1bfe1d560SDave Jiang /* SPDX-License-Identifier: GPL-2.0 */ 2bfe1d560SDave Jiang /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3bfe1d560SDave Jiang #ifndef _IDXD_REGISTERS_H_ 4bfe1d560SDave Jiang #define _IDXD_REGISTERS_H_ 5bfe1d560SDave Jiang 6bfe1d560SDave Jiang /* PCI Config */ 7bfe1d560SDave Jiang #define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25 8bfe1d560SDave Jiang 9bfe1d560SDave Jiang #define IDXD_MMIO_BAR 0 10bfe1d560SDave Jiang #define IDXD_WQ_BAR 2 11c52ca478SDave Jiang #define IDXD_PORTAL_SIZE 0x4000 12bfe1d560SDave Jiang 13bfe1d560SDave Jiang /* MMIO Device BAR0 Registers */ 14bfe1d560SDave Jiang #define IDXD_VER_OFFSET 0x00 15bfe1d560SDave Jiang #define IDXD_VER_MAJOR_MASK 0xf0 16bfe1d560SDave Jiang #define IDXD_VER_MINOR_MASK 0x0f 17bfe1d560SDave Jiang #define GET_IDXD_VER_MAJOR(x) (((x) & IDXD_VER_MAJOR_MASK) >> 4) 18bfe1d560SDave Jiang #define GET_IDXD_VER_MINOR(x) ((x) & IDXD_VER_MINOR_MASK) 19bfe1d560SDave Jiang 20bfe1d560SDave Jiang union gen_cap_reg { 21bfe1d560SDave Jiang struct { 22bfe1d560SDave Jiang u64 block_on_fault:1; 23bfe1d560SDave Jiang u64 overlap_copy:1; 24bfe1d560SDave Jiang u64 cache_control_mem:1; 25bfe1d560SDave Jiang u64 cache_control_cache:1; 26bfe1d560SDave Jiang u64 rsvd:3; 27bfe1d560SDave Jiang u64 int_handle_req:1; 28bfe1d560SDave Jiang u64 dest_readback:1; 29bfe1d560SDave Jiang u64 drain_readback:1; 30bfe1d560SDave Jiang u64 rsvd2:6; 31bfe1d560SDave Jiang u64 max_xfer_shift:5; 32bfe1d560SDave Jiang u64 max_batch_shift:4; 33bfe1d560SDave Jiang u64 max_ims_mult:6; 34bfe1d560SDave Jiang u64 config_en:1; 35bfe1d560SDave Jiang u64 max_descs_per_engine:8; 36bfe1d560SDave Jiang u64 rsvd3:24; 37bfe1d560SDave Jiang }; 38bfe1d560SDave Jiang u64 bits; 39bfe1d560SDave Jiang } __packed; 40bfe1d560SDave Jiang #define IDXD_GENCAP_OFFSET 0x10 41bfe1d560SDave Jiang 42bfe1d560SDave Jiang union wq_cap_reg { 43bfe1d560SDave Jiang struct { 44bfe1d560SDave Jiang u64 total_wq_size:16; 45bfe1d560SDave Jiang u64 num_wqs:8; 46d98793b5SDave Jiang u64 wqcfg_size:4; 47d98793b5SDave Jiang u64 rsvd:20; 48bfe1d560SDave Jiang u64 shared_mode:1; 49bfe1d560SDave Jiang u64 dedicated_mode:1; 50*92de5fa2SDave Jiang u64 wq_ats_support:1; 51bfe1d560SDave Jiang u64 priority:1; 52bfe1d560SDave Jiang u64 occupancy:1; 53bfe1d560SDave Jiang u64 occupancy_int:1; 54bfe1d560SDave Jiang u64 rsvd3:10; 55bfe1d560SDave Jiang }; 56bfe1d560SDave Jiang u64 bits; 57bfe1d560SDave Jiang } __packed; 58bfe1d560SDave Jiang #define IDXD_WQCAP_OFFSET 0x20 59d98793b5SDave Jiang #define IDXD_WQCFG_MIN 5 60bfe1d560SDave Jiang 61bfe1d560SDave Jiang union group_cap_reg { 62bfe1d560SDave Jiang struct { 63bfe1d560SDave Jiang u64 num_groups:8; 64bfe1d560SDave Jiang u64 total_tokens:8; 65bfe1d560SDave Jiang u64 token_en:1; 66bfe1d560SDave Jiang u64 token_limit:1; 67bfe1d560SDave Jiang u64 rsvd:46; 68bfe1d560SDave Jiang }; 69bfe1d560SDave Jiang u64 bits; 70bfe1d560SDave Jiang } __packed; 71bfe1d560SDave Jiang #define IDXD_GRPCAP_OFFSET 0x30 72bfe1d560SDave Jiang 73bfe1d560SDave Jiang union engine_cap_reg { 74bfe1d560SDave Jiang struct { 75bfe1d560SDave Jiang u64 num_engines:8; 76bfe1d560SDave Jiang u64 rsvd:56; 77bfe1d560SDave Jiang }; 78bfe1d560SDave Jiang u64 bits; 79bfe1d560SDave Jiang } __packed; 80bfe1d560SDave Jiang 81bfe1d560SDave Jiang #define IDXD_ENGCAP_OFFSET 0x38 82bfe1d560SDave Jiang 83bfe1d560SDave Jiang #define IDXD_OPCAP_NOOP 0x0001 84bfe1d560SDave Jiang #define IDXD_OPCAP_BATCH 0x0002 85bfe1d560SDave Jiang #define IDXD_OPCAP_MEMMOVE 0x0008 86bfe1d560SDave Jiang struct opcap { 87bfe1d560SDave Jiang u64 bits[4]; 88bfe1d560SDave Jiang }; 89bfe1d560SDave Jiang 90bfe1d560SDave Jiang #define IDXD_OPCAP_OFFSET 0x40 91bfe1d560SDave Jiang 92bfe1d560SDave Jiang #define IDXD_TABLE_OFFSET 0x60 93bfe1d560SDave Jiang union offsets_reg { 94bfe1d560SDave Jiang struct { 95bfe1d560SDave Jiang u64 grpcfg:16; 96bfe1d560SDave Jiang u64 wqcfg:16; 97bfe1d560SDave Jiang u64 msix_perm:16; 98bfe1d560SDave Jiang u64 ims:16; 99bfe1d560SDave Jiang u64 perfmon:16; 100bfe1d560SDave Jiang u64 rsvd:48; 101bfe1d560SDave Jiang }; 102bfe1d560SDave Jiang u64 bits[2]; 103bfe1d560SDave Jiang } __packed; 104bfe1d560SDave Jiang 1052f8417a9SDave Jiang #define IDXD_TABLE_MULT 0x100 1062f8417a9SDave Jiang 107bfe1d560SDave Jiang #define IDXD_GENCFG_OFFSET 0x80 108bfe1d560SDave Jiang union gencfg_reg { 109bfe1d560SDave Jiang struct { 110bfe1d560SDave Jiang u32 token_limit:8; 111bfe1d560SDave Jiang u32 rsvd:4; 112bfe1d560SDave Jiang u32 user_int_en:1; 113bfe1d560SDave Jiang u32 rsvd2:19; 114bfe1d560SDave Jiang }; 115bfe1d560SDave Jiang u32 bits; 116bfe1d560SDave Jiang } __packed; 117bfe1d560SDave Jiang 118bfe1d560SDave Jiang #define IDXD_GENCTRL_OFFSET 0x88 119bfe1d560SDave Jiang union genctrl_reg { 120bfe1d560SDave Jiang struct { 121bfe1d560SDave Jiang u32 softerr_int_en:1; 122bfe1d560SDave Jiang u32 rsvd:31; 123bfe1d560SDave Jiang }; 124bfe1d560SDave Jiang u32 bits; 125bfe1d560SDave Jiang } __packed; 126bfe1d560SDave Jiang 127bfe1d560SDave Jiang #define IDXD_GENSTATS_OFFSET 0x90 128bfe1d560SDave Jiang union gensts_reg { 129bfe1d560SDave Jiang struct { 130bfe1d560SDave Jiang u32 state:2; 131bfe1d560SDave Jiang u32 reset_type:2; 132bfe1d560SDave Jiang u32 rsvd:28; 133bfe1d560SDave Jiang }; 134bfe1d560SDave Jiang u32 bits; 135bfe1d560SDave Jiang } __packed; 136bfe1d560SDave Jiang 137bfe1d560SDave Jiang enum idxd_device_status_state { 138bfe1d560SDave Jiang IDXD_DEVICE_STATE_DISABLED = 0, 139bfe1d560SDave Jiang IDXD_DEVICE_STATE_ENABLED, 140bfe1d560SDave Jiang IDXD_DEVICE_STATE_DRAIN, 141bfe1d560SDave Jiang IDXD_DEVICE_STATE_HALT, 142bfe1d560SDave Jiang }; 143bfe1d560SDave Jiang 144bfe1d560SDave Jiang enum idxd_device_reset_type { 145bfe1d560SDave Jiang IDXD_DEVICE_RESET_SOFTWARE = 0, 146bfe1d560SDave Jiang IDXD_DEVICE_RESET_FLR, 147bfe1d560SDave Jiang IDXD_DEVICE_RESET_WARM, 148bfe1d560SDave Jiang IDXD_DEVICE_RESET_COLD, 149bfe1d560SDave Jiang }; 150bfe1d560SDave Jiang 151bfe1d560SDave Jiang #define IDXD_INTCAUSE_OFFSET 0x98 152bfe1d560SDave Jiang #define IDXD_INTC_ERR 0x01 153bfe1d560SDave Jiang #define IDXD_INTC_CMD 0x02 154bfe1d560SDave Jiang #define IDXD_INTC_OCCUPY 0x04 155bfe1d560SDave Jiang #define IDXD_INTC_PERFMON_OVFL 0x08 156bfe1d560SDave Jiang 157bfe1d560SDave Jiang #define IDXD_CMD_OFFSET 0xa0 158bfe1d560SDave Jiang union idxd_command_reg { 159bfe1d560SDave Jiang struct { 160bfe1d560SDave Jiang u32 operand:20; 161bfe1d560SDave Jiang u32 cmd:5; 162bfe1d560SDave Jiang u32 rsvd:6; 163bfe1d560SDave Jiang u32 int_req:1; 164bfe1d560SDave Jiang }; 165bfe1d560SDave Jiang u32 bits; 166bfe1d560SDave Jiang } __packed; 167bfe1d560SDave Jiang 168bfe1d560SDave Jiang enum idxd_cmd { 169bfe1d560SDave Jiang IDXD_CMD_ENABLE_DEVICE = 1, 170bfe1d560SDave Jiang IDXD_CMD_DISABLE_DEVICE, 171bfe1d560SDave Jiang IDXD_CMD_DRAIN_ALL, 172bfe1d560SDave Jiang IDXD_CMD_ABORT_ALL, 173bfe1d560SDave Jiang IDXD_CMD_RESET_DEVICE, 174bfe1d560SDave Jiang IDXD_CMD_ENABLE_WQ, 175bfe1d560SDave Jiang IDXD_CMD_DISABLE_WQ, 176bfe1d560SDave Jiang IDXD_CMD_DRAIN_WQ, 177bfe1d560SDave Jiang IDXD_CMD_ABORT_WQ, 178bfe1d560SDave Jiang IDXD_CMD_RESET_WQ, 179bfe1d560SDave Jiang IDXD_CMD_DRAIN_PASID, 180bfe1d560SDave Jiang IDXD_CMD_ABORT_PASID, 181bfe1d560SDave Jiang IDXD_CMD_REQUEST_INT_HANDLE, 182bfe1d560SDave Jiang }; 183bfe1d560SDave Jiang 184bfe1d560SDave Jiang #define IDXD_CMDSTS_OFFSET 0xa8 185bfe1d560SDave Jiang union cmdsts_reg { 186bfe1d560SDave Jiang struct { 187bfe1d560SDave Jiang u8 err; 188bfe1d560SDave Jiang u16 result; 189bfe1d560SDave Jiang u8 rsvd:7; 190bfe1d560SDave Jiang u8 active:1; 191bfe1d560SDave Jiang }; 192bfe1d560SDave Jiang u32 bits; 193bfe1d560SDave Jiang } __packed; 194bfe1d560SDave Jiang #define IDXD_CMDSTS_ACTIVE 0x80000000 195bfe1d560SDave Jiang 196bfe1d560SDave Jiang enum idxd_cmdsts_err { 197bfe1d560SDave Jiang IDXD_CMDSTS_SUCCESS = 0, 198bfe1d560SDave Jiang IDXD_CMDSTS_INVAL_CMD, 199bfe1d560SDave Jiang IDXD_CMDSTS_INVAL_WQIDX, 200bfe1d560SDave Jiang IDXD_CMDSTS_HW_ERR, 201bfe1d560SDave Jiang /* enable device errors */ 202bfe1d560SDave Jiang IDXD_CMDSTS_ERR_DEV_ENABLED = 0x10, 203bfe1d560SDave Jiang IDXD_CMDSTS_ERR_CONFIG, 204bfe1d560SDave Jiang IDXD_CMDSTS_ERR_BUSMASTER_EN, 205bfe1d560SDave Jiang IDXD_CMDSTS_ERR_PASID_INVAL, 206bfe1d560SDave Jiang IDXD_CMDSTS_ERR_WQ_SIZE_ERANGE, 207bfe1d560SDave Jiang IDXD_CMDSTS_ERR_GRP_CONFIG, 208bfe1d560SDave Jiang IDXD_CMDSTS_ERR_GRP_CONFIG2, 209bfe1d560SDave Jiang IDXD_CMDSTS_ERR_GRP_CONFIG3, 210bfe1d560SDave Jiang IDXD_CMDSTS_ERR_GRP_CONFIG4, 211bfe1d560SDave Jiang /* enable wq errors */ 212bfe1d560SDave Jiang IDXD_CMDSTS_ERR_DEV_NOTEN = 0x20, 213bfe1d560SDave Jiang IDXD_CMDSTS_ERR_WQ_ENABLED, 214bfe1d560SDave Jiang IDXD_CMDSTS_ERR_WQ_SIZE, 215bfe1d560SDave Jiang IDXD_CMDSTS_ERR_WQ_PRIOR, 216bfe1d560SDave Jiang IDXD_CMDSTS_ERR_WQ_MODE, 217bfe1d560SDave Jiang IDXD_CMDSTS_ERR_BOF_EN, 218bfe1d560SDave Jiang IDXD_CMDSTS_ERR_PASID_EN, 219bfe1d560SDave Jiang IDXD_CMDSTS_ERR_MAX_BATCH_SIZE, 220bfe1d560SDave Jiang IDXD_CMDSTS_ERR_MAX_XFER_SIZE, 221bfe1d560SDave Jiang /* disable device errors */ 222bfe1d560SDave Jiang IDXD_CMDSTS_ERR_DIS_DEV_EN = 0x31, 223bfe1d560SDave Jiang /* disable WQ, drain WQ, abort WQ, reset WQ */ 224bfe1d560SDave Jiang IDXD_CMDSTS_ERR_DEV_NOT_EN, 225bfe1d560SDave Jiang /* request interrupt handle */ 226bfe1d560SDave Jiang IDXD_CMDSTS_ERR_INVAL_INT_IDX = 0x41, 227bfe1d560SDave Jiang IDXD_CMDSTS_ERR_NO_HANDLE, 228bfe1d560SDave Jiang }; 229bfe1d560SDave Jiang 230bfe1d560SDave Jiang #define IDXD_SWERR_OFFSET 0xc0 231bfe1d560SDave Jiang #define IDXD_SWERR_VALID 0x00000001 232bfe1d560SDave Jiang #define IDXD_SWERR_OVERFLOW 0x00000002 233bfe1d560SDave Jiang #define IDXD_SWERR_ACK (IDXD_SWERR_VALID | IDXD_SWERR_OVERFLOW) 234bfe1d560SDave Jiang union sw_err_reg { 235bfe1d560SDave Jiang struct { 236bfe1d560SDave Jiang u64 valid:1; 237bfe1d560SDave Jiang u64 overflow:1; 238bfe1d560SDave Jiang u64 desc_valid:1; 239bfe1d560SDave Jiang u64 wq_idx_valid:1; 240bfe1d560SDave Jiang u64 batch:1; 241bfe1d560SDave Jiang u64 fault_rw:1; 242bfe1d560SDave Jiang u64 priv:1; 243bfe1d560SDave Jiang u64 rsvd:1; 244bfe1d560SDave Jiang u64 error:8; 245bfe1d560SDave Jiang u64 wq_idx:8; 246bfe1d560SDave Jiang u64 rsvd2:8; 247bfe1d560SDave Jiang u64 operation:8; 248bfe1d560SDave Jiang u64 pasid:20; 249bfe1d560SDave Jiang u64 rsvd3:4; 250bfe1d560SDave Jiang 251bfe1d560SDave Jiang u64 batch_idx:16; 252bfe1d560SDave Jiang u64 rsvd4:16; 253bfe1d560SDave Jiang u64 invalid_flags:32; 254bfe1d560SDave Jiang 255bfe1d560SDave Jiang u64 fault_addr; 256bfe1d560SDave Jiang 257bfe1d560SDave Jiang u64 rsvd5; 258bfe1d560SDave Jiang }; 259bfe1d560SDave Jiang u64 bits[4]; 260bfe1d560SDave Jiang } __packed; 261bfe1d560SDave Jiang 262bfe1d560SDave Jiang union msix_perm { 263bfe1d560SDave Jiang struct { 264bfe1d560SDave Jiang u32 rsvd:2; 265bfe1d560SDave Jiang u32 ignore:1; 266bfe1d560SDave Jiang u32 pasid_en:1; 267bfe1d560SDave Jiang u32 rsvd2:8; 268bfe1d560SDave Jiang u32 pasid:20; 269bfe1d560SDave Jiang }; 270bfe1d560SDave Jiang u32 bits; 271bfe1d560SDave Jiang } __packed; 272bfe1d560SDave Jiang 273bfe1d560SDave Jiang union group_flags { 274bfe1d560SDave Jiang struct { 275bfe1d560SDave Jiang u32 tc_a:3; 276bfe1d560SDave Jiang u32 tc_b:3; 277bfe1d560SDave Jiang u32 rsvd:1; 278bfe1d560SDave Jiang u32 use_token_limit:1; 279bfe1d560SDave Jiang u32 tokens_reserved:8; 280bfe1d560SDave Jiang u32 rsvd2:4; 281bfe1d560SDave Jiang u32 tokens_allowed:8; 282bfe1d560SDave Jiang u32 rsvd3:4; 283bfe1d560SDave Jiang }; 284bfe1d560SDave Jiang u32 bits; 285bfe1d560SDave Jiang } __packed; 286bfe1d560SDave Jiang 287bfe1d560SDave Jiang struct grpcfg { 288bfe1d560SDave Jiang u64 wqs[4]; 289bfe1d560SDave Jiang u64 engines; 290bfe1d560SDave Jiang union group_flags flags; 291bfe1d560SDave Jiang } __packed; 292bfe1d560SDave Jiang 293bfe1d560SDave Jiang union wqcfg { 294bfe1d560SDave Jiang struct { 295bfe1d560SDave Jiang /* bytes 0-3 */ 296bfe1d560SDave Jiang u16 wq_size; 297bfe1d560SDave Jiang u16 rsvd; 298bfe1d560SDave Jiang 299bfe1d560SDave Jiang /* bytes 4-7 */ 300bfe1d560SDave Jiang u16 wq_thresh; 301bfe1d560SDave Jiang u16 rsvd1; 302bfe1d560SDave Jiang 303bfe1d560SDave Jiang /* bytes 8-11 */ 304bfe1d560SDave Jiang u32 mode:1; /* shared or dedicated */ 305bfe1d560SDave Jiang u32 bof:1; /* block on fault */ 306*92de5fa2SDave Jiang u32 wq_ats_disable:1; 307*92de5fa2SDave Jiang u32 rsvd2:1; 308bfe1d560SDave Jiang u32 priority:4; 309bfe1d560SDave Jiang u32 pasid:20; 310bfe1d560SDave Jiang u32 pasid_en:1; 311bfe1d560SDave Jiang u32 priv:1; 312bfe1d560SDave Jiang u32 rsvd3:2; 313bfe1d560SDave Jiang 314bfe1d560SDave Jiang /* bytes 12-15 */ 315bfe1d560SDave Jiang u32 max_xfer_shift:5; 316bfe1d560SDave Jiang u32 max_batch_shift:4; 317bfe1d560SDave Jiang u32 rsvd4:23; 318bfe1d560SDave Jiang 319bfe1d560SDave Jiang /* bytes 16-19 */ 320bfe1d560SDave Jiang u16 occupancy_inth; 321bfe1d560SDave Jiang u16 occupancy_table_sel:1; 322bfe1d560SDave Jiang u16 rsvd5:15; 323bfe1d560SDave Jiang 324bfe1d560SDave Jiang /* bytes 20-23 */ 325bfe1d560SDave Jiang u16 occupancy_limit; 326bfe1d560SDave Jiang u16 occupancy_int_en:1; 327bfe1d560SDave Jiang u16 rsvd6:15; 328bfe1d560SDave Jiang 329bfe1d560SDave Jiang /* bytes 24-27 */ 330bfe1d560SDave Jiang u16 occupancy; 331bfe1d560SDave Jiang u16 occupancy_int:1; 332bfe1d560SDave Jiang u16 rsvd7:12; 333bfe1d560SDave Jiang u16 mode_support:1; 334bfe1d560SDave Jiang u16 wq_state:2; 335bfe1d560SDave Jiang 336bfe1d560SDave Jiang /* bytes 28-31 */ 337bfe1d560SDave Jiang u32 rsvd8; 338bfe1d560SDave Jiang }; 339bfe1d560SDave Jiang u32 bits[8]; 340bfe1d560SDave Jiang } __packed; 341d98793b5SDave Jiang 3428e50d392SDave Jiang #define WQCFG_PASID_IDX 2 3438e50d392SDave Jiang 344d98793b5SDave Jiang /* 345d98793b5SDave Jiang * This macro calculates the offset into the WQCFG register 346d98793b5SDave Jiang * idxd - struct idxd * 347d98793b5SDave Jiang * n - wq id 348d98793b5SDave Jiang * ofs - the index of the 32b dword for the config register 349d98793b5SDave Jiang * 350d98793b5SDave Jiang * The WQCFG register block is divided into groups per each wq. The n index 351d98793b5SDave Jiang * allows us to move to the register group that's for that particular wq. 352d98793b5SDave Jiang * Each register is 32bits. The ofs gives us the number of register to access. 353d98793b5SDave Jiang */ 354d98793b5SDave Jiang #define WQCFG_OFFSET(_idxd_dev, n, ofs) \ 355d98793b5SDave Jiang ({\ 356d98793b5SDave Jiang typeof(_idxd_dev) __idxd_dev = (_idxd_dev); \ 357d98793b5SDave Jiang (__idxd_dev)->wqcfg_offset + (n) * (__idxd_dev)->wqcfg_size + sizeof(u32) * (ofs); \ 358d98793b5SDave Jiang }) 359d98793b5SDave Jiang 360d98793b5SDave Jiang #define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32)) 361d98793b5SDave Jiang 3625a712701SDave Jiang #define GRPCFG_SIZE 64 3635a712701SDave Jiang #define GRPWQCFG_STRIDES 4 3645a712701SDave Jiang 3655a712701SDave Jiang /* 3665a712701SDave Jiang * This macro calculates the offset into the GRPCFG register 3675a712701SDave Jiang * idxd - struct idxd * 3685a712701SDave Jiang * n - wq id 3695a712701SDave Jiang * ofs - the index of the 32b dword for the config register 3705a712701SDave Jiang * 3715a712701SDave Jiang * The WQCFG register block is divided into groups per each wq. The n index 3725a712701SDave Jiang * allows us to move to the register group that's for that particular wq. 3735a712701SDave Jiang * Each register is 32bits. The ofs gives us the number of register to access. 3745a712701SDave Jiang */ 3755a712701SDave Jiang #define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\ 3765a712701SDave Jiang (n) * GRPCFG_SIZE + sizeof(u64) * (ofs)) 3775a712701SDave Jiang #define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32) 3785a712701SDave Jiang #define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40) 3795a712701SDave Jiang 380bfe1d560SDave Jiang #endif 381