1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/errno.h> 10 #include <linux/pci.h> 11 #include <linux/device.h> 12 #include <linux/slab.h> 13 #include <linux/vmalloc.h> 14 #include <linux/spinlock.h> 15 #include <linux/sizes.h> 16 #include <linux/atomic.h> 17 #include <linux/log2.h> 18 #include <linux/io.h> 19 #include <linux/completion.h> 20 #include <linux/err.h> 21 #include <asm/byteorder.h> 22 #include <asm/barrier.h> 23 24 #include "hinic_common.h" 25 #include "hinic_hw_if.h" 26 #include "hinic_hw_eqs.h" 27 #include "hinic_hw_mgmt.h" 28 #include "hinic_hw_wqe.h" 29 #include "hinic_hw_wq.h" 30 #include "hinic_hw_cmdq.h" 31 #include "hinic_hw_io.h" 32 #include "hinic_hw_dev.h" 33 34 #define CMDQ_CEQE_TYPE_SHIFT 0 35 36 #define CMDQ_CEQE_TYPE_MASK 0x7 37 38 #define CMDQ_CEQE_GET(val, member) \ 39 (((val) >> CMDQ_CEQE_##member##_SHIFT) \ 40 & CMDQ_CEQE_##member##_MASK) 41 42 #define CMDQ_WQE_ERRCODE_VAL_SHIFT 20 43 44 #define CMDQ_WQE_ERRCODE_VAL_MASK 0xF 45 46 #define CMDQ_WQE_ERRCODE_GET(val, member) \ 47 (((val) >> CMDQ_WQE_ERRCODE_##member##_SHIFT) \ 48 & CMDQ_WQE_ERRCODE_##member##_MASK) 49 50 #define CMDQ_DB_PI_OFF(pi) (((u16)LOWER_8_BITS(pi)) << 3) 51 52 #define CMDQ_DB_ADDR(db_base, pi) ((db_base) + CMDQ_DB_PI_OFF(pi)) 53 54 #define CMDQ_WQE_HEADER(wqe) ((struct hinic_cmdq_header *)(wqe)) 55 56 #define CMDQ_WQE_COMPLETED(ctrl_info) \ 57 HINIC_CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT) 58 59 #define FIRST_DATA_TO_WRITE_LAST sizeof(u64) 60 61 #define CMDQ_DB_OFF SZ_2K 62 63 #define CMDQ_WQEBB_SIZE 64 64 #define CMDQ_WQE_SIZE 64 65 #define CMDQ_DEPTH SZ_4K 66 67 #define CMDQ_WQ_PAGE_SIZE SZ_256K 68 69 #define WQE_LCMD_SIZE 64 70 #define WQE_SCMD_SIZE 64 71 72 #define COMPLETE_LEN 3 73 74 #define CMDQ_TIMEOUT 1000 75 76 #define CMDQ_PFN(addr, page_size) ((addr) >> (ilog2(page_size))) 77 78 #define cmdq_to_cmdqs(cmdq) container_of((cmdq) - (cmdq)->cmdq_type, \ 79 struct hinic_cmdqs, cmdq[0]) 80 81 #define cmdqs_to_func_to_io(cmdqs) container_of(cmdqs, \ 82 struct hinic_func_to_io, \ 83 cmdqs) 84 85 enum cmdq_wqe_type { 86 WQE_LCMD_TYPE = 0, 87 WQE_SCMD_TYPE = 1, 88 }; 89 90 enum completion_format { 91 COMPLETE_DIRECT = 0, 92 COMPLETE_SGE = 1, 93 }; 94 95 enum data_format { 96 DATA_SGE = 0, 97 DATA_DIRECT = 1, 98 }; 99 100 enum bufdesc_len { 101 BUFDESC_LCMD_LEN = 2, /* 16 bytes - 2(8 byte unit) */ 102 BUFDESC_SCMD_LEN = 3, /* 24 bytes - 3(8 byte unit) */ 103 }; 104 105 enum ctrl_sect_len { 106 CTRL_SECT_LEN = 1, /* 4 bytes (ctrl) - 1(8 byte unit) */ 107 CTRL_DIRECT_SECT_LEN = 2, /* 12 bytes (ctrl + rsvd) - 2(8 byte unit) */ 108 }; 109 110 enum cmdq_scmd_type { 111 CMDQ_SET_ARM_CMD = 2, 112 }; 113 114 enum cmdq_cmd_type { 115 CMDQ_CMD_SYNC_DIRECT_RESP = 0, 116 CMDQ_CMD_SYNC_SGE_RESP = 1, 117 }; 118 119 enum completion_request { 120 NO_CEQ = 0, 121 CEQ_SET = 1, 122 }; 123 124 /** 125 * hinic_alloc_cmdq_buf - alloc buffer for sending command 126 * @cmdqs: the cmdqs 127 * @cmdq_buf: the buffer returned in this struct 128 * 129 * Return 0 - Success, negative - Failure 130 **/ 131 int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs, 132 struct hinic_cmdq_buf *cmdq_buf) 133 { 134 struct hinic_hwif *hwif = cmdqs->hwif; 135 struct pci_dev *pdev = hwif->pdev; 136 137 cmdq_buf->buf = dma_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL, 138 &cmdq_buf->dma_addr); 139 if (!cmdq_buf->buf) { 140 dev_err(&pdev->dev, "Failed to allocate cmd from the pool\n"); 141 return -ENOMEM; 142 } 143 144 return 0; 145 } 146 147 /** 148 * hinic_free_cmdq_buf - free buffer 149 * @cmdqs: the cmdqs 150 * @cmdq_buf: the buffer to free that is in this struct 151 **/ 152 void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs, 153 struct hinic_cmdq_buf *cmdq_buf) 154 { 155 dma_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr); 156 } 157 158 static unsigned int cmdq_wqe_size_from_bdlen(enum bufdesc_len len) 159 { 160 unsigned int wqe_size = 0; 161 162 switch (len) { 163 case BUFDESC_LCMD_LEN: 164 wqe_size = WQE_LCMD_SIZE; 165 break; 166 case BUFDESC_SCMD_LEN: 167 wqe_size = WQE_SCMD_SIZE; 168 break; 169 } 170 171 return wqe_size; 172 } 173 174 static void cmdq_set_sge_completion(struct hinic_cmdq_completion *completion, 175 struct hinic_cmdq_buf *buf_out) 176 { 177 struct hinic_sge_resp *sge_resp = &completion->sge_resp; 178 179 hinic_set_sge(&sge_resp->sge, buf_out->dma_addr, buf_out->size); 180 } 181 182 static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped, 183 enum hinic_cmd_ack_type ack_type, 184 enum hinic_mod_type mod, u8 cmd, u16 prod_idx, 185 enum completion_format complete_format, 186 enum data_format data_format, 187 enum bufdesc_len buf_len) 188 { 189 struct hinic_cmdq_wqe_lcmd *wqe_lcmd; 190 struct hinic_cmdq_wqe_scmd *wqe_scmd; 191 enum ctrl_sect_len ctrl_len; 192 struct hinic_ctrl *ctrl; 193 u32 saved_data; 194 195 if (data_format == DATA_SGE) { 196 wqe_lcmd = &wqe->wqe_lcmd; 197 198 wqe_lcmd->status.status_info = 0; 199 ctrl = &wqe_lcmd->ctrl; 200 ctrl_len = CTRL_SECT_LEN; 201 } else { 202 wqe_scmd = &wqe->direct_wqe.wqe_scmd; 203 204 wqe_scmd->status.status_info = 0; 205 ctrl = &wqe_scmd->ctrl; 206 ctrl_len = CTRL_DIRECT_SECT_LEN; 207 } 208 209 ctrl->ctrl_info = HINIC_CMDQ_CTRL_SET(prod_idx, PI) | 210 HINIC_CMDQ_CTRL_SET(cmd, CMD) | 211 HINIC_CMDQ_CTRL_SET(mod, MOD) | 212 HINIC_CMDQ_CTRL_SET(ack_type, ACK_TYPE); 213 214 CMDQ_WQE_HEADER(wqe)->header_info = 215 HINIC_CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN) | 216 HINIC_CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT) | 217 HINIC_CMDQ_WQE_HEADER_SET(data_format, DATA_FMT) | 218 HINIC_CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ) | 219 HINIC_CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) | 220 HINIC_CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN) | 221 HINIC_CMDQ_WQE_HEADER_SET(wrapped, TOGGLED_WRAPPED); 222 223 saved_data = CMDQ_WQE_HEADER(wqe)->saved_data; 224 saved_data = HINIC_SAVED_DATA_CLEAR(saved_data, ARM); 225 226 if (cmd == CMDQ_SET_ARM_CMD && mod == HINIC_MOD_COMM) 227 CMDQ_WQE_HEADER(wqe)->saved_data |= 228 HINIC_SAVED_DATA_SET(1, ARM); 229 else 230 CMDQ_WQE_HEADER(wqe)->saved_data = saved_data; 231 } 232 233 static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe_lcmd, 234 struct hinic_cmdq_buf *buf_in) 235 { 236 hinic_set_sge(&wqe_lcmd->buf_desc.sge, buf_in->dma_addr, buf_in->size); 237 } 238 239 static void cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe *wqe, 240 void *buf_in, u32 in_size) 241 { 242 struct hinic_cmdq_wqe_scmd *wqe_scmd = &wqe->wqe_scmd; 243 244 wqe_scmd->buf_desc.buf_len = in_size; 245 memcpy(wqe_scmd->buf_desc.data, buf_in, in_size); 246 } 247 248 static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe, 249 enum cmdq_cmd_type cmd_type, 250 struct hinic_cmdq_buf *buf_in, 251 struct hinic_cmdq_buf *buf_out, int wrapped, 252 enum hinic_cmd_ack_type ack_type, 253 enum hinic_mod_type mod, u8 cmd, u16 prod_idx) 254 { 255 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; 256 enum completion_format complete_format; 257 258 switch (cmd_type) { 259 case CMDQ_CMD_SYNC_SGE_RESP: 260 complete_format = COMPLETE_SGE; 261 cmdq_set_sge_completion(&wqe_lcmd->completion, buf_out); 262 break; 263 case CMDQ_CMD_SYNC_DIRECT_RESP: 264 complete_format = COMPLETE_DIRECT; 265 wqe_lcmd->completion.direct_resp = 0; 266 break; 267 } 268 269 cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, 270 prod_idx, complete_format, DATA_SGE, 271 BUFDESC_LCMD_LEN); 272 273 cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in); 274 } 275 276 static void cmdq_set_direct_wqe(struct hinic_cmdq_wqe *wqe, 277 enum cmdq_cmd_type cmd_type, 278 void *buf_in, u16 in_size, 279 struct hinic_cmdq_buf *buf_out, int wrapped, 280 enum hinic_cmd_ack_type ack_type, 281 enum hinic_mod_type mod, u8 cmd, u16 prod_idx) 282 { 283 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; 284 enum completion_format complete_format; 285 struct hinic_cmdq_wqe_scmd *wqe_scmd; 286 287 wqe_scmd = &direct_wqe->wqe_scmd; 288 289 switch (cmd_type) { 290 case CMDQ_CMD_SYNC_SGE_RESP: 291 complete_format = COMPLETE_SGE; 292 cmdq_set_sge_completion(&wqe_scmd->completion, buf_out); 293 break; 294 case CMDQ_CMD_SYNC_DIRECT_RESP: 295 complete_format = COMPLETE_DIRECT; 296 wqe_scmd->completion.direct_resp = 0; 297 break; 298 } 299 300 cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, prod_idx, 301 complete_format, DATA_DIRECT, BUFDESC_SCMD_LEN); 302 303 cmdq_set_direct_wqe_data(direct_wqe, buf_in, in_size); 304 } 305 306 static void cmdq_wqe_fill(void *dst, void *src) 307 { 308 memcpy(dst + FIRST_DATA_TO_WRITE_LAST, src + FIRST_DATA_TO_WRITE_LAST, 309 CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST); 310 311 wmb(); /* The first 8 bytes should be written last */ 312 313 *(u64 *)dst = *(u64 *)src; 314 } 315 316 static void cmdq_fill_db(u32 *db_info, 317 enum hinic_cmdq_type cmdq_type, u16 prod_idx) 318 { 319 *db_info = HINIC_CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) | 320 HINIC_CMDQ_DB_INFO_SET(HINIC_CTRL_PATH, PATH) | 321 HINIC_CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE) | 322 HINIC_CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, DB_TYPE); 323 } 324 325 static void cmdq_set_db(struct hinic_cmdq *cmdq, 326 enum hinic_cmdq_type cmdq_type, u16 prod_idx) 327 { 328 u32 db_info; 329 330 cmdq_fill_db(&db_info, cmdq_type, prod_idx); 331 332 /* The data that is written to HW should be in Big Endian Format */ 333 db_info = cpu_to_be32(db_info); 334 335 wmb(); /* write all before the doorbell */ 336 337 writel(db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx)); 338 } 339 340 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq, 341 enum hinic_mod_type mod, u8 cmd, 342 struct hinic_cmdq_buf *buf_in, 343 u64 *resp) 344 { 345 struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe; 346 u16 curr_prod_idx, next_prod_idx; 347 int errcode, wrapped, num_wqebbs; 348 struct hinic_wq *wq = cmdq->wq; 349 struct hinic_hw_wqe *hw_wqe; 350 struct completion done; 351 352 /* Keep doorbell index correct. bh - for tasklet(ceq). */ 353 spin_lock_bh(&cmdq->cmdq_lock); 354 355 /* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/ 356 hw_wqe = hinic_get_wqe(wq, WQE_LCMD_SIZE, &curr_prod_idx); 357 if (IS_ERR(hw_wqe)) { 358 spin_unlock_bh(&cmdq->cmdq_lock); 359 return -EBUSY; 360 } 361 362 curr_cmdq_wqe = &hw_wqe->cmdq_wqe; 363 364 wrapped = cmdq->wrapped; 365 366 num_wqebbs = ALIGN(WQE_LCMD_SIZE, wq->wqebb_size) / wq->wqebb_size; 367 next_prod_idx = curr_prod_idx + num_wqebbs; 368 if (next_prod_idx >= wq->q_depth) { 369 cmdq->wrapped = !cmdq->wrapped; 370 next_prod_idx -= wq->q_depth; 371 } 372 373 cmdq->errcode[curr_prod_idx] = &errcode; 374 375 init_completion(&done); 376 cmdq->done[curr_prod_idx] = &done; 377 378 cmdq_set_lcmd_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, NULL, 379 wrapped, HINIC_CMD_ACK_TYPE_CMDQ, mod, cmd, 380 curr_prod_idx); 381 382 /* The data that is written to HW should be in Big Endian Format */ 383 hinic_cpu_to_be32(&cmdq_wqe, WQE_LCMD_SIZE); 384 385 /* CMDQ WQE is not shadow, therefore wqe will be written to wq */ 386 cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe); 387 388 cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx); 389 390 spin_unlock_bh(&cmdq->cmdq_lock); 391 392 if (!wait_for_completion_timeout(&done, 393 msecs_to_jiffies(CMDQ_TIMEOUT))) { 394 spin_lock_bh(&cmdq->cmdq_lock); 395 396 if (cmdq->errcode[curr_prod_idx] == &errcode) 397 cmdq->errcode[curr_prod_idx] = NULL; 398 399 if (cmdq->done[curr_prod_idx] == &done) 400 cmdq->done[curr_prod_idx] = NULL; 401 402 spin_unlock_bh(&cmdq->cmdq_lock); 403 404 hinic_dump_ceq_info(cmdq->hwdev); 405 return -ETIMEDOUT; 406 } 407 408 smp_rmb(); /* read error code after completion */ 409 410 if (resp) { 411 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &curr_cmdq_wqe->wqe_lcmd; 412 413 *resp = cpu_to_be64(wqe_lcmd->completion.direct_resp); 414 } 415 416 if (errcode != 0) 417 return -EFAULT; 418 419 return 0; 420 } 421 422 static int cmdq_set_arm_bit(struct hinic_cmdq *cmdq, void *buf_in, 423 u16 in_size) 424 { 425 struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe; 426 u16 curr_prod_idx, next_prod_idx; 427 struct hinic_wq *wq = cmdq->wq; 428 struct hinic_hw_wqe *hw_wqe; 429 int wrapped, num_wqebbs; 430 431 /* Keep doorbell index correct */ 432 spin_lock(&cmdq->cmdq_lock); 433 434 /* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/ 435 hw_wqe = hinic_get_wqe(wq, WQE_SCMD_SIZE, &curr_prod_idx); 436 if (IS_ERR(hw_wqe)) { 437 spin_unlock(&cmdq->cmdq_lock); 438 return -EBUSY; 439 } 440 441 curr_cmdq_wqe = &hw_wqe->cmdq_wqe; 442 443 wrapped = cmdq->wrapped; 444 445 num_wqebbs = ALIGN(WQE_SCMD_SIZE, wq->wqebb_size) / wq->wqebb_size; 446 next_prod_idx = curr_prod_idx + num_wqebbs; 447 if (next_prod_idx >= wq->q_depth) { 448 cmdq->wrapped = !cmdq->wrapped; 449 next_prod_idx -= wq->q_depth; 450 } 451 452 cmdq_set_direct_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, 453 in_size, NULL, wrapped, HINIC_CMD_ACK_TYPE_CMDQ, 454 HINIC_MOD_COMM, CMDQ_SET_ARM_CMD, curr_prod_idx); 455 456 /* The data that is written to HW should be in Big Endian Format */ 457 hinic_cpu_to_be32(&cmdq_wqe, WQE_SCMD_SIZE); 458 459 /* cmdq wqe is not shadow, therefore wqe will be written to wq */ 460 cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe); 461 462 cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx); 463 464 spin_unlock(&cmdq->cmdq_lock); 465 return 0; 466 } 467 468 static int cmdq_params_valid(struct hinic_cmdq_buf *buf_in) 469 { 470 if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE) 471 return -EINVAL; 472 473 return 0; 474 } 475 476 /** 477 * hinic_cmdq_direct_resp - send command with direct data as resp 478 * @cmdqs: the cmdqs 479 * @mod: module on the card that will handle the command 480 * @cmd: the command 481 * @buf_in: the buffer for the command 482 * @resp: the response to return 483 * 484 * Return 0 - Success, negative - Failure 485 **/ 486 int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs, 487 enum hinic_mod_type mod, u8 cmd, 488 struct hinic_cmdq_buf *buf_in, u64 *resp) 489 { 490 struct hinic_hwif *hwif = cmdqs->hwif; 491 struct pci_dev *pdev = hwif->pdev; 492 int err; 493 494 err = cmdq_params_valid(buf_in); 495 if (err) { 496 dev_err(&pdev->dev, "Invalid CMDQ parameters\n"); 497 return err; 498 } 499 500 return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC], 501 mod, cmd, buf_in, resp); 502 } 503 504 /** 505 * hinic_set_arm_bit - set arm bit for enable interrupt again 506 * @cmdqs: the cmdqs 507 * @q_type: type of queue to set the arm bit for 508 * @q_id: the queue number 509 * 510 * Return 0 - Success, negative - Failure 511 **/ 512 int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs, 513 enum hinic_set_arm_qtype q_type, u32 q_id) 514 { 515 struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC]; 516 struct hinic_hwif *hwif = cmdqs->hwif; 517 struct pci_dev *pdev = hwif->pdev; 518 struct hinic_cmdq_arm_bit arm_bit; 519 int err; 520 521 arm_bit.q_type = q_type; 522 arm_bit.q_id = q_id; 523 524 err = cmdq_set_arm_bit(cmdq, &arm_bit, sizeof(arm_bit)); 525 if (err) { 526 dev_err(&pdev->dev, "Failed to set arm for qid %d\n", q_id); 527 return err; 528 } 529 530 return 0; 531 } 532 533 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq, 534 struct hinic_cmdq_wqe *wqe) 535 { 536 u32 header_info = be32_to_cpu(CMDQ_WQE_HEADER(wqe)->header_info); 537 unsigned int bufdesc_len, wqe_size; 538 struct hinic_ctrl *ctrl; 539 540 bufdesc_len = HINIC_CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN); 541 wqe_size = cmdq_wqe_size_from_bdlen(bufdesc_len); 542 if (wqe_size == WQE_LCMD_SIZE) { 543 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; 544 545 ctrl = &wqe_lcmd->ctrl; 546 } else { 547 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; 548 struct hinic_cmdq_wqe_scmd *wqe_scmd; 549 550 wqe_scmd = &direct_wqe->wqe_scmd; 551 ctrl = &wqe_scmd->ctrl; 552 } 553 554 /* clear HW busy bit */ 555 ctrl->ctrl_info = 0; 556 557 wmb(); /* verify wqe is clear */ 558 } 559 560 /** 561 * cmdq_arm_ceq_handler - cmdq completion event handler for arm command 562 * @cmdq: the cmdq of the arm command 563 * @wqe: the wqe of the arm command 564 * 565 * Return 0 - Success, negative - Failure 566 **/ 567 static int cmdq_arm_ceq_handler(struct hinic_cmdq *cmdq, 568 struct hinic_cmdq_wqe *wqe) 569 { 570 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; 571 struct hinic_cmdq_wqe_scmd *wqe_scmd; 572 struct hinic_ctrl *ctrl; 573 u32 ctrl_info; 574 575 wqe_scmd = &direct_wqe->wqe_scmd; 576 ctrl = &wqe_scmd->ctrl; 577 ctrl_info = be32_to_cpu(ctrl->ctrl_info); 578 579 /* HW should toggle the HW BUSY BIT */ 580 if (!CMDQ_WQE_COMPLETED(ctrl_info)) 581 return -EBUSY; 582 583 clear_wqe_complete_bit(cmdq, wqe); 584 585 hinic_put_wqe(cmdq->wq, WQE_SCMD_SIZE); 586 return 0; 587 } 588 589 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx, 590 int errcode) 591 { 592 if (cmdq->errcode[prod_idx]) 593 *cmdq->errcode[prod_idx] = errcode; 594 } 595 596 /** 597 * cmdq_sync_cmd_handler - cmdq completion event handler for sync command 598 * @cmdq: the cmdq of the command 599 * @cons_idx: the consumer index to update the error code for 600 * @errcode: the error code 601 **/ 602 static void cmdq_sync_cmd_handler(struct hinic_cmdq *cmdq, u16 cons_idx, 603 int errcode) 604 { 605 u16 prod_idx = cons_idx; 606 607 spin_lock(&cmdq->cmdq_lock); 608 cmdq_update_errcode(cmdq, prod_idx, errcode); 609 610 wmb(); /* write all before update for the command request */ 611 612 if (cmdq->done[prod_idx]) 613 complete(cmdq->done[prod_idx]); 614 spin_unlock(&cmdq->cmdq_lock); 615 } 616 617 static int cmdq_cmd_ceq_handler(struct hinic_cmdq *cmdq, u16 ci, 618 struct hinic_cmdq_wqe *cmdq_wqe) 619 { 620 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &cmdq_wqe->wqe_lcmd; 621 struct hinic_status *status = &wqe_lcmd->status; 622 struct hinic_ctrl *ctrl = &wqe_lcmd->ctrl; 623 int errcode; 624 625 if (!CMDQ_WQE_COMPLETED(be32_to_cpu(ctrl->ctrl_info))) 626 return -EBUSY; 627 628 dma_rmb(); 629 630 errcode = CMDQ_WQE_ERRCODE_GET(be32_to_cpu(status->status_info), VAL); 631 632 cmdq_sync_cmd_handler(cmdq, ci, errcode); 633 634 clear_wqe_complete_bit(cmdq, cmdq_wqe); 635 hinic_put_wqe(cmdq->wq, WQE_LCMD_SIZE); 636 return 0; 637 } 638 639 /** 640 * cmdq_ceq_handler - cmdq completion event handler 641 * @handle: private data for the handler(cmdqs) 642 * @ceqe_data: ceq element data 643 **/ 644 static void cmdq_ceq_handler(void *handle, u32 ceqe_data) 645 { 646 enum hinic_cmdq_type cmdq_type = CMDQ_CEQE_GET(ceqe_data, TYPE); 647 struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)handle; 648 struct hinic_cmdq *cmdq = &cmdqs->cmdq[cmdq_type]; 649 struct hinic_cmdq_header *header; 650 struct hinic_hw_wqe *hw_wqe; 651 int err, set_arm = 0; 652 u32 saved_data; 653 u16 ci; 654 655 /* Read the smallest wqe size for getting wqe size */ 656 while ((hw_wqe = hinic_read_wqe(cmdq->wq, WQE_SCMD_SIZE, &ci))) { 657 if (IS_ERR(hw_wqe)) 658 break; 659 660 header = CMDQ_WQE_HEADER(&hw_wqe->cmdq_wqe); 661 saved_data = be32_to_cpu(header->saved_data); 662 663 if (HINIC_SAVED_DATA_GET(saved_data, ARM)) { 664 /* arm_bit was set until here */ 665 set_arm = 0; 666 667 if (cmdq_arm_ceq_handler(cmdq, &hw_wqe->cmdq_wqe)) 668 break; 669 } else { 670 set_arm = 1; 671 672 hw_wqe = hinic_read_wqe(cmdq->wq, WQE_LCMD_SIZE, &ci); 673 if (IS_ERR(hw_wqe)) 674 break; 675 676 if (cmdq_cmd_ceq_handler(cmdq, ci, &hw_wqe->cmdq_wqe)) 677 break; 678 } 679 } 680 681 if (set_arm) { 682 struct hinic_hwif *hwif = cmdqs->hwif; 683 struct pci_dev *pdev = hwif->pdev; 684 685 err = hinic_set_arm_bit(cmdqs, HINIC_SET_ARM_CMDQ, cmdq_type); 686 if (err) 687 dev_err(&pdev->dev, "Failed to set arm for CMDQ\n"); 688 } 689 } 690 691 /** 692 * cmdq_init_queue_ctxt - init the queue ctxt of a cmdq 693 * @cmdq_ctxt: cmdq ctxt to initialize 694 * @cmdq: the cmdq 695 * @cmdq_pages: the memory of the queue 696 **/ 697 static void cmdq_init_queue_ctxt(struct hinic_cmdq_ctxt *cmdq_ctxt, 698 struct hinic_cmdq *cmdq, 699 struct hinic_cmdq_pages *cmdq_pages) 700 { 701 struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info; 702 u64 wq_first_page_paddr, cmdq_first_block_paddr, pfn; 703 struct hinic_cmdqs *cmdqs = cmdq_to_cmdqs(cmdq); 704 struct hinic_wq *wq = cmdq->wq; 705 706 /* The data in the HW is in Big Endian Format */ 707 wq_first_page_paddr = be64_to_cpu(*wq->block_vaddr); 708 709 pfn = CMDQ_PFN(wq_first_page_paddr, SZ_4K); 710 711 ctxt_info->curr_wqe_page_pfn = 712 HINIC_CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN) | 713 HINIC_CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) | 714 HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_ARM) | 715 HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN) | 716 HINIC_CMDQ_CTXT_PAGE_INFO_SET(cmdq->wrapped, WRAPPED); 717 718 if (wq->num_q_pages != 1) { 719 /* block PFN - Read Modify Write */ 720 cmdq_first_block_paddr = cmdq_pages->page_paddr; 721 722 pfn = CMDQ_PFN(cmdq_first_block_paddr, wq->wq_page_size); 723 } 724 725 ctxt_info->wq_block_pfn = 726 HINIC_CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN) | 727 HINIC_CMDQ_CTXT_BLOCK_INFO_SET(atomic_read(&wq->cons_idx), CI); 728 729 cmdq_ctxt->func_idx = HINIC_HWIF_FUNC_IDX(cmdqs->hwif); 730 cmdq_ctxt->ppf_idx = HINIC_HWIF_PPF_IDX(cmdqs->hwif); 731 cmdq_ctxt->cmdq_type = cmdq->cmdq_type; 732 } 733 734 /** 735 * init_cmdq - initialize cmdq 736 * @cmdq: the cmdq 737 * @wq: the wq attaced to the cmdq 738 * @q_type: the cmdq type of the cmdq 739 * @db_area: doorbell area for the cmdq 740 * 741 * Return 0 - Success, negative - Failure 742 **/ 743 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_wq *wq, 744 enum hinic_cmdq_type q_type, void __iomem *db_area) 745 { 746 int err; 747 748 cmdq->wq = wq; 749 cmdq->cmdq_type = q_type; 750 cmdq->wrapped = 1; 751 752 spin_lock_init(&cmdq->cmdq_lock); 753 754 cmdq->done = vzalloc(array_size(sizeof(*cmdq->done), wq->q_depth)); 755 if (!cmdq->done) 756 return -ENOMEM; 757 758 cmdq->errcode = vzalloc(array_size(sizeof(*cmdq->errcode), 759 wq->q_depth)); 760 if (!cmdq->errcode) { 761 err = -ENOMEM; 762 goto err_errcode; 763 } 764 765 cmdq->db_base = db_area + CMDQ_DB_OFF; 766 return 0; 767 768 err_errcode: 769 vfree(cmdq->done); 770 return err; 771 } 772 773 /** 774 * free_cmdq - Free cmdq 775 * @cmdq: the cmdq to free 776 **/ 777 static void free_cmdq(struct hinic_cmdq *cmdq) 778 { 779 vfree(cmdq->errcode); 780 vfree(cmdq->done); 781 } 782 783 /** 784 * init_cmdqs_ctxt - write the cmdq ctxt to HW after init all cmdq 785 * @hwdev: the NIC HW device 786 * @cmdqs: cmdqs to write the ctxts for 787 * @db_area: db_area for all the cmdqs 788 * 789 * Return 0 - Success, negative - Failure 790 **/ 791 static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev, 792 struct hinic_cmdqs *cmdqs, void __iomem **db_area) 793 { 794 struct hinic_hwif *hwif = hwdev->hwif; 795 enum hinic_cmdq_type type, cmdq_type; 796 struct hinic_cmdq_ctxt *cmdq_ctxts; 797 struct pci_dev *pdev = hwif->pdev; 798 struct hinic_pfhwdev *pfhwdev; 799 size_t cmdq_ctxts_size; 800 int err; 801 802 cmdq_ctxts_size = HINIC_MAX_CMDQ_TYPES * sizeof(*cmdq_ctxts); 803 cmdq_ctxts = devm_kzalloc(&pdev->dev, cmdq_ctxts_size, GFP_KERNEL); 804 if (!cmdq_ctxts) 805 return -ENOMEM; 806 807 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 808 809 cmdq_type = HINIC_CMDQ_SYNC; 810 for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) { 811 cmdqs->cmdq[cmdq_type].hwdev = hwdev; 812 err = init_cmdq(&cmdqs->cmdq[cmdq_type], 813 &cmdqs->saved_wqs[cmdq_type], cmdq_type, 814 db_area[cmdq_type]); 815 if (err) { 816 dev_err(&pdev->dev, "Failed to initialize cmdq\n"); 817 goto err_init_cmdq; 818 } 819 820 cmdq_init_queue_ctxt(&cmdq_ctxts[cmdq_type], 821 &cmdqs->cmdq[cmdq_type], 822 &cmdqs->cmdq_pages); 823 } 824 825 /* Write the CMDQ ctxts */ 826 cmdq_type = HINIC_CMDQ_SYNC; 827 for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) { 828 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 829 HINIC_COMM_CMD_CMDQ_CTXT_SET, 830 &cmdq_ctxts[cmdq_type], 831 sizeof(cmdq_ctxts[cmdq_type]), 832 NULL, NULL, HINIC_MGMT_MSG_SYNC); 833 if (err) { 834 dev_err(&pdev->dev, "Failed to set CMDQ CTXT type = %d\n", 835 cmdq_type); 836 goto err_write_cmdq_ctxt; 837 } 838 } 839 840 devm_kfree(&pdev->dev, cmdq_ctxts); 841 return 0; 842 843 err_write_cmdq_ctxt: 844 cmdq_type = HINIC_MAX_CMDQ_TYPES; 845 846 err_init_cmdq: 847 for (type = HINIC_CMDQ_SYNC; type < cmdq_type; type++) 848 free_cmdq(&cmdqs->cmdq[type]); 849 850 devm_kfree(&pdev->dev, cmdq_ctxts); 851 return err; 852 } 853 854 static int hinic_set_cmdq_depth(struct hinic_hwdev *hwdev, u16 cmdq_depth) 855 { 856 struct hinic_cmd_hw_ioctxt hw_ioctxt = { 0 }; 857 struct hinic_pfhwdev *pfhwdev; 858 859 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 860 861 hw_ioctxt.func_idx = HINIC_HWIF_FUNC_IDX(hwdev->hwif); 862 hw_ioctxt.ppf_idx = HINIC_HWIF_PPF_IDX(hwdev->hwif); 863 864 hw_ioctxt.set_cmdq_depth = HW_IOCTXT_SET_CMDQ_DEPTH_ENABLE; 865 hw_ioctxt.cmdq_depth = (u8)ilog2(cmdq_depth); 866 867 return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 868 HINIC_COMM_CMD_HWCTXT_SET, 869 &hw_ioctxt, sizeof(hw_ioctxt), NULL, 870 NULL, HINIC_MGMT_MSG_SYNC); 871 } 872 873 /** 874 * hinic_init_cmdqs - init all cmdqs 875 * @cmdqs: cmdqs to init 876 * @hwif: HW interface for accessing cmdqs 877 * @db_area: doorbell areas for all the cmdqs 878 * 879 * Return 0 - Success, negative - Failure 880 **/ 881 int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif, 882 void __iomem **db_area) 883 { 884 struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs); 885 struct pci_dev *pdev = hwif->pdev; 886 struct hinic_hwdev *hwdev; 887 size_t saved_wqs_size; 888 u16 max_wqe_size; 889 int err; 890 891 cmdqs->hwif = hwif; 892 cmdqs->cmdq_buf_pool = dma_pool_create("hinic_cmdq", &pdev->dev, 893 HINIC_CMDQ_BUF_SIZE, 894 HINIC_CMDQ_BUF_SIZE, 0); 895 if (!cmdqs->cmdq_buf_pool) 896 return -ENOMEM; 897 898 saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq); 899 cmdqs->saved_wqs = devm_kzalloc(&pdev->dev, saved_wqs_size, GFP_KERNEL); 900 if (!cmdqs->saved_wqs) { 901 err = -ENOMEM; 902 goto err_saved_wqs; 903 } 904 905 max_wqe_size = WQE_LCMD_SIZE; 906 err = hinic_wqs_cmdq_alloc(&cmdqs->cmdq_pages, cmdqs->saved_wqs, hwif, 907 HINIC_MAX_CMDQ_TYPES, CMDQ_WQEBB_SIZE, 908 CMDQ_WQ_PAGE_SIZE, CMDQ_DEPTH, max_wqe_size); 909 if (err) { 910 dev_err(&pdev->dev, "Failed to allocate CMDQ wqs\n"); 911 goto err_cmdq_wqs; 912 } 913 914 hwdev = container_of(func_to_io, struct hinic_hwdev, func_to_io); 915 err = init_cmdqs_ctxt(hwdev, cmdqs, db_area); 916 if (err) { 917 dev_err(&pdev->dev, "Failed to write cmdq ctxt\n"); 918 goto err_cmdq_ctxt; 919 } 920 921 hinic_ceq_register_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ, cmdqs, 922 cmdq_ceq_handler); 923 924 err = hinic_set_cmdq_depth(hwdev, CMDQ_DEPTH); 925 if (err) { 926 dev_err(&hwif->pdev->dev, "Failed to set cmdq depth\n"); 927 goto err_set_cmdq_depth; 928 } 929 930 return 0; 931 932 err_set_cmdq_depth: 933 hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ); 934 935 err_cmdq_ctxt: 936 hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs, 937 HINIC_MAX_CMDQ_TYPES); 938 939 err_cmdq_wqs: 940 devm_kfree(&pdev->dev, cmdqs->saved_wqs); 941 942 err_saved_wqs: 943 dma_pool_destroy(cmdqs->cmdq_buf_pool); 944 return err; 945 } 946 947 /** 948 * hinic_free_cmdqs - free all cmdqs 949 * @cmdqs: cmdqs to free 950 **/ 951 void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs) 952 { 953 struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs); 954 struct hinic_hwif *hwif = cmdqs->hwif; 955 struct pci_dev *pdev = hwif->pdev; 956 enum hinic_cmdq_type cmdq_type; 957 958 hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ); 959 960 cmdq_type = HINIC_CMDQ_SYNC; 961 for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) 962 free_cmdq(&cmdqs->cmdq[cmdq_type]); 963 964 hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs, 965 HINIC_MAX_CMDQ_TYPES); 966 967 devm_kfree(&pdev->dev, cmdqs->saved_wqs); 968 969 dma_pool_destroy(cmdqs->cmdq_buf_pool); 970 } 971