1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/errno.h> 10 #include <linux/pci.h> 11 #include <linux/device.h> 12 #include <linux/slab.h> 13 #include <linux/vmalloc.h> 14 #include <linux/spinlock.h> 15 #include <linux/sizes.h> 16 #include <linux/atomic.h> 17 #include <linux/log2.h> 18 #include <linux/io.h> 19 #include <linux/completion.h> 20 #include <linux/err.h> 21 #include <asm/byteorder.h> 22 #include <asm/barrier.h> 23 24 #include "hinic_common.h" 25 #include "hinic_hw_if.h" 26 #include "hinic_hw_eqs.h" 27 #include "hinic_hw_mgmt.h" 28 #include "hinic_hw_wqe.h" 29 #include "hinic_hw_wq.h" 30 #include "hinic_hw_cmdq.h" 31 #include "hinic_hw_io.h" 32 #include "hinic_hw_dev.h" 33 34 #define CMDQ_CEQE_TYPE_SHIFT 0 35 36 #define CMDQ_CEQE_TYPE_MASK 0x7 37 38 #define CMDQ_CEQE_GET(val, member) \ 39 (((val) >> CMDQ_CEQE_##member##_SHIFT) \ 40 & CMDQ_CEQE_##member##_MASK) 41 42 #define CMDQ_WQE_ERRCODE_VAL_SHIFT 20 43 44 #define CMDQ_WQE_ERRCODE_VAL_MASK 0xF 45 46 #define CMDQ_WQE_ERRCODE_GET(val, member) \ 47 (((val) >> CMDQ_WQE_ERRCODE_##member##_SHIFT) \ 48 & CMDQ_WQE_ERRCODE_##member##_MASK) 49 50 #define CMDQ_DB_PI_OFF(pi) (((u16)LOWER_8_BITS(pi)) << 3) 51 52 #define CMDQ_DB_ADDR(db_base, pi) ((db_base) + CMDQ_DB_PI_OFF(pi)) 53 54 #define CMDQ_WQE_HEADER(wqe) ((struct hinic_cmdq_header *)(wqe)) 55 56 #define CMDQ_WQE_COMPLETED(ctrl_info) \ 57 HINIC_CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT) 58 59 #define FIRST_DATA_TO_WRITE_LAST sizeof(u64) 60 61 #define CMDQ_DB_OFF SZ_2K 62 63 #define CMDQ_WQEBB_SIZE 64 64 #define CMDQ_WQE_SIZE 64 65 #define CMDQ_DEPTH SZ_4K 66 67 #define CMDQ_WQ_PAGE_SIZE SZ_256K 68 69 #define WQE_LCMD_SIZE 64 70 #define WQE_SCMD_SIZE 64 71 72 #define COMPLETE_LEN 3 73 74 #define CMDQ_TIMEOUT 1000 75 76 #define CMDQ_PFN(addr, page_size) ((addr) >> (ilog2(page_size))) 77 78 #define cmdq_to_cmdqs(cmdq) container_of((cmdq) - (cmdq)->cmdq_type, \ 79 struct hinic_cmdqs, cmdq[0]) 80 81 #define cmdqs_to_func_to_io(cmdqs) container_of(cmdqs, \ 82 struct hinic_func_to_io, \ 83 cmdqs) 84 85 enum completion_format { 86 COMPLETE_DIRECT = 0, 87 COMPLETE_SGE = 1, 88 }; 89 90 enum data_format { 91 DATA_SGE = 0, 92 DATA_DIRECT = 1, 93 }; 94 95 enum bufdesc_len { 96 BUFDESC_LCMD_LEN = 2, /* 16 bytes - 2(8 byte unit) */ 97 BUFDESC_SCMD_LEN = 3, /* 24 bytes - 3(8 byte unit) */ 98 }; 99 100 enum ctrl_sect_len { 101 CTRL_SECT_LEN = 1, /* 4 bytes (ctrl) - 1(8 byte unit) */ 102 CTRL_DIRECT_SECT_LEN = 2, /* 12 bytes (ctrl + rsvd) - 2(8 byte unit) */ 103 }; 104 105 enum cmdq_scmd_type { 106 CMDQ_SET_ARM_CMD = 2, 107 }; 108 109 enum cmdq_cmd_type { 110 CMDQ_CMD_SYNC_DIRECT_RESP = 0, 111 CMDQ_CMD_SYNC_SGE_RESP = 1, 112 }; 113 114 enum completion_request { 115 NO_CEQ = 0, 116 CEQ_SET = 1, 117 }; 118 119 /** 120 * hinic_alloc_cmdq_buf - alloc buffer for sending command 121 * @cmdqs: the cmdqs 122 * @cmdq_buf: the buffer returned in this struct 123 * 124 * Return 0 - Success, negative - Failure 125 **/ 126 int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs, 127 struct hinic_cmdq_buf *cmdq_buf) 128 { 129 struct hinic_hwif *hwif = cmdqs->hwif; 130 struct pci_dev *pdev = hwif->pdev; 131 132 cmdq_buf->buf = dma_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL, 133 &cmdq_buf->dma_addr); 134 if (!cmdq_buf->buf) { 135 dev_err(&pdev->dev, "Failed to allocate cmd from the pool\n"); 136 return -ENOMEM; 137 } 138 139 return 0; 140 } 141 142 /** 143 * hinic_free_cmdq_buf - free buffer 144 * @cmdqs: the cmdqs 145 * @cmdq_buf: the buffer to free that is in this struct 146 **/ 147 void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs, 148 struct hinic_cmdq_buf *cmdq_buf) 149 { 150 dma_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr); 151 } 152 153 static unsigned int cmdq_wqe_size_from_bdlen(enum bufdesc_len len) 154 { 155 unsigned int wqe_size = 0; 156 157 switch (len) { 158 case BUFDESC_LCMD_LEN: 159 wqe_size = WQE_LCMD_SIZE; 160 break; 161 case BUFDESC_SCMD_LEN: 162 wqe_size = WQE_SCMD_SIZE; 163 break; 164 } 165 166 return wqe_size; 167 } 168 169 static void cmdq_set_sge_completion(struct hinic_cmdq_completion *completion, 170 struct hinic_cmdq_buf *buf_out) 171 { 172 struct hinic_sge_resp *sge_resp = &completion->sge_resp; 173 174 hinic_set_sge(&sge_resp->sge, buf_out->dma_addr, buf_out->size); 175 } 176 177 static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped, 178 enum hinic_cmd_ack_type ack_type, 179 enum hinic_mod_type mod, u8 cmd, u16 prod_idx, 180 enum completion_format complete_format, 181 enum data_format data_format, 182 enum bufdesc_len buf_len) 183 { 184 struct hinic_cmdq_wqe_lcmd *wqe_lcmd; 185 struct hinic_cmdq_wqe_scmd *wqe_scmd; 186 enum ctrl_sect_len ctrl_len; 187 struct hinic_ctrl *ctrl; 188 u32 saved_data; 189 190 if (data_format == DATA_SGE) { 191 wqe_lcmd = &wqe->wqe_lcmd; 192 193 wqe_lcmd->status.status_info = 0; 194 ctrl = &wqe_lcmd->ctrl; 195 ctrl_len = CTRL_SECT_LEN; 196 } else { 197 wqe_scmd = &wqe->direct_wqe.wqe_scmd; 198 199 wqe_scmd->status.status_info = 0; 200 ctrl = &wqe_scmd->ctrl; 201 ctrl_len = CTRL_DIRECT_SECT_LEN; 202 } 203 204 ctrl->ctrl_info = HINIC_CMDQ_CTRL_SET(prod_idx, PI) | 205 HINIC_CMDQ_CTRL_SET(cmd, CMD) | 206 HINIC_CMDQ_CTRL_SET(mod, MOD) | 207 HINIC_CMDQ_CTRL_SET(ack_type, ACK_TYPE); 208 209 CMDQ_WQE_HEADER(wqe)->header_info = 210 HINIC_CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN) | 211 HINIC_CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT) | 212 HINIC_CMDQ_WQE_HEADER_SET(data_format, DATA_FMT) | 213 HINIC_CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ) | 214 HINIC_CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) | 215 HINIC_CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN) | 216 HINIC_CMDQ_WQE_HEADER_SET(wrapped, TOGGLED_WRAPPED); 217 218 saved_data = CMDQ_WQE_HEADER(wqe)->saved_data; 219 saved_data = HINIC_SAVED_DATA_CLEAR(saved_data, ARM); 220 221 if (cmd == CMDQ_SET_ARM_CMD && mod == HINIC_MOD_COMM) 222 CMDQ_WQE_HEADER(wqe)->saved_data |= 223 HINIC_SAVED_DATA_SET(1, ARM); 224 else 225 CMDQ_WQE_HEADER(wqe)->saved_data = saved_data; 226 } 227 228 static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe_lcmd, 229 struct hinic_cmdq_buf *buf_in) 230 { 231 hinic_set_sge(&wqe_lcmd->buf_desc.sge, buf_in->dma_addr, buf_in->size); 232 } 233 234 static void cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe *wqe, 235 void *buf_in, u32 in_size) 236 { 237 struct hinic_cmdq_wqe_scmd *wqe_scmd = &wqe->wqe_scmd; 238 239 wqe_scmd->buf_desc.buf_len = in_size; 240 memcpy(wqe_scmd->buf_desc.data, buf_in, in_size); 241 } 242 243 static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe, 244 enum cmdq_cmd_type cmd_type, 245 struct hinic_cmdq_buf *buf_in, 246 struct hinic_cmdq_buf *buf_out, int wrapped, 247 enum hinic_cmd_ack_type ack_type, 248 enum hinic_mod_type mod, u8 cmd, u16 prod_idx) 249 { 250 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; 251 enum completion_format complete_format; 252 253 switch (cmd_type) { 254 case CMDQ_CMD_SYNC_SGE_RESP: 255 complete_format = COMPLETE_SGE; 256 cmdq_set_sge_completion(&wqe_lcmd->completion, buf_out); 257 break; 258 case CMDQ_CMD_SYNC_DIRECT_RESP: 259 complete_format = COMPLETE_DIRECT; 260 wqe_lcmd->completion.direct_resp = 0; 261 break; 262 } 263 264 cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, 265 prod_idx, complete_format, DATA_SGE, 266 BUFDESC_LCMD_LEN); 267 268 cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in); 269 } 270 271 static void cmdq_set_direct_wqe(struct hinic_cmdq_wqe *wqe, 272 enum cmdq_cmd_type cmd_type, 273 void *buf_in, u16 in_size, 274 struct hinic_cmdq_buf *buf_out, int wrapped, 275 enum hinic_cmd_ack_type ack_type, 276 enum hinic_mod_type mod, u8 cmd, u16 prod_idx) 277 { 278 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; 279 enum completion_format complete_format; 280 struct hinic_cmdq_wqe_scmd *wqe_scmd; 281 282 wqe_scmd = &direct_wqe->wqe_scmd; 283 284 switch (cmd_type) { 285 case CMDQ_CMD_SYNC_SGE_RESP: 286 complete_format = COMPLETE_SGE; 287 cmdq_set_sge_completion(&wqe_scmd->completion, buf_out); 288 break; 289 case CMDQ_CMD_SYNC_DIRECT_RESP: 290 complete_format = COMPLETE_DIRECT; 291 wqe_scmd->completion.direct_resp = 0; 292 break; 293 } 294 295 cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, prod_idx, 296 complete_format, DATA_DIRECT, BUFDESC_SCMD_LEN); 297 298 cmdq_set_direct_wqe_data(direct_wqe, buf_in, in_size); 299 } 300 301 static void cmdq_wqe_fill(void *dst, void *src) 302 { 303 memcpy(dst + FIRST_DATA_TO_WRITE_LAST, src + FIRST_DATA_TO_WRITE_LAST, 304 CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST); 305 306 wmb(); /* The first 8 bytes should be written last */ 307 308 *(u64 *)dst = *(u64 *)src; 309 } 310 311 static void cmdq_fill_db(u32 *db_info, 312 enum hinic_cmdq_type cmdq_type, u16 prod_idx) 313 { 314 *db_info = HINIC_CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) | 315 HINIC_CMDQ_DB_INFO_SET(HINIC_CTRL_PATH, PATH) | 316 HINIC_CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE) | 317 HINIC_CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, DB_TYPE); 318 } 319 320 static void cmdq_set_db(struct hinic_cmdq *cmdq, 321 enum hinic_cmdq_type cmdq_type, u16 prod_idx) 322 { 323 u32 db_info; 324 325 cmdq_fill_db(&db_info, cmdq_type, prod_idx); 326 327 /* The data that is written to HW should be in Big Endian Format */ 328 db_info = cpu_to_be32(db_info); 329 330 wmb(); /* write all before the doorbell */ 331 332 writel(db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx)); 333 } 334 335 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq, 336 enum hinic_mod_type mod, u8 cmd, 337 struct hinic_cmdq_buf *buf_in, 338 u64 *resp) 339 { 340 struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe; 341 u16 curr_prod_idx, next_prod_idx; 342 int errcode, wrapped, num_wqebbs; 343 struct hinic_wq *wq = cmdq->wq; 344 struct hinic_hw_wqe *hw_wqe; 345 struct completion done; 346 347 /* Keep doorbell index correct. bh - for tasklet(ceq). */ 348 spin_lock_bh(&cmdq->cmdq_lock); 349 350 /* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/ 351 hw_wqe = hinic_get_wqe(wq, WQE_LCMD_SIZE, &curr_prod_idx); 352 if (IS_ERR(hw_wqe)) { 353 spin_unlock_bh(&cmdq->cmdq_lock); 354 return -EBUSY; 355 } 356 357 curr_cmdq_wqe = &hw_wqe->cmdq_wqe; 358 359 wrapped = cmdq->wrapped; 360 361 num_wqebbs = ALIGN(WQE_LCMD_SIZE, wq->wqebb_size) / wq->wqebb_size; 362 next_prod_idx = curr_prod_idx + num_wqebbs; 363 if (next_prod_idx >= wq->q_depth) { 364 cmdq->wrapped = !cmdq->wrapped; 365 next_prod_idx -= wq->q_depth; 366 } 367 368 cmdq->errcode[curr_prod_idx] = &errcode; 369 370 init_completion(&done); 371 cmdq->done[curr_prod_idx] = &done; 372 373 cmdq_set_lcmd_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, NULL, 374 wrapped, HINIC_CMD_ACK_TYPE_CMDQ, mod, cmd, 375 curr_prod_idx); 376 377 /* The data that is written to HW should be in Big Endian Format */ 378 hinic_cpu_to_be32(&cmdq_wqe, WQE_LCMD_SIZE); 379 380 /* CMDQ WQE is not shadow, therefore wqe will be written to wq */ 381 cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe); 382 383 cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx); 384 385 spin_unlock_bh(&cmdq->cmdq_lock); 386 387 if (!wait_for_completion_timeout(&done, 388 msecs_to_jiffies(CMDQ_TIMEOUT))) { 389 spin_lock_bh(&cmdq->cmdq_lock); 390 391 if (cmdq->errcode[curr_prod_idx] == &errcode) 392 cmdq->errcode[curr_prod_idx] = NULL; 393 394 if (cmdq->done[curr_prod_idx] == &done) 395 cmdq->done[curr_prod_idx] = NULL; 396 397 spin_unlock_bh(&cmdq->cmdq_lock); 398 399 hinic_dump_ceq_info(cmdq->hwdev); 400 return -ETIMEDOUT; 401 } 402 403 smp_rmb(); /* read error code after completion */ 404 405 if (resp) { 406 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &curr_cmdq_wqe->wqe_lcmd; 407 408 *resp = cpu_to_be64(wqe_lcmd->completion.direct_resp); 409 } 410 411 if (errcode != 0) 412 return -EFAULT; 413 414 return 0; 415 } 416 417 static int cmdq_set_arm_bit(struct hinic_cmdq *cmdq, void *buf_in, 418 u16 in_size) 419 { 420 struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe; 421 u16 curr_prod_idx, next_prod_idx; 422 struct hinic_wq *wq = cmdq->wq; 423 struct hinic_hw_wqe *hw_wqe; 424 int wrapped, num_wqebbs; 425 426 /* Keep doorbell index correct */ 427 spin_lock(&cmdq->cmdq_lock); 428 429 /* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/ 430 hw_wqe = hinic_get_wqe(wq, WQE_SCMD_SIZE, &curr_prod_idx); 431 if (IS_ERR(hw_wqe)) { 432 spin_unlock(&cmdq->cmdq_lock); 433 return -EBUSY; 434 } 435 436 curr_cmdq_wqe = &hw_wqe->cmdq_wqe; 437 438 wrapped = cmdq->wrapped; 439 440 num_wqebbs = ALIGN(WQE_SCMD_SIZE, wq->wqebb_size) / wq->wqebb_size; 441 next_prod_idx = curr_prod_idx + num_wqebbs; 442 if (next_prod_idx >= wq->q_depth) { 443 cmdq->wrapped = !cmdq->wrapped; 444 next_prod_idx -= wq->q_depth; 445 } 446 447 cmdq_set_direct_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, 448 in_size, NULL, wrapped, HINIC_CMD_ACK_TYPE_CMDQ, 449 HINIC_MOD_COMM, CMDQ_SET_ARM_CMD, curr_prod_idx); 450 451 /* The data that is written to HW should be in Big Endian Format */ 452 hinic_cpu_to_be32(&cmdq_wqe, WQE_SCMD_SIZE); 453 454 /* cmdq wqe is not shadow, therefore wqe will be written to wq */ 455 cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe); 456 457 cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx); 458 459 spin_unlock(&cmdq->cmdq_lock); 460 return 0; 461 } 462 463 static int cmdq_params_valid(struct hinic_cmdq_buf *buf_in) 464 { 465 if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE) 466 return -EINVAL; 467 468 return 0; 469 } 470 471 /** 472 * hinic_cmdq_direct_resp - send command with direct data as resp 473 * @cmdqs: the cmdqs 474 * @mod: module on the card that will handle the command 475 * @cmd: the command 476 * @buf_in: the buffer for the command 477 * @resp: the response to return 478 * 479 * Return 0 - Success, negative - Failure 480 **/ 481 int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs, 482 enum hinic_mod_type mod, u8 cmd, 483 struct hinic_cmdq_buf *buf_in, u64 *resp) 484 { 485 struct hinic_hwif *hwif = cmdqs->hwif; 486 struct pci_dev *pdev = hwif->pdev; 487 int err; 488 489 err = cmdq_params_valid(buf_in); 490 if (err) { 491 dev_err(&pdev->dev, "Invalid CMDQ parameters\n"); 492 return err; 493 } 494 495 return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC], 496 mod, cmd, buf_in, resp); 497 } 498 499 /** 500 * hinic_set_arm_bit - set arm bit for enable interrupt again 501 * @cmdqs: the cmdqs 502 * @q_type: type of queue to set the arm bit for 503 * @q_id: the queue number 504 * 505 * Return 0 - Success, negative - Failure 506 **/ 507 static int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs, 508 enum hinic_set_arm_qtype q_type, u32 q_id) 509 { 510 struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC]; 511 struct hinic_hwif *hwif = cmdqs->hwif; 512 struct pci_dev *pdev = hwif->pdev; 513 struct hinic_cmdq_arm_bit arm_bit; 514 int err; 515 516 arm_bit.q_type = q_type; 517 arm_bit.q_id = q_id; 518 519 err = cmdq_set_arm_bit(cmdq, &arm_bit, sizeof(arm_bit)); 520 if (err) { 521 dev_err(&pdev->dev, "Failed to set arm for qid %d\n", q_id); 522 return err; 523 } 524 525 return 0; 526 } 527 528 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq, 529 struct hinic_cmdq_wqe *wqe) 530 { 531 u32 header_info = be32_to_cpu(CMDQ_WQE_HEADER(wqe)->header_info); 532 unsigned int bufdesc_len, wqe_size; 533 struct hinic_ctrl *ctrl; 534 535 bufdesc_len = HINIC_CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN); 536 wqe_size = cmdq_wqe_size_from_bdlen(bufdesc_len); 537 if (wqe_size == WQE_LCMD_SIZE) { 538 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; 539 540 ctrl = &wqe_lcmd->ctrl; 541 } else { 542 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; 543 struct hinic_cmdq_wqe_scmd *wqe_scmd; 544 545 wqe_scmd = &direct_wqe->wqe_scmd; 546 ctrl = &wqe_scmd->ctrl; 547 } 548 549 /* clear HW busy bit */ 550 ctrl->ctrl_info = 0; 551 552 wmb(); /* verify wqe is clear */ 553 } 554 555 /** 556 * cmdq_arm_ceq_handler - cmdq completion event handler for arm command 557 * @cmdq: the cmdq of the arm command 558 * @wqe: the wqe of the arm command 559 * 560 * Return 0 - Success, negative - Failure 561 **/ 562 static int cmdq_arm_ceq_handler(struct hinic_cmdq *cmdq, 563 struct hinic_cmdq_wqe *wqe) 564 { 565 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; 566 struct hinic_cmdq_wqe_scmd *wqe_scmd; 567 struct hinic_ctrl *ctrl; 568 u32 ctrl_info; 569 570 wqe_scmd = &direct_wqe->wqe_scmd; 571 ctrl = &wqe_scmd->ctrl; 572 ctrl_info = be32_to_cpu(ctrl->ctrl_info); 573 574 /* HW should toggle the HW BUSY BIT */ 575 if (!CMDQ_WQE_COMPLETED(ctrl_info)) 576 return -EBUSY; 577 578 clear_wqe_complete_bit(cmdq, wqe); 579 580 hinic_put_wqe(cmdq->wq, WQE_SCMD_SIZE); 581 return 0; 582 } 583 584 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx, 585 int errcode) 586 { 587 if (cmdq->errcode[prod_idx]) 588 *cmdq->errcode[prod_idx] = errcode; 589 } 590 591 /** 592 * cmdq_sync_cmd_handler - cmdq completion event handler for sync command 593 * @cmdq: the cmdq of the command 594 * @cons_idx: the consumer index to update the error code for 595 * @errcode: the error code 596 **/ 597 static void cmdq_sync_cmd_handler(struct hinic_cmdq *cmdq, u16 cons_idx, 598 int errcode) 599 { 600 u16 prod_idx = cons_idx; 601 602 spin_lock(&cmdq->cmdq_lock); 603 cmdq_update_errcode(cmdq, prod_idx, errcode); 604 605 wmb(); /* write all before update for the command request */ 606 607 if (cmdq->done[prod_idx]) 608 complete(cmdq->done[prod_idx]); 609 spin_unlock(&cmdq->cmdq_lock); 610 } 611 612 static int cmdq_cmd_ceq_handler(struct hinic_cmdq *cmdq, u16 ci, 613 struct hinic_cmdq_wqe *cmdq_wqe) 614 { 615 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &cmdq_wqe->wqe_lcmd; 616 struct hinic_status *status = &wqe_lcmd->status; 617 struct hinic_ctrl *ctrl = &wqe_lcmd->ctrl; 618 int errcode; 619 620 if (!CMDQ_WQE_COMPLETED(be32_to_cpu(ctrl->ctrl_info))) 621 return -EBUSY; 622 623 dma_rmb(); 624 625 errcode = CMDQ_WQE_ERRCODE_GET(be32_to_cpu(status->status_info), VAL); 626 627 cmdq_sync_cmd_handler(cmdq, ci, errcode); 628 629 clear_wqe_complete_bit(cmdq, cmdq_wqe); 630 hinic_put_wqe(cmdq->wq, WQE_LCMD_SIZE); 631 return 0; 632 } 633 634 /** 635 * cmdq_ceq_handler - cmdq completion event handler 636 * @handle: private data for the handler(cmdqs) 637 * @ceqe_data: ceq element data 638 **/ 639 static void cmdq_ceq_handler(void *handle, u32 ceqe_data) 640 { 641 enum hinic_cmdq_type cmdq_type = CMDQ_CEQE_GET(ceqe_data, TYPE); 642 struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)handle; 643 struct hinic_cmdq *cmdq = &cmdqs->cmdq[cmdq_type]; 644 struct hinic_cmdq_header *header; 645 struct hinic_hw_wqe *hw_wqe; 646 int err, set_arm = 0; 647 u32 saved_data; 648 u16 ci; 649 650 /* Read the smallest wqe size for getting wqe size */ 651 while ((hw_wqe = hinic_read_wqe(cmdq->wq, WQE_SCMD_SIZE, &ci))) { 652 if (IS_ERR(hw_wqe)) 653 break; 654 655 header = CMDQ_WQE_HEADER(&hw_wqe->cmdq_wqe); 656 saved_data = be32_to_cpu(header->saved_data); 657 658 if (HINIC_SAVED_DATA_GET(saved_data, ARM)) { 659 /* arm_bit was set until here */ 660 set_arm = 0; 661 662 if (cmdq_arm_ceq_handler(cmdq, &hw_wqe->cmdq_wqe)) 663 break; 664 } else { 665 set_arm = 1; 666 667 hw_wqe = hinic_read_wqe(cmdq->wq, WQE_LCMD_SIZE, &ci); 668 if (IS_ERR(hw_wqe)) 669 break; 670 671 if (cmdq_cmd_ceq_handler(cmdq, ci, &hw_wqe->cmdq_wqe)) 672 break; 673 } 674 } 675 676 if (set_arm) { 677 struct hinic_hwif *hwif = cmdqs->hwif; 678 struct pci_dev *pdev = hwif->pdev; 679 680 err = hinic_set_arm_bit(cmdqs, HINIC_SET_ARM_CMDQ, cmdq_type); 681 if (err) 682 dev_err(&pdev->dev, "Failed to set arm for CMDQ\n"); 683 } 684 } 685 686 /** 687 * cmdq_init_queue_ctxt - init the queue ctxt of a cmdq 688 * @cmdq_ctxt: cmdq ctxt to initialize 689 * @cmdq: the cmdq 690 * @cmdq_pages: the memory of the queue 691 **/ 692 static void cmdq_init_queue_ctxt(struct hinic_cmdq_ctxt *cmdq_ctxt, 693 struct hinic_cmdq *cmdq, 694 struct hinic_cmdq_pages *cmdq_pages) 695 { 696 struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info; 697 u64 wq_first_page_paddr, cmdq_first_block_paddr, pfn; 698 struct hinic_cmdqs *cmdqs = cmdq_to_cmdqs(cmdq); 699 struct hinic_wq *wq = cmdq->wq; 700 701 /* The data in the HW is in Big Endian Format */ 702 wq_first_page_paddr = be64_to_cpu(*wq->block_vaddr); 703 704 pfn = CMDQ_PFN(wq_first_page_paddr, SZ_4K); 705 706 ctxt_info->curr_wqe_page_pfn = 707 HINIC_CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN) | 708 HINIC_CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) | 709 HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_ARM) | 710 HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN) | 711 HINIC_CMDQ_CTXT_PAGE_INFO_SET(cmdq->wrapped, WRAPPED); 712 713 if (wq->num_q_pages != 1) { 714 /* block PFN - Read Modify Write */ 715 cmdq_first_block_paddr = cmdq_pages->page_paddr; 716 717 pfn = CMDQ_PFN(cmdq_first_block_paddr, wq->wq_page_size); 718 } 719 720 ctxt_info->wq_block_pfn = 721 HINIC_CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN) | 722 HINIC_CMDQ_CTXT_BLOCK_INFO_SET(atomic_read(&wq->cons_idx), CI); 723 724 cmdq_ctxt->func_idx = HINIC_HWIF_FUNC_IDX(cmdqs->hwif); 725 cmdq_ctxt->ppf_idx = HINIC_HWIF_PPF_IDX(cmdqs->hwif); 726 cmdq_ctxt->cmdq_type = cmdq->cmdq_type; 727 } 728 729 /** 730 * init_cmdq - initialize cmdq 731 * @cmdq: the cmdq 732 * @wq: the wq attaced to the cmdq 733 * @q_type: the cmdq type of the cmdq 734 * @db_area: doorbell area for the cmdq 735 * 736 * Return 0 - Success, negative - Failure 737 **/ 738 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_wq *wq, 739 enum hinic_cmdq_type q_type, void __iomem *db_area) 740 { 741 int err; 742 743 cmdq->wq = wq; 744 cmdq->cmdq_type = q_type; 745 cmdq->wrapped = 1; 746 747 spin_lock_init(&cmdq->cmdq_lock); 748 749 cmdq->done = vzalloc(array_size(sizeof(*cmdq->done), wq->q_depth)); 750 if (!cmdq->done) 751 return -ENOMEM; 752 753 cmdq->errcode = vzalloc(array_size(sizeof(*cmdq->errcode), 754 wq->q_depth)); 755 if (!cmdq->errcode) { 756 err = -ENOMEM; 757 goto err_errcode; 758 } 759 760 cmdq->db_base = db_area + CMDQ_DB_OFF; 761 return 0; 762 763 err_errcode: 764 vfree(cmdq->done); 765 return err; 766 } 767 768 /** 769 * free_cmdq - Free cmdq 770 * @cmdq: the cmdq to free 771 **/ 772 static void free_cmdq(struct hinic_cmdq *cmdq) 773 { 774 vfree(cmdq->errcode); 775 vfree(cmdq->done); 776 } 777 778 /** 779 * init_cmdqs_ctxt - write the cmdq ctxt to HW after init all cmdq 780 * @hwdev: the NIC HW device 781 * @cmdqs: cmdqs to write the ctxts for 782 * @db_area: db_area for all the cmdqs 783 * 784 * Return 0 - Success, negative - Failure 785 **/ 786 static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev, 787 struct hinic_cmdqs *cmdqs, void __iomem **db_area) 788 { 789 struct hinic_hwif *hwif = hwdev->hwif; 790 enum hinic_cmdq_type type, cmdq_type; 791 struct hinic_cmdq_ctxt *cmdq_ctxts; 792 struct pci_dev *pdev = hwif->pdev; 793 struct hinic_pfhwdev *pfhwdev; 794 int err; 795 796 cmdq_ctxts = devm_kcalloc(&pdev->dev, HINIC_MAX_CMDQ_TYPES, 797 sizeof(*cmdq_ctxts), GFP_KERNEL); 798 if (!cmdq_ctxts) 799 return -ENOMEM; 800 801 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 802 803 cmdq_type = HINIC_CMDQ_SYNC; 804 for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) { 805 cmdqs->cmdq[cmdq_type].hwdev = hwdev; 806 err = init_cmdq(&cmdqs->cmdq[cmdq_type], 807 &cmdqs->saved_wqs[cmdq_type], cmdq_type, 808 db_area[cmdq_type]); 809 if (err) { 810 dev_err(&pdev->dev, "Failed to initialize cmdq\n"); 811 goto err_init_cmdq; 812 } 813 814 cmdq_init_queue_ctxt(&cmdq_ctxts[cmdq_type], 815 &cmdqs->cmdq[cmdq_type], 816 &cmdqs->cmdq_pages); 817 } 818 819 /* Write the CMDQ ctxts */ 820 cmdq_type = HINIC_CMDQ_SYNC; 821 for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) { 822 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 823 HINIC_COMM_CMD_CMDQ_CTXT_SET, 824 &cmdq_ctxts[cmdq_type], 825 sizeof(cmdq_ctxts[cmdq_type]), 826 NULL, NULL, HINIC_MGMT_MSG_SYNC); 827 if (err) { 828 dev_err(&pdev->dev, "Failed to set CMDQ CTXT type = %d\n", 829 cmdq_type); 830 goto err_write_cmdq_ctxt; 831 } 832 } 833 834 devm_kfree(&pdev->dev, cmdq_ctxts); 835 return 0; 836 837 err_write_cmdq_ctxt: 838 cmdq_type = HINIC_MAX_CMDQ_TYPES; 839 840 err_init_cmdq: 841 for (type = HINIC_CMDQ_SYNC; type < cmdq_type; type++) 842 free_cmdq(&cmdqs->cmdq[type]); 843 844 devm_kfree(&pdev->dev, cmdq_ctxts); 845 return err; 846 } 847 848 static int hinic_set_cmdq_depth(struct hinic_hwdev *hwdev, u16 cmdq_depth) 849 { 850 struct hinic_cmd_hw_ioctxt hw_ioctxt = { 0 }; 851 struct hinic_pfhwdev *pfhwdev; 852 853 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 854 855 hw_ioctxt.func_idx = HINIC_HWIF_FUNC_IDX(hwdev->hwif); 856 hw_ioctxt.ppf_idx = HINIC_HWIF_PPF_IDX(hwdev->hwif); 857 858 hw_ioctxt.set_cmdq_depth = HW_IOCTXT_SET_CMDQ_DEPTH_ENABLE; 859 hw_ioctxt.cmdq_depth = (u8)ilog2(cmdq_depth); 860 861 return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 862 HINIC_COMM_CMD_HWCTXT_SET, 863 &hw_ioctxt, sizeof(hw_ioctxt), NULL, 864 NULL, HINIC_MGMT_MSG_SYNC); 865 } 866 867 /** 868 * hinic_init_cmdqs - init all cmdqs 869 * @cmdqs: cmdqs to init 870 * @hwif: HW interface for accessing cmdqs 871 * @db_area: doorbell areas for all the cmdqs 872 * 873 * Return 0 - Success, negative - Failure 874 **/ 875 int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif, 876 void __iomem **db_area) 877 { 878 struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs); 879 struct pci_dev *pdev = hwif->pdev; 880 struct hinic_hwdev *hwdev; 881 u16 max_wqe_size; 882 int err; 883 884 cmdqs->hwif = hwif; 885 cmdqs->cmdq_buf_pool = dma_pool_create("hinic_cmdq", &pdev->dev, 886 HINIC_CMDQ_BUF_SIZE, 887 HINIC_CMDQ_BUF_SIZE, 0); 888 if (!cmdqs->cmdq_buf_pool) 889 return -ENOMEM; 890 891 cmdqs->saved_wqs = devm_kcalloc(&pdev->dev, HINIC_MAX_CMDQ_TYPES, 892 sizeof(*cmdqs->saved_wqs), GFP_KERNEL); 893 if (!cmdqs->saved_wqs) { 894 err = -ENOMEM; 895 goto err_saved_wqs; 896 } 897 898 max_wqe_size = WQE_LCMD_SIZE; 899 err = hinic_wqs_cmdq_alloc(&cmdqs->cmdq_pages, cmdqs->saved_wqs, hwif, 900 HINIC_MAX_CMDQ_TYPES, CMDQ_WQEBB_SIZE, 901 CMDQ_WQ_PAGE_SIZE, CMDQ_DEPTH, max_wqe_size); 902 if (err) { 903 dev_err(&pdev->dev, "Failed to allocate CMDQ wqs\n"); 904 goto err_cmdq_wqs; 905 } 906 907 hwdev = container_of(func_to_io, struct hinic_hwdev, func_to_io); 908 err = init_cmdqs_ctxt(hwdev, cmdqs, db_area); 909 if (err) { 910 dev_err(&pdev->dev, "Failed to write cmdq ctxt\n"); 911 goto err_cmdq_ctxt; 912 } 913 914 hinic_ceq_register_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ, cmdqs, 915 cmdq_ceq_handler); 916 917 err = hinic_set_cmdq_depth(hwdev, CMDQ_DEPTH); 918 if (err) { 919 dev_err(&hwif->pdev->dev, "Failed to set cmdq depth\n"); 920 goto err_set_cmdq_depth; 921 } 922 923 return 0; 924 925 err_set_cmdq_depth: 926 hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ); 927 928 err_cmdq_ctxt: 929 hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs, 930 HINIC_MAX_CMDQ_TYPES); 931 932 err_cmdq_wqs: 933 devm_kfree(&pdev->dev, cmdqs->saved_wqs); 934 935 err_saved_wqs: 936 dma_pool_destroy(cmdqs->cmdq_buf_pool); 937 return err; 938 } 939 940 /** 941 * hinic_free_cmdqs - free all cmdqs 942 * @cmdqs: cmdqs to free 943 **/ 944 void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs) 945 { 946 struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs); 947 struct hinic_hwif *hwif = cmdqs->hwif; 948 struct pci_dev *pdev = hwif->pdev; 949 enum hinic_cmdq_type cmdq_type; 950 951 hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ); 952 953 cmdq_type = HINIC_CMDQ_SYNC; 954 for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) 955 free_cmdq(&cmdqs->cmdq[cmdq_type]); 956 957 hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs, 958 HINIC_MAX_CMDQ_TYPES); 959 960 devm_kfree(&pdev->dev, cmdqs->saved_wqs); 961 962 dma_pool_destroy(cmdqs->cmdq_buf_pool); 963 } 964