1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/errno.h> 10 #include <linux/pci.h> 11 #include <linux/device.h> 12 #include <linux/slab.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/bitops.h> 15 #include <linux/err.h> 16 #include <linux/jiffies.h> 17 #include <linux/delay.h> 18 #include <linux/log2.h> 19 #include <linux/semaphore.h> 20 #include <asm/byteorder.h> 21 #include <asm/barrier.h> 22 23 #include "hinic_hw_csr.h" 24 #include "hinic_hw_if.h" 25 #include "hinic_hw_api_cmd.h" 26 27 #define API_CHAIN_NUM_CELLS 32 28 29 #define API_CMD_CELL_SIZE_SHIFT 6 30 #define API_CMD_CELL_SIZE_MIN (BIT(API_CMD_CELL_SIZE_SHIFT)) 31 32 #define API_CMD_CELL_SIZE(cell_size) \ 33 (((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \ 34 (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN) 35 36 #define API_CMD_CELL_SIZE_VAL(size) \ 37 ilog2((size) >> API_CMD_CELL_SIZE_SHIFT) 38 39 #define API_CMD_BUF_SIZE 2048 40 41 /* Sizes of the members in hinic_api_cmd_cell */ 42 #define API_CMD_CELL_DESC_SIZE 8 43 #define API_CMD_CELL_DATA_ADDR_SIZE 8 44 45 #define API_CMD_CELL_ALIGNMENT 8 46 47 #define API_CMD_TIMEOUT 1000 48 49 #define MASKED_IDX(chain, idx) ((idx) & ((chain)->num_cells - 1)) 50 51 #define SIZE_8BYTES(size) (ALIGN((size), 8) >> 3) 52 #define SIZE_4BYTES(size) (ALIGN((size), 4) >> 2) 53 54 #define RD_DMA_ATTR_DEFAULT 0 55 #define WR_DMA_ATTR_DEFAULT 0 56 57 enum api_cmd_data_format { 58 SGE_DATA = 1, /* cell data is passed by hw address */ 59 }; 60 61 enum api_cmd_type { 62 API_CMD_WRITE = 0, 63 }; 64 65 enum api_cmd_bypass { 66 NO_BYPASS = 0, 67 BYPASS = 1, 68 }; 69 70 enum api_cmd_xor_chk_level { 71 XOR_CHK_DIS = 0, 72 73 XOR_CHK_ALL = 3, 74 }; 75 76 static u8 xor_chksum_set(void *data) 77 { 78 int idx; 79 u8 *val, checksum = 0; 80 81 val = data; 82 83 for (idx = 0; idx < 7; idx++) 84 checksum ^= val[idx]; 85 86 return checksum; 87 } 88 89 static void set_prod_idx(struct hinic_api_cmd_chain *chain) 90 { 91 enum hinic_api_cmd_chain_type chain_type = chain->chain_type; 92 struct hinic_hwif *hwif = chain->hwif; 93 u32 addr, prod_idx; 94 95 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type); 96 prod_idx = hinic_hwif_read_reg(hwif, addr); 97 98 prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX); 99 100 prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX); 101 102 hinic_hwif_write_reg(hwif, addr, prod_idx); 103 } 104 105 static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain) 106 { 107 u32 addr, val; 108 109 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type); 110 val = hinic_hwif_read_reg(chain->hwif, addr); 111 112 return HINIC_API_CMD_STATUS_GET(val, CONS_IDX); 113 } 114 115 /** 116 * chain_busy - check if the chain is still processing last requests 117 * @chain: chain to check 118 * 119 * Return 0 - Success, negative - Failure 120 **/ 121 static int chain_busy(struct hinic_api_cmd_chain *chain) 122 { 123 struct hinic_hwif *hwif = chain->hwif; 124 struct pci_dev *pdev = hwif->pdev; 125 u32 prod_idx; 126 127 switch (chain->chain_type) { 128 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 129 chain->cons_idx = get_hw_cons_idx(chain); 130 prod_idx = chain->prod_idx; 131 132 /* check for a space for a new command */ 133 if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) { 134 dev_err(&pdev->dev, "API CMD chain %d is busy\n", 135 chain->chain_type); 136 return -EBUSY; 137 } 138 break; 139 140 default: 141 dev_err(&pdev->dev, "Unknown API CMD Chain type\n"); 142 break; 143 } 144 145 return 0; 146 } 147 148 /** 149 * get_cell_data_size - get the data size of a specific cell type 150 * @type: chain type 151 * 152 * Return the data(Desc + Address) size in the cell 153 **/ 154 static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type) 155 { 156 u8 cell_data_size = 0; 157 158 switch (type) { 159 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 160 cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE + 161 API_CMD_CELL_DATA_ADDR_SIZE, 162 API_CMD_CELL_ALIGNMENT); 163 break; 164 default: 165 break; 166 } 167 168 return cell_data_size; 169 } 170 171 /** 172 * prepare_cell_ctrl - prepare the ctrl of the cell for the command 173 * @cell_ctrl: the control of the cell to set the control value into it 174 * @data_size: the size of the data in the cell 175 **/ 176 static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size) 177 { 178 u8 chksum; 179 u64 ctrl; 180 181 ctrl = HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ) | 182 HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) | 183 HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR); 184 185 chksum = xor_chksum_set(&ctrl); 186 187 ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM); 188 189 /* The data in the HW should be in Big Endian Format */ 190 *cell_ctrl = cpu_to_be64(ctrl); 191 } 192 193 /** 194 * prepare_api_cmd - prepare API CMD command 195 * @chain: chain for the command 196 * @dest: destination node on the card that will receive the command 197 * @cmd: command data 198 * @cmd_size: the command size 199 **/ 200 static void prepare_api_cmd(struct hinic_api_cmd_chain *chain, 201 enum hinic_node_id dest, 202 void *cmd, u16 cmd_size) 203 { 204 struct hinic_api_cmd_cell *cell = chain->curr_node; 205 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 206 struct hinic_hwif *hwif = chain->hwif; 207 struct pci_dev *pdev = hwif->pdev; 208 209 cell_ctxt = &chain->cell_ctxt[chain->prod_idx]; 210 211 switch (chain->chain_type) { 212 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 213 cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE) | 214 HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) | 215 HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS); 216 break; 217 218 default: 219 dev_err(&pdev->dev, "unknown Chain type\n"); 220 return; 221 } 222 223 cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST) | 224 HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE); 225 226 cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc), 227 XOR_CHKSUM); 228 229 /* The data in the HW should be in Big Endian Format */ 230 cell->desc = cpu_to_be64(cell->desc); 231 232 memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size); 233 } 234 235 /** 236 * prepare_cell - prepare cell ctrl and cmd in the current cell 237 * @chain: chain for the command 238 * @dest: destination node on the card that will receive the command 239 * @cmd: command data 240 * @cmd_size: the command size 241 * 242 * Return 0 - Success, negative - Failure 243 **/ 244 static void prepare_cell(struct hinic_api_cmd_chain *chain, 245 enum hinic_node_id dest, 246 void *cmd, u16 cmd_size) 247 { 248 struct hinic_api_cmd_cell *curr_node = chain->curr_node; 249 u16 data_size = get_cell_data_size(chain->chain_type); 250 251 prepare_cell_ctrl(&curr_node->ctrl, data_size); 252 prepare_api_cmd(chain, dest, cmd, cmd_size); 253 } 254 255 static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain) 256 { 257 chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1); 258 } 259 260 /** 261 * api_cmd_status_update - update the status in the chain struct 262 * @chain: chain to update 263 **/ 264 static void api_cmd_status_update(struct hinic_api_cmd_chain *chain) 265 { 266 enum hinic_api_cmd_chain_type chain_type; 267 struct hinic_api_cmd_status *wb_status; 268 struct hinic_hwif *hwif = chain->hwif; 269 struct pci_dev *pdev = hwif->pdev; 270 u64 status_header; 271 u32 status; 272 273 wb_status = chain->wb_status; 274 status_header = be64_to_cpu(wb_status->header); 275 276 status = be32_to_cpu(wb_status->status); 277 if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) { 278 dev_err(&pdev->dev, "API CMD status: Xor check error\n"); 279 return; 280 } 281 282 chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID); 283 if (chain_type >= HINIC_API_CMD_MAX) { 284 dev_err(&pdev->dev, "unknown API CMD Chain %d\n", chain_type); 285 return; 286 } 287 288 chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX); 289 } 290 291 /** 292 * wait_for_status_poll - wait for write to api cmd command to complete 293 * @chain: the chain of the command 294 * 295 * Return 0 - Success, negative - Failure 296 **/ 297 static int wait_for_status_poll(struct hinic_api_cmd_chain *chain) 298 { 299 int err = -ETIMEDOUT; 300 unsigned long end; 301 302 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); 303 do { 304 api_cmd_status_update(chain); 305 306 /* wait for CI to be updated - sign for completion */ 307 if (chain->cons_idx == chain->prod_idx) { 308 err = 0; 309 break; 310 } 311 312 msleep(20); 313 } while (time_before(jiffies, end)); 314 315 return err; 316 } 317 318 /** 319 * wait_for_api_cmd_completion - wait for command to complete 320 * @chain: chain for the command 321 * 322 * Return 0 - Success, negative - Failure 323 **/ 324 static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain) 325 { 326 struct hinic_hwif *hwif = chain->hwif; 327 struct pci_dev *pdev = hwif->pdev; 328 int err; 329 330 switch (chain->chain_type) { 331 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 332 err = wait_for_status_poll(chain); 333 if (err) { 334 dev_err(&pdev->dev, "API CMD Poll status timeout\n"); 335 break; 336 } 337 break; 338 339 default: 340 dev_err(&pdev->dev, "unknown API CMD Chain type\n"); 341 err = -EINVAL; 342 break; 343 } 344 345 return err; 346 } 347 348 /** 349 * api_cmd - API CMD command 350 * @chain: chain for the command 351 * @dest: destination node on the card that will receive the command 352 * @cmd: command data 353 * @size: the command size 354 * 355 * Return 0 - Success, negative - Failure 356 **/ 357 static int api_cmd(struct hinic_api_cmd_chain *chain, 358 enum hinic_node_id dest, u8 *cmd, u16 cmd_size) 359 { 360 struct hinic_api_cmd_cell_ctxt *ctxt; 361 int err; 362 363 down(&chain->sem); 364 if (chain_busy(chain)) { 365 up(&chain->sem); 366 return -EBUSY; 367 } 368 369 prepare_cell(chain, dest, cmd, cmd_size); 370 cmd_chain_prod_idx_inc(chain); 371 372 wmb(); /* inc pi before issue the command */ 373 374 set_prod_idx(chain); /* issue the command */ 375 376 ctxt = &chain->cell_ctxt[chain->prod_idx]; 377 378 chain->curr_node = ctxt->cell_vaddr; 379 380 err = wait_for_api_cmd_completion(chain); 381 382 up(&chain->sem); 383 return err; 384 } 385 386 /** 387 * hinic_api_cmd_write - Write API CMD command 388 * @chain: chain for write command 389 * @dest: destination node on the card that will receive the command 390 * @cmd: command data 391 * @size: the command size 392 * 393 * Return 0 - Success, negative - Failure 394 **/ 395 int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain, 396 enum hinic_node_id dest, u8 *cmd, u16 size) 397 { 398 /* Verify the chain type */ 399 if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU) 400 return api_cmd(chain, dest, cmd, size); 401 402 return -EINVAL; 403 } 404 405 /** 406 * api_cmd_hw_restart - restart the chain in the HW 407 * @chain: the API CMD specific chain to restart 408 * 409 * Return 0 - Success, negative - Failure 410 **/ 411 static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain) 412 { 413 struct hinic_hwif *hwif = chain->hwif; 414 int err = -ETIMEDOUT; 415 unsigned long end; 416 u32 reg_addr, val; 417 418 /* Read Modify Write */ 419 reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type); 420 val = hinic_hwif_read_reg(hwif, reg_addr); 421 422 val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART); 423 val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART); 424 425 hinic_hwif_write_reg(hwif, reg_addr, val); 426 427 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); 428 do { 429 val = hinic_hwif_read_reg(hwif, reg_addr); 430 431 if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) { 432 err = 0; 433 break; 434 } 435 436 msleep(20); 437 } while (time_before(jiffies, end)); 438 439 return err; 440 } 441 442 /** 443 * api_cmd_ctrl_init - set the control register of a chain 444 * @chain: the API CMD specific chain to set control register for 445 **/ 446 static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain) 447 { 448 struct hinic_hwif *hwif = chain->hwif; 449 u32 addr, ctrl; 450 u16 cell_size; 451 452 /* Read Modify Write */ 453 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); 454 455 cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size); 456 457 ctrl = hinic_hwif_read_reg(hwif, addr); 458 459 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) & 460 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) & 461 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & 462 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) & 463 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); 464 465 ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR) | 466 HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) | 467 HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE); 468 469 hinic_hwif_write_reg(hwif, addr, ctrl); 470 } 471 472 /** 473 * api_cmd_set_status_addr - set the status address of a chain in the HW 474 * @chain: the API CMD specific chain to set in HW status address for 475 **/ 476 static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain) 477 { 478 struct hinic_hwif *hwif = chain->hwif; 479 u32 addr, val; 480 481 addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type); 482 val = upper_32_bits(chain->wb_status_paddr); 483 hinic_hwif_write_reg(hwif, addr, val); 484 485 addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type); 486 val = lower_32_bits(chain->wb_status_paddr); 487 hinic_hwif_write_reg(hwif, addr, val); 488 } 489 490 /** 491 * api_cmd_set_num_cells - set the number cells of a chain in the HW 492 * @chain: the API CMD specific chain to set in HW the number of cells for 493 **/ 494 static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain) 495 { 496 struct hinic_hwif *hwif = chain->hwif; 497 u32 addr, val; 498 499 addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type); 500 val = chain->num_cells; 501 hinic_hwif_write_reg(hwif, addr, val); 502 } 503 504 /** 505 * api_cmd_head_init - set the head of a chain in the HW 506 * @chain: the API CMD specific chain to set in HW the head for 507 **/ 508 static void api_cmd_head_init(struct hinic_api_cmd_chain *chain) 509 { 510 struct hinic_hwif *hwif = chain->hwif; 511 u32 addr, val; 512 513 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type); 514 val = upper_32_bits(chain->head_cell_paddr); 515 hinic_hwif_write_reg(hwif, addr, val); 516 517 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type); 518 val = lower_32_bits(chain->head_cell_paddr); 519 hinic_hwif_write_reg(hwif, addr, val); 520 } 521 522 /** 523 * api_cmd_chain_hw_clean - clean the HW 524 * @chain: the API CMD specific chain 525 **/ 526 static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain) 527 { 528 struct hinic_hwif *hwif = chain->hwif; 529 u32 addr, ctrl; 530 531 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); 532 533 ctrl = hinic_hwif_read_reg(hwif, addr); 534 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) & 535 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) & 536 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & 537 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) & 538 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); 539 540 hinic_hwif_write_reg(hwif, addr, ctrl); 541 } 542 543 /** 544 * api_cmd_chain_hw_init - initialize the chain in the HW 545 * @chain: the API CMD specific chain to initialize in HW 546 * 547 * Return 0 - Success, negative - Failure 548 **/ 549 static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain) 550 { 551 struct hinic_hwif *hwif = chain->hwif; 552 struct pci_dev *pdev = hwif->pdev; 553 int err; 554 555 api_cmd_chain_hw_clean(chain); 556 557 api_cmd_set_status_addr(chain); 558 559 err = api_cmd_hw_restart(chain); 560 if (err) { 561 dev_err(&pdev->dev, "Failed to restart API CMD HW\n"); 562 return err; 563 } 564 565 api_cmd_ctrl_init(chain); 566 api_cmd_set_num_cells(chain); 567 api_cmd_head_init(chain); 568 return 0; 569 } 570 571 /** 572 * free_cmd_buf - free the dma buffer of API CMD command 573 * @chain: the API CMD specific chain of the cmd 574 * @cell_idx: the cell index of the cmd 575 **/ 576 static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx) 577 { 578 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 579 struct hinic_hwif *hwif = chain->hwif; 580 struct pci_dev *pdev = hwif->pdev; 581 582 cell_ctxt = &chain->cell_ctxt[cell_idx]; 583 584 dma_free_coherent(&pdev->dev, API_CMD_BUF_SIZE, 585 cell_ctxt->api_cmd_vaddr, 586 cell_ctxt->api_cmd_paddr); 587 } 588 589 /** 590 * alloc_cmd_buf - allocate a dma buffer for API CMD command 591 * @chain: the API CMD specific chain for the cmd 592 * @cell: the cell in the HW for the cmd 593 * @cell_idx: the index of the cell 594 * 595 * Return 0 - Success, negative - Failure 596 **/ 597 static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain, 598 struct hinic_api_cmd_cell *cell, int cell_idx) 599 { 600 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 601 struct hinic_hwif *hwif = chain->hwif; 602 struct pci_dev *pdev = hwif->pdev; 603 dma_addr_t cmd_paddr; 604 u8 *cmd_vaddr; 605 int err = 0; 606 607 cmd_vaddr = dma_alloc_coherent(&pdev->dev, API_CMD_BUF_SIZE, 608 &cmd_paddr, GFP_KERNEL); 609 if (!cmd_vaddr) { 610 dev_err(&pdev->dev, "Failed to allocate API CMD DMA memory\n"); 611 return -ENOMEM; 612 } 613 614 cell_ctxt = &chain->cell_ctxt[cell_idx]; 615 616 cell_ctxt->api_cmd_vaddr = cmd_vaddr; 617 cell_ctxt->api_cmd_paddr = cmd_paddr; 618 619 /* set the cmd DMA address in the cell */ 620 switch (chain->chain_type) { 621 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 622 /* The data in the HW should be in Big Endian Format */ 623 cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr); 624 break; 625 626 default: 627 dev_err(&pdev->dev, "Unsupported API CMD chain type\n"); 628 free_cmd_buf(chain, cell_idx); 629 err = -EINVAL; 630 break; 631 } 632 633 return err; 634 } 635 636 /** 637 * api_cmd_create_cell - create API CMD cell for specific chain 638 * @chain: the API CMD specific chain to create its cell 639 * @cell_idx: the index of the cell to create 640 * @pre_node: previous cell 641 * @node_vaddr: the returned virt addr of the cell 642 * 643 * Return 0 - Success, negative - Failure 644 **/ 645 static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain, 646 int cell_idx, 647 struct hinic_api_cmd_cell *pre_node, 648 struct hinic_api_cmd_cell **node_vaddr) 649 { 650 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 651 struct hinic_hwif *hwif = chain->hwif; 652 struct pci_dev *pdev = hwif->pdev; 653 struct hinic_api_cmd_cell *node; 654 dma_addr_t node_paddr; 655 int err; 656 657 node = dma_alloc_coherent(&pdev->dev, chain->cell_size, &node_paddr, 658 GFP_KERNEL); 659 if (!node) { 660 dev_err(&pdev->dev, "Failed to allocate dma API CMD cell\n"); 661 return -ENOMEM; 662 } 663 664 node->read.hw_wb_resp_paddr = 0; 665 666 cell_ctxt = &chain->cell_ctxt[cell_idx]; 667 cell_ctxt->cell_vaddr = node; 668 cell_ctxt->cell_paddr = node_paddr; 669 670 if (!pre_node) { 671 chain->head_cell_paddr = node_paddr; 672 chain->head_node = node; 673 } else { 674 /* The data in the HW should be in Big Endian Format */ 675 pre_node->next_cell_paddr = cpu_to_be64(node_paddr); 676 } 677 678 switch (chain->chain_type) { 679 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 680 err = alloc_cmd_buf(chain, node, cell_idx); 681 if (err) { 682 dev_err(&pdev->dev, "Failed to allocate cmd buffer\n"); 683 goto err_alloc_cmd_buf; 684 } 685 break; 686 687 default: 688 dev_err(&pdev->dev, "Unsupported API CMD chain type\n"); 689 err = -EINVAL; 690 goto err_alloc_cmd_buf; 691 } 692 693 *node_vaddr = node; 694 return 0; 695 696 err_alloc_cmd_buf: 697 dma_free_coherent(&pdev->dev, chain->cell_size, node, node_paddr); 698 return err; 699 } 700 701 /** 702 * api_cmd_destroy_cell - destroy API CMD cell of specific chain 703 * @chain: the API CMD specific chain to destroy its cell 704 * @cell_idx: the cell to destroy 705 **/ 706 static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain, 707 int cell_idx) 708 { 709 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 710 struct hinic_hwif *hwif = chain->hwif; 711 struct pci_dev *pdev = hwif->pdev; 712 struct hinic_api_cmd_cell *node; 713 dma_addr_t node_paddr; 714 size_t node_size; 715 716 cell_ctxt = &chain->cell_ctxt[cell_idx]; 717 718 node = cell_ctxt->cell_vaddr; 719 node_paddr = cell_ctxt->cell_paddr; 720 node_size = chain->cell_size; 721 722 if (cell_ctxt->api_cmd_vaddr) { 723 switch (chain->chain_type) { 724 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 725 free_cmd_buf(chain, cell_idx); 726 break; 727 default: 728 dev_err(&pdev->dev, "Unsupported API CMD chain type\n"); 729 break; 730 } 731 732 dma_free_coherent(&pdev->dev, node_size, node, 733 node_paddr); 734 } 735 } 736 737 /** 738 * api_cmd_destroy_cells - destroy API CMD cells of specific chain 739 * @chain: the API CMD specific chain to destroy its cells 740 * @num_cells: number of cells to destroy 741 **/ 742 static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain, 743 int num_cells) 744 { 745 int cell_idx; 746 747 for (cell_idx = 0; cell_idx < num_cells; cell_idx++) 748 api_cmd_destroy_cell(chain, cell_idx); 749 } 750 751 /** 752 * api_cmd_create_cells - create API CMD cells for specific chain 753 * @chain: the API CMD specific chain 754 * 755 * Return 0 - Success, negative - Failure 756 **/ 757 static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain) 758 { 759 struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL; 760 struct hinic_hwif *hwif = chain->hwif; 761 struct pci_dev *pdev = hwif->pdev; 762 int err, cell_idx; 763 764 for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) { 765 err = api_cmd_create_cell(chain, cell_idx, pre_node, &node); 766 if (err) { 767 dev_err(&pdev->dev, "Failed to create API CMD cell\n"); 768 goto err_create_cell; 769 } 770 771 pre_node = node; 772 } 773 774 /* set the Final node to point on the start */ 775 node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr); 776 777 /* set the current node to be the head */ 778 chain->curr_node = chain->head_node; 779 return 0; 780 781 err_create_cell: 782 api_cmd_destroy_cells(chain, cell_idx); 783 return err; 784 } 785 786 /** 787 * api_chain_init - initialize API CMD specific chain 788 * @chain: the API CMD specific chain to initialize 789 * @attr: attributes to set in the chain 790 * 791 * Return 0 - Success, negative - Failure 792 **/ 793 static int api_chain_init(struct hinic_api_cmd_chain *chain, 794 struct hinic_api_cmd_chain_attr *attr) 795 { 796 struct hinic_hwif *hwif = attr->hwif; 797 struct pci_dev *pdev = hwif->pdev; 798 size_t cell_ctxt_size; 799 800 chain->hwif = hwif; 801 chain->chain_type = attr->chain_type; 802 chain->num_cells = attr->num_cells; 803 chain->cell_size = attr->cell_size; 804 805 chain->prod_idx = 0; 806 chain->cons_idx = 0; 807 808 sema_init(&chain->sem, 1); 809 810 cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt); 811 chain->cell_ctxt = devm_kzalloc(&pdev->dev, cell_ctxt_size, GFP_KERNEL); 812 if (!chain->cell_ctxt) 813 return -ENOMEM; 814 815 chain->wb_status = dma_alloc_coherent(&pdev->dev, 816 sizeof(*chain->wb_status), 817 &chain->wb_status_paddr, 818 GFP_KERNEL); 819 if (!chain->wb_status) { 820 dev_err(&pdev->dev, "Failed to allocate DMA wb status\n"); 821 return -ENOMEM; 822 } 823 824 return 0; 825 } 826 827 /** 828 * api_chain_free - free API CMD specific chain 829 * @chain: the API CMD specific chain to free 830 **/ 831 static void api_chain_free(struct hinic_api_cmd_chain *chain) 832 { 833 struct hinic_hwif *hwif = chain->hwif; 834 struct pci_dev *pdev = hwif->pdev; 835 836 dma_free_coherent(&pdev->dev, sizeof(*chain->wb_status), 837 chain->wb_status, chain->wb_status_paddr); 838 } 839 840 /** 841 * api_cmd_create_chain - create API CMD specific chain 842 * @attr: attributes to set the chain 843 * 844 * Return the created chain 845 **/ 846 static struct hinic_api_cmd_chain * 847 api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr) 848 { 849 struct hinic_hwif *hwif = attr->hwif; 850 struct pci_dev *pdev = hwif->pdev; 851 struct hinic_api_cmd_chain *chain; 852 int err; 853 854 if (attr->num_cells & (attr->num_cells - 1)) { 855 dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n"); 856 return ERR_PTR(-EINVAL); 857 } 858 859 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL); 860 if (!chain) 861 return ERR_PTR(-ENOMEM); 862 863 err = api_chain_init(chain, attr); 864 if (err) { 865 dev_err(&pdev->dev, "Failed to initialize chain\n"); 866 return ERR_PTR(err); 867 } 868 869 err = api_cmd_create_cells(chain); 870 if (err) { 871 dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n"); 872 goto err_create_cells; 873 } 874 875 err = api_cmd_chain_hw_init(chain); 876 if (err) { 877 dev_err(&pdev->dev, "Failed to initialize chain HW\n"); 878 goto err_chain_hw_init; 879 } 880 881 return chain; 882 883 err_chain_hw_init: 884 api_cmd_destroy_cells(chain, chain->num_cells); 885 886 err_create_cells: 887 api_chain_free(chain); 888 return ERR_PTR(err); 889 } 890 891 /** 892 * api_cmd_destroy_chain - destroy API CMD specific chain 893 * @chain: the API CMD specific chain to destroy 894 **/ 895 static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain) 896 { 897 api_cmd_chain_hw_clean(chain); 898 api_cmd_destroy_cells(chain, chain->num_cells); 899 api_chain_free(chain); 900 } 901 902 /** 903 * hinic_api_cmd_init - Initialize all the API CMD chains 904 * @chain: the API CMD chains that are initialized 905 * @hwif: the hardware interface of a pci function device 906 * 907 * Return 0 - Success, negative - Failure 908 **/ 909 int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain, 910 struct hinic_hwif *hwif) 911 { 912 enum hinic_api_cmd_chain_type type, chain_type; 913 struct hinic_api_cmd_chain_attr attr; 914 struct pci_dev *pdev = hwif->pdev; 915 size_t hw_cell_sz; 916 int err; 917 918 hw_cell_sz = sizeof(struct hinic_api_cmd_cell); 919 920 attr.hwif = hwif; 921 attr.num_cells = API_CHAIN_NUM_CELLS; 922 attr.cell_size = API_CMD_CELL_SIZE(hw_cell_sz); 923 924 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; 925 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) { 926 attr.chain_type = chain_type; 927 928 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) 929 continue; 930 931 chain[chain_type] = api_cmd_create_chain(&attr); 932 if (IS_ERR(chain[chain_type])) { 933 dev_err(&pdev->dev, "Failed to create chain %d\n", 934 chain_type); 935 err = PTR_ERR(chain[chain_type]); 936 goto err_create_chain; 937 } 938 } 939 940 return 0; 941 942 err_create_chain: 943 type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; 944 for ( ; type < chain_type; type++) { 945 if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) 946 continue; 947 948 api_cmd_destroy_chain(chain[type]); 949 } 950 951 return err; 952 } 953 954 /** 955 * hinic_api_cmd_free - free the API CMD chains 956 * @chain: the API CMD chains that are freed 957 **/ 958 void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain) 959 { 960 enum hinic_api_cmd_chain_type chain_type; 961 962 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; 963 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) { 964 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) 965 continue; 966 967 api_cmd_destroy_chain(chain[chain_type]); 968 } 969 } 970