1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/errno.h> 10 #include <linux/pci.h> 11 #include <linux/device.h> 12 #include <linux/slab.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/bitops.h> 15 #include <linux/err.h> 16 #include <linux/jiffies.h> 17 #include <linux/delay.h> 18 #include <linux/log2.h> 19 #include <linux/semaphore.h> 20 #include <asm/byteorder.h> 21 #include <asm/barrier.h> 22 23 #include "hinic_hw_csr.h" 24 #include "hinic_hw_if.h" 25 #include "hinic_hw_api_cmd.h" 26 27 #define API_CHAIN_NUM_CELLS 32 28 29 #define API_CMD_CELL_SIZE_SHIFT 6 30 #define API_CMD_CELL_SIZE_MIN (BIT(API_CMD_CELL_SIZE_SHIFT)) 31 32 #define API_CMD_CELL_SIZE(cell_size) \ 33 (((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \ 34 (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN) 35 36 #define API_CMD_CELL_SIZE_VAL(size) \ 37 ilog2((size) >> API_CMD_CELL_SIZE_SHIFT) 38 39 #define API_CMD_BUF_SIZE 2048 40 41 /* Sizes of the members in hinic_api_cmd_cell */ 42 #define API_CMD_CELL_DESC_SIZE 8 43 #define API_CMD_CELL_DATA_ADDR_SIZE 8 44 45 #define API_CMD_CELL_ALIGNMENT 8 46 47 #define API_CMD_TIMEOUT 1000 48 49 #define MASKED_IDX(chain, idx) ((idx) & ((chain)->num_cells - 1)) 50 51 #define SIZE_8BYTES(size) (ALIGN((size), 8) >> 3) 52 #define SIZE_4BYTES(size) (ALIGN((size), 4) >> 2) 53 54 #define RD_DMA_ATTR_DEFAULT 0 55 #define WR_DMA_ATTR_DEFAULT 0 56 57 enum api_cmd_data_format { 58 SGE_DATA = 1, /* cell data is passed by hw address */ 59 }; 60 61 enum api_cmd_type { 62 API_CMD_WRITE = 0, 63 }; 64 65 enum api_cmd_bypass { 66 NO_BYPASS = 0, 67 BYPASS = 1, 68 }; 69 70 enum api_cmd_xor_chk_level { 71 XOR_CHK_DIS = 0, 72 73 XOR_CHK_ALL = 3, 74 }; 75 76 static u8 xor_chksum_set(void *data) 77 { 78 int idx; 79 u8 *val, checksum = 0; 80 81 val = data; 82 83 for (idx = 0; idx < 7; idx++) 84 checksum ^= val[idx]; 85 86 return checksum; 87 } 88 89 static void set_prod_idx(struct hinic_api_cmd_chain *chain) 90 { 91 enum hinic_api_cmd_chain_type chain_type = chain->chain_type; 92 struct hinic_hwif *hwif = chain->hwif; 93 u32 addr, prod_idx; 94 95 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type); 96 prod_idx = hinic_hwif_read_reg(hwif, addr); 97 98 prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX); 99 100 prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX); 101 102 hinic_hwif_write_reg(hwif, addr, prod_idx); 103 } 104 105 static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain) 106 { 107 u32 addr, val; 108 109 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type); 110 val = hinic_hwif_read_reg(chain->hwif, addr); 111 112 return HINIC_API_CMD_STATUS_GET(val, CONS_IDX); 113 } 114 115 static void dump_api_chain_reg(struct hinic_api_cmd_chain *chain) 116 { 117 u32 addr, val; 118 119 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type); 120 val = hinic_hwif_read_reg(chain->hwif, addr); 121 122 dev_err(&chain->hwif->pdev->dev, "Chain type: 0x%x, cpld error: 0x%x, check error: 0x%x, current fsm: 0x%x\n", 123 chain->chain_type, HINIC_API_CMD_STATUS_GET(val, CPLD_ERR), 124 HINIC_API_CMD_STATUS_GET(val, CHKSUM_ERR), 125 HINIC_API_CMD_STATUS_GET(val, FSM)); 126 127 dev_err(&chain->hwif->pdev->dev, "Chain hw current ci: 0x%x\n", 128 HINIC_API_CMD_STATUS_GET(val, CONS_IDX)); 129 130 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain->chain_type); 131 val = hinic_hwif_read_reg(chain->hwif, addr); 132 dev_err(&chain->hwif->pdev->dev, "Chain hw current pi: 0x%x\n", val); 133 } 134 135 /** 136 * chain_busy - check if the chain is still processing last requests 137 * @chain: chain to check 138 * 139 * Return 0 - Success, negative - Failure 140 **/ 141 static int chain_busy(struct hinic_api_cmd_chain *chain) 142 { 143 struct hinic_hwif *hwif = chain->hwif; 144 struct pci_dev *pdev = hwif->pdev; 145 u32 prod_idx; 146 147 switch (chain->chain_type) { 148 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 149 chain->cons_idx = get_hw_cons_idx(chain); 150 prod_idx = chain->prod_idx; 151 152 /* check for a space for a new command */ 153 if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) { 154 dev_err(&pdev->dev, "API CMD chain %d is busy, cons_idx: %d, prod_idx: %d\n", 155 chain->chain_type, chain->cons_idx, 156 chain->prod_idx); 157 dump_api_chain_reg(chain); 158 return -EBUSY; 159 } 160 break; 161 162 default: 163 dev_err(&pdev->dev, "Unknown API CMD Chain type\n"); 164 break; 165 } 166 167 return 0; 168 } 169 170 /** 171 * get_cell_data_size - get the data size of a specific cell type 172 * @type: chain type 173 * 174 * Return the data(Desc + Address) size in the cell 175 **/ 176 static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type) 177 { 178 u8 cell_data_size = 0; 179 180 switch (type) { 181 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 182 cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE + 183 API_CMD_CELL_DATA_ADDR_SIZE, 184 API_CMD_CELL_ALIGNMENT); 185 break; 186 default: 187 break; 188 } 189 190 return cell_data_size; 191 } 192 193 /** 194 * prepare_cell_ctrl - prepare the ctrl of the cell for the command 195 * @cell_ctrl: the control of the cell to set the control value into it 196 * @data_size: the size of the data in the cell 197 **/ 198 static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size) 199 { 200 u8 chksum; 201 u64 ctrl; 202 203 ctrl = HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ) | 204 HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) | 205 HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR); 206 207 chksum = xor_chksum_set(&ctrl); 208 209 ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM); 210 211 /* The data in the HW should be in Big Endian Format */ 212 *cell_ctrl = cpu_to_be64(ctrl); 213 } 214 215 /** 216 * prepare_api_cmd - prepare API CMD command 217 * @chain: chain for the command 218 * @dest: destination node on the card that will receive the command 219 * @cmd: command data 220 * @cmd_size: the command size 221 **/ 222 static void prepare_api_cmd(struct hinic_api_cmd_chain *chain, 223 enum hinic_node_id dest, 224 void *cmd, u16 cmd_size) 225 { 226 struct hinic_api_cmd_cell *cell = chain->curr_node; 227 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 228 struct hinic_hwif *hwif = chain->hwif; 229 struct pci_dev *pdev = hwif->pdev; 230 231 cell_ctxt = &chain->cell_ctxt[chain->prod_idx]; 232 233 switch (chain->chain_type) { 234 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 235 cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE) | 236 HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) | 237 HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS); 238 break; 239 240 default: 241 dev_err(&pdev->dev, "unknown Chain type\n"); 242 return; 243 } 244 245 cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST) | 246 HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE); 247 248 cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc), 249 XOR_CHKSUM); 250 251 /* The data in the HW should be in Big Endian Format */ 252 cell->desc = cpu_to_be64(cell->desc); 253 254 memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size); 255 } 256 257 /** 258 * prepare_cell - prepare cell ctrl and cmd in the current cell 259 * @chain: chain for the command 260 * @dest: destination node on the card that will receive the command 261 * @cmd: command data 262 * @cmd_size: the command size 263 * 264 * Return 0 - Success, negative - Failure 265 **/ 266 static void prepare_cell(struct hinic_api_cmd_chain *chain, 267 enum hinic_node_id dest, 268 void *cmd, u16 cmd_size) 269 { 270 struct hinic_api_cmd_cell *curr_node = chain->curr_node; 271 u16 data_size = get_cell_data_size(chain->chain_type); 272 273 prepare_cell_ctrl(&curr_node->ctrl, data_size); 274 prepare_api_cmd(chain, dest, cmd, cmd_size); 275 } 276 277 static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain) 278 { 279 chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1); 280 } 281 282 /** 283 * api_cmd_status_update - update the status in the chain struct 284 * @chain: chain to update 285 **/ 286 static void api_cmd_status_update(struct hinic_api_cmd_chain *chain) 287 { 288 enum hinic_api_cmd_chain_type chain_type; 289 struct hinic_api_cmd_status *wb_status; 290 struct hinic_hwif *hwif = chain->hwif; 291 struct pci_dev *pdev = hwif->pdev; 292 u64 status_header; 293 u32 status; 294 295 wb_status = chain->wb_status; 296 status_header = be64_to_cpu(wb_status->header); 297 298 status = be32_to_cpu(wb_status->status); 299 if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) { 300 dev_err(&pdev->dev, "API CMD status: Xor check error\n"); 301 return; 302 } 303 304 chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID); 305 if (chain_type >= HINIC_API_CMD_MAX) { 306 dev_err(&pdev->dev, "unknown API CMD Chain %d\n", chain_type); 307 return; 308 } 309 310 chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX); 311 } 312 313 /** 314 * wait_for_status_poll - wait for write to api cmd command to complete 315 * @chain: the chain of the command 316 * 317 * Return 0 - Success, negative - Failure 318 **/ 319 static int wait_for_status_poll(struct hinic_api_cmd_chain *chain) 320 { 321 int err = -ETIMEDOUT; 322 unsigned long end; 323 324 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); 325 do { 326 api_cmd_status_update(chain); 327 328 /* wait for CI to be updated - sign for completion */ 329 if (chain->cons_idx == chain->prod_idx) { 330 err = 0; 331 break; 332 } 333 334 msleep(20); 335 } while (time_before(jiffies, end)); 336 337 return err; 338 } 339 340 /** 341 * wait_for_api_cmd_completion - wait for command to complete 342 * @chain: chain for the command 343 * 344 * Return 0 - Success, negative - Failure 345 **/ 346 static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain) 347 { 348 struct hinic_hwif *hwif = chain->hwif; 349 struct pci_dev *pdev = hwif->pdev; 350 int err; 351 352 switch (chain->chain_type) { 353 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 354 err = wait_for_status_poll(chain); 355 if (err) { 356 dev_err(&pdev->dev, "API CMD Poll status timeout\n"); 357 dump_api_chain_reg(chain); 358 break; 359 } 360 break; 361 362 default: 363 dev_err(&pdev->dev, "unknown API CMD Chain type\n"); 364 err = -EINVAL; 365 break; 366 } 367 368 return err; 369 } 370 371 /** 372 * api_cmd - API CMD command 373 * @chain: chain for the command 374 * @dest: destination node on the card that will receive the command 375 * @cmd: command data 376 * @cmd_size: the command size 377 * 378 * Return 0 - Success, negative - Failure 379 **/ 380 static int api_cmd(struct hinic_api_cmd_chain *chain, 381 enum hinic_node_id dest, u8 *cmd, u16 cmd_size) 382 { 383 struct hinic_api_cmd_cell_ctxt *ctxt; 384 int err; 385 386 down(&chain->sem); 387 if (chain_busy(chain)) { 388 up(&chain->sem); 389 return -EBUSY; 390 } 391 392 prepare_cell(chain, dest, cmd, cmd_size); 393 cmd_chain_prod_idx_inc(chain); 394 395 wmb(); /* inc pi before issue the command */ 396 397 set_prod_idx(chain); /* issue the command */ 398 399 ctxt = &chain->cell_ctxt[chain->prod_idx]; 400 401 chain->curr_node = ctxt->cell_vaddr; 402 403 err = wait_for_api_cmd_completion(chain); 404 405 up(&chain->sem); 406 return err; 407 } 408 409 /** 410 * hinic_api_cmd_write - Write API CMD command 411 * @chain: chain for write command 412 * @dest: destination node on the card that will receive the command 413 * @cmd: command data 414 * @size: the command size 415 * 416 * Return 0 - Success, negative - Failure 417 **/ 418 int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain, 419 enum hinic_node_id dest, u8 *cmd, u16 size) 420 { 421 /* Verify the chain type */ 422 if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU) 423 return api_cmd(chain, dest, cmd, size); 424 425 return -EINVAL; 426 } 427 428 /** 429 * api_cmd_hw_restart - restart the chain in the HW 430 * @chain: the API CMD specific chain to restart 431 * 432 * Return 0 - Success, negative - Failure 433 **/ 434 static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain) 435 { 436 struct hinic_hwif *hwif = chain->hwif; 437 int err = -ETIMEDOUT; 438 unsigned long end; 439 u32 reg_addr, val; 440 441 /* Read Modify Write */ 442 reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type); 443 val = hinic_hwif_read_reg(hwif, reg_addr); 444 445 val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART); 446 val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART); 447 448 hinic_hwif_write_reg(hwif, reg_addr, val); 449 450 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); 451 do { 452 val = hinic_hwif_read_reg(hwif, reg_addr); 453 454 if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) { 455 err = 0; 456 break; 457 } 458 459 msleep(20); 460 } while (time_before(jiffies, end)); 461 462 return err; 463 } 464 465 /** 466 * api_cmd_ctrl_init - set the control register of a chain 467 * @chain: the API CMD specific chain to set control register for 468 **/ 469 static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain) 470 { 471 struct hinic_hwif *hwif = chain->hwif; 472 u32 addr, ctrl; 473 u16 cell_size; 474 475 /* Read Modify Write */ 476 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); 477 478 cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size); 479 480 ctrl = hinic_hwif_read_reg(hwif, addr); 481 482 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) & 483 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) & 484 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & 485 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) & 486 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); 487 488 ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR) | 489 HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) | 490 HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE); 491 492 hinic_hwif_write_reg(hwif, addr, ctrl); 493 } 494 495 /** 496 * api_cmd_set_status_addr - set the status address of a chain in the HW 497 * @chain: the API CMD specific chain to set in HW status address for 498 **/ 499 static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain) 500 { 501 struct hinic_hwif *hwif = chain->hwif; 502 u32 addr, val; 503 504 addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type); 505 val = upper_32_bits(chain->wb_status_paddr); 506 hinic_hwif_write_reg(hwif, addr, val); 507 508 addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type); 509 val = lower_32_bits(chain->wb_status_paddr); 510 hinic_hwif_write_reg(hwif, addr, val); 511 } 512 513 /** 514 * api_cmd_set_num_cells - set the number cells of a chain in the HW 515 * @chain: the API CMD specific chain to set in HW the number of cells for 516 **/ 517 static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain) 518 { 519 struct hinic_hwif *hwif = chain->hwif; 520 u32 addr, val; 521 522 addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type); 523 val = chain->num_cells; 524 hinic_hwif_write_reg(hwif, addr, val); 525 } 526 527 /** 528 * api_cmd_head_init - set the head of a chain in the HW 529 * @chain: the API CMD specific chain to set in HW the head for 530 **/ 531 static void api_cmd_head_init(struct hinic_api_cmd_chain *chain) 532 { 533 struct hinic_hwif *hwif = chain->hwif; 534 u32 addr, val; 535 536 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type); 537 val = upper_32_bits(chain->head_cell_paddr); 538 hinic_hwif_write_reg(hwif, addr, val); 539 540 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type); 541 val = lower_32_bits(chain->head_cell_paddr); 542 hinic_hwif_write_reg(hwif, addr, val); 543 } 544 545 /** 546 * api_cmd_chain_hw_clean - clean the HW 547 * @chain: the API CMD specific chain 548 **/ 549 static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain) 550 { 551 struct hinic_hwif *hwif = chain->hwif; 552 u32 addr, ctrl; 553 554 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); 555 556 ctrl = hinic_hwif_read_reg(hwif, addr); 557 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) & 558 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) & 559 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & 560 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) & 561 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); 562 563 hinic_hwif_write_reg(hwif, addr, ctrl); 564 } 565 566 /** 567 * api_cmd_chain_hw_init - initialize the chain in the HW 568 * @chain: the API CMD specific chain to initialize in HW 569 * 570 * Return 0 - Success, negative - Failure 571 **/ 572 static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain) 573 { 574 struct hinic_hwif *hwif = chain->hwif; 575 struct pci_dev *pdev = hwif->pdev; 576 int err; 577 578 api_cmd_chain_hw_clean(chain); 579 580 api_cmd_set_status_addr(chain); 581 582 err = api_cmd_hw_restart(chain); 583 if (err) { 584 dev_err(&pdev->dev, "Failed to restart API CMD HW\n"); 585 return err; 586 } 587 588 api_cmd_ctrl_init(chain); 589 api_cmd_set_num_cells(chain); 590 api_cmd_head_init(chain); 591 return 0; 592 } 593 594 /** 595 * free_cmd_buf - free the dma buffer of API CMD command 596 * @chain: the API CMD specific chain of the cmd 597 * @cell_idx: the cell index of the cmd 598 **/ 599 static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx) 600 { 601 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 602 struct hinic_hwif *hwif = chain->hwif; 603 struct pci_dev *pdev = hwif->pdev; 604 605 cell_ctxt = &chain->cell_ctxt[cell_idx]; 606 607 dma_free_coherent(&pdev->dev, API_CMD_BUF_SIZE, 608 cell_ctxt->api_cmd_vaddr, 609 cell_ctxt->api_cmd_paddr); 610 } 611 612 /** 613 * alloc_cmd_buf - allocate a dma buffer for API CMD command 614 * @chain: the API CMD specific chain for the cmd 615 * @cell: the cell in the HW for the cmd 616 * @cell_idx: the index of the cell 617 * 618 * Return 0 - Success, negative - Failure 619 **/ 620 static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain, 621 struct hinic_api_cmd_cell *cell, int cell_idx) 622 { 623 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 624 struct hinic_hwif *hwif = chain->hwif; 625 struct pci_dev *pdev = hwif->pdev; 626 dma_addr_t cmd_paddr; 627 u8 *cmd_vaddr; 628 int err = 0; 629 630 cmd_vaddr = dma_alloc_coherent(&pdev->dev, API_CMD_BUF_SIZE, 631 &cmd_paddr, GFP_KERNEL); 632 if (!cmd_vaddr) { 633 dev_err(&pdev->dev, "Failed to allocate API CMD DMA memory\n"); 634 return -ENOMEM; 635 } 636 637 cell_ctxt = &chain->cell_ctxt[cell_idx]; 638 639 cell_ctxt->api_cmd_vaddr = cmd_vaddr; 640 cell_ctxt->api_cmd_paddr = cmd_paddr; 641 642 /* set the cmd DMA address in the cell */ 643 switch (chain->chain_type) { 644 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 645 /* The data in the HW should be in Big Endian Format */ 646 cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr); 647 break; 648 649 default: 650 dev_err(&pdev->dev, "Unsupported API CMD chain type\n"); 651 free_cmd_buf(chain, cell_idx); 652 err = -EINVAL; 653 break; 654 } 655 656 return err; 657 } 658 659 /** 660 * api_cmd_create_cell - create API CMD cell for specific chain 661 * @chain: the API CMD specific chain to create its cell 662 * @cell_idx: the index of the cell to create 663 * @pre_node: previous cell 664 * @node_vaddr: the returned virt addr of the cell 665 * 666 * Return 0 - Success, negative - Failure 667 **/ 668 static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain, 669 int cell_idx, 670 struct hinic_api_cmd_cell *pre_node, 671 struct hinic_api_cmd_cell **node_vaddr) 672 { 673 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 674 struct hinic_hwif *hwif = chain->hwif; 675 struct pci_dev *pdev = hwif->pdev; 676 struct hinic_api_cmd_cell *node; 677 dma_addr_t node_paddr; 678 int err; 679 680 node = dma_alloc_coherent(&pdev->dev, chain->cell_size, &node_paddr, 681 GFP_KERNEL); 682 if (!node) { 683 dev_err(&pdev->dev, "Failed to allocate dma API CMD cell\n"); 684 return -ENOMEM; 685 } 686 687 node->read.hw_wb_resp_paddr = 0; 688 689 cell_ctxt = &chain->cell_ctxt[cell_idx]; 690 cell_ctxt->cell_vaddr = node; 691 cell_ctxt->cell_paddr = node_paddr; 692 693 if (!pre_node) { 694 chain->head_cell_paddr = node_paddr; 695 chain->head_node = node; 696 } else { 697 /* The data in the HW should be in Big Endian Format */ 698 pre_node->next_cell_paddr = cpu_to_be64(node_paddr); 699 } 700 701 switch (chain->chain_type) { 702 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 703 err = alloc_cmd_buf(chain, node, cell_idx); 704 if (err) { 705 dev_err(&pdev->dev, "Failed to allocate cmd buffer\n"); 706 goto err_alloc_cmd_buf; 707 } 708 break; 709 710 default: 711 dev_err(&pdev->dev, "Unsupported API CMD chain type\n"); 712 err = -EINVAL; 713 goto err_alloc_cmd_buf; 714 } 715 716 *node_vaddr = node; 717 return 0; 718 719 err_alloc_cmd_buf: 720 dma_free_coherent(&pdev->dev, chain->cell_size, node, node_paddr); 721 return err; 722 } 723 724 /** 725 * api_cmd_destroy_cell - destroy API CMD cell of specific chain 726 * @chain: the API CMD specific chain to destroy its cell 727 * @cell_idx: the cell to destroy 728 **/ 729 static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain, 730 int cell_idx) 731 { 732 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 733 struct hinic_hwif *hwif = chain->hwif; 734 struct pci_dev *pdev = hwif->pdev; 735 struct hinic_api_cmd_cell *node; 736 dma_addr_t node_paddr; 737 size_t node_size; 738 739 cell_ctxt = &chain->cell_ctxt[cell_idx]; 740 741 node = cell_ctxt->cell_vaddr; 742 node_paddr = cell_ctxt->cell_paddr; 743 node_size = chain->cell_size; 744 745 if (cell_ctxt->api_cmd_vaddr) { 746 switch (chain->chain_type) { 747 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 748 free_cmd_buf(chain, cell_idx); 749 break; 750 default: 751 dev_err(&pdev->dev, "Unsupported API CMD chain type\n"); 752 break; 753 } 754 755 dma_free_coherent(&pdev->dev, node_size, node, 756 node_paddr); 757 } 758 } 759 760 /** 761 * api_cmd_destroy_cells - destroy API CMD cells of specific chain 762 * @chain: the API CMD specific chain to destroy its cells 763 * @num_cells: number of cells to destroy 764 **/ 765 static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain, 766 int num_cells) 767 { 768 int cell_idx; 769 770 for (cell_idx = 0; cell_idx < num_cells; cell_idx++) 771 api_cmd_destroy_cell(chain, cell_idx); 772 } 773 774 /** 775 * api_cmd_create_cells - create API CMD cells for specific chain 776 * @chain: the API CMD specific chain 777 * 778 * Return 0 - Success, negative - Failure 779 **/ 780 static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain) 781 { 782 struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL; 783 struct hinic_hwif *hwif = chain->hwif; 784 struct pci_dev *pdev = hwif->pdev; 785 int err, cell_idx; 786 787 for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) { 788 err = api_cmd_create_cell(chain, cell_idx, pre_node, &node); 789 if (err) { 790 dev_err(&pdev->dev, "Failed to create API CMD cell\n"); 791 goto err_create_cell; 792 } 793 794 pre_node = node; 795 } 796 797 /* set the Final node to point on the start */ 798 node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr); 799 800 /* set the current node to be the head */ 801 chain->curr_node = chain->head_node; 802 return 0; 803 804 err_create_cell: 805 api_cmd_destroy_cells(chain, cell_idx); 806 return err; 807 } 808 809 /** 810 * api_chain_init - initialize API CMD specific chain 811 * @chain: the API CMD specific chain to initialize 812 * @attr: attributes to set in the chain 813 * 814 * Return 0 - Success, negative - Failure 815 **/ 816 static int api_chain_init(struct hinic_api_cmd_chain *chain, 817 struct hinic_api_cmd_chain_attr *attr) 818 { 819 struct hinic_hwif *hwif = attr->hwif; 820 struct pci_dev *pdev = hwif->pdev; 821 size_t cell_ctxt_size; 822 823 chain->hwif = hwif; 824 chain->chain_type = attr->chain_type; 825 chain->num_cells = attr->num_cells; 826 chain->cell_size = attr->cell_size; 827 828 chain->prod_idx = 0; 829 chain->cons_idx = 0; 830 831 sema_init(&chain->sem, 1); 832 833 cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt); 834 chain->cell_ctxt = devm_kzalloc(&pdev->dev, cell_ctxt_size, GFP_KERNEL); 835 if (!chain->cell_ctxt) 836 return -ENOMEM; 837 838 chain->wb_status = dma_alloc_coherent(&pdev->dev, 839 sizeof(*chain->wb_status), 840 &chain->wb_status_paddr, 841 GFP_KERNEL); 842 if (!chain->wb_status) { 843 dev_err(&pdev->dev, "Failed to allocate DMA wb status\n"); 844 return -ENOMEM; 845 } 846 847 return 0; 848 } 849 850 /** 851 * api_chain_free - free API CMD specific chain 852 * @chain: the API CMD specific chain to free 853 **/ 854 static void api_chain_free(struct hinic_api_cmd_chain *chain) 855 { 856 struct hinic_hwif *hwif = chain->hwif; 857 struct pci_dev *pdev = hwif->pdev; 858 859 dma_free_coherent(&pdev->dev, sizeof(*chain->wb_status), 860 chain->wb_status, chain->wb_status_paddr); 861 } 862 863 /** 864 * api_cmd_create_chain - create API CMD specific chain 865 * @attr: attributes to set the chain 866 * 867 * Return the created chain 868 **/ 869 static struct hinic_api_cmd_chain * 870 api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr) 871 { 872 struct hinic_hwif *hwif = attr->hwif; 873 struct pci_dev *pdev = hwif->pdev; 874 struct hinic_api_cmd_chain *chain; 875 int err; 876 877 if (attr->num_cells & (attr->num_cells - 1)) { 878 dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n"); 879 return ERR_PTR(-EINVAL); 880 } 881 882 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL); 883 if (!chain) 884 return ERR_PTR(-ENOMEM); 885 886 err = api_chain_init(chain, attr); 887 if (err) { 888 dev_err(&pdev->dev, "Failed to initialize chain\n"); 889 return ERR_PTR(err); 890 } 891 892 err = api_cmd_create_cells(chain); 893 if (err) { 894 dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n"); 895 goto err_create_cells; 896 } 897 898 err = api_cmd_chain_hw_init(chain); 899 if (err) { 900 dev_err(&pdev->dev, "Failed to initialize chain HW\n"); 901 goto err_chain_hw_init; 902 } 903 904 return chain; 905 906 err_chain_hw_init: 907 api_cmd_destroy_cells(chain, chain->num_cells); 908 909 err_create_cells: 910 api_chain_free(chain); 911 return ERR_PTR(err); 912 } 913 914 /** 915 * api_cmd_destroy_chain - destroy API CMD specific chain 916 * @chain: the API CMD specific chain to destroy 917 **/ 918 static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain) 919 { 920 api_cmd_chain_hw_clean(chain); 921 api_cmd_destroy_cells(chain, chain->num_cells); 922 api_chain_free(chain); 923 } 924 925 /** 926 * hinic_api_cmd_init - Initialize all the API CMD chains 927 * @chain: the API CMD chains that are initialized 928 * @hwif: the hardware interface of a pci function device 929 * 930 * Return 0 - Success, negative - Failure 931 **/ 932 int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain, 933 struct hinic_hwif *hwif) 934 { 935 enum hinic_api_cmd_chain_type type, chain_type; 936 struct hinic_api_cmd_chain_attr attr; 937 struct pci_dev *pdev = hwif->pdev; 938 size_t hw_cell_sz; 939 int err; 940 941 hw_cell_sz = sizeof(struct hinic_api_cmd_cell); 942 943 attr.hwif = hwif; 944 attr.num_cells = API_CHAIN_NUM_CELLS; 945 attr.cell_size = API_CMD_CELL_SIZE(hw_cell_sz); 946 947 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; 948 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) { 949 attr.chain_type = chain_type; 950 951 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) 952 continue; 953 954 chain[chain_type] = api_cmd_create_chain(&attr); 955 if (IS_ERR(chain[chain_type])) { 956 dev_err(&pdev->dev, "Failed to create chain %d\n", 957 chain_type); 958 err = PTR_ERR(chain[chain_type]); 959 goto err_create_chain; 960 } 961 } 962 963 return 0; 964 965 err_create_chain: 966 type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; 967 for ( ; type < chain_type; type++) { 968 if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) 969 continue; 970 971 api_cmd_destroy_chain(chain[type]); 972 } 973 974 return err; 975 } 976 977 /** 978 * hinic_api_cmd_free - free the API CMD chains 979 * @chain: the API CMD chains that are freed 980 **/ 981 void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain) 982 { 983 enum hinic_api_cmd_chain_type chain_type; 984 985 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; 986 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) { 987 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) 988 continue; 989 990 api_cmd_destroy_chain(chain[chain_type]); 991 } 992 } 993