1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/pci.h> 10 #include <linux/device.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/slab.h> 13 #include <linux/atomic.h> 14 #include <linux/semaphore.h> 15 #include <linux/errno.h> 16 #include <linux/vmalloc.h> 17 #include <linux/err.h> 18 #include <asm/byteorder.h> 19 20 #include "hinic_hw_if.h" 21 #include "hinic_hw_wqe.h" 22 #include "hinic_hw_wq.h" 23 #include "hinic_hw_cmdq.h" 24 25 #define WQS_BLOCKS_PER_PAGE 4 26 27 #define WQ_BLOCK_SIZE 4096 28 #define WQS_PAGE_SIZE (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE) 29 30 #define WQS_MAX_NUM_BLOCKS 128 31 #define WQS_FREE_BLOCKS_SIZE(wqs) (WQS_MAX_NUM_BLOCKS * \ 32 sizeof((wqs)->free_blocks[0])) 33 34 #define WQ_SIZE(wq) ((wq)->q_depth * (wq)->wqebb_size) 35 36 #define WQ_PAGE_ADDR_SIZE sizeof(u64) 37 #define WQ_MAX_PAGES (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) 38 39 #define CMDQ_BLOCK_SIZE 512 40 #define CMDQ_PAGE_SIZE 4096 41 42 #define CMDQ_WQ_MAX_PAGES (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) 43 44 #define WQ_BASE_VADDR(wqs, wq) \ 45 ((void *)((wqs)->page_vaddr[(wq)->page_idx]) \ 46 + (wq)->block_idx * WQ_BLOCK_SIZE) 47 48 #define WQ_BASE_PADDR(wqs, wq) \ 49 ((wqs)->page_paddr[(wq)->page_idx] \ 50 + (wq)->block_idx * WQ_BLOCK_SIZE) 51 52 #define WQ_BASE_ADDR(wqs, wq) \ 53 ((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \ 54 + (wq)->block_idx * WQ_BLOCK_SIZE) 55 56 #define CMDQ_BASE_VADDR(cmdq_pages, wq) \ 57 ((void *)((cmdq_pages)->page_vaddr) \ 58 + (wq)->block_idx * CMDQ_BLOCK_SIZE) 59 60 #define CMDQ_BASE_PADDR(cmdq_pages, wq) \ 61 ((cmdq_pages)->page_paddr \ 62 + (wq)->block_idx * CMDQ_BLOCK_SIZE) 63 64 #define CMDQ_BASE_ADDR(cmdq_pages, wq) \ 65 ((void *)((cmdq_pages)->shadow_page_vaddr) \ 66 + (wq)->block_idx * CMDQ_BLOCK_SIZE) 67 68 #define WQ_PAGE_ADDR(wq, idx) \ 69 ((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)]) 70 71 #define MASKED_WQE_IDX(wq, idx) ((idx) & (wq)->mask) 72 73 #define WQE_IN_RANGE(wqe, start, end) \ 74 (((unsigned long)(wqe) >= (unsigned long)(start)) && \ 75 ((unsigned long)(wqe) < (unsigned long)(end))) 76 77 #define WQE_SHADOW_PAGE(wq, wqe) \ 78 (((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \ 79 / (wq)->max_wqe_size) 80 81 static inline int WQE_PAGE_OFF(struct hinic_wq *wq, u16 idx) 82 { 83 return (((idx) & ((wq)->num_wqebbs_per_page - 1)) 84 << (wq)->wqebb_size_shift); 85 } 86 87 static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx) 88 { 89 return (((idx) >> ((wq)->wqebbs_per_page_shift)) 90 & ((wq)->num_q_pages - 1)); 91 } 92 93 /** 94 * queue_alloc_page - allocate page for Queue 95 * @hwif: HW interface for allocating DMA 96 * @vaddr: virtual address will be returned in this address 97 * @paddr: physical address will be returned in this address 98 * @shadow_vaddr: VM area will be return here for holding WQ page addresses 99 * @page_sz: page size of each WQ page 100 * 101 * Return 0 - Success, negative - Failure 102 **/ 103 static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr, 104 void ***shadow_vaddr, size_t page_sz) 105 { 106 struct pci_dev *pdev = hwif->pdev; 107 dma_addr_t dma_addr; 108 109 *vaddr = dma_alloc_coherent(&pdev->dev, page_sz, &dma_addr, 110 GFP_KERNEL); 111 if (!*vaddr) { 112 dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n"); 113 return -ENOMEM; 114 } 115 116 *paddr = (u64)dma_addr; 117 118 /* use vzalloc for big mem */ 119 *shadow_vaddr = vzalloc(page_sz); 120 if (!*shadow_vaddr) 121 goto err_shadow_vaddr; 122 123 return 0; 124 125 err_shadow_vaddr: 126 dma_free_coherent(&pdev->dev, page_sz, *vaddr, dma_addr); 127 return -ENOMEM; 128 } 129 130 /** 131 * wqs_allocate_page - allocate page for WQ set 132 * @wqs: Work Queue Set 133 * @page_idx: the page index of the page will be allocated 134 * 135 * Return 0 - Success, negative - Failure 136 **/ 137 static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx) 138 { 139 return queue_alloc_page(wqs->hwif, &wqs->page_vaddr[page_idx], 140 &wqs->page_paddr[page_idx], 141 &wqs->shadow_page_vaddr[page_idx], 142 WQS_PAGE_SIZE); 143 } 144 145 /** 146 * wqs_free_page - free page of WQ set 147 * @wqs: Work Queue Set 148 * @page_idx: the page index of the page will be freed 149 **/ 150 static void wqs_free_page(struct hinic_wqs *wqs, int page_idx) 151 { 152 struct hinic_hwif *hwif = wqs->hwif; 153 struct pci_dev *pdev = hwif->pdev; 154 155 dma_free_coherent(&pdev->dev, WQS_PAGE_SIZE, 156 wqs->page_vaddr[page_idx], 157 (dma_addr_t)wqs->page_paddr[page_idx]); 158 vfree(wqs->shadow_page_vaddr[page_idx]); 159 } 160 161 /** 162 * cmdq_allocate_page - allocate page for cmdq 163 * @cmdq_pages: the pages of the cmdq queue struct to hold the page 164 * 165 * Return 0 - Success, negative - Failure 166 **/ 167 static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages) 168 { 169 return queue_alloc_page(cmdq_pages->hwif, &cmdq_pages->page_vaddr, 170 &cmdq_pages->page_paddr, 171 &cmdq_pages->shadow_page_vaddr, 172 CMDQ_PAGE_SIZE); 173 } 174 175 /** 176 * cmdq_free_page - free page from cmdq 177 * @cmdq_pages: the pages of the cmdq queue struct that hold the page 178 * 179 * Return 0 - Success, negative - Failure 180 **/ 181 static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages) 182 { 183 struct hinic_hwif *hwif = cmdq_pages->hwif; 184 struct pci_dev *pdev = hwif->pdev; 185 186 dma_free_coherent(&pdev->dev, CMDQ_PAGE_SIZE, 187 cmdq_pages->page_vaddr, 188 (dma_addr_t)cmdq_pages->page_paddr); 189 vfree(cmdq_pages->shadow_page_vaddr); 190 } 191 192 static int alloc_page_arrays(struct hinic_wqs *wqs) 193 { 194 struct hinic_hwif *hwif = wqs->hwif; 195 struct pci_dev *pdev = hwif->pdev; 196 size_t size; 197 198 size = wqs->num_pages * sizeof(*wqs->page_paddr); 199 wqs->page_paddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 200 if (!wqs->page_paddr) 201 return -ENOMEM; 202 203 size = wqs->num_pages * sizeof(*wqs->page_vaddr); 204 wqs->page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 205 if (!wqs->page_vaddr) 206 goto err_page_vaddr; 207 208 size = wqs->num_pages * sizeof(*wqs->shadow_page_vaddr); 209 wqs->shadow_page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 210 if (!wqs->shadow_page_vaddr) 211 goto err_page_shadow_vaddr; 212 213 return 0; 214 215 err_page_shadow_vaddr: 216 devm_kfree(&pdev->dev, wqs->page_vaddr); 217 218 err_page_vaddr: 219 devm_kfree(&pdev->dev, wqs->page_paddr); 220 return -ENOMEM; 221 } 222 223 static void free_page_arrays(struct hinic_wqs *wqs) 224 { 225 struct hinic_hwif *hwif = wqs->hwif; 226 struct pci_dev *pdev = hwif->pdev; 227 228 devm_kfree(&pdev->dev, wqs->shadow_page_vaddr); 229 devm_kfree(&pdev->dev, wqs->page_vaddr); 230 devm_kfree(&pdev->dev, wqs->page_paddr); 231 } 232 233 static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx, 234 int *block_idx) 235 { 236 int pos; 237 238 down(&wqs->alloc_blocks_lock); 239 240 wqs->num_free_blks--; 241 242 if (wqs->num_free_blks < 0) { 243 wqs->num_free_blks++; 244 up(&wqs->alloc_blocks_lock); 245 return -ENOMEM; 246 } 247 248 pos = wqs->alloc_blk_pos++; 249 pos &= WQS_MAX_NUM_BLOCKS - 1; 250 251 *page_idx = wqs->free_blocks[pos].page_idx; 252 *block_idx = wqs->free_blocks[pos].block_idx; 253 254 wqs->free_blocks[pos].page_idx = -1; 255 wqs->free_blocks[pos].block_idx = -1; 256 257 up(&wqs->alloc_blocks_lock); 258 return 0; 259 } 260 261 static void wqs_return_block(struct hinic_wqs *wqs, int page_idx, 262 int block_idx) 263 { 264 int pos; 265 266 down(&wqs->alloc_blocks_lock); 267 268 pos = wqs->return_blk_pos++; 269 pos &= WQS_MAX_NUM_BLOCKS - 1; 270 271 wqs->free_blocks[pos].page_idx = page_idx; 272 wqs->free_blocks[pos].block_idx = block_idx; 273 274 wqs->num_free_blks++; 275 276 up(&wqs->alloc_blocks_lock); 277 } 278 279 static void init_wqs_blocks_arr(struct hinic_wqs *wqs) 280 { 281 int page_idx, blk_idx, pos = 0; 282 283 for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { 284 for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) { 285 wqs->free_blocks[pos].page_idx = page_idx; 286 wqs->free_blocks[pos].block_idx = blk_idx; 287 pos++; 288 } 289 } 290 291 wqs->alloc_blk_pos = 0; 292 wqs->return_blk_pos = pos; 293 wqs->num_free_blks = pos; 294 295 sema_init(&wqs->alloc_blocks_lock, 1); 296 } 297 298 /** 299 * hinic_wqs_alloc - allocate Work Queues set 300 * @wqs: Work Queue Set 301 * @max_wqs: maximum wqs to allocate 302 * @hwif: HW interface for use for the allocation 303 * 304 * Return 0 - Success, negative - Failure 305 **/ 306 int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs, 307 struct hinic_hwif *hwif) 308 { 309 struct pci_dev *pdev = hwif->pdev; 310 int err, i, page_idx; 311 312 max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE); 313 if (max_wqs > WQS_MAX_NUM_BLOCKS) { 314 dev_err(&pdev->dev, "Invalid max_wqs = %d\n", max_wqs); 315 return -EINVAL; 316 } 317 318 wqs->hwif = hwif; 319 wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE; 320 321 if (alloc_page_arrays(wqs)) { 322 dev_err(&pdev->dev, 323 "Failed to allocate mem for page addresses\n"); 324 return -ENOMEM; 325 } 326 327 for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { 328 err = wqs_allocate_page(wqs, page_idx); 329 if (err) { 330 dev_err(&pdev->dev, "Failed wq page allocation\n"); 331 goto err_wq_allocate_page; 332 } 333 } 334 335 wqs->free_blocks = devm_kzalloc(&pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs), 336 GFP_KERNEL); 337 if (!wqs->free_blocks) { 338 err = -ENOMEM; 339 goto err_alloc_blocks; 340 } 341 342 init_wqs_blocks_arr(wqs); 343 return 0; 344 345 err_alloc_blocks: 346 err_wq_allocate_page: 347 for (i = 0; i < page_idx; i++) 348 wqs_free_page(wqs, i); 349 350 free_page_arrays(wqs); 351 return err; 352 } 353 354 /** 355 * hinic_wqs_free - free Work Queues set 356 * @wqs: Work Queue Set 357 **/ 358 void hinic_wqs_free(struct hinic_wqs *wqs) 359 { 360 struct hinic_hwif *hwif = wqs->hwif; 361 struct pci_dev *pdev = hwif->pdev; 362 int page_idx; 363 364 devm_kfree(&pdev->dev, wqs->free_blocks); 365 366 for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) 367 wqs_free_page(wqs, page_idx); 368 369 free_page_arrays(wqs); 370 } 371 372 /** 373 * alloc_wqes_shadow - allocate WQE shadows for WQ 374 * @wq: WQ to allocate shadows for 375 * 376 * Return 0 - Success, negative - Failure 377 **/ 378 static int alloc_wqes_shadow(struct hinic_wq *wq) 379 { 380 struct hinic_hwif *hwif = wq->hwif; 381 struct pci_dev *pdev = hwif->pdev; 382 size_t size; 383 384 size = wq->num_q_pages * wq->max_wqe_size; 385 wq->shadow_wqe = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 386 if (!wq->shadow_wqe) 387 return -ENOMEM; 388 389 size = wq->num_q_pages * sizeof(wq->prod_idx); 390 wq->shadow_idx = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 391 if (!wq->shadow_idx) 392 goto err_shadow_idx; 393 394 return 0; 395 396 err_shadow_idx: 397 devm_kfree(&pdev->dev, wq->shadow_wqe); 398 return -ENOMEM; 399 } 400 401 /** 402 * free_wqes_shadow - free WQE shadows of WQ 403 * @wq: WQ to free shadows from 404 **/ 405 static void free_wqes_shadow(struct hinic_wq *wq) 406 { 407 struct hinic_hwif *hwif = wq->hwif; 408 struct pci_dev *pdev = hwif->pdev; 409 410 devm_kfree(&pdev->dev, wq->shadow_idx); 411 devm_kfree(&pdev->dev, wq->shadow_wqe); 412 } 413 414 /** 415 * free_wq_pages - free pages of WQ 416 * @hwif: HW interface for releasing dma addresses 417 * @wq: WQ to free pages from 418 * @num_q_pages: number pages to free 419 **/ 420 static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, 421 int num_q_pages) 422 { 423 struct pci_dev *pdev = hwif->pdev; 424 int i; 425 426 for (i = 0; i < num_q_pages; i++) { 427 void **vaddr = &wq->shadow_block_vaddr[i]; 428 u64 *paddr = &wq->block_vaddr[i]; 429 dma_addr_t dma_addr; 430 431 dma_addr = (dma_addr_t)be64_to_cpu(*paddr); 432 dma_free_coherent(&pdev->dev, wq->wq_page_size, *vaddr, 433 dma_addr); 434 } 435 436 free_wqes_shadow(wq); 437 } 438 439 /** 440 * alloc_wq_pages - alloc pages for WQ 441 * @hwif: HW interface for allocating dma addresses 442 * @wq: WQ to allocate pages for 443 * @max_pages: maximum pages allowed 444 * 445 * Return 0 - Success, negative - Failure 446 **/ 447 static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, 448 int max_pages) 449 { 450 struct pci_dev *pdev = hwif->pdev; 451 int i, err, num_q_pages; 452 453 num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size; 454 if (num_q_pages > max_pages) { 455 dev_err(&pdev->dev, "Number wq pages exceeds the limit\n"); 456 return -EINVAL; 457 } 458 459 if (num_q_pages & (num_q_pages - 1)) { 460 dev_err(&pdev->dev, "Number wq pages must be power of 2\n"); 461 return -EINVAL; 462 } 463 464 wq->num_q_pages = num_q_pages; 465 466 err = alloc_wqes_shadow(wq); 467 if (err) { 468 dev_err(&pdev->dev, "Failed to allocate wqe shadow\n"); 469 return err; 470 } 471 472 for (i = 0; i < num_q_pages; i++) { 473 void **vaddr = &wq->shadow_block_vaddr[i]; 474 u64 *paddr = &wq->block_vaddr[i]; 475 dma_addr_t dma_addr; 476 477 *vaddr = dma_alloc_coherent(&pdev->dev, wq->wq_page_size, 478 &dma_addr, GFP_KERNEL); 479 if (!*vaddr) { 480 dev_err(&pdev->dev, "Failed to allocate wq page\n"); 481 goto err_alloc_wq_pages; 482 } 483 484 /* HW uses Big Endian Format */ 485 *paddr = cpu_to_be64(dma_addr); 486 } 487 488 return 0; 489 490 err_alloc_wq_pages: 491 free_wq_pages(wq, hwif, i); 492 return -ENOMEM; 493 } 494 495 /** 496 * hinic_wq_allocate - Allocate the WQ resources from the WQS 497 * @wqs: WQ set from which to allocate the WQ resources 498 * @wq: WQ to allocate resources for it from the WQ set 499 * @wqebb_size: Work Queue Block Byte Size 500 * @wq_page_size: the page size in the Work Queue 501 * @q_depth: number of wqebbs in WQ 502 * @max_wqe_size: maximum WQE size that will be used in the WQ 503 * 504 * Return 0 - Success, negative - Failure 505 **/ 506 int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq, 507 u16 wqebb_size, u32 wq_page_size, u16 q_depth, 508 u16 max_wqe_size) 509 { 510 struct hinic_hwif *hwif = wqs->hwif; 511 struct pci_dev *pdev = hwif->pdev; 512 u16 num_wqebbs_per_page; 513 u16 wqebb_size_shift; 514 int err; 515 516 if (!is_power_of_2(wqebb_size)) { 517 dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); 518 return -EINVAL; 519 } 520 521 if (wq_page_size == 0) { 522 dev_err(&pdev->dev, "wq_page_size must be > 0\n"); 523 return -EINVAL; 524 } 525 526 if (q_depth & (q_depth - 1)) { 527 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); 528 return -EINVAL; 529 } 530 531 wqebb_size_shift = ilog2(wqebb_size); 532 num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) 533 >> wqebb_size_shift; 534 535 if (!is_power_of_2(num_wqebbs_per_page)) { 536 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); 537 return -EINVAL; 538 } 539 540 wq->hwif = hwif; 541 542 err = wqs_next_block(wqs, &wq->page_idx, &wq->block_idx); 543 if (err) { 544 dev_err(&pdev->dev, "Failed to get free wqs next block\n"); 545 return err; 546 } 547 548 wq->wqebb_size = wqebb_size; 549 wq->wq_page_size = wq_page_size; 550 wq->q_depth = q_depth; 551 wq->max_wqe_size = max_wqe_size; 552 wq->num_wqebbs_per_page = num_wqebbs_per_page; 553 wq->wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); 554 wq->wqebb_size_shift = wqebb_size_shift; 555 wq->block_vaddr = WQ_BASE_VADDR(wqs, wq); 556 wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq); 557 wq->block_paddr = WQ_BASE_PADDR(wqs, wq); 558 559 err = alloc_wq_pages(wq, wqs->hwif, WQ_MAX_PAGES); 560 if (err) { 561 dev_err(&pdev->dev, "Failed to allocate wq pages\n"); 562 goto err_alloc_wq_pages; 563 } 564 565 atomic_set(&wq->cons_idx, 0); 566 atomic_set(&wq->prod_idx, 0); 567 atomic_set(&wq->delta, q_depth); 568 wq->mask = q_depth - 1; 569 570 return 0; 571 572 err_alloc_wq_pages: 573 wqs_return_block(wqs, wq->page_idx, wq->block_idx); 574 return err; 575 } 576 577 /** 578 * hinic_wq_free - Free the WQ resources to the WQS 579 * @wqs: WQ set to free the WQ resources to it 580 * @wq: WQ to free its resources to the WQ set resources 581 **/ 582 void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq) 583 { 584 free_wq_pages(wq, wqs->hwif, wq->num_q_pages); 585 586 wqs_return_block(wqs, wq->page_idx, wq->block_idx); 587 } 588 589 /** 590 * hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs 591 * @cmdq_pages: will hold the pages of the cmdq 592 * @wq: returned wqs 593 * @hwif: HW interface 594 * @cmdq_blocks: number of cmdq blocks/wq to allocate 595 * @wqebb_size: Work Queue Block Byte Size 596 * @wq_page_size: the page size in the Work Queue 597 * @q_depth: number of wqebbs in WQ 598 * @max_wqe_size: maximum WQE size that will be used in the WQ 599 * 600 * Return 0 - Success, negative - Failure 601 **/ 602 int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages, 603 struct hinic_wq *wq, struct hinic_hwif *hwif, 604 int cmdq_blocks, u16 wqebb_size, u32 wq_page_size, 605 u16 q_depth, u16 max_wqe_size) 606 { 607 struct pci_dev *pdev = hwif->pdev; 608 u16 num_wqebbs_per_page_shift; 609 u16 num_wqebbs_per_page; 610 u16 wqebb_size_shift; 611 int i, j, err = -ENOMEM; 612 613 if (!is_power_of_2(wqebb_size)) { 614 dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); 615 return -EINVAL; 616 } 617 618 if (wq_page_size == 0) { 619 dev_err(&pdev->dev, "wq_page_size must be > 0\n"); 620 return -EINVAL; 621 } 622 623 if (q_depth & (q_depth - 1)) { 624 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); 625 return -EINVAL; 626 } 627 628 wqebb_size_shift = ilog2(wqebb_size); 629 num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) 630 >> wqebb_size_shift; 631 632 if (!is_power_of_2(num_wqebbs_per_page)) { 633 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); 634 return -EINVAL; 635 } 636 637 cmdq_pages->hwif = hwif; 638 639 err = cmdq_allocate_page(cmdq_pages); 640 if (err) { 641 dev_err(&pdev->dev, "Failed to allocate CMDQ page\n"); 642 return err; 643 } 644 num_wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); 645 646 for (i = 0; i < cmdq_blocks; i++) { 647 wq[i].hwif = hwif; 648 wq[i].page_idx = 0; 649 wq[i].block_idx = i; 650 651 wq[i].wqebb_size = wqebb_size; 652 wq[i].wq_page_size = wq_page_size; 653 wq[i].q_depth = q_depth; 654 wq[i].max_wqe_size = max_wqe_size; 655 wq[i].num_wqebbs_per_page = num_wqebbs_per_page; 656 wq[i].wqebbs_per_page_shift = num_wqebbs_per_page_shift; 657 wq[i].wqebb_size_shift = wqebb_size_shift; 658 wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]); 659 wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]); 660 wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]); 661 662 err = alloc_wq_pages(&wq[i], cmdq_pages->hwif, 663 CMDQ_WQ_MAX_PAGES); 664 if (err) { 665 dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n"); 666 goto err_cmdq_block; 667 } 668 669 atomic_set(&wq[i].cons_idx, 0); 670 atomic_set(&wq[i].prod_idx, 0); 671 atomic_set(&wq[i].delta, q_depth); 672 wq[i].mask = q_depth - 1; 673 } 674 675 return 0; 676 677 err_cmdq_block: 678 for (j = 0; j < i; j++) 679 free_wq_pages(&wq[j], cmdq_pages->hwif, wq[j].num_q_pages); 680 681 cmdq_free_page(cmdq_pages); 682 return err; 683 } 684 685 /** 686 * hinic_wqs_cmdq_free - Free wqs from cmdqs 687 * @cmdq_pages: hold the pages of the cmdq 688 * @wq: wqs to free 689 * @cmdq_blocks: number of wqs to free 690 **/ 691 void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages, 692 struct hinic_wq *wq, int cmdq_blocks) 693 { 694 int i; 695 696 for (i = 0; i < cmdq_blocks; i++) 697 free_wq_pages(&wq[i], cmdq_pages->hwif, wq[i].num_q_pages); 698 699 cmdq_free_page(cmdq_pages); 700 } 701 702 static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr, 703 int num_wqebbs, u16 idx) 704 { 705 void *wqebb_addr; 706 int i; 707 708 for (i = 0; i < num_wqebbs; i++, idx++) { 709 idx = MASKED_WQE_IDX(wq, idx); 710 wqebb_addr = WQ_PAGE_ADDR(wq, idx) + 711 WQE_PAGE_OFF(wq, idx); 712 713 memcpy(shadow_addr, wqebb_addr, wq->wqebb_size); 714 715 shadow_addr += wq->wqebb_size; 716 } 717 } 718 719 static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr, 720 int num_wqebbs, u16 idx) 721 { 722 void *wqebb_addr; 723 int i; 724 725 for (i = 0; i < num_wqebbs; i++, idx++) { 726 idx = MASKED_WQE_IDX(wq, idx); 727 wqebb_addr = WQ_PAGE_ADDR(wq, idx) + 728 WQE_PAGE_OFF(wq, idx); 729 730 memcpy(wqebb_addr, shadow_addr, wq->wqebb_size); 731 shadow_addr += wq->wqebb_size; 732 } 733 } 734 735 /** 736 * hinic_get_wqe - get wqe ptr in the current pi and update the pi 737 * @wq: wq to get wqe from 738 * @wqe_size: wqe size 739 * @prod_idx: returned pi 740 * 741 * Return wqe pointer 742 **/ 743 struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size, 744 u16 *prod_idx) 745 { 746 int curr_pg, end_pg, num_wqebbs; 747 u16 curr_prod_idx, end_prod_idx; 748 749 *prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx)); 750 751 num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) >> wq->wqebb_size_shift; 752 753 if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) { 754 atomic_add(num_wqebbs, &wq->delta); 755 return ERR_PTR(-EBUSY); 756 } 757 758 end_prod_idx = atomic_add_return(num_wqebbs, &wq->prod_idx); 759 760 end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx); 761 curr_prod_idx = end_prod_idx - num_wqebbs; 762 curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx); 763 764 /* end prod index points to the next wqebb, therefore minus 1 */ 765 end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1); 766 767 curr_pg = WQE_PAGE_NUM(wq, curr_prod_idx); 768 end_pg = WQE_PAGE_NUM(wq, end_prod_idx); 769 770 *prod_idx = curr_prod_idx; 771 772 /* If we only have one page, still need to get shadown wqe when 773 * wqe rolling-over page 774 */ 775 if (curr_pg != end_pg || MASKED_WQE_IDX(wq, end_prod_idx) < *prod_idx) { 776 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; 777 778 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx); 779 780 wq->shadow_idx[curr_pg] = *prod_idx; 781 return shadow_addr; 782 } 783 784 return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, *prod_idx); 785 } 786 787 /** 788 * hinic_return_wqe - return the wqe when transmit failed 789 * @wq: wq to return wqe 790 * @wqe_size: wqe size 791 **/ 792 void hinic_return_wqe(struct hinic_wq *wq, unsigned int wqe_size) 793 { 794 int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; 795 796 atomic_sub(num_wqebbs, &wq->prod_idx); 797 798 atomic_add(num_wqebbs, &wq->delta); 799 } 800 801 /** 802 * hinic_put_wqe - return the wqe place to use for a new wqe 803 * @wq: wq to return wqe 804 * @wqe_size: wqe size 805 **/ 806 void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size) 807 { 808 int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) 809 >> wq->wqebb_size_shift; 810 811 atomic_add(num_wqebbs, &wq->cons_idx); 812 813 atomic_add(num_wqebbs, &wq->delta); 814 } 815 816 /** 817 * hinic_read_wqe - read wqe ptr in the current ci 818 * @wq: wq to get read from 819 * @wqe_size: wqe size 820 * @cons_idx: returned ci 821 * 822 * Return wqe pointer 823 **/ 824 struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size, 825 u16 *cons_idx) 826 { 827 int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) 828 >> wq->wqebb_size_shift; 829 u16 curr_cons_idx, end_cons_idx; 830 int curr_pg, end_pg; 831 832 if ((atomic_read(&wq->delta) + num_wqebbs) > wq->q_depth) 833 return ERR_PTR(-EBUSY); 834 835 curr_cons_idx = atomic_read(&wq->cons_idx); 836 837 curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx); 838 end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1); 839 840 curr_pg = WQE_PAGE_NUM(wq, curr_cons_idx); 841 end_pg = WQE_PAGE_NUM(wq, end_cons_idx); 842 843 *cons_idx = curr_cons_idx; 844 845 if (curr_pg != end_pg) { 846 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; 847 848 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx); 849 return shadow_addr; 850 } 851 852 return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, *cons_idx); 853 } 854 855 /** 856 * hinic_read_wqe_direct - read wqe directly from ci position 857 * @wq: wq 858 * @cons_idx: ci position 859 * 860 * Return wqe 861 **/ 862 struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx) 863 { 864 return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, cons_idx); 865 } 866 867 /** 868 * wqe_shadow - check if a wqe is shadow 869 * @wq: wq of the wqe 870 * @wqe: the wqe for shadow checking 871 * 872 * Return true - shadow, false - Not shadow 873 **/ 874 static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe) 875 { 876 size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size; 877 878 return WQE_IN_RANGE(wqe, wq->shadow_wqe, 879 &wq->shadow_wqe[wqe_shadow_size]); 880 } 881 882 /** 883 * hinic_write_wqe - write the wqe to the wq 884 * @wq: wq to write wqe to 885 * @wqe: wqe to write 886 * @wqe_size: wqe size 887 **/ 888 void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe, 889 unsigned int wqe_size) 890 { 891 int curr_pg, num_wqebbs; 892 void *shadow_addr; 893 u16 prod_idx; 894 895 if (wqe_shadow(wq, wqe)) { 896 curr_pg = WQE_SHADOW_PAGE(wq, wqe); 897 898 prod_idx = wq->shadow_idx[curr_pg]; 899 num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; 900 shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; 901 902 copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, prod_idx); 903 } 904 } 905