1 /* 2 * Huawei HiNIC PCI Express Linux driver 3 * Copyright(c) 2017 Huawei Technologies Co., Ltd 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * for more details. 13 * 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/pci.h> 19 #include <linux/device.h> 20 #include <linux/dma-mapping.h> 21 #include <linux/slab.h> 22 #include <linux/atomic.h> 23 #include <linux/semaphore.h> 24 #include <linux/errno.h> 25 #include <linux/vmalloc.h> 26 #include <linux/err.h> 27 #include <asm/byteorder.h> 28 29 #include "hinic_hw_if.h" 30 #include "hinic_hw_wqe.h" 31 #include "hinic_hw_wq.h" 32 #include "hinic_hw_cmdq.h" 33 34 #define WQS_BLOCKS_PER_PAGE 4 35 36 #define WQ_BLOCK_SIZE 4096 37 #define WQS_PAGE_SIZE (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE) 38 39 #define WQS_MAX_NUM_BLOCKS 128 40 #define WQS_FREE_BLOCKS_SIZE(wqs) (WQS_MAX_NUM_BLOCKS * \ 41 sizeof((wqs)->free_blocks[0])) 42 43 #define WQ_SIZE(wq) ((wq)->q_depth * (wq)->wqebb_size) 44 45 #define WQ_PAGE_ADDR_SIZE sizeof(u64) 46 #define WQ_MAX_PAGES (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) 47 48 #define CMDQ_BLOCK_SIZE 512 49 #define CMDQ_PAGE_SIZE 4096 50 51 #define CMDQ_WQ_MAX_PAGES (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) 52 53 #define WQ_BASE_VADDR(wqs, wq) \ 54 ((void *)((wqs)->page_vaddr[(wq)->page_idx]) \ 55 + (wq)->block_idx * WQ_BLOCK_SIZE) 56 57 #define WQ_BASE_PADDR(wqs, wq) \ 58 ((wqs)->page_paddr[(wq)->page_idx] \ 59 + (wq)->block_idx * WQ_BLOCK_SIZE) 60 61 #define WQ_BASE_ADDR(wqs, wq) \ 62 ((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \ 63 + (wq)->block_idx * WQ_BLOCK_SIZE) 64 65 #define CMDQ_BASE_VADDR(cmdq_pages, wq) \ 66 ((void *)((cmdq_pages)->page_vaddr) \ 67 + (wq)->block_idx * CMDQ_BLOCK_SIZE) 68 69 #define CMDQ_BASE_PADDR(cmdq_pages, wq) \ 70 ((cmdq_pages)->page_paddr \ 71 + (wq)->block_idx * CMDQ_BLOCK_SIZE) 72 73 #define CMDQ_BASE_ADDR(cmdq_pages, wq) \ 74 ((void *)((cmdq_pages)->shadow_page_vaddr) \ 75 + (wq)->block_idx * CMDQ_BLOCK_SIZE) 76 77 #define WQ_PAGE_ADDR(wq, idx) \ 78 ((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)]) 79 80 #define MASKED_WQE_IDX(wq, idx) ((idx) & (wq)->mask) 81 82 #define WQE_IN_RANGE(wqe, start, end) \ 83 (((unsigned long)(wqe) >= (unsigned long)(start)) && \ 84 ((unsigned long)(wqe) < (unsigned long)(end))) 85 86 #define WQE_SHADOW_PAGE(wq, wqe) \ 87 (((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \ 88 / (wq)->max_wqe_size) 89 90 static inline int WQE_PAGE_OFF(struct hinic_wq *wq, u16 idx) 91 { 92 return (((idx) & ((wq)->num_wqebbs_per_page - 1)) 93 << (wq)->wqebb_size_shift); 94 } 95 96 static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx) 97 { 98 return (((idx) >> ((wq)->wqebbs_per_page_shift)) 99 & ((wq)->num_q_pages - 1)); 100 } 101 /** 102 * queue_alloc_page - allocate page for Queue 103 * @hwif: HW interface for allocating DMA 104 * @vaddr: virtual address will be returned in this address 105 * @paddr: physical address will be returned in this address 106 * @shadow_vaddr: VM area will be return here for holding WQ page addresses 107 * @page_sz: page size of each WQ page 108 * 109 * Return 0 - Success, negative - Failure 110 **/ 111 static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr, 112 void ***shadow_vaddr, size_t page_sz) 113 { 114 struct pci_dev *pdev = hwif->pdev; 115 dma_addr_t dma_addr; 116 117 *vaddr = dma_alloc_coherent(&pdev->dev, page_sz, &dma_addr, 118 GFP_KERNEL); 119 if (!*vaddr) { 120 dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n"); 121 return -ENOMEM; 122 } 123 124 *paddr = (u64)dma_addr; 125 126 /* use vzalloc for big mem */ 127 *shadow_vaddr = vzalloc(page_sz); 128 if (!*shadow_vaddr) 129 goto err_shadow_vaddr; 130 131 return 0; 132 133 err_shadow_vaddr: 134 dma_free_coherent(&pdev->dev, page_sz, *vaddr, dma_addr); 135 return -ENOMEM; 136 } 137 138 /** 139 * wqs_allocate_page - allocate page for WQ set 140 * @wqs: Work Queue Set 141 * @page_idx: the page index of the page will be allocated 142 * 143 * Return 0 - Success, negative - Failure 144 **/ 145 static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx) 146 { 147 return queue_alloc_page(wqs->hwif, &wqs->page_vaddr[page_idx], 148 &wqs->page_paddr[page_idx], 149 &wqs->shadow_page_vaddr[page_idx], 150 WQS_PAGE_SIZE); 151 } 152 153 /** 154 * wqs_free_page - free page of WQ set 155 * @wqs: Work Queue Set 156 * @page_idx: the page index of the page will be freed 157 **/ 158 static void wqs_free_page(struct hinic_wqs *wqs, int page_idx) 159 { 160 struct hinic_hwif *hwif = wqs->hwif; 161 struct pci_dev *pdev = hwif->pdev; 162 163 dma_free_coherent(&pdev->dev, WQS_PAGE_SIZE, 164 wqs->page_vaddr[page_idx], 165 (dma_addr_t)wqs->page_paddr[page_idx]); 166 vfree(wqs->shadow_page_vaddr[page_idx]); 167 } 168 169 /** 170 * cmdq_allocate_page - allocate page for cmdq 171 * @cmdq_pages: the pages of the cmdq queue struct to hold the page 172 * 173 * Return 0 - Success, negative - Failure 174 **/ 175 static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages) 176 { 177 return queue_alloc_page(cmdq_pages->hwif, &cmdq_pages->page_vaddr, 178 &cmdq_pages->page_paddr, 179 &cmdq_pages->shadow_page_vaddr, 180 CMDQ_PAGE_SIZE); 181 } 182 183 /** 184 * cmdq_free_page - free page from cmdq 185 * @cmdq_pages: the pages of the cmdq queue struct that hold the page 186 * 187 * Return 0 - Success, negative - Failure 188 **/ 189 static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages) 190 { 191 struct hinic_hwif *hwif = cmdq_pages->hwif; 192 struct pci_dev *pdev = hwif->pdev; 193 194 dma_free_coherent(&pdev->dev, CMDQ_PAGE_SIZE, 195 cmdq_pages->page_vaddr, 196 (dma_addr_t)cmdq_pages->page_paddr); 197 vfree(cmdq_pages->shadow_page_vaddr); 198 } 199 200 static int alloc_page_arrays(struct hinic_wqs *wqs) 201 { 202 struct hinic_hwif *hwif = wqs->hwif; 203 struct pci_dev *pdev = hwif->pdev; 204 size_t size; 205 206 size = wqs->num_pages * sizeof(*wqs->page_paddr); 207 wqs->page_paddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 208 if (!wqs->page_paddr) 209 return -ENOMEM; 210 211 size = wqs->num_pages * sizeof(*wqs->page_vaddr); 212 wqs->page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 213 if (!wqs->page_vaddr) 214 goto err_page_vaddr; 215 216 size = wqs->num_pages * sizeof(*wqs->shadow_page_vaddr); 217 wqs->shadow_page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 218 if (!wqs->shadow_page_vaddr) 219 goto err_page_shadow_vaddr; 220 221 return 0; 222 223 err_page_shadow_vaddr: 224 devm_kfree(&pdev->dev, wqs->page_vaddr); 225 226 err_page_vaddr: 227 devm_kfree(&pdev->dev, wqs->page_paddr); 228 return -ENOMEM; 229 } 230 231 static void free_page_arrays(struct hinic_wqs *wqs) 232 { 233 struct hinic_hwif *hwif = wqs->hwif; 234 struct pci_dev *pdev = hwif->pdev; 235 236 devm_kfree(&pdev->dev, wqs->shadow_page_vaddr); 237 devm_kfree(&pdev->dev, wqs->page_vaddr); 238 devm_kfree(&pdev->dev, wqs->page_paddr); 239 } 240 241 static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx, 242 int *block_idx) 243 { 244 int pos; 245 246 down(&wqs->alloc_blocks_lock); 247 248 wqs->num_free_blks--; 249 250 if (wqs->num_free_blks < 0) { 251 wqs->num_free_blks++; 252 up(&wqs->alloc_blocks_lock); 253 return -ENOMEM; 254 } 255 256 pos = wqs->alloc_blk_pos++; 257 pos &= WQS_MAX_NUM_BLOCKS - 1; 258 259 *page_idx = wqs->free_blocks[pos].page_idx; 260 *block_idx = wqs->free_blocks[pos].block_idx; 261 262 wqs->free_blocks[pos].page_idx = -1; 263 wqs->free_blocks[pos].block_idx = -1; 264 265 up(&wqs->alloc_blocks_lock); 266 return 0; 267 } 268 269 static void wqs_return_block(struct hinic_wqs *wqs, int page_idx, 270 int block_idx) 271 { 272 int pos; 273 274 down(&wqs->alloc_blocks_lock); 275 276 pos = wqs->return_blk_pos++; 277 pos &= WQS_MAX_NUM_BLOCKS - 1; 278 279 wqs->free_blocks[pos].page_idx = page_idx; 280 wqs->free_blocks[pos].block_idx = block_idx; 281 282 wqs->num_free_blks++; 283 284 up(&wqs->alloc_blocks_lock); 285 } 286 287 static void init_wqs_blocks_arr(struct hinic_wqs *wqs) 288 { 289 int page_idx, blk_idx, pos = 0; 290 291 for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { 292 for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) { 293 wqs->free_blocks[pos].page_idx = page_idx; 294 wqs->free_blocks[pos].block_idx = blk_idx; 295 pos++; 296 } 297 } 298 299 wqs->alloc_blk_pos = 0; 300 wqs->return_blk_pos = pos; 301 wqs->num_free_blks = pos; 302 303 sema_init(&wqs->alloc_blocks_lock, 1); 304 } 305 306 /** 307 * hinic_wqs_alloc - allocate Work Queues set 308 * @wqs: Work Queue Set 309 * @max_wqs: maximum wqs to allocate 310 * @hwif: HW interface for use for the allocation 311 * 312 * Return 0 - Success, negative - Failure 313 **/ 314 int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs, 315 struct hinic_hwif *hwif) 316 { 317 struct pci_dev *pdev = hwif->pdev; 318 int err, i, page_idx; 319 320 max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE); 321 if (max_wqs > WQS_MAX_NUM_BLOCKS) { 322 dev_err(&pdev->dev, "Invalid max_wqs = %d\n", max_wqs); 323 return -EINVAL; 324 } 325 326 wqs->hwif = hwif; 327 wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE; 328 329 if (alloc_page_arrays(wqs)) { 330 dev_err(&pdev->dev, 331 "Failed to allocate mem for page addresses\n"); 332 return -ENOMEM; 333 } 334 335 for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { 336 err = wqs_allocate_page(wqs, page_idx); 337 if (err) { 338 dev_err(&pdev->dev, "Failed wq page allocation\n"); 339 goto err_wq_allocate_page; 340 } 341 } 342 343 wqs->free_blocks = devm_kzalloc(&pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs), 344 GFP_KERNEL); 345 if (!wqs->free_blocks) { 346 err = -ENOMEM; 347 goto err_alloc_blocks; 348 } 349 350 init_wqs_blocks_arr(wqs); 351 return 0; 352 353 err_alloc_blocks: 354 err_wq_allocate_page: 355 for (i = 0; i < page_idx; i++) 356 wqs_free_page(wqs, i); 357 358 free_page_arrays(wqs); 359 return err; 360 } 361 362 /** 363 * hinic_wqs_free - free Work Queues set 364 * @wqs: Work Queue Set 365 **/ 366 void hinic_wqs_free(struct hinic_wqs *wqs) 367 { 368 struct hinic_hwif *hwif = wqs->hwif; 369 struct pci_dev *pdev = hwif->pdev; 370 int page_idx; 371 372 devm_kfree(&pdev->dev, wqs->free_blocks); 373 374 for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) 375 wqs_free_page(wqs, page_idx); 376 377 free_page_arrays(wqs); 378 } 379 380 /** 381 * alloc_wqes_shadow - allocate WQE shadows for WQ 382 * @wq: WQ to allocate shadows for 383 * 384 * Return 0 - Success, negative - Failure 385 **/ 386 static int alloc_wqes_shadow(struct hinic_wq *wq) 387 { 388 struct hinic_hwif *hwif = wq->hwif; 389 struct pci_dev *pdev = hwif->pdev; 390 size_t size; 391 392 size = wq->num_q_pages * wq->max_wqe_size; 393 wq->shadow_wqe = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 394 if (!wq->shadow_wqe) 395 return -ENOMEM; 396 397 size = wq->num_q_pages * sizeof(wq->prod_idx); 398 wq->shadow_idx = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 399 if (!wq->shadow_idx) 400 goto err_shadow_idx; 401 402 return 0; 403 404 err_shadow_idx: 405 devm_kfree(&pdev->dev, wq->shadow_wqe); 406 return -ENOMEM; 407 } 408 409 /** 410 * free_wqes_shadow - free WQE shadows of WQ 411 * @wq: WQ to free shadows from 412 **/ 413 static void free_wqes_shadow(struct hinic_wq *wq) 414 { 415 struct hinic_hwif *hwif = wq->hwif; 416 struct pci_dev *pdev = hwif->pdev; 417 418 devm_kfree(&pdev->dev, wq->shadow_idx); 419 devm_kfree(&pdev->dev, wq->shadow_wqe); 420 } 421 422 /** 423 * free_wq_pages - free pages of WQ 424 * @hwif: HW interface for releasing dma addresses 425 * @wq: WQ to free pages from 426 * @num_q_pages: number pages to free 427 **/ 428 static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, 429 int num_q_pages) 430 { 431 struct pci_dev *pdev = hwif->pdev; 432 int i; 433 434 for (i = 0; i < num_q_pages; i++) { 435 void **vaddr = &wq->shadow_block_vaddr[i]; 436 u64 *paddr = &wq->block_vaddr[i]; 437 dma_addr_t dma_addr; 438 439 dma_addr = (dma_addr_t)be64_to_cpu(*paddr); 440 dma_free_coherent(&pdev->dev, wq->wq_page_size, *vaddr, 441 dma_addr); 442 } 443 444 free_wqes_shadow(wq); 445 } 446 447 /** 448 * alloc_wq_pages - alloc pages for WQ 449 * @hwif: HW interface for allocating dma addresses 450 * @wq: WQ to allocate pages for 451 * @max_pages: maximum pages allowed 452 * 453 * Return 0 - Success, negative - Failure 454 **/ 455 static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, 456 int max_pages) 457 { 458 struct pci_dev *pdev = hwif->pdev; 459 int i, err, num_q_pages; 460 461 num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size; 462 if (num_q_pages > max_pages) { 463 dev_err(&pdev->dev, "Number wq pages exceeds the limit\n"); 464 return -EINVAL; 465 } 466 467 if (num_q_pages & (num_q_pages - 1)) { 468 dev_err(&pdev->dev, "Number wq pages must be power of 2\n"); 469 return -EINVAL; 470 } 471 472 wq->num_q_pages = num_q_pages; 473 474 err = alloc_wqes_shadow(wq); 475 if (err) { 476 dev_err(&pdev->dev, "Failed to allocate wqe shadow\n"); 477 return err; 478 } 479 480 for (i = 0; i < num_q_pages; i++) { 481 void **vaddr = &wq->shadow_block_vaddr[i]; 482 u64 *paddr = &wq->block_vaddr[i]; 483 dma_addr_t dma_addr; 484 485 *vaddr = dma_alloc_coherent(&pdev->dev, wq->wq_page_size, 486 &dma_addr, GFP_KERNEL); 487 if (!*vaddr) { 488 dev_err(&pdev->dev, "Failed to allocate wq page\n"); 489 goto err_alloc_wq_pages; 490 } 491 492 /* HW uses Big Endian Format */ 493 *paddr = cpu_to_be64(dma_addr); 494 } 495 496 return 0; 497 498 err_alloc_wq_pages: 499 free_wq_pages(wq, hwif, i); 500 return -ENOMEM; 501 } 502 503 /** 504 * hinic_wq_allocate - Allocate the WQ resources from the WQS 505 * @wqs: WQ set from which to allocate the WQ resources 506 * @wq: WQ to allocate resources for it from the WQ set 507 * @wqebb_size: Work Queue Block Byte Size 508 * @wq_page_size: the page size in the Work Queue 509 * @q_depth: number of wqebbs in WQ 510 * @max_wqe_size: maximum WQE size that will be used in the WQ 511 * 512 * Return 0 - Success, negative - Failure 513 **/ 514 int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq, 515 u16 wqebb_size, u16 wq_page_size, u16 q_depth, 516 u16 max_wqe_size) 517 { 518 struct hinic_hwif *hwif = wqs->hwif; 519 struct pci_dev *pdev = hwif->pdev; 520 u16 num_wqebbs_per_page; 521 u16 wqebb_size_shift; 522 int err; 523 524 if (!is_power_of_2(wqebb_size)) { 525 dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); 526 return -EINVAL; 527 } 528 529 if (wq_page_size == 0) { 530 dev_err(&pdev->dev, "wq_page_size must be > 0\n"); 531 return -EINVAL; 532 } 533 534 if (q_depth & (q_depth - 1)) { 535 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); 536 return -EINVAL; 537 } 538 539 wqebb_size_shift = ilog2(wqebb_size); 540 num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) 541 >> wqebb_size_shift; 542 543 if (!is_power_of_2(num_wqebbs_per_page)) { 544 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); 545 return -EINVAL; 546 } 547 548 wq->hwif = hwif; 549 550 err = wqs_next_block(wqs, &wq->page_idx, &wq->block_idx); 551 if (err) { 552 dev_err(&pdev->dev, "Failed to get free wqs next block\n"); 553 return err; 554 } 555 556 wq->wqebb_size = wqebb_size; 557 wq->wq_page_size = wq_page_size; 558 wq->q_depth = q_depth; 559 wq->max_wqe_size = max_wqe_size; 560 wq->num_wqebbs_per_page = num_wqebbs_per_page; 561 wq->wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); 562 wq->wqebb_size_shift = wqebb_size_shift; 563 wq->block_vaddr = WQ_BASE_VADDR(wqs, wq); 564 wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq); 565 wq->block_paddr = WQ_BASE_PADDR(wqs, wq); 566 567 err = alloc_wq_pages(wq, wqs->hwif, WQ_MAX_PAGES); 568 if (err) { 569 dev_err(&pdev->dev, "Failed to allocate wq pages\n"); 570 goto err_alloc_wq_pages; 571 } 572 573 atomic_set(&wq->cons_idx, 0); 574 atomic_set(&wq->prod_idx, 0); 575 atomic_set(&wq->delta, q_depth); 576 wq->mask = q_depth - 1; 577 578 return 0; 579 580 err_alloc_wq_pages: 581 wqs_return_block(wqs, wq->page_idx, wq->block_idx); 582 return err; 583 } 584 585 /** 586 * hinic_wq_free - Free the WQ resources to the WQS 587 * @wqs: WQ set to free the WQ resources to it 588 * @wq: WQ to free its resources to the WQ set resources 589 **/ 590 void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq) 591 { 592 free_wq_pages(wq, wqs->hwif, wq->num_q_pages); 593 594 wqs_return_block(wqs, wq->page_idx, wq->block_idx); 595 } 596 597 /** 598 * hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs 599 * @cmdq_pages: will hold the pages of the cmdq 600 * @wq: returned wqs 601 * @hwif: HW interface 602 * @cmdq_blocks: number of cmdq blocks/wq to allocate 603 * @wqebb_size: Work Queue Block Byte Size 604 * @wq_page_size: the page size in the Work Queue 605 * @q_depth: number of wqebbs in WQ 606 * @max_wqe_size: maximum WQE size that will be used in the WQ 607 * 608 * Return 0 - Success, negative - Failure 609 **/ 610 int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages, 611 struct hinic_wq *wq, struct hinic_hwif *hwif, 612 int cmdq_blocks, u16 wqebb_size, u16 wq_page_size, 613 u16 q_depth, u16 max_wqe_size) 614 { 615 struct pci_dev *pdev = hwif->pdev; 616 u16 num_wqebbs_per_page_shift; 617 u16 num_wqebbs_per_page; 618 u16 wqebb_size_shift; 619 int i, j, err = -ENOMEM; 620 621 if (!is_power_of_2(wqebb_size)) { 622 dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); 623 return -EINVAL; 624 } 625 626 if (wq_page_size == 0) { 627 dev_err(&pdev->dev, "wq_page_size must be > 0\n"); 628 return -EINVAL; 629 } 630 631 if (q_depth & (q_depth - 1)) { 632 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); 633 return -EINVAL; 634 } 635 636 wqebb_size_shift = ilog2(wqebb_size); 637 num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) 638 >> wqebb_size_shift; 639 640 if (!is_power_of_2(num_wqebbs_per_page)) { 641 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); 642 return -EINVAL; 643 } 644 645 cmdq_pages->hwif = hwif; 646 647 err = cmdq_allocate_page(cmdq_pages); 648 if (err) { 649 dev_err(&pdev->dev, "Failed to allocate CMDQ page\n"); 650 return err; 651 } 652 num_wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); 653 654 for (i = 0; i < cmdq_blocks; i++) { 655 wq[i].hwif = hwif; 656 wq[i].page_idx = 0; 657 wq[i].block_idx = i; 658 659 wq[i].wqebb_size = wqebb_size; 660 wq[i].wq_page_size = wq_page_size; 661 wq[i].q_depth = q_depth; 662 wq[i].max_wqe_size = max_wqe_size; 663 wq[i].num_wqebbs_per_page = num_wqebbs_per_page; 664 wq[i].wqebbs_per_page_shift = num_wqebbs_per_page_shift; 665 wq[i].wqebb_size_shift = wqebb_size_shift; 666 wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]); 667 wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]); 668 wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]); 669 670 err = alloc_wq_pages(&wq[i], cmdq_pages->hwif, 671 CMDQ_WQ_MAX_PAGES); 672 if (err) { 673 dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n"); 674 goto err_cmdq_block; 675 } 676 677 atomic_set(&wq[i].cons_idx, 0); 678 atomic_set(&wq[i].prod_idx, 0); 679 atomic_set(&wq[i].delta, q_depth); 680 wq[i].mask = q_depth - 1; 681 } 682 683 return 0; 684 685 err_cmdq_block: 686 for (j = 0; j < i; j++) 687 free_wq_pages(&wq[j], cmdq_pages->hwif, wq[j].num_q_pages); 688 689 cmdq_free_page(cmdq_pages); 690 return err; 691 } 692 693 /** 694 * hinic_wqs_cmdq_free - Free wqs from cmdqs 695 * @cmdq_pages: hold the pages of the cmdq 696 * @wq: wqs to free 697 * @cmdq_blocks: number of wqs to free 698 **/ 699 void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages, 700 struct hinic_wq *wq, int cmdq_blocks) 701 { 702 int i; 703 704 for (i = 0; i < cmdq_blocks; i++) 705 free_wq_pages(&wq[i], cmdq_pages->hwif, wq[i].num_q_pages); 706 707 cmdq_free_page(cmdq_pages); 708 } 709 710 static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr, 711 int num_wqebbs, u16 idx) 712 { 713 void *wqebb_addr; 714 int i; 715 716 for (i = 0; i < num_wqebbs; i++, idx++) { 717 idx = MASKED_WQE_IDX(wq, idx); 718 wqebb_addr = WQ_PAGE_ADDR(wq, idx) + 719 WQE_PAGE_OFF(wq, idx); 720 721 memcpy(shadow_addr, wqebb_addr, wq->wqebb_size); 722 723 shadow_addr += wq->wqebb_size; 724 } 725 } 726 727 static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr, 728 int num_wqebbs, u16 idx) 729 { 730 void *wqebb_addr; 731 int i; 732 733 for (i = 0; i < num_wqebbs; i++, idx++) { 734 idx = MASKED_WQE_IDX(wq, idx); 735 wqebb_addr = WQ_PAGE_ADDR(wq, idx) + 736 WQE_PAGE_OFF(wq, idx); 737 738 memcpy(wqebb_addr, shadow_addr, wq->wqebb_size); 739 shadow_addr += wq->wqebb_size; 740 } 741 } 742 743 /** 744 * hinic_get_wqe - get wqe ptr in the current pi and update the pi 745 * @wq: wq to get wqe from 746 * @wqe_size: wqe size 747 * @prod_idx: returned pi 748 * 749 * Return wqe pointer 750 **/ 751 struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size, 752 u16 *prod_idx) 753 { 754 int curr_pg, end_pg, num_wqebbs; 755 u16 curr_prod_idx, end_prod_idx; 756 757 *prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx)); 758 759 num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) >> wq->wqebb_size_shift; 760 761 if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) { 762 atomic_add(num_wqebbs, &wq->delta); 763 return ERR_PTR(-EBUSY); 764 } 765 766 end_prod_idx = atomic_add_return(num_wqebbs, &wq->prod_idx); 767 768 end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx); 769 curr_prod_idx = end_prod_idx - num_wqebbs; 770 curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx); 771 772 /* end prod index points to the next wqebb, therefore minus 1 */ 773 end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1); 774 775 curr_pg = WQE_PAGE_NUM(wq, curr_prod_idx); 776 end_pg = WQE_PAGE_NUM(wq, end_prod_idx); 777 778 *prod_idx = curr_prod_idx; 779 780 if (curr_pg != end_pg) { 781 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; 782 783 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx); 784 785 wq->shadow_idx[curr_pg] = *prod_idx; 786 return shadow_addr; 787 } 788 789 return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, *prod_idx); 790 } 791 792 /** 793 * hinic_return_wqe - return the wqe when transmit failed 794 * @wq: wq to return wqe 795 * @wqe_size: wqe size 796 **/ 797 void hinic_return_wqe(struct hinic_wq *wq, unsigned int wqe_size) 798 { 799 int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; 800 801 atomic_sub(num_wqebbs, &wq->prod_idx); 802 803 atomic_add(num_wqebbs, &wq->delta); 804 } 805 806 /** 807 * hinic_put_wqe - return the wqe place to use for a new wqe 808 * @wq: wq to return wqe 809 * @wqe_size: wqe size 810 **/ 811 void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size) 812 { 813 int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) 814 >> wq->wqebb_size_shift; 815 816 atomic_add(num_wqebbs, &wq->cons_idx); 817 818 atomic_add(num_wqebbs, &wq->delta); 819 } 820 821 /** 822 * hinic_read_wqe - read wqe ptr in the current ci 823 * @wq: wq to get read from 824 * @wqe_size: wqe size 825 * @cons_idx: returned ci 826 * 827 * Return wqe pointer 828 **/ 829 struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size, 830 u16 *cons_idx) 831 { 832 int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) 833 >> wq->wqebb_size_shift; 834 u16 curr_cons_idx, end_cons_idx; 835 int curr_pg, end_pg; 836 837 if ((atomic_read(&wq->delta) + num_wqebbs) > wq->q_depth) 838 return ERR_PTR(-EBUSY); 839 840 curr_cons_idx = atomic_read(&wq->cons_idx); 841 842 curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx); 843 end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1); 844 845 curr_pg = WQE_PAGE_NUM(wq, curr_cons_idx); 846 end_pg = WQE_PAGE_NUM(wq, end_cons_idx); 847 848 *cons_idx = curr_cons_idx; 849 850 if (curr_pg != end_pg) { 851 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; 852 853 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx); 854 return shadow_addr; 855 } 856 857 return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, *cons_idx); 858 } 859 860 /** 861 * hinic_read_wqe_direct - read wqe directly from ci position 862 * @wq: wq 863 * @cons_idx: ci position 864 * 865 * Return wqe 866 **/ 867 struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx) 868 { 869 return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, cons_idx); 870 } 871 872 /** 873 * wqe_shadow - check if a wqe is shadow 874 * @wq: wq of the wqe 875 * @wqe: the wqe for shadow checking 876 * 877 * Return true - shadow, false - Not shadow 878 **/ 879 static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe) 880 { 881 size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size; 882 883 return WQE_IN_RANGE(wqe, wq->shadow_wqe, 884 &wq->shadow_wqe[wqe_shadow_size]); 885 } 886 887 /** 888 * hinic_write_wqe - write the wqe to the wq 889 * @wq: wq to write wqe to 890 * @wqe: wqe to write 891 * @wqe_size: wqe size 892 **/ 893 void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe, 894 unsigned int wqe_size) 895 { 896 int curr_pg, num_wqebbs; 897 void *shadow_addr; 898 u16 prod_idx; 899 900 if (wqe_shadow(wq, wqe)) { 901 curr_pg = WQE_SHADOW_PAGE(wq, wqe); 902 903 prod_idx = wq->shadow_idx[curr_pg]; 904 num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; 905 shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; 906 907 copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, prod_idx); 908 } 909 } 910