1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Huawei HiNIC PCI Express Linux driver
4  * Copyright(c) 2017 Huawei Technologies Co., Ltd
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/pci.h>
10 #include <linux/device.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/slab.h>
13 #include <linux/atomic.h>
14 #include <linux/semaphore.h>
15 #include <linux/errno.h>
16 #include <linux/vmalloc.h>
17 #include <linux/err.h>
18 #include <asm/byteorder.h>
19 
20 #include "hinic_hw_if.h"
21 #include "hinic_hw_wqe.h"
22 #include "hinic_hw_wq.h"
23 #include "hinic_hw_cmdq.h"
24 
25 #define WQS_BLOCKS_PER_PAGE             4
26 
27 #define WQ_BLOCK_SIZE                   4096
28 #define WQS_PAGE_SIZE                   (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE)
29 
30 #define WQS_MAX_NUM_BLOCKS              128
31 #define WQS_FREE_BLOCKS_SIZE(wqs)       (WQS_MAX_NUM_BLOCKS * \
32 					 sizeof((wqs)->free_blocks[0]))
33 
34 #define WQ_SIZE(wq)                     ((wq)->q_depth * (wq)->wqebb_size)
35 
36 #define WQ_PAGE_ADDR_SIZE               sizeof(u64)
37 #define WQ_MAX_PAGES                    (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE)
38 
39 #define CMDQ_BLOCK_SIZE                 512
40 #define CMDQ_PAGE_SIZE                  4096
41 
42 #define CMDQ_WQ_MAX_PAGES               (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE)
43 
44 #define WQ_BASE_VADDR(wqs, wq)          \
45 			((void *)((wqs)->page_vaddr[(wq)->page_idx]) \
46 				+ (wq)->block_idx * WQ_BLOCK_SIZE)
47 
48 #define WQ_BASE_PADDR(wqs, wq)          \
49 			((wqs)->page_paddr[(wq)->page_idx] \
50 				+ (wq)->block_idx * WQ_BLOCK_SIZE)
51 
52 #define WQ_BASE_ADDR(wqs, wq)           \
53 			((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \
54 				+ (wq)->block_idx * WQ_BLOCK_SIZE)
55 
56 #define CMDQ_BASE_VADDR(cmdq_pages, wq) \
57 			((void *)((cmdq_pages)->page_vaddr) \
58 				+ (wq)->block_idx * CMDQ_BLOCK_SIZE)
59 
60 #define CMDQ_BASE_PADDR(cmdq_pages, wq) \
61 			((cmdq_pages)->page_paddr \
62 				+ (wq)->block_idx * CMDQ_BLOCK_SIZE)
63 
64 #define CMDQ_BASE_ADDR(cmdq_pages, wq)  \
65 			((void *)((cmdq_pages)->shadow_page_vaddr) \
66 				+ (wq)->block_idx * CMDQ_BLOCK_SIZE)
67 
68 #define WQ_PAGE_ADDR(wq, idx)           \
69 			((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)])
70 
71 #define MASKED_WQE_IDX(wq, idx)         ((idx) & (wq)->mask)
72 
73 #define WQE_IN_RANGE(wqe, start, end)   \
74 		(((unsigned long)(wqe) >= (unsigned long)(start)) && \
75 		 ((unsigned long)(wqe) < (unsigned long)(end)))
76 
77 #define WQE_SHADOW_PAGE(wq, wqe)        \
78 		(((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \
79 			/ (wq)->max_wqe_size)
80 
WQE_PAGE_OFF(struct hinic_wq * wq,u16 idx)81 static inline int WQE_PAGE_OFF(struct hinic_wq *wq, u16 idx)
82 {
83 	return (((idx) & ((wq)->num_wqebbs_per_page - 1))
84 		<< (wq)->wqebb_size_shift);
85 }
86 
WQE_PAGE_NUM(struct hinic_wq * wq,u16 idx)87 static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx)
88 {
89 	return (((idx) >> ((wq)->wqebbs_per_page_shift))
90 		& ((wq)->num_q_pages - 1));
91 }
92 
93 /**
94  * queue_alloc_page - allocate page for Queue
95  * @hwif: HW interface for allocating DMA
96  * @vaddr: virtual address will be returned in this address
97  * @paddr: physical address will be returned in this address
98  * @shadow_vaddr: VM area will be return here for holding WQ page addresses
99  * @page_sz: page size of each WQ page
100  *
101  * Return 0 - Success, negative - Failure
102  **/
queue_alloc_page(struct hinic_hwif * hwif,u64 ** vaddr,u64 * paddr,void *** shadow_vaddr,size_t page_sz)103 static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr,
104 			    void ***shadow_vaddr, size_t page_sz)
105 {
106 	struct pci_dev *pdev = hwif->pdev;
107 	dma_addr_t dma_addr;
108 
109 	*vaddr = dma_alloc_coherent(&pdev->dev, page_sz, &dma_addr,
110 				    GFP_KERNEL);
111 	if (!*vaddr) {
112 		dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n");
113 		return -ENOMEM;
114 	}
115 
116 	*paddr = (u64)dma_addr;
117 
118 	/* use vzalloc for big mem */
119 	*shadow_vaddr = vzalloc(page_sz);
120 	if (!*shadow_vaddr)
121 		goto err_shadow_vaddr;
122 
123 	return 0;
124 
125 err_shadow_vaddr:
126 	dma_free_coherent(&pdev->dev, page_sz, *vaddr, dma_addr);
127 	return -ENOMEM;
128 }
129 
130 /**
131  * wqs_allocate_page - allocate page for WQ set
132  * @wqs: Work Queue Set
133  * @page_idx: the page index of the page will be allocated
134  *
135  * Return 0 - Success, negative - Failure
136  **/
wqs_allocate_page(struct hinic_wqs * wqs,int page_idx)137 static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx)
138 {
139 	return queue_alloc_page(wqs->hwif, &wqs->page_vaddr[page_idx],
140 				&wqs->page_paddr[page_idx],
141 				&wqs->shadow_page_vaddr[page_idx],
142 				WQS_PAGE_SIZE);
143 }
144 
145 /**
146  * wqs_free_page - free page of WQ set
147  * @wqs: Work Queue Set
148  * @page_idx: the page index of the page will be freed
149  **/
wqs_free_page(struct hinic_wqs * wqs,int page_idx)150 static void wqs_free_page(struct hinic_wqs *wqs, int page_idx)
151 {
152 	struct hinic_hwif *hwif = wqs->hwif;
153 	struct pci_dev *pdev = hwif->pdev;
154 
155 	dma_free_coherent(&pdev->dev, WQS_PAGE_SIZE,
156 			  wqs->page_vaddr[page_idx],
157 			  (dma_addr_t)wqs->page_paddr[page_idx]);
158 	vfree(wqs->shadow_page_vaddr[page_idx]);
159 }
160 
161 /**
162  * cmdq_allocate_page - allocate page for cmdq
163  * @cmdq_pages: the pages of the cmdq queue struct to hold the page
164  *
165  * Return 0 - Success, negative - Failure
166  **/
cmdq_allocate_page(struct hinic_cmdq_pages * cmdq_pages)167 static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages)
168 {
169 	return queue_alloc_page(cmdq_pages->hwif, &cmdq_pages->page_vaddr,
170 				&cmdq_pages->page_paddr,
171 				&cmdq_pages->shadow_page_vaddr,
172 				CMDQ_PAGE_SIZE);
173 }
174 
175 /**
176  * cmdq_free_page - free page from cmdq
177  * @cmdq_pages: the pages of the cmdq queue struct that hold the page
178  **/
cmdq_free_page(struct hinic_cmdq_pages * cmdq_pages)179 static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages)
180 {
181 	struct hinic_hwif *hwif = cmdq_pages->hwif;
182 	struct pci_dev *pdev = hwif->pdev;
183 
184 	dma_free_coherent(&pdev->dev, CMDQ_PAGE_SIZE,
185 			  cmdq_pages->page_vaddr,
186 			  (dma_addr_t)cmdq_pages->page_paddr);
187 	vfree(cmdq_pages->shadow_page_vaddr);
188 }
189 
alloc_page_arrays(struct hinic_wqs * wqs)190 static int alloc_page_arrays(struct hinic_wqs *wqs)
191 {
192 	struct hinic_hwif *hwif = wqs->hwif;
193 	struct pci_dev *pdev = hwif->pdev;
194 
195 	wqs->page_paddr = devm_kcalloc(&pdev->dev, wqs->num_pages,
196 				       sizeof(*wqs->page_paddr), GFP_KERNEL);
197 	if (!wqs->page_paddr)
198 		return -ENOMEM;
199 
200 	wqs->page_vaddr = devm_kcalloc(&pdev->dev, wqs->num_pages,
201 				       sizeof(*wqs->page_vaddr), GFP_KERNEL);
202 	if (!wqs->page_vaddr)
203 		goto err_page_vaddr;
204 
205 	wqs->shadow_page_vaddr = devm_kcalloc(&pdev->dev, wqs->num_pages,
206 					      sizeof(*wqs->shadow_page_vaddr),
207 					      GFP_KERNEL);
208 	if (!wqs->shadow_page_vaddr)
209 		goto err_page_shadow_vaddr;
210 
211 	return 0;
212 
213 err_page_shadow_vaddr:
214 	devm_kfree(&pdev->dev, wqs->page_vaddr);
215 
216 err_page_vaddr:
217 	devm_kfree(&pdev->dev, wqs->page_paddr);
218 	return -ENOMEM;
219 }
220 
free_page_arrays(struct hinic_wqs * wqs)221 static void free_page_arrays(struct hinic_wqs *wqs)
222 {
223 	struct hinic_hwif *hwif = wqs->hwif;
224 	struct pci_dev *pdev = hwif->pdev;
225 
226 	devm_kfree(&pdev->dev, wqs->shadow_page_vaddr);
227 	devm_kfree(&pdev->dev, wqs->page_vaddr);
228 	devm_kfree(&pdev->dev, wqs->page_paddr);
229 }
230 
wqs_next_block(struct hinic_wqs * wqs,int * page_idx,int * block_idx)231 static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx,
232 			  int *block_idx)
233 {
234 	int pos;
235 
236 	down(&wqs->alloc_blocks_lock);
237 
238 	wqs->num_free_blks--;
239 
240 	if (wqs->num_free_blks < 0) {
241 		wqs->num_free_blks++;
242 		up(&wqs->alloc_blocks_lock);
243 		return -ENOMEM;
244 	}
245 
246 	pos = wqs->alloc_blk_pos++;
247 	pos &= WQS_MAX_NUM_BLOCKS - 1;
248 
249 	*page_idx = wqs->free_blocks[pos].page_idx;
250 	*block_idx = wqs->free_blocks[pos].block_idx;
251 
252 	wqs->free_blocks[pos].page_idx = -1;
253 	wqs->free_blocks[pos].block_idx = -1;
254 
255 	up(&wqs->alloc_blocks_lock);
256 	return 0;
257 }
258 
wqs_return_block(struct hinic_wqs * wqs,int page_idx,int block_idx)259 static void wqs_return_block(struct hinic_wqs *wqs, int page_idx,
260 			     int block_idx)
261 {
262 	int pos;
263 
264 	down(&wqs->alloc_blocks_lock);
265 
266 	pos = wqs->return_blk_pos++;
267 	pos &= WQS_MAX_NUM_BLOCKS - 1;
268 
269 	wqs->free_blocks[pos].page_idx = page_idx;
270 	wqs->free_blocks[pos].block_idx = block_idx;
271 
272 	wqs->num_free_blks++;
273 
274 	up(&wqs->alloc_blocks_lock);
275 }
276 
init_wqs_blocks_arr(struct hinic_wqs * wqs)277 static void init_wqs_blocks_arr(struct hinic_wqs *wqs)
278 {
279 	int page_idx, blk_idx, pos = 0;
280 
281 	for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) {
282 		for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) {
283 			wqs->free_blocks[pos].page_idx = page_idx;
284 			wqs->free_blocks[pos].block_idx = blk_idx;
285 			pos++;
286 		}
287 	}
288 
289 	wqs->alloc_blk_pos = 0;
290 	wqs->return_blk_pos = pos;
291 	wqs->num_free_blks = pos;
292 
293 	sema_init(&wqs->alloc_blocks_lock, 1);
294 }
295 
296 /**
297  * hinic_wqs_alloc - allocate Work Queues set
298  * @wqs: Work Queue Set
299  * @max_wqs: maximum wqs to allocate
300  * @hwif: HW interface for use for the allocation
301  *
302  * Return 0 - Success, negative - Failure
303  **/
hinic_wqs_alloc(struct hinic_wqs * wqs,int max_wqs,struct hinic_hwif * hwif)304 int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs,
305 		    struct hinic_hwif *hwif)
306 {
307 	struct pci_dev *pdev = hwif->pdev;
308 	int err, i, page_idx;
309 
310 	max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE);
311 	if (max_wqs > WQS_MAX_NUM_BLOCKS)  {
312 		dev_err(&pdev->dev, "Invalid max_wqs = %d\n", max_wqs);
313 		return -EINVAL;
314 	}
315 
316 	wqs->hwif = hwif;
317 	wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE;
318 
319 	if (alloc_page_arrays(wqs)) {
320 		dev_err(&pdev->dev,
321 			"Failed to allocate mem for page addresses\n");
322 		return -ENOMEM;
323 	}
324 
325 	for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) {
326 		err = wqs_allocate_page(wqs, page_idx);
327 		if (err) {
328 			dev_err(&pdev->dev, "Failed wq page allocation\n");
329 			goto err_wq_allocate_page;
330 		}
331 	}
332 
333 	wqs->free_blocks = devm_kzalloc(&pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs),
334 					GFP_KERNEL);
335 	if (!wqs->free_blocks) {
336 		err = -ENOMEM;
337 		goto err_alloc_blocks;
338 	}
339 
340 	init_wqs_blocks_arr(wqs);
341 	return 0;
342 
343 err_alloc_blocks:
344 err_wq_allocate_page:
345 	for (i = 0; i < page_idx; i++)
346 		wqs_free_page(wqs, i);
347 
348 	free_page_arrays(wqs);
349 	return err;
350 }
351 
352 /**
353  * hinic_wqs_free - free Work Queues set
354  * @wqs: Work Queue Set
355  **/
hinic_wqs_free(struct hinic_wqs * wqs)356 void hinic_wqs_free(struct hinic_wqs *wqs)
357 {
358 	struct hinic_hwif *hwif = wqs->hwif;
359 	struct pci_dev *pdev = hwif->pdev;
360 	int page_idx;
361 
362 	devm_kfree(&pdev->dev, wqs->free_blocks);
363 
364 	for (page_idx = 0; page_idx < wqs->num_pages; page_idx++)
365 		wqs_free_page(wqs, page_idx);
366 
367 	free_page_arrays(wqs);
368 }
369 
370 /**
371  * alloc_wqes_shadow - allocate WQE shadows for WQ
372  * @wq: WQ to allocate shadows for
373  *
374  * Return 0 - Success, negative - Failure
375  **/
alloc_wqes_shadow(struct hinic_wq * wq)376 static int alloc_wqes_shadow(struct hinic_wq *wq)
377 {
378 	struct hinic_hwif *hwif = wq->hwif;
379 	struct pci_dev *pdev = hwif->pdev;
380 
381 	wq->shadow_wqe = devm_kcalloc(&pdev->dev, wq->num_q_pages,
382 				      wq->max_wqe_size, GFP_KERNEL);
383 	if (!wq->shadow_wqe)
384 		return -ENOMEM;
385 
386 	wq->shadow_idx = devm_kcalloc(&pdev->dev, wq->num_q_pages,
387 				      sizeof(*wq->shadow_idx), GFP_KERNEL);
388 	if (!wq->shadow_idx)
389 		goto err_shadow_idx;
390 
391 	return 0;
392 
393 err_shadow_idx:
394 	devm_kfree(&pdev->dev, wq->shadow_wqe);
395 	return -ENOMEM;
396 }
397 
398 /**
399  * free_wqes_shadow - free WQE shadows of WQ
400  * @wq: WQ to free shadows from
401  **/
free_wqes_shadow(struct hinic_wq * wq)402 static void free_wqes_shadow(struct hinic_wq *wq)
403 {
404 	struct hinic_hwif *hwif = wq->hwif;
405 	struct pci_dev *pdev = hwif->pdev;
406 
407 	devm_kfree(&pdev->dev, wq->shadow_idx);
408 	devm_kfree(&pdev->dev, wq->shadow_wqe);
409 }
410 
411 /**
412  * free_wq_pages - free pages of WQ
413  * @hwif: HW interface for releasing dma addresses
414  * @wq: WQ to free pages from
415  * @num_q_pages: number pages to free
416  **/
free_wq_pages(struct hinic_wq * wq,struct hinic_hwif * hwif,int num_q_pages)417 static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif,
418 			  int num_q_pages)
419 {
420 	struct pci_dev *pdev = hwif->pdev;
421 	int i;
422 
423 	for (i = 0; i < num_q_pages; i++) {
424 		void **vaddr = &wq->shadow_block_vaddr[i];
425 		u64 *paddr = &wq->block_vaddr[i];
426 		dma_addr_t dma_addr;
427 
428 		dma_addr = (dma_addr_t)be64_to_cpu(*paddr);
429 		dma_free_coherent(&pdev->dev, wq->wq_page_size, *vaddr,
430 				  dma_addr);
431 	}
432 
433 	free_wqes_shadow(wq);
434 }
435 
436 /**
437  * alloc_wq_pages - alloc pages for WQ
438  * @hwif: HW interface for allocating dma addresses
439  * @wq: WQ to allocate pages for
440  * @max_pages: maximum pages allowed
441  *
442  * Return 0 - Success, negative - Failure
443  **/
alloc_wq_pages(struct hinic_wq * wq,struct hinic_hwif * hwif,int max_pages)444 static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif,
445 			  int max_pages)
446 {
447 	struct pci_dev *pdev = hwif->pdev;
448 	int i, err, num_q_pages;
449 
450 	num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size;
451 	if (num_q_pages > max_pages) {
452 		dev_err(&pdev->dev, "Number wq pages exceeds the limit\n");
453 		return -EINVAL;
454 	}
455 
456 	if (num_q_pages & (num_q_pages - 1)) {
457 		dev_err(&pdev->dev, "Number wq pages must be power of 2\n");
458 		return -EINVAL;
459 	}
460 
461 	wq->num_q_pages = num_q_pages;
462 
463 	err = alloc_wqes_shadow(wq);
464 	if (err) {
465 		dev_err(&pdev->dev, "Failed to allocate wqe shadow\n");
466 		return err;
467 	}
468 
469 	for (i = 0; i < num_q_pages; i++) {
470 		void **vaddr = &wq->shadow_block_vaddr[i];
471 		u64 *paddr = &wq->block_vaddr[i];
472 		dma_addr_t dma_addr;
473 
474 		*vaddr = dma_alloc_coherent(&pdev->dev, wq->wq_page_size,
475 					    &dma_addr, GFP_KERNEL);
476 		if (!*vaddr) {
477 			dev_err(&pdev->dev, "Failed to allocate wq page\n");
478 			goto err_alloc_wq_pages;
479 		}
480 
481 		/* HW uses Big Endian Format */
482 		*paddr = cpu_to_be64(dma_addr);
483 	}
484 
485 	return 0;
486 
487 err_alloc_wq_pages:
488 	free_wq_pages(wq, hwif, i);
489 	return -ENOMEM;
490 }
491 
492 /**
493  * hinic_wq_allocate - Allocate the WQ resources from the WQS
494  * @wqs: WQ set from which to allocate the WQ resources
495  * @wq: WQ to allocate resources for it from the WQ set
496  * @wqebb_size: Work Queue Block Byte Size
497  * @wq_page_size: the page size in the Work Queue
498  * @q_depth: number of wqebbs in WQ
499  * @max_wqe_size: maximum WQE size that will be used in the WQ
500  *
501  * Return 0 - Success, negative - Failure
502  **/
hinic_wq_allocate(struct hinic_wqs * wqs,struct hinic_wq * wq,u16 wqebb_size,u32 wq_page_size,u16 q_depth,u16 max_wqe_size)503 int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq,
504 		      u16 wqebb_size, u32 wq_page_size, u16 q_depth,
505 		      u16 max_wqe_size)
506 {
507 	struct hinic_hwif *hwif = wqs->hwif;
508 	struct pci_dev *pdev = hwif->pdev;
509 	u16 num_wqebbs_per_page;
510 	u16 wqebb_size_shift;
511 	int err;
512 
513 	if (!is_power_of_2(wqebb_size)) {
514 		dev_err(&pdev->dev, "wqebb_size must be power of 2\n");
515 		return -EINVAL;
516 	}
517 
518 	if (wq_page_size == 0) {
519 		dev_err(&pdev->dev, "wq_page_size must be > 0\n");
520 		return -EINVAL;
521 	}
522 
523 	if (q_depth & (q_depth - 1)) {
524 		dev_err(&pdev->dev, "WQ q_depth must be power of 2\n");
525 		return -EINVAL;
526 	}
527 
528 	wqebb_size_shift = ilog2(wqebb_size);
529 	num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size)
530 				>> wqebb_size_shift;
531 
532 	if (!is_power_of_2(num_wqebbs_per_page)) {
533 		dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
534 		return -EINVAL;
535 	}
536 
537 	wq->hwif = hwif;
538 
539 	err = wqs_next_block(wqs, &wq->page_idx, &wq->block_idx);
540 	if (err) {
541 		dev_err(&pdev->dev, "Failed to get free wqs next block\n");
542 		return err;
543 	}
544 
545 	wq->wqebb_size = wqebb_size;
546 	wq->wq_page_size = wq_page_size;
547 	wq->q_depth = q_depth;
548 	wq->max_wqe_size = max_wqe_size;
549 	wq->num_wqebbs_per_page = num_wqebbs_per_page;
550 	wq->wqebbs_per_page_shift = ilog2(num_wqebbs_per_page);
551 	wq->wqebb_size_shift = wqebb_size_shift;
552 	wq->block_vaddr = WQ_BASE_VADDR(wqs, wq);
553 	wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq);
554 	wq->block_paddr = WQ_BASE_PADDR(wqs, wq);
555 
556 	err = alloc_wq_pages(wq, wqs->hwif, WQ_MAX_PAGES);
557 	if (err) {
558 		dev_err(&pdev->dev, "Failed to allocate wq pages\n");
559 		goto err_alloc_wq_pages;
560 	}
561 
562 	atomic_set(&wq->cons_idx, 0);
563 	atomic_set(&wq->prod_idx, 0);
564 	atomic_set(&wq->delta, q_depth);
565 	wq->mask = q_depth - 1;
566 
567 	return 0;
568 
569 err_alloc_wq_pages:
570 	wqs_return_block(wqs, wq->page_idx, wq->block_idx);
571 	return err;
572 }
573 
574 /**
575  * hinic_wq_free - Free the WQ resources to the WQS
576  * @wqs: WQ set to free the WQ resources to it
577  * @wq: WQ to free its resources to the WQ set resources
578  **/
hinic_wq_free(struct hinic_wqs * wqs,struct hinic_wq * wq)579 void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq)
580 {
581 	free_wq_pages(wq, wqs->hwif, wq->num_q_pages);
582 
583 	wqs_return_block(wqs, wq->page_idx, wq->block_idx);
584 }
585 
586 /**
587  * hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs
588  * @cmdq_pages: will hold the pages of the cmdq
589  * @wq: returned wqs
590  * @hwif: HW interface
591  * @cmdq_blocks: number of cmdq blocks/wq to allocate
592  * @wqebb_size: Work Queue Block Byte Size
593  * @wq_page_size: the page size in the Work Queue
594  * @q_depth: number of wqebbs in WQ
595  * @max_wqe_size: maximum WQE size that will be used in the WQ
596  *
597  * Return 0 - Success, negative - Failure
598  **/
hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages * cmdq_pages,struct hinic_wq * wq,struct hinic_hwif * hwif,int cmdq_blocks,u16 wqebb_size,u32 wq_page_size,u16 q_depth,u16 max_wqe_size)599 int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages,
600 			 struct hinic_wq *wq, struct hinic_hwif *hwif,
601 			 int cmdq_blocks, u16 wqebb_size, u32 wq_page_size,
602 			 u16 q_depth, u16 max_wqe_size)
603 {
604 	struct pci_dev *pdev = hwif->pdev;
605 	u16 num_wqebbs_per_page_shift;
606 	u16 num_wqebbs_per_page;
607 	u16 wqebb_size_shift;
608 	int i, j, err = -ENOMEM;
609 
610 	if (!is_power_of_2(wqebb_size)) {
611 		dev_err(&pdev->dev, "wqebb_size must be power of 2\n");
612 		return -EINVAL;
613 	}
614 
615 	if (wq_page_size == 0) {
616 		dev_err(&pdev->dev, "wq_page_size must be > 0\n");
617 		return -EINVAL;
618 	}
619 
620 	if (q_depth & (q_depth - 1)) {
621 		dev_err(&pdev->dev, "WQ q_depth must be power of 2\n");
622 		return -EINVAL;
623 	}
624 
625 	wqebb_size_shift = ilog2(wqebb_size);
626 	num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size)
627 				>> wqebb_size_shift;
628 
629 	if (!is_power_of_2(num_wqebbs_per_page)) {
630 		dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
631 		return -EINVAL;
632 	}
633 
634 	cmdq_pages->hwif = hwif;
635 
636 	err = cmdq_allocate_page(cmdq_pages);
637 	if (err) {
638 		dev_err(&pdev->dev, "Failed to allocate CMDQ page\n");
639 		return err;
640 	}
641 	num_wqebbs_per_page_shift = ilog2(num_wqebbs_per_page);
642 
643 	for (i = 0; i < cmdq_blocks; i++) {
644 		wq[i].hwif = hwif;
645 		wq[i].page_idx = 0;
646 		wq[i].block_idx = i;
647 
648 		wq[i].wqebb_size = wqebb_size;
649 		wq[i].wq_page_size = wq_page_size;
650 		wq[i].q_depth = q_depth;
651 		wq[i].max_wqe_size = max_wqe_size;
652 		wq[i].num_wqebbs_per_page = num_wqebbs_per_page;
653 		wq[i].wqebbs_per_page_shift = num_wqebbs_per_page_shift;
654 		wq[i].wqebb_size_shift = wqebb_size_shift;
655 		wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]);
656 		wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]);
657 		wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]);
658 
659 		err = alloc_wq_pages(&wq[i], cmdq_pages->hwif,
660 				     CMDQ_WQ_MAX_PAGES);
661 		if (err) {
662 			dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n");
663 			goto err_cmdq_block;
664 		}
665 
666 		atomic_set(&wq[i].cons_idx, 0);
667 		atomic_set(&wq[i].prod_idx, 0);
668 		atomic_set(&wq[i].delta, q_depth);
669 		wq[i].mask = q_depth - 1;
670 	}
671 
672 	return 0;
673 
674 err_cmdq_block:
675 	for (j = 0; j < i; j++)
676 		free_wq_pages(&wq[j], cmdq_pages->hwif, wq[j].num_q_pages);
677 
678 	cmdq_free_page(cmdq_pages);
679 	return err;
680 }
681 
682 /**
683  * hinic_wqs_cmdq_free - Free wqs from cmdqs
684  * @cmdq_pages: hold the pages of the cmdq
685  * @wq: wqs to free
686  * @cmdq_blocks: number of wqs to free
687  **/
hinic_wqs_cmdq_free(struct hinic_cmdq_pages * cmdq_pages,struct hinic_wq * wq,int cmdq_blocks)688 void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages,
689 			 struct hinic_wq *wq, int cmdq_blocks)
690 {
691 	int i;
692 
693 	for (i = 0; i < cmdq_blocks; i++)
694 		free_wq_pages(&wq[i], cmdq_pages->hwif, wq[i].num_q_pages);
695 
696 	cmdq_free_page(cmdq_pages);
697 }
698 
copy_wqe_to_shadow(struct hinic_wq * wq,void * shadow_addr,int num_wqebbs,u16 idx)699 static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr,
700 			       int num_wqebbs, u16 idx)
701 {
702 	void *wqebb_addr;
703 	int i;
704 
705 	for (i = 0; i < num_wqebbs; i++, idx++) {
706 		idx = MASKED_WQE_IDX(wq, idx);
707 		wqebb_addr = WQ_PAGE_ADDR(wq, idx) +
708 			     WQE_PAGE_OFF(wq, idx);
709 
710 		memcpy(shadow_addr, wqebb_addr, wq->wqebb_size);
711 
712 		shadow_addr += wq->wqebb_size;
713 	}
714 }
715 
copy_wqe_from_shadow(struct hinic_wq * wq,void * shadow_addr,int num_wqebbs,u16 idx)716 static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr,
717 				 int num_wqebbs, u16 idx)
718 {
719 	void *wqebb_addr;
720 	int i;
721 
722 	for (i = 0; i < num_wqebbs; i++, idx++) {
723 		idx = MASKED_WQE_IDX(wq, idx);
724 		wqebb_addr = WQ_PAGE_ADDR(wq, idx) +
725 			     WQE_PAGE_OFF(wq, idx);
726 
727 		memcpy(wqebb_addr, shadow_addr, wq->wqebb_size);
728 		shadow_addr += wq->wqebb_size;
729 	}
730 }
731 
732 /**
733  * hinic_get_wqe - get wqe ptr in the current pi and update the pi
734  * @wq: wq to get wqe from
735  * @wqe_size: wqe size
736  * @prod_idx: returned pi
737  *
738  * Return wqe pointer
739  **/
hinic_get_wqe(struct hinic_wq * wq,unsigned int wqe_size,u16 * prod_idx)740 struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size,
741 				   u16 *prod_idx)
742 {
743 	int curr_pg, end_pg, num_wqebbs;
744 	u16 curr_prod_idx, end_prod_idx;
745 
746 	*prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx));
747 
748 	num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) >> wq->wqebb_size_shift;
749 
750 	if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) {
751 		atomic_add(num_wqebbs, &wq->delta);
752 		return ERR_PTR(-EBUSY);
753 	}
754 
755 	end_prod_idx = atomic_add_return(num_wqebbs, &wq->prod_idx);
756 
757 	end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx);
758 	curr_prod_idx = end_prod_idx - num_wqebbs;
759 	curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx);
760 
761 	/* end prod index points to the next wqebb, therefore minus 1 */
762 	end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1);
763 
764 	curr_pg = WQE_PAGE_NUM(wq, curr_prod_idx);
765 	end_pg = WQE_PAGE_NUM(wq, end_prod_idx);
766 
767 	*prod_idx = curr_prod_idx;
768 
769 	/* If we only have one page, still need to get shadown wqe when
770 	 * wqe rolling-over page
771 	 */
772 	if (curr_pg != end_pg || end_prod_idx < *prod_idx) {
773 		void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
774 
775 		copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx);
776 
777 		wq->shadow_idx[curr_pg] = *prod_idx;
778 		return shadow_addr;
779 	}
780 
781 	return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, *prod_idx);
782 }
783 
784 /**
785  * hinic_return_wqe - return the wqe when transmit failed
786  * @wq: wq to return wqe
787  * @wqe_size: wqe size
788  **/
hinic_return_wqe(struct hinic_wq * wq,unsigned int wqe_size)789 void hinic_return_wqe(struct hinic_wq *wq, unsigned int wqe_size)
790 {
791 	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
792 
793 	atomic_sub(num_wqebbs, &wq->prod_idx);
794 
795 	atomic_add(num_wqebbs, &wq->delta);
796 }
797 
798 /**
799  * hinic_put_wqe - return the wqe place to use for a new wqe
800  * @wq: wq to return wqe
801  * @wqe_size: wqe size
802  **/
hinic_put_wqe(struct hinic_wq * wq,unsigned int wqe_size)803 void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size)
804 {
805 	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size)
806 			>> wq->wqebb_size_shift;
807 
808 	atomic_add(num_wqebbs, &wq->cons_idx);
809 
810 	atomic_add(num_wqebbs, &wq->delta);
811 }
812 
813 /**
814  * hinic_read_wqe - read wqe ptr in the current ci
815  * @wq: wq to get read from
816  * @wqe_size: wqe size
817  * @cons_idx: returned ci
818  *
819  * Return wqe pointer
820  **/
hinic_read_wqe(struct hinic_wq * wq,unsigned int wqe_size,u16 * cons_idx)821 struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size,
822 				    u16 *cons_idx)
823 {
824 	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size)
825 			>> wq->wqebb_size_shift;
826 	u16 curr_cons_idx, end_cons_idx;
827 	int curr_pg, end_pg;
828 
829 	if ((atomic_read(&wq->delta) + num_wqebbs) > wq->q_depth)
830 		return ERR_PTR(-EBUSY);
831 
832 	curr_cons_idx = atomic_read(&wq->cons_idx);
833 
834 	curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx);
835 	end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1);
836 
837 	curr_pg = WQE_PAGE_NUM(wq, curr_cons_idx);
838 	end_pg = WQE_PAGE_NUM(wq, end_cons_idx);
839 
840 	*cons_idx = curr_cons_idx;
841 
842 	/* If we only have one page, still need to get shadown wqe when
843 	 * wqe rolling-over page
844 	 */
845 	if (curr_pg != end_pg || end_cons_idx < curr_cons_idx) {
846 		void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
847 
848 		copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx);
849 		return shadow_addr;
850 	}
851 
852 	return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, *cons_idx);
853 }
854 
855 /**
856  * hinic_read_wqe_direct - read wqe directly from ci position
857  * @wq: wq
858  * @cons_idx: ci position
859  *
860  * Return wqe
861  **/
hinic_read_wqe_direct(struct hinic_wq * wq,u16 cons_idx)862 struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx)
863 {
864 	return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, cons_idx);
865 }
866 
867 /**
868  * wqe_shadow - check if a wqe is shadow
869  * @wq: wq of the wqe
870  * @wqe: the wqe for shadow checking
871  *
872  * Return true - shadow, false - Not shadow
873  **/
wqe_shadow(struct hinic_wq * wq,struct hinic_hw_wqe * wqe)874 static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe)
875 {
876 	size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size;
877 
878 	return WQE_IN_RANGE(wqe, wq->shadow_wqe,
879 			    &wq->shadow_wqe[wqe_shadow_size]);
880 }
881 
882 /**
883  * hinic_write_wqe - write the wqe to the wq
884  * @wq: wq to write wqe to
885  * @wqe: wqe to write
886  * @wqe_size: wqe size
887  **/
hinic_write_wqe(struct hinic_wq * wq,struct hinic_hw_wqe * wqe,unsigned int wqe_size)888 void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe,
889 		     unsigned int wqe_size)
890 {
891 	int curr_pg, num_wqebbs;
892 	void *shadow_addr;
893 	u16 prod_idx;
894 
895 	if (wqe_shadow(wq, wqe)) {
896 		curr_pg = WQE_SHADOW_PAGE(wq, wqe);
897 
898 		prod_idx = wq->shadow_idx[curr_pg];
899 		num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
900 		shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
901 
902 		copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, prod_idx);
903 	}
904 }
905