1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Huawei HiNIC PCI Express Linux driver
4  * Copyright(c) 2017 Huawei Technologies Co., Ltd
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/pci.h>
10 #include <linux/device.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/vmalloc.h>
13 #include <linux/errno.h>
14 #include <linux/sizes.h>
15 #include <linux/atomic.h>
16 #include <linux/skbuff.h>
17 #include <linux/io.h>
18 #include <asm/barrier.h>
19 #include <asm/byteorder.h>
20 
21 #include "hinic_common.h"
22 #include "hinic_hw_if.h"
23 #include "hinic_hw_wqe.h"
24 #include "hinic_hw_wq.h"
25 #include "hinic_hw_qp_ctxt.h"
26 #include "hinic_hw_qp.h"
27 #include "hinic_hw_io.h"
28 
29 #define SQ_DB_OFF               SZ_2K
30 
31 /* The number of cache line to prefetch Until threshold state */
32 #define WQ_PREFETCH_MAX         2
33 /* The number of cache line to prefetch After threshold state */
34 #define WQ_PREFETCH_MIN         1
35 /* Threshold state */
36 #define WQ_PREFETCH_THRESHOLD   256
37 
38 /* sizes of the SQ/RQ ctxt */
39 #define Q_CTXT_SIZE             48
40 #define CTXT_RSVD               240
41 
42 #define SQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
43 		(((max_rqs) + (max_sqs)) * CTXT_RSVD + (q_id) * Q_CTXT_SIZE)
44 
45 #define RQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
46 		(((max_rqs) + (max_sqs)) * CTXT_RSVD + \
47 		 (max_sqs + (q_id)) * Q_CTXT_SIZE)
48 
49 #define SIZE_16BYTES(size)              (ALIGN(size, 16) >> 4)
50 #define SIZE_8BYTES(size)               (ALIGN(size, 8) >> 3)
51 #define SECT_SIZE_FROM_8BYTES(size)     ((size) << 3)
52 
53 #define SQ_DB_PI_HI_SHIFT       8
54 #define SQ_DB_PI_HI(prod_idx)   ((prod_idx) >> SQ_DB_PI_HI_SHIFT)
55 
56 #define SQ_DB_PI_LOW_MASK       0xFF
57 #define SQ_DB_PI_LOW(prod_idx)  ((prod_idx) & SQ_DB_PI_LOW_MASK)
58 
59 #define SQ_DB_ADDR(sq, pi)      ((u64 *)((sq)->db_base) + SQ_DB_PI_LOW(pi))
60 
61 #define SQ_MASKED_IDX(sq, idx)  ((idx) & (sq)->wq->mask)
62 #define RQ_MASKED_IDX(rq, idx)  ((idx) & (rq)->wq->mask)
63 
64 enum sq_wqe_type {
65 	SQ_NORMAL_WQE = 0,
66 };
67 
68 enum rq_completion_fmt {
69 	RQ_COMPLETE_SGE = 1
70 };
71 
72 void hinic_qp_prepare_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr,
73 			     enum hinic_qp_ctxt_type ctxt_type,
74 			     u16 num_queues, u16 max_queues)
75 {
76 	u16 max_sqs = max_queues;
77 	u16 max_rqs = max_queues;
78 
79 	qp_ctxt_hdr->num_queues = num_queues;
80 	qp_ctxt_hdr->queue_type = ctxt_type;
81 
82 	if (ctxt_type == HINIC_QP_CTXT_TYPE_SQ)
83 		qp_ctxt_hdr->addr_offset = SQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
84 	else
85 		qp_ctxt_hdr->addr_offset = RQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
86 
87 	qp_ctxt_hdr->addr_offset = SIZE_16BYTES(qp_ctxt_hdr->addr_offset);
88 
89 	hinic_cpu_to_be32(qp_ctxt_hdr, sizeof(*qp_ctxt_hdr));
90 }
91 
92 void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt,
93 			   struct hinic_sq *sq, u16 global_qid)
94 {
95 	u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
96 	u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
97 	u16 pi_start, ci_start;
98 	struct hinic_wq *wq;
99 
100 	wq = sq->wq;
101 	ci_start = atomic_read(&wq->cons_idx);
102 	pi_start = atomic_read(&wq->prod_idx);
103 
104 	/* Read the first page paddr from the WQ page paddr ptrs */
105 	wq_page_addr = be64_to_cpu(*wq->block_vaddr);
106 
107 	wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
108 	wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
109 	wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
110 
111 	/* If only one page, use 0-level CLA */
112 	if (wq->num_q_pages == 1)
113 		wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq_page_addr);
114 	else
115 		wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
116 
117 	wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
118 	wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
119 
120 	sq_ctxt->ceq_attr = HINIC_SQ_CTXT_CEQ_ATTR_SET(global_qid,
121 						       GLOBAL_SQ_ID) |
122 			    HINIC_SQ_CTXT_CEQ_ATTR_SET(0, EN);
123 
124 	sq_ctxt->ci_wrapped = HINIC_SQ_CTXT_CI_SET(ci_start, IDX) |
125 			      HINIC_SQ_CTXT_CI_SET(1, WRAPPED);
126 
127 	sq_ctxt->wq_hi_pfn_pi =
128 			HINIC_SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |
129 			HINIC_SQ_CTXT_WQ_PAGE_SET(pi_start, PI);
130 
131 	sq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
132 
133 	sq_ctxt->pref_cache =
134 		HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
135 		HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
136 		HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
137 
138 	sq_ctxt->pref_wrapped = 1;
139 
140 	sq_ctxt->pref_wq_hi_pfn_ci =
141 		HINIC_SQ_CTXT_PREF_SET(ci_start, CI) |
142 		HINIC_SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN);
143 
144 	sq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
145 
146 	sq_ctxt->wq_block_hi_pfn =
147 		HINIC_SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
148 
149 	sq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
150 
151 	hinic_cpu_to_be32(sq_ctxt, sizeof(*sq_ctxt));
152 }
153 
154 void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt,
155 			   struct hinic_rq *rq, u16 global_qid)
156 {
157 	u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
158 	u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
159 	u16 pi_start, ci_start;
160 	struct hinic_wq *wq;
161 
162 	wq = rq->wq;
163 	ci_start = atomic_read(&wq->cons_idx);
164 	pi_start = atomic_read(&wq->prod_idx);
165 
166 	/* Read the first page paddr from the WQ page paddr ptrs */
167 	wq_page_addr = be64_to_cpu(*wq->block_vaddr);
168 
169 	wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
170 	wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
171 	wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
172 
173 	wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
174 	wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
175 	wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
176 
177 	rq_ctxt->ceq_attr = HINIC_RQ_CTXT_CEQ_ATTR_SET(0, EN) |
178 			    HINIC_RQ_CTXT_CEQ_ATTR_SET(1, WRAPPED);
179 
180 	rq_ctxt->pi_intr_attr = HINIC_RQ_CTXT_PI_SET(pi_start, IDX) |
181 				HINIC_RQ_CTXT_PI_SET(rq->msix_entry, INTR);
182 
183 	rq_ctxt->wq_hi_pfn_ci = HINIC_RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi,
184 							  HI_PFN) |
185 				HINIC_RQ_CTXT_WQ_PAGE_SET(ci_start, CI);
186 
187 	rq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
188 
189 	rq_ctxt->pref_cache =
190 		HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
191 		HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
192 		HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
193 
194 	rq_ctxt->pref_wrapped = 1;
195 
196 	rq_ctxt->pref_wq_hi_pfn_ci =
197 		HINIC_RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN) |
198 		HINIC_RQ_CTXT_PREF_SET(ci_start, CI);
199 
200 	rq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
201 
202 	rq_ctxt->pi_paddr_hi = upper_32_bits(rq->pi_dma_addr);
203 	rq_ctxt->pi_paddr_lo = lower_32_bits(rq->pi_dma_addr);
204 
205 	rq_ctxt->wq_block_hi_pfn =
206 		HINIC_RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
207 
208 	rq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
209 
210 	hinic_cpu_to_be32(rq_ctxt, sizeof(*rq_ctxt));
211 }
212 
213 /**
214  * alloc_sq_skb_arr - allocate sq array for saved skb
215  * @sq: HW Send Queue
216  *
217  * Return 0 - Success, negative - Failure
218  **/
219 static int alloc_sq_skb_arr(struct hinic_sq *sq)
220 {
221 	struct hinic_wq *wq = sq->wq;
222 	size_t skb_arr_size;
223 
224 	skb_arr_size = wq->q_depth * sizeof(*sq->saved_skb);
225 	sq->saved_skb = vzalloc(skb_arr_size);
226 	if (!sq->saved_skb)
227 		return -ENOMEM;
228 
229 	return 0;
230 }
231 
232 /**
233  * free_sq_skb_arr - free sq array for saved skb
234  * @sq: HW Send Queue
235  **/
236 static void free_sq_skb_arr(struct hinic_sq *sq)
237 {
238 	vfree(sq->saved_skb);
239 }
240 
241 /**
242  * alloc_rq_skb_arr - allocate rq array for saved skb
243  * @rq: HW Receive Queue
244  *
245  * Return 0 - Success, negative - Failure
246  **/
247 static int alloc_rq_skb_arr(struct hinic_rq *rq)
248 {
249 	struct hinic_wq *wq = rq->wq;
250 	size_t skb_arr_size;
251 
252 	skb_arr_size = wq->q_depth * sizeof(*rq->saved_skb);
253 	rq->saved_skb = vzalloc(skb_arr_size);
254 	if (!rq->saved_skb)
255 		return -ENOMEM;
256 
257 	return 0;
258 }
259 
260 /**
261  * free_rq_skb_arr - free rq array for saved skb
262  * @rq: HW Receive Queue
263  **/
264 static void free_rq_skb_arr(struct hinic_rq *rq)
265 {
266 	vfree(rq->saved_skb);
267 }
268 
269 /**
270  * hinic_init_sq - Initialize HW Send Queue
271  * @sq: HW Send Queue
272  * @hwif: HW Interface for accessing HW
273  * @wq: Work Queue for the data of the SQ
274  * @entry: msix entry for sq
275  * @ci_addr: address for reading the current HW consumer index
276  * @ci_dma_addr: dma address for reading the current HW consumer index
277  * @db_base: doorbell base address
278  *
279  * Return 0 - Success, negative - Failure
280  **/
281 int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif,
282 		  struct hinic_wq *wq, struct msix_entry *entry,
283 		  void *ci_addr, dma_addr_t ci_dma_addr,
284 		  void __iomem *db_base)
285 {
286 	sq->hwif = hwif;
287 
288 	sq->wq = wq;
289 
290 	sq->irq = entry->vector;
291 	sq->msix_entry = entry->entry;
292 
293 	sq->hw_ci_addr = ci_addr;
294 	sq->hw_ci_dma_addr = ci_dma_addr;
295 
296 	sq->db_base = db_base + SQ_DB_OFF;
297 
298 	return alloc_sq_skb_arr(sq);
299 }
300 
301 /**
302  * hinic_clean_sq - Clean HW Send Queue's Resources
303  * @sq: Send Queue
304  **/
305 void hinic_clean_sq(struct hinic_sq *sq)
306 {
307 	free_sq_skb_arr(sq);
308 }
309 
310 /**
311  * alloc_rq_cqe - allocate rq completion queue elements
312  * @rq: HW Receive Queue
313  *
314  * Return 0 - Success, negative - Failure
315  **/
316 static int alloc_rq_cqe(struct hinic_rq *rq)
317 {
318 	struct hinic_hwif *hwif = rq->hwif;
319 	struct pci_dev *pdev = hwif->pdev;
320 	size_t cqe_dma_size, cqe_size;
321 	struct hinic_wq *wq = rq->wq;
322 	int j, i;
323 
324 	cqe_size = wq->q_depth * sizeof(*rq->cqe);
325 	rq->cqe = vzalloc(cqe_size);
326 	if (!rq->cqe)
327 		return -ENOMEM;
328 
329 	cqe_dma_size = wq->q_depth * sizeof(*rq->cqe_dma);
330 	rq->cqe_dma = vzalloc(cqe_dma_size);
331 	if (!rq->cqe_dma)
332 		goto err_cqe_dma_arr_alloc;
333 
334 	for (i = 0; i < wq->q_depth; i++) {
335 		rq->cqe[i] = dma_alloc_coherent(&pdev->dev,
336 						sizeof(*rq->cqe[i]),
337 						&rq->cqe_dma[i], GFP_KERNEL);
338 		if (!rq->cqe[i])
339 			goto err_cqe_alloc;
340 	}
341 
342 	return 0;
343 
344 err_cqe_alloc:
345 	for (j = 0; j < i; j++)
346 		dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[j]), rq->cqe[j],
347 				  rq->cqe_dma[j]);
348 
349 	vfree(rq->cqe_dma);
350 
351 err_cqe_dma_arr_alloc:
352 	vfree(rq->cqe);
353 	return -ENOMEM;
354 }
355 
356 /**
357  * free_rq_cqe - free rq completion queue elements
358  * @rq: HW Receive Queue
359  **/
360 static void free_rq_cqe(struct hinic_rq *rq)
361 {
362 	struct hinic_hwif *hwif = rq->hwif;
363 	struct pci_dev *pdev = hwif->pdev;
364 	struct hinic_wq *wq = rq->wq;
365 	int i;
366 
367 	for (i = 0; i < wq->q_depth; i++)
368 		dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[i]), rq->cqe[i],
369 				  rq->cqe_dma[i]);
370 
371 	vfree(rq->cqe_dma);
372 	vfree(rq->cqe);
373 }
374 
375 /**
376  * hinic_init_rq - Initialize HW Receive Queue
377  * @rq: HW Receive Queue
378  * @hwif: HW Interface for accessing HW
379  * @wq: Work Queue for the data of the RQ
380  * @entry: msix entry for rq
381  *
382  * Return 0 - Success, negative - Failure
383  **/
384 int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif,
385 		  struct hinic_wq *wq, struct msix_entry *entry)
386 {
387 	struct pci_dev *pdev = hwif->pdev;
388 	size_t pi_size;
389 	int err;
390 
391 	rq->hwif = hwif;
392 
393 	rq->wq = wq;
394 
395 	rq->irq = entry->vector;
396 	rq->msix_entry = entry->entry;
397 
398 	rq->buf_sz = HINIC_RX_BUF_SZ;
399 
400 	err = alloc_rq_skb_arr(rq);
401 	if (err) {
402 		dev_err(&pdev->dev, "Failed to allocate rq priv data\n");
403 		return err;
404 	}
405 
406 	err = alloc_rq_cqe(rq);
407 	if (err) {
408 		dev_err(&pdev->dev, "Failed to allocate rq cqe\n");
409 		goto err_alloc_rq_cqe;
410 	}
411 
412 	/* HW requirements: Must be at least 32 bit */
413 	pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
414 	rq->pi_virt_addr = dma_alloc_coherent(&pdev->dev, pi_size,
415 					      &rq->pi_dma_addr, GFP_KERNEL);
416 	if (!rq->pi_virt_addr) {
417 		err = -ENOMEM;
418 		goto err_pi_virt;
419 	}
420 
421 	return 0;
422 
423 err_pi_virt:
424 	free_rq_cqe(rq);
425 
426 err_alloc_rq_cqe:
427 	free_rq_skb_arr(rq);
428 	return err;
429 }
430 
431 /**
432  * hinic_clean_rq - Clean HW Receive Queue's Resources
433  * @rq: HW Receive Queue
434  **/
435 void hinic_clean_rq(struct hinic_rq *rq)
436 {
437 	struct hinic_hwif *hwif = rq->hwif;
438 	struct pci_dev *pdev = hwif->pdev;
439 	size_t pi_size;
440 
441 	pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
442 	dma_free_coherent(&pdev->dev, pi_size, rq->pi_virt_addr,
443 			  rq->pi_dma_addr);
444 
445 	free_rq_cqe(rq);
446 	free_rq_skb_arr(rq);
447 }
448 
449 /**
450  * hinic_get_sq_free_wqebbs - return number of free wqebbs for use
451  * @sq: send queue
452  *
453  * Return number of free wqebbs
454  **/
455 int hinic_get_sq_free_wqebbs(struct hinic_sq *sq)
456 {
457 	struct hinic_wq *wq = sq->wq;
458 
459 	return atomic_read(&wq->delta) - 1;
460 }
461 
462 /**
463  * hinic_get_rq_free_wqebbs - return number of free wqebbs for use
464  * @rq: recv queue
465  *
466  * Return number of free wqebbs
467  **/
468 int hinic_get_rq_free_wqebbs(struct hinic_rq *rq)
469 {
470 	struct hinic_wq *wq = rq->wq;
471 
472 	return atomic_read(&wq->delta) - 1;
473 }
474 
475 static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, u16 prod_idx,
476 			    int nr_descs)
477 {
478 	u32 ctrl_size, task_size, bufdesc_size;
479 
480 	ctrl_size = SIZE_8BYTES(sizeof(struct hinic_sq_ctrl));
481 	task_size = SIZE_8BYTES(sizeof(struct hinic_sq_task));
482 	bufdesc_size = nr_descs * sizeof(struct hinic_sq_bufdesc);
483 	bufdesc_size = SIZE_8BYTES(bufdesc_size);
484 
485 	ctrl->ctrl_info = HINIC_SQ_CTRL_SET(bufdesc_size, BUFDESC_SECT_LEN) |
486 			  HINIC_SQ_CTRL_SET(task_size, TASKSECT_LEN)        |
487 			  HINIC_SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT)     |
488 			  HINIC_SQ_CTRL_SET(ctrl_size, LEN);
489 
490 	ctrl->queue_info = HINIC_SQ_CTRL_SET(HINIC_MSS_DEFAULT,
491 					     QUEUE_INFO_MSS) |
492 			   HINIC_SQ_CTRL_SET(1, QUEUE_INFO_UC);
493 }
494 
495 static void sq_prepare_task(struct hinic_sq_task *task)
496 {
497 	task->pkt_info0 = 0;
498 	task->pkt_info1 = 0;
499 	task->pkt_info2 = 0;
500 
501 	task->ufo_v6_identify = 0;
502 
503 	task->pkt_info4 = HINIC_SQ_TASK_INFO4_SET(HINIC_L2TYPE_ETH, L2TYPE);
504 
505 	task->zero_pad = 0;
506 }
507 
508 void hinic_task_set_l2hdr(struct hinic_sq_task *task, u32 len)
509 {
510 	task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(len, L2HDR_LEN);
511 }
512 
513 void hinic_task_set_outter_l3(struct hinic_sq_task *task,
514 			      enum hinic_l3_offload_type l3_type,
515 			      u32 network_len)
516 {
517 	task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l3_type, OUTER_L3TYPE) |
518 			   HINIC_SQ_TASK_INFO2_SET(network_len, OUTER_L3LEN);
519 }
520 
521 void hinic_task_set_inner_l3(struct hinic_sq_task *task,
522 			     enum hinic_l3_offload_type l3_type,
523 			     u32 network_len)
524 {
525 	task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l3_type, INNER_L3TYPE);
526 	task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(network_len, INNER_L3LEN);
527 }
528 
529 void hinic_task_set_tunnel_l4(struct hinic_sq_task *task,
530 			      enum hinic_l4_tunnel_type l4_type,
531 			      u32 tunnel_len)
532 {
533 	task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l4_type, TUNNEL_L4TYPE) |
534 			   HINIC_SQ_TASK_INFO2_SET(tunnel_len, TUNNEL_L4LEN);
535 }
536 
537 void hinic_set_cs_inner_l4(struct hinic_sq_task *task, u32 *queue_info,
538 			   enum hinic_l4_offload_type l4_offload,
539 			   u32 l4_len, u32 offset)
540 {
541 	u32 tcp_udp_cs = 0, sctp = 0;
542 	u32 mss = HINIC_MSS_DEFAULT;
543 
544 	if (l4_offload == TCP_OFFLOAD_ENABLE ||
545 	    l4_offload == UDP_OFFLOAD_ENABLE)
546 		tcp_udp_cs = 1;
547 	else if (l4_offload == SCTP_OFFLOAD_ENABLE)
548 		sctp = 1;
549 
550 	task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD);
551 	task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN);
552 
553 	*queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) |
554 		       HINIC_SQ_CTRL_SET(tcp_udp_cs, QUEUE_INFO_TCPUDP_CS) |
555 		       HINIC_SQ_CTRL_SET(sctp, QUEUE_INFO_SCTP);
556 
557 	*queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);
558 	*queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS);
559 }
560 
561 void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 *queue_info,
562 			    enum hinic_l4_offload_type l4_offload,
563 			    u32 l4_len, u32 offset, u32 ip_ident, u32 mss)
564 {
565 	u32 tso = 0, ufo = 0;
566 
567 	if (l4_offload == TCP_OFFLOAD_ENABLE)
568 		tso = 1;
569 	else if (l4_offload == UDP_OFFLOAD_ENABLE)
570 		ufo = 1;
571 
572 	task->ufo_v6_identify = ip_ident;
573 
574 	task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD);
575 	task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(tso || ufo, TSO_FLAG);
576 	task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN);
577 
578 	*queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) |
579 		       HINIC_SQ_CTRL_SET(tso, QUEUE_INFO_TSO) |
580 		       HINIC_SQ_CTRL_SET(ufo, QUEUE_INFO_UFO) |
581 		       HINIC_SQ_CTRL_SET(!!l4_offload, QUEUE_INFO_TCPUDP_CS);
582 
583 	/* set MSS value */
584 	*queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);
585 	*queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS);
586 }
587 
588 /**
589  * hinic_sq_prepare_wqe - prepare wqe before insert to the queue
590  * @sq: send queue
591  * @prod_idx: pi value
592  * @sq_wqe: wqe to prepare
593  * @sges: sges for use by the wqe for send for buf addresses
594  * @nr_sges: number of sges
595  **/
596 void hinic_sq_prepare_wqe(struct hinic_sq *sq, u16 prod_idx,
597 			  struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
598 			  int nr_sges)
599 {
600 	int i;
601 
602 	sq_prepare_ctrl(&sq_wqe->ctrl, prod_idx, nr_sges);
603 
604 	sq_prepare_task(&sq_wqe->task);
605 
606 	for (i = 0; i < nr_sges; i++)
607 		sq_wqe->buf_descs[i].sge = sges[i];
608 }
609 
610 /**
611  * sq_prepare_db - prepare doorbell to write
612  * @sq: send queue
613  * @prod_idx: pi value for the doorbell
614  * @cos: cos of the doorbell
615  *
616  * Return db value
617  **/
618 static u32 sq_prepare_db(struct hinic_sq *sq, u16 prod_idx, unsigned int cos)
619 {
620 	struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq);
621 	u8 hi_prod_idx = SQ_DB_PI_HI(SQ_MASKED_IDX(sq, prod_idx));
622 
623 	/* Data should be written to HW in Big Endian Format */
624 	return cpu_to_be32(HINIC_SQ_DB_INFO_SET(hi_prod_idx, PI_HI)     |
625 			   HINIC_SQ_DB_INFO_SET(HINIC_DB_SQ_TYPE, TYPE) |
626 			   HINIC_SQ_DB_INFO_SET(HINIC_DATA_PATH, PATH)  |
627 			   HINIC_SQ_DB_INFO_SET(cos, COS)               |
628 			   HINIC_SQ_DB_INFO_SET(qp->q_id, QID));
629 }
630 
631 /**
632  * hinic_sq_write_db- write doorbell
633  * @sq: send queue
634  * @prod_idx: pi value for the doorbell
635  * @wqe_size: wqe size
636  * @cos: cos of the wqe
637  **/
638 void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size,
639 		       unsigned int cos)
640 {
641 	struct hinic_wq *wq = sq->wq;
642 
643 	/* increment prod_idx to the next */
644 	prod_idx += ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
645 	prod_idx = SQ_MASKED_IDX(sq, prod_idx);
646 
647 	wmb();  /* Write all before the doorbell */
648 
649 	writel(sq_prepare_db(sq, prod_idx, cos), SQ_DB_ADDR(sq, prod_idx));
650 }
651 
652 /**
653  * hinic_sq_get_wqe - get wqe ptr in the current pi and update the pi
654  * @sq: sq to get wqe from
655  * @wqe_size: wqe size
656  * @prod_idx: returned pi
657  *
658  * Return wqe pointer
659  **/
660 struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq,
661 				      unsigned int wqe_size, u16 *prod_idx)
662 {
663 	struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(sq->wq, wqe_size,
664 						    prod_idx);
665 
666 	if (IS_ERR(hw_wqe))
667 		return NULL;
668 
669 	return &hw_wqe->sq_wqe;
670 }
671 
672 /**
673  * hinic_sq_return_wqe - return the wqe to the sq
674  * @sq: send queue
675  * @wqe_size: the size of the wqe
676  **/
677 void hinic_sq_return_wqe(struct hinic_sq *sq, unsigned int wqe_size)
678 {
679 	hinic_return_wqe(sq->wq, wqe_size);
680 }
681 
682 /**
683  * hinic_sq_write_wqe - write the wqe to the sq
684  * @sq: send queue
685  * @prod_idx: pi of the wqe
686  * @sq_wqe: the wqe to write
687  * @skb: skb to save
688  * @wqe_size: the size of the wqe
689  **/
690 void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx,
691 			struct hinic_sq_wqe *sq_wqe,
692 			struct sk_buff *skb, unsigned int wqe_size)
693 {
694 	struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)sq_wqe;
695 
696 	sq->saved_skb[prod_idx] = skb;
697 
698 	/* The data in the HW should be in Big Endian Format */
699 	hinic_cpu_to_be32(sq_wqe, wqe_size);
700 
701 	hinic_write_wqe(sq->wq, hw_wqe, wqe_size);
702 }
703 
704 /**
705  * hinic_sq_read_wqebb - read wqe ptr in the current ci and update the ci, the
706  * wqe only have one wqebb
707  * @sq: send queue
708  * @skb: return skb that was saved
709  * @wqe_size: the wqe size ptr
710  * @cons_idx: consumer index of the wqe
711  *
712  * Return wqe in ci position
713  **/
714 struct hinic_sq_wqe *hinic_sq_read_wqebb(struct hinic_sq *sq,
715 					 struct sk_buff **skb,
716 					 unsigned int *wqe_size, u16 *cons_idx)
717 {
718 	struct hinic_hw_wqe *hw_wqe;
719 	struct hinic_sq_wqe *sq_wqe;
720 	struct hinic_sq_ctrl *ctrl;
721 	unsigned int buf_sect_len;
722 	u32 ctrl_info;
723 
724 	/* read the ctrl section for getting wqe size */
725 	hw_wqe = hinic_read_wqe(sq->wq, sizeof(*ctrl), cons_idx);
726 	if (IS_ERR(hw_wqe))
727 		return NULL;
728 
729 	*skb = sq->saved_skb[*cons_idx];
730 
731 	sq_wqe = &hw_wqe->sq_wqe;
732 	ctrl = &sq_wqe->ctrl;
733 	ctrl_info = be32_to_cpu(ctrl->ctrl_info);
734 	buf_sect_len = HINIC_SQ_CTRL_GET(ctrl_info, BUFDESC_SECT_LEN);
735 
736 	*wqe_size = sizeof(*ctrl) + sizeof(sq_wqe->task);
737 	*wqe_size += SECT_SIZE_FROM_8BYTES(buf_sect_len);
738 	*wqe_size = ALIGN(*wqe_size, sq->wq->wqebb_size);
739 
740 	return &hw_wqe->sq_wqe;
741 }
742 
743 /**
744  * hinic_sq_read_wqe - read wqe ptr in the current ci and update the ci
745  * @sq: send queue
746  * @skb: return skb that was saved
747  * @wqe_size: the size of the wqe
748  * @cons_idx: consumer index of the wqe
749  *
750  * Return wqe in ci position
751  **/
752 struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq,
753 				       struct sk_buff **skb,
754 				       unsigned int wqe_size, u16 *cons_idx)
755 {
756 	struct hinic_hw_wqe *hw_wqe;
757 
758 	hw_wqe = hinic_read_wqe(sq->wq, wqe_size, cons_idx);
759 	*skb = sq->saved_skb[*cons_idx];
760 
761 	return &hw_wqe->sq_wqe;
762 }
763 
764 /**
765  * hinic_sq_put_wqe - release the ci for new wqes
766  * @sq: send queue
767  * @wqe_size: the size of the wqe
768  **/
769 void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size)
770 {
771 	hinic_put_wqe(sq->wq, wqe_size);
772 }
773 
774 /**
775  * hinic_sq_get_sges - get sges from the wqe
776  * @sq_wqe: wqe to get the sges from its buffer addresses
777  * @sges: returned sges
778  * @nr_sges: number sges to return
779  **/
780 void hinic_sq_get_sges(struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
781 		       int nr_sges)
782 {
783 	int i;
784 
785 	for (i = 0; i < nr_sges && i < HINIC_MAX_SQ_BUFDESCS; i++) {
786 		sges[i] = sq_wqe->buf_descs[i].sge;
787 		hinic_be32_to_cpu(&sges[i], sizeof(sges[i]));
788 	}
789 }
790 
791 /**
792  * hinic_rq_get_wqe - get wqe ptr in the current pi and update the pi
793  * @rq: rq to get wqe from
794  * @wqe_size: wqe size
795  * @prod_idx: returned pi
796  *
797  * Return wqe pointer
798  **/
799 struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq,
800 				      unsigned int wqe_size, u16 *prod_idx)
801 {
802 	struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(rq->wq, wqe_size,
803 						    prod_idx);
804 
805 	if (IS_ERR(hw_wqe))
806 		return NULL;
807 
808 	return &hw_wqe->rq_wqe;
809 }
810 
811 /**
812  * hinic_rq_write_wqe - write the wqe to the rq
813  * @rq: recv queue
814  * @prod_idx: pi of the wqe
815  * @rq_wqe: the wqe to write
816  * @skb: skb to save
817  **/
818 void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx,
819 			struct hinic_rq_wqe *rq_wqe, struct sk_buff *skb)
820 {
821 	struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)rq_wqe;
822 
823 	rq->saved_skb[prod_idx] = skb;
824 
825 	/* The data in the HW should be in Big Endian Format */
826 	hinic_cpu_to_be32(rq_wqe, sizeof(*rq_wqe));
827 
828 	hinic_write_wqe(rq->wq, hw_wqe, sizeof(*rq_wqe));
829 }
830 
831 /**
832  * hinic_rq_read_wqe - read wqe ptr in the current ci and update the ci
833  * @rq: recv queue
834  * @wqe_size: the size of the wqe
835  * @skb: return saved skb
836  * @cons_idx: consumer index of the wqe
837  *
838  * Return wqe in ci position
839  **/
840 struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq,
841 				       unsigned int wqe_size,
842 				       struct sk_buff **skb, u16 *cons_idx)
843 {
844 	struct hinic_hw_wqe *hw_wqe;
845 	struct hinic_rq_cqe *cqe;
846 	int rx_done;
847 	u32 status;
848 
849 	hw_wqe = hinic_read_wqe(rq->wq, wqe_size, cons_idx);
850 	if (IS_ERR(hw_wqe))
851 		return NULL;
852 
853 	cqe = rq->cqe[*cons_idx];
854 
855 	status = be32_to_cpu(cqe->status);
856 
857 	rx_done = HINIC_RQ_CQE_STATUS_GET(status, RXDONE);
858 	if (!rx_done)
859 		return NULL;
860 
861 	*skb = rq->saved_skb[*cons_idx];
862 
863 	return &hw_wqe->rq_wqe;
864 }
865 
866 /**
867  * hinic_rq_read_next_wqe - increment ci and read the wqe in ci position
868  * @rq: recv queue
869  * @wqe_size: the size of the wqe
870  * @skb: return saved skb
871  * @cons_idx: consumer index in the wq
872  *
873  * Return wqe in incremented ci position
874  **/
875 struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq,
876 					    unsigned int wqe_size,
877 					    struct sk_buff **skb,
878 					    u16 *cons_idx)
879 {
880 	struct hinic_wq *wq = rq->wq;
881 	struct hinic_hw_wqe *hw_wqe;
882 	unsigned int num_wqebbs;
883 
884 	wqe_size = ALIGN(wqe_size, wq->wqebb_size);
885 	num_wqebbs = wqe_size / wq->wqebb_size;
886 
887 	*cons_idx = RQ_MASKED_IDX(rq, *cons_idx + num_wqebbs);
888 
889 	*skb = rq->saved_skb[*cons_idx];
890 
891 	hw_wqe = hinic_read_wqe_direct(wq, *cons_idx);
892 
893 	return &hw_wqe->rq_wqe;
894 }
895 
896 /**
897  * hinic_rq_put_wqe - release the ci for new wqes
898  * @rq: recv queue
899  * @cons_idx: consumer index of the wqe
900  * @wqe_size: the size of the wqe
901  **/
902 void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx,
903 		      unsigned int wqe_size)
904 {
905 	struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
906 	u32 status = be32_to_cpu(cqe->status);
907 
908 	status = HINIC_RQ_CQE_STATUS_CLEAR(status, RXDONE);
909 
910 	/* Rx WQE size is 1 WQEBB, no wq shadow*/
911 	cqe->status = cpu_to_be32(status);
912 
913 	wmb();          /* clear done flag */
914 
915 	hinic_put_wqe(rq->wq, wqe_size);
916 }
917 
918 /**
919  * hinic_rq_get_sge - get sge from the wqe
920  * @rq: recv queue
921  * @rq_wqe: wqe to get the sge from its buf address
922  * @cons_idx: consumer index
923  * @sge: returned sge
924  **/
925 void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *rq_wqe,
926 		      u16 cons_idx, struct hinic_sge *sge)
927 {
928 	struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
929 	u32 len = be32_to_cpu(cqe->len);
930 
931 	sge->hi_addr = be32_to_cpu(rq_wqe->buf_desc.hi_addr);
932 	sge->lo_addr = be32_to_cpu(rq_wqe->buf_desc.lo_addr);
933 	sge->len = HINIC_RQ_CQE_SGE_GET(len, LEN);
934 }
935 
936 /**
937  * hinic_rq_prepare_wqe - prepare wqe before insert to the queue
938  * @rq: recv queue
939  * @prod_idx: pi value
940  * @rq_wqe: the wqe
941  * @sge: sge for use by the wqe for recv buf address
942  **/
943 void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx,
944 			  struct hinic_rq_wqe *rq_wqe, struct hinic_sge *sge)
945 {
946 	struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
947 	struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
948 	struct hinic_rq_cqe *cqe = rq->cqe[prod_idx];
949 	struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
950 	dma_addr_t cqe_dma = rq->cqe_dma[prod_idx];
951 
952 	ctrl->ctrl_info =
953 		HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) |
954 		HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)),
955 				  COMPLETE_LEN)                    |
956 		HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)),
957 				  BUFDESC_SECT_LEN)                |
958 		HINIC_RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
959 
960 	hinic_set_sge(&cqe_sect->sge, cqe_dma, sizeof(*cqe));
961 
962 	buf_desc->hi_addr = sge->hi_addr;
963 	buf_desc->lo_addr = sge->lo_addr;
964 }
965 
966 /**
967  * hinic_rq_update - update pi of the rq
968  * @rq: recv queue
969  * @prod_idx: pi value
970  **/
971 void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx)
972 {
973 	*rq->pi_virt_addr = cpu_to_be16(RQ_MASKED_IDX(rq, prod_idx + 1));
974 }
975