1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Huawei HiNIC PCI Express Linux driver
4  * Copyright(c) 2017 Huawei Technologies Co., Ltd
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/errno.h>
10 #include <linux/pci.h>
11 #include <linux/device.h>
12 #include <linux/netdevice.h>
13 #include <linux/etherdevice.h>
14 #include <linux/u64_stats_sync.h>
15 #include <linux/slab.h>
16 #include <linux/interrupt.h>
17 #include <linux/skbuff.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/prefetch.h>
20 #include <linux/cpumask.h>
21 #include <linux/if_vlan.h>
22 #include <asm/barrier.h>
23 
24 #include "hinic_common.h"
25 #include "hinic_hw_if.h"
26 #include "hinic_hw_wqe.h"
27 #include "hinic_hw_wq.h"
28 #include "hinic_hw_qp.h"
29 #include "hinic_hw_dev.h"
30 #include "hinic_rx.h"
31 #include "hinic_dev.h"
32 
33 #define RX_IRQ_NO_PENDING               0
34 #define RX_IRQ_NO_COALESC               0
35 #define RX_IRQ_NO_LLI_TIMER             0
36 #define RX_IRQ_NO_CREDIT                0
37 #define RX_IRQ_NO_RESEND_TIMER          0
38 #define HINIC_RX_BUFFER_WRITE           16
39 
40 #define HINIC_RX_IPV6_PKT		7
41 #define LRO_PKT_HDR_LEN_IPV4		66
42 #define LRO_PKT_HDR_LEN_IPV6		86
43 #define LRO_REPLENISH_THLD		256
44 
45 #define LRO_PKT_HDR_LEN(cqe)		\
46 	(HINIC_GET_RX_PKT_TYPE(be32_to_cpu((cqe)->offload_type)) == \
47 	 HINIC_RX_IPV6_PKT ? LRO_PKT_HDR_LEN_IPV6 : LRO_PKT_HDR_LEN_IPV4)
48 
49 /**
50  * hinic_rxq_clean_stats - Clean the statistics of specific queue
51  * @rxq: Logical Rx Queue
52  **/
53 void hinic_rxq_clean_stats(struct hinic_rxq *rxq)
54 {
55 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
56 
57 	u64_stats_update_begin(&rxq_stats->syncp);
58 	rxq_stats->pkts  = 0;
59 	rxq_stats->bytes = 0;
60 	rxq_stats->errors = 0;
61 	rxq_stats->csum_errors = 0;
62 	rxq_stats->other_errors = 0;
63 	u64_stats_update_end(&rxq_stats->syncp);
64 }
65 
66 /**
67  * hinic_rxq_get_stats - get statistics of Rx Queue
68  * @rxq: Logical Rx Queue
69  * @stats: return updated stats here
70  **/
71 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
72 {
73 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
74 	unsigned int start;
75 
76 	u64_stats_update_begin(&stats->syncp);
77 	do {
78 		start = u64_stats_fetch_begin(&rxq_stats->syncp);
79 		stats->pkts = rxq_stats->pkts;
80 		stats->bytes = rxq_stats->bytes;
81 		stats->errors = rxq_stats->csum_errors +
82 				rxq_stats->other_errors;
83 		stats->csum_errors = rxq_stats->csum_errors;
84 		stats->other_errors = rxq_stats->other_errors;
85 	} while (u64_stats_fetch_retry(&rxq_stats->syncp, start));
86 	u64_stats_update_end(&stats->syncp);
87 }
88 
89 /**
90  * rxq_stats_init - Initialize the statistics of specific queue
91  * @rxq: Logical Rx Queue
92  **/
93 static void rxq_stats_init(struct hinic_rxq *rxq)
94 {
95 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
96 
97 	u64_stats_init(&rxq_stats->syncp);
98 	hinic_rxq_clean_stats(rxq);
99 }
100 
101 static void rx_csum(struct hinic_rxq *rxq, u32 status,
102 		    struct sk_buff *skb)
103 {
104 	struct net_device *netdev = rxq->netdev;
105 	u32 csum_err;
106 
107 	csum_err = HINIC_RQ_CQE_STATUS_GET(status, CSUM_ERR);
108 
109 	if (!(netdev->features & NETIF_F_RXCSUM))
110 		return;
111 
112 	if (!csum_err) {
113 		skb->ip_summed = CHECKSUM_UNNECESSARY;
114 	} else {
115 		if (!(csum_err & (HINIC_RX_CSUM_HW_CHECK_NONE |
116 			HINIC_RX_CSUM_IPSU_OTHER_ERR)))
117 			rxq->rxq_stats.csum_errors++;
118 		skb->ip_summed = CHECKSUM_NONE;
119 	}
120 }
121 /**
122  * rx_alloc_skb - allocate skb and map it to dma address
123  * @rxq: rx queue
124  * @dma_addr: returned dma address for the skb
125  *
126  * Return skb
127  **/
128 static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq,
129 				    dma_addr_t *dma_addr)
130 {
131 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
132 	struct hinic_hwdev *hwdev = nic_dev->hwdev;
133 	struct hinic_hwif *hwif = hwdev->hwif;
134 	struct pci_dev *pdev = hwif->pdev;
135 	struct sk_buff *skb;
136 	dma_addr_t addr;
137 	int err;
138 
139 	skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz);
140 	if (!skb) {
141 		netdev_err(rxq->netdev, "Failed to allocate Rx SKB\n");
142 		return NULL;
143 	}
144 
145 	addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz,
146 			      DMA_FROM_DEVICE);
147 	err = dma_mapping_error(&pdev->dev, addr);
148 	if (err) {
149 		dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err);
150 		goto err_rx_map;
151 	}
152 
153 	*dma_addr = addr;
154 	return skb;
155 
156 err_rx_map:
157 	dev_kfree_skb_any(skb);
158 	return NULL;
159 }
160 
161 /**
162  * rx_unmap_skb - unmap the dma address of the skb
163  * @rxq: rx queue
164  * @dma_addr: dma address of the skb
165  **/
166 static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr)
167 {
168 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
169 	struct hinic_hwdev *hwdev = nic_dev->hwdev;
170 	struct hinic_hwif *hwif = hwdev->hwif;
171 	struct pci_dev *pdev = hwif->pdev;
172 
173 	dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz,
174 			 DMA_FROM_DEVICE);
175 }
176 
177 /**
178  * rx_free_skb - unmap and free skb
179  * @rxq: rx queue
180  * @skb: skb to free
181  * @dma_addr: dma address of the skb
182  **/
183 static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb,
184 			dma_addr_t dma_addr)
185 {
186 	rx_unmap_skb(rxq, dma_addr);
187 	dev_kfree_skb_any(skb);
188 }
189 
190 /**
191  * rx_alloc_pkts - allocate pkts in rx queue
192  * @rxq: rx queue
193  *
194  * Return number of skbs allocated
195  **/
196 static int rx_alloc_pkts(struct hinic_rxq *rxq)
197 {
198 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
199 	struct hinic_rq_wqe *rq_wqe;
200 	unsigned int free_wqebbs;
201 	struct hinic_sge sge;
202 	dma_addr_t dma_addr;
203 	struct sk_buff *skb;
204 	u16 prod_idx;
205 	int i;
206 
207 	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
208 
209 	/* Limit the allocation chunks */
210 	if (free_wqebbs > nic_dev->rx_weight)
211 		free_wqebbs = nic_dev->rx_weight;
212 
213 	for (i = 0; i < free_wqebbs; i++) {
214 		skb = rx_alloc_skb(rxq, &dma_addr);
215 		if (!skb) {
216 			netdev_err(rxq->netdev, "Failed to alloc Rx skb\n");
217 			goto skb_out;
218 		}
219 
220 		hinic_set_sge(&sge, dma_addr, skb->len);
221 
222 		rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
223 					  &prod_idx);
224 		if (!rq_wqe) {
225 			rx_free_skb(rxq, skb, dma_addr);
226 			goto skb_out;
227 		}
228 
229 		hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge);
230 
231 		hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb);
232 	}
233 
234 skb_out:
235 	if (i) {
236 		wmb();  /* write all the wqes before update PI */
237 
238 		hinic_rq_update(rxq->rq, prod_idx);
239 	}
240 
241 	return i;
242 }
243 
244 /**
245  * free_all_rx_skbs - free all skbs in rx queue
246  * @rxq: rx queue
247  **/
248 static void free_all_rx_skbs(struct hinic_rxq *rxq)
249 {
250 	struct hinic_rq *rq = rxq->rq;
251 	struct hinic_hw_wqe *hw_wqe;
252 	struct hinic_sge sge;
253 	u16 ci;
254 
255 	while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) {
256 		if (IS_ERR(hw_wqe))
257 			break;
258 
259 		hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge);
260 
261 		hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE);
262 
263 		rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));
264 	}
265 }
266 
267 /**
268  * rx_recv_jumbo_pkt - Rx handler for jumbo pkt
269  * @rxq: rx queue
270  * @head_skb: the first skb in the list
271  * @left_pkt_len: left size of the pkt exclude head skb
272  * @ci: consumer index
273  *
274  * Return number of wqes that used for the left of the pkt
275  **/
276 static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb,
277 			     unsigned int left_pkt_len, u16 ci)
278 {
279 	struct sk_buff *skb, *curr_skb = head_skb;
280 	struct hinic_rq_wqe *rq_wqe;
281 	unsigned int curr_len;
282 	struct hinic_sge sge;
283 	int num_wqes = 0;
284 
285 	while (left_pkt_len > 0) {
286 		rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
287 						&skb, &ci);
288 
289 		num_wqes++;
290 
291 		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
292 
293 		rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
294 
295 		prefetch(skb->data);
296 
297 		curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ :
298 			    left_pkt_len;
299 
300 		left_pkt_len -= curr_len;
301 
302 		__skb_put(skb, curr_len);
303 
304 		if (curr_skb == head_skb)
305 			skb_shinfo(head_skb)->frag_list = skb;
306 		else
307 			curr_skb->next = skb;
308 
309 		head_skb->len += skb->len;
310 		head_skb->data_len += skb->len;
311 		head_skb->truesize += skb->truesize;
312 
313 		curr_skb = skb;
314 	}
315 
316 	return num_wqes;
317 }
318 
319 /**
320  * rxq_recv - Rx handler
321  * @rxq: rx queue
322  * @budget: maximum pkts to process
323  *
324  * Return number of pkts received
325  **/
326 static int rxq_recv(struct hinic_rxq *rxq, int budget)
327 {
328 	struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);
329 	struct net_device *netdev = rxq->netdev;
330 	u64 pkt_len = 0, rx_bytes = 0;
331 	struct hinic_rq *rq = rxq->rq;
332 	struct hinic_rq_wqe *rq_wqe;
333 	unsigned int free_wqebbs;
334 	struct hinic_rq_cqe *cqe;
335 	int num_wqes, pkts = 0;
336 	struct hinic_sge sge;
337 	unsigned int status;
338 	struct sk_buff *skb;
339 	u32 offload_type;
340 	u16 ci, num_lro;
341 	u16 num_wqe = 0;
342 	u32 vlan_len;
343 	u16 vid;
344 
345 	while (pkts < budget) {
346 		num_wqes = 0;
347 
348 		rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb,
349 					   &ci);
350 		if (!rq_wqe)
351 			break;
352 
353 		cqe = rq->cqe[ci];
354 		status =  be32_to_cpu(cqe->status);
355 		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
356 
357 		rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
358 
359 		rx_csum(rxq, status, skb);
360 
361 		prefetch(skb->data);
362 
363 		pkt_len = sge.len;
364 
365 		if (pkt_len <= HINIC_RX_BUF_SZ) {
366 			__skb_put(skb, pkt_len);
367 		} else {
368 			__skb_put(skb, HINIC_RX_BUF_SZ);
369 			num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len -
370 						     HINIC_RX_BUF_SZ, ci);
371 		}
372 
373 		hinic_rq_put_wqe(rq, ci,
374 				 (num_wqes + 1) * HINIC_RQ_WQE_SIZE);
375 
376 		offload_type = be32_to_cpu(cqe->offload_type);
377 		vlan_len = be32_to_cpu(cqe->len);
378 		if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
379 		    HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type)) {
380 			vid = HINIC_GET_RX_VLAN_TAG(vlan_len);
381 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
382 		}
383 
384 		skb_record_rx_queue(skb, qp->q_id);
385 		skb->protocol = eth_type_trans(skb, rxq->netdev);
386 
387 		napi_gro_receive(&rxq->napi, skb);
388 
389 		pkts++;
390 		rx_bytes += pkt_len;
391 
392 		num_lro = HINIC_GET_RX_NUM_LRO(status);
393 		if (num_lro) {
394 			rx_bytes += ((num_lro - 1) *
395 				     LRO_PKT_HDR_LEN(cqe));
396 
397 			num_wqe +=
398 			(u16)(pkt_len >> rxq->rx_buff_shift) +
399 			((pkt_len & (rxq->buf_len - 1)) ? 1 : 0);
400 		}
401 
402 		cqe->status = 0;
403 
404 		if (num_wqe >= LRO_REPLENISH_THLD)
405 			break;
406 	}
407 
408 	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
409 	if (free_wqebbs > HINIC_RX_BUFFER_WRITE)
410 		rx_alloc_pkts(rxq);
411 
412 	u64_stats_update_begin(&rxq->rxq_stats.syncp);
413 	rxq->rxq_stats.pkts += pkts;
414 	rxq->rxq_stats.bytes += rx_bytes;
415 	u64_stats_update_end(&rxq->rxq_stats.syncp);
416 
417 	return pkts;
418 }
419 
420 static int rx_poll(struct napi_struct *napi, int budget)
421 {
422 	struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi);
423 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
424 	struct hinic_rq *rq = rxq->rq;
425 	int pkts;
426 
427 	pkts = rxq_recv(rxq, budget);
428 	if (pkts >= budget)
429 		return budget;
430 
431 	napi_complete(napi);
432 	hinic_hwdev_set_msix_state(nic_dev->hwdev,
433 				   rq->msix_entry,
434 				   HINIC_MSIX_ENABLE);
435 
436 	return pkts;
437 }
438 
439 static void rx_add_napi(struct hinic_rxq *rxq)
440 {
441 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
442 
443 	netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight);
444 	napi_enable(&rxq->napi);
445 }
446 
447 static void rx_del_napi(struct hinic_rxq *rxq)
448 {
449 	napi_disable(&rxq->napi);
450 	netif_napi_del(&rxq->napi);
451 }
452 
453 static irqreturn_t rx_irq(int irq, void *data)
454 {
455 	struct hinic_rxq *rxq = (struct hinic_rxq *)data;
456 	struct hinic_rq *rq = rxq->rq;
457 	struct hinic_dev *nic_dev;
458 
459 	/* Disable the interrupt until napi will be completed */
460 	nic_dev = netdev_priv(rxq->netdev);
461 	hinic_hwdev_set_msix_state(nic_dev->hwdev,
462 				   rq->msix_entry,
463 				   HINIC_MSIX_DISABLE);
464 
465 	nic_dev = netdev_priv(rxq->netdev);
466 	hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry);
467 
468 	napi_schedule(&rxq->napi);
469 	return IRQ_HANDLED;
470 }
471 
472 static int rx_request_irq(struct hinic_rxq *rxq)
473 {
474 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
475 	struct hinic_hwdev *hwdev = nic_dev->hwdev;
476 	struct hinic_rq *rq = rxq->rq;
477 	struct hinic_qp *qp;
478 	struct cpumask mask;
479 	int err;
480 
481 	rx_add_napi(rxq);
482 
483 	hinic_hwdev_msix_set(hwdev, rq->msix_entry,
484 			     RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC,
485 			     RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT,
486 			     RX_IRQ_NO_RESEND_TIMER);
487 
488 	err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);
489 	if (err) {
490 		rx_del_napi(rxq);
491 		return err;
492 	}
493 
494 	qp = container_of(rq, struct hinic_qp, rq);
495 	cpumask_set_cpu(qp->q_id % num_online_cpus(), &mask);
496 	return irq_set_affinity_hint(rq->irq, &mask);
497 }
498 
499 static void rx_free_irq(struct hinic_rxq *rxq)
500 {
501 	struct hinic_rq *rq = rxq->rq;
502 
503 	irq_set_affinity_hint(rq->irq, NULL);
504 	free_irq(rq->irq, rxq);
505 	rx_del_napi(rxq);
506 }
507 
508 /**
509  * hinic_init_rxq - Initialize the Rx Queue
510  * @rxq: Logical Rx Queue
511  * @rq: Hardware Rx Queue to connect the Logical queue with
512  * @netdev: network device to connect the Logical queue with
513  *
514  * Return 0 - Success, negative - Failure
515  **/
516 int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq,
517 		   struct net_device *netdev)
518 {
519 	struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq);
520 	int err, pkts;
521 
522 	rxq->netdev = netdev;
523 	rxq->rq = rq;
524 	rxq->buf_len = HINIC_RX_BUF_SZ;
525 	rxq->rx_buff_shift = ilog2(HINIC_RX_BUF_SZ);
526 
527 	rxq_stats_init(rxq);
528 
529 	rxq->irq_name = devm_kasprintf(&netdev->dev, GFP_KERNEL,
530 				       "hinic_rxq%d", qp->q_id);
531 	if (!rxq->irq_name)
532 		return -ENOMEM;
533 
534 	pkts = rx_alloc_pkts(rxq);
535 	if (!pkts) {
536 		err = -ENOMEM;
537 		goto err_rx_pkts;
538 	}
539 
540 	err = rx_request_irq(rxq);
541 	if (err) {
542 		netdev_err(netdev, "Failed to request Rx irq\n");
543 		goto err_req_rx_irq;
544 	}
545 
546 	return 0;
547 
548 err_req_rx_irq:
549 err_rx_pkts:
550 	free_all_rx_skbs(rxq);
551 	devm_kfree(&netdev->dev, rxq->irq_name);
552 	return err;
553 }
554 
555 /**
556  * hinic_clean_rxq - Clean the Rx Queue
557  * @rxq: Logical Rx Queue
558  **/
559 void hinic_clean_rxq(struct hinic_rxq *rxq)
560 {
561 	struct net_device *netdev = rxq->netdev;
562 
563 	rx_free_irq(rxq);
564 
565 	free_all_rx_skbs(rxq);
566 	devm_kfree(&netdev->dev, rxq->irq_name);
567 }
568