1 /*
2  * Huawei HiNIC PCI Express Linux driver
3  * Copyright(c) 2017 Huawei Technologies Co., Ltd
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12  * for more details.
13  *
14  */
15 
16 #include <linux/kernel.h>
17 #include <linux/netdevice.h>
18 #include <linux/u64_stats_sync.h>
19 #include <linux/errno.h>
20 #include <linux/types.h>
21 #include <linux/pci.h>
22 #include <linux/device.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/skbuff.h>
27 #include <linux/smp.h>
28 #include <asm/byteorder.h>
29 #include <linux/ip.h>
30 #include <linux/tcp.h>
31 #include <linux/sctp.h>
32 #include <linux/ipv6.h>
33 #include <net/ipv6.h>
34 #include <net/checksum.h>
35 #include <net/ip6_checksum.h>
36 
37 #include "hinic_common.h"
38 #include "hinic_hw_if.h"
39 #include "hinic_hw_wqe.h"
40 #include "hinic_hw_wq.h"
41 #include "hinic_hw_qp.h"
42 #include "hinic_hw_dev.h"
43 #include "hinic_dev.h"
44 #include "hinic_tx.h"
45 
46 #define TX_IRQ_NO_PENDING               0
47 #define TX_IRQ_NO_COALESC               0
48 #define TX_IRQ_NO_LLI_TIMER             0
49 #define TX_IRQ_NO_CREDIT                0
50 #define TX_IRQ_NO_RESEND_TIMER          0
51 
52 #define CI_UPDATE_NO_PENDING            0
53 #define CI_UPDATE_NO_COALESC            0
54 
55 #define HW_CONS_IDX(sq)                 be16_to_cpu(*(u16 *)((sq)->hw_ci_addr))
56 
57 #define MIN_SKB_LEN                     17
58 
59 #define	MAX_PAYLOAD_OFFSET	        221
60 #define TRANSPORT_OFFSET(l4_hdr, skb)	((u32)((l4_hdr) - (skb)->data))
61 
62 union hinic_l3 {
63 	struct iphdr *v4;
64 	struct ipv6hdr *v6;
65 	unsigned char *hdr;
66 };
67 
68 union hinic_l4 {
69 	struct tcphdr *tcp;
70 	struct udphdr *udp;
71 	unsigned char *hdr;
72 };
73 
74 enum hinic_offload_type {
75 	TX_OFFLOAD_TSO     = BIT(0),
76 	TX_OFFLOAD_CSUM    = BIT(1),
77 	TX_OFFLOAD_VLAN    = BIT(2),
78 	TX_OFFLOAD_INVALID = BIT(3),
79 };
80 
81 /**
82  * hinic_txq_clean_stats - Clean the statistics of specific queue
83  * @txq: Logical Tx Queue
84  **/
85 void hinic_txq_clean_stats(struct hinic_txq *txq)
86 {
87 	struct hinic_txq_stats *txq_stats = &txq->txq_stats;
88 
89 	u64_stats_update_begin(&txq_stats->syncp);
90 	txq_stats->pkts    = 0;
91 	txq_stats->bytes   = 0;
92 	txq_stats->tx_busy = 0;
93 	txq_stats->tx_wake = 0;
94 	txq_stats->tx_dropped = 0;
95 	u64_stats_update_end(&txq_stats->syncp);
96 }
97 
98 /**
99  * hinic_txq_get_stats - get statistics of Tx Queue
100  * @txq: Logical Tx Queue
101  * @stats: return updated stats here
102  **/
103 void hinic_txq_get_stats(struct hinic_txq *txq, struct hinic_txq_stats *stats)
104 {
105 	struct hinic_txq_stats *txq_stats = &txq->txq_stats;
106 	unsigned int start;
107 
108 	u64_stats_update_begin(&stats->syncp);
109 	do {
110 		start = u64_stats_fetch_begin(&txq_stats->syncp);
111 		stats->pkts    = txq_stats->pkts;
112 		stats->bytes   = txq_stats->bytes;
113 		stats->tx_busy = txq_stats->tx_busy;
114 		stats->tx_wake = txq_stats->tx_wake;
115 		stats->tx_dropped = txq_stats->tx_dropped;
116 	} while (u64_stats_fetch_retry(&txq_stats->syncp, start));
117 	u64_stats_update_end(&stats->syncp);
118 }
119 
120 /**
121  * txq_stats_init - Initialize the statistics of specific queue
122  * @txq: Logical Tx Queue
123  **/
124 static void txq_stats_init(struct hinic_txq *txq)
125 {
126 	struct hinic_txq_stats *txq_stats = &txq->txq_stats;
127 
128 	u64_stats_init(&txq_stats->syncp);
129 	hinic_txq_clean_stats(txq);
130 }
131 
132 /**
133  * tx_map_skb - dma mapping for skb and return sges
134  * @nic_dev: nic device
135  * @skb: the skb
136  * @sges: returned sges
137  *
138  * Return 0 - Success, negative - Failure
139  **/
140 static int tx_map_skb(struct hinic_dev *nic_dev, struct sk_buff *skb,
141 		      struct hinic_sge *sges)
142 {
143 	struct hinic_hwdev *hwdev = nic_dev->hwdev;
144 	struct hinic_hwif *hwif = hwdev->hwif;
145 	struct pci_dev *pdev = hwif->pdev;
146 	struct skb_frag_struct *frag;
147 	dma_addr_t dma_addr;
148 	int i, j;
149 
150 	dma_addr = dma_map_single(&pdev->dev, skb->data, skb_headlen(skb),
151 				  DMA_TO_DEVICE);
152 	if (dma_mapping_error(&pdev->dev, dma_addr)) {
153 		dev_err(&pdev->dev, "Failed to map Tx skb data\n");
154 		return -EFAULT;
155 	}
156 
157 	hinic_set_sge(&sges[0], dma_addr, skb_headlen(skb));
158 
159 	for (i = 0 ; i < skb_shinfo(skb)->nr_frags; i++) {
160 		frag = &skb_shinfo(skb)->frags[i];
161 
162 		dma_addr = skb_frag_dma_map(&pdev->dev, frag, 0,
163 					    skb_frag_size(frag),
164 					    DMA_TO_DEVICE);
165 		if (dma_mapping_error(&pdev->dev, dma_addr)) {
166 			dev_err(&pdev->dev, "Failed to map Tx skb frag\n");
167 			goto err_tx_map;
168 		}
169 
170 		hinic_set_sge(&sges[i + 1], dma_addr, skb_frag_size(frag));
171 	}
172 
173 	return 0;
174 
175 err_tx_map:
176 	for (j = 0; j < i; j++)
177 		dma_unmap_page(&pdev->dev, hinic_sge_to_dma(&sges[j + 1]),
178 			       sges[j + 1].len, DMA_TO_DEVICE);
179 
180 	dma_unmap_single(&pdev->dev, hinic_sge_to_dma(&sges[0]), sges[0].len,
181 			 DMA_TO_DEVICE);
182 	return -EFAULT;
183 }
184 
185 /**
186  * tx_unmap_skb - unmap the dma address of the skb
187  * @nic_dev: nic device
188  * @skb: the skb
189  * @sges: the sges that are connected to the skb
190  **/
191 static void tx_unmap_skb(struct hinic_dev *nic_dev, struct sk_buff *skb,
192 			 struct hinic_sge *sges)
193 {
194 	struct hinic_hwdev *hwdev = nic_dev->hwdev;
195 	struct hinic_hwif *hwif = hwdev->hwif;
196 	struct pci_dev *pdev = hwif->pdev;
197 	int i;
198 
199 	for (i = 0; i < skb_shinfo(skb)->nr_frags ; i++)
200 		dma_unmap_page(&pdev->dev, hinic_sge_to_dma(&sges[i + 1]),
201 			       sges[i + 1].len, DMA_TO_DEVICE);
202 
203 	dma_unmap_single(&pdev->dev, hinic_sge_to_dma(&sges[0]), sges[0].len,
204 			 DMA_TO_DEVICE);
205 }
206 
207 static void get_inner_l3_l4_type(struct sk_buff *skb, union hinic_l3 *ip,
208 				 union hinic_l4 *l4,
209 				 enum hinic_offload_type offload_type,
210 				 enum hinic_l3_offload_type *l3_type,
211 				 u8 *l4_proto)
212 {
213 	u8 *exthdr;
214 
215 	if (ip->v4->version == 4) {
216 		*l3_type = (offload_type == TX_OFFLOAD_CSUM) ?
217 			   IPV4_PKT_NO_CHKSUM_OFFLOAD :
218 			   IPV4_PKT_WITH_CHKSUM_OFFLOAD;
219 		*l4_proto = ip->v4->protocol;
220 	} else if (ip->v4->version == 6) {
221 		*l3_type = IPV6_PKT;
222 		exthdr = ip->hdr + sizeof(*ip->v6);
223 		*l4_proto = ip->v6->nexthdr;
224 		if (exthdr != l4->hdr) {
225 			int start = exthdr - skb->data;
226 			__be16 frag_off;
227 
228 			ipv6_skip_exthdr(skb, start, l4_proto, &frag_off);
229 		}
230 	} else {
231 		*l3_type = L3TYPE_UNKNOWN;
232 		*l4_proto = 0;
233 	}
234 }
235 
236 static void get_inner_l4_info(struct sk_buff *skb, union hinic_l4 *l4,
237 			      enum hinic_offload_type offload_type, u8 l4_proto,
238 			      enum hinic_l4_offload_type *l4_offload,
239 			      u32 *l4_len, u32 *offset)
240 {
241 	*l4_offload = OFFLOAD_DISABLE;
242 	*offset = 0;
243 	*l4_len = 0;
244 
245 	switch (l4_proto) {
246 	case IPPROTO_TCP:
247 		*l4_offload = TCP_OFFLOAD_ENABLE;
248 		/* doff in unit of 4B */
249 		*l4_len = l4->tcp->doff * 4;
250 		*offset = *l4_len + TRANSPORT_OFFSET(l4->hdr, skb);
251 		break;
252 
253 	case IPPROTO_UDP:
254 		*l4_offload = UDP_OFFLOAD_ENABLE;
255 		*l4_len = sizeof(struct udphdr);
256 		*offset = TRANSPORT_OFFSET(l4->hdr, skb);
257 		break;
258 
259 	case IPPROTO_SCTP:
260 		/* only csum offload support sctp */
261 		if (offload_type != TX_OFFLOAD_CSUM)
262 			break;
263 
264 		*l4_offload = SCTP_OFFLOAD_ENABLE;
265 		*l4_len = sizeof(struct sctphdr);
266 		*offset = TRANSPORT_OFFSET(l4->hdr, skb);
267 		break;
268 
269 	default:
270 		break;
271 	}
272 }
273 
274 static __sum16 csum_magic(union hinic_l3 *ip, unsigned short proto)
275 {
276 	return (ip->v4->version == 4) ?
277 		csum_tcpudp_magic(ip->v4->saddr, ip->v4->daddr, 0, proto, 0) :
278 		csum_ipv6_magic(&ip->v6->saddr, &ip->v6->daddr, 0, proto, 0);
279 }
280 
281 static int offload_tso(struct hinic_sq_task *task, u32 *queue_info,
282 		       struct sk_buff *skb)
283 {
284 	u32 offset, l4_len, ip_identify, network_hdr_len;
285 	enum hinic_l3_offload_type l3_offload;
286 	enum hinic_l4_offload_type l4_offload;
287 	union hinic_l3 ip;
288 	union hinic_l4 l4;
289 	u8 l4_proto;
290 
291 	if (!skb_is_gso(skb))
292 		return 0;
293 
294 	if (skb_cow_head(skb, 0) < 0)
295 		return -EPROTONOSUPPORT;
296 
297 	if (skb->encapsulation) {
298 		u32 gso_type = skb_shinfo(skb)->gso_type;
299 		u32 tunnel_type = 0;
300 		u32 l4_tunnel_len;
301 
302 		ip.hdr = skb_network_header(skb);
303 		l4.hdr = skb_transport_header(skb);
304 		network_hdr_len = skb_inner_network_header_len(skb);
305 
306 		if (ip.v4->version == 4) {
307 			ip.v4->tot_len = 0;
308 			l3_offload = IPV4_PKT_WITH_CHKSUM_OFFLOAD;
309 		} else if (ip.v4->version == 6) {
310 			l3_offload = IPV6_PKT;
311 		} else {
312 			l3_offload = 0;
313 		}
314 
315 		hinic_task_set_outter_l3(task, l3_offload,
316 					 skb_network_header_len(skb));
317 
318 		if (gso_type & SKB_GSO_UDP_TUNNEL_CSUM) {
319 			l4.udp->check = ~csum_magic(&ip, IPPROTO_UDP);
320 			tunnel_type = TUNNEL_UDP_CSUM;
321 		} else if (gso_type & SKB_GSO_UDP_TUNNEL) {
322 			tunnel_type = TUNNEL_UDP_NO_CSUM;
323 		}
324 
325 		l4_tunnel_len = skb_inner_network_offset(skb) -
326 				skb_transport_offset(skb);
327 		hinic_task_set_tunnel_l4(task, tunnel_type, l4_tunnel_len);
328 
329 		ip.hdr = skb_inner_network_header(skb);
330 		l4.hdr = skb_inner_transport_header(skb);
331 	} else {
332 		ip.hdr = skb_network_header(skb);
333 		l4.hdr = skb_transport_header(skb);
334 		network_hdr_len = skb_network_header_len(skb);
335 	}
336 
337 	/* initialize inner IP header fields */
338 	if (ip.v4->version == 4)
339 		ip.v4->tot_len = 0;
340 	else
341 		ip.v6->payload_len = 0;
342 
343 	get_inner_l3_l4_type(skb, &ip, &l4, TX_OFFLOAD_TSO, &l3_offload,
344 			     &l4_proto);
345 
346 	hinic_task_set_inner_l3(task, l3_offload, network_hdr_len);
347 
348 	ip_identify = 0;
349 	if (l4_proto == IPPROTO_TCP)
350 		l4.tcp->check = ~csum_magic(&ip, IPPROTO_TCP);
351 
352 	get_inner_l4_info(skb, &l4, TX_OFFLOAD_TSO, l4_proto, &l4_offload,
353 			  &l4_len, &offset);
354 
355 	hinic_set_tso_inner_l4(task, queue_info, l4_offload, l4_len, offset,
356 			       ip_identify, skb_shinfo(skb)->gso_size);
357 
358 	return 1;
359 }
360 
361 static int offload_csum(struct hinic_sq_task *task, u32 *queue_info,
362 			struct sk_buff *skb)
363 {
364 	enum hinic_l4_offload_type l4_offload;
365 	u32 offset, l4_len, network_hdr_len;
366 	enum hinic_l3_offload_type l3_type;
367 	union hinic_l3 ip;
368 	union hinic_l4 l4;
369 	u8 l4_proto;
370 
371 	if (skb->ip_summed != CHECKSUM_PARTIAL)
372 		return 0;
373 
374 	if (skb->encapsulation) {
375 		u32 l4_tunnel_len;
376 
377 		ip.hdr = skb_network_header(skb);
378 
379 		if (ip.v4->version == 4)
380 			l3_type = IPV4_PKT_NO_CHKSUM_OFFLOAD;
381 		else if (ip.v4->version == 6)
382 			l3_type = IPV6_PKT;
383 		else
384 			l3_type = L3TYPE_UNKNOWN;
385 
386 		hinic_task_set_outter_l3(task, l3_type,
387 					 skb_network_header_len(skb));
388 
389 		l4_tunnel_len = skb_inner_network_offset(skb) -
390 				skb_transport_offset(skb);
391 
392 		hinic_task_set_tunnel_l4(task, TUNNEL_UDP_NO_CSUM,
393 					 l4_tunnel_len);
394 
395 		ip.hdr = skb_inner_network_header(skb);
396 		l4.hdr = skb_inner_transport_header(skb);
397 		network_hdr_len = skb_inner_network_header_len(skb);
398 	} else {
399 		ip.hdr = skb_network_header(skb);
400 		l4.hdr = skb_transport_header(skb);
401 		network_hdr_len = skb_network_header_len(skb);
402 	}
403 
404 	get_inner_l3_l4_type(skb, &ip, &l4, TX_OFFLOAD_CSUM, &l3_type,
405 			     &l4_proto);
406 
407 	hinic_task_set_inner_l3(task, l3_type, network_hdr_len);
408 
409 	get_inner_l4_info(skb, &l4, TX_OFFLOAD_CSUM, l4_proto, &l4_offload,
410 			  &l4_len, &offset);
411 
412 	hinic_set_cs_inner_l4(task, queue_info, l4_offload, l4_len, offset);
413 
414 	return 1;
415 }
416 
417 static int hinic_tx_offload(struct sk_buff *skb, struct hinic_sq_task *task,
418 			    u32 *queue_info)
419 {
420 	enum hinic_offload_type offload = 0;
421 	int enabled;
422 
423 	enabled = offload_tso(task, queue_info, skb);
424 	if (enabled > 0) {
425 		offload |= TX_OFFLOAD_TSO;
426 	} else if (enabled == 0) {
427 		enabled = offload_csum(task, queue_info, skb);
428 		if (enabled)
429 			offload |= TX_OFFLOAD_CSUM;
430 	} else {
431 		return -EPROTONOSUPPORT;
432 	}
433 
434 	if (offload)
435 		hinic_task_set_l2hdr(task, skb_network_offset(skb));
436 
437 	/* payload offset should not more than 221 */
438 	if (HINIC_SQ_CTRL_GET(*queue_info, QUEUE_INFO_PLDOFF) >
439 	    MAX_PAYLOAD_OFFSET) {
440 		return -EPROTONOSUPPORT;
441 	}
442 
443 	/* mss should not less than 80 */
444 	if (HINIC_SQ_CTRL_GET(*queue_info, QUEUE_INFO_MSS) < HINIC_MSS_MIN) {
445 		*queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);
446 		*queue_info |= HINIC_SQ_CTRL_SET(HINIC_MSS_MIN, QUEUE_INFO_MSS);
447 	}
448 
449 	return 0;
450 }
451 
452 netdev_tx_t hinic_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
453 {
454 	struct hinic_dev *nic_dev = netdev_priv(netdev);
455 	u16 prod_idx, q_id = skb->queue_mapping;
456 	struct netdev_queue *netdev_txq;
457 	int nr_sges, err = NETDEV_TX_OK;
458 	struct hinic_sq_wqe *sq_wqe;
459 	unsigned int wqe_size;
460 	struct hinic_txq *txq;
461 	struct hinic_qp *qp;
462 
463 	txq = &nic_dev->txqs[q_id];
464 	qp = container_of(txq->sq, struct hinic_qp, sq);
465 
466 	if (skb->len < MIN_SKB_LEN) {
467 		if (skb_pad(skb, MIN_SKB_LEN - skb->len)) {
468 			netdev_err(netdev, "Failed to pad skb\n");
469 			goto update_error_stats;
470 		}
471 
472 		skb->len = MIN_SKB_LEN;
473 	}
474 
475 	nr_sges = skb_shinfo(skb)->nr_frags + 1;
476 	if (nr_sges > txq->max_sges) {
477 		netdev_err(netdev, "Too many Tx sges\n");
478 		goto skb_error;
479 	}
480 
481 	err = tx_map_skb(nic_dev, skb, txq->sges);
482 	if (err)
483 		goto skb_error;
484 
485 	wqe_size = HINIC_SQ_WQE_SIZE(nr_sges);
486 
487 	sq_wqe = hinic_sq_get_wqe(txq->sq, wqe_size, &prod_idx);
488 	if (!sq_wqe) {
489 		netif_stop_subqueue(netdev, qp->q_id);
490 
491 		/* Check for the case free_tx_poll is called in another cpu
492 		 * and we stopped the subqueue after free_tx_poll check.
493 		 */
494 		sq_wqe = hinic_sq_get_wqe(txq->sq, wqe_size, &prod_idx);
495 		if (sq_wqe) {
496 			netif_wake_subqueue(nic_dev->netdev, qp->q_id);
497 			goto process_sq_wqe;
498 		}
499 
500 		tx_unmap_skb(nic_dev, skb, txq->sges);
501 
502 		u64_stats_update_begin(&txq->txq_stats.syncp);
503 		txq->txq_stats.tx_busy++;
504 		u64_stats_update_end(&txq->txq_stats.syncp);
505 		err = NETDEV_TX_BUSY;
506 		wqe_size = 0;
507 		goto flush_skbs;
508 	}
509 
510 process_sq_wqe:
511 	hinic_sq_prepare_wqe(txq->sq, prod_idx, sq_wqe, txq->sges, nr_sges);
512 
513 	err = hinic_tx_offload(skb, &sq_wqe->task, &sq_wqe->ctrl.queue_info);
514 	if (err)
515 		goto offload_error;
516 
517 	hinic_sq_write_wqe(txq->sq, prod_idx, sq_wqe, skb, wqe_size);
518 
519 flush_skbs:
520 	netdev_txq = netdev_get_tx_queue(netdev, q_id);
521 	if ((!skb->xmit_more) || (netif_xmit_stopped(netdev_txq)))
522 		hinic_sq_write_db(txq->sq, prod_idx, wqe_size, 0);
523 
524 	return err;
525 
526 offload_error:
527 	hinic_sq_return_wqe(txq->sq, wqe_size);
528 	tx_unmap_skb(nic_dev, skb, txq->sges);
529 
530 skb_error:
531 	dev_kfree_skb_any(skb);
532 
533 update_error_stats:
534 	u64_stats_update_begin(&txq->txq_stats.syncp);
535 	txq->txq_stats.tx_dropped++;
536 	u64_stats_update_end(&txq->txq_stats.syncp);
537 
538 	return NETDEV_TX_OK;
539 }
540 
541 /**
542  * tx_free_skb - unmap and free skb
543  * @nic_dev: nic device
544  * @skb: the skb
545  * @sges: the sges that are connected to the skb
546  **/
547 static void tx_free_skb(struct hinic_dev *nic_dev, struct sk_buff *skb,
548 			struct hinic_sge *sges)
549 {
550 	tx_unmap_skb(nic_dev, skb, sges);
551 
552 	dev_kfree_skb_any(skb);
553 }
554 
555 /**
556  * free_all_rx_skbs - free all skbs in tx queue
557  * @txq: tx queue
558  **/
559 static void free_all_tx_skbs(struct hinic_txq *txq)
560 {
561 	struct hinic_dev *nic_dev = netdev_priv(txq->netdev);
562 	struct hinic_sq *sq = txq->sq;
563 	struct hinic_sq_wqe *sq_wqe;
564 	unsigned int wqe_size;
565 	struct sk_buff *skb;
566 	int nr_sges;
567 	u16 ci;
568 
569 	while ((sq_wqe = hinic_sq_read_wqebb(sq, &skb, &wqe_size, &ci))) {
570 		sq_wqe = hinic_sq_read_wqe(sq, &skb, wqe_size, &ci);
571 		if (!sq_wqe)
572 			break;
573 
574 		nr_sges = skb_shinfo(skb)->nr_frags + 1;
575 
576 		hinic_sq_get_sges(sq_wqe, txq->free_sges, nr_sges);
577 
578 		hinic_sq_put_wqe(sq, wqe_size);
579 
580 		tx_free_skb(nic_dev, skb, txq->free_sges);
581 	}
582 }
583 
584 /**
585  * free_tx_poll - free finished tx skbs in tx queue that connected to napi
586  * @napi: napi
587  * @budget: number of tx
588  *
589  * Return 0 - Success, negative - Failure
590  **/
591 static int free_tx_poll(struct napi_struct *napi, int budget)
592 {
593 	struct hinic_txq *txq = container_of(napi, struct hinic_txq, napi);
594 	struct hinic_qp *qp = container_of(txq->sq, struct hinic_qp, sq);
595 	struct hinic_dev *nic_dev = netdev_priv(txq->netdev);
596 	struct netdev_queue *netdev_txq;
597 	struct hinic_sq *sq = txq->sq;
598 	struct hinic_wq *wq = sq->wq;
599 	struct hinic_sq_wqe *sq_wqe;
600 	unsigned int wqe_size;
601 	int nr_sges, pkts = 0;
602 	struct sk_buff *skb;
603 	u64 tx_bytes = 0;
604 	u16 hw_ci, sw_ci;
605 
606 	do {
607 		hw_ci = HW_CONS_IDX(sq) & wq->mask;
608 
609 		/* Reading a WQEBB to get real WQE size and consumer index. */
610 		sq_wqe = hinic_sq_read_wqebb(sq, &skb, &wqe_size, &sw_ci);
611 		if ((!sq_wqe) ||
612 		    (((hw_ci - sw_ci) & wq->mask) * wq->wqebb_size < wqe_size))
613 			break;
614 
615 		/* If this WQE have multiple WQEBBs, we will read again to get
616 		 * full size WQE.
617 		 */
618 		if (wqe_size > wq->wqebb_size) {
619 			sq_wqe = hinic_sq_read_wqe(sq, &skb, wqe_size, &sw_ci);
620 			if (unlikely(!sq_wqe))
621 				break;
622 		}
623 
624 		tx_bytes += skb->len;
625 		pkts++;
626 
627 		nr_sges = skb_shinfo(skb)->nr_frags + 1;
628 
629 		hinic_sq_get_sges(sq_wqe, txq->free_sges, nr_sges);
630 
631 		hinic_sq_put_wqe(sq, wqe_size);
632 
633 		tx_free_skb(nic_dev, skb, txq->free_sges);
634 	} while (pkts < budget);
635 
636 	if (__netif_subqueue_stopped(nic_dev->netdev, qp->q_id) &&
637 	    hinic_get_sq_free_wqebbs(sq) >= HINIC_MIN_TX_NUM_WQEBBS(sq)) {
638 		netdev_txq = netdev_get_tx_queue(txq->netdev, qp->q_id);
639 
640 		__netif_tx_lock(netdev_txq, smp_processor_id());
641 
642 		netif_wake_subqueue(nic_dev->netdev, qp->q_id);
643 
644 		__netif_tx_unlock(netdev_txq);
645 
646 		u64_stats_update_begin(&txq->txq_stats.syncp);
647 		txq->txq_stats.tx_wake++;
648 		u64_stats_update_end(&txq->txq_stats.syncp);
649 	}
650 
651 	u64_stats_update_begin(&txq->txq_stats.syncp);
652 	txq->txq_stats.bytes += tx_bytes;
653 	txq->txq_stats.pkts += pkts;
654 	u64_stats_update_end(&txq->txq_stats.syncp);
655 
656 	if (pkts < budget) {
657 		napi_complete(napi);
658 		hinic_hwdev_set_msix_state(nic_dev->hwdev,
659 					   sq->msix_entry,
660 					   HINIC_MSIX_ENABLE);
661 		return pkts;
662 	}
663 
664 	return budget;
665 }
666 
667 static void tx_napi_add(struct hinic_txq *txq, int weight)
668 {
669 	netif_napi_add(txq->netdev, &txq->napi, free_tx_poll, weight);
670 	napi_enable(&txq->napi);
671 }
672 
673 static void tx_napi_del(struct hinic_txq *txq)
674 {
675 	napi_disable(&txq->napi);
676 	netif_napi_del(&txq->napi);
677 }
678 
679 static irqreturn_t tx_irq(int irq, void *data)
680 {
681 	struct hinic_txq *txq = data;
682 	struct hinic_dev *nic_dev;
683 
684 	nic_dev = netdev_priv(txq->netdev);
685 
686 	/* Disable the interrupt until napi will be completed */
687 	hinic_hwdev_set_msix_state(nic_dev->hwdev,
688 				   txq->sq->msix_entry,
689 				   HINIC_MSIX_DISABLE);
690 
691 	hinic_hwdev_msix_cnt_set(nic_dev->hwdev, txq->sq->msix_entry);
692 
693 	napi_schedule(&txq->napi);
694 	return IRQ_HANDLED;
695 }
696 
697 static int tx_request_irq(struct hinic_txq *txq)
698 {
699 	struct hinic_dev *nic_dev = netdev_priv(txq->netdev);
700 	struct hinic_hwdev *hwdev = nic_dev->hwdev;
701 	struct hinic_hwif *hwif = hwdev->hwif;
702 	struct pci_dev *pdev = hwif->pdev;
703 	struct hinic_sq *sq = txq->sq;
704 	int err;
705 
706 	tx_napi_add(txq, nic_dev->tx_weight);
707 
708 	hinic_hwdev_msix_set(nic_dev->hwdev, sq->msix_entry,
709 			     TX_IRQ_NO_PENDING, TX_IRQ_NO_COALESC,
710 			     TX_IRQ_NO_LLI_TIMER, TX_IRQ_NO_CREDIT,
711 			     TX_IRQ_NO_RESEND_TIMER);
712 
713 	err = request_irq(sq->irq, tx_irq, 0, txq->irq_name, txq);
714 	if (err) {
715 		dev_err(&pdev->dev, "Failed to request Tx irq\n");
716 		tx_napi_del(txq);
717 		return err;
718 	}
719 
720 	return 0;
721 }
722 
723 static void tx_free_irq(struct hinic_txq *txq)
724 {
725 	struct hinic_sq *sq = txq->sq;
726 
727 	free_irq(sq->irq, txq);
728 	tx_napi_del(txq);
729 }
730 
731 /**
732  * hinic_init_txq - Initialize the Tx Queue
733  * @txq: Logical Tx Queue
734  * @sq: Hardware Tx Queue to connect the Logical queue with
735  * @netdev: network device to connect the Logical queue with
736  *
737  * Return 0 - Success, negative - Failure
738  **/
739 int hinic_init_txq(struct hinic_txq *txq, struct hinic_sq *sq,
740 		   struct net_device *netdev)
741 {
742 	struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq);
743 	struct hinic_dev *nic_dev = netdev_priv(netdev);
744 	struct hinic_hwdev *hwdev = nic_dev->hwdev;
745 	int err, irqname_len;
746 	size_t sges_size;
747 
748 	txq->netdev = netdev;
749 	txq->sq = sq;
750 
751 	txq_stats_init(txq);
752 
753 	txq->max_sges = HINIC_MAX_SQ_BUFDESCS;
754 
755 	sges_size = txq->max_sges * sizeof(*txq->sges);
756 	txq->sges = devm_kzalloc(&netdev->dev, sges_size, GFP_KERNEL);
757 	if (!txq->sges)
758 		return -ENOMEM;
759 
760 	sges_size = txq->max_sges * sizeof(*txq->free_sges);
761 	txq->free_sges = devm_kzalloc(&netdev->dev, sges_size, GFP_KERNEL);
762 	if (!txq->free_sges) {
763 		err = -ENOMEM;
764 		goto err_alloc_free_sges;
765 	}
766 
767 	irqname_len = snprintf(NULL, 0, "hinic_txq%d", qp->q_id) + 1;
768 	txq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL);
769 	if (!txq->irq_name) {
770 		err = -ENOMEM;
771 		goto err_alloc_irqname;
772 	}
773 
774 	sprintf(txq->irq_name, "hinic_txq%d", qp->q_id);
775 
776 	err = hinic_hwdev_hw_ci_addr_set(hwdev, sq, CI_UPDATE_NO_PENDING,
777 					 CI_UPDATE_NO_COALESC);
778 	if (err)
779 		goto err_hw_ci;
780 
781 	err = tx_request_irq(txq);
782 	if (err) {
783 		netdev_err(netdev, "Failed to request Tx irq\n");
784 		goto err_req_tx_irq;
785 	}
786 
787 	return 0;
788 
789 err_req_tx_irq:
790 err_hw_ci:
791 	devm_kfree(&netdev->dev, txq->irq_name);
792 
793 err_alloc_irqname:
794 	devm_kfree(&netdev->dev, txq->free_sges);
795 
796 err_alloc_free_sges:
797 	devm_kfree(&netdev->dev, txq->sges);
798 	return err;
799 }
800 
801 /**
802  * hinic_clean_txq - Clean the Tx Queue
803  * @txq: Logical Tx Queue
804  **/
805 void hinic_clean_txq(struct hinic_txq *txq)
806 {
807 	struct net_device *netdev = txq->netdev;
808 
809 	tx_free_irq(txq);
810 
811 	free_all_tx_skbs(txq);
812 
813 	devm_kfree(&netdev->dev, txq->irq_name);
814 	devm_kfree(&netdev->dev, txq->free_sges);
815 	devm_kfree(&netdev->dev, txq->sges);
816 }
817