1 // SPDX-License-Identifier: GPL-2.0+
2 // Copyright (c) 2016-2017 Hisilicon Limited.
3 
4 #include <linux/dma-mapping.h>
5 #include <linux/etherdevice.h>
6 #include <linux/interrupt.h>
7 #include <linux/if_vlan.h>
8 #include <linux/ip.h>
9 #include <linux/ipv6.h>
10 #include <linux/module.h>
11 #include <linux/pci.h>
12 #include <linux/aer.h>
13 #include <linux/skbuff.h>
14 #include <linux/sctp.h>
15 #include <linux/vermagic.h>
16 #include <net/gre.h>
17 #include <net/pkt_cls.h>
18 #include <net/tcp.h>
19 #include <net/vxlan.h>
20 
21 #include "hnae3.h"
22 #include "hns3_enet.h"
23 
24 static void hns3_clear_all_ring(struct hnae3_handle *h);
25 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
26 static void hns3_remove_hw_addr(struct net_device *netdev);
27 
28 static const char hns3_driver_name[] = "hns3";
29 const char hns3_driver_version[] = VERMAGIC_STRING;
30 static const char hns3_driver_string[] =
31 			"Hisilicon Ethernet Network Driver for Hip08 Family";
32 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
33 static struct hnae3_client client;
34 
35 /* hns3_pci_tbl - PCI Device ID Table
36  *
37  * Last entry must be all 0s
38  *
39  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
40  *   Class, Class Mask, private data (not used) }
41  */
42 static const struct pci_device_id hns3_pci_tbl[] = {
43 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
44 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
45 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
46 	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
47 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
48 	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
49 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
50 	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
51 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
52 	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
53 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
54 	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
55 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
56 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
57 	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
58 	/* required last entry */
59 	{0, }
60 };
61 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
62 
63 static irqreturn_t hns3_irq_handle(int irq, void *vector)
64 {
65 	struct hns3_enet_tqp_vector *tqp_vector = vector;
66 
67 	napi_schedule(&tqp_vector->napi);
68 
69 	return IRQ_HANDLED;
70 }
71 
72 /* This callback function is used to set affinity changes to the irq affinity
73  * masks when the irq_set_affinity_notifier function is used.
74  */
75 static void hns3_nic_irq_affinity_notify(struct irq_affinity_notify *notify,
76 					 const cpumask_t *mask)
77 {
78 	struct hns3_enet_tqp_vector *tqp_vectors =
79 		container_of(notify, struct hns3_enet_tqp_vector,
80 			     affinity_notify);
81 
82 	tqp_vectors->affinity_mask = *mask;
83 }
84 
85 static void hns3_nic_irq_affinity_release(struct kref *ref)
86 {
87 }
88 
89 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
90 {
91 	struct hns3_enet_tqp_vector *tqp_vectors;
92 	unsigned int i;
93 
94 	for (i = 0; i < priv->vector_num; i++) {
95 		tqp_vectors = &priv->tqp_vector[i];
96 
97 		if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
98 			continue;
99 
100 		/* clear the affinity notifier and affinity mask */
101 		irq_set_affinity_notifier(tqp_vectors->vector_irq, NULL);
102 		irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
103 
104 		/* release the irq resource */
105 		free_irq(tqp_vectors->vector_irq, tqp_vectors);
106 		tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
107 	}
108 }
109 
110 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
111 {
112 	struct hns3_enet_tqp_vector *tqp_vectors;
113 	int txrx_int_idx = 0;
114 	int rx_int_idx = 0;
115 	int tx_int_idx = 0;
116 	unsigned int i;
117 	int ret;
118 
119 	for (i = 0; i < priv->vector_num; i++) {
120 		tqp_vectors = &priv->tqp_vector[i];
121 
122 		if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
123 			continue;
124 
125 		if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
126 			snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
127 				 "%s-%s-%d", priv->netdev->name, "TxRx",
128 				 txrx_int_idx++);
129 			txrx_int_idx++;
130 		} else if (tqp_vectors->rx_group.ring) {
131 			snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
132 				 "%s-%s-%d", priv->netdev->name, "Rx",
133 				 rx_int_idx++);
134 		} else if (tqp_vectors->tx_group.ring) {
135 			snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
136 				 "%s-%s-%d", priv->netdev->name, "Tx",
137 				 tx_int_idx++);
138 		} else {
139 			/* Skip this unused q_vector */
140 			continue;
141 		}
142 
143 		tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
144 
145 		ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
146 				  tqp_vectors->name,
147 				       tqp_vectors);
148 		if (ret) {
149 			netdev_err(priv->netdev, "request irq(%d) fail\n",
150 				   tqp_vectors->vector_irq);
151 			return ret;
152 		}
153 
154 		tqp_vectors->affinity_notify.notify =
155 					hns3_nic_irq_affinity_notify;
156 		tqp_vectors->affinity_notify.release =
157 					hns3_nic_irq_affinity_release;
158 		irq_set_affinity_notifier(tqp_vectors->vector_irq,
159 					  &tqp_vectors->affinity_notify);
160 		irq_set_affinity_hint(tqp_vectors->vector_irq,
161 				      &tqp_vectors->affinity_mask);
162 
163 		tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
164 	}
165 
166 	return 0;
167 }
168 
169 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
170 				 u32 mask_en)
171 {
172 	writel(mask_en, tqp_vector->mask_addr);
173 }
174 
175 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
176 {
177 	napi_enable(&tqp_vector->napi);
178 
179 	/* enable vector */
180 	hns3_mask_vector_irq(tqp_vector, 1);
181 }
182 
183 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
184 {
185 	/* disable vector */
186 	hns3_mask_vector_irq(tqp_vector, 0);
187 
188 	disable_irq(tqp_vector->vector_irq);
189 	napi_disable(&tqp_vector->napi);
190 }
191 
192 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
193 				 u32 rl_value)
194 {
195 	u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
196 
197 	/* this defines the configuration for RL (Interrupt Rate Limiter).
198 	 * Rl defines rate of interrupts i.e. number of interrupts-per-second
199 	 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
200 	 */
201 
202 	if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
203 	    !tqp_vector->rx_group.coal.gl_adapt_enable)
204 		/* According to the hardware, the range of rl_reg is
205 		 * 0-59 and the unit is 4.
206 		 */
207 		rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
208 
209 	writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
210 }
211 
212 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
213 				    u32 gl_value)
214 {
215 	u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
216 
217 	writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
218 }
219 
220 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
221 				    u32 gl_value)
222 {
223 	u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
224 
225 	writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
226 }
227 
228 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
229 				   struct hns3_nic_priv *priv)
230 {
231 	/* initialize the configuration for interrupt coalescing.
232 	 * 1. GL (Interrupt Gap Limiter)
233 	 * 2. RL (Interrupt Rate Limiter)
234 	 */
235 
236 	/* Default: enable interrupt coalescing self-adaptive and GL */
237 	tqp_vector->tx_group.coal.gl_adapt_enable = 1;
238 	tqp_vector->rx_group.coal.gl_adapt_enable = 1;
239 
240 	tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
241 	tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
242 
243 	tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
244 	tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
245 }
246 
247 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
248 				      struct hns3_nic_priv *priv)
249 {
250 	struct hnae3_handle *h = priv->ae_handle;
251 
252 	hns3_set_vector_coalesce_tx_gl(tqp_vector,
253 				       tqp_vector->tx_group.coal.int_gl);
254 	hns3_set_vector_coalesce_rx_gl(tqp_vector,
255 				       tqp_vector->rx_group.coal.int_gl);
256 	hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
257 }
258 
259 static int hns3_nic_set_real_num_queue(struct net_device *netdev)
260 {
261 	struct hnae3_handle *h = hns3_get_handle(netdev);
262 	struct hnae3_knic_private_info *kinfo = &h->kinfo;
263 	unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
264 	int i, ret;
265 
266 	if (kinfo->num_tc <= 1) {
267 		netdev_reset_tc(netdev);
268 	} else {
269 		ret = netdev_set_num_tc(netdev, kinfo->num_tc);
270 		if (ret) {
271 			netdev_err(netdev,
272 				   "netdev_set_num_tc fail, ret=%d!\n", ret);
273 			return ret;
274 		}
275 
276 		for (i = 0; i < HNAE3_MAX_TC; i++) {
277 			if (!kinfo->tc_info[i].enable)
278 				continue;
279 
280 			netdev_set_tc_queue(netdev,
281 					    kinfo->tc_info[i].tc,
282 					    kinfo->tc_info[i].tqp_count,
283 					    kinfo->tc_info[i].tqp_offset);
284 		}
285 	}
286 
287 	ret = netif_set_real_num_tx_queues(netdev, queue_size);
288 	if (ret) {
289 		netdev_err(netdev,
290 			   "netif_set_real_num_tx_queues fail, ret=%d!\n",
291 			   ret);
292 		return ret;
293 	}
294 
295 	ret = netif_set_real_num_rx_queues(netdev, queue_size);
296 	if (ret) {
297 		netdev_err(netdev,
298 			   "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
299 		return ret;
300 	}
301 
302 	return 0;
303 }
304 
305 static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
306 {
307 	u16 alloc_tqps, max_rss_size, rss_size;
308 
309 	h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
310 	rss_size = alloc_tqps / h->kinfo.num_tc;
311 
312 	return min_t(u16, rss_size, max_rss_size);
313 }
314 
315 static void hns3_tqp_enable(struct hnae3_queue *tqp)
316 {
317 	u32 rcb_reg;
318 
319 	rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
320 	rcb_reg |= BIT(HNS3_RING_EN_B);
321 	hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
322 }
323 
324 static void hns3_tqp_disable(struct hnae3_queue *tqp)
325 {
326 	u32 rcb_reg;
327 
328 	rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
329 	rcb_reg &= ~BIT(HNS3_RING_EN_B);
330 	hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
331 }
332 
333 static int hns3_nic_net_up(struct net_device *netdev)
334 {
335 	struct hns3_nic_priv *priv = netdev_priv(netdev);
336 	struct hnae3_handle *h = priv->ae_handle;
337 	int i, j;
338 	int ret;
339 
340 	ret = hns3_nic_reset_all_ring(h);
341 	if (ret)
342 		return ret;
343 
344 	/* get irq resource for all vectors */
345 	ret = hns3_nic_init_irq(priv);
346 	if (ret) {
347 		netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
348 		return ret;
349 	}
350 
351 	/* enable the vectors */
352 	for (i = 0; i < priv->vector_num; i++)
353 		hns3_vector_enable(&priv->tqp_vector[i]);
354 
355 	/* enable rcb */
356 	for (j = 0; j < h->kinfo.num_tqps; j++)
357 		hns3_tqp_enable(h->kinfo.tqp[j]);
358 
359 	/* start the ae_dev */
360 	ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
361 	if (ret)
362 		goto out_start_err;
363 
364 	clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
365 
366 	return 0;
367 
368 out_start_err:
369 	while (j--)
370 		hns3_tqp_disable(h->kinfo.tqp[j]);
371 
372 	for (j = i - 1; j >= 0; j--)
373 		hns3_vector_disable(&priv->tqp_vector[j]);
374 
375 	hns3_nic_uninit_irq(priv);
376 
377 	return ret;
378 }
379 
380 static int hns3_nic_net_open(struct net_device *netdev)
381 {
382 	struct hns3_nic_priv *priv = netdev_priv(netdev);
383 	struct hnae3_handle *h = hns3_get_handle(netdev);
384 	struct hnae3_knic_private_info *kinfo;
385 	int i, ret;
386 
387 	if (hns3_nic_resetting(netdev))
388 		return -EBUSY;
389 
390 	netif_carrier_off(netdev);
391 
392 	ret = hns3_nic_set_real_num_queue(netdev);
393 	if (ret)
394 		return ret;
395 
396 	ret = hns3_nic_net_up(netdev);
397 	if (ret) {
398 		netdev_err(netdev,
399 			   "hns net up fail, ret=%d!\n", ret);
400 		return ret;
401 	}
402 
403 	kinfo = &h->kinfo;
404 	for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
405 		netdev_set_prio_tc_map(netdev, i,
406 				       kinfo->prio_tc[i]);
407 	}
408 
409 	if (h->ae_algo->ops->set_timer_task)
410 		h->ae_algo->ops->set_timer_task(priv->ae_handle, true);
411 
412 	return 0;
413 }
414 
415 static void hns3_nic_net_down(struct net_device *netdev)
416 {
417 	struct hns3_nic_priv *priv = netdev_priv(netdev);
418 	struct hnae3_handle *h = hns3_get_handle(netdev);
419 	const struct hnae3_ae_ops *ops;
420 	int i;
421 
422 	/* disable vectors */
423 	for (i = 0; i < priv->vector_num; i++)
424 		hns3_vector_disable(&priv->tqp_vector[i]);
425 
426 	/* disable rcb */
427 	for (i = 0; i < h->kinfo.num_tqps; i++)
428 		hns3_tqp_disable(h->kinfo.tqp[i]);
429 
430 	/* stop ae_dev */
431 	ops = priv->ae_handle->ae_algo->ops;
432 	if (ops->stop)
433 		ops->stop(priv->ae_handle);
434 
435 	/* free irq resources */
436 	hns3_nic_uninit_irq(priv);
437 
438 	hns3_clear_all_ring(priv->ae_handle);
439 }
440 
441 static int hns3_nic_net_stop(struct net_device *netdev)
442 {
443 	struct hns3_nic_priv *priv = netdev_priv(netdev);
444 	struct hnae3_handle *h = hns3_get_handle(netdev);
445 
446 	if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
447 		return 0;
448 
449 	if (h->ae_algo->ops->set_timer_task)
450 		h->ae_algo->ops->set_timer_task(priv->ae_handle, false);
451 
452 	netif_tx_stop_all_queues(netdev);
453 	netif_carrier_off(netdev);
454 
455 	hns3_nic_net_down(netdev);
456 
457 	return 0;
458 }
459 
460 static int hns3_nic_uc_sync(struct net_device *netdev,
461 			    const unsigned char *addr)
462 {
463 	struct hnae3_handle *h = hns3_get_handle(netdev);
464 
465 	if (h->ae_algo->ops->add_uc_addr)
466 		return h->ae_algo->ops->add_uc_addr(h, addr);
467 
468 	return 0;
469 }
470 
471 static int hns3_nic_uc_unsync(struct net_device *netdev,
472 			      const unsigned char *addr)
473 {
474 	struct hnae3_handle *h = hns3_get_handle(netdev);
475 
476 	if (h->ae_algo->ops->rm_uc_addr)
477 		return h->ae_algo->ops->rm_uc_addr(h, addr);
478 
479 	return 0;
480 }
481 
482 static int hns3_nic_mc_sync(struct net_device *netdev,
483 			    const unsigned char *addr)
484 {
485 	struct hnae3_handle *h = hns3_get_handle(netdev);
486 
487 	if (h->ae_algo->ops->add_mc_addr)
488 		return h->ae_algo->ops->add_mc_addr(h, addr);
489 
490 	return 0;
491 }
492 
493 static int hns3_nic_mc_unsync(struct net_device *netdev,
494 			      const unsigned char *addr)
495 {
496 	struct hnae3_handle *h = hns3_get_handle(netdev);
497 
498 	if (h->ae_algo->ops->rm_mc_addr)
499 		return h->ae_algo->ops->rm_mc_addr(h, addr);
500 
501 	return 0;
502 }
503 
504 static u8 hns3_get_netdev_flags(struct net_device *netdev)
505 {
506 	u8 flags = 0;
507 
508 	if (netdev->flags & IFF_PROMISC) {
509 		flags = HNAE3_USER_UPE | HNAE3_USER_MPE;
510 	} else {
511 		flags |= HNAE3_VLAN_FLTR;
512 		if (netdev->flags & IFF_ALLMULTI)
513 			flags |= HNAE3_USER_MPE;
514 	}
515 
516 	return flags;
517 }
518 
519 static void hns3_nic_set_rx_mode(struct net_device *netdev)
520 {
521 	struct hnae3_handle *h = hns3_get_handle(netdev);
522 	u8 new_flags;
523 	int ret;
524 
525 	new_flags = hns3_get_netdev_flags(netdev);
526 
527 	ret = __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
528 	if (ret) {
529 		netdev_err(netdev, "sync uc address fail\n");
530 		if (ret == -ENOSPC)
531 			new_flags |= HNAE3_OVERFLOW_UPE;
532 	}
533 
534 	if (netdev->flags & IFF_MULTICAST) {
535 		ret = __dev_mc_sync(netdev, hns3_nic_mc_sync,
536 				    hns3_nic_mc_unsync);
537 		if (ret) {
538 			netdev_err(netdev, "sync mc address fail\n");
539 			if (ret == -ENOSPC)
540 				new_flags |= HNAE3_OVERFLOW_MPE;
541 		}
542 	}
543 
544 	hns3_update_promisc_mode(netdev, new_flags);
545 	/* User mode Promisc mode enable and vlan filtering is disabled to
546 	 * let all packets in. MAC-VLAN Table overflow Promisc enabled and
547 	 * vlan fitering is enabled
548 	 */
549 	hns3_enable_vlan_filter(netdev, new_flags & HNAE3_VLAN_FLTR);
550 	h->netdev_flags = new_flags;
551 }
552 
553 int hns3_update_promisc_mode(struct net_device *netdev, u8 promisc_flags)
554 {
555 	struct hns3_nic_priv *priv = netdev_priv(netdev);
556 	struct hnae3_handle *h = priv->ae_handle;
557 
558 	if (h->ae_algo->ops->set_promisc_mode) {
559 		return h->ae_algo->ops->set_promisc_mode(h,
560 						promisc_flags & HNAE3_UPE,
561 						promisc_flags & HNAE3_MPE);
562 	}
563 
564 	return 0;
565 }
566 
567 void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
568 {
569 	struct hns3_nic_priv *priv = netdev_priv(netdev);
570 	struct hnae3_handle *h = priv->ae_handle;
571 	bool last_state;
572 
573 	if (h->pdev->revision >= 0x21 && h->ae_algo->ops->enable_vlan_filter) {
574 		last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
575 		if (enable != last_state) {
576 			netdev_info(netdev,
577 				    "%s vlan filter\n",
578 				    enable ? "enable" : "disable");
579 			h->ae_algo->ops->enable_vlan_filter(h, enable);
580 		}
581 	}
582 }
583 
584 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
585 			u16 *mss, u32 *type_cs_vlan_tso)
586 {
587 	u32 l4_offset, hdr_len;
588 	union l3_hdr_info l3;
589 	union l4_hdr_info l4;
590 	u32 l4_paylen;
591 	int ret;
592 
593 	if (!skb_is_gso(skb))
594 		return 0;
595 
596 	ret = skb_cow_head(skb, 0);
597 	if (ret)
598 		return ret;
599 
600 	l3.hdr = skb_network_header(skb);
601 	l4.hdr = skb_transport_header(skb);
602 
603 	/* Software should clear the IPv4's checksum field when tso is
604 	 * needed.
605 	 */
606 	if (l3.v4->version == 4)
607 		l3.v4->check = 0;
608 
609 	/* tunnel packet.*/
610 	if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
611 					 SKB_GSO_GRE_CSUM |
612 					 SKB_GSO_UDP_TUNNEL |
613 					 SKB_GSO_UDP_TUNNEL_CSUM)) {
614 		if ((!(skb_shinfo(skb)->gso_type &
615 		    SKB_GSO_PARTIAL)) &&
616 		    (skb_shinfo(skb)->gso_type &
617 		    SKB_GSO_UDP_TUNNEL_CSUM)) {
618 			/* Software should clear the udp's checksum
619 			 * field when tso is needed.
620 			 */
621 			l4.udp->check = 0;
622 		}
623 		/* reset l3&l4 pointers from outer to inner headers */
624 		l3.hdr = skb_inner_network_header(skb);
625 		l4.hdr = skb_inner_transport_header(skb);
626 
627 		/* Software should clear the IPv4's checksum field when
628 		 * tso is needed.
629 		 */
630 		if (l3.v4->version == 4)
631 			l3.v4->check = 0;
632 	}
633 
634 	/* normal or tunnel packet*/
635 	l4_offset = l4.hdr - skb->data;
636 	hdr_len = (l4.tcp->doff * 4) + l4_offset;
637 
638 	/* remove payload length from inner pseudo checksum when tso*/
639 	l4_paylen = skb->len - l4_offset;
640 	csum_replace_by_diff(&l4.tcp->check,
641 			     (__force __wsum)htonl(l4_paylen));
642 
643 	/* find the txbd field values */
644 	*paylen = skb->len - hdr_len;
645 	hnae3_set_bit(*type_cs_vlan_tso,
646 		      HNS3_TXD_TSO_B, 1);
647 
648 	/* get MSS for TSO */
649 	*mss = skb_shinfo(skb)->gso_size;
650 
651 	return 0;
652 }
653 
654 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
655 				u8 *il4_proto)
656 {
657 	union {
658 		struct iphdr *v4;
659 		struct ipv6hdr *v6;
660 		unsigned char *hdr;
661 	} l3;
662 	unsigned char *l4_hdr;
663 	unsigned char *exthdr;
664 	u8 l4_proto_tmp;
665 	__be16 frag_off;
666 
667 	/* find outer header point */
668 	l3.hdr = skb_network_header(skb);
669 	l4_hdr = skb_transport_header(skb);
670 
671 	if (skb->protocol == htons(ETH_P_IPV6)) {
672 		exthdr = l3.hdr + sizeof(*l3.v6);
673 		l4_proto_tmp = l3.v6->nexthdr;
674 		if (l4_hdr != exthdr)
675 			ipv6_skip_exthdr(skb, exthdr - skb->data,
676 					 &l4_proto_tmp, &frag_off);
677 	} else if (skb->protocol == htons(ETH_P_IP)) {
678 		l4_proto_tmp = l3.v4->protocol;
679 	} else {
680 		return -EINVAL;
681 	}
682 
683 	*ol4_proto = l4_proto_tmp;
684 
685 	/* tunnel packet */
686 	if (!skb->encapsulation) {
687 		*il4_proto = 0;
688 		return 0;
689 	}
690 
691 	/* find inner header point */
692 	l3.hdr = skb_inner_network_header(skb);
693 	l4_hdr = skb_inner_transport_header(skb);
694 
695 	if (l3.v6->version == 6) {
696 		exthdr = l3.hdr + sizeof(*l3.v6);
697 		l4_proto_tmp = l3.v6->nexthdr;
698 		if (l4_hdr != exthdr)
699 			ipv6_skip_exthdr(skb, exthdr - skb->data,
700 					 &l4_proto_tmp, &frag_off);
701 	} else if (l3.v4->version == 4) {
702 		l4_proto_tmp = l3.v4->protocol;
703 	}
704 
705 	*il4_proto = l4_proto_tmp;
706 
707 	return 0;
708 }
709 
710 static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
711 				u8 il4_proto, u32 *type_cs_vlan_tso,
712 				u32 *ol_type_vlan_len_msec)
713 {
714 	union {
715 		struct iphdr *v4;
716 		struct ipv6hdr *v6;
717 		unsigned char *hdr;
718 	} l3;
719 	union {
720 		struct tcphdr *tcp;
721 		struct udphdr *udp;
722 		struct gre_base_hdr *gre;
723 		unsigned char *hdr;
724 	} l4;
725 	unsigned char *l2_hdr;
726 	u8 l4_proto = ol4_proto;
727 	u32 ol2_len;
728 	u32 ol3_len;
729 	u32 ol4_len;
730 	u32 l2_len;
731 	u32 l3_len;
732 
733 	l3.hdr = skb_network_header(skb);
734 	l4.hdr = skb_transport_header(skb);
735 
736 	/* compute L2 header size for normal packet, defined in 2 Bytes */
737 	l2_len = l3.hdr - skb->data;
738 	hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
739 			HNS3_TXD_L2LEN_S, l2_len >> 1);
740 
741 	/* tunnel packet*/
742 	if (skb->encapsulation) {
743 		/* compute OL2 header size, defined in 2 Bytes */
744 		ol2_len = l2_len;
745 		hnae3_set_field(*ol_type_vlan_len_msec,
746 				HNS3_TXD_L2LEN_M,
747 				HNS3_TXD_L2LEN_S, ol2_len >> 1);
748 
749 		/* compute OL3 header size, defined in 4 Bytes */
750 		ol3_len = l4.hdr - l3.hdr;
751 		hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
752 				HNS3_TXD_L3LEN_S, ol3_len >> 2);
753 
754 		/* MAC in UDP, MAC in GRE (0x6558)*/
755 		if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
756 			/* switch MAC header ptr from outer to inner header.*/
757 			l2_hdr = skb_inner_mac_header(skb);
758 
759 			/* compute OL4 header size, defined in 4 Bytes. */
760 			ol4_len = l2_hdr - l4.hdr;
761 			hnae3_set_field(*ol_type_vlan_len_msec,
762 					HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
763 					ol4_len >> 2);
764 
765 			/* switch IP header ptr from outer to inner header */
766 			l3.hdr = skb_inner_network_header(skb);
767 
768 			/* compute inner l2 header size, defined in 2 Bytes. */
769 			l2_len = l3.hdr - l2_hdr;
770 			hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
771 					HNS3_TXD_L2LEN_S, l2_len >> 1);
772 		} else {
773 			/* skb packet types not supported by hardware,
774 			 * txbd len fild doesn't be filled.
775 			 */
776 			return;
777 		}
778 
779 		/* switch L4 header pointer from outer to inner */
780 		l4.hdr = skb_inner_transport_header(skb);
781 
782 		l4_proto = il4_proto;
783 	}
784 
785 	/* compute inner(/normal) L3 header size, defined in 4 Bytes */
786 	l3_len = l4.hdr - l3.hdr;
787 	hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
788 			HNS3_TXD_L3LEN_S, l3_len >> 2);
789 
790 	/* compute inner(/normal) L4 header size, defined in 4 Bytes */
791 	switch (l4_proto) {
792 	case IPPROTO_TCP:
793 		hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
794 				HNS3_TXD_L4LEN_S, l4.tcp->doff);
795 		break;
796 	case IPPROTO_SCTP:
797 		hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
798 				HNS3_TXD_L4LEN_S,
799 				(sizeof(struct sctphdr) >> 2));
800 		break;
801 	case IPPROTO_UDP:
802 		hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
803 				HNS3_TXD_L4LEN_S,
804 				(sizeof(struct udphdr) >> 2));
805 		break;
806 	default:
807 		/* skb packet types not supported by hardware,
808 		 * txbd len fild doesn't be filled.
809 		 */
810 		return;
811 	}
812 }
813 
814 /* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
815  * and it is udp packet, which has a dest port as the IANA assigned.
816  * the hardware is expected to do the checksum offload, but the
817  * hardware will not do the checksum offload when udp dest port is
818  * 4789.
819  */
820 static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
821 {
822 #define IANA_VXLAN_PORT	4789
823 	union {
824 		struct tcphdr *tcp;
825 		struct udphdr *udp;
826 		struct gre_base_hdr *gre;
827 		unsigned char *hdr;
828 	} l4;
829 
830 	l4.hdr = skb_transport_header(skb);
831 
832 	if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
833 		return false;
834 
835 	skb_checksum_help(skb);
836 
837 	return true;
838 }
839 
840 static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
841 				   u8 il4_proto, u32 *type_cs_vlan_tso,
842 				   u32 *ol_type_vlan_len_msec)
843 {
844 	union {
845 		struct iphdr *v4;
846 		struct ipv6hdr *v6;
847 		unsigned char *hdr;
848 	} l3;
849 	u32 l4_proto = ol4_proto;
850 
851 	l3.hdr = skb_network_header(skb);
852 
853 	/* define OL3 type and tunnel type(OL4).*/
854 	if (skb->encapsulation) {
855 		/* define outer network header type.*/
856 		if (skb->protocol == htons(ETH_P_IP)) {
857 			if (skb_is_gso(skb))
858 				hnae3_set_field(*ol_type_vlan_len_msec,
859 						HNS3_TXD_OL3T_M,
860 						HNS3_TXD_OL3T_S,
861 						HNS3_OL3T_IPV4_CSUM);
862 			else
863 				hnae3_set_field(*ol_type_vlan_len_msec,
864 						HNS3_TXD_OL3T_M,
865 						HNS3_TXD_OL3T_S,
866 						HNS3_OL3T_IPV4_NO_CSUM);
867 
868 		} else if (skb->protocol == htons(ETH_P_IPV6)) {
869 			hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
870 					HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
871 		}
872 
873 		/* define tunnel type(OL4).*/
874 		switch (l4_proto) {
875 		case IPPROTO_UDP:
876 			hnae3_set_field(*ol_type_vlan_len_msec,
877 					HNS3_TXD_TUNTYPE_M,
878 					HNS3_TXD_TUNTYPE_S,
879 					HNS3_TUN_MAC_IN_UDP);
880 			break;
881 		case IPPROTO_GRE:
882 			hnae3_set_field(*ol_type_vlan_len_msec,
883 					HNS3_TXD_TUNTYPE_M,
884 					HNS3_TXD_TUNTYPE_S,
885 					HNS3_TUN_NVGRE);
886 			break;
887 		default:
888 			/* drop the skb tunnel packet if hardware don't support,
889 			 * because hardware can't calculate csum when TSO.
890 			 */
891 			if (skb_is_gso(skb))
892 				return -EDOM;
893 
894 			/* the stack computes the IP header already,
895 			 * driver calculate l4 checksum when not TSO.
896 			 */
897 			skb_checksum_help(skb);
898 			return 0;
899 		}
900 
901 		l3.hdr = skb_inner_network_header(skb);
902 		l4_proto = il4_proto;
903 	}
904 
905 	if (l3.v4->version == 4) {
906 		hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
907 				HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
908 
909 		/* the stack computes the IP header already, the only time we
910 		 * need the hardware to recompute it is in the case of TSO.
911 		 */
912 		if (skb_is_gso(skb))
913 			hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
914 	} else if (l3.v6->version == 6) {
915 		hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
916 				HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
917 	}
918 
919 	switch (l4_proto) {
920 	case IPPROTO_TCP:
921 		hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
922 		hnae3_set_field(*type_cs_vlan_tso,
923 				HNS3_TXD_L4T_M,
924 				HNS3_TXD_L4T_S,
925 				HNS3_L4T_TCP);
926 		break;
927 	case IPPROTO_UDP:
928 		if (hns3_tunnel_csum_bug(skb))
929 			break;
930 
931 		hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
932 		hnae3_set_field(*type_cs_vlan_tso,
933 				HNS3_TXD_L4T_M,
934 				HNS3_TXD_L4T_S,
935 				HNS3_L4T_UDP);
936 		break;
937 	case IPPROTO_SCTP:
938 		hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
939 		hnae3_set_field(*type_cs_vlan_tso,
940 				HNS3_TXD_L4T_M,
941 				HNS3_TXD_L4T_S,
942 				HNS3_L4T_SCTP);
943 		break;
944 	default:
945 		/* drop the skb tunnel packet if hardware don't support,
946 		 * because hardware can't calculate csum when TSO.
947 		 */
948 		if (skb_is_gso(skb))
949 			return -EDOM;
950 
951 		/* the stack computes the IP header already,
952 		 * driver calculate l4 checksum when not TSO.
953 		 */
954 		skb_checksum_help(skb);
955 		return 0;
956 	}
957 
958 	return 0;
959 }
960 
961 static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
962 {
963 	/* Config bd buffer end */
964 	hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
965 			HNS3_TXD_BDTYPE_S, 0);
966 	hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
967 	hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
968 	hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
969 }
970 
971 static int hns3_fill_desc_vtags(struct sk_buff *skb,
972 				struct hns3_enet_ring *tx_ring,
973 				u32 *inner_vlan_flag,
974 				u32 *out_vlan_flag,
975 				u16 *inner_vtag,
976 				u16 *out_vtag)
977 {
978 #define HNS3_TX_VLAN_PRIO_SHIFT 13
979 
980 	if (skb->protocol == htons(ETH_P_8021Q) &&
981 	    !(tx_ring->tqp->handle->kinfo.netdev->features &
982 	    NETIF_F_HW_VLAN_CTAG_TX)) {
983 		/* When HW VLAN acceleration is turned off, and the stack
984 		 * sets the protocol to 802.1q, the driver just need to
985 		 * set the protocol to the encapsulated ethertype.
986 		 */
987 		skb->protocol = vlan_get_protocol(skb);
988 		return 0;
989 	}
990 
991 	if (skb_vlan_tag_present(skb)) {
992 		u16 vlan_tag;
993 
994 		vlan_tag = skb_vlan_tag_get(skb);
995 		vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
996 
997 		/* Based on hw strategy, use out_vtag in two layer tag case,
998 		 * and use inner_vtag in one tag case.
999 		 */
1000 		if (skb->protocol == htons(ETH_P_8021Q)) {
1001 			hnae3_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
1002 			*out_vtag = vlan_tag;
1003 		} else {
1004 			hnae3_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
1005 			*inner_vtag = vlan_tag;
1006 		}
1007 	} else if (skb->protocol == htons(ETH_P_8021Q)) {
1008 		struct vlan_ethhdr *vhdr;
1009 		int rc;
1010 
1011 		rc = skb_cow_head(skb, 0);
1012 		if (rc < 0)
1013 			return rc;
1014 		vhdr = (struct vlan_ethhdr *)skb->data;
1015 		vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
1016 					<< HNS3_TX_VLAN_PRIO_SHIFT);
1017 	}
1018 
1019 	skb->protocol = vlan_get_protocol(skb);
1020 	return 0;
1021 }
1022 
1023 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
1024 			  int size, int frag_end, enum hns_desc_type type)
1025 {
1026 	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1027 	struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1028 	struct device *dev = ring_to_dev(ring);
1029 	u32 ol_type_vlan_len_msec = 0;
1030 	u16 bdtp_fe_sc_vld_ra_ri = 0;
1031 	struct skb_frag_struct *frag;
1032 	unsigned int frag_buf_num;
1033 	u32 type_cs_vlan_tso = 0;
1034 	struct sk_buff *skb;
1035 	u16 inner_vtag = 0;
1036 	u16 out_vtag = 0;
1037 	unsigned int k;
1038 	int sizeoflast;
1039 	u32 paylen = 0;
1040 	dma_addr_t dma;
1041 	u16 mss = 0;
1042 	u8 ol4_proto;
1043 	u8 il4_proto;
1044 	int ret;
1045 
1046 	if (type == DESC_TYPE_SKB) {
1047 		skb = (struct sk_buff *)priv;
1048 		paylen = skb->len;
1049 
1050 		ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
1051 					   &ol_type_vlan_len_msec,
1052 					   &inner_vtag, &out_vtag);
1053 		if (unlikely(ret))
1054 			return ret;
1055 
1056 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
1057 			skb_reset_mac_len(skb);
1058 
1059 			ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1060 			if (ret)
1061 				return ret;
1062 			hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
1063 					    &type_cs_vlan_tso,
1064 					    &ol_type_vlan_len_msec);
1065 			ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
1066 						      &type_cs_vlan_tso,
1067 						      &ol_type_vlan_len_msec);
1068 			if (ret)
1069 				return ret;
1070 
1071 			ret = hns3_set_tso(skb, &paylen, &mss,
1072 					   &type_cs_vlan_tso);
1073 			if (ret)
1074 				return ret;
1075 		}
1076 
1077 		/* Set txbd */
1078 		desc->tx.ol_type_vlan_len_msec =
1079 			cpu_to_le32(ol_type_vlan_len_msec);
1080 		desc->tx.type_cs_vlan_tso_len =
1081 			cpu_to_le32(type_cs_vlan_tso);
1082 		desc->tx.paylen = cpu_to_le32(paylen);
1083 		desc->tx.mss = cpu_to_le16(mss);
1084 		desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1085 		desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1086 
1087 		dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1088 	} else {
1089 		frag = (struct skb_frag_struct *)priv;
1090 		dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1091 	}
1092 
1093 	if (dma_mapping_error(ring->dev, dma)) {
1094 		ring->stats.sw_err_cnt++;
1095 		return -ENOMEM;
1096 	}
1097 
1098 	desc_cb->length = size;
1099 
1100 	frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1101 	sizeoflast = size % HNS3_MAX_BD_SIZE;
1102 	sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1103 
1104 	/* When frag size is bigger than hardware limit, split this frag */
1105 	for (k = 0; k < frag_buf_num; k++) {
1106 		/* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
1107 		desc_cb->priv = priv;
1108 		desc_cb->dma = dma + HNS3_MAX_BD_SIZE * k;
1109 		desc_cb->type = (type == DESC_TYPE_SKB && !k) ?
1110 					DESC_TYPE_SKB : DESC_TYPE_PAGE;
1111 
1112 		/* now, fill the descriptor */
1113 		desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
1114 		desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1115 				(u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1116 		hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri,
1117 				       frag_end && (k == frag_buf_num - 1) ?
1118 						1 : 0);
1119 		desc->tx.bdtp_fe_sc_vld_ra_ri =
1120 				cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
1121 
1122 		/* move ring pointer to next.*/
1123 		ring_ptr_move_fw(ring, next_to_use);
1124 
1125 		desc_cb = &ring->desc_cb[ring->next_to_use];
1126 		desc = &ring->desc[ring->next_to_use];
1127 	}
1128 
1129 	return 0;
1130 }
1131 
1132 static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
1133 				   struct hns3_enet_ring *ring)
1134 {
1135 	struct sk_buff *skb = *out_skb;
1136 	struct skb_frag_struct *frag;
1137 	int bdnum_for_frag;
1138 	int frag_num;
1139 	int buf_num;
1140 	int size;
1141 	int i;
1142 
1143 	size = skb_headlen(skb);
1144 	buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1145 
1146 	frag_num = skb_shinfo(skb)->nr_frags;
1147 	for (i = 0; i < frag_num; i++) {
1148 		frag = &skb_shinfo(skb)->frags[i];
1149 		size = skb_frag_size(frag);
1150 		bdnum_for_frag =
1151 			(size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1152 		if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
1153 			return -ENOMEM;
1154 
1155 		buf_num += bdnum_for_frag;
1156 	}
1157 
1158 	if (buf_num > ring_space(ring))
1159 		return -EBUSY;
1160 
1161 	*bnum = buf_num;
1162 	return 0;
1163 }
1164 
1165 static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1166 				  struct hns3_enet_ring *ring)
1167 {
1168 	struct sk_buff *skb = *out_skb;
1169 	int buf_num;
1170 
1171 	/* No. of segments (plus a header) */
1172 	buf_num = skb_shinfo(skb)->nr_frags + 1;
1173 
1174 	if (unlikely(ring_space(ring) < buf_num))
1175 		return -EBUSY;
1176 
1177 	*bnum = buf_num;
1178 
1179 	return 0;
1180 }
1181 
1182 static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
1183 {
1184 	struct device *dev = ring_to_dev(ring);
1185 	unsigned int i;
1186 
1187 	for (i = 0; i < ring->desc_num; i++) {
1188 		/* check if this is where we started */
1189 		if (ring->next_to_use == next_to_use_orig)
1190 			break;
1191 
1192 		/* unmap the descriptor dma address */
1193 		if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1194 			dma_unmap_single(dev,
1195 					 ring->desc_cb[ring->next_to_use].dma,
1196 					ring->desc_cb[ring->next_to_use].length,
1197 					DMA_TO_DEVICE);
1198 		else if (ring->desc_cb[ring->next_to_use].length)
1199 			dma_unmap_page(dev,
1200 				       ring->desc_cb[ring->next_to_use].dma,
1201 				       ring->desc_cb[ring->next_to_use].length,
1202 				       DMA_TO_DEVICE);
1203 
1204 		ring->desc_cb[ring->next_to_use].length = 0;
1205 
1206 		/* rollback one */
1207 		ring_ptr_move_bw(ring, next_to_use);
1208 	}
1209 }
1210 
1211 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
1212 {
1213 	struct hns3_nic_priv *priv = netdev_priv(netdev);
1214 	struct hns3_nic_ring_data *ring_data =
1215 		&tx_ring_data(priv, skb->queue_mapping);
1216 	struct hns3_enet_ring *ring = ring_data->ring;
1217 	struct netdev_queue *dev_queue;
1218 	struct skb_frag_struct *frag;
1219 	int next_to_use_head;
1220 	int next_to_use_frag;
1221 	int buf_num;
1222 	int seg_num;
1223 	int size;
1224 	int ret;
1225 	int i;
1226 
1227 	/* Prefetch the data used later */
1228 	prefetch(skb->data);
1229 
1230 	switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1231 	case -EBUSY:
1232 		u64_stats_update_begin(&ring->syncp);
1233 		ring->stats.tx_busy++;
1234 		u64_stats_update_end(&ring->syncp);
1235 
1236 		goto out_net_tx_busy;
1237 	case -ENOMEM:
1238 		u64_stats_update_begin(&ring->syncp);
1239 		ring->stats.sw_err_cnt++;
1240 		u64_stats_update_end(&ring->syncp);
1241 		netdev_err(netdev, "no memory to xmit!\n");
1242 
1243 		goto out_err_tx_ok;
1244 	default:
1245 		break;
1246 	}
1247 
1248 	/* No. of segments (plus a header) */
1249 	seg_num = skb_shinfo(skb)->nr_frags + 1;
1250 	/* Fill the first part */
1251 	size = skb_headlen(skb);
1252 
1253 	next_to_use_head = ring->next_to_use;
1254 
1255 	ret = priv->ops.fill_desc(ring, skb, size, seg_num == 1 ? 1 : 0,
1256 				  DESC_TYPE_SKB);
1257 	if (ret)
1258 		goto head_fill_err;
1259 
1260 	next_to_use_frag = ring->next_to_use;
1261 	/* Fill the fragments */
1262 	for (i = 1; i < seg_num; i++) {
1263 		frag = &skb_shinfo(skb)->frags[i - 1];
1264 		size = skb_frag_size(frag);
1265 
1266 		ret = priv->ops.fill_desc(ring, frag, size,
1267 					  seg_num - 1 == i ? 1 : 0,
1268 					  DESC_TYPE_PAGE);
1269 
1270 		if (ret)
1271 			goto frag_fill_err;
1272 	}
1273 
1274 	/* Complete translate all packets */
1275 	dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1276 	netdev_tx_sent_queue(dev_queue, skb->len);
1277 
1278 	wmb(); /* Commit all data before submit */
1279 
1280 	hnae3_queue_xmit(ring->tqp, buf_num);
1281 
1282 	return NETDEV_TX_OK;
1283 
1284 frag_fill_err:
1285 	hns3_clear_desc(ring, next_to_use_frag);
1286 
1287 head_fill_err:
1288 	hns3_clear_desc(ring, next_to_use_head);
1289 
1290 out_err_tx_ok:
1291 	dev_kfree_skb_any(skb);
1292 	return NETDEV_TX_OK;
1293 
1294 out_net_tx_busy:
1295 	netif_stop_subqueue(netdev, ring_data->queue_index);
1296 	smp_mb(); /* Commit all data before submit */
1297 
1298 	return NETDEV_TX_BUSY;
1299 }
1300 
1301 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1302 {
1303 	struct hnae3_handle *h = hns3_get_handle(netdev);
1304 	struct sockaddr *mac_addr = p;
1305 	int ret;
1306 
1307 	if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1308 		return -EADDRNOTAVAIL;
1309 
1310 	if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1311 		netdev_info(netdev, "already using mac address %pM\n",
1312 			    mac_addr->sa_data);
1313 		return 0;
1314 	}
1315 
1316 	ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
1317 	if (ret) {
1318 		netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1319 		return ret;
1320 	}
1321 
1322 	ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1323 
1324 	return 0;
1325 }
1326 
1327 static int hns3_nic_do_ioctl(struct net_device *netdev,
1328 			     struct ifreq *ifr, int cmd)
1329 {
1330 	struct hnae3_handle *h = hns3_get_handle(netdev);
1331 
1332 	if (!netif_running(netdev))
1333 		return -EINVAL;
1334 
1335 	if (!h->ae_algo->ops->do_ioctl)
1336 		return -EOPNOTSUPP;
1337 
1338 	return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1339 }
1340 
1341 static int hns3_nic_set_features(struct net_device *netdev,
1342 				 netdev_features_t features)
1343 {
1344 	netdev_features_t changed = netdev->features ^ features;
1345 	struct hns3_nic_priv *priv = netdev_priv(netdev);
1346 	struct hnae3_handle *h = priv->ae_handle;
1347 	int ret;
1348 
1349 	if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1350 		if (features & (NETIF_F_TSO | NETIF_F_TSO6))
1351 			priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1352 		else
1353 			priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1354 	}
1355 
1356 	if (changed & (NETIF_F_GRO_HW) && h->ae_algo->ops->set_gro_en) {
1357 		if (features & NETIF_F_GRO_HW)
1358 			ret = h->ae_algo->ops->set_gro_en(h, true);
1359 		else
1360 			ret = h->ae_algo->ops->set_gro_en(h, false);
1361 		if (ret)
1362 			return ret;
1363 	}
1364 
1365 	if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1366 	    h->ae_algo->ops->enable_vlan_filter) {
1367 		if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1368 			h->ae_algo->ops->enable_vlan_filter(h, true);
1369 		else
1370 			h->ae_algo->ops->enable_vlan_filter(h, false);
1371 	}
1372 
1373 	if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1374 	    h->ae_algo->ops->enable_hw_strip_rxvtag) {
1375 		if (features & NETIF_F_HW_VLAN_CTAG_RX)
1376 			ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1377 		else
1378 			ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1379 
1380 		if (ret)
1381 			return ret;
1382 	}
1383 
1384 	if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1385 		if (features & NETIF_F_NTUPLE)
1386 			h->ae_algo->ops->enable_fd(h, true);
1387 		else
1388 			h->ae_algo->ops->enable_fd(h, false);
1389 	}
1390 
1391 	netdev->features = features;
1392 	return 0;
1393 }
1394 
1395 static void hns3_nic_get_stats64(struct net_device *netdev,
1396 				 struct rtnl_link_stats64 *stats)
1397 {
1398 	struct hns3_nic_priv *priv = netdev_priv(netdev);
1399 	int queue_num = priv->ae_handle->kinfo.num_tqps;
1400 	struct hnae3_handle *handle = priv->ae_handle;
1401 	struct hns3_enet_ring *ring;
1402 	unsigned int start;
1403 	unsigned int idx;
1404 	u64 tx_bytes = 0;
1405 	u64 rx_bytes = 0;
1406 	u64 tx_pkts = 0;
1407 	u64 rx_pkts = 0;
1408 	u64 tx_drop = 0;
1409 	u64 rx_drop = 0;
1410 
1411 	if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1412 		return;
1413 
1414 	handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1415 
1416 	for (idx = 0; idx < queue_num; idx++) {
1417 		/* fetch the tx stats */
1418 		ring = priv->ring_data[idx].ring;
1419 		do {
1420 			start = u64_stats_fetch_begin_irq(&ring->syncp);
1421 			tx_bytes += ring->stats.tx_bytes;
1422 			tx_pkts += ring->stats.tx_pkts;
1423 			tx_drop += ring->stats.tx_busy;
1424 			tx_drop += ring->stats.sw_err_cnt;
1425 		} while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1426 
1427 		/* fetch the rx stats */
1428 		ring = priv->ring_data[idx + queue_num].ring;
1429 		do {
1430 			start = u64_stats_fetch_begin_irq(&ring->syncp);
1431 			rx_bytes += ring->stats.rx_bytes;
1432 			rx_pkts += ring->stats.rx_pkts;
1433 			rx_drop += ring->stats.non_vld_descs;
1434 			rx_drop += ring->stats.err_pkt_len;
1435 			rx_drop += ring->stats.l2_err;
1436 		} while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1437 	}
1438 
1439 	stats->tx_bytes = tx_bytes;
1440 	stats->tx_packets = tx_pkts;
1441 	stats->rx_bytes = rx_bytes;
1442 	stats->rx_packets = rx_pkts;
1443 
1444 	stats->rx_errors = netdev->stats.rx_errors;
1445 	stats->multicast = netdev->stats.multicast;
1446 	stats->rx_length_errors = netdev->stats.rx_length_errors;
1447 	stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1448 	stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1449 
1450 	stats->tx_errors = netdev->stats.tx_errors;
1451 	stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1452 	stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
1453 	stats->collisions = netdev->stats.collisions;
1454 	stats->rx_over_errors = netdev->stats.rx_over_errors;
1455 	stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1456 	stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1457 	stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1458 	stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1459 	stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1460 	stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1461 	stats->tx_window_errors = netdev->stats.tx_window_errors;
1462 	stats->rx_compressed = netdev->stats.rx_compressed;
1463 	stats->tx_compressed = netdev->stats.tx_compressed;
1464 }
1465 
1466 static int hns3_setup_tc(struct net_device *netdev, void *type_data)
1467 {
1468 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
1469 	struct hnae3_handle *h = hns3_get_handle(netdev);
1470 	struct hnae3_knic_private_info *kinfo = &h->kinfo;
1471 	u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1472 	u8 tc = mqprio_qopt->qopt.num_tc;
1473 	u16 mode = mqprio_qopt->mode;
1474 	u8 hw = mqprio_qopt->qopt.hw;
1475 	bool if_running;
1476 	int ret;
1477 
1478 	if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1479 	       mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1480 		return -EOPNOTSUPP;
1481 
1482 	if (tc > HNAE3_MAX_TC)
1483 		return -EINVAL;
1484 
1485 	if (!netdev)
1486 		return -EINVAL;
1487 
1488 	if_running = netif_running(netdev);
1489 	if (if_running) {
1490 		hns3_nic_net_stop(netdev);
1491 		msleep(100);
1492 	}
1493 
1494 	ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1495 		kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
1496 	if (ret)
1497 		goto out;
1498 
1499 	ret = hns3_nic_set_real_num_queue(netdev);
1500 
1501 out:
1502 	if (if_running)
1503 		hns3_nic_net_open(netdev);
1504 
1505 	return ret;
1506 }
1507 
1508 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1509 			     void *type_data)
1510 {
1511 	if (type != TC_SETUP_QDISC_MQPRIO)
1512 		return -EOPNOTSUPP;
1513 
1514 	return hns3_setup_tc(dev, type_data);
1515 }
1516 
1517 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1518 				__be16 proto, u16 vid)
1519 {
1520 	struct hnae3_handle *h = hns3_get_handle(netdev);
1521 	struct hns3_nic_priv *priv = netdev_priv(netdev);
1522 	int ret = -EIO;
1523 
1524 	if (h->ae_algo->ops->set_vlan_filter)
1525 		ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1526 
1527 	if (!ret)
1528 		set_bit(vid, priv->active_vlans);
1529 
1530 	return ret;
1531 }
1532 
1533 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1534 				 __be16 proto, u16 vid)
1535 {
1536 	struct hnae3_handle *h = hns3_get_handle(netdev);
1537 	struct hns3_nic_priv *priv = netdev_priv(netdev);
1538 	int ret = -EIO;
1539 
1540 	if (h->ae_algo->ops->set_vlan_filter)
1541 		ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1542 
1543 	if (!ret)
1544 		clear_bit(vid, priv->active_vlans);
1545 
1546 	return ret;
1547 }
1548 
1549 static int hns3_restore_vlan(struct net_device *netdev)
1550 {
1551 	struct hns3_nic_priv *priv = netdev_priv(netdev);
1552 	int ret = 0;
1553 	u16 vid;
1554 
1555 	for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1556 		ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
1557 		if (ret) {
1558 			netdev_err(netdev, "Restore vlan: %d filter, ret:%d\n",
1559 				   vid, ret);
1560 			return ret;
1561 		}
1562 	}
1563 
1564 	return ret;
1565 }
1566 
1567 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1568 				u8 qos, __be16 vlan_proto)
1569 {
1570 	struct hnae3_handle *h = hns3_get_handle(netdev);
1571 	int ret = -EIO;
1572 
1573 	if (h->ae_algo->ops->set_vf_vlan_filter)
1574 		ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1575 						   qos, vlan_proto);
1576 
1577 	return ret;
1578 }
1579 
1580 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1581 {
1582 	struct hnae3_handle *h = hns3_get_handle(netdev);
1583 	int ret;
1584 
1585 	if (!h->ae_algo->ops->set_mtu)
1586 		return -EOPNOTSUPP;
1587 
1588 	ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1589 	if (ret)
1590 		netdev_err(netdev, "failed to change MTU in hardware %d\n",
1591 			   ret);
1592 	else
1593 		netdev->mtu = new_mtu;
1594 
1595 	return ret;
1596 }
1597 
1598 static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1599 {
1600 	struct hns3_nic_priv *priv = netdev_priv(ndev);
1601 	struct hns3_enet_ring *tx_ring = NULL;
1602 	int timeout_queue = 0;
1603 	int hw_head, hw_tail;
1604 	int i;
1605 
1606 	/* Find the stopped queue the same way the stack does */
1607 	for (i = 0; i < ndev->real_num_tx_queues; i++) {
1608 		struct netdev_queue *q;
1609 		unsigned long trans_start;
1610 
1611 		q = netdev_get_tx_queue(ndev, i);
1612 		trans_start = q->trans_start;
1613 		if (netif_xmit_stopped(q) &&
1614 		    time_after(jiffies,
1615 			       (trans_start + ndev->watchdog_timeo))) {
1616 			timeout_queue = i;
1617 			break;
1618 		}
1619 	}
1620 
1621 	if (i == ndev->num_tx_queues) {
1622 		netdev_info(ndev,
1623 			    "no netdev TX timeout queue found, timeout count: %llu\n",
1624 			    priv->tx_timeout_count);
1625 		return false;
1626 	}
1627 
1628 	tx_ring = priv->ring_data[timeout_queue].ring;
1629 
1630 	hw_head = readl_relaxed(tx_ring->tqp->io_base +
1631 				HNS3_RING_TX_RING_HEAD_REG);
1632 	hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1633 				HNS3_RING_TX_RING_TAIL_REG);
1634 	netdev_info(ndev,
1635 		    "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1636 		    priv->tx_timeout_count,
1637 		    timeout_queue,
1638 		    tx_ring->next_to_use,
1639 		    tx_ring->next_to_clean,
1640 		    hw_head,
1641 		    hw_tail,
1642 		    readl(tx_ring->tqp_vector->mask_addr));
1643 
1644 	return true;
1645 }
1646 
1647 static void hns3_nic_net_timeout(struct net_device *ndev)
1648 {
1649 	struct hns3_nic_priv *priv = netdev_priv(ndev);
1650 	struct hnae3_handle *h = priv->ae_handle;
1651 
1652 	if (!hns3_get_tx_timeo_queue_info(ndev))
1653 		return;
1654 
1655 	priv->tx_timeout_count++;
1656 
1657 	/* request the reset, and let the hclge to determine
1658 	 * which reset level should be done
1659 	 */
1660 	if (h->ae_algo->ops->reset_event)
1661 		h->ae_algo->ops->reset_event(h->pdev, h);
1662 }
1663 
1664 static const struct net_device_ops hns3_nic_netdev_ops = {
1665 	.ndo_open		= hns3_nic_net_open,
1666 	.ndo_stop		= hns3_nic_net_stop,
1667 	.ndo_start_xmit		= hns3_nic_net_xmit,
1668 	.ndo_tx_timeout		= hns3_nic_net_timeout,
1669 	.ndo_set_mac_address	= hns3_nic_net_set_mac_address,
1670 	.ndo_do_ioctl		= hns3_nic_do_ioctl,
1671 	.ndo_change_mtu		= hns3_nic_change_mtu,
1672 	.ndo_set_features	= hns3_nic_set_features,
1673 	.ndo_get_stats64	= hns3_nic_get_stats64,
1674 	.ndo_setup_tc		= hns3_nic_setup_tc,
1675 	.ndo_set_rx_mode	= hns3_nic_set_rx_mode,
1676 	.ndo_vlan_rx_add_vid	= hns3_vlan_rx_add_vid,
1677 	.ndo_vlan_rx_kill_vid	= hns3_vlan_rx_kill_vid,
1678 	.ndo_set_vf_vlan	= hns3_ndo_set_vf_vlan,
1679 };
1680 
1681 static bool hns3_is_phys_func(struct pci_dev *pdev)
1682 {
1683 	u32 dev_id = pdev->device;
1684 
1685 	switch (dev_id) {
1686 	case HNAE3_DEV_ID_GE:
1687 	case HNAE3_DEV_ID_25GE:
1688 	case HNAE3_DEV_ID_25GE_RDMA:
1689 	case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1690 	case HNAE3_DEV_ID_50GE_RDMA:
1691 	case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1692 	case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1693 		return true;
1694 	case HNAE3_DEV_ID_100G_VF:
1695 	case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1696 		return false;
1697 	default:
1698 		dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1699 			 dev_id);
1700 	}
1701 
1702 	return false;
1703 }
1704 
1705 static void hns3_disable_sriov(struct pci_dev *pdev)
1706 {
1707 	/* If our VFs are assigned we cannot shut down SR-IOV
1708 	 * without causing issues, so just leave the hardware
1709 	 * available but disabled
1710 	 */
1711 	if (pci_vfs_assigned(pdev)) {
1712 		dev_warn(&pdev->dev,
1713 			 "disabling driver while VFs are assigned\n");
1714 		return;
1715 	}
1716 
1717 	pci_disable_sriov(pdev);
1718 }
1719 
1720 static void hns3_get_dev_capability(struct pci_dev *pdev,
1721 				    struct hnae3_ae_dev *ae_dev)
1722 {
1723 	if (pdev->revision >= 0x21) {
1724 		hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_FD_B, 1);
1725 		hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_GRO_B, 1);
1726 	}
1727 }
1728 
1729 /* hns3_probe - Device initialization routine
1730  * @pdev: PCI device information struct
1731  * @ent: entry in hns3_pci_tbl
1732  *
1733  * hns3_probe initializes a PF identified by a pci_dev structure.
1734  * The OS initialization, configuring of the PF private structure,
1735  * and a hardware reset occur.
1736  *
1737  * Returns 0 on success, negative on failure
1738  */
1739 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1740 {
1741 	struct hnae3_ae_dev *ae_dev;
1742 	int ret;
1743 
1744 	ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1745 			      GFP_KERNEL);
1746 	if (!ae_dev) {
1747 		ret = -ENOMEM;
1748 		return ret;
1749 	}
1750 
1751 	ae_dev->pdev = pdev;
1752 	ae_dev->flag = ent->driver_data;
1753 	ae_dev->dev_type = HNAE3_DEV_KNIC;
1754 	ae_dev->reset_type = HNAE3_NONE_RESET;
1755 	hns3_get_dev_capability(pdev, ae_dev);
1756 	pci_set_drvdata(pdev, ae_dev);
1757 
1758 	hnae3_register_ae_dev(ae_dev);
1759 
1760 	return 0;
1761 }
1762 
1763 /* hns3_remove - Device removal routine
1764  * @pdev: PCI device information struct
1765  */
1766 static void hns3_remove(struct pci_dev *pdev)
1767 {
1768 	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1769 
1770 	if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1771 		hns3_disable_sriov(pdev);
1772 
1773 	hnae3_unregister_ae_dev(ae_dev);
1774 }
1775 
1776 /**
1777  * hns3_pci_sriov_configure
1778  * @pdev: pointer to a pci_dev structure
1779  * @num_vfs: number of VFs to allocate
1780  *
1781  * Enable or change the number of VFs. Called when the user updates the number
1782  * of VFs in sysfs.
1783  **/
1784 static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
1785 {
1786 	int ret;
1787 
1788 	if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1789 		dev_warn(&pdev->dev, "Can not config SRIOV\n");
1790 		return -EINVAL;
1791 	}
1792 
1793 	if (num_vfs) {
1794 		ret = pci_enable_sriov(pdev, num_vfs);
1795 		if (ret)
1796 			dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
1797 		else
1798 			return num_vfs;
1799 	} else if (!pci_vfs_assigned(pdev)) {
1800 		pci_disable_sriov(pdev);
1801 	} else {
1802 		dev_warn(&pdev->dev,
1803 			 "Unable to free VFs because some are assigned to VMs.\n");
1804 	}
1805 
1806 	return 0;
1807 }
1808 
1809 static void hns3_shutdown(struct pci_dev *pdev)
1810 {
1811 	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1812 
1813 	hnae3_unregister_ae_dev(ae_dev);
1814 	devm_kfree(&pdev->dev, ae_dev);
1815 	pci_set_drvdata(pdev, NULL);
1816 
1817 	if (system_state == SYSTEM_POWER_OFF)
1818 		pci_set_power_state(pdev, PCI_D3hot);
1819 }
1820 
1821 static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
1822 					    pci_channel_state_t state)
1823 {
1824 	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1825 	pci_ers_result_t ret;
1826 
1827 	dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
1828 
1829 	if (state == pci_channel_io_perm_failure)
1830 		return PCI_ERS_RESULT_DISCONNECT;
1831 
1832 	if (!ae_dev) {
1833 		dev_err(&pdev->dev,
1834 			"Can't recover - error happened during device init\n");
1835 		return PCI_ERS_RESULT_NONE;
1836 	}
1837 
1838 	if (ae_dev->ops->handle_hw_ras_error)
1839 		ret = ae_dev->ops->handle_hw_ras_error(ae_dev);
1840 	else
1841 		return PCI_ERS_RESULT_NONE;
1842 
1843 	return ret;
1844 }
1845 
1846 static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
1847 {
1848 	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1849 	struct device *dev = &pdev->dev;
1850 
1851 	dev_info(dev, "requesting reset due to PCI error\n");
1852 
1853 	/* request the reset */
1854 	if (ae_dev->ops->reset_event) {
1855 		ae_dev->ops->reset_event(pdev, NULL);
1856 		return PCI_ERS_RESULT_RECOVERED;
1857 	}
1858 
1859 	return PCI_ERS_RESULT_DISCONNECT;
1860 }
1861 
1862 static void hns3_reset_prepare(struct pci_dev *pdev)
1863 {
1864 	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1865 
1866 	dev_info(&pdev->dev, "hns3 flr prepare\n");
1867 	if (ae_dev && ae_dev->ops && ae_dev->ops->flr_prepare)
1868 		ae_dev->ops->flr_prepare(ae_dev);
1869 }
1870 
1871 static void hns3_reset_done(struct pci_dev *pdev)
1872 {
1873 	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1874 
1875 	dev_info(&pdev->dev, "hns3 flr done\n");
1876 	if (ae_dev && ae_dev->ops && ae_dev->ops->flr_done)
1877 		ae_dev->ops->flr_done(ae_dev);
1878 }
1879 
1880 static const struct pci_error_handlers hns3_err_handler = {
1881 	.error_detected = hns3_error_detected,
1882 	.slot_reset     = hns3_slot_reset,
1883 	.reset_prepare	= hns3_reset_prepare,
1884 	.reset_done	= hns3_reset_done,
1885 };
1886 
1887 static struct pci_driver hns3_driver = {
1888 	.name     = hns3_driver_name,
1889 	.id_table = hns3_pci_tbl,
1890 	.probe    = hns3_probe,
1891 	.remove   = hns3_remove,
1892 	.shutdown = hns3_shutdown,
1893 	.sriov_configure = hns3_pci_sriov_configure,
1894 	.err_handler    = &hns3_err_handler,
1895 };
1896 
1897 /* set default feature to hns3 */
1898 static void hns3_set_default_feature(struct net_device *netdev)
1899 {
1900 	struct hnae3_handle *h = hns3_get_handle(netdev);
1901 	struct pci_dev *pdev = h->pdev;
1902 
1903 	netdev->priv_flags |= IFF_UNICAST_FLT;
1904 
1905 	netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1906 		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1907 		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1908 		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1909 		NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1910 
1911 	netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1912 
1913 	netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1914 
1915 	netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1916 		NETIF_F_HW_VLAN_CTAG_FILTER |
1917 		NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1918 		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1919 		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1920 		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1921 		NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1922 
1923 	netdev->vlan_features |=
1924 		NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1925 		NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1926 		NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1927 		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1928 		NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1929 
1930 	netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1931 		NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1932 		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1933 		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1934 		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1935 		NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1936 
1937 	if (pdev->revision >= 0x21) {
1938 		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER |
1939 			NETIF_F_GRO_HW;
1940 		netdev->features |= NETIF_F_GRO_HW;
1941 
1942 		if (!(h->flags & HNAE3_SUPPORT_VF)) {
1943 			netdev->hw_features |= NETIF_F_NTUPLE;
1944 			netdev->features |= NETIF_F_NTUPLE;
1945 		}
1946 	}
1947 }
1948 
1949 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1950 			     struct hns3_desc_cb *cb)
1951 {
1952 	unsigned int order = hnae3_page_order(ring);
1953 	struct page *p;
1954 
1955 	p = dev_alloc_pages(order);
1956 	if (!p)
1957 		return -ENOMEM;
1958 
1959 	cb->priv = p;
1960 	cb->page_offset = 0;
1961 	cb->reuse_flag = 0;
1962 	cb->buf  = page_address(p);
1963 	cb->length = hnae3_page_size(ring);
1964 	cb->type = DESC_TYPE_PAGE;
1965 
1966 	return 0;
1967 }
1968 
1969 static void hns3_free_buffer(struct hns3_enet_ring *ring,
1970 			     struct hns3_desc_cb *cb)
1971 {
1972 	if (cb->type == DESC_TYPE_SKB)
1973 		dev_kfree_skb_any((struct sk_buff *)cb->priv);
1974 	else if (!HNAE3_IS_TX_RING(ring))
1975 		put_page((struct page *)cb->priv);
1976 	memset(cb, 0, sizeof(*cb));
1977 }
1978 
1979 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1980 {
1981 	cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1982 			       cb->length, ring_to_dma_dir(ring));
1983 
1984 	if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
1985 		return -EIO;
1986 
1987 	return 0;
1988 }
1989 
1990 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1991 			      struct hns3_desc_cb *cb)
1992 {
1993 	if (cb->type == DESC_TYPE_SKB)
1994 		dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1995 				 ring_to_dma_dir(ring));
1996 	else if (cb->length)
1997 		dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1998 			       ring_to_dma_dir(ring));
1999 }
2000 
2001 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
2002 {
2003 	hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2004 	ring->desc[i].addr = 0;
2005 }
2006 
2007 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
2008 {
2009 	struct hns3_desc_cb *cb = &ring->desc_cb[i];
2010 
2011 	if (!ring->desc_cb[i].dma)
2012 		return;
2013 
2014 	hns3_buffer_detach(ring, i);
2015 	hns3_free_buffer(ring, cb);
2016 }
2017 
2018 static void hns3_free_buffers(struct hns3_enet_ring *ring)
2019 {
2020 	int i;
2021 
2022 	for (i = 0; i < ring->desc_num; i++)
2023 		hns3_free_buffer_detach(ring, i);
2024 }
2025 
2026 /* free desc along with its attached buffer */
2027 static void hns3_free_desc(struct hns3_enet_ring *ring)
2028 {
2029 	int size = ring->desc_num * sizeof(ring->desc[0]);
2030 
2031 	hns3_free_buffers(ring);
2032 
2033 	if (ring->desc) {
2034 		dma_free_coherent(ring_to_dev(ring), size,
2035 				  ring->desc, ring->desc_dma_addr);
2036 		ring->desc = NULL;
2037 	}
2038 }
2039 
2040 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
2041 {
2042 	int size = ring->desc_num * sizeof(ring->desc[0]);
2043 
2044 	ring->desc = dma_alloc_coherent(ring_to_dev(ring), size,
2045 					&ring->desc_dma_addr, GFP_KERNEL);
2046 	if (!ring->desc)
2047 		return -ENOMEM;
2048 
2049 	return 0;
2050 }
2051 
2052 static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
2053 				   struct hns3_desc_cb *cb)
2054 {
2055 	int ret;
2056 
2057 	ret = hns3_alloc_buffer(ring, cb);
2058 	if (ret)
2059 		goto out;
2060 
2061 	ret = hns3_map_buffer(ring, cb);
2062 	if (ret)
2063 		goto out_with_buf;
2064 
2065 	return 0;
2066 
2067 out_with_buf:
2068 	hns3_free_buffer(ring, cb);
2069 out:
2070 	return ret;
2071 }
2072 
2073 static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
2074 {
2075 	int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
2076 
2077 	if (ret)
2078 		return ret;
2079 
2080 	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2081 
2082 	return 0;
2083 }
2084 
2085 /* Allocate memory for raw pkg, and map with dma */
2086 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2087 {
2088 	int i, j, ret;
2089 
2090 	for (i = 0; i < ring->desc_num; i++) {
2091 		ret = hns3_alloc_buffer_attach(ring, i);
2092 		if (ret)
2093 			goto out_buffer_fail;
2094 	}
2095 
2096 	return 0;
2097 
2098 out_buffer_fail:
2099 	for (j = i - 1; j >= 0; j--)
2100 		hns3_free_buffer_detach(ring, j);
2101 	return ret;
2102 }
2103 
2104 /* detach a in-used buffer and replace with a reserved one  */
2105 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2106 				struct hns3_desc_cb *res_cb)
2107 {
2108 	hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2109 	ring->desc_cb[i] = *res_cb;
2110 	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2111 	ring->desc[i].rx.bd_base_info = 0;
2112 }
2113 
2114 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2115 {
2116 	ring->desc_cb[i].reuse_flag = 0;
2117 	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
2118 		+ ring->desc_cb[i].page_offset);
2119 	ring->desc[i].rx.bd_base_info = 0;
2120 }
2121 
2122 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
2123 				      int *pkts)
2124 {
2125 	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2126 
2127 	(*pkts) += (desc_cb->type == DESC_TYPE_SKB);
2128 	(*bytes) += desc_cb->length;
2129 	/* desc_cb will be cleaned, after hnae3_free_buffer_detach*/
2130 	hns3_free_buffer_detach(ring, ring->next_to_clean);
2131 
2132 	ring_ptr_move_fw(ring, next_to_clean);
2133 }
2134 
2135 static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
2136 {
2137 	int u = ring->next_to_use;
2138 	int c = ring->next_to_clean;
2139 
2140 	if (unlikely(h > ring->desc_num))
2141 		return 0;
2142 
2143 	return u > c ? (h > c && h <= u) : (h > c || h <= u);
2144 }
2145 
2146 void hns3_clean_tx_ring(struct hns3_enet_ring *ring)
2147 {
2148 	struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2149 	struct hns3_nic_priv *priv = netdev_priv(netdev);
2150 	struct netdev_queue *dev_queue;
2151 	int bytes, pkts;
2152 	int head;
2153 
2154 	head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
2155 	rmb(); /* Make sure head is ready before touch any data */
2156 
2157 	if (is_ring_empty(ring) || head == ring->next_to_clean)
2158 		return; /* no data to poll */
2159 
2160 	if (unlikely(!is_valid_clean_head(ring, head))) {
2161 		netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
2162 			   ring->next_to_use, ring->next_to_clean);
2163 
2164 		u64_stats_update_begin(&ring->syncp);
2165 		ring->stats.io_err_cnt++;
2166 		u64_stats_update_end(&ring->syncp);
2167 		return;
2168 	}
2169 
2170 	bytes = 0;
2171 	pkts = 0;
2172 	while (head != ring->next_to_clean) {
2173 		hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
2174 		/* Issue prefetch for next Tx descriptor */
2175 		prefetch(&ring->desc_cb[ring->next_to_clean]);
2176 	}
2177 
2178 	ring->tqp_vector->tx_group.total_bytes += bytes;
2179 	ring->tqp_vector->tx_group.total_packets += pkts;
2180 
2181 	u64_stats_update_begin(&ring->syncp);
2182 	ring->stats.tx_bytes += bytes;
2183 	ring->stats.tx_pkts += pkts;
2184 	u64_stats_update_end(&ring->syncp);
2185 
2186 	dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2187 	netdev_tx_completed_queue(dev_queue, pkts, bytes);
2188 
2189 	if (unlikely(pkts && netif_carrier_ok(netdev) &&
2190 		     (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
2191 		/* Make sure that anybody stopping the queue after this
2192 		 * sees the new next_to_clean.
2193 		 */
2194 		smp_mb();
2195 		if (netif_tx_queue_stopped(dev_queue) &&
2196 		    !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
2197 			netif_tx_wake_queue(dev_queue);
2198 			ring->stats.restart_queue++;
2199 		}
2200 	}
2201 }
2202 
2203 static int hns3_desc_unused(struct hns3_enet_ring *ring)
2204 {
2205 	int ntc = ring->next_to_clean;
2206 	int ntu = ring->next_to_use;
2207 
2208 	return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2209 }
2210 
2211 static void
2212 hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
2213 {
2214 	struct hns3_desc_cb *desc_cb;
2215 	struct hns3_desc_cb res_cbs;
2216 	int i, ret;
2217 
2218 	for (i = 0; i < cleand_count; i++) {
2219 		desc_cb = &ring->desc_cb[ring->next_to_use];
2220 		if (desc_cb->reuse_flag) {
2221 			u64_stats_update_begin(&ring->syncp);
2222 			ring->stats.reuse_pg_cnt++;
2223 			u64_stats_update_end(&ring->syncp);
2224 
2225 			hns3_reuse_buffer(ring, ring->next_to_use);
2226 		} else {
2227 			ret = hns3_reserve_buffer_map(ring, &res_cbs);
2228 			if (ret) {
2229 				u64_stats_update_begin(&ring->syncp);
2230 				ring->stats.sw_err_cnt++;
2231 				u64_stats_update_end(&ring->syncp);
2232 
2233 				netdev_err(ring->tqp->handle->kinfo.netdev,
2234 					   "hnae reserve buffer map failed.\n");
2235 				break;
2236 			}
2237 			hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2238 		}
2239 
2240 		ring_ptr_move_fw(ring, next_to_use);
2241 	}
2242 
2243 	wmb(); /* Make all data has been write before submit */
2244 	writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2245 }
2246 
2247 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2248 				struct hns3_enet_ring *ring, int pull_len,
2249 				struct hns3_desc_cb *desc_cb)
2250 {
2251 	struct hns3_desc *desc;
2252 	u32 truesize;
2253 	int size;
2254 	int last_offset;
2255 	bool twobufs;
2256 
2257 	twobufs = ((PAGE_SIZE < 8192) &&
2258 		hnae3_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2259 
2260 	desc = &ring->desc[ring->next_to_clean];
2261 	size = le16_to_cpu(desc->rx.size);
2262 
2263 	truesize = hnae3_buf_size(ring);
2264 
2265 	if (!twobufs)
2266 		last_offset = hnae3_page_size(ring) - hnae3_buf_size(ring);
2267 
2268 	skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2269 			size - pull_len, truesize);
2270 
2271 	 /* Avoid re-using remote pages,flag default unreuse */
2272 	if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2273 		return;
2274 
2275 	if (twobufs) {
2276 		/* If we are only owner of page we can reuse it */
2277 		if (likely(page_count(desc_cb->priv) == 1)) {
2278 			/* Flip page offset to other buffer */
2279 			desc_cb->page_offset ^= truesize;
2280 
2281 			desc_cb->reuse_flag = 1;
2282 			/* bump ref count on page before it is given*/
2283 			get_page(desc_cb->priv);
2284 		}
2285 		return;
2286 	}
2287 
2288 	/* Move offset up to the next cache line */
2289 	desc_cb->page_offset += truesize;
2290 
2291 	if (desc_cb->page_offset <= last_offset) {
2292 		desc_cb->reuse_flag = 1;
2293 		/* Bump ref count on page before it is given*/
2294 		get_page(desc_cb->priv);
2295 	}
2296 }
2297 
2298 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2299 			     struct hns3_desc *desc)
2300 {
2301 	struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2302 	int l3_type, l4_type;
2303 	u32 bd_base_info;
2304 	int ol4_type;
2305 	u32 l234info;
2306 
2307 	bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2308 	l234info = le32_to_cpu(desc->rx.l234_info);
2309 
2310 	skb->ip_summed = CHECKSUM_NONE;
2311 
2312 	skb_checksum_none_assert(skb);
2313 
2314 	if (!(netdev->features & NETIF_F_RXCSUM))
2315 		return;
2316 
2317 	/* We MUST enable hardware checksum before enabling hardware GRO */
2318 	if (skb_shinfo(skb)->gso_size) {
2319 		skb->ip_summed = CHECKSUM_UNNECESSARY;
2320 		return;
2321 	}
2322 
2323 	/* check if hardware has done checksum */
2324 	if (!hnae3_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2325 		return;
2326 
2327 	if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L3E_B) ||
2328 		     hnae3_get_bit(l234info, HNS3_RXD_L4E_B) ||
2329 		     hnae3_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2330 		     hnae3_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2331 		u64_stats_update_begin(&ring->syncp);
2332 		ring->stats.l3l4_csum_err++;
2333 		u64_stats_update_end(&ring->syncp);
2334 
2335 		return;
2336 	}
2337 
2338 	l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2339 				  HNS3_RXD_L3ID_S);
2340 	l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2341 				  HNS3_RXD_L4ID_S);
2342 
2343 	ol4_type = hnae3_get_field(l234info, HNS3_RXD_OL4ID_M,
2344 				   HNS3_RXD_OL4ID_S);
2345 	switch (ol4_type) {
2346 	case HNS3_OL4_TYPE_MAC_IN_UDP:
2347 	case HNS3_OL4_TYPE_NVGRE:
2348 		skb->csum_level = 1;
2349 		/* fall through */
2350 	case HNS3_OL4_TYPE_NO_TUN:
2351 		/* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2352 		if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2353 		     l3_type == HNS3_L3_TYPE_IPV6) &&
2354 		    (l4_type == HNS3_L4_TYPE_UDP ||
2355 		     l4_type == HNS3_L4_TYPE_TCP ||
2356 		     l4_type == HNS3_L4_TYPE_SCTP))
2357 			skb->ip_summed = CHECKSUM_UNNECESSARY;
2358 		break;
2359 	default:
2360 		break;
2361 	}
2362 }
2363 
2364 static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2365 {
2366 	if (skb_has_frag_list(skb))
2367 		napi_gro_flush(&ring->tqp_vector->napi, false);
2368 
2369 	napi_gro_receive(&ring->tqp_vector->napi, skb);
2370 }
2371 
2372 static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2373 				struct hns3_desc *desc, u32 l234info,
2374 				u16 *vlan_tag)
2375 {
2376 	struct pci_dev *pdev = ring->tqp->handle->pdev;
2377 
2378 	if (pdev->revision == 0x20) {
2379 		*vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2380 		if (!(*vlan_tag & VLAN_VID_MASK))
2381 			*vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2382 
2383 		return (*vlan_tag != 0);
2384 	}
2385 
2386 #define HNS3_STRP_OUTER_VLAN	0x1
2387 #define HNS3_STRP_INNER_VLAN	0x2
2388 
2389 	switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2390 				HNS3_RXD_STRP_TAGP_S)) {
2391 	case HNS3_STRP_OUTER_VLAN:
2392 		*vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2393 		return true;
2394 	case HNS3_STRP_INNER_VLAN:
2395 		*vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2396 		return true;
2397 	default:
2398 		return false;
2399 	}
2400 }
2401 
2402 static int hns3_alloc_skb(struct hns3_enet_ring *ring, int length,
2403 			  unsigned char *va)
2404 {
2405 #define HNS3_NEED_ADD_FRAG	1
2406 	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2407 	struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2408 	struct sk_buff *skb;
2409 
2410 	ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);
2411 	skb = ring->skb;
2412 	if (unlikely(!skb)) {
2413 		netdev_err(netdev, "alloc rx skb fail\n");
2414 
2415 		u64_stats_update_begin(&ring->syncp);
2416 		ring->stats.sw_err_cnt++;
2417 		u64_stats_update_end(&ring->syncp);
2418 
2419 		return -ENOMEM;
2420 	}
2421 
2422 	prefetchw(skb->data);
2423 
2424 	ring->pending_buf = 1;
2425 	ring->frag_num = 0;
2426 	ring->tail_skb = NULL;
2427 	if (length <= HNS3_RX_HEAD_SIZE) {
2428 		memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2429 
2430 		/* We can reuse buffer as-is, just make sure it is local */
2431 		if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2432 			desc_cb->reuse_flag = 1;
2433 		else /* This page cannot be reused so discard it */
2434 			put_page(desc_cb->priv);
2435 
2436 		ring_ptr_move_fw(ring, next_to_clean);
2437 		return 0;
2438 	}
2439 	u64_stats_update_begin(&ring->syncp);
2440 	ring->stats.seg_pkt_cnt++;
2441 	u64_stats_update_end(&ring->syncp);
2442 
2443 	ring->pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2444 	__skb_put(skb, ring->pull_len);
2445 	hns3_nic_reuse_page(skb, ring->frag_num++, ring, ring->pull_len,
2446 			    desc_cb);
2447 	ring_ptr_move_fw(ring, next_to_clean);
2448 
2449 	return HNS3_NEED_ADD_FRAG;
2450 }
2451 
2452 static int hns3_add_frag(struct hns3_enet_ring *ring, struct hns3_desc *desc,
2453 			 struct sk_buff **out_skb, bool pending)
2454 {
2455 	struct sk_buff *skb = *out_skb;
2456 	struct sk_buff *head_skb = *out_skb;
2457 	struct sk_buff *new_skb;
2458 	struct hns3_desc_cb *desc_cb;
2459 	struct hns3_desc *pre_desc;
2460 	u32 bd_base_info;
2461 	int pre_bd;
2462 
2463 	/* if there is pending bd, the SW param next_to_clean has moved
2464 	 * to next and the next is NULL
2465 	 */
2466 	if (pending) {
2467 		pre_bd = (ring->next_to_clean - 1 + ring->desc_num) %
2468 			ring->desc_num;
2469 		pre_desc = &ring->desc[pre_bd];
2470 		bd_base_info = le32_to_cpu(pre_desc->rx.bd_base_info);
2471 	} else {
2472 		bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2473 	}
2474 
2475 	while (!hnae3_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2476 		desc = &ring->desc[ring->next_to_clean];
2477 		desc_cb = &ring->desc_cb[ring->next_to_clean];
2478 		bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2479 		if (!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))
2480 			return -ENXIO;
2481 
2482 		if (unlikely(ring->frag_num >= MAX_SKB_FRAGS)) {
2483 			new_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2484 						 HNS3_RX_HEAD_SIZE);
2485 			if (unlikely(!new_skb)) {
2486 				netdev_err(ring->tqp->handle->kinfo.netdev,
2487 					   "alloc rx skb frag fail\n");
2488 				return -ENXIO;
2489 			}
2490 			ring->frag_num = 0;
2491 
2492 			if (ring->tail_skb) {
2493 				ring->tail_skb->next = new_skb;
2494 				ring->tail_skb = new_skb;
2495 			} else {
2496 				skb_shinfo(skb)->frag_list = new_skb;
2497 				ring->tail_skb = new_skb;
2498 			}
2499 		}
2500 
2501 		if (ring->tail_skb) {
2502 			head_skb->truesize += hnae3_buf_size(ring);
2503 			head_skb->data_len += le16_to_cpu(desc->rx.size);
2504 			head_skb->len += le16_to_cpu(desc->rx.size);
2505 			skb = ring->tail_skb;
2506 		}
2507 
2508 		hns3_nic_reuse_page(skb, ring->frag_num++, ring, 0, desc_cb);
2509 		ring_ptr_move_fw(ring, next_to_clean);
2510 		ring->pending_buf++;
2511 	}
2512 
2513 	return 0;
2514 }
2515 
2516 static void hns3_set_gro_param(struct sk_buff *skb, u32 l234info,
2517 			       u32 bd_base_info)
2518 {
2519 	u16 gro_count;
2520 	u32 l3_type;
2521 
2522 	gro_count = hnae3_get_field(l234info, HNS3_RXD_GRO_COUNT_M,
2523 				    HNS3_RXD_GRO_COUNT_S);
2524 	/* if there is no HW GRO, do not set gro params */
2525 	if (!gro_count)
2526 		return;
2527 
2528 	/* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
2529 	 * to skb_shinfo(skb)->gso_segs
2530 	 */
2531 	NAPI_GRO_CB(skb)->count = gro_count;
2532 
2533 	l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2534 				  HNS3_RXD_L3ID_S);
2535 	if (l3_type == HNS3_L3_TYPE_IPV4)
2536 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
2537 	else if (l3_type == HNS3_L3_TYPE_IPV6)
2538 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
2539 	else
2540 		return;
2541 
2542 	skb_shinfo(skb)->gso_size = hnae3_get_field(bd_base_info,
2543 						    HNS3_RXD_GRO_SIZE_M,
2544 						    HNS3_RXD_GRO_SIZE_S);
2545 	if (skb_shinfo(skb)->gso_size)
2546 		tcp_gro_complete(skb);
2547 }
2548 
2549 static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
2550 				     struct sk_buff *skb)
2551 {
2552 	struct hnae3_handle *handle = ring->tqp->handle;
2553 	enum pkt_hash_types rss_type;
2554 	struct hns3_desc *desc;
2555 	int last_bd;
2556 
2557 	/* When driver handle the rss type, ring->next_to_clean indicates the
2558 	 * first descriptor of next packet, need -1 here.
2559 	 */
2560 	last_bd = (ring->next_to_clean - 1 + ring->desc_num) % ring->desc_num;
2561 	desc = &ring->desc[last_bd];
2562 
2563 	if (le32_to_cpu(desc->rx.rss_hash))
2564 		rss_type = handle->kinfo.rss_type;
2565 	else
2566 		rss_type = PKT_HASH_TYPE_NONE;
2567 
2568 	skb_set_hash(skb, le32_to_cpu(desc->rx.rss_hash), rss_type);
2569 }
2570 
2571 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2572 			     struct sk_buff **out_skb)
2573 {
2574 	struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2575 	struct sk_buff *skb = ring->skb;
2576 	struct hns3_desc_cb *desc_cb;
2577 	struct hns3_desc *desc;
2578 	u32 bd_base_info;
2579 	u32 l234info;
2580 	int length;
2581 	int ret;
2582 
2583 	desc = &ring->desc[ring->next_to_clean];
2584 	desc_cb = &ring->desc_cb[ring->next_to_clean];
2585 
2586 	prefetch(desc);
2587 
2588 	length = le16_to_cpu(desc->rx.size);
2589 	bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2590 
2591 	/* Check valid BD */
2592 	if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
2593 		return -ENXIO;
2594 
2595 	if (!skb)
2596 		ring->va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2597 
2598 	/* Prefetch first cache line of first page
2599 	 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2600 	 * line size is 64B so need to prefetch twice to make it 128B. But in
2601 	 * actual we can have greater size of caches with 128B Level 1 cache
2602 	 * lines. In such a case, single fetch would suffice to cache in the
2603 	 * relevant part of the header.
2604 	 */
2605 	prefetch(ring->va);
2606 #if L1_CACHE_BYTES < 128
2607 	prefetch(ring->va + L1_CACHE_BYTES);
2608 #endif
2609 
2610 	if (!skb) {
2611 		ret = hns3_alloc_skb(ring, length, ring->va);
2612 		*out_skb = skb = ring->skb;
2613 
2614 		if (ret < 0) /* alloc buffer fail */
2615 			return ret;
2616 		if (ret > 0) { /* need add frag */
2617 			ret = hns3_add_frag(ring, desc, &skb, false);
2618 			if (ret)
2619 				return ret;
2620 
2621 			/* As the head data may be changed when GRO enable, copy
2622 			 * the head data in after other data rx completed
2623 			 */
2624 			memcpy(skb->data, ring->va,
2625 			       ALIGN(ring->pull_len, sizeof(long)));
2626 		}
2627 	} else {
2628 		ret = hns3_add_frag(ring, desc, &skb, true);
2629 		if (ret)
2630 			return ret;
2631 
2632 		/* As the head data may be changed when GRO enable, copy
2633 		 * the head data in after other data rx completed
2634 		 */
2635 		memcpy(skb->data, ring->va,
2636 		       ALIGN(ring->pull_len, sizeof(long)));
2637 	}
2638 
2639 	l234info = le32_to_cpu(desc->rx.l234_info);
2640 	bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2641 
2642 	/* Based on hw strategy, the tag offloaded will be stored at
2643 	 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2644 	 * in one layer tag case.
2645 	 */
2646 	if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2647 		u16 vlan_tag;
2648 
2649 		if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
2650 			__vlan_hwaccel_put_tag(skb,
2651 					       htons(ETH_P_8021Q),
2652 					       vlan_tag);
2653 	}
2654 
2655 	if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2656 		u64_stats_update_begin(&ring->syncp);
2657 		ring->stats.non_vld_descs++;
2658 		u64_stats_update_end(&ring->syncp);
2659 
2660 		dev_kfree_skb_any(skb);
2661 		return -EINVAL;
2662 	}
2663 
2664 	if (unlikely((!desc->rx.pkt_len) ||
2665 		     hnae3_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2666 		u64_stats_update_begin(&ring->syncp);
2667 		ring->stats.err_pkt_len++;
2668 		u64_stats_update_end(&ring->syncp);
2669 
2670 		dev_kfree_skb_any(skb);
2671 		return -EFAULT;
2672 	}
2673 
2674 	if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L2E_B))) {
2675 		u64_stats_update_begin(&ring->syncp);
2676 		ring->stats.l2_err++;
2677 		u64_stats_update_end(&ring->syncp);
2678 
2679 		dev_kfree_skb_any(skb);
2680 		return -EFAULT;
2681 	}
2682 
2683 	u64_stats_update_begin(&ring->syncp);
2684 	ring->stats.rx_pkts++;
2685 	ring->stats.rx_bytes += skb->len;
2686 	u64_stats_update_end(&ring->syncp);
2687 
2688 	ring->tqp_vector->rx_group.total_bytes += skb->len;
2689 
2690 	/* This is needed in order to enable forwarding support */
2691 	hns3_set_gro_param(skb, l234info, bd_base_info);
2692 
2693 	hns3_rx_checksum(ring, skb, desc);
2694 	*out_skb = skb;
2695 	hns3_set_rx_skb_rss_type(ring, skb);
2696 
2697 	return 0;
2698 }
2699 
2700 int hns3_clean_rx_ring(
2701 		struct hns3_enet_ring *ring, int budget,
2702 		void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
2703 {
2704 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2705 	struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2706 	int recv_pkts, recv_bds, clean_count, err;
2707 	int unused_count = hns3_desc_unused(ring) - ring->pending_buf;
2708 	struct sk_buff *skb = ring->skb;
2709 	int num;
2710 
2711 	num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2712 	rmb(); /* Make sure num taken effect before the other data is touched */
2713 
2714 	recv_pkts = 0, recv_bds = 0, clean_count = 0;
2715 	num -= unused_count;
2716 
2717 	while (recv_pkts < budget && recv_bds < num) {
2718 		/* Reuse or realloc buffers */
2719 		if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2720 			hns3_nic_alloc_rx_buffers(ring,
2721 						  clean_count + unused_count);
2722 			clean_count = 0;
2723 			unused_count = hns3_desc_unused(ring) -
2724 					ring->pending_buf;
2725 		}
2726 
2727 		/* Poll one pkt */
2728 		err = hns3_handle_rx_bd(ring, &skb);
2729 		if (unlikely(!skb)) /* This fault cannot be repaired */
2730 			goto out;
2731 
2732 		if (err == -ENXIO) { /* Do not get FE for the packet */
2733 			goto out;
2734 		} else if (unlikely(err)) {  /* Do jump the err */
2735 			recv_bds += ring->pending_buf;
2736 			clean_count += ring->pending_buf;
2737 			ring->skb = NULL;
2738 			ring->pending_buf = 0;
2739 			continue;
2740 		}
2741 
2742 		/* Do update ip stack process */
2743 		skb->protocol = eth_type_trans(skb, netdev);
2744 		rx_fn(ring, skb);
2745 		recv_bds += ring->pending_buf;
2746 		clean_count += ring->pending_buf;
2747 		ring->skb = NULL;
2748 		ring->pending_buf = 0;
2749 
2750 		recv_pkts++;
2751 	}
2752 
2753 out:
2754 	/* Make all data has been write before submit */
2755 	if (clean_count + unused_count > 0)
2756 		hns3_nic_alloc_rx_buffers(ring,
2757 					  clean_count + unused_count);
2758 
2759 	return recv_pkts;
2760 }
2761 
2762 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2763 {
2764 	struct hns3_enet_tqp_vector *tqp_vector =
2765 					ring_group->ring->tqp_vector;
2766 	enum hns3_flow_level_range new_flow_level;
2767 	int packets_per_msecs;
2768 	int bytes_per_msecs;
2769 	u32 time_passed_ms;
2770 	u16 new_int_gl;
2771 
2772 	if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies)
2773 		return false;
2774 
2775 	if (ring_group->total_packets == 0) {
2776 		ring_group->coal.int_gl = HNS3_INT_GL_50K;
2777 		ring_group->coal.flow_level = HNS3_FLOW_LOW;
2778 		return true;
2779 	}
2780 
2781 	/* Simple throttlerate management
2782 	 * 0-10MB/s   lower     (50000 ints/s)
2783 	 * 10-20MB/s   middle    (20000 ints/s)
2784 	 * 20-1249MB/s high      (18000 ints/s)
2785 	 * > 40000pps  ultra     (8000 ints/s)
2786 	 */
2787 	new_flow_level = ring_group->coal.flow_level;
2788 	new_int_gl = ring_group->coal.int_gl;
2789 	time_passed_ms =
2790 		jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2791 
2792 	if (!time_passed_ms)
2793 		return false;
2794 
2795 	do_div(ring_group->total_packets, time_passed_ms);
2796 	packets_per_msecs = ring_group->total_packets;
2797 
2798 	do_div(ring_group->total_bytes, time_passed_ms);
2799 	bytes_per_msecs = ring_group->total_bytes;
2800 
2801 #define HNS3_RX_LOW_BYTE_RATE 10000
2802 #define HNS3_RX_MID_BYTE_RATE 20000
2803 
2804 	switch (new_flow_level) {
2805 	case HNS3_FLOW_LOW:
2806 		if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
2807 			new_flow_level = HNS3_FLOW_MID;
2808 		break;
2809 	case HNS3_FLOW_MID:
2810 		if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
2811 			new_flow_level = HNS3_FLOW_HIGH;
2812 		else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
2813 			new_flow_level = HNS3_FLOW_LOW;
2814 		break;
2815 	case HNS3_FLOW_HIGH:
2816 	case HNS3_FLOW_ULTRA:
2817 	default:
2818 		if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
2819 			new_flow_level = HNS3_FLOW_MID;
2820 		break;
2821 	}
2822 
2823 #define HNS3_RX_ULTRA_PACKET_RATE 40
2824 
2825 	if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2826 	    &tqp_vector->rx_group == ring_group)
2827 		new_flow_level = HNS3_FLOW_ULTRA;
2828 
2829 	switch (new_flow_level) {
2830 	case HNS3_FLOW_LOW:
2831 		new_int_gl = HNS3_INT_GL_50K;
2832 		break;
2833 	case HNS3_FLOW_MID:
2834 		new_int_gl = HNS3_INT_GL_20K;
2835 		break;
2836 	case HNS3_FLOW_HIGH:
2837 		new_int_gl = HNS3_INT_GL_18K;
2838 		break;
2839 	case HNS3_FLOW_ULTRA:
2840 		new_int_gl = HNS3_INT_GL_8K;
2841 		break;
2842 	default:
2843 		break;
2844 	}
2845 
2846 	ring_group->total_bytes = 0;
2847 	ring_group->total_packets = 0;
2848 	ring_group->coal.flow_level = new_flow_level;
2849 	if (new_int_gl != ring_group->coal.int_gl) {
2850 		ring_group->coal.int_gl = new_int_gl;
2851 		return true;
2852 	}
2853 	return false;
2854 }
2855 
2856 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2857 {
2858 	struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2859 	struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2860 	bool rx_update, tx_update;
2861 
2862 	/* update param every 1000ms */
2863 	if (time_before(jiffies,
2864 			tqp_vector->last_jiffies + msecs_to_jiffies(1000)))
2865 		return;
2866 
2867 	if (rx_group->coal.gl_adapt_enable) {
2868 		rx_update = hns3_get_new_int_gl(rx_group);
2869 		if (rx_update)
2870 			hns3_set_vector_coalesce_rx_gl(tqp_vector,
2871 						       rx_group->coal.int_gl);
2872 	}
2873 
2874 	if (tx_group->coal.gl_adapt_enable) {
2875 		tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2876 		if (tx_update)
2877 			hns3_set_vector_coalesce_tx_gl(tqp_vector,
2878 						       tx_group->coal.int_gl);
2879 	}
2880 
2881 	tqp_vector->last_jiffies = jiffies;
2882 }
2883 
2884 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2885 {
2886 	struct hns3_nic_priv *priv = netdev_priv(napi->dev);
2887 	struct hns3_enet_ring *ring;
2888 	int rx_pkt_total = 0;
2889 
2890 	struct hns3_enet_tqp_vector *tqp_vector =
2891 		container_of(napi, struct hns3_enet_tqp_vector, napi);
2892 	bool clean_complete = true;
2893 	int rx_budget;
2894 
2895 	if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
2896 		napi_complete(napi);
2897 		return 0;
2898 	}
2899 
2900 	/* Since the actual Tx work is minimal, we can give the Tx a larger
2901 	 * budget and be more aggressive about cleaning up the Tx descriptors.
2902 	 */
2903 	hns3_for_each_ring(ring, tqp_vector->tx_group)
2904 		hns3_clean_tx_ring(ring);
2905 
2906 	/* make sure rx ring budget not smaller than 1 */
2907 	rx_budget = max(budget / tqp_vector->num_tqps, 1);
2908 
2909 	hns3_for_each_ring(ring, tqp_vector->rx_group) {
2910 		int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2911 						    hns3_rx_skb);
2912 
2913 		if (rx_cleaned >= rx_budget)
2914 			clean_complete = false;
2915 
2916 		rx_pkt_total += rx_cleaned;
2917 	}
2918 
2919 	tqp_vector->rx_group.total_packets += rx_pkt_total;
2920 
2921 	if (!clean_complete)
2922 		return budget;
2923 
2924 	if (napi_complete(napi) &&
2925 	    likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
2926 		hns3_update_new_int_gl(tqp_vector);
2927 		hns3_mask_vector_irq(tqp_vector, 1);
2928 	}
2929 
2930 	return rx_pkt_total;
2931 }
2932 
2933 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2934 				      struct hnae3_ring_chain_node *head)
2935 {
2936 	struct pci_dev *pdev = tqp_vector->handle->pdev;
2937 	struct hnae3_ring_chain_node *cur_chain = head;
2938 	struct hnae3_ring_chain_node *chain;
2939 	struct hns3_enet_ring *tx_ring;
2940 	struct hns3_enet_ring *rx_ring;
2941 
2942 	tx_ring = tqp_vector->tx_group.ring;
2943 	if (tx_ring) {
2944 		cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2945 		hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2946 			      HNAE3_RING_TYPE_TX);
2947 		hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2948 				HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
2949 
2950 		cur_chain->next = NULL;
2951 
2952 		while (tx_ring->next) {
2953 			tx_ring = tx_ring->next;
2954 
2955 			chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2956 					     GFP_KERNEL);
2957 			if (!chain)
2958 				goto err_free_chain;
2959 
2960 			cur_chain->next = chain;
2961 			chain->tqp_index = tx_ring->tqp->tqp_index;
2962 			hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2963 				      HNAE3_RING_TYPE_TX);
2964 			hnae3_set_field(chain->int_gl_idx,
2965 					HNAE3_RING_GL_IDX_M,
2966 					HNAE3_RING_GL_IDX_S,
2967 					HNAE3_RING_GL_TX);
2968 
2969 			cur_chain = chain;
2970 		}
2971 	}
2972 
2973 	rx_ring = tqp_vector->rx_group.ring;
2974 	if (!tx_ring && rx_ring) {
2975 		cur_chain->next = NULL;
2976 		cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2977 		hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2978 			      HNAE3_RING_TYPE_RX);
2979 		hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2980 				HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2981 
2982 		rx_ring = rx_ring->next;
2983 	}
2984 
2985 	while (rx_ring) {
2986 		chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2987 		if (!chain)
2988 			goto err_free_chain;
2989 
2990 		cur_chain->next = chain;
2991 		chain->tqp_index = rx_ring->tqp->tqp_index;
2992 		hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2993 			      HNAE3_RING_TYPE_RX);
2994 		hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2995 				HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2996 
2997 		cur_chain = chain;
2998 
2999 		rx_ring = rx_ring->next;
3000 	}
3001 
3002 	return 0;
3003 
3004 err_free_chain:
3005 	cur_chain = head->next;
3006 	while (cur_chain) {
3007 		chain = cur_chain->next;
3008 		devm_kfree(&pdev->dev, cur_chain);
3009 		cur_chain = chain;
3010 	}
3011 	head->next = NULL;
3012 
3013 	return -ENOMEM;
3014 }
3015 
3016 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
3017 					struct hnae3_ring_chain_node *head)
3018 {
3019 	struct pci_dev *pdev = tqp_vector->handle->pdev;
3020 	struct hnae3_ring_chain_node *chain_tmp, *chain;
3021 
3022 	chain = head->next;
3023 
3024 	while (chain) {
3025 		chain_tmp = chain->next;
3026 		devm_kfree(&pdev->dev, chain);
3027 		chain = chain_tmp;
3028 	}
3029 }
3030 
3031 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
3032 				   struct hns3_enet_ring *ring)
3033 {
3034 	ring->next = group->ring;
3035 	group->ring = ring;
3036 
3037 	group->count++;
3038 }
3039 
3040 static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
3041 {
3042 	struct pci_dev *pdev = priv->ae_handle->pdev;
3043 	struct hns3_enet_tqp_vector *tqp_vector;
3044 	int num_vectors = priv->vector_num;
3045 	int numa_node;
3046 	int vector_i;
3047 
3048 	numa_node = dev_to_node(&pdev->dev);
3049 
3050 	for (vector_i = 0; vector_i < num_vectors; vector_i++) {
3051 		tqp_vector = &priv->tqp_vector[vector_i];
3052 		cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
3053 				&tqp_vector->affinity_mask);
3054 	}
3055 }
3056 
3057 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
3058 {
3059 	struct hnae3_ring_chain_node vector_ring_chain;
3060 	struct hnae3_handle *h = priv->ae_handle;
3061 	struct hns3_enet_tqp_vector *tqp_vector;
3062 	int ret = 0;
3063 	int i;
3064 
3065 	hns3_nic_set_cpumask(priv);
3066 
3067 	for (i = 0; i < priv->vector_num; i++) {
3068 		tqp_vector = &priv->tqp_vector[i];
3069 		hns3_vector_gl_rl_init_hw(tqp_vector, priv);
3070 		tqp_vector->num_tqps = 0;
3071 	}
3072 
3073 	for (i = 0; i < h->kinfo.num_tqps; i++) {
3074 		u16 vector_i = i % priv->vector_num;
3075 		u16 tqp_num = h->kinfo.num_tqps;
3076 
3077 		tqp_vector = &priv->tqp_vector[vector_i];
3078 
3079 		hns3_add_ring_to_group(&tqp_vector->tx_group,
3080 				       priv->ring_data[i].ring);
3081 
3082 		hns3_add_ring_to_group(&tqp_vector->rx_group,
3083 				       priv->ring_data[i + tqp_num].ring);
3084 
3085 		priv->ring_data[i].ring->tqp_vector = tqp_vector;
3086 		priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
3087 		tqp_vector->num_tqps++;
3088 	}
3089 
3090 	for (i = 0; i < priv->vector_num; i++) {
3091 		tqp_vector = &priv->tqp_vector[i];
3092 
3093 		tqp_vector->rx_group.total_bytes = 0;
3094 		tqp_vector->rx_group.total_packets = 0;
3095 		tqp_vector->tx_group.total_bytes = 0;
3096 		tqp_vector->tx_group.total_packets = 0;
3097 		tqp_vector->handle = h;
3098 
3099 		ret = hns3_get_vector_ring_chain(tqp_vector,
3100 						 &vector_ring_chain);
3101 		if (ret)
3102 			goto map_ring_fail;
3103 
3104 		ret = h->ae_algo->ops->map_ring_to_vector(h,
3105 			tqp_vector->vector_irq, &vector_ring_chain);
3106 
3107 		hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3108 
3109 		if (ret)
3110 			goto map_ring_fail;
3111 
3112 		netif_napi_add(priv->netdev, &tqp_vector->napi,
3113 			       hns3_nic_common_poll, NAPI_POLL_WEIGHT);
3114 	}
3115 
3116 	return 0;
3117 
3118 map_ring_fail:
3119 	while (i--)
3120 		netif_napi_del(&priv->tqp_vector[i].napi);
3121 
3122 	return ret;
3123 }
3124 
3125 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
3126 {
3127 #define HNS3_VECTOR_PF_MAX_NUM		64
3128 
3129 	struct hnae3_handle *h = priv->ae_handle;
3130 	struct hns3_enet_tqp_vector *tqp_vector;
3131 	struct hnae3_vector_info *vector;
3132 	struct pci_dev *pdev = h->pdev;
3133 	u16 tqp_num = h->kinfo.num_tqps;
3134 	u16 vector_num;
3135 	int ret = 0;
3136 	u16 i;
3137 
3138 	/* RSS size, cpu online and vector_num should be the same */
3139 	/* Should consider 2p/4p later */
3140 	vector_num = min_t(u16, num_online_cpus(), tqp_num);
3141 	vector_num = min_t(u16, vector_num, HNS3_VECTOR_PF_MAX_NUM);
3142 
3143 	vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
3144 			      GFP_KERNEL);
3145 	if (!vector)
3146 		return -ENOMEM;
3147 
3148 	vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
3149 
3150 	priv->vector_num = vector_num;
3151 	priv->tqp_vector = (struct hns3_enet_tqp_vector *)
3152 		devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
3153 			     GFP_KERNEL);
3154 	if (!priv->tqp_vector) {
3155 		ret = -ENOMEM;
3156 		goto out;
3157 	}
3158 
3159 	for (i = 0; i < priv->vector_num; i++) {
3160 		tqp_vector = &priv->tqp_vector[i];
3161 		tqp_vector->idx = i;
3162 		tqp_vector->mask_addr = vector[i].io_addr;
3163 		tqp_vector->vector_irq = vector[i].vector;
3164 		hns3_vector_gl_rl_init(tqp_vector, priv);
3165 	}
3166 
3167 out:
3168 	devm_kfree(&pdev->dev, vector);
3169 	return ret;
3170 }
3171 
3172 static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
3173 {
3174 	group->ring = NULL;
3175 	group->count = 0;
3176 }
3177 
3178 static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
3179 {
3180 	struct hnae3_ring_chain_node vector_ring_chain;
3181 	struct hnae3_handle *h = priv->ae_handle;
3182 	struct hns3_enet_tqp_vector *tqp_vector;
3183 	int i, ret;
3184 
3185 	for (i = 0; i < priv->vector_num; i++) {
3186 		tqp_vector = &priv->tqp_vector[i];
3187 
3188 		ret = hns3_get_vector_ring_chain(tqp_vector,
3189 						 &vector_ring_chain);
3190 		if (ret)
3191 			return ret;
3192 
3193 		ret = h->ae_algo->ops->unmap_ring_from_vector(h,
3194 			tqp_vector->vector_irq, &vector_ring_chain);
3195 		if (ret)
3196 			return ret;
3197 
3198 		hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3199 
3200 		if (tqp_vector->irq_init_flag == HNS3_VECTOR_INITED) {
3201 			irq_set_affinity_notifier(tqp_vector->vector_irq,
3202 						  NULL);
3203 			irq_set_affinity_hint(tqp_vector->vector_irq, NULL);
3204 			free_irq(tqp_vector->vector_irq, tqp_vector);
3205 			tqp_vector->irq_init_flag = HNS3_VECTOR_NOT_INITED;
3206 		}
3207 
3208 		priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
3209 		hns3_clear_ring_group(&tqp_vector->rx_group);
3210 		hns3_clear_ring_group(&tqp_vector->tx_group);
3211 		netif_napi_del(&priv->tqp_vector[i].napi);
3212 	}
3213 
3214 	return 0;
3215 }
3216 
3217 static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
3218 {
3219 	struct hnae3_handle *h = priv->ae_handle;
3220 	struct pci_dev *pdev = h->pdev;
3221 	int i, ret;
3222 
3223 	for (i = 0; i < priv->vector_num; i++) {
3224 		struct hns3_enet_tqp_vector *tqp_vector;
3225 
3226 		tqp_vector = &priv->tqp_vector[i];
3227 		ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
3228 		if (ret)
3229 			return ret;
3230 	}
3231 
3232 	devm_kfree(&pdev->dev, priv->tqp_vector);
3233 	return 0;
3234 }
3235 
3236 static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
3237 			     int ring_type)
3238 {
3239 	struct hns3_nic_ring_data *ring_data = priv->ring_data;
3240 	int queue_num = priv->ae_handle->kinfo.num_tqps;
3241 	struct pci_dev *pdev = priv->ae_handle->pdev;
3242 	struct hns3_enet_ring *ring;
3243 
3244 	ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
3245 	if (!ring)
3246 		return -ENOMEM;
3247 
3248 	if (ring_type == HNAE3_RING_TYPE_TX) {
3249 		ring_data[q->tqp_index].ring = ring;
3250 		ring_data[q->tqp_index].queue_index = q->tqp_index;
3251 		ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
3252 	} else {
3253 		ring_data[q->tqp_index + queue_num].ring = ring;
3254 		ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
3255 		ring->io_base = q->io_base;
3256 	}
3257 
3258 	hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
3259 
3260 	ring->tqp = q;
3261 	ring->desc = NULL;
3262 	ring->desc_cb = NULL;
3263 	ring->dev = priv->dev;
3264 	ring->desc_dma_addr = 0;
3265 	ring->buf_size = q->buf_size;
3266 	ring->desc_num = q->desc_num;
3267 	ring->next_to_use = 0;
3268 	ring->next_to_clean = 0;
3269 
3270 	return 0;
3271 }
3272 
3273 static int hns3_queue_to_ring(struct hnae3_queue *tqp,
3274 			      struct hns3_nic_priv *priv)
3275 {
3276 	int ret;
3277 
3278 	ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
3279 	if (ret)
3280 		return ret;
3281 
3282 	ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
3283 	if (ret) {
3284 		devm_kfree(priv->dev, priv->ring_data[tqp->tqp_index].ring);
3285 		return ret;
3286 	}
3287 
3288 	return 0;
3289 }
3290 
3291 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
3292 {
3293 	struct hnae3_handle *h = priv->ae_handle;
3294 	struct pci_dev *pdev = h->pdev;
3295 	int i, ret;
3296 
3297 	priv->ring_data =  devm_kzalloc(&pdev->dev,
3298 					array3_size(h->kinfo.num_tqps,
3299 						    sizeof(*priv->ring_data),
3300 						    2),
3301 					GFP_KERNEL);
3302 	if (!priv->ring_data)
3303 		return -ENOMEM;
3304 
3305 	for (i = 0; i < h->kinfo.num_tqps; i++) {
3306 		ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
3307 		if (ret)
3308 			goto err;
3309 	}
3310 
3311 	return 0;
3312 err:
3313 	while (i--) {
3314 		devm_kfree(priv->dev, priv->ring_data[i].ring);
3315 		devm_kfree(priv->dev,
3316 			   priv->ring_data[i + h->kinfo.num_tqps].ring);
3317 	}
3318 
3319 	devm_kfree(&pdev->dev, priv->ring_data);
3320 	return ret;
3321 }
3322 
3323 static void hns3_put_ring_config(struct hns3_nic_priv *priv)
3324 {
3325 	struct hnae3_handle *h = priv->ae_handle;
3326 	int i;
3327 
3328 	for (i = 0; i < h->kinfo.num_tqps; i++) {
3329 		devm_kfree(priv->dev, priv->ring_data[i].ring);
3330 		devm_kfree(priv->dev,
3331 			   priv->ring_data[i + h->kinfo.num_tqps].ring);
3332 	}
3333 	devm_kfree(priv->dev, priv->ring_data);
3334 }
3335 
3336 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
3337 {
3338 	int ret;
3339 
3340 	if (ring->desc_num <= 0 || ring->buf_size <= 0)
3341 		return -EINVAL;
3342 
3343 	ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
3344 				GFP_KERNEL);
3345 	if (!ring->desc_cb) {
3346 		ret = -ENOMEM;
3347 		goto out;
3348 	}
3349 
3350 	ret = hns3_alloc_desc(ring);
3351 	if (ret)
3352 		goto out_with_desc_cb;
3353 
3354 	if (!HNAE3_IS_TX_RING(ring)) {
3355 		ret = hns3_alloc_ring_buffers(ring);
3356 		if (ret)
3357 			goto out_with_desc;
3358 	}
3359 
3360 	return 0;
3361 
3362 out_with_desc:
3363 	hns3_free_desc(ring);
3364 out_with_desc_cb:
3365 	kfree(ring->desc_cb);
3366 	ring->desc_cb = NULL;
3367 out:
3368 	return ret;
3369 }
3370 
3371 static void hns3_fini_ring(struct hns3_enet_ring *ring)
3372 {
3373 	hns3_free_desc(ring);
3374 	kfree(ring->desc_cb);
3375 	ring->desc_cb = NULL;
3376 	ring->next_to_clean = 0;
3377 	ring->next_to_use = 0;
3378 }
3379 
3380 static int hns3_buf_size2type(u32 buf_size)
3381 {
3382 	int bd_size_type;
3383 
3384 	switch (buf_size) {
3385 	case 512:
3386 		bd_size_type = HNS3_BD_SIZE_512_TYPE;
3387 		break;
3388 	case 1024:
3389 		bd_size_type = HNS3_BD_SIZE_1024_TYPE;
3390 		break;
3391 	case 2048:
3392 		bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3393 		break;
3394 	case 4096:
3395 		bd_size_type = HNS3_BD_SIZE_4096_TYPE;
3396 		break;
3397 	default:
3398 		bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3399 	}
3400 
3401 	return bd_size_type;
3402 }
3403 
3404 static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
3405 {
3406 	dma_addr_t dma = ring->desc_dma_addr;
3407 	struct hnae3_queue *q = ring->tqp;
3408 
3409 	if (!HNAE3_IS_TX_RING(ring)) {
3410 		hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
3411 			       (u32)dma);
3412 		hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
3413 			       (u32)((dma >> 31) >> 1));
3414 
3415 		hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
3416 			       hns3_buf_size2type(ring->buf_size));
3417 		hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
3418 			       ring->desc_num / 8 - 1);
3419 
3420 	} else {
3421 		hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
3422 			       (u32)dma);
3423 		hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
3424 			       (u32)((dma >> 31) >> 1));
3425 
3426 		hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
3427 			       ring->desc_num / 8 - 1);
3428 	}
3429 }
3430 
3431 static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
3432 {
3433 	struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
3434 	int i;
3435 
3436 	for (i = 0; i < HNAE3_MAX_TC; i++) {
3437 		struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3438 		int j;
3439 
3440 		if (!tc_info->enable)
3441 			continue;
3442 
3443 		for (j = 0; j < tc_info->tqp_count; j++) {
3444 			struct hnae3_queue *q;
3445 
3446 			q = priv->ring_data[tc_info->tqp_offset + j].ring->tqp;
3447 			hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG,
3448 				       tc_info->tc);
3449 		}
3450 	}
3451 }
3452 
3453 int hns3_init_all_ring(struct hns3_nic_priv *priv)
3454 {
3455 	struct hnae3_handle *h = priv->ae_handle;
3456 	int ring_num = h->kinfo.num_tqps * 2;
3457 	int i, j;
3458 	int ret;
3459 
3460 	for (i = 0; i < ring_num; i++) {
3461 		ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
3462 		if (ret) {
3463 			dev_err(priv->dev,
3464 				"Alloc ring memory fail! ret=%d\n", ret);
3465 			goto out_when_alloc_ring_memory;
3466 		}
3467 
3468 		u64_stats_init(&priv->ring_data[i].ring->syncp);
3469 	}
3470 
3471 	return 0;
3472 
3473 out_when_alloc_ring_memory:
3474 	for (j = i - 1; j >= 0; j--)
3475 		hns3_fini_ring(priv->ring_data[j].ring);
3476 
3477 	return -ENOMEM;
3478 }
3479 
3480 int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
3481 {
3482 	struct hnae3_handle *h = priv->ae_handle;
3483 	int i;
3484 
3485 	for (i = 0; i < h->kinfo.num_tqps; i++) {
3486 		hns3_fini_ring(priv->ring_data[i].ring);
3487 		hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3488 	}
3489 	return 0;
3490 }
3491 
3492 /* Set mac addr if it is configured. or leave it to the AE driver */
3493 static int hns3_init_mac_addr(struct net_device *netdev, bool init)
3494 {
3495 	struct hns3_nic_priv *priv = netdev_priv(netdev);
3496 	struct hnae3_handle *h = priv->ae_handle;
3497 	u8 mac_addr_temp[ETH_ALEN];
3498 	int ret = 0;
3499 
3500 	if (h->ae_algo->ops->get_mac_addr && init) {
3501 		h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3502 		ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3503 	}
3504 
3505 	/* Check if the MAC address is valid, if not get a random one */
3506 	if (!is_valid_ether_addr(netdev->dev_addr)) {
3507 		eth_hw_addr_random(netdev);
3508 		dev_warn(priv->dev, "using random MAC address %pM\n",
3509 			 netdev->dev_addr);
3510 	}
3511 
3512 	if (h->ae_algo->ops->set_mac_addr)
3513 		ret = h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
3514 
3515 	return ret;
3516 }
3517 
3518 static int hns3_restore_fd_rules(struct net_device *netdev)
3519 {
3520 	struct hnae3_handle *h = hns3_get_handle(netdev);
3521 	int ret = 0;
3522 
3523 	if (h->ae_algo->ops->restore_fd_rules)
3524 		ret = h->ae_algo->ops->restore_fd_rules(h);
3525 
3526 	return ret;
3527 }
3528 
3529 static void hns3_del_all_fd_rules(struct net_device *netdev, bool clear_list)
3530 {
3531 	struct hnae3_handle *h = hns3_get_handle(netdev);
3532 
3533 	if (h->ae_algo->ops->del_all_fd_entries)
3534 		h->ae_algo->ops->del_all_fd_entries(h, clear_list);
3535 }
3536 
3537 static void hns3_nic_set_priv_ops(struct net_device *netdev)
3538 {
3539 	struct hns3_nic_priv *priv = netdev_priv(netdev);
3540 
3541 	priv->ops.fill_desc = hns3_fill_desc;
3542 	if ((netdev->features & NETIF_F_TSO) ||
3543 	    (netdev->features & NETIF_F_TSO6))
3544 		priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
3545 	else
3546 		priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
3547 }
3548 
3549 static int hns3_client_start(struct hnae3_handle *handle)
3550 {
3551 	if (!handle->ae_algo->ops->client_start)
3552 		return 0;
3553 
3554 	return handle->ae_algo->ops->client_start(handle);
3555 }
3556 
3557 static void hns3_client_stop(struct hnae3_handle *handle)
3558 {
3559 	if (!handle->ae_algo->ops->client_stop)
3560 		return;
3561 
3562 	handle->ae_algo->ops->client_stop(handle);
3563 }
3564 
3565 static int hns3_client_init(struct hnae3_handle *handle)
3566 {
3567 	struct pci_dev *pdev = handle->pdev;
3568 	u16 alloc_tqps, max_rss_size;
3569 	struct hns3_nic_priv *priv;
3570 	struct net_device *netdev;
3571 	int ret;
3572 
3573 	handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,
3574 						    &max_rss_size);
3575 	netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);
3576 	if (!netdev)
3577 		return -ENOMEM;
3578 
3579 	priv = netdev_priv(netdev);
3580 	priv->dev = &pdev->dev;
3581 	priv->netdev = netdev;
3582 	priv->ae_handle = handle;
3583 	priv->tx_timeout_count = 0;
3584 
3585 	handle->kinfo.netdev = netdev;
3586 	handle->priv = (void *)priv;
3587 
3588 	hns3_init_mac_addr(netdev, true);
3589 
3590 	hns3_set_default_feature(netdev);
3591 
3592 	netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3593 	netdev->priv_flags |= IFF_UNICAST_FLT;
3594 	netdev->netdev_ops = &hns3_nic_netdev_ops;
3595 	SET_NETDEV_DEV(netdev, &pdev->dev);
3596 	hns3_ethtool_set_ops(netdev);
3597 	hns3_nic_set_priv_ops(netdev);
3598 
3599 	/* Carrier off reporting is important to ethtool even BEFORE open */
3600 	netif_carrier_off(netdev);
3601 
3602 	ret = hns3_get_ring_config(priv);
3603 	if (ret) {
3604 		ret = -ENOMEM;
3605 		goto out_get_ring_cfg;
3606 	}
3607 
3608 	ret = hns3_nic_alloc_vector_data(priv);
3609 	if (ret) {
3610 		ret = -ENOMEM;
3611 		goto out_alloc_vector_data;
3612 	}
3613 
3614 	ret = hns3_nic_init_vector_data(priv);
3615 	if (ret) {
3616 		ret = -ENOMEM;
3617 		goto out_init_vector_data;
3618 	}
3619 
3620 	ret = hns3_init_all_ring(priv);
3621 	if (ret) {
3622 		ret = -ENOMEM;
3623 		goto out_init_ring_data;
3624 	}
3625 
3626 	ret = register_netdev(netdev);
3627 	if (ret) {
3628 		dev_err(priv->dev, "probe register netdev fail!\n");
3629 		goto out_reg_netdev_fail;
3630 	}
3631 
3632 	ret = hns3_client_start(handle);
3633 	if (ret) {
3634 		dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);
3635 			goto out_reg_netdev_fail;
3636 	}
3637 
3638 	hns3_dcbnl_setup(handle);
3639 
3640 	hns3_dbg_init(handle);
3641 
3642 	/* MTU range: (ETH_MIN_MTU(kernel default) - 9702) */
3643 	netdev->max_mtu = HNS3_MAX_MTU;
3644 
3645 	set_bit(HNS3_NIC_STATE_INITED, &priv->state);
3646 
3647 	return ret;
3648 
3649 out_reg_netdev_fail:
3650 out_init_ring_data:
3651 	(void)hns3_nic_uninit_vector_data(priv);
3652 out_init_vector_data:
3653 	hns3_nic_dealloc_vector_data(priv);
3654 out_alloc_vector_data:
3655 	priv->ring_data = NULL;
3656 out_get_ring_cfg:
3657 	priv->ae_handle = NULL;
3658 	free_netdev(netdev);
3659 	return ret;
3660 }
3661 
3662 static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3663 {
3664 	struct net_device *netdev = handle->kinfo.netdev;
3665 	struct hns3_nic_priv *priv = netdev_priv(netdev);
3666 	int ret;
3667 
3668 	hns3_client_stop(handle);
3669 
3670 	hns3_remove_hw_addr(netdev);
3671 
3672 	if (netdev->reg_state != NETREG_UNINITIALIZED)
3673 		unregister_netdev(netdev);
3674 
3675 	if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
3676 		netdev_warn(netdev, "already uninitialized\n");
3677 		goto out_netdev_free;
3678 	}
3679 
3680 	hns3_del_all_fd_rules(netdev, true);
3681 
3682 	hns3_force_clear_all_rx_ring(handle);
3683 
3684 	ret = hns3_nic_uninit_vector_data(priv);
3685 	if (ret)
3686 		netdev_err(netdev, "uninit vector error\n");
3687 
3688 	ret = hns3_nic_dealloc_vector_data(priv);
3689 	if (ret)
3690 		netdev_err(netdev, "dealloc vector error\n");
3691 
3692 	ret = hns3_uninit_all_ring(priv);
3693 	if (ret)
3694 		netdev_err(netdev, "uninit ring error\n");
3695 
3696 	hns3_put_ring_config(priv);
3697 
3698 	hns3_dbg_uninit(handle);
3699 
3700 	priv->ring_data = NULL;
3701 
3702 out_netdev_free:
3703 	free_netdev(netdev);
3704 }
3705 
3706 static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3707 {
3708 	struct net_device *netdev = handle->kinfo.netdev;
3709 
3710 	if (!netdev)
3711 		return;
3712 
3713 	if (linkup) {
3714 		netif_carrier_on(netdev);
3715 		netif_tx_wake_all_queues(netdev);
3716 		netdev_info(netdev, "link up\n");
3717 	} else {
3718 		netif_carrier_off(netdev);
3719 		netif_tx_stop_all_queues(netdev);
3720 		netdev_info(netdev, "link down\n");
3721 	}
3722 }
3723 
3724 static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3725 {
3726 	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3727 	struct net_device *ndev = kinfo->netdev;
3728 	bool if_running;
3729 	int ret;
3730 
3731 	if (tc > HNAE3_MAX_TC)
3732 		return -EINVAL;
3733 
3734 	if (!ndev)
3735 		return -ENODEV;
3736 
3737 	if_running = netif_running(ndev);
3738 
3739 	if (if_running) {
3740 		(void)hns3_nic_net_stop(ndev);
3741 		msleep(100);
3742 	}
3743 
3744 	ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3745 		kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3746 	if (ret)
3747 		goto err_out;
3748 
3749 	ret = hns3_nic_set_real_num_queue(ndev);
3750 
3751 err_out:
3752 	if (if_running)
3753 		(void)hns3_nic_net_open(ndev);
3754 
3755 	return ret;
3756 }
3757 
3758 static int hns3_recover_hw_addr(struct net_device *ndev)
3759 {
3760 	struct netdev_hw_addr_list *list;
3761 	struct netdev_hw_addr *ha, *tmp;
3762 	int ret = 0;
3763 
3764 	/* go through and sync uc_addr entries to the device */
3765 	list = &ndev->uc;
3766 	list_for_each_entry_safe(ha, tmp, &list->list, list) {
3767 		ret = hns3_nic_uc_sync(ndev, ha->addr);
3768 		if (ret)
3769 			return ret;
3770 	}
3771 
3772 	/* go through and sync mc_addr entries to the device */
3773 	list = &ndev->mc;
3774 	list_for_each_entry_safe(ha, tmp, &list->list, list) {
3775 		ret = hns3_nic_mc_sync(ndev, ha->addr);
3776 		if (ret)
3777 			return ret;
3778 	}
3779 
3780 	return ret;
3781 }
3782 
3783 static void hns3_remove_hw_addr(struct net_device *netdev)
3784 {
3785 	struct netdev_hw_addr_list *list;
3786 	struct netdev_hw_addr *ha, *tmp;
3787 
3788 	hns3_nic_uc_unsync(netdev, netdev->dev_addr);
3789 
3790 	/* go through and unsync uc_addr entries to the device */
3791 	list = &netdev->uc;
3792 	list_for_each_entry_safe(ha, tmp, &list->list, list)
3793 		hns3_nic_uc_unsync(netdev, ha->addr);
3794 
3795 	/* go through and unsync mc_addr entries to the device */
3796 	list = &netdev->mc;
3797 	list_for_each_entry_safe(ha, tmp, &list->list, list)
3798 		if (ha->refcount > 1)
3799 			hns3_nic_mc_unsync(netdev, ha->addr);
3800 }
3801 
3802 static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
3803 {
3804 	while (ring->next_to_clean != ring->next_to_use) {
3805 		ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
3806 		hns3_free_buffer_detach(ring, ring->next_to_clean);
3807 		ring_ptr_move_fw(ring, next_to_clean);
3808 	}
3809 }
3810 
3811 static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
3812 {
3813 	struct hns3_desc_cb res_cbs;
3814 	int ret;
3815 
3816 	while (ring->next_to_use != ring->next_to_clean) {
3817 		/* When a buffer is not reused, it's memory has been
3818 		 * freed in hns3_handle_rx_bd or will be freed by
3819 		 * stack, so we need to replace the buffer here.
3820 		 */
3821 		if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3822 			ret = hns3_reserve_buffer_map(ring, &res_cbs);
3823 			if (ret) {
3824 				u64_stats_update_begin(&ring->syncp);
3825 				ring->stats.sw_err_cnt++;
3826 				u64_stats_update_end(&ring->syncp);
3827 				/* if alloc new buffer fail, exit directly
3828 				 * and reclear in up flow.
3829 				 */
3830 				netdev_warn(ring->tqp->handle->kinfo.netdev,
3831 					    "reserve buffer map failed, ret = %d\n",
3832 					    ret);
3833 				return ret;
3834 			}
3835 			hns3_replace_buffer(ring, ring->next_to_use,
3836 					    &res_cbs);
3837 		}
3838 		ring_ptr_move_fw(ring, next_to_use);
3839 	}
3840 
3841 	return 0;
3842 }
3843 
3844 static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
3845 {
3846 	while (ring->next_to_use != ring->next_to_clean) {
3847 		/* When a buffer is not reused, it's memory has been
3848 		 * freed in hns3_handle_rx_bd or will be freed by
3849 		 * stack, so only need to unmap the buffer here.
3850 		 */
3851 		if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3852 			hns3_unmap_buffer(ring,
3853 					  &ring->desc_cb[ring->next_to_use]);
3854 			ring->desc_cb[ring->next_to_use].dma = 0;
3855 		}
3856 
3857 		ring_ptr_move_fw(ring, next_to_use);
3858 	}
3859 }
3860 
3861 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h)
3862 {
3863 	struct net_device *ndev = h->kinfo.netdev;
3864 	struct hns3_nic_priv *priv = netdev_priv(ndev);
3865 	struct hns3_enet_ring *ring;
3866 	u32 i;
3867 
3868 	for (i = 0; i < h->kinfo.num_tqps; i++) {
3869 		ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3870 		hns3_force_clear_rx_ring(ring);
3871 	}
3872 }
3873 
3874 static void hns3_clear_all_ring(struct hnae3_handle *h)
3875 {
3876 	struct net_device *ndev = h->kinfo.netdev;
3877 	struct hns3_nic_priv *priv = netdev_priv(ndev);
3878 	u32 i;
3879 
3880 	for (i = 0; i < h->kinfo.num_tqps; i++) {
3881 		struct netdev_queue *dev_queue;
3882 		struct hns3_enet_ring *ring;
3883 
3884 		ring = priv->ring_data[i].ring;
3885 		hns3_clear_tx_ring(ring);
3886 		dev_queue = netdev_get_tx_queue(ndev,
3887 						priv->ring_data[i].queue_index);
3888 		netdev_tx_reset_queue(dev_queue);
3889 
3890 		ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3891 		/* Continue to clear other rings even if clearing some
3892 		 * rings failed.
3893 		 */
3894 		hns3_clear_rx_ring(ring);
3895 	}
3896 }
3897 
3898 int hns3_nic_reset_all_ring(struct hnae3_handle *h)
3899 {
3900 	struct net_device *ndev = h->kinfo.netdev;
3901 	struct hns3_nic_priv *priv = netdev_priv(ndev);
3902 	struct hns3_enet_ring *rx_ring;
3903 	int i, j;
3904 	int ret;
3905 
3906 	for (i = 0; i < h->kinfo.num_tqps; i++) {
3907 		ret = h->ae_algo->ops->reset_queue(h, i);
3908 		if (ret)
3909 			return ret;
3910 
3911 		hns3_init_ring_hw(priv->ring_data[i].ring);
3912 
3913 		/* We need to clear tx ring here because self test will
3914 		 * use the ring and will not run down before up
3915 		 */
3916 		hns3_clear_tx_ring(priv->ring_data[i].ring);
3917 		priv->ring_data[i].ring->next_to_clean = 0;
3918 		priv->ring_data[i].ring->next_to_use = 0;
3919 
3920 		rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3921 		hns3_init_ring_hw(rx_ring);
3922 		ret = hns3_clear_rx_ring(rx_ring);
3923 		if (ret)
3924 			return ret;
3925 
3926 		/* We can not know the hardware head and tail when this
3927 		 * function is called in reset flow, so we reuse all desc.
3928 		 */
3929 		for (j = 0; j < rx_ring->desc_num; j++)
3930 			hns3_reuse_buffer(rx_ring, j);
3931 
3932 		rx_ring->next_to_clean = 0;
3933 		rx_ring->next_to_use = 0;
3934 	}
3935 
3936 	hns3_init_tx_ring_tc(priv);
3937 
3938 	return 0;
3939 }
3940 
3941 static void hns3_store_coal(struct hns3_nic_priv *priv)
3942 {
3943 	/* ethtool only support setting and querying one coal
3944 	 * configuation for now, so save the vector 0' coal
3945 	 * configuation here in order to restore it.
3946 	 */
3947 	memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal,
3948 	       sizeof(struct hns3_enet_coalesce));
3949 	memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal,
3950 	       sizeof(struct hns3_enet_coalesce));
3951 }
3952 
3953 static void hns3_restore_coal(struct hns3_nic_priv *priv)
3954 {
3955 	u16 vector_num = priv->vector_num;
3956 	int i;
3957 
3958 	for (i = 0; i < vector_num; i++) {
3959 		memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal,
3960 		       sizeof(struct hns3_enet_coalesce));
3961 		memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal,
3962 		       sizeof(struct hns3_enet_coalesce));
3963 	}
3964 }
3965 
3966 static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3967 {
3968 	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev);
3969 	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3970 	struct net_device *ndev = kinfo->netdev;
3971 	struct hns3_nic_priv *priv = netdev_priv(ndev);
3972 
3973 	if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state))
3974 		return 0;
3975 
3976 	/* it is cumbersome for hardware to pick-and-choose entries for deletion
3977 	 * from table space. Hence, for function reset software intervention is
3978 	 * required to delete the entries
3979 	 */
3980 	if (hns3_dev_ongoing_func_reset(ae_dev)) {
3981 		hns3_remove_hw_addr(ndev);
3982 		hns3_del_all_fd_rules(ndev, false);
3983 	}
3984 
3985 	if (!netif_running(ndev))
3986 		return 0;
3987 
3988 	return hns3_nic_net_stop(ndev);
3989 }
3990 
3991 static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3992 {
3993 	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3994 	struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
3995 	int ret = 0;
3996 
3997 	clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
3998 
3999 	if (netif_running(kinfo->netdev)) {
4000 		ret = hns3_nic_net_open(kinfo->netdev);
4001 		if (ret) {
4002 			set_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
4003 			netdev_err(kinfo->netdev,
4004 				   "hns net up fail, ret=%d!\n", ret);
4005 			return ret;
4006 		}
4007 	}
4008 
4009 	return ret;
4010 }
4011 
4012 static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
4013 {
4014 	struct net_device *netdev = handle->kinfo.netdev;
4015 	struct hns3_nic_priv *priv = netdev_priv(netdev);
4016 	bool vlan_filter_enable;
4017 	int ret;
4018 
4019 	ret = hns3_init_mac_addr(netdev, false);
4020 	if (ret)
4021 		return ret;
4022 
4023 	ret = hns3_recover_hw_addr(netdev);
4024 	if (ret)
4025 		return ret;
4026 
4027 	ret = hns3_update_promisc_mode(netdev, handle->netdev_flags);
4028 	if (ret)
4029 		return ret;
4030 
4031 	vlan_filter_enable = netdev->flags & IFF_PROMISC ? false : true;
4032 	hns3_enable_vlan_filter(netdev, vlan_filter_enable);
4033 
4034 	/* Hardware table is only clear when pf resets */
4035 	if (!(handle->flags & HNAE3_SUPPORT_VF)) {
4036 		ret = hns3_restore_vlan(netdev);
4037 		if (ret)
4038 			return ret;
4039 	}
4040 
4041 	ret = hns3_restore_fd_rules(netdev);
4042 	if (ret)
4043 		return ret;
4044 
4045 	/* Carrier off reporting is important to ethtool even BEFORE open */
4046 	netif_carrier_off(netdev);
4047 
4048 	ret = hns3_nic_alloc_vector_data(priv);
4049 	if (ret)
4050 		return ret;
4051 
4052 	hns3_restore_coal(priv);
4053 
4054 	ret = hns3_nic_init_vector_data(priv);
4055 	if (ret)
4056 		goto err_dealloc_vector;
4057 
4058 	ret = hns3_init_all_ring(priv);
4059 	if (ret)
4060 		goto err_uninit_vector;
4061 
4062 	set_bit(HNS3_NIC_STATE_INITED, &priv->state);
4063 
4064 	return ret;
4065 
4066 err_uninit_vector:
4067 	hns3_nic_uninit_vector_data(priv);
4068 	priv->ring_data = NULL;
4069 err_dealloc_vector:
4070 	hns3_nic_dealloc_vector_data(priv);
4071 
4072 	return ret;
4073 }
4074 
4075 static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
4076 {
4077 	struct net_device *netdev = handle->kinfo.netdev;
4078 	struct hns3_nic_priv *priv = netdev_priv(netdev);
4079 	int ret;
4080 
4081 	if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
4082 		netdev_warn(netdev, "already uninitialized\n");
4083 		return 0;
4084 	}
4085 
4086 	hns3_force_clear_all_rx_ring(handle);
4087 
4088 	ret = hns3_nic_uninit_vector_data(priv);
4089 	if (ret) {
4090 		netdev_err(netdev, "uninit vector error\n");
4091 		return ret;
4092 	}
4093 
4094 	hns3_store_coal(priv);
4095 
4096 	ret = hns3_nic_dealloc_vector_data(priv);
4097 	if (ret)
4098 		netdev_err(netdev, "dealloc vector error\n");
4099 
4100 	ret = hns3_uninit_all_ring(priv);
4101 	if (ret)
4102 		netdev_err(netdev, "uninit ring error\n");
4103 
4104 	clear_bit(HNS3_NIC_STATE_INITED, &priv->state);
4105 
4106 	return ret;
4107 }
4108 
4109 static int hns3_reset_notify(struct hnae3_handle *handle,
4110 			     enum hnae3_reset_notify_type type)
4111 {
4112 	int ret = 0;
4113 
4114 	switch (type) {
4115 	case HNAE3_UP_CLIENT:
4116 		ret = hns3_reset_notify_up_enet(handle);
4117 		break;
4118 	case HNAE3_DOWN_CLIENT:
4119 		ret = hns3_reset_notify_down_enet(handle);
4120 		break;
4121 	case HNAE3_INIT_CLIENT:
4122 		ret = hns3_reset_notify_init_enet(handle);
4123 		break;
4124 	case HNAE3_UNINIT_CLIENT:
4125 		ret = hns3_reset_notify_uninit_enet(handle);
4126 		break;
4127 	default:
4128 		break;
4129 	}
4130 
4131 	return ret;
4132 }
4133 
4134 static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
4135 {
4136 	struct hns3_nic_priv *priv = netdev_priv(netdev);
4137 	struct hnae3_handle *h = hns3_get_handle(netdev);
4138 	int ret;
4139 
4140 	ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
4141 	if (ret)
4142 		return ret;
4143 
4144 	ret = hns3_get_ring_config(priv);
4145 	if (ret)
4146 		return ret;
4147 
4148 	ret = hns3_nic_alloc_vector_data(priv);
4149 	if (ret)
4150 		goto err_alloc_vector;
4151 
4152 	hns3_restore_coal(priv);
4153 
4154 	ret = hns3_nic_init_vector_data(priv);
4155 	if (ret)
4156 		goto err_uninit_vector;
4157 
4158 	ret = hns3_init_all_ring(priv);
4159 	if (ret)
4160 		goto err_put_ring;
4161 
4162 	return 0;
4163 
4164 err_put_ring:
4165 	hns3_put_ring_config(priv);
4166 err_uninit_vector:
4167 	hns3_nic_uninit_vector_data(priv);
4168 err_alloc_vector:
4169 	hns3_nic_dealloc_vector_data(priv);
4170 	return ret;
4171 }
4172 
4173 static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
4174 {
4175 	return (new_tqp_num / num_tc) * num_tc;
4176 }
4177 
4178 int hns3_set_channels(struct net_device *netdev,
4179 		      struct ethtool_channels *ch)
4180 {
4181 	struct hns3_nic_priv *priv = netdev_priv(netdev);
4182 	struct hnae3_handle *h = hns3_get_handle(netdev);
4183 	struct hnae3_knic_private_info *kinfo = &h->kinfo;
4184 	bool if_running = netif_running(netdev);
4185 	u32 new_tqp_num = ch->combined_count;
4186 	u16 org_tqp_num;
4187 	int ret;
4188 
4189 	if (ch->rx_count || ch->tx_count)
4190 		return -EINVAL;
4191 
4192 	if (new_tqp_num > hns3_get_max_available_channels(h) ||
4193 	    new_tqp_num < kinfo->num_tc) {
4194 		dev_err(&netdev->dev,
4195 			"Change tqps fail, the tqp range is from %d to %d",
4196 			kinfo->num_tc,
4197 			hns3_get_max_available_channels(h));
4198 		return -EINVAL;
4199 	}
4200 
4201 	new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
4202 	if (kinfo->num_tqps == new_tqp_num)
4203 		return 0;
4204 
4205 	if (if_running)
4206 		hns3_nic_net_stop(netdev);
4207 
4208 	ret = hns3_nic_uninit_vector_data(priv);
4209 	if (ret) {
4210 		dev_err(&netdev->dev,
4211 			"Unbind vector with tqp fail, nothing is changed");
4212 		goto open_netdev;
4213 	}
4214 
4215 	hns3_store_coal(priv);
4216 
4217 	hns3_nic_dealloc_vector_data(priv);
4218 
4219 	hns3_uninit_all_ring(priv);
4220 	hns3_put_ring_config(priv);
4221 
4222 	org_tqp_num = h->kinfo.num_tqps;
4223 	ret = hns3_modify_tqp_num(netdev, new_tqp_num);
4224 	if (ret) {
4225 		ret = hns3_modify_tqp_num(netdev, org_tqp_num);
4226 		if (ret) {
4227 			/* If revert to old tqp failed, fatal error occurred */
4228 			dev_err(&netdev->dev,
4229 				"Revert to old tqp num fail, ret=%d", ret);
4230 			return ret;
4231 		}
4232 		dev_info(&netdev->dev,
4233 			 "Change tqp num fail, Revert to old tqp num");
4234 	}
4235 
4236 open_netdev:
4237 	if (if_running)
4238 		hns3_nic_net_open(netdev);
4239 
4240 	return ret;
4241 }
4242 
4243 static const struct hnae3_client_ops client_ops = {
4244 	.init_instance = hns3_client_init,
4245 	.uninit_instance = hns3_client_uninit,
4246 	.link_status_change = hns3_link_status_change,
4247 	.setup_tc = hns3_client_setup_tc,
4248 	.reset_notify = hns3_reset_notify,
4249 };
4250 
4251 /* hns3_init_module - Driver registration routine
4252  * hns3_init_module is the first routine called when the driver is
4253  * loaded. All it does is register with the PCI subsystem.
4254  */
4255 static int __init hns3_init_module(void)
4256 {
4257 	int ret;
4258 
4259 	pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
4260 	pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
4261 
4262 	client.type = HNAE3_CLIENT_KNIC;
4263 	snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
4264 		 hns3_driver_name);
4265 
4266 	client.ops = &client_ops;
4267 
4268 	INIT_LIST_HEAD(&client.node);
4269 
4270 	hns3_dbg_register_debugfs(hns3_driver_name);
4271 
4272 	ret = hnae3_register_client(&client);
4273 	if (ret)
4274 		goto err_reg_client;
4275 
4276 	ret = pci_register_driver(&hns3_driver);
4277 	if (ret)
4278 		goto err_reg_driver;
4279 
4280 	return ret;
4281 
4282 err_reg_driver:
4283 	hnae3_unregister_client(&client);
4284 err_reg_client:
4285 	hns3_dbg_unregister_debugfs();
4286 	return ret;
4287 }
4288 module_init(hns3_init_module);
4289 
4290 /* hns3_exit_module - Driver exit cleanup routine
4291  * hns3_exit_module is called just before the driver is removed
4292  * from memory.
4293  */
4294 static void __exit hns3_exit_module(void)
4295 {
4296 	pci_unregister_driver(&hns3_driver);
4297 	hnae3_unregister_client(&client);
4298 	hns3_dbg_unregister_debugfs();
4299 }
4300 module_exit(hns3_exit_module);
4301 
4302 MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
4303 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
4304 MODULE_LICENSE("GPL");
4305 MODULE_ALIAS("pci:hns-nic");
4306 MODULE_VERSION(HNS3_MOD_VERSION);
4307