1 /* 2 * Copyright (c) 2016~2017 Hisilicon Limited. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 */ 9 10 #include <linux/dma-mapping.h> 11 #include <linux/etherdevice.h> 12 #include <linux/interrupt.h> 13 #include <linux/if_vlan.h> 14 #include <linux/ip.h> 15 #include <linux/ipv6.h> 16 #include <linux/module.h> 17 #include <linux/pci.h> 18 #include <linux/skbuff.h> 19 #include <linux/sctp.h> 20 #include <linux/vermagic.h> 21 #include <net/gre.h> 22 #include <net/pkt_cls.h> 23 #include <net/vxlan.h> 24 25 #include "hnae3.h" 26 #include "hns3_enet.h" 27 28 static const char hns3_driver_name[] = "hns3"; 29 const char hns3_driver_version[] = VERMAGIC_STRING; 30 static const char hns3_driver_string[] = 31 "Hisilicon Ethernet Network Driver for Hip08 Family"; 32 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation."; 33 static struct hnae3_client client; 34 35 /* hns3_pci_tbl - PCI Device ID Table 36 * 37 * Last entry must be all 0s 38 * 39 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 40 * Class, Class Mask, private data (not used) } 41 */ 42 static const struct pci_device_id hns3_pci_tbl[] = { 43 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0}, 44 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0}, 45 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 46 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 47 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 48 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 49 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 50 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 51 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 52 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 53 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 54 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 55 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0}, 56 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF), 0}, 57 /* required last entry */ 58 {0, } 59 }; 60 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl); 61 62 static irqreturn_t hns3_irq_handle(int irq, void *dev) 63 { 64 struct hns3_enet_tqp_vector *tqp_vector = dev; 65 66 napi_schedule(&tqp_vector->napi); 67 68 return IRQ_HANDLED; 69 } 70 71 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv) 72 { 73 struct hns3_enet_tqp_vector *tqp_vectors; 74 unsigned int i; 75 76 for (i = 0; i < priv->vector_num; i++) { 77 tqp_vectors = &priv->tqp_vector[i]; 78 79 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED) 80 continue; 81 82 /* release the irq resource */ 83 free_irq(tqp_vectors->vector_irq, tqp_vectors); 84 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED; 85 } 86 } 87 88 static int hns3_nic_init_irq(struct hns3_nic_priv *priv) 89 { 90 struct hns3_enet_tqp_vector *tqp_vectors; 91 int txrx_int_idx = 0; 92 int rx_int_idx = 0; 93 int tx_int_idx = 0; 94 unsigned int i; 95 int ret; 96 97 for (i = 0; i < priv->vector_num; i++) { 98 tqp_vectors = &priv->tqp_vector[i]; 99 100 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED) 101 continue; 102 103 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) { 104 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1, 105 "%s-%s-%d", priv->netdev->name, "TxRx", 106 txrx_int_idx++); 107 txrx_int_idx++; 108 } else if (tqp_vectors->rx_group.ring) { 109 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1, 110 "%s-%s-%d", priv->netdev->name, "Rx", 111 rx_int_idx++); 112 } else if (tqp_vectors->tx_group.ring) { 113 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1, 114 "%s-%s-%d", priv->netdev->name, "Tx", 115 tx_int_idx++); 116 } else { 117 /* Skip this unused q_vector */ 118 continue; 119 } 120 121 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0'; 122 123 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0, 124 tqp_vectors->name, 125 tqp_vectors); 126 if (ret) { 127 netdev_err(priv->netdev, "request irq(%d) fail\n", 128 tqp_vectors->vector_irq); 129 return ret; 130 } 131 132 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED; 133 } 134 135 return 0; 136 } 137 138 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector, 139 u32 mask_en) 140 { 141 writel(mask_en, tqp_vector->mask_addr); 142 } 143 144 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector) 145 { 146 napi_enable(&tqp_vector->napi); 147 148 /* enable vector */ 149 hns3_mask_vector_irq(tqp_vector, 1); 150 } 151 152 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector) 153 { 154 /* disable vector */ 155 hns3_mask_vector_irq(tqp_vector, 0); 156 157 disable_irq(tqp_vector->vector_irq); 158 napi_disable(&tqp_vector->napi); 159 } 160 161 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector, 162 u32 rl_value) 163 { 164 u32 rl_reg = hns3_rl_usec_to_reg(rl_value); 165 166 /* this defines the configuration for RL (Interrupt Rate Limiter). 167 * Rl defines rate of interrupts i.e. number of interrupts-per-second 168 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing 169 */ 170 171 if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable && 172 !tqp_vector->rx_group.coal.gl_adapt_enable) 173 /* According to the hardware, the range of rl_reg is 174 * 0-59 and the unit is 4. 175 */ 176 rl_reg |= HNS3_INT_RL_ENABLE_MASK; 177 178 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET); 179 } 180 181 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector, 182 u32 gl_value) 183 { 184 u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value); 185 186 writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET); 187 } 188 189 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector, 190 u32 gl_value) 191 { 192 u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value); 193 194 writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET); 195 } 196 197 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector, 198 struct hns3_nic_priv *priv) 199 { 200 struct hnae3_handle *h = priv->ae_handle; 201 202 /* initialize the configuration for interrupt coalescing. 203 * 1. GL (Interrupt Gap Limiter) 204 * 2. RL (Interrupt Rate Limiter) 205 */ 206 207 /* Default: enable interrupt coalescing self-adaptive and GL */ 208 tqp_vector->tx_group.coal.gl_adapt_enable = 1; 209 tqp_vector->rx_group.coal.gl_adapt_enable = 1; 210 211 tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K; 212 tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K; 213 214 /* Default: disable RL */ 215 h->kinfo.int_rl_setting = 0; 216 217 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START; 218 tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW; 219 tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW; 220 } 221 222 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector, 223 struct hns3_nic_priv *priv) 224 { 225 struct hnae3_handle *h = priv->ae_handle; 226 227 hns3_set_vector_coalesce_tx_gl(tqp_vector, 228 tqp_vector->tx_group.coal.int_gl); 229 hns3_set_vector_coalesce_rx_gl(tqp_vector, 230 tqp_vector->rx_group.coal.int_gl); 231 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting); 232 } 233 234 static int hns3_nic_set_real_num_queue(struct net_device *netdev) 235 { 236 struct hnae3_handle *h = hns3_get_handle(netdev); 237 struct hnae3_knic_private_info *kinfo = &h->kinfo; 238 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc; 239 int ret; 240 241 ret = netif_set_real_num_tx_queues(netdev, queue_size); 242 if (ret) { 243 netdev_err(netdev, 244 "netif_set_real_num_tx_queues fail, ret=%d!\n", 245 ret); 246 return ret; 247 } 248 249 ret = netif_set_real_num_rx_queues(netdev, queue_size); 250 if (ret) { 251 netdev_err(netdev, 252 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret); 253 return ret; 254 } 255 256 return 0; 257 } 258 259 static u16 hns3_get_max_available_channels(struct hnae3_handle *h) 260 { 261 u16 free_tqps, max_rss_size, max_tqps; 262 263 h->ae_algo->ops->get_tqps_and_rss_info(h, &free_tqps, &max_rss_size); 264 max_tqps = h->kinfo.num_tc * max_rss_size; 265 266 return min_t(u16, max_tqps, (free_tqps + h->kinfo.num_tqps)); 267 } 268 269 static int hns3_nic_net_up(struct net_device *netdev) 270 { 271 struct hns3_nic_priv *priv = netdev_priv(netdev); 272 struct hnae3_handle *h = priv->ae_handle; 273 int i, j; 274 int ret; 275 276 /* get irq resource for all vectors */ 277 ret = hns3_nic_init_irq(priv); 278 if (ret) { 279 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret); 280 return ret; 281 } 282 283 /* enable the vectors */ 284 for (i = 0; i < priv->vector_num; i++) 285 hns3_vector_enable(&priv->tqp_vector[i]); 286 287 /* start the ae_dev */ 288 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0; 289 if (ret) 290 goto out_start_err; 291 292 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state); 293 294 return 0; 295 296 out_start_err: 297 for (j = i - 1; j >= 0; j--) 298 hns3_vector_disable(&priv->tqp_vector[j]); 299 300 hns3_nic_uninit_irq(priv); 301 302 return ret; 303 } 304 305 static int hns3_nic_net_open(struct net_device *netdev) 306 { 307 struct hns3_nic_priv *priv = netdev_priv(netdev); 308 int ret; 309 310 netif_carrier_off(netdev); 311 312 ret = hns3_nic_set_real_num_queue(netdev); 313 if (ret) 314 return ret; 315 316 ret = hns3_nic_net_up(netdev); 317 if (ret) { 318 netdev_err(netdev, 319 "hns net up fail, ret=%d!\n", ret); 320 return ret; 321 } 322 323 priv->ae_handle->last_reset_time = jiffies; 324 return 0; 325 } 326 327 static void hns3_nic_net_down(struct net_device *netdev) 328 { 329 struct hns3_nic_priv *priv = netdev_priv(netdev); 330 const struct hnae3_ae_ops *ops; 331 int i; 332 333 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state)) 334 return; 335 336 /* stop ae_dev */ 337 ops = priv->ae_handle->ae_algo->ops; 338 if (ops->stop) 339 ops->stop(priv->ae_handle); 340 341 /* disable vectors */ 342 for (i = 0; i < priv->vector_num; i++) 343 hns3_vector_disable(&priv->tqp_vector[i]); 344 345 /* free irq resources */ 346 hns3_nic_uninit_irq(priv); 347 } 348 349 static int hns3_nic_net_stop(struct net_device *netdev) 350 { 351 netif_tx_stop_all_queues(netdev); 352 netif_carrier_off(netdev); 353 354 hns3_nic_net_down(netdev); 355 356 return 0; 357 } 358 359 static int hns3_nic_uc_sync(struct net_device *netdev, 360 const unsigned char *addr) 361 { 362 struct hnae3_handle *h = hns3_get_handle(netdev); 363 364 if (h->ae_algo->ops->add_uc_addr) 365 return h->ae_algo->ops->add_uc_addr(h, addr); 366 367 return 0; 368 } 369 370 static int hns3_nic_uc_unsync(struct net_device *netdev, 371 const unsigned char *addr) 372 { 373 struct hnae3_handle *h = hns3_get_handle(netdev); 374 375 if (h->ae_algo->ops->rm_uc_addr) 376 return h->ae_algo->ops->rm_uc_addr(h, addr); 377 378 return 0; 379 } 380 381 static int hns3_nic_mc_sync(struct net_device *netdev, 382 const unsigned char *addr) 383 { 384 struct hnae3_handle *h = hns3_get_handle(netdev); 385 386 if (h->ae_algo->ops->add_mc_addr) 387 return h->ae_algo->ops->add_mc_addr(h, addr); 388 389 return 0; 390 } 391 392 static int hns3_nic_mc_unsync(struct net_device *netdev, 393 const unsigned char *addr) 394 { 395 struct hnae3_handle *h = hns3_get_handle(netdev); 396 397 if (h->ae_algo->ops->rm_mc_addr) 398 return h->ae_algo->ops->rm_mc_addr(h, addr); 399 400 return 0; 401 } 402 403 static void hns3_nic_set_rx_mode(struct net_device *netdev) 404 { 405 struct hnae3_handle *h = hns3_get_handle(netdev); 406 407 if (h->ae_algo->ops->set_promisc_mode) { 408 if (netdev->flags & IFF_PROMISC) 409 h->ae_algo->ops->set_promisc_mode(h, 1); 410 else 411 h->ae_algo->ops->set_promisc_mode(h, 0); 412 } 413 if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync)) 414 netdev_err(netdev, "sync uc address fail\n"); 415 if (netdev->flags & IFF_MULTICAST) 416 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync)) 417 netdev_err(netdev, "sync mc address fail\n"); 418 } 419 420 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen, 421 u16 *mss, u32 *type_cs_vlan_tso) 422 { 423 u32 l4_offset, hdr_len; 424 union l3_hdr_info l3; 425 union l4_hdr_info l4; 426 u32 l4_paylen; 427 int ret; 428 429 if (!skb_is_gso(skb)) 430 return 0; 431 432 ret = skb_cow_head(skb, 0); 433 if (ret) 434 return ret; 435 436 l3.hdr = skb_network_header(skb); 437 l4.hdr = skb_transport_header(skb); 438 439 /* Software should clear the IPv4's checksum field when tso is 440 * needed. 441 */ 442 if (l3.v4->version == 4) 443 l3.v4->check = 0; 444 445 /* tunnel packet.*/ 446 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 447 SKB_GSO_GRE_CSUM | 448 SKB_GSO_UDP_TUNNEL | 449 SKB_GSO_UDP_TUNNEL_CSUM)) { 450 if ((!(skb_shinfo(skb)->gso_type & 451 SKB_GSO_PARTIAL)) && 452 (skb_shinfo(skb)->gso_type & 453 SKB_GSO_UDP_TUNNEL_CSUM)) { 454 /* Software should clear the udp's checksum 455 * field when tso is needed. 456 */ 457 l4.udp->check = 0; 458 } 459 /* reset l3&l4 pointers from outer to inner headers */ 460 l3.hdr = skb_inner_network_header(skb); 461 l4.hdr = skb_inner_transport_header(skb); 462 463 /* Software should clear the IPv4's checksum field when 464 * tso is needed. 465 */ 466 if (l3.v4->version == 4) 467 l3.v4->check = 0; 468 } 469 470 /* normal or tunnel packet*/ 471 l4_offset = l4.hdr - skb->data; 472 hdr_len = (l4.tcp->doff * 4) + l4_offset; 473 474 /* remove payload length from inner pseudo checksum when tso*/ 475 l4_paylen = skb->len - l4_offset; 476 csum_replace_by_diff(&l4.tcp->check, 477 (__force __wsum)htonl(l4_paylen)); 478 479 /* find the txbd field values */ 480 *paylen = skb->len - hdr_len; 481 hnae_set_bit(*type_cs_vlan_tso, 482 HNS3_TXD_TSO_B, 1); 483 484 /* get MSS for TSO */ 485 *mss = skb_shinfo(skb)->gso_size; 486 487 return 0; 488 } 489 490 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto, 491 u8 *il4_proto) 492 { 493 union { 494 struct iphdr *v4; 495 struct ipv6hdr *v6; 496 unsigned char *hdr; 497 } l3; 498 unsigned char *l4_hdr; 499 unsigned char *exthdr; 500 u8 l4_proto_tmp; 501 __be16 frag_off; 502 503 /* find outer header point */ 504 l3.hdr = skb_network_header(skb); 505 l4_hdr = skb_inner_transport_header(skb); 506 507 if (skb->protocol == htons(ETH_P_IPV6)) { 508 exthdr = l3.hdr + sizeof(*l3.v6); 509 l4_proto_tmp = l3.v6->nexthdr; 510 if (l4_hdr != exthdr) 511 ipv6_skip_exthdr(skb, exthdr - skb->data, 512 &l4_proto_tmp, &frag_off); 513 } else if (skb->protocol == htons(ETH_P_IP)) { 514 l4_proto_tmp = l3.v4->protocol; 515 } else { 516 return -EINVAL; 517 } 518 519 *ol4_proto = l4_proto_tmp; 520 521 /* tunnel packet */ 522 if (!skb->encapsulation) { 523 *il4_proto = 0; 524 return 0; 525 } 526 527 /* find inner header point */ 528 l3.hdr = skb_inner_network_header(skb); 529 l4_hdr = skb_inner_transport_header(skb); 530 531 if (l3.v6->version == 6) { 532 exthdr = l3.hdr + sizeof(*l3.v6); 533 l4_proto_tmp = l3.v6->nexthdr; 534 if (l4_hdr != exthdr) 535 ipv6_skip_exthdr(skb, exthdr - skb->data, 536 &l4_proto_tmp, &frag_off); 537 } else if (l3.v4->version == 4) { 538 l4_proto_tmp = l3.v4->protocol; 539 } 540 541 *il4_proto = l4_proto_tmp; 542 543 return 0; 544 } 545 546 static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto, 547 u8 il4_proto, u32 *type_cs_vlan_tso, 548 u32 *ol_type_vlan_len_msec) 549 { 550 union { 551 struct iphdr *v4; 552 struct ipv6hdr *v6; 553 unsigned char *hdr; 554 } l3; 555 union { 556 struct tcphdr *tcp; 557 struct udphdr *udp; 558 struct gre_base_hdr *gre; 559 unsigned char *hdr; 560 } l4; 561 unsigned char *l2_hdr; 562 u8 l4_proto = ol4_proto; 563 u32 ol2_len; 564 u32 ol3_len; 565 u32 ol4_len; 566 u32 l2_len; 567 u32 l3_len; 568 569 l3.hdr = skb_network_header(skb); 570 l4.hdr = skb_transport_header(skb); 571 572 /* compute L2 header size for normal packet, defined in 2 Bytes */ 573 l2_len = l3.hdr - skb->data; 574 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M, 575 HNS3_TXD_L2LEN_S, l2_len >> 1); 576 577 /* tunnel packet*/ 578 if (skb->encapsulation) { 579 /* compute OL2 header size, defined in 2 Bytes */ 580 ol2_len = l2_len; 581 hnae_set_field(*ol_type_vlan_len_msec, 582 HNS3_TXD_L2LEN_M, 583 HNS3_TXD_L2LEN_S, ol2_len >> 1); 584 585 /* compute OL3 header size, defined in 4 Bytes */ 586 ol3_len = l4.hdr - l3.hdr; 587 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M, 588 HNS3_TXD_L3LEN_S, ol3_len >> 2); 589 590 /* MAC in UDP, MAC in GRE (0x6558)*/ 591 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) { 592 /* switch MAC header ptr from outer to inner header.*/ 593 l2_hdr = skb_inner_mac_header(skb); 594 595 /* compute OL4 header size, defined in 4 Bytes. */ 596 ol4_len = l2_hdr - l4.hdr; 597 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M, 598 HNS3_TXD_L4LEN_S, ol4_len >> 2); 599 600 /* switch IP header ptr from outer to inner header */ 601 l3.hdr = skb_inner_network_header(skb); 602 603 /* compute inner l2 header size, defined in 2 Bytes. */ 604 l2_len = l3.hdr - l2_hdr; 605 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M, 606 HNS3_TXD_L2LEN_S, l2_len >> 1); 607 } else { 608 /* skb packet types not supported by hardware, 609 * txbd len fild doesn't be filled. 610 */ 611 return; 612 } 613 614 /* switch L4 header pointer from outer to inner */ 615 l4.hdr = skb_inner_transport_header(skb); 616 617 l4_proto = il4_proto; 618 } 619 620 /* compute inner(/normal) L3 header size, defined in 4 Bytes */ 621 l3_len = l4.hdr - l3.hdr; 622 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M, 623 HNS3_TXD_L3LEN_S, l3_len >> 2); 624 625 /* compute inner(/normal) L4 header size, defined in 4 Bytes */ 626 switch (l4_proto) { 627 case IPPROTO_TCP: 628 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M, 629 HNS3_TXD_L4LEN_S, l4.tcp->doff); 630 break; 631 case IPPROTO_SCTP: 632 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M, 633 HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2)); 634 break; 635 case IPPROTO_UDP: 636 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M, 637 HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2)); 638 break; 639 default: 640 /* skb packet types not supported by hardware, 641 * txbd len fild doesn't be filled. 642 */ 643 return; 644 } 645 } 646 647 static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto, 648 u8 il4_proto, u32 *type_cs_vlan_tso, 649 u32 *ol_type_vlan_len_msec) 650 { 651 union { 652 struct iphdr *v4; 653 struct ipv6hdr *v6; 654 unsigned char *hdr; 655 } l3; 656 u32 l4_proto = ol4_proto; 657 658 l3.hdr = skb_network_header(skb); 659 660 /* define OL3 type and tunnel type(OL4).*/ 661 if (skb->encapsulation) { 662 /* define outer network header type.*/ 663 if (skb->protocol == htons(ETH_P_IP)) { 664 if (skb_is_gso(skb)) 665 hnae_set_field(*ol_type_vlan_len_msec, 666 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S, 667 HNS3_OL3T_IPV4_CSUM); 668 else 669 hnae_set_field(*ol_type_vlan_len_msec, 670 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S, 671 HNS3_OL3T_IPV4_NO_CSUM); 672 673 } else if (skb->protocol == htons(ETH_P_IPV6)) { 674 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M, 675 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6); 676 } 677 678 /* define tunnel type(OL4).*/ 679 switch (l4_proto) { 680 case IPPROTO_UDP: 681 hnae_set_field(*ol_type_vlan_len_msec, 682 HNS3_TXD_TUNTYPE_M, 683 HNS3_TXD_TUNTYPE_S, 684 HNS3_TUN_MAC_IN_UDP); 685 break; 686 case IPPROTO_GRE: 687 hnae_set_field(*ol_type_vlan_len_msec, 688 HNS3_TXD_TUNTYPE_M, 689 HNS3_TXD_TUNTYPE_S, 690 HNS3_TUN_NVGRE); 691 break; 692 default: 693 /* drop the skb tunnel packet if hardware don't support, 694 * because hardware can't calculate csum when TSO. 695 */ 696 if (skb_is_gso(skb)) 697 return -EDOM; 698 699 /* the stack computes the IP header already, 700 * driver calculate l4 checksum when not TSO. 701 */ 702 skb_checksum_help(skb); 703 return 0; 704 } 705 706 l3.hdr = skb_inner_network_header(skb); 707 l4_proto = il4_proto; 708 } 709 710 if (l3.v4->version == 4) { 711 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M, 712 HNS3_TXD_L3T_S, HNS3_L3T_IPV4); 713 714 /* the stack computes the IP header already, the only time we 715 * need the hardware to recompute it is in the case of TSO. 716 */ 717 if (skb_is_gso(skb)) 718 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1); 719 720 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1); 721 } else if (l3.v6->version == 6) { 722 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M, 723 HNS3_TXD_L3T_S, HNS3_L3T_IPV6); 724 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1); 725 } 726 727 switch (l4_proto) { 728 case IPPROTO_TCP: 729 hnae_set_field(*type_cs_vlan_tso, 730 HNS3_TXD_L4T_M, 731 HNS3_TXD_L4T_S, 732 HNS3_L4T_TCP); 733 break; 734 case IPPROTO_UDP: 735 hnae_set_field(*type_cs_vlan_tso, 736 HNS3_TXD_L4T_M, 737 HNS3_TXD_L4T_S, 738 HNS3_L4T_UDP); 739 break; 740 case IPPROTO_SCTP: 741 hnae_set_field(*type_cs_vlan_tso, 742 HNS3_TXD_L4T_M, 743 HNS3_TXD_L4T_S, 744 HNS3_L4T_SCTP); 745 break; 746 default: 747 /* drop the skb tunnel packet if hardware don't support, 748 * because hardware can't calculate csum when TSO. 749 */ 750 if (skb_is_gso(skb)) 751 return -EDOM; 752 753 /* the stack computes the IP header already, 754 * driver calculate l4 checksum when not TSO. 755 */ 756 skb_checksum_help(skb); 757 return 0; 758 } 759 760 return 0; 761 } 762 763 static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end) 764 { 765 /* Config bd buffer end */ 766 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M, 767 HNS3_TXD_BDTYPE_S, 0); 768 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end); 769 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1); 770 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0); 771 } 772 773 static int hns3_fill_desc_vtags(struct sk_buff *skb, 774 struct hns3_enet_ring *tx_ring, 775 u32 *inner_vlan_flag, 776 u32 *out_vlan_flag, 777 u16 *inner_vtag, 778 u16 *out_vtag) 779 { 780 #define HNS3_TX_VLAN_PRIO_SHIFT 13 781 782 if (skb->protocol == htons(ETH_P_8021Q) && 783 !(tx_ring->tqp->handle->kinfo.netdev->features & 784 NETIF_F_HW_VLAN_CTAG_TX)) { 785 /* When HW VLAN acceleration is turned off, and the stack 786 * sets the protocol to 802.1q, the driver just need to 787 * set the protocol to the encapsulated ethertype. 788 */ 789 skb->protocol = vlan_get_protocol(skb); 790 return 0; 791 } 792 793 if (skb_vlan_tag_present(skb)) { 794 u16 vlan_tag; 795 796 vlan_tag = skb_vlan_tag_get(skb); 797 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT; 798 799 /* Based on hw strategy, use out_vtag in two layer tag case, 800 * and use inner_vtag in one tag case. 801 */ 802 if (skb->protocol == htons(ETH_P_8021Q)) { 803 hnae_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1); 804 *out_vtag = vlan_tag; 805 } else { 806 hnae_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1); 807 *inner_vtag = vlan_tag; 808 } 809 } else if (skb->protocol == htons(ETH_P_8021Q)) { 810 struct vlan_ethhdr *vhdr; 811 int rc; 812 813 rc = skb_cow_head(skb, 0); 814 if (rc < 0) 815 return rc; 816 vhdr = (struct vlan_ethhdr *)skb->data; 817 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7) 818 << HNS3_TX_VLAN_PRIO_SHIFT); 819 } 820 821 skb->protocol = vlan_get_protocol(skb); 822 return 0; 823 } 824 825 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv, 826 int size, dma_addr_t dma, int frag_end, 827 enum hns_desc_type type) 828 { 829 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; 830 struct hns3_desc *desc = &ring->desc[ring->next_to_use]; 831 u32 ol_type_vlan_len_msec = 0; 832 u16 bdtp_fe_sc_vld_ra_ri = 0; 833 u32 type_cs_vlan_tso = 0; 834 struct sk_buff *skb; 835 u16 inner_vtag = 0; 836 u16 out_vtag = 0; 837 u32 paylen = 0; 838 u16 mss = 0; 839 __be16 protocol; 840 u8 ol4_proto; 841 u8 il4_proto; 842 int ret; 843 844 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */ 845 desc_cb->priv = priv; 846 desc_cb->length = size; 847 desc_cb->dma = dma; 848 desc_cb->type = type; 849 850 /* now, fill the descriptor */ 851 desc->addr = cpu_to_le64(dma); 852 desc->tx.send_size = cpu_to_le16((u16)size); 853 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end); 854 desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri); 855 856 if (type == DESC_TYPE_SKB) { 857 skb = (struct sk_buff *)priv; 858 paylen = skb->len; 859 860 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso, 861 &ol_type_vlan_len_msec, 862 &inner_vtag, &out_vtag); 863 if (unlikely(ret)) 864 return ret; 865 866 if (skb->ip_summed == CHECKSUM_PARTIAL) { 867 skb_reset_mac_len(skb); 868 protocol = skb->protocol; 869 870 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto); 871 if (ret) 872 return ret; 873 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto, 874 &type_cs_vlan_tso, 875 &ol_type_vlan_len_msec); 876 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto, 877 &type_cs_vlan_tso, 878 &ol_type_vlan_len_msec); 879 if (ret) 880 return ret; 881 882 ret = hns3_set_tso(skb, &paylen, &mss, 883 &type_cs_vlan_tso); 884 if (ret) 885 return ret; 886 } 887 888 /* Set txbd */ 889 desc->tx.ol_type_vlan_len_msec = 890 cpu_to_le32(ol_type_vlan_len_msec); 891 desc->tx.type_cs_vlan_tso_len = 892 cpu_to_le32(type_cs_vlan_tso); 893 desc->tx.paylen = cpu_to_le32(paylen); 894 desc->tx.mss = cpu_to_le16(mss); 895 desc->tx.vlan_tag = cpu_to_le16(inner_vtag); 896 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag); 897 } 898 899 /* move ring pointer to next.*/ 900 ring_ptr_move_fw(ring, next_to_use); 901 902 return 0; 903 } 904 905 static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv, 906 int size, dma_addr_t dma, int frag_end, 907 enum hns_desc_type type) 908 { 909 unsigned int frag_buf_num; 910 unsigned int k; 911 int sizeoflast; 912 int ret; 913 914 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE; 915 sizeoflast = size % HNS3_MAX_BD_SIZE; 916 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE; 917 918 /* When the frag size is bigger than hardware, split this frag */ 919 for (k = 0; k < frag_buf_num; k++) { 920 ret = hns3_fill_desc(ring, priv, 921 (k == frag_buf_num - 1) ? 922 sizeoflast : HNS3_MAX_BD_SIZE, 923 dma + HNS3_MAX_BD_SIZE * k, 924 frag_end && (k == frag_buf_num - 1) ? 1 : 0, 925 (type == DESC_TYPE_SKB && !k) ? 926 DESC_TYPE_SKB : DESC_TYPE_PAGE); 927 if (ret) 928 return ret; 929 } 930 931 return 0; 932 } 933 934 static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum, 935 struct hns3_enet_ring *ring) 936 { 937 struct sk_buff *skb = *out_skb; 938 struct skb_frag_struct *frag; 939 int bdnum_for_frag; 940 int frag_num; 941 int buf_num; 942 int size; 943 int i; 944 945 size = skb_headlen(skb); 946 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE; 947 948 frag_num = skb_shinfo(skb)->nr_frags; 949 for (i = 0; i < frag_num; i++) { 950 frag = &skb_shinfo(skb)->frags[i]; 951 size = skb_frag_size(frag); 952 bdnum_for_frag = 953 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE; 954 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG) 955 return -ENOMEM; 956 957 buf_num += bdnum_for_frag; 958 } 959 960 if (buf_num > ring_space(ring)) 961 return -EBUSY; 962 963 *bnum = buf_num; 964 return 0; 965 } 966 967 static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum, 968 struct hns3_enet_ring *ring) 969 { 970 struct sk_buff *skb = *out_skb; 971 int buf_num; 972 973 /* No. of segments (plus a header) */ 974 buf_num = skb_shinfo(skb)->nr_frags + 1; 975 976 if (buf_num > ring_space(ring)) 977 return -EBUSY; 978 979 *bnum = buf_num; 980 981 return 0; 982 } 983 984 static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig) 985 { 986 struct device *dev = ring_to_dev(ring); 987 unsigned int i; 988 989 for (i = 0; i < ring->desc_num; i++) { 990 /* check if this is where we started */ 991 if (ring->next_to_use == next_to_use_orig) 992 break; 993 994 /* unmap the descriptor dma address */ 995 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB) 996 dma_unmap_single(dev, 997 ring->desc_cb[ring->next_to_use].dma, 998 ring->desc_cb[ring->next_to_use].length, 999 DMA_TO_DEVICE); 1000 else 1001 dma_unmap_page(dev, 1002 ring->desc_cb[ring->next_to_use].dma, 1003 ring->desc_cb[ring->next_to_use].length, 1004 DMA_TO_DEVICE); 1005 1006 /* rollback one */ 1007 ring_ptr_move_bw(ring, next_to_use); 1008 } 1009 } 1010 1011 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) 1012 { 1013 struct hns3_nic_priv *priv = netdev_priv(netdev); 1014 struct hns3_nic_ring_data *ring_data = 1015 &tx_ring_data(priv, skb->queue_mapping); 1016 struct hns3_enet_ring *ring = ring_data->ring; 1017 struct device *dev = priv->dev; 1018 struct netdev_queue *dev_queue; 1019 struct skb_frag_struct *frag; 1020 int next_to_use_head; 1021 int next_to_use_frag; 1022 dma_addr_t dma; 1023 int buf_num; 1024 int seg_num; 1025 int size; 1026 int ret; 1027 int i; 1028 1029 /* Prefetch the data used later */ 1030 prefetch(skb->data); 1031 1032 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) { 1033 case -EBUSY: 1034 u64_stats_update_begin(&ring->syncp); 1035 ring->stats.tx_busy++; 1036 u64_stats_update_end(&ring->syncp); 1037 1038 goto out_net_tx_busy; 1039 case -ENOMEM: 1040 u64_stats_update_begin(&ring->syncp); 1041 ring->stats.sw_err_cnt++; 1042 u64_stats_update_end(&ring->syncp); 1043 netdev_err(netdev, "no memory to xmit!\n"); 1044 1045 goto out_err_tx_ok; 1046 default: 1047 break; 1048 } 1049 1050 /* No. of segments (plus a header) */ 1051 seg_num = skb_shinfo(skb)->nr_frags + 1; 1052 /* Fill the first part */ 1053 size = skb_headlen(skb); 1054 1055 next_to_use_head = ring->next_to_use; 1056 1057 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); 1058 if (dma_mapping_error(dev, dma)) { 1059 netdev_err(netdev, "TX head DMA map failed\n"); 1060 ring->stats.sw_err_cnt++; 1061 goto out_err_tx_ok; 1062 } 1063 1064 ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0, 1065 DESC_TYPE_SKB); 1066 if (ret) 1067 goto head_dma_map_err; 1068 1069 next_to_use_frag = ring->next_to_use; 1070 /* Fill the fragments */ 1071 for (i = 1; i < seg_num; i++) { 1072 frag = &skb_shinfo(skb)->frags[i - 1]; 1073 size = skb_frag_size(frag); 1074 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); 1075 if (dma_mapping_error(dev, dma)) { 1076 netdev_err(netdev, "TX frag(%d) DMA map failed\n", i); 1077 ring->stats.sw_err_cnt++; 1078 goto frag_dma_map_err; 1079 } 1080 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma, 1081 seg_num - 1 == i ? 1 : 0, 1082 DESC_TYPE_PAGE); 1083 1084 if (ret) 1085 goto frag_dma_map_err; 1086 } 1087 1088 /* Complete translate all packets */ 1089 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index); 1090 netdev_tx_sent_queue(dev_queue, skb->len); 1091 1092 wmb(); /* Commit all data before submit */ 1093 1094 hnae_queue_xmit(ring->tqp, buf_num); 1095 1096 return NETDEV_TX_OK; 1097 1098 frag_dma_map_err: 1099 hns_nic_dma_unmap(ring, next_to_use_frag); 1100 1101 head_dma_map_err: 1102 hns_nic_dma_unmap(ring, next_to_use_head); 1103 1104 out_err_tx_ok: 1105 dev_kfree_skb_any(skb); 1106 return NETDEV_TX_OK; 1107 1108 out_net_tx_busy: 1109 netif_stop_subqueue(netdev, ring_data->queue_index); 1110 smp_mb(); /* Commit all data before submit */ 1111 1112 return NETDEV_TX_BUSY; 1113 } 1114 1115 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p) 1116 { 1117 struct hnae3_handle *h = hns3_get_handle(netdev); 1118 struct sockaddr *mac_addr = p; 1119 int ret; 1120 1121 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data)) 1122 return -EADDRNOTAVAIL; 1123 1124 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false); 1125 if (ret) { 1126 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret); 1127 return ret; 1128 } 1129 1130 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data); 1131 1132 return 0; 1133 } 1134 1135 static int hns3_nic_set_features(struct net_device *netdev, 1136 netdev_features_t features) 1137 { 1138 netdev_features_t changed = netdev->features ^ features; 1139 struct hns3_nic_priv *priv = netdev_priv(netdev); 1140 struct hnae3_handle *h = priv->ae_handle; 1141 int ret; 1142 1143 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) { 1144 if (features & (NETIF_F_TSO | NETIF_F_TSO6)) { 1145 priv->ops.fill_desc = hns3_fill_desc_tso; 1146 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso; 1147 } else { 1148 priv->ops.fill_desc = hns3_fill_desc; 1149 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx; 1150 } 1151 } 1152 1153 if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) && 1154 h->ae_algo->ops->enable_vlan_filter) { 1155 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) 1156 h->ae_algo->ops->enable_vlan_filter(h, true); 1157 else 1158 h->ae_algo->ops->enable_vlan_filter(h, false); 1159 } 1160 1161 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && 1162 h->ae_algo->ops->enable_hw_strip_rxvtag) { 1163 if (features & NETIF_F_HW_VLAN_CTAG_RX) 1164 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true); 1165 else 1166 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false); 1167 1168 if (ret) 1169 return ret; 1170 } 1171 1172 netdev->features = features; 1173 return 0; 1174 } 1175 1176 static void hns3_nic_get_stats64(struct net_device *netdev, 1177 struct rtnl_link_stats64 *stats) 1178 { 1179 struct hns3_nic_priv *priv = netdev_priv(netdev); 1180 int queue_num = priv->ae_handle->kinfo.num_tqps; 1181 struct hnae3_handle *handle = priv->ae_handle; 1182 struct hns3_enet_ring *ring; 1183 unsigned int start; 1184 unsigned int idx; 1185 u64 tx_bytes = 0; 1186 u64 rx_bytes = 0; 1187 u64 tx_pkts = 0; 1188 u64 rx_pkts = 0; 1189 u64 tx_drop = 0; 1190 u64 rx_drop = 0; 1191 1192 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) 1193 return; 1194 1195 handle->ae_algo->ops->update_stats(handle, &netdev->stats); 1196 1197 for (idx = 0; idx < queue_num; idx++) { 1198 /* fetch the tx stats */ 1199 ring = priv->ring_data[idx].ring; 1200 do { 1201 start = u64_stats_fetch_begin_irq(&ring->syncp); 1202 tx_bytes += ring->stats.tx_bytes; 1203 tx_pkts += ring->stats.tx_pkts; 1204 tx_drop += ring->stats.tx_busy; 1205 tx_drop += ring->stats.sw_err_cnt; 1206 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 1207 1208 /* fetch the rx stats */ 1209 ring = priv->ring_data[idx + queue_num].ring; 1210 do { 1211 start = u64_stats_fetch_begin_irq(&ring->syncp); 1212 rx_bytes += ring->stats.rx_bytes; 1213 rx_pkts += ring->stats.rx_pkts; 1214 rx_drop += ring->stats.non_vld_descs; 1215 rx_drop += ring->stats.err_pkt_len; 1216 rx_drop += ring->stats.l2_err; 1217 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 1218 } 1219 1220 stats->tx_bytes = tx_bytes; 1221 stats->tx_packets = tx_pkts; 1222 stats->rx_bytes = rx_bytes; 1223 stats->rx_packets = rx_pkts; 1224 1225 stats->rx_errors = netdev->stats.rx_errors; 1226 stats->multicast = netdev->stats.multicast; 1227 stats->rx_length_errors = netdev->stats.rx_length_errors; 1228 stats->rx_crc_errors = netdev->stats.rx_crc_errors; 1229 stats->rx_missed_errors = netdev->stats.rx_missed_errors; 1230 1231 stats->tx_errors = netdev->stats.tx_errors; 1232 stats->rx_dropped = rx_drop + netdev->stats.rx_dropped; 1233 stats->tx_dropped = tx_drop + netdev->stats.tx_dropped; 1234 stats->collisions = netdev->stats.collisions; 1235 stats->rx_over_errors = netdev->stats.rx_over_errors; 1236 stats->rx_frame_errors = netdev->stats.rx_frame_errors; 1237 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors; 1238 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors; 1239 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors; 1240 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors; 1241 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors; 1242 stats->tx_window_errors = netdev->stats.tx_window_errors; 1243 stats->rx_compressed = netdev->stats.rx_compressed; 1244 stats->tx_compressed = netdev->stats.tx_compressed; 1245 } 1246 1247 static void hns3_add_tunnel_port(struct net_device *netdev, u16 port, 1248 enum hns3_udp_tnl_type type) 1249 { 1250 struct hns3_nic_priv *priv = netdev_priv(netdev); 1251 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type]; 1252 struct hnae3_handle *h = priv->ae_handle; 1253 1254 if (udp_tnl->used && udp_tnl->dst_port == port) { 1255 udp_tnl->used++; 1256 return; 1257 } 1258 1259 if (udp_tnl->used) { 1260 netdev_warn(netdev, 1261 "UDP tunnel [%d], port [%d] offload\n", type, port); 1262 return; 1263 } 1264 1265 udp_tnl->dst_port = port; 1266 udp_tnl->used = 1; 1267 /* TBD send command to hardware to add port */ 1268 if (h->ae_algo->ops->add_tunnel_udp) 1269 h->ae_algo->ops->add_tunnel_udp(h, port); 1270 } 1271 1272 static void hns3_del_tunnel_port(struct net_device *netdev, u16 port, 1273 enum hns3_udp_tnl_type type) 1274 { 1275 struct hns3_nic_priv *priv = netdev_priv(netdev); 1276 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type]; 1277 struct hnae3_handle *h = priv->ae_handle; 1278 1279 if (!udp_tnl->used || udp_tnl->dst_port != port) { 1280 netdev_warn(netdev, 1281 "Invalid UDP tunnel port %d\n", port); 1282 return; 1283 } 1284 1285 udp_tnl->used--; 1286 if (udp_tnl->used) 1287 return; 1288 1289 udp_tnl->dst_port = 0; 1290 /* TBD send command to hardware to del port */ 1291 if (h->ae_algo->ops->del_tunnel_udp) 1292 h->ae_algo->ops->del_tunnel_udp(h, port); 1293 } 1294 1295 /* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports 1296 * @netdev: This physical ports's netdev 1297 * @ti: Tunnel information 1298 */ 1299 static void hns3_nic_udp_tunnel_add(struct net_device *netdev, 1300 struct udp_tunnel_info *ti) 1301 { 1302 u16 port_n = ntohs(ti->port); 1303 1304 switch (ti->type) { 1305 case UDP_TUNNEL_TYPE_VXLAN: 1306 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN); 1307 break; 1308 case UDP_TUNNEL_TYPE_GENEVE: 1309 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE); 1310 break; 1311 default: 1312 netdev_err(netdev, "unsupported tunnel type %d\n", ti->type); 1313 break; 1314 } 1315 } 1316 1317 static void hns3_nic_udp_tunnel_del(struct net_device *netdev, 1318 struct udp_tunnel_info *ti) 1319 { 1320 u16 port_n = ntohs(ti->port); 1321 1322 switch (ti->type) { 1323 case UDP_TUNNEL_TYPE_VXLAN: 1324 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN); 1325 break; 1326 case UDP_TUNNEL_TYPE_GENEVE: 1327 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE); 1328 break; 1329 default: 1330 break; 1331 } 1332 } 1333 1334 static int hns3_setup_tc(struct net_device *netdev, void *type_data) 1335 { 1336 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 1337 struct hnae3_handle *h = hns3_get_handle(netdev); 1338 struct hnae3_knic_private_info *kinfo = &h->kinfo; 1339 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map; 1340 u8 tc = mqprio_qopt->qopt.num_tc; 1341 u16 mode = mqprio_qopt->mode; 1342 u8 hw = mqprio_qopt->qopt.hw; 1343 bool if_running; 1344 unsigned int i; 1345 int ret; 1346 1347 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS && 1348 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0))) 1349 return -EOPNOTSUPP; 1350 1351 if (tc > HNAE3_MAX_TC) 1352 return -EINVAL; 1353 1354 if (!netdev) 1355 return -EINVAL; 1356 1357 if_running = netif_running(netdev); 1358 if (if_running) { 1359 hns3_nic_net_stop(netdev); 1360 msleep(100); 1361 } 1362 1363 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ? 1364 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP; 1365 if (ret) 1366 goto out; 1367 1368 if (tc <= 1) { 1369 netdev_reset_tc(netdev); 1370 } else { 1371 ret = netdev_set_num_tc(netdev, tc); 1372 if (ret) 1373 goto out; 1374 1375 for (i = 0; i < HNAE3_MAX_TC; i++) { 1376 if (!kinfo->tc_info[i].enable) 1377 continue; 1378 1379 netdev_set_tc_queue(netdev, 1380 kinfo->tc_info[i].tc, 1381 kinfo->tc_info[i].tqp_count, 1382 kinfo->tc_info[i].tqp_offset); 1383 } 1384 } 1385 1386 ret = hns3_nic_set_real_num_queue(netdev); 1387 1388 out: 1389 if (if_running) 1390 hns3_nic_net_open(netdev); 1391 1392 return ret; 1393 } 1394 1395 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type, 1396 void *type_data) 1397 { 1398 if (type != TC_SETUP_QDISC_MQPRIO) 1399 return -EOPNOTSUPP; 1400 1401 return hns3_setup_tc(dev, type_data); 1402 } 1403 1404 static int hns3_vlan_rx_add_vid(struct net_device *netdev, 1405 __be16 proto, u16 vid) 1406 { 1407 struct hnae3_handle *h = hns3_get_handle(netdev); 1408 struct hns3_nic_priv *priv = netdev_priv(netdev); 1409 int ret = -EIO; 1410 1411 if (h->ae_algo->ops->set_vlan_filter) 1412 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false); 1413 1414 if (!ret) 1415 set_bit(vid, priv->active_vlans); 1416 1417 return ret; 1418 } 1419 1420 static int hns3_vlan_rx_kill_vid(struct net_device *netdev, 1421 __be16 proto, u16 vid) 1422 { 1423 struct hnae3_handle *h = hns3_get_handle(netdev); 1424 struct hns3_nic_priv *priv = netdev_priv(netdev); 1425 int ret = -EIO; 1426 1427 if (h->ae_algo->ops->set_vlan_filter) 1428 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true); 1429 1430 if (!ret) 1431 clear_bit(vid, priv->active_vlans); 1432 1433 return ret; 1434 } 1435 1436 static void hns3_restore_vlan(struct net_device *netdev) 1437 { 1438 struct hns3_nic_priv *priv = netdev_priv(netdev); 1439 u16 vid; 1440 int ret; 1441 1442 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 1443 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid); 1444 if (ret) 1445 netdev_warn(netdev, "Restore vlan: %d filter, ret:%d\n", 1446 vid, ret); 1447 } 1448 } 1449 1450 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, 1451 u8 qos, __be16 vlan_proto) 1452 { 1453 struct hnae3_handle *h = hns3_get_handle(netdev); 1454 int ret = -EIO; 1455 1456 if (h->ae_algo->ops->set_vf_vlan_filter) 1457 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan, 1458 qos, vlan_proto); 1459 1460 return ret; 1461 } 1462 1463 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu) 1464 { 1465 struct hnae3_handle *h = hns3_get_handle(netdev); 1466 bool if_running = netif_running(netdev); 1467 int ret; 1468 1469 if (!h->ae_algo->ops->set_mtu) 1470 return -EOPNOTSUPP; 1471 1472 /* if this was called with netdev up then bring netdevice down */ 1473 if (if_running) { 1474 (void)hns3_nic_net_stop(netdev); 1475 msleep(100); 1476 } 1477 1478 ret = h->ae_algo->ops->set_mtu(h, new_mtu); 1479 if (ret) { 1480 netdev_err(netdev, "failed to change MTU in hardware %d\n", 1481 ret); 1482 return ret; 1483 } 1484 1485 netdev->mtu = new_mtu; 1486 1487 /* if the netdev was running earlier, bring it up again */ 1488 if (if_running && hns3_nic_net_open(netdev)) 1489 ret = -EINVAL; 1490 1491 return ret; 1492 } 1493 1494 static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) 1495 { 1496 struct hns3_nic_priv *priv = netdev_priv(ndev); 1497 struct hns3_enet_ring *tx_ring = NULL; 1498 int timeout_queue = 0; 1499 int hw_head, hw_tail; 1500 int i; 1501 1502 /* Find the stopped queue the same way the stack does */ 1503 for (i = 0; i < ndev->real_num_tx_queues; i++) { 1504 struct netdev_queue *q; 1505 unsigned long trans_start; 1506 1507 q = netdev_get_tx_queue(ndev, i); 1508 trans_start = q->trans_start; 1509 if (netif_xmit_stopped(q) && 1510 time_after(jiffies, 1511 (trans_start + ndev->watchdog_timeo))) { 1512 timeout_queue = i; 1513 break; 1514 } 1515 } 1516 1517 if (i == ndev->num_tx_queues) { 1518 netdev_info(ndev, 1519 "no netdev TX timeout queue found, timeout count: %llu\n", 1520 priv->tx_timeout_count); 1521 return false; 1522 } 1523 1524 tx_ring = priv->ring_data[timeout_queue].ring; 1525 1526 hw_head = readl_relaxed(tx_ring->tqp->io_base + 1527 HNS3_RING_TX_RING_HEAD_REG); 1528 hw_tail = readl_relaxed(tx_ring->tqp->io_base + 1529 HNS3_RING_TX_RING_TAIL_REG); 1530 netdev_info(ndev, 1531 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n", 1532 priv->tx_timeout_count, 1533 timeout_queue, 1534 tx_ring->next_to_use, 1535 tx_ring->next_to_clean, 1536 hw_head, 1537 hw_tail, 1538 readl(tx_ring->tqp_vector->mask_addr)); 1539 1540 return true; 1541 } 1542 1543 static void hns3_nic_net_timeout(struct net_device *ndev) 1544 { 1545 struct hns3_nic_priv *priv = netdev_priv(ndev); 1546 struct hnae3_handle *h = priv->ae_handle; 1547 1548 if (!hns3_get_tx_timeo_queue_info(ndev)) 1549 return; 1550 1551 priv->tx_timeout_count++; 1552 1553 if (time_before(jiffies, (h->last_reset_time + ndev->watchdog_timeo))) 1554 return; 1555 1556 /* request the reset */ 1557 if (h->ae_algo->ops->reset_event) 1558 h->ae_algo->ops->reset_event(h); 1559 } 1560 1561 static const struct net_device_ops hns3_nic_netdev_ops = { 1562 .ndo_open = hns3_nic_net_open, 1563 .ndo_stop = hns3_nic_net_stop, 1564 .ndo_start_xmit = hns3_nic_net_xmit, 1565 .ndo_tx_timeout = hns3_nic_net_timeout, 1566 .ndo_set_mac_address = hns3_nic_net_set_mac_address, 1567 .ndo_change_mtu = hns3_nic_change_mtu, 1568 .ndo_set_features = hns3_nic_set_features, 1569 .ndo_get_stats64 = hns3_nic_get_stats64, 1570 .ndo_setup_tc = hns3_nic_setup_tc, 1571 .ndo_set_rx_mode = hns3_nic_set_rx_mode, 1572 .ndo_udp_tunnel_add = hns3_nic_udp_tunnel_add, 1573 .ndo_udp_tunnel_del = hns3_nic_udp_tunnel_del, 1574 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid, 1575 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid, 1576 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan, 1577 }; 1578 1579 /* hns3_probe - Device initialization routine 1580 * @pdev: PCI device information struct 1581 * @ent: entry in hns3_pci_tbl 1582 * 1583 * hns3_probe initializes a PF identified by a pci_dev structure. 1584 * The OS initialization, configuring of the PF private structure, 1585 * and a hardware reset occur. 1586 * 1587 * Returns 0 on success, negative on failure 1588 */ 1589 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1590 { 1591 struct hnae3_ae_dev *ae_dev; 1592 int ret; 1593 1594 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev), 1595 GFP_KERNEL); 1596 if (!ae_dev) { 1597 ret = -ENOMEM; 1598 return ret; 1599 } 1600 1601 ae_dev->pdev = pdev; 1602 ae_dev->flag = ent->driver_data; 1603 ae_dev->dev_type = HNAE3_DEV_KNIC; 1604 pci_set_drvdata(pdev, ae_dev); 1605 1606 return hnae3_register_ae_dev(ae_dev); 1607 } 1608 1609 /* hns3_remove - Device removal routine 1610 * @pdev: PCI device information struct 1611 */ 1612 static void hns3_remove(struct pci_dev *pdev) 1613 { 1614 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 1615 1616 hnae3_unregister_ae_dev(ae_dev); 1617 } 1618 1619 static struct pci_driver hns3_driver = { 1620 .name = hns3_driver_name, 1621 .id_table = hns3_pci_tbl, 1622 .probe = hns3_probe, 1623 .remove = hns3_remove, 1624 }; 1625 1626 /* set default feature to hns3 */ 1627 static void hns3_set_default_feature(struct net_device *netdev) 1628 { 1629 struct hnae3_handle *h = hns3_get_handle(netdev); 1630 1631 netdev->priv_flags |= IFF_UNICAST_FLT; 1632 1633 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 1634 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO | 1635 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE | 1636 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | 1637 NETIF_F_GSO_UDP_TUNNEL_CSUM; 1638 1639 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 1640 1641 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 1642 1643 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 1644 NETIF_F_HW_VLAN_CTAG_FILTER | 1645 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 1646 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO | 1647 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE | 1648 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | 1649 NETIF_F_GSO_UDP_TUNNEL_CSUM; 1650 1651 netdev->vlan_features |= 1652 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | 1653 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO | 1654 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE | 1655 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | 1656 NETIF_F_GSO_UDP_TUNNEL_CSUM; 1657 1658 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 1659 NETIF_F_HW_VLAN_CTAG_TX | 1660 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO | 1661 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE | 1662 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | 1663 NETIF_F_GSO_UDP_TUNNEL_CSUM; 1664 1665 if (!(h->flags & HNAE3_SUPPORT_VF)) 1666 netdev->hw_features |= 1667 NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX; 1668 } 1669 1670 static int hns3_alloc_buffer(struct hns3_enet_ring *ring, 1671 struct hns3_desc_cb *cb) 1672 { 1673 unsigned int order = hnae_page_order(ring); 1674 struct page *p; 1675 1676 p = dev_alloc_pages(order); 1677 if (!p) 1678 return -ENOMEM; 1679 1680 cb->priv = p; 1681 cb->page_offset = 0; 1682 cb->reuse_flag = 0; 1683 cb->buf = page_address(p); 1684 cb->length = hnae_page_size(ring); 1685 cb->type = DESC_TYPE_PAGE; 1686 1687 return 0; 1688 } 1689 1690 static void hns3_free_buffer(struct hns3_enet_ring *ring, 1691 struct hns3_desc_cb *cb) 1692 { 1693 if (cb->type == DESC_TYPE_SKB) 1694 dev_kfree_skb_any((struct sk_buff *)cb->priv); 1695 else if (!HNAE3_IS_TX_RING(ring)) 1696 put_page((struct page *)cb->priv); 1697 memset(cb, 0, sizeof(*cb)); 1698 } 1699 1700 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb) 1701 { 1702 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0, 1703 cb->length, ring_to_dma_dir(ring)); 1704 1705 if (dma_mapping_error(ring_to_dev(ring), cb->dma)) 1706 return -EIO; 1707 1708 return 0; 1709 } 1710 1711 static void hns3_unmap_buffer(struct hns3_enet_ring *ring, 1712 struct hns3_desc_cb *cb) 1713 { 1714 if (cb->type == DESC_TYPE_SKB) 1715 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length, 1716 ring_to_dma_dir(ring)); 1717 else 1718 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length, 1719 ring_to_dma_dir(ring)); 1720 } 1721 1722 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i) 1723 { 1724 hns3_unmap_buffer(ring, &ring->desc_cb[i]); 1725 ring->desc[i].addr = 0; 1726 } 1727 1728 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i) 1729 { 1730 struct hns3_desc_cb *cb = &ring->desc_cb[i]; 1731 1732 if (!ring->desc_cb[i].dma) 1733 return; 1734 1735 hns3_buffer_detach(ring, i); 1736 hns3_free_buffer(ring, cb); 1737 } 1738 1739 static void hns3_free_buffers(struct hns3_enet_ring *ring) 1740 { 1741 int i; 1742 1743 for (i = 0; i < ring->desc_num; i++) 1744 hns3_free_buffer_detach(ring, i); 1745 } 1746 1747 /* free desc along with its attached buffer */ 1748 static void hns3_free_desc(struct hns3_enet_ring *ring) 1749 { 1750 hns3_free_buffers(ring); 1751 1752 dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr, 1753 ring->desc_num * sizeof(ring->desc[0]), 1754 DMA_BIDIRECTIONAL); 1755 ring->desc_dma_addr = 0; 1756 kfree(ring->desc); 1757 ring->desc = NULL; 1758 } 1759 1760 static int hns3_alloc_desc(struct hns3_enet_ring *ring) 1761 { 1762 int size = ring->desc_num * sizeof(ring->desc[0]); 1763 1764 ring->desc = kzalloc(size, GFP_KERNEL); 1765 if (!ring->desc) 1766 return -ENOMEM; 1767 1768 ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc, 1769 size, DMA_BIDIRECTIONAL); 1770 if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) { 1771 ring->desc_dma_addr = 0; 1772 kfree(ring->desc); 1773 ring->desc = NULL; 1774 return -ENOMEM; 1775 } 1776 1777 return 0; 1778 } 1779 1780 static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring, 1781 struct hns3_desc_cb *cb) 1782 { 1783 int ret; 1784 1785 ret = hns3_alloc_buffer(ring, cb); 1786 if (ret) 1787 goto out; 1788 1789 ret = hns3_map_buffer(ring, cb); 1790 if (ret) 1791 goto out_with_buf; 1792 1793 return 0; 1794 1795 out_with_buf: 1796 hns3_free_buffer(ring, cb); 1797 out: 1798 return ret; 1799 } 1800 1801 static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i) 1802 { 1803 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]); 1804 1805 if (ret) 1806 return ret; 1807 1808 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma); 1809 1810 return 0; 1811 } 1812 1813 /* Allocate memory for raw pkg, and map with dma */ 1814 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring) 1815 { 1816 int i, j, ret; 1817 1818 for (i = 0; i < ring->desc_num; i++) { 1819 ret = hns3_alloc_buffer_attach(ring, i); 1820 if (ret) 1821 goto out_buffer_fail; 1822 } 1823 1824 return 0; 1825 1826 out_buffer_fail: 1827 for (j = i - 1; j >= 0; j--) 1828 hns3_free_buffer_detach(ring, j); 1829 return ret; 1830 } 1831 1832 /* detach a in-used buffer and replace with a reserved one */ 1833 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i, 1834 struct hns3_desc_cb *res_cb) 1835 { 1836 hns3_unmap_buffer(ring, &ring->desc_cb[i]); 1837 ring->desc_cb[i] = *res_cb; 1838 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma); 1839 } 1840 1841 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i) 1842 { 1843 ring->desc_cb[i].reuse_flag = 0; 1844 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma 1845 + ring->desc_cb[i].page_offset); 1846 } 1847 1848 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes, 1849 int *pkts) 1850 { 1851 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean]; 1852 1853 (*pkts) += (desc_cb->type == DESC_TYPE_SKB); 1854 (*bytes) += desc_cb->length; 1855 /* desc_cb will be cleaned, after hnae_free_buffer_detach*/ 1856 hns3_free_buffer_detach(ring, ring->next_to_clean); 1857 1858 ring_ptr_move_fw(ring, next_to_clean); 1859 } 1860 1861 static int is_valid_clean_head(struct hns3_enet_ring *ring, int h) 1862 { 1863 int u = ring->next_to_use; 1864 int c = ring->next_to_clean; 1865 1866 if (unlikely(h > ring->desc_num)) 1867 return 0; 1868 1869 return u > c ? (h > c && h <= u) : (h > c || h <= u); 1870 } 1871 1872 bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget) 1873 { 1874 struct net_device *netdev = ring->tqp->handle->kinfo.netdev; 1875 struct netdev_queue *dev_queue; 1876 int bytes, pkts; 1877 int head; 1878 1879 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG); 1880 rmb(); /* Make sure head is ready before touch any data */ 1881 1882 if (is_ring_empty(ring) || head == ring->next_to_clean) 1883 return true; /* no data to poll */ 1884 1885 if (!is_valid_clean_head(ring, head)) { 1886 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head, 1887 ring->next_to_use, ring->next_to_clean); 1888 1889 u64_stats_update_begin(&ring->syncp); 1890 ring->stats.io_err_cnt++; 1891 u64_stats_update_end(&ring->syncp); 1892 return true; 1893 } 1894 1895 bytes = 0; 1896 pkts = 0; 1897 while (head != ring->next_to_clean && budget) { 1898 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts); 1899 /* Issue prefetch for next Tx descriptor */ 1900 prefetch(&ring->desc_cb[ring->next_to_clean]); 1901 budget--; 1902 } 1903 1904 ring->tqp_vector->tx_group.total_bytes += bytes; 1905 ring->tqp_vector->tx_group.total_packets += pkts; 1906 1907 u64_stats_update_begin(&ring->syncp); 1908 ring->stats.tx_bytes += bytes; 1909 ring->stats.tx_pkts += pkts; 1910 u64_stats_update_end(&ring->syncp); 1911 1912 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index); 1913 netdev_tx_completed_queue(dev_queue, pkts, bytes); 1914 1915 if (unlikely(pkts && netif_carrier_ok(netdev) && 1916 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) { 1917 /* Make sure that anybody stopping the queue after this 1918 * sees the new next_to_clean. 1919 */ 1920 smp_mb(); 1921 if (netif_tx_queue_stopped(dev_queue)) { 1922 netif_tx_wake_queue(dev_queue); 1923 ring->stats.restart_queue++; 1924 } 1925 } 1926 1927 return !!budget; 1928 } 1929 1930 static int hns3_desc_unused(struct hns3_enet_ring *ring) 1931 { 1932 int ntc = ring->next_to_clean; 1933 int ntu = ring->next_to_use; 1934 1935 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu; 1936 } 1937 1938 static void 1939 hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count) 1940 { 1941 struct hns3_desc_cb *desc_cb; 1942 struct hns3_desc_cb res_cbs; 1943 int i, ret; 1944 1945 for (i = 0; i < cleand_count; i++) { 1946 desc_cb = &ring->desc_cb[ring->next_to_use]; 1947 if (desc_cb->reuse_flag) { 1948 u64_stats_update_begin(&ring->syncp); 1949 ring->stats.reuse_pg_cnt++; 1950 u64_stats_update_end(&ring->syncp); 1951 1952 hns3_reuse_buffer(ring, ring->next_to_use); 1953 } else { 1954 ret = hns3_reserve_buffer_map(ring, &res_cbs); 1955 if (ret) { 1956 u64_stats_update_begin(&ring->syncp); 1957 ring->stats.sw_err_cnt++; 1958 u64_stats_update_end(&ring->syncp); 1959 1960 netdev_err(ring->tqp->handle->kinfo.netdev, 1961 "hnae reserve buffer map failed.\n"); 1962 break; 1963 } 1964 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs); 1965 } 1966 1967 ring_ptr_move_fw(ring, next_to_use); 1968 } 1969 1970 wmb(); /* Make all data has been write before submit */ 1971 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG); 1972 } 1973 1974 /* hns3_nic_get_headlen - determine size of header for LRO/GRO 1975 * @data: pointer to the start of the headers 1976 * @max: total length of section to find headers in 1977 * 1978 * This function is meant to determine the length of headers that will 1979 * be recognized by hardware for LRO, GRO, and RSC offloads. The main 1980 * motivation of doing this is to only perform one pull for IPv4 TCP 1981 * packets so that we can do basic things like calculating the gso_size 1982 * based on the average data per packet. 1983 */ 1984 static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag, 1985 unsigned int max_size) 1986 { 1987 unsigned char *network; 1988 u8 hlen; 1989 1990 /* This should never happen, but better safe than sorry */ 1991 if (max_size < ETH_HLEN) 1992 return max_size; 1993 1994 /* Initialize network frame pointer */ 1995 network = data; 1996 1997 /* Set first protocol and move network header forward */ 1998 network += ETH_HLEN; 1999 2000 /* Handle any vlan tag if present */ 2001 if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S) 2002 == HNS3_RX_FLAG_VLAN_PRESENT) { 2003 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN)) 2004 return max_size; 2005 2006 network += VLAN_HLEN; 2007 } 2008 2009 /* Handle L3 protocols */ 2010 if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S) 2011 == HNS3_RX_FLAG_L3ID_IPV4) { 2012 if ((typeof(max_size))(network - data) > 2013 (max_size - sizeof(struct iphdr))) 2014 return max_size; 2015 2016 /* Access ihl as a u8 to avoid unaligned access on ia64 */ 2017 hlen = (network[0] & 0x0F) << 2; 2018 2019 /* Verify hlen meets minimum size requirements */ 2020 if (hlen < sizeof(struct iphdr)) 2021 return network - data; 2022 2023 /* Record next protocol if header is present */ 2024 } else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S) 2025 == HNS3_RX_FLAG_L3ID_IPV6) { 2026 if ((typeof(max_size))(network - data) > 2027 (max_size - sizeof(struct ipv6hdr))) 2028 return max_size; 2029 2030 /* Record next protocol */ 2031 hlen = sizeof(struct ipv6hdr); 2032 } else { 2033 return network - data; 2034 } 2035 2036 /* Relocate pointer to start of L4 header */ 2037 network += hlen; 2038 2039 /* Finally sort out TCP/UDP */ 2040 if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S) 2041 == HNS3_RX_FLAG_L4ID_TCP) { 2042 if ((typeof(max_size))(network - data) > 2043 (max_size - sizeof(struct tcphdr))) 2044 return max_size; 2045 2046 /* Access doff as a u8 to avoid unaligned access on ia64 */ 2047 hlen = (network[12] & 0xF0) >> 2; 2048 2049 /* Verify hlen meets minimum size requirements */ 2050 if (hlen < sizeof(struct tcphdr)) 2051 return network - data; 2052 2053 network += hlen; 2054 } else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S) 2055 == HNS3_RX_FLAG_L4ID_UDP) { 2056 if ((typeof(max_size))(network - data) > 2057 (max_size - sizeof(struct udphdr))) 2058 return max_size; 2059 2060 network += sizeof(struct udphdr); 2061 } 2062 2063 /* If everything has gone correctly network should be the 2064 * data section of the packet and will be the end of the header. 2065 * If not then it probably represents the end of the last recognized 2066 * header. 2067 */ 2068 if ((typeof(max_size))(network - data) < max_size) 2069 return network - data; 2070 else 2071 return max_size; 2072 } 2073 2074 static void hns3_nic_reuse_page(struct sk_buff *skb, int i, 2075 struct hns3_enet_ring *ring, int pull_len, 2076 struct hns3_desc_cb *desc_cb) 2077 { 2078 struct hns3_desc *desc; 2079 int truesize, size; 2080 int last_offset; 2081 bool twobufs; 2082 2083 twobufs = ((PAGE_SIZE < 8192) && 2084 hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048); 2085 2086 desc = &ring->desc[ring->next_to_clean]; 2087 size = le16_to_cpu(desc->rx.size); 2088 2089 truesize = hnae_buf_size(ring); 2090 2091 if (!twobufs) 2092 last_offset = hnae_page_size(ring) - hnae_buf_size(ring); 2093 2094 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len, 2095 size - pull_len, truesize); 2096 2097 /* Avoid re-using remote pages,flag default unreuse */ 2098 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id())) 2099 return; 2100 2101 if (twobufs) { 2102 /* If we are only owner of page we can reuse it */ 2103 if (likely(page_count(desc_cb->priv) == 1)) { 2104 /* Flip page offset to other buffer */ 2105 desc_cb->page_offset ^= truesize; 2106 2107 desc_cb->reuse_flag = 1; 2108 /* bump ref count on page before it is given*/ 2109 get_page(desc_cb->priv); 2110 } 2111 return; 2112 } 2113 2114 /* Move offset up to the next cache line */ 2115 desc_cb->page_offset += truesize; 2116 2117 if (desc_cb->page_offset <= last_offset) { 2118 desc_cb->reuse_flag = 1; 2119 /* Bump ref count on page before it is given*/ 2120 get_page(desc_cb->priv); 2121 } 2122 } 2123 2124 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb, 2125 struct hns3_desc *desc) 2126 { 2127 struct net_device *netdev = ring->tqp->handle->kinfo.netdev; 2128 int l3_type, l4_type; 2129 u32 bd_base_info; 2130 int ol4_type; 2131 u32 l234info; 2132 2133 bd_base_info = le32_to_cpu(desc->rx.bd_base_info); 2134 l234info = le32_to_cpu(desc->rx.l234_info); 2135 2136 skb->ip_summed = CHECKSUM_NONE; 2137 2138 skb_checksum_none_assert(skb); 2139 2140 if (!(netdev->features & NETIF_F_RXCSUM)) 2141 return; 2142 2143 /* check if hardware has done checksum */ 2144 if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B)) 2145 return; 2146 2147 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) || 2148 hnae_get_bit(l234info, HNS3_RXD_L4E_B) || 2149 hnae_get_bit(l234info, HNS3_RXD_OL3E_B) || 2150 hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) { 2151 netdev_err(netdev, "L3/L4 error pkt\n"); 2152 u64_stats_update_begin(&ring->syncp); 2153 ring->stats.l3l4_csum_err++; 2154 u64_stats_update_end(&ring->syncp); 2155 2156 return; 2157 } 2158 2159 l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M, 2160 HNS3_RXD_L3ID_S); 2161 l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M, 2162 HNS3_RXD_L4ID_S); 2163 2164 ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S); 2165 switch (ol4_type) { 2166 case HNS3_OL4_TYPE_MAC_IN_UDP: 2167 case HNS3_OL4_TYPE_NVGRE: 2168 skb->csum_level = 1; 2169 case HNS3_OL4_TYPE_NO_TUN: 2170 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */ 2171 if (l3_type == HNS3_L3_TYPE_IPV4 || 2172 (l3_type == HNS3_L3_TYPE_IPV6 && 2173 (l4_type == HNS3_L4_TYPE_UDP || 2174 l4_type == HNS3_L4_TYPE_TCP || 2175 l4_type == HNS3_L4_TYPE_SCTP))) 2176 skb->ip_summed = CHECKSUM_UNNECESSARY; 2177 break; 2178 } 2179 } 2180 2181 static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb) 2182 { 2183 napi_gro_receive(&ring->tqp_vector->napi, skb); 2184 } 2185 2186 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring, 2187 struct sk_buff **out_skb, int *out_bnum) 2188 { 2189 struct net_device *netdev = ring->tqp->handle->kinfo.netdev; 2190 struct hns3_desc_cb *desc_cb; 2191 struct hns3_desc *desc; 2192 struct sk_buff *skb; 2193 unsigned char *va; 2194 u32 bd_base_info; 2195 int pull_len; 2196 u32 l234info; 2197 int length; 2198 int bnum; 2199 2200 desc = &ring->desc[ring->next_to_clean]; 2201 desc_cb = &ring->desc_cb[ring->next_to_clean]; 2202 2203 prefetch(desc); 2204 2205 length = le16_to_cpu(desc->rx.pkt_len); 2206 bd_base_info = le32_to_cpu(desc->rx.bd_base_info); 2207 l234info = le32_to_cpu(desc->rx.l234_info); 2208 2209 /* Check valid BD */ 2210 if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B)) 2211 return -EFAULT; 2212 2213 va = (unsigned char *)desc_cb->buf + desc_cb->page_offset; 2214 2215 /* Prefetch first cache line of first page 2216 * Idea is to cache few bytes of the header of the packet. Our L1 Cache 2217 * line size is 64B so need to prefetch twice to make it 128B. But in 2218 * actual we can have greater size of caches with 128B Level 1 cache 2219 * lines. In such a case, single fetch would suffice to cache in the 2220 * relevant part of the header. 2221 */ 2222 prefetch(va); 2223 #if L1_CACHE_BYTES < 128 2224 prefetch(va + L1_CACHE_BYTES); 2225 #endif 2226 2227 skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi, 2228 HNS3_RX_HEAD_SIZE); 2229 if (unlikely(!skb)) { 2230 netdev_err(netdev, "alloc rx skb fail\n"); 2231 2232 u64_stats_update_begin(&ring->syncp); 2233 ring->stats.sw_err_cnt++; 2234 u64_stats_update_end(&ring->syncp); 2235 2236 return -ENOMEM; 2237 } 2238 2239 prefetchw(skb->data); 2240 2241 /* Based on hw strategy, the tag offloaded will be stored at 2242 * ot_vlan_tag in two layer tag case, and stored at vlan_tag 2243 * in one layer tag case. 2244 */ 2245 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) { 2246 u16 vlan_tag; 2247 2248 vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag); 2249 if (!(vlan_tag & VLAN_VID_MASK)) 2250 vlan_tag = le16_to_cpu(desc->rx.vlan_tag); 2251 if (vlan_tag & VLAN_VID_MASK) 2252 __vlan_hwaccel_put_tag(skb, 2253 htons(ETH_P_8021Q), 2254 vlan_tag); 2255 } 2256 2257 bnum = 1; 2258 if (length <= HNS3_RX_HEAD_SIZE) { 2259 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long))); 2260 2261 /* We can reuse buffer as-is, just make sure it is local */ 2262 if (likely(page_to_nid(desc_cb->priv) == numa_node_id())) 2263 desc_cb->reuse_flag = 1; 2264 else /* This page cannot be reused so discard it */ 2265 put_page(desc_cb->priv); 2266 2267 ring_ptr_move_fw(ring, next_to_clean); 2268 } else { 2269 u64_stats_update_begin(&ring->syncp); 2270 ring->stats.seg_pkt_cnt++; 2271 u64_stats_update_end(&ring->syncp); 2272 2273 pull_len = hns3_nic_get_headlen(va, l234info, 2274 HNS3_RX_HEAD_SIZE); 2275 memcpy(__skb_put(skb, pull_len), va, 2276 ALIGN(pull_len, sizeof(long))); 2277 2278 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb); 2279 ring_ptr_move_fw(ring, next_to_clean); 2280 2281 while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) { 2282 desc = &ring->desc[ring->next_to_clean]; 2283 desc_cb = &ring->desc_cb[ring->next_to_clean]; 2284 bd_base_info = le32_to_cpu(desc->rx.bd_base_info); 2285 hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb); 2286 ring_ptr_move_fw(ring, next_to_clean); 2287 bnum++; 2288 } 2289 } 2290 2291 *out_bnum = bnum; 2292 2293 if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) { 2294 netdev_err(netdev, "no valid bd,%016llx,%016llx\n", 2295 ((u64 *)desc)[0], ((u64 *)desc)[1]); 2296 u64_stats_update_begin(&ring->syncp); 2297 ring->stats.non_vld_descs++; 2298 u64_stats_update_end(&ring->syncp); 2299 2300 dev_kfree_skb_any(skb); 2301 return -EINVAL; 2302 } 2303 2304 if (unlikely((!desc->rx.pkt_len) || 2305 hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) { 2306 netdev_err(netdev, "truncated pkt\n"); 2307 u64_stats_update_begin(&ring->syncp); 2308 ring->stats.err_pkt_len++; 2309 u64_stats_update_end(&ring->syncp); 2310 2311 dev_kfree_skb_any(skb); 2312 return -EFAULT; 2313 } 2314 2315 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) { 2316 netdev_err(netdev, "L2 error pkt\n"); 2317 u64_stats_update_begin(&ring->syncp); 2318 ring->stats.l2_err++; 2319 u64_stats_update_end(&ring->syncp); 2320 2321 dev_kfree_skb_any(skb); 2322 return -EFAULT; 2323 } 2324 2325 u64_stats_update_begin(&ring->syncp); 2326 ring->stats.rx_pkts++; 2327 ring->stats.rx_bytes += skb->len; 2328 u64_stats_update_end(&ring->syncp); 2329 2330 ring->tqp_vector->rx_group.total_bytes += skb->len; 2331 2332 hns3_rx_checksum(ring, skb, desc); 2333 return 0; 2334 } 2335 2336 int hns3_clean_rx_ring( 2337 struct hns3_enet_ring *ring, int budget, 2338 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *)) 2339 { 2340 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16 2341 struct net_device *netdev = ring->tqp->handle->kinfo.netdev; 2342 int recv_pkts, recv_bds, clean_count, err; 2343 int unused_count = hns3_desc_unused(ring); 2344 struct sk_buff *skb = NULL; 2345 int num, bnum = 0; 2346 2347 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG); 2348 rmb(); /* Make sure num taken effect before the other data is touched */ 2349 2350 recv_pkts = 0, recv_bds = 0, clean_count = 0; 2351 num -= unused_count; 2352 2353 while (recv_pkts < budget && recv_bds < num) { 2354 /* Reuse or realloc buffers */ 2355 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) { 2356 hns3_nic_alloc_rx_buffers(ring, 2357 clean_count + unused_count); 2358 clean_count = 0; 2359 unused_count = hns3_desc_unused(ring); 2360 } 2361 2362 /* Poll one pkt */ 2363 err = hns3_handle_rx_bd(ring, &skb, &bnum); 2364 if (unlikely(!skb)) /* This fault cannot be repaired */ 2365 goto out; 2366 2367 recv_bds += bnum; 2368 clean_count += bnum; 2369 if (unlikely(err)) { /* Do jump the err */ 2370 recv_pkts++; 2371 continue; 2372 } 2373 2374 /* Do update ip stack process */ 2375 skb->protocol = eth_type_trans(skb, netdev); 2376 rx_fn(ring, skb); 2377 2378 recv_pkts++; 2379 } 2380 2381 out: 2382 /* Make all data has been write before submit */ 2383 if (clean_count + unused_count > 0) 2384 hns3_nic_alloc_rx_buffers(ring, 2385 clean_count + unused_count); 2386 2387 return recv_pkts; 2388 } 2389 2390 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group) 2391 { 2392 struct hns3_enet_tqp_vector *tqp_vector = 2393 ring_group->ring->tqp_vector; 2394 enum hns3_flow_level_range new_flow_level; 2395 int packets_per_msecs; 2396 int bytes_per_msecs; 2397 u32 time_passed_ms; 2398 u16 new_int_gl; 2399 2400 if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies) 2401 return false; 2402 2403 if (ring_group->total_packets == 0) { 2404 ring_group->coal.int_gl = HNS3_INT_GL_50K; 2405 ring_group->coal.flow_level = HNS3_FLOW_LOW; 2406 return true; 2407 } 2408 2409 /* Simple throttlerate management 2410 * 0-10MB/s lower (50000 ints/s) 2411 * 10-20MB/s middle (20000 ints/s) 2412 * 20-1249MB/s high (18000 ints/s) 2413 * > 40000pps ultra (8000 ints/s) 2414 */ 2415 new_flow_level = ring_group->coal.flow_level; 2416 new_int_gl = ring_group->coal.int_gl; 2417 time_passed_ms = 2418 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies); 2419 2420 if (!time_passed_ms) 2421 return false; 2422 2423 do_div(ring_group->total_packets, time_passed_ms); 2424 packets_per_msecs = ring_group->total_packets; 2425 2426 do_div(ring_group->total_bytes, time_passed_ms); 2427 bytes_per_msecs = ring_group->total_bytes; 2428 2429 #define HNS3_RX_LOW_BYTE_RATE 10000 2430 #define HNS3_RX_MID_BYTE_RATE 20000 2431 2432 switch (new_flow_level) { 2433 case HNS3_FLOW_LOW: 2434 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE) 2435 new_flow_level = HNS3_FLOW_MID; 2436 break; 2437 case HNS3_FLOW_MID: 2438 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE) 2439 new_flow_level = HNS3_FLOW_HIGH; 2440 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE) 2441 new_flow_level = HNS3_FLOW_LOW; 2442 break; 2443 case HNS3_FLOW_HIGH: 2444 case HNS3_FLOW_ULTRA: 2445 default: 2446 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE) 2447 new_flow_level = HNS3_FLOW_MID; 2448 break; 2449 } 2450 2451 #define HNS3_RX_ULTRA_PACKET_RATE 40 2452 2453 if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE && 2454 &tqp_vector->rx_group == ring_group) 2455 new_flow_level = HNS3_FLOW_ULTRA; 2456 2457 switch (new_flow_level) { 2458 case HNS3_FLOW_LOW: 2459 new_int_gl = HNS3_INT_GL_50K; 2460 break; 2461 case HNS3_FLOW_MID: 2462 new_int_gl = HNS3_INT_GL_20K; 2463 break; 2464 case HNS3_FLOW_HIGH: 2465 new_int_gl = HNS3_INT_GL_18K; 2466 break; 2467 case HNS3_FLOW_ULTRA: 2468 new_int_gl = HNS3_INT_GL_8K; 2469 break; 2470 default: 2471 break; 2472 } 2473 2474 ring_group->total_bytes = 0; 2475 ring_group->total_packets = 0; 2476 ring_group->coal.flow_level = new_flow_level; 2477 if (new_int_gl != ring_group->coal.int_gl) { 2478 ring_group->coal.int_gl = new_int_gl; 2479 return true; 2480 } 2481 return false; 2482 } 2483 2484 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector) 2485 { 2486 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group; 2487 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group; 2488 bool rx_update, tx_update; 2489 2490 if (tqp_vector->int_adapt_down > 0) { 2491 tqp_vector->int_adapt_down--; 2492 return; 2493 } 2494 2495 if (rx_group->coal.gl_adapt_enable) { 2496 rx_update = hns3_get_new_int_gl(rx_group); 2497 if (rx_update) 2498 hns3_set_vector_coalesce_rx_gl(tqp_vector, 2499 rx_group->coal.int_gl); 2500 } 2501 2502 if (tx_group->coal.gl_adapt_enable) { 2503 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group); 2504 if (tx_update) 2505 hns3_set_vector_coalesce_tx_gl(tqp_vector, 2506 tx_group->coal.int_gl); 2507 } 2508 2509 tqp_vector->last_jiffies = jiffies; 2510 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START; 2511 } 2512 2513 static int hns3_nic_common_poll(struct napi_struct *napi, int budget) 2514 { 2515 struct hns3_enet_ring *ring; 2516 int rx_pkt_total = 0; 2517 2518 struct hns3_enet_tqp_vector *tqp_vector = 2519 container_of(napi, struct hns3_enet_tqp_vector, napi); 2520 bool clean_complete = true; 2521 int rx_budget; 2522 2523 /* Since the actual Tx work is minimal, we can give the Tx a larger 2524 * budget and be more aggressive about cleaning up the Tx descriptors. 2525 */ 2526 hns3_for_each_ring(ring, tqp_vector->tx_group) { 2527 if (!hns3_clean_tx_ring(ring, budget)) 2528 clean_complete = false; 2529 } 2530 2531 /* make sure rx ring budget not smaller than 1 */ 2532 rx_budget = max(budget / tqp_vector->num_tqps, 1); 2533 2534 hns3_for_each_ring(ring, tqp_vector->rx_group) { 2535 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget, 2536 hns3_rx_skb); 2537 2538 if (rx_cleaned >= rx_budget) 2539 clean_complete = false; 2540 2541 rx_pkt_total += rx_cleaned; 2542 } 2543 2544 tqp_vector->rx_group.total_packets += rx_pkt_total; 2545 2546 if (!clean_complete) 2547 return budget; 2548 2549 napi_complete(napi); 2550 hns3_update_new_int_gl(tqp_vector); 2551 hns3_mask_vector_irq(tqp_vector, 1); 2552 2553 return rx_pkt_total; 2554 } 2555 2556 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector, 2557 struct hnae3_ring_chain_node *head) 2558 { 2559 struct pci_dev *pdev = tqp_vector->handle->pdev; 2560 struct hnae3_ring_chain_node *cur_chain = head; 2561 struct hnae3_ring_chain_node *chain; 2562 struct hns3_enet_ring *tx_ring; 2563 struct hns3_enet_ring *rx_ring; 2564 2565 tx_ring = tqp_vector->tx_group.ring; 2566 if (tx_ring) { 2567 cur_chain->tqp_index = tx_ring->tqp->tqp_index; 2568 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B, 2569 HNAE3_RING_TYPE_TX); 2570 hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M, 2571 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX); 2572 2573 cur_chain->next = NULL; 2574 2575 while (tx_ring->next) { 2576 tx_ring = tx_ring->next; 2577 2578 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), 2579 GFP_KERNEL); 2580 if (!chain) 2581 return -ENOMEM; 2582 2583 cur_chain->next = chain; 2584 chain->tqp_index = tx_ring->tqp->tqp_index; 2585 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B, 2586 HNAE3_RING_TYPE_TX); 2587 hnae_set_field(chain->int_gl_idx, 2588 HNAE3_RING_GL_IDX_M, 2589 HNAE3_RING_GL_IDX_S, 2590 HNAE3_RING_GL_TX); 2591 2592 cur_chain = chain; 2593 } 2594 } 2595 2596 rx_ring = tqp_vector->rx_group.ring; 2597 if (!tx_ring && rx_ring) { 2598 cur_chain->next = NULL; 2599 cur_chain->tqp_index = rx_ring->tqp->tqp_index; 2600 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B, 2601 HNAE3_RING_TYPE_RX); 2602 hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M, 2603 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX); 2604 2605 rx_ring = rx_ring->next; 2606 } 2607 2608 while (rx_ring) { 2609 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL); 2610 if (!chain) 2611 return -ENOMEM; 2612 2613 cur_chain->next = chain; 2614 chain->tqp_index = rx_ring->tqp->tqp_index; 2615 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B, 2616 HNAE3_RING_TYPE_RX); 2617 hnae_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M, 2618 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX); 2619 2620 cur_chain = chain; 2621 2622 rx_ring = rx_ring->next; 2623 } 2624 2625 return 0; 2626 } 2627 2628 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector, 2629 struct hnae3_ring_chain_node *head) 2630 { 2631 struct pci_dev *pdev = tqp_vector->handle->pdev; 2632 struct hnae3_ring_chain_node *chain_tmp, *chain; 2633 2634 chain = head->next; 2635 2636 while (chain) { 2637 chain_tmp = chain->next; 2638 devm_kfree(&pdev->dev, chain); 2639 chain = chain_tmp; 2640 } 2641 } 2642 2643 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group, 2644 struct hns3_enet_ring *ring) 2645 { 2646 ring->next = group->ring; 2647 group->ring = ring; 2648 2649 group->count++; 2650 } 2651 2652 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) 2653 { 2654 struct hnae3_ring_chain_node vector_ring_chain; 2655 struct hnae3_handle *h = priv->ae_handle; 2656 struct hns3_enet_tqp_vector *tqp_vector; 2657 int ret = 0; 2658 u16 i; 2659 2660 for (i = 0; i < priv->vector_num; i++) { 2661 tqp_vector = &priv->tqp_vector[i]; 2662 hns3_vector_gl_rl_init_hw(tqp_vector, priv); 2663 tqp_vector->num_tqps = 0; 2664 } 2665 2666 for (i = 0; i < h->kinfo.num_tqps; i++) { 2667 u16 vector_i = i % priv->vector_num; 2668 u16 tqp_num = h->kinfo.num_tqps; 2669 2670 tqp_vector = &priv->tqp_vector[vector_i]; 2671 2672 hns3_add_ring_to_group(&tqp_vector->tx_group, 2673 priv->ring_data[i].ring); 2674 2675 hns3_add_ring_to_group(&tqp_vector->rx_group, 2676 priv->ring_data[i + tqp_num].ring); 2677 2678 priv->ring_data[i].ring->tqp_vector = tqp_vector; 2679 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector; 2680 tqp_vector->num_tqps++; 2681 } 2682 2683 for (i = 0; i < priv->vector_num; i++) { 2684 tqp_vector = &priv->tqp_vector[i]; 2685 2686 tqp_vector->rx_group.total_bytes = 0; 2687 tqp_vector->rx_group.total_packets = 0; 2688 tqp_vector->tx_group.total_bytes = 0; 2689 tqp_vector->tx_group.total_packets = 0; 2690 tqp_vector->handle = h; 2691 2692 ret = hns3_get_vector_ring_chain(tqp_vector, 2693 &vector_ring_chain); 2694 if (ret) 2695 return ret; 2696 2697 ret = h->ae_algo->ops->map_ring_to_vector(h, 2698 tqp_vector->vector_irq, &vector_ring_chain); 2699 2700 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain); 2701 2702 if (ret) 2703 return ret; 2704 2705 netif_napi_add(priv->netdev, &tqp_vector->napi, 2706 hns3_nic_common_poll, NAPI_POLL_WEIGHT); 2707 } 2708 2709 return 0; 2710 } 2711 2712 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv) 2713 { 2714 struct hnae3_handle *h = priv->ae_handle; 2715 struct hns3_enet_tqp_vector *tqp_vector; 2716 struct hnae3_vector_info *vector; 2717 struct pci_dev *pdev = h->pdev; 2718 u16 tqp_num = h->kinfo.num_tqps; 2719 u16 vector_num; 2720 int ret = 0; 2721 u16 i; 2722 2723 /* RSS size, cpu online and vector_num should be the same */ 2724 /* Should consider 2p/4p later */ 2725 vector_num = min_t(u16, num_online_cpus(), tqp_num); 2726 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector), 2727 GFP_KERNEL); 2728 if (!vector) 2729 return -ENOMEM; 2730 2731 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector); 2732 2733 priv->vector_num = vector_num; 2734 priv->tqp_vector = (struct hns3_enet_tqp_vector *) 2735 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector), 2736 GFP_KERNEL); 2737 if (!priv->tqp_vector) { 2738 ret = -ENOMEM; 2739 goto out; 2740 } 2741 2742 for (i = 0; i < priv->vector_num; i++) { 2743 tqp_vector = &priv->tqp_vector[i]; 2744 tqp_vector->idx = i; 2745 tqp_vector->mask_addr = vector[i].io_addr; 2746 tqp_vector->vector_irq = vector[i].vector; 2747 hns3_vector_gl_rl_init(tqp_vector, priv); 2748 } 2749 2750 out: 2751 devm_kfree(&pdev->dev, vector); 2752 return ret; 2753 } 2754 2755 static void hns3_clear_ring_group(struct hns3_enet_ring_group *group) 2756 { 2757 group->ring = NULL; 2758 group->count = 0; 2759 } 2760 2761 static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv) 2762 { 2763 struct hnae3_ring_chain_node vector_ring_chain; 2764 struct hnae3_handle *h = priv->ae_handle; 2765 struct hns3_enet_tqp_vector *tqp_vector; 2766 int i, ret; 2767 2768 for (i = 0; i < priv->vector_num; i++) { 2769 tqp_vector = &priv->tqp_vector[i]; 2770 2771 ret = hns3_get_vector_ring_chain(tqp_vector, 2772 &vector_ring_chain); 2773 if (ret) 2774 return ret; 2775 2776 ret = h->ae_algo->ops->unmap_ring_from_vector(h, 2777 tqp_vector->vector_irq, &vector_ring_chain); 2778 if (ret) 2779 return ret; 2780 2781 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq); 2782 if (ret) 2783 return ret; 2784 2785 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain); 2786 2787 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) { 2788 (void)irq_set_affinity_hint( 2789 priv->tqp_vector[i].vector_irq, 2790 NULL); 2791 free_irq(priv->tqp_vector[i].vector_irq, 2792 &priv->tqp_vector[i]); 2793 } 2794 2795 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED; 2796 hns3_clear_ring_group(&tqp_vector->rx_group); 2797 hns3_clear_ring_group(&tqp_vector->tx_group); 2798 netif_napi_del(&priv->tqp_vector[i].napi); 2799 } 2800 2801 return 0; 2802 } 2803 2804 static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv) 2805 { 2806 struct hnae3_handle *h = priv->ae_handle; 2807 struct pci_dev *pdev = h->pdev; 2808 int i, ret; 2809 2810 for (i = 0; i < priv->vector_num; i++) { 2811 struct hns3_enet_tqp_vector *tqp_vector; 2812 2813 tqp_vector = &priv->tqp_vector[i]; 2814 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq); 2815 if (ret) 2816 return ret; 2817 } 2818 2819 devm_kfree(&pdev->dev, priv->tqp_vector); 2820 return 0; 2821 } 2822 2823 static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv, 2824 int ring_type) 2825 { 2826 struct hns3_nic_ring_data *ring_data = priv->ring_data; 2827 int queue_num = priv->ae_handle->kinfo.num_tqps; 2828 struct pci_dev *pdev = priv->ae_handle->pdev; 2829 struct hns3_enet_ring *ring; 2830 2831 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL); 2832 if (!ring) 2833 return -ENOMEM; 2834 2835 if (ring_type == HNAE3_RING_TYPE_TX) { 2836 ring_data[q->tqp_index].ring = ring; 2837 ring_data[q->tqp_index].queue_index = q->tqp_index; 2838 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET; 2839 } else { 2840 ring_data[q->tqp_index + queue_num].ring = ring; 2841 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index; 2842 ring->io_base = q->io_base; 2843 } 2844 2845 hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type); 2846 2847 ring->tqp = q; 2848 ring->desc = NULL; 2849 ring->desc_cb = NULL; 2850 ring->dev = priv->dev; 2851 ring->desc_dma_addr = 0; 2852 ring->buf_size = q->buf_size; 2853 ring->desc_num = q->desc_num; 2854 ring->next_to_use = 0; 2855 ring->next_to_clean = 0; 2856 2857 return 0; 2858 } 2859 2860 static int hns3_queue_to_ring(struct hnae3_queue *tqp, 2861 struct hns3_nic_priv *priv) 2862 { 2863 int ret; 2864 2865 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX); 2866 if (ret) 2867 return ret; 2868 2869 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX); 2870 if (ret) 2871 return ret; 2872 2873 return 0; 2874 } 2875 2876 static int hns3_get_ring_config(struct hns3_nic_priv *priv) 2877 { 2878 struct hnae3_handle *h = priv->ae_handle; 2879 struct pci_dev *pdev = h->pdev; 2880 int i, ret; 2881 2882 priv->ring_data = devm_kzalloc(&pdev->dev, h->kinfo.num_tqps * 2883 sizeof(*priv->ring_data) * 2, 2884 GFP_KERNEL); 2885 if (!priv->ring_data) 2886 return -ENOMEM; 2887 2888 for (i = 0; i < h->kinfo.num_tqps; i++) { 2889 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv); 2890 if (ret) 2891 goto err; 2892 } 2893 2894 return 0; 2895 err: 2896 devm_kfree(&pdev->dev, priv->ring_data); 2897 return ret; 2898 } 2899 2900 static void hns3_put_ring_config(struct hns3_nic_priv *priv) 2901 { 2902 struct hnae3_handle *h = priv->ae_handle; 2903 int i; 2904 2905 for (i = 0; i < h->kinfo.num_tqps; i++) { 2906 devm_kfree(priv->dev, priv->ring_data[i].ring); 2907 devm_kfree(priv->dev, 2908 priv->ring_data[i + h->kinfo.num_tqps].ring); 2909 } 2910 devm_kfree(priv->dev, priv->ring_data); 2911 } 2912 2913 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring) 2914 { 2915 int ret; 2916 2917 if (ring->desc_num <= 0 || ring->buf_size <= 0) 2918 return -EINVAL; 2919 2920 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]), 2921 GFP_KERNEL); 2922 if (!ring->desc_cb) { 2923 ret = -ENOMEM; 2924 goto out; 2925 } 2926 2927 ret = hns3_alloc_desc(ring); 2928 if (ret) 2929 goto out_with_desc_cb; 2930 2931 if (!HNAE3_IS_TX_RING(ring)) { 2932 ret = hns3_alloc_ring_buffers(ring); 2933 if (ret) 2934 goto out_with_desc; 2935 } 2936 2937 return 0; 2938 2939 out_with_desc: 2940 hns3_free_desc(ring); 2941 out_with_desc_cb: 2942 kfree(ring->desc_cb); 2943 ring->desc_cb = NULL; 2944 out: 2945 return ret; 2946 } 2947 2948 static void hns3_fini_ring(struct hns3_enet_ring *ring) 2949 { 2950 hns3_free_desc(ring); 2951 kfree(ring->desc_cb); 2952 ring->desc_cb = NULL; 2953 ring->next_to_clean = 0; 2954 ring->next_to_use = 0; 2955 } 2956 2957 static int hns3_buf_size2type(u32 buf_size) 2958 { 2959 int bd_size_type; 2960 2961 switch (buf_size) { 2962 case 512: 2963 bd_size_type = HNS3_BD_SIZE_512_TYPE; 2964 break; 2965 case 1024: 2966 bd_size_type = HNS3_BD_SIZE_1024_TYPE; 2967 break; 2968 case 2048: 2969 bd_size_type = HNS3_BD_SIZE_2048_TYPE; 2970 break; 2971 case 4096: 2972 bd_size_type = HNS3_BD_SIZE_4096_TYPE; 2973 break; 2974 default: 2975 bd_size_type = HNS3_BD_SIZE_2048_TYPE; 2976 } 2977 2978 return bd_size_type; 2979 } 2980 2981 static void hns3_init_ring_hw(struct hns3_enet_ring *ring) 2982 { 2983 dma_addr_t dma = ring->desc_dma_addr; 2984 struct hnae3_queue *q = ring->tqp; 2985 2986 if (!HNAE3_IS_TX_RING(ring)) { 2987 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG, 2988 (u32)dma); 2989 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG, 2990 (u32)((dma >> 31) >> 1)); 2991 2992 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG, 2993 hns3_buf_size2type(ring->buf_size)); 2994 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG, 2995 ring->desc_num / 8 - 1); 2996 2997 } else { 2998 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG, 2999 (u32)dma); 3000 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG, 3001 (u32)((dma >> 31) >> 1)); 3002 3003 hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG, 3004 hns3_buf_size2type(ring->buf_size)); 3005 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG, 3006 ring->desc_num / 8 - 1); 3007 } 3008 } 3009 3010 int hns3_init_all_ring(struct hns3_nic_priv *priv) 3011 { 3012 struct hnae3_handle *h = priv->ae_handle; 3013 int ring_num = h->kinfo.num_tqps * 2; 3014 int i, j; 3015 int ret; 3016 3017 for (i = 0; i < ring_num; i++) { 3018 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring); 3019 if (ret) { 3020 dev_err(priv->dev, 3021 "Alloc ring memory fail! ret=%d\n", ret); 3022 goto out_when_alloc_ring_memory; 3023 } 3024 3025 hns3_init_ring_hw(priv->ring_data[i].ring); 3026 3027 u64_stats_init(&priv->ring_data[i].ring->syncp); 3028 } 3029 3030 return 0; 3031 3032 out_when_alloc_ring_memory: 3033 for (j = i - 1; j >= 0; j--) 3034 hns3_fini_ring(priv->ring_data[j].ring); 3035 3036 return -ENOMEM; 3037 } 3038 3039 int hns3_uninit_all_ring(struct hns3_nic_priv *priv) 3040 { 3041 struct hnae3_handle *h = priv->ae_handle; 3042 int i; 3043 3044 for (i = 0; i < h->kinfo.num_tqps; i++) { 3045 if (h->ae_algo->ops->reset_queue) 3046 h->ae_algo->ops->reset_queue(h, i); 3047 3048 hns3_fini_ring(priv->ring_data[i].ring); 3049 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring); 3050 } 3051 return 0; 3052 } 3053 3054 /* Set mac addr if it is configured. or leave it to the AE driver */ 3055 static void hns3_init_mac_addr(struct net_device *netdev) 3056 { 3057 struct hns3_nic_priv *priv = netdev_priv(netdev); 3058 struct hnae3_handle *h = priv->ae_handle; 3059 u8 mac_addr_temp[ETH_ALEN]; 3060 3061 if (h->ae_algo->ops->get_mac_addr) { 3062 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp); 3063 ether_addr_copy(netdev->dev_addr, mac_addr_temp); 3064 } 3065 3066 /* Check if the MAC address is valid, if not get a random one */ 3067 if (!is_valid_ether_addr(netdev->dev_addr)) { 3068 eth_hw_addr_random(netdev); 3069 dev_warn(priv->dev, "using random MAC address %pM\n", 3070 netdev->dev_addr); 3071 } 3072 3073 if (h->ae_algo->ops->set_mac_addr) 3074 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true); 3075 3076 } 3077 3078 static void hns3_nic_set_priv_ops(struct net_device *netdev) 3079 { 3080 struct hns3_nic_priv *priv = netdev_priv(netdev); 3081 3082 if ((netdev->features & NETIF_F_TSO) || 3083 (netdev->features & NETIF_F_TSO6)) { 3084 priv->ops.fill_desc = hns3_fill_desc_tso; 3085 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso; 3086 } else { 3087 priv->ops.fill_desc = hns3_fill_desc; 3088 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx; 3089 } 3090 } 3091 3092 static int hns3_client_init(struct hnae3_handle *handle) 3093 { 3094 struct pci_dev *pdev = handle->pdev; 3095 struct hns3_nic_priv *priv; 3096 struct net_device *netdev; 3097 int ret; 3098 3099 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), 3100 hns3_get_max_available_channels(handle)); 3101 if (!netdev) 3102 return -ENOMEM; 3103 3104 priv = netdev_priv(netdev); 3105 priv->dev = &pdev->dev; 3106 priv->netdev = netdev; 3107 priv->ae_handle = handle; 3108 priv->ae_handle->reset_level = HNAE3_NONE_RESET; 3109 priv->ae_handle->last_reset_time = jiffies; 3110 priv->tx_timeout_count = 0; 3111 3112 handle->kinfo.netdev = netdev; 3113 handle->priv = (void *)priv; 3114 3115 hns3_init_mac_addr(netdev); 3116 3117 hns3_set_default_feature(netdev); 3118 3119 netdev->watchdog_timeo = HNS3_TX_TIMEOUT; 3120 netdev->priv_flags |= IFF_UNICAST_FLT; 3121 netdev->netdev_ops = &hns3_nic_netdev_ops; 3122 SET_NETDEV_DEV(netdev, &pdev->dev); 3123 hns3_ethtool_set_ops(netdev); 3124 hns3_nic_set_priv_ops(netdev); 3125 3126 /* Carrier off reporting is important to ethtool even BEFORE open */ 3127 netif_carrier_off(netdev); 3128 3129 ret = hns3_get_ring_config(priv); 3130 if (ret) { 3131 ret = -ENOMEM; 3132 goto out_get_ring_cfg; 3133 } 3134 3135 ret = hns3_nic_alloc_vector_data(priv); 3136 if (ret) { 3137 ret = -ENOMEM; 3138 goto out_alloc_vector_data; 3139 } 3140 3141 ret = hns3_nic_init_vector_data(priv); 3142 if (ret) { 3143 ret = -ENOMEM; 3144 goto out_init_vector_data; 3145 } 3146 3147 ret = hns3_init_all_ring(priv); 3148 if (ret) { 3149 ret = -ENOMEM; 3150 goto out_init_ring_data; 3151 } 3152 3153 ret = register_netdev(netdev); 3154 if (ret) { 3155 dev_err(priv->dev, "probe register netdev fail!\n"); 3156 goto out_reg_netdev_fail; 3157 } 3158 3159 hns3_dcbnl_setup(handle); 3160 3161 /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */ 3162 netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN); 3163 3164 return ret; 3165 3166 out_reg_netdev_fail: 3167 out_init_ring_data: 3168 (void)hns3_nic_uninit_vector_data(priv); 3169 out_init_vector_data: 3170 hns3_nic_dealloc_vector_data(priv); 3171 out_alloc_vector_data: 3172 priv->ring_data = NULL; 3173 out_get_ring_cfg: 3174 priv->ae_handle = NULL; 3175 free_netdev(netdev); 3176 return ret; 3177 } 3178 3179 static void hns3_client_uninit(struct hnae3_handle *handle, bool reset) 3180 { 3181 struct net_device *netdev = handle->kinfo.netdev; 3182 struct hns3_nic_priv *priv = netdev_priv(netdev); 3183 int ret; 3184 3185 if (netdev->reg_state != NETREG_UNINITIALIZED) 3186 unregister_netdev(netdev); 3187 3188 ret = hns3_nic_uninit_vector_data(priv); 3189 if (ret) 3190 netdev_err(netdev, "uninit vector error\n"); 3191 3192 ret = hns3_nic_dealloc_vector_data(priv); 3193 if (ret) 3194 netdev_err(netdev, "dealloc vector error\n"); 3195 3196 ret = hns3_uninit_all_ring(priv); 3197 if (ret) 3198 netdev_err(netdev, "uninit ring error\n"); 3199 3200 hns3_put_ring_config(priv); 3201 3202 priv->ring_data = NULL; 3203 3204 free_netdev(netdev); 3205 } 3206 3207 static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup) 3208 { 3209 struct net_device *netdev = handle->kinfo.netdev; 3210 3211 if (!netdev) 3212 return; 3213 3214 if (linkup) { 3215 netif_carrier_on(netdev); 3216 netif_tx_wake_all_queues(netdev); 3217 netdev_info(netdev, "link up\n"); 3218 } else { 3219 netif_carrier_off(netdev); 3220 netif_tx_stop_all_queues(netdev); 3221 netdev_info(netdev, "link down\n"); 3222 } 3223 } 3224 3225 static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc) 3226 { 3227 struct hnae3_knic_private_info *kinfo = &handle->kinfo; 3228 struct net_device *ndev = kinfo->netdev; 3229 bool if_running; 3230 int ret; 3231 u8 i; 3232 3233 if (tc > HNAE3_MAX_TC) 3234 return -EINVAL; 3235 3236 if (!ndev) 3237 return -ENODEV; 3238 3239 if_running = netif_running(ndev); 3240 3241 ret = netdev_set_num_tc(ndev, tc); 3242 if (ret) 3243 return ret; 3244 3245 if (if_running) { 3246 (void)hns3_nic_net_stop(ndev); 3247 msleep(100); 3248 } 3249 3250 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ? 3251 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP; 3252 if (ret) 3253 goto err_out; 3254 3255 if (tc <= 1) { 3256 netdev_reset_tc(ndev); 3257 goto out; 3258 } 3259 3260 for (i = 0; i < HNAE3_MAX_TC; i++) { 3261 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i]; 3262 3263 if (tc_info->enable) 3264 netdev_set_tc_queue(ndev, 3265 tc_info->tc, 3266 tc_info->tqp_count, 3267 tc_info->tqp_offset); 3268 } 3269 3270 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) { 3271 netdev_set_prio_tc_map(ndev, i, 3272 kinfo->prio_tc[i]); 3273 } 3274 3275 out: 3276 ret = hns3_nic_set_real_num_queue(ndev); 3277 3278 err_out: 3279 if (if_running) 3280 (void)hns3_nic_net_open(ndev); 3281 3282 return ret; 3283 } 3284 3285 static void hns3_recover_hw_addr(struct net_device *ndev) 3286 { 3287 struct netdev_hw_addr_list *list; 3288 struct netdev_hw_addr *ha, *tmp; 3289 3290 /* go through and sync uc_addr entries to the device */ 3291 list = &ndev->uc; 3292 list_for_each_entry_safe(ha, tmp, &list->list, list) 3293 hns3_nic_uc_sync(ndev, ha->addr); 3294 3295 /* go through and sync mc_addr entries to the device */ 3296 list = &ndev->mc; 3297 list_for_each_entry_safe(ha, tmp, &list->list, list) 3298 hns3_nic_mc_sync(ndev, ha->addr); 3299 } 3300 3301 static void hns3_drop_skb_data(struct hns3_enet_ring *ring, struct sk_buff *skb) 3302 { 3303 dev_kfree_skb_any(skb); 3304 } 3305 3306 static void hns3_clear_all_ring(struct hnae3_handle *h) 3307 { 3308 struct net_device *ndev = h->kinfo.netdev; 3309 struct hns3_nic_priv *priv = netdev_priv(ndev); 3310 u32 i; 3311 3312 for (i = 0; i < h->kinfo.num_tqps; i++) { 3313 struct netdev_queue *dev_queue; 3314 struct hns3_enet_ring *ring; 3315 3316 ring = priv->ring_data[i].ring; 3317 hns3_clean_tx_ring(ring, ring->desc_num); 3318 dev_queue = netdev_get_tx_queue(ndev, 3319 priv->ring_data[i].queue_index); 3320 netdev_tx_reset_queue(dev_queue); 3321 3322 ring = priv->ring_data[i + h->kinfo.num_tqps].ring; 3323 hns3_clean_rx_ring(ring, ring->desc_num, hns3_drop_skb_data); 3324 } 3325 } 3326 3327 static int hns3_reset_notify_down_enet(struct hnae3_handle *handle) 3328 { 3329 struct hnae3_knic_private_info *kinfo = &handle->kinfo; 3330 struct net_device *ndev = kinfo->netdev; 3331 3332 if (!netif_running(ndev)) 3333 return -EIO; 3334 3335 return hns3_nic_net_stop(ndev); 3336 } 3337 3338 static int hns3_reset_notify_up_enet(struct hnae3_handle *handle) 3339 { 3340 struct hnae3_knic_private_info *kinfo = &handle->kinfo; 3341 int ret = 0; 3342 3343 if (netif_running(kinfo->netdev)) { 3344 ret = hns3_nic_net_up(kinfo->netdev); 3345 if (ret) { 3346 netdev_err(kinfo->netdev, 3347 "hns net up fail, ret=%d!\n", ret); 3348 return ret; 3349 } 3350 handle->last_reset_time = jiffies; 3351 } 3352 3353 return ret; 3354 } 3355 3356 static int hns3_reset_notify_init_enet(struct hnae3_handle *handle) 3357 { 3358 struct net_device *netdev = handle->kinfo.netdev; 3359 struct hns3_nic_priv *priv = netdev_priv(netdev); 3360 int ret; 3361 3362 hns3_init_mac_addr(netdev); 3363 hns3_nic_set_rx_mode(netdev); 3364 hns3_recover_hw_addr(netdev); 3365 3366 /* Hardware table is only clear when pf resets */ 3367 if (!(handle->flags & HNAE3_SUPPORT_VF)) 3368 hns3_restore_vlan(netdev); 3369 3370 /* Carrier off reporting is important to ethtool even BEFORE open */ 3371 netif_carrier_off(netdev); 3372 3373 ret = hns3_get_ring_config(priv); 3374 if (ret) 3375 return ret; 3376 3377 ret = hns3_nic_init_vector_data(priv); 3378 if (ret) 3379 return ret; 3380 3381 ret = hns3_init_all_ring(priv); 3382 if (ret) { 3383 hns3_nic_uninit_vector_data(priv); 3384 priv->ring_data = NULL; 3385 } 3386 3387 return ret; 3388 } 3389 3390 static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) 3391 { 3392 struct net_device *netdev = handle->kinfo.netdev; 3393 struct hns3_nic_priv *priv = netdev_priv(netdev); 3394 int ret; 3395 3396 hns3_clear_all_ring(handle); 3397 3398 ret = hns3_nic_uninit_vector_data(priv); 3399 if (ret) { 3400 netdev_err(netdev, "uninit vector error\n"); 3401 return ret; 3402 } 3403 3404 ret = hns3_uninit_all_ring(priv); 3405 if (ret) 3406 netdev_err(netdev, "uninit ring error\n"); 3407 3408 hns3_put_ring_config(priv); 3409 3410 priv->ring_data = NULL; 3411 3412 return ret; 3413 } 3414 3415 static int hns3_reset_notify(struct hnae3_handle *handle, 3416 enum hnae3_reset_notify_type type) 3417 { 3418 int ret = 0; 3419 3420 switch (type) { 3421 case HNAE3_UP_CLIENT: 3422 ret = hns3_reset_notify_up_enet(handle); 3423 break; 3424 case HNAE3_DOWN_CLIENT: 3425 ret = hns3_reset_notify_down_enet(handle); 3426 break; 3427 case HNAE3_INIT_CLIENT: 3428 ret = hns3_reset_notify_init_enet(handle); 3429 break; 3430 case HNAE3_UNINIT_CLIENT: 3431 ret = hns3_reset_notify_uninit_enet(handle); 3432 break; 3433 default: 3434 break; 3435 } 3436 3437 return ret; 3438 } 3439 3440 static void hns3_restore_coal(struct hns3_nic_priv *priv, 3441 struct hns3_enet_coalesce *tx, 3442 struct hns3_enet_coalesce *rx) 3443 { 3444 u16 vector_num = priv->vector_num; 3445 int i; 3446 3447 for (i = 0; i < vector_num; i++) { 3448 memcpy(&priv->tqp_vector[i].tx_group.coal, tx, 3449 sizeof(struct hns3_enet_coalesce)); 3450 memcpy(&priv->tqp_vector[i].rx_group.coal, rx, 3451 sizeof(struct hns3_enet_coalesce)); 3452 } 3453 } 3454 3455 static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num, 3456 struct hns3_enet_coalesce *tx, 3457 struct hns3_enet_coalesce *rx) 3458 { 3459 struct hns3_nic_priv *priv = netdev_priv(netdev); 3460 struct hnae3_handle *h = hns3_get_handle(netdev); 3461 int ret; 3462 3463 ret = h->ae_algo->ops->set_channels(h, new_tqp_num); 3464 if (ret) 3465 return ret; 3466 3467 ret = hns3_get_ring_config(priv); 3468 if (ret) 3469 return ret; 3470 3471 ret = hns3_nic_alloc_vector_data(priv); 3472 if (ret) 3473 goto err_alloc_vector; 3474 3475 hns3_restore_coal(priv, tx, rx); 3476 3477 ret = hns3_nic_init_vector_data(priv); 3478 if (ret) 3479 goto err_uninit_vector; 3480 3481 ret = hns3_init_all_ring(priv); 3482 if (ret) 3483 goto err_put_ring; 3484 3485 return 0; 3486 3487 err_put_ring: 3488 hns3_put_ring_config(priv); 3489 err_uninit_vector: 3490 hns3_nic_uninit_vector_data(priv); 3491 err_alloc_vector: 3492 hns3_nic_dealloc_vector_data(priv); 3493 return ret; 3494 } 3495 3496 static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num) 3497 { 3498 return (new_tqp_num / num_tc) * num_tc; 3499 } 3500 3501 int hns3_set_channels(struct net_device *netdev, 3502 struct ethtool_channels *ch) 3503 { 3504 struct hns3_nic_priv *priv = netdev_priv(netdev); 3505 struct hnae3_handle *h = hns3_get_handle(netdev); 3506 struct hnae3_knic_private_info *kinfo = &h->kinfo; 3507 struct hns3_enet_coalesce tx_coal, rx_coal; 3508 bool if_running = netif_running(netdev); 3509 u32 new_tqp_num = ch->combined_count; 3510 u16 org_tqp_num; 3511 int ret; 3512 3513 if (ch->rx_count || ch->tx_count) 3514 return -EINVAL; 3515 3516 if (new_tqp_num > hns3_get_max_available_channels(h) || 3517 new_tqp_num < kinfo->num_tc) { 3518 dev_err(&netdev->dev, 3519 "Change tqps fail, the tqp range is from %d to %d", 3520 kinfo->num_tc, 3521 hns3_get_max_available_channels(h)); 3522 return -EINVAL; 3523 } 3524 3525 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num); 3526 if (kinfo->num_tqps == new_tqp_num) 3527 return 0; 3528 3529 if (if_running) 3530 hns3_nic_net_stop(netdev); 3531 3532 hns3_clear_all_ring(h); 3533 3534 ret = hns3_nic_uninit_vector_data(priv); 3535 if (ret) { 3536 dev_err(&netdev->dev, 3537 "Unbind vector with tqp fail, nothing is changed"); 3538 goto open_netdev; 3539 } 3540 3541 /* Changing the tqp num may also change the vector num, 3542 * ethtool only support setting and querying one coal 3543 * configuation for now, so save the vector 0' coal 3544 * configuation here in order to restore it. 3545 */ 3546 memcpy(&tx_coal, &priv->tqp_vector[0].tx_group.coal, 3547 sizeof(struct hns3_enet_coalesce)); 3548 memcpy(&rx_coal, &priv->tqp_vector[0].rx_group.coal, 3549 sizeof(struct hns3_enet_coalesce)); 3550 3551 hns3_nic_dealloc_vector_data(priv); 3552 3553 hns3_uninit_all_ring(priv); 3554 hns3_put_ring_config(priv); 3555 3556 org_tqp_num = h->kinfo.num_tqps; 3557 ret = hns3_modify_tqp_num(netdev, new_tqp_num, &tx_coal, &rx_coal); 3558 if (ret) { 3559 ret = hns3_modify_tqp_num(netdev, org_tqp_num, 3560 &tx_coal, &rx_coal); 3561 if (ret) { 3562 /* If revert to old tqp failed, fatal error occurred */ 3563 dev_err(&netdev->dev, 3564 "Revert to old tqp num fail, ret=%d", ret); 3565 return ret; 3566 } 3567 dev_info(&netdev->dev, 3568 "Change tqp num fail, Revert to old tqp num"); 3569 } 3570 3571 open_netdev: 3572 if (if_running) 3573 hns3_nic_net_open(netdev); 3574 3575 return ret; 3576 } 3577 3578 static const struct hnae3_client_ops client_ops = { 3579 .init_instance = hns3_client_init, 3580 .uninit_instance = hns3_client_uninit, 3581 .link_status_change = hns3_link_status_change, 3582 .setup_tc = hns3_client_setup_tc, 3583 .reset_notify = hns3_reset_notify, 3584 }; 3585 3586 /* hns3_init_module - Driver registration routine 3587 * hns3_init_module is the first routine called when the driver is 3588 * loaded. All it does is register with the PCI subsystem. 3589 */ 3590 static int __init hns3_init_module(void) 3591 { 3592 int ret; 3593 3594 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string); 3595 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright); 3596 3597 client.type = HNAE3_CLIENT_KNIC; 3598 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s", 3599 hns3_driver_name); 3600 3601 client.ops = &client_ops; 3602 3603 ret = hnae3_register_client(&client); 3604 if (ret) 3605 return ret; 3606 3607 ret = pci_register_driver(&hns3_driver); 3608 if (ret) 3609 hnae3_unregister_client(&client); 3610 3611 return ret; 3612 } 3613 module_init(hns3_init_module); 3614 3615 /* hns3_exit_module - Driver exit cleanup routine 3616 * hns3_exit_module is called just before the driver is removed 3617 * from memory. 3618 */ 3619 static void __exit hns3_exit_module(void) 3620 { 3621 pci_unregister_driver(&hns3_driver); 3622 hnae3_unregister_client(&client); 3623 } 3624 module_exit(hns3_exit_module); 3625 3626 MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver"); 3627 MODULE_AUTHOR("Huawei Tech. Co., Ltd."); 3628 MODULE_LICENSE("GPL"); 3629 MODULE_ALIAS("pci:hns-nic"); 3630