1 /* 2 * Copyright (c) 2014-2015 Hisilicon Limited. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 */ 9 10 #include <linux/clk.h> 11 #include <linux/cpumask.h> 12 #include <linux/etherdevice.h> 13 #include <linux/if_vlan.h> 14 #include <linux/interrupt.h> 15 #include <linux/io.h> 16 #include <linux/ip.h> 17 #include <linux/ipv6.h> 18 #include <linux/module.h> 19 #include <linux/phy.h> 20 #include <linux/platform_device.h> 21 #include <linux/skbuff.h> 22 23 #include "hnae.h" 24 #include "hns_enet.h" 25 26 #define NIC_MAX_Q_PER_VF 16 27 #define HNS_NIC_TX_TIMEOUT (5 * HZ) 28 29 #define SERVICE_TIMER_HZ (1 * HZ) 30 31 #define NIC_TX_CLEAN_MAX_NUM 256 32 #define NIC_RX_CLEAN_MAX_NUM 64 33 34 #define RCB_IRQ_NOT_INITED 0 35 #define RCB_IRQ_INITED 1 36 37 static void fill_desc(struct hnae_ring *ring, void *priv, 38 int size, dma_addr_t dma, int frag_end, 39 int buf_num, enum hns_desc_type type) 40 { 41 struct hnae_desc *desc = &ring->desc[ring->next_to_use]; 42 struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; 43 struct sk_buff *skb; 44 __be16 protocol; 45 u32 ip_offset; 46 u32 asid_bufnum_pid = 0; 47 u32 flag_ipoffset = 0; 48 49 desc_cb->priv = priv; 50 desc_cb->length = size; 51 desc_cb->dma = dma; 52 desc_cb->type = type; 53 54 desc->addr = cpu_to_le64(dma); 55 desc->tx.send_size = cpu_to_le16((u16)size); 56 57 /*config bd buffer end */ 58 flag_ipoffset |= 1 << HNS_TXD_VLD_B; 59 60 asid_bufnum_pid |= buf_num << HNS_TXD_BUFNUM_S; 61 62 if (type == DESC_TYPE_SKB) { 63 skb = (struct sk_buff *)priv; 64 65 if (skb->ip_summed == CHECKSUM_PARTIAL) { 66 protocol = skb->protocol; 67 ip_offset = ETH_HLEN; 68 69 /*if it is a SW VLAN check the next protocol*/ 70 if (protocol == htons(ETH_P_8021Q)) { 71 ip_offset += VLAN_HLEN; 72 protocol = vlan_get_protocol(skb); 73 skb->protocol = protocol; 74 } 75 76 if (skb->protocol == htons(ETH_P_IP)) { 77 flag_ipoffset |= 1 << HNS_TXD_L3CS_B; 78 /* check for tcp/udp header */ 79 flag_ipoffset |= 1 << HNS_TXD_L4CS_B; 80 81 } else if (skb->protocol == htons(ETH_P_IPV6)) { 82 /* ipv6 has not l3 cs, check for L4 header */ 83 flag_ipoffset |= 1 << HNS_TXD_L4CS_B; 84 } 85 86 flag_ipoffset |= ip_offset << HNS_TXD_IPOFFSET_S; 87 } 88 } 89 90 flag_ipoffset |= frag_end << HNS_TXD_FE_B; 91 92 desc->tx.asid_bufnum_pid = cpu_to_le16(asid_bufnum_pid); 93 desc->tx.flag_ipoffset = cpu_to_le32(flag_ipoffset); 94 95 ring_ptr_move_fw(ring, next_to_use); 96 } 97 98 static void unfill_desc(struct hnae_ring *ring) 99 { 100 ring_ptr_move_bw(ring, next_to_use); 101 } 102 103 int hns_nic_net_xmit_hw(struct net_device *ndev, 104 struct sk_buff *skb, 105 struct hns_nic_ring_data *ring_data) 106 { 107 struct hns_nic_priv *priv = netdev_priv(ndev); 108 struct device *dev = priv->dev; 109 struct hnae_ring *ring = ring_data->ring; 110 struct netdev_queue *dev_queue; 111 struct skb_frag_struct *frag; 112 int buf_num; 113 dma_addr_t dma; 114 int size, next_to_use; 115 int i, j; 116 struct sk_buff *new_skb; 117 118 assert(ring->max_desc_num_per_pkt <= ring->desc_num); 119 120 /* no. of segments (plus a header) */ 121 buf_num = skb_shinfo(skb)->nr_frags + 1; 122 123 if (unlikely(buf_num > ring->max_desc_num_per_pkt)) { 124 if (ring_space(ring) < 1) { 125 ring->stats.tx_busy++; 126 goto out_net_tx_busy; 127 } 128 129 new_skb = skb_copy(skb, GFP_ATOMIC); 130 if (!new_skb) { 131 ring->stats.sw_err_cnt++; 132 netdev_err(ndev, "no memory to xmit!\n"); 133 goto out_err_tx_ok; 134 } 135 136 dev_kfree_skb_any(skb); 137 skb = new_skb; 138 buf_num = 1; 139 assert(skb_shinfo(skb)->nr_frags == 1); 140 } else if (buf_num > ring_space(ring)) { 141 ring->stats.tx_busy++; 142 goto out_net_tx_busy; 143 } 144 next_to_use = ring->next_to_use; 145 146 /* fill the first part */ 147 size = skb_headlen(skb); 148 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); 149 if (dma_mapping_error(dev, dma)) { 150 netdev_err(ndev, "TX head DMA map failed\n"); 151 ring->stats.sw_err_cnt++; 152 goto out_err_tx_ok; 153 } 154 fill_desc(ring, skb, size, dma, buf_num == 1 ? 1 : 0, buf_num, 155 DESC_TYPE_SKB); 156 157 /* fill the fragments */ 158 for (i = 1; i < buf_num; i++) { 159 frag = &skb_shinfo(skb)->frags[i - 1]; 160 size = skb_frag_size(frag); 161 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); 162 if (dma_mapping_error(dev, dma)) { 163 netdev_err(ndev, "TX frag(%d) DMA map failed\n", i); 164 ring->stats.sw_err_cnt++; 165 goto out_map_frag_fail; 166 } 167 fill_desc(ring, skb_frag_page(frag), size, dma, 168 buf_num - 1 == i ? 1 : 0, buf_num, DESC_TYPE_PAGE); 169 } 170 171 /*complete translate all packets*/ 172 dev_queue = netdev_get_tx_queue(ndev, skb->queue_mapping); 173 netdev_tx_sent_queue(dev_queue, skb->len); 174 175 wmb(); /* commit all data before submit */ 176 assert(skb->queue_mapping < priv->ae_handle->q_num); 177 hnae_queue_xmit(priv->ae_handle->qs[skb->queue_mapping], buf_num); 178 ring->stats.tx_pkts++; 179 ring->stats.tx_bytes += skb->len; 180 181 return NETDEV_TX_OK; 182 183 out_map_frag_fail: 184 185 for (j = i - 1; j > 0; j--) { 186 unfill_desc(ring); 187 next_to_use = ring->next_to_use; 188 dma_unmap_page(dev, ring->desc_cb[next_to_use].dma, 189 ring->desc_cb[next_to_use].length, 190 DMA_TO_DEVICE); 191 } 192 193 unfill_desc(ring); 194 next_to_use = ring->next_to_use; 195 dma_unmap_single(dev, ring->desc_cb[next_to_use].dma, 196 ring->desc_cb[next_to_use].length, DMA_TO_DEVICE); 197 198 out_err_tx_ok: 199 200 dev_kfree_skb_any(skb); 201 return NETDEV_TX_OK; 202 203 out_net_tx_busy: 204 205 netif_stop_subqueue(ndev, skb->queue_mapping); 206 207 /* Herbert's original patch had: 208 * smp_mb__after_netif_stop_queue(); 209 * but since that doesn't exist yet, just open code it. 210 */ 211 smp_mb(); 212 return NETDEV_TX_BUSY; 213 } 214 215 /** 216 * hns_nic_get_headlen - determine size of header for RSC/LRO/GRO/FCOE 217 * @data: pointer to the start of the headers 218 * @max: total length of section to find headers in 219 * 220 * This function is meant to determine the length of headers that will 221 * be recognized by hardware for LRO, GRO, and RSC offloads. The main 222 * motivation of doing this is to only perform one pull for IPv4 TCP 223 * packets so that we can do basic things like calculating the gso_size 224 * based on the average data per packet. 225 **/ 226 static unsigned int hns_nic_get_headlen(unsigned char *data, u32 flag, 227 unsigned int max_size) 228 { 229 unsigned char *network; 230 u8 hlen; 231 232 /* this should never happen, but better safe than sorry */ 233 if (max_size < ETH_HLEN) 234 return max_size; 235 236 /* initialize network frame pointer */ 237 network = data; 238 239 /* set first protocol and move network header forward */ 240 network += ETH_HLEN; 241 242 /* handle any vlan tag if present */ 243 if (hnae_get_field(flag, HNS_RXD_VLAN_M, HNS_RXD_VLAN_S) 244 == HNS_RX_FLAG_VLAN_PRESENT) { 245 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN)) 246 return max_size; 247 248 network += VLAN_HLEN; 249 } 250 251 /* handle L3 protocols */ 252 if (hnae_get_field(flag, HNS_RXD_L3ID_M, HNS_RXD_L3ID_S) 253 == HNS_RX_FLAG_L3ID_IPV4) { 254 if ((typeof(max_size))(network - data) > 255 (max_size - sizeof(struct iphdr))) 256 return max_size; 257 258 /* access ihl as a u8 to avoid unaligned access on ia64 */ 259 hlen = (network[0] & 0x0F) << 2; 260 261 /* verify hlen meets minimum size requirements */ 262 if (hlen < sizeof(struct iphdr)) 263 return network - data; 264 265 /* record next protocol if header is present */ 266 } else if (hnae_get_field(flag, HNS_RXD_L3ID_M, HNS_RXD_L3ID_S) 267 == HNS_RX_FLAG_L3ID_IPV6) { 268 if ((typeof(max_size))(network - data) > 269 (max_size - sizeof(struct ipv6hdr))) 270 return max_size; 271 272 /* record next protocol */ 273 hlen = sizeof(struct ipv6hdr); 274 } else { 275 return network - data; 276 } 277 278 /* relocate pointer to start of L4 header */ 279 network += hlen; 280 281 /* finally sort out TCP/UDP */ 282 if (hnae_get_field(flag, HNS_RXD_L4ID_M, HNS_RXD_L4ID_S) 283 == HNS_RX_FLAG_L4ID_TCP) { 284 if ((typeof(max_size))(network - data) > 285 (max_size - sizeof(struct tcphdr))) 286 return max_size; 287 288 /* access doff as a u8 to avoid unaligned access on ia64 */ 289 hlen = (network[12] & 0xF0) >> 2; 290 291 /* verify hlen meets minimum size requirements */ 292 if (hlen < sizeof(struct tcphdr)) 293 return network - data; 294 295 network += hlen; 296 } else if (hnae_get_field(flag, HNS_RXD_L4ID_M, HNS_RXD_L4ID_S) 297 == HNS_RX_FLAG_L4ID_UDP) { 298 if ((typeof(max_size))(network - data) > 299 (max_size - sizeof(struct udphdr))) 300 return max_size; 301 302 network += sizeof(struct udphdr); 303 } 304 305 /* If everything has gone correctly network should be the 306 * data section of the packet and will be the end of the header. 307 * If not then it probably represents the end of the last recognized 308 * header. 309 */ 310 if ((typeof(max_size))(network - data) < max_size) 311 return network - data; 312 else 313 return max_size; 314 } 315 316 static void 317 hns_nic_reuse_page(struct hnae_desc_cb *desc_cb, int tsize, int last_offset) 318 { 319 /* avoid re-using remote pages,flag default unreuse */ 320 if (likely(page_to_nid(desc_cb->priv) == numa_node_id())) { 321 /* move offset up to the next cache line */ 322 desc_cb->page_offset += tsize; 323 324 if (desc_cb->page_offset <= last_offset) { 325 desc_cb->reuse_flag = 1; 326 /* bump ref count on page before it is given*/ 327 get_page(desc_cb->priv); 328 } 329 } 330 } 331 332 static int hns_nic_poll_rx_skb(struct hns_nic_ring_data *ring_data, 333 struct sk_buff **out_skb, int *out_bnum) 334 { 335 struct hnae_ring *ring = ring_data->ring; 336 struct net_device *ndev = ring_data->napi.dev; 337 struct sk_buff *skb; 338 struct hnae_desc *desc; 339 struct hnae_desc_cb *desc_cb; 340 unsigned char *va; 341 int bnum, length, size, i, truesize, last_offset; 342 int pull_len; 343 u32 bnum_flag; 344 345 last_offset = hnae_page_size(ring) - hnae_buf_size(ring); 346 desc = &ring->desc[ring->next_to_clean]; 347 desc_cb = &ring->desc_cb[ring->next_to_clean]; 348 length = le16_to_cpu(desc->rx.pkt_len); 349 bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag); 350 bnum = hnae_get_field(bnum_flag, HNS_RXD_BUFNUM_M, HNS_RXD_BUFNUM_S); 351 *out_bnum = bnum; 352 va = (unsigned char *)desc_cb->buf + desc_cb->page_offset; 353 354 skb = *out_skb = napi_alloc_skb(&ring_data->napi, HNS_RX_HEAD_SIZE); 355 if (unlikely(!skb)) { 356 netdev_err(ndev, "alloc rx skb fail\n"); 357 ring->stats.sw_err_cnt++; 358 return -ENOMEM; 359 } 360 361 if (length <= HNS_RX_HEAD_SIZE) { 362 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long))); 363 364 /* we can reuse buffer as-is, just make sure it is local */ 365 if (likely(page_to_nid(desc_cb->priv) == numa_node_id())) 366 desc_cb->reuse_flag = 1; 367 else /* this page cannot be reused so discard it */ 368 put_page(desc_cb->priv); 369 370 ring_ptr_move_fw(ring, next_to_clean); 371 372 if (unlikely(bnum != 1)) { /* check err*/ 373 *out_bnum = 1; 374 goto out_bnum_err; 375 } 376 } else { 377 ring->stats.seg_pkt_cnt++; 378 379 pull_len = hns_nic_get_headlen(va, bnum_flag, HNS_RX_HEAD_SIZE); 380 memcpy(__skb_put(skb, pull_len), va, 381 ALIGN(pull_len, sizeof(long))); 382 383 size = le16_to_cpu(desc->rx.size); 384 truesize = ALIGN(size, L1_CACHE_BYTES); 385 skb_add_rx_frag(skb, 0, desc_cb->priv, 386 desc_cb->page_offset + pull_len, 387 size - pull_len, truesize - pull_len); 388 389 hns_nic_reuse_page(desc_cb, truesize, last_offset); 390 ring_ptr_move_fw(ring, next_to_clean); 391 392 if (unlikely(bnum >= (int)MAX_SKB_FRAGS)) { /* check err*/ 393 *out_bnum = 1; 394 goto out_bnum_err; 395 } 396 for (i = 1; i < bnum; i++) { 397 desc = &ring->desc[ring->next_to_clean]; 398 desc_cb = &ring->desc_cb[ring->next_to_clean]; 399 size = le16_to_cpu(desc->rx.size); 400 truesize = ALIGN(size, L1_CACHE_BYTES); 401 skb_add_rx_frag(skb, i, desc_cb->priv, 402 desc_cb->page_offset, 403 size, truesize); 404 405 hns_nic_reuse_page(desc_cb, truesize, last_offset); 406 ring_ptr_move_fw(ring, next_to_clean); 407 } 408 } 409 410 /* check except process, free skb and jump the desc */ 411 if (unlikely((!bnum) || (bnum > ring->max_desc_num_per_pkt))) { 412 out_bnum_err: 413 *out_bnum = *out_bnum ? *out_bnum : 1; /* ntc moved,cannot 0*/ 414 netdev_err(ndev, "invalid bnum(%d,%d,%d,%d),%016llx,%016llx\n", 415 bnum, ring->max_desc_num_per_pkt, 416 length, (int)MAX_SKB_FRAGS, 417 ((u64 *)desc)[0], ((u64 *)desc)[1]); 418 ring->stats.err_bd_num++; 419 dev_kfree_skb_any(skb); 420 return -EDOM; 421 } 422 423 bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag); 424 425 if (unlikely(!hnae_get_bit(bnum_flag, HNS_RXD_VLD_B))) { 426 netdev_err(ndev, "no valid bd,%016llx,%016llx\n", 427 ((u64 *)desc)[0], ((u64 *)desc)[1]); 428 ring->stats.non_vld_descs++; 429 dev_kfree_skb_any(skb); 430 return -EINVAL; 431 } 432 433 if (unlikely((!desc->rx.pkt_len) || 434 hnae_get_bit(bnum_flag, HNS_RXD_DROP_B))) { 435 ring->stats.err_pkt_len++; 436 dev_kfree_skb_any(skb); 437 return -EFAULT; 438 } 439 440 if (unlikely(hnae_get_bit(bnum_flag, HNS_RXD_L2E_B))) { 441 ring->stats.l2_err++; 442 dev_kfree_skb_any(skb); 443 return -EFAULT; 444 } 445 446 ring->stats.rx_pkts++; 447 ring->stats.rx_bytes += skb->len; 448 449 if (unlikely(hnae_get_bit(bnum_flag, HNS_RXD_L3E_B) || 450 hnae_get_bit(bnum_flag, HNS_RXD_L4E_B))) { 451 ring->stats.l3l4_csum_err++; 452 return 0; 453 } 454 455 skb->ip_summed = CHECKSUM_UNNECESSARY; 456 457 return 0; 458 } 459 460 static void 461 hns_nic_alloc_rx_buffers(struct hns_nic_ring_data *ring_data, int cleand_count) 462 { 463 int i, ret; 464 struct hnae_desc_cb res_cbs; 465 struct hnae_desc_cb *desc_cb; 466 struct hnae_ring *ring = ring_data->ring; 467 struct net_device *ndev = ring_data->napi.dev; 468 469 for (i = 0; i < cleand_count; i++) { 470 desc_cb = &ring->desc_cb[ring->next_to_use]; 471 if (desc_cb->reuse_flag) { 472 ring->stats.reuse_pg_cnt++; 473 hnae_reuse_buffer(ring, ring->next_to_use); 474 } else { 475 ret = hnae_reserve_buffer_map(ring, &res_cbs); 476 if (ret) { 477 ring->stats.sw_err_cnt++; 478 netdev_err(ndev, "hnae reserve buffer map failed.\n"); 479 break; 480 } 481 hnae_replace_buffer(ring, ring->next_to_use, &res_cbs); 482 } 483 484 ring_ptr_move_fw(ring, next_to_use); 485 } 486 487 wmb(); /* make all data has been write before submit */ 488 writel_relaxed(i, ring->io_base + RCB_REG_HEAD); 489 } 490 491 /* return error number for error or number of desc left to take 492 */ 493 static void hns_nic_rx_up_pro(struct hns_nic_ring_data *ring_data, 494 struct sk_buff *skb) 495 { 496 struct net_device *ndev = ring_data->napi.dev; 497 498 skb->protocol = eth_type_trans(skb, ndev); 499 (void)napi_gro_receive(&ring_data->napi, skb); 500 ndev->last_rx = jiffies; 501 } 502 503 static int hns_nic_rx_poll_one(struct hns_nic_ring_data *ring_data, 504 int budget, void *v) 505 { 506 struct hnae_ring *ring = ring_data->ring; 507 struct sk_buff *skb; 508 int num, bnum, ex_num; 509 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16 510 int recv_pkts, recv_bds, clean_count, err; 511 512 num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM); 513 rmb(); /* make sure num taken effect before the other data is touched */ 514 515 recv_pkts = 0, recv_bds = 0, clean_count = 0; 516 recv: 517 while (recv_pkts < budget && recv_bds < num) { 518 /* reuse or realloc buffers*/ 519 if (clean_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) { 520 hns_nic_alloc_rx_buffers(ring_data, clean_count); 521 clean_count = 0; 522 } 523 524 /* poll one pkg*/ 525 err = hns_nic_poll_rx_skb(ring_data, &skb, &bnum); 526 if (unlikely(!skb)) /* this fault cannot be repaired */ 527 break; 528 529 recv_bds += bnum; 530 clean_count += bnum; 531 if (unlikely(err)) { /* do jump the err */ 532 recv_pkts++; 533 continue; 534 } 535 536 /* do update ip stack process*/ 537 ((void (*)(struct hns_nic_ring_data *, struct sk_buff *))v)( 538 ring_data, skb); 539 recv_pkts++; 540 } 541 542 /* make all data has been write before submit */ 543 if (clean_count > 0) { 544 hns_nic_alloc_rx_buffers(ring_data, clean_count); 545 clean_count = 0; 546 } 547 548 if (recv_pkts < budget) { 549 ex_num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM); 550 rmb(); /*complete read rx ring bd number*/ 551 if (ex_num > 0) { 552 num += ex_num; 553 goto recv; 554 } 555 } 556 557 return recv_pkts; 558 } 559 560 static void hns_nic_rx_fini_pro(struct hns_nic_ring_data *ring_data) 561 { 562 struct hnae_ring *ring = ring_data->ring; 563 int num = 0; 564 565 /* for hardware bug fixed */ 566 num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM); 567 568 if (num > 0) { 569 ring_data->ring->q->handle->dev->ops->toggle_ring_irq( 570 ring_data->ring, 1); 571 572 napi_schedule(&ring_data->napi); 573 } 574 } 575 576 static inline void hns_nic_reclaim_one_desc(struct hnae_ring *ring, 577 int *bytes, int *pkts) 578 { 579 struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean]; 580 581 (*pkts) += (desc_cb->type == DESC_TYPE_SKB); 582 (*bytes) += desc_cb->length; 583 /* desc_cb will be cleaned, after hnae_free_buffer_detach*/ 584 hnae_free_buffer_detach(ring, ring->next_to_clean); 585 586 ring_ptr_move_fw(ring, next_to_clean); 587 } 588 589 static int is_valid_clean_head(struct hnae_ring *ring, int h) 590 { 591 int u = ring->next_to_use; 592 int c = ring->next_to_clean; 593 594 if (unlikely(h > ring->desc_num)) 595 return 0; 596 597 assert(u > 0 && u < ring->desc_num); 598 assert(c > 0 && c < ring->desc_num); 599 assert(u != c && h != c); /* must be checked before call this func */ 600 601 return u > c ? (h > c && h <= u) : (h > c || h <= u); 602 } 603 604 /* netif_tx_lock will turn down the performance, set only when necessary */ 605 #ifdef CONFIG_NET_POLL_CONTROLLER 606 #define NETIF_TX_LOCK(ndev) netif_tx_lock(ndev) 607 #define NETIF_TX_UNLOCK(ndev) netif_tx_unlock(ndev) 608 #else 609 #define NETIF_TX_LOCK(ndev) 610 #define NETIF_TX_UNLOCK(ndev) 611 #endif 612 /* reclaim all desc in one budget 613 * return error or number of desc left 614 */ 615 static int hns_nic_tx_poll_one(struct hns_nic_ring_data *ring_data, 616 int budget, void *v) 617 { 618 struct hnae_ring *ring = ring_data->ring; 619 struct net_device *ndev = ring_data->napi.dev; 620 struct netdev_queue *dev_queue; 621 struct hns_nic_priv *priv = netdev_priv(ndev); 622 int head; 623 int bytes, pkts; 624 625 NETIF_TX_LOCK(ndev); 626 627 head = readl_relaxed(ring->io_base + RCB_REG_HEAD); 628 rmb(); /* make sure head is ready before touch any data */ 629 630 if (is_ring_empty(ring) || head == ring->next_to_clean) { 631 NETIF_TX_UNLOCK(ndev); 632 return 0; /* no data to poll */ 633 } 634 635 if (!is_valid_clean_head(ring, head)) { 636 netdev_err(ndev, "wrong head (%d, %d-%d)\n", head, 637 ring->next_to_use, ring->next_to_clean); 638 ring->stats.io_err_cnt++; 639 NETIF_TX_UNLOCK(ndev); 640 return -EIO; 641 } 642 643 bytes = 0; 644 pkts = 0; 645 while (head != ring->next_to_clean) 646 hns_nic_reclaim_one_desc(ring, &bytes, &pkts); 647 648 NETIF_TX_UNLOCK(ndev); 649 650 dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index); 651 netdev_tx_completed_queue(dev_queue, pkts, bytes); 652 653 if (unlikely(pkts && netif_carrier_ok(ndev) && 654 (ring_space(ring) >= ring->max_desc_num_per_pkt * 2))) { 655 /* Make sure that anybody stopping the queue after this 656 * sees the new next_to_clean. 657 */ 658 smp_mb(); 659 if (netif_tx_queue_stopped(dev_queue) && 660 !test_bit(NIC_STATE_DOWN, &priv->state)) { 661 netif_tx_wake_queue(dev_queue); 662 ring->stats.restart_queue++; 663 } 664 } 665 return 0; 666 } 667 668 static void hns_nic_tx_fini_pro(struct hns_nic_ring_data *ring_data) 669 { 670 struct hnae_ring *ring = ring_data->ring; 671 int head = ring->next_to_clean; 672 673 /* for hardware bug fixed */ 674 head = readl_relaxed(ring->io_base + RCB_REG_HEAD); 675 676 if (head != ring->next_to_clean) { 677 ring_data->ring->q->handle->dev->ops->toggle_ring_irq( 678 ring_data->ring, 1); 679 680 napi_schedule(&ring_data->napi); 681 } 682 } 683 684 static void hns_nic_tx_clr_all_bufs(struct hns_nic_ring_data *ring_data) 685 { 686 struct hnae_ring *ring = ring_data->ring; 687 struct net_device *ndev = ring_data->napi.dev; 688 struct netdev_queue *dev_queue; 689 int head; 690 int bytes, pkts; 691 692 NETIF_TX_LOCK(ndev); 693 694 head = ring->next_to_use; /* ntu :soft setted ring position*/ 695 bytes = 0; 696 pkts = 0; 697 while (head != ring->next_to_clean) 698 hns_nic_reclaim_one_desc(ring, &bytes, &pkts); 699 700 NETIF_TX_UNLOCK(ndev); 701 702 dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index); 703 netdev_tx_reset_queue(dev_queue); 704 } 705 706 static int hns_nic_common_poll(struct napi_struct *napi, int budget) 707 { 708 struct hns_nic_ring_data *ring_data = 709 container_of(napi, struct hns_nic_ring_data, napi); 710 int clean_complete = ring_data->poll_one( 711 ring_data, budget, ring_data->ex_process); 712 713 if (clean_complete >= 0 && clean_complete < budget) { 714 napi_complete(napi); 715 ring_data->ring->q->handle->dev->ops->toggle_ring_irq( 716 ring_data->ring, 0); 717 718 ring_data->fini_process(ring_data); 719 } 720 721 return clean_complete; 722 } 723 724 static irqreturn_t hns_irq_handle(int irq, void *dev) 725 { 726 struct hns_nic_ring_data *ring_data = (struct hns_nic_ring_data *)dev; 727 728 ring_data->ring->q->handle->dev->ops->toggle_ring_irq( 729 ring_data->ring, 1); 730 napi_schedule(&ring_data->napi); 731 732 return IRQ_HANDLED; 733 } 734 735 /** 736 *hns_nic_adjust_link - adjust net work mode by the phy stat or new param 737 *@ndev: net device 738 */ 739 static void hns_nic_adjust_link(struct net_device *ndev) 740 { 741 struct hns_nic_priv *priv = netdev_priv(ndev); 742 struct hnae_handle *h = priv->ae_handle; 743 744 h->dev->ops->adjust_link(h, ndev->phydev->speed, ndev->phydev->duplex); 745 } 746 747 /** 748 *hns_nic_init_phy - init phy 749 *@ndev: net device 750 *@h: ae handle 751 * Return 0 on success, negative on failure 752 */ 753 int hns_nic_init_phy(struct net_device *ndev, struct hnae_handle *h) 754 { 755 struct hns_nic_priv *priv = netdev_priv(ndev); 756 struct phy_device *phy_dev = NULL; 757 758 if (!h->phy_node) 759 return 0; 760 761 if (h->phy_if != PHY_INTERFACE_MODE_XGMII) 762 phy_dev = of_phy_connect(ndev, h->phy_node, 763 hns_nic_adjust_link, 0, h->phy_if); 764 else 765 phy_dev = of_phy_attach(ndev, h->phy_node, 0, h->phy_if); 766 767 if (unlikely(!phy_dev) || IS_ERR(phy_dev)) 768 return !phy_dev ? -ENODEV : PTR_ERR(phy_dev); 769 770 phy_dev->supported &= h->if_support; 771 phy_dev->advertising = phy_dev->supported; 772 773 if (h->phy_if == PHY_INTERFACE_MODE_XGMII) 774 phy_dev->autoneg = false; 775 776 priv->phy = phy_dev; 777 778 return 0; 779 } 780 781 static int hns_nic_ring_open(struct net_device *netdev, int idx) 782 { 783 struct hns_nic_priv *priv = netdev_priv(netdev); 784 struct hnae_handle *h = priv->ae_handle; 785 786 napi_enable(&priv->ring_data[idx].napi); 787 788 enable_irq(priv->ring_data[idx].ring->irq); 789 h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 0); 790 791 return 0; 792 } 793 794 static int hns_nic_net_set_mac_address(struct net_device *ndev, void *p) 795 { 796 struct hns_nic_priv *priv = netdev_priv(ndev); 797 struct hnae_handle *h = priv->ae_handle; 798 struct sockaddr *mac_addr = p; 799 int ret; 800 801 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data)) 802 return -EADDRNOTAVAIL; 803 804 ret = h->dev->ops->set_mac_addr(h, mac_addr->sa_data); 805 if (ret) { 806 netdev_err(ndev, "set_mac_address fail, ret=%d!\n", ret); 807 return ret; 808 } 809 810 memcpy(ndev->dev_addr, mac_addr->sa_data, ndev->addr_len); 811 812 return 0; 813 } 814 815 void hns_nic_update_stats(struct net_device *netdev) 816 { 817 struct hns_nic_priv *priv = netdev_priv(netdev); 818 struct hnae_handle *h = priv->ae_handle; 819 820 h->dev->ops->update_stats(h, &netdev->stats); 821 } 822 823 /* set mac addr if it is configed. or leave it to the AE driver */ 824 static void hns_init_mac_addr(struct net_device *ndev) 825 { 826 struct hns_nic_priv *priv = netdev_priv(ndev); 827 struct device_node *node = priv->dev->of_node; 828 const void *mac_addr_temp; 829 830 mac_addr_temp = of_get_mac_address(node); 831 if (mac_addr_temp && is_valid_ether_addr(mac_addr_temp)) { 832 memcpy(ndev->dev_addr, mac_addr_temp, ndev->addr_len); 833 } else { 834 eth_hw_addr_random(ndev); 835 dev_warn(priv->dev, "No valid mac, use random mac %pM", 836 ndev->dev_addr); 837 } 838 } 839 840 static void hns_nic_ring_close(struct net_device *netdev, int idx) 841 { 842 struct hns_nic_priv *priv = netdev_priv(netdev); 843 struct hnae_handle *h = priv->ae_handle; 844 845 h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 1); 846 disable_irq(priv->ring_data[idx].ring->irq); 847 848 napi_disable(&priv->ring_data[idx].napi); 849 } 850 851 static int hns_nic_init_irq(struct hns_nic_priv *priv) 852 { 853 struct hnae_handle *h = priv->ae_handle; 854 struct hns_nic_ring_data *rd; 855 int i; 856 int ret; 857 int cpu; 858 cpumask_t mask; 859 860 for (i = 0; i < h->q_num * 2; i++) { 861 rd = &priv->ring_data[i]; 862 863 if (rd->ring->irq_init_flag == RCB_IRQ_INITED) 864 break; 865 866 snprintf(rd->ring->ring_name, RCB_RING_NAME_LEN, 867 "%s-%s%d", priv->netdev->name, 868 (i < h->q_num ? "tx" : "rx"), rd->queue_index); 869 870 rd->ring->ring_name[RCB_RING_NAME_LEN - 1] = '\0'; 871 872 ret = request_irq(rd->ring->irq, 873 hns_irq_handle, 0, rd->ring->ring_name, rd); 874 if (ret) { 875 netdev_err(priv->netdev, "request irq(%d) fail\n", 876 rd->ring->irq); 877 return ret; 878 } 879 disable_irq(rd->ring->irq); 880 rd->ring->irq_init_flag = RCB_IRQ_INITED; 881 882 /*set cpu affinity*/ 883 if (cpu_online(rd->queue_index)) { 884 cpumask_clear(&mask); 885 cpu = rd->queue_index; 886 cpumask_set_cpu(cpu, &mask); 887 irq_set_affinity_hint(rd->ring->irq, &mask); 888 } 889 } 890 891 return 0; 892 } 893 894 static int hns_nic_net_up(struct net_device *ndev) 895 { 896 struct hns_nic_priv *priv = netdev_priv(ndev); 897 struct hnae_handle *h = priv->ae_handle; 898 int i, j, k; 899 int ret; 900 901 ret = hns_nic_init_irq(priv); 902 if (ret != 0) { 903 netdev_err(ndev, "hns init irq failed! ret=%d\n", ret); 904 return ret; 905 } 906 907 for (i = 0; i < h->q_num * 2; i++) { 908 ret = hns_nic_ring_open(ndev, i); 909 if (ret) 910 goto out_has_some_queues; 911 } 912 913 for (k = 0; k < h->q_num; k++) 914 h->dev->ops->toggle_queue_status(h->qs[k], 1); 915 916 ret = h->dev->ops->set_mac_addr(h, ndev->dev_addr); 917 if (ret) 918 goto out_set_mac_addr_err; 919 920 ret = h->dev->ops->start ? h->dev->ops->start(h) : 0; 921 if (ret) 922 goto out_start_err; 923 924 if (priv->phy) 925 phy_start(priv->phy); 926 927 clear_bit(NIC_STATE_DOWN, &priv->state); 928 (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ); 929 930 return 0; 931 932 out_start_err: 933 netif_stop_queue(ndev); 934 out_set_mac_addr_err: 935 for (k = 0; k < h->q_num; k++) 936 h->dev->ops->toggle_queue_status(h->qs[k], 0); 937 out_has_some_queues: 938 for (j = i - 1; j >= 0; j--) 939 hns_nic_ring_close(ndev, j); 940 941 set_bit(NIC_STATE_DOWN, &priv->state); 942 943 return ret; 944 } 945 946 static void hns_nic_net_down(struct net_device *ndev) 947 { 948 int i; 949 struct hnae_ae_ops *ops; 950 struct hns_nic_priv *priv = netdev_priv(ndev); 951 952 if (test_and_set_bit(NIC_STATE_DOWN, &priv->state)) 953 return; 954 955 (void)del_timer_sync(&priv->service_timer); 956 netif_tx_stop_all_queues(ndev); 957 netif_carrier_off(ndev); 958 netif_tx_disable(ndev); 959 priv->link = 0; 960 961 if (priv->phy) 962 phy_stop(priv->phy); 963 964 ops = priv->ae_handle->dev->ops; 965 966 if (ops->stop) 967 ops->stop(priv->ae_handle); 968 969 netif_tx_stop_all_queues(ndev); 970 971 for (i = priv->ae_handle->q_num - 1; i >= 0; i--) { 972 hns_nic_ring_close(ndev, i); 973 hns_nic_ring_close(ndev, i + priv->ae_handle->q_num); 974 975 /* clean tx buffers*/ 976 hns_nic_tx_clr_all_bufs(priv->ring_data + i); 977 } 978 } 979 980 void hns_nic_net_reset(struct net_device *ndev) 981 { 982 struct hns_nic_priv *priv = netdev_priv(ndev); 983 struct hnae_handle *handle = priv->ae_handle; 984 985 while (test_and_set_bit(NIC_STATE_RESETTING, &priv->state)) 986 usleep_range(1000, 2000); 987 988 (void)hnae_reinit_handle(handle); 989 990 clear_bit(NIC_STATE_RESETTING, &priv->state); 991 } 992 993 void hns_nic_net_reinit(struct net_device *netdev) 994 { 995 struct hns_nic_priv *priv = netdev_priv(netdev); 996 997 priv->netdev->trans_start = jiffies; 998 while (test_and_set_bit(NIC_STATE_REINITING, &priv->state)) 999 usleep_range(1000, 2000); 1000 1001 hns_nic_net_down(netdev); 1002 hns_nic_net_reset(netdev); 1003 (void)hns_nic_net_up(netdev); 1004 clear_bit(NIC_STATE_REINITING, &priv->state); 1005 } 1006 1007 static int hns_nic_net_open(struct net_device *ndev) 1008 { 1009 struct hns_nic_priv *priv = netdev_priv(ndev); 1010 struct hnae_handle *h = priv->ae_handle; 1011 int ret; 1012 1013 if (test_bit(NIC_STATE_TESTING, &priv->state)) 1014 return -EBUSY; 1015 1016 priv->link = 0; 1017 netif_carrier_off(ndev); 1018 1019 ret = netif_set_real_num_tx_queues(ndev, h->q_num); 1020 if (ret < 0) { 1021 netdev_err(ndev, "netif_set_real_num_tx_queues fail, ret=%d!\n", 1022 ret); 1023 return ret; 1024 } 1025 1026 ret = netif_set_real_num_rx_queues(ndev, h->q_num); 1027 if (ret < 0) { 1028 netdev_err(ndev, 1029 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret); 1030 return ret; 1031 } 1032 1033 ret = hns_nic_net_up(ndev); 1034 if (ret) { 1035 netdev_err(ndev, 1036 "hns net up fail, ret=%d!\n", ret); 1037 return ret; 1038 } 1039 1040 return 0; 1041 } 1042 1043 static int hns_nic_net_stop(struct net_device *ndev) 1044 { 1045 hns_nic_net_down(ndev); 1046 1047 return 0; 1048 } 1049 1050 static void hns_tx_timeout_reset(struct hns_nic_priv *priv); 1051 static void hns_nic_net_timeout(struct net_device *ndev) 1052 { 1053 struct hns_nic_priv *priv = netdev_priv(ndev); 1054 1055 hns_tx_timeout_reset(priv); 1056 } 1057 1058 static int hns_nic_do_ioctl(struct net_device *netdev, struct ifreq *ifr, 1059 int cmd) 1060 { 1061 struct hns_nic_priv *priv = netdev_priv(netdev); 1062 struct phy_device *phy_dev = priv->phy; 1063 1064 if (!netif_running(netdev)) 1065 return -EINVAL; 1066 1067 if (!phy_dev) 1068 return -ENOTSUPP; 1069 1070 return phy_mii_ioctl(phy_dev, ifr, cmd); 1071 } 1072 1073 /* use only for netconsole to poll with the device without interrupt */ 1074 #ifdef CONFIG_NET_POLL_CONTROLLER 1075 void hns_nic_poll_controller(struct net_device *ndev) 1076 { 1077 struct hns_nic_priv *priv = netdev_priv(ndev); 1078 unsigned long flags; 1079 int i; 1080 1081 local_irq_save(flags); 1082 for (i = 0; i < priv->ae_handle->q_num * 2; i++) 1083 napi_schedule(&priv->ring_data[i].napi); 1084 local_irq_restore(flags); 1085 } 1086 #endif 1087 1088 static netdev_tx_t hns_nic_net_xmit(struct sk_buff *skb, 1089 struct net_device *ndev) 1090 { 1091 struct hns_nic_priv *priv = netdev_priv(ndev); 1092 int ret; 1093 1094 assert(skb->queue_mapping < ndev->ae_handle->q_num); 1095 ret = hns_nic_net_xmit_hw(ndev, skb, 1096 &tx_ring_data(priv, skb->queue_mapping)); 1097 if (ret == NETDEV_TX_OK) { 1098 ndev->trans_start = jiffies; 1099 ndev->stats.tx_bytes += skb->len; 1100 ndev->stats.tx_packets++; 1101 } 1102 return (netdev_tx_t)ret; 1103 } 1104 1105 static int hns_nic_change_mtu(struct net_device *ndev, int new_mtu) 1106 { 1107 struct hns_nic_priv *priv = netdev_priv(ndev); 1108 struct hnae_handle *h = priv->ae_handle; 1109 int ret; 1110 1111 /* MTU < 68 is an error and causes problems on some kernels */ 1112 if (new_mtu < 68) 1113 return -EINVAL; 1114 1115 if (!h->dev->ops->set_mtu) 1116 return -ENOTSUPP; 1117 1118 if (netif_running(ndev)) { 1119 (void)hns_nic_net_stop(ndev); 1120 msleep(100); 1121 1122 ret = h->dev->ops->set_mtu(h, new_mtu); 1123 if (ret) 1124 netdev_err(ndev, "set mtu fail, return value %d\n", 1125 ret); 1126 1127 if (hns_nic_net_open(ndev)) 1128 netdev_err(ndev, "hns net open fail\n"); 1129 } else { 1130 ret = h->dev->ops->set_mtu(h, new_mtu); 1131 } 1132 1133 if (!ret) 1134 ndev->mtu = new_mtu; 1135 1136 return ret; 1137 } 1138 1139 /** 1140 * nic_set_multicast_list - set mutl mac address 1141 * @netdev: net device 1142 * @p: mac address 1143 * 1144 * return void 1145 */ 1146 void hns_set_multicast_list(struct net_device *ndev) 1147 { 1148 struct hns_nic_priv *priv = netdev_priv(ndev); 1149 struct hnae_handle *h = priv->ae_handle; 1150 struct netdev_hw_addr *ha = NULL; 1151 1152 if (!h) { 1153 netdev_err(ndev, "hnae handle is null\n"); 1154 return; 1155 } 1156 1157 if (h->dev->ops->set_mc_addr) { 1158 netdev_for_each_mc_addr(ha, ndev) 1159 if (h->dev->ops->set_mc_addr(h, ha->addr)) 1160 netdev_err(ndev, "set multicast fail\n"); 1161 } 1162 } 1163 1164 void hns_nic_set_rx_mode(struct net_device *ndev) 1165 { 1166 struct hns_nic_priv *priv = netdev_priv(ndev); 1167 struct hnae_handle *h = priv->ae_handle; 1168 1169 if (h->dev->ops->set_promisc_mode) { 1170 if (ndev->flags & IFF_PROMISC) 1171 h->dev->ops->set_promisc_mode(h, 1); 1172 else 1173 h->dev->ops->set_promisc_mode(h, 0); 1174 } 1175 1176 hns_set_multicast_list(ndev); 1177 } 1178 1179 struct rtnl_link_stats64 *hns_nic_get_stats64(struct net_device *ndev, 1180 struct rtnl_link_stats64 *stats) 1181 { 1182 int idx = 0; 1183 u64 tx_bytes = 0; 1184 u64 rx_bytes = 0; 1185 u64 tx_pkts = 0; 1186 u64 rx_pkts = 0; 1187 struct hns_nic_priv *priv = netdev_priv(ndev); 1188 struct hnae_handle *h = priv->ae_handle; 1189 1190 for (idx = 0; idx < h->q_num; idx++) { 1191 tx_bytes += h->qs[idx]->tx_ring.stats.tx_bytes; 1192 tx_pkts += h->qs[idx]->tx_ring.stats.tx_pkts; 1193 rx_bytes += h->qs[idx]->rx_ring.stats.rx_bytes; 1194 rx_pkts += h->qs[idx]->rx_ring.stats.rx_pkts; 1195 } 1196 1197 stats->tx_bytes = tx_bytes; 1198 stats->tx_packets = tx_pkts; 1199 stats->rx_bytes = rx_bytes; 1200 stats->rx_packets = rx_pkts; 1201 1202 stats->rx_errors = ndev->stats.rx_errors; 1203 stats->multicast = ndev->stats.multicast; 1204 stats->rx_length_errors = ndev->stats.rx_length_errors; 1205 stats->rx_crc_errors = ndev->stats.rx_crc_errors; 1206 stats->rx_missed_errors = ndev->stats.rx_missed_errors; 1207 1208 stats->tx_errors = ndev->stats.tx_errors; 1209 stats->rx_dropped = ndev->stats.rx_dropped; 1210 stats->tx_dropped = ndev->stats.tx_dropped; 1211 stats->collisions = ndev->stats.collisions; 1212 stats->rx_over_errors = ndev->stats.rx_over_errors; 1213 stats->rx_frame_errors = ndev->stats.rx_frame_errors; 1214 stats->rx_fifo_errors = ndev->stats.rx_fifo_errors; 1215 stats->tx_aborted_errors = ndev->stats.tx_aborted_errors; 1216 stats->tx_carrier_errors = ndev->stats.tx_carrier_errors; 1217 stats->tx_fifo_errors = ndev->stats.tx_fifo_errors; 1218 stats->tx_heartbeat_errors = ndev->stats.tx_heartbeat_errors; 1219 stats->tx_window_errors = ndev->stats.tx_window_errors; 1220 stats->rx_compressed = ndev->stats.rx_compressed; 1221 stats->tx_compressed = ndev->stats.tx_compressed; 1222 1223 return stats; 1224 } 1225 1226 static const struct net_device_ops hns_nic_netdev_ops = { 1227 .ndo_open = hns_nic_net_open, 1228 .ndo_stop = hns_nic_net_stop, 1229 .ndo_start_xmit = hns_nic_net_xmit, 1230 .ndo_tx_timeout = hns_nic_net_timeout, 1231 .ndo_set_mac_address = hns_nic_net_set_mac_address, 1232 .ndo_change_mtu = hns_nic_change_mtu, 1233 .ndo_do_ioctl = hns_nic_do_ioctl, 1234 .ndo_get_stats64 = hns_nic_get_stats64, 1235 #ifdef CONFIG_NET_POLL_CONTROLLER 1236 .ndo_poll_controller = hns_nic_poll_controller, 1237 #endif 1238 .ndo_set_rx_mode = hns_nic_set_rx_mode, 1239 }; 1240 1241 static void hns_nic_update_link_status(struct net_device *netdev) 1242 { 1243 struct hns_nic_priv *priv = netdev_priv(netdev); 1244 1245 struct hnae_handle *h = priv->ae_handle; 1246 int state = 1; 1247 1248 if (priv->phy) { 1249 if (!genphy_update_link(priv->phy)) 1250 state = priv->phy->link; 1251 else 1252 state = 0; 1253 } 1254 state = state && h->dev->ops->get_status(h); 1255 1256 if (state != priv->link) { 1257 if (state) { 1258 netif_carrier_on(netdev); 1259 netif_tx_wake_all_queues(netdev); 1260 netdev_info(netdev, "link up\n"); 1261 } else { 1262 netif_carrier_off(netdev); 1263 netdev_info(netdev, "link down\n"); 1264 } 1265 priv->link = state; 1266 } 1267 } 1268 1269 /* for dumping key regs*/ 1270 static void hns_nic_dump(struct hns_nic_priv *priv) 1271 { 1272 struct hnae_handle *h = priv->ae_handle; 1273 struct hnae_ae_ops *ops = h->dev->ops; 1274 u32 *data, reg_num, i; 1275 1276 if (ops->get_regs_len && ops->get_regs) { 1277 reg_num = ops->get_regs_len(priv->ae_handle); 1278 reg_num = (reg_num + 3ul) & ~3ul; 1279 data = kcalloc(reg_num, sizeof(u32), GFP_KERNEL); 1280 if (data) { 1281 ops->get_regs(priv->ae_handle, data); 1282 for (i = 0; i < reg_num; i += 4) 1283 pr_info("0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", 1284 i, data[i], data[i + 1], 1285 data[i + 2], data[i + 3]); 1286 kfree(data); 1287 } 1288 } 1289 1290 for (i = 0; i < h->q_num; i++) { 1291 pr_info("tx_queue%d_next_to_clean:%d\n", 1292 i, h->qs[i]->tx_ring.next_to_clean); 1293 pr_info("tx_queue%d_next_to_use:%d\n", 1294 i, h->qs[i]->tx_ring.next_to_use); 1295 pr_info("rx_queue%d_next_to_clean:%d\n", 1296 i, h->qs[i]->rx_ring.next_to_clean); 1297 pr_info("rx_queue%d_next_to_use:%d\n", 1298 i, h->qs[i]->rx_ring.next_to_use); 1299 } 1300 } 1301 1302 /* for resetting suntask*/ 1303 static void hns_nic_reset_subtask(struct hns_nic_priv *priv) 1304 { 1305 enum hnae_port_type type = priv->ae_handle->port_type; 1306 1307 if (!test_bit(NIC_STATE2_RESET_REQUESTED, &priv->state)) 1308 return; 1309 clear_bit(NIC_STATE2_RESET_REQUESTED, &priv->state); 1310 1311 /* If we're already down, removing or resetting, just bail */ 1312 if (test_bit(NIC_STATE_DOWN, &priv->state) || 1313 test_bit(NIC_STATE_REMOVING, &priv->state) || 1314 test_bit(NIC_STATE_RESETTING, &priv->state)) 1315 return; 1316 1317 hns_nic_dump(priv); 1318 netdev_info(priv->netdev, "Reset %s port\n", 1319 (type == HNAE_PORT_DEBUG ? "debug" : "business")); 1320 1321 rtnl_lock(); 1322 /* put off any impending NetWatchDogTimeout */ 1323 priv->netdev->trans_start = jiffies; 1324 1325 if (type == HNAE_PORT_DEBUG) 1326 hns_nic_net_reinit(priv->netdev); 1327 rtnl_unlock(); 1328 } 1329 1330 /* for doing service complete*/ 1331 static void hns_nic_service_event_complete(struct hns_nic_priv *priv) 1332 { 1333 assert(!test_bit(NIC_STATE_SERVICE_SCHED, &priv->state)); 1334 1335 smp_mb__before_atomic(); 1336 clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state); 1337 } 1338 1339 static void hns_nic_service_task(struct work_struct *work) 1340 { 1341 struct hns_nic_priv *priv 1342 = container_of(work, struct hns_nic_priv, service_task); 1343 struct hnae_handle *h = priv->ae_handle; 1344 1345 hns_nic_update_link_status(priv->netdev); 1346 h->dev->ops->update_led_status(h); 1347 hns_nic_update_stats(priv->netdev); 1348 1349 hns_nic_reset_subtask(priv); 1350 hns_nic_service_event_complete(priv); 1351 } 1352 1353 static void hns_nic_task_schedule(struct hns_nic_priv *priv) 1354 { 1355 if (!test_bit(NIC_STATE_DOWN, &priv->state) && 1356 !test_bit(NIC_STATE_REMOVING, &priv->state) && 1357 !test_and_set_bit(NIC_STATE_SERVICE_SCHED, &priv->state)) 1358 (void)schedule_work(&priv->service_task); 1359 } 1360 1361 static void hns_nic_service_timer(unsigned long data) 1362 { 1363 struct hns_nic_priv *priv = (struct hns_nic_priv *)data; 1364 1365 (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ); 1366 1367 hns_nic_task_schedule(priv); 1368 } 1369 1370 /** 1371 * hns_tx_timeout_reset - initiate reset due to Tx timeout 1372 * @priv: driver private struct 1373 **/ 1374 static void hns_tx_timeout_reset(struct hns_nic_priv *priv) 1375 { 1376 /* Do the reset outside of interrupt context */ 1377 if (!test_bit(NIC_STATE_DOWN, &priv->state)) { 1378 set_bit(NIC_STATE2_RESET_REQUESTED, &priv->state); 1379 netdev_warn(priv->netdev, 1380 "initiating reset due to tx timeout(%llu,0x%lx)\n", 1381 priv->tx_timeout_count, priv->state); 1382 priv->tx_timeout_count++; 1383 hns_nic_task_schedule(priv); 1384 } 1385 } 1386 1387 static int hns_nic_init_ring_data(struct hns_nic_priv *priv) 1388 { 1389 struct hnae_handle *h = priv->ae_handle; 1390 struct hns_nic_ring_data *rd; 1391 int i; 1392 1393 if (h->q_num > NIC_MAX_Q_PER_VF) { 1394 netdev_err(priv->netdev, "too much queue (%d)\n", h->q_num); 1395 return -EINVAL; 1396 } 1397 1398 priv->ring_data = kzalloc(h->q_num * sizeof(*priv->ring_data) * 2, 1399 GFP_KERNEL); 1400 if (!priv->ring_data) 1401 return -ENOMEM; 1402 1403 for (i = 0; i < h->q_num; i++) { 1404 rd = &priv->ring_data[i]; 1405 rd->queue_index = i; 1406 rd->ring = &h->qs[i]->tx_ring; 1407 rd->poll_one = hns_nic_tx_poll_one; 1408 rd->fini_process = hns_nic_tx_fini_pro; 1409 1410 netif_napi_add(priv->netdev, &rd->napi, 1411 hns_nic_common_poll, NIC_TX_CLEAN_MAX_NUM); 1412 rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED; 1413 } 1414 for (i = h->q_num; i < h->q_num * 2; i++) { 1415 rd = &priv->ring_data[i]; 1416 rd->queue_index = i - h->q_num; 1417 rd->ring = &h->qs[i - h->q_num]->rx_ring; 1418 rd->poll_one = hns_nic_rx_poll_one; 1419 rd->ex_process = hns_nic_rx_up_pro; 1420 rd->fini_process = hns_nic_rx_fini_pro; 1421 1422 netif_napi_add(priv->netdev, &rd->napi, 1423 hns_nic_common_poll, NIC_RX_CLEAN_MAX_NUM); 1424 rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED; 1425 } 1426 1427 return 0; 1428 } 1429 1430 static void hns_nic_uninit_ring_data(struct hns_nic_priv *priv) 1431 { 1432 struct hnae_handle *h = priv->ae_handle; 1433 int i; 1434 1435 for (i = 0; i < h->q_num * 2; i++) { 1436 netif_napi_del(&priv->ring_data[i].napi); 1437 if (priv->ring_data[i].ring->irq_init_flag == RCB_IRQ_INITED) { 1438 irq_set_affinity_hint(priv->ring_data[i].ring->irq, 1439 NULL); 1440 free_irq(priv->ring_data[i].ring->irq, 1441 &priv->ring_data[i]); 1442 } 1443 1444 priv->ring_data[i].ring->irq_init_flag = RCB_IRQ_NOT_INITED; 1445 } 1446 kfree(priv->ring_data); 1447 } 1448 1449 static int hns_nic_try_get_ae(struct net_device *ndev) 1450 { 1451 struct hns_nic_priv *priv = netdev_priv(ndev); 1452 struct hnae_handle *h; 1453 int ret; 1454 1455 h = hnae_get_handle(&priv->netdev->dev, 1456 priv->ae_name, priv->port_id, NULL); 1457 if (IS_ERR_OR_NULL(h)) { 1458 ret = PTR_ERR(h); 1459 dev_dbg(priv->dev, "has not handle, register notifier!\n"); 1460 goto out; 1461 } 1462 priv->ae_handle = h; 1463 1464 ret = hns_nic_init_phy(ndev, h); 1465 if (ret) { 1466 dev_err(priv->dev, "probe phy device fail!\n"); 1467 goto out_init_phy; 1468 } 1469 1470 ret = hns_nic_init_ring_data(priv); 1471 if (ret) { 1472 ret = -ENOMEM; 1473 goto out_init_ring_data; 1474 } 1475 1476 ret = register_netdev(ndev); 1477 if (ret) { 1478 dev_err(priv->dev, "probe register netdev fail!\n"); 1479 goto out_reg_ndev_fail; 1480 } 1481 return 0; 1482 1483 out_reg_ndev_fail: 1484 hns_nic_uninit_ring_data(priv); 1485 priv->ring_data = NULL; 1486 out_init_phy: 1487 out_init_ring_data: 1488 hnae_put_handle(priv->ae_handle); 1489 priv->ae_handle = NULL; 1490 out: 1491 return ret; 1492 } 1493 1494 static int hns_nic_notifier_action(struct notifier_block *nb, 1495 unsigned long action, void *data) 1496 { 1497 struct hns_nic_priv *priv = 1498 container_of(nb, struct hns_nic_priv, notifier_block); 1499 1500 assert(action == HNAE_AE_REGISTER); 1501 1502 if (!hns_nic_try_get_ae(priv->netdev)) { 1503 hnae_unregister_notifier(&priv->notifier_block); 1504 priv->notifier_block.notifier_call = NULL; 1505 } 1506 return 0; 1507 } 1508 1509 static int hns_nic_dev_probe(struct platform_device *pdev) 1510 { 1511 struct device *dev = &pdev->dev; 1512 struct net_device *ndev; 1513 struct hns_nic_priv *priv; 1514 struct device_node *node = dev->of_node; 1515 int ret; 1516 1517 ndev = alloc_etherdev_mq(sizeof(struct hns_nic_priv), NIC_MAX_Q_PER_VF); 1518 if (!ndev) 1519 return -ENOMEM; 1520 1521 platform_set_drvdata(pdev, ndev); 1522 1523 priv = netdev_priv(ndev); 1524 priv->dev = dev; 1525 priv->netdev = ndev; 1526 1527 if (of_device_is_compatible(node, "hisilicon,hns-nic-v2")) 1528 priv->enet_ver = AE_VERSION_2; 1529 else 1530 priv->enet_ver = AE_VERSION_1; 1531 1532 ret = of_property_read_string(node, "ae-name", &priv->ae_name); 1533 if (ret) 1534 goto out_read_string_fail; 1535 1536 ret = of_property_read_u32(node, "port-id", &priv->port_id); 1537 if (ret) 1538 goto out_read_string_fail; 1539 1540 hns_init_mac_addr(ndev); 1541 1542 ndev->watchdog_timeo = HNS_NIC_TX_TIMEOUT; 1543 ndev->priv_flags |= IFF_UNICAST_FLT; 1544 ndev->netdev_ops = &hns_nic_netdev_ops; 1545 hns_ethtool_set_ops(ndev); 1546 ndev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 1547 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO | 1548 NETIF_F_GRO; 1549 ndev->vlan_features |= 1550 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM; 1551 ndev->vlan_features |= NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO; 1552 1553 SET_NETDEV_DEV(ndev, dev); 1554 1555 if (!dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) 1556 dev_dbg(dev, "set mask to 64bit\n"); 1557 else 1558 dev_err(dev, "set mask to 32bit fail!\n"); 1559 1560 /* carrier off reporting is important to ethtool even BEFORE open */ 1561 netif_carrier_off(ndev); 1562 1563 setup_timer(&priv->service_timer, hns_nic_service_timer, 1564 (unsigned long)priv); 1565 INIT_WORK(&priv->service_task, hns_nic_service_task); 1566 1567 set_bit(NIC_STATE_SERVICE_INITED, &priv->state); 1568 clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state); 1569 set_bit(NIC_STATE_DOWN, &priv->state); 1570 1571 if (hns_nic_try_get_ae(priv->netdev)) { 1572 priv->notifier_block.notifier_call = hns_nic_notifier_action; 1573 ret = hnae_register_notifier(&priv->notifier_block); 1574 if (ret) { 1575 dev_err(dev, "register notifier fail!\n"); 1576 goto out_notify_fail; 1577 } 1578 dev_dbg(dev, "has not handle, register notifier!\n"); 1579 } 1580 1581 return 0; 1582 1583 out_notify_fail: 1584 (void)cancel_work_sync(&priv->service_task); 1585 out_read_string_fail: 1586 free_netdev(ndev); 1587 return ret; 1588 } 1589 1590 static int hns_nic_dev_remove(struct platform_device *pdev) 1591 { 1592 struct net_device *ndev = platform_get_drvdata(pdev); 1593 struct hns_nic_priv *priv = netdev_priv(ndev); 1594 1595 if (ndev->reg_state != NETREG_UNINITIALIZED) 1596 unregister_netdev(ndev); 1597 1598 if (priv->ring_data) 1599 hns_nic_uninit_ring_data(priv); 1600 priv->ring_data = NULL; 1601 1602 if (priv->phy) 1603 phy_disconnect(priv->phy); 1604 priv->phy = NULL; 1605 1606 if (!IS_ERR_OR_NULL(priv->ae_handle)) 1607 hnae_put_handle(priv->ae_handle); 1608 priv->ae_handle = NULL; 1609 if (priv->notifier_block.notifier_call) 1610 hnae_unregister_notifier(&priv->notifier_block); 1611 priv->notifier_block.notifier_call = NULL; 1612 1613 set_bit(NIC_STATE_REMOVING, &priv->state); 1614 (void)cancel_work_sync(&priv->service_task); 1615 1616 free_netdev(ndev); 1617 return 0; 1618 } 1619 1620 static const struct of_device_id hns_enet_of_match[] = { 1621 {.compatible = "hisilicon,hns-nic-v1",}, 1622 {.compatible = "hisilicon,hns-nic-v2",}, 1623 {}, 1624 }; 1625 1626 MODULE_DEVICE_TABLE(of, hns_enet_of_match); 1627 1628 static struct platform_driver hns_nic_dev_driver = { 1629 .driver = { 1630 .name = "hns-nic", 1631 .of_match_table = hns_enet_of_match, 1632 }, 1633 .probe = hns_nic_dev_probe, 1634 .remove = hns_nic_dev_remove, 1635 }; 1636 1637 module_platform_driver(hns_nic_dev_driver); 1638 1639 MODULE_DESCRIPTION("HISILICON HNS Ethernet driver"); 1640 MODULE_AUTHOR("Hisilicon, Inc."); 1641 MODULE_LICENSE("GPL"); 1642 MODULE_ALIAS("platform:hns-nic"); 1643