1 /* 2 * Copyright (c) 2009, Microsoft Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, see <http://www.gnu.org/licenses/>. 15 * 16 * Authors: 17 * Haiyang Zhang <haiyangz@microsoft.com> 18 * Hank Janssen <hjanssen@microsoft.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include <linux/init.h> 23 #include <linux/atomic.h> 24 #include <linux/module.h> 25 #include <linux/highmem.h> 26 #include <linux/device.h> 27 #include <linux/io.h> 28 #include <linux/delay.h> 29 #include <linux/netdevice.h> 30 #include <linux/inetdevice.h> 31 #include <linux/etherdevice.h> 32 #include <linux/skbuff.h> 33 #include <linux/if_vlan.h> 34 #include <linux/in.h> 35 #include <linux/slab.h> 36 #include <linux/rtnetlink.h> 37 #include <linux/netpoll.h> 38 39 #include <net/arp.h> 40 #include <net/route.h> 41 #include <net/sock.h> 42 #include <net/pkt_sched.h> 43 #include <net/checksum.h> 44 #include <net/ip6_checksum.h> 45 46 #include "hyperv_net.h" 47 48 #define RING_SIZE_MIN 64 49 #define RETRY_US_LO 5000 50 #define RETRY_US_HI 10000 51 #define RETRY_MAX 2000 /* >10 sec */ 52 53 #define LINKCHANGE_INT (2 * HZ) 54 #define VF_TAKEOVER_INT (HZ / 10) 55 56 static unsigned int ring_size __ro_after_init = 128; 57 module_param(ring_size, uint, 0444); 58 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)"); 59 unsigned int netvsc_ring_bytes __ro_after_init; 60 61 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | 62 NETIF_MSG_LINK | NETIF_MSG_IFUP | 63 NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR | 64 NETIF_MSG_TX_ERR; 65 66 static int debug = -1; 67 module_param(debug, int, 0444); 68 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 69 70 static LIST_HEAD(netvsc_dev_list); 71 72 static void netvsc_change_rx_flags(struct net_device *net, int change) 73 { 74 struct net_device_context *ndev_ctx = netdev_priv(net); 75 struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev); 76 int inc; 77 78 if (!vf_netdev) 79 return; 80 81 if (change & IFF_PROMISC) { 82 inc = (net->flags & IFF_PROMISC) ? 1 : -1; 83 dev_set_promiscuity(vf_netdev, inc); 84 } 85 86 if (change & IFF_ALLMULTI) { 87 inc = (net->flags & IFF_ALLMULTI) ? 1 : -1; 88 dev_set_allmulti(vf_netdev, inc); 89 } 90 } 91 92 static void netvsc_set_rx_mode(struct net_device *net) 93 { 94 struct net_device_context *ndev_ctx = netdev_priv(net); 95 struct net_device *vf_netdev; 96 struct netvsc_device *nvdev; 97 98 rcu_read_lock(); 99 vf_netdev = rcu_dereference(ndev_ctx->vf_netdev); 100 if (vf_netdev) { 101 dev_uc_sync(vf_netdev, net); 102 dev_mc_sync(vf_netdev, net); 103 } 104 105 nvdev = rcu_dereference(ndev_ctx->nvdev); 106 if (nvdev) 107 rndis_filter_update(nvdev); 108 rcu_read_unlock(); 109 } 110 111 static int netvsc_open(struct net_device *net) 112 { 113 struct net_device_context *ndev_ctx = netdev_priv(net); 114 struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev); 115 struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev); 116 struct rndis_device *rdev; 117 int ret = 0; 118 119 netif_carrier_off(net); 120 121 /* Open up the device */ 122 ret = rndis_filter_open(nvdev); 123 if (ret != 0) { 124 netdev_err(net, "unable to open device (ret %d).\n", ret); 125 return ret; 126 } 127 128 rdev = nvdev->extension; 129 if (!rdev->link_state) { 130 netif_carrier_on(net); 131 netif_tx_wake_all_queues(net); 132 } 133 134 if (vf_netdev) { 135 /* Setting synthetic device up transparently sets 136 * slave as up. If open fails, then slave will be 137 * still be offline (and not used). 138 */ 139 ret = dev_open(vf_netdev); 140 if (ret) 141 netdev_warn(net, 142 "unable to open slave: %s: %d\n", 143 vf_netdev->name, ret); 144 } 145 return 0; 146 } 147 148 static int netvsc_wait_until_empty(struct netvsc_device *nvdev) 149 { 150 unsigned int retry = 0; 151 int i; 152 153 /* Ensure pending bytes in ring are read */ 154 for (;;) { 155 u32 aread = 0; 156 157 for (i = 0; i < nvdev->num_chn; i++) { 158 struct vmbus_channel *chn 159 = nvdev->chan_table[i].channel; 160 161 if (!chn) 162 continue; 163 164 /* make sure receive not running now */ 165 napi_synchronize(&nvdev->chan_table[i].napi); 166 167 aread = hv_get_bytes_to_read(&chn->inbound); 168 if (aread) 169 break; 170 171 aread = hv_get_bytes_to_read(&chn->outbound); 172 if (aread) 173 break; 174 } 175 176 if (aread == 0) 177 return 0; 178 179 if (++retry > RETRY_MAX) 180 return -ETIMEDOUT; 181 182 usleep_range(RETRY_US_LO, RETRY_US_HI); 183 } 184 } 185 186 static int netvsc_close(struct net_device *net) 187 { 188 struct net_device_context *net_device_ctx = netdev_priv(net); 189 struct net_device *vf_netdev 190 = rtnl_dereference(net_device_ctx->vf_netdev); 191 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev); 192 int ret; 193 194 netif_tx_disable(net); 195 196 /* No need to close rndis filter if it is removed already */ 197 if (!nvdev) 198 return 0; 199 200 ret = rndis_filter_close(nvdev); 201 if (ret != 0) { 202 netdev_err(net, "unable to close device (ret %d).\n", ret); 203 return ret; 204 } 205 206 ret = netvsc_wait_until_empty(nvdev); 207 if (ret) 208 netdev_err(net, "Ring buffer not empty after closing rndis\n"); 209 210 if (vf_netdev) 211 dev_close(vf_netdev); 212 213 return ret; 214 } 215 216 static inline void *init_ppi_data(struct rndis_message *msg, 217 u32 ppi_size, u32 pkt_type) 218 { 219 struct rndis_packet *rndis_pkt = &msg->msg.pkt; 220 struct rndis_per_packet_info *ppi; 221 222 rndis_pkt->data_offset += ppi_size; 223 ppi = (void *)rndis_pkt + rndis_pkt->per_pkt_info_offset 224 + rndis_pkt->per_pkt_info_len; 225 226 ppi->size = ppi_size; 227 ppi->type = pkt_type; 228 ppi->ppi_offset = sizeof(struct rndis_per_packet_info); 229 230 rndis_pkt->per_pkt_info_len += ppi_size; 231 232 return ppi + 1; 233 } 234 235 /* Azure hosts don't support non-TCP port numbers in hashing for fragmented 236 * packets. We can use ethtool to change UDP hash level when necessary. 237 */ 238 static inline u32 netvsc_get_hash( 239 struct sk_buff *skb, 240 const struct net_device_context *ndc) 241 { 242 struct flow_keys flow; 243 u32 hash, pkt_proto = 0; 244 static u32 hashrnd __read_mostly; 245 246 net_get_random_once(&hashrnd, sizeof(hashrnd)); 247 248 if (!skb_flow_dissect_flow_keys(skb, &flow, 0)) 249 return 0; 250 251 switch (flow.basic.ip_proto) { 252 case IPPROTO_TCP: 253 if (flow.basic.n_proto == htons(ETH_P_IP)) 254 pkt_proto = HV_TCP4_L4HASH; 255 else if (flow.basic.n_proto == htons(ETH_P_IPV6)) 256 pkt_proto = HV_TCP6_L4HASH; 257 258 break; 259 260 case IPPROTO_UDP: 261 if (flow.basic.n_proto == htons(ETH_P_IP)) 262 pkt_proto = HV_UDP4_L4HASH; 263 else if (flow.basic.n_proto == htons(ETH_P_IPV6)) 264 pkt_proto = HV_UDP6_L4HASH; 265 266 break; 267 } 268 269 if (pkt_proto & ndc->l4_hash) { 270 return skb_get_hash(skb); 271 } else { 272 if (flow.basic.n_proto == htons(ETH_P_IP)) 273 hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd); 274 else if (flow.basic.n_proto == htons(ETH_P_IPV6)) 275 hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd); 276 else 277 hash = 0; 278 279 skb_set_hash(skb, hash, PKT_HASH_TYPE_L3); 280 } 281 282 return hash; 283 } 284 285 static inline int netvsc_get_tx_queue(struct net_device *ndev, 286 struct sk_buff *skb, int old_idx) 287 { 288 const struct net_device_context *ndc = netdev_priv(ndev); 289 struct sock *sk = skb->sk; 290 int q_idx; 291 292 q_idx = ndc->tx_table[netvsc_get_hash(skb, ndc) & 293 (VRSS_SEND_TAB_SIZE - 1)]; 294 295 /* If queue index changed record the new value */ 296 if (q_idx != old_idx && 297 sk && sk_fullsock(sk) && rcu_access_pointer(sk->sk_dst_cache)) 298 sk_tx_queue_set(sk, q_idx); 299 300 return q_idx; 301 } 302 303 /* 304 * Select queue for transmit. 305 * 306 * If a valid queue has already been assigned, then use that. 307 * Otherwise compute tx queue based on hash and the send table. 308 * 309 * This is basically similar to default (__netdev_pick_tx) with the added step 310 * of using the host send_table when no other queue has been assigned. 311 * 312 * TODO support XPS - but get_xps_queue not exported 313 */ 314 static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb) 315 { 316 int q_idx = sk_tx_queue_get(skb->sk); 317 318 if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) { 319 /* If forwarding a packet, we use the recorded queue when 320 * available for better cache locality. 321 */ 322 if (skb_rx_queue_recorded(skb)) 323 q_idx = skb_get_rx_queue(skb); 324 else 325 q_idx = netvsc_get_tx_queue(ndev, skb, q_idx); 326 } 327 328 return q_idx; 329 } 330 331 static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb, 332 void *accel_priv, 333 select_queue_fallback_t fallback) 334 { 335 struct net_device_context *ndc = netdev_priv(ndev); 336 struct net_device *vf_netdev; 337 u16 txq; 338 339 rcu_read_lock(); 340 vf_netdev = rcu_dereference(ndc->vf_netdev); 341 if (vf_netdev) { 342 const struct net_device_ops *vf_ops = vf_netdev->netdev_ops; 343 344 if (vf_ops->ndo_select_queue) 345 txq = vf_ops->ndo_select_queue(vf_netdev, skb, 346 accel_priv, fallback); 347 else 348 txq = fallback(vf_netdev, skb); 349 350 /* Record the queue selected by VF so that it can be 351 * used for common case where VF has more queues than 352 * the synthetic device. 353 */ 354 qdisc_skb_cb(skb)->slave_dev_queue_mapping = txq; 355 } else { 356 txq = netvsc_pick_tx(ndev, skb); 357 } 358 rcu_read_unlock(); 359 360 while (unlikely(txq >= ndev->real_num_tx_queues)) 361 txq -= ndev->real_num_tx_queues; 362 363 return txq; 364 } 365 366 static u32 fill_pg_buf(struct page *page, u32 offset, u32 len, 367 struct hv_page_buffer *pb) 368 { 369 int j = 0; 370 371 /* Deal with compund pages by ignoring unused part 372 * of the page. 373 */ 374 page += (offset >> PAGE_SHIFT); 375 offset &= ~PAGE_MASK; 376 377 while (len > 0) { 378 unsigned long bytes; 379 380 bytes = PAGE_SIZE - offset; 381 if (bytes > len) 382 bytes = len; 383 pb[j].pfn = page_to_pfn(page); 384 pb[j].offset = offset; 385 pb[j].len = bytes; 386 387 offset += bytes; 388 len -= bytes; 389 390 if (offset == PAGE_SIZE && len) { 391 page++; 392 offset = 0; 393 j++; 394 } 395 } 396 397 return j + 1; 398 } 399 400 static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb, 401 struct hv_netvsc_packet *packet, 402 struct hv_page_buffer *pb) 403 { 404 u32 slots_used = 0; 405 char *data = skb->data; 406 int frags = skb_shinfo(skb)->nr_frags; 407 int i; 408 409 /* The packet is laid out thus: 410 * 1. hdr: RNDIS header and PPI 411 * 2. skb linear data 412 * 3. skb fragment data 413 */ 414 slots_used += fill_pg_buf(virt_to_page(hdr), 415 offset_in_page(hdr), 416 len, &pb[slots_used]); 417 418 packet->rmsg_size = len; 419 packet->rmsg_pgcnt = slots_used; 420 421 slots_used += fill_pg_buf(virt_to_page(data), 422 offset_in_page(data), 423 skb_headlen(skb), &pb[slots_used]); 424 425 for (i = 0; i < frags; i++) { 426 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 427 428 slots_used += fill_pg_buf(skb_frag_page(frag), 429 frag->page_offset, 430 skb_frag_size(frag), &pb[slots_used]); 431 } 432 return slots_used; 433 } 434 435 static int count_skb_frag_slots(struct sk_buff *skb) 436 { 437 int i, frags = skb_shinfo(skb)->nr_frags; 438 int pages = 0; 439 440 for (i = 0; i < frags; i++) { 441 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 442 unsigned long size = skb_frag_size(frag); 443 unsigned long offset = frag->page_offset; 444 445 /* Skip unused frames from start of page */ 446 offset &= ~PAGE_MASK; 447 pages += PFN_UP(offset + size); 448 } 449 return pages; 450 } 451 452 static int netvsc_get_slots(struct sk_buff *skb) 453 { 454 char *data = skb->data; 455 unsigned int offset = offset_in_page(data); 456 unsigned int len = skb_headlen(skb); 457 int slots; 458 int frag_slots; 459 460 slots = DIV_ROUND_UP(offset + len, PAGE_SIZE); 461 frag_slots = count_skb_frag_slots(skb); 462 return slots + frag_slots; 463 } 464 465 static u32 net_checksum_info(struct sk_buff *skb) 466 { 467 if (skb->protocol == htons(ETH_P_IP)) { 468 struct iphdr *ip = ip_hdr(skb); 469 470 if (ip->protocol == IPPROTO_TCP) 471 return TRANSPORT_INFO_IPV4_TCP; 472 else if (ip->protocol == IPPROTO_UDP) 473 return TRANSPORT_INFO_IPV4_UDP; 474 } else { 475 struct ipv6hdr *ip6 = ipv6_hdr(skb); 476 477 if (ip6->nexthdr == IPPROTO_TCP) 478 return TRANSPORT_INFO_IPV6_TCP; 479 else if (ip6->nexthdr == IPPROTO_UDP) 480 return TRANSPORT_INFO_IPV6_UDP; 481 } 482 483 return TRANSPORT_INFO_NOT_IP; 484 } 485 486 /* Send skb on the slave VF device. */ 487 static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev, 488 struct sk_buff *skb) 489 { 490 struct net_device_context *ndev_ctx = netdev_priv(net); 491 unsigned int len = skb->len; 492 int rc; 493 494 skb->dev = vf_netdev; 495 skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping; 496 497 rc = dev_queue_xmit(skb); 498 if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) { 499 struct netvsc_vf_pcpu_stats *pcpu_stats 500 = this_cpu_ptr(ndev_ctx->vf_stats); 501 502 u64_stats_update_begin(&pcpu_stats->syncp); 503 pcpu_stats->tx_packets++; 504 pcpu_stats->tx_bytes += len; 505 u64_stats_update_end(&pcpu_stats->syncp); 506 } else { 507 this_cpu_inc(ndev_ctx->vf_stats->tx_dropped); 508 } 509 510 return rc; 511 } 512 513 static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net) 514 { 515 struct net_device_context *net_device_ctx = netdev_priv(net); 516 struct hv_netvsc_packet *packet = NULL; 517 int ret; 518 unsigned int num_data_pgs; 519 struct rndis_message *rndis_msg; 520 struct net_device *vf_netdev; 521 u32 rndis_msg_size; 522 u32 hash; 523 struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT]; 524 525 /* if VF is present and up then redirect packets 526 * already called with rcu_read_lock_bh 527 */ 528 vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev); 529 if (vf_netdev && netif_running(vf_netdev) && 530 !netpoll_tx_running(net)) 531 return netvsc_vf_xmit(net, vf_netdev, skb); 532 533 /* We will atmost need two pages to describe the rndis 534 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number 535 * of pages in a single packet. If skb is scattered around 536 * more pages we try linearizing it. 537 */ 538 539 num_data_pgs = netvsc_get_slots(skb) + 2; 540 541 if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) { 542 ++net_device_ctx->eth_stats.tx_scattered; 543 544 if (skb_linearize(skb)) 545 goto no_memory; 546 547 num_data_pgs = netvsc_get_slots(skb) + 2; 548 if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) { 549 ++net_device_ctx->eth_stats.tx_too_big; 550 goto drop; 551 } 552 } 553 554 /* 555 * Place the rndis header in the skb head room and 556 * the skb->cb will be used for hv_netvsc_packet 557 * structure. 558 */ 559 ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE); 560 if (ret) 561 goto no_memory; 562 563 /* Use the skb control buffer for building up the packet */ 564 BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) > 565 FIELD_SIZEOF(struct sk_buff, cb)); 566 packet = (struct hv_netvsc_packet *)skb->cb; 567 568 packet->q_idx = skb_get_queue_mapping(skb); 569 570 packet->total_data_buflen = skb->len; 571 packet->total_bytes = skb->len; 572 packet->total_packets = 1; 573 574 rndis_msg = (struct rndis_message *)skb->head; 575 576 /* Add the rndis header */ 577 rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET; 578 rndis_msg->msg_len = packet->total_data_buflen; 579 580 rndis_msg->msg.pkt = (struct rndis_packet) { 581 .data_offset = sizeof(struct rndis_packet), 582 .data_len = packet->total_data_buflen, 583 .per_pkt_info_offset = sizeof(struct rndis_packet), 584 }; 585 586 rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet); 587 588 hash = skb_get_hash_raw(skb); 589 if (hash != 0 && net->real_num_tx_queues > 1) { 590 u32 *hash_info; 591 592 rndis_msg_size += NDIS_HASH_PPI_SIZE; 593 hash_info = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE, 594 NBL_HASH_VALUE); 595 *hash_info = hash; 596 } 597 598 if (skb_vlan_tag_present(skb)) { 599 struct ndis_pkt_8021q_info *vlan; 600 601 rndis_msg_size += NDIS_VLAN_PPI_SIZE; 602 vlan = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE, 603 IEEE_8021Q_INFO); 604 605 vlan->value = 0; 606 vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK; 607 vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >> 608 VLAN_PRIO_SHIFT; 609 } 610 611 if (skb_is_gso(skb)) { 612 struct ndis_tcp_lso_info *lso_info; 613 614 rndis_msg_size += NDIS_LSO_PPI_SIZE; 615 lso_info = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE, 616 TCP_LARGESEND_PKTINFO); 617 618 lso_info->value = 0; 619 lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE; 620 if (skb->protocol == htons(ETH_P_IP)) { 621 lso_info->lso_v2_transmit.ip_version = 622 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4; 623 ip_hdr(skb)->tot_len = 0; 624 ip_hdr(skb)->check = 0; 625 tcp_hdr(skb)->check = 626 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 627 ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); 628 } else { 629 lso_info->lso_v2_transmit.ip_version = 630 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6; 631 ipv6_hdr(skb)->payload_len = 0; 632 tcp_hdr(skb)->check = 633 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 634 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); 635 } 636 lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb); 637 lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size; 638 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 639 if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) { 640 struct ndis_tcp_ip_checksum_info *csum_info; 641 642 rndis_msg_size += NDIS_CSUM_PPI_SIZE; 643 csum_info = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE, 644 TCPIP_CHKSUM_PKTINFO); 645 646 csum_info->value = 0; 647 csum_info->transmit.tcp_header_offset = skb_transport_offset(skb); 648 649 if (skb->protocol == htons(ETH_P_IP)) { 650 csum_info->transmit.is_ipv4 = 1; 651 652 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 653 csum_info->transmit.tcp_checksum = 1; 654 else 655 csum_info->transmit.udp_checksum = 1; 656 } else { 657 csum_info->transmit.is_ipv6 = 1; 658 659 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 660 csum_info->transmit.tcp_checksum = 1; 661 else 662 csum_info->transmit.udp_checksum = 1; 663 } 664 } else { 665 /* Can't do offload of this type of checksum */ 666 if (skb_checksum_help(skb)) 667 goto drop; 668 } 669 } 670 671 /* Start filling in the page buffers with the rndis hdr */ 672 rndis_msg->msg_len += rndis_msg_size; 673 packet->total_data_buflen = rndis_msg->msg_len; 674 packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size, 675 skb, packet, pb); 676 677 /* timestamp packet in software */ 678 skb_tx_timestamp(skb); 679 680 ret = netvsc_send(net, packet, rndis_msg, pb, skb); 681 if (likely(ret == 0)) 682 return NETDEV_TX_OK; 683 684 if (ret == -EAGAIN) { 685 ++net_device_ctx->eth_stats.tx_busy; 686 return NETDEV_TX_BUSY; 687 } 688 689 if (ret == -ENOSPC) 690 ++net_device_ctx->eth_stats.tx_no_space; 691 692 drop: 693 dev_kfree_skb_any(skb); 694 net->stats.tx_dropped++; 695 696 return NETDEV_TX_OK; 697 698 no_memory: 699 ++net_device_ctx->eth_stats.tx_no_memory; 700 goto drop; 701 } 702 703 /* 704 * netvsc_linkstatus_callback - Link up/down notification 705 */ 706 void netvsc_linkstatus_callback(struct net_device *net, 707 struct rndis_message *resp) 708 { 709 struct rndis_indicate_status *indicate = &resp->msg.indicate_status; 710 struct net_device_context *ndev_ctx = netdev_priv(net); 711 struct netvsc_reconfig *event; 712 unsigned long flags; 713 714 /* Update the physical link speed when changing to another vSwitch */ 715 if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) { 716 u32 speed; 717 718 speed = *(u32 *)((void *)indicate 719 + indicate->status_buf_offset) / 10000; 720 ndev_ctx->speed = speed; 721 return; 722 } 723 724 /* Handle these link change statuses below */ 725 if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE && 726 indicate->status != RNDIS_STATUS_MEDIA_CONNECT && 727 indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT) 728 return; 729 730 if (net->reg_state != NETREG_REGISTERED) 731 return; 732 733 event = kzalloc(sizeof(*event), GFP_ATOMIC); 734 if (!event) 735 return; 736 event->event = indicate->status; 737 738 spin_lock_irqsave(&ndev_ctx->lock, flags); 739 list_add_tail(&event->list, &ndev_ctx->reconfig_events); 740 spin_unlock_irqrestore(&ndev_ctx->lock, flags); 741 742 schedule_delayed_work(&ndev_ctx->dwork, 0); 743 } 744 745 static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net, 746 struct napi_struct *napi, 747 const struct ndis_tcp_ip_checksum_info *csum_info, 748 const struct ndis_pkt_8021q_info *vlan, 749 void *data, u32 buflen) 750 { 751 struct sk_buff *skb; 752 753 skb = napi_alloc_skb(napi, buflen); 754 if (!skb) 755 return skb; 756 757 /* 758 * Copy to skb. This copy is needed here since the memory pointed by 759 * hv_netvsc_packet cannot be deallocated 760 */ 761 skb_put_data(skb, data, buflen); 762 763 skb->protocol = eth_type_trans(skb, net); 764 765 /* skb is already created with CHECKSUM_NONE */ 766 skb_checksum_none_assert(skb); 767 768 /* 769 * In Linux, the IP checksum is always checked. 770 * Do L4 checksum offload if enabled and present. 771 */ 772 if (csum_info && (net->features & NETIF_F_RXCSUM)) { 773 if (csum_info->receive.tcp_checksum_succeeded || 774 csum_info->receive.udp_checksum_succeeded) 775 skb->ip_summed = CHECKSUM_UNNECESSARY; 776 } 777 778 if (vlan) { 779 u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT); 780 781 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 782 vlan_tci); 783 } 784 785 return skb; 786 } 787 788 /* 789 * netvsc_recv_callback - Callback when we receive a packet from the 790 * "wire" on the specified device. 791 */ 792 int netvsc_recv_callback(struct net_device *net, 793 struct netvsc_device *net_device, 794 struct vmbus_channel *channel, 795 void *data, u32 len, 796 const struct ndis_tcp_ip_checksum_info *csum_info, 797 const struct ndis_pkt_8021q_info *vlan) 798 { 799 struct net_device_context *net_device_ctx = netdev_priv(net); 800 u16 q_idx = channel->offermsg.offer.sub_channel_index; 801 struct netvsc_channel *nvchan = &net_device->chan_table[q_idx]; 802 struct sk_buff *skb; 803 struct netvsc_stats *rx_stats; 804 805 if (net->reg_state != NETREG_REGISTERED) 806 return NVSP_STAT_FAIL; 807 808 /* Allocate a skb - TODO direct I/O to pages? */ 809 skb = netvsc_alloc_recv_skb(net, &nvchan->napi, 810 csum_info, vlan, data, len); 811 if (unlikely(!skb)) { 812 ++net_device_ctx->eth_stats.rx_no_memory; 813 rcu_read_unlock(); 814 return NVSP_STAT_FAIL; 815 } 816 817 skb_record_rx_queue(skb, q_idx); 818 819 /* 820 * Even if injecting the packet, record the statistics 821 * on the synthetic device because modifying the VF device 822 * statistics will not work correctly. 823 */ 824 rx_stats = &nvchan->rx_stats; 825 u64_stats_update_begin(&rx_stats->syncp); 826 rx_stats->packets++; 827 rx_stats->bytes += len; 828 829 if (skb->pkt_type == PACKET_BROADCAST) 830 ++rx_stats->broadcast; 831 else if (skb->pkt_type == PACKET_MULTICAST) 832 ++rx_stats->multicast; 833 u64_stats_update_end(&rx_stats->syncp); 834 835 napi_gro_receive(&nvchan->napi, skb); 836 return NVSP_STAT_SUCCESS; 837 } 838 839 static void netvsc_get_drvinfo(struct net_device *net, 840 struct ethtool_drvinfo *info) 841 { 842 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 843 strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); 844 } 845 846 static void netvsc_get_channels(struct net_device *net, 847 struct ethtool_channels *channel) 848 { 849 struct net_device_context *net_device_ctx = netdev_priv(net); 850 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev); 851 852 if (nvdev) { 853 channel->max_combined = nvdev->max_chn; 854 channel->combined_count = nvdev->num_chn; 855 } 856 } 857 858 static int netvsc_detach(struct net_device *ndev, 859 struct netvsc_device *nvdev) 860 { 861 struct net_device_context *ndev_ctx = netdev_priv(ndev); 862 struct hv_device *hdev = ndev_ctx->device_ctx; 863 int ret; 864 865 /* Don't try continuing to try and setup sub channels */ 866 if (cancel_work_sync(&nvdev->subchan_work)) 867 nvdev->num_chn = 1; 868 869 /* If device was up (receiving) then shutdown */ 870 if (netif_running(ndev)) { 871 netif_tx_disable(ndev); 872 873 ret = rndis_filter_close(nvdev); 874 if (ret) { 875 netdev_err(ndev, 876 "unable to close device (ret %d).\n", ret); 877 return ret; 878 } 879 880 ret = netvsc_wait_until_empty(nvdev); 881 if (ret) { 882 netdev_err(ndev, 883 "Ring buffer not empty after closing rndis\n"); 884 return ret; 885 } 886 } 887 888 netif_device_detach(ndev); 889 890 rndis_filter_device_remove(hdev, nvdev); 891 892 return 0; 893 } 894 895 static int netvsc_attach(struct net_device *ndev, 896 struct netvsc_device_info *dev_info) 897 { 898 struct net_device_context *ndev_ctx = netdev_priv(ndev); 899 struct hv_device *hdev = ndev_ctx->device_ctx; 900 struct netvsc_device *nvdev; 901 struct rndis_device *rdev; 902 int ret; 903 904 nvdev = rndis_filter_device_add(hdev, dev_info); 905 if (IS_ERR(nvdev)) 906 return PTR_ERR(nvdev); 907 908 /* Note: enable and attach happen when sub-channels setup */ 909 910 netif_carrier_off(ndev); 911 912 if (netif_running(ndev)) { 913 ret = rndis_filter_open(nvdev); 914 if (ret) 915 return ret; 916 917 rdev = nvdev->extension; 918 if (!rdev->link_state) 919 netif_carrier_on(ndev); 920 } 921 922 return 0; 923 } 924 925 static int netvsc_set_channels(struct net_device *net, 926 struct ethtool_channels *channels) 927 { 928 struct net_device_context *net_device_ctx = netdev_priv(net); 929 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev); 930 unsigned int orig, count = channels->combined_count; 931 struct netvsc_device_info device_info; 932 int ret; 933 934 /* We do not support separate count for rx, tx, or other */ 935 if (count == 0 || 936 channels->rx_count || channels->tx_count || channels->other_count) 937 return -EINVAL; 938 939 if (!nvdev || nvdev->destroy) 940 return -ENODEV; 941 942 if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5) 943 return -EINVAL; 944 945 if (count > nvdev->max_chn) 946 return -EINVAL; 947 948 orig = nvdev->num_chn; 949 950 memset(&device_info, 0, sizeof(device_info)); 951 device_info.num_chn = count; 952 device_info.send_sections = nvdev->send_section_cnt; 953 device_info.send_section_size = nvdev->send_section_size; 954 device_info.recv_sections = nvdev->recv_section_cnt; 955 device_info.recv_section_size = nvdev->recv_section_size; 956 957 ret = netvsc_detach(net, nvdev); 958 if (ret) 959 return ret; 960 961 ret = netvsc_attach(net, &device_info); 962 if (ret) { 963 device_info.num_chn = orig; 964 if (netvsc_attach(net, &device_info)) 965 netdev_err(net, "restoring channel setting failed\n"); 966 } 967 968 return ret; 969 } 970 971 static bool 972 netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd) 973 { 974 struct ethtool_link_ksettings diff1 = *cmd; 975 struct ethtool_link_ksettings diff2 = {}; 976 977 diff1.base.speed = 0; 978 diff1.base.duplex = 0; 979 /* advertising and cmd are usually set */ 980 ethtool_link_ksettings_zero_link_mode(&diff1, advertising); 981 diff1.base.cmd = 0; 982 /* We set port to PORT_OTHER */ 983 diff2.base.port = PORT_OTHER; 984 985 return !memcmp(&diff1, &diff2, sizeof(diff1)); 986 } 987 988 static void netvsc_init_settings(struct net_device *dev) 989 { 990 struct net_device_context *ndc = netdev_priv(dev); 991 992 ndc->l4_hash = HV_DEFAULT_L4HASH; 993 994 ndc->speed = SPEED_UNKNOWN; 995 ndc->duplex = DUPLEX_FULL; 996 } 997 998 static int netvsc_get_link_ksettings(struct net_device *dev, 999 struct ethtool_link_ksettings *cmd) 1000 { 1001 struct net_device_context *ndc = netdev_priv(dev); 1002 1003 cmd->base.speed = ndc->speed; 1004 cmd->base.duplex = ndc->duplex; 1005 cmd->base.port = PORT_OTHER; 1006 1007 return 0; 1008 } 1009 1010 static int netvsc_set_link_ksettings(struct net_device *dev, 1011 const struct ethtool_link_ksettings *cmd) 1012 { 1013 struct net_device_context *ndc = netdev_priv(dev); 1014 u32 speed; 1015 1016 speed = cmd->base.speed; 1017 if (!ethtool_validate_speed(speed) || 1018 !ethtool_validate_duplex(cmd->base.duplex) || 1019 !netvsc_validate_ethtool_ss_cmd(cmd)) 1020 return -EINVAL; 1021 1022 ndc->speed = speed; 1023 ndc->duplex = cmd->base.duplex; 1024 1025 return 0; 1026 } 1027 1028 static int netvsc_change_mtu(struct net_device *ndev, int mtu) 1029 { 1030 struct net_device_context *ndevctx = netdev_priv(ndev); 1031 struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev); 1032 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); 1033 int orig_mtu = ndev->mtu; 1034 struct netvsc_device_info device_info; 1035 int ret = 0; 1036 1037 if (!nvdev || nvdev->destroy) 1038 return -ENODEV; 1039 1040 /* Change MTU of underlying VF netdev first. */ 1041 if (vf_netdev) { 1042 ret = dev_set_mtu(vf_netdev, mtu); 1043 if (ret) 1044 return ret; 1045 } 1046 1047 memset(&device_info, 0, sizeof(device_info)); 1048 device_info.num_chn = nvdev->num_chn; 1049 device_info.send_sections = nvdev->send_section_cnt; 1050 device_info.send_section_size = nvdev->send_section_size; 1051 device_info.recv_sections = nvdev->recv_section_cnt; 1052 device_info.recv_section_size = nvdev->recv_section_size; 1053 1054 ret = netvsc_detach(ndev, nvdev); 1055 if (ret) 1056 goto rollback_vf; 1057 1058 ndev->mtu = mtu; 1059 1060 ret = netvsc_attach(ndev, &device_info); 1061 if (ret) 1062 goto rollback; 1063 1064 return 0; 1065 1066 rollback: 1067 /* Attempt rollback to original MTU */ 1068 ndev->mtu = orig_mtu; 1069 1070 if (netvsc_attach(ndev, &device_info)) 1071 netdev_err(ndev, "restoring mtu failed\n"); 1072 rollback_vf: 1073 if (vf_netdev) 1074 dev_set_mtu(vf_netdev, orig_mtu); 1075 1076 return ret; 1077 } 1078 1079 static void netvsc_get_vf_stats(struct net_device *net, 1080 struct netvsc_vf_pcpu_stats *tot) 1081 { 1082 struct net_device_context *ndev_ctx = netdev_priv(net); 1083 int i; 1084 1085 memset(tot, 0, sizeof(*tot)); 1086 1087 for_each_possible_cpu(i) { 1088 const struct netvsc_vf_pcpu_stats *stats 1089 = per_cpu_ptr(ndev_ctx->vf_stats, i); 1090 u64 rx_packets, rx_bytes, tx_packets, tx_bytes; 1091 unsigned int start; 1092 1093 do { 1094 start = u64_stats_fetch_begin_irq(&stats->syncp); 1095 rx_packets = stats->rx_packets; 1096 tx_packets = stats->tx_packets; 1097 rx_bytes = stats->rx_bytes; 1098 tx_bytes = stats->tx_bytes; 1099 } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); 1100 1101 tot->rx_packets += rx_packets; 1102 tot->tx_packets += tx_packets; 1103 tot->rx_bytes += rx_bytes; 1104 tot->tx_bytes += tx_bytes; 1105 tot->tx_dropped += stats->tx_dropped; 1106 } 1107 } 1108 1109 static void netvsc_get_stats64(struct net_device *net, 1110 struct rtnl_link_stats64 *t) 1111 { 1112 struct net_device_context *ndev_ctx = netdev_priv(net); 1113 struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev); 1114 struct netvsc_vf_pcpu_stats vf_tot; 1115 int i; 1116 1117 if (!nvdev) 1118 return; 1119 1120 netdev_stats_to_stats64(t, &net->stats); 1121 1122 netvsc_get_vf_stats(net, &vf_tot); 1123 t->rx_packets += vf_tot.rx_packets; 1124 t->tx_packets += vf_tot.tx_packets; 1125 t->rx_bytes += vf_tot.rx_bytes; 1126 t->tx_bytes += vf_tot.tx_bytes; 1127 t->tx_dropped += vf_tot.tx_dropped; 1128 1129 for (i = 0; i < nvdev->num_chn; i++) { 1130 const struct netvsc_channel *nvchan = &nvdev->chan_table[i]; 1131 const struct netvsc_stats *stats; 1132 u64 packets, bytes, multicast; 1133 unsigned int start; 1134 1135 stats = &nvchan->tx_stats; 1136 do { 1137 start = u64_stats_fetch_begin_irq(&stats->syncp); 1138 packets = stats->packets; 1139 bytes = stats->bytes; 1140 } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); 1141 1142 t->tx_bytes += bytes; 1143 t->tx_packets += packets; 1144 1145 stats = &nvchan->rx_stats; 1146 do { 1147 start = u64_stats_fetch_begin_irq(&stats->syncp); 1148 packets = stats->packets; 1149 bytes = stats->bytes; 1150 multicast = stats->multicast + stats->broadcast; 1151 } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); 1152 1153 t->rx_bytes += bytes; 1154 t->rx_packets += packets; 1155 t->multicast += multicast; 1156 } 1157 } 1158 1159 static int netvsc_set_mac_addr(struct net_device *ndev, void *p) 1160 { 1161 struct net_device_context *ndc = netdev_priv(ndev); 1162 struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev); 1163 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); 1164 struct sockaddr *addr = p; 1165 int err; 1166 1167 err = eth_prepare_mac_addr_change(ndev, p); 1168 if (err) 1169 return err; 1170 1171 if (!nvdev) 1172 return -ENODEV; 1173 1174 if (vf_netdev) { 1175 err = dev_set_mac_address(vf_netdev, addr); 1176 if (err) 1177 return err; 1178 } 1179 1180 err = rndis_filter_set_device_mac(nvdev, addr->sa_data); 1181 if (!err) { 1182 eth_commit_mac_addr_change(ndev, p); 1183 } else if (vf_netdev) { 1184 /* rollback change on VF */ 1185 memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN); 1186 dev_set_mac_address(vf_netdev, addr); 1187 } 1188 1189 return err; 1190 } 1191 1192 static const struct { 1193 char name[ETH_GSTRING_LEN]; 1194 u16 offset; 1195 } netvsc_stats[] = { 1196 { "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) }, 1197 { "tx_no_memory", offsetof(struct netvsc_ethtool_stats, tx_no_memory) }, 1198 { "tx_no_space", offsetof(struct netvsc_ethtool_stats, tx_no_space) }, 1199 { "tx_too_big", offsetof(struct netvsc_ethtool_stats, tx_too_big) }, 1200 { "tx_busy", offsetof(struct netvsc_ethtool_stats, tx_busy) }, 1201 { "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) }, 1202 { "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) }, 1203 { "rx_no_memory", offsetof(struct netvsc_ethtool_stats, rx_no_memory) }, 1204 { "stop_queue", offsetof(struct netvsc_ethtool_stats, stop_queue) }, 1205 { "wake_queue", offsetof(struct netvsc_ethtool_stats, wake_queue) }, 1206 }, vf_stats[] = { 1207 { "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) }, 1208 { "vf_rx_bytes", offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) }, 1209 { "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) }, 1210 { "vf_tx_bytes", offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) }, 1211 { "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) }, 1212 }; 1213 1214 #define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats) 1215 #define NETVSC_VF_STATS_LEN ARRAY_SIZE(vf_stats) 1216 1217 /* 4 statistics per queue (rx/tx packets/bytes) */ 1218 #define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4) 1219 1220 static int netvsc_get_sset_count(struct net_device *dev, int string_set) 1221 { 1222 struct net_device_context *ndc = netdev_priv(dev); 1223 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); 1224 1225 if (!nvdev) 1226 return -ENODEV; 1227 1228 switch (string_set) { 1229 case ETH_SS_STATS: 1230 return NETVSC_GLOBAL_STATS_LEN 1231 + NETVSC_VF_STATS_LEN 1232 + NETVSC_QUEUE_STATS_LEN(nvdev); 1233 default: 1234 return -EINVAL; 1235 } 1236 } 1237 1238 static void netvsc_get_ethtool_stats(struct net_device *dev, 1239 struct ethtool_stats *stats, u64 *data) 1240 { 1241 struct net_device_context *ndc = netdev_priv(dev); 1242 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); 1243 const void *nds = &ndc->eth_stats; 1244 const struct netvsc_stats *qstats; 1245 struct netvsc_vf_pcpu_stats sum; 1246 unsigned int start; 1247 u64 packets, bytes; 1248 int i, j; 1249 1250 if (!nvdev) 1251 return; 1252 1253 for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++) 1254 data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset); 1255 1256 netvsc_get_vf_stats(dev, &sum); 1257 for (j = 0; j < NETVSC_VF_STATS_LEN; j++) 1258 data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset); 1259 1260 for (j = 0; j < nvdev->num_chn; j++) { 1261 qstats = &nvdev->chan_table[j].tx_stats; 1262 1263 do { 1264 start = u64_stats_fetch_begin_irq(&qstats->syncp); 1265 packets = qstats->packets; 1266 bytes = qstats->bytes; 1267 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start)); 1268 data[i++] = packets; 1269 data[i++] = bytes; 1270 1271 qstats = &nvdev->chan_table[j].rx_stats; 1272 do { 1273 start = u64_stats_fetch_begin_irq(&qstats->syncp); 1274 packets = qstats->packets; 1275 bytes = qstats->bytes; 1276 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start)); 1277 data[i++] = packets; 1278 data[i++] = bytes; 1279 } 1280 } 1281 1282 static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data) 1283 { 1284 struct net_device_context *ndc = netdev_priv(dev); 1285 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); 1286 u8 *p = data; 1287 int i; 1288 1289 if (!nvdev) 1290 return; 1291 1292 switch (stringset) { 1293 case ETH_SS_STATS: 1294 for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) { 1295 memcpy(p, netvsc_stats[i].name, ETH_GSTRING_LEN); 1296 p += ETH_GSTRING_LEN; 1297 } 1298 1299 for (i = 0; i < ARRAY_SIZE(vf_stats); i++) { 1300 memcpy(p, vf_stats[i].name, ETH_GSTRING_LEN); 1301 p += ETH_GSTRING_LEN; 1302 } 1303 1304 for (i = 0; i < nvdev->num_chn; i++) { 1305 sprintf(p, "tx_queue_%u_packets", i); 1306 p += ETH_GSTRING_LEN; 1307 sprintf(p, "tx_queue_%u_bytes", i); 1308 p += ETH_GSTRING_LEN; 1309 sprintf(p, "rx_queue_%u_packets", i); 1310 p += ETH_GSTRING_LEN; 1311 sprintf(p, "rx_queue_%u_bytes", i); 1312 p += ETH_GSTRING_LEN; 1313 } 1314 1315 break; 1316 } 1317 } 1318 1319 static int 1320 netvsc_get_rss_hash_opts(struct net_device_context *ndc, 1321 struct ethtool_rxnfc *info) 1322 { 1323 const u32 l4_flag = RXH_L4_B_0_1 | RXH_L4_B_2_3; 1324 1325 info->data = RXH_IP_SRC | RXH_IP_DST; 1326 1327 switch (info->flow_type) { 1328 case TCP_V4_FLOW: 1329 if (ndc->l4_hash & HV_TCP4_L4HASH) 1330 info->data |= l4_flag; 1331 1332 break; 1333 1334 case TCP_V6_FLOW: 1335 if (ndc->l4_hash & HV_TCP6_L4HASH) 1336 info->data |= l4_flag; 1337 1338 break; 1339 1340 case UDP_V4_FLOW: 1341 if (ndc->l4_hash & HV_UDP4_L4HASH) 1342 info->data |= l4_flag; 1343 1344 break; 1345 1346 case UDP_V6_FLOW: 1347 if (ndc->l4_hash & HV_UDP6_L4HASH) 1348 info->data |= l4_flag; 1349 1350 break; 1351 1352 case IPV4_FLOW: 1353 case IPV6_FLOW: 1354 break; 1355 default: 1356 info->data = 0; 1357 break; 1358 } 1359 1360 return 0; 1361 } 1362 1363 static int 1364 netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, 1365 u32 *rules) 1366 { 1367 struct net_device_context *ndc = netdev_priv(dev); 1368 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); 1369 1370 if (!nvdev) 1371 return -ENODEV; 1372 1373 switch (info->cmd) { 1374 case ETHTOOL_GRXRINGS: 1375 info->data = nvdev->num_chn; 1376 return 0; 1377 1378 case ETHTOOL_GRXFH: 1379 return netvsc_get_rss_hash_opts(ndc, info); 1380 } 1381 return -EOPNOTSUPP; 1382 } 1383 1384 static int netvsc_set_rss_hash_opts(struct net_device_context *ndc, 1385 struct ethtool_rxnfc *info) 1386 { 1387 if (info->data == (RXH_IP_SRC | RXH_IP_DST | 1388 RXH_L4_B_0_1 | RXH_L4_B_2_3)) { 1389 switch (info->flow_type) { 1390 case TCP_V4_FLOW: 1391 ndc->l4_hash |= HV_TCP4_L4HASH; 1392 break; 1393 1394 case TCP_V6_FLOW: 1395 ndc->l4_hash |= HV_TCP6_L4HASH; 1396 break; 1397 1398 case UDP_V4_FLOW: 1399 ndc->l4_hash |= HV_UDP4_L4HASH; 1400 break; 1401 1402 case UDP_V6_FLOW: 1403 ndc->l4_hash |= HV_UDP6_L4HASH; 1404 break; 1405 1406 default: 1407 return -EOPNOTSUPP; 1408 } 1409 1410 return 0; 1411 } 1412 1413 if (info->data == (RXH_IP_SRC | RXH_IP_DST)) { 1414 switch (info->flow_type) { 1415 case TCP_V4_FLOW: 1416 ndc->l4_hash &= ~HV_TCP4_L4HASH; 1417 break; 1418 1419 case TCP_V6_FLOW: 1420 ndc->l4_hash &= ~HV_TCP6_L4HASH; 1421 break; 1422 1423 case UDP_V4_FLOW: 1424 ndc->l4_hash &= ~HV_UDP4_L4HASH; 1425 break; 1426 1427 case UDP_V6_FLOW: 1428 ndc->l4_hash &= ~HV_UDP6_L4HASH; 1429 break; 1430 1431 default: 1432 return -EOPNOTSUPP; 1433 } 1434 1435 return 0; 1436 } 1437 1438 return -EOPNOTSUPP; 1439 } 1440 1441 static int 1442 netvsc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info) 1443 { 1444 struct net_device_context *ndc = netdev_priv(ndev); 1445 1446 if (info->cmd == ETHTOOL_SRXFH) 1447 return netvsc_set_rss_hash_opts(ndc, info); 1448 1449 return -EOPNOTSUPP; 1450 } 1451 1452 #ifdef CONFIG_NET_POLL_CONTROLLER 1453 static void netvsc_poll_controller(struct net_device *dev) 1454 { 1455 struct net_device_context *ndc = netdev_priv(dev); 1456 struct netvsc_device *ndev; 1457 int i; 1458 1459 rcu_read_lock(); 1460 ndev = rcu_dereference(ndc->nvdev); 1461 if (ndev) { 1462 for (i = 0; i < ndev->num_chn; i++) { 1463 struct netvsc_channel *nvchan = &ndev->chan_table[i]; 1464 1465 napi_schedule(&nvchan->napi); 1466 } 1467 } 1468 rcu_read_unlock(); 1469 } 1470 #endif 1471 1472 static u32 netvsc_get_rxfh_key_size(struct net_device *dev) 1473 { 1474 return NETVSC_HASH_KEYLEN; 1475 } 1476 1477 static u32 netvsc_rss_indir_size(struct net_device *dev) 1478 { 1479 return ITAB_NUM; 1480 } 1481 1482 static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, 1483 u8 *hfunc) 1484 { 1485 struct net_device_context *ndc = netdev_priv(dev); 1486 struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev); 1487 struct rndis_device *rndis_dev; 1488 int i; 1489 1490 if (!ndev) 1491 return -ENODEV; 1492 1493 if (hfunc) 1494 *hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ 1495 1496 rndis_dev = ndev->extension; 1497 if (indir) { 1498 for (i = 0; i < ITAB_NUM; i++) 1499 indir[i] = rndis_dev->rx_table[i]; 1500 } 1501 1502 if (key) 1503 memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN); 1504 1505 return 0; 1506 } 1507 1508 static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir, 1509 const u8 *key, const u8 hfunc) 1510 { 1511 struct net_device_context *ndc = netdev_priv(dev); 1512 struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev); 1513 struct rndis_device *rndis_dev; 1514 int i; 1515 1516 if (!ndev) 1517 return -ENODEV; 1518 1519 if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) 1520 return -EOPNOTSUPP; 1521 1522 rndis_dev = ndev->extension; 1523 if (indir) { 1524 for (i = 0; i < ITAB_NUM; i++) 1525 if (indir[i] >= ndev->num_chn) 1526 return -EINVAL; 1527 1528 for (i = 0; i < ITAB_NUM; i++) 1529 rndis_dev->rx_table[i] = indir[i]; 1530 } 1531 1532 if (!key) { 1533 if (!indir) 1534 return 0; 1535 1536 key = rndis_dev->rss_key; 1537 } 1538 1539 return rndis_filter_set_rss_param(rndis_dev, key); 1540 } 1541 1542 /* Hyper-V RNDIS protocol does not have ring in the HW sense. 1543 * It does have pre-allocated receive area which is divided into sections. 1544 */ 1545 static void __netvsc_get_ringparam(struct netvsc_device *nvdev, 1546 struct ethtool_ringparam *ring) 1547 { 1548 u32 max_buf_size; 1549 1550 ring->rx_pending = nvdev->recv_section_cnt; 1551 ring->tx_pending = nvdev->send_section_cnt; 1552 1553 if (nvdev->nvsp_version <= NVSP_PROTOCOL_VERSION_2) 1554 max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY; 1555 else 1556 max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE; 1557 1558 ring->rx_max_pending = max_buf_size / nvdev->recv_section_size; 1559 ring->tx_max_pending = NETVSC_SEND_BUFFER_SIZE 1560 / nvdev->send_section_size; 1561 } 1562 1563 static void netvsc_get_ringparam(struct net_device *ndev, 1564 struct ethtool_ringparam *ring) 1565 { 1566 struct net_device_context *ndevctx = netdev_priv(ndev); 1567 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); 1568 1569 if (!nvdev) 1570 return; 1571 1572 __netvsc_get_ringparam(nvdev, ring); 1573 } 1574 1575 static int netvsc_set_ringparam(struct net_device *ndev, 1576 struct ethtool_ringparam *ring) 1577 { 1578 struct net_device_context *ndevctx = netdev_priv(ndev); 1579 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); 1580 struct netvsc_device_info device_info; 1581 struct ethtool_ringparam orig; 1582 u32 new_tx, new_rx; 1583 int ret = 0; 1584 1585 if (!nvdev || nvdev->destroy) 1586 return -ENODEV; 1587 1588 memset(&orig, 0, sizeof(orig)); 1589 __netvsc_get_ringparam(nvdev, &orig); 1590 1591 new_tx = clamp_t(u32, ring->tx_pending, 1592 NETVSC_MIN_TX_SECTIONS, orig.tx_max_pending); 1593 new_rx = clamp_t(u32, ring->rx_pending, 1594 NETVSC_MIN_RX_SECTIONS, orig.rx_max_pending); 1595 1596 if (new_tx == orig.tx_pending && 1597 new_rx == orig.rx_pending) 1598 return 0; /* no change */ 1599 1600 memset(&device_info, 0, sizeof(device_info)); 1601 device_info.num_chn = nvdev->num_chn; 1602 device_info.send_sections = new_tx; 1603 device_info.send_section_size = nvdev->send_section_size; 1604 device_info.recv_sections = new_rx; 1605 device_info.recv_section_size = nvdev->recv_section_size; 1606 1607 ret = netvsc_detach(ndev, nvdev); 1608 if (ret) 1609 return ret; 1610 1611 ret = netvsc_attach(ndev, &device_info); 1612 if (ret) { 1613 device_info.send_sections = orig.tx_pending; 1614 device_info.recv_sections = orig.rx_pending; 1615 1616 if (netvsc_attach(ndev, &device_info)) 1617 netdev_err(ndev, "restoring ringparam failed"); 1618 } 1619 1620 return ret; 1621 } 1622 1623 static u32 netvsc_get_msglevel(struct net_device *ndev) 1624 { 1625 struct net_device_context *ndev_ctx = netdev_priv(ndev); 1626 1627 return ndev_ctx->msg_enable; 1628 } 1629 1630 static void netvsc_set_msglevel(struct net_device *ndev, u32 val) 1631 { 1632 struct net_device_context *ndev_ctx = netdev_priv(ndev); 1633 1634 ndev_ctx->msg_enable = val; 1635 } 1636 1637 static const struct ethtool_ops ethtool_ops = { 1638 .get_drvinfo = netvsc_get_drvinfo, 1639 .get_msglevel = netvsc_get_msglevel, 1640 .set_msglevel = netvsc_set_msglevel, 1641 .get_link = ethtool_op_get_link, 1642 .get_ethtool_stats = netvsc_get_ethtool_stats, 1643 .get_sset_count = netvsc_get_sset_count, 1644 .get_strings = netvsc_get_strings, 1645 .get_channels = netvsc_get_channels, 1646 .set_channels = netvsc_set_channels, 1647 .get_ts_info = ethtool_op_get_ts_info, 1648 .get_rxnfc = netvsc_get_rxnfc, 1649 .set_rxnfc = netvsc_set_rxnfc, 1650 .get_rxfh_key_size = netvsc_get_rxfh_key_size, 1651 .get_rxfh_indir_size = netvsc_rss_indir_size, 1652 .get_rxfh = netvsc_get_rxfh, 1653 .set_rxfh = netvsc_set_rxfh, 1654 .get_link_ksettings = netvsc_get_link_ksettings, 1655 .set_link_ksettings = netvsc_set_link_ksettings, 1656 .get_ringparam = netvsc_get_ringparam, 1657 .set_ringparam = netvsc_set_ringparam, 1658 }; 1659 1660 static const struct net_device_ops device_ops = { 1661 .ndo_open = netvsc_open, 1662 .ndo_stop = netvsc_close, 1663 .ndo_start_xmit = netvsc_start_xmit, 1664 .ndo_change_rx_flags = netvsc_change_rx_flags, 1665 .ndo_set_rx_mode = netvsc_set_rx_mode, 1666 .ndo_change_mtu = netvsc_change_mtu, 1667 .ndo_validate_addr = eth_validate_addr, 1668 .ndo_set_mac_address = netvsc_set_mac_addr, 1669 .ndo_select_queue = netvsc_select_queue, 1670 .ndo_get_stats64 = netvsc_get_stats64, 1671 #ifdef CONFIG_NET_POLL_CONTROLLER 1672 .ndo_poll_controller = netvsc_poll_controller, 1673 #endif 1674 }; 1675 1676 /* 1677 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link 1678 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is 1679 * present send GARP packet to network peers with netif_notify_peers(). 1680 */ 1681 static void netvsc_link_change(struct work_struct *w) 1682 { 1683 struct net_device_context *ndev_ctx = 1684 container_of(w, struct net_device_context, dwork.work); 1685 struct hv_device *device_obj = ndev_ctx->device_ctx; 1686 struct net_device *net = hv_get_drvdata(device_obj); 1687 struct netvsc_device *net_device; 1688 struct rndis_device *rdev; 1689 struct netvsc_reconfig *event = NULL; 1690 bool notify = false, reschedule = false; 1691 unsigned long flags, next_reconfig, delay; 1692 1693 /* if changes are happening, comeback later */ 1694 if (!rtnl_trylock()) { 1695 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT); 1696 return; 1697 } 1698 1699 net_device = rtnl_dereference(ndev_ctx->nvdev); 1700 if (!net_device) 1701 goto out_unlock; 1702 1703 rdev = net_device->extension; 1704 1705 next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT; 1706 if (time_is_after_jiffies(next_reconfig)) { 1707 /* link_watch only sends one notification with current state 1708 * per second, avoid doing reconfig more frequently. Handle 1709 * wrap around. 1710 */ 1711 delay = next_reconfig - jiffies; 1712 delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT; 1713 schedule_delayed_work(&ndev_ctx->dwork, delay); 1714 goto out_unlock; 1715 } 1716 ndev_ctx->last_reconfig = jiffies; 1717 1718 spin_lock_irqsave(&ndev_ctx->lock, flags); 1719 if (!list_empty(&ndev_ctx->reconfig_events)) { 1720 event = list_first_entry(&ndev_ctx->reconfig_events, 1721 struct netvsc_reconfig, list); 1722 list_del(&event->list); 1723 reschedule = !list_empty(&ndev_ctx->reconfig_events); 1724 } 1725 spin_unlock_irqrestore(&ndev_ctx->lock, flags); 1726 1727 if (!event) 1728 goto out_unlock; 1729 1730 switch (event->event) { 1731 /* Only the following events are possible due to the check in 1732 * netvsc_linkstatus_callback() 1733 */ 1734 case RNDIS_STATUS_MEDIA_CONNECT: 1735 if (rdev->link_state) { 1736 rdev->link_state = false; 1737 netif_carrier_on(net); 1738 netif_tx_wake_all_queues(net); 1739 } else { 1740 notify = true; 1741 } 1742 kfree(event); 1743 break; 1744 case RNDIS_STATUS_MEDIA_DISCONNECT: 1745 if (!rdev->link_state) { 1746 rdev->link_state = true; 1747 netif_carrier_off(net); 1748 netif_tx_stop_all_queues(net); 1749 } 1750 kfree(event); 1751 break; 1752 case RNDIS_STATUS_NETWORK_CHANGE: 1753 /* Only makes sense if carrier is present */ 1754 if (!rdev->link_state) { 1755 rdev->link_state = true; 1756 netif_carrier_off(net); 1757 netif_tx_stop_all_queues(net); 1758 event->event = RNDIS_STATUS_MEDIA_CONNECT; 1759 spin_lock_irqsave(&ndev_ctx->lock, flags); 1760 list_add(&event->list, &ndev_ctx->reconfig_events); 1761 spin_unlock_irqrestore(&ndev_ctx->lock, flags); 1762 reschedule = true; 1763 } 1764 break; 1765 } 1766 1767 rtnl_unlock(); 1768 1769 if (notify) 1770 netdev_notify_peers(net); 1771 1772 /* link_watch only sends one notification with current state per 1773 * second, handle next reconfig event in 2 seconds. 1774 */ 1775 if (reschedule) 1776 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT); 1777 1778 return; 1779 1780 out_unlock: 1781 rtnl_unlock(); 1782 } 1783 1784 static struct net_device *get_netvsc_bymac(const u8 *mac) 1785 { 1786 struct net_device_context *ndev_ctx; 1787 1788 list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) { 1789 struct net_device *dev = hv_get_drvdata(ndev_ctx->device_ctx); 1790 1791 if (ether_addr_equal(mac, dev->perm_addr)) 1792 return dev; 1793 } 1794 1795 return NULL; 1796 } 1797 1798 static struct net_device *get_netvsc_byref(struct net_device *vf_netdev) 1799 { 1800 struct net_device_context *net_device_ctx; 1801 struct net_device *dev; 1802 1803 dev = netdev_master_upper_dev_get(vf_netdev); 1804 if (!dev || dev->netdev_ops != &device_ops) 1805 return NULL; /* not a netvsc device */ 1806 1807 net_device_ctx = netdev_priv(dev); 1808 if (!rtnl_dereference(net_device_ctx->nvdev)) 1809 return NULL; /* device is removed */ 1810 1811 return dev; 1812 } 1813 1814 /* Called when VF is injecting data into network stack. 1815 * Change the associated network device from VF to netvsc. 1816 * note: already called with rcu_read_lock 1817 */ 1818 static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb) 1819 { 1820 struct sk_buff *skb = *pskb; 1821 struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data); 1822 struct net_device_context *ndev_ctx = netdev_priv(ndev); 1823 struct netvsc_vf_pcpu_stats *pcpu_stats 1824 = this_cpu_ptr(ndev_ctx->vf_stats); 1825 1826 skb->dev = ndev; 1827 1828 u64_stats_update_begin(&pcpu_stats->syncp); 1829 pcpu_stats->rx_packets++; 1830 pcpu_stats->rx_bytes += skb->len; 1831 u64_stats_update_end(&pcpu_stats->syncp); 1832 1833 return RX_HANDLER_ANOTHER; 1834 } 1835 1836 static int netvsc_vf_join(struct net_device *vf_netdev, 1837 struct net_device *ndev) 1838 { 1839 struct net_device_context *ndev_ctx = netdev_priv(ndev); 1840 int ret; 1841 1842 ret = netdev_rx_handler_register(vf_netdev, 1843 netvsc_vf_handle_frame, ndev); 1844 if (ret != 0) { 1845 netdev_err(vf_netdev, 1846 "can not register netvsc VF receive handler (err = %d)\n", 1847 ret); 1848 goto rx_handler_failed; 1849 } 1850 1851 ret = netdev_master_upper_dev_link(vf_netdev, ndev, 1852 NULL, NULL, NULL); 1853 if (ret != 0) { 1854 netdev_err(vf_netdev, 1855 "can not set master device %s (err = %d)\n", 1856 ndev->name, ret); 1857 goto upper_link_failed; 1858 } 1859 1860 /* set slave flag before open to prevent IPv6 addrconf */ 1861 vf_netdev->flags |= IFF_SLAVE; 1862 1863 schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT); 1864 1865 call_netdevice_notifiers(NETDEV_JOIN, vf_netdev); 1866 1867 netdev_info(vf_netdev, "joined to %s\n", ndev->name); 1868 return 0; 1869 1870 upper_link_failed: 1871 netdev_rx_handler_unregister(vf_netdev); 1872 rx_handler_failed: 1873 return ret; 1874 } 1875 1876 static void __netvsc_vf_setup(struct net_device *ndev, 1877 struct net_device *vf_netdev) 1878 { 1879 int ret; 1880 1881 /* Align MTU of VF with master */ 1882 ret = dev_set_mtu(vf_netdev, ndev->mtu); 1883 if (ret) 1884 netdev_warn(vf_netdev, 1885 "unable to change mtu to %u\n", ndev->mtu); 1886 1887 /* set multicast etc flags on VF */ 1888 dev_change_flags(vf_netdev, ndev->flags | IFF_SLAVE); 1889 1890 /* sync address list from ndev to VF */ 1891 netif_addr_lock_bh(ndev); 1892 dev_uc_sync(vf_netdev, ndev); 1893 dev_mc_sync(vf_netdev, ndev); 1894 netif_addr_unlock_bh(ndev); 1895 1896 if (netif_running(ndev)) { 1897 ret = dev_open(vf_netdev); 1898 if (ret) 1899 netdev_warn(vf_netdev, 1900 "unable to open: %d\n", ret); 1901 } 1902 } 1903 1904 /* Setup VF as slave of the synthetic device. 1905 * Runs in workqueue to avoid recursion in netlink callbacks. 1906 */ 1907 static void netvsc_vf_setup(struct work_struct *w) 1908 { 1909 struct net_device_context *ndev_ctx 1910 = container_of(w, struct net_device_context, vf_takeover.work); 1911 struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx); 1912 struct net_device *vf_netdev; 1913 1914 if (!rtnl_trylock()) { 1915 schedule_delayed_work(&ndev_ctx->vf_takeover, 0); 1916 return; 1917 } 1918 1919 vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev); 1920 if (vf_netdev) 1921 __netvsc_vf_setup(ndev, vf_netdev); 1922 1923 rtnl_unlock(); 1924 } 1925 1926 static int netvsc_register_vf(struct net_device *vf_netdev) 1927 { 1928 struct net_device *ndev; 1929 struct net_device_context *net_device_ctx; 1930 struct netvsc_device *netvsc_dev; 1931 int ret; 1932 1933 if (vf_netdev->addr_len != ETH_ALEN) 1934 return NOTIFY_DONE; 1935 1936 /* 1937 * We will use the MAC address to locate the synthetic interface to 1938 * associate with the VF interface. If we don't find a matching 1939 * synthetic interface, move on. 1940 */ 1941 ndev = get_netvsc_bymac(vf_netdev->perm_addr); 1942 if (!ndev) 1943 return NOTIFY_DONE; 1944 1945 net_device_ctx = netdev_priv(ndev); 1946 netvsc_dev = rtnl_dereference(net_device_ctx->nvdev); 1947 if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev)) 1948 return NOTIFY_DONE; 1949 1950 /* if syntihetic interface is a different namespace, 1951 * then move the VF to that namespace; join will be 1952 * done again in that context. 1953 */ 1954 if (!net_eq(dev_net(ndev), dev_net(vf_netdev))) { 1955 ret = dev_change_net_namespace(vf_netdev, 1956 dev_net(ndev), "eth%d"); 1957 if (ret) 1958 netdev_err(vf_netdev, 1959 "could not move to same namespace as %s: %d\n", 1960 ndev->name, ret); 1961 else 1962 netdev_info(vf_netdev, 1963 "VF moved to namespace with: %s\n", 1964 ndev->name); 1965 return NOTIFY_DONE; 1966 } 1967 1968 netdev_info(ndev, "VF registering: %s\n", vf_netdev->name); 1969 1970 if (netvsc_vf_join(vf_netdev, ndev) != 0) 1971 return NOTIFY_DONE; 1972 1973 dev_hold(vf_netdev); 1974 rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev); 1975 return NOTIFY_OK; 1976 } 1977 1978 /* VF up/down change detected, schedule to change data path */ 1979 static int netvsc_vf_changed(struct net_device *vf_netdev) 1980 { 1981 struct net_device_context *net_device_ctx; 1982 struct netvsc_device *netvsc_dev; 1983 struct net_device *ndev; 1984 bool vf_is_up = netif_running(vf_netdev); 1985 1986 ndev = get_netvsc_byref(vf_netdev); 1987 if (!ndev) 1988 return NOTIFY_DONE; 1989 1990 net_device_ctx = netdev_priv(ndev); 1991 netvsc_dev = rtnl_dereference(net_device_ctx->nvdev); 1992 if (!netvsc_dev) 1993 return NOTIFY_DONE; 1994 1995 netvsc_switch_datapath(ndev, vf_is_up); 1996 netdev_info(ndev, "Data path switched %s VF: %s\n", 1997 vf_is_up ? "to" : "from", vf_netdev->name); 1998 1999 return NOTIFY_OK; 2000 } 2001 2002 static int netvsc_unregister_vf(struct net_device *vf_netdev) 2003 { 2004 struct net_device *ndev; 2005 struct net_device_context *net_device_ctx; 2006 2007 ndev = get_netvsc_byref(vf_netdev); 2008 if (!ndev) 2009 return NOTIFY_DONE; 2010 2011 net_device_ctx = netdev_priv(ndev); 2012 cancel_delayed_work_sync(&net_device_ctx->vf_takeover); 2013 2014 netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name); 2015 2016 netdev_rx_handler_unregister(vf_netdev); 2017 netdev_upper_dev_unlink(vf_netdev, ndev); 2018 RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL); 2019 dev_put(vf_netdev); 2020 2021 return NOTIFY_OK; 2022 } 2023 2024 static int netvsc_probe(struct hv_device *dev, 2025 const struct hv_vmbus_device_id *dev_id) 2026 { 2027 struct net_device *net = NULL; 2028 struct net_device_context *net_device_ctx; 2029 struct netvsc_device_info device_info; 2030 struct netvsc_device *nvdev; 2031 int ret = -ENOMEM; 2032 2033 net = alloc_etherdev_mq(sizeof(struct net_device_context), 2034 VRSS_CHANNEL_MAX); 2035 if (!net) 2036 goto no_net; 2037 2038 netif_carrier_off(net); 2039 2040 netvsc_init_settings(net); 2041 2042 net_device_ctx = netdev_priv(net); 2043 net_device_ctx->device_ctx = dev; 2044 net_device_ctx->msg_enable = netif_msg_init(debug, default_msg); 2045 if (netif_msg_probe(net_device_ctx)) 2046 netdev_dbg(net, "netvsc msg_enable: %d\n", 2047 net_device_ctx->msg_enable); 2048 2049 hv_set_drvdata(dev, net); 2050 2051 INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change); 2052 2053 spin_lock_init(&net_device_ctx->lock); 2054 INIT_LIST_HEAD(&net_device_ctx->reconfig_events); 2055 INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup); 2056 2057 net_device_ctx->vf_stats 2058 = netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats); 2059 if (!net_device_ctx->vf_stats) 2060 goto no_stats; 2061 2062 net->netdev_ops = &device_ops; 2063 net->ethtool_ops = ðtool_ops; 2064 SET_NETDEV_DEV(net, &dev->device); 2065 2066 /* We always need headroom for rndis header */ 2067 net->needed_headroom = RNDIS_AND_PPI_SIZE; 2068 2069 /* Initialize the number of queues to be 1, we may change it if more 2070 * channels are offered later. 2071 */ 2072 netif_set_real_num_tx_queues(net, 1); 2073 netif_set_real_num_rx_queues(net, 1); 2074 2075 /* Notify the netvsc driver of the new device */ 2076 memset(&device_info, 0, sizeof(device_info)); 2077 device_info.num_chn = VRSS_CHANNEL_DEFAULT; 2078 device_info.send_sections = NETVSC_DEFAULT_TX; 2079 device_info.send_section_size = NETVSC_SEND_SECTION_SIZE; 2080 device_info.recv_sections = NETVSC_DEFAULT_RX; 2081 device_info.recv_section_size = NETVSC_RECV_SECTION_SIZE; 2082 2083 nvdev = rndis_filter_device_add(dev, &device_info); 2084 if (IS_ERR(nvdev)) { 2085 ret = PTR_ERR(nvdev); 2086 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret); 2087 goto rndis_failed; 2088 } 2089 2090 memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN); 2091 2092 /* hw_features computed in rndis_netdev_set_hwcaps() */ 2093 net->features = net->hw_features | 2094 NETIF_F_HIGHDMA | NETIF_F_SG | 2095 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 2096 net->vlan_features = net->features; 2097 2098 netdev_lockdep_set_classes(net); 2099 2100 /* MTU range: 68 - 1500 or 65521 */ 2101 net->min_mtu = NETVSC_MTU_MIN; 2102 if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2) 2103 net->max_mtu = NETVSC_MTU - ETH_HLEN; 2104 else 2105 net->max_mtu = ETH_DATA_LEN; 2106 2107 rtnl_lock(); 2108 ret = register_netdevice(net); 2109 if (ret != 0) { 2110 pr_err("Unable to register netdev.\n"); 2111 goto register_failed; 2112 } 2113 2114 list_add(&net_device_ctx->list, &netvsc_dev_list); 2115 rtnl_unlock(); 2116 return 0; 2117 2118 register_failed: 2119 rtnl_unlock(); 2120 rndis_filter_device_remove(dev, nvdev); 2121 rndis_failed: 2122 free_percpu(net_device_ctx->vf_stats); 2123 no_stats: 2124 hv_set_drvdata(dev, NULL); 2125 free_netdev(net); 2126 no_net: 2127 return ret; 2128 } 2129 2130 static int netvsc_remove(struct hv_device *dev) 2131 { 2132 struct net_device_context *ndev_ctx; 2133 struct net_device *vf_netdev, *net; 2134 struct netvsc_device *nvdev; 2135 2136 net = hv_get_drvdata(dev); 2137 if (net == NULL) { 2138 dev_err(&dev->device, "No net device to remove\n"); 2139 return 0; 2140 } 2141 2142 ndev_ctx = netdev_priv(net); 2143 2144 cancel_delayed_work_sync(&ndev_ctx->dwork); 2145 2146 rcu_read_lock(); 2147 nvdev = rcu_dereference(ndev_ctx->nvdev); 2148 2149 if (nvdev) 2150 cancel_work_sync(&nvdev->subchan_work); 2151 2152 /* 2153 * Call to the vsc driver to let it know that the device is being 2154 * removed. Also blocks mtu and channel changes. 2155 */ 2156 rtnl_lock(); 2157 vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev); 2158 if (vf_netdev) 2159 netvsc_unregister_vf(vf_netdev); 2160 2161 if (nvdev) 2162 rndis_filter_device_remove(dev, nvdev); 2163 2164 unregister_netdevice(net); 2165 list_del(&ndev_ctx->list); 2166 2167 rtnl_unlock(); 2168 rcu_read_unlock(); 2169 2170 hv_set_drvdata(dev, NULL); 2171 2172 free_percpu(ndev_ctx->vf_stats); 2173 free_netdev(net); 2174 return 0; 2175 } 2176 2177 static const struct hv_vmbus_device_id id_table[] = { 2178 /* Network guid */ 2179 { HV_NIC_GUID, }, 2180 { }, 2181 }; 2182 2183 MODULE_DEVICE_TABLE(vmbus, id_table); 2184 2185 /* The one and only one */ 2186 static struct hv_driver netvsc_drv = { 2187 .name = KBUILD_MODNAME, 2188 .id_table = id_table, 2189 .probe = netvsc_probe, 2190 .remove = netvsc_remove, 2191 }; 2192 2193 /* 2194 * On Hyper-V, every VF interface is matched with a corresponding 2195 * synthetic interface. The synthetic interface is presented first 2196 * to the guest. When the corresponding VF instance is registered, 2197 * we will take care of switching the data path. 2198 */ 2199 static int netvsc_netdev_event(struct notifier_block *this, 2200 unsigned long event, void *ptr) 2201 { 2202 struct net_device *event_dev = netdev_notifier_info_to_dev(ptr); 2203 2204 /* Skip our own events */ 2205 if (event_dev->netdev_ops == &device_ops) 2206 return NOTIFY_DONE; 2207 2208 /* Avoid non-Ethernet type devices */ 2209 if (event_dev->type != ARPHRD_ETHER) 2210 return NOTIFY_DONE; 2211 2212 /* Avoid Vlan dev with same MAC registering as VF */ 2213 if (is_vlan_dev(event_dev)) 2214 return NOTIFY_DONE; 2215 2216 /* Avoid Bonding master dev with same MAC registering as VF */ 2217 if ((event_dev->priv_flags & IFF_BONDING) && 2218 (event_dev->flags & IFF_MASTER)) 2219 return NOTIFY_DONE; 2220 2221 switch (event) { 2222 case NETDEV_REGISTER: 2223 return netvsc_register_vf(event_dev); 2224 case NETDEV_UNREGISTER: 2225 return netvsc_unregister_vf(event_dev); 2226 case NETDEV_UP: 2227 case NETDEV_DOWN: 2228 return netvsc_vf_changed(event_dev); 2229 default: 2230 return NOTIFY_DONE; 2231 } 2232 } 2233 2234 static struct notifier_block netvsc_netdev_notifier = { 2235 .notifier_call = netvsc_netdev_event, 2236 }; 2237 2238 static void __exit netvsc_drv_exit(void) 2239 { 2240 unregister_netdevice_notifier(&netvsc_netdev_notifier); 2241 vmbus_driver_unregister(&netvsc_drv); 2242 } 2243 2244 static int __init netvsc_drv_init(void) 2245 { 2246 int ret; 2247 2248 if (ring_size < RING_SIZE_MIN) { 2249 ring_size = RING_SIZE_MIN; 2250 pr_info("Increased ring_size to %u (min allowed)\n", 2251 ring_size); 2252 } 2253 netvsc_ring_bytes = ring_size * PAGE_SIZE; 2254 2255 ret = vmbus_driver_register(&netvsc_drv); 2256 if (ret) 2257 return ret; 2258 2259 register_netdevice_notifier(&netvsc_netdev_notifier); 2260 return 0; 2261 } 2262 2263 MODULE_LICENSE("GPL"); 2264 MODULE_DESCRIPTION("Microsoft Hyper-V network driver"); 2265 2266 module_init(netvsc_drv_init); 2267 module_exit(netvsc_drv_exit); 2268