1 /* 2 * Copyright (c) 2009, Microsoft Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, see <http://www.gnu.org/licenses/>. 15 * 16 * Authors: 17 * Haiyang Zhang <haiyangz@microsoft.com> 18 * Hank Janssen <hjanssen@microsoft.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include <linux/init.h> 23 #include <linux/atomic.h> 24 #include <linux/module.h> 25 #include <linux/highmem.h> 26 #include <linux/device.h> 27 #include <linux/io.h> 28 #include <linux/delay.h> 29 #include <linux/netdevice.h> 30 #include <linux/inetdevice.h> 31 #include <linux/etherdevice.h> 32 #include <linux/skbuff.h> 33 #include <linux/if_vlan.h> 34 #include <linux/in.h> 35 #include <linux/slab.h> 36 #include <net/arp.h> 37 #include <net/route.h> 38 #include <net/sock.h> 39 #include <net/pkt_sched.h> 40 41 #include "hyperv_net.h" 42 43 44 #define RING_SIZE_MIN 64 45 static int ring_size = 128; 46 module_param(ring_size, int, S_IRUGO); 47 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)"); 48 49 static int max_num_vrss_chns = 8; 50 51 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | 52 NETIF_MSG_LINK | NETIF_MSG_IFUP | 53 NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR | 54 NETIF_MSG_TX_ERR; 55 56 static int debug = -1; 57 module_param(debug, int, S_IRUGO); 58 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 59 60 static void do_set_multicast(struct work_struct *w) 61 { 62 struct net_device_context *ndevctx = 63 container_of(w, struct net_device_context, work); 64 struct netvsc_device *nvdev; 65 struct rndis_device *rdev; 66 67 nvdev = hv_get_drvdata(ndevctx->device_ctx); 68 if (nvdev == NULL || nvdev->ndev == NULL) 69 return; 70 71 rdev = nvdev->extension; 72 if (rdev == NULL) 73 return; 74 75 if (nvdev->ndev->flags & IFF_PROMISC) 76 rndis_filter_set_packet_filter(rdev, 77 NDIS_PACKET_TYPE_PROMISCUOUS); 78 else 79 rndis_filter_set_packet_filter(rdev, 80 NDIS_PACKET_TYPE_BROADCAST | 81 NDIS_PACKET_TYPE_ALL_MULTICAST | 82 NDIS_PACKET_TYPE_DIRECTED); 83 } 84 85 static void netvsc_set_multicast_list(struct net_device *net) 86 { 87 struct net_device_context *net_device_ctx = netdev_priv(net); 88 89 schedule_work(&net_device_ctx->work); 90 } 91 92 static int netvsc_open(struct net_device *net) 93 { 94 struct net_device_context *net_device_ctx = netdev_priv(net); 95 struct hv_device *device_obj = net_device_ctx->device_ctx; 96 struct netvsc_device *nvdev; 97 struct rndis_device *rdev; 98 int ret = 0; 99 100 netif_carrier_off(net); 101 102 /* Open up the device */ 103 ret = rndis_filter_open(device_obj); 104 if (ret != 0) { 105 netdev_err(net, "unable to open device (ret %d).\n", ret); 106 return ret; 107 } 108 109 netif_tx_wake_all_queues(net); 110 111 nvdev = hv_get_drvdata(device_obj); 112 rdev = nvdev->extension; 113 if (!rdev->link_state) 114 netif_carrier_on(net); 115 116 return ret; 117 } 118 119 static int netvsc_close(struct net_device *net) 120 { 121 struct net_device_context *net_device_ctx = netdev_priv(net); 122 struct hv_device *device_obj = net_device_ctx->device_ctx; 123 struct netvsc_device *nvdev = hv_get_drvdata(device_obj); 124 int ret; 125 u32 aread, awrite, i, msec = 10, retry = 0, retry_max = 20; 126 struct vmbus_channel *chn; 127 128 netif_tx_disable(net); 129 130 /* Make sure netvsc_set_multicast_list doesn't re-enable filter! */ 131 cancel_work_sync(&net_device_ctx->work); 132 ret = rndis_filter_close(device_obj); 133 if (ret != 0) { 134 netdev_err(net, "unable to close device (ret %d).\n", ret); 135 return ret; 136 } 137 138 /* Ensure pending bytes in ring are read */ 139 while (true) { 140 aread = 0; 141 for (i = 0; i < nvdev->num_chn; i++) { 142 chn = nvdev->chn_table[i]; 143 if (!chn) 144 continue; 145 146 hv_get_ringbuffer_availbytes(&chn->inbound, &aread, 147 &awrite); 148 149 if (aread) 150 break; 151 152 hv_get_ringbuffer_availbytes(&chn->outbound, &aread, 153 &awrite); 154 155 if (aread) 156 break; 157 } 158 159 retry++; 160 if (retry > retry_max || aread == 0) 161 break; 162 163 msleep(msec); 164 165 if (msec < 1000) 166 msec *= 2; 167 } 168 169 if (aread) { 170 netdev_err(net, "Ring buffer not empty after closing rndis\n"); 171 ret = -ETIMEDOUT; 172 } 173 174 return ret; 175 } 176 177 static void *init_ppi_data(struct rndis_message *msg, u32 ppi_size, 178 int pkt_type) 179 { 180 struct rndis_packet *rndis_pkt; 181 struct rndis_per_packet_info *ppi; 182 183 rndis_pkt = &msg->msg.pkt; 184 rndis_pkt->data_offset += ppi_size; 185 186 ppi = (struct rndis_per_packet_info *)((void *)rndis_pkt + 187 rndis_pkt->per_pkt_info_offset + rndis_pkt->per_pkt_info_len); 188 189 ppi->size = ppi_size; 190 ppi->type = pkt_type; 191 ppi->ppi_offset = sizeof(struct rndis_per_packet_info); 192 193 rndis_pkt->per_pkt_info_len += ppi_size; 194 195 return ppi; 196 } 197 198 union sub_key { 199 u64 k; 200 struct { 201 u8 pad[3]; 202 u8 kb; 203 u32 ka; 204 }; 205 }; 206 207 /* Toeplitz hash function 208 * data: network byte order 209 * return: host byte order 210 */ 211 static u32 comp_hash(u8 *key, int klen, void *data, int dlen) 212 { 213 union sub_key subk; 214 int k_next = 4; 215 u8 dt; 216 int i, j; 217 u32 ret = 0; 218 219 subk.k = 0; 220 subk.ka = ntohl(*(u32 *)key); 221 222 for (i = 0; i < dlen; i++) { 223 subk.kb = key[k_next]; 224 k_next = (k_next + 1) % klen; 225 dt = ((u8 *)data)[i]; 226 for (j = 0; j < 8; j++) { 227 if (dt & 0x80) 228 ret ^= subk.ka; 229 dt <<= 1; 230 subk.k <<= 1; 231 } 232 } 233 234 return ret; 235 } 236 237 static bool netvsc_set_hash(u32 *hash, struct sk_buff *skb) 238 { 239 struct flow_keys flow; 240 int data_len; 241 242 if (!skb_flow_dissect_flow_keys(skb, &flow, 0) || 243 !(flow.basic.n_proto == htons(ETH_P_IP) || 244 flow.basic.n_proto == htons(ETH_P_IPV6))) 245 return false; 246 247 if (flow.basic.ip_proto == IPPROTO_TCP) 248 data_len = 12; 249 else 250 data_len = 8; 251 252 *hash = comp_hash(netvsc_hash_key, HASH_KEYLEN, &flow, data_len); 253 254 return true; 255 } 256 257 static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb, 258 void *accel_priv, select_queue_fallback_t fallback) 259 { 260 struct net_device_context *net_device_ctx = netdev_priv(ndev); 261 struct hv_device *hdev = net_device_ctx->device_ctx; 262 struct netvsc_device *nvsc_dev = hv_get_drvdata(hdev); 263 u32 hash; 264 u16 q_idx = 0; 265 266 if (nvsc_dev == NULL || ndev->real_num_tx_queues <= 1) 267 return 0; 268 269 if (netvsc_set_hash(&hash, skb)) { 270 q_idx = nvsc_dev->send_table[hash % VRSS_SEND_TAB_SIZE] % 271 ndev->real_num_tx_queues; 272 skb_set_hash(skb, hash, PKT_HASH_TYPE_L3); 273 } 274 275 return q_idx; 276 } 277 278 void netvsc_xmit_completion(void *context) 279 { 280 struct hv_netvsc_packet *packet = (struct hv_netvsc_packet *)context; 281 struct sk_buff *skb = (struct sk_buff *) 282 (unsigned long)packet->send_completion_tid; 283 284 if (skb) 285 dev_kfree_skb_any(skb); 286 } 287 288 static u32 fill_pg_buf(struct page *page, u32 offset, u32 len, 289 struct hv_page_buffer *pb) 290 { 291 int j = 0; 292 293 /* Deal with compund pages by ignoring unused part 294 * of the page. 295 */ 296 page += (offset >> PAGE_SHIFT); 297 offset &= ~PAGE_MASK; 298 299 while (len > 0) { 300 unsigned long bytes; 301 302 bytes = PAGE_SIZE - offset; 303 if (bytes > len) 304 bytes = len; 305 pb[j].pfn = page_to_pfn(page); 306 pb[j].offset = offset; 307 pb[j].len = bytes; 308 309 offset += bytes; 310 len -= bytes; 311 312 if (offset == PAGE_SIZE && len) { 313 page++; 314 offset = 0; 315 j++; 316 } 317 } 318 319 return j + 1; 320 } 321 322 static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb, 323 struct hv_netvsc_packet *packet) 324 { 325 struct hv_page_buffer *pb = packet->page_buf; 326 u32 slots_used = 0; 327 char *data = skb->data; 328 int frags = skb_shinfo(skb)->nr_frags; 329 int i; 330 331 /* The packet is laid out thus: 332 * 1. hdr: RNDIS header and PPI 333 * 2. skb linear data 334 * 3. skb fragment data 335 */ 336 if (hdr != NULL) 337 slots_used += fill_pg_buf(virt_to_page(hdr), 338 offset_in_page(hdr), 339 len, &pb[slots_used]); 340 341 packet->rmsg_size = len; 342 packet->rmsg_pgcnt = slots_used; 343 344 slots_used += fill_pg_buf(virt_to_page(data), 345 offset_in_page(data), 346 skb_headlen(skb), &pb[slots_used]); 347 348 for (i = 0; i < frags; i++) { 349 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 350 351 slots_used += fill_pg_buf(skb_frag_page(frag), 352 frag->page_offset, 353 skb_frag_size(frag), &pb[slots_used]); 354 } 355 return slots_used; 356 } 357 358 static int count_skb_frag_slots(struct sk_buff *skb) 359 { 360 int i, frags = skb_shinfo(skb)->nr_frags; 361 int pages = 0; 362 363 for (i = 0; i < frags; i++) { 364 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 365 unsigned long size = skb_frag_size(frag); 366 unsigned long offset = frag->page_offset; 367 368 /* Skip unused frames from start of page */ 369 offset &= ~PAGE_MASK; 370 pages += PFN_UP(offset + size); 371 } 372 return pages; 373 } 374 375 static int netvsc_get_slots(struct sk_buff *skb) 376 { 377 char *data = skb->data; 378 unsigned int offset = offset_in_page(data); 379 unsigned int len = skb_headlen(skb); 380 int slots; 381 int frag_slots; 382 383 slots = DIV_ROUND_UP(offset + len, PAGE_SIZE); 384 frag_slots = count_skb_frag_slots(skb); 385 return slots + frag_slots; 386 } 387 388 static u32 get_net_transport_info(struct sk_buff *skb, u32 *trans_off) 389 { 390 u32 ret_val = TRANSPORT_INFO_NOT_IP; 391 392 if ((eth_hdr(skb)->h_proto != htons(ETH_P_IP)) && 393 (eth_hdr(skb)->h_proto != htons(ETH_P_IPV6))) { 394 goto not_ip; 395 } 396 397 *trans_off = skb_transport_offset(skb); 398 399 if ((eth_hdr(skb)->h_proto == htons(ETH_P_IP))) { 400 struct iphdr *iphdr = ip_hdr(skb); 401 402 if (iphdr->protocol == IPPROTO_TCP) 403 ret_val = TRANSPORT_INFO_IPV4_TCP; 404 else if (iphdr->protocol == IPPROTO_UDP) 405 ret_val = TRANSPORT_INFO_IPV4_UDP; 406 } else { 407 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 408 ret_val = TRANSPORT_INFO_IPV6_TCP; 409 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP) 410 ret_val = TRANSPORT_INFO_IPV6_UDP; 411 } 412 413 not_ip: 414 return ret_val; 415 } 416 417 static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net) 418 { 419 struct net_device_context *net_device_ctx = netdev_priv(net); 420 struct hv_netvsc_packet *packet = NULL; 421 int ret; 422 unsigned int num_data_pgs; 423 struct rndis_message *rndis_msg; 424 struct rndis_packet *rndis_pkt; 425 u32 rndis_msg_size; 426 bool isvlan; 427 bool linear = false; 428 struct rndis_per_packet_info *ppi; 429 struct ndis_tcp_ip_checksum_info *csum_info; 430 struct ndis_tcp_lso_info *lso_info; 431 int hdr_offset; 432 u32 net_trans_info; 433 u32 hash; 434 u32 skb_length; 435 u32 pkt_sz; 436 struct hv_page_buffer page_buf[MAX_PAGE_BUFFER_COUNT]; 437 struct netvsc_stats *tx_stats = this_cpu_ptr(net_device_ctx->tx_stats); 438 439 /* We will atmost need two pages to describe the rndis 440 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number 441 * of pages in a single packet. If skb is scattered around 442 * more pages we try linearizing it. 443 */ 444 445 check_size: 446 skb_length = skb->len; 447 num_data_pgs = netvsc_get_slots(skb) + 2; 448 if (num_data_pgs > MAX_PAGE_BUFFER_COUNT && linear) { 449 net_alert_ratelimited("packet too big: %u pages (%u bytes)\n", 450 num_data_pgs, skb->len); 451 ret = -EFAULT; 452 goto drop; 453 } else if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) { 454 if (skb_linearize(skb)) { 455 net_alert_ratelimited("failed to linearize skb\n"); 456 ret = -ENOMEM; 457 goto drop; 458 } 459 linear = true; 460 goto check_size; 461 } 462 463 pkt_sz = sizeof(struct hv_netvsc_packet) + RNDIS_AND_PPI_SIZE; 464 465 ret = skb_cow_head(skb, pkt_sz); 466 if (ret) { 467 netdev_err(net, "unable to alloc hv_netvsc_packet\n"); 468 ret = -ENOMEM; 469 goto drop; 470 } 471 /* Use the headroom for building up the packet */ 472 packet = (struct hv_netvsc_packet *)skb->head; 473 474 packet->status = 0; 475 packet->xmit_more = skb->xmit_more; 476 477 packet->vlan_tci = skb->vlan_tci; 478 packet->page_buf = page_buf; 479 480 packet->q_idx = skb_get_queue_mapping(skb); 481 482 packet->is_data_pkt = true; 483 packet->total_data_buflen = skb->len; 484 485 packet->rndis_msg = (struct rndis_message *)((unsigned long)packet + 486 sizeof(struct hv_netvsc_packet)); 487 488 memset(packet->rndis_msg, 0, RNDIS_AND_PPI_SIZE); 489 490 /* Set the completion routine */ 491 packet->send_completion = netvsc_xmit_completion; 492 packet->send_completion_ctx = packet; 493 packet->send_completion_tid = (unsigned long)skb; 494 495 isvlan = packet->vlan_tci & VLAN_TAG_PRESENT; 496 497 /* Add the rndis header */ 498 rndis_msg = packet->rndis_msg; 499 rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET; 500 rndis_msg->msg_len = packet->total_data_buflen; 501 rndis_pkt = &rndis_msg->msg.pkt; 502 rndis_pkt->data_offset = sizeof(struct rndis_packet); 503 rndis_pkt->data_len = packet->total_data_buflen; 504 rndis_pkt->per_pkt_info_offset = sizeof(struct rndis_packet); 505 506 rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet); 507 508 hash = skb_get_hash_raw(skb); 509 if (hash != 0 && net->real_num_tx_queues > 1) { 510 rndis_msg_size += NDIS_HASH_PPI_SIZE; 511 ppi = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE, 512 NBL_HASH_VALUE); 513 *(u32 *)((void *)ppi + ppi->ppi_offset) = hash; 514 } 515 516 if (isvlan) { 517 struct ndis_pkt_8021q_info *vlan; 518 519 rndis_msg_size += NDIS_VLAN_PPI_SIZE; 520 ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE, 521 IEEE_8021Q_INFO); 522 vlan = (struct ndis_pkt_8021q_info *)((void *)ppi + 523 ppi->ppi_offset); 524 vlan->vlanid = packet->vlan_tci & VLAN_VID_MASK; 525 vlan->pri = (packet->vlan_tci & VLAN_PRIO_MASK) >> 526 VLAN_PRIO_SHIFT; 527 } 528 529 net_trans_info = get_net_transport_info(skb, &hdr_offset); 530 if (net_trans_info == TRANSPORT_INFO_NOT_IP) 531 goto do_send; 532 533 /* 534 * Setup the sendside checksum offload only if this is not a 535 * GSO packet. 536 */ 537 if (skb_is_gso(skb)) 538 goto do_lso; 539 540 if ((skb->ip_summed == CHECKSUM_NONE) || 541 (skb->ip_summed == CHECKSUM_UNNECESSARY)) 542 goto do_send; 543 544 rndis_msg_size += NDIS_CSUM_PPI_SIZE; 545 ppi = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE, 546 TCPIP_CHKSUM_PKTINFO); 547 548 csum_info = (struct ndis_tcp_ip_checksum_info *)((void *)ppi + 549 ppi->ppi_offset); 550 551 if (net_trans_info & (INFO_IPV4 << 16)) 552 csum_info->transmit.is_ipv4 = 1; 553 else 554 csum_info->transmit.is_ipv6 = 1; 555 556 if (net_trans_info & INFO_TCP) { 557 csum_info->transmit.tcp_checksum = 1; 558 csum_info->transmit.tcp_header_offset = hdr_offset; 559 } else if (net_trans_info & INFO_UDP) { 560 /* UDP checksum offload is not supported on ws2008r2. 561 * Furthermore, on ws2012 and ws2012r2, there are some 562 * issues with udp checksum offload from Linux guests. 563 * (these are host issues). 564 * For now compute the checksum here. 565 */ 566 struct udphdr *uh; 567 u16 udp_len; 568 569 ret = skb_cow_head(skb, 0); 570 if (ret) 571 goto drop; 572 573 uh = udp_hdr(skb); 574 udp_len = ntohs(uh->len); 575 uh->check = 0; 576 uh->check = csum_tcpudp_magic(ip_hdr(skb)->saddr, 577 ip_hdr(skb)->daddr, 578 udp_len, IPPROTO_UDP, 579 csum_partial(uh, udp_len, 0)); 580 if (uh->check == 0) 581 uh->check = CSUM_MANGLED_0; 582 583 csum_info->transmit.udp_checksum = 0; 584 } 585 goto do_send; 586 587 do_lso: 588 rndis_msg_size += NDIS_LSO_PPI_SIZE; 589 ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE, 590 TCP_LARGESEND_PKTINFO); 591 592 lso_info = (struct ndis_tcp_lso_info *)((void *)ppi + 593 ppi->ppi_offset); 594 595 lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE; 596 if (net_trans_info & (INFO_IPV4 << 16)) { 597 lso_info->lso_v2_transmit.ip_version = 598 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4; 599 ip_hdr(skb)->tot_len = 0; 600 ip_hdr(skb)->check = 0; 601 tcp_hdr(skb)->check = 602 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 603 ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); 604 } else { 605 lso_info->lso_v2_transmit.ip_version = 606 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6; 607 ipv6_hdr(skb)->payload_len = 0; 608 tcp_hdr(skb)->check = 609 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 610 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); 611 } 612 lso_info->lso_v2_transmit.tcp_header_offset = hdr_offset; 613 lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size; 614 615 do_send: 616 /* Start filling in the page buffers with the rndis hdr */ 617 rndis_msg->msg_len += rndis_msg_size; 618 packet->total_data_buflen = rndis_msg->msg_len; 619 packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size, 620 skb, packet); 621 622 ret = netvsc_send(net_device_ctx->device_ctx, packet); 623 624 drop: 625 if (ret == 0) { 626 u64_stats_update_begin(&tx_stats->syncp); 627 tx_stats->packets++; 628 tx_stats->bytes += skb_length; 629 u64_stats_update_end(&tx_stats->syncp); 630 } else { 631 if (ret != -EAGAIN) { 632 dev_kfree_skb_any(skb); 633 net->stats.tx_dropped++; 634 } 635 } 636 637 return (ret == -EAGAIN) ? NETDEV_TX_BUSY : NETDEV_TX_OK; 638 } 639 640 /* 641 * netvsc_linkstatus_callback - Link up/down notification 642 */ 643 void netvsc_linkstatus_callback(struct hv_device *device_obj, 644 struct rndis_message *resp) 645 { 646 struct rndis_indicate_status *indicate = &resp->msg.indicate_status; 647 struct net_device *net; 648 struct net_device_context *ndev_ctx; 649 struct netvsc_device *net_device; 650 struct rndis_device *rdev; 651 652 net_device = hv_get_drvdata(device_obj); 653 rdev = net_device->extension; 654 655 switch (indicate->status) { 656 case RNDIS_STATUS_MEDIA_CONNECT: 657 rdev->link_state = false; 658 break; 659 case RNDIS_STATUS_MEDIA_DISCONNECT: 660 rdev->link_state = true; 661 break; 662 case RNDIS_STATUS_NETWORK_CHANGE: 663 rdev->link_change = true; 664 break; 665 default: 666 return; 667 } 668 669 net = net_device->ndev; 670 671 if (!net || net->reg_state != NETREG_REGISTERED) 672 return; 673 674 ndev_ctx = netdev_priv(net); 675 if (!rdev->link_state) { 676 schedule_delayed_work(&ndev_ctx->dwork, 0); 677 schedule_delayed_work(&ndev_ctx->dwork, msecs_to_jiffies(20)); 678 } else { 679 schedule_delayed_work(&ndev_ctx->dwork, 0); 680 } 681 } 682 683 /* 684 * netvsc_recv_callback - Callback when we receive a packet from the 685 * "wire" on the specified device. 686 */ 687 int netvsc_recv_callback(struct hv_device *device_obj, 688 struct hv_netvsc_packet *packet, 689 struct ndis_tcp_ip_checksum_info *csum_info) 690 { 691 struct net_device *net; 692 struct net_device_context *net_device_ctx; 693 struct sk_buff *skb; 694 struct netvsc_stats *rx_stats; 695 696 net = ((struct netvsc_device *)hv_get_drvdata(device_obj))->ndev; 697 if (!net || net->reg_state != NETREG_REGISTERED) { 698 packet->status = NVSP_STAT_FAIL; 699 return 0; 700 } 701 net_device_ctx = netdev_priv(net); 702 rx_stats = this_cpu_ptr(net_device_ctx->rx_stats); 703 704 /* Allocate a skb - TODO direct I/O to pages? */ 705 skb = netdev_alloc_skb_ip_align(net, packet->total_data_buflen); 706 if (unlikely(!skb)) { 707 ++net->stats.rx_dropped; 708 packet->status = NVSP_STAT_FAIL; 709 return 0; 710 } 711 712 /* 713 * Copy to skb. This copy is needed here since the memory pointed by 714 * hv_netvsc_packet cannot be deallocated 715 */ 716 memcpy(skb_put(skb, packet->total_data_buflen), packet->data, 717 packet->total_data_buflen); 718 719 skb->protocol = eth_type_trans(skb, net); 720 if (csum_info) { 721 /* We only look at the IP checksum here. 722 * Should we be dropping the packet if checksum 723 * failed? How do we deal with other checksums - TCP/UDP? 724 */ 725 if (csum_info->receive.ip_checksum_succeeded) 726 skb->ip_summed = CHECKSUM_UNNECESSARY; 727 else 728 skb->ip_summed = CHECKSUM_NONE; 729 } 730 731 if (packet->vlan_tci & VLAN_TAG_PRESENT) 732 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 733 packet->vlan_tci); 734 735 skb_record_rx_queue(skb, packet->channel-> 736 offermsg.offer.sub_channel_index); 737 738 u64_stats_update_begin(&rx_stats->syncp); 739 rx_stats->packets++; 740 rx_stats->bytes += packet->total_data_buflen; 741 u64_stats_update_end(&rx_stats->syncp); 742 743 /* 744 * Pass the skb back up. Network stack will deallocate the skb when it 745 * is done. 746 * TODO - use NAPI? 747 */ 748 netif_rx(skb); 749 750 return 0; 751 } 752 753 static void netvsc_get_drvinfo(struct net_device *net, 754 struct ethtool_drvinfo *info) 755 { 756 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 757 strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); 758 } 759 760 static void netvsc_get_channels(struct net_device *net, 761 struct ethtool_channels *channel) 762 { 763 struct net_device_context *net_device_ctx = netdev_priv(net); 764 struct hv_device *dev = net_device_ctx->device_ctx; 765 struct netvsc_device *nvdev = hv_get_drvdata(dev); 766 767 if (nvdev) { 768 channel->max_combined = nvdev->max_chn; 769 channel->combined_count = nvdev->num_chn; 770 } 771 } 772 773 static int netvsc_set_channels(struct net_device *net, 774 struct ethtool_channels *channels) 775 { 776 struct net_device_context *net_device_ctx = netdev_priv(net); 777 struct hv_device *dev = net_device_ctx->device_ctx; 778 struct netvsc_device *nvdev = hv_get_drvdata(dev); 779 struct netvsc_device_info device_info; 780 u32 num_chn; 781 u32 max_chn; 782 int ret = 0; 783 bool recovering = false; 784 785 if (!nvdev || nvdev->destroy) 786 return -ENODEV; 787 788 num_chn = nvdev->num_chn; 789 max_chn = min_t(u32, nvdev->max_chn, num_online_cpus()); 790 791 if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5) { 792 pr_info("vRSS unsupported before NVSP Version 5\n"); 793 return -EINVAL; 794 } 795 796 /* We do not support rx, tx, or other */ 797 if (!channels || 798 channels->rx_count || 799 channels->tx_count || 800 channels->other_count || 801 (channels->combined_count < 1)) 802 return -EINVAL; 803 804 if (channels->combined_count > max_chn) { 805 pr_info("combined channels too high, using %d\n", max_chn); 806 channels->combined_count = max_chn; 807 } 808 809 ret = netvsc_close(net); 810 if (ret) 811 goto out; 812 813 do_set: 814 nvdev->start_remove = true; 815 rndis_filter_device_remove(dev); 816 817 nvdev->num_chn = channels->combined_count; 818 819 net_device_ctx->device_ctx = dev; 820 hv_set_drvdata(dev, net); 821 822 memset(&device_info, 0, sizeof(device_info)); 823 device_info.num_chn = nvdev->num_chn; /* passed to RNDIS */ 824 device_info.ring_size = ring_size; 825 device_info.max_num_vrss_chns = max_num_vrss_chns; 826 827 ret = rndis_filter_device_add(dev, &device_info); 828 if (ret) { 829 if (recovering) { 830 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret); 831 return ret; 832 } 833 goto recover; 834 } 835 836 nvdev = hv_get_drvdata(dev); 837 838 ret = netif_set_real_num_tx_queues(net, nvdev->num_chn); 839 if (ret) { 840 if (recovering) { 841 netdev_err(net, "could not set tx queue count (ret %d)\n", ret); 842 return ret; 843 } 844 goto recover; 845 } 846 847 ret = netif_set_real_num_rx_queues(net, nvdev->num_chn); 848 if (ret) { 849 if (recovering) { 850 netdev_err(net, "could not set rx queue count (ret %d)\n", ret); 851 return ret; 852 } 853 goto recover; 854 } 855 856 out: 857 netvsc_open(net); 858 859 return ret; 860 861 recover: 862 /* If the above failed, we attempt to recover through the same 863 * process but with the original number of channels. 864 */ 865 netdev_err(net, "could not set channels, recovering\n"); 866 recovering = true; 867 channels->combined_count = num_chn; 868 goto do_set; 869 } 870 871 static int netvsc_change_mtu(struct net_device *ndev, int mtu) 872 { 873 struct net_device_context *ndevctx = netdev_priv(ndev); 874 struct hv_device *hdev = ndevctx->device_ctx; 875 struct netvsc_device *nvdev = hv_get_drvdata(hdev); 876 struct netvsc_device_info device_info; 877 int limit = ETH_DATA_LEN; 878 int ret = 0; 879 880 if (nvdev == NULL || nvdev->destroy) 881 return -ENODEV; 882 883 if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2) 884 limit = NETVSC_MTU - ETH_HLEN; 885 886 if (mtu < NETVSC_MTU_MIN || mtu > limit) 887 return -EINVAL; 888 889 ret = netvsc_close(ndev); 890 if (ret) 891 goto out; 892 893 nvdev->start_remove = true; 894 rndis_filter_device_remove(hdev); 895 896 ndev->mtu = mtu; 897 898 ndevctx->device_ctx = hdev; 899 hv_set_drvdata(hdev, ndev); 900 901 memset(&device_info, 0, sizeof(device_info)); 902 device_info.ring_size = ring_size; 903 device_info.num_chn = nvdev->num_chn; 904 device_info.max_num_vrss_chns = max_num_vrss_chns; 905 rndis_filter_device_add(hdev, &device_info); 906 907 out: 908 netvsc_open(ndev); 909 910 return ret; 911 } 912 913 static struct rtnl_link_stats64 *netvsc_get_stats64(struct net_device *net, 914 struct rtnl_link_stats64 *t) 915 { 916 struct net_device_context *ndev_ctx = netdev_priv(net); 917 int cpu; 918 919 for_each_possible_cpu(cpu) { 920 struct netvsc_stats *tx_stats = per_cpu_ptr(ndev_ctx->tx_stats, 921 cpu); 922 struct netvsc_stats *rx_stats = per_cpu_ptr(ndev_ctx->rx_stats, 923 cpu); 924 u64 tx_packets, tx_bytes, rx_packets, rx_bytes; 925 unsigned int start; 926 927 do { 928 start = u64_stats_fetch_begin_irq(&tx_stats->syncp); 929 tx_packets = tx_stats->packets; 930 tx_bytes = tx_stats->bytes; 931 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); 932 933 do { 934 start = u64_stats_fetch_begin_irq(&rx_stats->syncp); 935 rx_packets = rx_stats->packets; 936 rx_bytes = rx_stats->bytes; 937 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); 938 939 t->tx_bytes += tx_bytes; 940 t->tx_packets += tx_packets; 941 t->rx_bytes += rx_bytes; 942 t->rx_packets += rx_packets; 943 } 944 945 t->tx_dropped = net->stats.tx_dropped; 946 t->tx_errors = net->stats.tx_dropped; 947 948 t->rx_dropped = net->stats.rx_dropped; 949 t->rx_errors = net->stats.rx_errors; 950 951 return t; 952 } 953 954 static int netvsc_set_mac_addr(struct net_device *ndev, void *p) 955 { 956 struct net_device_context *ndevctx = netdev_priv(ndev); 957 struct hv_device *hdev = ndevctx->device_ctx; 958 struct sockaddr *addr = p; 959 char save_adr[ETH_ALEN]; 960 unsigned char save_aatype; 961 int err; 962 963 memcpy(save_adr, ndev->dev_addr, ETH_ALEN); 964 save_aatype = ndev->addr_assign_type; 965 966 err = eth_mac_addr(ndev, p); 967 if (err != 0) 968 return err; 969 970 err = rndis_filter_set_device_mac(hdev, addr->sa_data); 971 if (err != 0) { 972 /* roll back to saved MAC */ 973 memcpy(ndev->dev_addr, save_adr, ETH_ALEN); 974 ndev->addr_assign_type = save_aatype; 975 } 976 977 return err; 978 } 979 980 #ifdef CONFIG_NET_POLL_CONTROLLER 981 static void netvsc_poll_controller(struct net_device *net) 982 { 983 /* As netvsc_start_xmit() works synchronous we don't have to 984 * trigger anything here. 985 */ 986 } 987 #endif 988 989 static const struct ethtool_ops ethtool_ops = { 990 .get_drvinfo = netvsc_get_drvinfo, 991 .get_link = ethtool_op_get_link, 992 .get_channels = netvsc_get_channels, 993 .set_channels = netvsc_set_channels, 994 }; 995 996 static const struct net_device_ops device_ops = { 997 .ndo_open = netvsc_open, 998 .ndo_stop = netvsc_close, 999 .ndo_start_xmit = netvsc_start_xmit, 1000 .ndo_set_rx_mode = netvsc_set_multicast_list, 1001 .ndo_change_mtu = netvsc_change_mtu, 1002 .ndo_validate_addr = eth_validate_addr, 1003 .ndo_set_mac_address = netvsc_set_mac_addr, 1004 .ndo_select_queue = netvsc_select_queue, 1005 .ndo_get_stats64 = netvsc_get_stats64, 1006 #ifdef CONFIG_NET_POLL_CONTROLLER 1007 .ndo_poll_controller = netvsc_poll_controller, 1008 #endif 1009 }; 1010 1011 /* 1012 * Send GARP packet to network peers after migrations. 1013 * After Quick Migration, the network is not immediately operational in the 1014 * current context when receiving RNDIS_STATUS_MEDIA_CONNECT event. So, add 1015 * another netif_notify_peers() into a delayed work, otherwise GARP packet 1016 * will not be sent after quick migration, and cause network disconnection. 1017 * Also, we update the carrier status here. 1018 */ 1019 static void netvsc_link_change(struct work_struct *w) 1020 { 1021 struct net_device_context *ndev_ctx; 1022 struct net_device *net; 1023 struct netvsc_device *net_device; 1024 struct rndis_device *rdev; 1025 bool notify, refresh = false; 1026 char *argv[] = { "/etc/init.d/network", "restart", NULL }; 1027 char *envp[] = { "HOME=/", "PATH=/sbin:/usr/sbin:/bin:/usr/bin", NULL }; 1028 1029 rtnl_lock(); 1030 1031 ndev_ctx = container_of(w, struct net_device_context, dwork.work); 1032 net_device = hv_get_drvdata(ndev_ctx->device_ctx); 1033 rdev = net_device->extension; 1034 net = net_device->ndev; 1035 1036 if (rdev->link_state) { 1037 netif_carrier_off(net); 1038 notify = false; 1039 } else { 1040 netif_carrier_on(net); 1041 notify = true; 1042 if (rdev->link_change) { 1043 rdev->link_change = false; 1044 refresh = true; 1045 } 1046 } 1047 1048 rtnl_unlock(); 1049 1050 if (refresh) 1051 call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); 1052 1053 if (notify) 1054 netdev_notify_peers(net); 1055 } 1056 1057 static void netvsc_free_netdev(struct net_device *netdev) 1058 { 1059 struct net_device_context *net_device_ctx = netdev_priv(netdev); 1060 1061 free_percpu(net_device_ctx->tx_stats); 1062 free_percpu(net_device_ctx->rx_stats); 1063 free_netdev(netdev); 1064 } 1065 1066 static int netvsc_probe(struct hv_device *dev, 1067 const struct hv_vmbus_device_id *dev_id) 1068 { 1069 struct net_device *net = NULL; 1070 struct net_device_context *net_device_ctx; 1071 struct netvsc_device_info device_info; 1072 struct netvsc_device *nvdev; 1073 int ret; 1074 u32 max_needed_headroom; 1075 1076 net = alloc_etherdev_mq(sizeof(struct net_device_context), 1077 num_online_cpus()); 1078 if (!net) 1079 return -ENOMEM; 1080 1081 max_needed_headroom = sizeof(struct hv_netvsc_packet) + 1082 RNDIS_AND_PPI_SIZE; 1083 1084 netif_carrier_off(net); 1085 1086 net_device_ctx = netdev_priv(net); 1087 net_device_ctx->device_ctx = dev; 1088 net_device_ctx->msg_enable = netif_msg_init(debug, default_msg); 1089 if (netif_msg_probe(net_device_ctx)) 1090 netdev_dbg(net, "netvsc msg_enable: %d\n", 1091 net_device_ctx->msg_enable); 1092 1093 net_device_ctx->tx_stats = netdev_alloc_pcpu_stats(struct netvsc_stats); 1094 if (!net_device_ctx->tx_stats) { 1095 free_netdev(net); 1096 return -ENOMEM; 1097 } 1098 net_device_ctx->rx_stats = netdev_alloc_pcpu_stats(struct netvsc_stats); 1099 if (!net_device_ctx->rx_stats) { 1100 free_percpu(net_device_ctx->tx_stats); 1101 free_netdev(net); 1102 return -ENOMEM; 1103 } 1104 1105 hv_set_drvdata(dev, net); 1106 INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change); 1107 INIT_WORK(&net_device_ctx->work, do_set_multicast); 1108 1109 net->netdev_ops = &device_ops; 1110 1111 net->hw_features = NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_IP_CSUM | 1112 NETIF_F_TSO; 1113 net->features = NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_SG | NETIF_F_RXCSUM | 1114 NETIF_F_IP_CSUM | NETIF_F_TSO; 1115 1116 net->ethtool_ops = ðtool_ops; 1117 SET_NETDEV_DEV(net, &dev->device); 1118 1119 /* 1120 * Request additional head room in the skb. 1121 * We will use this space to build the rndis 1122 * heaser and other state we need to maintain. 1123 */ 1124 net->needed_headroom = max_needed_headroom; 1125 1126 /* Notify the netvsc driver of the new device */ 1127 memset(&device_info, 0, sizeof(device_info)); 1128 device_info.ring_size = ring_size; 1129 device_info.max_num_vrss_chns = max_num_vrss_chns; 1130 ret = rndis_filter_device_add(dev, &device_info); 1131 if (ret != 0) { 1132 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret); 1133 netvsc_free_netdev(net); 1134 hv_set_drvdata(dev, NULL); 1135 return ret; 1136 } 1137 memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN); 1138 1139 nvdev = hv_get_drvdata(dev); 1140 netif_set_real_num_tx_queues(net, nvdev->num_chn); 1141 netif_set_real_num_rx_queues(net, nvdev->num_chn); 1142 1143 ret = register_netdev(net); 1144 if (ret != 0) { 1145 pr_err("Unable to register netdev.\n"); 1146 rndis_filter_device_remove(dev); 1147 netvsc_free_netdev(net); 1148 } else { 1149 schedule_delayed_work(&net_device_ctx->dwork, 0); 1150 } 1151 1152 return ret; 1153 } 1154 1155 static int netvsc_remove(struct hv_device *dev) 1156 { 1157 struct net_device *net; 1158 struct net_device_context *ndev_ctx; 1159 struct netvsc_device *net_device; 1160 1161 net_device = hv_get_drvdata(dev); 1162 net = net_device->ndev; 1163 1164 if (net == NULL) { 1165 dev_err(&dev->device, "No net device to remove\n"); 1166 return 0; 1167 } 1168 1169 net_device->start_remove = true; 1170 1171 ndev_ctx = netdev_priv(net); 1172 cancel_delayed_work_sync(&ndev_ctx->dwork); 1173 cancel_work_sync(&ndev_ctx->work); 1174 1175 /* Stop outbound asap */ 1176 netif_tx_disable(net); 1177 1178 unregister_netdev(net); 1179 1180 /* 1181 * Call to the vsc driver to let it know that the device is being 1182 * removed 1183 */ 1184 rndis_filter_device_remove(dev); 1185 1186 netvsc_free_netdev(net); 1187 return 0; 1188 } 1189 1190 static const struct hv_vmbus_device_id id_table[] = { 1191 /* Network guid */ 1192 { HV_NIC_GUID, }, 1193 { }, 1194 }; 1195 1196 MODULE_DEVICE_TABLE(vmbus, id_table); 1197 1198 /* The one and only one */ 1199 static struct hv_driver netvsc_drv = { 1200 .name = KBUILD_MODNAME, 1201 .id_table = id_table, 1202 .probe = netvsc_probe, 1203 .remove = netvsc_remove, 1204 }; 1205 1206 static void __exit netvsc_drv_exit(void) 1207 { 1208 vmbus_driver_unregister(&netvsc_drv); 1209 } 1210 1211 static int __init netvsc_drv_init(void) 1212 { 1213 if (ring_size < RING_SIZE_MIN) { 1214 ring_size = RING_SIZE_MIN; 1215 pr_info("Increased ring_size to %d (min allowed)\n", 1216 ring_size); 1217 } 1218 return vmbus_driver_register(&netvsc_drv); 1219 } 1220 1221 MODULE_LICENSE("GPL"); 1222 MODULE_DESCRIPTION("Microsoft Hyper-V network driver"); 1223 1224 module_init(netvsc_drv_init); 1225 module_exit(netvsc_drv_exit); 1226