1 /* 2 * Copyright (c) 2009, Microsoft Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, see <http://www.gnu.org/licenses/>. 15 * 16 * Authors: 17 * Haiyang Zhang <haiyangz@microsoft.com> 18 * Hank Janssen <hjanssen@microsoft.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include <linux/init.h> 23 #include <linux/atomic.h> 24 #include <linux/module.h> 25 #include <linux/highmem.h> 26 #include <linux/device.h> 27 #include <linux/io.h> 28 #include <linux/delay.h> 29 #include <linux/netdevice.h> 30 #include <linux/inetdevice.h> 31 #include <linux/etherdevice.h> 32 #include <linux/skbuff.h> 33 #include <linux/if_vlan.h> 34 #include <linux/in.h> 35 #include <linux/slab.h> 36 #include <net/arp.h> 37 #include <net/route.h> 38 #include <net/sock.h> 39 #include <net/pkt_sched.h> 40 41 #include "hyperv_net.h" 42 43 44 #define RING_SIZE_MIN 64 45 static int ring_size = 128; 46 module_param(ring_size, int, S_IRUGO); 47 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)"); 48 49 static int max_num_vrss_chns = 8; 50 51 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | 52 NETIF_MSG_LINK | NETIF_MSG_IFUP | 53 NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR | 54 NETIF_MSG_TX_ERR; 55 56 static int debug = -1; 57 module_param(debug, int, S_IRUGO); 58 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 59 60 static void do_set_multicast(struct work_struct *w) 61 { 62 struct net_device_context *ndevctx = 63 container_of(w, struct net_device_context, work); 64 struct netvsc_device *nvdev; 65 struct rndis_device *rdev; 66 67 nvdev = hv_get_drvdata(ndevctx->device_ctx); 68 if (nvdev == NULL || nvdev->ndev == NULL) 69 return; 70 71 rdev = nvdev->extension; 72 if (rdev == NULL) 73 return; 74 75 if (nvdev->ndev->flags & IFF_PROMISC) 76 rndis_filter_set_packet_filter(rdev, 77 NDIS_PACKET_TYPE_PROMISCUOUS); 78 else 79 rndis_filter_set_packet_filter(rdev, 80 NDIS_PACKET_TYPE_BROADCAST | 81 NDIS_PACKET_TYPE_ALL_MULTICAST | 82 NDIS_PACKET_TYPE_DIRECTED); 83 } 84 85 static void netvsc_set_multicast_list(struct net_device *net) 86 { 87 struct net_device_context *net_device_ctx = netdev_priv(net); 88 89 schedule_work(&net_device_ctx->work); 90 } 91 92 static int netvsc_open(struct net_device *net) 93 { 94 struct net_device_context *net_device_ctx = netdev_priv(net); 95 struct hv_device *device_obj = net_device_ctx->device_ctx; 96 struct netvsc_device *nvdev; 97 struct rndis_device *rdev; 98 int ret = 0; 99 100 netif_carrier_off(net); 101 102 /* Open up the device */ 103 ret = rndis_filter_open(device_obj); 104 if (ret != 0) { 105 netdev_err(net, "unable to open device (ret %d).\n", ret); 106 return ret; 107 } 108 109 netif_tx_wake_all_queues(net); 110 111 nvdev = hv_get_drvdata(device_obj); 112 rdev = nvdev->extension; 113 if (!rdev->link_state) 114 netif_carrier_on(net); 115 116 return ret; 117 } 118 119 static int netvsc_close(struct net_device *net) 120 { 121 struct net_device_context *net_device_ctx = netdev_priv(net); 122 struct hv_device *device_obj = net_device_ctx->device_ctx; 123 struct netvsc_device *nvdev = hv_get_drvdata(device_obj); 124 int ret; 125 u32 aread, awrite, i, msec = 10, retry = 0, retry_max = 20; 126 struct vmbus_channel *chn; 127 128 netif_tx_disable(net); 129 130 /* Make sure netvsc_set_multicast_list doesn't re-enable filter! */ 131 cancel_work_sync(&net_device_ctx->work); 132 ret = rndis_filter_close(device_obj); 133 if (ret != 0) { 134 netdev_err(net, "unable to close device (ret %d).\n", ret); 135 return ret; 136 } 137 138 /* Ensure pending bytes in ring are read */ 139 while (true) { 140 aread = 0; 141 for (i = 0; i < nvdev->num_chn; i++) { 142 chn = nvdev->chn_table[i]; 143 if (!chn) 144 continue; 145 146 hv_get_ringbuffer_availbytes(&chn->inbound, &aread, 147 &awrite); 148 149 if (aread) 150 break; 151 152 hv_get_ringbuffer_availbytes(&chn->outbound, &aread, 153 &awrite); 154 155 if (aread) 156 break; 157 } 158 159 retry++; 160 if (retry > retry_max || aread == 0) 161 break; 162 163 msleep(msec); 164 165 if (msec < 1000) 166 msec *= 2; 167 } 168 169 if (aread) { 170 netdev_err(net, "Ring buffer not empty after closing rndis\n"); 171 ret = -ETIMEDOUT; 172 } 173 174 return ret; 175 } 176 177 static void *init_ppi_data(struct rndis_message *msg, u32 ppi_size, 178 int pkt_type) 179 { 180 struct rndis_packet *rndis_pkt; 181 struct rndis_per_packet_info *ppi; 182 183 rndis_pkt = &msg->msg.pkt; 184 rndis_pkt->data_offset += ppi_size; 185 186 ppi = (struct rndis_per_packet_info *)((void *)rndis_pkt + 187 rndis_pkt->per_pkt_info_offset + rndis_pkt->per_pkt_info_len); 188 189 ppi->size = ppi_size; 190 ppi->type = pkt_type; 191 ppi->ppi_offset = sizeof(struct rndis_per_packet_info); 192 193 rndis_pkt->per_pkt_info_len += ppi_size; 194 195 return ppi; 196 } 197 198 union sub_key { 199 u64 k; 200 struct { 201 u8 pad[3]; 202 u8 kb; 203 u32 ka; 204 }; 205 }; 206 207 /* Toeplitz hash function 208 * data: network byte order 209 * return: host byte order 210 */ 211 static u32 comp_hash(u8 *key, int klen, void *data, int dlen) 212 { 213 union sub_key subk; 214 int k_next = 4; 215 u8 dt; 216 int i, j; 217 u32 ret = 0; 218 219 subk.k = 0; 220 subk.ka = ntohl(*(u32 *)key); 221 222 for (i = 0; i < dlen; i++) { 223 subk.kb = key[k_next]; 224 k_next = (k_next + 1) % klen; 225 dt = ((u8 *)data)[i]; 226 for (j = 0; j < 8; j++) { 227 if (dt & 0x80) 228 ret ^= subk.ka; 229 dt <<= 1; 230 subk.k <<= 1; 231 } 232 } 233 234 return ret; 235 } 236 237 static bool netvsc_set_hash(u32 *hash, struct sk_buff *skb) 238 { 239 struct flow_keys flow; 240 int data_len; 241 242 if (!skb_flow_dissect_flow_keys(skb, &flow) || 243 !(flow.basic.n_proto == htons(ETH_P_IP) || 244 flow.basic.n_proto == htons(ETH_P_IPV6))) 245 return false; 246 247 if (flow.basic.ip_proto == IPPROTO_TCP) 248 data_len = 12; 249 else 250 data_len = 8; 251 252 *hash = comp_hash(netvsc_hash_key, HASH_KEYLEN, &flow, data_len); 253 254 return true; 255 } 256 257 static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb, 258 void *accel_priv, select_queue_fallback_t fallback) 259 { 260 struct net_device_context *net_device_ctx = netdev_priv(ndev); 261 struct hv_device *hdev = net_device_ctx->device_ctx; 262 struct netvsc_device *nvsc_dev = hv_get_drvdata(hdev); 263 u32 hash; 264 u16 q_idx = 0; 265 266 if (nvsc_dev == NULL || ndev->real_num_tx_queues <= 1) 267 return 0; 268 269 if (netvsc_set_hash(&hash, skb)) { 270 q_idx = nvsc_dev->send_table[hash % VRSS_SEND_TAB_SIZE] % 271 ndev->real_num_tx_queues; 272 skb_set_hash(skb, hash, PKT_HASH_TYPE_L3); 273 } 274 275 return q_idx; 276 } 277 278 void netvsc_xmit_completion(void *context) 279 { 280 struct hv_netvsc_packet *packet = (struct hv_netvsc_packet *)context; 281 struct sk_buff *skb = (struct sk_buff *) 282 (unsigned long)packet->send_completion_tid; 283 284 if (skb) 285 dev_kfree_skb_any(skb); 286 } 287 288 static u32 fill_pg_buf(struct page *page, u32 offset, u32 len, 289 struct hv_page_buffer *pb) 290 { 291 int j = 0; 292 293 /* Deal with compund pages by ignoring unused part 294 * of the page. 295 */ 296 page += (offset >> PAGE_SHIFT); 297 offset &= ~PAGE_MASK; 298 299 while (len > 0) { 300 unsigned long bytes; 301 302 bytes = PAGE_SIZE - offset; 303 if (bytes > len) 304 bytes = len; 305 pb[j].pfn = page_to_pfn(page); 306 pb[j].offset = offset; 307 pb[j].len = bytes; 308 309 offset += bytes; 310 len -= bytes; 311 312 if (offset == PAGE_SIZE && len) { 313 page++; 314 offset = 0; 315 j++; 316 } 317 } 318 319 return j + 1; 320 } 321 322 static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb, 323 struct hv_netvsc_packet *packet) 324 { 325 struct hv_page_buffer *pb = packet->page_buf; 326 u32 slots_used = 0; 327 char *data = skb->data; 328 int frags = skb_shinfo(skb)->nr_frags; 329 int i; 330 331 /* The packet is laid out thus: 332 * 1. hdr: RNDIS header and PPI 333 * 2. skb linear data 334 * 3. skb fragment data 335 */ 336 if (hdr != NULL) 337 slots_used += fill_pg_buf(virt_to_page(hdr), 338 offset_in_page(hdr), 339 len, &pb[slots_used]); 340 341 packet->rmsg_size = len; 342 packet->rmsg_pgcnt = slots_used; 343 344 slots_used += fill_pg_buf(virt_to_page(data), 345 offset_in_page(data), 346 skb_headlen(skb), &pb[slots_used]); 347 348 for (i = 0; i < frags; i++) { 349 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 350 351 slots_used += fill_pg_buf(skb_frag_page(frag), 352 frag->page_offset, 353 skb_frag_size(frag), &pb[slots_used]); 354 } 355 return slots_used; 356 } 357 358 static int count_skb_frag_slots(struct sk_buff *skb) 359 { 360 int i, frags = skb_shinfo(skb)->nr_frags; 361 int pages = 0; 362 363 for (i = 0; i < frags; i++) { 364 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 365 unsigned long size = skb_frag_size(frag); 366 unsigned long offset = frag->page_offset; 367 368 /* Skip unused frames from start of page */ 369 offset &= ~PAGE_MASK; 370 pages += PFN_UP(offset + size); 371 } 372 return pages; 373 } 374 375 static int netvsc_get_slots(struct sk_buff *skb) 376 { 377 char *data = skb->data; 378 unsigned int offset = offset_in_page(data); 379 unsigned int len = skb_headlen(skb); 380 int slots; 381 int frag_slots; 382 383 slots = DIV_ROUND_UP(offset + len, PAGE_SIZE); 384 frag_slots = count_skb_frag_slots(skb); 385 return slots + frag_slots; 386 } 387 388 static u32 get_net_transport_info(struct sk_buff *skb, u32 *trans_off) 389 { 390 u32 ret_val = TRANSPORT_INFO_NOT_IP; 391 392 if ((eth_hdr(skb)->h_proto != htons(ETH_P_IP)) && 393 (eth_hdr(skb)->h_proto != htons(ETH_P_IPV6))) { 394 goto not_ip; 395 } 396 397 *trans_off = skb_transport_offset(skb); 398 399 if ((eth_hdr(skb)->h_proto == htons(ETH_P_IP))) { 400 struct iphdr *iphdr = ip_hdr(skb); 401 402 if (iphdr->protocol == IPPROTO_TCP) 403 ret_val = TRANSPORT_INFO_IPV4_TCP; 404 else if (iphdr->protocol == IPPROTO_UDP) 405 ret_val = TRANSPORT_INFO_IPV4_UDP; 406 } else { 407 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 408 ret_val = TRANSPORT_INFO_IPV6_TCP; 409 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP) 410 ret_val = TRANSPORT_INFO_IPV6_UDP; 411 } 412 413 not_ip: 414 return ret_val; 415 } 416 417 static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net) 418 { 419 struct net_device_context *net_device_ctx = netdev_priv(net); 420 struct hv_netvsc_packet *packet = NULL; 421 int ret; 422 unsigned int num_data_pgs; 423 struct rndis_message *rndis_msg; 424 struct rndis_packet *rndis_pkt; 425 u32 rndis_msg_size; 426 bool isvlan; 427 bool linear = false; 428 struct rndis_per_packet_info *ppi; 429 struct ndis_tcp_ip_checksum_info *csum_info; 430 struct ndis_tcp_lso_info *lso_info; 431 int hdr_offset; 432 u32 net_trans_info; 433 u32 hash; 434 u32 skb_length; 435 u32 pkt_sz; 436 struct hv_page_buffer page_buf[MAX_PAGE_BUFFER_COUNT]; 437 struct netvsc_stats *tx_stats = this_cpu_ptr(net_device_ctx->tx_stats); 438 439 /* We will atmost need two pages to describe the rndis 440 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number 441 * of pages in a single packet. If skb is scattered around 442 * more pages we try linearizing it. 443 */ 444 445 check_size: 446 skb_length = skb->len; 447 num_data_pgs = netvsc_get_slots(skb) + 2; 448 if (num_data_pgs > MAX_PAGE_BUFFER_COUNT && linear) { 449 net_alert_ratelimited("packet too big: %u pages (%u bytes)\n", 450 num_data_pgs, skb->len); 451 ret = -EFAULT; 452 goto drop; 453 } else if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) { 454 if (skb_linearize(skb)) { 455 net_alert_ratelimited("failed to linearize skb\n"); 456 ret = -ENOMEM; 457 goto drop; 458 } 459 linear = true; 460 goto check_size; 461 } 462 463 pkt_sz = sizeof(struct hv_netvsc_packet) + RNDIS_AND_PPI_SIZE; 464 465 ret = skb_cow_head(skb, pkt_sz); 466 if (ret) { 467 netdev_err(net, "unable to alloc hv_netvsc_packet\n"); 468 ret = -ENOMEM; 469 goto drop; 470 } 471 /* Use the headroom for building up the packet */ 472 packet = (struct hv_netvsc_packet *)skb->head; 473 474 packet->status = 0; 475 packet->xmit_more = skb->xmit_more; 476 477 packet->vlan_tci = skb->vlan_tci; 478 packet->page_buf = page_buf; 479 480 packet->q_idx = skb_get_queue_mapping(skb); 481 482 packet->is_data_pkt = true; 483 packet->total_data_buflen = skb->len; 484 485 packet->rndis_msg = (struct rndis_message *)((unsigned long)packet + 486 sizeof(struct hv_netvsc_packet)); 487 488 memset(packet->rndis_msg, 0, RNDIS_AND_PPI_SIZE); 489 490 /* Set the completion routine */ 491 packet->send_completion = netvsc_xmit_completion; 492 packet->send_completion_ctx = packet; 493 packet->send_completion_tid = (unsigned long)skb; 494 495 isvlan = packet->vlan_tci & VLAN_TAG_PRESENT; 496 497 /* Add the rndis header */ 498 rndis_msg = packet->rndis_msg; 499 rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET; 500 rndis_msg->msg_len = packet->total_data_buflen; 501 rndis_pkt = &rndis_msg->msg.pkt; 502 rndis_pkt->data_offset = sizeof(struct rndis_packet); 503 rndis_pkt->data_len = packet->total_data_buflen; 504 rndis_pkt->per_pkt_info_offset = sizeof(struct rndis_packet); 505 506 rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet); 507 508 hash = skb_get_hash_raw(skb); 509 if (hash != 0 && net->real_num_tx_queues > 1) { 510 rndis_msg_size += NDIS_HASH_PPI_SIZE; 511 ppi = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE, 512 NBL_HASH_VALUE); 513 *(u32 *)((void *)ppi + ppi->ppi_offset) = hash; 514 } 515 516 if (isvlan) { 517 struct ndis_pkt_8021q_info *vlan; 518 519 rndis_msg_size += NDIS_VLAN_PPI_SIZE; 520 ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE, 521 IEEE_8021Q_INFO); 522 vlan = (struct ndis_pkt_8021q_info *)((void *)ppi + 523 ppi->ppi_offset); 524 vlan->vlanid = packet->vlan_tci & VLAN_VID_MASK; 525 vlan->pri = (packet->vlan_tci & VLAN_PRIO_MASK) >> 526 VLAN_PRIO_SHIFT; 527 } 528 529 net_trans_info = get_net_transport_info(skb, &hdr_offset); 530 if (net_trans_info == TRANSPORT_INFO_NOT_IP) 531 goto do_send; 532 533 /* 534 * Setup the sendside checksum offload only if this is not a 535 * GSO packet. 536 */ 537 if (skb_is_gso(skb)) 538 goto do_lso; 539 540 if ((skb->ip_summed == CHECKSUM_NONE) || 541 (skb->ip_summed == CHECKSUM_UNNECESSARY)) 542 goto do_send; 543 544 rndis_msg_size += NDIS_CSUM_PPI_SIZE; 545 ppi = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE, 546 TCPIP_CHKSUM_PKTINFO); 547 548 csum_info = (struct ndis_tcp_ip_checksum_info *)((void *)ppi + 549 ppi->ppi_offset); 550 551 if (net_trans_info & (INFO_IPV4 << 16)) 552 csum_info->transmit.is_ipv4 = 1; 553 else 554 csum_info->transmit.is_ipv6 = 1; 555 556 if (net_trans_info & INFO_TCP) { 557 csum_info->transmit.tcp_checksum = 1; 558 csum_info->transmit.tcp_header_offset = hdr_offset; 559 } else if (net_trans_info & INFO_UDP) { 560 /* UDP checksum offload is not supported on ws2008r2. 561 * Furthermore, on ws2012 and ws2012r2, there are some 562 * issues with udp checksum offload from Linux guests. 563 * (these are host issues). 564 * For now compute the checksum here. 565 */ 566 struct udphdr *uh; 567 u16 udp_len; 568 569 ret = skb_cow_head(skb, 0); 570 if (ret) 571 goto drop; 572 573 uh = udp_hdr(skb); 574 udp_len = ntohs(uh->len); 575 uh->check = 0; 576 uh->check = csum_tcpudp_magic(ip_hdr(skb)->saddr, 577 ip_hdr(skb)->daddr, 578 udp_len, IPPROTO_UDP, 579 csum_partial(uh, udp_len, 0)); 580 if (uh->check == 0) 581 uh->check = CSUM_MANGLED_0; 582 583 csum_info->transmit.udp_checksum = 0; 584 } 585 goto do_send; 586 587 do_lso: 588 rndis_msg_size += NDIS_LSO_PPI_SIZE; 589 ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE, 590 TCP_LARGESEND_PKTINFO); 591 592 lso_info = (struct ndis_tcp_lso_info *)((void *)ppi + 593 ppi->ppi_offset); 594 595 lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE; 596 if (net_trans_info & (INFO_IPV4 << 16)) { 597 lso_info->lso_v2_transmit.ip_version = 598 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4; 599 ip_hdr(skb)->tot_len = 0; 600 ip_hdr(skb)->check = 0; 601 tcp_hdr(skb)->check = 602 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 603 ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); 604 } else { 605 lso_info->lso_v2_transmit.ip_version = 606 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6; 607 ipv6_hdr(skb)->payload_len = 0; 608 tcp_hdr(skb)->check = 609 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 610 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); 611 } 612 lso_info->lso_v2_transmit.tcp_header_offset = hdr_offset; 613 lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size; 614 615 do_send: 616 /* Start filling in the page buffers with the rndis hdr */ 617 rndis_msg->msg_len += rndis_msg_size; 618 packet->total_data_buflen = rndis_msg->msg_len; 619 packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size, 620 skb, packet); 621 622 ret = netvsc_send(net_device_ctx->device_ctx, packet); 623 624 drop: 625 if (ret == 0) { 626 u64_stats_update_begin(&tx_stats->syncp); 627 tx_stats->packets++; 628 tx_stats->bytes += skb_length; 629 u64_stats_update_end(&tx_stats->syncp); 630 } else { 631 if (ret != -EAGAIN) { 632 dev_kfree_skb_any(skb); 633 net->stats.tx_dropped++; 634 } 635 } 636 637 return (ret == -EAGAIN) ? NETDEV_TX_BUSY : NETDEV_TX_OK; 638 } 639 640 /* 641 * netvsc_linkstatus_callback - Link up/down notification 642 */ 643 void netvsc_linkstatus_callback(struct hv_device *device_obj, 644 struct rndis_message *resp) 645 { 646 struct rndis_indicate_status *indicate = &resp->msg.indicate_status; 647 struct net_device *net; 648 struct net_device_context *ndev_ctx; 649 struct netvsc_device *net_device; 650 struct rndis_device *rdev; 651 652 net_device = hv_get_drvdata(device_obj); 653 rdev = net_device->extension; 654 655 switch (indicate->status) { 656 case RNDIS_STATUS_MEDIA_CONNECT: 657 rdev->link_state = false; 658 break; 659 case RNDIS_STATUS_MEDIA_DISCONNECT: 660 rdev->link_state = true; 661 break; 662 case RNDIS_STATUS_NETWORK_CHANGE: 663 rdev->link_change = true; 664 break; 665 default: 666 return; 667 } 668 669 net = net_device->ndev; 670 671 if (!net || net->reg_state != NETREG_REGISTERED) 672 return; 673 674 ndev_ctx = netdev_priv(net); 675 if (!rdev->link_state) { 676 schedule_delayed_work(&ndev_ctx->dwork, 0); 677 schedule_delayed_work(&ndev_ctx->dwork, msecs_to_jiffies(20)); 678 } else { 679 schedule_delayed_work(&ndev_ctx->dwork, 0); 680 } 681 } 682 683 /* 684 * netvsc_recv_callback - Callback when we receive a packet from the 685 * "wire" on the specified device. 686 */ 687 int netvsc_recv_callback(struct hv_device *device_obj, 688 struct hv_netvsc_packet *packet, 689 struct ndis_tcp_ip_checksum_info *csum_info) 690 { 691 struct net_device *net; 692 struct net_device_context *net_device_ctx; 693 struct sk_buff *skb; 694 struct netvsc_stats *rx_stats; 695 696 net = ((struct netvsc_device *)hv_get_drvdata(device_obj))->ndev; 697 if (!net || net->reg_state != NETREG_REGISTERED) { 698 packet->status = NVSP_STAT_FAIL; 699 return 0; 700 } 701 net_device_ctx = netdev_priv(net); 702 rx_stats = this_cpu_ptr(net_device_ctx->rx_stats); 703 704 /* Allocate a skb - TODO direct I/O to pages? */ 705 skb = netdev_alloc_skb_ip_align(net, packet->total_data_buflen); 706 if (unlikely(!skb)) { 707 ++net->stats.rx_dropped; 708 packet->status = NVSP_STAT_FAIL; 709 return 0; 710 } 711 712 /* 713 * Copy to skb. This copy is needed here since the memory pointed by 714 * hv_netvsc_packet cannot be deallocated 715 */ 716 memcpy(skb_put(skb, packet->total_data_buflen), packet->data, 717 packet->total_data_buflen); 718 719 skb->protocol = eth_type_trans(skb, net); 720 if (csum_info) { 721 /* We only look at the IP checksum here. 722 * Should we be dropping the packet if checksum 723 * failed? How do we deal with other checksums - TCP/UDP? 724 */ 725 if (csum_info->receive.ip_checksum_succeeded) 726 skb->ip_summed = CHECKSUM_UNNECESSARY; 727 else 728 skb->ip_summed = CHECKSUM_NONE; 729 } 730 731 if (packet->vlan_tci & VLAN_TAG_PRESENT) 732 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 733 packet->vlan_tci); 734 735 skb_record_rx_queue(skb, packet->channel-> 736 offermsg.offer.sub_channel_index); 737 738 u64_stats_update_begin(&rx_stats->syncp); 739 rx_stats->packets++; 740 rx_stats->bytes += packet->total_data_buflen; 741 u64_stats_update_end(&rx_stats->syncp); 742 743 /* 744 * Pass the skb back up. Network stack will deallocate the skb when it 745 * is done. 746 * TODO - use NAPI? 747 */ 748 netif_rx(skb); 749 750 return 0; 751 } 752 753 static void netvsc_get_drvinfo(struct net_device *net, 754 struct ethtool_drvinfo *info) 755 { 756 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 757 strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); 758 } 759 760 static void netvsc_get_channels(struct net_device *net, 761 struct ethtool_channels *channel) 762 { 763 struct net_device_context *net_device_ctx = netdev_priv(net); 764 struct hv_device *dev = net_device_ctx->device_ctx; 765 struct netvsc_device *nvdev = hv_get_drvdata(dev); 766 767 if (nvdev) { 768 channel->max_combined = nvdev->max_chn; 769 channel->combined_count = nvdev->num_chn; 770 } 771 } 772 773 static int netvsc_change_mtu(struct net_device *ndev, int mtu) 774 { 775 struct net_device_context *ndevctx = netdev_priv(ndev); 776 struct hv_device *hdev = ndevctx->device_ctx; 777 struct netvsc_device *nvdev = hv_get_drvdata(hdev); 778 struct netvsc_device_info device_info; 779 int limit = ETH_DATA_LEN; 780 int ret = 0; 781 782 if (nvdev == NULL || nvdev->destroy) 783 return -ENODEV; 784 785 if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2) 786 limit = NETVSC_MTU - ETH_HLEN; 787 788 if (mtu < NETVSC_MTU_MIN || mtu > limit) 789 return -EINVAL; 790 791 ret = netvsc_close(ndev); 792 if (ret) 793 goto out; 794 795 nvdev->start_remove = true; 796 rndis_filter_device_remove(hdev); 797 798 ndev->mtu = mtu; 799 800 ndevctx->device_ctx = hdev; 801 hv_set_drvdata(hdev, ndev); 802 device_info.ring_size = ring_size; 803 device_info.max_num_vrss_chns = max_num_vrss_chns; 804 rndis_filter_device_add(hdev, &device_info); 805 806 out: 807 netvsc_open(ndev); 808 809 return ret; 810 } 811 812 static struct rtnl_link_stats64 *netvsc_get_stats64(struct net_device *net, 813 struct rtnl_link_stats64 *t) 814 { 815 struct net_device_context *ndev_ctx = netdev_priv(net); 816 int cpu; 817 818 for_each_possible_cpu(cpu) { 819 struct netvsc_stats *tx_stats = per_cpu_ptr(ndev_ctx->tx_stats, 820 cpu); 821 struct netvsc_stats *rx_stats = per_cpu_ptr(ndev_ctx->rx_stats, 822 cpu); 823 u64 tx_packets, tx_bytes, rx_packets, rx_bytes; 824 unsigned int start; 825 826 do { 827 start = u64_stats_fetch_begin_irq(&tx_stats->syncp); 828 tx_packets = tx_stats->packets; 829 tx_bytes = tx_stats->bytes; 830 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); 831 832 do { 833 start = u64_stats_fetch_begin_irq(&rx_stats->syncp); 834 rx_packets = rx_stats->packets; 835 rx_bytes = rx_stats->bytes; 836 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); 837 838 t->tx_bytes += tx_bytes; 839 t->tx_packets += tx_packets; 840 t->rx_bytes += rx_bytes; 841 t->rx_packets += rx_packets; 842 } 843 844 t->tx_dropped = net->stats.tx_dropped; 845 t->tx_errors = net->stats.tx_dropped; 846 847 t->rx_dropped = net->stats.rx_dropped; 848 t->rx_errors = net->stats.rx_errors; 849 850 return t; 851 } 852 853 static int netvsc_set_mac_addr(struct net_device *ndev, void *p) 854 { 855 struct net_device_context *ndevctx = netdev_priv(ndev); 856 struct hv_device *hdev = ndevctx->device_ctx; 857 struct sockaddr *addr = p; 858 char save_adr[ETH_ALEN]; 859 unsigned char save_aatype; 860 int err; 861 862 memcpy(save_adr, ndev->dev_addr, ETH_ALEN); 863 save_aatype = ndev->addr_assign_type; 864 865 err = eth_mac_addr(ndev, p); 866 if (err != 0) 867 return err; 868 869 err = rndis_filter_set_device_mac(hdev, addr->sa_data); 870 if (err != 0) { 871 /* roll back to saved MAC */ 872 memcpy(ndev->dev_addr, save_adr, ETH_ALEN); 873 ndev->addr_assign_type = save_aatype; 874 } 875 876 return err; 877 } 878 879 #ifdef CONFIG_NET_POLL_CONTROLLER 880 static void netvsc_poll_controller(struct net_device *net) 881 { 882 /* As netvsc_start_xmit() works synchronous we don't have to 883 * trigger anything here. 884 */ 885 } 886 #endif 887 888 static const struct ethtool_ops ethtool_ops = { 889 .get_drvinfo = netvsc_get_drvinfo, 890 .get_link = ethtool_op_get_link, 891 .get_channels = netvsc_get_channels, 892 }; 893 894 static const struct net_device_ops device_ops = { 895 .ndo_open = netvsc_open, 896 .ndo_stop = netvsc_close, 897 .ndo_start_xmit = netvsc_start_xmit, 898 .ndo_set_rx_mode = netvsc_set_multicast_list, 899 .ndo_change_mtu = netvsc_change_mtu, 900 .ndo_validate_addr = eth_validate_addr, 901 .ndo_set_mac_address = netvsc_set_mac_addr, 902 .ndo_select_queue = netvsc_select_queue, 903 .ndo_get_stats64 = netvsc_get_stats64, 904 #ifdef CONFIG_NET_POLL_CONTROLLER 905 .ndo_poll_controller = netvsc_poll_controller, 906 #endif 907 }; 908 909 /* 910 * Send GARP packet to network peers after migrations. 911 * After Quick Migration, the network is not immediately operational in the 912 * current context when receiving RNDIS_STATUS_MEDIA_CONNECT event. So, add 913 * another netif_notify_peers() into a delayed work, otherwise GARP packet 914 * will not be sent after quick migration, and cause network disconnection. 915 * Also, we update the carrier status here. 916 */ 917 static void netvsc_link_change(struct work_struct *w) 918 { 919 struct net_device_context *ndev_ctx; 920 struct net_device *net; 921 struct netvsc_device *net_device; 922 struct rndis_device *rdev; 923 bool notify, refresh = false; 924 char *argv[] = { "/etc/init.d/network", "restart", NULL }; 925 char *envp[] = { "HOME=/", "PATH=/sbin:/usr/sbin:/bin:/usr/bin", NULL }; 926 927 rtnl_lock(); 928 929 ndev_ctx = container_of(w, struct net_device_context, dwork.work); 930 net_device = hv_get_drvdata(ndev_ctx->device_ctx); 931 rdev = net_device->extension; 932 net = net_device->ndev; 933 934 if (rdev->link_state) { 935 netif_carrier_off(net); 936 notify = false; 937 } else { 938 netif_carrier_on(net); 939 notify = true; 940 if (rdev->link_change) { 941 rdev->link_change = false; 942 refresh = true; 943 } 944 } 945 946 rtnl_unlock(); 947 948 if (refresh) 949 call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); 950 951 if (notify) 952 netdev_notify_peers(net); 953 } 954 955 static void netvsc_free_netdev(struct net_device *netdev) 956 { 957 struct net_device_context *net_device_ctx = netdev_priv(netdev); 958 959 free_percpu(net_device_ctx->tx_stats); 960 free_percpu(net_device_ctx->rx_stats); 961 free_netdev(netdev); 962 } 963 964 static int netvsc_probe(struct hv_device *dev, 965 const struct hv_vmbus_device_id *dev_id) 966 { 967 struct net_device *net = NULL; 968 struct net_device_context *net_device_ctx; 969 struct netvsc_device_info device_info; 970 struct netvsc_device *nvdev; 971 int ret; 972 u32 max_needed_headroom; 973 974 net = alloc_etherdev_mq(sizeof(struct net_device_context), 975 num_online_cpus()); 976 if (!net) 977 return -ENOMEM; 978 979 max_needed_headroom = sizeof(struct hv_netvsc_packet) + 980 RNDIS_AND_PPI_SIZE; 981 982 netif_carrier_off(net); 983 984 net_device_ctx = netdev_priv(net); 985 net_device_ctx->device_ctx = dev; 986 net_device_ctx->msg_enable = netif_msg_init(debug, default_msg); 987 if (netif_msg_probe(net_device_ctx)) 988 netdev_dbg(net, "netvsc msg_enable: %d\n", 989 net_device_ctx->msg_enable); 990 991 net_device_ctx->tx_stats = netdev_alloc_pcpu_stats(struct netvsc_stats); 992 if (!net_device_ctx->tx_stats) { 993 free_netdev(net); 994 return -ENOMEM; 995 } 996 net_device_ctx->rx_stats = netdev_alloc_pcpu_stats(struct netvsc_stats); 997 if (!net_device_ctx->rx_stats) { 998 free_percpu(net_device_ctx->tx_stats); 999 free_netdev(net); 1000 return -ENOMEM; 1001 } 1002 1003 hv_set_drvdata(dev, net); 1004 INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change); 1005 INIT_WORK(&net_device_ctx->work, do_set_multicast); 1006 1007 net->netdev_ops = &device_ops; 1008 1009 net->hw_features = NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_IP_CSUM | 1010 NETIF_F_TSO; 1011 net->features = NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_SG | NETIF_F_RXCSUM | 1012 NETIF_F_IP_CSUM | NETIF_F_TSO; 1013 1014 net->ethtool_ops = ðtool_ops; 1015 SET_NETDEV_DEV(net, &dev->device); 1016 1017 /* 1018 * Request additional head room in the skb. 1019 * We will use this space to build the rndis 1020 * heaser and other state we need to maintain. 1021 */ 1022 net->needed_headroom = max_needed_headroom; 1023 1024 /* Notify the netvsc driver of the new device */ 1025 device_info.ring_size = ring_size; 1026 device_info.max_num_vrss_chns = max_num_vrss_chns; 1027 ret = rndis_filter_device_add(dev, &device_info); 1028 if (ret != 0) { 1029 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret); 1030 netvsc_free_netdev(net); 1031 hv_set_drvdata(dev, NULL); 1032 return ret; 1033 } 1034 memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN); 1035 1036 nvdev = hv_get_drvdata(dev); 1037 netif_set_real_num_tx_queues(net, nvdev->num_chn); 1038 netif_set_real_num_rx_queues(net, nvdev->num_chn); 1039 1040 ret = register_netdev(net); 1041 if (ret != 0) { 1042 pr_err("Unable to register netdev.\n"); 1043 rndis_filter_device_remove(dev); 1044 netvsc_free_netdev(net); 1045 } else { 1046 schedule_delayed_work(&net_device_ctx->dwork, 0); 1047 } 1048 1049 return ret; 1050 } 1051 1052 static int netvsc_remove(struct hv_device *dev) 1053 { 1054 struct net_device *net; 1055 struct net_device_context *ndev_ctx; 1056 struct netvsc_device *net_device; 1057 1058 net_device = hv_get_drvdata(dev); 1059 net = net_device->ndev; 1060 1061 if (net == NULL) { 1062 dev_err(&dev->device, "No net device to remove\n"); 1063 return 0; 1064 } 1065 1066 net_device->start_remove = true; 1067 1068 ndev_ctx = netdev_priv(net); 1069 cancel_delayed_work_sync(&ndev_ctx->dwork); 1070 cancel_work_sync(&ndev_ctx->work); 1071 1072 /* Stop outbound asap */ 1073 netif_tx_disable(net); 1074 1075 unregister_netdev(net); 1076 1077 /* 1078 * Call to the vsc driver to let it know that the device is being 1079 * removed 1080 */ 1081 rndis_filter_device_remove(dev); 1082 1083 netvsc_free_netdev(net); 1084 return 0; 1085 } 1086 1087 static const struct hv_vmbus_device_id id_table[] = { 1088 /* Network guid */ 1089 { HV_NIC_GUID, }, 1090 { }, 1091 }; 1092 1093 MODULE_DEVICE_TABLE(vmbus, id_table); 1094 1095 /* The one and only one */ 1096 static struct hv_driver netvsc_drv = { 1097 .name = KBUILD_MODNAME, 1098 .id_table = id_table, 1099 .probe = netvsc_probe, 1100 .remove = netvsc_remove, 1101 }; 1102 1103 static void __exit netvsc_drv_exit(void) 1104 { 1105 vmbus_driver_unregister(&netvsc_drv); 1106 } 1107 1108 static int __init netvsc_drv_init(void) 1109 { 1110 if (ring_size < RING_SIZE_MIN) { 1111 ring_size = RING_SIZE_MIN; 1112 pr_info("Increased ring_size to %d (min allowed)\n", 1113 ring_size); 1114 } 1115 return vmbus_driver_register(&netvsc_drv); 1116 } 1117 1118 MODULE_LICENSE("GPL"); 1119 MODULE_DESCRIPTION("Microsoft Hyper-V network driver"); 1120 1121 module_init(netvsc_drv_init); 1122 module_exit(netvsc_drv_exit); 1123