1 /* 2 * Virtual network driver for conversing with remote driver backends. 3 * 4 * Copyright (c) 2002-2005, K A Fraser 5 * Copyright (c) 2005, XenSource Ltd 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License version 2 9 * as published by the Free Software Foundation; or, when distributed 10 * separately from the Linux kernel or incorporated into other 11 * software packages, subject to the following license: 12 * 13 * Permission is hereby granted, free of charge, to any person obtaining a copy 14 * of this source file (the "Software"), to deal in the Software without 15 * restriction, including without limitation the rights to use, copy, modify, 16 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 17 * and to permit persons to whom the Software is furnished to do so, subject to 18 * the following conditions: 19 * 20 * The above copyright notice and this permission notice shall be included in 21 * all copies or substantial portions of the Software. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 29 * IN THE SOFTWARE. 30 */ 31 32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 33 34 #include <linux/module.h> 35 #include <linux/kernel.h> 36 #include <linux/netdevice.h> 37 #include <linux/etherdevice.h> 38 #include <linux/skbuff.h> 39 #include <linux/ethtool.h> 40 #include <linux/if_ether.h> 41 #include <net/tcp.h> 42 #include <linux/udp.h> 43 #include <linux/moduleparam.h> 44 #include <linux/mm.h> 45 #include <linux/slab.h> 46 #include <net/ip.h> 47 #include <linux/bpf.h> 48 #include <net/page_pool.h> 49 #include <linux/bpf_trace.h> 50 51 #include <xen/xen.h> 52 #include <xen/xenbus.h> 53 #include <xen/events.h> 54 #include <xen/page.h> 55 #include <xen/platform_pci.h> 56 #include <xen/grant_table.h> 57 58 #include <xen/interface/io/netif.h> 59 #include <xen/interface/memory.h> 60 #include <xen/interface/grant_table.h> 61 62 /* Module parameters */ 63 #define MAX_QUEUES_DEFAULT 8 64 static unsigned int xennet_max_queues; 65 module_param_named(max_queues, xennet_max_queues, uint, 0644); 66 MODULE_PARM_DESC(max_queues, 67 "Maximum number of queues per virtual interface"); 68 69 #define XENNET_TIMEOUT (5 * HZ) 70 71 static const struct ethtool_ops xennet_ethtool_ops; 72 73 struct netfront_cb { 74 int pull_to; 75 }; 76 77 #define NETFRONT_SKB_CB(skb) ((struct netfront_cb *)((skb)->cb)) 78 79 #define RX_COPY_THRESHOLD 256 80 81 #define GRANT_INVALID_REF 0 82 83 #define NET_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE) 84 #define NET_RX_RING_SIZE __CONST_RING_SIZE(xen_netif_rx, XEN_PAGE_SIZE) 85 86 /* Minimum number of Rx slots (includes slot for GSO metadata). */ 87 #define NET_RX_SLOTS_MIN (XEN_NETIF_NR_SLOTS_MIN + 1) 88 89 /* Queue name is interface name with "-qNNN" appended */ 90 #define QUEUE_NAME_SIZE (IFNAMSIZ + 6) 91 92 /* IRQ name is queue name with "-tx" or "-rx" appended */ 93 #define IRQ_NAME_SIZE (QUEUE_NAME_SIZE + 3) 94 95 static DECLARE_WAIT_QUEUE_HEAD(module_wq); 96 97 struct netfront_stats { 98 u64 packets; 99 u64 bytes; 100 struct u64_stats_sync syncp; 101 }; 102 103 struct netfront_info; 104 105 struct netfront_queue { 106 unsigned int id; /* Queue ID, 0-based */ 107 char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */ 108 struct netfront_info *info; 109 110 struct bpf_prog __rcu *xdp_prog; 111 112 struct napi_struct napi; 113 114 /* Split event channels support, tx_* == rx_* when using 115 * single event channel. 116 */ 117 unsigned int tx_evtchn, rx_evtchn; 118 unsigned int tx_irq, rx_irq; 119 /* Only used when split event channels support is enabled */ 120 char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */ 121 char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */ 122 123 spinlock_t tx_lock; 124 struct xen_netif_tx_front_ring tx; 125 int tx_ring_ref; 126 127 /* 128 * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries 129 * are linked from tx_skb_freelist through tx_link. 130 */ 131 struct sk_buff *tx_skbs[NET_TX_RING_SIZE]; 132 unsigned short tx_link[NET_TX_RING_SIZE]; 133 #define TX_LINK_NONE 0xffff 134 #define TX_PENDING 0xfffe 135 grant_ref_t gref_tx_head; 136 grant_ref_t grant_tx_ref[NET_TX_RING_SIZE]; 137 struct page *grant_tx_page[NET_TX_RING_SIZE]; 138 unsigned tx_skb_freelist; 139 unsigned int tx_pend_queue; 140 141 spinlock_t rx_lock ____cacheline_aligned_in_smp; 142 struct xen_netif_rx_front_ring rx; 143 int rx_ring_ref; 144 145 struct timer_list rx_refill_timer; 146 147 struct sk_buff *rx_skbs[NET_RX_RING_SIZE]; 148 grant_ref_t gref_rx_head; 149 grant_ref_t grant_rx_ref[NET_RX_RING_SIZE]; 150 151 unsigned int rx_rsp_unconsumed; 152 spinlock_t rx_cons_lock; 153 154 struct page_pool *page_pool; 155 struct xdp_rxq_info xdp_rxq; 156 }; 157 158 struct netfront_info { 159 struct list_head list; 160 struct net_device *netdev; 161 162 struct xenbus_device *xbdev; 163 164 /* Multi-queue support */ 165 struct netfront_queue *queues; 166 167 /* Statistics */ 168 struct netfront_stats __percpu *rx_stats; 169 struct netfront_stats __percpu *tx_stats; 170 171 /* XDP state */ 172 bool netback_has_xdp_headroom; 173 bool netfront_xdp_enabled; 174 175 /* Is device behaving sane? */ 176 bool broken; 177 178 atomic_t rx_gso_checksum_fixup; 179 }; 180 181 struct netfront_rx_info { 182 struct xen_netif_rx_response rx; 183 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1]; 184 }; 185 186 /* 187 * Access macros for acquiring freeing slots in tx_skbs[]. 188 */ 189 190 static void add_id_to_list(unsigned *head, unsigned short *list, 191 unsigned short id) 192 { 193 list[id] = *head; 194 *head = id; 195 } 196 197 static unsigned short get_id_from_list(unsigned *head, unsigned short *list) 198 { 199 unsigned int id = *head; 200 201 if (id != TX_LINK_NONE) { 202 *head = list[id]; 203 list[id] = TX_LINK_NONE; 204 } 205 return id; 206 } 207 208 static int xennet_rxidx(RING_IDX idx) 209 { 210 return idx & (NET_RX_RING_SIZE - 1); 211 } 212 213 static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue, 214 RING_IDX ri) 215 { 216 int i = xennet_rxidx(ri); 217 struct sk_buff *skb = queue->rx_skbs[i]; 218 queue->rx_skbs[i] = NULL; 219 return skb; 220 } 221 222 static grant_ref_t xennet_get_rx_ref(struct netfront_queue *queue, 223 RING_IDX ri) 224 { 225 int i = xennet_rxidx(ri); 226 grant_ref_t ref = queue->grant_rx_ref[i]; 227 queue->grant_rx_ref[i] = GRANT_INVALID_REF; 228 return ref; 229 } 230 231 #ifdef CONFIG_SYSFS 232 static const struct attribute_group xennet_dev_group; 233 #endif 234 235 static bool xennet_can_sg(struct net_device *dev) 236 { 237 return dev->features & NETIF_F_SG; 238 } 239 240 241 static void rx_refill_timeout(struct timer_list *t) 242 { 243 struct netfront_queue *queue = from_timer(queue, t, rx_refill_timer); 244 napi_schedule(&queue->napi); 245 } 246 247 static int netfront_tx_slot_available(struct netfront_queue *queue) 248 { 249 return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) < 250 (NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1); 251 } 252 253 static void xennet_maybe_wake_tx(struct netfront_queue *queue) 254 { 255 struct net_device *dev = queue->info->netdev; 256 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id); 257 258 if (unlikely(netif_tx_queue_stopped(dev_queue)) && 259 netfront_tx_slot_available(queue) && 260 likely(netif_running(dev))) 261 netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id)); 262 } 263 264 265 static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue) 266 { 267 struct sk_buff *skb; 268 struct page *page; 269 270 skb = __netdev_alloc_skb(queue->info->netdev, 271 RX_COPY_THRESHOLD + NET_IP_ALIGN, 272 GFP_ATOMIC | __GFP_NOWARN); 273 if (unlikely(!skb)) 274 return NULL; 275 276 page = page_pool_dev_alloc_pages(queue->page_pool); 277 if (unlikely(!page)) { 278 kfree_skb(skb); 279 return NULL; 280 } 281 skb_add_rx_frag(skb, 0, page, 0, 0, PAGE_SIZE); 282 283 /* Align ip header to a 16 bytes boundary */ 284 skb_reserve(skb, NET_IP_ALIGN); 285 skb->dev = queue->info->netdev; 286 287 return skb; 288 } 289 290 291 static void xennet_alloc_rx_buffers(struct netfront_queue *queue) 292 { 293 RING_IDX req_prod = queue->rx.req_prod_pvt; 294 int notify; 295 int err = 0; 296 297 if (unlikely(!netif_carrier_ok(queue->info->netdev))) 298 return; 299 300 for (req_prod = queue->rx.req_prod_pvt; 301 req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE; 302 req_prod++) { 303 struct sk_buff *skb; 304 unsigned short id; 305 grant_ref_t ref; 306 struct page *page; 307 struct xen_netif_rx_request *req; 308 309 skb = xennet_alloc_one_rx_buffer(queue); 310 if (!skb) { 311 err = -ENOMEM; 312 break; 313 } 314 315 id = xennet_rxidx(req_prod); 316 317 BUG_ON(queue->rx_skbs[id]); 318 queue->rx_skbs[id] = skb; 319 320 ref = gnttab_claim_grant_reference(&queue->gref_rx_head); 321 WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref)); 322 queue->grant_rx_ref[id] = ref; 323 324 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); 325 326 req = RING_GET_REQUEST(&queue->rx, req_prod); 327 gnttab_page_grant_foreign_access_ref_one(ref, 328 queue->info->xbdev->otherend_id, 329 page, 330 0); 331 req->id = id; 332 req->gref = ref; 333 } 334 335 queue->rx.req_prod_pvt = req_prod; 336 337 /* Try again later if there are not enough requests or skb allocation 338 * failed. 339 * Enough requests is quantified as the sum of newly created slots and 340 * the unconsumed slots at the backend. 341 */ 342 if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN || 343 unlikely(err)) { 344 mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); 345 return; 346 } 347 348 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify); 349 if (notify) 350 notify_remote_via_irq(queue->rx_irq); 351 } 352 353 static int xennet_open(struct net_device *dev) 354 { 355 struct netfront_info *np = netdev_priv(dev); 356 unsigned int num_queues = dev->real_num_tx_queues; 357 unsigned int i = 0; 358 struct netfront_queue *queue = NULL; 359 360 if (!np->queues || np->broken) 361 return -ENODEV; 362 363 for (i = 0; i < num_queues; ++i) { 364 queue = &np->queues[i]; 365 napi_enable(&queue->napi); 366 367 spin_lock_bh(&queue->rx_lock); 368 if (netif_carrier_ok(dev)) { 369 xennet_alloc_rx_buffers(queue); 370 queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1; 371 if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)) 372 napi_schedule(&queue->napi); 373 } 374 spin_unlock_bh(&queue->rx_lock); 375 } 376 377 netif_tx_start_all_queues(dev); 378 379 return 0; 380 } 381 382 static bool xennet_tx_buf_gc(struct netfront_queue *queue) 383 { 384 RING_IDX cons, prod; 385 unsigned short id; 386 struct sk_buff *skb; 387 bool more_to_do; 388 bool work_done = false; 389 const struct device *dev = &queue->info->netdev->dev; 390 391 BUG_ON(!netif_carrier_ok(queue->info->netdev)); 392 393 do { 394 prod = queue->tx.sring->rsp_prod; 395 if (RING_RESPONSE_PROD_OVERFLOW(&queue->tx, prod)) { 396 dev_alert(dev, "Illegal number of responses %u\n", 397 prod - queue->tx.rsp_cons); 398 goto err; 399 } 400 rmb(); /* Ensure we see responses up to 'rp'. */ 401 402 for (cons = queue->tx.rsp_cons; cons != prod; cons++) { 403 struct xen_netif_tx_response txrsp; 404 405 work_done = true; 406 407 RING_COPY_RESPONSE(&queue->tx, cons, &txrsp); 408 if (txrsp.status == XEN_NETIF_RSP_NULL) 409 continue; 410 411 id = txrsp.id; 412 if (id >= RING_SIZE(&queue->tx)) { 413 dev_alert(dev, 414 "Response has incorrect id (%u)\n", 415 id); 416 goto err; 417 } 418 if (queue->tx_link[id] != TX_PENDING) { 419 dev_alert(dev, 420 "Response for inactive request\n"); 421 goto err; 422 } 423 424 queue->tx_link[id] = TX_LINK_NONE; 425 skb = queue->tx_skbs[id]; 426 queue->tx_skbs[id] = NULL; 427 if (unlikely(gnttab_query_foreign_access( 428 queue->grant_tx_ref[id]) != 0)) { 429 dev_alert(dev, 430 "Grant still in use by backend domain\n"); 431 goto err; 432 } 433 gnttab_end_foreign_access_ref( 434 queue->grant_tx_ref[id], GNTMAP_readonly); 435 gnttab_release_grant_reference( 436 &queue->gref_tx_head, queue->grant_tx_ref[id]); 437 queue->grant_tx_ref[id] = GRANT_INVALID_REF; 438 queue->grant_tx_page[id] = NULL; 439 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id); 440 dev_kfree_skb_irq(skb); 441 } 442 443 queue->tx.rsp_cons = prod; 444 445 RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do); 446 } while (more_to_do); 447 448 xennet_maybe_wake_tx(queue); 449 450 return work_done; 451 452 err: 453 queue->info->broken = true; 454 dev_alert(dev, "Disabled for further use\n"); 455 456 return work_done; 457 } 458 459 struct xennet_gnttab_make_txreq { 460 struct netfront_queue *queue; 461 struct sk_buff *skb; 462 struct page *page; 463 struct xen_netif_tx_request *tx; /* Last request on ring page */ 464 struct xen_netif_tx_request tx_local; /* Last request local copy*/ 465 unsigned int size; 466 }; 467 468 static void xennet_tx_setup_grant(unsigned long gfn, unsigned int offset, 469 unsigned int len, void *data) 470 { 471 struct xennet_gnttab_make_txreq *info = data; 472 unsigned int id; 473 struct xen_netif_tx_request *tx; 474 grant_ref_t ref; 475 /* convenient aliases */ 476 struct page *page = info->page; 477 struct netfront_queue *queue = info->queue; 478 struct sk_buff *skb = info->skb; 479 480 id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link); 481 tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); 482 ref = gnttab_claim_grant_reference(&queue->gref_tx_head); 483 WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref)); 484 485 gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, 486 gfn, GNTMAP_readonly); 487 488 queue->tx_skbs[id] = skb; 489 queue->grant_tx_page[id] = page; 490 queue->grant_tx_ref[id] = ref; 491 492 info->tx_local.id = id; 493 info->tx_local.gref = ref; 494 info->tx_local.offset = offset; 495 info->tx_local.size = len; 496 info->tx_local.flags = 0; 497 498 *tx = info->tx_local; 499 500 /* 501 * Put the request in the pending queue, it will be set to be pending 502 * when the producer index is about to be raised. 503 */ 504 add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id); 505 506 info->tx = tx; 507 info->size += info->tx_local.size; 508 } 509 510 static struct xen_netif_tx_request *xennet_make_first_txreq( 511 struct xennet_gnttab_make_txreq *info, 512 unsigned int offset, unsigned int len) 513 { 514 info->size = 0; 515 516 gnttab_for_one_grant(info->page, offset, len, xennet_tx_setup_grant, info); 517 518 return info->tx; 519 } 520 521 static void xennet_make_one_txreq(unsigned long gfn, unsigned int offset, 522 unsigned int len, void *data) 523 { 524 struct xennet_gnttab_make_txreq *info = data; 525 526 info->tx->flags |= XEN_NETTXF_more_data; 527 skb_get(info->skb); 528 xennet_tx_setup_grant(gfn, offset, len, data); 529 } 530 531 static void xennet_make_txreqs( 532 struct xennet_gnttab_make_txreq *info, 533 struct page *page, 534 unsigned int offset, unsigned int len) 535 { 536 /* Skip unused frames from start of page */ 537 page += offset >> PAGE_SHIFT; 538 offset &= ~PAGE_MASK; 539 540 while (len) { 541 info->page = page; 542 info->size = 0; 543 544 gnttab_foreach_grant_in_range(page, offset, len, 545 xennet_make_one_txreq, 546 info); 547 548 page++; 549 offset = 0; 550 len -= info->size; 551 } 552 } 553 554 /* 555 * Count how many ring slots are required to send this skb. Each frag 556 * might be a compound page. 557 */ 558 static int xennet_count_skb_slots(struct sk_buff *skb) 559 { 560 int i, frags = skb_shinfo(skb)->nr_frags; 561 int slots; 562 563 slots = gnttab_count_grant(offset_in_page(skb->data), 564 skb_headlen(skb)); 565 566 for (i = 0; i < frags; i++) { 567 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 568 unsigned long size = skb_frag_size(frag); 569 unsigned long offset = skb_frag_off(frag); 570 571 /* Skip unused frames from start of page */ 572 offset &= ~PAGE_MASK; 573 574 slots += gnttab_count_grant(offset, size); 575 } 576 577 return slots; 578 } 579 580 static u16 xennet_select_queue(struct net_device *dev, struct sk_buff *skb, 581 struct net_device *sb_dev) 582 { 583 unsigned int num_queues = dev->real_num_tx_queues; 584 u32 hash; 585 u16 queue_idx; 586 587 /* First, check if there is only one queue */ 588 if (num_queues == 1) { 589 queue_idx = 0; 590 } else { 591 hash = skb_get_hash(skb); 592 queue_idx = hash % num_queues; 593 } 594 595 return queue_idx; 596 } 597 598 static void xennet_mark_tx_pending(struct netfront_queue *queue) 599 { 600 unsigned int i; 601 602 while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) != 603 TX_LINK_NONE) 604 queue->tx_link[i] = TX_PENDING; 605 } 606 607 static int xennet_xdp_xmit_one(struct net_device *dev, 608 struct netfront_queue *queue, 609 struct xdp_frame *xdpf) 610 { 611 struct netfront_info *np = netdev_priv(dev); 612 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); 613 struct xennet_gnttab_make_txreq info = { 614 .queue = queue, 615 .skb = NULL, 616 .page = virt_to_page(xdpf->data), 617 }; 618 int notify; 619 620 xennet_make_first_txreq(&info, 621 offset_in_page(xdpf->data), 622 xdpf->len); 623 624 xennet_mark_tx_pending(queue); 625 626 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); 627 if (notify) 628 notify_remote_via_irq(queue->tx_irq); 629 630 u64_stats_update_begin(&tx_stats->syncp); 631 tx_stats->bytes += xdpf->len; 632 tx_stats->packets++; 633 u64_stats_update_end(&tx_stats->syncp); 634 635 xennet_tx_buf_gc(queue); 636 637 return 0; 638 } 639 640 static int xennet_xdp_xmit(struct net_device *dev, int n, 641 struct xdp_frame **frames, u32 flags) 642 { 643 unsigned int num_queues = dev->real_num_tx_queues; 644 struct netfront_info *np = netdev_priv(dev); 645 struct netfront_queue *queue = NULL; 646 unsigned long irq_flags; 647 int nxmit = 0; 648 int i; 649 650 if (unlikely(np->broken)) 651 return -ENODEV; 652 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 653 return -EINVAL; 654 655 queue = &np->queues[smp_processor_id() % num_queues]; 656 657 spin_lock_irqsave(&queue->tx_lock, irq_flags); 658 for (i = 0; i < n; i++) { 659 struct xdp_frame *xdpf = frames[i]; 660 661 if (!xdpf) 662 continue; 663 if (xennet_xdp_xmit_one(dev, queue, xdpf)) 664 break; 665 nxmit++; 666 } 667 spin_unlock_irqrestore(&queue->tx_lock, irq_flags); 668 669 return nxmit; 670 } 671 672 673 #define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1) 674 675 static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev) 676 { 677 struct netfront_info *np = netdev_priv(dev); 678 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); 679 struct xen_netif_tx_request *first_tx; 680 unsigned int i; 681 int notify; 682 int slots; 683 struct page *page; 684 unsigned int offset; 685 unsigned int len; 686 unsigned long flags; 687 struct netfront_queue *queue = NULL; 688 struct xennet_gnttab_make_txreq info = { }; 689 unsigned int num_queues = dev->real_num_tx_queues; 690 u16 queue_index; 691 struct sk_buff *nskb; 692 693 /* Drop the packet if no queues are set up */ 694 if (num_queues < 1) 695 goto drop; 696 if (unlikely(np->broken)) 697 goto drop; 698 /* Determine which queue to transmit this SKB on */ 699 queue_index = skb_get_queue_mapping(skb); 700 queue = &np->queues[queue_index]; 701 702 /* If skb->len is too big for wire format, drop skb and alert 703 * user about misconfiguration. 704 */ 705 if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) { 706 net_alert_ratelimited( 707 "xennet: skb->len = %u, too big for wire format\n", 708 skb->len); 709 goto drop; 710 } 711 712 slots = xennet_count_skb_slots(skb); 713 if (unlikely(slots > MAX_XEN_SKB_FRAGS + 1)) { 714 net_dbg_ratelimited("xennet: skb rides the rocket: %d slots, %d bytes\n", 715 slots, skb->len); 716 if (skb_linearize(skb)) 717 goto drop; 718 } 719 720 page = virt_to_page(skb->data); 721 offset = offset_in_page(skb->data); 722 723 /* The first req should be at least ETH_HLEN size or the packet will be 724 * dropped by netback. 725 */ 726 if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) { 727 nskb = skb_copy(skb, GFP_ATOMIC); 728 if (!nskb) 729 goto drop; 730 dev_consume_skb_any(skb); 731 skb = nskb; 732 page = virt_to_page(skb->data); 733 offset = offset_in_page(skb->data); 734 } 735 736 len = skb_headlen(skb); 737 738 spin_lock_irqsave(&queue->tx_lock, flags); 739 740 if (unlikely(!netif_carrier_ok(dev) || 741 (slots > 1 && !xennet_can_sg(dev)) || 742 netif_needs_gso(skb, netif_skb_features(skb)))) { 743 spin_unlock_irqrestore(&queue->tx_lock, flags); 744 goto drop; 745 } 746 747 /* First request for the linear area. */ 748 info.queue = queue; 749 info.skb = skb; 750 info.page = page; 751 first_tx = xennet_make_first_txreq(&info, offset, len); 752 offset += info.tx_local.size; 753 if (offset == PAGE_SIZE) { 754 page++; 755 offset = 0; 756 } 757 len -= info.tx_local.size; 758 759 if (skb->ip_summed == CHECKSUM_PARTIAL) 760 /* local packet? */ 761 first_tx->flags |= XEN_NETTXF_csum_blank | 762 XEN_NETTXF_data_validated; 763 else if (skb->ip_summed == CHECKSUM_UNNECESSARY) 764 /* remote but checksummed. */ 765 first_tx->flags |= XEN_NETTXF_data_validated; 766 767 /* Optional extra info after the first request. */ 768 if (skb_shinfo(skb)->gso_size) { 769 struct xen_netif_extra_info *gso; 770 771 gso = (struct xen_netif_extra_info *) 772 RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); 773 774 first_tx->flags |= XEN_NETTXF_extra_info; 775 776 gso->u.gso.size = skb_shinfo(skb)->gso_size; 777 gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ? 778 XEN_NETIF_GSO_TYPE_TCPV6 : 779 XEN_NETIF_GSO_TYPE_TCPV4; 780 gso->u.gso.pad = 0; 781 gso->u.gso.features = 0; 782 783 gso->type = XEN_NETIF_EXTRA_TYPE_GSO; 784 gso->flags = 0; 785 } 786 787 /* Requests for the rest of the linear area. */ 788 xennet_make_txreqs(&info, page, offset, len); 789 790 /* Requests for all the frags. */ 791 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 792 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 793 xennet_make_txreqs(&info, skb_frag_page(frag), 794 skb_frag_off(frag), 795 skb_frag_size(frag)); 796 } 797 798 /* First request has the packet length. */ 799 first_tx->size = skb->len; 800 801 /* timestamp packet in software */ 802 skb_tx_timestamp(skb); 803 804 xennet_mark_tx_pending(queue); 805 806 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); 807 if (notify) 808 notify_remote_via_irq(queue->tx_irq); 809 810 u64_stats_update_begin(&tx_stats->syncp); 811 tx_stats->bytes += skb->len; 812 tx_stats->packets++; 813 u64_stats_update_end(&tx_stats->syncp); 814 815 /* Note: It is not safe to access skb after xennet_tx_buf_gc()! */ 816 xennet_tx_buf_gc(queue); 817 818 if (!netfront_tx_slot_available(queue)) 819 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); 820 821 spin_unlock_irqrestore(&queue->tx_lock, flags); 822 823 return NETDEV_TX_OK; 824 825 drop: 826 dev->stats.tx_dropped++; 827 dev_kfree_skb_any(skb); 828 return NETDEV_TX_OK; 829 } 830 831 static int xennet_close(struct net_device *dev) 832 { 833 struct netfront_info *np = netdev_priv(dev); 834 unsigned int num_queues = dev->real_num_tx_queues; 835 unsigned int i; 836 struct netfront_queue *queue; 837 netif_tx_stop_all_queues(np->netdev); 838 for (i = 0; i < num_queues; ++i) { 839 queue = &np->queues[i]; 840 napi_disable(&queue->napi); 841 } 842 return 0; 843 } 844 845 static void xennet_destroy_queues(struct netfront_info *info) 846 { 847 unsigned int i; 848 849 for (i = 0; i < info->netdev->real_num_tx_queues; i++) { 850 struct netfront_queue *queue = &info->queues[i]; 851 852 if (netif_running(info->netdev)) 853 napi_disable(&queue->napi); 854 netif_napi_del(&queue->napi); 855 } 856 857 kfree(info->queues); 858 info->queues = NULL; 859 } 860 861 static void xennet_uninit(struct net_device *dev) 862 { 863 struct netfront_info *np = netdev_priv(dev); 864 xennet_destroy_queues(np); 865 } 866 867 static void xennet_set_rx_rsp_cons(struct netfront_queue *queue, RING_IDX val) 868 { 869 unsigned long flags; 870 871 spin_lock_irqsave(&queue->rx_cons_lock, flags); 872 queue->rx.rsp_cons = val; 873 queue->rx_rsp_unconsumed = RING_HAS_UNCONSUMED_RESPONSES(&queue->rx); 874 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); 875 } 876 877 static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb, 878 grant_ref_t ref) 879 { 880 int new = xennet_rxidx(queue->rx.req_prod_pvt); 881 882 BUG_ON(queue->rx_skbs[new]); 883 queue->rx_skbs[new] = skb; 884 queue->grant_rx_ref[new] = ref; 885 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new; 886 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref; 887 queue->rx.req_prod_pvt++; 888 } 889 890 static int xennet_get_extras(struct netfront_queue *queue, 891 struct xen_netif_extra_info *extras, 892 RING_IDX rp) 893 894 { 895 struct xen_netif_extra_info extra; 896 struct device *dev = &queue->info->netdev->dev; 897 RING_IDX cons = queue->rx.rsp_cons; 898 int err = 0; 899 900 do { 901 struct sk_buff *skb; 902 grant_ref_t ref; 903 904 if (unlikely(cons + 1 == rp)) { 905 if (net_ratelimit()) 906 dev_warn(dev, "Missing extra info\n"); 907 err = -EBADR; 908 break; 909 } 910 911 RING_COPY_RESPONSE(&queue->rx, ++cons, &extra); 912 913 if (unlikely(!extra.type || 914 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) { 915 if (net_ratelimit()) 916 dev_warn(dev, "Invalid extra type: %d\n", 917 extra.type); 918 err = -EINVAL; 919 } else { 920 extras[extra.type - 1] = extra; 921 } 922 923 skb = xennet_get_rx_skb(queue, cons); 924 ref = xennet_get_rx_ref(queue, cons); 925 xennet_move_rx_slot(queue, skb, ref); 926 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE); 927 928 xennet_set_rx_rsp_cons(queue, cons); 929 return err; 930 } 931 932 static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata, 933 struct xen_netif_rx_response *rx, struct bpf_prog *prog, 934 struct xdp_buff *xdp, bool *need_xdp_flush) 935 { 936 struct xdp_frame *xdpf; 937 u32 len = rx->status; 938 u32 act; 939 int err; 940 941 xdp_init_buff(xdp, XEN_PAGE_SIZE - XDP_PACKET_HEADROOM, 942 &queue->xdp_rxq); 943 xdp_prepare_buff(xdp, page_address(pdata), XDP_PACKET_HEADROOM, 944 len, false); 945 946 act = bpf_prog_run_xdp(prog, xdp); 947 switch (act) { 948 case XDP_TX: 949 get_page(pdata); 950 xdpf = xdp_convert_buff_to_frame(xdp); 951 err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0); 952 if (unlikely(!err)) 953 xdp_return_frame_rx_napi(xdpf); 954 else if (unlikely(err < 0)) 955 trace_xdp_exception(queue->info->netdev, prog, act); 956 break; 957 case XDP_REDIRECT: 958 get_page(pdata); 959 err = xdp_do_redirect(queue->info->netdev, xdp, prog); 960 *need_xdp_flush = true; 961 if (unlikely(err)) 962 trace_xdp_exception(queue->info->netdev, prog, act); 963 break; 964 case XDP_PASS: 965 case XDP_DROP: 966 break; 967 968 case XDP_ABORTED: 969 trace_xdp_exception(queue->info->netdev, prog, act); 970 break; 971 972 default: 973 bpf_warn_invalid_xdp_action(queue->info->netdev, prog, act); 974 } 975 976 return act; 977 } 978 979 static int xennet_get_responses(struct netfront_queue *queue, 980 struct netfront_rx_info *rinfo, RING_IDX rp, 981 struct sk_buff_head *list, 982 bool *need_xdp_flush) 983 { 984 struct xen_netif_rx_response *rx = &rinfo->rx, rx_local; 985 int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD); 986 RING_IDX cons = queue->rx.rsp_cons; 987 struct sk_buff *skb = xennet_get_rx_skb(queue, cons); 988 struct xen_netif_extra_info *extras = rinfo->extras; 989 grant_ref_t ref = xennet_get_rx_ref(queue, cons); 990 struct device *dev = &queue->info->netdev->dev; 991 struct bpf_prog *xdp_prog; 992 struct xdp_buff xdp; 993 unsigned long ret; 994 int slots = 1; 995 int err = 0; 996 u32 verdict; 997 998 if (rx->flags & XEN_NETRXF_extra_info) { 999 err = xennet_get_extras(queue, extras, rp); 1000 if (!err) { 1001 if (extras[XEN_NETIF_EXTRA_TYPE_XDP - 1].type) { 1002 struct xen_netif_extra_info *xdp; 1003 1004 xdp = &extras[XEN_NETIF_EXTRA_TYPE_XDP - 1]; 1005 rx->offset = xdp->u.xdp.headroom; 1006 } 1007 } 1008 cons = queue->rx.rsp_cons; 1009 } 1010 1011 for (;;) { 1012 if (unlikely(rx->status < 0 || 1013 rx->offset + rx->status > XEN_PAGE_SIZE)) { 1014 if (net_ratelimit()) 1015 dev_warn(dev, "rx->offset: %u, size: %d\n", 1016 rx->offset, rx->status); 1017 xennet_move_rx_slot(queue, skb, ref); 1018 err = -EINVAL; 1019 goto next; 1020 } 1021 1022 /* 1023 * This definitely indicates a bug, either in this driver or in 1024 * the backend driver. In future this should flag the bad 1025 * situation to the system controller to reboot the backend. 1026 */ 1027 if (ref == GRANT_INVALID_REF) { 1028 if (net_ratelimit()) 1029 dev_warn(dev, "Bad rx response id %d.\n", 1030 rx->id); 1031 err = -EINVAL; 1032 goto next; 1033 } 1034 1035 ret = gnttab_end_foreign_access_ref(ref, 0); 1036 BUG_ON(!ret); 1037 1038 gnttab_release_grant_reference(&queue->gref_rx_head, ref); 1039 1040 rcu_read_lock(); 1041 xdp_prog = rcu_dereference(queue->xdp_prog); 1042 if (xdp_prog) { 1043 if (!(rx->flags & XEN_NETRXF_more_data)) { 1044 /* currently only a single page contains data */ 1045 verdict = xennet_run_xdp(queue, 1046 skb_frag_page(&skb_shinfo(skb)->frags[0]), 1047 rx, xdp_prog, &xdp, need_xdp_flush); 1048 if (verdict != XDP_PASS) 1049 err = -EINVAL; 1050 } else { 1051 /* drop the frame */ 1052 err = -EINVAL; 1053 } 1054 } 1055 rcu_read_unlock(); 1056 next: 1057 __skb_queue_tail(list, skb); 1058 if (!(rx->flags & XEN_NETRXF_more_data)) 1059 break; 1060 1061 if (cons + slots == rp) { 1062 if (net_ratelimit()) 1063 dev_warn(dev, "Need more slots\n"); 1064 err = -ENOENT; 1065 break; 1066 } 1067 1068 RING_COPY_RESPONSE(&queue->rx, cons + slots, &rx_local); 1069 rx = &rx_local; 1070 skb = xennet_get_rx_skb(queue, cons + slots); 1071 ref = xennet_get_rx_ref(queue, cons + slots); 1072 slots++; 1073 } 1074 1075 if (unlikely(slots > max)) { 1076 if (net_ratelimit()) 1077 dev_warn(dev, "Too many slots\n"); 1078 err = -E2BIG; 1079 } 1080 1081 if (unlikely(err)) 1082 xennet_set_rx_rsp_cons(queue, cons + slots); 1083 1084 return err; 1085 } 1086 1087 static int xennet_set_skb_gso(struct sk_buff *skb, 1088 struct xen_netif_extra_info *gso) 1089 { 1090 if (!gso->u.gso.size) { 1091 if (net_ratelimit()) 1092 pr_warn("GSO size must not be zero\n"); 1093 return -EINVAL; 1094 } 1095 1096 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 && 1097 gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) { 1098 if (net_ratelimit()) 1099 pr_warn("Bad GSO type %d\n", gso->u.gso.type); 1100 return -EINVAL; 1101 } 1102 1103 skb_shinfo(skb)->gso_size = gso->u.gso.size; 1104 skb_shinfo(skb)->gso_type = 1105 (gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ? 1106 SKB_GSO_TCPV4 : 1107 SKB_GSO_TCPV6; 1108 1109 /* Header must be checked, and gso_segs computed. */ 1110 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 1111 skb_shinfo(skb)->gso_segs = 0; 1112 1113 return 0; 1114 } 1115 1116 static int xennet_fill_frags(struct netfront_queue *queue, 1117 struct sk_buff *skb, 1118 struct sk_buff_head *list) 1119 { 1120 RING_IDX cons = queue->rx.rsp_cons; 1121 struct sk_buff *nskb; 1122 1123 while ((nskb = __skb_dequeue(list))) { 1124 struct xen_netif_rx_response rx; 1125 skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0]; 1126 1127 RING_COPY_RESPONSE(&queue->rx, ++cons, &rx); 1128 1129 if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) { 1130 unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to; 1131 1132 BUG_ON(pull_to < skb_headlen(skb)); 1133 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); 1134 } 1135 if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) { 1136 xennet_set_rx_rsp_cons(queue, 1137 ++cons + skb_queue_len(list)); 1138 kfree_skb(nskb); 1139 return -ENOENT; 1140 } 1141 1142 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 1143 skb_frag_page(nfrag), 1144 rx.offset, rx.status, PAGE_SIZE); 1145 1146 skb_shinfo(nskb)->nr_frags = 0; 1147 kfree_skb(nskb); 1148 } 1149 1150 xennet_set_rx_rsp_cons(queue, cons); 1151 1152 return 0; 1153 } 1154 1155 static int checksum_setup(struct net_device *dev, struct sk_buff *skb) 1156 { 1157 bool recalculate_partial_csum = false; 1158 1159 /* 1160 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy 1161 * peers can fail to set NETRXF_csum_blank when sending a GSO 1162 * frame. In this case force the SKB to CHECKSUM_PARTIAL and 1163 * recalculate the partial checksum. 1164 */ 1165 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { 1166 struct netfront_info *np = netdev_priv(dev); 1167 atomic_inc(&np->rx_gso_checksum_fixup); 1168 skb->ip_summed = CHECKSUM_PARTIAL; 1169 recalculate_partial_csum = true; 1170 } 1171 1172 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */ 1173 if (skb->ip_summed != CHECKSUM_PARTIAL) 1174 return 0; 1175 1176 return skb_checksum_setup(skb, recalculate_partial_csum); 1177 } 1178 1179 static int handle_incoming_queue(struct netfront_queue *queue, 1180 struct sk_buff_head *rxq) 1181 { 1182 struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats); 1183 int packets_dropped = 0; 1184 struct sk_buff *skb; 1185 1186 while ((skb = __skb_dequeue(rxq)) != NULL) { 1187 int pull_to = NETFRONT_SKB_CB(skb)->pull_to; 1188 1189 if (pull_to > skb_headlen(skb)) 1190 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); 1191 1192 /* Ethernet work: Delayed to here as it peeks the header. */ 1193 skb->protocol = eth_type_trans(skb, queue->info->netdev); 1194 skb_reset_network_header(skb); 1195 1196 if (checksum_setup(queue->info->netdev, skb)) { 1197 kfree_skb(skb); 1198 packets_dropped++; 1199 queue->info->netdev->stats.rx_errors++; 1200 continue; 1201 } 1202 1203 u64_stats_update_begin(&rx_stats->syncp); 1204 rx_stats->packets++; 1205 rx_stats->bytes += skb->len; 1206 u64_stats_update_end(&rx_stats->syncp); 1207 1208 /* Pass it up. */ 1209 napi_gro_receive(&queue->napi, skb); 1210 } 1211 1212 return packets_dropped; 1213 } 1214 1215 static int xennet_poll(struct napi_struct *napi, int budget) 1216 { 1217 struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi); 1218 struct net_device *dev = queue->info->netdev; 1219 struct sk_buff *skb; 1220 struct netfront_rx_info rinfo; 1221 struct xen_netif_rx_response *rx = &rinfo.rx; 1222 struct xen_netif_extra_info *extras = rinfo.extras; 1223 RING_IDX i, rp; 1224 int work_done; 1225 struct sk_buff_head rxq; 1226 struct sk_buff_head errq; 1227 struct sk_buff_head tmpq; 1228 int err; 1229 bool need_xdp_flush = false; 1230 1231 spin_lock(&queue->rx_lock); 1232 1233 skb_queue_head_init(&rxq); 1234 skb_queue_head_init(&errq); 1235 skb_queue_head_init(&tmpq); 1236 1237 rp = queue->rx.sring->rsp_prod; 1238 if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) { 1239 dev_alert(&dev->dev, "Illegal number of responses %u\n", 1240 rp - queue->rx.rsp_cons); 1241 queue->info->broken = true; 1242 spin_unlock(&queue->rx_lock); 1243 return 0; 1244 } 1245 rmb(); /* Ensure we see queued responses up to 'rp'. */ 1246 1247 i = queue->rx.rsp_cons; 1248 work_done = 0; 1249 while ((i != rp) && (work_done < budget)) { 1250 RING_COPY_RESPONSE(&queue->rx, i, rx); 1251 memset(extras, 0, sizeof(rinfo.extras)); 1252 1253 err = xennet_get_responses(queue, &rinfo, rp, &tmpq, 1254 &need_xdp_flush); 1255 1256 if (unlikely(err)) { 1257 err: 1258 while ((skb = __skb_dequeue(&tmpq))) 1259 __skb_queue_tail(&errq, skb); 1260 dev->stats.rx_errors++; 1261 i = queue->rx.rsp_cons; 1262 continue; 1263 } 1264 1265 skb = __skb_dequeue(&tmpq); 1266 1267 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { 1268 struct xen_netif_extra_info *gso; 1269 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; 1270 1271 if (unlikely(xennet_set_skb_gso(skb, gso))) { 1272 __skb_queue_head(&tmpq, skb); 1273 xennet_set_rx_rsp_cons(queue, 1274 queue->rx.rsp_cons + 1275 skb_queue_len(&tmpq)); 1276 goto err; 1277 } 1278 } 1279 1280 NETFRONT_SKB_CB(skb)->pull_to = rx->status; 1281 if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD) 1282 NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD; 1283 1284 skb_frag_off_set(&skb_shinfo(skb)->frags[0], rx->offset); 1285 skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status); 1286 skb->data_len = rx->status; 1287 skb->len += rx->status; 1288 1289 if (unlikely(xennet_fill_frags(queue, skb, &tmpq))) 1290 goto err; 1291 1292 if (rx->flags & XEN_NETRXF_csum_blank) 1293 skb->ip_summed = CHECKSUM_PARTIAL; 1294 else if (rx->flags & XEN_NETRXF_data_validated) 1295 skb->ip_summed = CHECKSUM_UNNECESSARY; 1296 1297 __skb_queue_tail(&rxq, skb); 1298 1299 i = queue->rx.rsp_cons + 1; 1300 xennet_set_rx_rsp_cons(queue, i); 1301 work_done++; 1302 } 1303 if (need_xdp_flush) 1304 xdp_do_flush(); 1305 1306 __skb_queue_purge(&errq); 1307 1308 work_done -= handle_incoming_queue(queue, &rxq); 1309 1310 xennet_alloc_rx_buffers(queue); 1311 1312 if (work_done < budget) { 1313 int more_to_do = 0; 1314 1315 napi_complete_done(napi, work_done); 1316 1317 RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do); 1318 if (more_to_do) 1319 napi_schedule(napi); 1320 } 1321 1322 spin_unlock(&queue->rx_lock); 1323 1324 return work_done; 1325 } 1326 1327 static int xennet_change_mtu(struct net_device *dev, int mtu) 1328 { 1329 int max = xennet_can_sg(dev) ? XEN_NETIF_MAX_TX_SIZE : ETH_DATA_LEN; 1330 1331 if (mtu > max) 1332 return -EINVAL; 1333 dev->mtu = mtu; 1334 return 0; 1335 } 1336 1337 static void xennet_get_stats64(struct net_device *dev, 1338 struct rtnl_link_stats64 *tot) 1339 { 1340 struct netfront_info *np = netdev_priv(dev); 1341 int cpu; 1342 1343 for_each_possible_cpu(cpu) { 1344 struct netfront_stats *rx_stats = per_cpu_ptr(np->rx_stats, cpu); 1345 struct netfront_stats *tx_stats = per_cpu_ptr(np->tx_stats, cpu); 1346 u64 rx_packets, rx_bytes, tx_packets, tx_bytes; 1347 unsigned int start; 1348 1349 do { 1350 start = u64_stats_fetch_begin_irq(&tx_stats->syncp); 1351 tx_packets = tx_stats->packets; 1352 tx_bytes = tx_stats->bytes; 1353 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); 1354 1355 do { 1356 start = u64_stats_fetch_begin_irq(&rx_stats->syncp); 1357 rx_packets = rx_stats->packets; 1358 rx_bytes = rx_stats->bytes; 1359 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); 1360 1361 tot->rx_packets += rx_packets; 1362 tot->tx_packets += tx_packets; 1363 tot->rx_bytes += rx_bytes; 1364 tot->tx_bytes += tx_bytes; 1365 } 1366 1367 tot->rx_errors = dev->stats.rx_errors; 1368 tot->tx_dropped = dev->stats.tx_dropped; 1369 } 1370 1371 static void xennet_release_tx_bufs(struct netfront_queue *queue) 1372 { 1373 struct sk_buff *skb; 1374 int i; 1375 1376 for (i = 0; i < NET_TX_RING_SIZE; i++) { 1377 /* Skip over entries which are actually freelist references */ 1378 if (!queue->tx_skbs[i]) 1379 continue; 1380 1381 skb = queue->tx_skbs[i]; 1382 queue->tx_skbs[i] = NULL; 1383 get_page(queue->grant_tx_page[i]); 1384 gnttab_end_foreign_access(queue->grant_tx_ref[i], 1385 GNTMAP_readonly, 1386 (unsigned long)page_address(queue->grant_tx_page[i])); 1387 queue->grant_tx_page[i] = NULL; 1388 queue->grant_tx_ref[i] = GRANT_INVALID_REF; 1389 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, i); 1390 dev_kfree_skb_irq(skb); 1391 } 1392 } 1393 1394 static void xennet_release_rx_bufs(struct netfront_queue *queue) 1395 { 1396 int id, ref; 1397 1398 spin_lock_bh(&queue->rx_lock); 1399 1400 for (id = 0; id < NET_RX_RING_SIZE; id++) { 1401 struct sk_buff *skb; 1402 struct page *page; 1403 1404 skb = queue->rx_skbs[id]; 1405 if (!skb) 1406 continue; 1407 1408 ref = queue->grant_rx_ref[id]; 1409 if (ref == GRANT_INVALID_REF) 1410 continue; 1411 1412 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); 1413 1414 /* gnttab_end_foreign_access() needs a page ref until 1415 * foreign access is ended (which may be deferred). 1416 */ 1417 get_page(page); 1418 gnttab_end_foreign_access(ref, 0, 1419 (unsigned long)page_address(page)); 1420 queue->grant_rx_ref[id] = GRANT_INVALID_REF; 1421 1422 kfree_skb(skb); 1423 } 1424 1425 spin_unlock_bh(&queue->rx_lock); 1426 } 1427 1428 static netdev_features_t xennet_fix_features(struct net_device *dev, 1429 netdev_features_t features) 1430 { 1431 struct netfront_info *np = netdev_priv(dev); 1432 1433 if (features & NETIF_F_SG && 1434 !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0)) 1435 features &= ~NETIF_F_SG; 1436 1437 if (features & NETIF_F_IPV6_CSUM && 1438 !xenbus_read_unsigned(np->xbdev->otherend, 1439 "feature-ipv6-csum-offload", 0)) 1440 features &= ~NETIF_F_IPV6_CSUM; 1441 1442 if (features & NETIF_F_TSO && 1443 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0)) 1444 features &= ~NETIF_F_TSO; 1445 1446 if (features & NETIF_F_TSO6 && 1447 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0)) 1448 features &= ~NETIF_F_TSO6; 1449 1450 return features; 1451 } 1452 1453 static int xennet_set_features(struct net_device *dev, 1454 netdev_features_t features) 1455 { 1456 if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) { 1457 netdev_info(dev, "Reducing MTU because no SG offload"); 1458 dev->mtu = ETH_DATA_LEN; 1459 } 1460 1461 return 0; 1462 } 1463 1464 static bool xennet_handle_tx(struct netfront_queue *queue, unsigned int *eoi) 1465 { 1466 unsigned long flags; 1467 1468 if (unlikely(queue->info->broken)) 1469 return false; 1470 1471 spin_lock_irqsave(&queue->tx_lock, flags); 1472 if (xennet_tx_buf_gc(queue)) 1473 *eoi = 0; 1474 spin_unlock_irqrestore(&queue->tx_lock, flags); 1475 1476 return true; 1477 } 1478 1479 static irqreturn_t xennet_tx_interrupt(int irq, void *dev_id) 1480 { 1481 unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS; 1482 1483 if (likely(xennet_handle_tx(dev_id, &eoiflag))) 1484 xen_irq_lateeoi(irq, eoiflag); 1485 1486 return IRQ_HANDLED; 1487 } 1488 1489 static bool xennet_handle_rx(struct netfront_queue *queue, unsigned int *eoi) 1490 { 1491 unsigned int work_queued; 1492 unsigned long flags; 1493 1494 if (unlikely(queue->info->broken)) 1495 return false; 1496 1497 spin_lock_irqsave(&queue->rx_cons_lock, flags); 1498 work_queued = RING_HAS_UNCONSUMED_RESPONSES(&queue->rx); 1499 if (work_queued > queue->rx_rsp_unconsumed) { 1500 queue->rx_rsp_unconsumed = work_queued; 1501 *eoi = 0; 1502 } else if (unlikely(work_queued < queue->rx_rsp_unconsumed)) { 1503 const struct device *dev = &queue->info->netdev->dev; 1504 1505 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); 1506 dev_alert(dev, "RX producer index going backwards\n"); 1507 dev_alert(dev, "Disabled for further use\n"); 1508 queue->info->broken = true; 1509 return false; 1510 } 1511 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); 1512 1513 if (likely(netif_carrier_ok(queue->info->netdev) && work_queued)) 1514 napi_schedule(&queue->napi); 1515 1516 return true; 1517 } 1518 1519 static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id) 1520 { 1521 unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS; 1522 1523 if (likely(xennet_handle_rx(dev_id, &eoiflag))) 1524 xen_irq_lateeoi(irq, eoiflag); 1525 1526 return IRQ_HANDLED; 1527 } 1528 1529 static irqreturn_t xennet_interrupt(int irq, void *dev_id) 1530 { 1531 unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS; 1532 1533 if (xennet_handle_tx(dev_id, &eoiflag) && 1534 xennet_handle_rx(dev_id, &eoiflag)) 1535 xen_irq_lateeoi(irq, eoiflag); 1536 1537 return IRQ_HANDLED; 1538 } 1539 1540 #ifdef CONFIG_NET_POLL_CONTROLLER 1541 static void xennet_poll_controller(struct net_device *dev) 1542 { 1543 /* Poll each queue */ 1544 struct netfront_info *info = netdev_priv(dev); 1545 unsigned int num_queues = dev->real_num_tx_queues; 1546 unsigned int i; 1547 1548 if (info->broken) 1549 return; 1550 1551 for (i = 0; i < num_queues; ++i) 1552 xennet_interrupt(0, &info->queues[i]); 1553 } 1554 #endif 1555 1556 #define NETBACK_XDP_HEADROOM_DISABLE 0 1557 #define NETBACK_XDP_HEADROOM_ENABLE 1 1558 1559 static int talk_to_netback_xdp(struct netfront_info *np, int xdp) 1560 { 1561 int err; 1562 unsigned short headroom; 1563 1564 headroom = xdp ? XDP_PACKET_HEADROOM : 0; 1565 err = xenbus_printf(XBT_NIL, np->xbdev->nodename, 1566 "xdp-headroom", "%hu", 1567 headroom); 1568 if (err) 1569 pr_warn("Error writing xdp-headroom\n"); 1570 1571 return err; 1572 } 1573 1574 static int xennet_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1575 struct netlink_ext_ack *extack) 1576 { 1577 unsigned long max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM; 1578 struct netfront_info *np = netdev_priv(dev); 1579 struct bpf_prog *old_prog; 1580 unsigned int i, err; 1581 1582 if (dev->mtu > max_mtu) { 1583 netdev_warn(dev, "XDP requires MTU less than %lu\n", max_mtu); 1584 return -EINVAL; 1585 } 1586 1587 if (!np->netback_has_xdp_headroom) 1588 return 0; 1589 1590 xenbus_switch_state(np->xbdev, XenbusStateReconfiguring); 1591 1592 err = talk_to_netback_xdp(np, prog ? NETBACK_XDP_HEADROOM_ENABLE : 1593 NETBACK_XDP_HEADROOM_DISABLE); 1594 if (err) 1595 return err; 1596 1597 /* avoid the race with XDP headroom adjustment */ 1598 wait_event(module_wq, 1599 xenbus_read_driver_state(np->xbdev->otherend) == 1600 XenbusStateReconfigured); 1601 np->netfront_xdp_enabled = true; 1602 1603 old_prog = rtnl_dereference(np->queues[0].xdp_prog); 1604 1605 if (prog) 1606 bpf_prog_add(prog, dev->real_num_tx_queues); 1607 1608 for (i = 0; i < dev->real_num_tx_queues; ++i) 1609 rcu_assign_pointer(np->queues[i].xdp_prog, prog); 1610 1611 if (old_prog) 1612 for (i = 0; i < dev->real_num_tx_queues; ++i) 1613 bpf_prog_put(old_prog); 1614 1615 xenbus_switch_state(np->xbdev, XenbusStateConnected); 1616 1617 return 0; 1618 } 1619 1620 static int xennet_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1621 { 1622 struct netfront_info *np = netdev_priv(dev); 1623 1624 if (np->broken) 1625 return -ENODEV; 1626 1627 switch (xdp->command) { 1628 case XDP_SETUP_PROG: 1629 return xennet_xdp_set(dev, xdp->prog, xdp->extack); 1630 default: 1631 return -EINVAL; 1632 } 1633 } 1634 1635 static const struct net_device_ops xennet_netdev_ops = { 1636 .ndo_uninit = xennet_uninit, 1637 .ndo_open = xennet_open, 1638 .ndo_stop = xennet_close, 1639 .ndo_start_xmit = xennet_start_xmit, 1640 .ndo_change_mtu = xennet_change_mtu, 1641 .ndo_get_stats64 = xennet_get_stats64, 1642 .ndo_set_mac_address = eth_mac_addr, 1643 .ndo_validate_addr = eth_validate_addr, 1644 .ndo_fix_features = xennet_fix_features, 1645 .ndo_set_features = xennet_set_features, 1646 .ndo_select_queue = xennet_select_queue, 1647 .ndo_bpf = xennet_xdp, 1648 .ndo_xdp_xmit = xennet_xdp_xmit, 1649 #ifdef CONFIG_NET_POLL_CONTROLLER 1650 .ndo_poll_controller = xennet_poll_controller, 1651 #endif 1652 }; 1653 1654 static void xennet_free_netdev(struct net_device *netdev) 1655 { 1656 struct netfront_info *np = netdev_priv(netdev); 1657 1658 free_percpu(np->rx_stats); 1659 free_percpu(np->tx_stats); 1660 free_netdev(netdev); 1661 } 1662 1663 static struct net_device *xennet_create_dev(struct xenbus_device *dev) 1664 { 1665 int err; 1666 struct net_device *netdev; 1667 struct netfront_info *np; 1668 1669 netdev = alloc_etherdev_mq(sizeof(struct netfront_info), xennet_max_queues); 1670 if (!netdev) 1671 return ERR_PTR(-ENOMEM); 1672 1673 np = netdev_priv(netdev); 1674 np->xbdev = dev; 1675 1676 np->queues = NULL; 1677 1678 err = -ENOMEM; 1679 np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); 1680 if (np->rx_stats == NULL) 1681 goto exit; 1682 np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); 1683 if (np->tx_stats == NULL) 1684 goto exit; 1685 1686 netdev->netdev_ops = &xennet_netdev_ops; 1687 1688 netdev->features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | 1689 NETIF_F_GSO_ROBUST; 1690 netdev->hw_features = NETIF_F_SG | 1691 NETIF_F_IPV6_CSUM | 1692 NETIF_F_TSO | NETIF_F_TSO6; 1693 1694 /* 1695 * Assume that all hw features are available for now. This set 1696 * will be adjusted by the call to netdev_update_features() in 1697 * xennet_connect() which is the earliest point where we can 1698 * negotiate with the backend regarding supported features. 1699 */ 1700 netdev->features |= netdev->hw_features; 1701 1702 netdev->ethtool_ops = &xennet_ethtool_ops; 1703 netdev->min_mtu = ETH_MIN_MTU; 1704 netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE; 1705 SET_NETDEV_DEV(netdev, &dev->dev); 1706 1707 np->netdev = netdev; 1708 np->netfront_xdp_enabled = false; 1709 1710 netif_carrier_off(netdev); 1711 1712 do { 1713 xenbus_switch_state(dev, XenbusStateInitialising); 1714 err = wait_event_timeout(module_wq, 1715 xenbus_read_driver_state(dev->otherend) != 1716 XenbusStateClosed && 1717 xenbus_read_driver_state(dev->otherend) != 1718 XenbusStateUnknown, XENNET_TIMEOUT); 1719 } while (!err); 1720 1721 return netdev; 1722 1723 exit: 1724 xennet_free_netdev(netdev); 1725 return ERR_PTR(err); 1726 } 1727 1728 /* 1729 * Entry point to this code when a new device is created. Allocate the basic 1730 * structures and the ring buffers for communication with the backend, and 1731 * inform the backend of the appropriate details for those. 1732 */ 1733 static int netfront_probe(struct xenbus_device *dev, 1734 const struct xenbus_device_id *id) 1735 { 1736 int err; 1737 struct net_device *netdev; 1738 struct netfront_info *info; 1739 1740 netdev = xennet_create_dev(dev); 1741 if (IS_ERR(netdev)) { 1742 err = PTR_ERR(netdev); 1743 xenbus_dev_fatal(dev, err, "creating netdev"); 1744 return err; 1745 } 1746 1747 info = netdev_priv(netdev); 1748 dev_set_drvdata(&dev->dev, info); 1749 #ifdef CONFIG_SYSFS 1750 info->netdev->sysfs_groups[0] = &xennet_dev_group; 1751 #endif 1752 1753 return 0; 1754 } 1755 1756 static void xennet_end_access(int ref, void *page) 1757 { 1758 /* This frees the page as a side-effect */ 1759 if (ref != GRANT_INVALID_REF) 1760 gnttab_end_foreign_access(ref, 0, (unsigned long)page); 1761 } 1762 1763 static void xennet_disconnect_backend(struct netfront_info *info) 1764 { 1765 unsigned int i = 0; 1766 unsigned int num_queues = info->netdev->real_num_tx_queues; 1767 1768 netif_carrier_off(info->netdev); 1769 1770 for (i = 0; i < num_queues && info->queues; ++i) { 1771 struct netfront_queue *queue = &info->queues[i]; 1772 1773 del_timer_sync(&queue->rx_refill_timer); 1774 1775 if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) 1776 unbind_from_irqhandler(queue->tx_irq, queue); 1777 if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { 1778 unbind_from_irqhandler(queue->tx_irq, queue); 1779 unbind_from_irqhandler(queue->rx_irq, queue); 1780 } 1781 queue->tx_evtchn = queue->rx_evtchn = 0; 1782 queue->tx_irq = queue->rx_irq = 0; 1783 1784 if (netif_running(info->netdev)) 1785 napi_synchronize(&queue->napi); 1786 1787 xennet_release_tx_bufs(queue); 1788 xennet_release_rx_bufs(queue); 1789 gnttab_free_grant_references(queue->gref_tx_head); 1790 gnttab_free_grant_references(queue->gref_rx_head); 1791 1792 /* End access and free the pages */ 1793 xennet_end_access(queue->tx_ring_ref, queue->tx.sring); 1794 xennet_end_access(queue->rx_ring_ref, queue->rx.sring); 1795 1796 queue->tx_ring_ref = GRANT_INVALID_REF; 1797 queue->rx_ring_ref = GRANT_INVALID_REF; 1798 queue->tx.sring = NULL; 1799 queue->rx.sring = NULL; 1800 1801 page_pool_destroy(queue->page_pool); 1802 } 1803 } 1804 1805 /* 1806 * We are reconnecting to the backend, due to a suspend/resume, or a backend 1807 * driver restart. We tear down our netif structure and recreate it, but 1808 * leave the device-layer structures intact so that this is transparent to the 1809 * rest of the kernel. 1810 */ 1811 static int netfront_resume(struct xenbus_device *dev) 1812 { 1813 struct netfront_info *info = dev_get_drvdata(&dev->dev); 1814 1815 dev_dbg(&dev->dev, "%s\n", dev->nodename); 1816 1817 netif_tx_lock_bh(info->netdev); 1818 netif_device_detach(info->netdev); 1819 netif_tx_unlock_bh(info->netdev); 1820 1821 xennet_disconnect_backend(info); 1822 return 0; 1823 } 1824 1825 static int xen_net_read_mac(struct xenbus_device *dev, u8 mac[]) 1826 { 1827 char *s, *e, *macstr; 1828 int i; 1829 1830 macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL); 1831 if (IS_ERR(macstr)) 1832 return PTR_ERR(macstr); 1833 1834 for (i = 0; i < ETH_ALEN; i++) { 1835 mac[i] = simple_strtoul(s, &e, 16); 1836 if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) { 1837 kfree(macstr); 1838 return -ENOENT; 1839 } 1840 s = e+1; 1841 } 1842 1843 kfree(macstr); 1844 return 0; 1845 } 1846 1847 static int setup_netfront_single(struct netfront_queue *queue) 1848 { 1849 int err; 1850 1851 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); 1852 if (err < 0) 1853 goto fail; 1854 1855 err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, 1856 xennet_interrupt, 0, 1857 queue->info->netdev->name, 1858 queue); 1859 if (err < 0) 1860 goto bind_fail; 1861 queue->rx_evtchn = queue->tx_evtchn; 1862 queue->rx_irq = queue->tx_irq = err; 1863 1864 return 0; 1865 1866 bind_fail: 1867 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); 1868 queue->tx_evtchn = 0; 1869 fail: 1870 return err; 1871 } 1872 1873 static int setup_netfront_split(struct netfront_queue *queue) 1874 { 1875 int err; 1876 1877 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); 1878 if (err < 0) 1879 goto fail; 1880 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn); 1881 if (err < 0) 1882 goto alloc_rx_evtchn_fail; 1883 1884 snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name), 1885 "%s-tx", queue->name); 1886 err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, 1887 xennet_tx_interrupt, 0, 1888 queue->tx_irq_name, queue); 1889 if (err < 0) 1890 goto bind_tx_fail; 1891 queue->tx_irq = err; 1892 1893 snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name), 1894 "%s-rx", queue->name); 1895 err = bind_evtchn_to_irqhandler_lateeoi(queue->rx_evtchn, 1896 xennet_rx_interrupt, 0, 1897 queue->rx_irq_name, queue); 1898 if (err < 0) 1899 goto bind_rx_fail; 1900 queue->rx_irq = err; 1901 1902 return 0; 1903 1904 bind_rx_fail: 1905 unbind_from_irqhandler(queue->tx_irq, queue); 1906 queue->tx_irq = 0; 1907 bind_tx_fail: 1908 xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn); 1909 queue->rx_evtchn = 0; 1910 alloc_rx_evtchn_fail: 1911 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); 1912 queue->tx_evtchn = 0; 1913 fail: 1914 return err; 1915 } 1916 1917 static int setup_netfront(struct xenbus_device *dev, 1918 struct netfront_queue *queue, unsigned int feature_split_evtchn) 1919 { 1920 struct xen_netif_tx_sring *txs; 1921 struct xen_netif_rx_sring *rxs; 1922 grant_ref_t gref; 1923 int err; 1924 1925 queue->tx_ring_ref = GRANT_INVALID_REF; 1926 queue->rx_ring_ref = GRANT_INVALID_REF; 1927 queue->rx.sring = NULL; 1928 queue->tx.sring = NULL; 1929 1930 txs = (struct xen_netif_tx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); 1931 if (!txs) { 1932 err = -ENOMEM; 1933 xenbus_dev_fatal(dev, err, "allocating tx ring page"); 1934 goto fail; 1935 } 1936 SHARED_RING_INIT(txs); 1937 FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE); 1938 1939 err = xenbus_grant_ring(dev, txs, 1, &gref); 1940 if (err < 0) 1941 goto grant_tx_ring_fail; 1942 queue->tx_ring_ref = gref; 1943 1944 rxs = (struct xen_netif_rx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); 1945 if (!rxs) { 1946 err = -ENOMEM; 1947 xenbus_dev_fatal(dev, err, "allocating rx ring page"); 1948 goto alloc_rx_ring_fail; 1949 } 1950 SHARED_RING_INIT(rxs); 1951 FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); 1952 1953 err = xenbus_grant_ring(dev, rxs, 1, &gref); 1954 if (err < 0) 1955 goto grant_rx_ring_fail; 1956 queue->rx_ring_ref = gref; 1957 1958 if (feature_split_evtchn) 1959 err = setup_netfront_split(queue); 1960 /* setup single event channel if 1961 * a) feature-split-event-channels == 0 1962 * b) feature-split-event-channels == 1 but failed to setup 1963 */ 1964 if (!feature_split_evtchn || err) 1965 err = setup_netfront_single(queue); 1966 1967 if (err) 1968 goto alloc_evtchn_fail; 1969 1970 return 0; 1971 1972 /* If we fail to setup netfront, it is safe to just revoke access to 1973 * granted pages because backend is not accessing it at this point. 1974 */ 1975 alloc_evtchn_fail: 1976 gnttab_end_foreign_access_ref(queue->rx_ring_ref, 0); 1977 grant_rx_ring_fail: 1978 free_page((unsigned long)rxs); 1979 alloc_rx_ring_fail: 1980 gnttab_end_foreign_access_ref(queue->tx_ring_ref, 0); 1981 grant_tx_ring_fail: 1982 free_page((unsigned long)txs); 1983 fail: 1984 return err; 1985 } 1986 1987 /* Queue-specific initialisation 1988 * This used to be done in xennet_create_dev() but must now 1989 * be run per-queue. 1990 */ 1991 static int xennet_init_queue(struct netfront_queue *queue) 1992 { 1993 unsigned short i; 1994 int err = 0; 1995 char *devid; 1996 1997 spin_lock_init(&queue->tx_lock); 1998 spin_lock_init(&queue->rx_lock); 1999 spin_lock_init(&queue->rx_cons_lock); 2000 2001 timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0); 2002 2003 devid = strrchr(queue->info->xbdev->nodename, '/') + 1; 2004 snprintf(queue->name, sizeof(queue->name), "vif%s-q%u", 2005 devid, queue->id); 2006 2007 /* Initialise tx_skb_freelist as a free chain containing every entry. */ 2008 queue->tx_skb_freelist = 0; 2009 queue->tx_pend_queue = TX_LINK_NONE; 2010 for (i = 0; i < NET_TX_RING_SIZE; i++) { 2011 queue->tx_link[i] = i + 1; 2012 queue->grant_tx_ref[i] = GRANT_INVALID_REF; 2013 queue->grant_tx_page[i] = NULL; 2014 } 2015 queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE; 2016 2017 /* Clear out rx_skbs */ 2018 for (i = 0; i < NET_RX_RING_SIZE; i++) { 2019 queue->rx_skbs[i] = NULL; 2020 queue->grant_rx_ref[i] = GRANT_INVALID_REF; 2021 } 2022 2023 /* A grant for every tx ring slot */ 2024 if (gnttab_alloc_grant_references(NET_TX_RING_SIZE, 2025 &queue->gref_tx_head) < 0) { 2026 pr_alert("can't alloc tx grant refs\n"); 2027 err = -ENOMEM; 2028 goto exit; 2029 } 2030 2031 /* A grant for every rx ring slot */ 2032 if (gnttab_alloc_grant_references(NET_RX_RING_SIZE, 2033 &queue->gref_rx_head) < 0) { 2034 pr_alert("can't alloc rx grant refs\n"); 2035 err = -ENOMEM; 2036 goto exit_free_tx; 2037 } 2038 2039 return 0; 2040 2041 exit_free_tx: 2042 gnttab_free_grant_references(queue->gref_tx_head); 2043 exit: 2044 return err; 2045 } 2046 2047 static int write_queue_xenstore_keys(struct netfront_queue *queue, 2048 struct xenbus_transaction *xbt, int write_hierarchical) 2049 { 2050 /* Write the queue-specific keys into XenStore in the traditional 2051 * way for a single queue, or in a queue subkeys for multiple 2052 * queues. 2053 */ 2054 struct xenbus_device *dev = queue->info->xbdev; 2055 int err; 2056 const char *message; 2057 char *path; 2058 size_t pathsize; 2059 2060 /* Choose the correct place to write the keys */ 2061 if (write_hierarchical) { 2062 pathsize = strlen(dev->nodename) + 10; 2063 path = kzalloc(pathsize, GFP_KERNEL); 2064 if (!path) { 2065 err = -ENOMEM; 2066 message = "out of memory while writing ring references"; 2067 goto error; 2068 } 2069 snprintf(path, pathsize, "%s/queue-%u", 2070 dev->nodename, queue->id); 2071 } else { 2072 path = (char *)dev->nodename; 2073 } 2074 2075 /* Write ring references */ 2076 err = xenbus_printf(*xbt, path, "tx-ring-ref", "%u", 2077 queue->tx_ring_ref); 2078 if (err) { 2079 message = "writing tx-ring-ref"; 2080 goto error; 2081 } 2082 2083 err = xenbus_printf(*xbt, path, "rx-ring-ref", "%u", 2084 queue->rx_ring_ref); 2085 if (err) { 2086 message = "writing rx-ring-ref"; 2087 goto error; 2088 } 2089 2090 /* Write event channels; taking into account both shared 2091 * and split event channel scenarios. 2092 */ 2093 if (queue->tx_evtchn == queue->rx_evtchn) { 2094 /* Shared event channel */ 2095 err = xenbus_printf(*xbt, path, 2096 "event-channel", "%u", queue->tx_evtchn); 2097 if (err) { 2098 message = "writing event-channel"; 2099 goto error; 2100 } 2101 } else { 2102 /* Split event channels */ 2103 err = xenbus_printf(*xbt, path, 2104 "event-channel-tx", "%u", queue->tx_evtchn); 2105 if (err) { 2106 message = "writing event-channel-tx"; 2107 goto error; 2108 } 2109 2110 err = xenbus_printf(*xbt, path, 2111 "event-channel-rx", "%u", queue->rx_evtchn); 2112 if (err) { 2113 message = "writing event-channel-rx"; 2114 goto error; 2115 } 2116 } 2117 2118 if (write_hierarchical) 2119 kfree(path); 2120 return 0; 2121 2122 error: 2123 if (write_hierarchical) 2124 kfree(path); 2125 xenbus_dev_fatal(dev, err, "%s", message); 2126 return err; 2127 } 2128 2129 2130 2131 static int xennet_create_page_pool(struct netfront_queue *queue) 2132 { 2133 int err; 2134 struct page_pool_params pp_params = { 2135 .order = 0, 2136 .flags = 0, 2137 .pool_size = NET_RX_RING_SIZE, 2138 .nid = NUMA_NO_NODE, 2139 .dev = &queue->info->netdev->dev, 2140 .offset = XDP_PACKET_HEADROOM, 2141 .max_len = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM, 2142 }; 2143 2144 queue->page_pool = page_pool_create(&pp_params); 2145 if (IS_ERR(queue->page_pool)) { 2146 err = PTR_ERR(queue->page_pool); 2147 queue->page_pool = NULL; 2148 return err; 2149 } 2150 2151 err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, 2152 queue->id, 0); 2153 if (err) { 2154 netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); 2155 goto err_free_pp; 2156 } 2157 2158 err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq, 2159 MEM_TYPE_PAGE_POOL, queue->page_pool); 2160 if (err) { 2161 netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n"); 2162 goto err_unregister_rxq; 2163 } 2164 return 0; 2165 2166 err_unregister_rxq: 2167 xdp_rxq_info_unreg(&queue->xdp_rxq); 2168 err_free_pp: 2169 page_pool_destroy(queue->page_pool); 2170 queue->page_pool = NULL; 2171 return err; 2172 } 2173 2174 static int xennet_create_queues(struct netfront_info *info, 2175 unsigned int *num_queues) 2176 { 2177 unsigned int i; 2178 int ret; 2179 2180 info->queues = kcalloc(*num_queues, sizeof(struct netfront_queue), 2181 GFP_KERNEL); 2182 if (!info->queues) 2183 return -ENOMEM; 2184 2185 for (i = 0; i < *num_queues; i++) { 2186 struct netfront_queue *queue = &info->queues[i]; 2187 2188 queue->id = i; 2189 queue->info = info; 2190 2191 ret = xennet_init_queue(queue); 2192 if (ret < 0) { 2193 dev_warn(&info->xbdev->dev, 2194 "only created %d queues\n", i); 2195 *num_queues = i; 2196 break; 2197 } 2198 2199 /* use page pool recycling instead of buddy allocator */ 2200 ret = xennet_create_page_pool(queue); 2201 if (ret < 0) { 2202 dev_err(&info->xbdev->dev, "can't allocate page pool\n"); 2203 *num_queues = i; 2204 return ret; 2205 } 2206 2207 netif_napi_add(queue->info->netdev, &queue->napi, 2208 xennet_poll, 64); 2209 if (netif_running(info->netdev)) 2210 napi_enable(&queue->napi); 2211 } 2212 2213 netif_set_real_num_tx_queues(info->netdev, *num_queues); 2214 2215 if (*num_queues == 0) { 2216 dev_err(&info->xbdev->dev, "no queues\n"); 2217 return -EINVAL; 2218 } 2219 return 0; 2220 } 2221 2222 /* Common code used when first setting up, and when resuming. */ 2223 static int talk_to_netback(struct xenbus_device *dev, 2224 struct netfront_info *info) 2225 { 2226 const char *message; 2227 struct xenbus_transaction xbt; 2228 int err; 2229 unsigned int feature_split_evtchn; 2230 unsigned int i = 0; 2231 unsigned int max_queues = 0; 2232 struct netfront_queue *queue = NULL; 2233 unsigned int num_queues = 1; 2234 u8 addr[ETH_ALEN]; 2235 2236 info->netdev->irq = 0; 2237 2238 /* Check if backend supports multiple queues */ 2239 max_queues = xenbus_read_unsigned(info->xbdev->otherend, 2240 "multi-queue-max-queues", 1); 2241 num_queues = min(max_queues, xennet_max_queues); 2242 2243 /* Check feature-split-event-channels */ 2244 feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend, 2245 "feature-split-event-channels", 0); 2246 2247 /* Read mac addr. */ 2248 err = xen_net_read_mac(dev, addr); 2249 if (err) { 2250 xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename); 2251 goto out_unlocked; 2252 } 2253 eth_hw_addr_set(info->netdev, addr); 2254 2255 info->netback_has_xdp_headroom = xenbus_read_unsigned(info->xbdev->otherend, 2256 "feature-xdp-headroom", 0); 2257 if (info->netback_has_xdp_headroom) { 2258 /* set the current xen-netfront xdp state */ 2259 err = talk_to_netback_xdp(info, info->netfront_xdp_enabled ? 2260 NETBACK_XDP_HEADROOM_ENABLE : 2261 NETBACK_XDP_HEADROOM_DISABLE); 2262 if (err) 2263 goto out_unlocked; 2264 } 2265 2266 rtnl_lock(); 2267 if (info->queues) 2268 xennet_destroy_queues(info); 2269 2270 /* For the case of a reconnect reset the "broken" indicator. */ 2271 info->broken = false; 2272 2273 err = xennet_create_queues(info, &num_queues); 2274 if (err < 0) { 2275 xenbus_dev_fatal(dev, err, "creating queues"); 2276 kfree(info->queues); 2277 info->queues = NULL; 2278 goto out; 2279 } 2280 rtnl_unlock(); 2281 2282 /* Create shared ring, alloc event channel -- for each queue */ 2283 for (i = 0; i < num_queues; ++i) { 2284 queue = &info->queues[i]; 2285 err = setup_netfront(dev, queue, feature_split_evtchn); 2286 if (err) 2287 goto destroy_ring; 2288 } 2289 2290 again: 2291 err = xenbus_transaction_start(&xbt); 2292 if (err) { 2293 xenbus_dev_fatal(dev, err, "starting transaction"); 2294 goto destroy_ring; 2295 } 2296 2297 if (xenbus_exists(XBT_NIL, 2298 info->xbdev->otherend, "multi-queue-max-queues")) { 2299 /* Write the number of queues */ 2300 err = xenbus_printf(xbt, dev->nodename, 2301 "multi-queue-num-queues", "%u", num_queues); 2302 if (err) { 2303 message = "writing multi-queue-num-queues"; 2304 goto abort_transaction_no_dev_fatal; 2305 } 2306 } 2307 2308 if (num_queues == 1) { 2309 err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */ 2310 if (err) 2311 goto abort_transaction_no_dev_fatal; 2312 } else { 2313 /* Write the keys for each queue */ 2314 for (i = 0; i < num_queues; ++i) { 2315 queue = &info->queues[i]; 2316 err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */ 2317 if (err) 2318 goto abort_transaction_no_dev_fatal; 2319 } 2320 } 2321 2322 /* The remaining keys are not queue-specific */ 2323 err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u", 2324 1); 2325 if (err) { 2326 message = "writing request-rx-copy"; 2327 goto abort_transaction; 2328 } 2329 2330 err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1); 2331 if (err) { 2332 message = "writing feature-rx-notify"; 2333 goto abort_transaction; 2334 } 2335 2336 err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1); 2337 if (err) { 2338 message = "writing feature-sg"; 2339 goto abort_transaction; 2340 } 2341 2342 err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1); 2343 if (err) { 2344 message = "writing feature-gso-tcpv4"; 2345 goto abort_transaction; 2346 } 2347 2348 err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1"); 2349 if (err) { 2350 message = "writing feature-gso-tcpv6"; 2351 goto abort_transaction; 2352 } 2353 2354 err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload", 2355 "1"); 2356 if (err) { 2357 message = "writing feature-ipv6-csum-offload"; 2358 goto abort_transaction; 2359 } 2360 2361 err = xenbus_transaction_end(xbt, 0); 2362 if (err) { 2363 if (err == -EAGAIN) 2364 goto again; 2365 xenbus_dev_fatal(dev, err, "completing transaction"); 2366 goto destroy_ring; 2367 } 2368 2369 return 0; 2370 2371 abort_transaction: 2372 xenbus_dev_fatal(dev, err, "%s", message); 2373 abort_transaction_no_dev_fatal: 2374 xenbus_transaction_end(xbt, 1); 2375 destroy_ring: 2376 xennet_disconnect_backend(info); 2377 rtnl_lock(); 2378 xennet_destroy_queues(info); 2379 out: 2380 rtnl_unlock(); 2381 out_unlocked: 2382 device_unregister(&dev->dev); 2383 return err; 2384 } 2385 2386 static int xennet_connect(struct net_device *dev) 2387 { 2388 struct netfront_info *np = netdev_priv(dev); 2389 unsigned int num_queues = 0; 2390 int err; 2391 unsigned int j = 0; 2392 struct netfront_queue *queue = NULL; 2393 2394 if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) { 2395 dev_info(&dev->dev, 2396 "backend does not support copying receive path\n"); 2397 return -ENODEV; 2398 } 2399 2400 err = talk_to_netback(np->xbdev, np); 2401 if (err) 2402 return err; 2403 if (np->netback_has_xdp_headroom) 2404 pr_info("backend supports XDP headroom\n"); 2405 2406 /* talk_to_netback() sets the correct number of queues */ 2407 num_queues = dev->real_num_tx_queues; 2408 2409 if (dev->reg_state == NETREG_UNINITIALIZED) { 2410 err = register_netdev(dev); 2411 if (err) { 2412 pr_warn("%s: register_netdev err=%d\n", __func__, err); 2413 device_unregister(&np->xbdev->dev); 2414 return err; 2415 } 2416 } 2417 2418 rtnl_lock(); 2419 netdev_update_features(dev); 2420 rtnl_unlock(); 2421 2422 /* 2423 * All public and private state should now be sane. Get 2424 * ready to start sending and receiving packets and give the driver 2425 * domain a kick because we've probably just requeued some 2426 * packets. 2427 */ 2428 netif_tx_lock_bh(np->netdev); 2429 netif_device_attach(np->netdev); 2430 netif_tx_unlock_bh(np->netdev); 2431 2432 netif_carrier_on(np->netdev); 2433 for (j = 0; j < num_queues; ++j) { 2434 queue = &np->queues[j]; 2435 2436 notify_remote_via_irq(queue->tx_irq); 2437 if (queue->tx_irq != queue->rx_irq) 2438 notify_remote_via_irq(queue->rx_irq); 2439 2440 spin_lock_irq(&queue->tx_lock); 2441 xennet_tx_buf_gc(queue); 2442 spin_unlock_irq(&queue->tx_lock); 2443 2444 spin_lock_bh(&queue->rx_lock); 2445 xennet_alloc_rx_buffers(queue); 2446 spin_unlock_bh(&queue->rx_lock); 2447 } 2448 2449 return 0; 2450 } 2451 2452 /* 2453 * Callback received when the backend's state changes. 2454 */ 2455 static void netback_changed(struct xenbus_device *dev, 2456 enum xenbus_state backend_state) 2457 { 2458 struct netfront_info *np = dev_get_drvdata(&dev->dev); 2459 struct net_device *netdev = np->netdev; 2460 2461 dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state)); 2462 2463 wake_up_all(&module_wq); 2464 2465 switch (backend_state) { 2466 case XenbusStateInitialising: 2467 case XenbusStateInitialised: 2468 case XenbusStateReconfiguring: 2469 case XenbusStateReconfigured: 2470 case XenbusStateUnknown: 2471 break; 2472 2473 case XenbusStateInitWait: 2474 if (dev->state != XenbusStateInitialising) 2475 break; 2476 if (xennet_connect(netdev) != 0) 2477 break; 2478 xenbus_switch_state(dev, XenbusStateConnected); 2479 break; 2480 2481 case XenbusStateConnected: 2482 netdev_notify_peers(netdev); 2483 break; 2484 2485 case XenbusStateClosed: 2486 if (dev->state == XenbusStateClosed) 2487 break; 2488 fallthrough; /* Missed the backend's CLOSING state */ 2489 case XenbusStateClosing: 2490 xenbus_frontend_closed(dev); 2491 break; 2492 } 2493 } 2494 2495 static const struct xennet_stat { 2496 char name[ETH_GSTRING_LEN]; 2497 u16 offset; 2498 } xennet_stats[] = { 2499 { 2500 "rx_gso_checksum_fixup", 2501 offsetof(struct netfront_info, rx_gso_checksum_fixup) 2502 }, 2503 }; 2504 2505 static int xennet_get_sset_count(struct net_device *dev, int string_set) 2506 { 2507 switch (string_set) { 2508 case ETH_SS_STATS: 2509 return ARRAY_SIZE(xennet_stats); 2510 default: 2511 return -EINVAL; 2512 } 2513 } 2514 2515 static void xennet_get_ethtool_stats(struct net_device *dev, 2516 struct ethtool_stats *stats, u64 * data) 2517 { 2518 void *np = netdev_priv(dev); 2519 int i; 2520 2521 for (i = 0; i < ARRAY_SIZE(xennet_stats); i++) 2522 data[i] = atomic_read((atomic_t *)(np + xennet_stats[i].offset)); 2523 } 2524 2525 static void xennet_get_strings(struct net_device *dev, u32 stringset, u8 * data) 2526 { 2527 int i; 2528 2529 switch (stringset) { 2530 case ETH_SS_STATS: 2531 for (i = 0; i < ARRAY_SIZE(xennet_stats); i++) 2532 memcpy(data + i * ETH_GSTRING_LEN, 2533 xennet_stats[i].name, ETH_GSTRING_LEN); 2534 break; 2535 } 2536 } 2537 2538 static const struct ethtool_ops xennet_ethtool_ops = 2539 { 2540 .get_link = ethtool_op_get_link, 2541 2542 .get_sset_count = xennet_get_sset_count, 2543 .get_ethtool_stats = xennet_get_ethtool_stats, 2544 .get_strings = xennet_get_strings, 2545 .get_ts_info = ethtool_op_get_ts_info, 2546 }; 2547 2548 #ifdef CONFIG_SYSFS 2549 static ssize_t show_rxbuf(struct device *dev, 2550 struct device_attribute *attr, char *buf) 2551 { 2552 return sprintf(buf, "%lu\n", NET_RX_RING_SIZE); 2553 } 2554 2555 static ssize_t store_rxbuf(struct device *dev, 2556 struct device_attribute *attr, 2557 const char *buf, size_t len) 2558 { 2559 char *endp; 2560 2561 if (!capable(CAP_NET_ADMIN)) 2562 return -EPERM; 2563 2564 simple_strtoul(buf, &endp, 0); 2565 if (endp == buf) 2566 return -EBADMSG; 2567 2568 /* rxbuf_min and rxbuf_max are no longer configurable. */ 2569 2570 return len; 2571 } 2572 2573 static DEVICE_ATTR(rxbuf_min, 0644, show_rxbuf, store_rxbuf); 2574 static DEVICE_ATTR(rxbuf_max, 0644, show_rxbuf, store_rxbuf); 2575 static DEVICE_ATTR(rxbuf_cur, 0444, show_rxbuf, NULL); 2576 2577 static struct attribute *xennet_dev_attrs[] = { 2578 &dev_attr_rxbuf_min.attr, 2579 &dev_attr_rxbuf_max.attr, 2580 &dev_attr_rxbuf_cur.attr, 2581 NULL 2582 }; 2583 2584 static const struct attribute_group xennet_dev_group = { 2585 .attrs = xennet_dev_attrs 2586 }; 2587 #endif /* CONFIG_SYSFS */ 2588 2589 static void xennet_bus_close(struct xenbus_device *dev) 2590 { 2591 int ret; 2592 2593 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) 2594 return; 2595 do { 2596 xenbus_switch_state(dev, XenbusStateClosing); 2597 ret = wait_event_timeout(module_wq, 2598 xenbus_read_driver_state(dev->otherend) == 2599 XenbusStateClosing || 2600 xenbus_read_driver_state(dev->otherend) == 2601 XenbusStateClosed || 2602 xenbus_read_driver_state(dev->otherend) == 2603 XenbusStateUnknown, 2604 XENNET_TIMEOUT); 2605 } while (!ret); 2606 2607 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) 2608 return; 2609 2610 do { 2611 xenbus_switch_state(dev, XenbusStateClosed); 2612 ret = wait_event_timeout(module_wq, 2613 xenbus_read_driver_state(dev->otherend) == 2614 XenbusStateClosed || 2615 xenbus_read_driver_state(dev->otherend) == 2616 XenbusStateUnknown, 2617 XENNET_TIMEOUT); 2618 } while (!ret); 2619 } 2620 2621 static int xennet_remove(struct xenbus_device *dev) 2622 { 2623 struct netfront_info *info = dev_get_drvdata(&dev->dev); 2624 2625 xennet_bus_close(dev); 2626 xennet_disconnect_backend(info); 2627 2628 if (info->netdev->reg_state == NETREG_REGISTERED) 2629 unregister_netdev(info->netdev); 2630 2631 if (info->queues) { 2632 rtnl_lock(); 2633 xennet_destroy_queues(info); 2634 rtnl_unlock(); 2635 } 2636 xennet_free_netdev(info->netdev); 2637 2638 return 0; 2639 } 2640 2641 static const struct xenbus_device_id netfront_ids[] = { 2642 { "vif" }, 2643 { "" } 2644 }; 2645 2646 static struct xenbus_driver netfront_driver = { 2647 .ids = netfront_ids, 2648 .probe = netfront_probe, 2649 .remove = xennet_remove, 2650 .resume = netfront_resume, 2651 .otherend_changed = netback_changed, 2652 }; 2653 2654 static int __init netif_init(void) 2655 { 2656 if (!xen_domain()) 2657 return -ENODEV; 2658 2659 if (!xen_has_pv_nic_devices()) 2660 return -ENODEV; 2661 2662 pr_info("Initialising Xen virtual ethernet driver\n"); 2663 2664 /* Allow as many queues as there are CPUs inut max. 8 if user has not 2665 * specified a value. 2666 */ 2667 if (xennet_max_queues == 0) 2668 xennet_max_queues = min_t(unsigned int, MAX_QUEUES_DEFAULT, 2669 num_online_cpus()); 2670 2671 return xenbus_register_frontend(&netfront_driver); 2672 } 2673 module_init(netif_init); 2674 2675 2676 static void __exit netif_exit(void) 2677 { 2678 xenbus_unregister_driver(&netfront_driver); 2679 } 2680 module_exit(netif_exit); 2681 2682 MODULE_DESCRIPTION("Xen virtual network device frontend"); 2683 MODULE_LICENSE("GPL"); 2684 MODULE_ALIAS("xen:vif"); 2685 MODULE_ALIAS("xennet"); 2686