1 /* 2 * Back-end of the driver for virtual network devices. This portion of the 3 * driver exports a 'unified' network-device interface that can be accessed 4 * by any operating system that implements a compatible front end. A 5 * reference front-end implementation can be found in: 6 * drivers/net/xen-netfront.c 7 * 8 * Copyright (c) 2002-2005, K A Fraser 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License version 2 12 * as published by the Free Software Foundation; or, when distributed 13 * separately from the Linux kernel or incorporated into other 14 * software packages, subject to the following license: 15 * 16 * Permission is hereby granted, free of charge, to any person obtaining a copy 17 * of this source file (the "Software"), to deal in the Software without 18 * restriction, including without limitation the rights to use, copy, modify, 19 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 20 * and to permit persons to whom the Software is furnished to do so, subject to 21 * the following conditions: 22 * 23 * The above copyright notice and this permission notice shall be included in 24 * all copies or substantial portions of the Software. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 32 * IN THE SOFTWARE. 33 */ 34 35 #include "common.h" 36 37 #include <linux/kthread.h> 38 #include <linux/if_vlan.h> 39 #include <linux/udp.h> 40 #include <linux/highmem.h> 41 42 #include <net/tcp.h> 43 44 #include <xen/xen.h> 45 #include <xen/events.h> 46 #include <xen/interface/memory.h> 47 #include <xen/page.h> 48 49 #include <asm/xen/hypercall.h> 50 51 /* Provide an option to disable split event channels at load time as 52 * event channels are limited resource. Split event channels are 53 * enabled by default. 54 */ 55 bool separate_tx_rx_irq = true; 56 module_param(separate_tx_rx_irq, bool, 0644); 57 58 /* The time that packets can stay on the guest Rx internal queue 59 * before they are dropped. 60 */ 61 unsigned int rx_drain_timeout_msecs = 10000; 62 module_param(rx_drain_timeout_msecs, uint, 0444); 63 64 /* The length of time before the frontend is considered unresponsive 65 * because it isn't providing Rx slots. 66 */ 67 unsigned int rx_stall_timeout_msecs = 60000; 68 module_param(rx_stall_timeout_msecs, uint, 0444); 69 70 #define MAX_QUEUES_DEFAULT 8 71 unsigned int xenvif_max_queues; 72 module_param_named(max_queues, xenvif_max_queues, uint, 0644); 73 MODULE_PARM_DESC(max_queues, 74 "Maximum number of queues per virtual interface"); 75 76 /* 77 * This is the maximum slots a skb can have. If a guest sends a skb 78 * which exceeds this limit it is considered malicious. 79 */ 80 #define FATAL_SKB_SLOTS_DEFAULT 20 81 static unsigned int fatal_skb_slots = FATAL_SKB_SLOTS_DEFAULT; 82 module_param(fatal_skb_slots, uint, 0444); 83 84 /* The amount to copy out of the first guest Tx slot into the skb's 85 * linear area. If the first slot has more data, it will be mapped 86 * and put into the first frag. 87 * 88 * This is sized to avoid pulling headers from the frags for most 89 * TCP/IP packets. 90 */ 91 #define XEN_NETBACK_TX_COPY_LEN 128 92 93 /* This is the maximum number of flows in the hash cache. */ 94 #define XENVIF_HASH_CACHE_SIZE_DEFAULT 64 95 unsigned int xenvif_hash_cache_size = XENVIF_HASH_CACHE_SIZE_DEFAULT; 96 module_param_named(hash_cache_size, xenvif_hash_cache_size, uint, 0644); 97 MODULE_PARM_DESC(hash_cache_size, "Number of flows in the hash cache"); 98 99 /* The module parameter tells that we have to put data 100 * for xen-netfront with the XDP_PACKET_HEADROOM offset 101 * needed for XDP processing 102 */ 103 bool provides_xdp_headroom = true; 104 module_param(provides_xdp_headroom, bool, 0644); 105 106 static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx, 107 u8 status); 108 109 static void make_tx_response(struct xenvif_queue *queue, 110 struct xen_netif_tx_request *txp, 111 unsigned int extra_count, 112 s8 st); 113 static void push_tx_responses(struct xenvif_queue *queue); 114 115 static inline int tx_work_todo(struct xenvif_queue *queue); 116 117 static inline unsigned long idx_to_pfn(struct xenvif_queue *queue, 118 u16 idx) 119 { 120 return page_to_pfn(queue->mmap_pages[idx]); 121 } 122 123 static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue, 124 u16 idx) 125 { 126 return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue, idx)); 127 } 128 129 #define callback_param(vif, pending_idx) \ 130 (vif->pending_tx_info[pending_idx].callback_struct) 131 132 /* Find the containing VIF's structure from a pointer in pending_tx_info array 133 */ 134 static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info *ubuf) 135 { 136 u16 pending_idx = ubuf->desc; 137 struct pending_tx_info *temp = 138 container_of(ubuf, struct pending_tx_info, callback_struct); 139 return container_of(temp - pending_idx, 140 struct xenvif_queue, 141 pending_tx_info[0]); 142 } 143 144 static u16 frag_get_pending_idx(skb_frag_t *frag) 145 { 146 return (u16)skb_frag_off(frag); 147 } 148 149 static void frag_set_pending_idx(skb_frag_t *frag, u16 pending_idx) 150 { 151 skb_frag_off_set(frag, pending_idx); 152 } 153 154 static inline pending_ring_idx_t pending_index(unsigned i) 155 { 156 return i & (MAX_PENDING_REQS-1); 157 } 158 159 void xenvif_kick_thread(struct xenvif_queue *queue) 160 { 161 wake_up(&queue->wq); 162 } 163 164 void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue) 165 { 166 int more_to_do; 167 168 RING_FINAL_CHECK_FOR_REQUESTS(&queue->tx, more_to_do); 169 170 if (more_to_do) 171 napi_schedule(&queue->napi); 172 } 173 174 static void tx_add_credit(struct xenvif_queue *queue) 175 { 176 unsigned long max_burst, max_credit; 177 178 /* 179 * Allow a burst big enough to transmit a jumbo packet of up to 128kB. 180 * Otherwise the interface can seize up due to insufficient credit. 181 */ 182 max_burst = max(131072UL, queue->credit_bytes); 183 184 /* Take care that adding a new chunk of credit doesn't wrap to zero. */ 185 max_credit = queue->remaining_credit + queue->credit_bytes; 186 if (max_credit < queue->remaining_credit) 187 max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */ 188 189 queue->remaining_credit = min(max_credit, max_burst); 190 queue->rate_limited = false; 191 } 192 193 void xenvif_tx_credit_callback(struct timer_list *t) 194 { 195 struct xenvif_queue *queue = from_timer(queue, t, credit_timeout); 196 tx_add_credit(queue); 197 xenvif_napi_schedule_or_enable_events(queue); 198 } 199 200 static void xenvif_tx_err(struct xenvif_queue *queue, 201 struct xen_netif_tx_request *txp, 202 unsigned int extra_count, RING_IDX end) 203 { 204 RING_IDX cons = queue->tx.req_cons; 205 unsigned long flags; 206 207 do { 208 spin_lock_irqsave(&queue->response_lock, flags); 209 make_tx_response(queue, txp, extra_count, XEN_NETIF_RSP_ERROR); 210 push_tx_responses(queue); 211 spin_unlock_irqrestore(&queue->response_lock, flags); 212 if (cons == end) 213 break; 214 RING_COPY_REQUEST(&queue->tx, cons++, txp); 215 extra_count = 0; /* only the first frag can have extras */ 216 } while (1); 217 queue->tx.req_cons = cons; 218 } 219 220 static void xenvif_fatal_tx_err(struct xenvif *vif) 221 { 222 netdev_err(vif->dev, "fatal error; disabling device\n"); 223 vif->disabled = true; 224 /* Disable the vif from queue 0's kthread */ 225 if (vif->num_queues) 226 xenvif_kick_thread(&vif->queues[0]); 227 } 228 229 static int xenvif_count_requests(struct xenvif_queue *queue, 230 struct xen_netif_tx_request *first, 231 unsigned int extra_count, 232 struct xen_netif_tx_request *txp, 233 int work_to_do) 234 { 235 RING_IDX cons = queue->tx.req_cons; 236 int slots = 0; 237 int drop_err = 0; 238 int more_data; 239 240 if (!(first->flags & XEN_NETTXF_more_data)) 241 return 0; 242 243 do { 244 struct xen_netif_tx_request dropped_tx = { 0 }; 245 246 if (slots >= work_to_do) { 247 netdev_err(queue->vif->dev, 248 "Asked for %d slots but exceeds this limit\n", 249 work_to_do); 250 xenvif_fatal_tx_err(queue->vif); 251 return -ENODATA; 252 } 253 254 /* This guest is really using too many slots and 255 * considered malicious. 256 */ 257 if (unlikely(slots >= fatal_skb_slots)) { 258 netdev_err(queue->vif->dev, 259 "Malicious frontend using %d slots, threshold %u\n", 260 slots, fatal_skb_slots); 261 xenvif_fatal_tx_err(queue->vif); 262 return -E2BIG; 263 } 264 265 /* Xen network protocol had implicit dependency on 266 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to 267 * the historical MAX_SKB_FRAGS value 18 to honor the 268 * same behavior as before. Any packet using more than 269 * 18 slots but less than fatal_skb_slots slots is 270 * dropped 271 */ 272 if (!drop_err && slots >= XEN_NETBK_LEGACY_SLOTS_MAX) { 273 if (net_ratelimit()) 274 netdev_dbg(queue->vif->dev, 275 "Too many slots (%d) exceeding limit (%d), dropping packet\n", 276 slots, XEN_NETBK_LEGACY_SLOTS_MAX); 277 drop_err = -E2BIG; 278 } 279 280 if (drop_err) 281 txp = &dropped_tx; 282 283 RING_COPY_REQUEST(&queue->tx, cons + slots, txp); 284 285 /* If the guest submitted a frame >= 64 KiB then 286 * first->size overflowed and following slots will 287 * appear to be larger than the frame. 288 * 289 * This cannot be fatal error as there are buggy 290 * frontends that do this. 291 * 292 * Consume all slots and drop the packet. 293 */ 294 if (!drop_err && txp->size > first->size) { 295 if (net_ratelimit()) 296 netdev_dbg(queue->vif->dev, 297 "Invalid tx request, slot size %u > remaining size %u\n", 298 txp->size, first->size); 299 drop_err = -EIO; 300 } 301 302 first->size -= txp->size; 303 slots++; 304 305 if (unlikely((txp->offset + txp->size) > XEN_PAGE_SIZE)) { 306 netdev_err(queue->vif->dev, "Cross page boundary, txp->offset: %u, size: %u\n", 307 txp->offset, txp->size); 308 xenvif_fatal_tx_err(queue->vif); 309 return -EINVAL; 310 } 311 312 more_data = txp->flags & XEN_NETTXF_more_data; 313 314 if (!drop_err) 315 txp++; 316 317 } while (more_data); 318 319 if (drop_err) { 320 xenvif_tx_err(queue, first, extra_count, cons + slots); 321 return drop_err; 322 } 323 324 return slots; 325 } 326 327 328 struct xenvif_tx_cb { 329 u16 pending_idx; 330 }; 331 332 #define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb) 333 334 static inline void xenvif_tx_create_map_op(struct xenvif_queue *queue, 335 u16 pending_idx, 336 struct xen_netif_tx_request *txp, 337 unsigned int extra_count, 338 struct gnttab_map_grant_ref *mop) 339 { 340 queue->pages_to_map[mop-queue->tx_map_ops] = queue->mmap_pages[pending_idx]; 341 gnttab_set_map_op(mop, idx_to_kaddr(queue, pending_idx), 342 GNTMAP_host_map | GNTMAP_readonly, 343 txp->gref, queue->vif->domid); 344 345 memcpy(&queue->pending_tx_info[pending_idx].req, txp, 346 sizeof(*txp)); 347 queue->pending_tx_info[pending_idx].extra_count = extra_count; 348 } 349 350 static inline struct sk_buff *xenvif_alloc_skb(unsigned int size) 351 { 352 struct sk_buff *skb = 353 alloc_skb(size + NET_SKB_PAD + NET_IP_ALIGN, 354 GFP_ATOMIC | __GFP_NOWARN); 355 if (unlikely(skb == NULL)) 356 return NULL; 357 358 /* Packets passed to netif_rx() must have some headroom. */ 359 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 360 361 /* Initialize it here to avoid later surprises */ 362 skb_shinfo(skb)->destructor_arg = NULL; 363 364 return skb; 365 } 366 367 static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif_queue *queue, 368 struct sk_buff *skb, 369 struct xen_netif_tx_request *txp, 370 struct gnttab_map_grant_ref *gop, 371 unsigned int frag_overflow, 372 struct sk_buff *nskb) 373 { 374 struct skb_shared_info *shinfo = skb_shinfo(skb); 375 skb_frag_t *frags = shinfo->frags; 376 u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx; 377 int start; 378 pending_ring_idx_t index; 379 unsigned int nr_slots; 380 381 nr_slots = shinfo->nr_frags; 382 383 /* Skip first skb fragment if it is on same page as header fragment. */ 384 start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx); 385 386 for (shinfo->nr_frags = start; shinfo->nr_frags < nr_slots; 387 shinfo->nr_frags++, txp++, gop++) { 388 index = pending_index(queue->pending_cons++); 389 pending_idx = queue->pending_ring[index]; 390 xenvif_tx_create_map_op(queue, pending_idx, txp, 0, gop); 391 frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx); 392 } 393 394 if (frag_overflow) { 395 396 shinfo = skb_shinfo(nskb); 397 frags = shinfo->frags; 398 399 for (shinfo->nr_frags = 0; shinfo->nr_frags < frag_overflow; 400 shinfo->nr_frags++, txp++, gop++) { 401 index = pending_index(queue->pending_cons++); 402 pending_idx = queue->pending_ring[index]; 403 xenvif_tx_create_map_op(queue, pending_idx, txp, 0, 404 gop); 405 frag_set_pending_idx(&frags[shinfo->nr_frags], 406 pending_idx); 407 } 408 409 skb_shinfo(skb)->frag_list = nskb; 410 } 411 412 return gop; 413 } 414 415 static inline void xenvif_grant_handle_set(struct xenvif_queue *queue, 416 u16 pending_idx, 417 grant_handle_t handle) 418 { 419 if (unlikely(queue->grant_tx_handle[pending_idx] != 420 NETBACK_INVALID_HANDLE)) { 421 netdev_err(queue->vif->dev, 422 "Trying to overwrite active handle! pending_idx: 0x%x\n", 423 pending_idx); 424 BUG(); 425 } 426 queue->grant_tx_handle[pending_idx] = handle; 427 } 428 429 static inline void xenvif_grant_handle_reset(struct xenvif_queue *queue, 430 u16 pending_idx) 431 { 432 if (unlikely(queue->grant_tx_handle[pending_idx] == 433 NETBACK_INVALID_HANDLE)) { 434 netdev_err(queue->vif->dev, 435 "Trying to unmap invalid handle! pending_idx: 0x%x\n", 436 pending_idx); 437 BUG(); 438 } 439 queue->grant_tx_handle[pending_idx] = NETBACK_INVALID_HANDLE; 440 } 441 442 static int xenvif_tx_check_gop(struct xenvif_queue *queue, 443 struct sk_buff *skb, 444 struct gnttab_map_grant_ref **gopp_map, 445 struct gnttab_copy **gopp_copy) 446 { 447 struct gnttab_map_grant_ref *gop_map = *gopp_map; 448 u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx; 449 /* This always points to the shinfo of the skb being checked, which 450 * could be either the first or the one on the frag_list 451 */ 452 struct skb_shared_info *shinfo = skb_shinfo(skb); 453 /* If this is non-NULL, we are currently checking the frag_list skb, and 454 * this points to the shinfo of the first one 455 */ 456 struct skb_shared_info *first_shinfo = NULL; 457 int nr_frags = shinfo->nr_frags; 458 const bool sharedslot = nr_frags && 459 frag_get_pending_idx(&shinfo->frags[0]) == pending_idx; 460 int i, err; 461 462 /* Check status of header. */ 463 err = (*gopp_copy)->status; 464 if (unlikely(err)) { 465 if (net_ratelimit()) 466 netdev_dbg(queue->vif->dev, 467 "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n", 468 (*gopp_copy)->status, 469 pending_idx, 470 (*gopp_copy)->source.u.ref); 471 /* The first frag might still have this slot mapped */ 472 if (!sharedslot) 473 xenvif_idx_release(queue, pending_idx, 474 XEN_NETIF_RSP_ERROR); 475 } 476 (*gopp_copy)++; 477 478 check_frags: 479 for (i = 0; i < nr_frags; i++, gop_map++) { 480 int j, newerr; 481 482 pending_idx = frag_get_pending_idx(&shinfo->frags[i]); 483 484 /* Check error status: if okay then remember grant handle. */ 485 newerr = gop_map->status; 486 487 if (likely(!newerr)) { 488 xenvif_grant_handle_set(queue, 489 pending_idx, 490 gop_map->handle); 491 /* Had a previous error? Invalidate this fragment. */ 492 if (unlikely(err)) { 493 xenvif_idx_unmap(queue, pending_idx); 494 /* If the mapping of the first frag was OK, but 495 * the header's copy failed, and they are 496 * sharing a slot, send an error 497 */ 498 if (i == 0 && sharedslot) 499 xenvif_idx_release(queue, pending_idx, 500 XEN_NETIF_RSP_ERROR); 501 else 502 xenvif_idx_release(queue, pending_idx, 503 XEN_NETIF_RSP_OKAY); 504 } 505 continue; 506 } 507 508 /* Error on this fragment: respond to client with an error. */ 509 if (net_ratelimit()) 510 netdev_dbg(queue->vif->dev, 511 "Grant map of %d. frag failed! status: %d pending_idx: %u ref: %u\n", 512 i, 513 gop_map->status, 514 pending_idx, 515 gop_map->ref); 516 517 xenvif_idx_release(queue, pending_idx, XEN_NETIF_RSP_ERROR); 518 519 /* Not the first error? Preceding frags already invalidated. */ 520 if (err) 521 continue; 522 523 /* First error: if the header haven't shared a slot with the 524 * first frag, release it as well. 525 */ 526 if (!sharedslot) 527 xenvif_idx_release(queue, 528 XENVIF_TX_CB(skb)->pending_idx, 529 XEN_NETIF_RSP_OKAY); 530 531 /* Invalidate preceding fragments of this skb. */ 532 for (j = 0; j < i; j++) { 533 pending_idx = frag_get_pending_idx(&shinfo->frags[j]); 534 xenvif_idx_unmap(queue, pending_idx); 535 xenvif_idx_release(queue, pending_idx, 536 XEN_NETIF_RSP_OKAY); 537 } 538 539 /* And if we found the error while checking the frag_list, unmap 540 * the first skb's frags 541 */ 542 if (first_shinfo) { 543 for (j = 0; j < first_shinfo->nr_frags; j++) { 544 pending_idx = frag_get_pending_idx(&first_shinfo->frags[j]); 545 xenvif_idx_unmap(queue, pending_idx); 546 xenvif_idx_release(queue, pending_idx, 547 XEN_NETIF_RSP_OKAY); 548 } 549 } 550 551 /* Remember the error: invalidate all subsequent fragments. */ 552 err = newerr; 553 } 554 555 if (skb_has_frag_list(skb) && !first_shinfo) { 556 first_shinfo = skb_shinfo(skb); 557 shinfo = skb_shinfo(skb_shinfo(skb)->frag_list); 558 nr_frags = shinfo->nr_frags; 559 560 goto check_frags; 561 } 562 563 *gopp_map = gop_map; 564 return err; 565 } 566 567 static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb) 568 { 569 struct skb_shared_info *shinfo = skb_shinfo(skb); 570 int nr_frags = shinfo->nr_frags; 571 int i; 572 u16 prev_pending_idx = INVALID_PENDING_IDX; 573 574 for (i = 0; i < nr_frags; i++) { 575 skb_frag_t *frag = shinfo->frags + i; 576 struct xen_netif_tx_request *txp; 577 struct page *page; 578 u16 pending_idx; 579 580 pending_idx = frag_get_pending_idx(frag); 581 582 /* If this is not the first frag, chain it to the previous*/ 583 if (prev_pending_idx == INVALID_PENDING_IDX) 584 skb_shinfo(skb)->destructor_arg = 585 &callback_param(queue, pending_idx); 586 else 587 callback_param(queue, prev_pending_idx).ctx = 588 &callback_param(queue, pending_idx); 589 590 callback_param(queue, pending_idx).ctx = NULL; 591 prev_pending_idx = pending_idx; 592 593 txp = &queue->pending_tx_info[pending_idx].req; 594 page = virt_to_page(idx_to_kaddr(queue, pending_idx)); 595 __skb_fill_page_desc(skb, i, page, txp->offset, txp->size); 596 skb->len += txp->size; 597 skb->data_len += txp->size; 598 skb->truesize += txp->size; 599 600 /* Take an extra reference to offset network stack's put_page */ 601 get_page(queue->mmap_pages[pending_idx]); 602 } 603 } 604 605 static int xenvif_get_extras(struct xenvif_queue *queue, 606 struct xen_netif_extra_info *extras, 607 unsigned int *extra_count, 608 int work_to_do) 609 { 610 struct xen_netif_extra_info extra; 611 RING_IDX cons = queue->tx.req_cons; 612 613 do { 614 if (unlikely(work_to_do-- <= 0)) { 615 netdev_err(queue->vif->dev, "Missing extra info\n"); 616 xenvif_fatal_tx_err(queue->vif); 617 return -EBADR; 618 } 619 620 RING_COPY_REQUEST(&queue->tx, cons, &extra); 621 622 queue->tx.req_cons = ++cons; 623 (*extra_count)++; 624 625 if (unlikely(!extra.type || 626 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) { 627 netdev_err(queue->vif->dev, 628 "Invalid extra type: %d\n", extra.type); 629 xenvif_fatal_tx_err(queue->vif); 630 return -EINVAL; 631 } 632 633 memcpy(&extras[extra.type - 1], &extra, sizeof(extra)); 634 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE); 635 636 return work_to_do; 637 } 638 639 static int xenvif_set_skb_gso(struct xenvif *vif, 640 struct sk_buff *skb, 641 struct xen_netif_extra_info *gso) 642 { 643 if (!gso->u.gso.size) { 644 netdev_err(vif->dev, "GSO size must not be zero.\n"); 645 xenvif_fatal_tx_err(vif); 646 return -EINVAL; 647 } 648 649 switch (gso->u.gso.type) { 650 case XEN_NETIF_GSO_TYPE_TCPV4: 651 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 652 break; 653 case XEN_NETIF_GSO_TYPE_TCPV6: 654 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; 655 break; 656 default: 657 netdev_err(vif->dev, "Bad GSO type %d.\n", gso->u.gso.type); 658 xenvif_fatal_tx_err(vif); 659 return -EINVAL; 660 } 661 662 skb_shinfo(skb)->gso_size = gso->u.gso.size; 663 /* gso_segs will be calculated later */ 664 665 return 0; 666 } 667 668 static int checksum_setup(struct xenvif_queue *queue, struct sk_buff *skb) 669 { 670 bool recalculate_partial_csum = false; 671 672 /* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy 673 * peers can fail to set NETRXF_csum_blank when sending a GSO 674 * frame. In this case force the SKB to CHECKSUM_PARTIAL and 675 * recalculate the partial checksum. 676 */ 677 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { 678 queue->stats.rx_gso_checksum_fixup++; 679 skb->ip_summed = CHECKSUM_PARTIAL; 680 recalculate_partial_csum = true; 681 } 682 683 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */ 684 if (skb->ip_summed != CHECKSUM_PARTIAL) 685 return 0; 686 687 return skb_checksum_setup(skb, recalculate_partial_csum); 688 } 689 690 static bool tx_credit_exceeded(struct xenvif_queue *queue, unsigned size) 691 { 692 u64 now = get_jiffies_64(); 693 u64 next_credit = queue->credit_window_start + 694 msecs_to_jiffies(queue->credit_usec / 1000); 695 696 /* Timer could already be pending in rare cases. */ 697 if (timer_pending(&queue->credit_timeout)) { 698 queue->rate_limited = true; 699 return true; 700 } 701 702 /* Passed the point where we can replenish credit? */ 703 if (time_after_eq64(now, next_credit)) { 704 queue->credit_window_start = now; 705 tx_add_credit(queue); 706 } 707 708 /* Still too big to send right now? Set a callback. */ 709 if (size > queue->remaining_credit) { 710 mod_timer(&queue->credit_timeout, 711 next_credit); 712 queue->credit_window_start = next_credit; 713 queue->rate_limited = true; 714 715 return true; 716 } 717 718 return false; 719 } 720 721 /* No locking is required in xenvif_mcast_add/del() as they are 722 * only ever invoked from NAPI poll. An RCU list is used because 723 * xenvif_mcast_match() is called asynchronously, during start_xmit. 724 */ 725 726 static int xenvif_mcast_add(struct xenvif *vif, const u8 *addr) 727 { 728 struct xenvif_mcast_addr *mcast; 729 730 if (vif->fe_mcast_count == XEN_NETBK_MCAST_MAX) { 731 if (net_ratelimit()) 732 netdev_err(vif->dev, 733 "Too many multicast addresses\n"); 734 return -ENOSPC; 735 } 736 737 mcast = kzalloc(sizeof(*mcast), GFP_ATOMIC); 738 if (!mcast) 739 return -ENOMEM; 740 741 ether_addr_copy(mcast->addr, addr); 742 list_add_tail_rcu(&mcast->entry, &vif->fe_mcast_addr); 743 vif->fe_mcast_count++; 744 745 return 0; 746 } 747 748 static void xenvif_mcast_del(struct xenvif *vif, const u8 *addr) 749 { 750 struct xenvif_mcast_addr *mcast; 751 752 list_for_each_entry_rcu(mcast, &vif->fe_mcast_addr, entry) { 753 if (ether_addr_equal(addr, mcast->addr)) { 754 --vif->fe_mcast_count; 755 list_del_rcu(&mcast->entry); 756 kfree_rcu(mcast, rcu); 757 break; 758 } 759 } 760 } 761 762 bool xenvif_mcast_match(struct xenvif *vif, const u8 *addr) 763 { 764 struct xenvif_mcast_addr *mcast; 765 766 rcu_read_lock(); 767 list_for_each_entry_rcu(mcast, &vif->fe_mcast_addr, entry) { 768 if (ether_addr_equal(addr, mcast->addr)) { 769 rcu_read_unlock(); 770 return true; 771 } 772 } 773 rcu_read_unlock(); 774 775 return false; 776 } 777 778 void xenvif_mcast_addr_list_free(struct xenvif *vif) 779 { 780 /* No need for locking or RCU here. NAPI poll and TX queue 781 * are stopped. 782 */ 783 while (!list_empty(&vif->fe_mcast_addr)) { 784 struct xenvif_mcast_addr *mcast; 785 786 mcast = list_first_entry(&vif->fe_mcast_addr, 787 struct xenvif_mcast_addr, 788 entry); 789 --vif->fe_mcast_count; 790 list_del(&mcast->entry); 791 kfree(mcast); 792 } 793 } 794 795 static void xenvif_tx_build_gops(struct xenvif_queue *queue, 796 int budget, 797 unsigned *copy_ops, 798 unsigned *map_ops) 799 { 800 struct gnttab_map_grant_ref *gop = queue->tx_map_ops; 801 struct sk_buff *skb, *nskb; 802 int ret; 803 unsigned int frag_overflow; 804 805 while (skb_queue_len(&queue->tx_queue) < budget) { 806 struct xen_netif_tx_request txreq; 807 struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX]; 808 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX-1]; 809 unsigned int extra_count; 810 u16 pending_idx; 811 RING_IDX idx; 812 int work_to_do; 813 unsigned int data_len; 814 pending_ring_idx_t index; 815 816 if (queue->tx.sring->req_prod - queue->tx.req_cons > 817 XEN_NETIF_TX_RING_SIZE) { 818 netdev_err(queue->vif->dev, 819 "Impossible number of requests. " 820 "req_prod %d, req_cons %d, size %ld\n", 821 queue->tx.sring->req_prod, queue->tx.req_cons, 822 XEN_NETIF_TX_RING_SIZE); 823 xenvif_fatal_tx_err(queue->vif); 824 break; 825 } 826 827 work_to_do = RING_HAS_UNCONSUMED_REQUESTS(&queue->tx); 828 if (!work_to_do) 829 break; 830 831 idx = queue->tx.req_cons; 832 rmb(); /* Ensure that we see the request before we copy it. */ 833 RING_COPY_REQUEST(&queue->tx, idx, &txreq); 834 835 /* Credit-based scheduling. */ 836 if (txreq.size > queue->remaining_credit && 837 tx_credit_exceeded(queue, txreq.size)) 838 break; 839 840 queue->remaining_credit -= txreq.size; 841 842 work_to_do--; 843 queue->tx.req_cons = ++idx; 844 845 memset(extras, 0, sizeof(extras)); 846 extra_count = 0; 847 if (txreq.flags & XEN_NETTXF_extra_info) { 848 work_to_do = xenvif_get_extras(queue, extras, 849 &extra_count, 850 work_to_do); 851 idx = queue->tx.req_cons; 852 if (unlikely(work_to_do < 0)) 853 break; 854 } 855 856 if (extras[XEN_NETIF_EXTRA_TYPE_MCAST_ADD - 1].type) { 857 struct xen_netif_extra_info *extra; 858 859 extra = &extras[XEN_NETIF_EXTRA_TYPE_MCAST_ADD - 1]; 860 ret = xenvif_mcast_add(queue->vif, extra->u.mcast.addr); 861 862 make_tx_response(queue, &txreq, extra_count, 863 (ret == 0) ? 864 XEN_NETIF_RSP_OKAY : 865 XEN_NETIF_RSP_ERROR); 866 push_tx_responses(queue); 867 continue; 868 } 869 870 if (extras[XEN_NETIF_EXTRA_TYPE_MCAST_DEL - 1].type) { 871 struct xen_netif_extra_info *extra; 872 873 extra = &extras[XEN_NETIF_EXTRA_TYPE_MCAST_DEL - 1]; 874 xenvif_mcast_del(queue->vif, extra->u.mcast.addr); 875 876 make_tx_response(queue, &txreq, extra_count, 877 XEN_NETIF_RSP_OKAY); 878 push_tx_responses(queue); 879 continue; 880 } 881 882 ret = xenvif_count_requests(queue, &txreq, extra_count, 883 txfrags, work_to_do); 884 if (unlikely(ret < 0)) 885 break; 886 887 idx += ret; 888 889 if (unlikely(txreq.size < ETH_HLEN)) { 890 netdev_dbg(queue->vif->dev, 891 "Bad packet size: %d\n", txreq.size); 892 xenvif_tx_err(queue, &txreq, extra_count, idx); 893 break; 894 } 895 896 /* No crossing a page as the payload mustn't fragment. */ 897 if (unlikely((txreq.offset + txreq.size) > XEN_PAGE_SIZE)) { 898 netdev_err(queue->vif->dev, 899 "txreq.offset: %u, size: %u, end: %lu\n", 900 txreq.offset, txreq.size, 901 (unsigned long)(txreq.offset&~XEN_PAGE_MASK) + txreq.size); 902 xenvif_fatal_tx_err(queue->vif); 903 break; 904 } 905 906 index = pending_index(queue->pending_cons); 907 pending_idx = queue->pending_ring[index]; 908 909 data_len = (txreq.size > XEN_NETBACK_TX_COPY_LEN && 910 ret < XEN_NETBK_LEGACY_SLOTS_MAX) ? 911 XEN_NETBACK_TX_COPY_LEN : txreq.size; 912 913 skb = xenvif_alloc_skb(data_len); 914 if (unlikely(skb == NULL)) { 915 netdev_dbg(queue->vif->dev, 916 "Can't allocate a skb in start_xmit.\n"); 917 xenvif_tx_err(queue, &txreq, extra_count, idx); 918 break; 919 } 920 921 skb_shinfo(skb)->nr_frags = ret; 922 if (data_len < txreq.size) 923 skb_shinfo(skb)->nr_frags++; 924 /* At this point shinfo->nr_frags is in fact the number of 925 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX. 926 */ 927 frag_overflow = 0; 928 nskb = NULL; 929 if (skb_shinfo(skb)->nr_frags > MAX_SKB_FRAGS) { 930 frag_overflow = skb_shinfo(skb)->nr_frags - MAX_SKB_FRAGS; 931 BUG_ON(frag_overflow > MAX_SKB_FRAGS); 932 skb_shinfo(skb)->nr_frags = MAX_SKB_FRAGS; 933 nskb = xenvif_alloc_skb(0); 934 if (unlikely(nskb == NULL)) { 935 skb_shinfo(skb)->nr_frags = 0; 936 kfree_skb(skb); 937 xenvif_tx_err(queue, &txreq, extra_count, idx); 938 if (net_ratelimit()) 939 netdev_err(queue->vif->dev, 940 "Can't allocate the frag_list skb.\n"); 941 break; 942 } 943 } 944 945 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { 946 struct xen_netif_extra_info *gso; 947 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; 948 949 if (xenvif_set_skb_gso(queue->vif, skb, gso)) { 950 /* Failure in xenvif_set_skb_gso is fatal. */ 951 skb_shinfo(skb)->nr_frags = 0; 952 kfree_skb(skb); 953 kfree_skb(nskb); 954 break; 955 } 956 } 957 958 if (extras[XEN_NETIF_EXTRA_TYPE_HASH - 1].type) { 959 struct xen_netif_extra_info *extra; 960 enum pkt_hash_types type = PKT_HASH_TYPE_NONE; 961 962 extra = &extras[XEN_NETIF_EXTRA_TYPE_HASH - 1]; 963 964 switch (extra->u.hash.type) { 965 case _XEN_NETIF_CTRL_HASH_TYPE_IPV4: 966 case _XEN_NETIF_CTRL_HASH_TYPE_IPV6: 967 type = PKT_HASH_TYPE_L3; 968 break; 969 970 case _XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP: 971 case _XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP: 972 type = PKT_HASH_TYPE_L4; 973 break; 974 975 default: 976 break; 977 } 978 979 if (type != PKT_HASH_TYPE_NONE) 980 skb_set_hash(skb, 981 *(u32 *)extra->u.hash.value, 982 type); 983 } 984 985 XENVIF_TX_CB(skb)->pending_idx = pending_idx; 986 987 __skb_put(skb, data_len); 988 queue->tx_copy_ops[*copy_ops].source.u.ref = txreq.gref; 989 queue->tx_copy_ops[*copy_ops].source.domid = queue->vif->domid; 990 queue->tx_copy_ops[*copy_ops].source.offset = txreq.offset; 991 992 queue->tx_copy_ops[*copy_ops].dest.u.gmfn = 993 virt_to_gfn(skb->data); 994 queue->tx_copy_ops[*copy_ops].dest.domid = DOMID_SELF; 995 queue->tx_copy_ops[*copy_ops].dest.offset = 996 offset_in_page(skb->data) & ~XEN_PAGE_MASK; 997 998 queue->tx_copy_ops[*copy_ops].len = data_len; 999 queue->tx_copy_ops[*copy_ops].flags = GNTCOPY_source_gref; 1000 1001 (*copy_ops)++; 1002 1003 if (data_len < txreq.size) { 1004 frag_set_pending_idx(&skb_shinfo(skb)->frags[0], 1005 pending_idx); 1006 xenvif_tx_create_map_op(queue, pending_idx, &txreq, 1007 extra_count, gop); 1008 gop++; 1009 } else { 1010 frag_set_pending_idx(&skb_shinfo(skb)->frags[0], 1011 INVALID_PENDING_IDX); 1012 memcpy(&queue->pending_tx_info[pending_idx].req, 1013 &txreq, sizeof(txreq)); 1014 queue->pending_tx_info[pending_idx].extra_count = 1015 extra_count; 1016 } 1017 1018 queue->pending_cons++; 1019 1020 gop = xenvif_get_requests(queue, skb, txfrags, gop, 1021 frag_overflow, nskb); 1022 1023 __skb_queue_tail(&queue->tx_queue, skb); 1024 1025 queue->tx.req_cons = idx; 1026 1027 if (((gop-queue->tx_map_ops) >= ARRAY_SIZE(queue->tx_map_ops)) || 1028 (*copy_ops >= ARRAY_SIZE(queue->tx_copy_ops))) 1029 break; 1030 } 1031 1032 (*map_ops) = gop - queue->tx_map_ops; 1033 return; 1034 } 1035 1036 /* Consolidate skb with a frag_list into a brand new one with local pages on 1037 * frags. Returns 0 or -ENOMEM if can't allocate new pages. 1038 */ 1039 static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *skb) 1040 { 1041 unsigned int offset = skb_headlen(skb); 1042 skb_frag_t frags[MAX_SKB_FRAGS]; 1043 int i, f; 1044 struct ubuf_info *uarg; 1045 struct sk_buff *nskb = skb_shinfo(skb)->frag_list; 1046 1047 queue->stats.tx_zerocopy_sent += 2; 1048 queue->stats.tx_frag_overflow++; 1049 1050 xenvif_fill_frags(queue, nskb); 1051 /* Subtract frags size, we will correct it later */ 1052 skb->truesize -= skb->data_len; 1053 skb->len += nskb->len; 1054 skb->data_len += nskb->len; 1055 1056 /* create a brand new frags array and coalesce there */ 1057 for (i = 0; offset < skb->len; i++) { 1058 struct page *page; 1059 unsigned int len; 1060 1061 BUG_ON(i >= MAX_SKB_FRAGS); 1062 page = alloc_page(GFP_ATOMIC); 1063 if (!page) { 1064 int j; 1065 skb->truesize += skb->data_len; 1066 for (j = 0; j < i; j++) 1067 put_page(skb_frag_page(&frags[j])); 1068 return -ENOMEM; 1069 } 1070 1071 if (offset + PAGE_SIZE < skb->len) 1072 len = PAGE_SIZE; 1073 else 1074 len = skb->len - offset; 1075 if (skb_copy_bits(skb, offset, page_address(page), len)) 1076 BUG(); 1077 1078 offset += len; 1079 __skb_frag_set_page(&frags[i], page); 1080 skb_frag_off_set(&frags[i], 0); 1081 skb_frag_size_set(&frags[i], len); 1082 } 1083 1084 /* Release all the original (foreign) frags. */ 1085 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1086 skb_frag_unref(skb, f); 1087 uarg = skb_shinfo(skb)->destructor_arg; 1088 /* increase inflight counter to offset decrement in callback */ 1089 atomic_inc(&queue->inflight_packets); 1090 uarg->callback(uarg, true); 1091 skb_shinfo(skb)->destructor_arg = NULL; 1092 1093 /* Fill the skb with the new (local) frags. */ 1094 memcpy(skb_shinfo(skb)->frags, frags, i * sizeof(skb_frag_t)); 1095 skb_shinfo(skb)->nr_frags = i; 1096 skb->truesize += i * PAGE_SIZE; 1097 1098 return 0; 1099 } 1100 1101 static int xenvif_tx_submit(struct xenvif_queue *queue) 1102 { 1103 struct gnttab_map_grant_ref *gop_map = queue->tx_map_ops; 1104 struct gnttab_copy *gop_copy = queue->tx_copy_ops; 1105 struct sk_buff *skb; 1106 int work_done = 0; 1107 1108 while ((skb = __skb_dequeue(&queue->tx_queue)) != NULL) { 1109 struct xen_netif_tx_request *txp; 1110 u16 pending_idx; 1111 unsigned data_len; 1112 1113 pending_idx = XENVIF_TX_CB(skb)->pending_idx; 1114 txp = &queue->pending_tx_info[pending_idx].req; 1115 1116 /* Check the remap error code. */ 1117 if (unlikely(xenvif_tx_check_gop(queue, skb, &gop_map, &gop_copy))) { 1118 /* If there was an error, xenvif_tx_check_gop is 1119 * expected to release all the frags which were mapped, 1120 * so kfree_skb shouldn't do it again 1121 */ 1122 skb_shinfo(skb)->nr_frags = 0; 1123 if (skb_has_frag_list(skb)) { 1124 struct sk_buff *nskb = 1125 skb_shinfo(skb)->frag_list; 1126 skb_shinfo(nskb)->nr_frags = 0; 1127 } 1128 kfree_skb(skb); 1129 continue; 1130 } 1131 1132 data_len = skb->len; 1133 callback_param(queue, pending_idx).ctx = NULL; 1134 if (data_len < txp->size) { 1135 /* Append the packet payload as a fragment. */ 1136 txp->offset += data_len; 1137 txp->size -= data_len; 1138 } else { 1139 /* Schedule a response immediately. */ 1140 xenvif_idx_release(queue, pending_idx, 1141 XEN_NETIF_RSP_OKAY); 1142 } 1143 1144 if (txp->flags & XEN_NETTXF_csum_blank) 1145 skb->ip_summed = CHECKSUM_PARTIAL; 1146 else if (txp->flags & XEN_NETTXF_data_validated) 1147 skb->ip_summed = CHECKSUM_UNNECESSARY; 1148 1149 xenvif_fill_frags(queue, skb); 1150 1151 if (unlikely(skb_has_frag_list(skb))) { 1152 struct sk_buff *nskb = skb_shinfo(skb)->frag_list; 1153 xenvif_skb_zerocopy_prepare(queue, nskb); 1154 if (xenvif_handle_frag_list(queue, skb)) { 1155 if (net_ratelimit()) 1156 netdev_err(queue->vif->dev, 1157 "Not enough memory to consolidate frag_list!\n"); 1158 xenvif_skb_zerocopy_prepare(queue, skb); 1159 kfree_skb(skb); 1160 continue; 1161 } 1162 /* Copied all the bits from the frag list -- free it. */ 1163 skb_frag_list_init(skb); 1164 kfree_skb(nskb); 1165 } 1166 1167 skb->dev = queue->vif->dev; 1168 skb->protocol = eth_type_trans(skb, skb->dev); 1169 skb_reset_network_header(skb); 1170 1171 if (checksum_setup(queue, skb)) { 1172 netdev_dbg(queue->vif->dev, 1173 "Can't setup checksum in net_tx_action\n"); 1174 /* We have to set this flag to trigger the callback */ 1175 if (skb_shinfo(skb)->destructor_arg) 1176 xenvif_skb_zerocopy_prepare(queue, skb); 1177 kfree_skb(skb); 1178 continue; 1179 } 1180 1181 skb_probe_transport_header(skb); 1182 1183 /* If the packet is GSO then we will have just set up the 1184 * transport header offset in checksum_setup so it's now 1185 * straightforward to calculate gso_segs. 1186 */ 1187 if (skb_is_gso(skb)) { 1188 int mss, hdrlen; 1189 1190 /* GSO implies having the L4 header. */ 1191 WARN_ON_ONCE(!skb_transport_header_was_set(skb)); 1192 if (unlikely(!skb_transport_header_was_set(skb))) { 1193 kfree_skb(skb); 1194 continue; 1195 } 1196 1197 mss = skb_shinfo(skb)->gso_size; 1198 hdrlen = skb_transport_header(skb) - 1199 skb_mac_header(skb) + 1200 tcp_hdrlen(skb); 1201 1202 skb_shinfo(skb)->gso_segs = 1203 DIV_ROUND_UP(skb->len - hdrlen, mss); 1204 } 1205 1206 queue->stats.rx_bytes += skb->len; 1207 queue->stats.rx_packets++; 1208 1209 work_done++; 1210 1211 /* Set this flag right before netif_receive_skb, otherwise 1212 * someone might think this packet already left netback, and 1213 * do a skb_copy_ubufs while we are still in control of the 1214 * skb. E.g. the __pskb_pull_tail earlier can do such thing. 1215 */ 1216 if (skb_shinfo(skb)->destructor_arg) { 1217 xenvif_skb_zerocopy_prepare(queue, skb); 1218 queue->stats.tx_zerocopy_sent++; 1219 } 1220 1221 netif_receive_skb(skb); 1222 } 1223 1224 return work_done; 1225 } 1226 1227 void xenvif_zerocopy_callback(struct ubuf_info *ubuf, bool zerocopy_success) 1228 { 1229 unsigned long flags; 1230 pending_ring_idx_t index; 1231 struct xenvif_queue *queue = ubuf_to_queue(ubuf); 1232 1233 /* This is the only place where we grab this lock, to protect callbacks 1234 * from each other. 1235 */ 1236 spin_lock_irqsave(&queue->callback_lock, flags); 1237 do { 1238 u16 pending_idx = ubuf->desc; 1239 ubuf = (struct ubuf_info *) ubuf->ctx; 1240 BUG_ON(queue->dealloc_prod - queue->dealloc_cons >= 1241 MAX_PENDING_REQS); 1242 index = pending_index(queue->dealloc_prod); 1243 queue->dealloc_ring[index] = pending_idx; 1244 /* Sync with xenvif_tx_dealloc_action: 1245 * insert idx then incr producer. 1246 */ 1247 smp_wmb(); 1248 queue->dealloc_prod++; 1249 } while (ubuf); 1250 spin_unlock_irqrestore(&queue->callback_lock, flags); 1251 1252 if (likely(zerocopy_success)) 1253 queue->stats.tx_zerocopy_success++; 1254 else 1255 queue->stats.tx_zerocopy_fail++; 1256 xenvif_skb_zerocopy_complete(queue); 1257 } 1258 1259 static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue) 1260 { 1261 struct gnttab_unmap_grant_ref *gop; 1262 pending_ring_idx_t dc, dp; 1263 u16 pending_idx, pending_idx_release[MAX_PENDING_REQS]; 1264 unsigned int i = 0; 1265 1266 dc = queue->dealloc_cons; 1267 gop = queue->tx_unmap_ops; 1268 1269 /* Free up any grants we have finished using */ 1270 do { 1271 dp = queue->dealloc_prod; 1272 1273 /* Ensure we see all indices enqueued by all 1274 * xenvif_zerocopy_callback(). 1275 */ 1276 smp_rmb(); 1277 1278 while (dc != dp) { 1279 BUG_ON(gop - queue->tx_unmap_ops >= MAX_PENDING_REQS); 1280 pending_idx = 1281 queue->dealloc_ring[pending_index(dc++)]; 1282 1283 pending_idx_release[gop - queue->tx_unmap_ops] = 1284 pending_idx; 1285 queue->pages_to_unmap[gop - queue->tx_unmap_ops] = 1286 queue->mmap_pages[pending_idx]; 1287 gnttab_set_unmap_op(gop, 1288 idx_to_kaddr(queue, pending_idx), 1289 GNTMAP_host_map, 1290 queue->grant_tx_handle[pending_idx]); 1291 xenvif_grant_handle_reset(queue, pending_idx); 1292 ++gop; 1293 } 1294 1295 } while (dp != queue->dealloc_prod); 1296 1297 queue->dealloc_cons = dc; 1298 1299 if (gop - queue->tx_unmap_ops > 0) { 1300 int ret; 1301 ret = gnttab_unmap_refs(queue->tx_unmap_ops, 1302 NULL, 1303 queue->pages_to_unmap, 1304 gop - queue->tx_unmap_ops); 1305 if (ret) { 1306 netdev_err(queue->vif->dev, "Unmap fail: nr_ops %tu ret %d\n", 1307 gop - queue->tx_unmap_ops, ret); 1308 for (i = 0; i < gop - queue->tx_unmap_ops; ++i) { 1309 if (gop[i].status != GNTST_okay) 1310 netdev_err(queue->vif->dev, 1311 " host_addr: 0x%llx handle: 0x%x status: %d\n", 1312 gop[i].host_addr, 1313 gop[i].handle, 1314 gop[i].status); 1315 } 1316 BUG(); 1317 } 1318 } 1319 1320 for (i = 0; i < gop - queue->tx_unmap_ops; ++i) 1321 xenvif_idx_release(queue, pending_idx_release[i], 1322 XEN_NETIF_RSP_OKAY); 1323 } 1324 1325 1326 /* Called after netfront has transmitted */ 1327 int xenvif_tx_action(struct xenvif_queue *queue, int budget) 1328 { 1329 unsigned nr_mops, nr_cops = 0; 1330 int work_done, ret; 1331 1332 if (unlikely(!tx_work_todo(queue))) 1333 return 0; 1334 1335 xenvif_tx_build_gops(queue, budget, &nr_cops, &nr_mops); 1336 1337 if (nr_cops == 0) 1338 return 0; 1339 1340 gnttab_batch_copy(queue->tx_copy_ops, nr_cops); 1341 if (nr_mops != 0) { 1342 ret = gnttab_map_refs(queue->tx_map_ops, 1343 NULL, 1344 queue->pages_to_map, 1345 nr_mops); 1346 BUG_ON(ret); 1347 } 1348 1349 work_done = xenvif_tx_submit(queue); 1350 1351 return work_done; 1352 } 1353 1354 static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx, 1355 u8 status) 1356 { 1357 struct pending_tx_info *pending_tx_info; 1358 pending_ring_idx_t index; 1359 unsigned long flags; 1360 1361 pending_tx_info = &queue->pending_tx_info[pending_idx]; 1362 1363 spin_lock_irqsave(&queue->response_lock, flags); 1364 1365 make_tx_response(queue, &pending_tx_info->req, 1366 pending_tx_info->extra_count, status); 1367 1368 /* Release the pending index before pusing the Tx response so 1369 * its available before a new Tx request is pushed by the 1370 * frontend. 1371 */ 1372 index = pending_index(queue->pending_prod++); 1373 queue->pending_ring[index] = pending_idx; 1374 1375 push_tx_responses(queue); 1376 1377 spin_unlock_irqrestore(&queue->response_lock, flags); 1378 } 1379 1380 1381 static void make_tx_response(struct xenvif_queue *queue, 1382 struct xen_netif_tx_request *txp, 1383 unsigned int extra_count, 1384 s8 st) 1385 { 1386 RING_IDX i = queue->tx.rsp_prod_pvt; 1387 struct xen_netif_tx_response *resp; 1388 1389 resp = RING_GET_RESPONSE(&queue->tx, i); 1390 resp->id = txp->id; 1391 resp->status = st; 1392 1393 while (extra_count-- != 0) 1394 RING_GET_RESPONSE(&queue->tx, ++i)->status = XEN_NETIF_RSP_NULL; 1395 1396 queue->tx.rsp_prod_pvt = ++i; 1397 } 1398 1399 static void push_tx_responses(struct xenvif_queue *queue) 1400 { 1401 int notify; 1402 1403 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->tx, notify); 1404 if (notify) 1405 notify_remote_via_irq(queue->tx_irq); 1406 } 1407 1408 void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx) 1409 { 1410 int ret; 1411 struct gnttab_unmap_grant_ref tx_unmap_op; 1412 1413 gnttab_set_unmap_op(&tx_unmap_op, 1414 idx_to_kaddr(queue, pending_idx), 1415 GNTMAP_host_map, 1416 queue->grant_tx_handle[pending_idx]); 1417 xenvif_grant_handle_reset(queue, pending_idx); 1418 1419 ret = gnttab_unmap_refs(&tx_unmap_op, NULL, 1420 &queue->mmap_pages[pending_idx], 1); 1421 if (ret) { 1422 netdev_err(queue->vif->dev, 1423 "Unmap fail: ret: %d pending_idx: %d host_addr: %llx handle: 0x%x status: %d\n", 1424 ret, 1425 pending_idx, 1426 tx_unmap_op.host_addr, 1427 tx_unmap_op.handle, 1428 tx_unmap_op.status); 1429 BUG(); 1430 } 1431 } 1432 1433 static inline int tx_work_todo(struct xenvif_queue *queue) 1434 { 1435 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue->tx))) 1436 return 1; 1437 1438 return 0; 1439 } 1440 1441 static inline bool tx_dealloc_work_todo(struct xenvif_queue *queue) 1442 { 1443 return queue->dealloc_cons != queue->dealloc_prod; 1444 } 1445 1446 void xenvif_unmap_frontend_data_rings(struct xenvif_queue *queue) 1447 { 1448 if (queue->tx.sring) 1449 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif), 1450 queue->tx.sring); 1451 if (queue->rx.sring) 1452 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif), 1453 queue->rx.sring); 1454 } 1455 1456 int xenvif_map_frontend_data_rings(struct xenvif_queue *queue, 1457 grant_ref_t tx_ring_ref, 1458 grant_ref_t rx_ring_ref) 1459 { 1460 void *addr; 1461 struct xen_netif_tx_sring *txs; 1462 struct xen_netif_rx_sring *rxs; 1463 RING_IDX rsp_prod, req_prod; 1464 int err = -ENOMEM; 1465 1466 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif), 1467 &tx_ring_ref, 1, &addr); 1468 if (err) 1469 goto err; 1470 1471 txs = (struct xen_netif_tx_sring *)addr; 1472 rsp_prod = READ_ONCE(txs->rsp_prod); 1473 req_prod = READ_ONCE(txs->req_prod); 1474 1475 BACK_RING_ATTACH(&queue->tx, txs, rsp_prod, XEN_PAGE_SIZE); 1476 1477 err = -EIO; 1478 if (req_prod - rsp_prod > RING_SIZE(&queue->tx)) 1479 goto err; 1480 1481 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif), 1482 &rx_ring_ref, 1, &addr); 1483 if (err) 1484 goto err; 1485 1486 rxs = (struct xen_netif_rx_sring *)addr; 1487 rsp_prod = READ_ONCE(rxs->rsp_prod); 1488 req_prod = READ_ONCE(rxs->req_prod); 1489 1490 BACK_RING_ATTACH(&queue->rx, rxs, rsp_prod, XEN_PAGE_SIZE); 1491 1492 err = -EIO; 1493 if (req_prod - rsp_prod > RING_SIZE(&queue->rx)) 1494 goto err; 1495 1496 return 0; 1497 1498 err: 1499 xenvif_unmap_frontend_data_rings(queue); 1500 return err; 1501 } 1502 1503 static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue *queue) 1504 { 1505 /* Dealloc thread must remain running until all inflight 1506 * packets complete. 1507 */ 1508 return kthread_should_stop() && 1509 !atomic_read(&queue->inflight_packets); 1510 } 1511 1512 int xenvif_dealloc_kthread(void *data) 1513 { 1514 struct xenvif_queue *queue = data; 1515 1516 for (;;) { 1517 wait_event_interruptible(queue->dealloc_wq, 1518 tx_dealloc_work_todo(queue) || 1519 xenvif_dealloc_kthread_should_stop(queue)); 1520 if (xenvif_dealloc_kthread_should_stop(queue)) 1521 break; 1522 1523 xenvif_tx_dealloc_action(queue); 1524 cond_resched(); 1525 } 1526 1527 /* Unmap anything remaining*/ 1528 if (tx_dealloc_work_todo(queue)) 1529 xenvif_tx_dealloc_action(queue); 1530 1531 return 0; 1532 } 1533 1534 static void make_ctrl_response(struct xenvif *vif, 1535 const struct xen_netif_ctrl_request *req, 1536 u32 status, u32 data) 1537 { 1538 RING_IDX idx = vif->ctrl.rsp_prod_pvt; 1539 struct xen_netif_ctrl_response rsp = { 1540 .id = req->id, 1541 .type = req->type, 1542 .status = status, 1543 .data = data, 1544 }; 1545 1546 *RING_GET_RESPONSE(&vif->ctrl, idx) = rsp; 1547 vif->ctrl.rsp_prod_pvt = ++idx; 1548 } 1549 1550 static void push_ctrl_response(struct xenvif *vif) 1551 { 1552 int notify; 1553 1554 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->ctrl, notify); 1555 if (notify) 1556 notify_remote_via_irq(vif->ctrl_irq); 1557 } 1558 1559 static void process_ctrl_request(struct xenvif *vif, 1560 const struct xen_netif_ctrl_request *req) 1561 { 1562 u32 status = XEN_NETIF_CTRL_STATUS_NOT_SUPPORTED; 1563 u32 data = 0; 1564 1565 switch (req->type) { 1566 case XEN_NETIF_CTRL_TYPE_SET_HASH_ALGORITHM: 1567 status = xenvif_set_hash_alg(vif, req->data[0]); 1568 break; 1569 1570 case XEN_NETIF_CTRL_TYPE_GET_HASH_FLAGS: 1571 status = xenvif_get_hash_flags(vif, &data); 1572 break; 1573 1574 case XEN_NETIF_CTRL_TYPE_SET_HASH_FLAGS: 1575 status = xenvif_set_hash_flags(vif, req->data[0]); 1576 break; 1577 1578 case XEN_NETIF_CTRL_TYPE_SET_HASH_KEY: 1579 status = xenvif_set_hash_key(vif, req->data[0], 1580 req->data[1]); 1581 break; 1582 1583 case XEN_NETIF_CTRL_TYPE_GET_HASH_MAPPING_SIZE: 1584 status = XEN_NETIF_CTRL_STATUS_SUCCESS; 1585 data = XEN_NETBK_MAX_HASH_MAPPING_SIZE; 1586 break; 1587 1588 case XEN_NETIF_CTRL_TYPE_SET_HASH_MAPPING_SIZE: 1589 status = xenvif_set_hash_mapping_size(vif, 1590 req->data[0]); 1591 break; 1592 1593 case XEN_NETIF_CTRL_TYPE_SET_HASH_MAPPING: 1594 status = xenvif_set_hash_mapping(vif, req->data[0], 1595 req->data[1], 1596 req->data[2]); 1597 break; 1598 1599 default: 1600 break; 1601 } 1602 1603 make_ctrl_response(vif, req, status, data); 1604 push_ctrl_response(vif); 1605 } 1606 1607 static void xenvif_ctrl_action(struct xenvif *vif) 1608 { 1609 for (;;) { 1610 RING_IDX req_prod, req_cons; 1611 1612 req_prod = vif->ctrl.sring->req_prod; 1613 req_cons = vif->ctrl.req_cons; 1614 1615 /* Make sure we can see requests before we process them. */ 1616 rmb(); 1617 1618 if (req_cons == req_prod) 1619 break; 1620 1621 while (req_cons != req_prod) { 1622 struct xen_netif_ctrl_request req; 1623 1624 RING_COPY_REQUEST(&vif->ctrl, req_cons, &req); 1625 req_cons++; 1626 1627 process_ctrl_request(vif, &req); 1628 } 1629 1630 vif->ctrl.req_cons = req_cons; 1631 vif->ctrl.sring->req_event = req_cons + 1; 1632 } 1633 } 1634 1635 static bool xenvif_ctrl_work_todo(struct xenvif *vif) 1636 { 1637 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&vif->ctrl))) 1638 return true; 1639 1640 return false; 1641 } 1642 1643 irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data) 1644 { 1645 struct xenvif *vif = data; 1646 1647 while (xenvif_ctrl_work_todo(vif)) 1648 xenvif_ctrl_action(vif); 1649 1650 return IRQ_HANDLED; 1651 } 1652 1653 static int __init netback_init(void) 1654 { 1655 int rc = 0; 1656 1657 if (!xen_domain()) 1658 return -ENODEV; 1659 1660 /* Allow as many queues as there are CPUs but max. 8 if user has not 1661 * specified a value. 1662 */ 1663 if (xenvif_max_queues == 0) 1664 xenvif_max_queues = min_t(unsigned int, MAX_QUEUES_DEFAULT, 1665 num_online_cpus()); 1666 1667 if (fatal_skb_slots < XEN_NETBK_LEGACY_SLOTS_MAX) { 1668 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n", 1669 fatal_skb_slots, XEN_NETBK_LEGACY_SLOTS_MAX); 1670 fatal_skb_slots = XEN_NETBK_LEGACY_SLOTS_MAX; 1671 } 1672 1673 rc = xenvif_xenbus_init(); 1674 if (rc) 1675 goto failed_init; 1676 1677 #ifdef CONFIG_DEBUG_FS 1678 xen_netback_dbg_root = debugfs_create_dir("xen-netback", NULL); 1679 #endif /* CONFIG_DEBUG_FS */ 1680 1681 return 0; 1682 1683 failed_init: 1684 return rc; 1685 } 1686 1687 module_init(netback_init); 1688 1689 static void __exit netback_fini(void) 1690 { 1691 #ifdef CONFIG_DEBUG_FS 1692 debugfs_remove_recursive(xen_netback_dbg_root); 1693 #endif /* CONFIG_DEBUG_FS */ 1694 xenvif_xenbus_fini(); 1695 } 1696 module_exit(netback_fini); 1697 1698 MODULE_LICENSE("Dual BSD/GPL"); 1699 MODULE_ALIAS("xen-backend:vif"); 1700