1 /* 2 * Back-end of the driver for virtual network devices. This portion of the 3 * driver exports a 'unified' network-device interface that can be accessed 4 * by any operating system that implements a compatible front end. A 5 * reference front-end implementation can be found in: 6 * drivers/net/xen-netfront.c 7 * 8 * Copyright (c) 2002-2005, K A Fraser 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License version 2 12 * as published by the Free Software Foundation; or, when distributed 13 * separately from the Linux kernel or incorporated into other 14 * software packages, subject to the following license: 15 * 16 * Permission is hereby granted, free of charge, to any person obtaining a copy 17 * of this source file (the "Software"), to deal in the Software without 18 * restriction, including without limitation the rights to use, copy, modify, 19 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 20 * and to permit persons to whom the Software is furnished to do so, subject to 21 * the following conditions: 22 * 23 * The above copyright notice and this permission notice shall be included in 24 * all copies or substantial portions of the Software. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 32 * IN THE SOFTWARE. 33 */ 34 35 #include "common.h" 36 37 #include <linux/kthread.h> 38 #include <linux/if_vlan.h> 39 #include <linux/udp.h> 40 41 #include <net/tcp.h> 42 43 #include <xen/xen.h> 44 #include <xen/events.h> 45 #include <xen/interface/memory.h> 46 47 #include <asm/xen/hypercall.h> 48 #include <asm/xen/page.h> 49 50 struct pending_tx_info { 51 struct xen_netif_tx_request req; 52 struct xenvif *vif; 53 }; 54 typedef unsigned int pending_ring_idx_t; 55 56 struct netbk_rx_meta { 57 int id; 58 int size; 59 int gso_size; 60 }; 61 62 #define MAX_PENDING_REQS 256 63 64 /* Discriminate from any valid pending_idx value. */ 65 #define INVALID_PENDING_IDX 0xFFFF 66 67 #define MAX_BUFFER_OFFSET PAGE_SIZE 68 69 /* extra field used in struct page */ 70 union page_ext { 71 struct { 72 #if BITS_PER_LONG < 64 73 #define IDX_WIDTH 8 74 #define GROUP_WIDTH (BITS_PER_LONG - IDX_WIDTH) 75 unsigned int group:GROUP_WIDTH; 76 unsigned int idx:IDX_WIDTH; 77 #else 78 unsigned int group, idx; 79 #endif 80 } e; 81 void *mapping; 82 }; 83 84 struct xen_netbk { 85 wait_queue_head_t wq; 86 struct task_struct *task; 87 88 struct sk_buff_head rx_queue; 89 struct sk_buff_head tx_queue; 90 91 struct timer_list net_timer; 92 93 struct page *mmap_pages[MAX_PENDING_REQS]; 94 95 pending_ring_idx_t pending_prod; 96 pending_ring_idx_t pending_cons; 97 struct list_head net_schedule_list; 98 99 /* Protect the net_schedule_list in netif. */ 100 spinlock_t net_schedule_list_lock; 101 102 atomic_t netfront_count; 103 104 struct pending_tx_info pending_tx_info[MAX_PENDING_REQS]; 105 struct gnttab_copy tx_copy_ops[MAX_PENDING_REQS]; 106 107 u16 pending_ring[MAX_PENDING_REQS]; 108 109 /* 110 * Given MAX_BUFFER_OFFSET of 4096 the worst case is that each 111 * head/fragment page uses 2 copy operations because it 112 * straddles two buffers in the frontend. 113 */ 114 struct gnttab_copy grant_copy_op[2*XEN_NETIF_RX_RING_SIZE]; 115 struct netbk_rx_meta meta[2*XEN_NETIF_RX_RING_SIZE]; 116 }; 117 118 static struct xen_netbk *xen_netbk; 119 static int xen_netbk_group_nr; 120 121 void xen_netbk_add_xenvif(struct xenvif *vif) 122 { 123 int i; 124 int min_netfront_count; 125 int min_group = 0; 126 struct xen_netbk *netbk; 127 128 min_netfront_count = atomic_read(&xen_netbk[0].netfront_count); 129 for (i = 0; i < xen_netbk_group_nr; i++) { 130 int netfront_count = atomic_read(&xen_netbk[i].netfront_count); 131 if (netfront_count < min_netfront_count) { 132 min_group = i; 133 min_netfront_count = netfront_count; 134 } 135 } 136 137 netbk = &xen_netbk[min_group]; 138 139 vif->netbk = netbk; 140 atomic_inc(&netbk->netfront_count); 141 } 142 143 void xen_netbk_remove_xenvif(struct xenvif *vif) 144 { 145 struct xen_netbk *netbk = vif->netbk; 146 vif->netbk = NULL; 147 atomic_dec(&netbk->netfront_count); 148 } 149 150 static void xen_netbk_idx_release(struct xen_netbk *netbk, u16 pending_idx, 151 u8 status); 152 static void make_tx_response(struct xenvif *vif, 153 struct xen_netif_tx_request *txp, 154 s8 st); 155 static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif, 156 u16 id, 157 s8 st, 158 u16 offset, 159 u16 size, 160 u16 flags); 161 162 static inline unsigned long idx_to_pfn(struct xen_netbk *netbk, 163 u16 idx) 164 { 165 return page_to_pfn(netbk->mmap_pages[idx]); 166 } 167 168 static inline unsigned long idx_to_kaddr(struct xen_netbk *netbk, 169 u16 idx) 170 { 171 return (unsigned long)pfn_to_kaddr(idx_to_pfn(netbk, idx)); 172 } 173 174 /* extra field used in struct page */ 175 static inline void set_page_ext(struct page *pg, struct xen_netbk *netbk, 176 unsigned int idx) 177 { 178 unsigned int group = netbk - xen_netbk; 179 union page_ext ext = { .e = { .group = group + 1, .idx = idx } }; 180 181 BUILD_BUG_ON(sizeof(ext) > sizeof(ext.mapping)); 182 pg->mapping = ext.mapping; 183 } 184 185 static int get_page_ext(struct page *pg, 186 unsigned int *pgroup, unsigned int *pidx) 187 { 188 union page_ext ext = { .mapping = pg->mapping }; 189 struct xen_netbk *netbk; 190 unsigned int group, idx; 191 192 group = ext.e.group - 1; 193 194 if (group < 0 || group >= xen_netbk_group_nr) 195 return 0; 196 197 netbk = &xen_netbk[group]; 198 199 idx = ext.e.idx; 200 201 if ((idx < 0) || (idx >= MAX_PENDING_REQS)) 202 return 0; 203 204 if (netbk->mmap_pages[idx] != pg) 205 return 0; 206 207 *pgroup = group; 208 *pidx = idx; 209 210 return 1; 211 } 212 213 /* 214 * This is the amount of packet we copy rather than map, so that the 215 * guest can't fiddle with the contents of the headers while we do 216 * packet processing on them (netfilter, routing, etc). 217 */ 218 #define PKT_PROT_LEN (ETH_HLEN + \ 219 VLAN_HLEN + \ 220 sizeof(struct iphdr) + MAX_IPOPTLEN + \ 221 sizeof(struct tcphdr) + MAX_TCP_OPTION_SPACE) 222 223 static u16 frag_get_pending_idx(skb_frag_t *frag) 224 { 225 return (u16)frag->page_offset; 226 } 227 228 static void frag_set_pending_idx(skb_frag_t *frag, u16 pending_idx) 229 { 230 frag->page_offset = pending_idx; 231 } 232 233 static inline pending_ring_idx_t pending_index(unsigned i) 234 { 235 return i & (MAX_PENDING_REQS-1); 236 } 237 238 static inline pending_ring_idx_t nr_pending_reqs(struct xen_netbk *netbk) 239 { 240 return MAX_PENDING_REQS - 241 netbk->pending_prod + netbk->pending_cons; 242 } 243 244 static void xen_netbk_kick_thread(struct xen_netbk *netbk) 245 { 246 wake_up(&netbk->wq); 247 } 248 249 static int max_required_rx_slots(struct xenvif *vif) 250 { 251 int max = DIV_ROUND_UP(vif->dev->mtu, PAGE_SIZE); 252 253 if (vif->can_sg || vif->gso || vif->gso_prefix) 254 max += MAX_SKB_FRAGS + 1; /* extra_info + frags */ 255 256 return max; 257 } 258 259 int xen_netbk_rx_ring_full(struct xenvif *vif) 260 { 261 RING_IDX peek = vif->rx_req_cons_peek; 262 RING_IDX needed = max_required_rx_slots(vif); 263 264 return ((vif->rx.sring->req_prod - peek) < needed) || 265 ((vif->rx.rsp_prod_pvt + XEN_NETIF_RX_RING_SIZE - peek) < needed); 266 } 267 268 int xen_netbk_must_stop_queue(struct xenvif *vif) 269 { 270 if (!xen_netbk_rx_ring_full(vif)) 271 return 0; 272 273 vif->rx.sring->req_event = vif->rx_req_cons_peek + 274 max_required_rx_slots(vif); 275 mb(); /* request notification /then/ check the queue */ 276 277 return xen_netbk_rx_ring_full(vif); 278 } 279 280 /* 281 * Returns true if we should start a new receive buffer instead of 282 * adding 'size' bytes to a buffer which currently contains 'offset' 283 * bytes. 284 */ 285 static bool start_new_rx_buffer(int offset, unsigned long size, int head) 286 { 287 /* simple case: we have completely filled the current buffer. */ 288 if (offset == MAX_BUFFER_OFFSET) 289 return true; 290 291 /* 292 * complex case: start a fresh buffer if the current frag 293 * would overflow the current buffer but only if: 294 * (i) this frag would fit completely in the next buffer 295 * and (ii) there is already some data in the current buffer 296 * and (iii) this is not the head buffer. 297 * 298 * Where: 299 * - (i) stops us splitting a frag into two copies 300 * unless the frag is too large for a single buffer. 301 * - (ii) stops us from leaving a buffer pointlessly empty. 302 * - (iii) stops us leaving the first buffer 303 * empty. Strictly speaking this is already covered 304 * by (ii) but is explicitly checked because 305 * netfront relies on the first buffer being 306 * non-empty and can crash otherwise. 307 * 308 * This means we will effectively linearise small 309 * frags but do not needlessly split large buffers 310 * into multiple copies tend to give large frags their 311 * own buffers as before. 312 */ 313 if ((offset + size > MAX_BUFFER_OFFSET) && 314 (size <= MAX_BUFFER_OFFSET) && offset && !head) 315 return true; 316 317 return false; 318 } 319 320 /* 321 * Figure out how many ring slots we're going to need to send @skb to 322 * the guest. This function is essentially a dry run of 323 * netbk_gop_frag_copy. 324 */ 325 unsigned int xen_netbk_count_skb_slots(struct xenvif *vif, struct sk_buff *skb) 326 { 327 unsigned int count; 328 int i, copy_off; 329 330 count = DIV_ROUND_UP(skb_headlen(skb), PAGE_SIZE); 331 332 copy_off = skb_headlen(skb) % PAGE_SIZE; 333 334 if (skb_shinfo(skb)->gso_size) 335 count++; 336 337 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 338 unsigned long size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 339 unsigned long offset = skb_shinfo(skb)->frags[i].page_offset; 340 unsigned long bytes; 341 342 offset &= ~PAGE_MASK; 343 344 while (size > 0) { 345 BUG_ON(offset >= PAGE_SIZE); 346 BUG_ON(copy_off > MAX_BUFFER_OFFSET); 347 348 bytes = PAGE_SIZE - offset; 349 350 if (bytes > size) 351 bytes = size; 352 353 if (start_new_rx_buffer(copy_off, bytes, 0)) { 354 count++; 355 copy_off = 0; 356 } 357 358 if (copy_off + bytes > MAX_BUFFER_OFFSET) 359 bytes = MAX_BUFFER_OFFSET - copy_off; 360 361 copy_off += bytes; 362 363 offset += bytes; 364 size -= bytes; 365 366 if (offset == PAGE_SIZE) 367 offset = 0; 368 } 369 } 370 return count; 371 } 372 373 struct netrx_pending_operations { 374 unsigned copy_prod, copy_cons; 375 unsigned meta_prod, meta_cons; 376 struct gnttab_copy *copy; 377 struct netbk_rx_meta *meta; 378 int copy_off; 379 grant_ref_t copy_gref; 380 }; 381 382 static struct netbk_rx_meta *get_next_rx_buffer(struct xenvif *vif, 383 struct netrx_pending_operations *npo) 384 { 385 struct netbk_rx_meta *meta; 386 struct xen_netif_rx_request *req; 387 388 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++); 389 390 meta = npo->meta + npo->meta_prod++; 391 meta->gso_size = 0; 392 meta->size = 0; 393 meta->id = req->id; 394 395 npo->copy_off = 0; 396 npo->copy_gref = req->gref; 397 398 return meta; 399 } 400 401 /* 402 * Set up the grant operations for this fragment. If it's a flipping 403 * interface, we also set up the unmap request from here. 404 */ 405 static void netbk_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb, 406 struct netrx_pending_operations *npo, 407 struct page *page, unsigned long size, 408 unsigned long offset, int *head) 409 { 410 struct gnttab_copy *copy_gop; 411 struct netbk_rx_meta *meta; 412 /* 413 * These variables are used iff get_page_ext returns true, 414 * in which case they are guaranteed to be initialized. 415 */ 416 unsigned int uninitialized_var(group), uninitialized_var(idx); 417 int foreign = get_page_ext(page, &group, &idx); 418 unsigned long bytes; 419 420 /* Data must not cross a page boundary. */ 421 BUG_ON(size + offset > PAGE_SIZE<<compound_order(page)); 422 423 meta = npo->meta + npo->meta_prod - 1; 424 425 /* Skip unused frames from start of page */ 426 page += offset >> PAGE_SHIFT; 427 offset &= ~PAGE_MASK; 428 429 while (size > 0) { 430 BUG_ON(offset >= PAGE_SIZE); 431 BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET); 432 433 bytes = PAGE_SIZE - offset; 434 435 if (bytes > size) 436 bytes = size; 437 438 if (start_new_rx_buffer(npo->copy_off, bytes, *head)) { 439 /* 440 * Netfront requires there to be some data in the head 441 * buffer. 442 */ 443 BUG_ON(*head); 444 445 meta = get_next_rx_buffer(vif, npo); 446 } 447 448 if (npo->copy_off + bytes > MAX_BUFFER_OFFSET) 449 bytes = MAX_BUFFER_OFFSET - npo->copy_off; 450 451 copy_gop = npo->copy + npo->copy_prod++; 452 copy_gop->flags = GNTCOPY_dest_gref; 453 if (foreign) { 454 struct xen_netbk *netbk = &xen_netbk[group]; 455 struct pending_tx_info *src_pend; 456 457 src_pend = &netbk->pending_tx_info[idx]; 458 459 copy_gop->source.domid = src_pend->vif->domid; 460 copy_gop->source.u.ref = src_pend->req.gref; 461 copy_gop->flags |= GNTCOPY_source_gref; 462 } else { 463 void *vaddr = page_address(page); 464 copy_gop->source.domid = DOMID_SELF; 465 copy_gop->source.u.gmfn = virt_to_mfn(vaddr); 466 } 467 copy_gop->source.offset = offset; 468 copy_gop->dest.domid = vif->domid; 469 470 copy_gop->dest.offset = npo->copy_off; 471 copy_gop->dest.u.ref = npo->copy_gref; 472 copy_gop->len = bytes; 473 474 npo->copy_off += bytes; 475 meta->size += bytes; 476 477 offset += bytes; 478 size -= bytes; 479 480 /* Next frame */ 481 if (offset == PAGE_SIZE && size) { 482 BUG_ON(!PageCompound(page)); 483 page++; 484 offset = 0; 485 } 486 487 /* Leave a gap for the GSO descriptor. */ 488 if (*head && skb_shinfo(skb)->gso_size && !vif->gso_prefix) 489 vif->rx.req_cons++; 490 491 *head = 0; /* There must be something in this buffer now. */ 492 493 } 494 } 495 496 /* 497 * Prepare an SKB to be transmitted to the frontend. 498 * 499 * This function is responsible for allocating grant operations, meta 500 * structures, etc. 501 * 502 * It returns the number of meta structures consumed. The number of 503 * ring slots used is always equal to the number of meta slots used 504 * plus the number of GSO descriptors used. Currently, we use either 505 * zero GSO descriptors (for non-GSO packets) or one descriptor (for 506 * frontend-side LRO). 507 */ 508 static int netbk_gop_skb(struct sk_buff *skb, 509 struct netrx_pending_operations *npo) 510 { 511 struct xenvif *vif = netdev_priv(skb->dev); 512 int nr_frags = skb_shinfo(skb)->nr_frags; 513 int i; 514 struct xen_netif_rx_request *req; 515 struct netbk_rx_meta *meta; 516 unsigned char *data; 517 int head = 1; 518 int old_meta_prod; 519 520 old_meta_prod = npo->meta_prod; 521 522 /* Set up a GSO prefix descriptor, if necessary */ 523 if (skb_shinfo(skb)->gso_size && vif->gso_prefix) { 524 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++); 525 meta = npo->meta + npo->meta_prod++; 526 meta->gso_size = skb_shinfo(skb)->gso_size; 527 meta->size = 0; 528 meta->id = req->id; 529 } 530 531 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++); 532 meta = npo->meta + npo->meta_prod++; 533 534 if (!vif->gso_prefix) 535 meta->gso_size = skb_shinfo(skb)->gso_size; 536 else 537 meta->gso_size = 0; 538 539 meta->size = 0; 540 meta->id = req->id; 541 npo->copy_off = 0; 542 npo->copy_gref = req->gref; 543 544 data = skb->data; 545 while (data < skb_tail_pointer(skb)) { 546 unsigned int offset = offset_in_page(data); 547 unsigned int len = PAGE_SIZE - offset; 548 549 if (data + len > skb_tail_pointer(skb)) 550 len = skb_tail_pointer(skb) - data; 551 552 netbk_gop_frag_copy(vif, skb, npo, 553 virt_to_page(data), len, offset, &head); 554 data += len; 555 } 556 557 for (i = 0; i < nr_frags; i++) { 558 netbk_gop_frag_copy(vif, skb, npo, 559 skb_frag_page(&skb_shinfo(skb)->frags[i]), 560 skb_frag_size(&skb_shinfo(skb)->frags[i]), 561 skb_shinfo(skb)->frags[i].page_offset, 562 &head); 563 } 564 565 return npo->meta_prod - old_meta_prod; 566 } 567 568 /* 569 * This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was 570 * used to set up the operations on the top of 571 * netrx_pending_operations, which have since been done. Check that 572 * they didn't give any errors and advance over them. 573 */ 574 static int netbk_check_gop(struct xenvif *vif, int nr_meta_slots, 575 struct netrx_pending_operations *npo) 576 { 577 struct gnttab_copy *copy_op; 578 int status = XEN_NETIF_RSP_OKAY; 579 int i; 580 581 for (i = 0; i < nr_meta_slots; i++) { 582 copy_op = npo->copy + npo->copy_cons++; 583 if (copy_op->status != GNTST_okay) { 584 netdev_dbg(vif->dev, 585 "Bad status %d from copy to DOM%d.\n", 586 copy_op->status, vif->domid); 587 status = XEN_NETIF_RSP_ERROR; 588 } 589 } 590 591 return status; 592 } 593 594 static void netbk_add_frag_responses(struct xenvif *vif, int status, 595 struct netbk_rx_meta *meta, 596 int nr_meta_slots) 597 { 598 int i; 599 unsigned long offset; 600 601 /* No fragments used */ 602 if (nr_meta_slots <= 1) 603 return; 604 605 nr_meta_slots--; 606 607 for (i = 0; i < nr_meta_slots; i++) { 608 int flags; 609 if (i == nr_meta_slots - 1) 610 flags = 0; 611 else 612 flags = XEN_NETRXF_more_data; 613 614 offset = 0; 615 make_rx_response(vif, meta[i].id, status, offset, 616 meta[i].size, flags); 617 } 618 } 619 620 struct skb_cb_overlay { 621 int meta_slots_used; 622 }; 623 624 static void xen_netbk_rx_action(struct xen_netbk *netbk) 625 { 626 struct xenvif *vif = NULL, *tmp; 627 s8 status; 628 u16 irq, flags; 629 struct xen_netif_rx_response *resp; 630 struct sk_buff_head rxq; 631 struct sk_buff *skb; 632 LIST_HEAD(notify); 633 int ret; 634 int nr_frags; 635 int count; 636 unsigned long offset; 637 struct skb_cb_overlay *sco; 638 639 struct netrx_pending_operations npo = { 640 .copy = netbk->grant_copy_op, 641 .meta = netbk->meta, 642 }; 643 644 skb_queue_head_init(&rxq); 645 646 count = 0; 647 648 while ((skb = skb_dequeue(&netbk->rx_queue)) != NULL) { 649 vif = netdev_priv(skb->dev); 650 nr_frags = skb_shinfo(skb)->nr_frags; 651 652 sco = (struct skb_cb_overlay *)skb->cb; 653 sco->meta_slots_used = netbk_gop_skb(skb, &npo); 654 655 count += nr_frags + 1; 656 657 __skb_queue_tail(&rxq, skb); 658 659 /* Filled the batch queue? */ 660 if (count + MAX_SKB_FRAGS >= XEN_NETIF_RX_RING_SIZE) 661 break; 662 } 663 664 BUG_ON(npo.meta_prod > ARRAY_SIZE(netbk->meta)); 665 666 if (!npo.copy_prod) 667 return; 668 669 BUG_ON(npo.copy_prod > ARRAY_SIZE(netbk->grant_copy_op)); 670 gnttab_batch_copy(netbk->grant_copy_op, npo.copy_prod); 671 672 while ((skb = __skb_dequeue(&rxq)) != NULL) { 673 sco = (struct skb_cb_overlay *)skb->cb; 674 675 vif = netdev_priv(skb->dev); 676 677 if (netbk->meta[npo.meta_cons].gso_size && vif->gso_prefix) { 678 resp = RING_GET_RESPONSE(&vif->rx, 679 vif->rx.rsp_prod_pvt++); 680 681 resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data; 682 683 resp->offset = netbk->meta[npo.meta_cons].gso_size; 684 resp->id = netbk->meta[npo.meta_cons].id; 685 resp->status = sco->meta_slots_used; 686 687 npo.meta_cons++; 688 sco->meta_slots_used--; 689 } 690 691 692 vif->dev->stats.tx_bytes += skb->len; 693 vif->dev->stats.tx_packets++; 694 695 status = netbk_check_gop(vif, sco->meta_slots_used, &npo); 696 697 if (sco->meta_slots_used == 1) 698 flags = 0; 699 else 700 flags = XEN_NETRXF_more_data; 701 702 if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */ 703 flags |= XEN_NETRXF_csum_blank | XEN_NETRXF_data_validated; 704 else if (skb->ip_summed == CHECKSUM_UNNECESSARY) 705 /* remote but checksummed. */ 706 flags |= XEN_NETRXF_data_validated; 707 708 offset = 0; 709 resp = make_rx_response(vif, netbk->meta[npo.meta_cons].id, 710 status, offset, 711 netbk->meta[npo.meta_cons].size, 712 flags); 713 714 if (netbk->meta[npo.meta_cons].gso_size && !vif->gso_prefix) { 715 struct xen_netif_extra_info *gso = 716 (struct xen_netif_extra_info *) 717 RING_GET_RESPONSE(&vif->rx, 718 vif->rx.rsp_prod_pvt++); 719 720 resp->flags |= XEN_NETRXF_extra_info; 721 722 gso->u.gso.size = netbk->meta[npo.meta_cons].gso_size; 723 gso->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4; 724 gso->u.gso.pad = 0; 725 gso->u.gso.features = 0; 726 727 gso->type = XEN_NETIF_EXTRA_TYPE_GSO; 728 gso->flags = 0; 729 } 730 731 netbk_add_frag_responses(vif, status, 732 netbk->meta + npo.meta_cons + 1, 733 sco->meta_slots_used); 734 735 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->rx, ret); 736 irq = vif->irq; 737 if (ret && list_empty(&vif->notify_list)) 738 list_add_tail(&vif->notify_list, ¬ify); 739 740 xenvif_notify_tx_completion(vif); 741 742 xenvif_put(vif); 743 npo.meta_cons += sco->meta_slots_used; 744 dev_kfree_skb(skb); 745 } 746 747 list_for_each_entry_safe(vif, tmp, ¬ify, notify_list) { 748 notify_remote_via_irq(vif->irq); 749 list_del_init(&vif->notify_list); 750 } 751 752 /* More work to do? */ 753 if (!skb_queue_empty(&netbk->rx_queue) && 754 !timer_pending(&netbk->net_timer)) 755 xen_netbk_kick_thread(netbk); 756 } 757 758 void xen_netbk_queue_tx_skb(struct xenvif *vif, struct sk_buff *skb) 759 { 760 struct xen_netbk *netbk = vif->netbk; 761 762 skb_queue_tail(&netbk->rx_queue, skb); 763 764 xen_netbk_kick_thread(netbk); 765 } 766 767 static void xen_netbk_alarm(unsigned long data) 768 { 769 struct xen_netbk *netbk = (struct xen_netbk *)data; 770 xen_netbk_kick_thread(netbk); 771 } 772 773 static int __on_net_schedule_list(struct xenvif *vif) 774 { 775 return !list_empty(&vif->schedule_list); 776 } 777 778 /* Must be called with net_schedule_list_lock held */ 779 static void remove_from_net_schedule_list(struct xenvif *vif) 780 { 781 if (likely(__on_net_schedule_list(vif))) { 782 list_del_init(&vif->schedule_list); 783 xenvif_put(vif); 784 } 785 } 786 787 static struct xenvif *poll_net_schedule_list(struct xen_netbk *netbk) 788 { 789 struct xenvif *vif = NULL; 790 791 spin_lock_irq(&netbk->net_schedule_list_lock); 792 if (list_empty(&netbk->net_schedule_list)) 793 goto out; 794 795 vif = list_first_entry(&netbk->net_schedule_list, 796 struct xenvif, schedule_list); 797 if (!vif) 798 goto out; 799 800 xenvif_get(vif); 801 802 remove_from_net_schedule_list(vif); 803 out: 804 spin_unlock_irq(&netbk->net_schedule_list_lock); 805 return vif; 806 } 807 808 void xen_netbk_schedule_xenvif(struct xenvif *vif) 809 { 810 unsigned long flags; 811 struct xen_netbk *netbk = vif->netbk; 812 813 if (__on_net_schedule_list(vif)) 814 goto kick; 815 816 spin_lock_irqsave(&netbk->net_schedule_list_lock, flags); 817 if (!__on_net_schedule_list(vif) && 818 likely(xenvif_schedulable(vif))) { 819 list_add_tail(&vif->schedule_list, &netbk->net_schedule_list); 820 xenvif_get(vif); 821 } 822 spin_unlock_irqrestore(&netbk->net_schedule_list_lock, flags); 823 824 kick: 825 smp_mb(); 826 if ((nr_pending_reqs(netbk) < (MAX_PENDING_REQS/2)) && 827 !list_empty(&netbk->net_schedule_list)) 828 xen_netbk_kick_thread(netbk); 829 } 830 831 void xen_netbk_deschedule_xenvif(struct xenvif *vif) 832 { 833 struct xen_netbk *netbk = vif->netbk; 834 spin_lock_irq(&netbk->net_schedule_list_lock); 835 remove_from_net_schedule_list(vif); 836 spin_unlock_irq(&netbk->net_schedule_list_lock); 837 } 838 839 void xen_netbk_check_rx_xenvif(struct xenvif *vif) 840 { 841 int more_to_do; 842 843 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do); 844 845 if (more_to_do) 846 xen_netbk_schedule_xenvif(vif); 847 } 848 849 static void tx_add_credit(struct xenvif *vif) 850 { 851 unsigned long max_burst, max_credit; 852 853 /* 854 * Allow a burst big enough to transmit a jumbo packet of up to 128kB. 855 * Otherwise the interface can seize up due to insufficient credit. 856 */ 857 max_burst = RING_GET_REQUEST(&vif->tx, vif->tx.req_cons)->size; 858 max_burst = min(max_burst, 131072UL); 859 max_burst = max(max_burst, vif->credit_bytes); 860 861 /* Take care that adding a new chunk of credit doesn't wrap to zero. */ 862 max_credit = vif->remaining_credit + vif->credit_bytes; 863 if (max_credit < vif->remaining_credit) 864 max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */ 865 866 vif->remaining_credit = min(max_credit, max_burst); 867 } 868 869 static void tx_credit_callback(unsigned long data) 870 { 871 struct xenvif *vif = (struct xenvif *)data; 872 tx_add_credit(vif); 873 xen_netbk_check_rx_xenvif(vif); 874 } 875 876 static void netbk_tx_err(struct xenvif *vif, 877 struct xen_netif_tx_request *txp, RING_IDX end) 878 { 879 RING_IDX cons = vif->tx.req_cons; 880 881 do { 882 make_tx_response(vif, txp, XEN_NETIF_RSP_ERROR); 883 if (cons == end) 884 break; 885 txp = RING_GET_REQUEST(&vif->tx, cons++); 886 } while (1); 887 vif->tx.req_cons = cons; 888 xen_netbk_check_rx_xenvif(vif); 889 xenvif_put(vif); 890 } 891 892 static void netbk_fatal_tx_err(struct xenvif *vif) 893 { 894 netdev_err(vif->dev, "fatal error; disabling device\n"); 895 xenvif_carrier_off(vif); 896 xenvif_put(vif); 897 } 898 899 static int netbk_count_requests(struct xenvif *vif, 900 struct xen_netif_tx_request *first, 901 struct xen_netif_tx_request *txp, 902 int work_to_do) 903 { 904 RING_IDX cons = vif->tx.req_cons; 905 int frags = 0; 906 907 if (!(first->flags & XEN_NETTXF_more_data)) 908 return 0; 909 910 do { 911 if (frags >= work_to_do) { 912 netdev_err(vif->dev, "Need more frags\n"); 913 netbk_fatal_tx_err(vif); 914 return -frags; 915 } 916 917 if (unlikely(frags >= MAX_SKB_FRAGS)) { 918 netdev_err(vif->dev, "Too many frags\n"); 919 netbk_fatal_tx_err(vif); 920 return -frags; 921 } 922 923 memcpy(txp, RING_GET_REQUEST(&vif->tx, cons + frags), 924 sizeof(*txp)); 925 if (txp->size > first->size) { 926 netdev_err(vif->dev, "Frag is bigger than frame.\n"); 927 netbk_fatal_tx_err(vif); 928 return -frags; 929 } 930 931 first->size -= txp->size; 932 frags++; 933 934 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) { 935 netdev_err(vif->dev, "txp->offset: %x, size: %u\n", 936 txp->offset, txp->size); 937 netbk_fatal_tx_err(vif); 938 return -frags; 939 } 940 } while ((txp++)->flags & XEN_NETTXF_more_data); 941 return frags; 942 } 943 944 static struct page *xen_netbk_alloc_page(struct xen_netbk *netbk, 945 struct sk_buff *skb, 946 u16 pending_idx) 947 { 948 struct page *page; 949 page = alloc_page(GFP_KERNEL|__GFP_COLD); 950 if (!page) 951 return NULL; 952 set_page_ext(page, netbk, pending_idx); 953 netbk->mmap_pages[pending_idx] = page; 954 return page; 955 } 956 957 static struct gnttab_copy *xen_netbk_get_requests(struct xen_netbk *netbk, 958 struct xenvif *vif, 959 struct sk_buff *skb, 960 struct xen_netif_tx_request *txp, 961 struct gnttab_copy *gop) 962 { 963 struct skb_shared_info *shinfo = skb_shinfo(skb); 964 skb_frag_t *frags = shinfo->frags; 965 u16 pending_idx = *((u16 *)skb->data); 966 int i, start; 967 968 /* Skip first skb fragment if it is on same page as header fragment. */ 969 start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx); 970 971 for (i = start; i < shinfo->nr_frags; i++, txp++) { 972 struct page *page; 973 pending_ring_idx_t index; 974 struct pending_tx_info *pending_tx_info = 975 netbk->pending_tx_info; 976 977 index = pending_index(netbk->pending_cons++); 978 pending_idx = netbk->pending_ring[index]; 979 page = xen_netbk_alloc_page(netbk, skb, pending_idx); 980 if (!page) 981 goto err; 982 983 gop->source.u.ref = txp->gref; 984 gop->source.domid = vif->domid; 985 gop->source.offset = txp->offset; 986 987 gop->dest.u.gmfn = virt_to_mfn(page_address(page)); 988 gop->dest.domid = DOMID_SELF; 989 gop->dest.offset = txp->offset; 990 991 gop->len = txp->size; 992 gop->flags = GNTCOPY_source_gref; 993 994 gop++; 995 996 memcpy(&pending_tx_info[pending_idx].req, txp, sizeof(*txp)); 997 xenvif_get(vif); 998 pending_tx_info[pending_idx].vif = vif; 999 frag_set_pending_idx(&frags[i], pending_idx); 1000 } 1001 1002 return gop; 1003 err: 1004 /* Unwind, freeing all pages and sending error responses. */ 1005 while (i-- > start) { 1006 xen_netbk_idx_release(netbk, frag_get_pending_idx(&frags[i]), 1007 XEN_NETIF_RSP_ERROR); 1008 } 1009 /* The head too, if necessary. */ 1010 if (start) 1011 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_ERROR); 1012 1013 return NULL; 1014 } 1015 1016 static int xen_netbk_tx_check_gop(struct xen_netbk *netbk, 1017 struct sk_buff *skb, 1018 struct gnttab_copy **gopp) 1019 { 1020 struct gnttab_copy *gop = *gopp; 1021 u16 pending_idx = *((u16 *)skb->data); 1022 struct skb_shared_info *shinfo = skb_shinfo(skb); 1023 int nr_frags = shinfo->nr_frags; 1024 int i, err, start; 1025 1026 /* Check status of header. */ 1027 err = gop->status; 1028 if (unlikely(err)) 1029 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_ERROR); 1030 1031 /* Skip first skb fragment if it is on same page as header fragment. */ 1032 start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx); 1033 1034 for (i = start; i < nr_frags; i++) { 1035 int j, newerr; 1036 1037 pending_idx = frag_get_pending_idx(&shinfo->frags[i]); 1038 1039 /* Check error status: if okay then remember grant handle. */ 1040 newerr = (++gop)->status; 1041 if (likely(!newerr)) { 1042 /* Had a previous error? Invalidate this fragment. */ 1043 if (unlikely(err)) 1044 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1045 continue; 1046 } 1047 1048 /* Error on this fragment: respond to client with an error. */ 1049 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_ERROR); 1050 1051 /* Not the first error? Preceding frags already invalidated. */ 1052 if (err) 1053 continue; 1054 1055 /* First error: invalidate header and preceding fragments. */ 1056 pending_idx = *((u16 *)skb->data); 1057 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1058 for (j = start; j < i; j++) { 1059 pending_idx = frag_get_pending_idx(&shinfo->frags[j]); 1060 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1061 } 1062 1063 /* Remember the error: invalidate all subsequent fragments. */ 1064 err = newerr; 1065 } 1066 1067 *gopp = gop + 1; 1068 return err; 1069 } 1070 1071 static void xen_netbk_fill_frags(struct xen_netbk *netbk, struct sk_buff *skb) 1072 { 1073 struct skb_shared_info *shinfo = skb_shinfo(skb); 1074 int nr_frags = shinfo->nr_frags; 1075 int i; 1076 1077 for (i = 0; i < nr_frags; i++) { 1078 skb_frag_t *frag = shinfo->frags + i; 1079 struct xen_netif_tx_request *txp; 1080 struct page *page; 1081 u16 pending_idx; 1082 1083 pending_idx = frag_get_pending_idx(frag); 1084 1085 txp = &netbk->pending_tx_info[pending_idx].req; 1086 page = virt_to_page(idx_to_kaddr(netbk, pending_idx)); 1087 __skb_fill_page_desc(skb, i, page, txp->offset, txp->size); 1088 skb->len += txp->size; 1089 skb->data_len += txp->size; 1090 skb->truesize += txp->size; 1091 1092 /* Take an extra reference to offset xen_netbk_idx_release */ 1093 get_page(netbk->mmap_pages[pending_idx]); 1094 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1095 } 1096 } 1097 1098 static int xen_netbk_get_extras(struct xenvif *vif, 1099 struct xen_netif_extra_info *extras, 1100 int work_to_do) 1101 { 1102 struct xen_netif_extra_info extra; 1103 RING_IDX cons = vif->tx.req_cons; 1104 1105 do { 1106 if (unlikely(work_to_do-- <= 0)) { 1107 netdev_err(vif->dev, "Missing extra info\n"); 1108 netbk_fatal_tx_err(vif); 1109 return -EBADR; 1110 } 1111 1112 memcpy(&extra, RING_GET_REQUEST(&vif->tx, cons), 1113 sizeof(extra)); 1114 if (unlikely(!extra.type || 1115 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) { 1116 vif->tx.req_cons = ++cons; 1117 netdev_err(vif->dev, 1118 "Invalid extra type: %d\n", extra.type); 1119 netbk_fatal_tx_err(vif); 1120 return -EINVAL; 1121 } 1122 1123 memcpy(&extras[extra.type - 1], &extra, sizeof(extra)); 1124 vif->tx.req_cons = ++cons; 1125 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE); 1126 1127 return work_to_do; 1128 } 1129 1130 static int netbk_set_skb_gso(struct xenvif *vif, 1131 struct sk_buff *skb, 1132 struct xen_netif_extra_info *gso) 1133 { 1134 if (!gso->u.gso.size) { 1135 netdev_err(vif->dev, "GSO size must not be zero.\n"); 1136 netbk_fatal_tx_err(vif); 1137 return -EINVAL; 1138 } 1139 1140 /* Currently only TCPv4 S.O. is supported. */ 1141 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4) { 1142 netdev_err(vif->dev, "Bad GSO type %d.\n", gso->u.gso.type); 1143 netbk_fatal_tx_err(vif); 1144 return -EINVAL; 1145 } 1146 1147 skb_shinfo(skb)->gso_size = gso->u.gso.size; 1148 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 1149 1150 /* Header must be checked, and gso_segs computed. */ 1151 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 1152 skb_shinfo(skb)->gso_segs = 0; 1153 1154 return 0; 1155 } 1156 1157 static int checksum_setup(struct xenvif *vif, struct sk_buff *skb) 1158 { 1159 struct iphdr *iph; 1160 unsigned char *th; 1161 int err = -EPROTO; 1162 int recalculate_partial_csum = 0; 1163 1164 /* 1165 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy 1166 * peers can fail to set NETRXF_csum_blank when sending a GSO 1167 * frame. In this case force the SKB to CHECKSUM_PARTIAL and 1168 * recalculate the partial checksum. 1169 */ 1170 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { 1171 vif->rx_gso_checksum_fixup++; 1172 skb->ip_summed = CHECKSUM_PARTIAL; 1173 recalculate_partial_csum = 1; 1174 } 1175 1176 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */ 1177 if (skb->ip_summed != CHECKSUM_PARTIAL) 1178 return 0; 1179 1180 if (skb->protocol != htons(ETH_P_IP)) 1181 goto out; 1182 1183 iph = (void *)skb->data; 1184 th = skb->data + 4 * iph->ihl; 1185 if (th >= skb_tail_pointer(skb)) 1186 goto out; 1187 1188 skb->csum_start = th - skb->head; 1189 switch (iph->protocol) { 1190 case IPPROTO_TCP: 1191 skb->csum_offset = offsetof(struct tcphdr, check); 1192 1193 if (recalculate_partial_csum) { 1194 struct tcphdr *tcph = (struct tcphdr *)th; 1195 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 1196 skb->len - iph->ihl*4, 1197 IPPROTO_TCP, 0); 1198 } 1199 break; 1200 case IPPROTO_UDP: 1201 skb->csum_offset = offsetof(struct udphdr, check); 1202 1203 if (recalculate_partial_csum) { 1204 struct udphdr *udph = (struct udphdr *)th; 1205 udph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 1206 skb->len - iph->ihl*4, 1207 IPPROTO_UDP, 0); 1208 } 1209 break; 1210 default: 1211 if (net_ratelimit()) 1212 netdev_err(vif->dev, 1213 "Attempting to checksum a non-TCP/UDP packet, dropping a protocol %d packet\n", 1214 iph->protocol); 1215 goto out; 1216 } 1217 1218 if ((th + skb->csum_offset + 2) > skb_tail_pointer(skb)) 1219 goto out; 1220 1221 err = 0; 1222 1223 out: 1224 return err; 1225 } 1226 1227 static bool tx_credit_exceeded(struct xenvif *vif, unsigned size) 1228 { 1229 unsigned long now = jiffies; 1230 unsigned long next_credit = 1231 vif->credit_timeout.expires + 1232 msecs_to_jiffies(vif->credit_usec / 1000); 1233 1234 /* Timer could already be pending in rare cases. */ 1235 if (timer_pending(&vif->credit_timeout)) 1236 return true; 1237 1238 /* Passed the point where we can replenish credit? */ 1239 if (time_after_eq(now, next_credit)) { 1240 vif->credit_timeout.expires = now; 1241 tx_add_credit(vif); 1242 } 1243 1244 /* Still too big to send right now? Set a callback. */ 1245 if (size > vif->remaining_credit) { 1246 vif->credit_timeout.data = 1247 (unsigned long)vif; 1248 vif->credit_timeout.function = 1249 tx_credit_callback; 1250 mod_timer(&vif->credit_timeout, 1251 next_credit); 1252 1253 return true; 1254 } 1255 1256 return false; 1257 } 1258 1259 static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk) 1260 { 1261 struct gnttab_copy *gop = netbk->tx_copy_ops, *request_gop; 1262 struct sk_buff *skb; 1263 int ret; 1264 1265 while (((nr_pending_reqs(netbk) + MAX_SKB_FRAGS) < MAX_PENDING_REQS) && 1266 !list_empty(&netbk->net_schedule_list)) { 1267 struct xenvif *vif; 1268 struct xen_netif_tx_request txreq; 1269 struct xen_netif_tx_request txfrags[MAX_SKB_FRAGS]; 1270 struct page *page; 1271 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX-1]; 1272 u16 pending_idx; 1273 RING_IDX idx; 1274 int work_to_do; 1275 unsigned int data_len; 1276 pending_ring_idx_t index; 1277 1278 /* Get a netif from the list with work to do. */ 1279 vif = poll_net_schedule_list(netbk); 1280 /* This can sometimes happen because the test of 1281 * list_empty(net_schedule_list) at the top of the 1282 * loop is unlocked. Just go back and have another 1283 * look. 1284 */ 1285 if (!vif) 1286 continue; 1287 1288 if (vif->tx.sring->req_prod - vif->tx.req_cons > 1289 XEN_NETIF_TX_RING_SIZE) { 1290 netdev_err(vif->dev, 1291 "Impossible number of requests. " 1292 "req_prod %d, req_cons %d, size %ld\n", 1293 vif->tx.sring->req_prod, vif->tx.req_cons, 1294 XEN_NETIF_TX_RING_SIZE); 1295 netbk_fatal_tx_err(vif); 1296 continue; 1297 } 1298 1299 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, work_to_do); 1300 if (!work_to_do) { 1301 xenvif_put(vif); 1302 continue; 1303 } 1304 1305 idx = vif->tx.req_cons; 1306 rmb(); /* Ensure that we see the request before we copy it. */ 1307 memcpy(&txreq, RING_GET_REQUEST(&vif->tx, idx), sizeof(txreq)); 1308 1309 /* Credit-based scheduling. */ 1310 if (txreq.size > vif->remaining_credit && 1311 tx_credit_exceeded(vif, txreq.size)) { 1312 xenvif_put(vif); 1313 continue; 1314 } 1315 1316 vif->remaining_credit -= txreq.size; 1317 1318 work_to_do--; 1319 vif->tx.req_cons = ++idx; 1320 1321 memset(extras, 0, sizeof(extras)); 1322 if (txreq.flags & XEN_NETTXF_extra_info) { 1323 work_to_do = xen_netbk_get_extras(vif, extras, 1324 work_to_do); 1325 idx = vif->tx.req_cons; 1326 if (unlikely(work_to_do < 0)) 1327 continue; 1328 } 1329 1330 ret = netbk_count_requests(vif, &txreq, txfrags, work_to_do); 1331 if (unlikely(ret < 0)) 1332 continue; 1333 1334 idx += ret; 1335 1336 if (unlikely(txreq.size < ETH_HLEN)) { 1337 netdev_dbg(vif->dev, 1338 "Bad packet size: %d\n", txreq.size); 1339 netbk_tx_err(vif, &txreq, idx); 1340 continue; 1341 } 1342 1343 /* No crossing a page as the payload mustn't fragment. */ 1344 if (unlikely((txreq.offset + txreq.size) > PAGE_SIZE)) { 1345 netdev_err(vif->dev, 1346 "txreq.offset: %x, size: %u, end: %lu\n", 1347 txreq.offset, txreq.size, 1348 (txreq.offset&~PAGE_MASK) + txreq.size); 1349 netbk_fatal_tx_err(vif); 1350 continue; 1351 } 1352 1353 index = pending_index(netbk->pending_cons); 1354 pending_idx = netbk->pending_ring[index]; 1355 1356 data_len = (txreq.size > PKT_PROT_LEN && 1357 ret < MAX_SKB_FRAGS) ? 1358 PKT_PROT_LEN : txreq.size; 1359 1360 skb = alloc_skb(data_len + NET_SKB_PAD + NET_IP_ALIGN, 1361 GFP_ATOMIC | __GFP_NOWARN); 1362 if (unlikely(skb == NULL)) { 1363 netdev_dbg(vif->dev, 1364 "Can't allocate a skb in start_xmit.\n"); 1365 netbk_tx_err(vif, &txreq, idx); 1366 break; 1367 } 1368 1369 /* Packets passed to netif_rx() must have some headroom. */ 1370 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 1371 1372 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { 1373 struct xen_netif_extra_info *gso; 1374 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; 1375 1376 if (netbk_set_skb_gso(vif, skb, gso)) { 1377 /* Failure in netbk_set_skb_gso is fatal. */ 1378 kfree_skb(skb); 1379 continue; 1380 } 1381 } 1382 1383 /* XXX could copy straight to head */ 1384 page = xen_netbk_alloc_page(netbk, skb, pending_idx); 1385 if (!page) { 1386 kfree_skb(skb); 1387 netbk_tx_err(vif, &txreq, idx); 1388 continue; 1389 } 1390 1391 gop->source.u.ref = txreq.gref; 1392 gop->source.domid = vif->domid; 1393 gop->source.offset = txreq.offset; 1394 1395 gop->dest.u.gmfn = virt_to_mfn(page_address(page)); 1396 gop->dest.domid = DOMID_SELF; 1397 gop->dest.offset = txreq.offset; 1398 1399 gop->len = txreq.size; 1400 gop->flags = GNTCOPY_source_gref; 1401 1402 gop++; 1403 1404 memcpy(&netbk->pending_tx_info[pending_idx].req, 1405 &txreq, sizeof(txreq)); 1406 netbk->pending_tx_info[pending_idx].vif = vif; 1407 *((u16 *)skb->data) = pending_idx; 1408 1409 __skb_put(skb, data_len); 1410 1411 skb_shinfo(skb)->nr_frags = ret; 1412 if (data_len < txreq.size) { 1413 skb_shinfo(skb)->nr_frags++; 1414 frag_set_pending_idx(&skb_shinfo(skb)->frags[0], 1415 pending_idx); 1416 } else { 1417 frag_set_pending_idx(&skb_shinfo(skb)->frags[0], 1418 INVALID_PENDING_IDX); 1419 } 1420 1421 netbk->pending_cons++; 1422 1423 request_gop = xen_netbk_get_requests(netbk, vif, 1424 skb, txfrags, gop); 1425 if (request_gop == NULL) { 1426 kfree_skb(skb); 1427 netbk_tx_err(vif, &txreq, idx); 1428 continue; 1429 } 1430 gop = request_gop; 1431 1432 __skb_queue_tail(&netbk->tx_queue, skb); 1433 1434 vif->tx.req_cons = idx; 1435 xen_netbk_check_rx_xenvif(vif); 1436 1437 if ((gop-netbk->tx_copy_ops) >= ARRAY_SIZE(netbk->tx_copy_ops)) 1438 break; 1439 } 1440 1441 return gop - netbk->tx_copy_ops; 1442 } 1443 1444 static void xen_netbk_tx_submit(struct xen_netbk *netbk) 1445 { 1446 struct gnttab_copy *gop = netbk->tx_copy_ops; 1447 struct sk_buff *skb; 1448 1449 while ((skb = __skb_dequeue(&netbk->tx_queue)) != NULL) { 1450 struct xen_netif_tx_request *txp; 1451 struct xenvif *vif; 1452 u16 pending_idx; 1453 unsigned data_len; 1454 1455 pending_idx = *((u16 *)skb->data); 1456 vif = netbk->pending_tx_info[pending_idx].vif; 1457 txp = &netbk->pending_tx_info[pending_idx].req; 1458 1459 /* Check the remap error code. */ 1460 if (unlikely(xen_netbk_tx_check_gop(netbk, skb, &gop))) { 1461 netdev_dbg(vif->dev, "netback grant failed.\n"); 1462 skb_shinfo(skb)->nr_frags = 0; 1463 kfree_skb(skb); 1464 continue; 1465 } 1466 1467 data_len = skb->len; 1468 memcpy(skb->data, 1469 (void *)(idx_to_kaddr(netbk, pending_idx)|txp->offset), 1470 data_len); 1471 if (data_len < txp->size) { 1472 /* Append the packet payload as a fragment. */ 1473 txp->offset += data_len; 1474 txp->size -= data_len; 1475 } else { 1476 /* Schedule a response immediately. */ 1477 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1478 } 1479 1480 if (txp->flags & XEN_NETTXF_csum_blank) 1481 skb->ip_summed = CHECKSUM_PARTIAL; 1482 else if (txp->flags & XEN_NETTXF_data_validated) 1483 skb->ip_summed = CHECKSUM_UNNECESSARY; 1484 1485 xen_netbk_fill_frags(netbk, skb); 1486 1487 /* 1488 * If the initial fragment was < PKT_PROT_LEN then 1489 * pull through some bytes from the other fragments to 1490 * increase the linear region to PKT_PROT_LEN bytes. 1491 */ 1492 if (skb_headlen(skb) < PKT_PROT_LEN && skb_is_nonlinear(skb)) { 1493 int target = min_t(int, skb->len, PKT_PROT_LEN); 1494 __pskb_pull_tail(skb, target - skb_headlen(skb)); 1495 } 1496 1497 skb->dev = vif->dev; 1498 skb->protocol = eth_type_trans(skb, skb->dev); 1499 1500 if (checksum_setup(vif, skb)) { 1501 netdev_dbg(vif->dev, 1502 "Can't setup checksum in net_tx_action\n"); 1503 kfree_skb(skb); 1504 continue; 1505 } 1506 1507 vif->dev->stats.rx_bytes += skb->len; 1508 vif->dev->stats.rx_packets++; 1509 1510 xenvif_receive_skb(vif, skb); 1511 } 1512 } 1513 1514 /* Called after netfront has transmitted */ 1515 static void xen_netbk_tx_action(struct xen_netbk *netbk) 1516 { 1517 unsigned nr_gops; 1518 1519 nr_gops = xen_netbk_tx_build_gops(netbk); 1520 1521 if (nr_gops == 0) 1522 return; 1523 1524 gnttab_batch_copy(netbk->tx_copy_ops, nr_gops); 1525 1526 xen_netbk_tx_submit(netbk); 1527 } 1528 1529 static void xen_netbk_idx_release(struct xen_netbk *netbk, u16 pending_idx, 1530 u8 status) 1531 { 1532 struct xenvif *vif; 1533 struct pending_tx_info *pending_tx_info; 1534 pending_ring_idx_t index; 1535 1536 /* Already complete? */ 1537 if (netbk->mmap_pages[pending_idx] == NULL) 1538 return; 1539 1540 pending_tx_info = &netbk->pending_tx_info[pending_idx]; 1541 1542 vif = pending_tx_info->vif; 1543 1544 make_tx_response(vif, &pending_tx_info->req, status); 1545 1546 index = pending_index(netbk->pending_prod++); 1547 netbk->pending_ring[index] = pending_idx; 1548 1549 xenvif_put(vif); 1550 1551 netbk->mmap_pages[pending_idx]->mapping = 0; 1552 put_page(netbk->mmap_pages[pending_idx]); 1553 netbk->mmap_pages[pending_idx] = NULL; 1554 } 1555 1556 static void make_tx_response(struct xenvif *vif, 1557 struct xen_netif_tx_request *txp, 1558 s8 st) 1559 { 1560 RING_IDX i = vif->tx.rsp_prod_pvt; 1561 struct xen_netif_tx_response *resp; 1562 int notify; 1563 1564 resp = RING_GET_RESPONSE(&vif->tx, i); 1565 resp->id = txp->id; 1566 resp->status = st; 1567 1568 if (txp->flags & XEN_NETTXF_extra_info) 1569 RING_GET_RESPONSE(&vif->tx, ++i)->status = XEN_NETIF_RSP_NULL; 1570 1571 vif->tx.rsp_prod_pvt = ++i; 1572 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->tx, notify); 1573 if (notify) 1574 notify_remote_via_irq(vif->irq); 1575 } 1576 1577 static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif, 1578 u16 id, 1579 s8 st, 1580 u16 offset, 1581 u16 size, 1582 u16 flags) 1583 { 1584 RING_IDX i = vif->rx.rsp_prod_pvt; 1585 struct xen_netif_rx_response *resp; 1586 1587 resp = RING_GET_RESPONSE(&vif->rx, i); 1588 resp->offset = offset; 1589 resp->flags = flags; 1590 resp->id = id; 1591 resp->status = (s16)size; 1592 if (st < 0) 1593 resp->status = (s16)st; 1594 1595 vif->rx.rsp_prod_pvt = ++i; 1596 1597 return resp; 1598 } 1599 1600 static inline int rx_work_todo(struct xen_netbk *netbk) 1601 { 1602 return !skb_queue_empty(&netbk->rx_queue); 1603 } 1604 1605 static inline int tx_work_todo(struct xen_netbk *netbk) 1606 { 1607 1608 if (((nr_pending_reqs(netbk) + MAX_SKB_FRAGS) < MAX_PENDING_REQS) && 1609 !list_empty(&netbk->net_schedule_list)) 1610 return 1; 1611 1612 return 0; 1613 } 1614 1615 static int xen_netbk_kthread(void *data) 1616 { 1617 struct xen_netbk *netbk = data; 1618 while (!kthread_should_stop()) { 1619 wait_event_interruptible(netbk->wq, 1620 rx_work_todo(netbk) || 1621 tx_work_todo(netbk) || 1622 kthread_should_stop()); 1623 cond_resched(); 1624 1625 if (kthread_should_stop()) 1626 break; 1627 1628 if (rx_work_todo(netbk)) 1629 xen_netbk_rx_action(netbk); 1630 1631 if (tx_work_todo(netbk)) 1632 xen_netbk_tx_action(netbk); 1633 } 1634 1635 return 0; 1636 } 1637 1638 void xen_netbk_unmap_frontend_rings(struct xenvif *vif) 1639 { 1640 if (vif->tx.sring) 1641 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif), 1642 vif->tx.sring); 1643 if (vif->rx.sring) 1644 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif), 1645 vif->rx.sring); 1646 } 1647 1648 int xen_netbk_map_frontend_rings(struct xenvif *vif, 1649 grant_ref_t tx_ring_ref, 1650 grant_ref_t rx_ring_ref) 1651 { 1652 void *addr; 1653 struct xen_netif_tx_sring *txs; 1654 struct xen_netif_rx_sring *rxs; 1655 1656 int err = -ENOMEM; 1657 1658 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif), 1659 tx_ring_ref, &addr); 1660 if (err) 1661 goto err; 1662 1663 txs = (struct xen_netif_tx_sring *)addr; 1664 BACK_RING_INIT(&vif->tx, txs, PAGE_SIZE); 1665 1666 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif), 1667 rx_ring_ref, &addr); 1668 if (err) 1669 goto err; 1670 1671 rxs = (struct xen_netif_rx_sring *)addr; 1672 BACK_RING_INIT(&vif->rx, rxs, PAGE_SIZE); 1673 1674 vif->rx_req_cons_peek = 0; 1675 1676 return 0; 1677 1678 err: 1679 xen_netbk_unmap_frontend_rings(vif); 1680 return err; 1681 } 1682 1683 static int __init netback_init(void) 1684 { 1685 int i; 1686 int rc = 0; 1687 int group; 1688 1689 if (!xen_domain()) 1690 return -ENODEV; 1691 1692 xen_netbk_group_nr = num_online_cpus(); 1693 xen_netbk = vzalloc(sizeof(struct xen_netbk) * xen_netbk_group_nr); 1694 if (!xen_netbk) 1695 return -ENOMEM; 1696 1697 for (group = 0; group < xen_netbk_group_nr; group++) { 1698 struct xen_netbk *netbk = &xen_netbk[group]; 1699 skb_queue_head_init(&netbk->rx_queue); 1700 skb_queue_head_init(&netbk->tx_queue); 1701 1702 init_timer(&netbk->net_timer); 1703 netbk->net_timer.data = (unsigned long)netbk; 1704 netbk->net_timer.function = xen_netbk_alarm; 1705 1706 netbk->pending_cons = 0; 1707 netbk->pending_prod = MAX_PENDING_REQS; 1708 for (i = 0; i < MAX_PENDING_REQS; i++) 1709 netbk->pending_ring[i] = i; 1710 1711 init_waitqueue_head(&netbk->wq); 1712 netbk->task = kthread_create(xen_netbk_kthread, 1713 (void *)netbk, 1714 "netback/%u", group); 1715 1716 if (IS_ERR(netbk->task)) { 1717 printk(KERN_ALERT "kthread_create() fails at netback\n"); 1718 del_timer(&netbk->net_timer); 1719 rc = PTR_ERR(netbk->task); 1720 goto failed_init; 1721 } 1722 1723 kthread_bind(netbk->task, group); 1724 1725 INIT_LIST_HEAD(&netbk->net_schedule_list); 1726 1727 spin_lock_init(&netbk->net_schedule_list_lock); 1728 1729 atomic_set(&netbk->netfront_count, 0); 1730 1731 wake_up_process(netbk->task); 1732 } 1733 1734 rc = xenvif_xenbus_init(); 1735 if (rc) 1736 goto failed_init; 1737 1738 return 0; 1739 1740 failed_init: 1741 while (--group >= 0) { 1742 struct xen_netbk *netbk = &xen_netbk[group]; 1743 for (i = 0; i < MAX_PENDING_REQS; i++) { 1744 if (netbk->mmap_pages[i]) 1745 __free_page(netbk->mmap_pages[i]); 1746 } 1747 del_timer(&netbk->net_timer); 1748 kthread_stop(netbk->task); 1749 } 1750 vfree(xen_netbk); 1751 return rc; 1752 1753 } 1754 1755 module_init(netback_init); 1756 1757 MODULE_LICENSE("Dual BSD/GPL"); 1758 MODULE_ALIAS("xen-backend:vif"); 1759