1 /* 2 * Routines having to do with the 'struct sk_buff' memory handlers. 3 * 4 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 5 * Florian La Roche <rzsfl@rz.uni-sb.de> 6 * 7 * Fixes: 8 * Alan Cox : Fixed the worst of the load 9 * balancer bugs. 10 * Dave Platt : Interrupt stacking fix. 11 * Richard Kooijman : Timestamp fixes. 12 * Alan Cox : Changed buffer format. 13 * Alan Cox : destructor hook for AF_UNIX etc. 14 * Linus Torvalds : Better skb_clone. 15 * Alan Cox : Added skb_copy. 16 * Alan Cox : Added all the changed routines Linus 17 * only put in the headers 18 * Ray VanTassle : Fixed --skb->lock in free 19 * Alan Cox : skb_copy copy arp field 20 * Andi Kleen : slabified it. 21 * Robert Olsson : Removed skb_head_pool 22 * 23 * NOTE: 24 * The __skb_ routines should be called with interrupts 25 * disabled, or you better be *real* sure that the operation is atomic 26 * with respect to whatever list is being frobbed (e.g. via lock_sock() 27 * or via disabling bottom half handlers, etc). 28 * 29 * This program is free software; you can redistribute it and/or 30 * modify it under the terms of the GNU General Public License 31 * as published by the Free Software Foundation; either version 32 * 2 of the License, or (at your option) any later version. 33 */ 34 35 /* 36 * The functions in this file will not compile correctly with gcc 2.4.x 37 */ 38 39 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 40 41 #include <linux/module.h> 42 #include <linux/types.h> 43 #include <linux/kernel.h> 44 #include <linux/mm.h> 45 #include <linux/interrupt.h> 46 #include <linux/in.h> 47 #include <linux/inet.h> 48 #include <linux/slab.h> 49 #include <linux/tcp.h> 50 #include <linux/udp.h> 51 #include <linux/sctp.h> 52 #include <linux/netdevice.h> 53 #ifdef CONFIG_NET_CLS_ACT 54 #include <net/pkt_sched.h> 55 #endif 56 #include <linux/string.h> 57 #include <linux/skbuff.h> 58 #include <linux/splice.h> 59 #include <linux/cache.h> 60 #include <linux/rtnetlink.h> 61 #include <linux/init.h> 62 #include <linux/scatterlist.h> 63 #include <linux/errqueue.h> 64 #include <linux/prefetch.h> 65 #include <linux/if_vlan.h> 66 67 #include <net/protocol.h> 68 #include <net/dst.h> 69 #include <net/sock.h> 70 #include <net/checksum.h> 71 #include <net/ip6_checksum.h> 72 #include <net/xfrm.h> 73 74 #include <linux/uaccess.h> 75 #include <trace/events/skb.h> 76 #include <linux/highmem.h> 77 #include <linux/capability.h> 78 #include <linux/user_namespace.h> 79 80 struct kmem_cache *skbuff_head_cache __read_mostly; 81 static struct kmem_cache *skbuff_fclone_cache __read_mostly; 82 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; 83 EXPORT_SYMBOL(sysctl_max_skb_frags); 84 85 /** 86 * skb_panic - private function for out-of-line support 87 * @skb: buffer 88 * @sz: size 89 * @addr: address 90 * @msg: skb_over_panic or skb_under_panic 91 * 92 * Out-of-line support for skb_put() and skb_push(). 93 * Called via the wrapper skb_over_panic() or skb_under_panic(). 94 * Keep out of line to prevent kernel bloat. 95 * __builtin_return_address is not used because it is not always reliable. 96 */ 97 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 98 const char msg[]) 99 { 100 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n", 101 msg, addr, skb->len, sz, skb->head, skb->data, 102 (unsigned long)skb->tail, (unsigned long)skb->end, 103 skb->dev ? skb->dev->name : "<NULL>"); 104 BUG(); 105 } 106 107 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 108 { 109 skb_panic(skb, sz, addr, __func__); 110 } 111 112 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 113 { 114 skb_panic(skb, sz, addr, __func__); 115 } 116 117 /* 118 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 119 * the caller if emergency pfmemalloc reserves are being used. If it is and 120 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 121 * may be used. Otherwise, the packet data may be discarded until enough 122 * memory is free 123 */ 124 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \ 125 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc) 126 127 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node, 128 unsigned long ip, bool *pfmemalloc) 129 { 130 void *obj; 131 bool ret_pfmemalloc = false; 132 133 /* 134 * Try a regular allocation, when that fails and we're not entitled 135 * to the reserves, fail. 136 */ 137 obj = kmalloc_node_track_caller(size, 138 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 139 node); 140 if (obj || !(gfp_pfmemalloc_allowed(flags))) 141 goto out; 142 143 /* Try again but now we are using pfmemalloc reserves */ 144 ret_pfmemalloc = true; 145 obj = kmalloc_node_track_caller(size, flags, node); 146 147 out: 148 if (pfmemalloc) 149 *pfmemalloc = ret_pfmemalloc; 150 151 return obj; 152 } 153 154 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 155 * 'private' fields and also do memory statistics to find all the 156 * [BEEP] leaks. 157 * 158 */ 159 160 /** 161 * __alloc_skb - allocate a network buffer 162 * @size: size to allocate 163 * @gfp_mask: allocation mask 164 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 165 * instead of head cache and allocate a cloned (child) skb. 166 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 167 * allocations in case the data is required for writeback 168 * @node: numa node to allocate memory on 169 * 170 * Allocate a new &sk_buff. The returned buffer has no headroom and a 171 * tail room of at least size bytes. The object has a reference count 172 * of one. The return is the buffer. On a failure the return is %NULL. 173 * 174 * Buffers may only be allocated from interrupts using a @gfp_mask of 175 * %GFP_ATOMIC. 176 */ 177 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 178 int flags, int node) 179 { 180 struct kmem_cache *cache; 181 struct skb_shared_info *shinfo; 182 struct sk_buff *skb; 183 u8 *data; 184 bool pfmemalloc; 185 186 cache = (flags & SKB_ALLOC_FCLONE) 187 ? skbuff_fclone_cache : skbuff_head_cache; 188 189 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 190 gfp_mask |= __GFP_MEMALLOC; 191 192 /* Get the HEAD */ 193 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); 194 if (!skb) 195 goto out; 196 prefetchw(skb); 197 198 /* We do our best to align skb_shared_info on a separate cache 199 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 200 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 201 * Both skb->head and skb_shared_info are cache line aligned. 202 */ 203 size = SKB_DATA_ALIGN(size); 204 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 205 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc); 206 if (!data) 207 goto nodata; 208 /* kmalloc(size) might give us more room than requested. 209 * Put skb_shared_info exactly at the end of allocated zone, 210 * to allow max possible filling before reallocation. 211 */ 212 size = SKB_WITH_OVERHEAD(ksize(data)); 213 prefetchw(data + size); 214 215 /* 216 * Only clear those fields we need to clear, not those that we will 217 * actually initialise below. Hence, don't put any more fields after 218 * the tail pointer in struct sk_buff! 219 */ 220 memset(skb, 0, offsetof(struct sk_buff, tail)); 221 /* Account for allocated memory : skb + skb->head */ 222 skb->truesize = SKB_TRUESIZE(size); 223 skb->pfmemalloc = pfmemalloc; 224 refcount_set(&skb->users, 1); 225 skb->head = data; 226 skb->data = data; 227 skb_reset_tail_pointer(skb); 228 skb->end = skb->tail + size; 229 skb->mac_header = (typeof(skb->mac_header))~0U; 230 skb->transport_header = (typeof(skb->transport_header))~0U; 231 232 /* make sure we initialize shinfo sequentially */ 233 shinfo = skb_shinfo(skb); 234 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 235 atomic_set(&shinfo->dataref, 1); 236 237 if (flags & SKB_ALLOC_FCLONE) { 238 struct sk_buff_fclones *fclones; 239 240 fclones = container_of(skb, struct sk_buff_fclones, skb1); 241 242 skb->fclone = SKB_FCLONE_ORIG; 243 refcount_set(&fclones->fclone_ref, 1); 244 245 fclones->skb2.fclone = SKB_FCLONE_CLONE; 246 } 247 out: 248 return skb; 249 nodata: 250 kmem_cache_free(cache, skb); 251 skb = NULL; 252 goto out; 253 } 254 EXPORT_SYMBOL(__alloc_skb); 255 256 /** 257 * __build_skb - build a network buffer 258 * @data: data buffer provided by caller 259 * @frag_size: size of data, or 0 if head was kmalloced 260 * 261 * Allocate a new &sk_buff. Caller provides space holding head and 262 * skb_shared_info. @data must have been allocated by kmalloc() only if 263 * @frag_size is 0, otherwise data should come from the page allocator 264 * or vmalloc() 265 * The return is the new skb buffer. 266 * On a failure the return is %NULL, and @data is not freed. 267 * Notes : 268 * Before IO, driver allocates only data buffer where NIC put incoming frame 269 * Driver should add room at head (NET_SKB_PAD) and 270 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 271 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 272 * before giving packet to stack. 273 * RX rings only contains data buffers, not full skbs. 274 */ 275 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 276 { 277 struct skb_shared_info *shinfo; 278 struct sk_buff *skb; 279 unsigned int size = frag_size ? : ksize(data); 280 281 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); 282 if (!skb) 283 return NULL; 284 285 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 286 287 memset(skb, 0, offsetof(struct sk_buff, tail)); 288 skb->truesize = SKB_TRUESIZE(size); 289 refcount_set(&skb->users, 1); 290 skb->head = data; 291 skb->data = data; 292 skb_reset_tail_pointer(skb); 293 skb->end = skb->tail + size; 294 skb->mac_header = (typeof(skb->mac_header))~0U; 295 skb->transport_header = (typeof(skb->transport_header))~0U; 296 297 /* make sure we initialize shinfo sequentially */ 298 shinfo = skb_shinfo(skb); 299 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 300 atomic_set(&shinfo->dataref, 1); 301 302 return skb; 303 } 304 305 /* build_skb() is wrapper over __build_skb(), that specifically 306 * takes care of skb->head and skb->pfmemalloc 307 * This means that if @frag_size is not zero, then @data must be backed 308 * by a page fragment, not kmalloc() or vmalloc() 309 */ 310 struct sk_buff *build_skb(void *data, unsigned int frag_size) 311 { 312 struct sk_buff *skb = __build_skb(data, frag_size); 313 314 if (skb && frag_size) { 315 skb->head_frag = 1; 316 if (page_is_pfmemalloc(virt_to_head_page(data))) 317 skb->pfmemalloc = 1; 318 } 319 return skb; 320 } 321 EXPORT_SYMBOL(build_skb); 322 323 #define NAPI_SKB_CACHE_SIZE 64 324 325 struct napi_alloc_cache { 326 struct page_frag_cache page; 327 unsigned int skb_count; 328 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 329 }; 330 331 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 332 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 333 334 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) 335 { 336 struct page_frag_cache *nc; 337 unsigned long flags; 338 void *data; 339 340 local_irq_save(flags); 341 nc = this_cpu_ptr(&netdev_alloc_cache); 342 data = page_frag_alloc(nc, fragsz, gfp_mask); 343 local_irq_restore(flags); 344 return data; 345 } 346 347 /** 348 * netdev_alloc_frag - allocate a page fragment 349 * @fragsz: fragment size 350 * 351 * Allocates a frag from a page for receive buffer. 352 * Uses GFP_ATOMIC allocations. 353 */ 354 void *netdev_alloc_frag(unsigned int fragsz) 355 { 356 return __netdev_alloc_frag(fragsz, GFP_ATOMIC); 357 } 358 EXPORT_SYMBOL(netdev_alloc_frag); 359 360 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) 361 { 362 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 363 364 return page_frag_alloc(&nc->page, fragsz, gfp_mask); 365 } 366 367 void *napi_alloc_frag(unsigned int fragsz) 368 { 369 return __napi_alloc_frag(fragsz, GFP_ATOMIC); 370 } 371 EXPORT_SYMBOL(napi_alloc_frag); 372 373 /** 374 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 375 * @dev: network device to receive on 376 * @len: length to allocate 377 * @gfp_mask: get_free_pages mask, passed to alloc_skb 378 * 379 * Allocate a new &sk_buff and assign it a usage count of one. The 380 * buffer has NET_SKB_PAD headroom built in. Users should allocate 381 * the headroom they think they need without accounting for the 382 * built in space. The built in space is used for optimisations. 383 * 384 * %NULL is returned if there is no free memory. 385 */ 386 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 387 gfp_t gfp_mask) 388 { 389 struct page_frag_cache *nc; 390 unsigned long flags; 391 struct sk_buff *skb; 392 bool pfmemalloc; 393 void *data; 394 395 len += NET_SKB_PAD; 396 397 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) || 398 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 399 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 400 if (!skb) 401 goto skb_fail; 402 goto skb_success; 403 } 404 405 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 406 len = SKB_DATA_ALIGN(len); 407 408 if (sk_memalloc_socks()) 409 gfp_mask |= __GFP_MEMALLOC; 410 411 local_irq_save(flags); 412 413 nc = this_cpu_ptr(&netdev_alloc_cache); 414 data = page_frag_alloc(nc, len, gfp_mask); 415 pfmemalloc = nc->pfmemalloc; 416 417 local_irq_restore(flags); 418 419 if (unlikely(!data)) 420 return NULL; 421 422 skb = __build_skb(data, len); 423 if (unlikely(!skb)) { 424 skb_free_frag(data); 425 return NULL; 426 } 427 428 /* use OR instead of assignment to avoid clearing of bits in mask */ 429 if (pfmemalloc) 430 skb->pfmemalloc = 1; 431 skb->head_frag = 1; 432 433 skb_success: 434 skb_reserve(skb, NET_SKB_PAD); 435 skb->dev = dev; 436 437 skb_fail: 438 return skb; 439 } 440 EXPORT_SYMBOL(__netdev_alloc_skb); 441 442 /** 443 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 444 * @napi: napi instance this buffer was allocated for 445 * @len: length to allocate 446 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages 447 * 448 * Allocate a new sk_buff for use in NAPI receive. This buffer will 449 * attempt to allocate the head from a special reserved region used 450 * only for NAPI Rx allocation. By doing this we can save several 451 * CPU cycles by avoiding having to disable and re-enable IRQs. 452 * 453 * %NULL is returned if there is no free memory. 454 */ 455 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, 456 gfp_t gfp_mask) 457 { 458 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 459 struct sk_buff *skb; 460 void *data; 461 462 len += NET_SKB_PAD + NET_IP_ALIGN; 463 464 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) || 465 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 466 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 467 if (!skb) 468 goto skb_fail; 469 goto skb_success; 470 } 471 472 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 473 len = SKB_DATA_ALIGN(len); 474 475 if (sk_memalloc_socks()) 476 gfp_mask |= __GFP_MEMALLOC; 477 478 data = page_frag_alloc(&nc->page, len, gfp_mask); 479 if (unlikely(!data)) 480 return NULL; 481 482 skb = __build_skb(data, len); 483 if (unlikely(!skb)) { 484 skb_free_frag(data); 485 return NULL; 486 } 487 488 /* use OR instead of assignment to avoid clearing of bits in mask */ 489 if (nc->page.pfmemalloc) 490 skb->pfmemalloc = 1; 491 skb->head_frag = 1; 492 493 skb_success: 494 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 495 skb->dev = napi->dev; 496 497 skb_fail: 498 return skb; 499 } 500 EXPORT_SYMBOL(__napi_alloc_skb); 501 502 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 503 int size, unsigned int truesize) 504 { 505 skb_fill_page_desc(skb, i, page, off, size); 506 skb->len += size; 507 skb->data_len += size; 508 skb->truesize += truesize; 509 } 510 EXPORT_SYMBOL(skb_add_rx_frag); 511 512 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 513 unsigned int truesize) 514 { 515 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 516 517 skb_frag_size_add(frag, size); 518 skb->len += size; 519 skb->data_len += size; 520 skb->truesize += truesize; 521 } 522 EXPORT_SYMBOL(skb_coalesce_rx_frag); 523 524 static void skb_drop_list(struct sk_buff **listp) 525 { 526 kfree_skb_list(*listp); 527 *listp = NULL; 528 } 529 530 static inline void skb_drop_fraglist(struct sk_buff *skb) 531 { 532 skb_drop_list(&skb_shinfo(skb)->frag_list); 533 } 534 535 static void skb_clone_fraglist(struct sk_buff *skb) 536 { 537 struct sk_buff *list; 538 539 skb_walk_frags(skb, list) 540 skb_get(list); 541 } 542 543 static void skb_free_head(struct sk_buff *skb) 544 { 545 unsigned char *head = skb->head; 546 547 if (skb->head_frag) 548 skb_free_frag(head); 549 else 550 kfree(head); 551 } 552 553 static void skb_release_data(struct sk_buff *skb) 554 { 555 struct skb_shared_info *shinfo = skb_shinfo(skb); 556 int i; 557 558 if (skb->cloned && 559 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 560 &shinfo->dataref)) 561 return; 562 563 for (i = 0; i < shinfo->nr_frags; i++) 564 __skb_frag_unref(&shinfo->frags[i]); 565 566 if (shinfo->frag_list) 567 kfree_skb_list(shinfo->frag_list); 568 569 skb_zcopy_clear(skb, true); 570 skb_free_head(skb); 571 } 572 573 /* 574 * Free an skbuff by memory without cleaning the state. 575 */ 576 static void kfree_skbmem(struct sk_buff *skb) 577 { 578 struct sk_buff_fclones *fclones; 579 580 switch (skb->fclone) { 581 case SKB_FCLONE_UNAVAILABLE: 582 kmem_cache_free(skbuff_head_cache, skb); 583 return; 584 585 case SKB_FCLONE_ORIG: 586 fclones = container_of(skb, struct sk_buff_fclones, skb1); 587 588 /* We usually free the clone (TX completion) before original skb 589 * This test would have no chance to be true for the clone, 590 * while here, branch prediction will be good. 591 */ 592 if (refcount_read(&fclones->fclone_ref) == 1) 593 goto fastpath; 594 break; 595 596 default: /* SKB_FCLONE_CLONE */ 597 fclones = container_of(skb, struct sk_buff_fclones, skb2); 598 break; 599 } 600 if (!refcount_dec_and_test(&fclones->fclone_ref)) 601 return; 602 fastpath: 603 kmem_cache_free(skbuff_fclone_cache, fclones); 604 } 605 606 void skb_release_head_state(struct sk_buff *skb) 607 { 608 skb_dst_drop(skb); 609 secpath_reset(skb); 610 if (skb->destructor) { 611 WARN_ON(in_irq()); 612 skb->destructor(skb); 613 } 614 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 615 nf_conntrack_put(skb_nfct(skb)); 616 #endif 617 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 618 nf_bridge_put(skb->nf_bridge); 619 #endif 620 } 621 622 /* Free everything but the sk_buff shell. */ 623 static void skb_release_all(struct sk_buff *skb) 624 { 625 skb_release_head_state(skb); 626 if (likely(skb->head)) 627 skb_release_data(skb); 628 } 629 630 /** 631 * __kfree_skb - private function 632 * @skb: buffer 633 * 634 * Free an sk_buff. Release anything attached to the buffer. 635 * Clean the state. This is an internal helper function. Users should 636 * always call kfree_skb 637 */ 638 639 void __kfree_skb(struct sk_buff *skb) 640 { 641 skb_release_all(skb); 642 kfree_skbmem(skb); 643 } 644 EXPORT_SYMBOL(__kfree_skb); 645 646 /** 647 * kfree_skb - free an sk_buff 648 * @skb: buffer to free 649 * 650 * Drop a reference to the buffer and free it if the usage count has 651 * hit zero. 652 */ 653 void kfree_skb(struct sk_buff *skb) 654 { 655 if (!skb_unref(skb)) 656 return; 657 658 trace_kfree_skb(skb, __builtin_return_address(0)); 659 __kfree_skb(skb); 660 } 661 EXPORT_SYMBOL(kfree_skb); 662 663 void kfree_skb_list(struct sk_buff *segs) 664 { 665 while (segs) { 666 struct sk_buff *next = segs->next; 667 668 kfree_skb(segs); 669 segs = next; 670 } 671 } 672 EXPORT_SYMBOL(kfree_skb_list); 673 674 /** 675 * skb_tx_error - report an sk_buff xmit error 676 * @skb: buffer that triggered an error 677 * 678 * Report xmit error if a device callback is tracking this skb. 679 * skb must be freed afterwards. 680 */ 681 void skb_tx_error(struct sk_buff *skb) 682 { 683 skb_zcopy_clear(skb, true); 684 } 685 EXPORT_SYMBOL(skb_tx_error); 686 687 /** 688 * consume_skb - free an skbuff 689 * @skb: buffer to free 690 * 691 * Drop a ref to the buffer and free it if the usage count has hit zero 692 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 693 * is being dropped after a failure and notes that 694 */ 695 void consume_skb(struct sk_buff *skb) 696 { 697 if (!skb_unref(skb)) 698 return; 699 700 trace_consume_skb(skb); 701 __kfree_skb(skb); 702 } 703 EXPORT_SYMBOL(consume_skb); 704 705 /** 706 * consume_stateless_skb - free an skbuff, assuming it is stateless 707 * @skb: buffer to free 708 * 709 * Alike consume_skb(), but this variant assumes that this is the last 710 * skb reference and all the head states have been already dropped 711 */ 712 void __consume_stateless_skb(struct sk_buff *skb) 713 { 714 trace_consume_skb(skb); 715 skb_release_data(skb); 716 kfree_skbmem(skb); 717 } 718 719 void __kfree_skb_flush(void) 720 { 721 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 722 723 /* flush skb_cache if containing objects */ 724 if (nc->skb_count) { 725 kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count, 726 nc->skb_cache); 727 nc->skb_count = 0; 728 } 729 } 730 731 static inline void _kfree_skb_defer(struct sk_buff *skb) 732 { 733 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 734 735 /* drop skb->head and call any destructors for packet */ 736 skb_release_all(skb); 737 738 /* record skb to CPU local list */ 739 nc->skb_cache[nc->skb_count++] = skb; 740 741 #ifdef CONFIG_SLUB 742 /* SLUB writes into objects when freeing */ 743 prefetchw(skb); 744 #endif 745 746 /* flush skb_cache if it is filled */ 747 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 748 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE, 749 nc->skb_cache); 750 nc->skb_count = 0; 751 } 752 } 753 void __kfree_skb_defer(struct sk_buff *skb) 754 { 755 _kfree_skb_defer(skb); 756 } 757 758 void napi_consume_skb(struct sk_buff *skb, int budget) 759 { 760 if (unlikely(!skb)) 761 return; 762 763 /* Zero budget indicate non-NAPI context called us, like netpoll */ 764 if (unlikely(!budget)) { 765 dev_consume_skb_any(skb); 766 return; 767 } 768 769 if (!skb_unref(skb)) 770 return; 771 772 /* if reaching here SKB is ready to free */ 773 trace_consume_skb(skb); 774 775 /* if SKB is a clone, don't handle this case */ 776 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 777 __kfree_skb(skb); 778 return; 779 } 780 781 _kfree_skb_defer(skb); 782 } 783 EXPORT_SYMBOL(napi_consume_skb); 784 785 /* Make sure a field is enclosed inside headers_start/headers_end section */ 786 #define CHECK_SKB_FIELD(field) \ 787 BUILD_BUG_ON(offsetof(struct sk_buff, field) < \ 788 offsetof(struct sk_buff, headers_start)); \ 789 BUILD_BUG_ON(offsetof(struct sk_buff, field) > \ 790 offsetof(struct sk_buff, headers_end)); \ 791 792 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 793 { 794 new->tstamp = old->tstamp; 795 /* We do not copy old->sk */ 796 new->dev = old->dev; 797 memcpy(new->cb, old->cb, sizeof(old->cb)); 798 skb_dst_copy(new, old); 799 #ifdef CONFIG_XFRM 800 new->sp = secpath_get(old->sp); 801 #endif 802 __nf_copy(new, old, false); 803 804 /* Note : this field could be in headers_start/headers_end section 805 * It is not yet because we do not want to have a 16 bit hole 806 */ 807 new->queue_mapping = old->queue_mapping; 808 809 memcpy(&new->headers_start, &old->headers_start, 810 offsetof(struct sk_buff, headers_end) - 811 offsetof(struct sk_buff, headers_start)); 812 CHECK_SKB_FIELD(protocol); 813 CHECK_SKB_FIELD(csum); 814 CHECK_SKB_FIELD(hash); 815 CHECK_SKB_FIELD(priority); 816 CHECK_SKB_FIELD(skb_iif); 817 CHECK_SKB_FIELD(vlan_proto); 818 CHECK_SKB_FIELD(vlan_tci); 819 CHECK_SKB_FIELD(transport_header); 820 CHECK_SKB_FIELD(network_header); 821 CHECK_SKB_FIELD(mac_header); 822 CHECK_SKB_FIELD(inner_protocol); 823 CHECK_SKB_FIELD(inner_transport_header); 824 CHECK_SKB_FIELD(inner_network_header); 825 CHECK_SKB_FIELD(inner_mac_header); 826 CHECK_SKB_FIELD(mark); 827 #ifdef CONFIG_NETWORK_SECMARK 828 CHECK_SKB_FIELD(secmark); 829 #endif 830 #ifdef CONFIG_NET_RX_BUSY_POLL 831 CHECK_SKB_FIELD(napi_id); 832 #endif 833 #ifdef CONFIG_XPS 834 CHECK_SKB_FIELD(sender_cpu); 835 #endif 836 #ifdef CONFIG_NET_SCHED 837 CHECK_SKB_FIELD(tc_index); 838 #endif 839 840 } 841 842 /* 843 * You should not add any new code to this function. Add it to 844 * __copy_skb_header above instead. 845 */ 846 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 847 { 848 #define C(x) n->x = skb->x 849 850 n->next = n->prev = NULL; 851 n->sk = NULL; 852 __copy_skb_header(n, skb); 853 854 C(len); 855 C(data_len); 856 C(mac_len); 857 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 858 n->cloned = 1; 859 n->nohdr = 0; 860 n->destructor = NULL; 861 C(tail); 862 C(end); 863 C(head); 864 C(head_frag); 865 C(data); 866 C(truesize); 867 refcount_set(&n->users, 1); 868 869 atomic_inc(&(skb_shinfo(skb)->dataref)); 870 skb->cloned = 1; 871 872 return n; 873 #undef C 874 } 875 876 /** 877 * skb_morph - morph one skb into another 878 * @dst: the skb to receive the contents 879 * @src: the skb to supply the contents 880 * 881 * This is identical to skb_clone except that the target skb is 882 * supplied by the user. 883 * 884 * The target skb is returned upon exit. 885 */ 886 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 887 { 888 skb_release_all(dst); 889 return __skb_clone(dst, src); 890 } 891 EXPORT_SYMBOL_GPL(skb_morph); 892 893 static int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 894 { 895 unsigned long max_pg, num_pg, new_pg, old_pg; 896 struct user_struct *user; 897 898 if (capable(CAP_IPC_LOCK) || !size) 899 return 0; 900 901 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 902 max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 903 user = mmp->user ? : current_user(); 904 905 do { 906 old_pg = atomic_long_read(&user->locked_vm); 907 new_pg = old_pg + num_pg; 908 if (new_pg > max_pg) 909 return -ENOBUFS; 910 } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) != 911 old_pg); 912 913 if (!mmp->user) { 914 mmp->user = get_uid(user); 915 mmp->num_pg = num_pg; 916 } else { 917 mmp->num_pg += num_pg; 918 } 919 920 return 0; 921 } 922 923 static void mm_unaccount_pinned_pages(struct mmpin *mmp) 924 { 925 if (mmp->user) { 926 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 927 free_uid(mmp->user); 928 } 929 } 930 931 struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size) 932 { 933 struct ubuf_info *uarg; 934 struct sk_buff *skb; 935 936 WARN_ON_ONCE(!in_task()); 937 938 if (!sock_flag(sk, SOCK_ZEROCOPY)) 939 return NULL; 940 941 skb = sock_omalloc(sk, 0, GFP_KERNEL); 942 if (!skb) 943 return NULL; 944 945 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 946 uarg = (void *)skb->cb; 947 uarg->mmp.user = NULL; 948 949 if (mm_account_pinned_pages(&uarg->mmp, size)) { 950 kfree_skb(skb); 951 return NULL; 952 } 953 954 uarg->callback = sock_zerocopy_callback; 955 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 956 uarg->len = 1; 957 uarg->bytelen = size; 958 uarg->zerocopy = 1; 959 refcount_set(&uarg->refcnt, 1); 960 sock_hold(sk); 961 962 return uarg; 963 } 964 EXPORT_SYMBOL_GPL(sock_zerocopy_alloc); 965 966 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg) 967 { 968 return container_of((void *)uarg, struct sk_buff, cb); 969 } 970 971 struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size, 972 struct ubuf_info *uarg) 973 { 974 if (uarg) { 975 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 976 u32 bytelen, next; 977 978 /* realloc only when socket is locked (TCP, UDP cork), 979 * so uarg->len and sk_zckey access is serialized 980 */ 981 if (!sock_owned_by_user(sk)) { 982 WARN_ON_ONCE(1); 983 return NULL; 984 } 985 986 bytelen = uarg->bytelen + size; 987 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) { 988 /* TCP can create new skb to attach new uarg */ 989 if (sk->sk_type == SOCK_STREAM) 990 goto new_alloc; 991 return NULL; 992 } 993 994 next = (u32)atomic_read(&sk->sk_zckey); 995 if ((u32)(uarg->id + uarg->len) == next) { 996 if (mm_account_pinned_pages(&uarg->mmp, size)) 997 return NULL; 998 uarg->len++; 999 uarg->bytelen = bytelen; 1000 atomic_set(&sk->sk_zckey, ++next); 1001 sock_zerocopy_get(uarg); 1002 return uarg; 1003 } 1004 } 1005 1006 new_alloc: 1007 return sock_zerocopy_alloc(sk, size); 1008 } 1009 EXPORT_SYMBOL_GPL(sock_zerocopy_realloc); 1010 1011 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1012 { 1013 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1014 u32 old_lo, old_hi; 1015 u64 sum_len; 1016 1017 old_lo = serr->ee.ee_info; 1018 old_hi = serr->ee.ee_data; 1019 sum_len = old_hi - old_lo + 1ULL + len; 1020 1021 if (sum_len >= (1ULL << 32)) 1022 return false; 1023 1024 if (lo != old_hi + 1) 1025 return false; 1026 1027 serr->ee.ee_data += len; 1028 return true; 1029 } 1030 1031 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success) 1032 { 1033 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1034 struct sock_exterr_skb *serr; 1035 struct sock *sk = skb->sk; 1036 struct sk_buff_head *q; 1037 unsigned long flags; 1038 u32 lo, hi; 1039 u16 len; 1040 1041 mm_unaccount_pinned_pages(&uarg->mmp); 1042 1043 /* if !len, there was only 1 call, and it was aborted 1044 * so do not queue a completion notification 1045 */ 1046 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1047 goto release; 1048 1049 len = uarg->len; 1050 lo = uarg->id; 1051 hi = uarg->id + len - 1; 1052 1053 serr = SKB_EXT_ERR(skb); 1054 memset(serr, 0, sizeof(*serr)); 1055 serr->ee.ee_errno = 0; 1056 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1057 serr->ee.ee_data = hi; 1058 serr->ee.ee_info = lo; 1059 if (!success) 1060 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1061 1062 q = &sk->sk_error_queue; 1063 spin_lock_irqsave(&q->lock, flags); 1064 tail = skb_peek_tail(q); 1065 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1066 !skb_zerocopy_notify_extend(tail, lo, len)) { 1067 __skb_queue_tail(q, skb); 1068 skb = NULL; 1069 } 1070 spin_unlock_irqrestore(&q->lock, flags); 1071 1072 sk->sk_error_report(sk); 1073 1074 release: 1075 consume_skb(skb); 1076 sock_put(sk); 1077 } 1078 EXPORT_SYMBOL_GPL(sock_zerocopy_callback); 1079 1080 void sock_zerocopy_put(struct ubuf_info *uarg) 1081 { 1082 if (uarg && refcount_dec_and_test(&uarg->refcnt)) { 1083 if (uarg->callback) 1084 uarg->callback(uarg, uarg->zerocopy); 1085 else 1086 consume_skb(skb_from_uarg(uarg)); 1087 } 1088 } 1089 EXPORT_SYMBOL_GPL(sock_zerocopy_put); 1090 1091 void sock_zerocopy_put_abort(struct ubuf_info *uarg) 1092 { 1093 if (uarg) { 1094 struct sock *sk = skb_from_uarg(uarg)->sk; 1095 1096 atomic_dec(&sk->sk_zckey); 1097 uarg->len--; 1098 1099 sock_zerocopy_put(uarg); 1100 } 1101 } 1102 EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort); 1103 1104 extern int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb, 1105 struct iov_iter *from, size_t length); 1106 1107 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1108 struct msghdr *msg, int len, 1109 struct ubuf_info *uarg) 1110 { 1111 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1112 struct iov_iter orig_iter = msg->msg_iter; 1113 int err, orig_len = skb->len; 1114 1115 /* An skb can only point to one uarg. This edge case happens when 1116 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc. 1117 */ 1118 if (orig_uarg && uarg != orig_uarg) 1119 return -EEXIST; 1120 1121 err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len); 1122 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1123 struct sock *save_sk = skb->sk; 1124 1125 /* Streams do not free skb on error. Reset to prev state. */ 1126 msg->msg_iter = orig_iter; 1127 skb->sk = sk; 1128 ___pskb_trim(skb, orig_len); 1129 skb->sk = save_sk; 1130 return err; 1131 } 1132 1133 skb_zcopy_set(skb, uarg); 1134 return skb->len - orig_len; 1135 } 1136 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1137 1138 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1139 gfp_t gfp_mask) 1140 { 1141 if (skb_zcopy(orig)) { 1142 if (skb_zcopy(nskb)) { 1143 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1144 if (!gfp_mask) { 1145 WARN_ON_ONCE(1); 1146 return -ENOMEM; 1147 } 1148 if (skb_uarg(nskb) == skb_uarg(orig)) 1149 return 0; 1150 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1151 return -EIO; 1152 } 1153 skb_zcopy_set(nskb, skb_uarg(orig)); 1154 } 1155 return 0; 1156 } 1157 1158 /** 1159 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1160 * @skb: the skb to modify 1161 * @gfp_mask: allocation priority 1162 * 1163 * This must be called on SKBTX_DEV_ZEROCOPY skb. 1164 * It will copy all frags into kernel and drop the reference 1165 * to userspace pages. 1166 * 1167 * If this function is called from an interrupt gfp_mask() must be 1168 * %GFP_ATOMIC. 1169 * 1170 * Returns 0 on success or a negative error code on failure 1171 * to allocate kernel memory to copy to. 1172 */ 1173 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 1174 { 1175 int num_frags = skb_shinfo(skb)->nr_frags; 1176 struct page *page, *head = NULL; 1177 int i, new_frags; 1178 u32 d_off; 1179 1180 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 1181 return -EINVAL; 1182 1183 if (!num_frags) 1184 goto release; 1185 1186 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT; 1187 for (i = 0; i < new_frags; i++) { 1188 page = alloc_page(gfp_mask); 1189 if (!page) { 1190 while (head) { 1191 struct page *next = (struct page *)page_private(head); 1192 put_page(head); 1193 head = next; 1194 } 1195 return -ENOMEM; 1196 } 1197 set_page_private(page, (unsigned long)head); 1198 head = page; 1199 } 1200 1201 page = head; 1202 d_off = 0; 1203 for (i = 0; i < num_frags; i++) { 1204 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1205 u32 p_off, p_len, copied; 1206 struct page *p; 1207 u8 *vaddr; 1208 1209 skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f), 1210 p, p_off, p_len, copied) { 1211 u32 copy, done = 0; 1212 vaddr = kmap_atomic(p); 1213 1214 while (done < p_len) { 1215 if (d_off == PAGE_SIZE) { 1216 d_off = 0; 1217 page = (struct page *)page_private(page); 1218 } 1219 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done); 1220 memcpy(page_address(page) + d_off, 1221 vaddr + p_off + done, copy); 1222 done += copy; 1223 d_off += copy; 1224 } 1225 kunmap_atomic(vaddr); 1226 } 1227 } 1228 1229 /* skb frags release userspace buffers */ 1230 for (i = 0; i < num_frags; i++) 1231 skb_frag_unref(skb, i); 1232 1233 /* skb frags point to kernel buffers */ 1234 for (i = 0; i < new_frags - 1; i++) { 1235 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE); 1236 head = (struct page *)page_private(head); 1237 } 1238 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); 1239 skb_shinfo(skb)->nr_frags = new_frags; 1240 1241 release: 1242 skb_zcopy_clear(skb, false); 1243 return 0; 1244 } 1245 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 1246 1247 /** 1248 * skb_clone - duplicate an sk_buff 1249 * @skb: buffer to clone 1250 * @gfp_mask: allocation priority 1251 * 1252 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 1253 * copies share the same packet data but not structure. The new 1254 * buffer has a reference count of 1. If the allocation fails the 1255 * function returns %NULL otherwise the new buffer is returned. 1256 * 1257 * If this function is called from an interrupt gfp_mask() must be 1258 * %GFP_ATOMIC. 1259 */ 1260 1261 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 1262 { 1263 struct sk_buff_fclones *fclones = container_of(skb, 1264 struct sk_buff_fclones, 1265 skb1); 1266 struct sk_buff *n; 1267 1268 if (skb_orphan_frags(skb, gfp_mask)) 1269 return NULL; 1270 1271 if (skb->fclone == SKB_FCLONE_ORIG && 1272 refcount_read(&fclones->fclone_ref) == 1) { 1273 n = &fclones->skb2; 1274 refcount_set(&fclones->fclone_ref, 2); 1275 } else { 1276 if (skb_pfmemalloc(skb)) 1277 gfp_mask |= __GFP_MEMALLOC; 1278 1279 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); 1280 if (!n) 1281 return NULL; 1282 1283 n->fclone = SKB_FCLONE_UNAVAILABLE; 1284 } 1285 1286 return __skb_clone(n, skb); 1287 } 1288 EXPORT_SYMBOL(skb_clone); 1289 1290 static void skb_headers_offset_update(struct sk_buff *skb, int off) 1291 { 1292 /* Only adjust this if it actually is csum_start rather than csum */ 1293 if (skb->ip_summed == CHECKSUM_PARTIAL) 1294 skb->csum_start += off; 1295 /* {transport,network,mac}_header and tail are relative to skb->head */ 1296 skb->transport_header += off; 1297 skb->network_header += off; 1298 if (skb_mac_header_was_set(skb)) 1299 skb->mac_header += off; 1300 skb->inner_transport_header += off; 1301 skb->inner_network_header += off; 1302 skb->inner_mac_header += off; 1303 } 1304 1305 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 1306 { 1307 __copy_skb_header(new, old); 1308 1309 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 1310 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 1311 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 1312 } 1313 1314 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 1315 { 1316 if (skb_pfmemalloc(skb)) 1317 return SKB_ALLOC_RX; 1318 return 0; 1319 } 1320 1321 /** 1322 * skb_copy - create private copy of an sk_buff 1323 * @skb: buffer to copy 1324 * @gfp_mask: allocation priority 1325 * 1326 * Make a copy of both an &sk_buff and its data. This is used when the 1327 * caller wishes to modify the data and needs a private copy of the 1328 * data to alter. Returns %NULL on failure or the pointer to the buffer 1329 * on success. The returned buffer has a reference count of 1. 1330 * 1331 * As by-product this function converts non-linear &sk_buff to linear 1332 * one, so that &sk_buff becomes completely private and caller is allowed 1333 * to modify all the data of returned buffer. This means that this 1334 * function is not recommended for use in circumstances when only 1335 * header is going to be modified. Use pskb_copy() instead. 1336 */ 1337 1338 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 1339 { 1340 int headerlen = skb_headroom(skb); 1341 unsigned int size = skb_end_offset(skb) + skb->data_len; 1342 struct sk_buff *n = __alloc_skb(size, gfp_mask, 1343 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 1344 1345 if (!n) 1346 return NULL; 1347 1348 /* Set the data pointer */ 1349 skb_reserve(n, headerlen); 1350 /* Set the tail pointer and length */ 1351 skb_put(n, skb->len); 1352 1353 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); 1354 1355 copy_skb_header(n, skb); 1356 return n; 1357 } 1358 EXPORT_SYMBOL(skb_copy); 1359 1360 /** 1361 * __pskb_copy_fclone - create copy of an sk_buff with private head. 1362 * @skb: buffer to copy 1363 * @headroom: headroom of new skb 1364 * @gfp_mask: allocation priority 1365 * @fclone: if true allocate the copy of the skb from the fclone 1366 * cache instead of the head cache; it is recommended to set this 1367 * to true for the cases where the copy will likely be cloned 1368 * 1369 * Make a copy of both an &sk_buff and part of its data, located 1370 * in header. Fragmented data remain shared. This is used when 1371 * the caller wishes to modify only header of &sk_buff and needs 1372 * private copy of the header to alter. Returns %NULL on failure 1373 * or the pointer to the buffer on success. 1374 * The returned buffer has a reference count of 1. 1375 */ 1376 1377 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 1378 gfp_t gfp_mask, bool fclone) 1379 { 1380 unsigned int size = skb_headlen(skb) + headroom; 1381 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 1382 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 1383 1384 if (!n) 1385 goto out; 1386 1387 /* Set the data pointer */ 1388 skb_reserve(n, headroom); 1389 /* Set the tail pointer and length */ 1390 skb_put(n, skb_headlen(skb)); 1391 /* Copy the bytes */ 1392 skb_copy_from_linear_data(skb, n->data, n->len); 1393 1394 n->truesize += skb->data_len; 1395 n->data_len = skb->data_len; 1396 n->len = skb->len; 1397 1398 if (skb_shinfo(skb)->nr_frags) { 1399 int i; 1400 1401 if (skb_orphan_frags(skb, gfp_mask) || 1402 skb_zerocopy_clone(n, skb, gfp_mask)) { 1403 kfree_skb(n); 1404 n = NULL; 1405 goto out; 1406 } 1407 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1408 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 1409 skb_frag_ref(skb, i); 1410 } 1411 skb_shinfo(n)->nr_frags = i; 1412 } 1413 1414 if (skb_has_frag_list(skb)) { 1415 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 1416 skb_clone_fraglist(n); 1417 } 1418 1419 copy_skb_header(n, skb); 1420 out: 1421 return n; 1422 } 1423 EXPORT_SYMBOL(__pskb_copy_fclone); 1424 1425 /** 1426 * pskb_expand_head - reallocate header of &sk_buff 1427 * @skb: buffer to reallocate 1428 * @nhead: room to add at head 1429 * @ntail: room to add at tail 1430 * @gfp_mask: allocation priority 1431 * 1432 * Expands (or creates identical copy, if @nhead and @ntail are zero) 1433 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 1434 * reference count of 1. Returns zero in the case of success or error, 1435 * if expansion failed. In the last case, &sk_buff is not changed. 1436 * 1437 * All the pointers pointing into skb header may change and must be 1438 * reloaded after call to this function. 1439 */ 1440 1441 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 1442 gfp_t gfp_mask) 1443 { 1444 int i, osize = skb_end_offset(skb); 1445 int size = osize + nhead + ntail; 1446 long off; 1447 u8 *data; 1448 1449 BUG_ON(nhead < 0); 1450 1451 BUG_ON(skb_shared(skb)); 1452 1453 size = SKB_DATA_ALIGN(size); 1454 1455 if (skb_pfmemalloc(skb)) 1456 gfp_mask |= __GFP_MEMALLOC; 1457 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 1458 gfp_mask, NUMA_NO_NODE, NULL); 1459 if (!data) 1460 goto nodata; 1461 size = SKB_WITH_OVERHEAD(ksize(data)); 1462 1463 /* Copy only real data... and, alas, header. This should be 1464 * optimized for the cases when header is void. 1465 */ 1466 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 1467 1468 memcpy((struct skb_shared_info *)(data + size), 1469 skb_shinfo(skb), 1470 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 1471 1472 /* 1473 * if shinfo is shared we must drop the old head gracefully, but if it 1474 * is not we can just drop the old head and let the existing refcount 1475 * be since all we did is relocate the values 1476 */ 1477 if (skb_cloned(skb)) { 1478 if (skb_orphan_frags(skb, gfp_mask)) 1479 goto nofrags; 1480 if (skb_zcopy(skb)) 1481 refcount_inc(&skb_uarg(skb)->refcnt); 1482 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1483 skb_frag_ref(skb, i); 1484 1485 if (skb_has_frag_list(skb)) 1486 skb_clone_fraglist(skb); 1487 1488 skb_release_data(skb); 1489 } else { 1490 skb_free_head(skb); 1491 } 1492 off = (data + nhead) - skb->head; 1493 1494 skb->head = data; 1495 skb->head_frag = 0; 1496 skb->data += off; 1497 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1498 skb->end = size; 1499 off = nhead; 1500 #else 1501 skb->end = skb->head + size; 1502 #endif 1503 skb->tail += off; 1504 skb_headers_offset_update(skb, nhead); 1505 skb->cloned = 0; 1506 skb->hdr_len = 0; 1507 skb->nohdr = 0; 1508 atomic_set(&skb_shinfo(skb)->dataref, 1); 1509 1510 skb_metadata_clear(skb); 1511 1512 /* It is not generally safe to change skb->truesize. 1513 * For the moment, we really care of rx path, or 1514 * when skb is orphaned (not attached to a socket). 1515 */ 1516 if (!skb->sk || skb->destructor == sock_edemux) 1517 skb->truesize += size - osize; 1518 1519 return 0; 1520 1521 nofrags: 1522 kfree(data); 1523 nodata: 1524 return -ENOMEM; 1525 } 1526 EXPORT_SYMBOL(pskb_expand_head); 1527 1528 /* Make private copy of skb with writable head and some headroom */ 1529 1530 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 1531 { 1532 struct sk_buff *skb2; 1533 int delta = headroom - skb_headroom(skb); 1534 1535 if (delta <= 0) 1536 skb2 = pskb_copy(skb, GFP_ATOMIC); 1537 else { 1538 skb2 = skb_clone(skb, GFP_ATOMIC); 1539 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 1540 GFP_ATOMIC)) { 1541 kfree_skb(skb2); 1542 skb2 = NULL; 1543 } 1544 } 1545 return skb2; 1546 } 1547 EXPORT_SYMBOL(skb_realloc_headroom); 1548 1549 /** 1550 * skb_copy_expand - copy and expand sk_buff 1551 * @skb: buffer to copy 1552 * @newheadroom: new free bytes at head 1553 * @newtailroom: new free bytes at tail 1554 * @gfp_mask: allocation priority 1555 * 1556 * Make a copy of both an &sk_buff and its data and while doing so 1557 * allocate additional space. 1558 * 1559 * This is used when the caller wishes to modify the data and needs a 1560 * private copy of the data to alter as well as more space for new fields. 1561 * Returns %NULL on failure or the pointer to the buffer 1562 * on success. The returned buffer has a reference count of 1. 1563 * 1564 * You must pass %GFP_ATOMIC as the allocation priority if this function 1565 * is called from an interrupt. 1566 */ 1567 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 1568 int newheadroom, int newtailroom, 1569 gfp_t gfp_mask) 1570 { 1571 /* 1572 * Allocate the copy buffer 1573 */ 1574 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 1575 gfp_mask, skb_alloc_rx_flag(skb), 1576 NUMA_NO_NODE); 1577 int oldheadroom = skb_headroom(skb); 1578 int head_copy_len, head_copy_off; 1579 1580 if (!n) 1581 return NULL; 1582 1583 skb_reserve(n, newheadroom); 1584 1585 /* Set the tail pointer and length */ 1586 skb_put(n, skb->len); 1587 1588 head_copy_len = oldheadroom; 1589 head_copy_off = 0; 1590 if (newheadroom <= head_copy_len) 1591 head_copy_len = newheadroom; 1592 else 1593 head_copy_off = newheadroom - head_copy_len; 1594 1595 /* Copy the linear header and data. */ 1596 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 1597 skb->len + head_copy_len)); 1598 1599 copy_skb_header(n, skb); 1600 1601 skb_headers_offset_update(n, newheadroom - oldheadroom); 1602 1603 return n; 1604 } 1605 EXPORT_SYMBOL(skb_copy_expand); 1606 1607 /** 1608 * __skb_pad - zero pad the tail of an skb 1609 * @skb: buffer to pad 1610 * @pad: space to pad 1611 * @free_on_error: free buffer on error 1612 * 1613 * Ensure that a buffer is followed by a padding area that is zero 1614 * filled. Used by network drivers which may DMA or transfer data 1615 * beyond the buffer end onto the wire. 1616 * 1617 * May return error in out of memory cases. The skb is freed on error 1618 * if @free_on_error is true. 1619 */ 1620 1621 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 1622 { 1623 int err; 1624 int ntail; 1625 1626 /* If the skbuff is non linear tailroom is always zero.. */ 1627 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 1628 memset(skb->data+skb->len, 0, pad); 1629 return 0; 1630 } 1631 1632 ntail = skb->data_len + pad - (skb->end - skb->tail); 1633 if (likely(skb_cloned(skb) || ntail > 0)) { 1634 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 1635 if (unlikely(err)) 1636 goto free_skb; 1637 } 1638 1639 /* FIXME: The use of this function with non-linear skb's really needs 1640 * to be audited. 1641 */ 1642 err = skb_linearize(skb); 1643 if (unlikely(err)) 1644 goto free_skb; 1645 1646 memset(skb->data + skb->len, 0, pad); 1647 return 0; 1648 1649 free_skb: 1650 if (free_on_error) 1651 kfree_skb(skb); 1652 return err; 1653 } 1654 EXPORT_SYMBOL(__skb_pad); 1655 1656 /** 1657 * pskb_put - add data to the tail of a potentially fragmented buffer 1658 * @skb: start of the buffer to use 1659 * @tail: tail fragment of the buffer to use 1660 * @len: amount of data to add 1661 * 1662 * This function extends the used data area of the potentially 1663 * fragmented buffer. @tail must be the last fragment of @skb -- or 1664 * @skb itself. If this would exceed the total buffer size the kernel 1665 * will panic. A pointer to the first byte of the extra data is 1666 * returned. 1667 */ 1668 1669 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 1670 { 1671 if (tail != skb) { 1672 skb->data_len += len; 1673 skb->len += len; 1674 } 1675 return skb_put(tail, len); 1676 } 1677 EXPORT_SYMBOL_GPL(pskb_put); 1678 1679 /** 1680 * skb_put - add data to a buffer 1681 * @skb: buffer to use 1682 * @len: amount of data to add 1683 * 1684 * This function extends the used data area of the buffer. If this would 1685 * exceed the total buffer size the kernel will panic. A pointer to the 1686 * first byte of the extra data is returned. 1687 */ 1688 void *skb_put(struct sk_buff *skb, unsigned int len) 1689 { 1690 void *tmp = skb_tail_pointer(skb); 1691 SKB_LINEAR_ASSERT(skb); 1692 skb->tail += len; 1693 skb->len += len; 1694 if (unlikely(skb->tail > skb->end)) 1695 skb_over_panic(skb, len, __builtin_return_address(0)); 1696 return tmp; 1697 } 1698 EXPORT_SYMBOL(skb_put); 1699 1700 /** 1701 * skb_push - add data to the start of a buffer 1702 * @skb: buffer to use 1703 * @len: amount of data to add 1704 * 1705 * This function extends the used data area of the buffer at the buffer 1706 * start. If this would exceed the total buffer headroom the kernel will 1707 * panic. A pointer to the first byte of the extra data is returned. 1708 */ 1709 void *skb_push(struct sk_buff *skb, unsigned int len) 1710 { 1711 skb->data -= len; 1712 skb->len += len; 1713 if (unlikely(skb->data<skb->head)) 1714 skb_under_panic(skb, len, __builtin_return_address(0)); 1715 return skb->data; 1716 } 1717 EXPORT_SYMBOL(skb_push); 1718 1719 /** 1720 * skb_pull - remove data from the start of a buffer 1721 * @skb: buffer to use 1722 * @len: amount of data to remove 1723 * 1724 * This function removes data from the start of a buffer, returning 1725 * the memory to the headroom. A pointer to the next data in the buffer 1726 * is returned. Once the data has been pulled future pushes will overwrite 1727 * the old data. 1728 */ 1729 void *skb_pull(struct sk_buff *skb, unsigned int len) 1730 { 1731 return skb_pull_inline(skb, len); 1732 } 1733 EXPORT_SYMBOL(skb_pull); 1734 1735 /** 1736 * skb_trim - remove end from a buffer 1737 * @skb: buffer to alter 1738 * @len: new length 1739 * 1740 * Cut the length of a buffer down by removing data from the tail. If 1741 * the buffer is already under the length specified it is not modified. 1742 * The skb must be linear. 1743 */ 1744 void skb_trim(struct sk_buff *skb, unsigned int len) 1745 { 1746 if (skb->len > len) 1747 __skb_trim(skb, len); 1748 } 1749 EXPORT_SYMBOL(skb_trim); 1750 1751 /* Trims skb to length len. It can change skb pointers. 1752 */ 1753 1754 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 1755 { 1756 struct sk_buff **fragp; 1757 struct sk_buff *frag; 1758 int offset = skb_headlen(skb); 1759 int nfrags = skb_shinfo(skb)->nr_frags; 1760 int i; 1761 int err; 1762 1763 if (skb_cloned(skb) && 1764 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 1765 return err; 1766 1767 i = 0; 1768 if (offset >= len) 1769 goto drop_pages; 1770 1771 for (; i < nfrags; i++) { 1772 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 1773 1774 if (end < len) { 1775 offset = end; 1776 continue; 1777 } 1778 1779 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 1780 1781 drop_pages: 1782 skb_shinfo(skb)->nr_frags = i; 1783 1784 for (; i < nfrags; i++) 1785 skb_frag_unref(skb, i); 1786 1787 if (skb_has_frag_list(skb)) 1788 skb_drop_fraglist(skb); 1789 goto done; 1790 } 1791 1792 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 1793 fragp = &frag->next) { 1794 int end = offset + frag->len; 1795 1796 if (skb_shared(frag)) { 1797 struct sk_buff *nfrag; 1798 1799 nfrag = skb_clone(frag, GFP_ATOMIC); 1800 if (unlikely(!nfrag)) 1801 return -ENOMEM; 1802 1803 nfrag->next = frag->next; 1804 consume_skb(frag); 1805 frag = nfrag; 1806 *fragp = frag; 1807 } 1808 1809 if (end < len) { 1810 offset = end; 1811 continue; 1812 } 1813 1814 if (end > len && 1815 unlikely((err = pskb_trim(frag, len - offset)))) 1816 return err; 1817 1818 if (frag->next) 1819 skb_drop_list(&frag->next); 1820 break; 1821 } 1822 1823 done: 1824 if (len > skb_headlen(skb)) { 1825 skb->data_len -= skb->len - len; 1826 skb->len = len; 1827 } else { 1828 skb->len = len; 1829 skb->data_len = 0; 1830 skb_set_tail_pointer(skb, len); 1831 } 1832 1833 if (!skb->sk || skb->destructor == sock_edemux) 1834 skb_condense(skb); 1835 return 0; 1836 } 1837 EXPORT_SYMBOL(___pskb_trim); 1838 1839 /** 1840 * __pskb_pull_tail - advance tail of skb header 1841 * @skb: buffer to reallocate 1842 * @delta: number of bytes to advance tail 1843 * 1844 * The function makes a sense only on a fragmented &sk_buff, 1845 * it expands header moving its tail forward and copying necessary 1846 * data from fragmented part. 1847 * 1848 * &sk_buff MUST have reference count of 1. 1849 * 1850 * Returns %NULL (and &sk_buff does not change) if pull failed 1851 * or value of new tail of skb in the case of success. 1852 * 1853 * All the pointers pointing into skb header may change and must be 1854 * reloaded after call to this function. 1855 */ 1856 1857 /* Moves tail of skb head forward, copying data from fragmented part, 1858 * when it is necessary. 1859 * 1. It may fail due to malloc failure. 1860 * 2. It may change skb pointers. 1861 * 1862 * It is pretty complicated. Luckily, it is called only in exceptional cases. 1863 */ 1864 void *__pskb_pull_tail(struct sk_buff *skb, int delta) 1865 { 1866 /* If skb has not enough free space at tail, get new one 1867 * plus 128 bytes for future expansions. If we have enough 1868 * room at tail, reallocate without expansion only if skb is cloned. 1869 */ 1870 int i, k, eat = (skb->tail + delta) - skb->end; 1871 1872 if (eat > 0 || skb_cloned(skb)) { 1873 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 1874 GFP_ATOMIC)) 1875 return NULL; 1876 } 1877 1878 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 1879 skb_tail_pointer(skb), delta)); 1880 1881 /* Optimization: no fragments, no reasons to preestimate 1882 * size of pulled pages. Superb. 1883 */ 1884 if (!skb_has_frag_list(skb)) 1885 goto pull_pages; 1886 1887 /* Estimate size of pulled pages. */ 1888 eat = delta; 1889 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1890 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 1891 1892 if (size >= eat) 1893 goto pull_pages; 1894 eat -= size; 1895 } 1896 1897 /* If we need update frag list, we are in troubles. 1898 * Certainly, it is possible to add an offset to skb data, 1899 * but taking into account that pulling is expected to 1900 * be very rare operation, it is worth to fight against 1901 * further bloating skb head and crucify ourselves here instead. 1902 * Pure masohism, indeed. 8)8) 1903 */ 1904 if (eat) { 1905 struct sk_buff *list = skb_shinfo(skb)->frag_list; 1906 struct sk_buff *clone = NULL; 1907 struct sk_buff *insp = NULL; 1908 1909 do { 1910 BUG_ON(!list); 1911 1912 if (list->len <= eat) { 1913 /* Eaten as whole. */ 1914 eat -= list->len; 1915 list = list->next; 1916 insp = list; 1917 } else { 1918 /* Eaten partially. */ 1919 1920 if (skb_shared(list)) { 1921 /* Sucks! We need to fork list. :-( */ 1922 clone = skb_clone(list, GFP_ATOMIC); 1923 if (!clone) 1924 return NULL; 1925 insp = list->next; 1926 list = clone; 1927 } else { 1928 /* This may be pulled without 1929 * problems. */ 1930 insp = list; 1931 } 1932 if (!pskb_pull(list, eat)) { 1933 kfree_skb(clone); 1934 return NULL; 1935 } 1936 break; 1937 } 1938 } while (eat); 1939 1940 /* Free pulled out fragments. */ 1941 while ((list = skb_shinfo(skb)->frag_list) != insp) { 1942 skb_shinfo(skb)->frag_list = list->next; 1943 kfree_skb(list); 1944 } 1945 /* And insert new clone at head. */ 1946 if (clone) { 1947 clone->next = list; 1948 skb_shinfo(skb)->frag_list = clone; 1949 } 1950 } 1951 /* Success! Now we may commit changes to skb data. */ 1952 1953 pull_pages: 1954 eat = delta; 1955 k = 0; 1956 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1957 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 1958 1959 if (size <= eat) { 1960 skb_frag_unref(skb, i); 1961 eat -= size; 1962 } else { 1963 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; 1964 if (eat) { 1965 skb_shinfo(skb)->frags[k].page_offset += eat; 1966 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat); 1967 if (!i) 1968 goto end; 1969 eat = 0; 1970 } 1971 k++; 1972 } 1973 } 1974 skb_shinfo(skb)->nr_frags = k; 1975 1976 end: 1977 skb->tail += delta; 1978 skb->data_len -= delta; 1979 1980 if (!skb->data_len) 1981 skb_zcopy_clear(skb, false); 1982 1983 return skb_tail_pointer(skb); 1984 } 1985 EXPORT_SYMBOL(__pskb_pull_tail); 1986 1987 /** 1988 * skb_copy_bits - copy bits from skb to kernel buffer 1989 * @skb: source skb 1990 * @offset: offset in source 1991 * @to: destination buffer 1992 * @len: number of bytes to copy 1993 * 1994 * Copy the specified number of bytes from the source skb to the 1995 * destination buffer. 1996 * 1997 * CAUTION ! : 1998 * If its prototype is ever changed, 1999 * check arch/{*}/net/{*}.S files, 2000 * since it is called from BPF assembly code. 2001 */ 2002 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 2003 { 2004 int start = skb_headlen(skb); 2005 struct sk_buff *frag_iter; 2006 int i, copy; 2007 2008 if (offset > (int)skb->len - len) 2009 goto fault; 2010 2011 /* Copy header. */ 2012 if ((copy = start - offset) > 0) { 2013 if (copy > len) 2014 copy = len; 2015 skb_copy_from_linear_data_offset(skb, offset, to, copy); 2016 if ((len -= copy) == 0) 2017 return 0; 2018 offset += copy; 2019 to += copy; 2020 } 2021 2022 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2023 int end; 2024 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2025 2026 WARN_ON(start > offset + len); 2027 2028 end = start + skb_frag_size(f); 2029 if ((copy = end - offset) > 0) { 2030 u32 p_off, p_len, copied; 2031 struct page *p; 2032 u8 *vaddr; 2033 2034 if (copy > len) 2035 copy = len; 2036 2037 skb_frag_foreach_page(f, 2038 f->page_offset + offset - start, 2039 copy, p, p_off, p_len, copied) { 2040 vaddr = kmap_atomic(p); 2041 memcpy(to + copied, vaddr + p_off, p_len); 2042 kunmap_atomic(vaddr); 2043 } 2044 2045 if ((len -= copy) == 0) 2046 return 0; 2047 offset += copy; 2048 to += copy; 2049 } 2050 start = end; 2051 } 2052 2053 skb_walk_frags(skb, frag_iter) { 2054 int end; 2055 2056 WARN_ON(start > offset + len); 2057 2058 end = start + frag_iter->len; 2059 if ((copy = end - offset) > 0) { 2060 if (copy > len) 2061 copy = len; 2062 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 2063 goto fault; 2064 if ((len -= copy) == 0) 2065 return 0; 2066 offset += copy; 2067 to += copy; 2068 } 2069 start = end; 2070 } 2071 2072 if (!len) 2073 return 0; 2074 2075 fault: 2076 return -EFAULT; 2077 } 2078 EXPORT_SYMBOL(skb_copy_bits); 2079 2080 /* 2081 * Callback from splice_to_pipe(), if we need to release some pages 2082 * at the end of the spd in case we error'ed out in filling the pipe. 2083 */ 2084 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 2085 { 2086 put_page(spd->pages[i]); 2087 } 2088 2089 static struct page *linear_to_page(struct page *page, unsigned int *len, 2090 unsigned int *offset, 2091 struct sock *sk) 2092 { 2093 struct page_frag *pfrag = sk_page_frag(sk); 2094 2095 if (!sk_page_frag_refill(sk, pfrag)) 2096 return NULL; 2097 2098 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 2099 2100 memcpy(page_address(pfrag->page) + pfrag->offset, 2101 page_address(page) + *offset, *len); 2102 *offset = pfrag->offset; 2103 pfrag->offset += *len; 2104 2105 return pfrag->page; 2106 } 2107 2108 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 2109 struct page *page, 2110 unsigned int offset) 2111 { 2112 return spd->nr_pages && 2113 spd->pages[spd->nr_pages - 1] == page && 2114 (spd->partial[spd->nr_pages - 1].offset + 2115 spd->partial[spd->nr_pages - 1].len == offset); 2116 } 2117 2118 /* 2119 * Fill page/offset/length into spd, if it can hold more pages. 2120 */ 2121 static bool spd_fill_page(struct splice_pipe_desc *spd, 2122 struct pipe_inode_info *pipe, struct page *page, 2123 unsigned int *len, unsigned int offset, 2124 bool linear, 2125 struct sock *sk) 2126 { 2127 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 2128 return true; 2129 2130 if (linear) { 2131 page = linear_to_page(page, len, &offset, sk); 2132 if (!page) 2133 return true; 2134 } 2135 if (spd_can_coalesce(spd, page, offset)) { 2136 spd->partial[spd->nr_pages - 1].len += *len; 2137 return false; 2138 } 2139 get_page(page); 2140 spd->pages[spd->nr_pages] = page; 2141 spd->partial[spd->nr_pages].len = *len; 2142 spd->partial[spd->nr_pages].offset = offset; 2143 spd->nr_pages++; 2144 2145 return false; 2146 } 2147 2148 static bool __splice_segment(struct page *page, unsigned int poff, 2149 unsigned int plen, unsigned int *off, 2150 unsigned int *len, 2151 struct splice_pipe_desc *spd, bool linear, 2152 struct sock *sk, 2153 struct pipe_inode_info *pipe) 2154 { 2155 if (!*len) 2156 return true; 2157 2158 /* skip this segment if already processed */ 2159 if (*off >= plen) { 2160 *off -= plen; 2161 return false; 2162 } 2163 2164 /* ignore any bits we already processed */ 2165 poff += *off; 2166 plen -= *off; 2167 *off = 0; 2168 2169 do { 2170 unsigned int flen = min(*len, plen); 2171 2172 if (spd_fill_page(spd, pipe, page, &flen, poff, 2173 linear, sk)) 2174 return true; 2175 poff += flen; 2176 plen -= flen; 2177 *len -= flen; 2178 } while (*len && plen); 2179 2180 return false; 2181 } 2182 2183 /* 2184 * Map linear and fragment data from the skb to spd. It reports true if the 2185 * pipe is full or if we already spliced the requested length. 2186 */ 2187 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 2188 unsigned int *offset, unsigned int *len, 2189 struct splice_pipe_desc *spd, struct sock *sk) 2190 { 2191 int seg; 2192 struct sk_buff *iter; 2193 2194 /* map the linear part : 2195 * If skb->head_frag is set, this 'linear' part is backed by a 2196 * fragment, and if the head is not shared with any clones then 2197 * we can avoid a copy since we own the head portion of this page. 2198 */ 2199 if (__splice_segment(virt_to_page(skb->data), 2200 (unsigned long) skb->data & (PAGE_SIZE - 1), 2201 skb_headlen(skb), 2202 offset, len, spd, 2203 skb_head_is_locked(skb), 2204 sk, pipe)) 2205 return true; 2206 2207 /* 2208 * then map the fragments 2209 */ 2210 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 2211 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 2212 2213 if (__splice_segment(skb_frag_page(f), 2214 f->page_offset, skb_frag_size(f), 2215 offset, len, spd, false, sk, pipe)) 2216 return true; 2217 } 2218 2219 skb_walk_frags(skb, iter) { 2220 if (*offset >= iter->len) { 2221 *offset -= iter->len; 2222 continue; 2223 } 2224 /* __skb_splice_bits() only fails if the output has no room 2225 * left, so no point in going over the frag_list for the error 2226 * case. 2227 */ 2228 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 2229 return true; 2230 } 2231 2232 return false; 2233 } 2234 2235 /* 2236 * Map data from the skb to a pipe. Should handle both the linear part, 2237 * the fragments, and the frag list. 2238 */ 2239 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 2240 struct pipe_inode_info *pipe, unsigned int tlen, 2241 unsigned int flags) 2242 { 2243 struct partial_page partial[MAX_SKB_FRAGS]; 2244 struct page *pages[MAX_SKB_FRAGS]; 2245 struct splice_pipe_desc spd = { 2246 .pages = pages, 2247 .partial = partial, 2248 .nr_pages_max = MAX_SKB_FRAGS, 2249 .ops = &nosteal_pipe_buf_ops, 2250 .spd_release = sock_spd_release, 2251 }; 2252 int ret = 0; 2253 2254 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 2255 2256 if (spd.nr_pages) 2257 ret = splice_to_pipe(pipe, &spd); 2258 2259 return ret; 2260 } 2261 EXPORT_SYMBOL_GPL(skb_splice_bits); 2262 2263 /* Send skb data on a socket. Socket must be locked. */ 2264 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 2265 int len) 2266 { 2267 unsigned int orig_len = len; 2268 struct sk_buff *head = skb; 2269 unsigned short fragidx; 2270 int slen, ret; 2271 2272 do_frag_list: 2273 2274 /* Deal with head data */ 2275 while (offset < skb_headlen(skb) && len) { 2276 struct kvec kv; 2277 struct msghdr msg; 2278 2279 slen = min_t(int, len, skb_headlen(skb) - offset); 2280 kv.iov_base = skb->data + offset; 2281 kv.iov_len = slen; 2282 memset(&msg, 0, sizeof(msg)); 2283 2284 ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen); 2285 if (ret <= 0) 2286 goto error; 2287 2288 offset += ret; 2289 len -= ret; 2290 } 2291 2292 /* All the data was skb head? */ 2293 if (!len) 2294 goto out; 2295 2296 /* Make offset relative to start of frags */ 2297 offset -= skb_headlen(skb); 2298 2299 /* Find where we are in frag list */ 2300 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2301 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2302 2303 if (offset < frag->size) 2304 break; 2305 2306 offset -= frag->size; 2307 } 2308 2309 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2310 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2311 2312 slen = min_t(size_t, len, frag->size - offset); 2313 2314 while (slen) { 2315 ret = kernel_sendpage_locked(sk, frag->page.p, 2316 frag->page_offset + offset, 2317 slen, MSG_DONTWAIT); 2318 if (ret <= 0) 2319 goto error; 2320 2321 len -= ret; 2322 offset += ret; 2323 slen -= ret; 2324 } 2325 2326 offset = 0; 2327 } 2328 2329 if (len) { 2330 /* Process any frag lists */ 2331 2332 if (skb == head) { 2333 if (skb_has_frag_list(skb)) { 2334 skb = skb_shinfo(skb)->frag_list; 2335 goto do_frag_list; 2336 } 2337 } else if (skb->next) { 2338 skb = skb->next; 2339 goto do_frag_list; 2340 } 2341 } 2342 2343 out: 2344 return orig_len - len; 2345 2346 error: 2347 return orig_len == len ? ret : orig_len - len; 2348 } 2349 EXPORT_SYMBOL_GPL(skb_send_sock_locked); 2350 2351 /* Send skb data on a socket. */ 2352 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) 2353 { 2354 int ret = 0; 2355 2356 lock_sock(sk); 2357 ret = skb_send_sock_locked(sk, skb, offset, len); 2358 release_sock(sk); 2359 2360 return ret; 2361 } 2362 EXPORT_SYMBOL_GPL(skb_send_sock); 2363 2364 /** 2365 * skb_store_bits - store bits from kernel buffer to skb 2366 * @skb: destination buffer 2367 * @offset: offset in destination 2368 * @from: source buffer 2369 * @len: number of bytes to copy 2370 * 2371 * Copy the specified number of bytes from the source buffer to the 2372 * destination skb. This function handles all the messy bits of 2373 * traversing fragment lists and such. 2374 */ 2375 2376 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 2377 { 2378 int start = skb_headlen(skb); 2379 struct sk_buff *frag_iter; 2380 int i, copy; 2381 2382 if (offset > (int)skb->len - len) 2383 goto fault; 2384 2385 if ((copy = start - offset) > 0) { 2386 if (copy > len) 2387 copy = len; 2388 skb_copy_to_linear_data_offset(skb, offset, from, copy); 2389 if ((len -= copy) == 0) 2390 return 0; 2391 offset += copy; 2392 from += copy; 2393 } 2394 2395 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2396 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2397 int end; 2398 2399 WARN_ON(start > offset + len); 2400 2401 end = start + skb_frag_size(frag); 2402 if ((copy = end - offset) > 0) { 2403 u32 p_off, p_len, copied; 2404 struct page *p; 2405 u8 *vaddr; 2406 2407 if (copy > len) 2408 copy = len; 2409 2410 skb_frag_foreach_page(frag, 2411 frag->page_offset + offset - start, 2412 copy, p, p_off, p_len, copied) { 2413 vaddr = kmap_atomic(p); 2414 memcpy(vaddr + p_off, from + copied, p_len); 2415 kunmap_atomic(vaddr); 2416 } 2417 2418 if ((len -= copy) == 0) 2419 return 0; 2420 offset += copy; 2421 from += copy; 2422 } 2423 start = end; 2424 } 2425 2426 skb_walk_frags(skb, frag_iter) { 2427 int end; 2428 2429 WARN_ON(start > offset + len); 2430 2431 end = start + frag_iter->len; 2432 if ((copy = end - offset) > 0) { 2433 if (copy > len) 2434 copy = len; 2435 if (skb_store_bits(frag_iter, offset - start, 2436 from, copy)) 2437 goto fault; 2438 if ((len -= copy) == 0) 2439 return 0; 2440 offset += copy; 2441 from += copy; 2442 } 2443 start = end; 2444 } 2445 if (!len) 2446 return 0; 2447 2448 fault: 2449 return -EFAULT; 2450 } 2451 EXPORT_SYMBOL(skb_store_bits); 2452 2453 /* Checksum skb data. */ 2454 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 2455 __wsum csum, const struct skb_checksum_ops *ops) 2456 { 2457 int start = skb_headlen(skb); 2458 int i, copy = start - offset; 2459 struct sk_buff *frag_iter; 2460 int pos = 0; 2461 2462 /* Checksum header. */ 2463 if (copy > 0) { 2464 if (copy > len) 2465 copy = len; 2466 csum = ops->update(skb->data + offset, copy, csum); 2467 if ((len -= copy) == 0) 2468 return csum; 2469 offset += copy; 2470 pos = copy; 2471 } 2472 2473 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2474 int end; 2475 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2476 2477 WARN_ON(start > offset + len); 2478 2479 end = start + skb_frag_size(frag); 2480 if ((copy = end - offset) > 0) { 2481 u32 p_off, p_len, copied; 2482 struct page *p; 2483 __wsum csum2; 2484 u8 *vaddr; 2485 2486 if (copy > len) 2487 copy = len; 2488 2489 skb_frag_foreach_page(frag, 2490 frag->page_offset + offset - start, 2491 copy, p, p_off, p_len, copied) { 2492 vaddr = kmap_atomic(p); 2493 csum2 = ops->update(vaddr + p_off, p_len, 0); 2494 kunmap_atomic(vaddr); 2495 csum = ops->combine(csum, csum2, pos, p_len); 2496 pos += p_len; 2497 } 2498 2499 if (!(len -= copy)) 2500 return csum; 2501 offset += copy; 2502 } 2503 start = end; 2504 } 2505 2506 skb_walk_frags(skb, frag_iter) { 2507 int end; 2508 2509 WARN_ON(start > offset + len); 2510 2511 end = start + frag_iter->len; 2512 if ((copy = end - offset) > 0) { 2513 __wsum csum2; 2514 if (copy > len) 2515 copy = len; 2516 csum2 = __skb_checksum(frag_iter, offset - start, 2517 copy, 0, ops); 2518 csum = ops->combine(csum, csum2, pos, copy); 2519 if ((len -= copy) == 0) 2520 return csum; 2521 offset += copy; 2522 pos += copy; 2523 } 2524 start = end; 2525 } 2526 BUG_ON(len); 2527 2528 return csum; 2529 } 2530 EXPORT_SYMBOL(__skb_checksum); 2531 2532 __wsum skb_checksum(const struct sk_buff *skb, int offset, 2533 int len, __wsum csum) 2534 { 2535 const struct skb_checksum_ops ops = { 2536 .update = csum_partial_ext, 2537 .combine = csum_block_add_ext, 2538 }; 2539 2540 return __skb_checksum(skb, offset, len, csum, &ops); 2541 } 2542 EXPORT_SYMBOL(skb_checksum); 2543 2544 /* Both of above in one bottle. */ 2545 2546 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 2547 u8 *to, int len, __wsum csum) 2548 { 2549 int start = skb_headlen(skb); 2550 int i, copy = start - offset; 2551 struct sk_buff *frag_iter; 2552 int pos = 0; 2553 2554 /* Copy header. */ 2555 if (copy > 0) { 2556 if (copy > len) 2557 copy = len; 2558 csum = csum_partial_copy_nocheck(skb->data + offset, to, 2559 copy, csum); 2560 if ((len -= copy) == 0) 2561 return csum; 2562 offset += copy; 2563 to += copy; 2564 pos = copy; 2565 } 2566 2567 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2568 int end; 2569 2570 WARN_ON(start > offset + len); 2571 2572 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2573 if ((copy = end - offset) > 0) { 2574 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2575 u32 p_off, p_len, copied; 2576 struct page *p; 2577 __wsum csum2; 2578 u8 *vaddr; 2579 2580 if (copy > len) 2581 copy = len; 2582 2583 skb_frag_foreach_page(frag, 2584 frag->page_offset + offset - start, 2585 copy, p, p_off, p_len, copied) { 2586 vaddr = kmap_atomic(p); 2587 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 2588 to + copied, 2589 p_len, 0); 2590 kunmap_atomic(vaddr); 2591 csum = csum_block_add(csum, csum2, pos); 2592 pos += p_len; 2593 } 2594 2595 if (!(len -= copy)) 2596 return csum; 2597 offset += copy; 2598 to += copy; 2599 } 2600 start = end; 2601 } 2602 2603 skb_walk_frags(skb, frag_iter) { 2604 __wsum csum2; 2605 int end; 2606 2607 WARN_ON(start > offset + len); 2608 2609 end = start + frag_iter->len; 2610 if ((copy = end - offset) > 0) { 2611 if (copy > len) 2612 copy = len; 2613 csum2 = skb_copy_and_csum_bits(frag_iter, 2614 offset - start, 2615 to, copy, 0); 2616 csum = csum_block_add(csum, csum2, pos); 2617 if ((len -= copy) == 0) 2618 return csum; 2619 offset += copy; 2620 to += copy; 2621 pos += copy; 2622 } 2623 start = end; 2624 } 2625 BUG_ON(len); 2626 return csum; 2627 } 2628 EXPORT_SYMBOL(skb_copy_and_csum_bits); 2629 2630 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum) 2631 { 2632 net_warn_ratelimited( 2633 "%s: attempt to compute crc32c without libcrc32c.ko\n", 2634 __func__); 2635 return 0; 2636 } 2637 2638 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2, 2639 int offset, int len) 2640 { 2641 net_warn_ratelimited( 2642 "%s: attempt to compute crc32c without libcrc32c.ko\n", 2643 __func__); 2644 return 0; 2645 } 2646 2647 static const struct skb_checksum_ops default_crc32c_ops = { 2648 .update = warn_crc32c_csum_update, 2649 .combine = warn_crc32c_csum_combine, 2650 }; 2651 2652 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly = 2653 &default_crc32c_ops; 2654 EXPORT_SYMBOL(crc32c_csum_stub); 2655 2656 /** 2657 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 2658 * @from: source buffer 2659 * 2660 * Calculates the amount of linear headroom needed in the 'to' skb passed 2661 * into skb_zerocopy(). 2662 */ 2663 unsigned int 2664 skb_zerocopy_headlen(const struct sk_buff *from) 2665 { 2666 unsigned int hlen = 0; 2667 2668 if (!from->head_frag || 2669 skb_headlen(from) < L1_CACHE_BYTES || 2670 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) 2671 hlen = skb_headlen(from); 2672 2673 if (skb_has_frag_list(from)) 2674 hlen = from->len; 2675 2676 return hlen; 2677 } 2678 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 2679 2680 /** 2681 * skb_zerocopy - Zero copy skb to skb 2682 * @to: destination buffer 2683 * @from: source buffer 2684 * @len: number of bytes to copy from source buffer 2685 * @hlen: size of linear headroom in destination buffer 2686 * 2687 * Copies up to `len` bytes from `from` to `to` by creating references 2688 * to the frags in the source buffer. 2689 * 2690 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 2691 * headroom in the `to` buffer. 2692 * 2693 * Return value: 2694 * 0: everything is OK 2695 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 2696 * -EFAULT: skb_copy_bits() found some problem with skb geometry 2697 */ 2698 int 2699 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 2700 { 2701 int i, j = 0; 2702 int plen = 0; /* length of skb->head fragment */ 2703 int ret; 2704 struct page *page; 2705 unsigned int offset; 2706 2707 BUG_ON(!from->head_frag && !hlen); 2708 2709 /* dont bother with small payloads */ 2710 if (len <= skb_tailroom(to)) 2711 return skb_copy_bits(from, 0, skb_put(to, len), len); 2712 2713 if (hlen) { 2714 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 2715 if (unlikely(ret)) 2716 return ret; 2717 len -= hlen; 2718 } else { 2719 plen = min_t(int, skb_headlen(from), len); 2720 if (plen) { 2721 page = virt_to_head_page(from->head); 2722 offset = from->data - (unsigned char *)page_address(page); 2723 __skb_fill_page_desc(to, 0, page, offset, plen); 2724 get_page(page); 2725 j = 1; 2726 len -= plen; 2727 } 2728 } 2729 2730 to->truesize += len + plen; 2731 to->len += len + plen; 2732 to->data_len += len + plen; 2733 2734 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 2735 skb_tx_error(from); 2736 return -ENOMEM; 2737 } 2738 skb_zerocopy_clone(to, from, GFP_ATOMIC); 2739 2740 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 2741 if (!len) 2742 break; 2743 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 2744 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len); 2745 len -= skb_shinfo(to)->frags[j].size; 2746 skb_frag_ref(to, j); 2747 j++; 2748 } 2749 skb_shinfo(to)->nr_frags = j; 2750 2751 return 0; 2752 } 2753 EXPORT_SYMBOL_GPL(skb_zerocopy); 2754 2755 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 2756 { 2757 __wsum csum; 2758 long csstart; 2759 2760 if (skb->ip_summed == CHECKSUM_PARTIAL) 2761 csstart = skb_checksum_start_offset(skb); 2762 else 2763 csstart = skb_headlen(skb); 2764 2765 BUG_ON(csstart > skb_headlen(skb)); 2766 2767 skb_copy_from_linear_data(skb, to, csstart); 2768 2769 csum = 0; 2770 if (csstart != skb->len) 2771 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 2772 skb->len - csstart, 0); 2773 2774 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2775 long csstuff = csstart + skb->csum_offset; 2776 2777 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 2778 } 2779 } 2780 EXPORT_SYMBOL(skb_copy_and_csum_dev); 2781 2782 /** 2783 * skb_dequeue - remove from the head of the queue 2784 * @list: list to dequeue from 2785 * 2786 * Remove the head of the list. The list lock is taken so the function 2787 * may be used safely with other locking list functions. The head item is 2788 * returned or %NULL if the list is empty. 2789 */ 2790 2791 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 2792 { 2793 unsigned long flags; 2794 struct sk_buff *result; 2795 2796 spin_lock_irqsave(&list->lock, flags); 2797 result = __skb_dequeue(list); 2798 spin_unlock_irqrestore(&list->lock, flags); 2799 return result; 2800 } 2801 EXPORT_SYMBOL(skb_dequeue); 2802 2803 /** 2804 * skb_dequeue_tail - remove from the tail of the queue 2805 * @list: list to dequeue from 2806 * 2807 * Remove the tail of the list. The list lock is taken so the function 2808 * may be used safely with other locking list functions. The tail item is 2809 * returned or %NULL if the list is empty. 2810 */ 2811 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 2812 { 2813 unsigned long flags; 2814 struct sk_buff *result; 2815 2816 spin_lock_irqsave(&list->lock, flags); 2817 result = __skb_dequeue_tail(list); 2818 spin_unlock_irqrestore(&list->lock, flags); 2819 return result; 2820 } 2821 EXPORT_SYMBOL(skb_dequeue_tail); 2822 2823 /** 2824 * skb_queue_purge - empty a list 2825 * @list: list to empty 2826 * 2827 * Delete all buffers on an &sk_buff list. Each buffer is removed from 2828 * the list and one reference dropped. This function takes the list 2829 * lock and is atomic with respect to other list locking functions. 2830 */ 2831 void skb_queue_purge(struct sk_buff_head *list) 2832 { 2833 struct sk_buff *skb; 2834 while ((skb = skb_dequeue(list)) != NULL) 2835 kfree_skb(skb); 2836 } 2837 EXPORT_SYMBOL(skb_queue_purge); 2838 2839 /** 2840 * skb_rbtree_purge - empty a skb rbtree 2841 * @root: root of the rbtree to empty 2842 * 2843 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 2844 * the list and one reference dropped. This function does not take 2845 * any lock. Synchronization should be handled by the caller (e.g., TCP 2846 * out-of-order queue is protected by the socket lock). 2847 */ 2848 void skb_rbtree_purge(struct rb_root *root) 2849 { 2850 struct rb_node *p = rb_first(root); 2851 2852 while (p) { 2853 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 2854 2855 p = rb_next(p); 2856 rb_erase(&skb->rbnode, root); 2857 kfree_skb(skb); 2858 } 2859 } 2860 2861 /** 2862 * skb_queue_head - queue a buffer at the list head 2863 * @list: list to use 2864 * @newsk: buffer to queue 2865 * 2866 * Queue a buffer at the start of the list. This function takes the 2867 * list lock and can be used safely with other locking &sk_buff functions 2868 * safely. 2869 * 2870 * A buffer cannot be placed on two lists at the same time. 2871 */ 2872 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 2873 { 2874 unsigned long flags; 2875 2876 spin_lock_irqsave(&list->lock, flags); 2877 __skb_queue_head(list, newsk); 2878 spin_unlock_irqrestore(&list->lock, flags); 2879 } 2880 EXPORT_SYMBOL(skb_queue_head); 2881 2882 /** 2883 * skb_queue_tail - queue a buffer at the list tail 2884 * @list: list to use 2885 * @newsk: buffer to queue 2886 * 2887 * Queue a buffer at the tail of the list. This function takes the 2888 * list lock and can be used safely with other locking &sk_buff functions 2889 * safely. 2890 * 2891 * A buffer cannot be placed on two lists at the same time. 2892 */ 2893 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 2894 { 2895 unsigned long flags; 2896 2897 spin_lock_irqsave(&list->lock, flags); 2898 __skb_queue_tail(list, newsk); 2899 spin_unlock_irqrestore(&list->lock, flags); 2900 } 2901 EXPORT_SYMBOL(skb_queue_tail); 2902 2903 /** 2904 * skb_unlink - remove a buffer from a list 2905 * @skb: buffer to remove 2906 * @list: list to use 2907 * 2908 * Remove a packet from a list. The list locks are taken and this 2909 * function is atomic with respect to other list locked calls 2910 * 2911 * You must know what list the SKB is on. 2912 */ 2913 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 2914 { 2915 unsigned long flags; 2916 2917 spin_lock_irqsave(&list->lock, flags); 2918 __skb_unlink(skb, list); 2919 spin_unlock_irqrestore(&list->lock, flags); 2920 } 2921 EXPORT_SYMBOL(skb_unlink); 2922 2923 /** 2924 * skb_append - append a buffer 2925 * @old: buffer to insert after 2926 * @newsk: buffer to insert 2927 * @list: list to use 2928 * 2929 * Place a packet after a given packet in a list. The list locks are taken 2930 * and this function is atomic with respect to other list locked calls. 2931 * A buffer cannot be placed on two lists at the same time. 2932 */ 2933 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 2934 { 2935 unsigned long flags; 2936 2937 spin_lock_irqsave(&list->lock, flags); 2938 __skb_queue_after(list, old, newsk); 2939 spin_unlock_irqrestore(&list->lock, flags); 2940 } 2941 EXPORT_SYMBOL(skb_append); 2942 2943 /** 2944 * skb_insert - insert a buffer 2945 * @old: buffer to insert before 2946 * @newsk: buffer to insert 2947 * @list: list to use 2948 * 2949 * Place a packet before a given packet in a list. The list locks are 2950 * taken and this function is atomic with respect to other list locked 2951 * calls. 2952 * 2953 * A buffer cannot be placed on two lists at the same time. 2954 */ 2955 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 2956 { 2957 unsigned long flags; 2958 2959 spin_lock_irqsave(&list->lock, flags); 2960 __skb_insert(newsk, old->prev, old, list); 2961 spin_unlock_irqrestore(&list->lock, flags); 2962 } 2963 EXPORT_SYMBOL(skb_insert); 2964 2965 static inline void skb_split_inside_header(struct sk_buff *skb, 2966 struct sk_buff* skb1, 2967 const u32 len, const int pos) 2968 { 2969 int i; 2970 2971 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 2972 pos - len); 2973 /* And move data appendix as is. */ 2974 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2975 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 2976 2977 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 2978 skb_shinfo(skb)->nr_frags = 0; 2979 skb1->data_len = skb->data_len; 2980 skb1->len += skb1->data_len; 2981 skb->data_len = 0; 2982 skb->len = len; 2983 skb_set_tail_pointer(skb, len); 2984 } 2985 2986 static inline void skb_split_no_header(struct sk_buff *skb, 2987 struct sk_buff* skb1, 2988 const u32 len, int pos) 2989 { 2990 int i, k = 0; 2991 const int nfrags = skb_shinfo(skb)->nr_frags; 2992 2993 skb_shinfo(skb)->nr_frags = 0; 2994 skb1->len = skb1->data_len = skb->len - len; 2995 skb->len = len; 2996 skb->data_len = len - pos; 2997 2998 for (i = 0; i < nfrags; i++) { 2999 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 3000 3001 if (pos + size > len) { 3002 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 3003 3004 if (pos < len) { 3005 /* Split frag. 3006 * We have two variants in this case: 3007 * 1. Move all the frag to the second 3008 * part, if it is possible. F.e. 3009 * this approach is mandatory for TUX, 3010 * where splitting is expensive. 3011 * 2. Split is accurately. We make this. 3012 */ 3013 skb_frag_ref(skb, i); 3014 skb_shinfo(skb1)->frags[0].page_offset += len - pos; 3015 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 3016 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 3017 skb_shinfo(skb)->nr_frags++; 3018 } 3019 k++; 3020 } else 3021 skb_shinfo(skb)->nr_frags++; 3022 pos += size; 3023 } 3024 skb_shinfo(skb1)->nr_frags = k; 3025 } 3026 3027 /** 3028 * skb_split - Split fragmented skb to two parts at length len. 3029 * @skb: the buffer to split 3030 * @skb1: the buffer to receive the second part 3031 * @len: new length for skb 3032 */ 3033 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 3034 { 3035 int pos = skb_headlen(skb); 3036 3037 skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags & 3038 SKBTX_SHARED_FRAG; 3039 skb_zerocopy_clone(skb1, skb, 0); 3040 if (len < pos) /* Split line is inside header. */ 3041 skb_split_inside_header(skb, skb1, len, pos); 3042 else /* Second chunk has no header, nothing to copy. */ 3043 skb_split_no_header(skb, skb1, len, pos); 3044 } 3045 EXPORT_SYMBOL(skb_split); 3046 3047 /* Shifting from/to a cloned skb is a no-go. 3048 * 3049 * Caller cannot keep skb_shinfo related pointers past calling here! 3050 */ 3051 static int skb_prepare_for_shift(struct sk_buff *skb) 3052 { 3053 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 3054 } 3055 3056 /** 3057 * skb_shift - Shifts paged data partially from skb to another 3058 * @tgt: buffer into which tail data gets added 3059 * @skb: buffer from which the paged data comes from 3060 * @shiftlen: shift up to this many bytes 3061 * 3062 * Attempts to shift up to shiftlen worth of bytes, which may be less than 3063 * the length of the skb, from skb to tgt. Returns number bytes shifted. 3064 * It's up to caller to free skb if everything was shifted. 3065 * 3066 * If @tgt runs out of frags, the whole operation is aborted. 3067 * 3068 * Skb cannot include anything else but paged data while tgt is allowed 3069 * to have non-paged data as well. 3070 * 3071 * TODO: full sized shift could be optimized but that would need 3072 * specialized skb free'er to handle frags without up-to-date nr_frags. 3073 */ 3074 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 3075 { 3076 int from, to, merge, todo; 3077 struct skb_frag_struct *fragfrom, *fragto; 3078 3079 BUG_ON(shiftlen > skb->len); 3080 3081 if (skb_headlen(skb)) 3082 return 0; 3083 if (skb_zcopy(tgt) || skb_zcopy(skb)) 3084 return 0; 3085 3086 todo = shiftlen; 3087 from = 0; 3088 to = skb_shinfo(tgt)->nr_frags; 3089 fragfrom = &skb_shinfo(skb)->frags[from]; 3090 3091 /* Actual merge is delayed until the point when we know we can 3092 * commit all, so that we don't have to undo partial changes 3093 */ 3094 if (!to || 3095 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 3096 fragfrom->page_offset)) { 3097 merge = -1; 3098 } else { 3099 merge = to - 1; 3100 3101 todo -= skb_frag_size(fragfrom); 3102 if (todo < 0) { 3103 if (skb_prepare_for_shift(skb) || 3104 skb_prepare_for_shift(tgt)) 3105 return 0; 3106 3107 /* All previous frag pointers might be stale! */ 3108 fragfrom = &skb_shinfo(skb)->frags[from]; 3109 fragto = &skb_shinfo(tgt)->frags[merge]; 3110 3111 skb_frag_size_add(fragto, shiftlen); 3112 skb_frag_size_sub(fragfrom, shiftlen); 3113 fragfrom->page_offset += shiftlen; 3114 3115 goto onlymerged; 3116 } 3117 3118 from++; 3119 } 3120 3121 /* Skip full, not-fitting skb to avoid expensive operations */ 3122 if ((shiftlen == skb->len) && 3123 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 3124 return 0; 3125 3126 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 3127 return 0; 3128 3129 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 3130 if (to == MAX_SKB_FRAGS) 3131 return 0; 3132 3133 fragfrom = &skb_shinfo(skb)->frags[from]; 3134 fragto = &skb_shinfo(tgt)->frags[to]; 3135 3136 if (todo >= skb_frag_size(fragfrom)) { 3137 *fragto = *fragfrom; 3138 todo -= skb_frag_size(fragfrom); 3139 from++; 3140 to++; 3141 3142 } else { 3143 __skb_frag_ref(fragfrom); 3144 fragto->page = fragfrom->page; 3145 fragto->page_offset = fragfrom->page_offset; 3146 skb_frag_size_set(fragto, todo); 3147 3148 fragfrom->page_offset += todo; 3149 skb_frag_size_sub(fragfrom, todo); 3150 todo = 0; 3151 3152 to++; 3153 break; 3154 } 3155 } 3156 3157 /* Ready to "commit" this state change to tgt */ 3158 skb_shinfo(tgt)->nr_frags = to; 3159 3160 if (merge >= 0) { 3161 fragfrom = &skb_shinfo(skb)->frags[0]; 3162 fragto = &skb_shinfo(tgt)->frags[merge]; 3163 3164 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 3165 __skb_frag_unref(fragfrom); 3166 } 3167 3168 /* Reposition in the original skb */ 3169 to = 0; 3170 while (from < skb_shinfo(skb)->nr_frags) 3171 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 3172 skb_shinfo(skb)->nr_frags = to; 3173 3174 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 3175 3176 onlymerged: 3177 /* Most likely the tgt won't ever need its checksum anymore, skb on 3178 * the other hand might need it if it needs to be resent 3179 */ 3180 tgt->ip_summed = CHECKSUM_PARTIAL; 3181 skb->ip_summed = CHECKSUM_PARTIAL; 3182 3183 /* Yak, is it really working this way? Some helper please? */ 3184 skb->len -= shiftlen; 3185 skb->data_len -= shiftlen; 3186 skb->truesize -= shiftlen; 3187 tgt->len += shiftlen; 3188 tgt->data_len += shiftlen; 3189 tgt->truesize += shiftlen; 3190 3191 return shiftlen; 3192 } 3193 3194 /** 3195 * skb_prepare_seq_read - Prepare a sequential read of skb data 3196 * @skb: the buffer to read 3197 * @from: lower offset of data to be read 3198 * @to: upper offset of data to be read 3199 * @st: state variable 3200 * 3201 * Initializes the specified state variable. Must be called before 3202 * invoking skb_seq_read() for the first time. 3203 */ 3204 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 3205 unsigned int to, struct skb_seq_state *st) 3206 { 3207 st->lower_offset = from; 3208 st->upper_offset = to; 3209 st->root_skb = st->cur_skb = skb; 3210 st->frag_idx = st->stepped_offset = 0; 3211 st->frag_data = NULL; 3212 } 3213 EXPORT_SYMBOL(skb_prepare_seq_read); 3214 3215 /** 3216 * skb_seq_read - Sequentially read skb data 3217 * @consumed: number of bytes consumed by the caller so far 3218 * @data: destination pointer for data to be returned 3219 * @st: state variable 3220 * 3221 * Reads a block of skb data at @consumed relative to the 3222 * lower offset specified to skb_prepare_seq_read(). Assigns 3223 * the head of the data block to @data and returns the length 3224 * of the block or 0 if the end of the skb data or the upper 3225 * offset has been reached. 3226 * 3227 * The caller is not required to consume all of the data 3228 * returned, i.e. @consumed is typically set to the number 3229 * of bytes already consumed and the next call to 3230 * skb_seq_read() will return the remaining part of the block. 3231 * 3232 * Note 1: The size of each block of data returned can be arbitrary, 3233 * this limitation is the cost for zerocopy sequential 3234 * reads of potentially non linear data. 3235 * 3236 * Note 2: Fragment lists within fragments are not implemented 3237 * at the moment, state->root_skb could be replaced with 3238 * a stack for this purpose. 3239 */ 3240 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 3241 struct skb_seq_state *st) 3242 { 3243 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 3244 skb_frag_t *frag; 3245 3246 if (unlikely(abs_offset >= st->upper_offset)) { 3247 if (st->frag_data) { 3248 kunmap_atomic(st->frag_data); 3249 st->frag_data = NULL; 3250 } 3251 return 0; 3252 } 3253 3254 next_skb: 3255 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 3256 3257 if (abs_offset < block_limit && !st->frag_data) { 3258 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 3259 return block_limit - abs_offset; 3260 } 3261 3262 if (st->frag_idx == 0 && !st->frag_data) 3263 st->stepped_offset += skb_headlen(st->cur_skb); 3264 3265 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 3266 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 3267 block_limit = skb_frag_size(frag) + st->stepped_offset; 3268 3269 if (abs_offset < block_limit) { 3270 if (!st->frag_data) 3271 st->frag_data = kmap_atomic(skb_frag_page(frag)); 3272 3273 *data = (u8 *) st->frag_data + frag->page_offset + 3274 (abs_offset - st->stepped_offset); 3275 3276 return block_limit - abs_offset; 3277 } 3278 3279 if (st->frag_data) { 3280 kunmap_atomic(st->frag_data); 3281 st->frag_data = NULL; 3282 } 3283 3284 st->frag_idx++; 3285 st->stepped_offset += skb_frag_size(frag); 3286 } 3287 3288 if (st->frag_data) { 3289 kunmap_atomic(st->frag_data); 3290 st->frag_data = NULL; 3291 } 3292 3293 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 3294 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 3295 st->frag_idx = 0; 3296 goto next_skb; 3297 } else if (st->cur_skb->next) { 3298 st->cur_skb = st->cur_skb->next; 3299 st->frag_idx = 0; 3300 goto next_skb; 3301 } 3302 3303 return 0; 3304 } 3305 EXPORT_SYMBOL(skb_seq_read); 3306 3307 /** 3308 * skb_abort_seq_read - Abort a sequential read of skb data 3309 * @st: state variable 3310 * 3311 * Must be called if skb_seq_read() was not called until it 3312 * returned 0. 3313 */ 3314 void skb_abort_seq_read(struct skb_seq_state *st) 3315 { 3316 if (st->frag_data) 3317 kunmap_atomic(st->frag_data); 3318 } 3319 EXPORT_SYMBOL(skb_abort_seq_read); 3320 3321 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 3322 3323 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 3324 struct ts_config *conf, 3325 struct ts_state *state) 3326 { 3327 return skb_seq_read(offset, text, TS_SKB_CB(state)); 3328 } 3329 3330 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 3331 { 3332 skb_abort_seq_read(TS_SKB_CB(state)); 3333 } 3334 3335 /** 3336 * skb_find_text - Find a text pattern in skb data 3337 * @skb: the buffer to look in 3338 * @from: search offset 3339 * @to: search limit 3340 * @config: textsearch configuration 3341 * 3342 * Finds a pattern in the skb data according to the specified 3343 * textsearch configuration. Use textsearch_next() to retrieve 3344 * subsequent occurrences of the pattern. Returns the offset 3345 * to the first occurrence or UINT_MAX if no match was found. 3346 */ 3347 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 3348 unsigned int to, struct ts_config *config) 3349 { 3350 struct ts_state state; 3351 unsigned int ret; 3352 3353 config->get_next_block = skb_ts_get_next_block; 3354 config->finish = skb_ts_finish; 3355 3356 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 3357 3358 ret = textsearch_find(config, &state); 3359 return (ret <= to - from ? ret : UINT_MAX); 3360 } 3361 EXPORT_SYMBOL(skb_find_text); 3362 3363 /** 3364 * skb_append_datato_frags - append the user data to a skb 3365 * @sk: sock structure 3366 * @skb: skb structure to be appended with user data. 3367 * @getfrag: call back function to be used for getting the user data 3368 * @from: pointer to user message iov 3369 * @length: length of the iov message 3370 * 3371 * Description: This procedure append the user data in the fragment part 3372 * of the skb if any page alloc fails user this procedure returns -ENOMEM 3373 */ 3374 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb, 3375 int (*getfrag)(void *from, char *to, int offset, 3376 int len, int odd, struct sk_buff *skb), 3377 void *from, int length) 3378 { 3379 int frg_cnt = skb_shinfo(skb)->nr_frags; 3380 int copy; 3381 int offset = 0; 3382 int ret; 3383 struct page_frag *pfrag = ¤t->task_frag; 3384 3385 do { 3386 /* Return error if we don't have space for new frag */ 3387 if (frg_cnt >= MAX_SKB_FRAGS) 3388 return -EMSGSIZE; 3389 3390 if (!sk_page_frag_refill(sk, pfrag)) 3391 return -ENOMEM; 3392 3393 /* copy the user data to page */ 3394 copy = min_t(int, length, pfrag->size - pfrag->offset); 3395 3396 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset, 3397 offset, copy, 0, skb); 3398 if (ret < 0) 3399 return -EFAULT; 3400 3401 /* copy was successful so update the size parameters */ 3402 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset, 3403 copy); 3404 frg_cnt++; 3405 pfrag->offset += copy; 3406 get_page(pfrag->page); 3407 3408 skb->truesize += copy; 3409 refcount_add(copy, &sk->sk_wmem_alloc); 3410 skb->len += copy; 3411 skb->data_len += copy; 3412 offset += copy; 3413 length -= copy; 3414 3415 } while (length > 0); 3416 3417 return 0; 3418 } 3419 EXPORT_SYMBOL(skb_append_datato_frags); 3420 3421 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 3422 int offset, size_t size) 3423 { 3424 int i = skb_shinfo(skb)->nr_frags; 3425 3426 if (skb_can_coalesce(skb, i, page, offset)) { 3427 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 3428 } else if (i < MAX_SKB_FRAGS) { 3429 get_page(page); 3430 skb_fill_page_desc(skb, i, page, offset, size); 3431 } else { 3432 return -EMSGSIZE; 3433 } 3434 3435 return 0; 3436 } 3437 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 3438 3439 /** 3440 * skb_pull_rcsum - pull skb and update receive checksum 3441 * @skb: buffer to update 3442 * @len: length of data pulled 3443 * 3444 * This function performs an skb_pull on the packet and updates 3445 * the CHECKSUM_COMPLETE checksum. It should be used on 3446 * receive path processing instead of skb_pull unless you know 3447 * that the checksum difference is zero (e.g., a valid IP header) 3448 * or you are setting ip_summed to CHECKSUM_NONE. 3449 */ 3450 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 3451 { 3452 unsigned char *data = skb->data; 3453 3454 BUG_ON(len > skb->len); 3455 __skb_pull(skb, len); 3456 skb_postpull_rcsum(skb, data, len); 3457 return skb->data; 3458 } 3459 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 3460 3461 /** 3462 * skb_segment - Perform protocol segmentation on skb. 3463 * @head_skb: buffer to segment 3464 * @features: features for the output path (see dev->features) 3465 * 3466 * This function performs segmentation on the given skb. It returns 3467 * a pointer to the first in a list of new skbs for the segments. 3468 * In case of error it returns ERR_PTR(err). 3469 */ 3470 struct sk_buff *skb_segment(struct sk_buff *head_skb, 3471 netdev_features_t features) 3472 { 3473 struct sk_buff *segs = NULL; 3474 struct sk_buff *tail = NULL; 3475 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 3476 skb_frag_t *frag = skb_shinfo(head_skb)->frags; 3477 unsigned int mss = skb_shinfo(head_skb)->gso_size; 3478 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 3479 struct sk_buff *frag_skb = head_skb; 3480 unsigned int offset = doffset; 3481 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 3482 unsigned int partial_segs = 0; 3483 unsigned int headroom; 3484 unsigned int len = head_skb->len; 3485 __be16 proto; 3486 bool csum, sg; 3487 int nfrags = skb_shinfo(head_skb)->nr_frags; 3488 int err = -ENOMEM; 3489 int i = 0; 3490 int pos; 3491 int dummy; 3492 3493 __skb_push(head_skb, doffset); 3494 proto = skb_network_protocol(head_skb, &dummy); 3495 if (unlikely(!proto)) 3496 return ERR_PTR(-EINVAL); 3497 3498 sg = !!(features & NETIF_F_SG); 3499 csum = !!can_checksum_protocol(features, proto); 3500 3501 if (sg && csum && (mss != GSO_BY_FRAGS)) { 3502 if (!(features & NETIF_F_GSO_PARTIAL)) { 3503 struct sk_buff *iter; 3504 unsigned int frag_len; 3505 3506 if (!list_skb || 3507 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 3508 goto normal; 3509 3510 /* If we get here then all the required 3511 * GSO features except frag_list are supported. 3512 * Try to split the SKB to multiple GSO SKBs 3513 * with no frag_list. 3514 * Currently we can do that only when the buffers don't 3515 * have a linear part and all the buffers except 3516 * the last are of the same length. 3517 */ 3518 frag_len = list_skb->len; 3519 skb_walk_frags(head_skb, iter) { 3520 if (frag_len != iter->len && iter->next) 3521 goto normal; 3522 if (skb_headlen(iter) && !iter->head_frag) 3523 goto normal; 3524 3525 len -= iter->len; 3526 } 3527 3528 if (len != frag_len) 3529 goto normal; 3530 } 3531 3532 /* GSO partial only requires that we trim off any excess that 3533 * doesn't fit into an MSS sized block, so take care of that 3534 * now. 3535 */ 3536 partial_segs = len / mss; 3537 if (partial_segs > 1) 3538 mss *= partial_segs; 3539 else 3540 partial_segs = 0; 3541 } 3542 3543 normal: 3544 headroom = skb_headroom(head_skb); 3545 pos = skb_headlen(head_skb); 3546 3547 do { 3548 struct sk_buff *nskb; 3549 skb_frag_t *nskb_frag; 3550 int hsize; 3551 int size; 3552 3553 if (unlikely(mss == GSO_BY_FRAGS)) { 3554 len = list_skb->len; 3555 } else { 3556 len = head_skb->len - offset; 3557 if (len > mss) 3558 len = mss; 3559 } 3560 3561 hsize = skb_headlen(head_skb) - offset; 3562 if (hsize < 0) 3563 hsize = 0; 3564 if (hsize > len || !sg) 3565 hsize = len; 3566 3567 if (!hsize && i >= nfrags && skb_headlen(list_skb) && 3568 (skb_headlen(list_skb) == len || sg)) { 3569 BUG_ON(skb_headlen(list_skb) > len); 3570 3571 i = 0; 3572 nfrags = skb_shinfo(list_skb)->nr_frags; 3573 frag = skb_shinfo(list_skb)->frags; 3574 frag_skb = list_skb; 3575 pos += skb_headlen(list_skb); 3576 3577 while (pos < offset + len) { 3578 BUG_ON(i >= nfrags); 3579 3580 size = skb_frag_size(frag); 3581 if (pos + size > offset + len) 3582 break; 3583 3584 i++; 3585 pos += size; 3586 frag++; 3587 } 3588 3589 nskb = skb_clone(list_skb, GFP_ATOMIC); 3590 list_skb = list_skb->next; 3591 3592 if (unlikely(!nskb)) 3593 goto err; 3594 3595 if (unlikely(pskb_trim(nskb, len))) { 3596 kfree_skb(nskb); 3597 goto err; 3598 } 3599 3600 hsize = skb_end_offset(nskb); 3601 if (skb_cow_head(nskb, doffset + headroom)) { 3602 kfree_skb(nskb); 3603 goto err; 3604 } 3605 3606 nskb->truesize += skb_end_offset(nskb) - hsize; 3607 skb_release_head_state(nskb); 3608 __skb_push(nskb, doffset); 3609 } else { 3610 nskb = __alloc_skb(hsize + doffset + headroom, 3611 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 3612 NUMA_NO_NODE); 3613 3614 if (unlikely(!nskb)) 3615 goto err; 3616 3617 skb_reserve(nskb, headroom); 3618 __skb_put(nskb, doffset); 3619 } 3620 3621 if (segs) 3622 tail->next = nskb; 3623 else 3624 segs = nskb; 3625 tail = nskb; 3626 3627 __copy_skb_header(nskb, head_skb); 3628 3629 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 3630 skb_reset_mac_len(nskb); 3631 3632 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 3633 nskb->data - tnl_hlen, 3634 doffset + tnl_hlen); 3635 3636 if (nskb->len == len + doffset) 3637 goto perform_csum_check; 3638 3639 if (!sg) { 3640 if (!nskb->remcsum_offload) 3641 nskb->ip_summed = CHECKSUM_NONE; 3642 SKB_GSO_CB(nskb)->csum = 3643 skb_copy_and_csum_bits(head_skb, offset, 3644 skb_put(nskb, len), 3645 len, 0); 3646 SKB_GSO_CB(nskb)->csum_start = 3647 skb_headroom(nskb) + doffset; 3648 continue; 3649 } 3650 3651 nskb_frag = skb_shinfo(nskb)->frags; 3652 3653 skb_copy_from_linear_data_offset(head_skb, offset, 3654 skb_put(nskb, hsize), hsize); 3655 3656 skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags & 3657 SKBTX_SHARED_FRAG; 3658 3659 while (pos < offset + len) { 3660 if (i >= nfrags) { 3661 BUG_ON(skb_headlen(list_skb)); 3662 3663 i = 0; 3664 nfrags = skb_shinfo(list_skb)->nr_frags; 3665 frag = skb_shinfo(list_skb)->frags; 3666 frag_skb = list_skb; 3667 3668 BUG_ON(!nfrags); 3669 3670 list_skb = list_skb->next; 3671 } 3672 3673 if (unlikely(skb_shinfo(nskb)->nr_frags >= 3674 MAX_SKB_FRAGS)) { 3675 net_warn_ratelimited( 3676 "skb_segment: too many frags: %u %u\n", 3677 pos, mss); 3678 goto err; 3679 } 3680 3681 if (unlikely(skb_orphan_frags(frag_skb, GFP_ATOMIC))) 3682 goto err; 3683 if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 3684 goto err; 3685 3686 *nskb_frag = *frag; 3687 __skb_frag_ref(nskb_frag); 3688 size = skb_frag_size(nskb_frag); 3689 3690 if (pos < offset) { 3691 nskb_frag->page_offset += offset - pos; 3692 skb_frag_size_sub(nskb_frag, offset - pos); 3693 } 3694 3695 skb_shinfo(nskb)->nr_frags++; 3696 3697 if (pos + size <= offset + len) { 3698 i++; 3699 frag++; 3700 pos += size; 3701 } else { 3702 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 3703 goto skip_fraglist; 3704 } 3705 3706 nskb_frag++; 3707 } 3708 3709 skip_fraglist: 3710 nskb->data_len = len - hsize; 3711 nskb->len += nskb->data_len; 3712 nskb->truesize += nskb->data_len; 3713 3714 perform_csum_check: 3715 if (!csum) { 3716 if (skb_has_shared_frag(nskb)) { 3717 err = __skb_linearize(nskb); 3718 if (err) 3719 goto err; 3720 } 3721 if (!nskb->remcsum_offload) 3722 nskb->ip_summed = CHECKSUM_NONE; 3723 SKB_GSO_CB(nskb)->csum = 3724 skb_checksum(nskb, doffset, 3725 nskb->len - doffset, 0); 3726 SKB_GSO_CB(nskb)->csum_start = 3727 skb_headroom(nskb) + doffset; 3728 } 3729 } while ((offset += len) < head_skb->len); 3730 3731 /* Some callers want to get the end of the list. 3732 * Put it in segs->prev to avoid walking the list. 3733 * (see validate_xmit_skb_list() for example) 3734 */ 3735 segs->prev = tail; 3736 3737 if (partial_segs) { 3738 struct sk_buff *iter; 3739 int type = skb_shinfo(head_skb)->gso_type; 3740 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 3741 3742 /* Update type to add partial and then remove dodgy if set */ 3743 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 3744 type &= ~SKB_GSO_DODGY; 3745 3746 /* Update GSO info and prepare to start updating headers on 3747 * our way back down the stack of protocols. 3748 */ 3749 for (iter = segs; iter; iter = iter->next) { 3750 skb_shinfo(iter)->gso_size = gso_size; 3751 skb_shinfo(iter)->gso_segs = partial_segs; 3752 skb_shinfo(iter)->gso_type = type; 3753 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 3754 } 3755 3756 if (tail->len - doffset <= gso_size) 3757 skb_shinfo(tail)->gso_size = 0; 3758 else if (tail != segs) 3759 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 3760 } 3761 3762 /* Following permits correct backpressure, for protocols 3763 * using skb_set_owner_w(). 3764 * Idea is to tranfert ownership from head_skb to last segment. 3765 */ 3766 if (head_skb->destructor == sock_wfree) { 3767 swap(tail->truesize, head_skb->truesize); 3768 swap(tail->destructor, head_skb->destructor); 3769 swap(tail->sk, head_skb->sk); 3770 } 3771 return segs; 3772 3773 err: 3774 kfree_skb_list(segs); 3775 return ERR_PTR(err); 3776 } 3777 EXPORT_SYMBOL_GPL(skb_segment); 3778 3779 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb) 3780 { 3781 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); 3782 unsigned int offset = skb_gro_offset(skb); 3783 unsigned int headlen = skb_headlen(skb); 3784 unsigned int len = skb_gro_len(skb); 3785 struct sk_buff *lp, *p = *head; 3786 unsigned int delta_truesize; 3787 3788 if (unlikely(p->len + len >= 65536)) 3789 return -E2BIG; 3790 3791 lp = NAPI_GRO_CB(p)->last; 3792 pinfo = skb_shinfo(lp); 3793 3794 if (headlen <= offset) { 3795 skb_frag_t *frag; 3796 skb_frag_t *frag2; 3797 int i = skbinfo->nr_frags; 3798 int nr_frags = pinfo->nr_frags + i; 3799 3800 if (nr_frags > MAX_SKB_FRAGS) 3801 goto merge; 3802 3803 offset -= headlen; 3804 pinfo->nr_frags = nr_frags; 3805 skbinfo->nr_frags = 0; 3806 3807 frag = pinfo->frags + nr_frags; 3808 frag2 = skbinfo->frags + i; 3809 do { 3810 *--frag = *--frag2; 3811 } while (--i); 3812 3813 frag->page_offset += offset; 3814 skb_frag_size_sub(frag, offset); 3815 3816 /* all fragments truesize : remove (head size + sk_buff) */ 3817 delta_truesize = skb->truesize - 3818 SKB_TRUESIZE(skb_end_offset(skb)); 3819 3820 skb->truesize -= skb->data_len; 3821 skb->len -= skb->data_len; 3822 skb->data_len = 0; 3823 3824 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; 3825 goto done; 3826 } else if (skb->head_frag) { 3827 int nr_frags = pinfo->nr_frags; 3828 skb_frag_t *frag = pinfo->frags + nr_frags; 3829 struct page *page = virt_to_head_page(skb->head); 3830 unsigned int first_size = headlen - offset; 3831 unsigned int first_offset; 3832 3833 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) 3834 goto merge; 3835 3836 first_offset = skb->data - 3837 (unsigned char *)page_address(page) + 3838 offset; 3839 3840 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; 3841 3842 frag->page.p = page; 3843 frag->page_offset = first_offset; 3844 skb_frag_size_set(frag, first_size); 3845 3846 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); 3847 /* We dont need to clear skbinfo->nr_frags here */ 3848 3849 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 3850 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; 3851 goto done; 3852 } 3853 3854 merge: 3855 delta_truesize = skb->truesize; 3856 if (offset > headlen) { 3857 unsigned int eat = offset - headlen; 3858 3859 skbinfo->frags[0].page_offset += eat; 3860 skb_frag_size_sub(&skbinfo->frags[0], eat); 3861 skb->data_len -= eat; 3862 skb->len -= eat; 3863 offset = headlen; 3864 } 3865 3866 __skb_pull(skb, offset); 3867 3868 if (NAPI_GRO_CB(p)->last == p) 3869 skb_shinfo(p)->frag_list = skb; 3870 else 3871 NAPI_GRO_CB(p)->last->next = skb; 3872 NAPI_GRO_CB(p)->last = skb; 3873 __skb_header_release(skb); 3874 lp = p; 3875 3876 done: 3877 NAPI_GRO_CB(p)->count++; 3878 p->data_len += len; 3879 p->truesize += delta_truesize; 3880 p->len += len; 3881 if (lp != p) { 3882 lp->data_len += len; 3883 lp->truesize += delta_truesize; 3884 lp->len += len; 3885 } 3886 NAPI_GRO_CB(skb)->same_flow = 1; 3887 return 0; 3888 } 3889 EXPORT_SYMBOL_GPL(skb_gro_receive); 3890 3891 void __init skb_init(void) 3892 { 3893 skbuff_head_cache = kmem_cache_create("skbuff_head_cache", 3894 sizeof(struct sk_buff), 3895 0, 3896 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3897 NULL); 3898 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 3899 sizeof(struct sk_buff_fclones), 3900 0, 3901 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3902 NULL); 3903 } 3904 3905 static int 3906 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 3907 unsigned int recursion_level) 3908 { 3909 int start = skb_headlen(skb); 3910 int i, copy = start - offset; 3911 struct sk_buff *frag_iter; 3912 int elt = 0; 3913 3914 if (unlikely(recursion_level >= 24)) 3915 return -EMSGSIZE; 3916 3917 if (copy > 0) { 3918 if (copy > len) 3919 copy = len; 3920 sg_set_buf(sg, skb->data + offset, copy); 3921 elt++; 3922 if ((len -= copy) == 0) 3923 return elt; 3924 offset += copy; 3925 } 3926 3927 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3928 int end; 3929 3930 WARN_ON(start > offset + len); 3931 3932 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3933 if ((copy = end - offset) > 0) { 3934 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3935 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 3936 return -EMSGSIZE; 3937 3938 if (copy > len) 3939 copy = len; 3940 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 3941 frag->page_offset+offset-start); 3942 elt++; 3943 if (!(len -= copy)) 3944 return elt; 3945 offset += copy; 3946 } 3947 start = end; 3948 } 3949 3950 skb_walk_frags(skb, frag_iter) { 3951 int end, ret; 3952 3953 WARN_ON(start > offset + len); 3954 3955 end = start + frag_iter->len; 3956 if ((copy = end - offset) > 0) { 3957 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 3958 return -EMSGSIZE; 3959 3960 if (copy > len) 3961 copy = len; 3962 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 3963 copy, recursion_level + 1); 3964 if (unlikely(ret < 0)) 3965 return ret; 3966 elt += ret; 3967 if ((len -= copy) == 0) 3968 return elt; 3969 offset += copy; 3970 } 3971 start = end; 3972 } 3973 BUG_ON(len); 3974 return elt; 3975 } 3976 3977 /** 3978 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 3979 * @skb: Socket buffer containing the buffers to be mapped 3980 * @sg: The scatter-gather list to map into 3981 * @offset: The offset into the buffer's contents to start mapping 3982 * @len: Length of buffer space to be mapped 3983 * 3984 * Fill the specified scatter-gather list with mappings/pointers into a 3985 * region of the buffer space attached to a socket buffer. Returns either 3986 * the number of scatterlist items used, or -EMSGSIZE if the contents 3987 * could not fit. 3988 */ 3989 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 3990 { 3991 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 3992 3993 if (nsg <= 0) 3994 return nsg; 3995 3996 sg_mark_end(&sg[nsg - 1]); 3997 3998 return nsg; 3999 } 4000 EXPORT_SYMBOL_GPL(skb_to_sgvec); 4001 4002 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 4003 * sglist without mark the sg which contain last skb data as the end. 4004 * So the caller can mannipulate sg list as will when padding new data after 4005 * the first call without calling sg_unmark_end to expend sg list. 4006 * 4007 * Scenario to use skb_to_sgvec_nomark: 4008 * 1. sg_init_table 4009 * 2. skb_to_sgvec_nomark(payload1) 4010 * 3. skb_to_sgvec_nomark(payload2) 4011 * 4012 * This is equivalent to: 4013 * 1. sg_init_table 4014 * 2. skb_to_sgvec(payload1) 4015 * 3. sg_unmark_end 4016 * 4. skb_to_sgvec(payload2) 4017 * 4018 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 4019 * is more preferable. 4020 */ 4021 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 4022 int offset, int len) 4023 { 4024 return __skb_to_sgvec(skb, sg, offset, len, 0); 4025 } 4026 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 4027 4028 4029 4030 /** 4031 * skb_cow_data - Check that a socket buffer's data buffers are writable 4032 * @skb: The socket buffer to check. 4033 * @tailbits: Amount of trailing space to be added 4034 * @trailer: Returned pointer to the skb where the @tailbits space begins 4035 * 4036 * Make sure that the data buffers attached to a socket buffer are 4037 * writable. If they are not, private copies are made of the data buffers 4038 * and the socket buffer is set to use these instead. 4039 * 4040 * If @tailbits is given, make sure that there is space to write @tailbits 4041 * bytes of data beyond current end of socket buffer. @trailer will be 4042 * set to point to the skb in which this space begins. 4043 * 4044 * The number of scatterlist elements required to completely map the 4045 * COW'd and extended socket buffer will be returned. 4046 */ 4047 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 4048 { 4049 int copyflag; 4050 int elt; 4051 struct sk_buff *skb1, **skb_p; 4052 4053 /* If skb is cloned or its head is paged, reallocate 4054 * head pulling out all the pages (pages are considered not writable 4055 * at the moment even if they are anonymous). 4056 */ 4057 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 4058 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL) 4059 return -ENOMEM; 4060 4061 /* Easy case. Most of packets will go this way. */ 4062 if (!skb_has_frag_list(skb)) { 4063 /* A little of trouble, not enough of space for trailer. 4064 * This should not happen, when stack is tuned to generate 4065 * good frames. OK, on miss we reallocate and reserve even more 4066 * space, 128 bytes is fair. */ 4067 4068 if (skb_tailroom(skb) < tailbits && 4069 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 4070 return -ENOMEM; 4071 4072 /* Voila! */ 4073 *trailer = skb; 4074 return 1; 4075 } 4076 4077 /* Misery. We are in troubles, going to mincer fragments... */ 4078 4079 elt = 1; 4080 skb_p = &skb_shinfo(skb)->frag_list; 4081 copyflag = 0; 4082 4083 while ((skb1 = *skb_p) != NULL) { 4084 int ntail = 0; 4085 4086 /* The fragment is partially pulled by someone, 4087 * this can happen on input. Copy it and everything 4088 * after it. */ 4089 4090 if (skb_shared(skb1)) 4091 copyflag = 1; 4092 4093 /* If the skb is the last, worry about trailer. */ 4094 4095 if (skb1->next == NULL && tailbits) { 4096 if (skb_shinfo(skb1)->nr_frags || 4097 skb_has_frag_list(skb1) || 4098 skb_tailroom(skb1) < tailbits) 4099 ntail = tailbits + 128; 4100 } 4101 4102 if (copyflag || 4103 skb_cloned(skb1) || 4104 ntail || 4105 skb_shinfo(skb1)->nr_frags || 4106 skb_has_frag_list(skb1)) { 4107 struct sk_buff *skb2; 4108 4109 /* Fuck, we are miserable poor guys... */ 4110 if (ntail == 0) 4111 skb2 = skb_copy(skb1, GFP_ATOMIC); 4112 else 4113 skb2 = skb_copy_expand(skb1, 4114 skb_headroom(skb1), 4115 ntail, 4116 GFP_ATOMIC); 4117 if (unlikely(skb2 == NULL)) 4118 return -ENOMEM; 4119 4120 if (skb1->sk) 4121 skb_set_owner_w(skb2, skb1->sk); 4122 4123 /* Looking around. Are we still alive? 4124 * OK, link new skb, drop old one */ 4125 4126 skb2->next = skb1->next; 4127 *skb_p = skb2; 4128 kfree_skb(skb1); 4129 skb1 = skb2; 4130 } 4131 elt++; 4132 *trailer = skb1; 4133 skb_p = &skb1->next; 4134 } 4135 4136 return elt; 4137 } 4138 EXPORT_SYMBOL_GPL(skb_cow_data); 4139 4140 static void sock_rmem_free(struct sk_buff *skb) 4141 { 4142 struct sock *sk = skb->sk; 4143 4144 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 4145 } 4146 4147 static void skb_set_err_queue(struct sk_buff *skb) 4148 { 4149 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING. 4150 * So, it is safe to (mis)use it to mark skbs on the error queue. 4151 */ 4152 skb->pkt_type = PACKET_OUTGOING; 4153 BUILD_BUG_ON(PACKET_OUTGOING == 0); 4154 } 4155 4156 /* 4157 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 4158 */ 4159 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 4160 { 4161 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 4162 (unsigned int)sk->sk_rcvbuf) 4163 return -ENOMEM; 4164 4165 skb_orphan(skb); 4166 skb->sk = sk; 4167 skb->destructor = sock_rmem_free; 4168 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 4169 skb_set_err_queue(skb); 4170 4171 /* before exiting rcu section, make sure dst is refcounted */ 4172 skb_dst_force(skb); 4173 4174 skb_queue_tail(&sk->sk_error_queue, skb); 4175 if (!sock_flag(sk, SOCK_DEAD)) 4176 sk->sk_data_ready(sk); 4177 return 0; 4178 } 4179 EXPORT_SYMBOL(sock_queue_err_skb); 4180 4181 static bool is_icmp_err_skb(const struct sk_buff *skb) 4182 { 4183 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 4184 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 4185 } 4186 4187 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 4188 { 4189 struct sk_buff_head *q = &sk->sk_error_queue; 4190 struct sk_buff *skb, *skb_next = NULL; 4191 bool icmp_next = false; 4192 unsigned long flags; 4193 4194 spin_lock_irqsave(&q->lock, flags); 4195 skb = __skb_dequeue(q); 4196 if (skb && (skb_next = skb_peek(q))) { 4197 icmp_next = is_icmp_err_skb(skb_next); 4198 if (icmp_next) 4199 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_origin; 4200 } 4201 spin_unlock_irqrestore(&q->lock, flags); 4202 4203 if (is_icmp_err_skb(skb) && !icmp_next) 4204 sk->sk_err = 0; 4205 4206 if (skb_next) 4207 sk->sk_error_report(sk); 4208 4209 return skb; 4210 } 4211 EXPORT_SYMBOL(sock_dequeue_err_skb); 4212 4213 /** 4214 * skb_clone_sk - create clone of skb, and take reference to socket 4215 * @skb: the skb to clone 4216 * 4217 * This function creates a clone of a buffer that holds a reference on 4218 * sk_refcnt. Buffers created via this function are meant to be 4219 * returned using sock_queue_err_skb, or free via kfree_skb. 4220 * 4221 * When passing buffers allocated with this function to sock_queue_err_skb 4222 * it is necessary to wrap the call with sock_hold/sock_put in order to 4223 * prevent the socket from being released prior to being enqueued on 4224 * the sk_error_queue. 4225 */ 4226 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 4227 { 4228 struct sock *sk = skb->sk; 4229 struct sk_buff *clone; 4230 4231 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 4232 return NULL; 4233 4234 clone = skb_clone(skb, GFP_ATOMIC); 4235 if (!clone) { 4236 sock_put(sk); 4237 return NULL; 4238 } 4239 4240 clone->sk = sk; 4241 clone->destructor = sock_efree; 4242 4243 return clone; 4244 } 4245 EXPORT_SYMBOL(skb_clone_sk); 4246 4247 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 4248 struct sock *sk, 4249 int tstype, 4250 bool opt_stats) 4251 { 4252 struct sock_exterr_skb *serr; 4253 int err; 4254 4255 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 4256 4257 serr = SKB_EXT_ERR(skb); 4258 memset(serr, 0, sizeof(*serr)); 4259 serr->ee.ee_errno = ENOMSG; 4260 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 4261 serr->ee.ee_info = tstype; 4262 serr->opt_stats = opt_stats; 4263 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 4264 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { 4265 serr->ee.ee_data = skb_shinfo(skb)->tskey; 4266 if (sk->sk_protocol == IPPROTO_TCP && 4267 sk->sk_type == SOCK_STREAM) 4268 serr->ee.ee_data -= sk->sk_tskey; 4269 } 4270 4271 err = sock_queue_err_skb(sk, skb); 4272 4273 if (err) 4274 kfree_skb(skb); 4275 } 4276 4277 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 4278 { 4279 bool ret; 4280 4281 if (likely(sysctl_tstamp_allow_data || tsonly)) 4282 return true; 4283 4284 read_lock_bh(&sk->sk_callback_lock); 4285 ret = sk->sk_socket && sk->sk_socket->file && 4286 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); 4287 read_unlock_bh(&sk->sk_callback_lock); 4288 return ret; 4289 } 4290 4291 void skb_complete_tx_timestamp(struct sk_buff *skb, 4292 struct skb_shared_hwtstamps *hwtstamps) 4293 { 4294 struct sock *sk = skb->sk; 4295 4296 if (!skb_may_tx_timestamp(sk, false)) 4297 goto err; 4298 4299 /* Take a reference to prevent skb_orphan() from freeing the socket, 4300 * but only if the socket refcount is not zero. 4301 */ 4302 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 4303 *skb_hwtstamps(skb) = *hwtstamps; 4304 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 4305 sock_put(sk); 4306 return; 4307 } 4308 4309 err: 4310 kfree_skb(skb); 4311 } 4312 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 4313 4314 void __skb_tstamp_tx(struct sk_buff *orig_skb, 4315 struct skb_shared_hwtstamps *hwtstamps, 4316 struct sock *sk, int tstype) 4317 { 4318 struct sk_buff *skb; 4319 bool tsonly, opt_stats = false; 4320 4321 if (!sk) 4322 return; 4323 4324 if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 4325 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 4326 return; 4327 4328 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 4329 if (!skb_may_tx_timestamp(sk, tsonly)) 4330 return; 4331 4332 if (tsonly) { 4333 #ifdef CONFIG_INET 4334 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && 4335 sk->sk_protocol == IPPROTO_TCP && 4336 sk->sk_type == SOCK_STREAM) { 4337 skb = tcp_get_timestamping_opt_stats(sk); 4338 opt_stats = true; 4339 } else 4340 #endif 4341 skb = alloc_skb(0, GFP_ATOMIC); 4342 } else { 4343 skb = skb_clone(orig_skb, GFP_ATOMIC); 4344 } 4345 if (!skb) 4346 return; 4347 4348 if (tsonly) { 4349 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 4350 SKBTX_ANY_TSTAMP; 4351 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 4352 } 4353 4354 if (hwtstamps) 4355 *skb_hwtstamps(skb) = *hwtstamps; 4356 else 4357 skb->tstamp = ktime_get_real(); 4358 4359 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 4360 } 4361 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 4362 4363 void skb_tstamp_tx(struct sk_buff *orig_skb, 4364 struct skb_shared_hwtstamps *hwtstamps) 4365 { 4366 return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk, 4367 SCM_TSTAMP_SND); 4368 } 4369 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 4370 4371 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 4372 { 4373 struct sock *sk = skb->sk; 4374 struct sock_exterr_skb *serr; 4375 int err = 1; 4376 4377 skb->wifi_acked_valid = 1; 4378 skb->wifi_acked = acked; 4379 4380 serr = SKB_EXT_ERR(skb); 4381 memset(serr, 0, sizeof(*serr)); 4382 serr->ee.ee_errno = ENOMSG; 4383 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 4384 4385 /* Take a reference to prevent skb_orphan() from freeing the socket, 4386 * but only if the socket refcount is not zero. 4387 */ 4388 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 4389 err = sock_queue_err_skb(sk, skb); 4390 sock_put(sk); 4391 } 4392 if (err) 4393 kfree_skb(skb); 4394 } 4395 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 4396 4397 /** 4398 * skb_partial_csum_set - set up and verify partial csum values for packet 4399 * @skb: the skb to set 4400 * @start: the number of bytes after skb->data to start checksumming. 4401 * @off: the offset from start to place the checksum. 4402 * 4403 * For untrusted partially-checksummed packets, we need to make sure the values 4404 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 4405 * 4406 * This function checks and sets those values and skb->ip_summed: if this 4407 * returns false you should drop the packet. 4408 */ 4409 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 4410 { 4411 if (unlikely(start > skb_headlen(skb)) || 4412 unlikely((int)start + off > skb_headlen(skb) - 2)) { 4413 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n", 4414 start, off, skb_headlen(skb)); 4415 return false; 4416 } 4417 skb->ip_summed = CHECKSUM_PARTIAL; 4418 skb->csum_start = skb_headroom(skb) + start; 4419 skb->csum_offset = off; 4420 skb_set_transport_header(skb, start); 4421 return true; 4422 } 4423 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 4424 4425 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 4426 unsigned int max) 4427 { 4428 if (skb_headlen(skb) >= len) 4429 return 0; 4430 4431 /* If we need to pullup then pullup to the max, so we 4432 * won't need to do it again. 4433 */ 4434 if (max > skb->len) 4435 max = skb->len; 4436 4437 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 4438 return -ENOMEM; 4439 4440 if (skb_headlen(skb) < len) 4441 return -EPROTO; 4442 4443 return 0; 4444 } 4445 4446 #define MAX_TCP_HDR_LEN (15 * 4) 4447 4448 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 4449 typeof(IPPROTO_IP) proto, 4450 unsigned int off) 4451 { 4452 switch (proto) { 4453 int err; 4454 4455 case IPPROTO_TCP: 4456 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 4457 off + MAX_TCP_HDR_LEN); 4458 if (!err && !skb_partial_csum_set(skb, off, 4459 offsetof(struct tcphdr, 4460 check))) 4461 err = -EPROTO; 4462 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 4463 4464 case IPPROTO_UDP: 4465 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 4466 off + sizeof(struct udphdr)); 4467 if (!err && !skb_partial_csum_set(skb, off, 4468 offsetof(struct udphdr, 4469 check))) 4470 err = -EPROTO; 4471 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 4472 } 4473 4474 return ERR_PTR(-EPROTO); 4475 } 4476 4477 /* This value should be large enough to cover a tagged ethernet header plus 4478 * maximally sized IP and TCP or UDP headers. 4479 */ 4480 #define MAX_IP_HDR_LEN 128 4481 4482 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 4483 { 4484 unsigned int off; 4485 bool fragment; 4486 __sum16 *csum; 4487 int err; 4488 4489 fragment = false; 4490 4491 err = skb_maybe_pull_tail(skb, 4492 sizeof(struct iphdr), 4493 MAX_IP_HDR_LEN); 4494 if (err < 0) 4495 goto out; 4496 4497 if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF)) 4498 fragment = true; 4499 4500 off = ip_hdrlen(skb); 4501 4502 err = -EPROTO; 4503 4504 if (fragment) 4505 goto out; 4506 4507 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 4508 if (IS_ERR(csum)) 4509 return PTR_ERR(csum); 4510 4511 if (recalculate) 4512 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 4513 ip_hdr(skb)->daddr, 4514 skb->len - off, 4515 ip_hdr(skb)->protocol, 0); 4516 err = 0; 4517 4518 out: 4519 return err; 4520 } 4521 4522 /* This value should be large enough to cover a tagged ethernet header plus 4523 * an IPv6 header, all options, and a maximal TCP or UDP header. 4524 */ 4525 #define MAX_IPV6_HDR_LEN 256 4526 4527 #define OPT_HDR(type, skb, off) \ 4528 (type *)(skb_network_header(skb) + (off)) 4529 4530 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 4531 { 4532 int err; 4533 u8 nexthdr; 4534 unsigned int off; 4535 unsigned int len; 4536 bool fragment; 4537 bool done; 4538 __sum16 *csum; 4539 4540 fragment = false; 4541 done = false; 4542 4543 off = sizeof(struct ipv6hdr); 4544 4545 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 4546 if (err < 0) 4547 goto out; 4548 4549 nexthdr = ipv6_hdr(skb)->nexthdr; 4550 4551 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 4552 while (off <= len && !done) { 4553 switch (nexthdr) { 4554 case IPPROTO_DSTOPTS: 4555 case IPPROTO_HOPOPTS: 4556 case IPPROTO_ROUTING: { 4557 struct ipv6_opt_hdr *hp; 4558 4559 err = skb_maybe_pull_tail(skb, 4560 off + 4561 sizeof(struct ipv6_opt_hdr), 4562 MAX_IPV6_HDR_LEN); 4563 if (err < 0) 4564 goto out; 4565 4566 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 4567 nexthdr = hp->nexthdr; 4568 off += ipv6_optlen(hp); 4569 break; 4570 } 4571 case IPPROTO_AH: { 4572 struct ip_auth_hdr *hp; 4573 4574 err = skb_maybe_pull_tail(skb, 4575 off + 4576 sizeof(struct ip_auth_hdr), 4577 MAX_IPV6_HDR_LEN); 4578 if (err < 0) 4579 goto out; 4580 4581 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 4582 nexthdr = hp->nexthdr; 4583 off += ipv6_authlen(hp); 4584 break; 4585 } 4586 case IPPROTO_FRAGMENT: { 4587 struct frag_hdr *hp; 4588 4589 err = skb_maybe_pull_tail(skb, 4590 off + 4591 sizeof(struct frag_hdr), 4592 MAX_IPV6_HDR_LEN); 4593 if (err < 0) 4594 goto out; 4595 4596 hp = OPT_HDR(struct frag_hdr, skb, off); 4597 4598 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 4599 fragment = true; 4600 4601 nexthdr = hp->nexthdr; 4602 off += sizeof(struct frag_hdr); 4603 break; 4604 } 4605 default: 4606 done = true; 4607 break; 4608 } 4609 } 4610 4611 err = -EPROTO; 4612 4613 if (!done || fragment) 4614 goto out; 4615 4616 csum = skb_checksum_setup_ip(skb, nexthdr, off); 4617 if (IS_ERR(csum)) 4618 return PTR_ERR(csum); 4619 4620 if (recalculate) 4621 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 4622 &ipv6_hdr(skb)->daddr, 4623 skb->len - off, nexthdr, 0); 4624 err = 0; 4625 4626 out: 4627 return err; 4628 } 4629 4630 /** 4631 * skb_checksum_setup - set up partial checksum offset 4632 * @skb: the skb to set up 4633 * @recalculate: if true the pseudo-header checksum will be recalculated 4634 */ 4635 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 4636 { 4637 int err; 4638 4639 switch (skb->protocol) { 4640 case htons(ETH_P_IP): 4641 err = skb_checksum_setup_ipv4(skb, recalculate); 4642 break; 4643 4644 case htons(ETH_P_IPV6): 4645 err = skb_checksum_setup_ipv6(skb, recalculate); 4646 break; 4647 4648 default: 4649 err = -EPROTO; 4650 break; 4651 } 4652 4653 return err; 4654 } 4655 EXPORT_SYMBOL(skb_checksum_setup); 4656 4657 /** 4658 * skb_checksum_maybe_trim - maybe trims the given skb 4659 * @skb: the skb to check 4660 * @transport_len: the data length beyond the network header 4661 * 4662 * Checks whether the given skb has data beyond the given transport length. 4663 * If so, returns a cloned skb trimmed to this transport length. 4664 * Otherwise returns the provided skb. Returns NULL in error cases 4665 * (e.g. transport_len exceeds skb length or out-of-memory). 4666 * 4667 * Caller needs to set the skb transport header and free any returned skb if it 4668 * differs from the provided skb. 4669 */ 4670 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 4671 unsigned int transport_len) 4672 { 4673 struct sk_buff *skb_chk; 4674 unsigned int len = skb_transport_offset(skb) + transport_len; 4675 int ret; 4676 4677 if (skb->len < len) 4678 return NULL; 4679 else if (skb->len == len) 4680 return skb; 4681 4682 skb_chk = skb_clone(skb, GFP_ATOMIC); 4683 if (!skb_chk) 4684 return NULL; 4685 4686 ret = pskb_trim_rcsum(skb_chk, len); 4687 if (ret) { 4688 kfree_skb(skb_chk); 4689 return NULL; 4690 } 4691 4692 return skb_chk; 4693 } 4694 4695 /** 4696 * skb_checksum_trimmed - validate checksum of an skb 4697 * @skb: the skb to check 4698 * @transport_len: the data length beyond the network header 4699 * @skb_chkf: checksum function to use 4700 * 4701 * Applies the given checksum function skb_chkf to the provided skb. 4702 * Returns a checked and maybe trimmed skb. Returns NULL on error. 4703 * 4704 * If the skb has data beyond the given transport length, then a 4705 * trimmed & cloned skb is checked and returned. 4706 * 4707 * Caller needs to set the skb transport header and free any returned skb if it 4708 * differs from the provided skb. 4709 */ 4710 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 4711 unsigned int transport_len, 4712 __sum16(*skb_chkf)(struct sk_buff *skb)) 4713 { 4714 struct sk_buff *skb_chk; 4715 unsigned int offset = skb_transport_offset(skb); 4716 __sum16 ret; 4717 4718 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 4719 if (!skb_chk) 4720 goto err; 4721 4722 if (!pskb_may_pull(skb_chk, offset)) 4723 goto err; 4724 4725 skb_pull_rcsum(skb_chk, offset); 4726 ret = skb_chkf(skb_chk); 4727 skb_push_rcsum(skb_chk, offset); 4728 4729 if (ret) 4730 goto err; 4731 4732 return skb_chk; 4733 4734 err: 4735 if (skb_chk && skb_chk != skb) 4736 kfree_skb(skb_chk); 4737 4738 return NULL; 4739 4740 } 4741 EXPORT_SYMBOL(skb_checksum_trimmed); 4742 4743 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 4744 { 4745 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 4746 skb->dev->name); 4747 } 4748 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 4749 4750 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 4751 { 4752 if (head_stolen) { 4753 skb_release_head_state(skb); 4754 kmem_cache_free(skbuff_head_cache, skb); 4755 } else { 4756 __kfree_skb(skb); 4757 } 4758 } 4759 EXPORT_SYMBOL(kfree_skb_partial); 4760 4761 /** 4762 * skb_try_coalesce - try to merge skb to prior one 4763 * @to: prior buffer 4764 * @from: buffer to add 4765 * @fragstolen: pointer to boolean 4766 * @delta_truesize: how much more was allocated than was requested 4767 */ 4768 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 4769 bool *fragstolen, int *delta_truesize) 4770 { 4771 struct skb_shared_info *to_shinfo, *from_shinfo; 4772 int i, delta, len = from->len; 4773 4774 *fragstolen = false; 4775 4776 if (skb_cloned(to)) 4777 return false; 4778 4779 if (len <= skb_tailroom(to)) { 4780 if (len) 4781 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 4782 *delta_truesize = 0; 4783 return true; 4784 } 4785 4786 to_shinfo = skb_shinfo(to); 4787 from_shinfo = skb_shinfo(from); 4788 if (to_shinfo->frag_list || from_shinfo->frag_list) 4789 return false; 4790 if (skb_zcopy(to) || skb_zcopy(from)) 4791 return false; 4792 4793 if (skb_headlen(from) != 0) { 4794 struct page *page; 4795 unsigned int offset; 4796 4797 if (to_shinfo->nr_frags + 4798 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 4799 return false; 4800 4801 if (skb_head_is_locked(from)) 4802 return false; 4803 4804 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 4805 4806 page = virt_to_head_page(from->head); 4807 offset = from->data - (unsigned char *)page_address(page); 4808 4809 skb_fill_page_desc(to, to_shinfo->nr_frags, 4810 page, offset, skb_headlen(from)); 4811 *fragstolen = true; 4812 } else { 4813 if (to_shinfo->nr_frags + 4814 from_shinfo->nr_frags > MAX_SKB_FRAGS) 4815 return false; 4816 4817 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 4818 } 4819 4820 WARN_ON_ONCE(delta < len); 4821 4822 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 4823 from_shinfo->frags, 4824 from_shinfo->nr_frags * sizeof(skb_frag_t)); 4825 to_shinfo->nr_frags += from_shinfo->nr_frags; 4826 4827 if (!skb_cloned(from)) 4828 from_shinfo->nr_frags = 0; 4829 4830 /* if the skb is not cloned this does nothing 4831 * since we set nr_frags to 0. 4832 */ 4833 for (i = 0; i < from_shinfo->nr_frags; i++) 4834 __skb_frag_ref(&from_shinfo->frags[i]); 4835 4836 to->truesize += delta; 4837 to->len += len; 4838 to->data_len += len; 4839 4840 *delta_truesize = delta; 4841 return true; 4842 } 4843 EXPORT_SYMBOL(skb_try_coalesce); 4844 4845 /** 4846 * skb_scrub_packet - scrub an skb 4847 * 4848 * @skb: buffer to clean 4849 * @xnet: packet is crossing netns 4850 * 4851 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 4852 * into/from a tunnel. Some information have to be cleared during these 4853 * operations. 4854 * skb_scrub_packet can also be used to clean a skb before injecting it in 4855 * another namespace (@xnet == true). We have to clear all information in the 4856 * skb that could impact namespace isolation. 4857 */ 4858 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 4859 { 4860 skb->tstamp = 0; 4861 skb->pkt_type = PACKET_HOST; 4862 skb->skb_iif = 0; 4863 skb->ignore_df = 0; 4864 skb_dst_drop(skb); 4865 secpath_reset(skb); 4866 nf_reset(skb); 4867 nf_reset_trace(skb); 4868 4869 if (!xnet) 4870 return; 4871 4872 ipvs_reset(skb); 4873 skb_orphan(skb); 4874 skb->mark = 0; 4875 } 4876 EXPORT_SYMBOL_GPL(skb_scrub_packet); 4877 4878 /** 4879 * skb_gso_transport_seglen - Return length of individual segments of a gso packet 4880 * 4881 * @skb: GSO skb 4882 * 4883 * skb_gso_transport_seglen is used to determine the real size of the 4884 * individual segments, including Layer4 headers (TCP/UDP). 4885 * 4886 * The MAC/L2 or network (IP, IPv6) headers are not accounted for. 4887 */ 4888 unsigned int skb_gso_transport_seglen(const struct sk_buff *skb) 4889 { 4890 const struct skb_shared_info *shinfo = skb_shinfo(skb); 4891 unsigned int thlen = 0; 4892 4893 if (skb->encapsulation) { 4894 thlen = skb_inner_transport_header(skb) - 4895 skb_transport_header(skb); 4896 4897 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 4898 thlen += inner_tcp_hdrlen(skb); 4899 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 4900 thlen = tcp_hdrlen(skb); 4901 } else if (unlikely(shinfo->gso_type & SKB_GSO_SCTP)) { 4902 thlen = sizeof(struct sctphdr); 4903 } 4904 /* UFO sets gso_size to the size of the fragmentation 4905 * payload, i.e. the size of the L4 (UDP) header is already 4906 * accounted for. 4907 */ 4908 return thlen + shinfo->gso_size; 4909 } 4910 EXPORT_SYMBOL_GPL(skb_gso_transport_seglen); 4911 4912 /** 4913 * skb_gso_validate_mtu - Return in case such skb fits a given MTU 4914 * 4915 * @skb: GSO skb 4916 * @mtu: MTU to validate against 4917 * 4918 * skb_gso_validate_mtu validates if a given skb will fit a wanted MTU 4919 * once split. 4920 */ 4921 bool skb_gso_validate_mtu(const struct sk_buff *skb, unsigned int mtu) 4922 { 4923 const struct skb_shared_info *shinfo = skb_shinfo(skb); 4924 const struct sk_buff *iter; 4925 unsigned int hlen; 4926 4927 hlen = skb_gso_network_seglen(skb); 4928 4929 if (shinfo->gso_size != GSO_BY_FRAGS) 4930 return hlen <= mtu; 4931 4932 /* Undo this so we can re-use header sizes */ 4933 hlen -= GSO_BY_FRAGS; 4934 4935 skb_walk_frags(skb, iter) { 4936 if (hlen + skb_headlen(iter) > mtu) 4937 return false; 4938 } 4939 4940 return true; 4941 } 4942 EXPORT_SYMBOL_GPL(skb_gso_validate_mtu); 4943 4944 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 4945 { 4946 if (skb_cow(skb, skb_headroom(skb)) < 0) { 4947 kfree_skb(skb); 4948 return NULL; 4949 } 4950 4951 memmove(skb->data - ETH_HLEN, skb->data - skb->mac_len - VLAN_HLEN, 4952 2 * ETH_ALEN); 4953 skb->mac_header += VLAN_HLEN; 4954 return skb; 4955 } 4956 4957 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 4958 { 4959 struct vlan_hdr *vhdr; 4960 u16 vlan_tci; 4961 4962 if (unlikely(skb_vlan_tag_present(skb))) { 4963 /* vlan_tci is already set-up so leave this for another time */ 4964 return skb; 4965 } 4966 4967 skb = skb_share_check(skb, GFP_ATOMIC); 4968 if (unlikely(!skb)) 4969 goto err_free; 4970 4971 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN))) 4972 goto err_free; 4973 4974 vhdr = (struct vlan_hdr *)skb->data; 4975 vlan_tci = ntohs(vhdr->h_vlan_TCI); 4976 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 4977 4978 skb_pull_rcsum(skb, VLAN_HLEN); 4979 vlan_set_encap_proto(skb, vhdr); 4980 4981 skb = skb_reorder_vlan_header(skb); 4982 if (unlikely(!skb)) 4983 goto err_free; 4984 4985 skb_reset_network_header(skb); 4986 skb_reset_transport_header(skb); 4987 skb_reset_mac_len(skb); 4988 4989 return skb; 4990 4991 err_free: 4992 kfree_skb(skb); 4993 return NULL; 4994 } 4995 EXPORT_SYMBOL(skb_vlan_untag); 4996 4997 int skb_ensure_writable(struct sk_buff *skb, int write_len) 4998 { 4999 if (!pskb_may_pull(skb, write_len)) 5000 return -ENOMEM; 5001 5002 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 5003 return 0; 5004 5005 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 5006 } 5007 EXPORT_SYMBOL(skb_ensure_writable); 5008 5009 /* remove VLAN header from packet and update csum accordingly. 5010 * expects a non skb_vlan_tag_present skb with a vlan tag payload 5011 */ 5012 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 5013 { 5014 struct vlan_hdr *vhdr; 5015 int offset = skb->data - skb_mac_header(skb); 5016 int err; 5017 5018 if (WARN_ONCE(offset, 5019 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 5020 offset)) { 5021 return -EINVAL; 5022 } 5023 5024 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 5025 if (unlikely(err)) 5026 return err; 5027 5028 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5029 5030 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN); 5031 *vlan_tci = ntohs(vhdr->h_vlan_TCI); 5032 5033 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN); 5034 __skb_pull(skb, VLAN_HLEN); 5035 5036 vlan_set_encap_proto(skb, vhdr); 5037 skb->mac_header += VLAN_HLEN; 5038 5039 if (skb_network_offset(skb) < ETH_HLEN) 5040 skb_set_network_header(skb, ETH_HLEN); 5041 5042 skb_reset_mac_len(skb); 5043 5044 return err; 5045 } 5046 EXPORT_SYMBOL(__skb_vlan_pop); 5047 5048 /* Pop a vlan tag either from hwaccel or from payload. 5049 * Expects skb->data at mac header. 5050 */ 5051 int skb_vlan_pop(struct sk_buff *skb) 5052 { 5053 u16 vlan_tci; 5054 __be16 vlan_proto; 5055 int err; 5056 5057 if (likely(skb_vlan_tag_present(skb))) { 5058 skb->vlan_tci = 0; 5059 } else { 5060 if (unlikely(!eth_type_vlan(skb->protocol))) 5061 return 0; 5062 5063 err = __skb_vlan_pop(skb, &vlan_tci); 5064 if (err) 5065 return err; 5066 } 5067 /* move next vlan tag to hw accel tag */ 5068 if (likely(!eth_type_vlan(skb->protocol))) 5069 return 0; 5070 5071 vlan_proto = skb->protocol; 5072 err = __skb_vlan_pop(skb, &vlan_tci); 5073 if (unlikely(err)) 5074 return err; 5075 5076 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5077 return 0; 5078 } 5079 EXPORT_SYMBOL(skb_vlan_pop); 5080 5081 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 5082 * Expects skb->data at mac header. 5083 */ 5084 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 5085 { 5086 if (skb_vlan_tag_present(skb)) { 5087 int offset = skb->data - skb_mac_header(skb); 5088 int err; 5089 5090 if (WARN_ONCE(offset, 5091 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 5092 offset)) { 5093 return -EINVAL; 5094 } 5095 5096 err = __vlan_insert_tag(skb, skb->vlan_proto, 5097 skb_vlan_tag_get(skb)); 5098 if (err) 5099 return err; 5100 5101 skb->protocol = skb->vlan_proto; 5102 skb->mac_len += VLAN_HLEN; 5103 5104 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5105 } 5106 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5107 return 0; 5108 } 5109 EXPORT_SYMBOL(skb_vlan_push); 5110 5111 /** 5112 * alloc_skb_with_frags - allocate skb with page frags 5113 * 5114 * @header_len: size of linear part 5115 * @data_len: needed length in frags 5116 * @max_page_order: max page order desired. 5117 * @errcode: pointer to error code if any 5118 * @gfp_mask: allocation mask 5119 * 5120 * This can be used to allocate a paged skb, given a maximal order for frags. 5121 */ 5122 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 5123 unsigned long data_len, 5124 int max_page_order, 5125 int *errcode, 5126 gfp_t gfp_mask) 5127 { 5128 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 5129 unsigned long chunk; 5130 struct sk_buff *skb; 5131 struct page *page; 5132 gfp_t gfp_head; 5133 int i; 5134 5135 *errcode = -EMSGSIZE; 5136 /* Note this test could be relaxed, if we succeed to allocate 5137 * high order pages... 5138 */ 5139 if (npages > MAX_SKB_FRAGS) 5140 return NULL; 5141 5142 gfp_head = gfp_mask; 5143 if (gfp_head & __GFP_DIRECT_RECLAIM) 5144 gfp_head |= __GFP_RETRY_MAYFAIL; 5145 5146 *errcode = -ENOBUFS; 5147 skb = alloc_skb(header_len, gfp_head); 5148 if (!skb) 5149 return NULL; 5150 5151 skb->truesize += npages << PAGE_SHIFT; 5152 5153 for (i = 0; npages > 0; i++) { 5154 int order = max_page_order; 5155 5156 while (order) { 5157 if (npages >= 1 << order) { 5158 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 5159 __GFP_COMP | 5160 __GFP_NOWARN | 5161 __GFP_NORETRY, 5162 order); 5163 if (page) 5164 goto fill_page; 5165 /* Do not retry other high order allocations */ 5166 order = 1; 5167 max_page_order = 0; 5168 } 5169 order--; 5170 } 5171 page = alloc_page(gfp_mask); 5172 if (!page) 5173 goto failure; 5174 fill_page: 5175 chunk = min_t(unsigned long, data_len, 5176 PAGE_SIZE << order); 5177 skb_fill_page_desc(skb, i, page, 0, chunk); 5178 data_len -= chunk; 5179 npages -= 1 << order; 5180 } 5181 return skb; 5182 5183 failure: 5184 kfree_skb(skb); 5185 return NULL; 5186 } 5187 EXPORT_SYMBOL(alloc_skb_with_frags); 5188 5189 /* carve out the first off bytes from skb when off < headlen */ 5190 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 5191 const int headlen, gfp_t gfp_mask) 5192 { 5193 int i; 5194 int size = skb_end_offset(skb); 5195 int new_hlen = headlen - off; 5196 u8 *data; 5197 5198 size = SKB_DATA_ALIGN(size); 5199 5200 if (skb_pfmemalloc(skb)) 5201 gfp_mask |= __GFP_MEMALLOC; 5202 data = kmalloc_reserve(size + 5203 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 5204 gfp_mask, NUMA_NO_NODE, NULL); 5205 if (!data) 5206 return -ENOMEM; 5207 5208 size = SKB_WITH_OVERHEAD(ksize(data)); 5209 5210 /* Copy real data, and all frags */ 5211 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 5212 skb->len -= off; 5213 5214 memcpy((struct skb_shared_info *)(data + size), 5215 skb_shinfo(skb), 5216 offsetof(struct skb_shared_info, 5217 frags[skb_shinfo(skb)->nr_frags])); 5218 if (skb_cloned(skb)) { 5219 /* drop the old head gracefully */ 5220 if (skb_orphan_frags(skb, gfp_mask)) { 5221 kfree(data); 5222 return -ENOMEM; 5223 } 5224 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 5225 skb_frag_ref(skb, i); 5226 if (skb_has_frag_list(skb)) 5227 skb_clone_fraglist(skb); 5228 skb_release_data(skb); 5229 } else { 5230 /* we can reuse existing recount- all we did was 5231 * relocate values 5232 */ 5233 skb_free_head(skb); 5234 } 5235 5236 skb->head = data; 5237 skb->data = data; 5238 skb->head_frag = 0; 5239 #ifdef NET_SKBUFF_DATA_USES_OFFSET 5240 skb->end = size; 5241 #else 5242 skb->end = skb->head + size; 5243 #endif 5244 skb_set_tail_pointer(skb, skb_headlen(skb)); 5245 skb_headers_offset_update(skb, 0); 5246 skb->cloned = 0; 5247 skb->hdr_len = 0; 5248 skb->nohdr = 0; 5249 atomic_set(&skb_shinfo(skb)->dataref, 1); 5250 5251 return 0; 5252 } 5253 5254 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 5255 5256 /* carve out the first eat bytes from skb's frag_list. May recurse into 5257 * pskb_carve() 5258 */ 5259 static int pskb_carve_frag_list(struct sk_buff *skb, 5260 struct skb_shared_info *shinfo, int eat, 5261 gfp_t gfp_mask) 5262 { 5263 struct sk_buff *list = shinfo->frag_list; 5264 struct sk_buff *clone = NULL; 5265 struct sk_buff *insp = NULL; 5266 5267 do { 5268 if (!list) { 5269 pr_err("Not enough bytes to eat. Want %d\n", eat); 5270 return -EFAULT; 5271 } 5272 if (list->len <= eat) { 5273 /* Eaten as whole. */ 5274 eat -= list->len; 5275 list = list->next; 5276 insp = list; 5277 } else { 5278 /* Eaten partially. */ 5279 if (skb_shared(list)) { 5280 clone = skb_clone(list, gfp_mask); 5281 if (!clone) 5282 return -ENOMEM; 5283 insp = list->next; 5284 list = clone; 5285 } else { 5286 /* This may be pulled without problems. */ 5287 insp = list; 5288 } 5289 if (pskb_carve(list, eat, gfp_mask) < 0) { 5290 kfree_skb(clone); 5291 return -ENOMEM; 5292 } 5293 break; 5294 } 5295 } while (eat); 5296 5297 /* Free pulled out fragments. */ 5298 while ((list = shinfo->frag_list) != insp) { 5299 shinfo->frag_list = list->next; 5300 kfree_skb(list); 5301 } 5302 /* And insert new clone at head. */ 5303 if (clone) { 5304 clone->next = list; 5305 shinfo->frag_list = clone; 5306 } 5307 return 0; 5308 } 5309 5310 /* carve off first len bytes from skb. Split line (off) is in the 5311 * non-linear part of skb 5312 */ 5313 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 5314 int pos, gfp_t gfp_mask) 5315 { 5316 int i, k = 0; 5317 int size = skb_end_offset(skb); 5318 u8 *data; 5319 const int nfrags = skb_shinfo(skb)->nr_frags; 5320 struct skb_shared_info *shinfo; 5321 5322 size = SKB_DATA_ALIGN(size); 5323 5324 if (skb_pfmemalloc(skb)) 5325 gfp_mask |= __GFP_MEMALLOC; 5326 data = kmalloc_reserve(size + 5327 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 5328 gfp_mask, NUMA_NO_NODE, NULL); 5329 if (!data) 5330 return -ENOMEM; 5331 5332 size = SKB_WITH_OVERHEAD(ksize(data)); 5333 5334 memcpy((struct skb_shared_info *)(data + size), 5335 skb_shinfo(skb), offsetof(struct skb_shared_info, 5336 frags[skb_shinfo(skb)->nr_frags])); 5337 if (skb_orphan_frags(skb, gfp_mask)) { 5338 kfree(data); 5339 return -ENOMEM; 5340 } 5341 shinfo = (struct skb_shared_info *)(data + size); 5342 for (i = 0; i < nfrags; i++) { 5343 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 5344 5345 if (pos + fsize > off) { 5346 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 5347 5348 if (pos < off) { 5349 /* Split frag. 5350 * We have two variants in this case: 5351 * 1. Move all the frag to the second 5352 * part, if it is possible. F.e. 5353 * this approach is mandatory for TUX, 5354 * where splitting is expensive. 5355 * 2. Split is accurately. We make this. 5356 */ 5357 shinfo->frags[0].page_offset += off - pos; 5358 skb_frag_size_sub(&shinfo->frags[0], off - pos); 5359 } 5360 skb_frag_ref(skb, i); 5361 k++; 5362 } 5363 pos += fsize; 5364 } 5365 shinfo->nr_frags = k; 5366 if (skb_has_frag_list(skb)) 5367 skb_clone_fraglist(skb); 5368 5369 if (k == 0) { 5370 /* split line is in frag list */ 5371 pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask); 5372 } 5373 skb_release_data(skb); 5374 5375 skb->head = data; 5376 skb->head_frag = 0; 5377 skb->data = data; 5378 #ifdef NET_SKBUFF_DATA_USES_OFFSET 5379 skb->end = size; 5380 #else 5381 skb->end = skb->head + size; 5382 #endif 5383 skb_reset_tail_pointer(skb); 5384 skb_headers_offset_update(skb, 0); 5385 skb->cloned = 0; 5386 skb->hdr_len = 0; 5387 skb->nohdr = 0; 5388 skb->len -= off; 5389 skb->data_len = skb->len; 5390 atomic_set(&skb_shinfo(skb)->dataref, 1); 5391 return 0; 5392 } 5393 5394 /* remove len bytes from the beginning of the skb */ 5395 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 5396 { 5397 int headlen = skb_headlen(skb); 5398 5399 if (len < headlen) 5400 return pskb_carve_inside_header(skb, len, headlen, gfp); 5401 else 5402 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 5403 } 5404 5405 /* Extract to_copy bytes starting at off from skb, and return this in 5406 * a new skb 5407 */ 5408 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 5409 int to_copy, gfp_t gfp) 5410 { 5411 struct sk_buff *clone = skb_clone(skb, gfp); 5412 5413 if (!clone) 5414 return NULL; 5415 5416 if (pskb_carve(clone, off, gfp) < 0 || 5417 pskb_trim(clone, to_copy)) { 5418 kfree_skb(clone); 5419 return NULL; 5420 } 5421 return clone; 5422 } 5423 EXPORT_SYMBOL(pskb_extract); 5424 5425 /** 5426 * skb_condense - try to get rid of fragments/frag_list if possible 5427 * @skb: buffer 5428 * 5429 * Can be used to save memory before skb is added to a busy queue. 5430 * If packet has bytes in frags and enough tail room in skb->head, 5431 * pull all of them, so that we can free the frags right now and adjust 5432 * truesize. 5433 * Notes: 5434 * We do not reallocate skb->head thus can not fail. 5435 * Caller must re-evaluate skb->truesize if needed. 5436 */ 5437 void skb_condense(struct sk_buff *skb) 5438 { 5439 if (skb->data_len) { 5440 if (skb->data_len > skb->end - skb->tail || 5441 skb_cloned(skb)) 5442 return; 5443 5444 /* Nice, we can free page frag(s) right now */ 5445 __pskb_pull_tail(skb, skb->data_len); 5446 } 5447 /* At this point, skb->truesize might be over estimated, 5448 * because skb had a fragment, and fragments do not tell 5449 * their truesize. 5450 * When we pulled its content into skb->head, fragment 5451 * was freed, but __pskb_pull_tail() could not possibly 5452 * adjust skb->truesize, not knowing the frag truesize. 5453 */ 5454 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 5455 } 5456