1 /* 2 * Routines having to do with the 'struct sk_buff' memory handlers. 3 * 4 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 5 * Florian La Roche <rzsfl@rz.uni-sb.de> 6 * 7 * Fixes: 8 * Alan Cox : Fixed the worst of the load 9 * balancer bugs. 10 * Dave Platt : Interrupt stacking fix. 11 * Richard Kooijman : Timestamp fixes. 12 * Alan Cox : Changed buffer format. 13 * Alan Cox : destructor hook for AF_UNIX etc. 14 * Linus Torvalds : Better skb_clone. 15 * Alan Cox : Added skb_copy. 16 * Alan Cox : Added all the changed routines Linus 17 * only put in the headers 18 * Ray VanTassle : Fixed --skb->lock in free 19 * Alan Cox : skb_copy copy arp field 20 * Andi Kleen : slabified it. 21 * Robert Olsson : Removed skb_head_pool 22 * 23 * NOTE: 24 * The __skb_ routines should be called with interrupts 25 * disabled, or you better be *real* sure that the operation is atomic 26 * with respect to whatever list is being frobbed (e.g. via lock_sock() 27 * or via disabling bottom half handlers, etc). 28 * 29 * This program is free software; you can redistribute it and/or 30 * modify it under the terms of the GNU General Public License 31 * as published by the Free Software Foundation; either version 32 * 2 of the License, or (at your option) any later version. 33 */ 34 35 /* 36 * The functions in this file will not compile correctly with gcc 2.4.x 37 */ 38 39 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 40 41 #include <linux/module.h> 42 #include <linux/types.h> 43 #include <linux/kernel.h> 44 #include <linux/kmemcheck.h> 45 #include <linux/mm.h> 46 #include <linux/interrupt.h> 47 #include <linux/in.h> 48 #include <linux/inet.h> 49 #include <linux/slab.h> 50 #include <linux/tcp.h> 51 #include <linux/udp.h> 52 #include <linux/sctp.h> 53 #include <linux/netdevice.h> 54 #ifdef CONFIG_NET_CLS_ACT 55 #include <net/pkt_sched.h> 56 #endif 57 #include <linux/string.h> 58 #include <linux/skbuff.h> 59 #include <linux/splice.h> 60 #include <linux/cache.h> 61 #include <linux/rtnetlink.h> 62 #include <linux/init.h> 63 #include <linux/scatterlist.h> 64 #include <linux/errqueue.h> 65 #include <linux/prefetch.h> 66 #include <linux/if_vlan.h> 67 68 #include <net/protocol.h> 69 #include <net/dst.h> 70 #include <net/sock.h> 71 #include <net/checksum.h> 72 #include <net/ip6_checksum.h> 73 #include <net/xfrm.h> 74 75 #include <asm/uaccess.h> 76 #include <trace/events/skb.h> 77 #include <linux/highmem.h> 78 #include <linux/capability.h> 79 #include <linux/user_namespace.h> 80 81 struct kmem_cache *skbuff_head_cache __read_mostly; 82 static struct kmem_cache *skbuff_fclone_cache __read_mostly; 83 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; 84 EXPORT_SYMBOL(sysctl_max_skb_frags); 85 86 /** 87 * skb_panic - private function for out-of-line support 88 * @skb: buffer 89 * @sz: size 90 * @addr: address 91 * @msg: skb_over_panic or skb_under_panic 92 * 93 * Out-of-line support for skb_put() and skb_push(). 94 * Called via the wrapper skb_over_panic() or skb_under_panic(). 95 * Keep out of line to prevent kernel bloat. 96 * __builtin_return_address is not used because it is not always reliable. 97 */ 98 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 99 const char msg[]) 100 { 101 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n", 102 msg, addr, skb->len, sz, skb->head, skb->data, 103 (unsigned long)skb->tail, (unsigned long)skb->end, 104 skb->dev ? skb->dev->name : "<NULL>"); 105 BUG(); 106 } 107 108 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 109 { 110 skb_panic(skb, sz, addr, __func__); 111 } 112 113 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 114 { 115 skb_panic(skb, sz, addr, __func__); 116 } 117 118 /* 119 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 120 * the caller if emergency pfmemalloc reserves are being used. If it is and 121 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 122 * may be used. Otherwise, the packet data may be discarded until enough 123 * memory is free 124 */ 125 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \ 126 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc) 127 128 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node, 129 unsigned long ip, bool *pfmemalloc) 130 { 131 void *obj; 132 bool ret_pfmemalloc = false; 133 134 /* 135 * Try a regular allocation, when that fails and we're not entitled 136 * to the reserves, fail. 137 */ 138 obj = kmalloc_node_track_caller(size, 139 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 140 node); 141 if (obj || !(gfp_pfmemalloc_allowed(flags))) 142 goto out; 143 144 /* Try again but now we are using pfmemalloc reserves */ 145 ret_pfmemalloc = true; 146 obj = kmalloc_node_track_caller(size, flags, node); 147 148 out: 149 if (pfmemalloc) 150 *pfmemalloc = ret_pfmemalloc; 151 152 return obj; 153 } 154 155 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 156 * 'private' fields and also do memory statistics to find all the 157 * [BEEP] leaks. 158 * 159 */ 160 161 struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node) 162 { 163 struct sk_buff *skb; 164 165 /* Get the HEAD */ 166 skb = kmem_cache_alloc_node(skbuff_head_cache, 167 gfp_mask & ~__GFP_DMA, node); 168 if (!skb) 169 goto out; 170 171 /* 172 * Only clear those fields we need to clear, not those that we will 173 * actually initialise below. Hence, don't put any more fields after 174 * the tail pointer in struct sk_buff! 175 */ 176 memset(skb, 0, offsetof(struct sk_buff, tail)); 177 skb->head = NULL; 178 skb->truesize = sizeof(struct sk_buff); 179 atomic_set(&skb->users, 1); 180 181 skb->mac_header = (typeof(skb->mac_header))~0U; 182 out: 183 return skb; 184 } 185 186 /** 187 * __alloc_skb - allocate a network buffer 188 * @size: size to allocate 189 * @gfp_mask: allocation mask 190 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 191 * instead of head cache and allocate a cloned (child) skb. 192 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 193 * allocations in case the data is required for writeback 194 * @node: numa node to allocate memory on 195 * 196 * Allocate a new &sk_buff. The returned buffer has no headroom and a 197 * tail room of at least size bytes. The object has a reference count 198 * of one. The return is the buffer. On a failure the return is %NULL. 199 * 200 * Buffers may only be allocated from interrupts using a @gfp_mask of 201 * %GFP_ATOMIC. 202 */ 203 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 204 int flags, int node) 205 { 206 struct kmem_cache *cache; 207 struct skb_shared_info *shinfo; 208 struct sk_buff *skb; 209 u8 *data; 210 bool pfmemalloc; 211 212 cache = (flags & SKB_ALLOC_FCLONE) 213 ? skbuff_fclone_cache : skbuff_head_cache; 214 215 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 216 gfp_mask |= __GFP_MEMALLOC; 217 218 /* Get the HEAD */ 219 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); 220 if (!skb) 221 goto out; 222 prefetchw(skb); 223 224 /* We do our best to align skb_shared_info on a separate cache 225 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 226 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 227 * Both skb->head and skb_shared_info are cache line aligned. 228 */ 229 size = SKB_DATA_ALIGN(size); 230 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 231 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc); 232 if (!data) 233 goto nodata; 234 /* kmalloc(size) might give us more room than requested. 235 * Put skb_shared_info exactly at the end of allocated zone, 236 * to allow max possible filling before reallocation. 237 */ 238 size = SKB_WITH_OVERHEAD(ksize(data)); 239 prefetchw(data + size); 240 241 /* 242 * Only clear those fields we need to clear, not those that we will 243 * actually initialise below. Hence, don't put any more fields after 244 * the tail pointer in struct sk_buff! 245 */ 246 memset(skb, 0, offsetof(struct sk_buff, tail)); 247 /* Account for allocated memory : skb + skb->head */ 248 skb->truesize = SKB_TRUESIZE(size); 249 skb->pfmemalloc = pfmemalloc; 250 atomic_set(&skb->users, 1); 251 skb->head = data; 252 skb->data = data; 253 skb_reset_tail_pointer(skb); 254 skb->end = skb->tail + size; 255 skb->mac_header = (typeof(skb->mac_header))~0U; 256 skb->transport_header = (typeof(skb->transport_header))~0U; 257 258 /* make sure we initialize shinfo sequentially */ 259 shinfo = skb_shinfo(skb); 260 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 261 atomic_set(&shinfo->dataref, 1); 262 kmemcheck_annotate_variable(shinfo->destructor_arg); 263 264 if (flags & SKB_ALLOC_FCLONE) { 265 struct sk_buff_fclones *fclones; 266 267 fclones = container_of(skb, struct sk_buff_fclones, skb1); 268 269 kmemcheck_annotate_bitfield(&fclones->skb2, flags1); 270 skb->fclone = SKB_FCLONE_ORIG; 271 atomic_set(&fclones->fclone_ref, 1); 272 273 fclones->skb2.fclone = SKB_FCLONE_CLONE; 274 fclones->skb2.pfmemalloc = pfmemalloc; 275 } 276 out: 277 return skb; 278 nodata: 279 kmem_cache_free(cache, skb); 280 skb = NULL; 281 goto out; 282 } 283 EXPORT_SYMBOL(__alloc_skb); 284 285 /** 286 * __build_skb - build a network buffer 287 * @data: data buffer provided by caller 288 * @frag_size: size of data, or 0 if head was kmalloced 289 * 290 * Allocate a new &sk_buff. Caller provides space holding head and 291 * skb_shared_info. @data must have been allocated by kmalloc() only if 292 * @frag_size is 0, otherwise data should come from the page allocator 293 * or vmalloc() 294 * The return is the new skb buffer. 295 * On a failure the return is %NULL, and @data is not freed. 296 * Notes : 297 * Before IO, driver allocates only data buffer where NIC put incoming frame 298 * Driver should add room at head (NET_SKB_PAD) and 299 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 300 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 301 * before giving packet to stack. 302 * RX rings only contains data buffers, not full skbs. 303 */ 304 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 305 { 306 struct skb_shared_info *shinfo; 307 struct sk_buff *skb; 308 unsigned int size = frag_size ? : ksize(data); 309 310 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); 311 if (!skb) 312 return NULL; 313 314 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 315 316 memset(skb, 0, offsetof(struct sk_buff, tail)); 317 skb->truesize = SKB_TRUESIZE(size); 318 atomic_set(&skb->users, 1); 319 skb->head = data; 320 skb->data = data; 321 skb_reset_tail_pointer(skb); 322 skb->end = skb->tail + size; 323 skb->mac_header = (typeof(skb->mac_header))~0U; 324 skb->transport_header = (typeof(skb->transport_header))~0U; 325 326 /* make sure we initialize shinfo sequentially */ 327 shinfo = skb_shinfo(skb); 328 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 329 atomic_set(&shinfo->dataref, 1); 330 kmemcheck_annotate_variable(shinfo->destructor_arg); 331 332 return skb; 333 } 334 335 /* build_skb() is wrapper over __build_skb(), that specifically 336 * takes care of skb->head and skb->pfmemalloc 337 * This means that if @frag_size is not zero, then @data must be backed 338 * by a page fragment, not kmalloc() or vmalloc() 339 */ 340 struct sk_buff *build_skb(void *data, unsigned int frag_size) 341 { 342 struct sk_buff *skb = __build_skb(data, frag_size); 343 344 if (skb && frag_size) { 345 skb->head_frag = 1; 346 if (page_is_pfmemalloc(virt_to_head_page(data))) 347 skb->pfmemalloc = 1; 348 } 349 return skb; 350 } 351 EXPORT_SYMBOL(build_skb); 352 353 #define NAPI_SKB_CACHE_SIZE 64 354 355 struct napi_alloc_cache { 356 struct page_frag_cache page; 357 size_t skb_count; 358 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 359 }; 360 361 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 362 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 363 364 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) 365 { 366 struct page_frag_cache *nc; 367 unsigned long flags; 368 void *data; 369 370 local_irq_save(flags); 371 nc = this_cpu_ptr(&netdev_alloc_cache); 372 data = __alloc_page_frag(nc, fragsz, gfp_mask); 373 local_irq_restore(flags); 374 return data; 375 } 376 377 /** 378 * netdev_alloc_frag - allocate a page fragment 379 * @fragsz: fragment size 380 * 381 * Allocates a frag from a page for receive buffer. 382 * Uses GFP_ATOMIC allocations. 383 */ 384 void *netdev_alloc_frag(unsigned int fragsz) 385 { 386 return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD); 387 } 388 EXPORT_SYMBOL(netdev_alloc_frag); 389 390 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) 391 { 392 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 393 394 return __alloc_page_frag(&nc->page, fragsz, gfp_mask); 395 } 396 397 void *napi_alloc_frag(unsigned int fragsz) 398 { 399 return __napi_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD); 400 } 401 EXPORT_SYMBOL(napi_alloc_frag); 402 403 /** 404 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 405 * @dev: network device to receive on 406 * @len: length to allocate 407 * @gfp_mask: get_free_pages mask, passed to alloc_skb 408 * 409 * Allocate a new &sk_buff and assign it a usage count of one. The 410 * buffer has NET_SKB_PAD headroom built in. Users should allocate 411 * the headroom they think they need without accounting for the 412 * built in space. The built in space is used for optimisations. 413 * 414 * %NULL is returned if there is no free memory. 415 */ 416 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 417 gfp_t gfp_mask) 418 { 419 struct page_frag_cache *nc; 420 unsigned long flags; 421 struct sk_buff *skb; 422 bool pfmemalloc; 423 void *data; 424 425 len += NET_SKB_PAD; 426 427 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) || 428 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 429 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 430 if (!skb) 431 goto skb_fail; 432 goto skb_success; 433 } 434 435 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 436 len = SKB_DATA_ALIGN(len); 437 438 if (sk_memalloc_socks()) 439 gfp_mask |= __GFP_MEMALLOC; 440 441 local_irq_save(flags); 442 443 nc = this_cpu_ptr(&netdev_alloc_cache); 444 data = __alloc_page_frag(nc, len, gfp_mask); 445 pfmemalloc = nc->pfmemalloc; 446 447 local_irq_restore(flags); 448 449 if (unlikely(!data)) 450 return NULL; 451 452 skb = __build_skb(data, len); 453 if (unlikely(!skb)) { 454 skb_free_frag(data); 455 return NULL; 456 } 457 458 /* use OR instead of assignment to avoid clearing of bits in mask */ 459 if (pfmemalloc) 460 skb->pfmemalloc = 1; 461 skb->head_frag = 1; 462 463 skb_success: 464 skb_reserve(skb, NET_SKB_PAD); 465 skb->dev = dev; 466 467 skb_fail: 468 return skb; 469 } 470 EXPORT_SYMBOL(__netdev_alloc_skb); 471 472 /** 473 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 474 * @napi: napi instance this buffer was allocated for 475 * @len: length to allocate 476 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages 477 * 478 * Allocate a new sk_buff for use in NAPI receive. This buffer will 479 * attempt to allocate the head from a special reserved region used 480 * only for NAPI Rx allocation. By doing this we can save several 481 * CPU cycles by avoiding having to disable and re-enable IRQs. 482 * 483 * %NULL is returned if there is no free memory. 484 */ 485 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, 486 gfp_t gfp_mask) 487 { 488 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 489 struct sk_buff *skb; 490 void *data; 491 492 len += NET_SKB_PAD + NET_IP_ALIGN; 493 494 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) || 495 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 496 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 497 if (!skb) 498 goto skb_fail; 499 goto skb_success; 500 } 501 502 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 503 len = SKB_DATA_ALIGN(len); 504 505 if (sk_memalloc_socks()) 506 gfp_mask |= __GFP_MEMALLOC; 507 508 data = __alloc_page_frag(&nc->page, len, gfp_mask); 509 if (unlikely(!data)) 510 return NULL; 511 512 skb = __build_skb(data, len); 513 if (unlikely(!skb)) { 514 skb_free_frag(data); 515 return NULL; 516 } 517 518 /* use OR instead of assignment to avoid clearing of bits in mask */ 519 if (nc->page.pfmemalloc) 520 skb->pfmemalloc = 1; 521 skb->head_frag = 1; 522 523 skb_success: 524 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 525 skb->dev = napi->dev; 526 527 skb_fail: 528 return skb; 529 } 530 EXPORT_SYMBOL(__napi_alloc_skb); 531 532 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 533 int size, unsigned int truesize) 534 { 535 skb_fill_page_desc(skb, i, page, off, size); 536 skb->len += size; 537 skb->data_len += size; 538 skb->truesize += truesize; 539 } 540 EXPORT_SYMBOL(skb_add_rx_frag); 541 542 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 543 unsigned int truesize) 544 { 545 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 546 547 skb_frag_size_add(frag, size); 548 skb->len += size; 549 skb->data_len += size; 550 skb->truesize += truesize; 551 } 552 EXPORT_SYMBOL(skb_coalesce_rx_frag); 553 554 static void skb_drop_list(struct sk_buff **listp) 555 { 556 kfree_skb_list(*listp); 557 *listp = NULL; 558 } 559 560 static inline void skb_drop_fraglist(struct sk_buff *skb) 561 { 562 skb_drop_list(&skb_shinfo(skb)->frag_list); 563 } 564 565 static void skb_clone_fraglist(struct sk_buff *skb) 566 { 567 struct sk_buff *list; 568 569 skb_walk_frags(skb, list) 570 skb_get(list); 571 } 572 573 static void skb_free_head(struct sk_buff *skb) 574 { 575 unsigned char *head = skb->head; 576 577 if (skb->head_frag) 578 skb_free_frag(head); 579 else 580 kfree(head); 581 } 582 583 static void skb_release_data(struct sk_buff *skb) 584 { 585 struct skb_shared_info *shinfo = skb_shinfo(skb); 586 int i; 587 588 if (skb->cloned && 589 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 590 &shinfo->dataref)) 591 return; 592 593 for (i = 0; i < shinfo->nr_frags; i++) 594 __skb_frag_unref(&shinfo->frags[i]); 595 596 /* 597 * If skb buf is from userspace, we need to notify the caller 598 * the lower device DMA has done; 599 */ 600 if (shinfo->tx_flags & SKBTX_DEV_ZEROCOPY) { 601 struct ubuf_info *uarg; 602 603 uarg = shinfo->destructor_arg; 604 if (uarg->callback) 605 uarg->callback(uarg, true); 606 } 607 608 if (shinfo->frag_list) 609 kfree_skb_list(shinfo->frag_list); 610 611 skb_free_head(skb); 612 } 613 614 /* 615 * Free an skbuff by memory without cleaning the state. 616 */ 617 static void kfree_skbmem(struct sk_buff *skb) 618 { 619 struct sk_buff_fclones *fclones; 620 621 switch (skb->fclone) { 622 case SKB_FCLONE_UNAVAILABLE: 623 kmem_cache_free(skbuff_head_cache, skb); 624 return; 625 626 case SKB_FCLONE_ORIG: 627 fclones = container_of(skb, struct sk_buff_fclones, skb1); 628 629 /* We usually free the clone (TX completion) before original skb 630 * This test would have no chance to be true for the clone, 631 * while here, branch prediction will be good. 632 */ 633 if (atomic_read(&fclones->fclone_ref) == 1) 634 goto fastpath; 635 break; 636 637 default: /* SKB_FCLONE_CLONE */ 638 fclones = container_of(skb, struct sk_buff_fclones, skb2); 639 break; 640 } 641 if (!atomic_dec_and_test(&fclones->fclone_ref)) 642 return; 643 fastpath: 644 kmem_cache_free(skbuff_fclone_cache, fclones); 645 } 646 647 static void skb_release_head_state(struct sk_buff *skb) 648 { 649 skb_dst_drop(skb); 650 #ifdef CONFIG_XFRM 651 secpath_put(skb->sp); 652 #endif 653 if (skb->destructor) { 654 WARN_ON(in_irq()); 655 skb->destructor(skb); 656 } 657 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 658 nf_conntrack_put(skb->nfct); 659 #endif 660 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 661 nf_bridge_put(skb->nf_bridge); 662 #endif 663 } 664 665 /* Free everything but the sk_buff shell. */ 666 static void skb_release_all(struct sk_buff *skb) 667 { 668 skb_release_head_state(skb); 669 if (likely(skb->head)) 670 skb_release_data(skb); 671 } 672 673 /** 674 * __kfree_skb - private function 675 * @skb: buffer 676 * 677 * Free an sk_buff. Release anything attached to the buffer. 678 * Clean the state. This is an internal helper function. Users should 679 * always call kfree_skb 680 */ 681 682 void __kfree_skb(struct sk_buff *skb) 683 { 684 skb_release_all(skb); 685 kfree_skbmem(skb); 686 } 687 EXPORT_SYMBOL(__kfree_skb); 688 689 /** 690 * kfree_skb - free an sk_buff 691 * @skb: buffer to free 692 * 693 * Drop a reference to the buffer and free it if the usage count has 694 * hit zero. 695 */ 696 void kfree_skb(struct sk_buff *skb) 697 { 698 if (unlikely(!skb)) 699 return; 700 if (likely(atomic_read(&skb->users) == 1)) 701 smp_rmb(); 702 else if (likely(!atomic_dec_and_test(&skb->users))) 703 return; 704 trace_kfree_skb(skb, __builtin_return_address(0)); 705 __kfree_skb(skb); 706 } 707 EXPORT_SYMBOL(kfree_skb); 708 709 void kfree_skb_list(struct sk_buff *segs) 710 { 711 while (segs) { 712 struct sk_buff *next = segs->next; 713 714 kfree_skb(segs); 715 segs = next; 716 } 717 } 718 EXPORT_SYMBOL(kfree_skb_list); 719 720 /** 721 * skb_tx_error - report an sk_buff xmit error 722 * @skb: buffer that triggered an error 723 * 724 * Report xmit error if a device callback is tracking this skb. 725 * skb must be freed afterwards. 726 */ 727 void skb_tx_error(struct sk_buff *skb) 728 { 729 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) { 730 struct ubuf_info *uarg; 731 732 uarg = skb_shinfo(skb)->destructor_arg; 733 if (uarg->callback) 734 uarg->callback(uarg, false); 735 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY; 736 } 737 } 738 EXPORT_SYMBOL(skb_tx_error); 739 740 /** 741 * consume_skb - free an skbuff 742 * @skb: buffer to free 743 * 744 * Drop a ref to the buffer and free it if the usage count has hit zero 745 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 746 * is being dropped after a failure and notes that 747 */ 748 void consume_skb(struct sk_buff *skb) 749 { 750 if (unlikely(!skb)) 751 return; 752 if (likely(atomic_read(&skb->users) == 1)) 753 smp_rmb(); 754 else if (likely(!atomic_dec_and_test(&skb->users))) 755 return; 756 trace_consume_skb(skb); 757 __kfree_skb(skb); 758 } 759 EXPORT_SYMBOL(consume_skb); 760 761 void __kfree_skb_flush(void) 762 { 763 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 764 765 /* flush skb_cache if containing objects */ 766 if (nc->skb_count) { 767 kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count, 768 nc->skb_cache); 769 nc->skb_count = 0; 770 } 771 } 772 773 static inline void _kfree_skb_defer(struct sk_buff *skb) 774 { 775 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 776 777 /* drop skb->head and call any destructors for packet */ 778 skb_release_all(skb); 779 780 /* record skb to CPU local list */ 781 nc->skb_cache[nc->skb_count++] = skb; 782 783 #ifdef CONFIG_SLUB 784 /* SLUB writes into objects when freeing */ 785 prefetchw(skb); 786 #endif 787 788 /* flush skb_cache if it is filled */ 789 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 790 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE, 791 nc->skb_cache); 792 nc->skb_count = 0; 793 } 794 } 795 void __kfree_skb_defer(struct sk_buff *skb) 796 { 797 _kfree_skb_defer(skb); 798 } 799 800 void napi_consume_skb(struct sk_buff *skb, int budget) 801 { 802 if (unlikely(!skb)) 803 return; 804 805 /* Zero budget indicate non-NAPI context called us, like netpoll */ 806 if (unlikely(!budget)) { 807 dev_consume_skb_any(skb); 808 return; 809 } 810 811 if (likely(atomic_read(&skb->users) == 1)) 812 smp_rmb(); 813 else if (likely(!atomic_dec_and_test(&skb->users))) 814 return; 815 /* if reaching here SKB is ready to free */ 816 trace_consume_skb(skb); 817 818 /* if SKB is a clone, don't handle this case */ 819 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 820 __kfree_skb(skb); 821 return; 822 } 823 824 _kfree_skb_defer(skb); 825 } 826 EXPORT_SYMBOL(napi_consume_skb); 827 828 /* Make sure a field is enclosed inside headers_start/headers_end section */ 829 #define CHECK_SKB_FIELD(field) \ 830 BUILD_BUG_ON(offsetof(struct sk_buff, field) < \ 831 offsetof(struct sk_buff, headers_start)); \ 832 BUILD_BUG_ON(offsetof(struct sk_buff, field) > \ 833 offsetof(struct sk_buff, headers_end)); \ 834 835 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 836 { 837 new->tstamp = old->tstamp; 838 /* We do not copy old->sk */ 839 new->dev = old->dev; 840 memcpy(new->cb, old->cb, sizeof(old->cb)); 841 skb_dst_copy(new, old); 842 #ifdef CONFIG_XFRM 843 new->sp = secpath_get(old->sp); 844 #endif 845 __nf_copy(new, old, false); 846 847 /* Note : this field could be in headers_start/headers_end section 848 * It is not yet because we do not want to have a 16 bit hole 849 */ 850 new->queue_mapping = old->queue_mapping; 851 852 memcpy(&new->headers_start, &old->headers_start, 853 offsetof(struct sk_buff, headers_end) - 854 offsetof(struct sk_buff, headers_start)); 855 CHECK_SKB_FIELD(protocol); 856 CHECK_SKB_FIELD(csum); 857 CHECK_SKB_FIELD(hash); 858 CHECK_SKB_FIELD(priority); 859 CHECK_SKB_FIELD(skb_iif); 860 CHECK_SKB_FIELD(vlan_proto); 861 CHECK_SKB_FIELD(vlan_tci); 862 CHECK_SKB_FIELD(transport_header); 863 CHECK_SKB_FIELD(network_header); 864 CHECK_SKB_FIELD(mac_header); 865 CHECK_SKB_FIELD(inner_protocol); 866 CHECK_SKB_FIELD(inner_transport_header); 867 CHECK_SKB_FIELD(inner_network_header); 868 CHECK_SKB_FIELD(inner_mac_header); 869 CHECK_SKB_FIELD(mark); 870 #ifdef CONFIG_NETWORK_SECMARK 871 CHECK_SKB_FIELD(secmark); 872 #endif 873 #ifdef CONFIG_NET_RX_BUSY_POLL 874 CHECK_SKB_FIELD(napi_id); 875 #endif 876 #ifdef CONFIG_XPS 877 CHECK_SKB_FIELD(sender_cpu); 878 #endif 879 #ifdef CONFIG_NET_SCHED 880 CHECK_SKB_FIELD(tc_index); 881 #ifdef CONFIG_NET_CLS_ACT 882 CHECK_SKB_FIELD(tc_verd); 883 #endif 884 #endif 885 886 } 887 888 /* 889 * You should not add any new code to this function. Add it to 890 * __copy_skb_header above instead. 891 */ 892 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 893 { 894 #define C(x) n->x = skb->x 895 896 n->next = n->prev = NULL; 897 n->sk = NULL; 898 __copy_skb_header(n, skb); 899 900 C(len); 901 C(data_len); 902 C(mac_len); 903 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 904 n->cloned = 1; 905 n->nohdr = 0; 906 n->destructor = NULL; 907 C(tail); 908 C(end); 909 C(head); 910 C(head_frag); 911 C(data); 912 C(truesize); 913 atomic_set(&n->users, 1); 914 915 atomic_inc(&(skb_shinfo(skb)->dataref)); 916 skb->cloned = 1; 917 918 return n; 919 #undef C 920 } 921 922 /** 923 * skb_morph - morph one skb into another 924 * @dst: the skb to receive the contents 925 * @src: the skb to supply the contents 926 * 927 * This is identical to skb_clone except that the target skb is 928 * supplied by the user. 929 * 930 * The target skb is returned upon exit. 931 */ 932 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 933 { 934 skb_release_all(dst); 935 return __skb_clone(dst, src); 936 } 937 EXPORT_SYMBOL_GPL(skb_morph); 938 939 /** 940 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 941 * @skb: the skb to modify 942 * @gfp_mask: allocation priority 943 * 944 * This must be called on SKBTX_DEV_ZEROCOPY skb. 945 * It will copy all frags into kernel and drop the reference 946 * to userspace pages. 947 * 948 * If this function is called from an interrupt gfp_mask() must be 949 * %GFP_ATOMIC. 950 * 951 * Returns 0 on success or a negative error code on failure 952 * to allocate kernel memory to copy to. 953 */ 954 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 955 { 956 int i; 957 int num_frags = skb_shinfo(skb)->nr_frags; 958 struct page *page, *head = NULL; 959 struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg; 960 961 for (i = 0; i < num_frags; i++) { 962 u8 *vaddr; 963 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 964 965 page = alloc_page(gfp_mask); 966 if (!page) { 967 while (head) { 968 struct page *next = (struct page *)page_private(head); 969 put_page(head); 970 head = next; 971 } 972 return -ENOMEM; 973 } 974 vaddr = kmap_atomic(skb_frag_page(f)); 975 memcpy(page_address(page), 976 vaddr + f->page_offset, skb_frag_size(f)); 977 kunmap_atomic(vaddr); 978 set_page_private(page, (unsigned long)head); 979 head = page; 980 } 981 982 /* skb frags release userspace buffers */ 983 for (i = 0; i < num_frags; i++) 984 skb_frag_unref(skb, i); 985 986 uarg->callback(uarg, false); 987 988 /* skb frags point to kernel buffers */ 989 for (i = num_frags - 1; i >= 0; i--) { 990 __skb_fill_page_desc(skb, i, head, 0, 991 skb_shinfo(skb)->frags[i].size); 992 head = (struct page *)page_private(head); 993 } 994 995 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY; 996 return 0; 997 } 998 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 999 1000 /** 1001 * skb_clone - duplicate an sk_buff 1002 * @skb: buffer to clone 1003 * @gfp_mask: allocation priority 1004 * 1005 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 1006 * copies share the same packet data but not structure. The new 1007 * buffer has a reference count of 1. If the allocation fails the 1008 * function returns %NULL otherwise the new buffer is returned. 1009 * 1010 * If this function is called from an interrupt gfp_mask() must be 1011 * %GFP_ATOMIC. 1012 */ 1013 1014 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 1015 { 1016 struct sk_buff_fclones *fclones = container_of(skb, 1017 struct sk_buff_fclones, 1018 skb1); 1019 struct sk_buff *n; 1020 1021 if (skb_orphan_frags(skb, gfp_mask)) 1022 return NULL; 1023 1024 if (skb->fclone == SKB_FCLONE_ORIG && 1025 atomic_read(&fclones->fclone_ref) == 1) { 1026 n = &fclones->skb2; 1027 atomic_set(&fclones->fclone_ref, 2); 1028 } else { 1029 if (skb_pfmemalloc(skb)) 1030 gfp_mask |= __GFP_MEMALLOC; 1031 1032 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); 1033 if (!n) 1034 return NULL; 1035 1036 kmemcheck_annotate_bitfield(n, flags1); 1037 n->fclone = SKB_FCLONE_UNAVAILABLE; 1038 } 1039 1040 return __skb_clone(n, skb); 1041 } 1042 EXPORT_SYMBOL(skb_clone); 1043 1044 static void skb_headers_offset_update(struct sk_buff *skb, int off) 1045 { 1046 /* Only adjust this if it actually is csum_start rather than csum */ 1047 if (skb->ip_summed == CHECKSUM_PARTIAL) 1048 skb->csum_start += off; 1049 /* {transport,network,mac}_header and tail are relative to skb->head */ 1050 skb->transport_header += off; 1051 skb->network_header += off; 1052 if (skb_mac_header_was_set(skb)) 1053 skb->mac_header += off; 1054 skb->inner_transport_header += off; 1055 skb->inner_network_header += off; 1056 skb->inner_mac_header += off; 1057 } 1058 1059 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 1060 { 1061 __copy_skb_header(new, old); 1062 1063 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 1064 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 1065 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 1066 } 1067 1068 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 1069 { 1070 if (skb_pfmemalloc(skb)) 1071 return SKB_ALLOC_RX; 1072 return 0; 1073 } 1074 1075 /** 1076 * skb_copy - create private copy of an sk_buff 1077 * @skb: buffer to copy 1078 * @gfp_mask: allocation priority 1079 * 1080 * Make a copy of both an &sk_buff and its data. This is used when the 1081 * caller wishes to modify the data and needs a private copy of the 1082 * data to alter. Returns %NULL on failure or the pointer to the buffer 1083 * on success. The returned buffer has a reference count of 1. 1084 * 1085 * As by-product this function converts non-linear &sk_buff to linear 1086 * one, so that &sk_buff becomes completely private and caller is allowed 1087 * to modify all the data of returned buffer. This means that this 1088 * function is not recommended for use in circumstances when only 1089 * header is going to be modified. Use pskb_copy() instead. 1090 */ 1091 1092 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 1093 { 1094 int headerlen = skb_headroom(skb); 1095 unsigned int size = skb_end_offset(skb) + skb->data_len; 1096 struct sk_buff *n = __alloc_skb(size, gfp_mask, 1097 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 1098 1099 if (!n) 1100 return NULL; 1101 1102 /* Set the data pointer */ 1103 skb_reserve(n, headerlen); 1104 /* Set the tail pointer and length */ 1105 skb_put(n, skb->len); 1106 1107 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)) 1108 BUG(); 1109 1110 copy_skb_header(n, skb); 1111 return n; 1112 } 1113 EXPORT_SYMBOL(skb_copy); 1114 1115 /** 1116 * __pskb_copy_fclone - create copy of an sk_buff with private head. 1117 * @skb: buffer to copy 1118 * @headroom: headroom of new skb 1119 * @gfp_mask: allocation priority 1120 * @fclone: if true allocate the copy of the skb from the fclone 1121 * cache instead of the head cache; it is recommended to set this 1122 * to true for the cases where the copy will likely be cloned 1123 * 1124 * Make a copy of both an &sk_buff and part of its data, located 1125 * in header. Fragmented data remain shared. This is used when 1126 * the caller wishes to modify only header of &sk_buff and needs 1127 * private copy of the header to alter. Returns %NULL on failure 1128 * or the pointer to the buffer on success. 1129 * The returned buffer has a reference count of 1. 1130 */ 1131 1132 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 1133 gfp_t gfp_mask, bool fclone) 1134 { 1135 unsigned int size = skb_headlen(skb) + headroom; 1136 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 1137 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 1138 1139 if (!n) 1140 goto out; 1141 1142 /* Set the data pointer */ 1143 skb_reserve(n, headroom); 1144 /* Set the tail pointer and length */ 1145 skb_put(n, skb_headlen(skb)); 1146 /* Copy the bytes */ 1147 skb_copy_from_linear_data(skb, n->data, n->len); 1148 1149 n->truesize += skb->data_len; 1150 n->data_len = skb->data_len; 1151 n->len = skb->len; 1152 1153 if (skb_shinfo(skb)->nr_frags) { 1154 int i; 1155 1156 if (skb_orphan_frags(skb, gfp_mask)) { 1157 kfree_skb(n); 1158 n = NULL; 1159 goto out; 1160 } 1161 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1162 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 1163 skb_frag_ref(skb, i); 1164 } 1165 skb_shinfo(n)->nr_frags = i; 1166 } 1167 1168 if (skb_has_frag_list(skb)) { 1169 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 1170 skb_clone_fraglist(n); 1171 } 1172 1173 copy_skb_header(n, skb); 1174 out: 1175 return n; 1176 } 1177 EXPORT_SYMBOL(__pskb_copy_fclone); 1178 1179 /** 1180 * pskb_expand_head - reallocate header of &sk_buff 1181 * @skb: buffer to reallocate 1182 * @nhead: room to add at head 1183 * @ntail: room to add at tail 1184 * @gfp_mask: allocation priority 1185 * 1186 * Expands (or creates identical copy, if @nhead and @ntail are zero) 1187 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 1188 * reference count of 1. Returns zero in the case of success or error, 1189 * if expansion failed. In the last case, &sk_buff is not changed. 1190 * 1191 * All the pointers pointing into skb header may change and must be 1192 * reloaded after call to this function. 1193 */ 1194 1195 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 1196 gfp_t gfp_mask) 1197 { 1198 int i; 1199 u8 *data; 1200 int size = nhead + skb_end_offset(skb) + ntail; 1201 long off; 1202 1203 BUG_ON(nhead < 0); 1204 1205 if (skb_shared(skb)) 1206 BUG(); 1207 1208 size = SKB_DATA_ALIGN(size); 1209 1210 if (skb_pfmemalloc(skb)) 1211 gfp_mask |= __GFP_MEMALLOC; 1212 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 1213 gfp_mask, NUMA_NO_NODE, NULL); 1214 if (!data) 1215 goto nodata; 1216 size = SKB_WITH_OVERHEAD(ksize(data)); 1217 1218 /* Copy only real data... and, alas, header. This should be 1219 * optimized for the cases when header is void. 1220 */ 1221 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 1222 1223 memcpy((struct skb_shared_info *)(data + size), 1224 skb_shinfo(skb), 1225 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 1226 1227 /* 1228 * if shinfo is shared we must drop the old head gracefully, but if it 1229 * is not we can just drop the old head and let the existing refcount 1230 * be since all we did is relocate the values 1231 */ 1232 if (skb_cloned(skb)) { 1233 /* copy this zero copy skb frags */ 1234 if (skb_orphan_frags(skb, gfp_mask)) 1235 goto nofrags; 1236 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1237 skb_frag_ref(skb, i); 1238 1239 if (skb_has_frag_list(skb)) 1240 skb_clone_fraglist(skb); 1241 1242 skb_release_data(skb); 1243 } else { 1244 skb_free_head(skb); 1245 } 1246 off = (data + nhead) - skb->head; 1247 1248 skb->head = data; 1249 skb->head_frag = 0; 1250 skb->data += off; 1251 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1252 skb->end = size; 1253 off = nhead; 1254 #else 1255 skb->end = skb->head + size; 1256 #endif 1257 skb->tail += off; 1258 skb_headers_offset_update(skb, nhead); 1259 skb->cloned = 0; 1260 skb->hdr_len = 0; 1261 skb->nohdr = 0; 1262 atomic_set(&skb_shinfo(skb)->dataref, 1); 1263 return 0; 1264 1265 nofrags: 1266 kfree(data); 1267 nodata: 1268 return -ENOMEM; 1269 } 1270 EXPORT_SYMBOL(pskb_expand_head); 1271 1272 /* Make private copy of skb with writable head and some headroom */ 1273 1274 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 1275 { 1276 struct sk_buff *skb2; 1277 int delta = headroom - skb_headroom(skb); 1278 1279 if (delta <= 0) 1280 skb2 = pskb_copy(skb, GFP_ATOMIC); 1281 else { 1282 skb2 = skb_clone(skb, GFP_ATOMIC); 1283 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 1284 GFP_ATOMIC)) { 1285 kfree_skb(skb2); 1286 skb2 = NULL; 1287 } 1288 } 1289 return skb2; 1290 } 1291 EXPORT_SYMBOL(skb_realloc_headroom); 1292 1293 /** 1294 * skb_copy_expand - copy and expand sk_buff 1295 * @skb: buffer to copy 1296 * @newheadroom: new free bytes at head 1297 * @newtailroom: new free bytes at tail 1298 * @gfp_mask: allocation priority 1299 * 1300 * Make a copy of both an &sk_buff and its data and while doing so 1301 * allocate additional space. 1302 * 1303 * This is used when the caller wishes to modify the data and needs a 1304 * private copy of the data to alter as well as more space for new fields. 1305 * Returns %NULL on failure or the pointer to the buffer 1306 * on success. The returned buffer has a reference count of 1. 1307 * 1308 * You must pass %GFP_ATOMIC as the allocation priority if this function 1309 * is called from an interrupt. 1310 */ 1311 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 1312 int newheadroom, int newtailroom, 1313 gfp_t gfp_mask) 1314 { 1315 /* 1316 * Allocate the copy buffer 1317 */ 1318 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 1319 gfp_mask, skb_alloc_rx_flag(skb), 1320 NUMA_NO_NODE); 1321 int oldheadroom = skb_headroom(skb); 1322 int head_copy_len, head_copy_off; 1323 1324 if (!n) 1325 return NULL; 1326 1327 skb_reserve(n, newheadroom); 1328 1329 /* Set the tail pointer and length */ 1330 skb_put(n, skb->len); 1331 1332 head_copy_len = oldheadroom; 1333 head_copy_off = 0; 1334 if (newheadroom <= head_copy_len) 1335 head_copy_len = newheadroom; 1336 else 1337 head_copy_off = newheadroom - head_copy_len; 1338 1339 /* Copy the linear header and data. */ 1340 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 1341 skb->len + head_copy_len)) 1342 BUG(); 1343 1344 copy_skb_header(n, skb); 1345 1346 skb_headers_offset_update(n, newheadroom - oldheadroom); 1347 1348 return n; 1349 } 1350 EXPORT_SYMBOL(skb_copy_expand); 1351 1352 /** 1353 * skb_pad - zero pad the tail of an skb 1354 * @skb: buffer to pad 1355 * @pad: space to pad 1356 * 1357 * Ensure that a buffer is followed by a padding area that is zero 1358 * filled. Used by network drivers which may DMA or transfer data 1359 * beyond the buffer end onto the wire. 1360 * 1361 * May return error in out of memory cases. The skb is freed on error. 1362 */ 1363 1364 int skb_pad(struct sk_buff *skb, int pad) 1365 { 1366 int err; 1367 int ntail; 1368 1369 /* If the skbuff is non linear tailroom is always zero.. */ 1370 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 1371 memset(skb->data+skb->len, 0, pad); 1372 return 0; 1373 } 1374 1375 ntail = skb->data_len + pad - (skb->end - skb->tail); 1376 if (likely(skb_cloned(skb) || ntail > 0)) { 1377 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 1378 if (unlikely(err)) 1379 goto free_skb; 1380 } 1381 1382 /* FIXME: The use of this function with non-linear skb's really needs 1383 * to be audited. 1384 */ 1385 err = skb_linearize(skb); 1386 if (unlikely(err)) 1387 goto free_skb; 1388 1389 memset(skb->data + skb->len, 0, pad); 1390 return 0; 1391 1392 free_skb: 1393 kfree_skb(skb); 1394 return err; 1395 } 1396 EXPORT_SYMBOL(skb_pad); 1397 1398 /** 1399 * pskb_put - add data to the tail of a potentially fragmented buffer 1400 * @skb: start of the buffer to use 1401 * @tail: tail fragment of the buffer to use 1402 * @len: amount of data to add 1403 * 1404 * This function extends the used data area of the potentially 1405 * fragmented buffer. @tail must be the last fragment of @skb -- or 1406 * @skb itself. If this would exceed the total buffer size the kernel 1407 * will panic. A pointer to the first byte of the extra data is 1408 * returned. 1409 */ 1410 1411 unsigned char *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 1412 { 1413 if (tail != skb) { 1414 skb->data_len += len; 1415 skb->len += len; 1416 } 1417 return skb_put(tail, len); 1418 } 1419 EXPORT_SYMBOL_GPL(pskb_put); 1420 1421 /** 1422 * skb_put - add data to a buffer 1423 * @skb: buffer to use 1424 * @len: amount of data to add 1425 * 1426 * This function extends the used data area of the buffer. If this would 1427 * exceed the total buffer size the kernel will panic. A pointer to the 1428 * first byte of the extra data is returned. 1429 */ 1430 unsigned char *skb_put(struct sk_buff *skb, unsigned int len) 1431 { 1432 unsigned char *tmp = skb_tail_pointer(skb); 1433 SKB_LINEAR_ASSERT(skb); 1434 skb->tail += len; 1435 skb->len += len; 1436 if (unlikely(skb->tail > skb->end)) 1437 skb_over_panic(skb, len, __builtin_return_address(0)); 1438 return tmp; 1439 } 1440 EXPORT_SYMBOL(skb_put); 1441 1442 /** 1443 * skb_push - add data to the start of a buffer 1444 * @skb: buffer to use 1445 * @len: amount of data to add 1446 * 1447 * This function extends the used data area of the buffer at the buffer 1448 * start. If this would exceed the total buffer headroom the kernel will 1449 * panic. A pointer to the first byte of the extra data is returned. 1450 */ 1451 unsigned char *skb_push(struct sk_buff *skb, unsigned int len) 1452 { 1453 skb->data -= len; 1454 skb->len += len; 1455 if (unlikely(skb->data<skb->head)) 1456 skb_under_panic(skb, len, __builtin_return_address(0)); 1457 return skb->data; 1458 } 1459 EXPORT_SYMBOL(skb_push); 1460 1461 /** 1462 * skb_pull - remove data from the start of a buffer 1463 * @skb: buffer to use 1464 * @len: amount of data to remove 1465 * 1466 * This function removes data from the start of a buffer, returning 1467 * the memory to the headroom. A pointer to the next data in the buffer 1468 * is returned. Once the data has been pulled future pushes will overwrite 1469 * the old data. 1470 */ 1471 unsigned char *skb_pull(struct sk_buff *skb, unsigned int len) 1472 { 1473 return skb_pull_inline(skb, len); 1474 } 1475 EXPORT_SYMBOL(skb_pull); 1476 1477 /** 1478 * skb_trim - remove end from a buffer 1479 * @skb: buffer to alter 1480 * @len: new length 1481 * 1482 * Cut the length of a buffer down by removing data from the tail. If 1483 * the buffer is already under the length specified it is not modified. 1484 * The skb must be linear. 1485 */ 1486 void skb_trim(struct sk_buff *skb, unsigned int len) 1487 { 1488 if (skb->len > len) 1489 __skb_trim(skb, len); 1490 } 1491 EXPORT_SYMBOL(skb_trim); 1492 1493 /* Trims skb to length len. It can change skb pointers. 1494 */ 1495 1496 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 1497 { 1498 struct sk_buff **fragp; 1499 struct sk_buff *frag; 1500 int offset = skb_headlen(skb); 1501 int nfrags = skb_shinfo(skb)->nr_frags; 1502 int i; 1503 int err; 1504 1505 if (skb_cloned(skb) && 1506 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 1507 return err; 1508 1509 i = 0; 1510 if (offset >= len) 1511 goto drop_pages; 1512 1513 for (; i < nfrags; i++) { 1514 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 1515 1516 if (end < len) { 1517 offset = end; 1518 continue; 1519 } 1520 1521 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 1522 1523 drop_pages: 1524 skb_shinfo(skb)->nr_frags = i; 1525 1526 for (; i < nfrags; i++) 1527 skb_frag_unref(skb, i); 1528 1529 if (skb_has_frag_list(skb)) 1530 skb_drop_fraglist(skb); 1531 goto done; 1532 } 1533 1534 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 1535 fragp = &frag->next) { 1536 int end = offset + frag->len; 1537 1538 if (skb_shared(frag)) { 1539 struct sk_buff *nfrag; 1540 1541 nfrag = skb_clone(frag, GFP_ATOMIC); 1542 if (unlikely(!nfrag)) 1543 return -ENOMEM; 1544 1545 nfrag->next = frag->next; 1546 consume_skb(frag); 1547 frag = nfrag; 1548 *fragp = frag; 1549 } 1550 1551 if (end < len) { 1552 offset = end; 1553 continue; 1554 } 1555 1556 if (end > len && 1557 unlikely((err = pskb_trim(frag, len - offset)))) 1558 return err; 1559 1560 if (frag->next) 1561 skb_drop_list(&frag->next); 1562 break; 1563 } 1564 1565 done: 1566 if (len > skb_headlen(skb)) { 1567 skb->data_len -= skb->len - len; 1568 skb->len = len; 1569 } else { 1570 skb->len = len; 1571 skb->data_len = 0; 1572 skb_set_tail_pointer(skb, len); 1573 } 1574 1575 return 0; 1576 } 1577 EXPORT_SYMBOL(___pskb_trim); 1578 1579 /** 1580 * __pskb_pull_tail - advance tail of skb header 1581 * @skb: buffer to reallocate 1582 * @delta: number of bytes to advance tail 1583 * 1584 * The function makes a sense only on a fragmented &sk_buff, 1585 * it expands header moving its tail forward and copying necessary 1586 * data from fragmented part. 1587 * 1588 * &sk_buff MUST have reference count of 1. 1589 * 1590 * Returns %NULL (and &sk_buff does not change) if pull failed 1591 * or value of new tail of skb in the case of success. 1592 * 1593 * All the pointers pointing into skb header may change and must be 1594 * reloaded after call to this function. 1595 */ 1596 1597 /* Moves tail of skb head forward, copying data from fragmented part, 1598 * when it is necessary. 1599 * 1. It may fail due to malloc failure. 1600 * 2. It may change skb pointers. 1601 * 1602 * It is pretty complicated. Luckily, it is called only in exceptional cases. 1603 */ 1604 unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta) 1605 { 1606 /* If skb has not enough free space at tail, get new one 1607 * plus 128 bytes for future expansions. If we have enough 1608 * room at tail, reallocate without expansion only if skb is cloned. 1609 */ 1610 int i, k, eat = (skb->tail + delta) - skb->end; 1611 1612 if (eat > 0 || skb_cloned(skb)) { 1613 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 1614 GFP_ATOMIC)) 1615 return NULL; 1616 } 1617 1618 if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta)) 1619 BUG(); 1620 1621 /* Optimization: no fragments, no reasons to preestimate 1622 * size of pulled pages. Superb. 1623 */ 1624 if (!skb_has_frag_list(skb)) 1625 goto pull_pages; 1626 1627 /* Estimate size of pulled pages. */ 1628 eat = delta; 1629 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1630 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 1631 1632 if (size >= eat) 1633 goto pull_pages; 1634 eat -= size; 1635 } 1636 1637 /* If we need update frag list, we are in troubles. 1638 * Certainly, it possible to add an offset to skb data, 1639 * but taking into account that pulling is expected to 1640 * be very rare operation, it is worth to fight against 1641 * further bloating skb head and crucify ourselves here instead. 1642 * Pure masohism, indeed. 8)8) 1643 */ 1644 if (eat) { 1645 struct sk_buff *list = skb_shinfo(skb)->frag_list; 1646 struct sk_buff *clone = NULL; 1647 struct sk_buff *insp = NULL; 1648 1649 do { 1650 BUG_ON(!list); 1651 1652 if (list->len <= eat) { 1653 /* Eaten as whole. */ 1654 eat -= list->len; 1655 list = list->next; 1656 insp = list; 1657 } else { 1658 /* Eaten partially. */ 1659 1660 if (skb_shared(list)) { 1661 /* Sucks! We need to fork list. :-( */ 1662 clone = skb_clone(list, GFP_ATOMIC); 1663 if (!clone) 1664 return NULL; 1665 insp = list->next; 1666 list = clone; 1667 } else { 1668 /* This may be pulled without 1669 * problems. */ 1670 insp = list; 1671 } 1672 if (!pskb_pull(list, eat)) { 1673 kfree_skb(clone); 1674 return NULL; 1675 } 1676 break; 1677 } 1678 } while (eat); 1679 1680 /* Free pulled out fragments. */ 1681 while ((list = skb_shinfo(skb)->frag_list) != insp) { 1682 skb_shinfo(skb)->frag_list = list->next; 1683 kfree_skb(list); 1684 } 1685 /* And insert new clone at head. */ 1686 if (clone) { 1687 clone->next = list; 1688 skb_shinfo(skb)->frag_list = clone; 1689 } 1690 } 1691 /* Success! Now we may commit changes to skb data. */ 1692 1693 pull_pages: 1694 eat = delta; 1695 k = 0; 1696 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1697 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 1698 1699 if (size <= eat) { 1700 skb_frag_unref(skb, i); 1701 eat -= size; 1702 } else { 1703 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; 1704 if (eat) { 1705 skb_shinfo(skb)->frags[k].page_offset += eat; 1706 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat); 1707 eat = 0; 1708 } 1709 k++; 1710 } 1711 } 1712 skb_shinfo(skb)->nr_frags = k; 1713 1714 skb->tail += delta; 1715 skb->data_len -= delta; 1716 1717 return skb_tail_pointer(skb); 1718 } 1719 EXPORT_SYMBOL(__pskb_pull_tail); 1720 1721 /** 1722 * skb_copy_bits - copy bits from skb to kernel buffer 1723 * @skb: source skb 1724 * @offset: offset in source 1725 * @to: destination buffer 1726 * @len: number of bytes to copy 1727 * 1728 * Copy the specified number of bytes from the source skb to the 1729 * destination buffer. 1730 * 1731 * CAUTION ! : 1732 * If its prototype is ever changed, 1733 * check arch/{*}/net/{*}.S files, 1734 * since it is called from BPF assembly code. 1735 */ 1736 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 1737 { 1738 int start = skb_headlen(skb); 1739 struct sk_buff *frag_iter; 1740 int i, copy; 1741 1742 if (offset > (int)skb->len - len) 1743 goto fault; 1744 1745 /* Copy header. */ 1746 if ((copy = start - offset) > 0) { 1747 if (copy > len) 1748 copy = len; 1749 skb_copy_from_linear_data_offset(skb, offset, to, copy); 1750 if ((len -= copy) == 0) 1751 return 0; 1752 offset += copy; 1753 to += copy; 1754 } 1755 1756 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1757 int end; 1758 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1759 1760 WARN_ON(start > offset + len); 1761 1762 end = start + skb_frag_size(f); 1763 if ((copy = end - offset) > 0) { 1764 u8 *vaddr; 1765 1766 if (copy > len) 1767 copy = len; 1768 1769 vaddr = kmap_atomic(skb_frag_page(f)); 1770 memcpy(to, 1771 vaddr + f->page_offset + offset - start, 1772 copy); 1773 kunmap_atomic(vaddr); 1774 1775 if ((len -= copy) == 0) 1776 return 0; 1777 offset += copy; 1778 to += copy; 1779 } 1780 start = end; 1781 } 1782 1783 skb_walk_frags(skb, frag_iter) { 1784 int end; 1785 1786 WARN_ON(start > offset + len); 1787 1788 end = start + frag_iter->len; 1789 if ((copy = end - offset) > 0) { 1790 if (copy > len) 1791 copy = len; 1792 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 1793 goto fault; 1794 if ((len -= copy) == 0) 1795 return 0; 1796 offset += copy; 1797 to += copy; 1798 } 1799 start = end; 1800 } 1801 1802 if (!len) 1803 return 0; 1804 1805 fault: 1806 return -EFAULT; 1807 } 1808 EXPORT_SYMBOL(skb_copy_bits); 1809 1810 /* 1811 * Callback from splice_to_pipe(), if we need to release some pages 1812 * at the end of the spd in case we error'ed out in filling the pipe. 1813 */ 1814 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 1815 { 1816 put_page(spd->pages[i]); 1817 } 1818 1819 static struct page *linear_to_page(struct page *page, unsigned int *len, 1820 unsigned int *offset, 1821 struct sock *sk) 1822 { 1823 struct page_frag *pfrag = sk_page_frag(sk); 1824 1825 if (!sk_page_frag_refill(sk, pfrag)) 1826 return NULL; 1827 1828 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 1829 1830 memcpy(page_address(pfrag->page) + pfrag->offset, 1831 page_address(page) + *offset, *len); 1832 *offset = pfrag->offset; 1833 pfrag->offset += *len; 1834 1835 return pfrag->page; 1836 } 1837 1838 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 1839 struct page *page, 1840 unsigned int offset) 1841 { 1842 return spd->nr_pages && 1843 spd->pages[spd->nr_pages - 1] == page && 1844 (spd->partial[spd->nr_pages - 1].offset + 1845 spd->partial[spd->nr_pages - 1].len == offset); 1846 } 1847 1848 /* 1849 * Fill page/offset/length into spd, if it can hold more pages. 1850 */ 1851 static bool spd_fill_page(struct splice_pipe_desc *spd, 1852 struct pipe_inode_info *pipe, struct page *page, 1853 unsigned int *len, unsigned int offset, 1854 bool linear, 1855 struct sock *sk) 1856 { 1857 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 1858 return true; 1859 1860 if (linear) { 1861 page = linear_to_page(page, len, &offset, sk); 1862 if (!page) 1863 return true; 1864 } 1865 if (spd_can_coalesce(spd, page, offset)) { 1866 spd->partial[spd->nr_pages - 1].len += *len; 1867 return false; 1868 } 1869 get_page(page); 1870 spd->pages[spd->nr_pages] = page; 1871 spd->partial[spd->nr_pages].len = *len; 1872 spd->partial[spd->nr_pages].offset = offset; 1873 spd->nr_pages++; 1874 1875 return false; 1876 } 1877 1878 static bool __splice_segment(struct page *page, unsigned int poff, 1879 unsigned int plen, unsigned int *off, 1880 unsigned int *len, 1881 struct splice_pipe_desc *spd, bool linear, 1882 struct sock *sk, 1883 struct pipe_inode_info *pipe) 1884 { 1885 if (!*len) 1886 return true; 1887 1888 /* skip this segment if already processed */ 1889 if (*off >= plen) { 1890 *off -= plen; 1891 return false; 1892 } 1893 1894 /* ignore any bits we already processed */ 1895 poff += *off; 1896 plen -= *off; 1897 *off = 0; 1898 1899 do { 1900 unsigned int flen = min(*len, plen); 1901 1902 if (spd_fill_page(spd, pipe, page, &flen, poff, 1903 linear, sk)) 1904 return true; 1905 poff += flen; 1906 plen -= flen; 1907 *len -= flen; 1908 } while (*len && plen); 1909 1910 return false; 1911 } 1912 1913 /* 1914 * Map linear and fragment data from the skb to spd. It reports true if the 1915 * pipe is full or if we already spliced the requested length. 1916 */ 1917 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 1918 unsigned int *offset, unsigned int *len, 1919 struct splice_pipe_desc *spd, struct sock *sk) 1920 { 1921 int seg; 1922 struct sk_buff *iter; 1923 1924 /* map the linear part : 1925 * If skb->head_frag is set, this 'linear' part is backed by a 1926 * fragment, and if the head is not shared with any clones then 1927 * we can avoid a copy since we own the head portion of this page. 1928 */ 1929 if (__splice_segment(virt_to_page(skb->data), 1930 (unsigned long) skb->data & (PAGE_SIZE - 1), 1931 skb_headlen(skb), 1932 offset, len, spd, 1933 skb_head_is_locked(skb), 1934 sk, pipe)) 1935 return true; 1936 1937 /* 1938 * then map the fragments 1939 */ 1940 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 1941 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 1942 1943 if (__splice_segment(skb_frag_page(f), 1944 f->page_offset, skb_frag_size(f), 1945 offset, len, spd, false, sk, pipe)) 1946 return true; 1947 } 1948 1949 skb_walk_frags(skb, iter) { 1950 if (*offset >= iter->len) { 1951 *offset -= iter->len; 1952 continue; 1953 } 1954 /* __skb_splice_bits() only fails if the output has no room 1955 * left, so no point in going over the frag_list for the error 1956 * case. 1957 */ 1958 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 1959 return true; 1960 } 1961 1962 return false; 1963 } 1964 1965 /* 1966 * Map data from the skb to a pipe. Should handle both the linear part, 1967 * the fragments, and the frag list. 1968 */ 1969 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 1970 struct pipe_inode_info *pipe, unsigned int tlen, 1971 unsigned int flags) 1972 { 1973 struct partial_page partial[MAX_SKB_FRAGS]; 1974 struct page *pages[MAX_SKB_FRAGS]; 1975 struct splice_pipe_desc spd = { 1976 .pages = pages, 1977 .partial = partial, 1978 .nr_pages_max = MAX_SKB_FRAGS, 1979 .flags = flags, 1980 .ops = &nosteal_pipe_buf_ops, 1981 .spd_release = sock_spd_release, 1982 }; 1983 int ret = 0; 1984 1985 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 1986 1987 if (spd.nr_pages) 1988 ret = splice_to_pipe(pipe, &spd); 1989 1990 return ret; 1991 } 1992 EXPORT_SYMBOL_GPL(skb_splice_bits); 1993 1994 /** 1995 * skb_store_bits - store bits from kernel buffer to skb 1996 * @skb: destination buffer 1997 * @offset: offset in destination 1998 * @from: source buffer 1999 * @len: number of bytes to copy 2000 * 2001 * Copy the specified number of bytes from the source buffer to the 2002 * destination skb. This function handles all the messy bits of 2003 * traversing fragment lists and such. 2004 */ 2005 2006 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 2007 { 2008 int start = skb_headlen(skb); 2009 struct sk_buff *frag_iter; 2010 int i, copy; 2011 2012 if (offset > (int)skb->len - len) 2013 goto fault; 2014 2015 if ((copy = start - offset) > 0) { 2016 if (copy > len) 2017 copy = len; 2018 skb_copy_to_linear_data_offset(skb, offset, from, copy); 2019 if ((len -= copy) == 0) 2020 return 0; 2021 offset += copy; 2022 from += copy; 2023 } 2024 2025 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2026 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2027 int end; 2028 2029 WARN_ON(start > offset + len); 2030 2031 end = start + skb_frag_size(frag); 2032 if ((copy = end - offset) > 0) { 2033 u8 *vaddr; 2034 2035 if (copy > len) 2036 copy = len; 2037 2038 vaddr = kmap_atomic(skb_frag_page(frag)); 2039 memcpy(vaddr + frag->page_offset + offset - start, 2040 from, copy); 2041 kunmap_atomic(vaddr); 2042 2043 if ((len -= copy) == 0) 2044 return 0; 2045 offset += copy; 2046 from += copy; 2047 } 2048 start = end; 2049 } 2050 2051 skb_walk_frags(skb, frag_iter) { 2052 int end; 2053 2054 WARN_ON(start > offset + len); 2055 2056 end = start + frag_iter->len; 2057 if ((copy = end - offset) > 0) { 2058 if (copy > len) 2059 copy = len; 2060 if (skb_store_bits(frag_iter, offset - start, 2061 from, copy)) 2062 goto fault; 2063 if ((len -= copy) == 0) 2064 return 0; 2065 offset += copy; 2066 from += copy; 2067 } 2068 start = end; 2069 } 2070 if (!len) 2071 return 0; 2072 2073 fault: 2074 return -EFAULT; 2075 } 2076 EXPORT_SYMBOL(skb_store_bits); 2077 2078 /* Checksum skb data. */ 2079 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 2080 __wsum csum, const struct skb_checksum_ops *ops) 2081 { 2082 int start = skb_headlen(skb); 2083 int i, copy = start - offset; 2084 struct sk_buff *frag_iter; 2085 int pos = 0; 2086 2087 /* Checksum header. */ 2088 if (copy > 0) { 2089 if (copy > len) 2090 copy = len; 2091 csum = ops->update(skb->data + offset, copy, csum); 2092 if ((len -= copy) == 0) 2093 return csum; 2094 offset += copy; 2095 pos = copy; 2096 } 2097 2098 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2099 int end; 2100 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2101 2102 WARN_ON(start > offset + len); 2103 2104 end = start + skb_frag_size(frag); 2105 if ((copy = end - offset) > 0) { 2106 __wsum csum2; 2107 u8 *vaddr; 2108 2109 if (copy > len) 2110 copy = len; 2111 vaddr = kmap_atomic(skb_frag_page(frag)); 2112 csum2 = ops->update(vaddr + frag->page_offset + 2113 offset - start, copy, 0); 2114 kunmap_atomic(vaddr); 2115 csum = ops->combine(csum, csum2, pos, copy); 2116 if (!(len -= copy)) 2117 return csum; 2118 offset += copy; 2119 pos += copy; 2120 } 2121 start = end; 2122 } 2123 2124 skb_walk_frags(skb, frag_iter) { 2125 int end; 2126 2127 WARN_ON(start > offset + len); 2128 2129 end = start + frag_iter->len; 2130 if ((copy = end - offset) > 0) { 2131 __wsum csum2; 2132 if (copy > len) 2133 copy = len; 2134 csum2 = __skb_checksum(frag_iter, offset - start, 2135 copy, 0, ops); 2136 csum = ops->combine(csum, csum2, pos, copy); 2137 if ((len -= copy) == 0) 2138 return csum; 2139 offset += copy; 2140 pos += copy; 2141 } 2142 start = end; 2143 } 2144 BUG_ON(len); 2145 2146 return csum; 2147 } 2148 EXPORT_SYMBOL(__skb_checksum); 2149 2150 __wsum skb_checksum(const struct sk_buff *skb, int offset, 2151 int len, __wsum csum) 2152 { 2153 const struct skb_checksum_ops ops = { 2154 .update = csum_partial_ext, 2155 .combine = csum_block_add_ext, 2156 }; 2157 2158 return __skb_checksum(skb, offset, len, csum, &ops); 2159 } 2160 EXPORT_SYMBOL(skb_checksum); 2161 2162 /* Both of above in one bottle. */ 2163 2164 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 2165 u8 *to, int len, __wsum csum) 2166 { 2167 int start = skb_headlen(skb); 2168 int i, copy = start - offset; 2169 struct sk_buff *frag_iter; 2170 int pos = 0; 2171 2172 /* Copy header. */ 2173 if (copy > 0) { 2174 if (copy > len) 2175 copy = len; 2176 csum = csum_partial_copy_nocheck(skb->data + offset, to, 2177 copy, csum); 2178 if ((len -= copy) == 0) 2179 return csum; 2180 offset += copy; 2181 to += copy; 2182 pos = copy; 2183 } 2184 2185 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2186 int end; 2187 2188 WARN_ON(start > offset + len); 2189 2190 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2191 if ((copy = end - offset) > 0) { 2192 __wsum csum2; 2193 u8 *vaddr; 2194 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2195 2196 if (copy > len) 2197 copy = len; 2198 vaddr = kmap_atomic(skb_frag_page(frag)); 2199 csum2 = csum_partial_copy_nocheck(vaddr + 2200 frag->page_offset + 2201 offset - start, to, 2202 copy, 0); 2203 kunmap_atomic(vaddr); 2204 csum = csum_block_add(csum, csum2, pos); 2205 if (!(len -= copy)) 2206 return csum; 2207 offset += copy; 2208 to += copy; 2209 pos += copy; 2210 } 2211 start = end; 2212 } 2213 2214 skb_walk_frags(skb, frag_iter) { 2215 __wsum csum2; 2216 int end; 2217 2218 WARN_ON(start > offset + len); 2219 2220 end = start + frag_iter->len; 2221 if ((copy = end - offset) > 0) { 2222 if (copy > len) 2223 copy = len; 2224 csum2 = skb_copy_and_csum_bits(frag_iter, 2225 offset - start, 2226 to, copy, 0); 2227 csum = csum_block_add(csum, csum2, pos); 2228 if ((len -= copy) == 0) 2229 return csum; 2230 offset += copy; 2231 to += copy; 2232 pos += copy; 2233 } 2234 start = end; 2235 } 2236 BUG_ON(len); 2237 return csum; 2238 } 2239 EXPORT_SYMBOL(skb_copy_and_csum_bits); 2240 2241 /** 2242 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 2243 * @from: source buffer 2244 * 2245 * Calculates the amount of linear headroom needed in the 'to' skb passed 2246 * into skb_zerocopy(). 2247 */ 2248 unsigned int 2249 skb_zerocopy_headlen(const struct sk_buff *from) 2250 { 2251 unsigned int hlen = 0; 2252 2253 if (!from->head_frag || 2254 skb_headlen(from) < L1_CACHE_BYTES || 2255 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) 2256 hlen = skb_headlen(from); 2257 2258 if (skb_has_frag_list(from)) 2259 hlen = from->len; 2260 2261 return hlen; 2262 } 2263 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 2264 2265 /** 2266 * skb_zerocopy - Zero copy skb to skb 2267 * @to: destination buffer 2268 * @from: source buffer 2269 * @len: number of bytes to copy from source buffer 2270 * @hlen: size of linear headroom in destination buffer 2271 * 2272 * Copies up to `len` bytes from `from` to `to` by creating references 2273 * to the frags in the source buffer. 2274 * 2275 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 2276 * headroom in the `to` buffer. 2277 * 2278 * Return value: 2279 * 0: everything is OK 2280 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 2281 * -EFAULT: skb_copy_bits() found some problem with skb geometry 2282 */ 2283 int 2284 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 2285 { 2286 int i, j = 0; 2287 int plen = 0; /* length of skb->head fragment */ 2288 int ret; 2289 struct page *page; 2290 unsigned int offset; 2291 2292 BUG_ON(!from->head_frag && !hlen); 2293 2294 /* dont bother with small payloads */ 2295 if (len <= skb_tailroom(to)) 2296 return skb_copy_bits(from, 0, skb_put(to, len), len); 2297 2298 if (hlen) { 2299 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 2300 if (unlikely(ret)) 2301 return ret; 2302 len -= hlen; 2303 } else { 2304 plen = min_t(int, skb_headlen(from), len); 2305 if (plen) { 2306 page = virt_to_head_page(from->head); 2307 offset = from->data - (unsigned char *)page_address(page); 2308 __skb_fill_page_desc(to, 0, page, offset, plen); 2309 get_page(page); 2310 j = 1; 2311 len -= plen; 2312 } 2313 } 2314 2315 to->truesize += len + plen; 2316 to->len += len + plen; 2317 to->data_len += len + plen; 2318 2319 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 2320 skb_tx_error(from); 2321 return -ENOMEM; 2322 } 2323 2324 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 2325 if (!len) 2326 break; 2327 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 2328 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len); 2329 len -= skb_shinfo(to)->frags[j].size; 2330 skb_frag_ref(to, j); 2331 j++; 2332 } 2333 skb_shinfo(to)->nr_frags = j; 2334 2335 return 0; 2336 } 2337 EXPORT_SYMBOL_GPL(skb_zerocopy); 2338 2339 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 2340 { 2341 __wsum csum; 2342 long csstart; 2343 2344 if (skb->ip_summed == CHECKSUM_PARTIAL) 2345 csstart = skb_checksum_start_offset(skb); 2346 else 2347 csstart = skb_headlen(skb); 2348 2349 BUG_ON(csstart > skb_headlen(skb)); 2350 2351 skb_copy_from_linear_data(skb, to, csstart); 2352 2353 csum = 0; 2354 if (csstart != skb->len) 2355 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 2356 skb->len - csstart, 0); 2357 2358 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2359 long csstuff = csstart + skb->csum_offset; 2360 2361 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 2362 } 2363 } 2364 EXPORT_SYMBOL(skb_copy_and_csum_dev); 2365 2366 /** 2367 * skb_dequeue - remove from the head of the queue 2368 * @list: list to dequeue from 2369 * 2370 * Remove the head of the list. The list lock is taken so the function 2371 * may be used safely with other locking list functions. The head item is 2372 * returned or %NULL if the list is empty. 2373 */ 2374 2375 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 2376 { 2377 unsigned long flags; 2378 struct sk_buff *result; 2379 2380 spin_lock_irqsave(&list->lock, flags); 2381 result = __skb_dequeue(list); 2382 spin_unlock_irqrestore(&list->lock, flags); 2383 return result; 2384 } 2385 EXPORT_SYMBOL(skb_dequeue); 2386 2387 /** 2388 * skb_dequeue_tail - remove from the tail of the queue 2389 * @list: list to dequeue from 2390 * 2391 * Remove the tail of the list. The list lock is taken so the function 2392 * may be used safely with other locking list functions. The tail item is 2393 * returned or %NULL if the list is empty. 2394 */ 2395 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 2396 { 2397 unsigned long flags; 2398 struct sk_buff *result; 2399 2400 spin_lock_irqsave(&list->lock, flags); 2401 result = __skb_dequeue_tail(list); 2402 spin_unlock_irqrestore(&list->lock, flags); 2403 return result; 2404 } 2405 EXPORT_SYMBOL(skb_dequeue_tail); 2406 2407 /** 2408 * skb_queue_purge - empty a list 2409 * @list: list to empty 2410 * 2411 * Delete all buffers on an &sk_buff list. Each buffer is removed from 2412 * the list and one reference dropped. This function takes the list 2413 * lock and is atomic with respect to other list locking functions. 2414 */ 2415 void skb_queue_purge(struct sk_buff_head *list) 2416 { 2417 struct sk_buff *skb; 2418 while ((skb = skb_dequeue(list)) != NULL) 2419 kfree_skb(skb); 2420 } 2421 EXPORT_SYMBOL(skb_queue_purge); 2422 2423 /** 2424 * skb_rbtree_purge - empty a skb rbtree 2425 * @root: root of the rbtree to empty 2426 * 2427 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 2428 * the list and one reference dropped. This function does not take 2429 * any lock. Synchronization should be handled by the caller (e.g., TCP 2430 * out-of-order queue is protected by the socket lock). 2431 */ 2432 void skb_rbtree_purge(struct rb_root *root) 2433 { 2434 struct sk_buff *skb, *next; 2435 2436 rbtree_postorder_for_each_entry_safe(skb, next, root, rbnode) 2437 kfree_skb(skb); 2438 2439 *root = RB_ROOT; 2440 } 2441 2442 /** 2443 * skb_queue_head - queue a buffer at the list head 2444 * @list: list to use 2445 * @newsk: buffer to queue 2446 * 2447 * Queue a buffer at the start of the list. This function takes the 2448 * list lock and can be used safely with other locking &sk_buff functions 2449 * safely. 2450 * 2451 * A buffer cannot be placed on two lists at the same time. 2452 */ 2453 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 2454 { 2455 unsigned long flags; 2456 2457 spin_lock_irqsave(&list->lock, flags); 2458 __skb_queue_head(list, newsk); 2459 spin_unlock_irqrestore(&list->lock, flags); 2460 } 2461 EXPORT_SYMBOL(skb_queue_head); 2462 2463 /** 2464 * skb_queue_tail - queue a buffer at the list tail 2465 * @list: list to use 2466 * @newsk: buffer to queue 2467 * 2468 * Queue a buffer at the tail of the list. This function takes the 2469 * list lock and can be used safely with other locking &sk_buff functions 2470 * safely. 2471 * 2472 * A buffer cannot be placed on two lists at the same time. 2473 */ 2474 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 2475 { 2476 unsigned long flags; 2477 2478 spin_lock_irqsave(&list->lock, flags); 2479 __skb_queue_tail(list, newsk); 2480 spin_unlock_irqrestore(&list->lock, flags); 2481 } 2482 EXPORT_SYMBOL(skb_queue_tail); 2483 2484 /** 2485 * skb_unlink - remove a buffer from a list 2486 * @skb: buffer to remove 2487 * @list: list to use 2488 * 2489 * Remove a packet from a list. The list locks are taken and this 2490 * function is atomic with respect to other list locked calls 2491 * 2492 * You must know what list the SKB is on. 2493 */ 2494 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 2495 { 2496 unsigned long flags; 2497 2498 spin_lock_irqsave(&list->lock, flags); 2499 __skb_unlink(skb, list); 2500 spin_unlock_irqrestore(&list->lock, flags); 2501 } 2502 EXPORT_SYMBOL(skb_unlink); 2503 2504 /** 2505 * skb_append - append a buffer 2506 * @old: buffer to insert after 2507 * @newsk: buffer to insert 2508 * @list: list to use 2509 * 2510 * Place a packet after a given packet in a list. The list locks are taken 2511 * and this function is atomic with respect to other list locked calls. 2512 * A buffer cannot be placed on two lists at the same time. 2513 */ 2514 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 2515 { 2516 unsigned long flags; 2517 2518 spin_lock_irqsave(&list->lock, flags); 2519 __skb_queue_after(list, old, newsk); 2520 spin_unlock_irqrestore(&list->lock, flags); 2521 } 2522 EXPORT_SYMBOL(skb_append); 2523 2524 /** 2525 * skb_insert - insert a buffer 2526 * @old: buffer to insert before 2527 * @newsk: buffer to insert 2528 * @list: list to use 2529 * 2530 * Place a packet before a given packet in a list. The list locks are 2531 * taken and this function is atomic with respect to other list locked 2532 * calls. 2533 * 2534 * A buffer cannot be placed on two lists at the same time. 2535 */ 2536 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 2537 { 2538 unsigned long flags; 2539 2540 spin_lock_irqsave(&list->lock, flags); 2541 __skb_insert(newsk, old->prev, old, list); 2542 spin_unlock_irqrestore(&list->lock, flags); 2543 } 2544 EXPORT_SYMBOL(skb_insert); 2545 2546 static inline void skb_split_inside_header(struct sk_buff *skb, 2547 struct sk_buff* skb1, 2548 const u32 len, const int pos) 2549 { 2550 int i; 2551 2552 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 2553 pos - len); 2554 /* And move data appendix as is. */ 2555 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2556 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 2557 2558 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 2559 skb_shinfo(skb)->nr_frags = 0; 2560 skb1->data_len = skb->data_len; 2561 skb1->len += skb1->data_len; 2562 skb->data_len = 0; 2563 skb->len = len; 2564 skb_set_tail_pointer(skb, len); 2565 } 2566 2567 static inline void skb_split_no_header(struct sk_buff *skb, 2568 struct sk_buff* skb1, 2569 const u32 len, int pos) 2570 { 2571 int i, k = 0; 2572 const int nfrags = skb_shinfo(skb)->nr_frags; 2573 2574 skb_shinfo(skb)->nr_frags = 0; 2575 skb1->len = skb1->data_len = skb->len - len; 2576 skb->len = len; 2577 skb->data_len = len - pos; 2578 2579 for (i = 0; i < nfrags; i++) { 2580 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2581 2582 if (pos + size > len) { 2583 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 2584 2585 if (pos < len) { 2586 /* Split frag. 2587 * We have two variants in this case: 2588 * 1. Move all the frag to the second 2589 * part, if it is possible. F.e. 2590 * this approach is mandatory for TUX, 2591 * where splitting is expensive. 2592 * 2. Split is accurately. We make this. 2593 */ 2594 skb_frag_ref(skb, i); 2595 skb_shinfo(skb1)->frags[0].page_offset += len - pos; 2596 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 2597 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 2598 skb_shinfo(skb)->nr_frags++; 2599 } 2600 k++; 2601 } else 2602 skb_shinfo(skb)->nr_frags++; 2603 pos += size; 2604 } 2605 skb_shinfo(skb1)->nr_frags = k; 2606 } 2607 2608 /** 2609 * skb_split - Split fragmented skb to two parts at length len. 2610 * @skb: the buffer to split 2611 * @skb1: the buffer to receive the second part 2612 * @len: new length for skb 2613 */ 2614 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 2615 { 2616 int pos = skb_headlen(skb); 2617 2618 skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG; 2619 if (len < pos) /* Split line is inside header. */ 2620 skb_split_inside_header(skb, skb1, len, pos); 2621 else /* Second chunk has no header, nothing to copy. */ 2622 skb_split_no_header(skb, skb1, len, pos); 2623 } 2624 EXPORT_SYMBOL(skb_split); 2625 2626 /* Shifting from/to a cloned skb is a no-go. 2627 * 2628 * Caller cannot keep skb_shinfo related pointers past calling here! 2629 */ 2630 static int skb_prepare_for_shift(struct sk_buff *skb) 2631 { 2632 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2633 } 2634 2635 /** 2636 * skb_shift - Shifts paged data partially from skb to another 2637 * @tgt: buffer into which tail data gets added 2638 * @skb: buffer from which the paged data comes from 2639 * @shiftlen: shift up to this many bytes 2640 * 2641 * Attempts to shift up to shiftlen worth of bytes, which may be less than 2642 * the length of the skb, from skb to tgt. Returns number bytes shifted. 2643 * It's up to caller to free skb if everything was shifted. 2644 * 2645 * If @tgt runs out of frags, the whole operation is aborted. 2646 * 2647 * Skb cannot include anything else but paged data while tgt is allowed 2648 * to have non-paged data as well. 2649 * 2650 * TODO: full sized shift could be optimized but that would need 2651 * specialized skb free'er to handle frags without up-to-date nr_frags. 2652 */ 2653 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 2654 { 2655 int from, to, merge, todo; 2656 struct skb_frag_struct *fragfrom, *fragto; 2657 2658 BUG_ON(shiftlen > skb->len); 2659 BUG_ON(skb_headlen(skb)); /* Would corrupt stream */ 2660 2661 todo = shiftlen; 2662 from = 0; 2663 to = skb_shinfo(tgt)->nr_frags; 2664 fragfrom = &skb_shinfo(skb)->frags[from]; 2665 2666 /* Actual merge is delayed until the point when we know we can 2667 * commit all, so that we don't have to undo partial changes 2668 */ 2669 if (!to || 2670 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 2671 fragfrom->page_offset)) { 2672 merge = -1; 2673 } else { 2674 merge = to - 1; 2675 2676 todo -= skb_frag_size(fragfrom); 2677 if (todo < 0) { 2678 if (skb_prepare_for_shift(skb) || 2679 skb_prepare_for_shift(tgt)) 2680 return 0; 2681 2682 /* All previous frag pointers might be stale! */ 2683 fragfrom = &skb_shinfo(skb)->frags[from]; 2684 fragto = &skb_shinfo(tgt)->frags[merge]; 2685 2686 skb_frag_size_add(fragto, shiftlen); 2687 skb_frag_size_sub(fragfrom, shiftlen); 2688 fragfrom->page_offset += shiftlen; 2689 2690 goto onlymerged; 2691 } 2692 2693 from++; 2694 } 2695 2696 /* Skip full, not-fitting skb to avoid expensive operations */ 2697 if ((shiftlen == skb->len) && 2698 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 2699 return 0; 2700 2701 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 2702 return 0; 2703 2704 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 2705 if (to == MAX_SKB_FRAGS) 2706 return 0; 2707 2708 fragfrom = &skb_shinfo(skb)->frags[from]; 2709 fragto = &skb_shinfo(tgt)->frags[to]; 2710 2711 if (todo >= skb_frag_size(fragfrom)) { 2712 *fragto = *fragfrom; 2713 todo -= skb_frag_size(fragfrom); 2714 from++; 2715 to++; 2716 2717 } else { 2718 __skb_frag_ref(fragfrom); 2719 fragto->page = fragfrom->page; 2720 fragto->page_offset = fragfrom->page_offset; 2721 skb_frag_size_set(fragto, todo); 2722 2723 fragfrom->page_offset += todo; 2724 skb_frag_size_sub(fragfrom, todo); 2725 todo = 0; 2726 2727 to++; 2728 break; 2729 } 2730 } 2731 2732 /* Ready to "commit" this state change to tgt */ 2733 skb_shinfo(tgt)->nr_frags = to; 2734 2735 if (merge >= 0) { 2736 fragfrom = &skb_shinfo(skb)->frags[0]; 2737 fragto = &skb_shinfo(tgt)->frags[merge]; 2738 2739 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 2740 __skb_frag_unref(fragfrom); 2741 } 2742 2743 /* Reposition in the original skb */ 2744 to = 0; 2745 while (from < skb_shinfo(skb)->nr_frags) 2746 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 2747 skb_shinfo(skb)->nr_frags = to; 2748 2749 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 2750 2751 onlymerged: 2752 /* Most likely the tgt won't ever need its checksum anymore, skb on 2753 * the other hand might need it if it needs to be resent 2754 */ 2755 tgt->ip_summed = CHECKSUM_PARTIAL; 2756 skb->ip_summed = CHECKSUM_PARTIAL; 2757 2758 /* Yak, is it really working this way? Some helper please? */ 2759 skb->len -= shiftlen; 2760 skb->data_len -= shiftlen; 2761 skb->truesize -= shiftlen; 2762 tgt->len += shiftlen; 2763 tgt->data_len += shiftlen; 2764 tgt->truesize += shiftlen; 2765 2766 return shiftlen; 2767 } 2768 2769 /** 2770 * skb_prepare_seq_read - Prepare a sequential read of skb data 2771 * @skb: the buffer to read 2772 * @from: lower offset of data to be read 2773 * @to: upper offset of data to be read 2774 * @st: state variable 2775 * 2776 * Initializes the specified state variable. Must be called before 2777 * invoking skb_seq_read() for the first time. 2778 */ 2779 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 2780 unsigned int to, struct skb_seq_state *st) 2781 { 2782 st->lower_offset = from; 2783 st->upper_offset = to; 2784 st->root_skb = st->cur_skb = skb; 2785 st->frag_idx = st->stepped_offset = 0; 2786 st->frag_data = NULL; 2787 } 2788 EXPORT_SYMBOL(skb_prepare_seq_read); 2789 2790 /** 2791 * skb_seq_read - Sequentially read skb data 2792 * @consumed: number of bytes consumed by the caller so far 2793 * @data: destination pointer for data to be returned 2794 * @st: state variable 2795 * 2796 * Reads a block of skb data at @consumed relative to the 2797 * lower offset specified to skb_prepare_seq_read(). Assigns 2798 * the head of the data block to @data and returns the length 2799 * of the block or 0 if the end of the skb data or the upper 2800 * offset has been reached. 2801 * 2802 * The caller is not required to consume all of the data 2803 * returned, i.e. @consumed is typically set to the number 2804 * of bytes already consumed and the next call to 2805 * skb_seq_read() will return the remaining part of the block. 2806 * 2807 * Note 1: The size of each block of data returned can be arbitrary, 2808 * this limitation is the cost for zerocopy sequential 2809 * reads of potentially non linear data. 2810 * 2811 * Note 2: Fragment lists within fragments are not implemented 2812 * at the moment, state->root_skb could be replaced with 2813 * a stack for this purpose. 2814 */ 2815 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 2816 struct skb_seq_state *st) 2817 { 2818 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 2819 skb_frag_t *frag; 2820 2821 if (unlikely(abs_offset >= st->upper_offset)) { 2822 if (st->frag_data) { 2823 kunmap_atomic(st->frag_data); 2824 st->frag_data = NULL; 2825 } 2826 return 0; 2827 } 2828 2829 next_skb: 2830 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 2831 2832 if (abs_offset < block_limit && !st->frag_data) { 2833 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 2834 return block_limit - abs_offset; 2835 } 2836 2837 if (st->frag_idx == 0 && !st->frag_data) 2838 st->stepped_offset += skb_headlen(st->cur_skb); 2839 2840 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 2841 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 2842 block_limit = skb_frag_size(frag) + st->stepped_offset; 2843 2844 if (abs_offset < block_limit) { 2845 if (!st->frag_data) 2846 st->frag_data = kmap_atomic(skb_frag_page(frag)); 2847 2848 *data = (u8 *) st->frag_data + frag->page_offset + 2849 (abs_offset - st->stepped_offset); 2850 2851 return block_limit - abs_offset; 2852 } 2853 2854 if (st->frag_data) { 2855 kunmap_atomic(st->frag_data); 2856 st->frag_data = NULL; 2857 } 2858 2859 st->frag_idx++; 2860 st->stepped_offset += skb_frag_size(frag); 2861 } 2862 2863 if (st->frag_data) { 2864 kunmap_atomic(st->frag_data); 2865 st->frag_data = NULL; 2866 } 2867 2868 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 2869 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 2870 st->frag_idx = 0; 2871 goto next_skb; 2872 } else if (st->cur_skb->next) { 2873 st->cur_skb = st->cur_skb->next; 2874 st->frag_idx = 0; 2875 goto next_skb; 2876 } 2877 2878 return 0; 2879 } 2880 EXPORT_SYMBOL(skb_seq_read); 2881 2882 /** 2883 * skb_abort_seq_read - Abort a sequential read of skb data 2884 * @st: state variable 2885 * 2886 * Must be called if skb_seq_read() was not called until it 2887 * returned 0. 2888 */ 2889 void skb_abort_seq_read(struct skb_seq_state *st) 2890 { 2891 if (st->frag_data) 2892 kunmap_atomic(st->frag_data); 2893 } 2894 EXPORT_SYMBOL(skb_abort_seq_read); 2895 2896 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 2897 2898 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 2899 struct ts_config *conf, 2900 struct ts_state *state) 2901 { 2902 return skb_seq_read(offset, text, TS_SKB_CB(state)); 2903 } 2904 2905 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 2906 { 2907 skb_abort_seq_read(TS_SKB_CB(state)); 2908 } 2909 2910 /** 2911 * skb_find_text - Find a text pattern in skb data 2912 * @skb: the buffer to look in 2913 * @from: search offset 2914 * @to: search limit 2915 * @config: textsearch configuration 2916 * 2917 * Finds a pattern in the skb data according to the specified 2918 * textsearch configuration. Use textsearch_next() to retrieve 2919 * subsequent occurrences of the pattern. Returns the offset 2920 * to the first occurrence or UINT_MAX if no match was found. 2921 */ 2922 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 2923 unsigned int to, struct ts_config *config) 2924 { 2925 struct ts_state state; 2926 unsigned int ret; 2927 2928 config->get_next_block = skb_ts_get_next_block; 2929 config->finish = skb_ts_finish; 2930 2931 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 2932 2933 ret = textsearch_find(config, &state); 2934 return (ret <= to - from ? ret : UINT_MAX); 2935 } 2936 EXPORT_SYMBOL(skb_find_text); 2937 2938 /** 2939 * skb_append_datato_frags - append the user data to a skb 2940 * @sk: sock structure 2941 * @skb: skb structure to be appended with user data. 2942 * @getfrag: call back function to be used for getting the user data 2943 * @from: pointer to user message iov 2944 * @length: length of the iov message 2945 * 2946 * Description: This procedure append the user data in the fragment part 2947 * of the skb if any page alloc fails user this procedure returns -ENOMEM 2948 */ 2949 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb, 2950 int (*getfrag)(void *from, char *to, int offset, 2951 int len, int odd, struct sk_buff *skb), 2952 void *from, int length) 2953 { 2954 int frg_cnt = skb_shinfo(skb)->nr_frags; 2955 int copy; 2956 int offset = 0; 2957 int ret; 2958 struct page_frag *pfrag = ¤t->task_frag; 2959 2960 do { 2961 /* Return error if we don't have space for new frag */ 2962 if (frg_cnt >= MAX_SKB_FRAGS) 2963 return -EMSGSIZE; 2964 2965 if (!sk_page_frag_refill(sk, pfrag)) 2966 return -ENOMEM; 2967 2968 /* copy the user data to page */ 2969 copy = min_t(int, length, pfrag->size - pfrag->offset); 2970 2971 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset, 2972 offset, copy, 0, skb); 2973 if (ret < 0) 2974 return -EFAULT; 2975 2976 /* copy was successful so update the size parameters */ 2977 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset, 2978 copy); 2979 frg_cnt++; 2980 pfrag->offset += copy; 2981 get_page(pfrag->page); 2982 2983 skb->truesize += copy; 2984 atomic_add(copy, &sk->sk_wmem_alloc); 2985 skb->len += copy; 2986 skb->data_len += copy; 2987 offset += copy; 2988 length -= copy; 2989 2990 } while (length > 0); 2991 2992 return 0; 2993 } 2994 EXPORT_SYMBOL(skb_append_datato_frags); 2995 2996 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 2997 int offset, size_t size) 2998 { 2999 int i = skb_shinfo(skb)->nr_frags; 3000 3001 if (skb_can_coalesce(skb, i, page, offset)) { 3002 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 3003 } else if (i < MAX_SKB_FRAGS) { 3004 get_page(page); 3005 skb_fill_page_desc(skb, i, page, offset, size); 3006 } else { 3007 return -EMSGSIZE; 3008 } 3009 3010 return 0; 3011 } 3012 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 3013 3014 /** 3015 * skb_pull_rcsum - pull skb and update receive checksum 3016 * @skb: buffer to update 3017 * @len: length of data pulled 3018 * 3019 * This function performs an skb_pull on the packet and updates 3020 * the CHECKSUM_COMPLETE checksum. It should be used on 3021 * receive path processing instead of skb_pull unless you know 3022 * that the checksum difference is zero (e.g., a valid IP header) 3023 * or you are setting ip_summed to CHECKSUM_NONE. 3024 */ 3025 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 3026 { 3027 unsigned char *data = skb->data; 3028 3029 BUG_ON(len > skb->len); 3030 __skb_pull(skb, len); 3031 skb_postpull_rcsum(skb, data, len); 3032 return skb->data; 3033 } 3034 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 3035 3036 /** 3037 * skb_segment - Perform protocol segmentation on skb. 3038 * @head_skb: buffer to segment 3039 * @features: features for the output path (see dev->features) 3040 * 3041 * This function performs segmentation on the given skb. It returns 3042 * a pointer to the first in a list of new skbs for the segments. 3043 * In case of error it returns ERR_PTR(err). 3044 */ 3045 struct sk_buff *skb_segment(struct sk_buff *head_skb, 3046 netdev_features_t features) 3047 { 3048 struct sk_buff *segs = NULL; 3049 struct sk_buff *tail = NULL; 3050 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 3051 skb_frag_t *frag = skb_shinfo(head_skb)->frags; 3052 unsigned int mss = skb_shinfo(head_skb)->gso_size; 3053 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 3054 struct sk_buff *frag_skb = head_skb; 3055 unsigned int offset = doffset; 3056 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 3057 unsigned int partial_segs = 0; 3058 unsigned int headroom; 3059 unsigned int len = head_skb->len; 3060 __be16 proto; 3061 bool csum, sg; 3062 int nfrags = skb_shinfo(head_skb)->nr_frags; 3063 int err = -ENOMEM; 3064 int i = 0; 3065 int pos; 3066 int dummy; 3067 3068 __skb_push(head_skb, doffset); 3069 proto = skb_network_protocol(head_skb, &dummy); 3070 if (unlikely(!proto)) 3071 return ERR_PTR(-EINVAL); 3072 3073 sg = !!(features & NETIF_F_SG); 3074 csum = !!can_checksum_protocol(features, proto); 3075 3076 if (sg && csum && (mss != GSO_BY_FRAGS)) { 3077 if (!(features & NETIF_F_GSO_PARTIAL)) { 3078 struct sk_buff *iter; 3079 3080 if (!list_skb || 3081 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 3082 goto normal; 3083 3084 /* Split the buffer at the frag_list pointer. 3085 * This is based on the assumption that all 3086 * buffers in the chain excluding the last 3087 * containing the same amount of data. 3088 */ 3089 skb_walk_frags(head_skb, iter) { 3090 if (skb_headlen(iter)) 3091 goto normal; 3092 3093 len -= iter->len; 3094 } 3095 } 3096 3097 /* GSO partial only requires that we trim off any excess that 3098 * doesn't fit into an MSS sized block, so take care of that 3099 * now. 3100 */ 3101 partial_segs = len / mss; 3102 if (partial_segs > 1) 3103 mss *= partial_segs; 3104 else 3105 partial_segs = 0; 3106 } 3107 3108 normal: 3109 headroom = skb_headroom(head_skb); 3110 pos = skb_headlen(head_skb); 3111 3112 do { 3113 struct sk_buff *nskb; 3114 skb_frag_t *nskb_frag; 3115 int hsize; 3116 int size; 3117 3118 if (unlikely(mss == GSO_BY_FRAGS)) { 3119 len = list_skb->len; 3120 } else { 3121 len = head_skb->len - offset; 3122 if (len > mss) 3123 len = mss; 3124 } 3125 3126 hsize = skb_headlen(head_skb) - offset; 3127 if (hsize < 0) 3128 hsize = 0; 3129 if (hsize > len || !sg) 3130 hsize = len; 3131 3132 if (!hsize && i >= nfrags && skb_headlen(list_skb) && 3133 (skb_headlen(list_skb) == len || sg)) { 3134 BUG_ON(skb_headlen(list_skb) > len); 3135 3136 i = 0; 3137 nfrags = skb_shinfo(list_skb)->nr_frags; 3138 frag = skb_shinfo(list_skb)->frags; 3139 frag_skb = list_skb; 3140 pos += skb_headlen(list_skb); 3141 3142 while (pos < offset + len) { 3143 BUG_ON(i >= nfrags); 3144 3145 size = skb_frag_size(frag); 3146 if (pos + size > offset + len) 3147 break; 3148 3149 i++; 3150 pos += size; 3151 frag++; 3152 } 3153 3154 nskb = skb_clone(list_skb, GFP_ATOMIC); 3155 list_skb = list_skb->next; 3156 3157 if (unlikely(!nskb)) 3158 goto err; 3159 3160 if (unlikely(pskb_trim(nskb, len))) { 3161 kfree_skb(nskb); 3162 goto err; 3163 } 3164 3165 hsize = skb_end_offset(nskb); 3166 if (skb_cow_head(nskb, doffset + headroom)) { 3167 kfree_skb(nskb); 3168 goto err; 3169 } 3170 3171 nskb->truesize += skb_end_offset(nskb) - hsize; 3172 skb_release_head_state(nskb); 3173 __skb_push(nskb, doffset); 3174 } else { 3175 nskb = __alloc_skb(hsize + doffset + headroom, 3176 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 3177 NUMA_NO_NODE); 3178 3179 if (unlikely(!nskb)) 3180 goto err; 3181 3182 skb_reserve(nskb, headroom); 3183 __skb_put(nskb, doffset); 3184 } 3185 3186 if (segs) 3187 tail->next = nskb; 3188 else 3189 segs = nskb; 3190 tail = nskb; 3191 3192 __copy_skb_header(nskb, head_skb); 3193 3194 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 3195 skb_reset_mac_len(nskb); 3196 3197 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 3198 nskb->data - tnl_hlen, 3199 doffset + tnl_hlen); 3200 3201 if (nskb->len == len + doffset) 3202 goto perform_csum_check; 3203 3204 if (!sg) { 3205 if (!nskb->remcsum_offload) 3206 nskb->ip_summed = CHECKSUM_NONE; 3207 SKB_GSO_CB(nskb)->csum = 3208 skb_copy_and_csum_bits(head_skb, offset, 3209 skb_put(nskb, len), 3210 len, 0); 3211 SKB_GSO_CB(nskb)->csum_start = 3212 skb_headroom(nskb) + doffset; 3213 continue; 3214 } 3215 3216 nskb_frag = skb_shinfo(nskb)->frags; 3217 3218 skb_copy_from_linear_data_offset(head_skb, offset, 3219 skb_put(nskb, hsize), hsize); 3220 3221 skb_shinfo(nskb)->tx_flags = skb_shinfo(head_skb)->tx_flags & 3222 SKBTX_SHARED_FRAG; 3223 3224 while (pos < offset + len) { 3225 if (i >= nfrags) { 3226 BUG_ON(skb_headlen(list_skb)); 3227 3228 i = 0; 3229 nfrags = skb_shinfo(list_skb)->nr_frags; 3230 frag = skb_shinfo(list_skb)->frags; 3231 frag_skb = list_skb; 3232 3233 BUG_ON(!nfrags); 3234 3235 list_skb = list_skb->next; 3236 } 3237 3238 if (unlikely(skb_shinfo(nskb)->nr_frags >= 3239 MAX_SKB_FRAGS)) { 3240 net_warn_ratelimited( 3241 "skb_segment: too many frags: %u %u\n", 3242 pos, mss); 3243 goto err; 3244 } 3245 3246 if (unlikely(skb_orphan_frags(frag_skb, GFP_ATOMIC))) 3247 goto err; 3248 3249 *nskb_frag = *frag; 3250 __skb_frag_ref(nskb_frag); 3251 size = skb_frag_size(nskb_frag); 3252 3253 if (pos < offset) { 3254 nskb_frag->page_offset += offset - pos; 3255 skb_frag_size_sub(nskb_frag, offset - pos); 3256 } 3257 3258 skb_shinfo(nskb)->nr_frags++; 3259 3260 if (pos + size <= offset + len) { 3261 i++; 3262 frag++; 3263 pos += size; 3264 } else { 3265 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 3266 goto skip_fraglist; 3267 } 3268 3269 nskb_frag++; 3270 } 3271 3272 skip_fraglist: 3273 nskb->data_len = len - hsize; 3274 nskb->len += nskb->data_len; 3275 nskb->truesize += nskb->data_len; 3276 3277 perform_csum_check: 3278 if (!csum) { 3279 if (skb_has_shared_frag(nskb)) { 3280 err = __skb_linearize(nskb); 3281 if (err) 3282 goto err; 3283 } 3284 if (!nskb->remcsum_offload) 3285 nskb->ip_summed = CHECKSUM_NONE; 3286 SKB_GSO_CB(nskb)->csum = 3287 skb_checksum(nskb, doffset, 3288 nskb->len - doffset, 0); 3289 SKB_GSO_CB(nskb)->csum_start = 3290 skb_headroom(nskb) + doffset; 3291 } 3292 } while ((offset += len) < head_skb->len); 3293 3294 /* Some callers want to get the end of the list. 3295 * Put it in segs->prev to avoid walking the list. 3296 * (see validate_xmit_skb_list() for example) 3297 */ 3298 segs->prev = tail; 3299 3300 if (partial_segs) { 3301 struct sk_buff *iter; 3302 int type = skb_shinfo(head_skb)->gso_type; 3303 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 3304 3305 /* Update type to add partial and then remove dodgy if set */ 3306 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 3307 type &= ~SKB_GSO_DODGY; 3308 3309 /* Update GSO info and prepare to start updating headers on 3310 * our way back down the stack of protocols. 3311 */ 3312 for (iter = segs; iter; iter = iter->next) { 3313 skb_shinfo(iter)->gso_size = gso_size; 3314 skb_shinfo(iter)->gso_segs = partial_segs; 3315 skb_shinfo(iter)->gso_type = type; 3316 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 3317 } 3318 3319 if (tail->len - doffset <= gso_size) 3320 skb_shinfo(tail)->gso_size = 0; 3321 else if (tail != segs) 3322 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 3323 } 3324 3325 /* Following permits correct backpressure, for protocols 3326 * using skb_set_owner_w(). 3327 * Idea is to tranfert ownership from head_skb to last segment. 3328 */ 3329 if (head_skb->destructor == sock_wfree) { 3330 swap(tail->truesize, head_skb->truesize); 3331 swap(tail->destructor, head_skb->destructor); 3332 swap(tail->sk, head_skb->sk); 3333 } 3334 return segs; 3335 3336 err: 3337 kfree_skb_list(segs); 3338 return ERR_PTR(err); 3339 } 3340 EXPORT_SYMBOL_GPL(skb_segment); 3341 3342 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb) 3343 { 3344 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); 3345 unsigned int offset = skb_gro_offset(skb); 3346 unsigned int headlen = skb_headlen(skb); 3347 unsigned int len = skb_gro_len(skb); 3348 struct sk_buff *lp, *p = *head; 3349 unsigned int delta_truesize; 3350 3351 if (unlikely(p->len + len >= 65536)) 3352 return -E2BIG; 3353 3354 lp = NAPI_GRO_CB(p)->last; 3355 pinfo = skb_shinfo(lp); 3356 3357 if (headlen <= offset) { 3358 skb_frag_t *frag; 3359 skb_frag_t *frag2; 3360 int i = skbinfo->nr_frags; 3361 int nr_frags = pinfo->nr_frags + i; 3362 3363 if (nr_frags > MAX_SKB_FRAGS) 3364 goto merge; 3365 3366 offset -= headlen; 3367 pinfo->nr_frags = nr_frags; 3368 skbinfo->nr_frags = 0; 3369 3370 frag = pinfo->frags + nr_frags; 3371 frag2 = skbinfo->frags + i; 3372 do { 3373 *--frag = *--frag2; 3374 } while (--i); 3375 3376 frag->page_offset += offset; 3377 skb_frag_size_sub(frag, offset); 3378 3379 /* all fragments truesize : remove (head size + sk_buff) */ 3380 delta_truesize = skb->truesize - 3381 SKB_TRUESIZE(skb_end_offset(skb)); 3382 3383 skb->truesize -= skb->data_len; 3384 skb->len -= skb->data_len; 3385 skb->data_len = 0; 3386 3387 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; 3388 goto done; 3389 } else if (skb->head_frag) { 3390 int nr_frags = pinfo->nr_frags; 3391 skb_frag_t *frag = pinfo->frags + nr_frags; 3392 struct page *page = virt_to_head_page(skb->head); 3393 unsigned int first_size = headlen - offset; 3394 unsigned int first_offset; 3395 3396 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) 3397 goto merge; 3398 3399 first_offset = skb->data - 3400 (unsigned char *)page_address(page) + 3401 offset; 3402 3403 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; 3404 3405 frag->page.p = page; 3406 frag->page_offset = first_offset; 3407 skb_frag_size_set(frag, first_size); 3408 3409 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); 3410 /* We dont need to clear skbinfo->nr_frags here */ 3411 3412 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 3413 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; 3414 goto done; 3415 } 3416 3417 merge: 3418 delta_truesize = skb->truesize; 3419 if (offset > headlen) { 3420 unsigned int eat = offset - headlen; 3421 3422 skbinfo->frags[0].page_offset += eat; 3423 skb_frag_size_sub(&skbinfo->frags[0], eat); 3424 skb->data_len -= eat; 3425 skb->len -= eat; 3426 offset = headlen; 3427 } 3428 3429 __skb_pull(skb, offset); 3430 3431 if (NAPI_GRO_CB(p)->last == p) 3432 skb_shinfo(p)->frag_list = skb; 3433 else 3434 NAPI_GRO_CB(p)->last->next = skb; 3435 NAPI_GRO_CB(p)->last = skb; 3436 __skb_header_release(skb); 3437 lp = p; 3438 3439 done: 3440 NAPI_GRO_CB(p)->count++; 3441 p->data_len += len; 3442 p->truesize += delta_truesize; 3443 p->len += len; 3444 if (lp != p) { 3445 lp->data_len += len; 3446 lp->truesize += delta_truesize; 3447 lp->len += len; 3448 } 3449 NAPI_GRO_CB(skb)->same_flow = 1; 3450 return 0; 3451 } 3452 EXPORT_SYMBOL_GPL(skb_gro_receive); 3453 3454 void __init skb_init(void) 3455 { 3456 skbuff_head_cache = kmem_cache_create("skbuff_head_cache", 3457 sizeof(struct sk_buff), 3458 0, 3459 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3460 NULL); 3461 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 3462 sizeof(struct sk_buff_fclones), 3463 0, 3464 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3465 NULL); 3466 } 3467 3468 /** 3469 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 3470 * @skb: Socket buffer containing the buffers to be mapped 3471 * @sg: The scatter-gather list to map into 3472 * @offset: The offset into the buffer's contents to start mapping 3473 * @len: Length of buffer space to be mapped 3474 * 3475 * Fill the specified scatter-gather list with mappings/pointers into a 3476 * region of the buffer space attached to a socket buffer. 3477 */ 3478 static int 3479 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 3480 { 3481 int start = skb_headlen(skb); 3482 int i, copy = start - offset; 3483 struct sk_buff *frag_iter; 3484 int elt = 0; 3485 3486 if (copy > 0) { 3487 if (copy > len) 3488 copy = len; 3489 sg_set_buf(sg, skb->data + offset, copy); 3490 elt++; 3491 if ((len -= copy) == 0) 3492 return elt; 3493 offset += copy; 3494 } 3495 3496 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3497 int end; 3498 3499 WARN_ON(start > offset + len); 3500 3501 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3502 if ((copy = end - offset) > 0) { 3503 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3504 3505 if (copy > len) 3506 copy = len; 3507 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 3508 frag->page_offset+offset-start); 3509 elt++; 3510 if (!(len -= copy)) 3511 return elt; 3512 offset += copy; 3513 } 3514 start = end; 3515 } 3516 3517 skb_walk_frags(skb, frag_iter) { 3518 int end; 3519 3520 WARN_ON(start > offset + len); 3521 3522 end = start + frag_iter->len; 3523 if ((copy = end - offset) > 0) { 3524 if (copy > len) 3525 copy = len; 3526 elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start, 3527 copy); 3528 if ((len -= copy) == 0) 3529 return elt; 3530 offset += copy; 3531 } 3532 start = end; 3533 } 3534 BUG_ON(len); 3535 return elt; 3536 } 3537 3538 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 3539 * sglist without mark the sg which contain last skb data as the end. 3540 * So the caller can mannipulate sg list as will when padding new data after 3541 * the first call without calling sg_unmark_end to expend sg list. 3542 * 3543 * Scenario to use skb_to_sgvec_nomark: 3544 * 1. sg_init_table 3545 * 2. skb_to_sgvec_nomark(payload1) 3546 * 3. skb_to_sgvec_nomark(payload2) 3547 * 3548 * This is equivalent to: 3549 * 1. sg_init_table 3550 * 2. skb_to_sgvec(payload1) 3551 * 3. sg_unmark_end 3552 * 4. skb_to_sgvec(payload2) 3553 * 3554 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 3555 * is more preferable. 3556 */ 3557 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 3558 int offset, int len) 3559 { 3560 return __skb_to_sgvec(skb, sg, offset, len); 3561 } 3562 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 3563 3564 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 3565 { 3566 int nsg = __skb_to_sgvec(skb, sg, offset, len); 3567 3568 sg_mark_end(&sg[nsg - 1]); 3569 3570 return nsg; 3571 } 3572 EXPORT_SYMBOL_GPL(skb_to_sgvec); 3573 3574 /** 3575 * skb_cow_data - Check that a socket buffer's data buffers are writable 3576 * @skb: The socket buffer to check. 3577 * @tailbits: Amount of trailing space to be added 3578 * @trailer: Returned pointer to the skb where the @tailbits space begins 3579 * 3580 * Make sure that the data buffers attached to a socket buffer are 3581 * writable. If they are not, private copies are made of the data buffers 3582 * and the socket buffer is set to use these instead. 3583 * 3584 * If @tailbits is given, make sure that there is space to write @tailbits 3585 * bytes of data beyond current end of socket buffer. @trailer will be 3586 * set to point to the skb in which this space begins. 3587 * 3588 * The number of scatterlist elements required to completely map the 3589 * COW'd and extended socket buffer will be returned. 3590 */ 3591 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 3592 { 3593 int copyflag; 3594 int elt; 3595 struct sk_buff *skb1, **skb_p; 3596 3597 /* If skb is cloned or its head is paged, reallocate 3598 * head pulling out all the pages (pages are considered not writable 3599 * at the moment even if they are anonymous). 3600 */ 3601 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 3602 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL) 3603 return -ENOMEM; 3604 3605 /* Easy case. Most of packets will go this way. */ 3606 if (!skb_has_frag_list(skb)) { 3607 /* A little of trouble, not enough of space for trailer. 3608 * This should not happen, when stack is tuned to generate 3609 * good frames. OK, on miss we reallocate and reserve even more 3610 * space, 128 bytes is fair. */ 3611 3612 if (skb_tailroom(skb) < tailbits && 3613 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 3614 return -ENOMEM; 3615 3616 /* Voila! */ 3617 *trailer = skb; 3618 return 1; 3619 } 3620 3621 /* Misery. We are in troubles, going to mincer fragments... */ 3622 3623 elt = 1; 3624 skb_p = &skb_shinfo(skb)->frag_list; 3625 copyflag = 0; 3626 3627 while ((skb1 = *skb_p) != NULL) { 3628 int ntail = 0; 3629 3630 /* The fragment is partially pulled by someone, 3631 * this can happen on input. Copy it and everything 3632 * after it. */ 3633 3634 if (skb_shared(skb1)) 3635 copyflag = 1; 3636 3637 /* If the skb is the last, worry about trailer. */ 3638 3639 if (skb1->next == NULL && tailbits) { 3640 if (skb_shinfo(skb1)->nr_frags || 3641 skb_has_frag_list(skb1) || 3642 skb_tailroom(skb1) < tailbits) 3643 ntail = tailbits + 128; 3644 } 3645 3646 if (copyflag || 3647 skb_cloned(skb1) || 3648 ntail || 3649 skb_shinfo(skb1)->nr_frags || 3650 skb_has_frag_list(skb1)) { 3651 struct sk_buff *skb2; 3652 3653 /* Fuck, we are miserable poor guys... */ 3654 if (ntail == 0) 3655 skb2 = skb_copy(skb1, GFP_ATOMIC); 3656 else 3657 skb2 = skb_copy_expand(skb1, 3658 skb_headroom(skb1), 3659 ntail, 3660 GFP_ATOMIC); 3661 if (unlikely(skb2 == NULL)) 3662 return -ENOMEM; 3663 3664 if (skb1->sk) 3665 skb_set_owner_w(skb2, skb1->sk); 3666 3667 /* Looking around. Are we still alive? 3668 * OK, link new skb, drop old one */ 3669 3670 skb2->next = skb1->next; 3671 *skb_p = skb2; 3672 kfree_skb(skb1); 3673 skb1 = skb2; 3674 } 3675 elt++; 3676 *trailer = skb1; 3677 skb_p = &skb1->next; 3678 } 3679 3680 return elt; 3681 } 3682 EXPORT_SYMBOL_GPL(skb_cow_data); 3683 3684 static void sock_rmem_free(struct sk_buff *skb) 3685 { 3686 struct sock *sk = skb->sk; 3687 3688 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 3689 } 3690 3691 /* 3692 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 3693 */ 3694 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 3695 { 3696 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 3697 (unsigned int)sk->sk_rcvbuf) 3698 return -ENOMEM; 3699 3700 skb_orphan(skb); 3701 skb->sk = sk; 3702 skb->destructor = sock_rmem_free; 3703 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 3704 3705 /* before exiting rcu section, make sure dst is refcounted */ 3706 skb_dst_force(skb); 3707 3708 skb_queue_tail(&sk->sk_error_queue, skb); 3709 if (!sock_flag(sk, SOCK_DEAD)) 3710 sk->sk_data_ready(sk); 3711 return 0; 3712 } 3713 EXPORT_SYMBOL(sock_queue_err_skb); 3714 3715 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 3716 { 3717 struct sk_buff_head *q = &sk->sk_error_queue; 3718 struct sk_buff *skb, *skb_next; 3719 unsigned long flags; 3720 int err = 0; 3721 3722 spin_lock_irqsave(&q->lock, flags); 3723 skb = __skb_dequeue(q); 3724 if (skb && (skb_next = skb_peek(q))) 3725 err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 3726 spin_unlock_irqrestore(&q->lock, flags); 3727 3728 sk->sk_err = err; 3729 if (err) 3730 sk->sk_error_report(sk); 3731 3732 return skb; 3733 } 3734 EXPORT_SYMBOL(sock_dequeue_err_skb); 3735 3736 /** 3737 * skb_clone_sk - create clone of skb, and take reference to socket 3738 * @skb: the skb to clone 3739 * 3740 * This function creates a clone of a buffer that holds a reference on 3741 * sk_refcnt. Buffers created via this function are meant to be 3742 * returned using sock_queue_err_skb, or free via kfree_skb. 3743 * 3744 * When passing buffers allocated with this function to sock_queue_err_skb 3745 * it is necessary to wrap the call with sock_hold/sock_put in order to 3746 * prevent the socket from being released prior to being enqueued on 3747 * the sk_error_queue. 3748 */ 3749 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 3750 { 3751 struct sock *sk = skb->sk; 3752 struct sk_buff *clone; 3753 3754 if (!sk || !atomic_inc_not_zero(&sk->sk_refcnt)) 3755 return NULL; 3756 3757 clone = skb_clone(skb, GFP_ATOMIC); 3758 if (!clone) { 3759 sock_put(sk); 3760 return NULL; 3761 } 3762 3763 clone->sk = sk; 3764 clone->destructor = sock_efree; 3765 3766 return clone; 3767 } 3768 EXPORT_SYMBOL(skb_clone_sk); 3769 3770 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 3771 struct sock *sk, 3772 int tstype) 3773 { 3774 struct sock_exterr_skb *serr; 3775 int err; 3776 3777 serr = SKB_EXT_ERR(skb); 3778 memset(serr, 0, sizeof(*serr)); 3779 serr->ee.ee_errno = ENOMSG; 3780 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 3781 serr->ee.ee_info = tstype; 3782 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { 3783 serr->ee.ee_data = skb_shinfo(skb)->tskey; 3784 if (sk->sk_protocol == IPPROTO_TCP && 3785 sk->sk_type == SOCK_STREAM) 3786 serr->ee.ee_data -= sk->sk_tskey; 3787 } 3788 3789 err = sock_queue_err_skb(sk, skb); 3790 3791 if (err) 3792 kfree_skb(skb); 3793 } 3794 3795 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 3796 { 3797 bool ret; 3798 3799 if (likely(sysctl_tstamp_allow_data || tsonly)) 3800 return true; 3801 3802 read_lock_bh(&sk->sk_callback_lock); 3803 ret = sk->sk_socket && sk->sk_socket->file && 3804 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); 3805 read_unlock_bh(&sk->sk_callback_lock); 3806 return ret; 3807 } 3808 3809 void skb_complete_tx_timestamp(struct sk_buff *skb, 3810 struct skb_shared_hwtstamps *hwtstamps) 3811 { 3812 struct sock *sk = skb->sk; 3813 3814 if (!skb_may_tx_timestamp(sk, false)) 3815 return; 3816 3817 /* take a reference to prevent skb_orphan() from freeing the socket */ 3818 sock_hold(sk); 3819 3820 *skb_hwtstamps(skb) = *hwtstamps; 3821 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND); 3822 3823 sock_put(sk); 3824 } 3825 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 3826 3827 void __skb_tstamp_tx(struct sk_buff *orig_skb, 3828 struct skb_shared_hwtstamps *hwtstamps, 3829 struct sock *sk, int tstype) 3830 { 3831 struct sk_buff *skb; 3832 bool tsonly; 3833 3834 if (!sk) 3835 return; 3836 3837 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 3838 if (!skb_may_tx_timestamp(sk, tsonly)) 3839 return; 3840 3841 if (tsonly) 3842 skb = alloc_skb(0, GFP_ATOMIC); 3843 else 3844 skb = skb_clone(orig_skb, GFP_ATOMIC); 3845 if (!skb) 3846 return; 3847 3848 if (tsonly) { 3849 skb_shinfo(skb)->tx_flags = skb_shinfo(orig_skb)->tx_flags; 3850 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 3851 } 3852 3853 if (hwtstamps) 3854 *skb_hwtstamps(skb) = *hwtstamps; 3855 else 3856 skb->tstamp = ktime_get_real(); 3857 3858 __skb_complete_tx_timestamp(skb, sk, tstype); 3859 } 3860 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 3861 3862 void skb_tstamp_tx(struct sk_buff *orig_skb, 3863 struct skb_shared_hwtstamps *hwtstamps) 3864 { 3865 return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk, 3866 SCM_TSTAMP_SND); 3867 } 3868 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 3869 3870 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 3871 { 3872 struct sock *sk = skb->sk; 3873 struct sock_exterr_skb *serr; 3874 int err; 3875 3876 skb->wifi_acked_valid = 1; 3877 skb->wifi_acked = acked; 3878 3879 serr = SKB_EXT_ERR(skb); 3880 memset(serr, 0, sizeof(*serr)); 3881 serr->ee.ee_errno = ENOMSG; 3882 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 3883 3884 /* take a reference to prevent skb_orphan() from freeing the socket */ 3885 sock_hold(sk); 3886 3887 err = sock_queue_err_skb(sk, skb); 3888 if (err) 3889 kfree_skb(skb); 3890 3891 sock_put(sk); 3892 } 3893 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 3894 3895 /** 3896 * skb_partial_csum_set - set up and verify partial csum values for packet 3897 * @skb: the skb to set 3898 * @start: the number of bytes after skb->data to start checksumming. 3899 * @off: the offset from start to place the checksum. 3900 * 3901 * For untrusted partially-checksummed packets, we need to make sure the values 3902 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 3903 * 3904 * This function checks and sets those values and skb->ip_summed: if this 3905 * returns false you should drop the packet. 3906 */ 3907 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 3908 { 3909 if (unlikely(start > skb_headlen(skb)) || 3910 unlikely((int)start + off > skb_headlen(skb) - 2)) { 3911 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n", 3912 start, off, skb_headlen(skb)); 3913 return false; 3914 } 3915 skb->ip_summed = CHECKSUM_PARTIAL; 3916 skb->csum_start = skb_headroom(skb) + start; 3917 skb->csum_offset = off; 3918 skb_set_transport_header(skb, start); 3919 return true; 3920 } 3921 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 3922 3923 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 3924 unsigned int max) 3925 { 3926 if (skb_headlen(skb) >= len) 3927 return 0; 3928 3929 /* If we need to pullup then pullup to the max, so we 3930 * won't need to do it again. 3931 */ 3932 if (max > skb->len) 3933 max = skb->len; 3934 3935 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 3936 return -ENOMEM; 3937 3938 if (skb_headlen(skb) < len) 3939 return -EPROTO; 3940 3941 return 0; 3942 } 3943 3944 #define MAX_TCP_HDR_LEN (15 * 4) 3945 3946 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 3947 typeof(IPPROTO_IP) proto, 3948 unsigned int off) 3949 { 3950 switch (proto) { 3951 int err; 3952 3953 case IPPROTO_TCP: 3954 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 3955 off + MAX_TCP_HDR_LEN); 3956 if (!err && !skb_partial_csum_set(skb, off, 3957 offsetof(struct tcphdr, 3958 check))) 3959 err = -EPROTO; 3960 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 3961 3962 case IPPROTO_UDP: 3963 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 3964 off + sizeof(struct udphdr)); 3965 if (!err && !skb_partial_csum_set(skb, off, 3966 offsetof(struct udphdr, 3967 check))) 3968 err = -EPROTO; 3969 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 3970 } 3971 3972 return ERR_PTR(-EPROTO); 3973 } 3974 3975 /* This value should be large enough to cover a tagged ethernet header plus 3976 * maximally sized IP and TCP or UDP headers. 3977 */ 3978 #define MAX_IP_HDR_LEN 128 3979 3980 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 3981 { 3982 unsigned int off; 3983 bool fragment; 3984 __sum16 *csum; 3985 int err; 3986 3987 fragment = false; 3988 3989 err = skb_maybe_pull_tail(skb, 3990 sizeof(struct iphdr), 3991 MAX_IP_HDR_LEN); 3992 if (err < 0) 3993 goto out; 3994 3995 if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF)) 3996 fragment = true; 3997 3998 off = ip_hdrlen(skb); 3999 4000 err = -EPROTO; 4001 4002 if (fragment) 4003 goto out; 4004 4005 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 4006 if (IS_ERR(csum)) 4007 return PTR_ERR(csum); 4008 4009 if (recalculate) 4010 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 4011 ip_hdr(skb)->daddr, 4012 skb->len - off, 4013 ip_hdr(skb)->protocol, 0); 4014 err = 0; 4015 4016 out: 4017 return err; 4018 } 4019 4020 /* This value should be large enough to cover a tagged ethernet header plus 4021 * an IPv6 header, all options, and a maximal TCP or UDP header. 4022 */ 4023 #define MAX_IPV6_HDR_LEN 256 4024 4025 #define OPT_HDR(type, skb, off) \ 4026 (type *)(skb_network_header(skb) + (off)) 4027 4028 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 4029 { 4030 int err; 4031 u8 nexthdr; 4032 unsigned int off; 4033 unsigned int len; 4034 bool fragment; 4035 bool done; 4036 __sum16 *csum; 4037 4038 fragment = false; 4039 done = false; 4040 4041 off = sizeof(struct ipv6hdr); 4042 4043 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 4044 if (err < 0) 4045 goto out; 4046 4047 nexthdr = ipv6_hdr(skb)->nexthdr; 4048 4049 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 4050 while (off <= len && !done) { 4051 switch (nexthdr) { 4052 case IPPROTO_DSTOPTS: 4053 case IPPROTO_HOPOPTS: 4054 case IPPROTO_ROUTING: { 4055 struct ipv6_opt_hdr *hp; 4056 4057 err = skb_maybe_pull_tail(skb, 4058 off + 4059 sizeof(struct ipv6_opt_hdr), 4060 MAX_IPV6_HDR_LEN); 4061 if (err < 0) 4062 goto out; 4063 4064 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 4065 nexthdr = hp->nexthdr; 4066 off += ipv6_optlen(hp); 4067 break; 4068 } 4069 case IPPROTO_AH: { 4070 struct ip_auth_hdr *hp; 4071 4072 err = skb_maybe_pull_tail(skb, 4073 off + 4074 sizeof(struct ip_auth_hdr), 4075 MAX_IPV6_HDR_LEN); 4076 if (err < 0) 4077 goto out; 4078 4079 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 4080 nexthdr = hp->nexthdr; 4081 off += ipv6_authlen(hp); 4082 break; 4083 } 4084 case IPPROTO_FRAGMENT: { 4085 struct frag_hdr *hp; 4086 4087 err = skb_maybe_pull_tail(skb, 4088 off + 4089 sizeof(struct frag_hdr), 4090 MAX_IPV6_HDR_LEN); 4091 if (err < 0) 4092 goto out; 4093 4094 hp = OPT_HDR(struct frag_hdr, skb, off); 4095 4096 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 4097 fragment = true; 4098 4099 nexthdr = hp->nexthdr; 4100 off += sizeof(struct frag_hdr); 4101 break; 4102 } 4103 default: 4104 done = true; 4105 break; 4106 } 4107 } 4108 4109 err = -EPROTO; 4110 4111 if (!done || fragment) 4112 goto out; 4113 4114 csum = skb_checksum_setup_ip(skb, nexthdr, off); 4115 if (IS_ERR(csum)) 4116 return PTR_ERR(csum); 4117 4118 if (recalculate) 4119 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 4120 &ipv6_hdr(skb)->daddr, 4121 skb->len - off, nexthdr, 0); 4122 err = 0; 4123 4124 out: 4125 return err; 4126 } 4127 4128 /** 4129 * skb_checksum_setup - set up partial checksum offset 4130 * @skb: the skb to set up 4131 * @recalculate: if true the pseudo-header checksum will be recalculated 4132 */ 4133 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 4134 { 4135 int err; 4136 4137 switch (skb->protocol) { 4138 case htons(ETH_P_IP): 4139 err = skb_checksum_setup_ipv4(skb, recalculate); 4140 break; 4141 4142 case htons(ETH_P_IPV6): 4143 err = skb_checksum_setup_ipv6(skb, recalculate); 4144 break; 4145 4146 default: 4147 err = -EPROTO; 4148 break; 4149 } 4150 4151 return err; 4152 } 4153 EXPORT_SYMBOL(skb_checksum_setup); 4154 4155 /** 4156 * skb_checksum_maybe_trim - maybe trims the given skb 4157 * @skb: the skb to check 4158 * @transport_len: the data length beyond the network header 4159 * 4160 * Checks whether the given skb has data beyond the given transport length. 4161 * If so, returns a cloned skb trimmed to this transport length. 4162 * Otherwise returns the provided skb. Returns NULL in error cases 4163 * (e.g. transport_len exceeds skb length or out-of-memory). 4164 * 4165 * Caller needs to set the skb transport header and free any returned skb if it 4166 * differs from the provided skb. 4167 */ 4168 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 4169 unsigned int transport_len) 4170 { 4171 struct sk_buff *skb_chk; 4172 unsigned int len = skb_transport_offset(skb) + transport_len; 4173 int ret; 4174 4175 if (skb->len < len) 4176 return NULL; 4177 else if (skb->len == len) 4178 return skb; 4179 4180 skb_chk = skb_clone(skb, GFP_ATOMIC); 4181 if (!skb_chk) 4182 return NULL; 4183 4184 ret = pskb_trim_rcsum(skb_chk, len); 4185 if (ret) { 4186 kfree_skb(skb_chk); 4187 return NULL; 4188 } 4189 4190 return skb_chk; 4191 } 4192 4193 /** 4194 * skb_checksum_trimmed - validate checksum of an skb 4195 * @skb: the skb to check 4196 * @transport_len: the data length beyond the network header 4197 * @skb_chkf: checksum function to use 4198 * 4199 * Applies the given checksum function skb_chkf to the provided skb. 4200 * Returns a checked and maybe trimmed skb. Returns NULL on error. 4201 * 4202 * If the skb has data beyond the given transport length, then a 4203 * trimmed & cloned skb is checked and returned. 4204 * 4205 * Caller needs to set the skb transport header and free any returned skb if it 4206 * differs from the provided skb. 4207 */ 4208 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 4209 unsigned int transport_len, 4210 __sum16(*skb_chkf)(struct sk_buff *skb)) 4211 { 4212 struct sk_buff *skb_chk; 4213 unsigned int offset = skb_transport_offset(skb); 4214 __sum16 ret; 4215 4216 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 4217 if (!skb_chk) 4218 goto err; 4219 4220 if (!pskb_may_pull(skb_chk, offset)) 4221 goto err; 4222 4223 skb_pull_rcsum(skb_chk, offset); 4224 ret = skb_chkf(skb_chk); 4225 skb_push_rcsum(skb_chk, offset); 4226 4227 if (ret) 4228 goto err; 4229 4230 return skb_chk; 4231 4232 err: 4233 if (skb_chk && skb_chk != skb) 4234 kfree_skb(skb_chk); 4235 4236 return NULL; 4237 4238 } 4239 EXPORT_SYMBOL(skb_checksum_trimmed); 4240 4241 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 4242 { 4243 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 4244 skb->dev->name); 4245 } 4246 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 4247 4248 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 4249 { 4250 if (head_stolen) { 4251 skb_release_head_state(skb); 4252 kmem_cache_free(skbuff_head_cache, skb); 4253 } else { 4254 __kfree_skb(skb); 4255 } 4256 } 4257 EXPORT_SYMBOL(kfree_skb_partial); 4258 4259 /** 4260 * skb_try_coalesce - try to merge skb to prior one 4261 * @to: prior buffer 4262 * @from: buffer to add 4263 * @fragstolen: pointer to boolean 4264 * @delta_truesize: how much more was allocated than was requested 4265 */ 4266 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 4267 bool *fragstolen, int *delta_truesize) 4268 { 4269 int i, delta, len = from->len; 4270 4271 *fragstolen = false; 4272 4273 if (skb_cloned(to)) 4274 return false; 4275 4276 if (len <= skb_tailroom(to)) { 4277 if (len) 4278 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 4279 *delta_truesize = 0; 4280 return true; 4281 } 4282 4283 if (skb_has_frag_list(to) || skb_has_frag_list(from)) 4284 return false; 4285 4286 if (skb_headlen(from) != 0) { 4287 struct page *page; 4288 unsigned int offset; 4289 4290 if (skb_shinfo(to)->nr_frags + 4291 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) 4292 return false; 4293 4294 if (skb_head_is_locked(from)) 4295 return false; 4296 4297 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 4298 4299 page = virt_to_head_page(from->head); 4300 offset = from->data - (unsigned char *)page_address(page); 4301 4302 skb_fill_page_desc(to, skb_shinfo(to)->nr_frags, 4303 page, offset, skb_headlen(from)); 4304 *fragstolen = true; 4305 } else { 4306 if (skb_shinfo(to)->nr_frags + 4307 skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS) 4308 return false; 4309 4310 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 4311 } 4312 4313 WARN_ON_ONCE(delta < len); 4314 4315 memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags, 4316 skb_shinfo(from)->frags, 4317 skb_shinfo(from)->nr_frags * sizeof(skb_frag_t)); 4318 skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags; 4319 4320 if (!skb_cloned(from)) 4321 skb_shinfo(from)->nr_frags = 0; 4322 4323 /* if the skb is not cloned this does nothing 4324 * since we set nr_frags to 0. 4325 */ 4326 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) 4327 skb_frag_ref(from, i); 4328 4329 to->truesize += delta; 4330 to->len += len; 4331 to->data_len += len; 4332 4333 *delta_truesize = delta; 4334 return true; 4335 } 4336 EXPORT_SYMBOL(skb_try_coalesce); 4337 4338 /** 4339 * skb_scrub_packet - scrub an skb 4340 * 4341 * @skb: buffer to clean 4342 * @xnet: packet is crossing netns 4343 * 4344 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 4345 * into/from a tunnel. Some information have to be cleared during these 4346 * operations. 4347 * skb_scrub_packet can also be used to clean a skb before injecting it in 4348 * another namespace (@xnet == true). We have to clear all information in the 4349 * skb that could impact namespace isolation. 4350 */ 4351 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 4352 { 4353 skb->tstamp.tv64 = 0; 4354 skb->pkt_type = PACKET_HOST; 4355 skb->skb_iif = 0; 4356 skb->ignore_df = 0; 4357 skb_dst_drop(skb); 4358 secpath_reset(skb); 4359 nf_reset(skb); 4360 nf_reset_trace(skb); 4361 4362 if (!xnet) 4363 return; 4364 4365 skb_orphan(skb); 4366 skb->mark = 0; 4367 } 4368 EXPORT_SYMBOL_GPL(skb_scrub_packet); 4369 4370 /** 4371 * skb_gso_transport_seglen - Return length of individual segments of a gso packet 4372 * 4373 * @skb: GSO skb 4374 * 4375 * skb_gso_transport_seglen is used to determine the real size of the 4376 * individual segments, including Layer4 headers (TCP/UDP). 4377 * 4378 * The MAC/L2 or network (IP, IPv6) headers are not accounted for. 4379 */ 4380 unsigned int skb_gso_transport_seglen(const struct sk_buff *skb) 4381 { 4382 const struct skb_shared_info *shinfo = skb_shinfo(skb); 4383 unsigned int thlen = 0; 4384 4385 if (skb->encapsulation) { 4386 thlen = skb_inner_transport_header(skb) - 4387 skb_transport_header(skb); 4388 4389 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 4390 thlen += inner_tcp_hdrlen(skb); 4391 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 4392 thlen = tcp_hdrlen(skb); 4393 } else if (unlikely(shinfo->gso_type & SKB_GSO_SCTP)) { 4394 thlen = sizeof(struct sctphdr); 4395 } 4396 /* UFO sets gso_size to the size of the fragmentation 4397 * payload, i.e. the size of the L4 (UDP) header is already 4398 * accounted for. 4399 */ 4400 return thlen + shinfo->gso_size; 4401 } 4402 EXPORT_SYMBOL_GPL(skb_gso_transport_seglen); 4403 4404 /** 4405 * skb_gso_validate_mtu - Return in case such skb fits a given MTU 4406 * 4407 * @skb: GSO skb 4408 * @mtu: MTU to validate against 4409 * 4410 * skb_gso_validate_mtu validates if a given skb will fit a wanted MTU 4411 * once split. 4412 */ 4413 bool skb_gso_validate_mtu(const struct sk_buff *skb, unsigned int mtu) 4414 { 4415 const struct skb_shared_info *shinfo = skb_shinfo(skb); 4416 const struct sk_buff *iter; 4417 unsigned int hlen; 4418 4419 hlen = skb_gso_network_seglen(skb); 4420 4421 if (shinfo->gso_size != GSO_BY_FRAGS) 4422 return hlen <= mtu; 4423 4424 /* Undo this so we can re-use header sizes */ 4425 hlen -= GSO_BY_FRAGS; 4426 4427 skb_walk_frags(skb, iter) { 4428 if (hlen + skb_headlen(iter) > mtu) 4429 return false; 4430 } 4431 4432 return true; 4433 } 4434 EXPORT_SYMBOL_GPL(skb_gso_validate_mtu); 4435 4436 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 4437 { 4438 if (skb_cow(skb, skb_headroom(skb)) < 0) { 4439 kfree_skb(skb); 4440 return NULL; 4441 } 4442 4443 memmove(skb->data - ETH_HLEN, skb->data - skb->mac_len - VLAN_HLEN, 4444 2 * ETH_ALEN); 4445 skb->mac_header += VLAN_HLEN; 4446 return skb; 4447 } 4448 4449 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 4450 { 4451 struct vlan_hdr *vhdr; 4452 u16 vlan_tci; 4453 4454 if (unlikely(skb_vlan_tag_present(skb))) { 4455 /* vlan_tci is already set-up so leave this for another time */ 4456 return skb; 4457 } 4458 4459 skb = skb_share_check(skb, GFP_ATOMIC); 4460 if (unlikely(!skb)) 4461 goto err_free; 4462 4463 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN))) 4464 goto err_free; 4465 4466 vhdr = (struct vlan_hdr *)skb->data; 4467 vlan_tci = ntohs(vhdr->h_vlan_TCI); 4468 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 4469 4470 skb_pull_rcsum(skb, VLAN_HLEN); 4471 vlan_set_encap_proto(skb, vhdr); 4472 4473 skb = skb_reorder_vlan_header(skb); 4474 if (unlikely(!skb)) 4475 goto err_free; 4476 4477 skb_reset_network_header(skb); 4478 skb_reset_transport_header(skb); 4479 skb_reset_mac_len(skb); 4480 4481 return skb; 4482 4483 err_free: 4484 kfree_skb(skb); 4485 return NULL; 4486 } 4487 EXPORT_SYMBOL(skb_vlan_untag); 4488 4489 int skb_ensure_writable(struct sk_buff *skb, int write_len) 4490 { 4491 if (!pskb_may_pull(skb, write_len)) 4492 return -ENOMEM; 4493 4494 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 4495 return 0; 4496 4497 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 4498 } 4499 EXPORT_SYMBOL(skb_ensure_writable); 4500 4501 /* remove VLAN header from packet and update csum accordingly. 4502 * expects a non skb_vlan_tag_present skb with a vlan tag payload 4503 */ 4504 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 4505 { 4506 struct vlan_hdr *vhdr; 4507 int offset = skb->data - skb_mac_header(skb); 4508 int err; 4509 4510 if (WARN_ONCE(offset, 4511 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 4512 offset)) { 4513 return -EINVAL; 4514 } 4515 4516 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 4517 if (unlikely(err)) 4518 return err; 4519 4520 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 4521 4522 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN); 4523 *vlan_tci = ntohs(vhdr->h_vlan_TCI); 4524 4525 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN); 4526 __skb_pull(skb, VLAN_HLEN); 4527 4528 vlan_set_encap_proto(skb, vhdr); 4529 skb->mac_header += VLAN_HLEN; 4530 4531 if (skb_network_offset(skb) < ETH_HLEN) 4532 skb_set_network_header(skb, ETH_HLEN); 4533 4534 skb_reset_mac_len(skb); 4535 4536 return err; 4537 } 4538 EXPORT_SYMBOL(__skb_vlan_pop); 4539 4540 /* Pop a vlan tag either from hwaccel or from payload. 4541 * Expects skb->data at mac header. 4542 */ 4543 int skb_vlan_pop(struct sk_buff *skb) 4544 { 4545 u16 vlan_tci; 4546 __be16 vlan_proto; 4547 int err; 4548 4549 if (likely(skb_vlan_tag_present(skb))) { 4550 skb->vlan_tci = 0; 4551 } else { 4552 if (unlikely(!eth_type_vlan(skb->protocol))) 4553 return 0; 4554 4555 err = __skb_vlan_pop(skb, &vlan_tci); 4556 if (err) 4557 return err; 4558 } 4559 /* move next vlan tag to hw accel tag */ 4560 if (likely(!eth_type_vlan(skb->protocol))) 4561 return 0; 4562 4563 vlan_proto = skb->protocol; 4564 err = __skb_vlan_pop(skb, &vlan_tci); 4565 if (unlikely(err)) 4566 return err; 4567 4568 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 4569 return 0; 4570 } 4571 EXPORT_SYMBOL(skb_vlan_pop); 4572 4573 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 4574 * Expects skb->data at mac header. 4575 */ 4576 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 4577 { 4578 if (skb_vlan_tag_present(skb)) { 4579 int offset = skb->data - skb_mac_header(skb); 4580 int err; 4581 4582 if (WARN_ONCE(offset, 4583 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 4584 offset)) { 4585 return -EINVAL; 4586 } 4587 4588 err = __vlan_insert_tag(skb, skb->vlan_proto, 4589 skb_vlan_tag_get(skb)); 4590 if (err) 4591 return err; 4592 4593 skb->protocol = skb->vlan_proto; 4594 skb->mac_len += VLAN_HLEN; 4595 4596 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 4597 } 4598 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 4599 return 0; 4600 } 4601 EXPORT_SYMBOL(skb_vlan_push); 4602 4603 /** 4604 * alloc_skb_with_frags - allocate skb with page frags 4605 * 4606 * @header_len: size of linear part 4607 * @data_len: needed length in frags 4608 * @max_page_order: max page order desired. 4609 * @errcode: pointer to error code if any 4610 * @gfp_mask: allocation mask 4611 * 4612 * This can be used to allocate a paged skb, given a maximal order for frags. 4613 */ 4614 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 4615 unsigned long data_len, 4616 int max_page_order, 4617 int *errcode, 4618 gfp_t gfp_mask) 4619 { 4620 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 4621 unsigned long chunk; 4622 struct sk_buff *skb; 4623 struct page *page; 4624 gfp_t gfp_head; 4625 int i; 4626 4627 *errcode = -EMSGSIZE; 4628 /* Note this test could be relaxed, if we succeed to allocate 4629 * high order pages... 4630 */ 4631 if (npages > MAX_SKB_FRAGS) 4632 return NULL; 4633 4634 gfp_head = gfp_mask; 4635 if (gfp_head & __GFP_DIRECT_RECLAIM) 4636 gfp_head |= __GFP_REPEAT; 4637 4638 *errcode = -ENOBUFS; 4639 skb = alloc_skb(header_len, gfp_head); 4640 if (!skb) 4641 return NULL; 4642 4643 skb->truesize += npages << PAGE_SHIFT; 4644 4645 for (i = 0; npages > 0; i++) { 4646 int order = max_page_order; 4647 4648 while (order) { 4649 if (npages >= 1 << order) { 4650 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 4651 __GFP_COMP | 4652 __GFP_NOWARN | 4653 __GFP_NORETRY, 4654 order); 4655 if (page) 4656 goto fill_page; 4657 /* Do not retry other high order allocations */ 4658 order = 1; 4659 max_page_order = 0; 4660 } 4661 order--; 4662 } 4663 page = alloc_page(gfp_mask); 4664 if (!page) 4665 goto failure; 4666 fill_page: 4667 chunk = min_t(unsigned long, data_len, 4668 PAGE_SIZE << order); 4669 skb_fill_page_desc(skb, i, page, 0, chunk); 4670 data_len -= chunk; 4671 npages -= 1 << order; 4672 } 4673 return skb; 4674 4675 failure: 4676 kfree_skb(skb); 4677 return NULL; 4678 } 4679 EXPORT_SYMBOL(alloc_skb_with_frags); 4680 4681 /* carve out the first off bytes from skb when off < headlen */ 4682 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 4683 const int headlen, gfp_t gfp_mask) 4684 { 4685 int i; 4686 int size = skb_end_offset(skb); 4687 int new_hlen = headlen - off; 4688 u8 *data; 4689 4690 size = SKB_DATA_ALIGN(size); 4691 4692 if (skb_pfmemalloc(skb)) 4693 gfp_mask |= __GFP_MEMALLOC; 4694 data = kmalloc_reserve(size + 4695 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 4696 gfp_mask, NUMA_NO_NODE, NULL); 4697 if (!data) 4698 return -ENOMEM; 4699 4700 size = SKB_WITH_OVERHEAD(ksize(data)); 4701 4702 /* Copy real data, and all frags */ 4703 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 4704 skb->len -= off; 4705 4706 memcpy((struct skb_shared_info *)(data + size), 4707 skb_shinfo(skb), 4708 offsetof(struct skb_shared_info, 4709 frags[skb_shinfo(skb)->nr_frags])); 4710 if (skb_cloned(skb)) { 4711 /* drop the old head gracefully */ 4712 if (skb_orphan_frags(skb, gfp_mask)) { 4713 kfree(data); 4714 return -ENOMEM; 4715 } 4716 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 4717 skb_frag_ref(skb, i); 4718 if (skb_has_frag_list(skb)) 4719 skb_clone_fraglist(skb); 4720 skb_release_data(skb); 4721 } else { 4722 /* we can reuse existing recount- all we did was 4723 * relocate values 4724 */ 4725 skb_free_head(skb); 4726 } 4727 4728 skb->head = data; 4729 skb->data = data; 4730 skb->head_frag = 0; 4731 #ifdef NET_SKBUFF_DATA_USES_OFFSET 4732 skb->end = size; 4733 #else 4734 skb->end = skb->head + size; 4735 #endif 4736 skb_set_tail_pointer(skb, skb_headlen(skb)); 4737 skb_headers_offset_update(skb, 0); 4738 skb->cloned = 0; 4739 skb->hdr_len = 0; 4740 skb->nohdr = 0; 4741 atomic_set(&skb_shinfo(skb)->dataref, 1); 4742 4743 return 0; 4744 } 4745 4746 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 4747 4748 /* carve out the first eat bytes from skb's frag_list. May recurse into 4749 * pskb_carve() 4750 */ 4751 static int pskb_carve_frag_list(struct sk_buff *skb, 4752 struct skb_shared_info *shinfo, int eat, 4753 gfp_t gfp_mask) 4754 { 4755 struct sk_buff *list = shinfo->frag_list; 4756 struct sk_buff *clone = NULL; 4757 struct sk_buff *insp = NULL; 4758 4759 do { 4760 if (!list) { 4761 pr_err("Not enough bytes to eat. Want %d\n", eat); 4762 return -EFAULT; 4763 } 4764 if (list->len <= eat) { 4765 /* Eaten as whole. */ 4766 eat -= list->len; 4767 list = list->next; 4768 insp = list; 4769 } else { 4770 /* Eaten partially. */ 4771 if (skb_shared(list)) { 4772 clone = skb_clone(list, gfp_mask); 4773 if (!clone) 4774 return -ENOMEM; 4775 insp = list->next; 4776 list = clone; 4777 } else { 4778 /* This may be pulled without problems. */ 4779 insp = list; 4780 } 4781 if (pskb_carve(list, eat, gfp_mask) < 0) { 4782 kfree_skb(clone); 4783 return -ENOMEM; 4784 } 4785 break; 4786 } 4787 } while (eat); 4788 4789 /* Free pulled out fragments. */ 4790 while ((list = shinfo->frag_list) != insp) { 4791 shinfo->frag_list = list->next; 4792 kfree_skb(list); 4793 } 4794 /* And insert new clone at head. */ 4795 if (clone) { 4796 clone->next = list; 4797 shinfo->frag_list = clone; 4798 } 4799 return 0; 4800 } 4801 4802 /* carve off first len bytes from skb. Split line (off) is in the 4803 * non-linear part of skb 4804 */ 4805 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 4806 int pos, gfp_t gfp_mask) 4807 { 4808 int i, k = 0; 4809 int size = skb_end_offset(skb); 4810 u8 *data; 4811 const int nfrags = skb_shinfo(skb)->nr_frags; 4812 struct skb_shared_info *shinfo; 4813 4814 size = SKB_DATA_ALIGN(size); 4815 4816 if (skb_pfmemalloc(skb)) 4817 gfp_mask |= __GFP_MEMALLOC; 4818 data = kmalloc_reserve(size + 4819 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 4820 gfp_mask, NUMA_NO_NODE, NULL); 4821 if (!data) 4822 return -ENOMEM; 4823 4824 size = SKB_WITH_OVERHEAD(ksize(data)); 4825 4826 memcpy((struct skb_shared_info *)(data + size), 4827 skb_shinfo(skb), offsetof(struct skb_shared_info, 4828 frags[skb_shinfo(skb)->nr_frags])); 4829 if (skb_orphan_frags(skb, gfp_mask)) { 4830 kfree(data); 4831 return -ENOMEM; 4832 } 4833 shinfo = (struct skb_shared_info *)(data + size); 4834 for (i = 0; i < nfrags; i++) { 4835 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 4836 4837 if (pos + fsize > off) { 4838 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 4839 4840 if (pos < off) { 4841 /* Split frag. 4842 * We have two variants in this case: 4843 * 1. Move all the frag to the second 4844 * part, if it is possible. F.e. 4845 * this approach is mandatory for TUX, 4846 * where splitting is expensive. 4847 * 2. Split is accurately. We make this. 4848 */ 4849 shinfo->frags[0].page_offset += off - pos; 4850 skb_frag_size_sub(&shinfo->frags[0], off - pos); 4851 } 4852 skb_frag_ref(skb, i); 4853 k++; 4854 } 4855 pos += fsize; 4856 } 4857 shinfo->nr_frags = k; 4858 if (skb_has_frag_list(skb)) 4859 skb_clone_fraglist(skb); 4860 4861 if (k == 0) { 4862 /* split line is in frag list */ 4863 pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask); 4864 } 4865 skb_release_data(skb); 4866 4867 skb->head = data; 4868 skb->head_frag = 0; 4869 skb->data = data; 4870 #ifdef NET_SKBUFF_DATA_USES_OFFSET 4871 skb->end = size; 4872 #else 4873 skb->end = skb->head + size; 4874 #endif 4875 skb_reset_tail_pointer(skb); 4876 skb_headers_offset_update(skb, 0); 4877 skb->cloned = 0; 4878 skb->hdr_len = 0; 4879 skb->nohdr = 0; 4880 skb->len -= off; 4881 skb->data_len = skb->len; 4882 atomic_set(&skb_shinfo(skb)->dataref, 1); 4883 return 0; 4884 } 4885 4886 /* remove len bytes from the beginning of the skb */ 4887 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 4888 { 4889 int headlen = skb_headlen(skb); 4890 4891 if (len < headlen) 4892 return pskb_carve_inside_header(skb, len, headlen, gfp); 4893 else 4894 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 4895 } 4896 4897 /* Extract to_copy bytes starting at off from skb, and return this in 4898 * a new skb 4899 */ 4900 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 4901 int to_copy, gfp_t gfp) 4902 { 4903 struct sk_buff *clone = skb_clone(skb, gfp); 4904 4905 if (!clone) 4906 return NULL; 4907 4908 if (pskb_carve(clone, off, gfp) < 0 || 4909 pskb_trim(clone, to_copy)) { 4910 kfree_skb(clone); 4911 return NULL; 4912 } 4913 return clone; 4914 } 4915 EXPORT_SYMBOL(pskb_extract); 4916