1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Routines having to do with the 'struct sk_buff' memory handlers. 4 * 5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 6 * Florian La Roche <rzsfl@rz.uni-sb.de> 7 * 8 * Fixes: 9 * Alan Cox : Fixed the worst of the load 10 * balancer bugs. 11 * Dave Platt : Interrupt stacking fix. 12 * Richard Kooijman : Timestamp fixes. 13 * Alan Cox : Changed buffer format. 14 * Alan Cox : destructor hook for AF_UNIX etc. 15 * Linus Torvalds : Better skb_clone. 16 * Alan Cox : Added skb_copy. 17 * Alan Cox : Added all the changed routines Linus 18 * only put in the headers 19 * Ray VanTassle : Fixed --skb->lock in free 20 * Alan Cox : skb_copy copy arp field 21 * Andi Kleen : slabified it. 22 * Robert Olsson : Removed skb_head_pool 23 * 24 * NOTE: 25 * The __skb_ routines should be called with interrupts 26 * disabled, or you better be *real* sure that the operation is atomic 27 * with respect to whatever list is being frobbed (e.g. via lock_sock() 28 * or via disabling bottom half handlers, etc). 29 */ 30 31 /* 32 * The functions in this file will not compile correctly with gcc 2.4.x 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/module.h> 38 #include <linux/types.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/interrupt.h> 42 #include <linux/in.h> 43 #include <linux/inet.h> 44 #include <linux/slab.h> 45 #include <linux/tcp.h> 46 #include <linux/udp.h> 47 #include <linux/sctp.h> 48 #include <linux/netdevice.h> 49 #ifdef CONFIG_NET_CLS_ACT 50 #include <net/pkt_sched.h> 51 #endif 52 #include <linux/string.h> 53 #include <linux/skbuff.h> 54 #include <linux/splice.h> 55 #include <linux/cache.h> 56 #include <linux/rtnetlink.h> 57 #include <linux/init.h> 58 #include <linux/scatterlist.h> 59 #include <linux/errqueue.h> 60 #include <linux/prefetch.h> 61 #include <linux/if_vlan.h> 62 63 #include <net/protocol.h> 64 #include <net/dst.h> 65 #include <net/sock.h> 66 #include <net/checksum.h> 67 #include <net/ip6_checksum.h> 68 #include <net/xfrm.h> 69 70 #include <linux/uaccess.h> 71 #include <trace/events/skb.h> 72 #include <linux/highmem.h> 73 #include <linux/capability.h> 74 #include <linux/user_namespace.h> 75 76 #include "datagram.h" 77 78 struct kmem_cache *skbuff_head_cache __ro_after_init; 79 static struct kmem_cache *skbuff_fclone_cache __ro_after_init; 80 #ifdef CONFIG_SKB_EXTENSIONS 81 static struct kmem_cache *skbuff_ext_cache __ro_after_init; 82 #endif 83 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; 84 EXPORT_SYMBOL(sysctl_max_skb_frags); 85 86 /** 87 * skb_panic - private function for out-of-line support 88 * @skb: buffer 89 * @sz: size 90 * @addr: address 91 * @msg: skb_over_panic or skb_under_panic 92 * 93 * Out-of-line support for skb_put() and skb_push(). 94 * Called via the wrapper skb_over_panic() or skb_under_panic(). 95 * Keep out of line to prevent kernel bloat. 96 * __builtin_return_address is not used because it is not always reliable. 97 */ 98 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 99 const char msg[]) 100 { 101 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n", 102 msg, addr, skb->len, sz, skb->head, skb->data, 103 (unsigned long)skb->tail, (unsigned long)skb->end, 104 skb->dev ? skb->dev->name : "<NULL>"); 105 BUG(); 106 } 107 108 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 109 { 110 skb_panic(skb, sz, addr, __func__); 111 } 112 113 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 114 { 115 skb_panic(skb, sz, addr, __func__); 116 } 117 118 /* 119 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 120 * the caller if emergency pfmemalloc reserves are being used. If it is and 121 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 122 * may be used. Otherwise, the packet data may be discarded until enough 123 * memory is free 124 */ 125 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \ 126 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc) 127 128 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node, 129 unsigned long ip, bool *pfmemalloc) 130 { 131 void *obj; 132 bool ret_pfmemalloc = false; 133 134 /* 135 * Try a regular allocation, when that fails and we're not entitled 136 * to the reserves, fail. 137 */ 138 obj = kmalloc_node_track_caller(size, 139 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 140 node); 141 if (obj || !(gfp_pfmemalloc_allowed(flags))) 142 goto out; 143 144 /* Try again but now we are using pfmemalloc reserves */ 145 ret_pfmemalloc = true; 146 obj = kmalloc_node_track_caller(size, flags, node); 147 148 out: 149 if (pfmemalloc) 150 *pfmemalloc = ret_pfmemalloc; 151 152 return obj; 153 } 154 155 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 156 * 'private' fields and also do memory statistics to find all the 157 * [BEEP] leaks. 158 * 159 */ 160 161 /** 162 * __alloc_skb - allocate a network buffer 163 * @size: size to allocate 164 * @gfp_mask: allocation mask 165 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 166 * instead of head cache and allocate a cloned (child) skb. 167 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 168 * allocations in case the data is required for writeback 169 * @node: numa node to allocate memory on 170 * 171 * Allocate a new &sk_buff. The returned buffer has no headroom and a 172 * tail room of at least size bytes. The object has a reference count 173 * of one. The return is the buffer. On a failure the return is %NULL. 174 * 175 * Buffers may only be allocated from interrupts using a @gfp_mask of 176 * %GFP_ATOMIC. 177 */ 178 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 179 int flags, int node) 180 { 181 struct kmem_cache *cache; 182 struct skb_shared_info *shinfo; 183 struct sk_buff *skb; 184 u8 *data; 185 bool pfmemalloc; 186 187 cache = (flags & SKB_ALLOC_FCLONE) 188 ? skbuff_fclone_cache : skbuff_head_cache; 189 190 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 191 gfp_mask |= __GFP_MEMALLOC; 192 193 /* Get the HEAD */ 194 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); 195 if (!skb) 196 goto out; 197 prefetchw(skb); 198 199 /* We do our best to align skb_shared_info on a separate cache 200 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 201 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 202 * Both skb->head and skb_shared_info are cache line aligned. 203 */ 204 size = SKB_DATA_ALIGN(size); 205 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 206 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc); 207 if (!data) 208 goto nodata; 209 /* kmalloc(size) might give us more room than requested. 210 * Put skb_shared_info exactly at the end of allocated zone, 211 * to allow max possible filling before reallocation. 212 */ 213 size = SKB_WITH_OVERHEAD(ksize(data)); 214 prefetchw(data + size); 215 216 /* 217 * Only clear those fields we need to clear, not those that we will 218 * actually initialise below. Hence, don't put any more fields after 219 * the tail pointer in struct sk_buff! 220 */ 221 memset(skb, 0, offsetof(struct sk_buff, tail)); 222 /* Account for allocated memory : skb + skb->head */ 223 skb->truesize = SKB_TRUESIZE(size); 224 skb->pfmemalloc = pfmemalloc; 225 refcount_set(&skb->users, 1); 226 skb->head = data; 227 skb->data = data; 228 skb_reset_tail_pointer(skb); 229 skb->end = skb->tail + size; 230 skb->mac_header = (typeof(skb->mac_header))~0U; 231 skb->transport_header = (typeof(skb->transport_header))~0U; 232 233 /* make sure we initialize shinfo sequentially */ 234 shinfo = skb_shinfo(skb); 235 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 236 atomic_set(&shinfo->dataref, 1); 237 238 if (flags & SKB_ALLOC_FCLONE) { 239 struct sk_buff_fclones *fclones; 240 241 fclones = container_of(skb, struct sk_buff_fclones, skb1); 242 243 skb->fclone = SKB_FCLONE_ORIG; 244 refcount_set(&fclones->fclone_ref, 1); 245 246 fclones->skb2.fclone = SKB_FCLONE_CLONE; 247 } 248 out: 249 return skb; 250 nodata: 251 kmem_cache_free(cache, skb); 252 skb = NULL; 253 goto out; 254 } 255 EXPORT_SYMBOL(__alloc_skb); 256 257 /* Caller must provide SKB that is memset cleared */ 258 static struct sk_buff *__build_skb_around(struct sk_buff *skb, 259 void *data, unsigned int frag_size) 260 { 261 struct skb_shared_info *shinfo; 262 unsigned int size = frag_size ? : ksize(data); 263 264 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 265 266 /* Assumes caller memset cleared SKB */ 267 skb->truesize = SKB_TRUESIZE(size); 268 refcount_set(&skb->users, 1); 269 skb->head = data; 270 skb->data = data; 271 skb_reset_tail_pointer(skb); 272 skb->end = skb->tail + size; 273 skb->mac_header = (typeof(skb->mac_header))~0U; 274 skb->transport_header = (typeof(skb->transport_header))~0U; 275 276 /* make sure we initialize shinfo sequentially */ 277 shinfo = skb_shinfo(skb); 278 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 279 atomic_set(&shinfo->dataref, 1); 280 281 return skb; 282 } 283 284 /** 285 * __build_skb - build a network buffer 286 * @data: data buffer provided by caller 287 * @frag_size: size of data, or 0 if head was kmalloced 288 * 289 * Allocate a new &sk_buff. Caller provides space holding head and 290 * skb_shared_info. @data must have been allocated by kmalloc() only if 291 * @frag_size is 0, otherwise data should come from the page allocator 292 * or vmalloc() 293 * The return is the new skb buffer. 294 * On a failure the return is %NULL, and @data is not freed. 295 * Notes : 296 * Before IO, driver allocates only data buffer where NIC put incoming frame 297 * Driver should add room at head (NET_SKB_PAD) and 298 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 299 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 300 * before giving packet to stack. 301 * RX rings only contains data buffers, not full skbs. 302 */ 303 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 304 { 305 struct sk_buff *skb; 306 307 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); 308 if (unlikely(!skb)) 309 return NULL; 310 311 memset(skb, 0, offsetof(struct sk_buff, tail)); 312 313 return __build_skb_around(skb, data, frag_size); 314 } 315 316 /* build_skb() is wrapper over __build_skb(), that specifically 317 * takes care of skb->head and skb->pfmemalloc 318 * This means that if @frag_size is not zero, then @data must be backed 319 * by a page fragment, not kmalloc() or vmalloc() 320 */ 321 struct sk_buff *build_skb(void *data, unsigned int frag_size) 322 { 323 struct sk_buff *skb = __build_skb(data, frag_size); 324 325 if (skb && frag_size) { 326 skb->head_frag = 1; 327 if (page_is_pfmemalloc(virt_to_head_page(data))) 328 skb->pfmemalloc = 1; 329 } 330 return skb; 331 } 332 EXPORT_SYMBOL(build_skb); 333 334 /** 335 * build_skb_around - build a network buffer around provided skb 336 * @skb: sk_buff provide by caller, must be memset cleared 337 * @data: data buffer provided by caller 338 * @frag_size: size of data, or 0 if head was kmalloced 339 */ 340 struct sk_buff *build_skb_around(struct sk_buff *skb, 341 void *data, unsigned int frag_size) 342 { 343 if (unlikely(!skb)) 344 return NULL; 345 346 skb = __build_skb_around(skb, data, frag_size); 347 348 if (skb && frag_size) { 349 skb->head_frag = 1; 350 if (page_is_pfmemalloc(virt_to_head_page(data))) 351 skb->pfmemalloc = 1; 352 } 353 return skb; 354 } 355 EXPORT_SYMBOL(build_skb_around); 356 357 #define NAPI_SKB_CACHE_SIZE 64 358 359 struct napi_alloc_cache { 360 struct page_frag_cache page; 361 unsigned int skb_count; 362 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 363 }; 364 365 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 366 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 367 368 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) 369 { 370 struct page_frag_cache *nc; 371 unsigned long flags; 372 void *data; 373 374 local_irq_save(flags); 375 nc = this_cpu_ptr(&netdev_alloc_cache); 376 data = page_frag_alloc(nc, fragsz, gfp_mask); 377 local_irq_restore(flags); 378 return data; 379 } 380 381 /** 382 * netdev_alloc_frag - allocate a page fragment 383 * @fragsz: fragment size 384 * 385 * Allocates a frag from a page for receive buffer. 386 * Uses GFP_ATOMIC allocations. 387 */ 388 void *netdev_alloc_frag(unsigned int fragsz) 389 { 390 fragsz = SKB_DATA_ALIGN(fragsz); 391 392 return __netdev_alloc_frag(fragsz, GFP_ATOMIC); 393 } 394 EXPORT_SYMBOL(netdev_alloc_frag); 395 396 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) 397 { 398 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 399 400 return page_frag_alloc(&nc->page, fragsz, gfp_mask); 401 } 402 403 void *napi_alloc_frag(unsigned int fragsz) 404 { 405 fragsz = SKB_DATA_ALIGN(fragsz); 406 407 return __napi_alloc_frag(fragsz, GFP_ATOMIC); 408 } 409 EXPORT_SYMBOL(napi_alloc_frag); 410 411 /** 412 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 413 * @dev: network device to receive on 414 * @len: length to allocate 415 * @gfp_mask: get_free_pages mask, passed to alloc_skb 416 * 417 * Allocate a new &sk_buff and assign it a usage count of one. The 418 * buffer has NET_SKB_PAD headroom built in. Users should allocate 419 * the headroom they think they need without accounting for the 420 * built in space. The built in space is used for optimisations. 421 * 422 * %NULL is returned if there is no free memory. 423 */ 424 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 425 gfp_t gfp_mask) 426 { 427 struct page_frag_cache *nc; 428 unsigned long flags; 429 struct sk_buff *skb; 430 bool pfmemalloc; 431 void *data; 432 433 len += NET_SKB_PAD; 434 435 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) || 436 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 437 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 438 if (!skb) 439 goto skb_fail; 440 goto skb_success; 441 } 442 443 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 444 len = SKB_DATA_ALIGN(len); 445 446 if (sk_memalloc_socks()) 447 gfp_mask |= __GFP_MEMALLOC; 448 449 local_irq_save(flags); 450 451 nc = this_cpu_ptr(&netdev_alloc_cache); 452 data = page_frag_alloc(nc, len, gfp_mask); 453 pfmemalloc = nc->pfmemalloc; 454 455 local_irq_restore(flags); 456 457 if (unlikely(!data)) 458 return NULL; 459 460 skb = __build_skb(data, len); 461 if (unlikely(!skb)) { 462 skb_free_frag(data); 463 return NULL; 464 } 465 466 /* use OR instead of assignment to avoid clearing of bits in mask */ 467 if (pfmemalloc) 468 skb->pfmemalloc = 1; 469 skb->head_frag = 1; 470 471 skb_success: 472 skb_reserve(skb, NET_SKB_PAD); 473 skb->dev = dev; 474 475 skb_fail: 476 return skb; 477 } 478 EXPORT_SYMBOL(__netdev_alloc_skb); 479 480 /** 481 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 482 * @napi: napi instance this buffer was allocated for 483 * @len: length to allocate 484 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages 485 * 486 * Allocate a new sk_buff for use in NAPI receive. This buffer will 487 * attempt to allocate the head from a special reserved region used 488 * only for NAPI Rx allocation. By doing this we can save several 489 * CPU cycles by avoiding having to disable and re-enable IRQs. 490 * 491 * %NULL is returned if there is no free memory. 492 */ 493 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, 494 gfp_t gfp_mask) 495 { 496 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 497 struct sk_buff *skb; 498 void *data; 499 500 len += NET_SKB_PAD + NET_IP_ALIGN; 501 502 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) || 503 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 504 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 505 if (!skb) 506 goto skb_fail; 507 goto skb_success; 508 } 509 510 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 511 len = SKB_DATA_ALIGN(len); 512 513 if (sk_memalloc_socks()) 514 gfp_mask |= __GFP_MEMALLOC; 515 516 data = page_frag_alloc(&nc->page, len, gfp_mask); 517 if (unlikely(!data)) 518 return NULL; 519 520 skb = __build_skb(data, len); 521 if (unlikely(!skb)) { 522 skb_free_frag(data); 523 return NULL; 524 } 525 526 /* use OR instead of assignment to avoid clearing of bits in mask */ 527 if (nc->page.pfmemalloc) 528 skb->pfmemalloc = 1; 529 skb->head_frag = 1; 530 531 skb_success: 532 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 533 skb->dev = napi->dev; 534 535 skb_fail: 536 return skb; 537 } 538 EXPORT_SYMBOL(__napi_alloc_skb); 539 540 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 541 int size, unsigned int truesize) 542 { 543 skb_fill_page_desc(skb, i, page, off, size); 544 skb->len += size; 545 skb->data_len += size; 546 skb->truesize += truesize; 547 } 548 EXPORT_SYMBOL(skb_add_rx_frag); 549 550 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 551 unsigned int truesize) 552 { 553 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 554 555 skb_frag_size_add(frag, size); 556 skb->len += size; 557 skb->data_len += size; 558 skb->truesize += truesize; 559 } 560 EXPORT_SYMBOL(skb_coalesce_rx_frag); 561 562 static void skb_drop_list(struct sk_buff **listp) 563 { 564 kfree_skb_list(*listp); 565 *listp = NULL; 566 } 567 568 static inline void skb_drop_fraglist(struct sk_buff *skb) 569 { 570 skb_drop_list(&skb_shinfo(skb)->frag_list); 571 } 572 573 static void skb_clone_fraglist(struct sk_buff *skb) 574 { 575 struct sk_buff *list; 576 577 skb_walk_frags(skb, list) 578 skb_get(list); 579 } 580 581 static void skb_free_head(struct sk_buff *skb) 582 { 583 unsigned char *head = skb->head; 584 585 if (skb->head_frag) 586 skb_free_frag(head); 587 else 588 kfree(head); 589 } 590 591 static void skb_release_data(struct sk_buff *skb) 592 { 593 struct skb_shared_info *shinfo = skb_shinfo(skb); 594 int i; 595 596 if (skb->cloned && 597 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 598 &shinfo->dataref)) 599 return; 600 601 for (i = 0; i < shinfo->nr_frags; i++) 602 __skb_frag_unref(&shinfo->frags[i]); 603 604 if (shinfo->frag_list) 605 kfree_skb_list(shinfo->frag_list); 606 607 skb_zcopy_clear(skb, true); 608 skb_free_head(skb); 609 } 610 611 /* 612 * Free an skbuff by memory without cleaning the state. 613 */ 614 static void kfree_skbmem(struct sk_buff *skb) 615 { 616 struct sk_buff_fclones *fclones; 617 618 switch (skb->fclone) { 619 case SKB_FCLONE_UNAVAILABLE: 620 kmem_cache_free(skbuff_head_cache, skb); 621 return; 622 623 case SKB_FCLONE_ORIG: 624 fclones = container_of(skb, struct sk_buff_fclones, skb1); 625 626 /* We usually free the clone (TX completion) before original skb 627 * This test would have no chance to be true for the clone, 628 * while here, branch prediction will be good. 629 */ 630 if (refcount_read(&fclones->fclone_ref) == 1) 631 goto fastpath; 632 break; 633 634 default: /* SKB_FCLONE_CLONE */ 635 fclones = container_of(skb, struct sk_buff_fclones, skb2); 636 break; 637 } 638 if (!refcount_dec_and_test(&fclones->fclone_ref)) 639 return; 640 fastpath: 641 kmem_cache_free(skbuff_fclone_cache, fclones); 642 } 643 644 void skb_release_head_state(struct sk_buff *skb) 645 { 646 skb_dst_drop(skb); 647 if (skb->destructor) { 648 WARN_ON(in_irq()); 649 skb->destructor(skb); 650 } 651 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 652 nf_conntrack_put(skb_nfct(skb)); 653 #endif 654 skb_ext_put(skb); 655 } 656 657 /* Free everything but the sk_buff shell. */ 658 static void skb_release_all(struct sk_buff *skb) 659 { 660 skb_release_head_state(skb); 661 if (likely(skb->head)) 662 skb_release_data(skb); 663 } 664 665 /** 666 * __kfree_skb - private function 667 * @skb: buffer 668 * 669 * Free an sk_buff. Release anything attached to the buffer. 670 * Clean the state. This is an internal helper function. Users should 671 * always call kfree_skb 672 */ 673 674 void __kfree_skb(struct sk_buff *skb) 675 { 676 skb_release_all(skb); 677 kfree_skbmem(skb); 678 } 679 EXPORT_SYMBOL(__kfree_skb); 680 681 /** 682 * kfree_skb - free an sk_buff 683 * @skb: buffer to free 684 * 685 * Drop a reference to the buffer and free it if the usage count has 686 * hit zero. 687 */ 688 void kfree_skb(struct sk_buff *skb) 689 { 690 if (!skb_unref(skb)) 691 return; 692 693 trace_kfree_skb(skb, __builtin_return_address(0)); 694 __kfree_skb(skb); 695 } 696 EXPORT_SYMBOL(kfree_skb); 697 698 void kfree_skb_list(struct sk_buff *segs) 699 { 700 while (segs) { 701 struct sk_buff *next = segs->next; 702 703 kfree_skb(segs); 704 segs = next; 705 } 706 } 707 EXPORT_SYMBOL(kfree_skb_list); 708 709 /** 710 * skb_tx_error - report an sk_buff xmit error 711 * @skb: buffer that triggered an error 712 * 713 * Report xmit error if a device callback is tracking this skb. 714 * skb must be freed afterwards. 715 */ 716 void skb_tx_error(struct sk_buff *skb) 717 { 718 skb_zcopy_clear(skb, true); 719 } 720 EXPORT_SYMBOL(skb_tx_error); 721 722 /** 723 * consume_skb - free an skbuff 724 * @skb: buffer to free 725 * 726 * Drop a ref to the buffer and free it if the usage count has hit zero 727 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 728 * is being dropped after a failure and notes that 729 */ 730 void consume_skb(struct sk_buff *skb) 731 { 732 if (!skb_unref(skb)) 733 return; 734 735 trace_consume_skb(skb); 736 __kfree_skb(skb); 737 } 738 EXPORT_SYMBOL(consume_skb); 739 740 /** 741 * consume_stateless_skb - free an skbuff, assuming it is stateless 742 * @skb: buffer to free 743 * 744 * Alike consume_skb(), but this variant assumes that this is the last 745 * skb reference and all the head states have been already dropped 746 */ 747 void __consume_stateless_skb(struct sk_buff *skb) 748 { 749 trace_consume_skb(skb); 750 skb_release_data(skb); 751 kfree_skbmem(skb); 752 } 753 754 void __kfree_skb_flush(void) 755 { 756 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 757 758 /* flush skb_cache if containing objects */ 759 if (nc->skb_count) { 760 kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count, 761 nc->skb_cache); 762 nc->skb_count = 0; 763 } 764 } 765 766 static inline void _kfree_skb_defer(struct sk_buff *skb) 767 { 768 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 769 770 /* drop skb->head and call any destructors for packet */ 771 skb_release_all(skb); 772 773 /* record skb to CPU local list */ 774 nc->skb_cache[nc->skb_count++] = skb; 775 776 #ifdef CONFIG_SLUB 777 /* SLUB writes into objects when freeing */ 778 prefetchw(skb); 779 #endif 780 781 /* flush skb_cache if it is filled */ 782 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 783 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE, 784 nc->skb_cache); 785 nc->skb_count = 0; 786 } 787 } 788 void __kfree_skb_defer(struct sk_buff *skb) 789 { 790 _kfree_skb_defer(skb); 791 } 792 793 void napi_consume_skb(struct sk_buff *skb, int budget) 794 { 795 if (unlikely(!skb)) 796 return; 797 798 /* Zero budget indicate non-NAPI context called us, like netpoll */ 799 if (unlikely(!budget)) { 800 dev_consume_skb_any(skb); 801 return; 802 } 803 804 if (!skb_unref(skb)) 805 return; 806 807 /* if reaching here SKB is ready to free */ 808 trace_consume_skb(skb); 809 810 /* if SKB is a clone, don't handle this case */ 811 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 812 __kfree_skb(skb); 813 return; 814 } 815 816 _kfree_skb_defer(skb); 817 } 818 EXPORT_SYMBOL(napi_consume_skb); 819 820 /* Make sure a field is enclosed inside headers_start/headers_end section */ 821 #define CHECK_SKB_FIELD(field) \ 822 BUILD_BUG_ON(offsetof(struct sk_buff, field) < \ 823 offsetof(struct sk_buff, headers_start)); \ 824 BUILD_BUG_ON(offsetof(struct sk_buff, field) > \ 825 offsetof(struct sk_buff, headers_end)); \ 826 827 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 828 { 829 new->tstamp = old->tstamp; 830 /* We do not copy old->sk */ 831 new->dev = old->dev; 832 memcpy(new->cb, old->cb, sizeof(old->cb)); 833 skb_dst_copy(new, old); 834 __skb_ext_copy(new, old); 835 __nf_copy(new, old, false); 836 837 /* Note : this field could be in headers_start/headers_end section 838 * It is not yet because we do not want to have a 16 bit hole 839 */ 840 new->queue_mapping = old->queue_mapping; 841 842 memcpy(&new->headers_start, &old->headers_start, 843 offsetof(struct sk_buff, headers_end) - 844 offsetof(struct sk_buff, headers_start)); 845 CHECK_SKB_FIELD(protocol); 846 CHECK_SKB_FIELD(csum); 847 CHECK_SKB_FIELD(hash); 848 CHECK_SKB_FIELD(priority); 849 CHECK_SKB_FIELD(skb_iif); 850 CHECK_SKB_FIELD(vlan_proto); 851 CHECK_SKB_FIELD(vlan_tci); 852 CHECK_SKB_FIELD(transport_header); 853 CHECK_SKB_FIELD(network_header); 854 CHECK_SKB_FIELD(mac_header); 855 CHECK_SKB_FIELD(inner_protocol); 856 CHECK_SKB_FIELD(inner_transport_header); 857 CHECK_SKB_FIELD(inner_network_header); 858 CHECK_SKB_FIELD(inner_mac_header); 859 CHECK_SKB_FIELD(mark); 860 #ifdef CONFIG_NETWORK_SECMARK 861 CHECK_SKB_FIELD(secmark); 862 #endif 863 #ifdef CONFIG_NET_RX_BUSY_POLL 864 CHECK_SKB_FIELD(napi_id); 865 #endif 866 #ifdef CONFIG_XPS 867 CHECK_SKB_FIELD(sender_cpu); 868 #endif 869 #ifdef CONFIG_NET_SCHED 870 CHECK_SKB_FIELD(tc_index); 871 #endif 872 873 } 874 875 /* 876 * You should not add any new code to this function. Add it to 877 * __copy_skb_header above instead. 878 */ 879 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 880 { 881 #define C(x) n->x = skb->x 882 883 n->next = n->prev = NULL; 884 n->sk = NULL; 885 __copy_skb_header(n, skb); 886 887 C(len); 888 C(data_len); 889 C(mac_len); 890 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 891 n->cloned = 1; 892 n->nohdr = 0; 893 n->peeked = 0; 894 C(pfmemalloc); 895 n->destructor = NULL; 896 C(tail); 897 C(end); 898 C(head); 899 C(head_frag); 900 C(data); 901 C(truesize); 902 refcount_set(&n->users, 1); 903 904 atomic_inc(&(skb_shinfo(skb)->dataref)); 905 skb->cloned = 1; 906 907 return n; 908 #undef C 909 } 910 911 /** 912 * skb_morph - morph one skb into another 913 * @dst: the skb to receive the contents 914 * @src: the skb to supply the contents 915 * 916 * This is identical to skb_clone except that the target skb is 917 * supplied by the user. 918 * 919 * The target skb is returned upon exit. 920 */ 921 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 922 { 923 skb_release_all(dst); 924 return __skb_clone(dst, src); 925 } 926 EXPORT_SYMBOL_GPL(skb_morph); 927 928 int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 929 { 930 unsigned long max_pg, num_pg, new_pg, old_pg; 931 struct user_struct *user; 932 933 if (capable(CAP_IPC_LOCK) || !size) 934 return 0; 935 936 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 937 max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 938 user = mmp->user ? : current_user(); 939 940 do { 941 old_pg = atomic_long_read(&user->locked_vm); 942 new_pg = old_pg + num_pg; 943 if (new_pg > max_pg) 944 return -ENOBUFS; 945 } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) != 946 old_pg); 947 948 if (!mmp->user) { 949 mmp->user = get_uid(user); 950 mmp->num_pg = num_pg; 951 } else { 952 mmp->num_pg += num_pg; 953 } 954 955 return 0; 956 } 957 EXPORT_SYMBOL_GPL(mm_account_pinned_pages); 958 959 void mm_unaccount_pinned_pages(struct mmpin *mmp) 960 { 961 if (mmp->user) { 962 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 963 free_uid(mmp->user); 964 } 965 } 966 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); 967 968 struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size) 969 { 970 struct ubuf_info *uarg; 971 struct sk_buff *skb; 972 973 WARN_ON_ONCE(!in_task()); 974 975 skb = sock_omalloc(sk, 0, GFP_KERNEL); 976 if (!skb) 977 return NULL; 978 979 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 980 uarg = (void *)skb->cb; 981 uarg->mmp.user = NULL; 982 983 if (mm_account_pinned_pages(&uarg->mmp, size)) { 984 kfree_skb(skb); 985 return NULL; 986 } 987 988 uarg->callback = sock_zerocopy_callback; 989 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 990 uarg->len = 1; 991 uarg->bytelen = size; 992 uarg->zerocopy = 1; 993 refcount_set(&uarg->refcnt, 1); 994 sock_hold(sk); 995 996 return uarg; 997 } 998 EXPORT_SYMBOL_GPL(sock_zerocopy_alloc); 999 1000 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg) 1001 { 1002 return container_of((void *)uarg, struct sk_buff, cb); 1003 } 1004 1005 struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size, 1006 struct ubuf_info *uarg) 1007 { 1008 if (uarg) { 1009 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 1010 u32 bytelen, next; 1011 1012 /* realloc only when socket is locked (TCP, UDP cork), 1013 * so uarg->len and sk_zckey access is serialized 1014 */ 1015 if (!sock_owned_by_user(sk)) { 1016 WARN_ON_ONCE(1); 1017 return NULL; 1018 } 1019 1020 bytelen = uarg->bytelen + size; 1021 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) { 1022 /* TCP can create new skb to attach new uarg */ 1023 if (sk->sk_type == SOCK_STREAM) 1024 goto new_alloc; 1025 return NULL; 1026 } 1027 1028 next = (u32)atomic_read(&sk->sk_zckey); 1029 if ((u32)(uarg->id + uarg->len) == next) { 1030 if (mm_account_pinned_pages(&uarg->mmp, size)) 1031 return NULL; 1032 uarg->len++; 1033 uarg->bytelen = bytelen; 1034 atomic_set(&sk->sk_zckey, ++next); 1035 sock_zerocopy_get(uarg); 1036 return uarg; 1037 } 1038 } 1039 1040 new_alloc: 1041 return sock_zerocopy_alloc(sk, size); 1042 } 1043 EXPORT_SYMBOL_GPL(sock_zerocopy_realloc); 1044 1045 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1046 { 1047 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1048 u32 old_lo, old_hi; 1049 u64 sum_len; 1050 1051 old_lo = serr->ee.ee_info; 1052 old_hi = serr->ee.ee_data; 1053 sum_len = old_hi - old_lo + 1ULL + len; 1054 1055 if (sum_len >= (1ULL << 32)) 1056 return false; 1057 1058 if (lo != old_hi + 1) 1059 return false; 1060 1061 serr->ee.ee_data += len; 1062 return true; 1063 } 1064 1065 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success) 1066 { 1067 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1068 struct sock_exterr_skb *serr; 1069 struct sock *sk = skb->sk; 1070 struct sk_buff_head *q; 1071 unsigned long flags; 1072 u32 lo, hi; 1073 u16 len; 1074 1075 mm_unaccount_pinned_pages(&uarg->mmp); 1076 1077 /* if !len, there was only 1 call, and it was aborted 1078 * so do not queue a completion notification 1079 */ 1080 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1081 goto release; 1082 1083 len = uarg->len; 1084 lo = uarg->id; 1085 hi = uarg->id + len - 1; 1086 1087 serr = SKB_EXT_ERR(skb); 1088 memset(serr, 0, sizeof(*serr)); 1089 serr->ee.ee_errno = 0; 1090 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1091 serr->ee.ee_data = hi; 1092 serr->ee.ee_info = lo; 1093 if (!success) 1094 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1095 1096 q = &sk->sk_error_queue; 1097 spin_lock_irqsave(&q->lock, flags); 1098 tail = skb_peek_tail(q); 1099 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1100 !skb_zerocopy_notify_extend(tail, lo, len)) { 1101 __skb_queue_tail(q, skb); 1102 skb = NULL; 1103 } 1104 spin_unlock_irqrestore(&q->lock, flags); 1105 1106 sk->sk_error_report(sk); 1107 1108 release: 1109 consume_skb(skb); 1110 sock_put(sk); 1111 } 1112 EXPORT_SYMBOL_GPL(sock_zerocopy_callback); 1113 1114 void sock_zerocopy_put(struct ubuf_info *uarg) 1115 { 1116 if (uarg && refcount_dec_and_test(&uarg->refcnt)) { 1117 if (uarg->callback) 1118 uarg->callback(uarg, uarg->zerocopy); 1119 else 1120 consume_skb(skb_from_uarg(uarg)); 1121 } 1122 } 1123 EXPORT_SYMBOL_GPL(sock_zerocopy_put); 1124 1125 void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1126 { 1127 if (uarg) { 1128 struct sock *sk = skb_from_uarg(uarg)->sk; 1129 1130 atomic_dec(&sk->sk_zckey); 1131 uarg->len--; 1132 1133 if (have_uref) 1134 sock_zerocopy_put(uarg); 1135 } 1136 } 1137 EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort); 1138 1139 int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len) 1140 { 1141 return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len); 1142 } 1143 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram); 1144 1145 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1146 struct msghdr *msg, int len, 1147 struct ubuf_info *uarg) 1148 { 1149 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1150 struct iov_iter orig_iter = msg->msg_iter; 1151 int err, orig_len = skb->len; 1152 1153 /* An skb can only point to one uarg. This edge case happens when 1154 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc. 1155 */ 1156 if (orig_uarg && uarg != orig_uarg) 1157 return -EEXIST; 1158 1159 err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len); 1160 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1161 struct sock *save_sk = skb->sk; 1162 1163 /* Streams do not free skb on error. Reset to prev state. */ 1164 msg->msg_iter = orig_iter; 1165 skb->sk = sk; 1166 ___pskb_trim(skb, orig_len); 1167 skb->sk = save_sk; 1168 return err; 1169 } 1170 1171 skb_zcopy_set(skb, uarg, NULL); 1172 return skb->len - orig_len; 1173 } 1174 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1175 1176 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1177 gfp_t gfp_mask) 1178 { 1179 if (skb_zcopy(orig)) { 1180 if (skb_zcopy(nskb)) { 1181 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1182 if (!gfp_mask) { 1183 WARN_ON_ONCE(1); 1184 return -ENOMEM; 1185 } 1186 if (skb_uarg(nskb) == skb_uarg(orig)) 1187 return 0; 1188 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1189 return -EIO; 1190 } 1191 skb_zcopy_set(nskb, skb_uarg(orig), NULL); 1192 } 1193 return 0; 1194 } 1195 1196 /** 1197 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1198 * @skb: the skb to modify 1199 * @gfp_mask: allocation priority 1200 * 1201 * This must be called on SKBTX_DEV_ZEROCOPY skb. 1202 * It will copy all frags into kernel and drop the reference 1203 * to userspace pages. 1204 * 1205 * If this function is called from an interrupt gfp_mask() must be 1206 * %GFP_ATOMIC. 1207 * 1208 * Returns 0 on success or a negative error code on failure 1209 * to allocate kernel memory to copy to. 1210 */ 1211 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 1212 { 1213 int num_frags = skb_shinfo(skb)->nr_frags; 1214 struct page *page, *head = NULL; 1215 int i, new_frags; 1216 u32 d_off; 1217 1218 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 1219 return -EINVAL; 1220 1221 if (!num_frags) 1222 goto release; 1223 1224 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT; 1225 for (i = 0; i < new_frags; i++) { 1226 page = alloc_page(gfp_mask); 1227 if (!page) { 1228 while (head) { 1229 struct page *next = (struct page *)page_private(head); 1230 put_page(head); 1231 head = next; 1232 } 1233 return -ENOMEM; 1234 } 1235 set_page_private(page, (unsigned long)head); 1236 head = page; 1237 } 1238 1239 page = head; 1240 d_off = 0; 1241 for (i = 0; i < num_frags; i++) { 1242 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1243 u32 p_off, p_len, copied; 1244 struct page *p; 1245 u8 *vaddr; 1246 1247 skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f), 1248 p, p_off, p_len, copied) { 1249 u32 copy, done = 0; 1250 vaddr = kmap_atomic(p); 1251 1252 while (done < p_len) { 1253 if (d_off == PAGE_SIZE) { 1254 d_off = 0; 1255 page = (struct page *)page_private(page); 1256 } 1257 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done); 1258 memcpy(page_address(page) + d_off, 1259 vaddr + p_off + done, copy); 1260 done += copy; 1261 d_off += copy; 1262 } 1263 kunmap_atomic(vaddr); 1264 } 1265 } 1266 1267 /* skb frags release userspace buffers */ 1268 for (i = 0; i < num_frags; i++) 1269 skb_frag_unref(skb, i); 1270 1271 /* skb frags point to kernel buffers */ 1272 for (i = 0; i < new_frags - 1; i++) { 1273 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE); 1274 head = (struct page *)page_private(head); 1275 } 1276 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); 1277 skb_shinfo(skb)->nr_frags = new_frags; 1278 1279 release: 1280 skb_zcopy_clear(skb, false); 1281 return 0; 1282 } 1283 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 1284 1285 /** 1286 * skb_clone - duplicate an sk_buff 1287 * @skb: buffer to clone 1288 * @gfp_mask: allocation priority 1289 * 1290 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 1291 * copies share the same packet data but not structure. The new 1292 * buffer has a reference count of 1. If the allocation fails the 1293 * function returns %NULL otherwise the new buffer is returned. 1294 * 1295 * If this function is called from an interrupt gfp_mask() must be 1296 * %GFP_ATOMIC. 1297 */ 1298 1299 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 1300 { 1301 struct sk_buff_fclones *fclones = container_of(skb, 1302 struct sk_buff_fclones, 1303 skb1); 1304 struct sk_buff *n; 1305 1306 if (skb_orphan_frags(skb, gfp_mask)) 1307 return NULL; 1308 1309 if (skb->fclone == SKB_FCLONE_ORIG && 1310 refcount_read(&fclones->fclone_ref) == 1) { 1311 n = &fclones->skb2; 1312 refcount_set(&fclones->fclone_ref, 2); 1313 } else { 1314 if (skb_pfmemalloc(skb)) 1315 gfp_mask |= __GFP_MEMALLOC; 1316 1317 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); 1318 if (!n) 1319 return NULL; 1320 1321 n->fclone = SKB_FCLONE_UNAVAILABLE; 1322 } 1323 1324 return __skb_clone(n, skb); 1325 } 1326 EXPORT_SYMBOL(skb_clone); 1327 1328 void skb_headers_offset_update(struct sk_buff *skb, int off) 1329 { 1330 /* Only adjust this if it actually is csum_start rather than csum */ 1331 if (skb->ip_summed == CHECKSUM_PARTIAL) 1332 skb->csum_start += off; 1333 /* {transport,network,mac}_header and tail are relative to skb->head */ 1334 skb->transport_header += off; 1335 skb->network_header += off; 1336 if (skb_mac_header_was_set(skb)) 1337 skb->mac_header += off; 1338 skb->inner_transport_header += off; 1339 skb->inner_network_header += off; 1340 skb->inner_mac_header += off; 1341 } 1342 EXPORT_SYMBOL(skb_headers_offset_update); 1343 1344 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) 1345 { 1346 __copy_skb_header(new, old); 1347 1348 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 1349 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 1350 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 1351 } 1352 EXPORT_SYMBOL(skb_copy_header); 1353 1354 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 1355 { 1356 if (skb_pfmemalloc(skb)) 1357 return SKB_ALLOC_RX; 1358 return 0; 1359 } 1360 1361 /** 1362 * skb_copy - create private copy of an sk_buff 1363 * @skb: buffer to copy 1364 * @gfp_mask: allocation priority 1365 * 1366 * Make a copy of both an &sk_buff and its data. This is used when the 1367 * caller wishes to modify the data and needs a private copy of the 1368 * data to alter. Returns %NULL on failure or the pointer to the buffer 1369 * on success. The returned buffer has a reference count of 1. 1370 * 1371 * As by-product this function converts non-linear &sk_buff to linear 1372 * one, so that &sk_buff becomes completely private and caller is allowed 1373 * to modify all the data of returned buffer. This means that this 1374 * function is not recommended for use in circumstances when only 1375 * header is going to be modified. Use pskb_copy() instead. 1376 */ 1377 1378 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 1379 { 1380 int headerlen = skb_headroom(skb); 1381 unsigned int size = skb_end_offset(skb) + skb->data_len; 1382 struct sk_buff *n = __alloc_skb(size, gfp_mask, 1383 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 1384 1385 if (!n) 1386 return NULL; 1387 1388 /* Set the data pointer */ 1389 skb_reserve(n, headerlen); 1390 /* Set the tail pointer and length */ 1391 skb_put(n, skb->len); 1392 1393 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); 1394 1395 skb_copy_header(n, skb); 1396 return n; 1397 } 1398 EXPORT_SYMBOL(skb_copy); 1399 1400 /** 1401 * __pskb_copy_fclone - create copy of an sk_buff with private head. 1402 * @skb: buffer to copy 1403 * @headroom: headroom of new skb 1404 * @gfp_mask: allocation priority 1405 * @fclone: if true allocate the copy of the skb from the fclone 1406 * cache instead of the head cache; it is recommended to set this 1407 * to true for the cases where the copy will likely be cloned 1408 * 1409 * Make a copy of both an &sk_buff and part of its data, located 1410 * in header. Fragmented data remain shared. This is used when 1411 * the caller wishes to modify only header of &sk_buff and needs 1412 * private copy of the header to alter. Returns %NULL on failure 1413 * or the pointer to the buffer on success. 1414 * The returned buffer has a reference count of 1. 1415 */ 1416 1417 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 1418 gfp_t gfp_mask, bool fclone) 1419 { 1420 unsigned int size = skb_headlen(skb) + headroom; 1421 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 1422 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 1423 1424 if (!n) 1425 goto out; 1426 1427 /* Set the data pointer */ 1428 skb_reserve(n, headroom); 1429 /* Set the tail pointer and length */ 1430 skb_put(n, skb_headlen(skb)); 1431 /* Copy the bytes */ 1432 skb_copy_from_linear_data(skb, n->data, n->len); 1433 1434 n->truesize += skb->data_len; 1435 n->data_len = skb->data_len; 1436 n->len = skb->len; 1437 1438 if (skb_shinfo(skb)->nr_frags) { 1439 int i; 1440 1441 if (skb_orphan_frags(skb, gfp_mask) || 1442 skb_zerocopy_clone(n, skb, gfp_mask)) { 1443 kfree_skb(n); 1444 n = NULL; 1445 goto out; 1446 } 1447 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1448 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 1449 skb_frag_ref(skb, i); 1450 } 1451 skb_shinfo(n)->nr_frags = i; 1452 } 1453 1454 if (skb_has_frag_list(skb)) { 1455 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 1456 skb_clone_fraglist(n); 1457 } 1458 1459 skb_copy_header(n, skb); 1460 out: 1461 return n; 1462 } 1463 EXPORT_SYMBOL(__pskb_copy_fclone); 1464 1465 /** 1466 * pskb_expand_head - reallocate header of &sk_buff 1467 * @skb: buffer to reallocate 1468 * @nhead: room to add at head 1469 * @ntail: room to add at tail 1470 * @gfp_mask: allocation priority 1471 * 1472 * Expands (or creates identical copy, if @nhead and @ntail are zero) 1473 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 1474 * reference count of 1. Returns zero in the case of success or error, 1475 * if expansion failed. In the last case, &sk_buff is not changed. 1476 * 1477 * All the pointers pointing into skb header may change and must be 1478 * reloaded after call to this function. 1479 */ 1480 1481 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 1482 gfp_t gfp_mask) 1483 { 1484 int i, osize = skb_end_offset(skb); 1485 int size = osize + nhead + ntail; 1486 long off; 1487 u8 *data; 1488 1489 BUG_ON(nhead < 0); 1490 1491 BUG_ON(skb_shared(skb)); 1492 1493 size = SKB_DATA_ALIGN(size); 1494 1495 if (skb_pfmemalloc(skb)) 1496 gfp_mask |= __GFP_MEMALLOC; 1497 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 1498 gfp_mask, NUMA_NO_NODE, NULL); 1499 if (!data) 1500 goto nodata; 1501 size = SKB_WITH_OVERHEAD(ksize(data)); 1502 1503 /* Copy only real data... and, alas, header. This should be 1504 * optimized for the cases when header is void. 1505 */ 1506 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 1507 1508 memcpy((struct skb_shared_info *)(data + size), 1509 skb_shinfo(skb), 1510 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 1511 1512 /* 1513 * if shinfo is shared we must drop the old head gracefully, but if it 1514 * is not we can just drop the old head and let the existing refcount 1515 * be since all we did is relocate the values 1516 */ 1517 if (skb_cloned(skb)) { 1518 if (skb_orphan_frags(skb, gfp_mask)) 1519 goto nofrags; 1520 if (skb_zcopy(skb)) 1521 refcount_inc(&skb_uarg(skb)->refcnt); 1522 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1523 skb_frag_ref(skb, i); 1524 1525 if (skb_has_frag_list(skb)) 1526 skb_clone_fraglist(skb); 1527 1528 skb_release_data(skb); 1529 } else { 1530 skb_free_head(skb); 1531 } 1532 off = (data + nhead) - skb->head; 1533 1534 skb->head = data; 1535 skb->head_frag = 0; 1536 skb->data += off; 1537 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1538 skb->end = size; 1539 off = nhead; 1540 #else 1541 skb->end = skb->head + size; 1542 #endif 1543 skb->tail += off; 1544 skb_headers_offset_update(skb, nhead); 1545 skb->cloned = 0; 1546 skb->hdr_len = 0; 1547 skb->nohdr = 0; 1548 atomic_set(&skb_shinfo(skb)->dataref, 1); 1549 1550 skb_metadata_clear(skb); 1551 1552 /* It is not generally safe to change skb->truesize. 1553 * For the moment, we really care of rx path, or 1554 * when skb is orphaned (not attached to a socket). 1555 */ 1556 if (!skb->sk || skb->destructor == sock_edemux) 1557 skb->truesize += size - osize; 1558 1559 return 0; 1560 1561 nofrags: 1562 kfree(data); 1563 nodata: 1564 return -ENOMEM; 1565 } 1566 EXPORT_SYMBOL(pskb_expand_head); 1567 1568 /* Make private copy of skb with writable head and some headroom */ 1569 1570 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 1571 { 1572 struct sk_buff *skb2; 1573 int delta = headroom - skb_headroom(skb); 1574 1575 if (delta <= 0) 1576 skb2 = pskb_copy(skb, GFP_ATOMIC); 1577 else { 1578 skb2 = skb_clone(skb, GFP_ATOMIC); 1579 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 1580 GFP_ATOMIC)) { 1581 kfree_skb(skb2); 1582 skb2 = NULL; 1583 } 1584 } 1585 return skb2; 1586 } 1587 EXPORT_SYMBOL(skb_realloc_headroom); 1588 1589 /** 1590 * skb_copy_expand - copy and expand sk_buff 1591 * @skb: buffer to copy 1592 * @newheadroom: new free bytes at head 1593 * @newtailroom: new free bytes at tail 1594 * @gfp_mask: allocation priority 1595 * 1596 * Make a copy of both an &sk_buff and its data and while doing so 1597 * allocate additional space. 1598 * 1599 * This is used when the caller wishes to modify the data and needs a 1600 * private copy of the data to alter as well as more space for new fields. 1601 * Returns %NULL on failure or the pointer to the buffer 1602 * on success. The returned buffer has a reference count of 1. 1603 * 1604 * You must pass %GFP_ATOMIC as the allocation priority if this function 1605 * is called from an interrupt. 1606 */ 1607 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 1608 int newheadroom, int newtailroom, 1609 gfp_t gfp_mask) 1610 { 1611 /* 1612 * Allocate the copy buffer 1613 */ 1614 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 1615 gfp_mask, skb_alloc_rx_flag(skb), 1616 NUMA_NO_NODE); 1617 int oldheadroom = skb_headroom(skb); 1618 int head_copy_len, head_copy_off; 1619 1620 if (!n) 1621 return NULL; 1622 1623 skb_reserve(n, newheadroom); 1624 1625 /* Set the tail pointer and length */ 1626 skb_put(n, skb->len); 1627 1628 head_copy_len = oldheadroom; 1629 head_copy_off = 0; 1630 if (newheadroom <= head_copy_len) 1631 head_copy_len = newheadroom; 1632 else 1633 head_copy_off = newheadroom - head_copy_len; 1634 1635 /* Copy the linear header and data. */ 1636 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 1637 skb->len + head_copy_len)); 1638 1639 skb_copy_header(n, skb); 1640 1641 skb_headers_offset_update(n, newheadroom - oldheadroom); 1642 1643 return n; 1644 } 1645 EXPORT_SYMBOL(skb_copy_expand); 1646 1647 /** 1648 * __skb_pad - zero pad the tail of an skb 1649 * @skb: buffer to pad 1650 * @pad: space to pad 1651 * @free_on_error: free buffer on error 1652 * 1653 * Ensure that a buffer is followed by a padding area that is zero 1654 * filled. Used by network drivers which may DMA or transfer data 1655 * beyond the buffer end onto the wire. 1656 * 1657 * May return error in out of memory cases. The skb is freed on error 1658 * if @free_on_error is true. 1659 */ 1660 1661 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 1662 { 1663 int err; 1664 int ntail; 1665 1666 /* If the skbuff is non linear tailroom is always zero.. */ 1667 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 1668 memset(skb->data+skb->len, 0, pad); 1669 return 0; 1670 } 1671 1672 ntail = skb->data_len + pad - (skb->end - skb->tail); 1673 if (likely(skb_cloned(skb) || ntail > 0)) { 1674 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 1675 if (unlikely(err)) 1676 goto free_skb; 1677 } 1678 1679 /* FIXME: The use of this function with non-linear skb's really needs 1680 * to be audited. 1681 */ 1682 err = skb_linearize(skb); 1683 if (unlikely(err)) 1684 goto free_skb; 1685 1686 memset(skb->data + skb->len, 0, pad); 1687 return 0; 1688 1689 free_skb: 1690 if (free_on_error) 1691 kfree_skb(skb); 1692 return err; 1693 } 1694 EXPORT_SYMBOL(__skb_pad); 1695 1696 /** 1697 * pskb_put - add data to the tail of a potentially fragmented buffer 1698 * @skb: start of the buffer to use 1699 * @tail: tail fragment of the buffer to use 1700 * @len: amount of data to add 1701 * 1702 * This function extends the used data area of the potentially 1703 * fragmented buffer. @tail must be the last fragment of @skb -- or 1704 * @skb itself. If this would exceed the total buffer size the kernel 1705 * will panic. A pointer to the first byte of the extra data is 1706 * returned. 1707 */ 1708 1709 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 1710 { 1711 if (tail != skb) { 1712 skb->data_len += len; 1713 skb->len += len; 1714 } 1715 return skb_put(tail, len); 1716 } 1717 EXPORT_SYMBOL_GPL(pskb_put); 1718 1719 /** 1720 * skb_put - add data to a buffer 1721 * @skb: buffer to use 1722 * @len: amount of data to add 1723 * 1724 * This function extends the used data area of the buffer. If this would 1725 * exceed the total buffer size the kernel will panic. A pointer to the 1726 * first byte of the extra data is returned. 1727 */ 1728 void *skb_put(struct sk_buff *skb, unsigned int len) 1729 { 1730 void *tmp = skb_tail_pointer(skb); 1731 SKB_LINEAR_ASSERT(skb); 1732 skb->tail += len; 1733 skb->len += len; 1734 if (unlikely(skb->tail > skb->end)) 1735 skb_over_panic(skb, len, __builtin_return_address(0)); 1736 return tmp; 1737 } 1738 EXPORT_SYMBOL(skb_put); 1739 1740 /** 1741 * skb_push - add data to the start of a buffer 1742 * @skb: buffer to use 1743 * @len: amount of data to add 1744 * 1745 * This function extends the used data area of the buffer at the buffer 1746 * start. If this would exceed the total buffer headroom the kernel will 1747 * panic. A pointer to the first byte of the extra data is returned. 1748 */ 1749 void *skb_push(struct sk_buff *skb, unsigned int len) 1750 { 1751 skb->data -= len; 1752 skb->len += len; 1753 if (unlikely(skb->data < skb->head)) 1754 skb_under_panic(skb, len, __builtin_return_address(0)); 1755 return skb->data; 1756 } 1757 EXPORT_SYMBOL(skb_push); 1758 1759 /** 1760 * skb_pull - remove data from the start of a buffer 1761 * @skb: buffer to use 1762 * @len: amount of data to remove 1763 * 1764 * This function removes data from the start of a buffer, returning 1765 * the memory to the headroom. A pointer to the next data in the buffer 1766 * is returned. Once the data has been pulled future pushes will overwrite 1767 * the old data. 1768 */ 1769 void *skb_pull(struct sk_buff *skb, unsigned int len) 1770 { 1771 return skb_pull_inline(skb, len); 1772 } 1773 EXPORT_SYMBOL(skb_pull); 1774 1775 /** 1776 * skb_trim - remove end from a buffer 1777 * @skb: buffer to alter 1778 * @len: new length 1779 * 1780 * Cut the length of a buffer down by removing data from the tail. If 1781 * the buffer is already under the length specified it is not modified. 1782 * The skb must be linear. 1783 */ 1784 void skb_trim(struct sk_buff *skb, unsigned int len) 1785 { 1786 if (skb->len > len) 1787 __skb_trim(skb, len); 1788 } 1789 EXPORT_SYMBOL(skb_trim); 1790 1791 /* Trims skb to length len. It can change skb pointers. 1792 */ 1793 1794 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 1795 { 1796 struct sk_buff **fragp; 1797 struct sk_buff *frag; 1798 int offset = skb_headlen(skb); 1799 int nfrags = skb_shinfo(skb)->nr_frags; 1800 int i; 1801 int err; 1802 1803 if (skb_cloned(skb) && 1804 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 1805 return err; 1806 1807 i = 0; 1808 if (offset >= len) 1809 goto drop_pages; 1810 1811 for (; i < nfrags; i++) { 1812 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 1813 1814 if (end < len) { 1815 offset = end; 1816 continue; 1817 } 1818 1819 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 1820 1821 drop_pages: 1822 skb_shinfo(skb)->nr_frags = i; 1823 1824 for (; i < nfrags; i++) 1825 skb_frag_unref(skb, i); 1826 1827 if (skb_has_frag_list(skb)) 1828 skb_drop_fraglist(skb); 1829 goto done; 1830 } 1831 1832 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 1833 fragp = &frag->next) { 1834 int end = offset + frag->len; 1835 1836 if (skb_shared(frag)) { 1837 struct sk_buff *nfrag; 1838 1839 nfrag = skb_clone(frag, GFP_ATOMIC); 1840 if (unlikely(!nfrag)) 1841 return -ENOMEM; 1842 1843 nfrag->next = frag->next; 1844 consume_skb(frag); 1845 frag = nfrag; 1846 *fragp = frag; 1847 } 1848 1849 if (end < len) { 1850 offset = end; 1851 continue; 1852 } 1853 1854 if (end > len && 1855 unlikely((err = pskb_trim(frag, len - offset)))) 1856 return err; 1857 1858 if (frag->next) 1859 skb_drop_list(&frag->next); 1860 break; 1861 } 1862 1863 done: 1864 if (len > skb_headlen(skb)) { 1865 skb->data_len -= skb->len - len; 1866 skb->len = len; 1867 } else { 1868 skb->len = len; 1869 skb->data_len = 0; 1870 skb_set_tail_pointer(skb, len); 1871 } 1872 1873 if (!skb->sk || skb->destructor == sock_edemux) 1874 skb_condense(skb); 1875 return 0; 1876 } 1877 EXPORT_SYMBOL(___pskb_trim); 1878 1879 /* Note : use pskb_trim_rcsum() instead of calling this directly 1880 */ 1881 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) 1882 { 1883 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1884 int delta = skb->len - len; 1885 1886 skb->csum = csum_block_sub(skb->csum, 1887 skb_checksum(skb, len, delta, 0), 1888 len); 1889 } 1890 return __pskb_trim(skb, len); 1891 } 1892 EXPORT_SYMBOL(pskb_trim_rcsum_slow); 1893 1894 /** 1895 * __pskb_pull_tail - advance tail of skb header 1896 * @skb: buffer to reallocate 1897 * @delta: number of bytes to advance tail 1898 * 1899 * The function makes a sense only on a fragmented &sk_buff, 1900 * it expands header moving its tail forward and copying necessary 1901 * data from fragmented part. 1902 * 1903 * &sk_buff MUST have reference count of 1. 1904 * 1905 * Returns %NULL (and &sk_buff does not change) if pull failed 1906 * or value of new tail of skb in the case of success. 1907 * 1908 * All the pointers pointing into skb header may change and must be 1909 * reloaded after call to this function. 1910 */ 1911 1912 /* Moves tail of skb head forward, copying data from fragmented part, 1913 * when it is necessary. 1914 * 1. It may fail due to malloc failure. 1915 * 2. It may change skb pointers. 1916 * 1917 * It is pretty complicated. Luckily, it is called only in exceptional cases. 1918 */ 1919 void *__pskb_pull_tail(struct sk_buff *skb, int delta) 1920 { 1921 /* If skb has not enough free space at tail, get new one 1922 * plus 128 bytes for future expansions. If we have enough 1923 * room at tail, reallocate without expansion only if skb is cloned. 1924 */ 1925 int i, k, eat = (skb->tail + delta) - skb->end; 1926 1927 if (eat > 0 || skb_cloned(skb)) { 1928 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 1929 GFP_ATOMIC)) 1930 return NULL; 1931 } 1932 1933 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 1934 skb_tail_pointer(skb), delta)); 1935 1936 /* Optimization: no fragments, no reasons to preestimate 1937 * size of pulled pages. Superb. 1938 */ 1939 if (!skb_has_frag_list(skb)) 1940 goto pull_pages; 1941 1942 /* Estimate size of pulled pages. */ 1943 eat = delta; 1944 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1945 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 1946 1947 if (size >= eat) 1948 goto pull_pages; 1949 eat -= size; 1950 } 1951 1952 /* If we need update frag list, we are in troubles. 1953 * Certainly, it is possible to add an offset to skb data, 1954 * but taking into account that pulling is expected to 1955 * be very rare operation, it is worth to fight against 1956 * further bloating skb head and crucify ourselves here instead. 1957 * Pure masohism, indeed. 8)8) 1958 */ 1959 if (eat) { 1960 struct sk_buff *list = skb_shinfo(skb)->frag_list; 1961 struct sk_buff *clone = NULL; 1962 struct sk_buff *insp = NULL; 1963 1964 do { 1965 if (list->len <= eat) { 1966 /* Eaten as whole. */ 1967 eat -= list->len; 1968 list = list->next; 1969 insp = list; 1970 } else { 1971 /* Eaten partially. */ 1972 1973 if (skb_shared(list)) { 1974 /* Sucks! We need to fork list. :-( */ 1975 clone = skb_clone(list, GFP_ATOMIC); 1976 if (!clone) 1977 return NULL; 1978 insp = list->next; 1979 list = clone; 1980 } else { 1981 /* This may be pulled without 1982 * problems. */ 1983 insp = list; 1984 } 1985 if (!pskb_pull(list, eat)) { 1986 kfree_skb(clone); 1987 return NULL; 1988 } 1989 break; 1990 } 1991 } while (eat); 1992 1993 /* Free pulled out fragments. */ 1994 while ((list = skb_shinfo(skb)->frag_list) != insp) { 1995 skb_shinfo(skb)->frag_list = list->next; 1996 kfree_skb(list); 1997 } 1998 /* And insert new clone at head. */ 1999 if (clone) { 2000 clone->next = list; 2001 skb_shinfo(skb)->frag_list = clone; 2002 } 2003 } 2004 /* Success! Now we may commit changes to skb data. */ 2005 2006 pull_pages: 2007 eat = delta; 2008 k = 0; 2009 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2010 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2011 2012 if (size <= eat) { 2013 skb_frag_unref(skb, i); 2014 eat -= size; 2015 } else { 2016 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; 2017 if (eat) { 2018 skb_shinfo(skb)->frags[k].page_offset += eat; 2019 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat); 2020 if (!i) 2021 goto end; 2022 eat = 0; 2023 } 2024 k++; 2025 } 2026 } 2027 skb_shinfo(skb)->nr_frags = k; 2028 2029 end: 2030 skb->tail += delta; 2031 skb->data_len -= delta; 2032 2033 if (!skb->data_len) 2034 skb_zcopy_clear(skb, false); 2035 2036 return skb_tail_pointer(skb); 2037 } 2038 EXPORT_SYMBOL(__pskb_pull_tail); 2039 2040 /** 2041 * skb_copy_bits - copy bits from skb to kernel buffer 2042 * @skb: source skb 2043 * @offset: offset in source 2044 * @to: destination buffer 2045 * @len: number of bytes to copy 2046 * 2047 * Copy the specified number of bytes from the source skb to the 2048 * destination buffer. 2049 * 2050 * CAUTION ! : 2051 * If its prototype is ever changed, 2052 * check arch/{*}/net/{*}.S files, 2053 * since it is called from BPF assembly code. 2054 */ 2055 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 2056 { 2057 int start = skb_headlen(skb); 2058 struct sk_buff *frag_iter; 2059 int i, copy; 2060 2061 if (offset > (int)skb->len - len) 2062 goto fault; 2063 2064 /* Copy header. */ 2065 if ((copy = start - offset) > 0) { 2066 if (copy > len) 2067 copy = len; 2068 skb_copy_from_linear_data_offset(skb, offset, to, copy); 2069 if ((len -= copy) == 0) 2070 return 0; 2071 offset += copy; 2072 to += copy; 2073 } 2074 2075 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2076 int end; 2077 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2078 2079 WARN_ON(start > offset + len); 2080 2081 end = start + skb_frag_size(f); 2082 if ((copy = end - offset) > 0) { 2083 u32 p_off, p_len, copied; 2084 struct page *p; 2085 u8 *vaddr; 2086 2087 if (copy > len) 2088 copy = len; 2089 2090 skb_frag_foreach_page(f, 2091 f->page_offset + offset - start, 2092 copy, p, p_off, p_len, copied) { 2093 vaddr = kmap_atomic(p); 2094 memcpy(to + copied, vaddr + p_off, p_len); 2095 kunmap_atomic(vaddr); 2096 } 2097 2098 if ((len -= copy) == 0) 2099 return 0; 2100 offset += copy; 2101 to += copy; 2102 } 2103 start = end; 2104 } 2105 2106 skb_walk_frags(skb, frag_iter) { 2107 int end; 2108 2109 WARN_ON(start > offset + len); 2110 2111 end = start + frag_iter->len; 2112 if ((copy = end - offset) > 0) { 2113 if (copy > len) 2114 copy = len; 2115 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 2116 goto fault; 2117 if ((len -= copy) == 0) 2118 return 0; 2119 offset += copy; 2120 to += copy; 2121 } 2122 start = end; 2123 } 2124 2125 if (!len) 2126 return 0; 2127 2128 fault: 2129 return -EFAULT; 2130 } 2131 EXPORT_SYMBOL(skb_copy_bits); 2132 2133 /* 2134 * Callback from splice_to_pipe(), if we need to release some pages 2135 * at the end of the spd in case we error'ed out in filling the pipe. 2136 */ 2137 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 2138 { 2139 put_page(spd->pages[i]); 2140 } 2141 2142 static struct page *linear_to_page(struct page *page, unsigned int *len, 2143 unsigned int *offset, 2144 struct sock *sk) 2145 { 2146 struct page_frag *pfrag = sk_page_frag(sk); 2147 2148 if (!sk_page_frag_refill(sk, pfrag)) 2149 return NULL; 2150 2151 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 2152 2153 memcpy(page_address(pfrag->page) + pfrag->offset, 2154 page_address(page) + *offset, *len); 2155 *offset = pfrag->offset; 2156 pfrag->offset += *len; 2157 2158 return pfrag->page; 2159 } 2160 2161 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 2162 struct page *page, 2163 unsigned int offset) 2164 { 2165 return spd->nr_pages && 2166 spd->pages[spd->nr_pages - 1] == page && 2167 (spd->partial[spd->nr_pages - 1].offset + 2168 spd->partial[spd->nr_pages - 1].len == offset); 2169 } 2170 2171 /* 2172 * Fill page/offset/length into spd, if it can hold more pages. 2173 */ 2174 static bool spd_fill_page(struct splice_pipe_desc *spd, 2175 struct pipe_inode_info *pipe, struct page *page, 2176 unsigned int *len, unsigned int offset, 2177 bool linear, 2178 struct sock *sk) 2179 { 2180 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 2181 return true; 2182 2183 if (linear) { 2184 page = linear_to_page(page, len, &offset, sk); 2185 if (!page) 2186 return true; 2187 } 2188 if (spd_can_coalesce(spd, page, offset)) { 2189 spd->partial[spd->nr_pages - 1].len += *len; 2190 return false; 2191 } 2192 get_page(page); 2193 spd->pages[spd->nr_pages] = page; 2194 spd->partial[spd->nr_pages].len = *len; 2195 spd->partial[spd->nr_pages].offset = offset; 2196 spd->nr_pages++; 2197 2198 return false; 2199 } 2200 2201 static bool __splice_segment(struct page *page, unsigned int poff, 2202 unsigned int plen, unsigned int *off, 2203 unsigned int *len, 2204 struct splice_pipe_desc *spd, bool linear, 2205 struct sock *sk, 2206 struct pipe_inode_info *pipe) 2207 { 2208 if (!*len) 2209 return true; 2210 2211 /* skip this segment if already processed */ 2212 if (*off >= plen) { 2213 *off -= plen; 2214 return false; 2215 } 2216 2217 /* ignore any bits we already processed */ 2218 poff += *off; 2219 plen -= *off; 2220 *off = 0; 2221 2222 do { 2223 unsigned int flen = min(*len, plen); 2224 2225 if (spd_fill_page(spd, pipe, page, &flen, poff, 2226 linear, sk)) 2227 return true; 2228 poff += flen; 2229 plen -= flen; 2230 *len -= flen; 2231 } while (*len && plen); 2232 2233 return false; 2234 } 2235 2236 /* 2237 * Map linear and fragment data from the skb to spd. It reports true if the 2238 * pipe is full or if we already spliced the requested length. 2239 */ 2240 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 2241 unsigned int *offset, unsigned int *len, 2242 struct splice_pipe_desc *spd, struct sock *sk) 2243 { 2244 int seg; 2245 struct sk_buff *iter; 2246 2247 /* map the linear part : 2248 * If skb->head_frag is set, this 'linear' part is backed by a 2249 * fragment, and if the head is not shared with any clones then 2250 * we can avoid a copy since we own the head portion of this page. 2251 */ 2252 if (__splice_segment(virt_to_page(skb->data), 2253 (unsigned long) skb->data & (PAGE_SIZE - 1), 2254 skb_headlen(skb), 2255 offset, len, spd, 2256 skb_head_is_locked(skb), 2257 sk, pipe)) 2258 return true; 2259 2260 /* 2261 * then map the fragments 2262 */ 2263 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 2264 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 2265 2266 if (__splice_segment(skb_frag_page(f), 2267 f->page_offset, skb_frag_size(f), 2268 offset, len, spd, false, sk, pipe)) 2269 return true; 2270 } 2271 2272 skb_walk_frags(skb, iter) { 2273 if (*offset >= iter->len) { 2274 *offset -= iter->len; 2275 continue; 2276 } 2277 /* __skb_splice_bits() only fails if the output has no room 2278 * left, so no point in going over the frag_list for the error 2279 * case. 2280 */ 2281 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 2282 return true; 2283 } 2284 2285 return false; 2286 } 2287 2288 /* 2289 * Map data from the skb to a pipe. Should handle both the linear part, 2290 * the fragments, and the frag list. 2291 */ 2292 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 2293 struct pipe_inode_info *pipe, unsigned int tlen, 2294 unsigned int flags) 2295 { 2296 struct partial_page partial[MAX_SKB_FRAGS]; 2297 struct page *pages[MAX_SKB_FRAGS]; 2298 struct splice_pipe_desc spd = { 2299 .pages = pages, 2300 .partial = partial, 2301 .nr_pages_max = MAX_SKB_FRAGS, 2302 .ops = &nosteal_pipe_buf_ops, 2303 .spd_release = sock_spd_release, 2304 }; 2305 int ret = 0; 2306 2307 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 2308 2309 if (spd.nr_pages) 2310 ret = splice_to_pipe(pipe, &spd); 2311 2312 return ret; 2313 } 2314 EXPORT_SYMBOL_GPL(skb_splice_bits); 2315 2316 /* Send skb data on a socket. Socket must be locked. */ 2317 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 2318 int len) 2319 { 2320 unsigned int orig_len = len; 2321 struct sk_buff *head = skb; 2322 unsigned short fragidx; 2323 int slen, ret; 2324 2325 do_frag_list: 2326 2327 /* Deal with head data */ 2328 while (offset < skb_headlen(skb) && len) { 2329 struct kvec kv; 2330 struct msghdr msg; 2331 2332 slen = min_t(int, len, skb_headlen(skb) - offset); 2333 kv.iov_base = skb->data + offset; 2334 kv.iov_len = slen; 2335 memset(&msg, 0, sizeof(msg)); 2336 2337 ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen); 2338 if (ret <= 0) 2339 goto error; 2340 2341 offset += ret; 2342 len -= ret; 2343 } 2344 2345 /* All the data was skb head? */ 2346 if (!len) 2347 goto out; 2348 2349 /* Make offset relative to start of frags */ 2350 offset -= skb_headlen(skb); 2351 2352 /* Find where we are in frag list */ 2353 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2354 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2355 2356 if (offset < frag->size) 2357 break; 2358 2359 offset -= frag->size; 2360 } 2361 2362 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2363 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2364 2365 slen = min_t(size_t, len, frag->size - offset); 2366 2367 while (slen) { 2368 ret = kernel_sendpage_locked(sk, frag->page.p, 2369 frag->page_offset + offset, 2370 slen, MSG_DONTWAIT); 2371 if (ret <= 0) 2372 goto error; 2373 2374 len -= ret; 2375 offset += ret; 2376 slen -= ret; 2377 } 2378 2379 offset = 0; 2380 } 2381 2382 if (len) { 2383 /* Process any frag lists */ 2384 2385 if (skb == head) { 2386 if (skb_has_frag_list(skb)) { 2387 skb = skb_shinfo(skb)->frag_list; 2388 goto do_frag_list; 2389 } 2390 } else if (skb->next) { 2391 skb = skb->next; 2392 goto do_frag_list; 2393 } 2394 } 2395 2396 out: 2397 return orig_len - len; 2398 2399 error: 2400 return orig_len == len ? ret : orig_len - len; 2401 } 2402 EXPORT_SYMBOL_GPL(skb_send_sock_locked); 2403 2404 /** 2405 * skb_store_bits - store bits from kernel buffer to skb 2406 * @skb: destination buffer 2407 * @offset: offset in destination 2408 * @from: source buffer 2409 * @len: number of bytes to copy 2410 * 2411 * Copy the specified number of bytes from the source buffer to the 2412 * destination skb. This function handles all the messy bits of 2413 * traversing fragment lists and such. 2414 */ 2415 2416 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 2417 { 2418 int start = skb_headlen(skb); 2419 struct sk_buff *frag_iter; 2420 int i, copy; 2421 2422 if (offset > (int)skb->len - len) 2423 goto fault; 2424 2425 if ((copy = start - offset) > 0) { 2426 if (copy > len) 2427 copy = len; 2428 skb_copy_to_linear_data_offset(skb, offset, from, copy); 2429 if ((len -= copy) == 0) 2430 return 0; 2431 offset += copy; 2432 from += copy; 2433 } 2434 2435 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2436 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2437 int end; 2438 2439 WARN_ON(start > offset + len); 2440 2441 end = start + skb_frag_size(frag); 2442 if ((copy = end - offset) > 0) { 2443 u32 p_off, p_len, copied; 2444 struct page *p; 2445 u8 *vaddr; 2446 2447 if (copy > len) 2448 copy = len; 2449 2450 skb_frag_foreach_page(frag, 2451 frag->page_offset + offset - start, 2452 copy, p, p_off, p_len, copied) { 2453 vaddr = kmap_atomic(p); 2454 memcpy(vaddr + p_off, from + copied, p_len); 2455 kunmap_atomic(vaddr); 2456 } 2457 2458 if ((len -= copy) == 0) 2459 return 0; 2460 offset += copy; 2461 from += copy; 2462 } 2463 start = end; 2464 } 2465 2466 skb_walk_frags(skb, frag_iter) { 2467 int end; 2468 2469 WARN_ON(start > offset + len); 2470 2471 end = start + frag_iter->len; 2472 if ((copy = end - offset) > 0) { 2473 if (copy > len) 2474 copy = len; 2475 if (skb_store_bits(frag_iter, offset - start, 2476 from, copy)) 2477 goto fault; 2478 if ((len -= copy) == 0) 2479 return 0; 2480 offset += copy; 2481 from += copy; 2482 } 2483 start = end; 2484 } 2485 if (!len) 2486 return 0; 2487 2488 fault: 2489 return -EFAULT; 2490 } 2491 EXPORT_SYMBOL(skb_store_bits); 2492 2493 /* Checksum skb data. */ 2494 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 2495 __wsum csum, const struct skb_checksum_ops *ops) 2496 { 2497 int start = skb_headlen(skb); 2498 int i, copy = start - offset; 2499 struct sk_buff *frag_iter; 2500 int pos = 0; 2501 2502 /* Checksum header. */ 2503 if (copy > 0) { 2504 if (copy > len) 2505 copy = len; 2506 csum = ops->update(skb->data + offset, copy, csum); 2507 if ((len -= copy) == 0) 2508 return csum; 2509 offset += copy; 2510 pos = copy; 2511 } 2512 2513 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2514 int end; 2515 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2516 2517 WARN_ON(start > offset + len); 2518 2519 end = start + skb_frag_size(frag); 2520 if ((copy = end - offset) > 0) { 2521 u32 p_off, p_len, copied; 2522 struct page *p; 2523 __wsum csum2; 2524 u8 *vaddr; 2525 2526 if (copy > len) 2527 copy = len; 2528 2529 skb_frag_foreach_page(frag, 2530 frag->page_offset + offset - start, 2531 copy, p, p_off, p_len, copied) { 2532 vaddr = kmap_atomic(p); 2533 csum2 = ops->update(vaddr + p_off, p_len, 0); 2534 kunmap_atomic(vaddr); 2535 csum = ops->combine(csum, csum2, pos, p_len); 2536 pos += p_len; 2537 } 2538 2539 if (!(len -= copy)) 2540 return csum; 2541 offset += copy; 2542 } 2543 start = end; 2544 } 2545 2546 skb_walk_frags(skb, frag_iter) { 2547 int end; 2548 2549 WARN_ON(start > offset + len); 2550 2551 end = start + frag_iter->len; 2552 if ((copy = end - offset) > 0) { 2553 __wsum csum2; 2554 if (copy > len) 2555 copy = len; 2556 csum2 = __skb_checksum(frag_iter, offset - start, 2557 copy, 0, ops); 2558 csum = ops->combine(csum, csum2, pos, copy); 2559 if ((len -= copy) == 0) 2560 return csum; 2561 offset += copy; 2562 pos += copy; 2563 } 2564 start = end; 2565 } 2566 BUG_ON(len); 2567 2568 return csum; 2569 } 2570 EXPORT_SYMBOL(__skb_checksum); 2571 2572 __wsum skb_checksum(const struct sk_buff *skb, int offset, 2573 int len, __wsum csum) 2574 { 2575 const struct skb_checksum_ops ops = { 2576 .update = csum_partial_ext, 2577 .combine = csum_block_add_ext, 2578 }; 2579 2580 return __skb_checksum(skb, offset, len, csum, &ops); 2581 } 2582 EXPORT_SYMBOL(skb_checksum); 2583 2584 /* Both of above in one bottle. */ 2585 2586 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 2587 u8 *to, int len, __wsum csum) 2588 { 2589 int start = skb_headlen(skb); 2590 int i, copy = start - offset; 2591 struct sk_buff *frag_iter; 2592 int pos = 0; 2593 2594 /* Copy header. */ 2595 if (copy > 0) { 2596 if (copy > len) 2597 copy = len; 2598 csum = csum_partial_copy_nocheck(skb->data + offset, to, 2599 copy, csum); 2600 if ((len -= copy) == 0) 2601 return csum; 2602 offset += copy; 2603 to += copy; 2604 pos = copy; 2605 } 2606 2607 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2608 int end; 2609 2610 WARN_ON(start > offset + len); 2611 2612 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2613 if ((copy = end - offset) > 0) { 2614 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2615 u32 p_off, p_len, copied; 2616 struct page *p; 2617 __wsum csum2; 2618 u8 *vaddr; 2619 2620 if (copy > len) 2621 copy = len; 2622 2623 skb_frag_foreach_page(frag, 2624 frag->page_offset + offset - start, 2625 copy, p, p_off, p_len, copied) { 2626 vaddr = kmap_atomic(p); 2627 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 2628 to + copied, 2629 p_len, 0); 2630 kunmap_atomic(vaddr); 2631 csum = csum_block_add(csum, csum2, pos); 2632 pos += p_len; 2633 } 2634 2635 if (!(len -= copy)) 2636 return csum; 2637 offset += copy; 2638 to += copy; 2639 } 2640 start = end; 2641 } 2642 2643 skb_walk_frags(skb, frag_iter) { 2644 __wsum csum2; 2645 int end; 2646 2647 WARN_ON(start > offset + len); 2648 2649 end = start + frag_iter->len; 2650 if ((copy = end - offset) > 0) { 2651 if (copy > len) 2652 copy = len; 2653 csum2 = skb_copy_and_csum_bits(frag_iter, 2654 offset - start, 2655 to, copy, 0); 2656 csum = csum_block_add(csum, csum2, pos); 2657 if ((len -= copy) == 0) 2658 return csum; 2659 offset += copy; 2660 to += copy; 2661 pos += copy; 2662 } 2663 start = end; 2664 } 2665 BUG_ON(len); 2666 return csum; 2667 } 2668 EXPORT_SYMBOL(skb_copy_and_csum_bits); 2669 2670 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 2671 { 2672 __sum16 sum; 2673 2674 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 2675 /* See comments in __skb_checksum_complete(). */ 2676 if (likely(!sum)) { 2677 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 2678 !skb->csum_complete_sw) 2679 netdev_rx_csum_fault(skb->dev, skb); 2680 } 2681 if (!skb_shared(skb)) 2682 skb->csum_valid = !sum; 2683 return sum; 2684 } 2685 EXPORT_SYMBOL(__skb_checksum_complete_head); 2686 2687 /* This function assumes skb->csum already holds pseudo header's checksum, 2688 * which has been changed from the hardware checksum, for example, by 2689 * __skb_checksum_validate_complete(). And, the original skb->csum must 2690 * have been validated unsuccessfully for CHECKSUM_COMPLETE case. 2691 * 2692 * It returns non-zero if the recomputed checksum is still invalid, otherwise 2693 * zero. The new checksum is stored back into skb->csum unless the skb is 2694 * shared. 2695 */ 2696 __sum16 __skb_checksum_complete(struct sk_buff *skb) 2697 { 2698 __wsum csum; 2699 __sum16 sum; 2700 2701 csum = skb_checksum(skb, 0, skb->len, 0); 2702 2703 sum = csum_fold(csum_add(skb->csum, csum)); 2704 /* This check is inverted, because we already knew the hardware 2705 * checksum is invalid before calling this function. So, if the 2706 * re-computed checksum is valid instead, then we have a mismatch 2707 * between the original skb->csum and skb_checksum(). This means either 2708 * the original hardware checksum is incorrect or we screw up skb->csum 2709 * when moving skb->data around. 2710 */ 2711 if (likely(!sum)) { 2712 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 2713 !skb->csum_complete_sw) 2714 netdev_rx_csum_fault(skb->dev, skb); 2715 } 2716 2717 if (!skb_shared(skb)) { 2718 /* Save full packet checksum */ 2719 skb->csum = csum; 2720 skb->ip_summed = CHECKSUM_COMPLETE; 2721 skb->csum_complete_sw = 1; 2722 skb->csum_valid = !sum; 2723 } 2724 2725 return sum; 2726 } 2727 EXPORT_SYMBOL(__skb_checksum_complete); 2728 2729 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum) 2730 { 2731 net_warn_ratelimited( 2732 "%s: attempt to compute crc32c without libcrc32c.ko\n", 2733 __func__); 2734 return 0; 2735 } 2736 2737 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2, 2738 int offset, int len) 2739 { 2740 net_warn_ratelimited( 2741 "%s: attempt to compute crc32c without libcrc32c.ko\n", 2742 __func__); 2743 return 0; 2744 } 2745 2746 static const struct skb_checksum_ops default_crc32c_ops = { 2747 .update = warn_crc32c_csum_update, 2748 .combine = warn_crc32c_csum_combine, 2749 }; 2750 2751 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly = 2752 &default_crc32c_ops; 2753 EXPORT_SYMBOL(crc32c_csum_stub); 2754 2755 /** 2756 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 2757 * @from: source buffer 2758 * 2759 * Calculates the amount of linear headroom needed in the 'to' skb passed 2760 * into skb_zerocopy(). 2761 */ 2762 unsigned int 2763 skb_zerocopy_headlen(const struct sk_buff *from) 2764 { 2765 unsigned int hlen = 0; 2766 2767 if (!from->head_frag || 2768 skb_headlen(from) < L1_CACHE_BYTES || 2769 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) 2770 hlen = skb_headlen(from); 2771 2772 if (skb_has_frag_list(from)) 2773 hlen = from->len; 2774 2775 return hlen; 2776 } 2777 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 2778 2779 /** 2780 * skb_zerocopy - Zero copy skb to skb 2781 * @to: destination buffer 2782 * @from: source buffer 2783 * @len: number of bytes to copy from source buffer 2784 * @hlen: size of linear headroom in destination buffer 2785 * 2786 * Copies up to `len` bytes from `from` to `to` by creating references 2787 * to the frags in the source buffer. 2788 * 2789 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 2790 * headroom in the `to` buffer. 2791 * 2792 * Return value: 2793 * 0: everything is OK 2794 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 2795 * -EFAULT: skb_copy_bits() found some problem with skb geometry 2796 */ 2797 int 2798 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 2799 { 2800 int i, j = 0; 2801 int plen = 0; /* length of skb->head fragment */ 2802 int ret; 2803 struct page *page; 2804 unsigned int offset; 2805 2806 BUG_ON(!from->head_frag && !hlen); 2807 2808 /* dont bother with small payloads */ 2809 if (len <= skb_tailroom(to)) 2810 return skb_copy_bits(from, 0, skb_put(to, len), len); 2811 2812 if (hlen) { 2813 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 2814 if (unlikely(ret)) 2815 return ret; 2816 len -= hlen; 2817 } else { 2818 plen = min_t(int, skb_headlen(from), len); 2819 if (plen) { 2820 page = virt_to_head_page(from->head); 2821 offset = from->data - (unsigned char *)page_address(page); 2822 __skb_fill_page_desc(to, 0, page, offset, plen); 2823 get_page(page); 2824 j = 1; 2825 len -= plen; 2826 } 2827 } 2828 2829 to->truesize += len + plen; 2830 to->len += len + plen; 2831 to->data_len += len + plen; 2832 2833 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 2834 skb_tx_error(from); 2835 return -ENOMEM; 2836 } 2837 skb_zerocopy_clone(to, from, GFP_ATOMIC); 2838 2839 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 2840 if (!len) 2841 break; 2842 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 2843 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len); 2844 len -= skb_shinfo(to)->frags[j].size; 2845 skb_frag_ref(to, j); 2846 j++; 2847 } 2848 skb_shinfo(to)->nr_frags = j; 2849 2850 return 0; 2851 } 2852 EXPORT_SYMBOL_GPL(skb_zerocopy); 2853 2854 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 2855 { 2856 __wsum csum; 2857 long csstart; 2858 2859 if (skb->ip_summed == CHECKSUM_PARTIAL) 2860 csstart = skb_checksum_start_offset(skb); 2861 else 2862 csstart = skb_headlen(skb); 2863 2864 BUG_ON(csstart > skb_headlen(skb)); 2865 2866 skb_copy_from_linear_data(skb, to, csstart); 2867 2868 csum = 0; 2869 if (csstart != skb->len) 2870 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 2871 skb->len - csstart, 0); 2872 2873 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2874 long csstuff = csstart + skb->csum_offset; 2875 2876 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 2877 } 2878 } 2879 EXPORT_SYMBOL(skb_copy_and_csum_dev); 2880 2881 /** 2882 * skb_dequeue - remove from the head of the queue 2883 * @list: list to dequeue from 2884 * 2885 * Remove the head of the list. The list lock is taken so the function 2886 * may be used safely with other locking list functions. The head item is 2887 * returned or %NULL if the list is empty. 2888 */ 2889 2890 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 2891 { 2892 unsigned long flags; 2893 struct sk_buff *result; 2894 2895 spin_lock_irqsave(&list->lock, flags); 2896 result = __skb_dequeue(list); 2897 spin_unlock_irqrestore(&list->lock, flags); 2898 return result; 2899 } 2900 EXPORT_SYMBOL(skb_dequeue); 2901 2902 /** 2903 * skb_dequeue_tail - remove from the tail of the queue 2904 * @list: list to dequeue from 2905 * 2906 * Remove the tail of the list. The list lock is taken so the function 2907 * may be used safely with other locking list functions. The tail item is 2908 * returned or %NULL if the list is empty. 2909 */ 2910 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 2911 { 2912 unsigned long flags; 2913 struct sk_buff *result; 2914 2915 spin_lock_irqsave(&list->lock, flags); 2916 result = __skb_dequeue_tail(list); 2917 spin_unlock_irqrestore(&list->lock, flags); 2918 return result; 2919 } 2920 EXPORT_SYMBOL(skb_dequeue_tail); 2921 2922 /** 2923 * skb_queue_purge - empty a list 2924 * @list: list to empty 2925 * 2926 * Delete all buffers on an &sk_buff list. Each buffer is removed from 2927 * the list and one reference dropped. This function takes the list 2928 * lock and is atomic with respect to other list locking functions. 2929 */ 2930 void skb_queue_purge(struct sk_buff_head *list) 2931 { 2932 struct sk_buff *skb; 2933 while ((skb = skb_dequeue(list)) != NULL) 2934 kfree_skb(skb); 2935 } 2936 EXPORT_SYMBOL(skb_queue_purge); 2937 2938 /** 2939 * skb_rbtree_purge - empty a skb rbtree 2940 * @root: root of the rbtree to empty 2941 * Return value: the sum of truesizes of all purged skbs. 2942 * 2943 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 2944 * the list and one reference dropped. This function does not take 2945 * any lock. Synchronization should be handled by the caller (e.g., TCP 2946 * out-of-order queue is protected by the socket lock). 2947 */ 2948 unsigned int skb_rbtree_purge(struct rb_root *root) 2949 { 2950 struct rb_node *p = rb_first(root); 2951 unsigned int sum = 0; 2952 2953 while (p) { 2954 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 2955 2956 p = rb_next(p); 2957 rb_erase(&skb->rbnode, root); 2958 sum += skb->truesize; 2959 kfree_skb(skb); 2960 } 2961 return sum; 2962 } 2963 2964 /** 2965 * skb_queue_head - queue a buffer at the list head 2966 * @list: list to use 2967 * @newsk: buffer to queue 2968 * 2969 * Queue a buffer at the start of the list. This function takes the 2970 * list lock and can be used safely with other locking &sk_buff functions 2971 * safely. 2972 * 2973 * A buffer cannot be placed on two lists at the same time. 2974 */ 2975 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 2976 { 2977 unsigned long flags; 2978 2979 spin_lock_irqsave(&list->lock, flags); 2980 __skb_queue_head(list, newsk); 2981 spin_unlock_irqrestore(&list->lock, flags); 2982 } 2983 EXPORT_SYMBOL(skb_queue_head); 2984 2985 /** 2986 * skb_queue_tail - queue a buffer at the list tail 2987 * @list: list to use 2988 * @newsk: buffer to queue 2989 * 2990 * Queue a buffer at the tail of the list. This function takes the 2991 * list lock and can be used safely with other locking &sk_buff functions 2992 * safely. 2993 * 2994 * A buffer cannot be placed on two lists at the same time. 2995 */ 2996 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 2997 { 2998 unsigned long flags; 2999 3000 spin_lock_irqsave(&list->lock, flags); 3001 __skb_queue_tail(list, newsk); 3002 spin_unlock_irqrestore(&list->lock, flags); 3003 } 3004 EXPORT_SYMBOL(skb_queue_tail); 3005 3006 /** 3007 * skb_unlink - remove a buffer from a list 3008 * @skb: buffer to remove 3009 * @list: list to use 3010 * 3011 * Remove a packet from a list. The list locks are taken and this 3012 * function is atomic with respect to other list locked calls 3013 * 3014 * You must know what list the SKB is on. 3015 */ 3016 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 3017 { 3018 unsigned long flags; 3019 3020 spin_lock_irqsave(&list->lock, flags); 3021 __skb_unlink(skb, list); 3022 spin_unlock_irqrestore(&list->lock, flags); 3023 } 3024 EXPORT_SYMBOL(skb_unlink); 3025 3026 /** 3027 * skb_append - append a buffer 3028 * @old: buffer to insert after 3029 * @newsk: buffer to insert 3030 * @list: list to use 3031 * 3032 * Place a packet after a given packet in a list. The list locks are taken 3033 * and this function is atomic with respect to other list locked calls. 3034 * A buffer cannot be placed on two lists at the same time. 3035 */ 3036 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 3037 { 3038 unsigned long flags; 3039 3040 spin_lock_irqsave(&list->lock, flags); 3041 __skb_queue_after(list, old, newsk); 3042 spin_unlock_irqrestore(&list->lock, flags); 3043 } 3044 EXPORT_SYMBOL(skb_append); 3045 3046 static inline void skb_split_inside_header(struct sk_buff *skb, 3047 struct sk_buff* skb1, 3048 const u32 len, const int pos) 3049 { 3050 int i; 3051 3052 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 3053 pos - len); 3054 /* And move data appendix as is. */ 3055 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 3056 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 3057 3058 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 3059 skb_shinfo(skb)->nr_frags = 0; 3060 skb1->data_len = skb->data_len; 3061 skb1->len += skb1->data_len; 3062 skb->data_len = 0; 3063 skb->len = len; 3064 skb_set_tail_pointer(skb, len); 3065 } 3066 3067 static inline void skb_split_no_header(struct sk_buff *skb, 3068 struct sk_buff* skb1, 3069 const u32 len, int pos) 3070 { 3071 int i, k = 0; 3072 const int nfrags = skb_shinfo(skb)->nr_frags; 3073 3074 skb_shinfo(skb)->nr_frags = 0; 3075 skb1->len = skb1->data_len = skb->len - len; 3076 skb->len = len; 3077 skb->data_len = len - pos; 3078 3079 for (i = 0; i < nfrags; i++) { 3080 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 3081 3082 if (pos + size > len) { 3083 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 3084 3085 if (pos < len) { 3086 /* Split frag. 3087 * We have two variants in this case: 3088 * 1. Move all the frag to the second 3089 * part, if it is possible. F.e. 3090 * this approach is mandatory for TUX, 3091 * where splitting is expensive. 3092 * 2. Split is accurately. We make this. 3093 */ 3094 skb_frag_ref(skb, i); 3095 skb_shinfo(skb1)->frags[0].page_offset += len - pos; 3096 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 3097 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 3098 skb_shinfo(skb)->nr_frags++; 3099 } 3100 k++; 3101 } else 3102 skb_shinfo(skb)->nr_frags++; 3103 pos += size; 3104 } 3105 skb_shinfo(skb1)->nr_frags = k; 3106 } 3107 3108 /** 3109 * skb_split - Split fragmented skb to two parts at length len. 3110 * @skb: the buffer to split 3111 * @skb1: the buffer to receive the second part 3112 * @len: new length for skb 3113 */ 3114 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 3115 { 3116 int pos = skb_headlen(skb); 3117 3118 skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags & 3119 SKBTX_SHARED_FRAG; 3120 skb_zerocopy_clone(skb1, skb, 0); 3121 if (len < pos) /* Split line is inside header. */ 3122 skb_split_inside_header(skb, skb1, len, pos); 3123 else /* Second chunk has no header, nothing to copy. */ 3124 skb_split_no_header(skb, skb1, len, pos); 3125 } 3126 EXPORT_SYMBOL(skb_split); 3127 3128 /* Shifting from/to a cloned skb is a no-go. 3129 * 3130 * Caller cannot keep skb_shinfo related pointers past calling here! 3131 */ 3132 static int skb_prepare_for_shift(struct sk_buff *skb) 3133 { 3134 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 3135 } 3136 3137 /** 3138 * skb_shift - Shifts paged data partially from skb to another 3139 * @tgt: buffer into which tail data gets added 3140 * @skb: buffer from which the paged data comes from 3141 * @shiftlen: shift up to this many bytes 3142 * 3143 * Attempts to shift up to shiftlen worth of bytes, which may be less than 3144 * the length of the skb, from skb to tgt. Returns number bytes shifted. 3145 * It's up to caller to free skb if everything was shifted. 3146 * 3147 * If @tgt runs out of frags, the whole operation is aborted. 3148 * 3149 * Skb cannot include anything else but paged data while tgt is allowed 3150 * to have non-paged data as well. 3151 * 3152 * TODO: full sized shift could be optimized but that would need 3153 * specialized skb free'er to handle frags without up-to-date nr_frags. 3154 */ 3155 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 3156 { 3157 int from, to, merge, todo; 3158 struct skb_frag_struct *fragfrom, *fragto; 3159 3160 BUG_ON(shiftlen > skb->len); 3161 3162 if (skb_headlen(skb)) 3163 return 0; 3164 if (skb_zcopy(tgt) || skb_zcopy(skb)) 3165 return 0; 3166 3167 todo = shiftlen; 3168 from = 0; 3169 to = skb_shinfo(tgt)->nr_frags; 3170 fragfrom = &skb_shinfo(skb)->frags[from]; 3171 3172 /* Actual merge is delayed until the point when we know we can 3173 * commit all, so that we don't have to undo partial changes 3174 */ 3175 if (!to || 3176 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 3177 fragfrom->page_offset)) { 3178 merge = -1; 3179 } else { 3180 merge = to - 1; 3181 3182 todo -= skb_frag_size(fragfrom); 3183 if (todo < 0) { 3184 if (skb_prepare_for_shift(skb) || 3185 skb_prepare_for_shift(tgt)) 3186 return 0; 3187 3188 /* All previous frag pointers might be stale! */ 3189 fragfrom = &skb_shinfo(skb)->frags[from]; 3190 fragto = &skb_shinfo(tgt)->frags[merge]; 3191 3192 skb_frag_size_add(fragto, shiftlen); 3193 skb_frag_size_sub(fragfrom, shiftlen); 3194 fragfrom->page_offset += shiftlen; 3195 3196 goto onlymerged; 3197 } 3198 3199 from++; 3200 } 3201 3202 /* Skip full, not-fitting skb to avoid expensive operations */ 3203 if ((shiftlen == skb->len) && 3204 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 3205 return 0; 3206 3207 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 3208 return 0; 3209 3210 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 3211 if (to == MAX_SKB_FRAGS) 3212 return 0; 3213 3214 fragfrom = &skb_shinfo(skb)->frags[from]; 3215 fragto = &skb_shinfo(tgt)->frags[to]; 3216 3217 if (todo >= skb_frag_size(fragfrom)) { 3218 *fragto = *fragfrom; 3219 todo -= skb_frag_size(fragfrom); 3220 from++; 3221 to++; 3222 3223 } else { 3224 __skb_frag_ref(fragfrom); 3225 fragto->page = fragfrom->page; 3226 fragto->page_offset = fragfrom->page_offset; 3227 skb_frag_size_set(fragto, todo); 3228 3229 fragfrom->page_offset += todo; 3230 skb_frag_size_sub(fragfrom, todo); 3231 todo = 0; 3232 3233 to++; 3234 break; 3235 } 3236 } 3237 3238 /* Ready to "commit" this state change to tgt */ 3239 skb_shinfo(tgt)->nr_frags = to; 3240 3241 if (merge >= 0) { 3242 fragfrom = &skb_shinfo(skb)->frags[0]; 3243 fragto = &skb_shinfo(tgt)->frags[merge]; 3244 3245 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 3246 __skb_frag_unref(fragfrom); 3247 } 3248 3249 /* Reposition in the original skb */ 3250 to = 0; 3251 while (from < skb_shinfo(skb)->nr_frags) 3252 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 3253 skb_shinfo(skb)->nr_frags = to; 3254 3255 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 3256 3257 onlymerged: 3258 /* Most likely the tgt won't ever need its checksum anymore, skb on 3259 * the other hand might need it if it needs to be resent 3260 */ 3261 tgt->ip_summed = CHECKSUM_PARTIAL; 3262 skb->ip_summed = CHECKSUM_PARTIAL; 3263 3264 /* Yak, is it really working this way? Some helper please? */ 3265 skb->len -= shiftlen; 3266 skb->data_len -= shiftlen; 3267 skb->truesize -= shiftlen; 3268 tgt->len += shiftlen; 3269 tgt->data_len += shiftlen; 3270 tgt->truesize += shiftlen; 3271 3272 return shiftlen; 3273 } 3274 3275 /** 3276 * skb_prepare_seq_read - Prepare a sequential read of skb data 3277 * @skb: the buffer to read 3278 * @from: lower offset of data to be read 3279 * @to: upper offset of data to be read 3280 * @st: state variable 3281 * 3282 * Initializes the specified state variable. Must be called before 3283 * invoking skb_seq_read() for the first time. 3284 */ 3285 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 3286 unsigned int to, struct skb_seq_state *st) 3287 { 3288 st->lower_offset = from; 3289 st->upper_offset = to; 3290 st->root_skb = st->cur_skb = skb; 3291 st->frag_idx = st->stepped_offset = 0; 3292 st->frag_data = NULL; 3293 } 3294 EXPORT_SYMBOL(skb_prepare_seq_read); 3295 3296 /** 3297 * skb_seq_read - Sequentially read skb data 3298 * @consumed: number of bytes consumed by the caller so far 3299 * @data: destination pointer for data to be returned 3300 * @st: state variable 3301 * 3302 * Reads a block of skb data at @consumed relative to the 3303 * lower offset specified to skb_prepare_seq_read(). Assigns 3304 * the head of the data block to @data and returns the length 3305 * of the block or 0 if the end of the skb data or the upper 3306 * offset has been reached. 3307 * 3308 * The caller is not required to consume all of the data 3309 * returned, i.e. @consumed is typically set to the number 3310 * of bytes already consumed and the next call to 3311 * skb_seq_read() will return the remaining part of the block. 3312 * 3313 * Note 1: The size of each block of data returned can be arbitrary, 3314 * this limitation is the cost for zerocopy sequential 3315 * reads of potentially non linear data. 3316 * 3317 * Note 2: Fragment lists within fragments are not implemented 3318 * at the moment, state->root_skb could be replaced with 3319 * a stack for this purpose. 3320 */ 3321 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 3322 struct skb_seq_state *st) 3323 { 3324 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 3325 skb_frag_t *frag; 3326 3327 if (unlikely(abs_offset >= st->upper_offset)) { 3328 if (st->frag_data) { 3329 kunmap_atomic(st->frag_data); 3330 st->frag_data = NULL; 3331 } 3332 return 0; 3333 } 3334 3335 next_skb: 3336 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 3337 3338 if (abs_offset < block_limit && !st->frag_data) { 3339 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 3340 return block_limit - abs_offset; 3341 } 3342 3343 if (st->frag_idx == 0 && !st->frag_data) 3344 st->stepped_offset += skb_headlen(st->cur_skb); 3345 3346 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 3347 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 3348 block_limit = skb_frag_size(frag) + st->stepped_offset; 3349 3350 if (abs_offset < block_limit) { 3351 if (!st->frag_data) 3352 st->frag_data = kmap_atomic(skb_frag_page(frag)); 3353 3354 *data = (u8 *) st->frag_data + frag->page_offset + 3355 (abs_offset - st->stepped_offset); 3356 3357 return block_limit - abs_offset; 3358 } 3359 3360 if (st->frag_data) { 3361 kunmap_atomic(st->frag_data); 3362 st->frag_data = NULL; 3363 } 3364 3365 st->frag_idx++; 3366 st->stepped_offset += skb_frag_size(frag); 3367 } 3368 3369 if (st->frag_data) { 3370 kunmap_atomic(st->frag_data); 3371 st->frag_data = NULL; 3372 } 3373 3374 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 3375 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 3376 st->frag_idx = 0; 3377 goto next_skb; 3378 } else if (st->cur_skb->next) { 3379 st->cur_skb = st->cur_skb->next; 3380 st->frag_idx = 0; 3381 goto next_skb; 3382 } 3383 3384 return 0; 3385 } 3386 EXPORT_SYMBOL(skb_seq_read); 3387 3388 /** 3389 * skb_abort_seq_read - Abort a sequential read of skb data 3390 * @st: state variable 3391 * 3392 * Must be called if skb_seq_read() was not called until it 3393 * returned 0. 3394 */ 3395 void skb_abort_seq_read(struct skb_seq_state *st) 3396 { 3397 if (st->frag_data) 3398 kunmap_atomic(st->frag_data); 3399 } 3400 EXPORT_SYMBOL(skb_abort_seq_read); 3401 3402 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 3403 3404 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 3405 struct ts_config *conf, 3406 struct ts_state *state) 3407 { 3408 return skb_seq_read(offset, text, TS_SKB_CB(state)); 3409 } 3410 3411 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 3412 { 3413 skb_abort_seq_read(TS_SKB_CB(state)); 3414 } 3415 3416 /** 3417 * skb_find_text - Find a text pattern in skb data 3418 * @skb: the buffer to look in 3419 * @from: search offset 3420 * @to: search limit 3421 * @config: textsearch configuration 3422 * 3423 * Finds a pattern in the skb data according to the specified 3424 * textsearch configuration. Use textsearch_next() to retrieve 3425 * subsequent occurrences of the pattern. Returns the offset 3426 * to the first occurrence or UINT_MAX if no match was found. 3427 */ 3428 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 3429 unsigned int to, struct ts_config *config) 3430 { 3431 struct ts_state state; 3432 unsigned int ret; 3433 3434 config->get_next_block = skb_ts_get_next_block; 3435 config->finish = skb_ts_finish; 3436 3437 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 3438 3439 ret = textsearch_find(config, &state); 3440 return (ret <= to - from ? ret : UINT_MAX); 3441 } 3442 EXPORT_SYMBOL(skb_find_text); 3443 3444 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 3445 int offset, size_t size) 3446 { 3447 int i = skb_shinfo(skb)->nr_frags; 3448 3449 if (skb_can_coalesce(skb, i, page, offset)) { 3450 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 3451 } else if (i < MAX_SKB_FRAGS) { 3452 get_page(page); 3453 skb_fill_page_desc(skb, i, page, offset, size); 3454 } else { 3455 return -EMSGSIZE; 3456 } 3457 3458 return 0; 3459 } 3460 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 3461 3462 /** 3463 * skb_pull_rcsum - pull skb and update receive checksum 3464 * @skb: buffer to update 3465 * @len: length of data pulled 3466 * 3467 * This function performs an skb_pull on the packet and updates 3468 * the CHECKSUM_COMPLETE checksum. It should be used on 3469 * receive path processing instead of skb_pull unless you know 3470 * that the checksum difference is zero (e.g., a valid IP header) 3471 * or you are setting ip_summed to CHECKSUM_NONE. 3472 */ 3473 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 3474 { 3475 unsigned char *data = skb->data; 3476 3477 BUG_ON(len > skb->len); 3478 __skb_pull(skb, len); 3479 skb_postpull_rcsum(skb, data, len); 3480 return skb->data; 3481 } 3482 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 3483 3484 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb) 3485 { 3486 skb_frag_t head_frag; 3487 struct page *page; 3488 3489 page = virt_to_head_page(frag_skb->head); 3490 head_frag.page.p = page; 3491 head_frag.page_offset = frag_skb->data - 3492 (unsigned char *)page_address(page); 3493 head_frag.size = skb_headlen(frag_skb); 3494 return head_frag; 3495 } 3496 3497 /** 3498 * skb_segment - Perform protocol segmentation on skb. 3499 * @head_skb: buffer to segment 3500 * @features: features for the output path (see dev->features) 3501 * 3502 * This function performs segmentation on the given skb. It returns 3503 * a pointer to the first in a list of new skbs for the segments. 3504 * In case of error it returns ERR_PTR(err). 3505 */ 3506 struct sk_buff *skb_segment(struct sk_buff *head_skb, 3507 netdev_features_t features) 3508 { 3509 struct sk_buff *segs = NULL; 3510 struct sk_buff *tail = NULL; 3511 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 3512 skb_frag_t *frag = skb_shinfo(head_skb)->frags; 3513 unsigned int mss = skb_shinfo(head_skb)->gso_size; 3514 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 3515 struct sk_buff *frag_skb = head_skb; 3516 unsigned int offset = doffset; 3517 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 3518 unsigned int partial_segs = 0; 3519 unsigned int headroom; 3520 unsigned int len = head_skb->len; 3521 __be16 proto; 3522 bool csum, sg; 3523 int nfrags = skb_shinfo(head_skb)->nr_frags; 3524 int err = -ENOMEM; 3525 int i = 0; 3526 int pos; 3527 int dummy; 3528 3529 __skb_push(head_skb, doffset); 3530 proto = skb_network_protocol(head_skb, &dummy); 3531 if (unlikely(!proto)) 3532 return ERR_PTR(-EINVAL); 3533 3534 sg = !!(features & NETIF_F_SG); 3535 csum = !!can_checksum_protocol(features, proto); 3536 3537 if (sg && csum && (mss != GSO_BY_FRAGS)) { 3538 if (!(features & NETIF_F_GSO_PARTIAL)) { 3539 struct sk_buff *iter; 3540 unsigned int frag_len; 3541 3542 if (!list_skb || 3543 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 3544 goto normal; 3545 3546 /* If we get here then all the required 3547 * GSO features except frag_list are supported. 3548 * Try to split the SKB to multiple GSO SKBs 3549 * with no frag_list. 3550 * Currently we can do that only when the buffers don't 3551 * have a linear part and all the buffers except 3552 * the last are of the same length. 3553 */ 3554 frag_len = list_skb->len; 3555 skb_walk_frags(head_skb, iter) { 3556 if (frag_len != iter->len && iter->next) 3557 goto normal; 3558 if (skb_headlen(iter) && !iter->head_frag) 3559 goto normal; 3560 3561 len -= iter->len; 3562 } 3563 3564 if (len != frag_len) 3565 goto normal; 3566 } 3567 3568 /* GSO partial only requires that we trim off any excess that 3569 * doesn't fit into an MSS sized block, so take care of that 3570 * now. 3571 */ 3572 partial_segs = len / mss; 3573 if (partial_segs > 1) 3574 mss *= partial_segs; 3575 else 3576 partial_segs = 0; 3577 } 3578 3579 normal: 3580 headroom = skb_headroom(head_skb); 3581 pos = skb_headlen(head_skb); 3582 3583 do { 3584 struct sk_buff *nskb; 3585 skb_frag_t *nskb_frag; 3586 int hsize; 3587 int size; 3588 3589 if (unlikely(mss == GSO_BY_FRAGS)) { 3590 len = list_skb->len; 3591 } else { 3592 len = head_skb->len - offset; 3593 if (len > mss) 3594 len = mss; 3595 } 3596 3597 hsize = skb_headlen(head_skb) - offset; 3598 if (hsize < 0) 3599 hsize = 0; 3600 if (hsize > len || !sg) 3601 hsize = len; 3602 3603 if (!hsize && i >= nfrags && skb_headlen(list_skb) && 3604 (skb_headlen(list_skb) == len || sg)) { 3605 BUG_ON(skb_headlen(list_skb) > len); 3606 3607 i = 0; 3608 nfrags = skb_shinfo(list_skb)->nr_frags; 3609 frag = skb_shinfo(list_skb)->frags; 3610 frag_skb = list_skb; 3611 pos += skb_headlen(list_skb); 3612 3613 while (pos < offset + len) { 3614 BUG_ON(i >= nfrags); 3615 3616 size = skb_frag_size(frag); 3617 if (pos + size > offset + len) 3618 break; 3619 3620 i++; 3621 pos += size; 3622 frag++; 3623 } 3624 3625 nskb = skb_clone(list_skb, GFP_ATOMIC); 3626 list_skb = list_skb->next; 3627 3628 if (unlikely(!nskb)) 3629 goto err; 3630 3631 if (unlikely(pskb_trim(nskb, len))) { 3632 kfree_skb(nskb); 3633 goto err; 3634 } 3635 3636 hsize = skb_end_offset(nskb); 3637 if (skb_cow_head(nskb, doffset + headroom)) { 3638 kfree_skb(nskb); 3639 goto err; 3640 } 3641 3642 nskb->truesize += skb_end_offset(nskb) - hsize; 3643 skb_release_head_state(nskb); 3644 __skb_push(nskb, doffset); 3645 } else { 3646 nskb = __alloc_skb(hsize + doffset + headroom, 3647 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 3648 NUMA_NO_NODE); 3649 3650 if (unlikely(!nskb)) 3651 goto err; 3652 3653 skb_reserve(nskb, headroom); 3654 __skb_put(nskb, doffset); 3655 } 3656 3657 if (segs) 3658 tail->next = nskb; 3659 else 3660 segs = nskb; 3661 tail = nskb; 3662 3663 __copy_skb_header(nskb, head_skb); 3664 3665 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 3666 skb_reset_mac_len(nskb); 3667 3668 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 3669 nskb->data - tnl_hlen, 3670 doffset + tnl_hlen); 3671 3672 if (nskb->len == len + doffset) 3673 goto perform_csum_check; 3674 3675 if (!sg) { 3676 if (!nskb->remcsum_offload) 3677 nskb->ip_summed = CHECKSUM_NONE; 3678 SKB_GSO_CB(nskb)->csum = 3679 skb_copy_and_csum_bits(head_skb, offset, 3680 skb_put(nskb, len), 3681 len, 0); 3682 SKB_GSO_CB(nskb)->csum_start = 3683 skb_headroom(nskb) + doffset; 3684 continue; 3685 } 3686 3687 nskb_frag = skb_shinfo(nskb)->frags; 3688 3689 skb_copy_from_linear_data_offset(head_skb, offset, 3690 skb_put(nskb, hsize), hsize); 3691 3692 skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags & 3693 SKBTX_SHARED_FRAG; 3694 3695 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 3696 skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 3697 goto err; 3698 3699 while (pos < offset + len) { 3700 if (i >= nfrags) { 3701 i = 0; 3702 nfrags = skb_shinfo(list_skb)->nr_frags; 3703 frag = skb_shinfo(list_skb)->frags; 3704 frag_skb = list_skb; 3705 if (!skb_headlen(list_skb)) { 3706 BUG_ON(!nfrags); 3707 } else { 3708 BUG_ON(!list_skb->head_frag); 3709 3710 /* to make room for head_frag. */ 3711 i--; 3712 frag--; 3713 } 3714 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 3715 skb_zerocopy_clone(nskb, frag_skb, 3716 GFP_ATOMIC)) 3717 goto err; 3718 3719 list_skb = list_skb->next; 3720 } 3721 3722 if (unlikely(skb_shinfo(nskb)->nr_frags >= 3723 MAX_SKB_FRAGS)) { 3724 net_warn_ratelimited( 3725 "skb_segment: too many frags: %u %u\n", 3726 pos, mss); 3727 err = -EINVAL; 3728 goto err; 3729 } 3730 3731 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; 3732 __skb_frag_ref(nskb_frag); 3733 size = skb_frag_size(nskb_frag); 3734 3735 if (pos < offset) { 3736 nskb_frag->page_offset += offset - pos; 3737 skb_frag_size_sub(nskb_frag, offset - pos); 3738 } 3739 3740 skb_shinfo(nskb)->nr_frags++; 3741 3742 if (pos + size <= offset + len) { 3743 i++; 3744 frag++; 3745 pos += size; 3746 } else { 3747 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 3748 goto skip_fraglist; 3749 } 3750 3751 nskb_frag++; 3752 } 3753 3754 skip_fraglist: 3755 nskb->data_len = len - hsize; 3756 nskb->len += nskb->data_len; 3757 nskb->truesize += nskb->data_len; 3758 3759 perform_csum_check: 3760 if (!csum) { 3761 if (skb_has_shared_frag(nskb) && 3762 __skb_linearize(nskb)) 3763 goto err; 3764 3765 if (!nskb->remcsum_offload) 3766 nskb->ip_summed = CHECKSUM_NONE; 3767 SKB_GSO_CB(nskb)->csum = 3768 skb_checksum(nskb, doffset, 3769 nskb->len - doffset, 0); 3770 SKB_GSO_CB(nskb)->csum_start = 3771 skb_headroom(nskb) + doffset; 3772 } 3773 } while ((offset += len) < head_skb->len); 3774 3775 /* Some callers want to get the end of the list. 3776 * Put it in segs->prev to avoid walking the list. 3777 * (see validate_xmit_skb_list() for example) 3778 */ 3779 segs->prev = tail; 3780 3781 if (partial_segs) { 3782 struct sk_buff *iter; 3783 int type = skb_shinfo(head_skb)->gso_type; 3784 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 3785 3786 /* Update type to add partial and then remove dodgy if set */ 3787 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 3788 type &= ~SKB_GSO_DODGY; 3789 3790 /* Update GSO info and prepare to start updating headers on 3791 * our way back down the stack of protocols. 3792 */ 3793 for (iter = segs; iter; iter = iter->next) { 3794 skb_shinfo(iter)->gso_size = gso_size; 3795 skb_shinfo(iter)->gso_segs = partial_segs; 3796 skb_shinfo(iter)->gso_type = type; 3797 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 3798 } 3799 3800 if (tail->len - doffset <= gso_size) 3801 skb_shinfo(tail)->gso_size = 0; 3802 else if (tail != segs) 3803 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 3804 } 3805 3806 /* Following permits correct backpressure, for protocols 3807 * using skb_set_owner_w(). 3808 * Idea is to tranfert ownership from head_skb to last segment. 3809 */ 3810 if (head_skb->destructor == sock_wfree) { 3811 swap(tail->truesize, head_skb->truesize); 3812 swap(tail->destructor, head_skb->destructor); 3813 swap(tail->sk, head_skb->sk); 3814 } 3815 return segs; 3816 3817 err: 3818 kfree_skb_list(segs); 3819 return ERR_PTR(err); 3820 } 3821 EXPORT_SYMBOL_GPL(skb_segment); 3822 3823 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) 3824 { 3825 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); 3826 unsigned int offset = skb_gro_offset(skb); 3827 unsigned int headlen = skb_headlen(skb); 3828 unsigned int len = skb_gro_len(skb); 3829 unsigned int delta_truesize; 3830 struct sk_buff *lp; 3831 3832 if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush)) 3833 return -E2BIG; 3834 3835 lp = NAPI_GRO_CB(p)->last; 3836 pinfo = skb_shinfo(lp); 3837 3838 if (headlen <= offset) { 3839 skb_frag_t *frag; 3840 skb_frag_t *frag2; 3841 int i = skbinfo->nr_frags; 3842 int nr_frags = pinfo->nr_frags + i; 3843 3844 if (nr_frags > MAX_SKB_FRAGS) 3845 goto merge; 3846 3847 offset -= headlen; 3848 pinfo->nr_frags = nr_frags; 3849 skbinfo->nr_frags = 0; 3850 3851 frag = pinfo->frags + nr_frags; 3852 frag2 = skbinfo->frags + i; 3853 do { 3854 *--frag = *--frag2; 3855 } while (--i); 3856 3857 frag->page_offset += offset; 3858 skb_frag_size_sub(frag, offset); 3859 3860 /* all fragments truesize : remove (head size + sk_buff) */ 3861 delta_truesize = skb->truesize - 3862 SKB_TRUESIZE(skb_end_offset(skb)); 3863 3864 skb->truesize -= skb->data_len; 3865 skb->len -= skb->data_len; 3866 skb->data_len = 0; 3867 3868 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; 3869 goto done; 3870 } else if (skb->head_frag) { 3871 int nr_frags = pinfo->nr_frags; 3872 skb_frag_t *frag = pinfo->frags + nr_frags; 3873 struct page *page = virt_to_head_page(skb->head); 3874 unsigned int first_size = headlen - offset; 3875 unsigned int first_offset; 3876 3877 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) 3878 goto merge; 3879 3880 first_offset = skb->data - 3881 (unsigned char *)page_address(page) + 3882 offset; 3883 3884 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; 3885 3886 frag->page.p = page; 3887 frag->page_offset = first_offset; 3888 skb_frag_size_set(frag, first_size); 3889 3890 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); 3891 /* We dont need to clear skbinfo->nr_frags here */ 3892 3893 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 3894 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; 3895 goto done; 3896 } 3897 3898 merge: 3899 delta_truesize = skb->truesize; 3900 if (offset > headlen) { 3901 unsigned int eat = offset - headlen; 3902 3903 skbinfo->frags[0].page_offset += eat; 3904 skb_frag_size_sub(&skbinfo->frags[0], eat); 3905 skb->data_len -= eat; 3906 skb->len -= eat; 3907 offset = headlen; 3908 } 3909 3910 __skb_pull(skb, offset); 3911 3912 if (NAPI_GRO_CB(p)->last == p) 3913 skb_shinfo(p)->frag_list = skb; 3914 else 3915 NAPI_GRO_CB(p)->last->next = skb; 3916 NAPI_GRO_CB(p)->last = skb; 3917 __skb_header_release(skb); 3918 lp = p; 3919 3920 done: 3921 NAPI_GRO_CB(p)->count++; 3922 p->data_len += len; 3923 p->truesize += delta_truesize; 3924 p->len += len; 3925 if (lp != p) { 3926 lp->data_len += len; 3927 lp->truesize += delta_truesize; 3928 lp->len += len; 3929 } 3930 NAPI_GRO_CB(skb)->same_flow = 1; 3931 return 0; 3932 } 3933 EXPORT_SYMBOL_GPL(skb_gro_receive); 3934 3935 #ifdef CONFIG_SKB_EXTENSIONS 3936 #define SKB_EXT_ALIGN_VALUE 8 3937 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) 3938 3939 static const u8 skb_ext_type_len[] = { 3940 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 3941 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info), 3942 #endif 3943 #ifdef CONFIG_XFRM 3944 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path), 3945 #endif 3946 }; 3947 3948 static __always_inline unsigned int skb_ext_total_length(void) 3949 { 3950 return SKB_EXT_CHUNKSIZEOF(struct skb_ext) + 3951 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 3952 skb_ext_type_len[SKB_EXT_BRIDGE_NF] + 3953 #endif 3954 #ifdef CONFIG_XFRM 3955 skb_ext_type_len[SKB_EXT_SEC_PATH] + 3956 #endif 3957 0; 3958 } 3959 3960 static void skb_extensions_init(void) 3961 { 3962 BUILD_BUG_ON(SKB_EXT_NUM >= 8); 3963 BUILD_BUG_ON(skb_ext_total_length() > 255); 3964 3965 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", 3966 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), 3967 0, 3968 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3969 NULL); 3970 } 3971 #else 3972 static void skb_extensions_init(void) {} 3973 #endif 3974 3975 void __init skb_init(void) 3976 { 3977 skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache", 3978 sizeof(struct sk_buff), 3979 0, 3980 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3981 offsetof(struct sk_buff, cb), 3982 sizeof_field(struct sk_buff, cb), 3983 NULL); 3984 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 3985 sizeof(struct sk_buff_fclones), 3986 0, 3987 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3988 NULL); 3989 skb_extensions_init(); 3990 } 3991 3992 static int 3993 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 3994 unsigned int recursion_level) 3995 { 3996 int start = skb_headlen(skb); 3997 int i, copy = start - offset; 3998 struct sk_buff *frag_iter; 3999 int elt = 0; 4000 4001 if (unlikely(recursion_level >= 24)) 4002 return -EMSGSIZE; 4003 4004 if (copy > 0) { 4005 if (copy > len) 4006 copy = len; 4007 sg_set_buf(sg, skb->data + offset, copy); 4008 elt++; 4009 if ((len -= copy) == 0) 4010 return elt; 4011 offset += copy; 4012 } 4013 4014 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4015 int end; 4016 4017 WARN_ON(start > offset + len); 4018 4019 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 4020 if ((copy = end - offset) > 0) { 4021 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4022 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4023 return -EMSGSIZE; 4024 4025 if (copy > len) 4026 copy = len; 4027 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 4028 frag->page_offset+offset-start); 4029 elt++; 4030 if (!(len -= copy)) 4031 return elt; 4032 offset += copy; 4033 } 4034 start = end; 4035 } 4036 4037 skb_walk_frags(skb, frag_iter) { 4038 int end, ret; 4039 4040 WARN_ON(start > offset + len); 4041 4042 end = start + frag_iter->len; 4043 if ((copy = end - offset) > 0) { 4044 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4045 return -EMSGSIZE; 4046 4047 if (copy > len) 4048 copy = len; 4049 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 4050 copy, recursion_level + 1); 4051 if (unlikely(ret < 0)) 4052 return ret; 4053 elt += ret; 4054 if ((len -= copy) == 0) 4055 return elt; 4056 offset += copy; 4057 } 4058 start = end; 4059 } 4060 BUG_ON(len); 4061 return elt; 4062 } 4063 4064 /** 4065 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 4066 * @skb: Socket buffer containing the buffers to be mapped 4067 * @sg: The scatter-gather list to map into 4068 * @offset: The offset into the buffer's contents to start mapping 4069 * @len: Length of buffer space to be mapped 4070 * 4071 * Fill the specified scatter-gather list with mappings/pointers into a 4072 * region of the buffer space attached to a socket buffer. Returns either 4073 * the number of scatterlist items used, or -EMSGSIZE if the contents 4074 * could not fit. 4075 */ 4076 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 4077 { 4078 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 4079 4080 if (nsg <= 0) 4081 return nsg; 4082 4083 sg_mark_end(&sg[nsg - 1]); 4084 4085 return nsg; 4086 } 4087 EXPORT_SYMBOL_GPL(skb_to_sgvec); 4088 4089 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 4090 * sglist without mark the sg which contain last skb data as the end. 4091 * So the caller can mannipulate sg list as will when padding new data after 4092 * the first call without calling sg_unmark_end to expend sg list. 4093 * 4094 * Scenario to use skb_to_sgvec_nomark: 4095 * 1. sg_init_table 4096 * 2. skb_to_sgvec_nomark(payload1) 4097 * 3. skb_to_sgvec_nomark(payload2) 4098 * 4099 * This is equivalent to: 4100 * 1. sg_init_table 4101 * 2. skb_to_sgvec(payload1) 4102 * 3. sg_unmark_end 4103 * 4. skb_to_sgvec(payload2) 4104 * 4105 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 4106 * is more preferable. 4107 */ 4108 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 4109 int offset, int len) 4110 { 4111 return __skb_to_sgvec(skb, sg, offset, len, 0); 4112 } 4113 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 4114 4115 4116 4117 /** 4118 * skb_cow_data - Check that a socket buffer's data buffers are writable 4119 * @skb: The socket buffer to check. 4120 * @tailbits: Amount of trailing space to be added 4121 * @trailer: Returned pointer to the skb where the @tailbits space begins 4122 * 4123 * Make sure that the data buffers attached to a socket buffer are 4124 * writable. If they are not, private copies are made of the data buffers 4125 * and the socket buffer is set to use these instead. 4126 * 4127 * If @tailbits is given, make sure that there is space to write @tailbits 4128 * bytes of data beyond current end of socket buffer. @trailer will be 4129 * set to point to the skb in which this space begins. 4130 * 4131 * The number of scatterlist elements required to completely map the 4132 * COW'd and extended socket buffer will be returned. 4133 */ 4134 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 4135 { 4136 int copyflag; 4137 int elt; 4138 struct sk_buff *skb1, **skb_p; 4139 4140 /* If skb is cloned or its head is paged, reallocate 4141 * head pulling out all the pages (pages are considered not writable 4142 * at the moment even if they are anonymous). 4143 */ 4144 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 4145 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL) 4146 return -ENOMEM; 4147 4148 /* Easy case. Most of packets will go this way. */ 4149 if (!skb_has_frag_list(skb)) { 4150 /* A little of trouble, not enough of space for trailer. 4151 * This should not happen, when stack is tuned to generate 4152 * good frames. OK, on miss we reallocate and reserve even more 4153 * space, 128 bytes is fair. */ 4154 4155 if (skb_tailroom(skb) < tailbits && 4156 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 4157 return -ENOMEM; 4158 4159 /* Voila! */ 4160 *trailer = skb; 4161 return 1; 4162 } 4163 4164 /* Misery. We are in troubles, going to mincer fragments... */ 4165 4166 elt = 1; 4167 skb_p = &skb_shinfo(skb)->frag_list; 4168 copyflag = 0; 4169 4170 while ((skb1 = *skb_p) != NULL) { 4171 int ntail = 0; 4172 4173 /* The fragment is partially pulled by someone, 4174 * this can happen on input. Copy it and everything 4175 * after it. */ 4176 4177 if (skb_shared(skb1)) 4178 copyflag = 1; 4179 4180 /* If the skb is the last, worry about trailer. */ 4181 4182 if (skb1->next == NULL && tailbits) { 4183 if (skb_shinfo(skb1)->nr_frags || 4184 skb_has_frag_list(skb1) || 4185 skb_tailroom(skb1) < tailbits) 4186 ntail = tailbits + 128; 4187 } 4188 4189 if (copyflag || 4190 skb_cloned(skb1) || 4191 ntail || 4192 skb_shinfo(skb1)->nr_frags || 4193 skb_has_frag_list(skb1)) { 4194 struct sk_buff *skb2; 4195 4196 /* Fuck, we are miserable poor guys... */ 4197 if (ntail == 0) 4198 skb2 = skb_copy(skb1, GFP_ATOMIC); 4199 else 4200 skb2 = skb_copy_expand(skb1, 4201 skb_headroom(skb1), 4202 ntail, 4203 GFP_ATOMIC); 4204 if (unlikely(skb2 == NULL)) 4205 return -ENOMEM; 4206 4207 if (skb1->sk) 4208 skb_set_owner_w(skb2, skb1->sk); 4209 4210 /* Looking around. Are we still alive? 4211 * OK, link new skb, drop old one */ 4212 4213 skb2->next = skb1->next; 4214 *skb_p = skb2; 4215 kfree_skb(skb1); 4216 skb1 = skb2; 4217 } 4218 elt++; 4219 *trailer = skb1; 4220 skb_p = &skb1->next; 4221 } 4222 4223 return elt; 4224 } 4225 EXPORT_SYMBOL_GPL(skb_cow_data); 4226 4227 static void sock_rmem_free(struct sk_buff *skb) 4228 { 4229 struct sock *sk = skb->sk; 4230 4231 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 4232 } 4233 4234 static void skb_set_err_queue(struct sk_buff *skb) 4235 { 4236 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING. 4237 * So, it is safe to (mis)use it to mark skbs on the error queue. 4238 */ 4239 skb->pkt_type = PACKET_OUTGOING; 4240 BUILD_BUG_ON(PACKET_OUTGOING == 0); 4241 } 4242 4243 /* 4244 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 4245 */ 4246 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 4247 { 4248 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 4249 (unsigned int)sk->sk_rcvbuf) 4250 return -ENOMEM; 4251 4252 skb_orphan(skb); 4253 skb->sk = sk; 4254 skb->destructor = sock_rmem_free; 4255 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 4256 skb_set_err_queue(skb); 4257 4258 /* before exiting rcu section, make sure dst is refcounted */ 4259 skb_dst_force(skb); 4260 4261 skb_queue_tail(&sk->sk_error_queue, skb); 4262 if (!sock_flag(sk, SOCK_DEAD)) 4263 sk->sk_error_report(sk); 4264 return 0; 4265 } 4266 EXPORT_SYMBOL(sock_queue_err_skb); 4267 4268 static bool is_icmp_err_skb(const struct sk_buff *skb) 4269 { 4270 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 4271 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 4272 } 4273 4274 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 4275 { 4276 struct sk_buff_head *q = &sk->sk_error_queue; 4277 struct sk_buff *skb, *skb_next = NULL; 4278 bool icmp_next = false; 4279 unsigned long flags; 4280 4281 spin_lock_irqsave(&q->lock, flags); 4282 skb = __skb_dequeue(q); 4283 if (skb && (skb_next = skb_peek(q))) { 4284 icmp_next = is_icmp_err_skb(skb_next); 4285 if (icmp_next) 4286 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_origin; 4287 } 4288 spin_unlock_irqrestore(&q->lock, flags); 4289 4290 if (is_icmp_err_skb(skb) && !icmp_next) 4291 sk->sk_err = 0; 4292 4293 if (skb_next) 4294 sk->sk_error_report(sk); 4295 4296 return skb; 4297 } 4298 EXPORT_SYMBOL(sock_dequeue_err_skb); 4299 4300 /** 4301 * skb_clone_sk - create clone of skb, and take reference to socket 4302 * @skb: the skb to clone 4303 * 4304 * This function creates a clone of a buffer that holds a reference on 4305 * sk_refcnt. Buffers created via this function are meant to be 4306 * returned using sock_queue_err_skb, or free via kfree_skb. 4307 * 4308 * When passing buffers allocated with this function to sock_queue_err_skb 4309 * it is necessary to wrap the call with sock_hold/sock_put in order to 4310 * prevent the socket from being released prior to being enqueued on 4311 * the sk_error_queue. 4312 */ 4313 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 4314 { 4315 struct sock *sk = skb->sk; 4316 struct sk_buff *clone; 4317 4318 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 4319 return NULL; 4320 4321 clone = skb_clone(skb, GFP_ATOMIC); 4322 if (!clone) { 4323 sock_put(sk); 4324 return NULL; 4325 } 4326 4327 clone->sk = sk; 4328 clone->destructor = sock_efree; 4329 4330 return clone; 4331 } 4332 EXPORT_SYMBOL(skb_clone_sk); 4333 4334 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 4335 struct sock *sk, 4336 int tstype, 4337 bool opt_stats) 4338 { 4339 struct sock_exterr_skb *serr; 4340 int err; 4341 4342 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 4343 4344 serr = SKB_EXT_ERR(skb); 4345 memset(serr, 0, sizeof(*serr)); 4346 serr->ee.ee_errno = ENOMSG; 4347 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 4348 serr->ee.ee_info = tstype; 4349 serr->opt_stats = opt_stats; 4350 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 4351 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { 4352 serr->ee.ee_data = skb_shinfo(skb)->tskey; 4353 if (sk->sk_protocol == IPPROTO_TCP && 4354 sk->sk_type == SOCK_STREAM) 4355 serr->ee.ee_data -= sk->sk_tskey; 4356 } 4357 4358 err = sock_queue_err_skb(sk, skb); 4359 4360 if (err) 4361 kfree_skb(skb); 4362 } 4363 4364 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 4365 { 4366 bool ret; 4367 4368 if (likely(sysctl_tstamp_allow_data || tsonly)) 4369 return true; 4370 4371 read_lock_bh(&sk->sk_callback_lock); 4372 ret = sk->sk_socket && sk->sk_socket->file && 4373 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); 4374 read_unlock_bh(&sk->sk_callback_lock); 4375 return ret; 4376 } 4377 4378 void skb_complete_tx_timestamp(struct sk_buff *skb, 4379 struct skb_shared_hwtstamps *hwtstamps) 4380 { 4381 struct sock *sk = skb->sk; 4382 4383 if (!skb_may_tx_timestamp(sk, false)) 4384 goto err; 4385 4386 /* Take a reference to prevent skb_orphan() from freeing the socket, 4387 * but only if the socket refcount is not zero. 4388 */ 4389 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 4390 *skb_hwtstamps(skb) = *hwtstamps; 4391 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 4392 sock_put(sk); 4393 return; 4394 } 4395 4396 err: 4397 kfree_skb(skb); 4398 } 4399 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 4400 4401 void __skb_tstamp_tx(struct sk_buff *orig_skb, 4402 struct skb_shared_hwtstamps *hwtstamps, 4403 struct sock *sk, int tstype) 4404 { 4405 struct sk_buff *skb; 4406 bool tsonly, opt_stats = false; 4407 4408 if (!sk) 4409 return; 4410 4411 if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 4412 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 4413 return; 4414 4415 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 4416 if (!skb_may_tx_timestamp(sk, tsonly)) 4417 return; 4418 4419 if (tsonly) { 4420 #ifdef CONFIG_INET 4421 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && 4422 sk->sk_protocol == IPPROTO_TCP && 4423 sk->sk_type == SOCK_STREAM) { 4424 skb = tcp_get_timestamping_opt_stats(sk); 4425 opt_stats = true; 4426 } else 4427 #endif 4428 skb = alloc_skb(0, GFP_ATOMIC); 4429 } else { 4430 skb = skb_clone(orig_skb, GFP_ATOMIC); 4431 } 4432 if (!skb) 4433 return; 4434 4435 if (tsonly) { 4436 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 4437 SKBTX_ANY_TSTAMP; 4438 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 4439 } 4440 4441 if (hwtstamps) 4442 *skb_hwtstamps(skb) = *hwtstamps; 4443 else 4444 skb->tstamp = ktime_get_real(); 4445 4446 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 4447 } 4448 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 4449 4450 void skb_tstamp_tx(struct sk_buff *orig_skb, 4451 struct skb_shared_hwtstamps *hwtstamps) 4452 { 4453 return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk, 4454 SCM_TSTAMP_SND); 4455 } 4456 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 4457 4458 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 4459 { 4460 struct sock *sk = skb->sk; 4461 struct sock_exterr_skb *serr; 4462 int err = 1; 4463 4464 skb->wifi_acked_valid = 1; 4465 skb->wifi_acked = acked; 4466 4467 serr = SKB_EXT_ERR(skb); 4468 memset(serr, 0, sizeof(*serr)); 4469 serr->ee.ee_errno = ENOMSG; 4470 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 4471 4472 /* Take a reference to prevent skb_orphan() from freeing the socket, 4473 * but only if the socket refcount is not zero. 4474 */ 4475 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 4476 err = sock_queue_err_skb(sk, skb); 4477 sock_put(sk); 4478 } 4479 if (err) 4480 kfree_skb(skb); 4481 } 4482 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 4483 4484 /** 4485 * skb_partial_csum_set - set up and verify partial csum values for packet 4486 * @skb: the skb to set 4487 * @start: the number of bytes after skb->data to start checksumming. 4488 * @off: the offset from start to place the checksum. 4489 * 4490 * For untrusted partially-checksummed packets, we need to make sure the values 4491 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 4492 * 4493 * This function checks and sets those values and skb->ip_summed: if this 4494 * returns false you should drop the packet. 4495 */ 4496 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 4497 { 4498 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16); 4499 u32 csum_start = skb_headroom(skb) + (u32)start; 4500 4501 if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) { 4502 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n", 4503 start, off, skb_headroom(skb), skb_headlen(skb)); 4504 return false; 4505 } 4506 skb->ip_summed = CHECKSUM_PARTIAL; 4507 skb->csum_start = csum_start; 4508 skb->csum_offset = off; 4509 skb_set_transport_header(skb, start); 4510 return true; 4511 } 4512 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 4513 4514 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 4515 unsigned int max) 4516 { 4517 if (skb_headlen(skb) >= len) 4518 return 0; 4519 4520 /* If we need to pullup then pullup to the max, so we 4521 * won't need to do it again. 4522 */ 4523 if (max > skb->len) 4524 max = skb->len; 4525 4526 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 4527 return -ENOMEM; 4528 4529 if (skb_headlen(skb) < len) 4530 return -EPROTO; 4531 4532 return 0; 4533 } 4534 4535 #define MAX_TCP_HDR_LEN (15 * 4) 4536 4537 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 4538 typeof(IPPROTO_IP) proto, 4539 unsigned int off) 4540 { 4541 switch (proto) { 4542 int err; 4543 4544 case IPPROTO_TCP: 4545 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 4546 off + MAX_TCP_HDR_LEN); 4547 if (!err && !skb_partial_csum_set(skb, off, 4548 offsetof(struct tcphdr, 4549 check))) 4550 err = -EPROTO; 4551 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 4552 4553 case IPPROTO_UDP: 4554 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 4555 off + sizeof(struct udphdr)); 4556 if (!err && !skb_partial_csum_set(skb, off, 4557 offsetof(struct udphdr, 4558 check))) 4559 err = -EPROTO; 4560 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 4561 } 4562 4563 return ERR_PTR(-EPROTO); 4564 } 4565 4566 /* This value should be large enough to cover a tagged ethernet header plus 4567 * maximally sized IP and TCP or UDP headers. 4568 */ 4569 #define MAX_IP_HDR_LEN 128 4570 4571 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 4572 { 4573 unsigned int off; 4574 bool fragment; 4575 __sum16 *csum; 4576 int err; 4577 4578 fragment = false; 4579 4580 err = skb_maybe_pull_tail(skb, 4581 sizeof(struct iphdr), 4582 MAX_IP_HDR_LEN); 4583 if (err < 0) 4584 goto out; 4585 4586 if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF)) 4587 fragment = true; 4588 4589 off = ip_hdrlen(skb); 4590 4591 err = -EPROTO; 4592 4593 if (fragment) 4594 goto out; 4595 4596 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 4597 if (IS_ERR(csum)) 4598 return PTR_ERR(csum); 4599 4600 if (recalculate) 4601 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 4602 ip_hdr(skb)->daddr, 4603 skb->len - off, 4604 ip_hdr(skb)->protocol, 0); 4605 err = 0; 4606 4607 out: 4608 return err; 4609 } 4610 4611 /* This value should be large enough to cover a tagged ethernet header plus 4612 * an IPv6 header, all options, and a maximal TCP or UDP header. 4613 */ 4614 #define MAX_IPV6_HDR_LEN 256 4615 4616 #define OPT_HDR(type, skb, off) \ 4617 (type *)(skb_network_header(skb) + (off)) 4618 4619 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 4620 { 4621 int err; 4622 u8 nexthdr; 4623 unsigned int off; 4624 unsigned int len; 4625 bool fragment; 4626 bool done; 4627 __sum16 *csum; 4628 4629 fragment = false; 4630 done = false; 4631 4632 off = sizeof(struct ipv6hdr); 4633 4634 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 4635 if (err < 0) 4636 goto out; 4637 4638 nexthdr = ipv6_hdr(skb)->nexthdr; 4639 4640 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 4641 while (off <= len && !done) { 4642 switch (nexthdr) { 4643 case IPPROTO_DSTOPTS: 4644 case IPPROTO_HOPOPTS: 4645 case IPPROTO_ROUTING: { 4646 struct ipv6_opt_hdr *hp; 4647 4648 err = skb_maybe_pull_tail(skb, 4649 off + 4650 sizeof(struct ipv6_opt_hdr), 4651 MAX_IPV6_HDR_LEN); 4652 if (err < 0) 4653 goto out; 4654 4655 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 4656 nexthdr = hp->nexthdr; 4657 off += ipv6_optlen(hp); 4658 break; 4659 } 4660 case IPPROTO_AH: { 4661 struct ip_auth_hdr *hp; 4662 4663 err = skb_maybe_pull_tail(skb, 4664 off + 4665 sizeof(struct ip_auth_hdr), 4666 MAX_IPV6_HDR_LEN); 4667 if (err < 0) 4668 goto out; 4669 4670 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 4671 nexthdr = hp->nexthdr; 4672 off += ipv6_authlen(hp); 4673 break; 4674 } 4675 case IPPROTO_FRAGMENT: { 4676 struct frag_hdr *hp; 4677 4678 err = skb_maybe_pull_tail(skb, 4679 off + 4680 sizeof(struct frag_hdr), 4681 MAX_IPV6_HDR_LEN); 4682 if (err < 0) 4683 goto out; 4684 4685 hp = OPT_HDR(struct frag_hdr, skb, off); 4686 4687 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 4688 fragment = true; 4689 4690 nexthdr = hp->nexthdr; 4691 off += sizeof(struct frag_hdr); 4692 break; 4693 } 4694 default: 4695 done = true; 4696 break; 4697 } 4698 } 4699 4700 err = -EPROTO; 4701 4702 if (!done || fragment) 4703 goto out; 4704 4705 csum = skb_checksum_setup_ip(skb, nexthdr, off); 4706 if (IS_ERR(csum)) 4707 return PTR_ERR(csum); 4708 4709 if (recalculate) 4710 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 4711 &ipv6_hdr(skb)->daddr, 4712 skb->len - off, nexthdr, 0); 4713 err = 0; 4714 4715 out: 4716 return err; 4717 } 4718 4719 /** 4720 * skb_checksum_setup - set up partial checksum offset 4721 * @skb: the skb to set up 4722 * @recalculate: if true the pseudo-header checksum will be recalculated 4723 */ 4724 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 4725 { 4726 int err; 4727 4728 switch (skb->protocol) { 4729 case htons(ETH_P_IP): 4730 err = skb_checksum_setup_ipv4(skb, recalculate); 4731 break; 4732 4733 case htons(ETH_P_IPV6): 4734 err = skb_checksum_setup_ipv6(skb, recalculate); 4735 break; 4736 4737 default: 4738 err = -EPROTO; 4739 break; 4740 } 4741 4742 return err; 4743 } 4744 EXPORT_SYMBOL(skb_checksum_setup); 4745 4746 /** 4747 * skb_checksum_maybe_trim - maybe trims the given skb 4748 * @skb: the skb to check 4749 * @transport_len: the data length beyond the network header 4750 * 4751 * Checks whether the given skb has data beyond the given transport length. 4752 * If so, returns a cloned skb trimmed to this transport length. 4753 * Otherwise returns the provided skb. Returns NULL in error cases 4754 * (e.g. transport_len exceeds skb length or out-of-memory). 4755 * 4756 * Caller needs to set the skb transport header and free any returned skb if it 4757 * differs from the provided skb. 4758 */ 4759 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 4760 unsigned int transport_len) 4761 { 4762 struct sk_buff *skb_chk; 4763 unsigned int len = skb_transport_offset(skb) + transport_len; 4764 int ret; 4765 4766 if (skb->len < len) 4767 return NULL; 4768 else if (skb->len == len) 4769 return skb; 4770 4771 skb_chk = skb_clone(skb, GFP_ATOMIC); 4772 if (!skb_chk) 4773 return NULL; 4774 4775 ret = pskb_trim_rcsum(skb_chk, len); 4776 if (ret) { 4777 kfree_skb(skb_chk); 4778 return NULL; 4779 } 4780 4781 return skb_chk; 4782 } 4783 4784 /** 4785 * skb_checksum_trimmed - validate checksum of an skb 4786 * @skb: the skb to check 4787 * @transport_len: the data length beyond the network header 4788 * @skb_chkf: checksum function to use 4789 * 4790 * Applies the given checksum function skb_chkf to the provided skb. 4791 * Returns a checked and maybe trimmed skb. Returns NULL on error. 4792 * 4793 * If the skb has data beyond the given transport length, then a 4794 * trimmed & cloned skb is checked and returned. 4795 * 4796 * Caller needs to set the skb transport header and free any returned skb if it 4797 * differs from the provided skb. 4798 */ 4799 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 4800 unsigned int transport_len, 4801 __sum16(*skb_chkf)(struct sk_buff *skb)) 4802 { 4803 struct sk_buff *skb_chk; 4804 unsigned int offset = skb_transport_offset(skb); 4805 __sum16 ret; 4806 4807 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 4808 if (!skb_chk) 4809 goto err; 4810 4811 if (!pskb_may_pull(skb_chk, offset)) 4812 goto err; 4813 4814 skb_pull_rcsum(skb_chk, offset); 4815 ret = skb_chkf(skb_chk); 4816 skb_push_rcsum(skb_chk, offset); 4817 4818 if (ret) 4819 goto err; 4820 4821 return skb_chk; 4822 4823 err: 4824 if (skb_chk && skb_chk != skb) 4825 kfree_skb(skb_chk); 4826 4827 return NULL; 4828 4829 } 4830 EXPORT_SYMBOL(skb_checksum_trimmed); 4831 4832 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 4833 { 4834 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 4835 skb->dev->name); 4836 } 4837 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 4838 4839 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 4840 { 4841 if (head_stolen) { 4842 skb_release_head_state(skb); 4843 kmem_cache_free(skbuff_head_cache, skb); 4844 } else { 4845 __kfree_skb(skb); 4846 } 4847 } 4848 EXPORT_SYMBOL(kfree_skb_partial); 4849 4850 /** 4851 * skb_try_coalesce - try to merge skb to prior one 4852 * @to: prior buffer 4853 * @from: buffer to add 4854 * @fragstolen: pointer to boolean 4855 * @delta_truesize: how much more was allocated than was requested 4856 */ 4857 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 4858 bool *fragstolen, int *delta_truesize) 4859 { 4860 struct skb_shared_info *to_shinfo, *from_shinfo; 4861 int i, delta, len = from->len; 4862 4863 *fragstolen = false; 4864 4865 if (skb_cloned(to)) 4866 return false; 4867 4868 if (len <= skb_tailroom(to)) { 4869 if (len) 4870 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 4871 *delta_truesize = 0; 4872 return true; 4873 } 4874 4875 to_shinfo = skb_shinfo(to); 4876 from_shinfo = skb_shinfo(from); 4877 if (to_shinfo->frag_list || from_shinfo->frag_list) 4878 return false; 4879 if (skb_zcopy(to) || skb_zcopy(from)) 4880 return false; 4881 4882 if (skb_headlen(from) != 0) { 4883 struct page *page; 4884 unsigned int offset; 4885 4886 if (to_shinfo->nr_frags + 4887 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 4888 return false; 4889 4890 if (skb_head_is_locked(from)) 4891 return false; 4892 4893 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 4894 4895 page = virt_to_head_page(from->head); 4896 offset = from->data - (unsigned char *)page_address(page); 4897 4898 skb_fill_page_desc(to, to_shinfo->nr_frags, 4899 page, offset, skb_headlen(from)); 4900 *fragstolen = true; 4901 } else { 4902 if (to_shinfo->nr_frags + 4903 from_shinfo->nr_frags > MAX_SKB_FRAGS) 4904 return false; 4905 4906 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 4907 } 4908 4909 WARN_ON_ONCE(delta < len); 4910 4911 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 4912 from_shinfo->frags, 4913 from_shinfo->nr_frags * sizeof(skb_frag_t)); 4914 to_shinfo->nr_frags += from_shinfo->nr_frags; 4915 4916 if (!skb_cloned(from)) 4917 from_shinfo->nr_frags = 0; 4918 4919 /* if the skb is not cloned this does nothing 4920 * since we set nr_frags to 0. 4921 */ 4922 for (i = 0; i < from_shinfo->nr_frags; i++) 4923 __skb_frag_ref(&from_shinfo->frags[i]); 4924 4925 to->truesize += delta; 4926 to->len += len; 4927 to->data_len += len; 4928 4929 *delta_truesize = delta; 4930 return true; 4931 } 4932 EXPORT_SYMBOL(skb_try_coalesce); 4933 4934 /** 4935 * skb_scrub_packet - scrub an skb 4936 * 4937 * @skb: buffer to clean 4938 * @xnet: packet is crossing netns 4939 * 4940 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 4941 * into/from a tunnel. Some information have to be cleared during these 4942 * operations. 4943 * skb_scrub_packet can also be used to clean a skb before injecting it in 4944 * another namespace (@xnet == true). We have to clear all information in the 4945 * skb that could impact namespace isolation. 4946 */ 4947 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 4948 { 4949 skb->pkt_type = PACKET_HOST; 4950 skb->skb_iif = 0; 4951 skb->ignore_df = 0; 4952 skb_dst_drop(skb); 4953 secpath_reset(skb); 4954 nf_reset(skb); 4955 nf_reset_trace(skb); 4956 4957 #ifdef CONFIG_NET_SWITCHDEV 4958 skb->offload_fwd_mark = 0; 4959 skb->offload_l3_fwd_mark = 0; 4960 #endif 4961 4962 if (!xnet) 4963 return; 4964 4965 ipvs_reset(skb); 4966 skb->mark = 0; 4967 skb->tstamp = 0; 4968 } 4969 EXPORT_SYMBOL_GPL(skb_scrub_packet); 4970 4971 /** 4972 * skb_gso_transport_seglen - Return length of individual segments of a gso packet 4973 * 4974 * @skb: GSO skb 4975 * 4976 * skb_gso_transport_seglen is used to determine the real size of the 4977 * individual segments, including Layer4 headers (TCP/UDP). 4978 * 4979 * The MAC/L2 or network (IP, IPv6) headers are not accounted for. 4980 */ 4981 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb) 4982 { 4983 const struct skb_shared_info *shinfo = skb_shinfo(skb); 4984 unsigned int thlen = 0; 4985 4986 if (skb->encapsulation) { 4987 thlen = skb_inner_transport_header(skb) - 4988 skb_transport_header(skb); 4989 4990 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 4991 thlen += inner_tcp_hdrlen(skb); 4992 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 4993 thlen = tcp_hdrlen(skb); 4994 } else if (unlikely(skb_is_gso_sctp(skb))) { 4995 thlen = sizeof(struct sctphdr); 4996 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { 4997 thlen = sizeof(struct udphdr); 4998 } 4999 /* UFO sets gso_size to the size of the fragmentation 5000 * payload, i.e. the size of the L4 (UDP) header is already 5001 * accounted for. 5002 */ 5003 return thlen + shinfo->gso_size; 5004 } 5005 5006 /** 5007 * skb_gso_network_seglen - Return length of individual segments of a gso packet 5008 * 5009 * @skb: GSO skb 5010 * 5011 * skb_gso_network_seglen is used to determine the real size of the 5012 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP). 5013 * 5014 * The MAC/L2 header is not accounted for. 5015 */ 5016 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb) 5017 { 5018 unsigned int hdr_len = skb_transport_header(skb) - 5019 skb_network_header(skb); 5020 5021 return hdr_len + skb_gso_transport_seglen(skb); 5022 } 5023 5024 /** 5025 * skb_gso_mac_seglen - Return length of individual segments of a gso packet 5026 * 5027 * @skb: GSO skb 5028 * 5029 * skb_gso_mac_seglen is used to determine the real size of the 5030 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4 5031 * headers (TCP/UDP). 5032 */ 5033 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb) 5034 { 5035 unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 5036 5037 return hdr_len + skb_gso_transport_seglen(skb); 5038 } 5039 5040 /** 5041 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS 5042 * 5043 * There are a couple of instances where we have a GSO skb, and we 5044 * want to determine what size it would be after it is segmented. 5045 * 5046 * We might want to check: 5047 * - L3+L4+payload size (e.g. IP forwarding) 5048 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver) 5049 * 5050 * This is a helper to do that correctly considering GSO_BY_FRAGS. 5051 * 5052 * @skb: GSO skb 5053 * 5054 * @seg_len: The segmented length (from skb_gso_*_seglen). In the 5055 * GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS]. 5056 * 5057 * @max_len: The maximum permissible length. 5058 * 5059 * Returns true if the segmented length <= max length. 5060 */ 5061 static inline bool skb_gso_size_check(const struct sk_buff *skb, 5062 unsigned int seg_len, 5063 unsigned int max_len) { 5064 const struct skb_shared_info *shinfo = skb_shinfo(skb); 5065 const struct sk_buff *iter; 5066 5067 if (shinfo->gso_size != GSO_BY_FRAGS) 5068 return seg_len <= max_len; 5069 5070 /* Undo this so we can re-use header sizes */ 5071 seg_len -= GSO_BY_FRAGS; 5072 5073 skb_walk_frags(skb, iter) { 5074 if (seg_len + skb_headlen(iter) > max_len) 5075 return false; 5076 } 5077 5078 return true; 5079 } 5080 5081 /** 5082 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU? 5083 * 5084 * @skb: GSO skb 5085 * @mtu: MTU to validate against 5086 * 5087 * skb_gso_validate_network_len validates if a given skb will fit a 5088 * wanted MTU once split. It considers L3 headers, L4 headers, and the 5089 * payload. 5090 */ 5091 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu) 5092 { 5093 return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu); 5094 } 5095 EXPORT_SYMBOL_GPL(skb_gso_validate_network_len); 5096 5097 /** 5098 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length? 5099 * 5100 * @skb: GSO skb 5101 * @len: length to validate against 5102 * 5103 * skb_gso_validate_mac_len validates if a given skb will fit a wanted 5104 * length once split, including L2, L3 and L4 headers and the payload. 5105 */ 5106 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len) 5107 { 5108 return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len); 5109 } 5110 EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len); 5111 5112 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 5113 { 5114 int mac_len, meta_len; 5115 void *meta; 5116 5117 if (skb_cow(skb, skb_headroom(skb)) < 0) { 5118 kfree_skb(skb); 5119 return NULL; 5120 } 5121 5122 mac_len = skb->data - skb_mac_header(skb); 5123 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) { 5124 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), 5125 mac_len - VLAN_HLEN - ETH_TLEN); 5126 } 5127 5128 meta_len = skb_metadata_len(skb); 5129 if (meta_len) { 5130 meta = skb_metadata_end(skb) - meta_len; 5131 memmove(meta + VLAN_HLEN, meta, meta_len); 5132 } 5133 5134 skb->mac_header += VLAN_HLEN; 5135 return skb; 5136 } 5137 5138 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 5139 { 5140 struct vlan_hdr *vhdr; 5141 u16 vlan_tci; 5142 5143 if (unlikely(skb_vlan_tag_present(skb))) { 5144 /* vlan_tci is already set-up so leave this for another time */ 5145 return skb; 5146 } 5147 5148 skb = skb_share_check(skb, GFP_ATOMIC); 5149 if (unlikely(!skb)) 5150 goto err_free; 5151 5152 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN))) 5153 goto err_free; 5154 5155 vhdr = (struct vlan_hdr *)skb->data; 5156 vlan_tci = ntohs(vhdr->h_vlan_TCI); 5157 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 5158 5159 skb_pull_rcsum(skb, VLAN_HLEN); 5160 vlan_set_encap_proto(skb, vhdr); 5161 5162 skb = skb_reorder_vlan_header(skb); 5163 if (unlikely(!skb)) 5164 goto err_free; 5165 5166 skb_reset_network_header(skb); 5167 skb_reset_transport_header(skb); 5168 skb_reset_mac_len(skb); 5169 5170 return skb; 5171 5172 err_free: 5173 kfree_skb(skb); 5174 return NULL; 5175 } 5176 EXPORT_SYMBOL(skb_vlan_untag); 5177 5178 int skb_ensure_writable(struct sk_buff *skb, int write_len) 5179 { 5180 if (!pskb_may_pull(skb, write_len)) 5181 return -ENOMEM; 5182 5183 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 5184 return 0; 5185 5186 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 5187 } 5188 EXPORT_SYMBOL(skb_ensure_writable); 5189 5190 /* remove VLAN header from packet and update csum accordingly. 5191 * expects a non skb_vlan_tag_present skb with a vlan tag payload 5192 */ 5193 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 5194 { 5195 struct vlan_hdr *vhdr; 5196 int offset = skb->data - skb_mac_header(skb); 5197 int err; 5198 5199 if (WARN_ONCE(offset, 5200 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 5201 offset)) { 5202 return -EINVAL; 5203 } 5204 5205 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 5206 if (unlikely(err)) 5207 return err; 5208 5209 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5210 5211 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN); 5212 *vlan_tci = ntohs(vhdr->h_vlan_TCI); 5213 5214 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN); 5215 __skb_pull(skb, VLAN_HLEN); 5216 5217 vlan_set_encap_proto(skb, vhdr); 5218 skb->mac_header += VLAN_HLEN; 5219 5220 if (skb_network_offset(skb) < ETH_HLEN) 5221 skb_set_network_header(skb, ETH_HLEN); 5222 5223 skb_reset_mac_len(skb); 5224 5225 return err; 5226 } 5227 EXPORT_SYMBOL(__skb_vlan_pop); 5228 5229 /* Pop a vlan tag either from hwaccel or from payload. 5230 * Expects skb->data at mac header. 5231 */ 5232 int skb_vlan_pop(struct sk_buff *skb) 5233 { 5234 u16 vlan_tci; 5235 __be16 vlan_proto; 5236 int err; 5237 5238 if (likely(skb_vlan_tag_present(skb))) { 5239 __vlan_hwaccel_clear_tag(skb); 5240 } else { 5241 if (unlikely(!eth_type_vlan(skb->protocol))) 5242 return 0; 5243 5244 err = __skb_vlan_pop(skb, &vlan_tci); 5245 if (err) 5246 return err; 5247 } 5248 /* move next vlan tag to hw accel tag */ 5249 if (likely(!eth_type_vlan(skb->protocol))) 5250 return 0; 5251 5252 vlan_proto = skb->protocol; 5253 err = __skb_vlan_pop(skb, &vlan_tci); 5254 if (unlikely(err)) 5255 return err; 5256 5257 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5258 return 0; 5259 } 5260 EXPORT_SYMBOL(skb_vlan_pop); 5261 5262 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 5263 * Expects skb->data at mac header. 5264 */ 5265 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 5266 { 5267 if (skb_vlan_tag_present(skb)) { 5268 int offset = skb->data - skb_mac_header(skb); 5269 int err; 5270 5271 if (WARN_ONCE(offset, 5272 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 5273 offset)) { 5274 return -EINVAL; 5275 } 5276 5277 err = __vlan_insert_tag(skb, skb->vlan_proto, 5278 skb_vlan_tag_get(skb)); 5279 if (err) 5280 return err; 5281 5282 skb->protocol = skb->vlan_proto; 5283 skb->mac_len += VLAN_HLEN; 5284 5285 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5286 } 5287 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5288 return 0; 5289 } 5290 EXPORT_SYMBOL(skb_vlan_push); 5291 5292 /** 5293 * alloc_skb_with_frags - allocate skb with page frags 5294 * 5295 * @header_len: size of linear part 5296 * @data_len: needed length in frags 5297 * @max_page_order: max page order desired. 5298 * @errcode: pointer to error code if any 5299 * @gfp_mask: allocation mask 5300 * 5301 * This can be used to allocate a paged skb, given a maximal order for frags. 5302 */ 5303 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 5304 unsigned long data_len, 5305 int max_page_order, 5306 int *errcode, 5307 gfp_t gfp_mask) 5308 { 5309 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 5310 unsigned long chunk; 5311 struct sk_buff *skb; 5312 struct page *page; 5313 int i; 5314 5315 *errcode = -EMSGSIZE; 5316 /* Note this test could be relaxed, if we succeed to allocate 5317 * high order pages... 5318 */ 5319 if (npages > MAX_SKB_FRAGS) 5320 return NULL; 5321 5322 *errcode = -ENOBUFS; 5323 skb = alloc_skb(header_len, gfp_mask); 5324 if (!skb) 5325 return NULL; 5326 5327 skb->truesize += npages << PAGE_SHIFT; 5328 5329 for (i = 0; npages > 0; i++) { 5330 int order = max_page_order; 5331 5332 while (order) { 5333 if (npages >= 1 << order) { 5334 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 5335 __GFP_COMP | 5336 __GFP_NOWARN, 5337 order); 5338 if (page) 5339 goto fill_page; 5340 /* Do not retry other high order allocations */ 5341 order = 1; 5342 max_page_order = 0; 5343 } 5344 order--; 5345 } 5346 page = alloc_page(gfp_mask); 5347 if (!page) 5348 goto failure; 5349 fill_page: 5350 chunk = min_t(unsigned long, data_len, 5351 PAGE_SIZE << order); 5352 skb_fill_page_desc(skb, i, page, 0, chunk); 5353 data_len -= chunk; 5354 npages -= 1 << order; 5355 } 5356 return skb; 5357 5358 failure: 5359 kfree_skb(skb); 5360 return NULL; 5361 } 5362 EXPORT_SYMBOL(alloc_skb_with_frags); 5363 5364 /* carve out the first off bytes from skb when off < headlen */ 5365 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 5366 const int headlen, gfp_t gfp_mask) 5367 { 5368 int i; 5369 int size = skb_end_offset(skb); 5370 int new_hlen = headlen - off; 5371 u8 *data; 5372 5373 size = SKB_DATA_ALIGN(size); 5374 5375 if (skb_pfmemalloc(skb)) 5376 gfp_mask |= __GFP_MEMALLOC; 5377 data = kmalloc_reserve(size + 5378 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 5379 gfp_mask, NUMA_NO_NODE, NULL); 5380 if (!data) 5381 return -ENOMEM; 5382 5383 size = SKB_WITH_OVERHEAD(ksize(data)); 5384 5385 /* Copy real data, and all frags */ 5386 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 5387 skb->len -= off; 5388 5389 memcpy((struct skb_shared_info *)(data + size), 5390 skb_shinfo(skb), 5391 offsetof(struct skb_shared_info, 5392 frags[skb_shinfo(skb)->nr_frags])); 5393 if (skb_cloned(skb)) { 5394 /* drop the old head gracefully */ 5395 if (skb_orphan_frags(skb, gfp_mask)) { 5396 kfree(data); 5397 return -ENOMEM; 5398 } 5399 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 5400 skb_frag_ref(skb, i); 5401 if (skb_has_frag_list(skb)) 5402 skb_clone_fraglist(skb); 5403 skb_release_data(skb); 5404 } else { 5405 /* we can reuse existing recount- all we did was 5406 * relocate values 5407 */ 5408 skb_free_head(skb); 5409 } 5410 5411 skb->head = data; 5412 skb->data = data; 5413 skb->head_frag = 0; 5414 #ifdef NET_SKBUFF_DATA_USES_OFFSET 5415 skb->end = size; 5416 #else 5417 skb->end = skb->head + size; 5418 #endif 5419 skb_set_tail_pointer(skb, skb_headlen(skb)); 5420 skb_headers_offset_update(skb, 0); 5421 skb->cloned = 0; 5422 skb->hdr_len = 0; 5423 skb->nohdr = 0; 5424 atomic_set(&skb_shinfo(skb)->dataref, 1); 5425 5426 return 0; 5427 } 5428 5429 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 5430 5431 /* carve out the first eat bytes from skb's frag_list. May recurse into 5432 * pskb_carve() 5433 */ 5434 static int pskb_carve_frag_list(struct sk_buff *skb, 5435 struct skb_shared_info *shinfo, int eat, 5436 gfp_t gfp_mask) 5437 { 5438 struct sk_buff *list = shinfo->frag_list; 5439 struct sk_buff *clone = NULL; 5440 struct sk_buff *insp = NULL; 5441 5442 do { 5443 if (!list) { 5444 pr_err("Not enough bytes to eat. Want %d\n", eat); 5445 return -EFAULT; 5446 } 5447 if (list->len <= eat) { 5448 /* Eaten as whole. */ 5449 eat -= list->len; 5450 list = list->next; 5451 insp = list; 5452 } else { 5453 /* Eaten partially. */ 5454 if (skb_shared(list)) { 5455 clone = skb_clone(list, gfp_mask); 5456 if (!clone) 5457 return -ENOMEM; 5458 insp = list->next; 5459 list = clone; 5460 } else { 5461 /* This may be pulled without problems. */ 5462 insp = list; 5463 } 5464 if (pskb_carve(list, eat, gfp_mask) < 0) { 5465 kfree_skb(clone); 5466 return -ENOMEM; 5467 } 5468 break; 5469 } 5470 } while (eat); 5471 5472 /* Free pulled out fragments. */ 5473 while ((list = shinfo->frag_list) != insp) { 5474 shinfo->frag_list = list->next; 5475 kfree_skb(list); 5476 } 5477 /* And insert new clone at head. */ 5478 if (clone) { 5479 clone->next = list; 5480 shinfo->frag_list = clone; 5481 } 5482 return 0; 5483 } 5484 5485 /* carve off first len bytes from skb. Split line (off) is in the 5486 * non-linear part of skb 5487 */ 5488 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 5489 int pos, gfp_t gfp_mask) 5490 { 5491 int i, k = 0; 5492 int size = skb_end_offset(skb); 5493 u8 *data; 5494 const int nfrags = skb_shinfo(skb)->nr_frags; 5495 struct skb_shared_info *shinfo; 5496 5497 size = SKB_DATA_ALIGN(size); 5498 5499 if (skb_pfmemalloc(skb)) 5500 gfp_mask |= __GFP_MEMALLOC; 5501 data = kmalloc_reserve(size + 5502 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 5503 gfp_mask, NUMA_NO_NODE, NULL); 5504 if (!data) 5505 return -ENOMEM; 5506 5507 size = SKB_WITH_OVERHEAD(ksize(data)); 5508 5509 memcpy((struct skb_shared_info *)(data + size), 5510 skb_shinfo(skb), offsetof(struct skb_shared_info, 5511 frags[skb_shinfo(skb)->nr_frags])); 5512 if (skb_orphan_frags(skb, gfp_mask)) { 5513 kfree(data); 5514 return -ENOMEM; 5515 } 5516 shinfo = (struct skb_shared_info *)(data + size); 5517 for (i = 0; i < nfrags; i++) { 5518 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 5519 5520 if (pos + fsize > off) { 5521 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 5522 5523 if (pos < off) { 5524 /* Split frag. 5525 * We have two variants in this case: 5526 * 1. Move all the frag to the second 5527 * part, if it is possible. F.e. 5528 * this approach is mandatory for TUX, 5529 * where splitting is expensive. 5530 * 2. Split is accurately. We make this. 5531 */ 5532 shinfo->frags[0].page_offset += off - pos; 5533 skb_frag_size_sub(&shinfo->frags[0], off - pos); 5534 } 5535 skb_frag_ref(skb, i); 5536 k++; 5537 } 5538 pos += fsize; 5539 } 5540 shinfo->nr_frags = k; 5541 if (skb_has_frag_list(skb)) 5542 skb_clone_fraglist(skb); 5543 5544 if (k == 0) { 5545 /* split line is in frag list */ 5546 pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask); 5547 } 5548 skb_release_data(skb); 5549 5550 skb->head = data; 5551 skb->head_frag = 0; 5552 skb->data = data; 5553 #ifdef NET_SKBUFF_DATA_USES_OFFSET 5554 skb->end = size; 5555 #else 5556 skb->end = skb->head + size; 5557 #endif 5558 skb_reset_tail_pointer(skb); 5559 skb_headers_offset_update(skb, 0); 5560 skb->cloned = 0; 5561 skb->hdr_len = 0; 5562 skb->nohdr = 0; 5563 skb->len -= off; 5564 skb->data_len = skb->len; 5565 atomic_set(&skb_shinfo(skb)->dataref, 1); 5566 return 0; 5567 } 5568 5569 /* remove len bytes from the beginning of the skb */ 5570 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 5571 { 5572 int headlen = skb_headlen(skb); 5573 5574 if (len < headlen) 5575 return pskb_carve_inside_header(skb, len, headlen, gfp); 5576 else 5577 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 5578 } 5579 5580 /* Extract to_copy bytes starting at off from skb, and return this in 5581 * a new skb 5582 */ 5583 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 5584 int to_copy, gfp_t gfp) 5585 { 5586 struct sk_buff *clone = skb_clone(skb, gfp); 5587 5588 if (!clone) 5589 return NULL; 5590 5591 if (pskb_carve(clone, off, gfp) < 0 || 5592 pskb_trim(clone, to_copy)) { 5593 kfree_skb(clone); 5594 return NULL; 5595 } 5596 return clone; 5597 } 5598 EXPORT_SYMBOL(pskb_extract); 5599 5600 /** 5601 * skb_condense - try to get rid of fragments/frag_list if possible 5602 * @skb: buffer 5603 * 5604 * Can be used to save memory before skb is added to a busy queue. 5605 * If packet has bytes in frags and enough tail room in skb->head, 5606 * pull all of them, so that we can free the frags right now and adjust 5607 * truesize. 5608 * Notes: 5609 * We do not reallocate skb->head thus can not fail. 5610 * Caller must re-evaluate skb->truesize if needed. 5611 */ 5612 void skb_condense(struct sk_buff *skb) 5613 { 5614 if (skb->data_len) { 5615 if (skb->data_len > skb->end - skb->tail || 5616 skb_cloned(skb)) 5617 return; 5618 5619 /* Nice, we can free page frag(s) right now */ 5620 __pskb_pull_tail(skb, skb->data_len); 5621 } 5622 /* At this point, skb->truesize might be over estimated, 5623 * because skb had a fragment, and fragments do not tell 5624 * their truesize. 5625 * When we pulled its content into skb->head, fragment 5626 * was freed, but __pskb_pull_tail() could not possibly 5627 * adjust skb->truesize, not knowing the frag truesize. 5628 */ 5629 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 5630 } 5631 5632 #ifdef CONFIG_SKB_EXTENSIONS 5633 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id) 5634 { 5635 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE); 5636 } 5637 5638 static struct skb_ext *skb_ext_alloc(void) 5639 { 5640 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 5641 5642 if (new) { 5643 memset(new->offset, 0, sizeof(new->offset)); 5644 refcount_set(&new->refcnt, 1); 5645 } 5646 5647 return new; 5648 } 5649 5650 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old, 5651 unsigned int old_active) 5652 { 5653 struct skb_ext *new; 5654 5655 if (refcount_read(&old->refcnt) == 1) 5656 return old; 5657 5658 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 5659 if (!new) 5660 return NULL; 5661 5662 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE); 5663 refcount_set(&new->refcnt, 1); 5664 5665 #ifdef CONFIG_XFRM 5666 if (old_active & (1 << SKB_EXT_SEC_PATH)) { 5667 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH); 5668 unsigned int i; 5669 5670 for (i = 0; i < sp->len; i++) 5671 xfrm_state_hold(sp->xvec[i]); 5672 } 5673 #endif 5674 __skb_ext_put(old); 5675 return new; 5676 } 5677 5678 /** 5679 * skb_ext_add - allocate space for given extension, COW if needed 5680 * @skb: buffer 5681 * @id: extension to allocate space for 5682 * 5683 * Allocates enough space for the given extension. 5684 * If the extension is already present, a pointer to that extension 5685 * is returned. 5686 * 5687 * If the skb was cloned, COW applies and the returned memory can be 5688 * modified without changing the extension space of clones buffers. 5689 * 5690 * Returns pointer to the extension or NULL on allocation failure. 5691 */ 5692 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id) 5693 { 5694 struct skb_ext *new, *old = NULL; 5695 unsigned int newlen, newoff; 5696 5697 if (skb->active_extensions) { 5698 old = skb->extensions; 5699 5700 new = skb_ext_maybe_cow(old, skb->active_extensions); 5701 if (!new) 5702 return NULL; 5703 5704 if (__skb_ext_exist(new, id)) 5705 goto set_active; 5706 5707 newoff = new->chunks; 5708 } else { 5709 newoff = SKB_EXT_CHUNKSIZEOF(*new); 5710 5711 new = skb_ext_alloc(); 5712 if (!new) 5713 return NULL; 5714 } 5715 5716 newlen = newoff + skb_ext_type_len[id]; 5717 new->chunks = newlen; 5718 new->offset[id] = newoff; 5719 set_active: 5720 skb->extensions = new; 5721 skb->active_extensions |= 1 << id; 5722 return skb_ext_get_ptr(new, id); 5723 } 5724 EXPORT_SYMBOL(skb_ext_add); 5725 5726 #ifdef CONFIG_XFRM 5727 static void skb_ext_put_sp(struct sec_path *sp) 5728 { 5729 unsigned int i; 5730 5731 for (i = 0; i < sp->len; i++) 5732 xfrm_state_put(sp->xvec[i]); 5733 } 5734 #endif 5735 5736 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 5737 { 5738 struct skb_ext *ext = skb->extensions; 5739 5740 skb->active_extensions &= ~(1 << id); 5741 if (skb->active_extensions == 0) { 5742 skb->extensions = NULL; 5743 __skb_ext_put(ext); 5744 #ifdef CONFIG_XFRM 5745 } else if (id == SKB_EXT_SEC_PATH && 5746 refcount_read(&ext->refcnt) == 1) { 5747 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH); 5748 5749 skb_ext_put_sp(sp); 5750 sp->len = 0; 5751 #endif 5752 } 5753 } 5754 EXPORT_SYMBOL(__skb_ext_del); 5755 5756 void __skb_ext_put(struct skb_ext *ext) 5757 { 5758 /* If this is last clone, nothing can increment 5759 * it after check passes. Avoids one atomic op. 5760 */ 5761 if (refcount_read(&ext->refcnt) == 1) 5762 goto free_now; 5763 5764 if (!refcount_dec_and_test(&ext->refcnt)) 5765 return; 5766 free_now: 5767 #ifdef CONFIG_XFRM 5768 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH)) 5769 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 5770 #endif 5771 5772 kmem_cache_free(skbuff_ext_cache, ext); 5773 } 5774 EXPORT_SYMBOL(__skb_ext_put); 5775 #endif /* CONFIG_SKB_EXTENSIONS */ 5776