1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Routines having to do with the 'struct sk_buff' memory handlers. 4 * 5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 6 * Florian La Roche <rzsfl@rz.uni-sb.de> 7 * 8 * Fixes: 9 * Alan Cox : Fixed the worst of the load 10 * balancer bugs. 11 * Dave Platt : Interrupt stacking fix. 12 * Richard Kooijman : Timestamp fixes. 13 * Alan Cox : Changed buffer format. 14 * Alan Cox : destructor hook for AF_UNIX etc. 15 * Linus Torvalds : Better skb_clone. 16 * Alan Cox : Added skb_copy. 17 * Alan Cox : Added all the changed routines Linus 18 * only put in the headers 19 * Ray VanTassle : Fixed --skb->lock in free 20 * Alan Cox : skb_copy copy arp field 21 * Andi Kleen : slabified it. 22 * Robert Olsson : Removed skb_head_pool 23 * 24 * NOTE: 25 * The __skb_ routines should be called with interrupts 26 * disabled, or you better be *real* sure that the operation is atomic 27 * with respect to whatever list is being frobbed (e.g. via lock_sock() 28 * or via disabling bottom half handlers, etc). 29 */ 30 31 /* 32 * The functions in this file will not compile correctly with gcc 2.4.x 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/module.h> 38 #include <linux/types.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/interrupt.h> 42 #include <linux/in.h> 43 #include <linux/inet.h> 44 #include <linux/slab.h> 45 #include <linux/tcp.h> 46 #include <linux/udp.h> 47 #include <linux/sctp.h> 48 #include <linux/netdevice.h> 49 #ifdef CONFIG_NET_CLS_ACT 50 #include <net/pkt_sched.h> 51 #endif 52 #include <linux/string.h> 53 #include <linux/skbuff.h> 54 #include <linux/splice.h> 55 #include <linux/cache.h> 56 #include <linux/rtnetlink.h> 57 #include <linux/init.h> 58 #include <linux/scatterlist.h> 59 #include <linux/errqueue.h> 60 #include <linux/prefetch.h> 61 #include <linux/if_vlan.h> 62 #include <linux/mpls.h> 63 #include <linux/kcov.h> 64 65 #include <net/protocol.h> 66 #include <net/dst.h> 67 #include <net/sock.h> 68 #include <net/checksum.h> 69 #include <net/ip6_checksum.h> 70 #include <net/xfrm.h> 71 #include <net/mpls.h> 72 #include <net/mptcp.h> 73 #include <net/mctp.h> 74 #include <net/page_pool.h> 75 76 #include <linux/uaccess.h> 77 #include <trace/events/skb.h> 78 #include <linux/highmem.h> 79 #include <linux/capability.h> 80 #include <linux/user_namespace.h> 81 #include <linux/indirect_call_wrapper.h> 82 83 #include "dev.h" 84 #include "sock_destructor.h" 85 86 struct kmem_cache *skbuff_head_cache __ro_after_init; 87 static struct kmem_cache *skbuff_fclone_cache __ro_after_init; 88 #ifdef CONFIG_SKB_EXTENSIONS 89 static struct kmem_cache *skbuff_ext_cache __ro_after_init; 90 #endif 91 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; 92 EXPORT_SYMBOL(sysctl_max_skb_frags); 93 94 #undef FN 95 #define FN(reason) [SKB_DROP_REASON_##reason] = #reason, 96 const char * const drop_reasons[] = { 97 [SKB_CONSUMED] = "CONSUMED", 98 DEFINE_DROP_REASON(FN, FN) 99 }; 100 EXPORT_SYMBOL(drop_reasons); 101 102 /** 103 * skb_panic - private function for out-of-line support 104 * @skb: buffer 105 * @sz: size 106 * @addr: address 107 * @msg: skb_over_panic or skb_under_panic 108 * 109 * Out-of-line support for skb_put() and skb_push(). 110 * Called via the wrapper skb_over_panic() or skb_under_panic(). 111 * Keep out of line to prevent kernel bloat. 112 * __builtin_return_address is not used because it is not always reliable. 113 */ 114 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 115 const char msg[]) 116 { 117 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n", 118 msg, addr, skb->len, sz, skb->head, skb->data, 119 (unsigned long)skb->tail, (unsigned long)skb->end, 120 skb->dev ? skb->dev->name : "<NULL>"); 121 BUG(); 122 } 123 124 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 125 { 126 skb_panic(skb, sz, addr, __func__); 127 } 128 129 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 130 { 131 skb_panic(skb, sz, addr, __func__); 132 } 133 134 #define NAPI_SKB_CACHE_SIZE 64 135 #define NAPI_SKB_CACHE_BULK 16 136 #define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2) 137 138 #if PAGE_SIZE == SZ_4K 139 140 #define NAPI_HAS_SMALL_PAGE_FRAG 1 141 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) ((nc).pfmemalloc) 142 143 /* specialized page frag allocator using a single order 0 page 144 * and slicing it into 1K sized fragment. Constrained to systems 145 * with a very limited amount of 1K fragments fitting a single 146 * page - to avoid excessive truesize underestimation 147 */ 148 149 struct page_frag_1k { 150 void *va; 151 u16 offset; 152 bool pfmemalloc; 153 }; 154 155 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp) 156 { 157 struct page *page; 158 int offset; 159 160 offset = nc->offset - SZ_1K; 161 if (likely(offset >= 0)) 162 goto use_frag; 163 164 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 165 if (!page) 166 return NULL; 167 168 nc->va = page_address(page); 169 nc->pfmemalloc = page_is_pfmemalloc(page); 170 offset = PAGE_SIZE - SZ_1K; 171 page_ref_add(page, offset / SZ_1K); 172 173 use_frag: 174 nc->offset = offset; 175 return nc->va + offset; 176 } 177 #else 178 179 /* the small page is actually unused in this build; add dummy helpers 180 * to please the compiler and avoid later preprocessor's conditionals 181 */ 182 #define NAPI_HAS_SMALL_PAGE_FRAG 0 183 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) false 184 185 struct page_frag_1k { 186 }; 187 188 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp_mask) 189 { 190 return NULL; 191 } 192 193 #endif 194 195 struct napi_alloc_cache { 196 struct page_frag_cache page; 197 struct page_frag_1k page_small; 198 unsigned int skb_count; 199 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 200 }; 201 202 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 203 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 204 205 /* Double check that napi_get_frags() allocates skbs with 206 * skb->head being backed by slab, not a page fragment. 207 * This is to make sure bug fixed in 3226b158e67c 208 * ("net: avoid 32 x truesize under-estimation for tiny skbs") 209 * does not accidentally come back. 210 */ 211 void napi_get_frags_check(struct napi_struct *napi) 212 { 213 struct sk_buff *skb; 214 215 local_bh_disable(); 216 skb = napi_get_frags(napi); 217 WARN_ON_ONCE(!NAPI_HAS_SMALL_PAGE_FRAG && skb && skb->head_frag); 218 napi_free_frags(napi); 219 local_bh_enable(); 220 } 221 222 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 223 { 224 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 225 226 fragsz = SKB_DATA_ALIGN(fragsz); 227 228 return page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); 229 } 230 EXPORT_SYMBOL(__napi_alloc_frag_align); 231 232 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 233 { 234 void *data; 235 236 fragsz = SKB_DATA_ALIGN(fragsz); 237 if (in_hardirq() || irqs_disabled()) { 238 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache); 239 240 data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask); 241 } else { 242 struct napi_alloc_cache *nc; 243 244 local_bh_disable(); 245 nc = this_cpu_ptr(&napi_alloc_cache); 246 data = page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); 247 local_bh_enable(); 248 } 249 return data; 250 } 251 EXPORT_SYMBOL(__netdev_alloc_frag_align); 252 253 static struct sk_buff *napi_skb_cache_get(void) 254 { 255 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 256 struct sk_buff *skb; 257 258 if (unlikely(!nc->skb_count)) { 259 nc->skb_count = kmem_cache_alloc_bulk(skbuff_head_cache, 260 GFP_ATOMIC, 261 NAPI_SKB_CACHE_BULK, 262 nc->skb_cache); 263 if (unlikely(!nc->skb_count)) 264 return NULL; 265 } 266 267 skb = nc->skb_cache[--nc->skb_count]; 268 kasan_unpoison_object_data(skbuff_head_cache, skb); 269 270 return skb; 271 } 272 273 static inline void __finalize_skb_around(struct sk_buff *skb, void *data, 274 unsigned int size) 275 { 276 struct skb_shared_info *shinfo; 277 278 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 279 280 /* Assumes caller memset cleared SKB */ 281 skb->truesize = SKB_TRUESIZE(size); 282 refcount_set(&skb->users, 1); 283 skb->head = data; 284 skb->data = data; 285 skb_reset_tail_pointer(skb); 286 skb_set_end_offset(skb, size); 287 skb->mac_header = (typeof(skb->mac_header))~0U; 288 skb->transport_header = (typeof(skb->transport_header))~0U; 289 skb->alloc_cpu = raw_smp_processor_id(); 290 /* make sure we initialize shinfo sequentially */ 291 shinfo = skb_shinfo(skb); 292 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 293 atomic_set(&shinfo->dataref, 1); 294 295 skb_set_kcov_handle(skb, kcov_common_handle()); 296 } 297 298 static inline void *__slab_build_skb(struct sk_buff *skb, void *data, 299 unsigned int *size) 300 { 301 void *resized; 302 303 /* Must find the allocation size (and grow it to match). */ 304 *size = ksize(data); 305 /* krealloc() will immediately return "data" when 306 * "ksize(data)" is requested: it is the existing upper 307 * bounds. As a result, GFP_ATOMIC will be ignored. Note 308 * that this "new" pointer needs to be passed back to the 309 * caller for use so the __alloc_size hinting will be 310 * tracked correctly. 311 */ 312 resized = krealloc(data, *size, GFP_ATOMIC); 313 WARN_ON_ONCE(resized != data); 314 return resized; 315 } 316 317 /* build_skb() variant which can operate on slab buffers. 318 * Note that this should be used sparingly as slab buffers 319 * cannot be combined efficiently by GRO! 320 */ 321 struct sk_buff *slab_build_skb(void *data) 322 { 323 struct sk_buff *skb; 324 unsigned int size; 325 326 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); 327 if (unlikely(!skb)) 328 return NULL; 329 330 memset(skb, 0, offsetof(struct sk_buff, tail)); 331 data = __slab_build_skb(skb, data, &size); 332 __finalize_skb_around(skb, data, size); 333 334 return skb; 335 } 336 EXPORT_SYMBOL(slab_build_skb); 337 338 /* Caller must provide SKB that is memset cleared */ 339 static void __build_skb_around(struct sk_buff *skb, void *data, 340 unsigned int frag_size) 341 { 342 unsigned int size = frag_size; 343 344 /* frag_size == 0 is considered deprecated now. Callers 345 * using slab buffer should use slab_build_skb() instead. 346 */ 347 if (WARN_ONCE(size == 0, "Use slab_build_skb() instead")) 348 data = __slab_build_skb(skb, data, &size); 349 350 __finalize_skb_around(skb, data, size); 351 } 352 353 /** 354 * __build_skb - build a network buffer 355 * @data: data buffer provided by caller 356 * @frag_size: size of data (must not be 0) 357 * 358 * Allocate a new &sk_buff. Caller provides space holding head and 359 * skb_shared_info. @data must have been allocated from the page 360 * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc() 361 * allocation is deprecated, and callers should use slab_build_skb() 362 * instead.) 363 * The return is the new skb buffer. 364 * On a failure the return is %NULL, and @data is not freed. 365 * Notes : 366 * Before IO, driver allocates only data buffer where NIC put incoming frame 367 * Driver should add room at head (NET_SKB_PAD) and 368 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 369 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 370 * before giving packet to stack. 371 * RX rings only contains data buffers, not full skbs. 372 */ 373 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 374 { 375 struct sk_buff *skb; 376 377 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); 378 if (unlikely(!skb)) 379 return NULL; 380 381 memset(skb, 0, offsetof(struct sk_buff, tail)); 382 __build_skb_around(skb, data, frag_size); 383 384 return skb; 385 } 386 387 /* build_skb() is wrapper over __build_skb(), that specifically 388 * takes care of skb->head and skb->pfmemalloc 389 * This means that if @frag_size is not zero, then @data must be backed 390 * by a page fragment, not kmalloc() or vmalloc() 391 */ 392 struct sk_buff *build_skb(void *data, unsigned int frag_size) 393 { 394 struct sk_buff *skb = __build_skb(data, frag_size); 395 396 if (skb && frag_size) { 397 skb->head_frag = 1; 398 if (page_is_pfmemalloc(virt_to_head_page(data))) 399 skb->pfmemalloc = 1; 400 } 401 return skb; 402 } 403 EXPORT_SYMBOL(build_skb); 404 405 /** 406 * build_skb_around - build a network buffer around provided skb 407 * @skb: sk_buff provide by caller, must be memset cleared 408 * @data: data buffer provided by caller 409 * @frag_size: size of data, or 0 if head was kmalloced 410 */ 411 struct sk_buff *build_skb_around(struct sk_buff *skb, 412 void *data, unsigned int frag_size) 413 { 414 if (unlikely(!skb)) 415 return NULL; 416 417 __build_skb_around(skb, data, frag_size); 418 419 if (frag_size) { 420 skb->head_frag = 1; 421 if (page_is_pfmemalloc(virt_to_head_page(data))) 422 skb->pfmemalloc = 1; 423 } 424 return skb; 425 } 426 EXPORT_SYMBOL(build_skb_around); 427 428 /** 429 * __napi_build_skb - build a network buffer 430 * @data: data buffer provided by caller 431 * @frag_size: size of data, or 0 if head was kmalloced 432 * 433 * Version of __build_skb() that uses NAPI percpu caches to obtain 434 * skbuff_head instead of inplace allocation. 435 * 436 * Returns a new &sk_buff on success, %NULL on allocation failure. 437 */ 438 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 439 { 440 struct sk_buff *skb; 441 442 skb = napi_skb_cache_get(); 443 if (unlikely(!skb)) 444 return NULL; 445 446 memset(skb, 0, offsetof(struct sk_buff, tail)); 447 __build_skb_around(skb, data, frag_size); 448 449 return skb; 450 } 451 452 /** 453 * napi_build_skb - build a network buffer 454 * @data: data buffer provided by caller 455 * @frag_size: size of data, or 0 if head was kmalloced 456 * 457 * Version of __napi_build_skb() that takes care of skb->head_frag 458 * and skb->pfmemalloc when the data is a page or page fragment. 459 * 460 * Returns a new &sk_buff on success, %NULL on allocation failure. 461 */ 462 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) 463 { 464 struct sk_buff *skb = __napi_build_skb(data, frag_size); 465 466 if (likely(skb) && frag_size) { 467 skb->head_frag = 1; 468 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 469 } 470 471 return skb; 472 } 473 EXPORT_SYMBOL(napi_build_skb); 474 475 /* 476 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 477 * the caller if emergency pfmemalloc reserves are being used. If it is and 478 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 479 * may be used. Otherwise, the packet data may be discarded until enough 480 * memory is free 481 */ 482 static void *kmalloc_reserve(size_t size, gfp_t flags, int node, 483 bool *pfmemalloc) 484 { 485 void *obj; 486 bool ret_pfmemalloc = false; 487 488 /* 489 * Try a regular allocation, when that fails and we're not entitled 490 * to the reserves, fail. 491 */ 492 obj = kmalloc_node_track_caller(size, 493 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 494 node); 495 if (obj || !(gfp_pfmemalloc_allowed(flags))) 496 goto out; 497 498 /* Try again but now we are using pfmemalloc reserves */ 499 ret_pfmemalloc = true; 500 obj = kmalloc_node_track_caller(size, flags, node); 501 502 out: 503 if (pfmemalloc) 504 *pfmemalloc = ret_pfmemalloc; 505 506 return obj; 507 } 508 509 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 510 * 'private' fields and also do memory statistics to find all the 511 * [BEEP] leaks. 512 * 513 */ 514 515 /** 516 * __alloc_skb - allocate a network buffer 517 * @size: size to allocate 518 * @gfp_mask: allocation mask 519 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 520 * instead of head cache and allocate a cloned (child) skb. 521 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 522 * allocations in case the data is required for writeback 523 * @node: numa node to allocate memory on 524 * 525 * Allocate a new &sk_buff. The returned buffer has no headroom and a 526 * tail room of at least size bytes. The object has a reference count 527 * of one. The return is the buffer. On a failure the return is %NULL. 528 * 529 * Buffers may only be allocated from interrupts using a @gfp_mask of 530 * %GFP_ATOMIC. 531 */ 532 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 533 int flags, int node) 534 { 535 struct kmem_cache *cache; 536 struct sk_buff *skb; 537 unsigned int osize; 538 bool pfmemalloc; 539 u8 *data; 540 541 cache = (flags & SKB_ALLOC_FCLONE) 542 ? skbuff_fclone_cache : skbuff_head_cache; 543 544 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 545 gfp_mask |= __GFP_MEMALLOC; 546 547 /* Get the HEAD */ 548 if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI && 549 likely(node == NUMA_NO_NODE || node == numa_mem_id())) 550 skb = napi_skb_cache_get(); 551 else 552 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); 553 if (unlikely(!skb)) 554 return NULL; 555 prefetchw(skb); 556 557 /* We do our best to align skb_shared_info on a separate cache 558 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 559 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 560 * Both skb->head and skb_shared_info are cache line aligned. 561 */ 562 size = SKB_DATA_ALIGN(size); 563 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 564 osize = kmalloc_size_roundup(size); 565 data = kmalloc_reserve(osize, gfp_mask, node, &pfmemalloc); 566 if (unlikely(!data)) 567 goto nodata; 568 /* kmalloc_size_roundup() might give us more room than requested. 569 * Put skb_shared_info exactly at the end of allocated zone, 570 * to allow max possible filling before reallocation. 571 */ 572 size = SKB_WITH_OVERHEAD(osize); 573 prefetchw(data + size); 574 575 /* 576 * Only clear those fields we need to clear, not those that we will 577 * actually initialise below. Hence, don't put any more fields after 578 * the tail pointer in struct sk_buff! 579 */ 580 memset(skb, 0, offsetof(struct sk_buff, tail)); 581 __build_skb_around(skb, data, osize); 582 skb->pfmemalloc = pfmemalloc; 583 584 if (flags & SKB_ALLOC_FCLONE) { 585 struct sk_buff_fclones *fclones; 586 587 fclones = container_of(skb, struct sk_buff_fclones, skb1); 588 589 skb->fclone = SKB_FCLONE_ORIG; 590 refcount_set(&fclones->fclone_ref, 1); 591 } 592 593 return skb; 594 595 nodata: 596 kmem_cache_free(cache, skb); 597 return NULL; 598 } 599 EXPORT_SYMBOL(__alloc_skb); 600 601 /** 602 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 603 * @dev: network device to receive on 604 * @len: length to allocate 605 * @gfp_mask: get_free_pages mask, passed to alloc_skb 606 * 607 * Allocate a new &sk_buff and assign it a usage count of one. The 608 * buffer has NET_SKB_PAD headroom built in. Users should allocate 609 * the headroom they think they need without accounting for the 610 * built in space. The built in space is used for optimisations. 611 * 612 * %NULL is returned if there is no free memory. 613 */ 614 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 615 gfp_t gfp_mask) 616 { 617 struct page_frag_cache *nc; 618 struct sk_buff *skb; 619 bool pfmemalloc; 620 void *data; 621 622 len += NET_SKB_PAD; 623 624 /* If requested length is either too small or too big, 625 * we use kmalloc() for skb->head allocation. 626 */ 627 if (len <= SKB_WITH_OVERHEAD(1024) || 628 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 629 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 630 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 631 if (!skb) 632 goto skb_fail; 633 goto skb_success; 634 } 635 636 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 637 len = SKB_DATA_ALIGN(len); 638 639 if (sk_memalloc_socks()) 640 gfp_mask |= __GFP_MEMALLOC; 641 642 if (in_hardirq() || irqs_disabled()) { 643 nc = this_cpu_ptr(&netdev_alloc_cache); 644 data = page_frag_alloc(nc, len, gfp_mask); 645 pfmemalloc = nc->pfmemalloc; 646 } else { 647 local_bh_disable(); 648 nc = this_cpu_ptr(&napi_alloc_cache.page); 649 data = page_frag_alloc(nc, len, gfp_mask); 650 pfmemalloc = nc->pfmemalloc; 651 local_bh_enable(); 652 } 653 654 if (unlikely(!data)) 655 return NULL; 656 657 skb = __build_skb(data, len); 658 if (unlikely(!skb)) { 659 skb_free_frag(data); 660 return NULL; 661 } 662 663 if (pfmemalloc) 664 skb->pfmemalloc = 1; 665 skb->head_frag = 1; 666 667 skb_success: 668 skb_reserve(skb, NET_SKB_PAD); 669 skb->dev = dev; 670 671 skb_fail: 672 return skb; 673 } 674 EXPORT_SYMBOL(__netdev_alloc_skb); 675 676 /** 677 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 678 * @napi: napi instance this buffer was allocated for 679 * @len: length to allocate 680 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages 681 * 682 * Allocate a new sk_buff for use in NAPI receive. This buffer will 683 * attempt to allocate the head from a special reserved region used 684 * only for NAPI Rx allocation. By doing this we can save several 685 * CPU cycles by avoiding having to disable and re-enable IRQs. 686 * 687 * %NULL is returned if there is no free memory. 688 */ 689 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, 690 gfp_t gfp_mask) 691 { 692 struct napi_alloc_cache *nc; 693 struct sk_buff *skb; 694 bool pfmemalloc; 695 void *data; 696 697 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 698 len += NET_SKB_PAD + NET_IP_ALIGN; 699 700 /* If requested length is either too small or too big, 701 * we use kmalloc() for skb->head allocation. 702 * When the small frag allocator is available, prefer it over kmalloc 703 * for small fragments 704 */ 705 if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) || 706 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 707 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 708 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, 709 NUMA_NO_NODE); 710 if (!skb) 711 goto skb_fail; 712 goto skb_success; 713 } 714 715 nc = this_cpu_ptr(&napi_alloc_cache); 716 717 if (sk_memalloc_socks()) 718 gfp_mask |= __GFP_MEMALLOC; 719 720 if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) { 721 /* we are artificially inflating the allocation size, but 722 * that is not as bad as it may look like, as: 723 * - 'len' less than GRO_MAX_HEAD makes little sense 724 * - On most systems, larger 'len' values lead to fragment 725 * size above 512 bytes 726 * - kmalloc would use the kmalloc-1k slab for such values 727 * - Builds with smaller GRO_MAX_HEAD will very likely do 728 * little networking, as that implies no WiFi and no 729 * tunnels support, and 32 bits arches. 730 */ 731 len = SZ_1K; 732 733 data = page_frag_alloc_1k(&nc->page_small, gfp_mask); 734 pfmemalloc = NAPI_SMALL_PAGE_PFMEMALLOC(nc->page_small); 735 } else { 736 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 737 len = SKB_DATA_ALIGN(len); 738 739 data = page_frag_alloc(&nc->page, len, gfp_mask); 740 pfmemalloc = nc->page.pfmemalloc; 741 } 742 743 if (unlikely(!data)) 744 return NULL; 745 746 skb = __napi_build_skb(data, len); 747 if (unlikely(!skb)) { 748 skb_free_frag(data); 749 return NULL; 750 } 751 752 if (pfmemalloc) 753 skb->pfmemalloc = 1; 754 skb->head_frag = 1; 755 756 skb_success: 757 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 758 skb->dev = napi->dev; 759 760 skb_fail: 761 return skb; 762 } 763 EXPORT_SYMBOL(__napi_alloc_skb); 764 765 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 766 int size, unsigned int truesize) 767 { 768 skb_fill_page_desc(skb, i, page, off, size); 769 skb->len += size; 770 skb->data_len += size; 771 skb->truesize += truesize; 772 } 773 EXPORT_SYMBOL(skb_add_rx_frag); 774 775 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 776 unsigned int truesize) 777 { 778 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 779 780 skb_frag_size_add(frag, size); 781 skb->len += size; 782 skb->data_len += size; 783 skb->truesize += truesize; 784 } 785 EXPORT_SYMBOL(skb_coalesce_rx_frag); 786 787 static void skb_drop_list(struct sk_buff **listp) 788 { 789 kfree_skb_list(*listp); 790 *listp = NULL; 791 } 792 793 static inline void skb_drop_fraglist(struct sk_buff *skb) 794 { 795 skb_drop_list(&skb_shinfo(skb)->frag_list); 796 } 797 798 static void skb_clone_fraglist(struct sk_buff *skb) 799 { 800 struct sk_buff *list; 801 802 skb_walk_frags(skb, list) 803 skb_get(list); 804 } 805 806 static bool skb_pp_recycle(struct sk_buff *skb, void *data) 807 { 808 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) 809 return false; 810 return page_pool_return_skb_page(virt_to_page(data)); 811 } 812 813 static void skb_free_head(struct sk_buff *skb) 814 { 815 unsigned char *head = skb->head; 816 817 if (skb->head_frag) { 818 if (skb_pp_recycle(skb, head)) 819 return; 820 skb_free_frag(head); 821 } else { 822 kfree(head); 823 } 824 } 825 826 static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason) 827 { 828 struct skb_shared_info *shinfo = skb_shinfo(skb); 829 int i; 830 831 if (skb->cloned && 832 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 833 &shinfo->dataref)) 834 goto exit; 835 836 if (skb_zcopy(skb)) { 837 bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS; 838 839 skb_zcopy_clear(skb, true); 840 if (skip_unref) 841 goto free_head; 842 } 843 844 for (i = 0; i < shinfo->nr_frags; i++) 845 __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle); 846 847 free_head: 848 if (shinfo->frag_list) 849 kfree_skb_list_reason(shinfo->frag_list, reason); 850 851 skb_free_head(skb); 852 exit: 853 /* When we clone an SKB we copy the reycling bit. The pp_recycle 854 * bit is only set on the head though, so in order to avoid races 855 * while trying to recycle fragments on __skb_frag_unref() we need 856 * to make one SKB responsible for triggering the recycle path. 857 * So disable the recycling bit if an SKB is cloned and we have 858 * additional references to the fragmented part of the SKB. 859 * Eventually the last SKB will have the recycling bit set and it's 860 * dataref set to 0, which will trigger the recycling 861 */ 862 skb->pp_recycle = 0; 863 } 864 865 /* 866 * Free an skbuff by memory without cleaning the state. 867 */ 868 static void kfree_skbmem(struct sk_buff *skb) 869 { 870 struct sk_buff_fclones *fclones; 871 872 switch (skb->fclone) { 873 case SKB_FCLONE_UNAVAILABLE: 874 kmem_cache_free(skbuff_head_cache, skb); 875 return; 876 877 case SKB_FCLONE_ORIG: 878 fclones = container_of(skb, struct sk_buff_fclones, skb1); 879 880 /* We usually free the clone (TX completion) before original skb 881 * This test would have no chance to be true for the clone, 882 * while here, branch prediction will be good. 883 */ 884 if (refcount_read(&fclones->fclone_ref) == 1) 885 goto fastpath; 886 break; 887 888 default: /* SKB_FCLONE_CLONE */ 889 fclones = container_of(skb, struct sk_buff_fclones, skb2); 890 break; 891 } 892 if (!refcount_dec_and_test(&fclones->fclone_ref)) 893 return; 894 fastpath: 895 kmem_cache_free(skbuff_fclone_cache, fclones); 896 } 897 898 void skb_release_head_state(struct sk_buff *skb) 899 { 900 skb_dst_drop(skb); 901 if (skb->destructor) { 902 DEBUG_NET_WARN_ON_ONCE(in_hardirq()); 903 skb->destructor(skb); 904 } 905 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 906 nf_conntrack_put(skb_nfct(skb)); 907 #endif 908 skb_ext_put(skb); 909 } 910 911 /* Free everything but the sk_buff shell. */ 912 static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason) 913 { 914 skb_release_head_state(skb); 915 if (likely(skb->head)) 916 skb_release_data(skb, reason); 917 } 918 919 /** 920 * __kfree_skb - private function 921 * @skb: buffer 922 * 923 * Free an sk_buff. Release anything attached to the buffer. 924 * Clean the state. This is an internal helper function. Users should 925 * always call kfree_skb 926 */ 927 928 void __kfree_skb(struct sk_buff *skb) 929 { 930 skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED); 931 kfree_skbmem(skb); 932 } 933 EXPORT_SYMBOL(__kfree_skb); 934 935 /** 936 * kfree_skb_reason - free an sk_buff with special reason 937 * @skb: buffer to free 938 * @reason: reason why this skb is dropped 939 * 940 * Drop a reference to the buffer and free it if the usage count has 941 * hit zero. Meanwhile, pass the drop reason to 'kfree_skb' 942 * tracepoint. 943 */ 944 void __fix_address 945 kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) 946 { 947 if (unlikely(!skb_unref(skb))) 948 return; 949 950 DEBUG_NET_WARN_ON_ONCE(reason <= 0 || reason >= SKB_DROP_REASON_MAX); 951 952 if (reason == SKB_CONSUMED) 953 trace_consume_skb(skb); 954 else 955 trace_kfree_skb(skb, __builtin_return_address(0), reason); 956 __kfree_skb(skb); 957 } 958 EXPORT_SYMBOL(kfree_skb_reason); 959 960 void kfree_skb_list_reason(struct sk_buff *segs, 961 enum skb_drop_reason reason) 962 { 963 while (segs) { 964 struct sk_buff *next = segs->next; 965 966 kfree_skb_reason(segs, reason); 967 segs = next; 968 } 969 } 970 EXPORT_SYMBOL(kfree_skb_list_reason); 971 972 /* Dump skb information and contents. 973 * 974 * Must only be called from net_ratelimit()-ed paths. 975 * 976 * Dumps whole packets if full_pkt, only headers otherwise. 977 */ 978 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt) 979 { 980 struct skb_shared_info *sh = skb_shinfo(skb); 981 struct net_device *dev = skb->dev; 982 struct sock *sk = skb->sk; 983 struct sk_buff *list_skb; 984 bool has_mac, has_trans; 985 int headroom, tailroom; 986 int i, len, seg_len; 987 988 if (full_pkt) 989 len = skb->len; 990 else 991 len = min_t(int, skb->len, MAX_HEADER + 128); 992 993 headroom = skb_headroom(skb); 994 tailroom = skb_tailroom(skb); 995 996 has_mac = skb_mac_header_was_set(skb); 997 has_trans = skb_transport_header_was_set(skb); 998 999 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n" 1000 "mac=(%d,%d) net=(%d,%d) trans=%d\n" 1001 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" 1002 "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n" 1003 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n", 1004 level, skb->len, headroom, skb_headlen(skb), tailroom, 1005 has_mac ? skb->mac_header : -1, 1006 has_mac ? skb_mac_header_len(skb) : -1, 1007 skb->network_header, 1008 has_trans ? skb_network_header_len(skb) : -1, 1009 has_trans ? skb->transport_header : -1, 1010 sh->tx_flags, sh->nr_frags, 1011 sh->gso_size, sh->gso_type, sh->gso_segs, 1012 skb->csum, skb->ip_summed, skb->csum_complete_sw, 1013 skb->csum_valid, skb->csum_level, 1014 skb->hash, skb->sw_hash, skb->l4_hash, 1015 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif); 1016 1017 if (dev) 1018 printk("%sdev name=%s feat=%pNF\n", 1019 level, dev->name, &dev->features); 1020 if (sk) 1021 printk("%ssk family=%hu type=%u proto=%u\n", 1022 level, sk->sk_family, sk->sk_type, sk->sk_protocol); 1023 1024 if (full_pkt && headroom) 1025 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET, 1026 16, 1, skb->head, headroom, false); 1027 1028 seg_len = min_t(int, skb_headlen(skb), len); 1029 if (seg_len) 1030 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, 1031 16, 1, skb->data, seg_len, false); 1032 len -= seg_len; 1033 1034 if (full_pkt && tailroom) 1035 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET, 1036 16, 1, skb_tail_pointer(skb), tailroom, false); 1037 1038 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { 1039 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1040 u32 p_off, p_len, copied; 1041 struct page *p; 1042 u8 *vaddr; 1043 1044 skb_frag_foreach_page(frag, skb_frag_off(frag), 1045 skb_frag_size(frag), p, p_off, p_len, 1046 copied) { 1047 seg_len = min_t(int, p_len, len); 1048 vaddr = kmap_atomic(p); 1049 print_hex_dump(level, "skb frag: ", 1050 DUMP_PREFIX_OFFSET, 1051 16, 1, vaddr + p_off, seg_len, false); 1052 kunmap_atomic(vaddr); 1053 len -= seg_len; 1054 if (!len) 1055 break; 1056 } 1057 } 1058 1059 if (full_pkt && skb_has_frag_list(skb)) { 1060 printk("skb fraglist:\n"); 1061 skb_walk_frags(skb, list_skb) 1062 skb_dump(level, list_skb, true); 1063 } 1064 } 1065 EXPORT_SYMBOL(skb_dump); 1066 1067 /** 1068 * skb_tx_error - report an sk_buff xmit error 1069 * @skb: buffer that triggered an error 1070 * 1071 * Report xmit error if a device callback is tracking this skb. 1072 * skb must be freed afterwards. 1073 */ 1074 void skb_tx_error(struct sk_buff *skb) 1075 { 1076 if (skb) { 1077 skb_zcopy_downgrade_managed(skb); 1078 skb_zcopy_clear(skb, true); 1079 } 1080 } 1081 EXPORT_SYMBOL(skb_tx_error); 1082 1083 #ifdef CONFIG_TRACEPOINTS 1084 /** 1085 * consume_skb - free an skbuff 1086 * @skb: buffer to free 1087 * 1088 * Drop a ref to the buffer and free it if the usage count has hit zero 1089 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 1090 * is being dropped after a failure and notes that 1091 */ 1092 void consume_skb(struct sk_buff *skb) 1093 { 1094 if (!skb_unref(skb)) 1095 return; 1096 1097 trace_consume_skb(skb); 1098 __kfree_skb(skb); 1099 } 1100 EXPORT_SYMBOL(consume_skb); 1101 #endif 1102 1103 /** 1104 * __consume_stateless_skb - free an skbuff, assuming it is stateless 1105 * @skb: buffer to free 1106 * 1107 * Alike consume_skb(), but this variant assumes that this is the last 1108 * skb reference and all the head states have been already dropped 1109 */ 1110 void __consume_stateless_skb(struct sk_buff *skb) 1111 { 1112 trace_consume_skb(skb); 1113 skb_release_data(skb, SKB_CONSUMED); 1114 kfree_skbmem(skb); 1115 } 1116 1117 static void napi_skb_cache_put(struct sk_buff *skb) 1118 { 1119 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 1120 u32 i; 1121 1122 kasan_poison_object_data(skbuff_head_cache, skb); 1123 nc->skb_cache[nc->skb_count++] = skb; 1124 1125 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 1126 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) 1127 kasan_unpoison_object_data(skbuff_head_cache, 1128 nc->skb_cache[i]); 1129 1130 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_HALF, 1131 nc->skb_cache + NAPI_SKB_CACHE_HALF); 1132 nc->skb_count = NAPI_SKB_CACHE_HALF; 1133 } 1134 } 1135 1136 void __kfree_skb_defer(struct sk_buff *skb) 1137 { 1138 skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED); 1139 napi_skb_cache_put(skb); 1140 } 1141 1142 void napi_skb_free_stolen_head(struct sk_buff *skb) 1143 { 1144 if (unlikely(skb->slow_gro)) { 1145 nf_reset_ct(skb); 1146 skb_dst_drop(skb); 1147 skb_ext_put(skb); 1148 skb_orphan(skb); 1149 skb->slow_gro = 0; 1150 } 1151 napi_skb_cache_put(skb); 1152 } 1153 1154 void napi_consume_skb(struct sk_buff *skb, int budget) 1155 { 1156 /* Zero budget indicate non-NAPI context called us, like netpoll */ 1157 if (unlikely(!budget)) { 1158 dev_consume_skb_any(skb); 1159 return; 1160 } 1161 1162 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 1163 1164 if (!skb_unref(skb)) 1165 return; 1166 1167 /* if reaching here SKB is ready to free */ 1168 trace_consume_skb(skb); 1169 1170 /* if SKB is a clone, don't handle this case */ 1171 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 1172 __kfree_skb(skb); 1173 return; 1174 } 1175 1176 skb_release_all(skb, SKB_CONSUMED); 1177 napi_skb_cache_put(skb); 1178 } 1179 EXPORT_SYMBOL(napi_consume_skb); 1180 1181 /* Make sure a field is contained by headers group */ 1182 #define CHECK_SKB_FIELD(field) \ 1183 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \ 1184 offsetof(struct sk_buff, headers.field)); \ 1185 1186 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 1187 { 1188 new->tstamp = old->tstamp; 1189 /* We do not copy old->sk */ 1190 new->dev = old->dev; 1191 memcpy(new->cb, old->cb, sizeof(old->cb)); 1192 skb_dst_copy(new, old); 1193 __skb_ext_copy(new, old); 1194 __nf_copy(new, old, false); 1195 1196 /* Note : this field could be in the headers group. 1197 * It is not yet because we do not want to have a 16 bit hole 1198 */ 1199 new->queue_mapping = old->queue_mapping; 1200 1201 memcpy(&new->headers, &old->headers, sizeof(new->headers)); 1202 CHECK_SKB_FIELD(protocol); 1203 CHECK_SKB_FIELD(csum); 1204 CHECK_SKB_FIELD(hash); 1205 CHECK_SKB_FIELD(priority); 1206 CHECK_SKB_FIELD(skb_iif); 1207 CHECK_SKB_FIELD(vlan_proto); 1208 CHECK_SKB_FIELD(vlan_tci); 1209 CHECK_SKB_FIELD(transport_header); 1210 CHECK_SKB_FIELD(network_header); 1211 CHECK_SKB_FIELD(mac_header); 1212 CHECK_SKB_FIELD(inner_protocol); 1213 CHECK_SKB_FIELD(inner_transport_header); 1214 CHECK_SKB_FIELD(inner_network_header); 1215 CHECK_SKB_FIELD(inner_mac_header); 1216 CHECK_SKB_FIELD(mark); 1217 #ifdef CONFIG_NETWORK_SECMARK 1218 CHECK_SKB_FIELD(secmark); 1219 #endif 1220 #ifdef CONFIG_NET_RX_BUSY_POLL 1221 CHECK_SKB_FIELD(napi_id); 1222 #endif 1223 CHECK_SKB_FIELD(alloc_cpu); 1224 #ifdef CONFIG_XPS 1225 CHECK_SKB_FIELD(sender_cpu); 1226 #endif 1227 #ifdef CONFIG_NET_SCHED 1228 CHECK_SKB_FIELD(tc_index); 1229 #endif 1230 1231 } 1232 1233 /* 1234 * You should not add any new code to this function. Add it to 1235 * __copy_skb_header above instead. 1236 */ 1237 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 1238 { 1239 #define C(x) n->x = skb->x 1240 1241 n->next = n->prev = NULL; 1242 n->sk = NULL; 1243 __copy_skb_header(n, skb); 1244 1245 C(len); 1246 C(data_len); 1247 C(mac_len); 1248 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 1249 n->cloned = 1; 1250 n->nohdr = 0; 1251 n->peeked = 0; 1252 C(pfmemalloc); 1253 C(pp_recycle); 1254 n->destructor = NULL; 1255 C(tail); 1256 C(end); 1257 C(head); 1258 C(head_frag); 1259 C(data); 1260 C(truesize); 1261 refcount_set(&n->users, 1); 1262 1263 atomic_inc(&(skb_shinfo(skb)->dataref)); 1264 skb->cloned = 1; 1265 1266 return n; 1267 #undef C 1268 } 1269 1270 /** 1271 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg 1272 * @first: first sk_buff of the msg 1273 */ 1274 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first) 1275 { 1276 struct sk_buff *n; 1277 1278 n = alloc_skb(0, GFP_ATOMIC); 1279 if (!n) 1280 return NULL; 1281 1282 n->len = first->len; 1283 n->data_len = first->len; 1284 n->truesize = first->truesize; 1285 1286 skb_shinfo(n)->frag_list = first; 1287 1288 __copy_skb_header(n, first); 1289 n->destructor = NULL; 1290 1291 return n; 1292 } 1293 EXPORT_SYMBOL_GPL(alloc_skb_for_msg); 1294 1295 /** 1296 * skb_morph - morph one skb into another 1297 * @dst: the skb to receive the contents 1298 * @src: the skb to supply the contents 1299 * 1300 * This is identical to skb_clone except that the target skb is 1301 * supplied by the user. 1302 * 1303 * The target skb is returned upon exit. 1304 */ 1305 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 1306 { 1307 skb_release_all(dst, SKB_CONSUMED); 1308 return __skb_clone(dst, src); 1309 } 1310 EXPORT_SYMBOL_GPL(skb_morph); 1311 1312 int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 1313 { 1314 unsigned long max_pg, num_pg, new_pg, old_pg; 1315 struct user_struct *user; 1316 1317 if (capable(CAP_IPC_LOCK) || !size) 1318 return 0; 1319 1320 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 1321 max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1322 user = mmp->user ? : current_user(); 1323 1324 old_pg = atomic_long_read(&user->locked_vm); 1325 do { 1326 new_pg = old_pg + num_pg; 1327 if (new_pg > max_pg) 1328 return -ENOBUFS; 1329 } while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg)); 1330 1331 if (!mmp->user) { 1332 mmp->user = get_uid(user); 1333 mmp->num_pg = num_pg; 1334 } else { 1335 mmp->num_pg += num_pg; 1336 } 1337 1338 return 0; 1339 } 1340 EXPORT_SYMBOL_GPL(mm_account_pinned_pages); 1341 1342 void mm_unaccount_pinned_pages(struct mmpin *mmp) 1343 { 1344 if (mmp->user) { 1345 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 1346 free_uid(mmp->user); 1347 } 1348 } 1349 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); 1350 1351 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) 1352 { 1353 struct ubuf_info_msgzc *uarg; 1354 struct sk_buff *skb; 1355 1356 WARN_ON_ONCE(!in_task()); 1357 1358 skb = sock_omalloc(sk, 0, GFP_KERNEL); 1359 if (!skb) 1360 return NULL; 1361 1362 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 1363 uarg = (void *)skb->cb; 1364 uarg->mmp.user = NULL; 1365 1366 if (mm_account_pinned_pages(&uarg->mmp, size)) { 1367 kfree_skb(skb); 1368 return NULL; 1369 } 1370 1371 uarg->ubuf.callback = msg_zerocopy_callback; 1372 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 1373 uarg->len = 1; 1374 uarg->bytelen = size; 1375 uarg->zerocopy = 1; 1376 uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; 1377 refcount_set(&uarg->ubuf.refcnt, 1); 1378 sock_hold(sk); 1379 1380 return &uarg->ubuf; 1381 } 1382 1383 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg) 1384 { 1385 return container_of((void *)uarg, struct sk_buff, cb); 1386 } 1387 1388 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, 1389 struct ubuf_info *uarg) 1390 { 1391 if (uarg) { 1392 struct ubuf_info_msgzc *uarg_zc; 1393 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 1394 u32 bytelen, next; 1395 1396 /* there might be non MSG_ZEROCOPY users */ 1397 if (uarg->callback != msg_zerocopy_callback) 1398 return NULL; 1399 1400 /* realloc only when socket is locked (TCP, UDP cork), 1401 * so uarg->len and sk_zckey access is serialized 1402 */ 1403 if (!sock_owned_by_user(sk)) { 1404 WARN_ON_ONCE(1); 1405 return NULL; 1406 } 1407 1408 uarg_zc = uarg_to_msgzc(uarg); 1409 bytelen = uarg_zc->bytelen + size; 1410 if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) { 1411 /* TCP can create new skb to attach new uarg */ 1412 if (sk->sk_type == SOCK_STREAM) 1413 goto new_alloc; 1414 return NULL; 1415 } 1416 1417 next = (u32)atomic_read(&sk->sk_zckey); 1418 if ((u32)(uarg_zc->id + uarg_zc->len) == next) { 1419 if (mm_account_pinned_pages(&uarg_zc->mmp, size)) 1420 return NULL; 1421 uarg_zc->len++; 1422 uarg_zc->bytelen = bytelen; 1423 atomic_set(&sk->sk_zckey, ++next); 1424 1425 /* no extra ref when appending to datagram (MSG_MORE) */ 1426 if (sk->sk_type == SOCK_STREAM) 1427 net_zcopy_get(uarg); 1428 1429 return uarg; 1430 } 1431 } 1432 1433 new_alloc: 1434 return msg_zerocopy_alloc(sk, size); 1435 } 1436 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); 1437 1438 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1439 { 1440 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1441 u32 old_lo, old_hi; 1442 u64 sum_len; 1443 1444 old_lo = serr->ee.ee_info; 1445 old_hi = serr->ee.ee_data; 1446 sum_len = old_hi - old_lo + 1ULL + len; 1447 1448 if (sum_len >= (1ULL << 32)) 1449 return false; 1450 1451 if (lo != old_hi + 1) 1452 return false; 1453 1454 serr->ee.ee_data += len; 1455 return true; 1456 } 1457 1458 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg) 1459 { 1460 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1461 struct sock_exterr_skb *serr; 1462 struct sock *sk = skb->sk; 1463 struct sk_buff_head *q; 1464 unsigned long flags; 1465 bool is_zerocopy; 1466 u32 lo, hi; 1467 u16 len; 1468 1469 mm_unaccount_pinned_pages(&uarg->mmp); 1470 1471 /* if !len, there was only 1 call, and it was aborted 1472 * so do not queue a completion notification 1473 */ 1474 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1475 goto release; 1476 1477 len = uarg->len; 1478 lo = uarg->id; 1479 hi = uarg->id + len - 1; 1480 is_zerocopy = uarg->zerocopy; 1481 1482 serr = SKB_EXT_ERR(skb); 1483 memset(serr, 0, sizeof(*serr)); 1484 serr->ee.ee_errno = 0; 1485 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1486 serr->ee.ee_data = hi; 1487 serr->ee.ee_info = lo; 1488 if (!is_zerocopy) 1489 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1490 1491 q = &sk->sk_error_queue; 1492 spin_lock_irqsave(&q->lock, flags); 1493 tail = skb_peek_tail(q); 1494 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1495 !skb_zerocopy_notify_extend(tail, lo, len)) { 1496 __skb_queue_tail(q, skb); 1497 skb = NULL; 1498 } 1499 spin_unlock_irqrestore(&q->lock, flags); 1500 1501 sk_error_report(sk); 1502 1503 release: 1504 consume_skb(skb); 1505 sock_put(sk); 1506 } 1507 1508 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, 1509 bool success) 1510 { 1511 struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg); 1512 1513 uarg_zc->zerocopy = uarg_zc->zerocopy & success; 1514 1515 if (refcount_dec_and_test(&uarg->refcnt)) 1516 __msg_zerocopy_callback(uarg_zc); 1517 } 1518 EXPORT_SYMBOL_GPL(msg_zerocopy_callback); 1519 1520 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1521 { 1522 struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk; 1523 1524 atomic_dec(&sk->sk_zckey); 1525 uarg_to_msgzc(uarg)->len--; 1526 1527 if (have_uref) 1528 msg_zerocopy_callback(NULL, uarg, true); 1529 } 1530 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); 1531 1532 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1533 struct msghdr *msg, int len, 1534 struct ubuf_info *uarg) 1535 { 1536 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1537 int err, orig_len = skb->len; 1538 1539 /* An skb can only point to one uarg. This edge case happens when 1540 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc. 1541 */ 1542 if (orig_uarg && uarg != orig_uarg) 1543 return -EEXIST; 1544 1545 err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len); 1546 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1547 struct sock *save_sk = skb->sk; 1548 1549 /* Streams do not free skb on error. Reset to prev state. */ 1550 iov_iter_revert(&msg->msg_iter, skb->len - orig_len); 1551 skb->sk = sk; 1552 ___pskb_trim(skb, orig_len); 1553 skb->sk = save_sk; 1554 return err; 1555 } 1556 1557 skb_zcopy_set(skb, uarg, NULL); 1558 return skb->len - orig_len; 1559 } 1560 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1561 1562 void __skb_zcopy_downgrade_managed(struct sk_buff *skb) 1563 { 1564 int i; 1565 1566 skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS; 1567 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1568 skb_frag_ref(skb, i); 1569 } 1570 EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed); 1571 1572 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1573 gfp_t gfp_mask) 1574 { 1575 if (skb_zcopy(orig)) { 1576 if (skb_zcopy(nskb)) { 1577 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1578 if (!gfp_mask) { 1579 WARN_ON_ONCE(1); 1580 return -ENOMEM; 1581 } 1582 if (skb_uarg(nskb) == skb_uarg(orig)) 1583 return 0; 1584 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1585 return -EIO; 1586 } 1587 skb_zcopy_set(nskb, skb_uarg(orig), NULL); 1588 } 1589 return 0; 1590 } 1591 1592 /** 1593 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1594 * @skb: the skb to modify 1595 * @gfp_mask: allocation priority 1596 * 1597 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. 1598 * It will copy all frags into kernel and drop the reference 1599 * to userspace pages. 1600 * 1601 * If this function is called from an interrupt gfp_mask() must be 1602 * %GFP_ATOMIC. 1603 * 1604 * Returns 0 on success or a negative error code on failure 1605 * to allocate kernel memory to copy to. 1606 */ 1607 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 1608 { 1609 int num_frags = skb_shinfo(skb)->nr_frags; 1610 struct page *page, *head = NULL; 1611 int i, new_frags; 1612 u32 d_off; 1613 1614 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 1615 return -EINVAL; 1616 1617 if (!num_frags) 1618 goto release; 1619 1620 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT; 1621 for (i = 0; i < new_frags; i++) { 1622 page = alloc_page(gfp_mask); 1623 if (!page) { 1624 while (head) { 1625 struct page *next = (struct page *)page_private(head); 1626 put_page(head); 1627 head = next; 1628 } 1629 return -ENOMEM; 1630 } 1631 set_page_private(page, (unsigned long)head); 1632 head = page; 1633 } 1634 1635 page = head; 1636 d_off = 0; 1637 for (i = 0; i < num_frags; i++) { 1638 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1639 u32 p_off, p_len, copied; 1640 struct page *p; 1641 u8 *vaddr; 1642 1643 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f), 1644 p, p_off, p_len, copied) { 1645 u32 copy, done = 0; 1646 vaddr = kmap_atomic(p); 1647 1648 while (done < p_len) { 1649 if (d_off == PAGE_SIZE) { 1650 d_off = 0; 1651 page = (struct page *)page_private(page); 1652 } 1653 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done); 1654 memcpy(page_address(page) + d_off, 1655 vaddr + p_off + done, copy); 1656 done += copy; 1657 d_off += copy; 1658 } 1659 kunmap_atomic(vaddr); 1660 } 1661 } 1662 1663 /* skb frags release userspace buffers */ 1664 for (i = 0; i < num_frags; i++) 1665 skb_frag_unref(skb, i); 1666 1667 /* skb frags point to kernel buffers */ 1668 for (i = 0; i < new_frags - 1; i++) { 1669 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE); 1670 head = (struct page *)page_private(head); 1671 } 1672 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); 1673 skb_shinfo(skb)->nr_frags = new_frags; 1674 1675 release: 1676 skb_zcopy_clear(skb, false); 1677 return 0; 1678 } 1679 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 1680 1681 /** 1682 * skb_clone - duplicate an sk_buff 1683 * @skb: buffer to clone 1684 * @gfp_mask: allocation priority 1685 * 1686 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 1687 * copies share the same packet data but not structure. The new 1688 * buffer has a reference count of 1. If the allocation fails the 1689 * function returns %NULL otherwise the new buffer is returned. 1690 * 1691 * If this function is called from an interrupt gfp_mask() must be 1692 * %GFP_ATOMIC. 1693 */ 1694 1695 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 1696 { 1697 struct sk_buff_fclones *fclones = container_of(skb, 1698 struct sk_buff_fclones, 1699 skb1); 1700 struct sk_buff *n; 1701 1702 if (skb_orphan_frags(skb, gfp_mask)) 1703 return NULL; 1704 1705 if (skb->fclone == SKB_FCLONE_ORIG && 1706 refcount_read(&fclones->fclone_ref) == 1) { 1707 n = &fclones->skb2; 1708 refcount_set(&fclones->fclone_ref, 2); 1709 n->fclone = SKB_FCLONE_CLONE; 1710 } else { 1711 if (skb_pfmemalloc(skb)) 1712 gfp_mask |= __GFP_MEMALLOC; 1713 1714 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); 1715 if (!n) 1716 return NULL; 1717 1718 n->fclone = SKB_FCLONE_UNAVAILABLE; 1719 } 1720 1721 return __skb_clone(n, skb); 1722 } 1723 EXPORT_SYMBOL(skb_clone); 1724 1725 void skb_headers_offset_update(struct sk_buff *skb, int off) 1726 { 1727 /* Only adjust this if it actually is csum_start rather than csum */ 1728 if (skb->ip_summed == CHECKSUM_PARTIAL) 1729 skb->csum_start += off; 1730 /* {transport,network,mac}_header and tail are relative to skb->head */ 1731 skb->transport_header += off; 1732 skb->network_header += off; 1733 if (skb_mac_header_was_set(skb)) 1734 skb->mac_header += off; 1735 skb->inner_transport_header += off; 1736 skb->inner_network_header += off; 1737 skb->inner_mac_header += off; 1738 } 1739 EXPORT_SYMBOL(skb_headers_offset_update); 1740 1741 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) 1742 { 1743 __copy_skb_header(new, old); 1744 1745 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 1746 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 1747 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 1748 } 1749 EXPORT_SYMBOL(skb_copy_header); 1750 1751 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 1752 { 1753 if (skb_pfmemalloc(skb)) 1754 return SKB_ALLOC_RX; 1755 return 0; 1756 } 1757 1758 /** 1759 * skb_copy - create private copy of an sk_buff 1760 * @skb: buffer to copy 1761 * @gfp_mask: allocation priority 1762 * 1763 * Make a copy of both an &sk_buff and its data. This is used when the 1764 * caller wishes to modify the data and needs a private copy of the 1765 * data to alter. Returns %NULL on failure or the pointer to the buffer 1766 * on success. The returned buffer has a reference count of 1. 1767 * 1768 * As by-product this function converts non-linear &sk_buff to linear 1769 * one, so that &sk_buff becomes completely private and caller is allowed 1770 * to modify all the data of returned buffer. This means that this 1771 * function is not recommended for use in circumstances when only 1772 * header is going to be modified. Use pskb_copy() instead. 1773 */ 1774 1775 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 1776 { 1777 int headerlen = skb_headroom(skb); 1778 unsigned int size = skb_end_offset(skb) + skb->data_len; 1779 struct sk_buff *n = __alloc_skb(size, gfp_mask, 1780 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 1781 1782 if (!n) 1783 return NULL; 1784 1785 /* Set the data pointer */ 1786 skb_reserve(n, headerlen); 1787 /* Set the tail pointer and length */ 1788 skb_put(n, skb->len); 1789 1790 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); 1791 1792 skb_copy_header(n, skb); 1793 return n; 1794 } 1795 EXPORT_SYMBOL(skb_copy); 1796 1797 /** 1798 * __pskb_copy_fclone - create copy of an sk_buff with private head. 1799 * @skb: buffer to copy 1800 * @headroom: headroom of new skb 1801 * @gfp_mask: allocation priority 1802 * @fclone: if true allocate the copy of the skb from the fclone 1803 * cache instead of the head cache; it is recommended to set this 1804 * to true for the cases where the copy will likely be cloned 1805 * 1806 * Make a copy of both an &sk_buff and part of its data, located 1807 * in header. Fragmented data remain shared. This is used when 1808 * the caller wishes to modify only header of &sk_buff and needs 1809 * private copy of the header to alter. Returns %NULL on failure 1810 * or the pointer to the buffer on success. 1811 * The returned buffer has a reference count of 1. 1812 */ 1813 1814 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 1815 gfp_t gfp_mask, bool fclone) 1816 { 1817 unsigned int size = skb_headlen(skb) + headroom; 1818 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 1819 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 1820 1821 if (!n) 1822 goto out; 1823 1824 /* Set the data pointer */ 1825 skb_reserve(n, headroom); 1826 /* Set the tail pointer and length */ 1827 skb_put(n, skb_headlen(skb)); 1828 /* Copy the bytes */ 1829 skb_copy_from_linear_data(skb, n->data, n->len); 1830 1831 n->truesize += skb->data_len; 1832 n->data_len = skb->data_len; 1833 n->len = skb->len; 1834 1835 if (skb_shinfo(skb)->nr_frags) { 1836 int i; 1837 1838 if (skb_orphan_frags(skb, gfp_mask) || 1839 skb_zerocopy_clone(n, skb, gfp_mask)) { 1840 kfree_skb(n); 1841 n = NULL; 1842 goto out; 1843 } 1844 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1845 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 1846 skb_frag_ref(skb, i); 1847 } 1848 skb_shinfo(n)->nr_frags = i; 1849 } 1850 1851 if (skb_has_frag_list(skb)) { 1852 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 1853 skb_clone_fraglist(n); 1854 } 1855 1856 skb_copy_header(n, skb); 1857 out: 1858 return n; 1859 } 1860 EXPORT_SYMBOL(__pskb_copy_fclone); 1861 1862 /** 1863 * pskb_expand_head - reallocate header of &sk_buff 1864 * @skb: buffer to reallocate 1865 * @nhead: room to add at head 1866 * @ntail: room to add at tail 1867 * @gfp_mask: allocation priority 1868 * 1869 * Expands (or creates identical copy, if @nhead and @ntail are zero) 1870 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 1871 * reference count of 1. Returns zero in the case of success or error, 1872 * if expansion failed. In the last case, &sk_buff is not changed. 1873 * 1874 * All the pointers pointing into skb header may change and must be 1875 * reloaded after call to this function. 1876 */ 1877 1878 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 1879 gfp_t gfp_mask) 1880 { 1881 unsigned int osize = skb_end_offset(skb); 1882 unsigned int size = osize + nhead + ntail; 1883 long off; 1884 u8 *data; 1885 int i; 1886 1887 BUG_ON(nhead < 0); 1888 1889 BUG_ON(skb_shared(skb)); 1890 1891 skb_zcopy_downgrade_managed(skb); 1892 1893 if (skb_pfmemalloc(skb)) 1894 gfp_mask |= __GFP_MEMALLOC; 1895 1896 size = SKB_DATA_ALIGN(size); 1897 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1898 size = kmalloc_size_roundup(size); 1899 data = kmalloc_reserve(size, gfp_mask, NUMA_NO_NODE, NULL); 1900 if (!data) 1901 goto nodata; 1902 size = SKB_WITH_OVERHEAD(size); 1903 1904 /* Copy only real data... and, alas, header. This should be 1905 * optimized for the cases when header is void. 1906 */ 1907 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 1908 1909 memcpy((struct skb_shared_info *)(data + size), 1910 skb_shinfo(skb), 1911 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 1912 1913 /* 1914 * if shinfo is shared we must drop the old head gracefully, but if it 1915 * is not we can just drop the old head and let the existing refcount 1916 * be since all we did is relocate the values 1917 */ 1918 if (skb_cloned(skb)) { 1919 if (skb_orphan_frags(skb, gfp_mask)) 1920 goto nofrags; 1921 if (skb_zcopy(skb)) 1922 refcount_inc(&skb_uarg(skb)->refcnt); 1923 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1924 skb_frag_ref(skb, i); 1925 1926 if (skb_has_frag_list(skb)) 1927 skb_clone_fraglist(skb); 1928 1929 skb_release_data(skb, SKB_CONSUMED); 1930 } else { 1931 skb_free_head(skb); 1932 } 1933 off = (data + nhead) - skb->head; 1934 1935 skb->head = data; 1936 skb->head_frag = 0; 1937 skb->data += off; 1938 1939 skb_set_end_offset(skb, size); 1940 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1941 off = nhead; 1942 #endif 1943 skb->tail += off; 1944 skb_headers_offset_update(skb, nhead); 1945 skb->cloned = 0; 1946 skb->hdr_len = 0; 1947 skb->nohdr = 0; 1948 atomic_set(&skb_shinfo(skb)->dataref, 1); 1949 1950 skb_metadata_clear(skb); 1951 1952 /* It is not generally safe to change skb->truesize. 1953 * For the moment, we really care of rx path, or 1954 * when skb is orphaned (not attached to a socket). 1955 */ 1956 if (!skb->sk || skb->destructor == sock_edemux) 1957 skb->truesize += size - osize; 1958 1959 return 0; 1960 1961 nofrags: 1962 kfree(data); 1963 nodata: 1964 return -ENOMEM; 1965 } 1966 EXPORT_SYMBOL(pskb_expand_head); 1967 1968 /* Make private copy of skb with writable head and some headroom */ 1969 1970 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 1971 { 1972 struct sk_buff *skb2; 1973 int delta = headroom - skb_headroom(skb); 1974 1975 if (delta <= 0) 1976 skb2 = pskb_copy(skb, GFP_ATOMIC); 1977 else { 1978 skb2 = skb_clone(skb, GFP_ATOMIC); 1979 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 1980 GFP_ATOMIC)) { 1981 kfree_skb(skb2); 1982 skb2 = NULL; 1983 } 1984 } 1985 return skb2; 1986 } 1987 EXPORT_SYMBOL(skb_realloc_headroom); 1988 1989 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) 1990 { 1991 unsigned int saved_end_offset, saved_truesize; 1992 struct skb_shared_info *shinfo; 1993 int res; 1994 1995 saved_end_offset = skb_end_offset(skb); 1996 saved_truesize = skb->truesize; 1997 1998 res = pskb_expand_head(skb, 0, 0, pri); 1999 if (res) 2000 return res; 2001 2002 skb->truesize = saved_truesize; 2003 2004 if (likely(skb_end_offset(skb) == saved_end_offset)) 2005 return 0; 2006 2007 shinfo = skb_shinfo(skb); 2008 2009 /* We are about to change back skb->end, 2010 * we need to move skb_shinfo() to its new location. 2011 */ 2012 memmove(skb->head + saved_end_offset, 2013 shinfo, 2014 offsetof(struct skb_shared_info, frags[shinfo->nr_frags])); 2015 2016 skb_set_end_offset(skb, saved_end_offset); 2017 2018 return 0; 2019 } 2020 2021 /** 2022 * skb_expand_head - reallocate header of &sk_buff 2023 * @skb: buffer to reallocate 2024 * @headroom: needed headroom 2025 * 2026 * Unlike skb_realloc_headroom, this one does not allocate a new skb 2027 * if possible; copies skb->sk to new skb as needed 2028 * and frees original skb in case of failures. 2029 * 2030 * It expect increased headroom and generates warning otherwise. 2031 */ 2032 2033 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom) 2034 { 2035 int delta = headroom - skb_headroom(skb); 2036 int osize = skb_end_offset(skb); 2037 struct sock *sk = skb->sk; 2038 2039 if (WARN_ONCE(delta <= 0, 2040 "%s is expecting an increase in the headroom", __func__)) 2041 return skb; 2042 2043 delta = SKB_DATA_ALIGN(delta); 2044 /* pskb_expand_head() might crash, if skb is shared. */ 2045 if (skb_shared(skb) || !is_skb_wmem(skb)) { 2046 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); 2047 2048 if (unlikely(!nskb)) 2049 goto fail; 2050 2051 if (sk) 2052 skb_set_owner_w(nskb, sk); 2053 consume_skb(skb); 2054 skb = nskb; 2055 } 2056 if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC)) 2057 goto fail; 2058 2059 if (sk && is_skb_wmem(skb)) { 2060 delta = skb_end_offset(skb) - osize; 2061 refcount_add(delta, &sk->sk_wmem_alloc); 2062 skb->truesize += delta; 2063 } 2064 return skb; 2065 2066 fail: 2067 kfree_skb(skb); 2068 return NULL; 2069 } 2070 EXPORT_SYMBOL(skb_expand_head); 2071 2072 /** 2073 * skb_copy_expand - copy and expand sk_buff 2074 * @skb: buffer to copy 2075 * @newheadroom: new free bytes at head 2076 * @newtailroom: new free bytes at tail 2077 * @gfp_mask: allocation priority 2078 * 2079 * Make a copy of both an &sk_buff and its data and while doing so 2080 * allocate additional space. 2081 * 2082 * This is used when the caller wishes to modify the data and needs a 2083 * private copy of the data to alter as well as more space for new fields. 2084 * Returns %NULL on failure or the pointer to the buffer 2085 * on success. The returned buffer has a reference count of 1. 2086 * 2087 * You must pass %GFP_ATOMIC as the allocation priority if this function 2088 * is called from an interrupt. 2089 */ 2090 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 2091 int newheadroom, int newtailroom, 2092 gfp_t gfp_mask) 2093 { 2094 /* 2095 * Allocate the copy buffer 2096 */ 2097 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 2098 gfp_mask, skb_alloc_rx_flag(skb), 2099 NUMA_NO_NODE); 2100 int oldheadroom = skb_headroom(skb); 2101 int head_copy_len, head_copy_off; 2102 2103 if (!n) 2104 return NULL; 2105 2106 skb_reserve(n, newheadroom); 2107 2108 /* Set the tail pointer and length */ 2109 skb_put(n, skb->len); 2110 2111 head_copy_len = oldheadroom; 2112 head_copy_off = 0; 2113 if (newheadroom <= head_copy_len) 2114 head_copy_len = newheadroom; 2115 else 2116 head_copy_off = newheadroom - head_copy_len; 2117 2118 /* Copy the linear header and data. */ 2119 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 2120 skb->len + head_copy_len)); 2121 2122 skb_copy_header(n, skb); 2123 2124 skb_headers_offset_update(n, newheadroom - oldheadroom); 2125 2126 return n; 2127 } 2128 EXPORT_SYMBOL(skb_copy_expand); 2129 2130 /** 2131 * __skb_pad - zero pad the tail of an skb 2132 * @skb: buffer to pad 2133 * @pad: space to pad 2134 * @free_on_error: free buffer on error 2135 * 2136 * Ensure that a buffer is followed by a padding area that is zero 2137 * filled. Used by network drivers which may DMA or transfer data 2138 * beyond the buffer end onto the wire. 2139 * 2140 * May return error in out of memory cases. The skb is freed on error 2141 * if @free_on_error is true. 2142 */ 2143 2144 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 2145 { 2146 int err; 2147 int ntail; 2148 2149 /* If the skbuff is non linear tailroom is always zero.. */ 2150 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 2151 memset(skb->data+skb->len, 0, pad); 2152 return 0; 2153 } 2154 2155 ntail = skb->data_len + pad - (skb->end - skb->tail); 2156 if (likely(skb_cloned(skb) || ntail > 0)) { 2157 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 2158 if (unlikely(err)) 2159 goto free_skb; 2160 } 2161 2162 /* FIXME: The use of this function with non-linear skb's really needs 2163 * to be audited. 2164 */ 2165 err = skb_linearize(skb); 2166 if (unlikely(err)) 2167 goto free_skb; 2168 2169 memset(skb->data + skb->len, 0, pad); 2170 return 0; 2171 2172 free_skb: 2173 if (free_on_error) 2174 kfree_skb(skb); 2175 return err; 2176 } 2177 EXPORT_SYMBOL(__skb_pad); 2178 2179 /** 2180 * pskb_put - add data to the tail of a potentially fragmented buffer 2181 * @skb: start of the buffer to use 2182 * @tail: tail fragment of the buffer to use 2183 * @len: amount of data to add 2184 * 2185 * This function extends the used data area of the potentially 2186 * fragmented buffer. @tail must be the last fragment of @skb -- or 2187 * @skb itself. If this would exceed the total buffer size the kernel 2188 * will panic. A pointer to the first byte of the extra data is 2189 * returned. 2190 */ 2191 2192 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 2193 { 2194 if (tail != skb) { 2195 skb->data_len += len; 2196 skb->len += len; 2197 } 2198 return skb_put(tail, len); 2199 } 2200 EXPORT_SYMBOL_GPL(pskb_put); 2201 2202 /** 2203 * skb_put - add data to a buffer 2204 * @skb: buffer to use 2205 * @len: amount of data to add 2206 * 2207 * This function extends the used data area of the buffer. If this would 2208 * exceed the total buffer size the kernel will panic. A pointer to the 2209 * first byte of the extra data is returned. 2210 */ 2211 void *skb_put(struct sk_buff *skb, unsigned int len) 2212 { 2213 void *tmp = skb_tail_pointer(skb); 2214 SKB_LINEAR_ASSERT(skb); 2215 skb->tail += len; 2216 skb->len += len; 2217 if (unlikely(skb->tail > skb->end)) 2218 skb_over_panic(skb, len, __builtin_return_address(0)); 2219 return tmp; 2220 } 2221 EXPORT_SYMBOL(skb_put); 2222 2223 /** 2224 * skb_push - add data to the start of a buffer 2225 * @skb: buffer to use 2226 * @len: amount of data to add 2227 * 2228 * This function extends the used data area of the buffer at the buffer 2229 * start. If this would exceed the total buffer headroom the kernel will 2230 * panic. A pointer to the first byte of the extra data is returned. 2231 */ 2232 void *skb_push(struct sk_buff *skb, unsigned int len) 2233 { 2234 skb->data -= len; 2235 skb->len += len; 2236 if (unlikely(skb->data < skb->head)) 2237 skb_under_panic(skb, len, __builtin_return_address(0)); 2238 return skb->data; 2239 } 2240 EXPORT_SYMBOL(skb_push); 2241 2242 /** 2243 * skb_pull - remove data from the start of a buffer 2244 * @skb: buffer to use 2245 * @len: amount of data to remove 2246 * 2247 * This function removes data from the start of a buffer, returning 2248 * the memory to the headroom. A pointer to the next data in the buffer 2249 * is returned. Once the data has been pulled future pushes will overwrite 2250 * the old data. 2251 */ 2252 void *skb_pull(struct sk_buff *skb, unsigned int len) 2253 { 2254 return skb_pull_inline(skb, len); 2255 } 2256 EXPORT_SYMBOL(skb_pull); 2257 2258 /** 2259 * skb_pull_data - remove data from the start of a buffer returning its 2260 * original position. 2261 * @skb: buffer to use 2262 * @len: amount of data to remove 2263 * 2264 * This function removes data from the start of a buffer, returning 2265 * the memory to the headroom. A pointer to the original data in the buffer 2266 * is returned after checking if there is enough data to pull. Once the 2267 * data has been pulled future pushes will overwrite the old data. 2268 */ 2269 void *skb_pull_data(struct sk_buff *skb, size_t len) 2270 { 2271 void *data = skb->data; 2272 2273 if (skb->len < len) 2274 return NULL; 2275 2276 skb_pull(skb, len); 2277 2278 return data; 2279 } 2280 EXPORT_SYMBOL(skb_pull_data); 2281 2282 /** 2283 * skb_trim - remove end from a buffer 2284 * @skb: buffer to alter 2285 * @len: new length 2286 * 2287 * Cut the length of a buffer down by removing data from the tail. If 2288 * the buffer is already under the length specified it is not modified. 2289 * The skb must be linear. 2290 */ 2291 void skb_trim(struct sk_buff *skb, unsigned int len) 2292 { 2293 if (skb->len > len) 2294 __skb_trim(skb, len); 2295 } 2296 EXPORT_SYMBOL(skb_trim); 2297 2298 /* Trims skb to length len. It can change skb pointers. 2299 */ 2300 2301 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 2302 { 2303 struct sk_buff **fragp; 2304 struct sk_buff *frag; 2305 int offset = skb_headlen(skb); 2306 int nfrags = skb_shinfo(skb)->nr_frags; 2307 int i; 2308 int err; 2309 2310 if (skb_cloned(skb) && 2311 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 2312 return err; 2313 2314 i = 0; 2315 if (offset >= len) 2316 goto drop_pages; 2317 2318 for (; i < nfrags; i++) { 2319 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2320 2321 if (end < len) { 2322 offset = end; 2323 continue; 2324 } 2325 2326 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 2327 2328 drop_pages: 2329 skb_shinfo(skb)->nr_frags = i; 2330 2331 for (; i < nfrags; i++) 2332 skb_frag_unref(skb, i); 2333 2334 if (skb_has_frag_list(skb)) 2335 skb_drop_fraglist(skb); 2336 goto done; 2337 } 2338 2339 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 2340 fragp = &frag->next) { 2341 int end = offset + frag->len; 2342 2343 if (skb_shared(frag)) { 2344 struct sk_buff *nfrag; 2345 2346 nfrag = skb_clone(frag, GFP_ATOMIC); 2347 if (unlikely(!nfrag)) 2348 return -ENOMEM; 2349 2350 nfrag->next = frag->next; 2351 consume_skb(frag); 2352 frag = nfrag; 2353 *fragp = frag; 2354 } 2355 2356 if (end < len) { 2357 offset = end; 2358 continue; 2359 } 2360 2361 if (end > len && 2362 unlikely((err = pskb_trim(frag, len - offset)))) 2363 return err; 2364 2365 if (frag->next) 2366 skb_drop_list(&frag->next); 2367 break; 2368 } 2369 2370 done: 2371 if (len > skb_headlen(skb)) { 2372 skb->data_len -= skb->len - len; 2373 skb->len = len; 2374 } else { 2375 skb->len = len; 2376 skb->data_len = 0; 2377 skb_set_tail_pointer(skb, len); 2378 } 2379 2380 if (!skb->sk || skb->destructor == sock_edemux) 2381 skb_condense(skb); 2382 return 0; 2383 } 2384 EXPORT_SYMBOL(___pskb_trim); 2385 2386 /* Note : use pskb_trim_rcsum() instead of calling this directly 2387 */ 2388 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) 2389 { 2390 if (skb->ip_summed == CHECKSUM_COMPLETE) { 2391 int delta = skb->len - len; 2392 2393 skb->csum = csum_block_sub(skb->csum, 2394 skb_checksum(skb, len, delta, 0), 2395 len); 2396 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 2397 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; 2398 int offset = skb_checksum_start_offset(skb) + skb->csum_offset; 2399 2400 if (offset + sizeof(__sum16) > hdlen) 2401 return -EINVAL; 2402 } 2403 return __pskb_trim(skb, len); 2404 } 2405 EXPORT_SYMBOL(pskb_trim_rcsum_slow); 2406 2407 /** 2408 * __pskb_pull_tail - advance tail of skb header 2409 * @skb: buffer to reallocate 2410 * @delta: number of bytes to advance tail 2411 * 2412 * The function makes a sense only on a fragmented &sk_buff, 2413 * it expands header moving its tail forward and copying necessary 2414 * data from fragmented part. 2415 * 2416 * &sk_buff MUST have reference count of 1. 2417 * 2418 * Returns %NULL (and &sk_buff does not change) if pull failed 2419 * or value of new tail of skb in the case of success. 2420 * 2421 * All the pointers pointing into skb header may change and must be 2422 * reloaded after call to this function. 2423 */ 2424 2425 /* Moves tail of skb head forward, copying data from fragmented part, 2426 * when it is necessary. 2427 * 1. It may fail due to malloc failure. 2428 * 2. It may change skb pointers. 2429 * 2430 * It is pretty complicated. Luckily, it is called only in exceptional cases. 2431 */ 2432 void *__pskb_pull_tail(struct sk_buff *skb, int delta) 2433 { 2434 /* If skb has not enough free space at tail, get new one 2435 * plus 128 bytes for future expansions. If we have enough 2436 * room at tail, reallocate without expansion only if skb is cloned. 2437 */ 2438 int i, k, eat = (skb->tail + delta) - skb->end; 2439 2440 if (eat > 0 || skb_cloned(skb)) { 2441 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 2442 GFP_ATOMIC)) 2443 return NULL; 2444 } 2445 2446 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 2447 skb_tail_pointer(skb), delta)); 2448 2449 /* Optimization: no fragments, no reasons to preestimate 2450 * size of pulled pages. Superb. 2451 */ 2452 if (!skb_has_frag_list(skb)) 2453 goto pull_pages; 2454 2455 /* Estimate size of pulled pages. */ 2456 eat = delta; 2457 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2458 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2459 2460 if (size >= eat) 2461 goto pull_pages; 2462 eat -= size; 2463 } 2464 2465 /* If we need update frag list, we are in troubles. 2466 * Certainly, it is possible to add an offset to skb data, 2467 * but taking into account that pulling is expected to 2468 * be very rare operation, it is worth to fight against 2469 * further bloating skb head and crucify ourselves here instead. 2470 * Pure masohism, indeed. 8)8) 2471 */ 2472 if (eat) { 2473 struct sk_buff *list = skb_shinfo(skb)->frag_list; 2474 struct sk_buff *clone = NULL; 2475 struct sk_buff *insp = NULL; 2476 2477 do { 2478 if (list->len <= eat) { 2479 /* Eaten as whole. */ 2480 eat -= list->len; 2481 list = list->next; 2482 insp = list; 2483 } else { 2484 /* Eaten partially. */ 2485 if (skb_is_gso(skb) && !list->head_frag && 2486 skb_headlen(list)) 2487 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 2488 2489 if (skb_shared(list)) { 2490 /* Sucks! We need to fork list. :-( */ 2491 clone = skb_clone(list, GFP_ATOMIC); 2492 if (!clone) 2493 return NULL; 2494 insp = list->next; 2495 list = clone; 2496 } else { 2497 /* This may be pulled without 2498 * problems. */ 2499 insp = list; 2500 } 2501 if (!pskb_pull(list, eat)) { 2502 kfree_skb(clone); 2503 return NULL; 2504 } 2505 break; 2506 } 2507 } while (eat); 2508 2509 /* Free pulled out fragments. */ 2510 while ((list = skb_shinfo(skb)->frag_list) != insp) { 2511 skb_shinfo(skb)->frag_list = list->next; 2512 consume_skb(list); 2513 } 2514 /* And insert new clone at head. */ 2515 if (clone) { 2516 clone->next = list; 2517 skb_shinfo(skb)->frag_list = clone; 2518 } 2519 } 2520 /* Success! Now we may commit changes to skb data. */ 2521 2522 pull_pages: 2523 eat = delta; 2524 k = 0; 2525 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2526 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2527 2528 if (size <= eat) { 2529 skb_frag_unref(skb, i); 2530 eat -= size; 2531 } else { 2532 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2533 2534 *frag = skb_shinfo(skb)->frags[i]; 2535 if (eat) { 2536 skb_frag_off_add(frag, eat); 2537 skb_frag_size_sub(frag, eat); 2538 if (!i) 2539 goto end; 2540 eat = 0; 2541 } 2542 k++; 2543 } 2544 } 2545 skb_shinfo(skb)->nr_frags = k; 2546 2547 end: 2548 skb->tail += delta; 2549 skb->data_len -= delta; 2550 2551 if (!skb->data_len) 2552 skb_zcopy_clear(skb, false); 2553 2554 return skb_tail_pointer(skb); 2555 } 2556 EXPORT_SYMBOL(__pskb_pull_tail); 2557 2558 /** 2559 * skb_copy_bits - copy bits from skb to kernel buffer 2560 * @skb: source skb 2561 * @offset: offset in source 2562 * @to: destination buffer 2563 * @len: number of bytes to copy 2564 * 2565 * Copy the specified number of bytes from the source skb to the 2566 * destination buffer. 2567 * 2568 * CAUTION ! : 2569 * If its prototype is ever changed, 2570 * check arch/{*}/net/{*}.S files, 2571 * since it is called from BPF assembly code. 2572 */ 2573 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 2574 { 2575 int start = skb_headlen(skb); 2576 struct sk_buff *frag_iter; 2577 int i, copy; 2578 2579 if (offset > (int)skb->len - len) 2580 goto fault; 2581 2582 /* Copy header. */ 2583 if ((copy = start - offset) > 0) { 2584 if (copy > len) 2585 copy = len; 2586 skb_copy_from_linear_data_offset(skb, offset, to, copy); 2587 if ((len -= copy) == 0) 2588 return 0; 2589 offset += copy; 2590 to += copy; 2591 } 2592 2593 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2594 int end; 2595 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2596 2597 WARN_ON(start > offset + len); 2598 2599 end = start + skb_frag_size(f); 2600 if ((copy = end - offset) > 0) { 2601 u32 p_off, p_len, copied; 2602 struct page *p; 2603 u8 *vaddr; 2604 2605 if (copy > len) 2606 copy = len; 2607 2608 skb_frag_foreach_page(f, 2609 skb_frag_off(f) + offset - start, 2610 copy, p, p_off, p_len, copied) { 2611 vaddr = kmap_atomic(p); 2612 memcpy(to + copied, vaddr + p_off, p_len); 2613 kunmap_atomic(vaddr); 2614 } 2615 2616 if ((len -= copy) == 0) 2617 return 0; 2618 offset += copy; 2619 to += copy; 2620 } 2621 start = end; 2622 } 2623 2624 skb_walk_frags(skb, frag_iter) { 2625 int end; 2626 2627 WARN_ON(start > offset + len); 2628 2629 end = start + frag_iter->len; 2630 if ((copy = end - offset) > 0) { 2631 if (copy > len) 2632 copy = len; 2633 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 2634 goto fault; 2635 if ((len -= copy) == 0) 2636 return 0; 2637 offset += copy; 2638 to += copy; 2639 } 2640 start = end; 2641 } 2642 2643 if (!len) 2644 return 0; 2645 2646 fault: 2647 return -EFAULT; 2648 } 2649 EXPORT_SYMBOL(skb_copy_bits); 2650 2651 /* 2652 * Callback from splice_to_pipe(), if we need to release some pages 2653 * at the end of the spd in case we error'ed out in filling the pipe. 2654 */ 2655 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 2656 { 2657 put_page(spd->pages[i]); 2658 } 2659 2660 static struct page *linear_to_page(struct page *page, unsigned int *len, 2661 unsigned int *offset, 2662 struct sock *sk) 2663 { 2664 struct page_frag *pfrag = sk_page_frag(sk); 2665 2666 if (!sk_page_frag_refill(sk, pfrag)) 2667 return NULL; 2668 2669 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 2670 2671 memcpy(page_address(pfrag->page) + pfrag->offset, 2672 page_address(page) + *offset, *len); 2673 *offset = pfrag->offset; 2674 pfrag->offset += *len; 2675 2676 return pfrag->page; 2677 } 2678 2679 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 2680 struct page *page, 2681 unsigned int offset) 2682 { 2683 return spd->nr_pages && 2684 spd->pages[spd->nr_pages - 1] == page && 2685 (spd->partial[spd->nr_pages - 1].offset + 2686 spd->partial[spd->nr_pages - 1].len == offset); 2687 } 2688 2689 /* 2690 * Fill page/offset/length into spd, if it can hold more pages. 2691 */ 2692 static bool spd_fill_page(struct splice_pipe_desc *spd, 2693 struct pipe_inode_info *pipe, struct page *page, 2694 unsigned int *len, unsigned int offset, 2695 bool linear, 2696 struct sock *sk) 2697 { 2698 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 2699 return true; 2700 2701 if (linear) { 2702 page = linear_to_page(page, len, &offset, sk); 2703 if (!page) 2704 return true; 2705 } 2706 if (spd_can_coalesce(spd, page, offset)) { 2707 spd->partial[spd->nr_pages - 1].len += *len; 2708 return false; 2709 } 2710 get_page(page); 2711 spd->pages[spd->nr_pages] = page; 2712 spd->partial[spd->nr_pages].len = *len; 2713 spd->partial[spd->nr_pages].offset = offset; 2714 spd->nr_pages++; 2715 2716 return false; 2717 } 2718 2719 static bool __splice_segment(struct page *page, unsigned int poff, 2720 unsigned int plen, unsigned int *off, 2721 unsigned int *len, 2722 struct splice_pipe_desc *spd, bool linear, 2723 struct sock *sk, 2724 struct pipe_inode_info *pipe) 2725 { 2726 if (!*len) 2727 return true; 2728 2729 /* skip this segment if already processed */ 2730 if (*off >= plen) { 2731 *off -= plen; 2732 return false; 2733 } 2734 2735 /* ignore any bits we already processed */ 2736 poff += *off; 2737 plen -= *off; 2738 *off = 0; 2739 2740 do { 2741 unsigned int flen = min(*len, plen); 2742 2743 if (spd_fill_page(spd, pipe, page, &flen, poff, 2744 linear, sk)) 2745 return true; 2746 poff += flen; 2747 plen -= flen; 2748 *len -= flen; 2749 } while (*len && plen); 2750 2751 return false; 2752 } 2753 2754 /* 2755 * Map linear and fragment data from the skb to spd. It reports true if the 2756 * pipe is full or if we already spliced the requested length. 2757 */ 2758 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 2759 unsigned int *offset, unsigned int *len, 2760 struct splice_pipe_desc *spd, struct sock *sk) 2761 { 2762 int seg; 2763 struct sk_buff *iter; 2764 2765 /* map the linear part : 2766 * If skb->head_frag is set, this 'linear' part is backed by a 2767 * fragment, and if the head is not shared with any clones then 2768 * we can avoid a copy since we own the head portion of this page. 2769 */ 2770 if (__splice_segment(virt_to_page(skb->data), 2771 (unsigned long) skb->data & (PAGE_SIZE - 1), 2772 skb_headlen(skb), 2773 offset, len, spd, 2774 skb_head_is_locked(skb), 2775 sk, pipe)) 2776 return true; 2777 2778 /* 2779 * then map the fragments 2780 */ 2781 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 2782 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 2783 2784 if (__splice_segment(skb_frag_page(f), 2785 skb_frag_off(f), skb_frag_size(f), 2786 offset, len, spd, false, sk, pipe)) 2787 return true; 2788 } 2789 2790 skb_walk_frags(skb, iter) { 2791 if (*offset >= iter->len) { 2792 *offset -= iter->len; 2793 continue; 2794 } 2795 /* __skb_splice_bits() only fails if the output has no room 2796 * left, so no point in going over the frag_list for the error 2797 * case. 2798 */ 2799 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 2800 return true; 2801 } 2802 2803 return false; 2804 } 2805 2806 /* 2807 * Map data from the skb to a pipe. Should handle both the linear part, 2808 * the fragments, and the frag list. 2809 */ 2810 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 2811 struct pipe_inode_info *pipe, unsigned int tlen, 2812 unsigned int flags) 2813 { 2814 struct partial_page partial[MAX_SKB_FRAGS]; 2815 struct page *pages[MAX_SKB_FRAGS]; 2816 struct splice_pipe_desc spd = { 2817 .pages = pages, 2818 .partial = partial, 2819 .nr_pages_max = MAX_SKB_FRAGS, 2820 .ops = &nosteal_pipe_buf_ops, 2821 .spd_release = sock_spd_release, 2822 }; 2823 int ret = 0; 2824 2825 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 2826 2827 if (spd.nr_pages) 2828 ret = splice_to_pipe(pipe, &spd); 2829 2830 return ret; 2831 } 2832 EXPORT_SYMBOL_GPL(skb_splice_bits); 2833 2834 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg, 2835 struct kvec *vec, size_t num, size_t size) 2836 { 2837 struct socket *sock = sk->sk_socket; 2838 2839 if (!sock) 2840 return -EINVAL; 2841 return kernel_sendmsg(sock, msg, vec, num, size); 2842 } 2843 2844 static int sendpage_unlocked(struct sock *sk, struct page *page, int offset, 2845 size_t size, int flags) 2846 { 2847 struct socket *sock = sk->sk_socket; 2848 2849 if (!sock) 2850 return -EINVAL; 2851 return kernel_sendpage(sock, page, offset, size, flags); 2852 } 2853 2854 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg, 2855 struct kvec *vec, size_t num, size_t size); 2856 typedef int (*sendpage_func)(struct sock *sk, struct page *page, int offset, 2857 size_t size, int flags); 2858 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, 2859 int len, sendmsg_func sendmsg, sendpage_func sendpage) 2860 { 2861 unsigned int orig_len = len; 2862 struct sk_buff *head = skb; 2863 unsigned short fragidx; 2864 int slen, ret; 2865 2866 do_frag_list: 2867 2868 /* Deal with head data */ 2869 while (offset < skb_headlen(skb) && len) { 2870 struct kvec kv; 2871 struct msghdr msg; 2872 2873 slen = min_t(int, len, skb_headlen(skb) - offset); 2874 kv.iov_base = skb->data + offset; 2875 kv.iov_len = slen; 2876 memset(&msg, 0, sizeof(msg)); 2877 msg.msg_flags = MSG_DONTWAIT; 2878 2879 ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked, 2880 sendmsg_unlocked, sk, &msg, &kv, 1, slen); 2881 if (ret <= 0) 2882 goto error; 2883 2884 offset += ret; 2885 len -= ret; 2886 } 2887 2888 /* All the data was skb head? */ 2889 if (!len) 2890 goto out; 2891 2892 /* Make offset relative to start of frags */ 2893 offset -= skb_headlen(skb); 2894 2895 /* Find where we are in frag list */ 2896 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2897 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2898 2899 if (offset < skb_frag_size(frag)) 2900 break; 2901 2902 offset -= skb_frag_size(frag); 2903 } 2904 2905 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2906 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2907 2908 slen = min_t(size_t, len, skb_frag_size(frag) - offset); 2909 2910 while (slen) { 2911 ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked, 2912 sendpage_unlocked, sk, 2913 skb_frag_page(frag), 2914 skb_frag_off(frag) + offset, 2915 slen, MSG_DONTWAIT); 2916 if (ret <= 0) 2917 goto error; 2918 2919 len -= ret; 2920 offset += ret; 2921 slen -= ret; 2922 } 2923 2924 offset = 0; 2925 } 2926 2927 if (len) { 2928 /* Process any frag lists */ 2929 2930 if (skb == head) { 2931 if (skb_has_frag_list(skb)) { 2932 skb = skb_shinfo(skb)->frag_list; 2933 goto do_frag_list; 2934 } 2935 } else if (skb->next) { 2936 skb = skb->next; 2937 goto do_frag_list; 2938 } 2939 } 2940 2941 out: 2942 return orig_len - len; 2943 2944 error: 2945 return orig_len == len ? ret : orig_len - len; 2946 } 2947 2948 /* Send skb data on a socket. Socket must be locked. */ 2949 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 2950 int len) 2951 { 2952 return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked, 2953 kernel_sendpage_locked); 2954 } 2955 EXPORT_SYMBOL_GPL(skb_send_sock_locked); 2956 2957 /* Send skb data on a socket. Socket must be unlocked. */ 2958 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) 2959 { 2960 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked, 2961 sendpage_unlocked); 2962 } 2963 2964 /** 2965 * skb_store_bits - store bits from kernel buffer to skb 2966 * @skb: destination buffer 2967 * @offset: offset in destination 2968 * @from: source buffer 2969 * @len: number of bytes to copy 2970 * 2971 * Copy the specified number of bytes from the source buffer to the 2972 * destination skb. This function handles all the messy bits of 2973 * traversing fragment lists and such. 2974 */ 2975 2976 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 2977 { 2978 int start = skb_headlen(skb); 2979 struct sk_buff *frag_iter; 2980 int i, copy; 2981 2982 if (offset > (int)skb->len - len) 2983 goto fault; 2984 2985 if ((copy = start - offset) > 0) { 2986 if (copy > len) 2987 copy = len; 2988 skb_copy_to_linear_data_offset(skb, offset, from, copy); 2989 if ((len -= copy) == 0) 2990 return 0; 2991 offset += copy; 2992 from += copy; 2993 } 2994 2995 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2996 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2997 int end; 2998 2999 WARN_ON(start > offset + len); 3000 3001 end = start + skb_frag_size(frag); 3002 if ((copy = end - offset) > 0) { 3003 u32 p_off, p_len, copied; 3004 struct page *p; 3005 u8 *vaddr; 3006 3007 if (copy > len) 3008 copy = len; 3009 3010 skb_frag_foreach_page(frag, 3011 skb_frag_off(frag) + offset - start, 3012 copy, p, p_off, p_len, copied) { 3013 vaddr = kmap_atomic(p); 3014 memcpy(vaddr + p_off, from + copied, p_len); 3015 kunmap_atomic(vaddr); 3016 } 3017 3018 if ((len -= copy) == 0) 3019 return 0; 3020 offset += copy; 3021 from += copy; 3022 } 3023 start = end; 3024 } 3025 3026 skb_walk_frags(skb, frag_iter) { 3027 int end; 3028 3029 WARN_ON(start > offset + len); 3030 3031 end = start + frag_iter->len; 3032 if ((copy = end - offset) > 0) { 3033 if (copy > len) 3034 copy = len; 3035 if (skb_store_bits(frag_iter, offset - start, 3036 from, copy)) 3037 goto fault; 3038 if ((len -= copy) == 0) 3039 return 0; 3040 offset += copy; 3041 from += copy; 3042 } 3043 start = end; 3044 } 3045 if (!len) 3046 return 0; 3047 3048 fault: 3049 return -EFAULT; 3050 } 3051 EXPORT_SYMBOL(skb_store_bits); 3052 3053 /* Checksum skb data. */ 3054 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 3055 __wsum csum, const struct skb_checksum_ops *ops) 3056 { 3057 int start = skb_headlen(skb); 3058 int i, copy = start - offset; 3059 struct sk_buff *frag_iter; 3060 int pos = 0; 3061 3062 /* Checksum header. */ 3063 if (copy > 0) { 3064 if (copy > len) 3065 copy = len; 3066 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext, 3067 skb->data + offset, copy, csum); 3068 if ((len -= copy) == 0) 3069 return csum; 3070 offset += copy; 3071 pos = copy; 3072 } 3073 3074 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3075 int end; 3076 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3077 3078 WARN_ON(start > offset + len); 3079 3080 end = start + skb_frag_size(frag); 3081 if ((copy = end - offset) > 0) { 3082 u32 p_off, p_len, copied; 3083 struct page *p; 3084 __wsum csum2; 3085 u8 *vaddr; 3086 3087 if (copy > len) 3088 copy = len; 3089 3090 skb_frag_foreach_page(frag, 3091 skb_frag_off(frag) + offset - start, 3092 copy, p, p_off, p_len, copied) { 3093 vaddr = kmap_atomic(p); 3094 csum2 = INDIRECT_CALL_1(ops->update, 3095 csum_partial_ext, 3096 vaddr + p_off, p_len, 0); 3097 kunmap_atomic(vaddr); 3098 csum = INDIRECT_CALL_1(ops->combine, 3099 csum_block_add_ext, csum, 3100 csum2, pos, p_len); 3101 pos += p_len; 3102 } 3103 3104 if (!(len -= copy)) 3105 return csum; 3106 offset += copy; 3107 } 3108 start = end; 3109 } 3110 3111 skb_walk_frags(skb, frag_iter) { 3112 int end; 3113 3114 WARN_ON(start > offset + len); 3115 3116 end = start + frag_iter->len; 3117 if ((copy = end - offset) > 0) { 3118 __wsum csum2; 3119 if (copy > len) 3120 copy = len; 3121 csum2 = __skb_checksum(frag_iter, offset - start, 3122 copy, 0, ops); 3123 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext, 3124 csum, csum2, pos, copy); 3125 if ((len -= copy) == 0) 3126 return csum; 3127 offset += copy; 3128 pos += copy; 3129 } 3130 start = end; 3131 } 3132 BUG_ON(len); 3133 3134 return csum; 3135 } 3136 EXPORT_SYMBOL(__skb_checksum); 3137 3138 __wsum skb_checksum(const struct sk_buff *skb, int offset, 3139 int len, __wsum csum) 3140 { 3141 const struct skb_checksum_ops ops = { 3142 .update = csum_partial_ext, 3143 .combine = csum_block_add_ext, 3144 }; 3145 3146 return __skb_checksum(skb, offset, len, csum, &ops); 3147 } 3148 EXPORT_SYMBOL(skb_checksum); 3149 3150 /* Both of above in one bottle. */ 3151 3152 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 3153 u8 *to, int len) 3154 { 3155 int start = skb_headlen(skb); 3156 int i, copy = start - offset; 3157 struct sk_buff *frag_iter; 3158 int pos = 0; 3159 __wsum csum = 0; 3160 3161 /* Copy header. */ 3162 if (copy > 0) { 3163 if (copy > len) 3164 copy = len; 3165 csum = csum_partial_copy_nocheck(skb->data + offset, to, 3166 copy); 3167 if ((len -= copy) == 0) 3168 return csum; 3169 offset += copy; 3170 to += copy; 3171 pos = copy; 3172 } 3173 3174 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3175 int end; 3176 3177 WARN_ON(start > offset + len); 3178 3179 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3180 if ((copy = end - offset) > 0) { 3181 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3182 u32 p_off, p_len, copied; 3183 struct page *p; 3184 __wsum csum2; 3185 u8 *vaddr; 3186 3187 if (copy > len) 3188 copy = len; 3189 3190 skb_frag_foreach_page(frag, 3191 skb_frag_off(frag) + offset - start, 3192 copy, p, p_off, p_len, copied) { 3193 vaddr = kmap_atomic(p); 3194 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 3195 to + copied, 3196 p_len); 3197 kunmap_atomic(vaddr); 3198 csum = csum_block_add(csum, csum2, pos); 3199 pos += p_len; 3200 } 3201 3202 if (!(len -= copy)) 3203 return csum; 3204 offset += copy; 3205 to += copy; 3206 } 3207 start = end; 3208 } 3209 3210 skb_walk_frags(skb, frag_iter) { 3211 __wsum csum2; 3212 int end; 3213 3214 WARN_ON(start > offset + len); 3215 3216 end = start + frag_iter->len; 3217 if ((copy = end - offset) > 0) { 3218 if (copy > len) 3219 copy = len; 3220 csum2 = skb_copy_and_csum_bits(frag_iter, 3221 offset - start, 3222 to, copy); 3223 csum = csum_block_add(csum, csum2, pos); 3224 if ((len -= copy) == 0) 3225 return csum; 3226 offset += copy; 3227 to += copy; 3228 pos += copy; 3229 } 3230 start = end; 3231 } 3232 BUG_ON(len); 3233 return csum; 3234 } 3235 EXPORT_SYMBOL(skb_copy_and_csum_bits); 3236 3237 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 3238 { 3239 __sum16 sum; 3240 3241 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 3242 /* See comments in __skb_checksum_complete(). */ 3243 if (likely(!sum)) { 3244 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3245 !skb->csum_complete_sw) 3246 netdev_rx_csum_fault(skb->dev, skb); 3247 } 3248 if (!skb_shared(skb)) 3249 skb->csum_valid = !sum; 3250 return sum; 3251 } 3252 EXPORT_SYMBOL(__skb_checksum_complete_head); 3253 3254 /* This function assumes skb->csum already holds pseudo header's checksum, 3255 * which has been changed from the hardware checksum, for example, by 3256 * __skb_checksum_validate_complete(). And, the original skb->csum must 3257 * have been validated unsuccessfully for CHECKSUM_COMPLETE case. 3258 * 3259 * It returns non-zero if the recomputed checksum is still invalid, otherwise 3260 * zero. The new checksum is stored back into skb->csum unless the skb is 3261 * shared. 3262 */ 3263 __sum16 __skb_checksum_complete(struct sk_buff *skb) 3264 { 3265 __wsum csum; 3266 __sum16 sum; 3267 3268 csum = skb_checksum(skb, 0, skb->len, 0); 3269 3270 sum = csum_fold(csum_add(skb->csum, csum)); 3271 /* This check is inverted, because we already knew the hardware 3272 * checksum is invalid before calling this function. So, if the 3273 * re-computed checksum is valid instead, then we have a mismatch 3274 * between the original skb->csum and skb_checksum(). This means either 3275 * the original hardware checksum is incorrect or we screw up skb->csum 3276 * when moving skb->data around. 3277 */ 3278 if (likely(!sum)) { 3279 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3280 !skb->csum_complete_sw) 3281 netdev_rx_csum_fault(skb->dev, skb); 3282 } 3283 3284 if (!skb_shared(skb)) { 3285 /* Save full packet checksum */ 3286 skb->csum = csum; 3287 skb->ip_summed = CHECKSUM_COMPLETE; 3288 skb->csum_complete_sw = 1; 3289 skb->csum_valid = !sum; 3290 } 3291 3292 return sum; 3293 } 3294 EXPORT_SYMBOL(__skb_checksum_complete); 3295 3296 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum) 3297 { 3298 net_warn_ratelimited( 3299 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3300 __func__); 3301 return 0; 3302 } 3303 3304 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2, 3305 int offset, int len) 3306 { 3307 net_warn_ratelimited( 3308 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3309 __func__); 3310 return 0; 3311 } 3312 3313 static const struct skb_checksum_ops default_crc32c_ops = { 3314 .update = warn_crc32c_csum_update, 3315 .combine = warn_crc32c_csum_combine, 3316 }; 3317 3318 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly = 3319 &default_crc32c_ops; 3320 EXPORT_SYMBOL(crc32c_csum_stub); 3321 3322 /** 3323 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 3324 * @from: source buffer 3325 * 3326 * Calculates the amount of linear headroom needed in the 'to' skb passed 3327 * into skb_zerocopy(). 3328 */ 3329 unsigned int 3330 skb_zerocopy_headlen(const struct sk_buff *from) 3331 { 3332 unsigned int hlen = 0; 3333 3334 if (!from->head_frag || 3335 skb_headlen(from) < L1_CACHE_BYTES || 3336 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) { 3337 hlen = skb_headlen(from); 3338 if (!hlen) 3339 hlen = from->len; 3340 } 3341 3342 if (skb_has_frag_list(from)) 3343 hlen = from->len; 3344 3345 return hlen; 3346 } 3347 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 3348 3349 /** 3350 * skb_zerocopy - Zero copy skb to skb 3351 * @to: destination buffer 3352 * @from: source buffer 3353 * @len: number of bytes to copy from source buffer 3354 * @hlen: size of linear headroom in destination buffer 3355 * 3356 * Copies up to `len` bytes from `from` to `to` by creating references 3357 * to the frags in the source buffer. 3358 * 3359 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 3360 * headroom in the `to` buffer. 3361 * 3362 * Return value: 3363 * 0: everything is OK 3364 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 3365 * -EFAULT: skb_copy_bits() found some problem with skb geometry 3366 */ 3367 int 3368 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 3369 { 3370 int i, j = 0; 3371 int plen = 0; /* length of skb->head fragment */ 3372 int ret; 3373 struct page *page; 3374 unsigned int offset; 3375 3376 BUG_ON(!from->head_frag && !hlen); 3377 3378 /* dont bother with small payloads */ 3379 if (len <= skb_tailroom(to)) 3380 return skb_copy_bits(from, 0, skb_put(to, len), len); 3381 3382 if (hlen) { 3383 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 3384 if (unlikely(ret)) 3385 return ret; 3386 len -= hlen; 3387 } else { 3388 plen = min_t(int, skb_headlen(from), len); 3389 if (plen) { 3390 page = virt_to_head_page(from->head); 3391 offset = from->data - (unsigned char *)page_address(page); 3392 __skb_fill_page_desc(to, 0, page, offset, plen); 3393 get_page(page); 3394 j = 1; 3395 len -= plen; 3396 } 3397 } 3398 3399 skb_len_add(to, len + plen); 3400 3401 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 3402 skb_tx_error(from); 3403 return -ENOMEM; 3404 } 3405 skb_zerocopy_clone(to, from, GFP_ATOMIC); 3406 3407 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 3408 int size; 3409 3410 if (!len) 3411 break; 3412 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 3413 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]), 3414 len); 3415 skb_frag_size_set(&skb_shinfo(to)->frags[j], size); 3416 len -= size; 3417 skb_frag_ref(to, j); 3418 j++; 3419 } 3420 skb_shinfo(to)->nr_frags = j; 3421 3422 return 0; 3423 } 3424 EXPORT_SYMBOL_GPL(skb_zerocopy); 3425 3426 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 3427 { 3428 __wsum csum; 3429 long csstart; 3430 3431 if (skb->ip_summed == CHECKSUM_PARTIAL) 3432 csstart = skb_checksum_start_offset(skb); 3433 else 3434 csstart = skb_headlen(skb); 3435 3436 BUG_ON(csstart > skb_headlen(skb)); 3437 3438 skb_copy_from_linear_data(skb, to, csstart); 3439 3440 csum = 0; 3441 if (csstart != skb->len) 3442 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 3443 skb->len - csstart); 3444 3445 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3446 long csstuff = csstart + skb->csum_offset; 3447 3448 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 3449 } 3450 } 3451 EXPORT_SYMBOL(skb_copy_and_csum_dev); 3452 3453 /** 3454 * skb_dequeue - remove from the head of the queue 3455 * @list: list to dequeue from 3456 * 3457 * Remove the head of the list. The list lock is taken so the function 3458 * may be used safely with other locking list functions. The head item is 3459 * returned or %NULL if the list is empty. 3460 */ 3461 3462 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 3463 { 3464 unsigned long flags; 3465 struct sk_buff *result; 3466 3467 spin_lock_irqsave(&list->lock, flags); 3468 result = __skb_dequeue(list); 3469 spin_unlock_irqrestore(&list->lock, flags); 3470 return result; 3471 } 3472 EXPORT_SYMBOL(skb_dequeue); 3473 3474 /** 3475 * skb_dequeue_tail - remove from the tail of the queue 3476 * @list: list to dequeue from 3477 * 3478 * Remove the tail of the list. The list lock is taken so the function 3479 * may be used safely with other locking list functions. The tail item is 3480 * returned or %NULL if the list is empty. 3481 */ 3482 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 3483 { 3484 unsigned long flags; 3485 struct sk_buff *result; 3486 3487 spin_lock_irqsave(&list->lock, flags); 3488 result = __skb_dequeue_tail(list); 3489 spin_unlock_irqrestore(&list->lock, flags); 3490 return result; 3491 } 3492 EXPORT_SYMBOL(skb_dequeue_tail); 3493 3494 /** 3495 * skb_queue_purge - empty a list 3496 * @list: list to empty 3497 * 3498 * Delete all buffers on an &sk_buff list. Each buffer is removed from 3499 * the list and one reference dropped. This function takes the list 3500 * lock and is atomic with respect to other list locking functions. 3501 */ 3502 void skb_queue_purge(struct sk_buff_head *list) 3503 { 3504 struct sk_buff *skb; 3505 while ((skb = skb_dequeue(list)) != NULL) 3506 kfree_skb(skb); 3507 } 3508 EXPORT_SYMBOL(skb_queue_purge); 3509 3510 /** 3511 * skb_rbtree_purge - empty a skb rbtree 3512 * @root: root of the rbtree to empty 3513 * Return value: the sum of truesizes of all purged skbs. 3514 * 3515 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 3516 * the list and one reference dropped. This function does not take 3517 * any lock. Synchronization should be handled by the caller (e.g., TCP 3518 * out-of-order queue is protected by the socket lock). 3519 */ 3520 unsigned int skb_rbtree_purge(struct rb_root *root) 3521 { 3522 struct rb_node *p = rb_first(root); 3523 unsigned int sum = 0; 3524 3525 while (p) { 3526 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 3527 3528 p = rb_next(p); 3529 rb_erase(&skb->rbnode, root); 3530 sum += skb->truesize; 3531 kfree_skb(skb); 3532 } 3533 return sum; 3534 } 3535 3536 /** 3537 * skb_queue_head - queue a buffer at the list head 3538 * @list: list to use 3539 * @newsk: buffer to queue 3540 * 3541 * Queue a buffer at the start of the list. This function takes the 3542 * list lock and can be used safely with other locking &sk_buff functions 3543 * safely. 3544 * 3545 * A buffer cannot be placed on two lists at the same time. 3546 */ 3547 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 3548 { 3549 unsigned long flags; 3550 3551 spin_lock_irqsave(&list->lock, flags); 3552 __skb_queue_head(list, newsk); 3553 spin_unlock_irqrestore(&list->lock, flags); 3554 } 3555 EXPORT_SYMBOL(skb_queue_head); 3556 3557 /** 3558 * skb_queue_tail - queue a buffer at the list tail 3559 * @list: list to use 3560 * @newsk: buffer to queue 3561 * 3562 * Queue a buffer at the tail of the list. This function takes the 3563 * list lock and can be used safely with other locking &sk_buff functions 3564 * safely. 3565 * 3566 * A buffer cannot be placed on two lists at the same time. 3567 */ 3568 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 3569 { 3570 unsigned long flags; 3571 3572 spin_lock_irqsave(&list->lock, flags); 3573 __skb_queue_tail(list, newsk); 3574 spin_unlock_irqrestore(&list->lock, flags); 3575 } 3576 EXPORT_SYMBOL(skb_queue_tail); 3577 3578 /** 3579 * skb_unlink - remove a buffer from a list 3580 * @skb: buffer to remove 3581 * @list: list to use 3582 * 3583 * Remove a packet from a list. The list locks are taken and this 3584 * function is atomic with respect to other list locked calls 3585 * 3586 * You must know what list the SKB is on. 3587 */ 3588 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 3589 { 3590 unsigned long flags; 3591 3592 spin_lock_irqsave(&list->lock, flags); 3593 __skb_unlink(skb, list); 3594 spin_unlock_irqrestore(&list->lock, flags); 3595 } 3596 EXPORT_SYMBOL(skb_unlink); 3597 3598 /** 3599 * skb_append - append a buffer 3600 * @old: buffer to insert after 3601 * @newsk: buffer to insert 3602 * @list: list to use 3603 * 3604 * Place a packet after a given packet in a list. The list locks are taken 3605 * and this function is atomic with respect to other list locked calls. 3606 * A buffer cannot be placed on two lists at the same time. 3607 */ 3608 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 3609 { 3610 unsigned long flags; 3611 3612 spin_lock_irqsave(&list->lock, flags); 3613 __skb_queue_after(list, old, newsk); 3614 spin_unlock_irqrestore(&list->lock, flags); 3615 } 3616 EXPORT_SYMBOL(skb_append); 3617 3618 static inline void skb_split_inside_header(struct sk_buff *skb, 3619 struct sk_buff* skb1, 3620 const u32 len, const int pos) 3621 { 3622 int i; 3623 3624 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 3625 pos - len); 3626 /* And move data appendix as is. */ 3627 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 3628 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 3629 3630 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 3631 skb_shinfo(skb)->nr_frags = 0; 3632 skb1->data_len = skb->data_len; 3633 skb1->len += skb1->data_len; 3634 skb->data_len = 0; 3635 skb->len = len; 3636 skb_set_tail_pointer(skb, len); 3637 } 3638 3639 static inline void skb_split_no_header(struct sk_buff *skb, 3640 struct sk_buff* skb1, 3641 const u32 len, int pos) 3642 { 3643 int i, k = 0; 3644 const int nfrags = skb_shinfo(skb)->nr_frags; 3645 3646 skb_shinfo(skb)->nr_frags = 0; 3647 skb1->len = skb1->data_len = skb->len - len; 3648 skb->len = len; 3649 skb->data_len = len - pos; 3650 3651 for (i = 0; i < nfrags; i++) { 3652 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 3653 3654 if (pos + size > len) { 3655 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 3656 3657 if (pos < len) { 3658 /* Split frag. 3659 * We have two variants in this case: 3660 * 1. Move all the frag to the second 3661 * part, if it is possible. F.e. 3662 * this approach is mandatory for TUX, 3663 * where splitting is expensive. 3664 * 2. Split is accurately. We make this. 3665 */ 3666 skb_frag_ref(skb, i); 3667 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos); 3668 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 3669 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 3670 skb_shinfo(skb)->nr_frags++; 3671 } 3672 k++; 3673 } else 3674 skb_shinfo(skb)->nr_frags++; 3675 pos += size; 3676 } 3677 skb_shinfo(skb1)->nr_frags = k; 3678 } 3679 3680 /** 3681 * skb_split - Split fragmented skb to two parts at length len. 3682 * @skb: the buffer to split 3683 * @skb1: the buffer to receive the second part 3684 * @len: new length for skb 3685 */ 3686 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 3687 { 3688 int pos = skb_headlen(skb); 3689 const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY; 3690 3691 skb_zcopy_downgrade_managed(skb); 3692 3693 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags; 3694 skb_zerocopy_clone(skb1, skb, 0); 3695 if (len < pos) /* Split line is inside header. */ 3696 skb_split_inside_header(skb, skb1, len, pos); 3697 else /* Second chunk has no header, nothing to copy. */ 3698 skb_split_no_header(skb, skb1, len, pos); 3699 } 3700 EXPORT_SYMBOL(skb_split); 3701 3702 /* Shifting from/to a cloned skb is a no-go. 3703 * 3704 * Caller cannot keep skb_shinfo related pointers past calling here! 3705 */ 3706 static int skb_prepare_for_shift(struct sk_buff *skb) 3707 { 3708 return skb_unclone_keeptruesize(skb, GFP_ATOMIC); 3709 } 3710 3711 /** 3712 * skb_shift - Shifts paged data partially from skb to another 3713 * @tgt: buffer into which tail data gets added 3714 * @skb: buffer from which the paged data comes from 3715 * @shiftlen: shift up to this many bytes 3716 * 3717 * Attempts to shift up to shiftlen worth of bytes, which may be less than 3718 * the length of the skb, from skb to tgt. Returns number bytes shifted. 3719 * It's up to caller to free skb if everything was shifted. 3720 * 3721 * If @tgt runs out of frags, the whole operation is aborted. 3722 * 3723 * Skb cannot include anything else but paged data while tgt is allowed 3724 * to have non-paged data as well. 3725 * 3726 * TODO: full sized shift could be optimized but that would need 3727 * specialized skb free'er to handle frags without up-to-date nr_frags. 3728 */ 3729 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 3730 { 3731 int from, to, merge, todo; 3732 skb_frag_t *fragfrom, *fragto; 3733 3734 BUG_ON(shiftlen > skb->len); 3735 3736 if (skb_headlen(skb)) 3737 return 0; 3738 if (skb_zcopy(tgt) || skb_zcopy(skb)) 3739 return 0; 3740 3741 todo = shiftlen; 3742 from = 0; 3743 to = skb_shinfo(tgt)->nr_frags; 3744 fragfrom = &skb_shinfo(skb)->frags[from]; 3745 3746 /* Actual merge is delayed until the point when we know we can 3747 * commit all, so that we don't have to undo partial changes 3748 */ 3749 if (!to || 3750 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 3751 skb_frag_off(fragfrom))) { 3752 merge = -1; 3753 } else { 3754 merge = to - 1; 3755 3756 todo -= skb_frag_size(fragfrom); 3757 if (todo < 0) { 3758 if (skb_prepare_for_shift(skb) || 3759 skb_prepare_for_shift(tgt)) 3760 return 0; 3761 3762 /* All previous frag pointers might be stale! */ 3763 fragfrom = &skb_shinfo(skb)->frags[from]; 3764 fragto = &skb_shinfo(tgt)->frags[merge]; 3765 3766 skb_frag_size_add(fragto, shiftlen); 3767 skb_frag_size_sub(fragfrom, shiftlen); 3768 skb_frag_off_add(fragfrom, shiftlen); 3769 3770 goto onlymerged; 3771 } 3772 3773 from++; 3774 } 3775 3776 /* Skip full, not-fitting skb to avoid expensive operations */ 3777 if ((shiftlen == skb->len) && 3778 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 3779 return 0; 3780 3781 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 3782 return 0; 3783 3784 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 3785 if (to == MAX_SKB_FRAGS) 3786 return 0; 3787 3788 fragfrom = &skb_shinfo(skb)->frags[from]; 3789 fragto = &skb_shinfo(tgt)->frags[to]; 3790 3791 if (todo >= skb_frag_size(fragfrom)) { 3792 *fragto = *fragfrom; 3793 todo -= skb_frag_size(fragfrom); 3794 from++; 3795 to++; 3796 3797 } else { 3798 __skb_frag_ref(fragfrom); 3799 skb_frag_page_copy(fragto, fragfrom); 3800 skb_frag_off_copy(fragto, fragfrom); 3801 skb_frag_size_set(fragto, todo); 3802 3803 skb_frag_off_add(fragfrom, todo); 3804 skb_frag_size_sub(fragfrom, todo); 3805 todo = 0; 3806 3807 to++; 3808 break; 3809 } 3810 } 3811 3812 /* Ready to "commit" this state change to tgt */ 3813 skb_shinfo(tgt)->nr_frags = to; 3814 3815 if (merge >= 0) { 3816 fragfrom = &skb_shinfo(skb)->frags[0]; 3817 fragto = &skb_shinfo(tgt)->frags[merge]; 3818 3819 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 3820 __skb_frag_unref(fragfrom, skb->pp_recycle); 3821 } 3822 3823 /* Reposition in the original skb */ 3824 to = 0; 3825 while (from < skb_shinfo(skb)->nr_frags) 3826 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 3827 skb_shinfo(skb)->nr_frags = to; 3828 3829 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 3830 3831 onlymerged: 3832 /* Most likely the tgt won't ever need its checksum anymore, skb on 3833 * the other hand might need it if it needs to be resent 3834 */ 3835 tgt->ip_summed = CHECKSUM_PARTIAL; 3836 skb->ip_summed = CHECKSUM_PARTIAL; 3837 3838 skb_len_add(skb, -shiftlen); 3839 skb_len_add(tgt, shiftlen); 3840 3841 return shiftlen; 3842 } 3843 3844 /** 3845 * skb_prepare_seq_read - Prepare a sequential read of skb data 3846 * @skb: the buffer to read 3847 * @from: lower offset of data to be read 3848 * @to: upper offset of data to be read 3849 * @st: state variable 3850 * 3851 * Initializes the specified state variable. Must be called before 3852 * invoking skb_seq_read() for the first time. 3853 */ 3854 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 3855 unsigned int to, struct skb_seq_state *st) 3856 { 3857 st->lower_offset = from; 3858 st->upper_offset = to; 3859 st->root_skb = st->cur_skb = skb; 3860 st->frag_idx = st->stepped_offset = 0; 3861 st->frag_data = NULL; 3862 st->frag_off = 0; 3863 } 3864 EXPORT_SYMBOL(skb_prepare_seq_read); 3865 3866 /** 3867 * skb_seq_read - Sequentially read skb data 3868 * @consumed: number of bytes consumed by the caller so far 3869 * @data: destination pointer for data to be returned 3870 * @st: state variable 3871 * 3872 * Reads a block of skb data at @consumed relative to the 3873 * lower offset specified to skb_prepare_seq_read(). Assigns 3874 * the head of the data block to @data and returns the length 3875 * of the block or 0 if the end of the skb data or the upper 3876 * offset has been reached. 3877 * 3878 * The caller is not required to consume all of the data 3879 * returned, i.e. @consumed is typically set to the number 3880 * of bytes already consumed and the next call to 3881 * skb_seq_read() will return the remaining part of the block. 3882 * 3883 * Note 1: The size of each block of data returned can be arbitrary, 3884 * this limitation is the cost for zerocopy sequential 3885 * reads of potentially non linear data. 3886 * 3887 * Note 2: Fragment lists within fragments are not implemented 3888 * at the moment, state->root_skb could be replaced with 3889 * a stack for this purpose. 3890 */ 3891 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 3892 struct skb_seq_state *st) 3893 { 3894 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 3895 skb_frag_t *frag; 3896 3897 if (unlikely(abs_offset >= st->upper_offset)) { 3898 if (st->frag_data) { 3899 kunmap_atomic(st->frag_data); 3900 st->frag_data = NULL; 3901 } 3902 return 0; 3903 } 3904 3905 next_skb: 3906 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 3907 3908 if (abs_offset < block_limit && !st->frag_data) { 3909 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 3910 return block_limit - abs_offset; 3911 } 3912 3913 if (st->frag_idx == 0 && !st->frag_data) 3914 st->stepped_offset += skb_headlen(st->cur_skb); 3915 3916 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 3917 unsigned int pg_idx, pg_off, pg_sz; 3918 3919 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 3920 3921 pg_idx = 0; 3922 pg_off = skb_frag_off(frag); 3923 pg_sz = skb_frag_size(frag); 3924 3925 if (skb_frag_must_loop(skb_frag_page(frag))) { 3926 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT; 3927 pg_off = offset_in_page(pg_off + st->frag_off); 3928 pg_sz = min_t(unsigned int, pg_sz - st->frag_off, 3929 PAGE_SIZE - pg_off); 3930 } 3931 3932 block_limit = pg_sz + st->stepped_offset; 3933 if (abs_offset < block_limit) { 3934 if (!st->frag_data) 3935 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx); 3936 3937 *data = (u8 *)st->frag_data + pg_off + 3938 (abs_offset - st->stepped_offset); 3939 3940 return block_limit - abs_offset; 3941 } 3942 3943 if (st->frag_data) { 3944 kunmap_atomic(st->frag_data); 3945 st->frag_data = NULL; 3946 } 3947 3948 st->stepped_offset += pg_sz; 3949 st->frag_off += pg_sz; 3950 if (st->frag_off == skb_frag_size(frag)) { 3951 st->frag_off = 0; 3952 st->frag_idx++; 3953 } 3954 } 3955 3956 if (st->frag_data) { 3957 kunmap_atomic(st->frag_data); 3958 st->frag_data = NULL; 3959 } 3960 3961 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 3962 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 3963 st->frag_idx = 0; 3964 goto next_skb; 3965 } else if (st->cur_skb->next) { 3966 st->cur_skb = st->cur_skb->next; 3967 st->frag_idx = 0; 3968 goto next_skb; 3969 } 3970 3971 return 0; 3972 } 3973 EXPORT_SYMBOL(skb_seq_read); 3974 3975 /** 3976 * skb_abort_seq_read - Abort a sequential read of skb data 3977 * @st: state variable 3978 * 3979 * Must be called if skb_seq_read() was not called until it 3980 * returned 0. 3981 */ 3982 void skb_abort_seq_read(struct skb_seq_state *st) 3983 { 3984 if (st->frag_data) 3985 kunmap_atomic(st->frag_data); 3986 } 3987 EXPORT_SYMBOL(skb_abort_seq_read); 3988 3989 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 3990 3991 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 3992 struct ts_config *conf, 3993 struct ts_state *state) 3994 { 3995 return skb_seq_read(offset, text, TS_SKB_CB(state)); 3996 } 3997 3998 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 3999 { 4000 skb_abort_seq_read(TS_SKB_CB(state)); 4001 } 4002 4003 /** 4004 * skb_find_text - Find a text pattern in skb data 4005 * @skb: the buffer to look in 4006 * @from: search offset 4007 * @to: search limit 4008 * @config: textsearch configuration 4009 * 4010 * Finds a pattern in the skb data according to the specified 4011 * textsearch configuration. Use textsearch_next() to retrieve 4012 * subsequent occurrences of the pattern. Returns the offset 4013 * to the first occurrence or UINT_MAX if no match was found. 4014 */ 4015 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 4016 unsigned int to, struct ts_config *config) 4017 { 4018 struct ts_state state; 4019 unsigned int ret; 4020 4021 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb)); 4022 4023 config->get_next_block = skb_ts_get_next_block; 4024 config->finish = skb_ts_finish; 4025 4026 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 4027 4028 ret = textsearch_find(config, &state); 4029 return (ret <= to - from ? ret : UINT_MAX); 4030 } 4031 EXPORT_SYMBOL(skb_find_text); 4032 4033 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 4034 int offset, size_t size) 4035 { 4036 int i = skb_shinfo(skb)->nr_frags; 4037 4038 if (skb_can_coalesce(skb, i, page, offset)) { 4039 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 4040 } else if (i < MAX_SKB_FRAGS) { 4041 skb_zcopy_downgrade_managed(skb); 4042 get_page(page); 4043 skb_fill_page_desc_noacc(skb, i, page, offset, size); 4044 } else { 4045 return -EMSGSIZE; 4046 } 4047 4048 return 0; 4049 } 4050 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 4051 4052 /** 4053 * skb_pull_rcsum - pull skb and update receive checksum 4054 * @skb: buffer to update 4055 * @len: length of data pulled 4056 * 4057 * This function performs an skb_pull on the packet and updates 4058 * the CHECKSUM_COMPLETE checksum. It should be used on 4059 * receive path processing instead of skb_pull unless you know 4060 * that the checksum difference is zero (e.g., a valid IP header) 4061 * or you are setting ip_summed to CHECKSUM_NONE. 4062 */ 4063 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 4064 { 4065 unsigned char *data = skb->data; 4066 4067 BUG_ON(len > skb->len); 4068 __skb_pull(skb, len); 4069 skb_postpull_rcsum(skb, data, len); 4070 return skb->data; 4071 } 4072 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 4073 4074 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb) 4075 { 4076 skb_frag_t head_frag; 4077 struct page *page; 4078 4079 page = virt_to_head_page(frag_skb->head); 4080 __skb_frag_set_page(&head_frag, page); 4081 skb_frag_off_set(&head_frag, frag_skb->data - 4082 (unsigned char *)page_address(page)); 4083 skb_frag_size_set(&head_frag, skb_headlen(frag_skb)); 4084 return head_frag; 4085 } 4086 4087 struct sk_buff *skb_segment_list(struct sk_buff *skb, 4088 netdev_features_t features, 4089 unsigned int offset) 4090 { 4091 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list; 4092 unsigned int tnl_hlen = skb_tnl_header_len(skb); 4093 unsigned int delta_truesize = 0; 4094 unsigned int delta_len = 0; 4095 struct sk_buff *tail = NULL; 4096 struct sk_buff *nskb, *tmp; 4097 int len_diff, err; 4098 4099 skb_push(skb, -skb_network_offset(skb) + offset); 4100 4101 skb_shinfo(skb)->frag_list = NULL; 4102 4103 while (list_skb) { 4104 nskb = list_skb; 4105 list_skb = list_skb->next; 4106 4107 err = 0; 4108 delta_truesize += nskb->truesize; 4109 if (skb_shared(nskb)) { 4110 tmp = skb_clone(nskb, GFP_ATOMIC); 4111 if (tmp) { 4112 consume_skb(nskb); 4113 nskb = tmp; 4114 err = skb_unclone(nskb, GFP_ATOMIC); 4115 } else { 4116 err = -ENOMEM; 4117 } 4118 } 4119 4120 if (!tail) 4121 skb->next = nskb; 4122 else 4123 tail->next = nskb; 4124 4125 if (unlikely(err)) { 4126 nskb->next = list_skb; 4127 goto err_linearize; 4128 } 4129 4130 tail = nskb; 4131 4132 delta_len += nskb->len; 4133 4134 skb_push(nskb, -skb_network_offset(nskb) + offset); 4135 4136 skb_release_head_state(nskb); 4137 len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb); 4138 __copy_skb_header(nskb, skb); 4139 4140 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb)); 4141 nskb->transport_header += len_diff; 4142 skb_copy_from_linear_data_offset(skb, -tnl_hlen, 4143 nskb->data - tnl_hlen, 4144 offset + tnl_hlen); 4145 4146 if (skb_needs_linearize(nskb, features) && 4147 __skb_linearize(nskb)) 4148 goto err_linearize; 4149 } 4150 4151 skb->truesize = skb->truesize - delta_truesize; 4152 skb->data_len = skb->data_len - delta_len; 4153 skb->len = skb->len - delta_len; 4154 4155 skb_gso_reset(skb); 4156 4157 skb->prev = tail; 4158 4159 if (skb_needs_linearize(skb, features) && 4160 __skb_linearize(skb)) 4161 goto err_linearize; 4162 4163 skb_get(skb); 4164 4165 return skb; 4166 4167 err_linearize: 4168 kfree_skb_list(skb->next); 4169 skb->next = NULL; 4170 return ERR_PTR(-ENOMEM); 4171 } 4172 EXPORT_SYMBOL_GPL(skb_segment_list); 4173 4174 /** 4175 * skb_segment - Perform protocol segmentation on skb. 4176 * @head_skb: buffer to segment 4177 * @features: features for the output path (see dev->features) 4178 * 4179 * This function performs segmentation on the given skb. It returns 4180 * a pointer to the first in a list of new skbs for the segments. 4181 * In case of error it returns ERR_PTR(err). 4182 */ 4183 struct sk_buff *skb_segment(struct sk_buff *head_skb, 4184 netdev_features_t features) 4185 { 4186 struct sk_buff *segs = NULL; 4187 struct sk_buff *tail = NULL; 4188 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 4189 skb_frag_t *frag = skb_shinfo(head_skb)->frags; 4190 unsigned int mss = skb_shinfo(head_skb)->gso_size; 4191 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 4192 struct sk_buff *frag_skb = head_skb; 4193 unsigned int offset = doffset; 4194 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 4195 unsigned int partial_segs = 0; 4196 unsigned int headroom; 4197 unsigned int len = head_skb->len; 4198 __be16 proto; 4199 bool csum, sg; 4200 int nfrags = skb_shinfo(head_skb)->nr_frags; 4201 int err = -ENOMEM; 4202 int i = 0; 4203 int pos; 4204 4205 if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) && 4206 mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) { 4207 struct sk_buff *check_skb; 4208 4209 for (check_skb = list_skb; check_skb; check_skb = check_skb->next) { 4210 if (skb_headlen(check_skb) && !check_skb->head_frag) { 4211 /* gso_size is untrusted, and we have a frag_list with 4212 * a linear non head_frag item. 4213 * 4214 * If head_skb's headlen does not fit requested gso_size, 4215 * it means that the frag_list members do NOT terminate 4216 * on exact gso_size boundaries. Hence we cannot perform 4217 * skb_frag_t page sharing. Therefore we must fallback to 4218 * copying the frag_list skbs; we do so by disabling SG. 4219 */ 4220 features &= ~NETIF_F_SG; 4221 break; 4222 } 4223 } 4224 } 4225 4226 __skb_push(head_skb, doffset); 4227 proto = skb_network_protocol(head_skb, NULL); 4228 if (unlikely(!proto)) 4229 return ERR_PTR(-EINVAL); 4230 4231 sg = !!(features & NETIF_F_SG); 4232 csum = !!can_checksum_protocol(features, proto); 4233 4234 if (sg && csum && (mss != GSO_BY_FRAGS)) { 4235 if (!(features & NETIF_F_GSO_PARTIAL)) { 4236 struct sk_buff *iter; 4237 unsigned int frag_len; 4238 4239 if (!list_skb || 4240 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 4241 goto normal; 4242 4243 /* If we get here then all the required 4244 * GSO features except frag_list are supported. 4245 * Try to split the SKB to multiple GSO SKBs 4246 * with no frag_list. 4247 * Currently we can do that only when the buffers don't 4248 * have a linear part and all the buffers except 4249 * the last are of the same length. 4250 */ 4251 frag_len = list_skb->len; 4252 skb_walk_frags(head_skb, iter) { 4253 if (frag_len != iter->len && iter->next) 4254 goto normal; 4255 if (skb_headlen(iter) && !iter->head_frag) 4256 goto normal; 4257 4258 len -= iter->len; 4259 } 4260 4261 if (len != frag_len) 4262 goto normal; 4263 } 4264 4265 /* GSO partial only requires that we trim off any excess that 4266 * doesn't fit into an MSS sized block, so take care of that 4267 * now. 4268 */ 4269 partial_segs = len / mss; 4270 if (partial_segs > 1) 4271 mss *= partial_segs; 4272 else 4273 partial_segs = 0; 4274 } 4275 4276 normal: 4277 headroom = skb_headroom(head_skb); 4278 pos = skb_headlen(head_skb); 4279 4280 do { 4281 struct sk_buff *nskb; 4282 skb_frag_t *nskb_frag; 4283 int hsize; 4284 int size; 4285 4286 if (unlikely(mss == GSO_BY_FRAGS)) { 4287 len = list_skb->len; 4288 } else { 4289 len = head_skb->len - offset; 4290 if (len > mss) 4291 len = mss; 4292 } 4293 4294 hsize = skb_headlen(head_skb) - offset; 4295 4296 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) && 4297 (skb_headlen(list_skb) == len || sg)) { 4298 BUG_ON(skb_headlen(list_skb) > len); 4299 4300 i = 0; 4301 nfrags = skb_shinfo(list_skb)->nr_frags; 4302 frag = skb_shinfo(list_skb)->frags; 4303 frag_skb = list_skb; 4304 pos += skb_headlen(list_skb); 4305 4306 while (pos < offset + len) { 4307 BUG_ON(i >= nfrags); 4308 4309 size = skb_frag_size(frag); 4310 if (pos + size > offset + len) 4311 break; 4312 4313 i++; 4314 pos += size; 4315 frag++; 4316 } 4317 4318 nskb = skb_clone(list_skb, GFP_ATOMIC); 4319 list_skb = list_skb->next; 4320 4321 if (unlikely(!nskb)) 4322 goto err; 4323 4324 if (unlikely(pskb_trim(nskb, len))) { 4325 kfree_skb(nskb); 4326 goto err; 4327 } 4328 4329 hsize = skb_end_offset(nskb); 4330 if (skb_cow_head(nskb, doffset + headroom)) { 4331 kfree_skb(nskb); 4332 goto err; 4333 } 4334 4335 nskb->truesize += skb_end_offset(nskb) - hsize; 4336 skb_release_head_state(nskb); 4337 __skb_push(nskb, doffset); 4338 } else { 4339 if (hsize < 0) 4340 hsize = 0; 4341 if (hsize > len || !sg) 4342 hsize = len; 4343 4344 nskb = __alloc_skb(hsize + doffset + headroom, 4345 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 4346 NUMA_NO_NODE); 4347 4348 if (unlikely(!nskb)) 4349 goto err; 4350 4351 skb_reserve(nskb, headroom); 4352 __skb_put(nskb, doffset); 4353 } 4354 4355 if (segs) 4356 tail->next = nskb; 4357 else 4358 segs = nskb; 4359 tail = nskb; 4360 4361 __copy_skb_header(nskb, head_skb); 4362 4363 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 4364 skb_reset_mac_len(nskb); 4365 4366 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 4367 nskb->data - tnl_hlen, 4368 doffset + tnl_hlen); 4369 4370 if (nskb->len == len + doffset) 4371 goto perform_csum_check; 4372 4373 if (!sg) { 4374 if (!csum) { 4375 if (!nskb->remcsum_offload) 4376 nskb->ip_summed = CHECKSUM_NONE; 4377 SKB_GSO_CB(nskb)->csum = 4378 skb_copy_and_csum_bits(head_skb, offset, 4379 skb_put(nskb, 4380 len), 4381 len); 4382 SKB_GSO_CB(nskb)->csum_start = 4383 skb_headroom(nskb) + doffset; 4384 } else { 4385 if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len)) 4386 goto err; 4387 } 4388 continue; 4389 } 4390 4391 nskb_frag = skb_shinfo(nskb)->frags; 4392 4393 skb_copy_from_linear_data_offset(head_skb, offset, 4394 skb_put(nskb, hsize), hsize); 4395 4396 skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags & 4397 SKBFL_SHARED_FRAG; 4398 4399 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4400 skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 4401 goto err; 4402 4403 while (pos < offset + len) { 4404 if (i >= nfrags) { 4405 i = 0; 4406 nfrags = skb_shinfo(list_skb)->nr_frags; 4407 frag = skb_shinfo(list_skb)->frags; 4408 frag_skb = list_skb; 4409 if (!skb_headlen(list_skb)) { 4410 BUG_ON(!nfrags); 4411 } else { 4412 BUG_ON(!list_skb->head_frag); 4413 4414 /* to make room for head_frag. */ 4415 i--; 4416 frag--; 4417 } 4418 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4419 skb_zerocopy_clone(nskb, frag_skb, 4420 GFP_ATOMIC)) 4421 goto err; 4422 4423 list_skb = list_skb->next; 4424 } 4425 4426 if (unlikely(skb_shinfo(nskb)->nr_frags >= 4427 MAX_SKB_FRAGS)) { 4428 net_warn_ratelimited( 4429 "skb_segment: too many frags: %u %u\n", 4430 pos, mss); 4431 err = -EINVAL; 4432 goto err; 4433 } 4434 4435 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; 4436 __skb_frag_ref(nskb_frag); 4437 size = skb_frag_size(nskb_frag); 4438 4439 if (pos < offset) { 4440 skb_frag_off_add(nskb_frag, offset - pos); 4441 skb_frag_size_sub(nskb_frag, offset - pos); 4442 } 4443 4444 skb_shinfo(nskb)->nr_frags++; 4445 4446 if (pos + size <= offset + len) { 4447 i++; 4448 frag++; 4449 pos += size; 4450 } else { 4451 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 4452 goto skip_fraglist; 4453 } 4454 4455 nskb_frag++; 4456 } 4457 4458 skip_fraglist: 4459 nskb->data_len = len - hsize; 4460 nskb->len += nskb->data_len; 4461 nskb->truesize += nskb->data_len; 4462 4463 perform_csum_check: 4464 if (!csum) { 4465 if (skb_has_shared_frag(nskb) && 4466 __skb_linearize(nskb)) 4467 goto err; 4468 4469 if (!nskb->remcsum_offload) 4470 nskb->ip_summed = CHECKSUM_NONE; 4471 SKB_GSO_CB(nskb)->csum = 4472 skb_checksum(nskb, doffset, 4473 nskb->len - doffset, 0); 4474 SKB_GSO_CB(nskb)->csum_start = 4475 skb_headroom(nskb) + doffset; 4476 } 4477 } while ((offset += len) < head_skb->len); 4478 4479 /* Some callers want to get the end of the list. 4480 * Put it in segs->prev to avoid walking the list. 4481 * (see validate_xmit_skb_list() for example) 4482 */ 4483 segs->prev = tail; 4484 4485 if (partial_segs) { 4486 struct sk_buff *iter; 4487 int type = skb_shinfo(head_skb)->gso_type; 4488 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 4489 4490 /* Update type to add partial and then remove dodgy if set */ 4491 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 4492 type &= ~SKB_GSO_DODGY; 4493 4494 /* Update GSO info and prepare to start updating headers on 4495 * our way back down the stack of protocols. 4496 */ 4497 for (iter = segs; iter; iter = iter->next) { 4498 skb_shinfo(iter)->gso_size = gso_size; 4499 skb_shinfo(iter)->gso_segs = partial_segs; 4500 skb_shinfo(iter)->gso_type = type; 4501 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 4502 } 4503 4504 if (tail->len - doffset <= gso_size) 4505 skb_shinfo(tail)->gso_size = 0; 4506 else if (tail != segs) 4507 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 4508 } 4509 4510 /* Following permits correct backpressure, for protocols 4511 * using skb_set_owner_w(). 4512 * Idea is to tranfert ownership from head_skb to last segment. 4513 */ 4514 if (head_skb->destructor == sock_wfree) { 4515 swap(tail->truesize, head_skb->truesize); 4516 swap(tail->destructor, head_skb->destructor); 4517 swap(tail->sk, head_skb->sk); 4518 } 4519 return segs; 4520 4521 err: 4522 kfree_skb_list(segs); 4523 return ERR_PTR(err); 4524 } 4525 EXPORT_SYMBOL_GPL(skb_segment); 4526 4527 #ifdef CONFIG_SKB_EXTENSIONS 4528 #define SKB_EXT_ALIGN_VALUE 8 4529 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) 4530 4531 static const u8 skb_ext_type_len[] = { 4532 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4533 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info), 4534 #endif 4535 #ifdef CONFIG_XFRM 4536 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path), 4537 #endif 4538 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4539 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext), 4540 #endif 4541 #if IS_ENABLED(CONFIG_MPTCP) 4542 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext), 4543 #endif 4544 #if IS_ENABLED(CONFIG_MCTP_FLOWS) 4545 [SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow), 4546 #endif 4547 }; 4548 4549 static __always_inline unsigned int skb_ext_total_length(void) 4550 { 4551 return SKB_EXT_CHUNKSIZEOF(struct skb_ext) + 4552 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4553 skb_ext_type_len[SKB_EXT_BRIDGE_NF] + 4554 #endif 4555 #ifdef CONFIG_XFRM 4556 skb_ext_type_len[SKB_EXT_SEC_PATH] + 4557 #endif 4558 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4559 skb_ext_type_len[TC_SKB_EXT] + 4560 #endif 4561 #if IS_ENABLED(CONFIG_MPTCP) 4562 skb_ext_type_len[SKB_EXT_MPTCP] + 4563 #endif 4564 #if IS_ENABLED(CONFIG_MCTP_FLOWS) 4565 skb_ext_type_len[SKB_EXT_MCTP] + 4566 #endif 4567 0; 4568 } 4569 4570 static void skb_extensions_init(void) 4571 { 4572 BUILD_BUG_ON(SKB_EXT_NUM >= 8); 4573 BUILD_BUG_ON(skb_ext_total_length() > 255); 4574 4575 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", 4576 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), 4577 0, 4578 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4579 NULL); 4580 } 4581 #else 4582 static void skb_extensions_init(void) {} 4583 #endif 4584 4585 void __init skb_init(void) 4586 { 4587 skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache", 4588 sizeof(struct sk_buff), 4589 0, 4590 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4591 offsetof(struct sk_buff, cb), 4592 sizeof_field(struct sk_buff, cb), 4593 NULL); 4594 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 4595 sizeof(struct sk_buff_fclones), 4596 0, 4597 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4598 NULL); 4599 skb_extensions_init(); 4600 } 4601 4602 static int 4603 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 4604 unsigned int recursion_level) 4605 { 4606 int start = skb_headlen(skb); 4607 int i, copy = start - offset; 4608 struct sk_buff *frag_iter; 4609 int elt = 0; 4610 4611 if (unlikely(recursion_level >= 24)) 4612 return -EMSGSIZE; 4613 4614 if (copy > 0) { 4615 if (copy > len) 4616 copy = len; 4617 sg_set_buf(sg, skb->data + offset, copy); 4618 elt++; 4619 if ((len -= copy) == 0) 4620 return elt; 4621 offset += copy; 4622 } 4623 4624 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4625 int end; 4626 4627 WARN_ON(start > offset + len); 4628 4629 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 4630 if ((copy = end - offset) > 0) { 4631 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4632 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4633 return -EMSGSIZE; 4634 4635 if (copy > len) 4636 copy = len; 4637 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 4638 skb_frag_off(frag) + offset - start); 4639 elt++; 4640 if (!(len -= copy)) 4641 return elt; 4642 offset += copy; 4643 } 4644 start = end; 4645 } 4646 4647 skb_walk_frags(skb, frag_iter) { 4648 int end, ret; 4649 4650 WARN_ON(start > offset + len); 4651 4652 end = start + frag_iter->len; 4653 if ((copy = end - offset) > 0) { 4654 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4655 return -EMSGSIZE; 4656 4657 if (copy > len) 4658 copy = len; 4659 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 4660 copy, recursion_level + 1); 4661 if (unlikely(ret < 0)) 4662 return ret; 4663 elt += ret; 4664 if ((len -= copy) == 0) 4665 return elt; 4666 offset += copy; 4667 } 4668 start = end; 4669 } 4670 BUG_ON(len); 4671 return elt; 4672 } 4673 4674 /** 4675 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 4676 * @skb: Socket buffer containing the buffers to be mapped 4677 * @sg: The scatter-gather list to map into 4678 * @offset: The offset into the buffer's contents to start mapping 4679 * @len: Length of buffer space to be mapped 4680 * 4681 * Fill the specified scatter-gather list with mappings/pointers into a 4682 * region of the buffer space attached to a socket buffer. Returns either 4683 * the number of scatterlist items used, or -EMSGSIZE if the contents 4684 * could not fit. 4685 */ 4686 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 4687 { 4688 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 4689 4690 if (nsg <= 0) 4691 return nsg; 4692 4693 sg_mark_end(&sg[nsg - 1]); 4694 4695 return nsg; 4696 } 4697 EXPORT_SYMBOL_GPL(skb_to_sgvec); 4698 4699 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 4700 * sglist without mark the sg which contain last skb data as the end. 4701 * So the caller can mannipulate sg list as will when padding new data after 4702 * the first call without calling sg_unmark_end to expend sg list. 4703 * 4704 * Scenario to use skb_to_sgvec_nomark: 4705 * 1. sg_init_table 4706 * 2. skb_to_sgvec_nomark(payload1) 4707 * 3. skb_to_sgvec_nomark(payload2) 4708 * 4709 * This is equivalent to: 4710 * 1. sg_init_table 4711 * 2. skb_to_sgvec(payload1) 4712 * 3. sg_unmark_end 4713 * 4. skb_to_sgvec(payload2) 4714 * 4715 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 4716 * is more preferable. 4717 */ 4718 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 4719 int offset, int len) 4720 { 4721 return __skb_to_sgvec(skb, sg, offset, len, 0); 4722 } 4723 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 4724 4725 4726 4727 /** 4728 * skb_cow_data - Check that a socket buffer's data buffers are writable 4729 * @skb: The socket buffer to check. 4730 * @tailbits: Amount of trailing space to be added 4731 * @trailer: Returned pointer to the skb where the @tailbits space begins 4732 * 4733 * Make sure that the data buffers attached to a socket buffer are 4734 * writable. If they are not, private copies are made of the data buffers 4735 * and the socket buffer is set to use these instead. 4736 * 4737 * If @tailbits is given, make sure that there is space to write @tailbits 4738 * bytes of data beyond current end of socket buffer. @trailer will be 4739 * set to point to the skb in which this space begins. 4740 * 4741 * The number of scatterlist elements required to completely map the 4742 * COW'd and extended socket buffer will be returned. 4743 */ 4744 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 4745 { 4746 int copyflag; 4747 int elt; 4748 struct sk_buff *skb1, **skb_p; 4749 4750 /* If skb is cloned or its head is paged, reallocate 4751 * head pulling out all the pages (pages are considered not writable 4752 * at the moment even if they are anonymous). 4753 */ 4754 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 4755 !__pskb_pull_tail(skb, __skb_pagelen(skb))) 4756 return -ENOMEM; 4757 4758 /* Easy case. Most of packets will go this way. */ 4759 if (!skb_has_frag_list(skb)) { 4760 /* A little of trouble, not enough of space for trailer. 4761 * This should not happen, when stack is tuned to generate 4762 * good frames. OK, on miss we reallocate and reserve even more 4763 * space, 128 bytes is fair. */ 4764 4765 if (skb_tailroom(skb) < tailbits && 4766 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 4767 return -ENOMEM; 4768 4769 /* Voila! */ 4770 *trailer = skb; 4771 return 1; 4772 } 4773 4774 /* Misery. We are in troubles, going to mincer fragments... */ 4775 4776 elt = 1; 4777 skb_p = &skb_shinfo(skb)->frag_list; 4778 copyflag = 0; 4779 4780 while ((skb1 = *skb_p) != NULL) { 4781 int ntail = 0; 4782 4783 /* The fragment is partially pulled by someone, 4784 * this can happen on input. Copy it and everything 4785 * after it. */ 4786 4787 if (skb_shared(skb1)) 4788 copyflag = 1; 4789 4790 /* If the skb is the last, worry about trailer. */ 4791 4792 if (skb1->next == NULL && tailbits) { 4793 if (skb_shinfo(skb1)->nr_frags || 4794 skb_has_frag_list(skb1) || 4795 skb_tailroom(skb1) < tailbits) 4796 ntail = tailbits + 128; 4797 } 4798 4799 if (copyflag || 4800 skb_cloned(skb1) || 4801 ntail || 4802 skb_shinfo(skb1)->nr_frags || 4803 skb_has_frag_list(skb1)) { 4804 struct sk_buff *skb2; 4805 4806 /* Fuck, we are miserable poor guys... */ 4807 if (ntail == 0) 4808 skb2 = skb_copy(skb1, GFP_ATOMIC); 4809 else 4810 skb2 = skb_copy_expand(skb1, 4811 skb_headroom(skb1), 4812 ntail, 4813 GFP_ATOMIC); 4814 if (unlikely(skb2 == NULL)) 4815 return -ENOMEM; 4816 4817 if (skb1->sk) 4818 skb_set_owner_w(skb2, skb1->sk); 4819 4820 /* Looking around. Are we still alive? 4821 * OK, link new skb, drop old one */ 4822 4823 skb2->next = skb1->next; 4824 *skb_p = skb2; 4825 kfree_skb(skb1); 4826 skb1 = skb2; 4827 } 4828 elt++; 4829 *trailer = skb1; 4830 skb_p = &skb1->next; 4831 } 4832 4833 return elt; 4834 } 4835 EXPORT_SYMBOL_GPL(skb_cow_data); 4836 4837 static void sock_rmem_free(struct sk_buff *skb) 4838 { 4839 struct sock *sk = skb->sk; 4840 4841 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 4842 } 4843 4844 static void skb_set_err_queue(struct sk_buff *skb) 4845 { 4846 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING. 4847 * So, it is safe to (mis)use it to mark skbs on the error queue. 4848 */ 4849 skb->pkt_type = PACKET_OUTGOING; 4850 BUILD_BUG_ON(PACKET_OUTGOING == 0); 4851 } 4852 4853 /* 4854 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 4855 */ 4856 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 4857 { 4858 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 4859 (unsigned int)READ_ONCE(sk->sk_rcvbuf)) 4860 return -ENOMEM; 4861 4862 skb_orphan(skb); 4863 skb->sk = sk; 4864 skb->destructor = sock_rmem_free; 4865 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 4866 skb_set_err_queue(skb); 4867 4868 /* before exiting rcu section, make sure dst is refcounted */ 4869 skb_dst_force(skb); 4870 4871 skb_queue_tail(&sk->sk_error_queue, skb); 4872 if (!sock_flag(sk, SOCK_DEAD)) 4873 sk_error_report(sk); 4874 return 0; 4875 } 4876 EXPORT_SYMBOL(sock_queue_err_skb); 4877 4878 static bool is_icmp_err_skb(const struct sk_buff *skb) 4879 { 4880 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 4881 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 4882 } 4883 4884 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 4885 { 4886 struct sk_buff_head *q = &sk->sk_error_queue; 4887 struct sk_buff *skb, *skb_next = NULL; 4888 bool icmp_next = false; 4889 unsigned long flags; 4890 4891 spin_lock_irqsave(&q->lock, flags); 4892 skb = __skb_dequeue(q); 4893 if (skb && (skb_next = skb_peek(q))) { 4894 icmp_next = is_icmp_err_skb(skb_next); 4895 if (icmp_next) 4896 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 4897 } 4898 spin_unlock_irqrestore(&q->lock, flags); 4899 4900 if (is_icmp_err_skb(skb) && !icmp_next) 4901 sk->sk_err = 0; 4902 4903 if (skb_next) 4904 sk_error_report(sk); 4905 4906 return skb; 4907 } 4908 EXPORT_SYMBOL(sock_dequeue_err_skb); 4909 4910 /** 4911 * skb_clone_sk - create clone of skb, and take reference to socket 4912 * @skb: the skb to clone 4913 * 4914 * This function creates a clone of a buffer that holds a reference on 4915 * sk_refcnt. Buffers created via this function are meant to be 4916 * returned using sock_queue_err_skb, or free via kfree_skb. 4917 * 4918 * When passing buffers allocated with this function to sock_queue_err_skb 4919 * it is necessary to wrap the call with sock_hold/sock_put in order to 4920 * prevent the socket from being released prior to being enqueued on 4921 * the sk_error_queue. 4922 */ 4923 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 4924 { 4925 struct sock *sk = skb->sk; 4926 struct sk_buff *clone; 4927 4928 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 4929 return NULL; 4930 4931 clone = skb_clone(skb, GFP_ATOMIC); 4932 if (!clone) { 4933 sock_put(sk); 4934 return NULL; 4935 } 4936 4937 clone->sk = sk; 4938 clone->destructor = sock_efree; 4939 4940 return clone; 4941 } 4942 EXPORT_SYMBOL(skb_clone_sk); 4943 4944 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 4945 struct sock *sk, 4946 int tstype, 4947 bool opt_stats) 4948 { 4949 struct sock_exterr_skb *serr; 4950 int err; 4951 4952 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 4953 4954 serr = SKB_EXT_ERR(skb); 4955 memset(serr, 0, sizeof(*serr)); 4956 serr->ee.ee_errno = ENOMSG; 4957 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 4958 serr->ee.ee_info = tstype; 4959 serr->opt_stats = opt_stats; 4960 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 4961 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { 4962 serr->ee.ee_data = skb_shinfo(skb)->tskey; 4963 if (sk_is_tcp(sk)) 4964 serr->ee.ee_data -= atomic_read(&sk->sk_tskey); 4965 } 4966 4967 err = sock_queue_err_skb(sk, skb); 4968 4969 if (err) 4970 kfree_skb(skb); 4971 } 4972 4973 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 4974 { 4975 bool ret; 4976 4977 if (likely(READ_ONCE(sysctl_tstamp_allow_data) || tsonly)) 4978 return true; 4979 4980 read_lock_bh(&sk->sk_callback_lock); 4981 ret = sk->sk_socket && sk->sk_socket->file && 4982 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); 4983 read_unlock_bh(&sk->sk_callback_lock); 4984 return ret; 4985 } 4986 4987 void skb_complete_tx_timestamp(struct sk_buff *skb, 4988 struct skb_shared_hwtstamps *hwtstamps) 4989 { 4990 struct sock *sk = skb->sk; 4991 4992 if (!skb_may_tx_timestamp(sk, false)) 4993 goto err; 4994 4995 /* Take a reference to prevent skb_orphan() from freeing the socket, 4996 * but only if the socket refcount is not zero. 4997 */ 4998 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 4999 *skb_hwtstamps(skb) = *hwtstamps; 5000 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 5001 sock_put(sk); 5002 return; 5003 } 5004 5005 err: 5006 kfree_skb(skb); 5007 } 5008 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 5009 5010 void __skb_tstamp_tx(struct sk_buff *orig_skb, 5011 const struct sk_buff *ack_skb, 5012 struct skb_shared_hwtstamps *hwtstamps, 5013 struct sock *sk, int tstype) 5014 { 5015 struct sk_buff *skb; 5016 bool tsonly, opt_stats = false; 5017 5018 if (!sk) 5019 return; 5020 5021 if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 5022 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 5023 return; 5024 5025 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 5026 if (!skb_may_tx_timestamp(sk, tsonly)) 5027 return; 5028 5029 if (tsonly) { 5030 #ifdef CONFIG_INET 5031 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && 5032 sk_is_tcp(sk)) { 5033 skb = tcp_get_timestamping_opt_stats(sk, orig_skb, 5034 ack_skb); 5035 opt_stats = true; 5036 } else 5037 #endif 5038 skb = alloc_skb(0, GFP_ATOMIC); 5039 } else { 5040 skb = skb_clone(orig_skb, GFP_ATOMIC); 5041 } 5042 if (!skb) 5043 return; 5044 5045 if (tsonly) { 5046 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 5047 SKBTX_ANY_TSTAMP; 5048 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 5049 } 5050 5051 if (hwtstamps) 5052 *skb_hwtstamps(skb) = *hwtstamps; 5053 else 5054 __net_timestamp(skb); 5055 5056 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 5057 } 5058 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 5059 5060 void skb_tstamp_tx(struct sk_buff *orig_skb, 5061 struct skb_shared_hwtstamps *hwtstamps) 5062 { 5063 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk, 5064 SCM_TSTAMP_SND); 5065 } 5066 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 5067 5068 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 5069 { 5070 struct sock *sk = skb->sk; 5071 struct sock_exterr_skb *serr; 5072 int err = 1; 5073 5074 skb->wifi_acked_valid = 1; 5075 skb->wifi_acked = acked; 5076 5077 serr = SKB_EXT_ERR(skb); 5078 memset(serr, 0, sizeof(*serr)); 5079 serr->ee.ee_errno = ENOMSG; 5080 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 5081 5082 /* Take a reference to prevent skb_orphan() from freeing the socket, 5083 * but only if the socket refcount is not zero. 5084 */ 5085 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5086 err = sock_queue_err_skb(sk, skb); 5087 sock_put(sk); 5088 } 5089 if (err) 5090 kfree_skb(skb); 5091 } 5092 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 5093 5094 /** 5095 * skb_partial_csum_set - set up and verify partial csum values for packet 5096 * @skb: the skb to set 5097 * @start: the number of bytes after skb->data to start checksumming. 5098 * @off: the offset from start to place the checksum. 5099 * 5100 * For untrusted partially-checksummed packets, we need to make sure the values 5101 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 5102 * 5103 * This function checks and sets those values and skb->ip_summed: if this 5104 * returns false you should drop the packet. 5105 */ 5106 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 5107 { 5108 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16); 5109 u32 csum_start = skb_headroom(skb) + (u32)start; 5110 5111 if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) { 5112 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n", 5113 start, off, skb_headroom(skb), skb_headlen(skb)); 5114 return false; 5115 } 5116 skb->ip_summed = CHECKSUM_PARTIAL; 5117 skb->csum_start = csum_start; 5118 skb->csum_offset = off; 5119 skb_set_transport_header(skb, start); 5120 return true; 5121 } 5122 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 5123 5124 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 5125 unsigned int max) 5126 { 5127 if (skb_headlen(skb) >= len) 5128 return 0; 5129 5130 /* If we need to pullup then pullup to the max, so we 5131 * won't need to do it again. 5132 */ 5133 if (max > skb->len) 5134 max = skb->len; 5135 5136 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 5137 return -ENOMEM; 5138 5139 if (skb_headlen(skb) < len) 5140 return -EPROTO; 5141 5142 return 0; 5143 } 5144 5145 #define MAX_TCP_HDR_LEN (15 * 4) 5146 5147 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 5148 typeof(IPPROTO_IP) proto, 5149 unsigned int off) 5150 { 5151 int err; 5152 5153 switch (proto) { 5154 case IPPROTO_TCP: 5155 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 5156 off + MAX_TCP_HDR_LEN); 5157 if (!err && !skb_partial_csum_set(skb, off, 5158 offsetof(struct tcphdr, 5159 check))) 5160 err = -EPROTO; 5161 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 5162 5163 case IPPROTO_UDP: 5164 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 5165 off + sizeof(struct udphdr)); 5166 if (!err && !skb_partial_csum_set(skb, off, 5167 offsetof(struct udphdr, 5168 check))) 5169 err = -EPROTO; 5170 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 5171 } 5172 5173 return ERR_PTR(-EPROTO); 5174 } 5175 5176 /* This value should be large enough to cover a tagged ethernet header plus 5177 * maximally sized IP and TCP or UDP headers. 5178 */ 5179 #define MAX_IP_HDR_LEN 128 5180 5181 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 5182 { 5183 unsigned int off; 5184 bool fragment; 5185 __sum16 *csum; 5186 int err; 5187 5188 fragment = false; 5189 5190 err = skb_maybe_pull_tail(skb, 5191 sizeof(struct iphdr), 5192 MAX_IP_HDR_LEN); 5193 if (err < 0) 5194 goto out; 5195 5196 if (ip_is_fragment(ip_hdr(skb))) 5197 fragment = true; 5198 5199 off = ip_hdrlen(skb); 5200 5201 err = -EPROTO; 5202 5203 if (fragment) 5204 goto out; 5205 5206 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 5207 if (IS_ERR(csum)) 5208 return PTR_ERR(csum); 5209 5210 if (recalculate) 5211 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 5212 ip_hdr(skb)->daddr, 5213 skb->len - off, 5214 ip_hdr(skb)->protocol, 0); 5215 err = 0; 5216 5217 out: 5218 return err; 5219 } 5220 5221 /* This value should be large enough to cover a tagged ethernet header plus 5222 * an IPv6 header, all options, and a maximal TCP or UDP header. 5223 */ 5224 #define MAX_IPV6_HDR_LEN 256 5225 5226 #define OPT_HDR(type, skb, off) \ 5227 (type *)(skb_network_header(skb) + (off)) 5228 5229 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 5230 { 5231 int err; 5232 u8 nexthdr; 5233 unsigned int off; 5234 unsigned int len; 5235 bool fragment; 5236 bool done; 5237 __sum16 *csum; 5238 5239 fragment = false; 5240 done = false; 5241 5242 off = sizeof(struct ipv6hdr); 5243 5244 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 5245 if (err < 0) 5246 goto out; 5247 5248 nexthdr = ipv6_hdr(skb)->nexthdr; 5249 5250 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 5251 while (off <= len && !done) { 5252 switch (nexthdr) { 5253 case IPPROTO_DSTOPTS: 5254 case IPPROTO_HOPOPTS: 5255 case IPPROTO_ROUTING: { 5256 struct ipv6_opt_hdr *hp; 5257 5258 err = skb_maybe_pull_tail(skb, 5259 off + 5260 sizeof(struct ipv6_opt_hdr), 5261 MAX_IPV6_HDR_LEN); 5262 if (err < 0) 5263 goto out; 5264 5265 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 5266 nexthdr = hp->nexthdr; 5267 off += ipv6_optlen(hp); 5268 break; 5269 } 5270 case IPPROTO_AH: { 5271 struct ip_auth_hdr *hp; 5272 5273 err = skb_maybe_pull_tail(skb, 5274 off + 5275 sizeof(struct ip_auth_hdr), 5276 MAX_IPV6_HDR_LEN); 5277 if (err < 0) 5278 goto out; 5279 5280 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 5281 nexthdr = hp->nexthdr; 5282 off += ipv6_authlen(hp); 5283 break; 5284 } 5285 case IPPROTO_FRAGMENT: { 5286 struct frag_hdr *hp; 5287 5288 err = skb_maybe_pull_tail(skb, 5289 off + 5290 sizeof(struct frag_hdr), 5291 MAX_IPV6_HDR_LEN); 5292 if (err < 0) 5293 goto out; 5294 5295 hp = OPT_HDR(struct frag_hdr, skb, off); 5296 5297 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 5298 fragment = true; 5299 5300 nexthdr = hp->nexthdr; 5301 off += sizeof(struct frag_hdr); 5302 break; 5303 } 5304 default: 5305 done = true; 5306 break; 5307 } 5308 } 5309 5310 err = -EPROTO; 5311 5312 if (!done || fragment) 5313 goto out; 5314 5315 csum = skb_checksum_setup_ip(skb, nexthdr, off); 5316 if (IS_ERR(csum)) 5317 return PTR_ERR(csum); 5318 5319 if (recalculate) 5320 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 5321 &ipv6_hdr(skb)->daddr, 5322 skb->len - off, nexthdr, 0); 5323 err = 0; 5324 5325 out: 5326 return err; 5327 } 5328 5329 /** 5330 * skb_checksum_setup - set up partial checksum offset 5331 * @skb: the skb to set up 5332 * @recalculate: if true the pseudo-header checksum will be recalculated 5333 */ 5334 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 5335 { 5336 int err; 5337 5338 switch (skb->protocol) { 5339 case htons(ETH_P_IP): 5340 err = skb_checksum_setup_ipv4(skb, recalculate); 5341 break; 5342 5343 case htons(ETH_P_IPV6): 5344 err = skb_checksum_setup_ipv6(skb, recalculate); 5345 break; 5346 5347 default: 5348 err = -EPROTO; 5349 break; 5350 } 5351 5352 return err; 5353 } 5354 EXPORT_SYMBOL(skb_checksum_setup); 5355 5356 /** 5357 * skb_checksum_maybe_trim - maybe trims the given skb 5358 * @skb: the skb to check 5359 * @transport_len: the data length beyond the network header 5360 * 5361 * Checks whether the given skb has data beyond the given transport length. 5362 * If so, returns a cloned skb trimmed to this transport length. 5363 * Otherwise returns the provided skb. Returns NULL in error cases 5364 * (e.g. transport_len exceeds skb length or out-of-memory). 5365 * 5366 * Caller needs to set the skb transport header and free any returned skb if it 5367 * differs from the provided skb. 5368 */ 5369 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 5370 unsigned int transport_len) 5371 { 5372 struct sk_buff *skb_chk; 5373 unsigned int len = skb_transport_offset(skb) + transport_len; 5374 int ret; 5375 5376 if (skb->len < len) 5377 return NULL; 5378 else if (skb->len == len) 5379 return skb; 5380 5381 skb_chk = skb_clone(skb, GFP_ATOMIC); 5382 if (!skb_chk) 5383 return NULL; 5384 5385 ret = pskb_trim_rcsum(skb_chk, len); 5386 if (ret) { 5387 kfree_skb(skb_chk); 5388 return NULL; 5389 } 5390 5391 return skb_chk; 5392 } 5393 5394 /** 5395 * skb_checksum_trimmed - validate checksum of an skb 5396 * @skb: the skb to check 5397 * @transport_len: the data length beyond the network header 5398 * @skb_chkf: checksum function to use 5399 * 5400 * Applies the given checksum function skb_chkf to the provided skb. 5401 * Returns a checked and maybe trimmed skb. Returns NULL on error. 5402 * 5403 * If the skb has data beyond the given transport length, then a 5404 * trimmed & cloned skb is checked and returned. 5405 * 5406 * Caller needs to set the skb transport header and free any returned skb if it 5407 * differs from the provided skb. 5408 */ 5409 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 5410 unsigned int transport_len, 5411 __sum16(*skb_chkf)(struct sk_buff *skb)) 5412 { 5413 struct sk_buff *skb_chk; 5414 unsigned int offset = skb_transport_offset(skb); 5415 __sum16 ret; 5416 5417 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 5418 if (!skb_chk) 5419 goto err; 5420 5421 if (!pskb_may_pull(skb_chk, offset)) 5422 goto err; 5423 5424 skb_pull_rcsum(skb_chk, offset); 5425 ret = skb_chkf(skb_chk); 5426 skb_push_rcsum(skb_chk, offset); 5427 5428 if (ret) 5429 goto err; 5430 5431 return skb_chk; 5432 5433 err: 5434 if (skb_chk && skb_chk != skb) 5435 kfree_skb(skb_chk); 5436 5437 return NULL; 5438 5439 } 5440 EXPORT_SYMBOL(skb_checksum_trimmed); 5441 5442 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 5443 { 5444 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 5445 skb->dev->name); 5446 } 5447 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 5448 5449 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 5450 { 5451 if (head_stolen) { 5452 skb_release_head_state(skb); 5453 kmem_cache_free(skbuff_head_cache, skb); 5454 } else { 5455 __kfree_skb(skb); 5456 } 5457 } 5458 EXPORT_SYMBOL(kfree_skb_partial); 5459 5460 /** 5461 * skb_try_coalesce - try to merge skb to prior one 5462 * @to: prior buffer 5463 * @from: buffer to add 5464 * @fragstolen: pointer to boolean 5465 * @delta_truesize: how much more was allocated than was requested 5466 */ 5467 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 5468 bool *fragstolen, int *delta_truesize) 5469 { 5470 struct skb_shared_info *to_shinfo, *from_shinfo; 5471 int i, delta, len = from->len; 5472 5473 *fragstolen = false; 5474 5475 if (skb_cloned(to)) 5476 return false; 5477 5478 /* In general, avoid mixing slab allocated and page_pool allocated 5479 * pages within the same SKB. However when @to is not pp_recycle and 5480 * @from is cloned, we can transition frag pages from page_pool to 5481 * reference counted. 5482 * 5483 * On the other hand, don't allow coalescing two pp_recycle SKBs if 5484 * @from is cloned, in case the SKB is using page_pool fragment 5485 * references (PP_FLAG_PAGE_FRAG). Since we only take full page 5486 * references for cloned SKBs at the moment that would result in 5487 * inconsistent reference counts. 5488 */ 5489 if (to->pp_recycle != (from->pp_recycle && !skb_cloned(from))) 5490 return false; 5491 5492 if (len <= skb_tailroom(to)) { 5493 if (len) 5494 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 5495 *delta_truesize = 0; 5496 return true; 5497 } 5498 5499 to_shinfo = skb_shinfo(to); 5500 from_shinfo = skb_shinfo(from); 5501 if (to_shinfo->frag_list || from_shinfo->frag_list) 5502 return false; 5503 if (skb_zcopy(to) || skb_zcopy(from)) 5504 return false; 5505 5506 if (skb_headlen(from) != 0) { 5507 struct page *page; 5508 unsigned int offset; 5509 5510 if (to_shinfo->nr_frags + 5511 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 5512 return false; 5513 5514 if (skb_head_is_locked(from)) 5515 return false; 5516 5517 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 5518 5519 page = virt_to_head_page(from->head); 5520 offset = from->data - (unsigned char *)page_address(page); 5521 5522 skb_fill_page_desc(to, to_shinfo->nr_frags, 5523 page, offset, skb_headlen(from)); 5524 *fragstolen = true; 5525 } else { 5526 if (to_shinfo->nr_frags + 5527 from_shinfo->nr_frags > MAX_SKB_FRAGS) 5528 return false; 5529 5530 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 5531 } 5532 5533 WARN_ON_ONCE(delta < len); 5534 5535 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 5536 from_shinfo->frags, 5537 from_shinfo->nr_frags * sizeof(skb_frag_t)); 5538 to_shinfo->nr_frags += from_shinfo->nr_frags; 5539 5540 if (!skb_cloned(from)) 5541 from_shinfo->nr_frags = 0; 5542 5543 /* if the skb is not cloned this does nothing 5544 * since we set nr_frags to 0. 5545 */ 5546 for (i = 0; i < from_shinfo->nr_frags; i++) 5547 __skb_frag_ref(&from_shinfo->frags[i]); 5548 5549 to->truesize += delta; 5550 to->len += len; 5551 to->data_len += len; 5552 5553 *delta_truesize = delta; 5554 return true; 5555 } 5556 EXPORT_SYMBOL(skb_try_coalesce); 5557 5558 /** 5559 * skb_scrub_packet - scrub an skb 5560 * 5561 * @skb: buffer to clean 5562 * @xnet: packet is crossing netns 5563 * 5564 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 5565 * into/from a tunnel. Some information have to be cleared during these 5566 * operations. 5567 * skb_scrub_packet can also be used to clean a skb before injecting it in 5568 * another namespace (@xnet == true). We have to clear all information in the 5569 * skb that could impact namespace isolation. 5570 */ 5571 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 5572 { 5573 skb->pkt_type = PACKET_HOST; 5574 skb->skb_iif = 0; 5575 skb->ignore_df = 0; 5576 skb_dst_drop(skb); 5577 skb_ext_reset(skb); 5578 nf_reset_ct(skb); 5579 nf_reset_trace(skb); 5580 5581 #ifdef CONFIG_NET_SWITCHDEV 5582 skb->offload_fwd_mark = 0; 5583 skb->offload_l3_fwd_mark = 0; 5584 #endif 5585 5586 if (!xnet) 5587 return; 5588 5589 ipvs_reset(skb); 5590 skb->mark = 0; 5591 skb_clear_tstamp(skb); 5592 } 5593 EXPORT_SYMBOL_GPL(skb_scrub_packet); 5594 5595 /** 5596 * skb_gso_transport_seglen - Return length of individual segments of a gso packet 5597 * 5598 * @skb: GSO skb 5599 * 5600 * skb_gso_transport_seglen is used to determine the real size of the 5601 * individual segments, including Layer4 headers (TCP/UDP). 5602 * 5603 * The MAC/L2 or network (IP, IPv6) headers are not accounted for. 5604 */ 5605 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb) 5606 { 5607 const struct skb_shared_info *shinfo = skb_shinfo(skb); 5608 unsigned int thlen = 0; 5609 5610 if (skb->encapsulation) { 5611 thlen = skb_inner_transport_header(skb) - 5612 skb_transport_header(skb); 5613 5614 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 5615 thlen += inner_tcp_hdrlen(skb); 5616 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 5617 thlen = tcp_hdrlen(skb); 5618 } else if (unlikely(skb_is_gso_sctp(skb))) { 5619 thlen = sizeof(struct sctphdr); 5620 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { 5621 thlen = sizeof(struct udphdr); 5622 } 5623 /* UFO sets gso_size to the size of the fragmentation 5624 * payload, i.e. the size of the L4 (UDP) header is already 5625 * accounted for. 5626 */ 5627 return thlen + shinfo->gso_size; 5628 } 5629 5630 /** 5631 * skb_gso_network_seglen - Return length of individual segments of a gso packet 5632 * 5633 * @skb: GSO skb 5634 * 5635 * skb_gso_network_seglen is used to determine the real size of the 5636 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP). 5637 * 5638 * The MAC/L2 header is not accounted for. 5639 */ 5640 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb) 5641 { 5642 unsigned int hdr_len = skb_transport_header(skb) - 5643 skb_network_header(skb); 5644 5645 return hdr_len + skb_gso_transport_seglen(skb); 5646 } 5647 5648 /** 5649 * skb_gso_mac_seglen - Return length of individual segments of a gso packet 5650 * 5651 * @skb: GSO skb 5652 * 5653 * skb_gso_mac_seglen is used to determine the real size of the 5654 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4 5655 * headers (TCP/UDP). 5656 */ 5657 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb) 5658 { 5659 unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 5660 5661 return hdr_len + skb_gso_transport_seglen(skb); 5662 } 5663 5664 /** 5665 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS 5666 * 5667 * There are a couple of instances where we have a GSO skb, and we 5668 * want to determine what size it would be after it is segmented. 5669 * 5670 * We might want to check: 5671 * - L3+L4+payload size (e.g. IP forwarding) 5672 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver) 5673 * 5674 * This is a helper to do that correctly considering GSO_BY_FRAGS. 5675 * 5676 * @skb: GSO skb 5677 * 5678 * @seg_len: The segmented length (from skb_gso_*_seglen). In the 5679 * GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS]. 5680 * 5681 * @max_len: The maximum permissible length. 5682 * 5683 * Returns true if the segmented length <= max length. 5684 */ 5685 static inline bool skb_gso_size_check(const struct sk_buff *skb, 5686 unsigned int seg_len, 5687 unsigned int max_len) { 5688 const struct skb_shared_info *shinfo = skb_shinfo(skb); 5689 const struct sk_buff *iter; 5690 5691 if (shinfo->gso_size != GSO_BY_FRAGS) 5692 return seg_len <= max_len; 5693 5694 /* Undo this so we can re-use header sizes */ 5695 seg_len -= GSO_BY_FRAGS; 5696 5697 skb_walk_frags(skb, iter) { 5698 if (seg_len + skb_headlen(iter) > max_len) 5699 return false; 5700 } 5701 5702 return true; 5703 } 5704 5705 /** 5706 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU? 5707 * 5708 * @skb: GSO skb 5709 * @mtu: MTU to validate against 5710 * 5711 * skb_gso_validate_network_len validates if a given skb will fit a 5712 * wanted MTU once split. It considers L3 headers, L4 headers, and the 5713 * payload. 5714 */ 5715 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu) 5716 { 5717 return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu); 5718 } 5719 EXPORT_SYMBOL_GPL(skb_gso_validate_network_len); 5720 5721 /** 5722 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length? 5723 * 5724 * @skb: GSO skb 5725 * @len: length to validate against 5726 * 5727 * skb_gso_validate_mac_len validates if a given skb will fit a wanted 5728 * length once split, including L2, L3 and L4 headers and the payload. 5729 */ 5730 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len) 5731 { 5732 return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len); 5733 } 5734 EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len); 5735 5736 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 5737 { 5738 int mac_len, meta_len; 5739 void *meta; 5740 5741 if (skb_cow(skb, skb_headroom(skb)) < 0) { 5742 kfree_skb(skb); 5743 return NULL; 5744 } 5745 5746 mac_len = skb->data - skb_mac_header(skb); 5747 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) { 5748 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), 5749 mac_len - VLAN_HLEN - ETH_TLEN); 5750 } 5751 5752 meta_len = skb_metadata_len(skb); 5753 if (meta_len) { 5754 meta = skb_metadata_end(skb) - meta_len; 5755 memmove(meta + VLAN_HLEN, meta, meta_len); 5756 } 5757 5758 skb->mac_header += VLAN_HLEN; 5759 return skb; 5760 } 5761 5762 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 5763 { 5764 struct vlan_hdr *vhdr; 5765 u16 vlan_tci; 5766 5767 if (unlikely(skb_vlan_tag_present(skb))) { 5768 /* vlan_tci is already set-up so leave this for another time */ 5769 return skb; 5770 } 5771 5772 skb = skb_share_check(skb, GFP_ATOMIC); 5773 if (unlikely(!skb)) 5774 goto err_free; 5775 /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */ 5776 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short)))) 5777 goto err_free; 5778 5779 vhdr = (struct vlan_hdr *)skb->data; 5780 vlan_tci = ntohs(vhdr->h_vlan_TCI); 5781 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 5782 5783 skb_pull_rcsum(skb, VLAN_HLEN); 5784 vlan_set_encap_proto(skb, vhdr); 5785 5786 skb = skb_reorder_vlan_header(skb); 5787 if (unlikely(!skb)) 5788 goto err_free; 5789 5790 skb_reset_network_header(skb); 5791 if (!skb_transport_header_was_set(skb)) 5792 skb_reset_transport_header(skb); 5793 skb_reset_mac_len(skb); 5794 5795 return skb; 5796 5797 err_free: 5798 kfree_skb(skb); 5799 return NULL; 5800 } 5801 EXPORT_SYMBOL(skb_vlan_untag); 5802 5803 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len) 5804 { 5805 if (!pskb_may_pull(skb, write_len)) 5806 return -ENOMEM; 5807 5808 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 5809 return 0; 5810 5811 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 5812 } 5813 EXPORT_SYMBOL(skb_ensure_writable); 5814 5815 /* remove VLAN header from packet and update csum accordingly. 5816 * expects a non skb_vlan_tag_present skb with a vlan tag payload 5817 */ 5818 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 5819 { 5820 struct vlan_hdr *vhdr; 5821 int offset = skb->data - skb_mac_header(skb); 5822 int err; 5823 5824 if (WARN_ONCE(offset, 5825 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 5826 offset)) { 5827 return -EINVAL; 5828 } 5829 5830 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 5831 if (unlikely(err)) 5832 return err; 5833 5834 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5835 5836 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN); 5837 *vlan_tci = ntohs(vhdr->h_vlan_TCI); 5838 5839 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN); 5840 __skb_pull(skb, VLAN_HLEN); 5841 5842 vlan_set_encap_proto(skb, vhdr); 5843 skb->mac_header += VLAN_HLEN; 5844 5845 if (skb_network_offset(skb) < ETH_HLEN) 5846 skb_set_network_header(skb, ETH_HLEN); 5847 5848 skb_reset_mac_len(skb); 5849 5850 return err; 5851 } 5852 EXPORT_SYMBOL(__skb_vlan_pop); 5853 5854 /* Pop a vlan tag either from hwaccel or from payload. 5855 * Expects skb->data at mac header. 5856 */ 5857 int skb_vlan_pop(struct sk_buff *skb) 5858 { 5859 u16 vlan_tci; 5860 __be16 vlan_proto; 5861 int err; 5862 5863 if (likely(skb_vlan_tag_present(skb))) { 5864 __vlan_hwaccel_clear_tag(skb); 5865 } else { 5866 if (unlikely(!eth_type_vlan(skb->protocol))) 5867 return 0; 5868 5869 err = __skb_vlan_pop(skb, &vlan_tci); 5870 if (err) 5871 return err; 5872 } 5873 /* move next vlan tag to hw accel tag */ 5874 if (likely(!eth_type_vlan(skb->protocol))) 5875 return 0; 5876 5877 vlan_proto = skb->protocol; 5878 err = __skb_vlan_pop(skb, &vlan_tci); 5879 if (unlikely(err)) 5880 return err; 5881 5882 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5883 return 0; 5884 } 5885 EXPORT_SYMBOL(skb_vlan_pop); 5886 5887 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 5888 * Expects skb->data at mac header. 5889 */ 5890 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 5891 { 5892 if (skb_vlan_tag_present(skb)) { 5893 int offset = skb->data - skb_mac_header(skb); 5894 int err; 5895 5896 if (WARN_ONCE(offset, 5897 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 5898 offset)) { 5899 return -EINVAL; 5900 } 5901 5902 err = __vlan_insert_tag(skb, skb->vlan_proto, 5903 skb_vlan_tag_get(skb)); 5904 if (err) 5905 return err; 5906 5907 skb->protocol = skb->vlan_proto; 5908 skb->mac_len += VLAN_HLEN; 5909 5910 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5911 } 5912 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5913 return 0; 5914 } 5915 EXPORT_SYMBOL(skb_vlan_push); 5916 5917 /** 5918 * skb_eth_pop() - Drop the Ethernet header at the head of a packet 5919 * 5920 * @skb: Socket buffer to modify 5921 * 5922 * Drop the Ethernet header of @skb. 5923 * 5924 * Expects that skb->data points to the mac header and that no VLAN tags are 5925 * present. 5926 * 5927 * Returns 0 on success, -errno otherwise. 5928 */ 5929 int skb_eth_pop(struct sk_buff *skb) 5930 { 5931 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) || 5932 skb_network_offset(skb) < ETH_HLEN) 5933 return -EPROTO; 5934 5935 skb_pull_rcsum(skb, ETH_HLEN); 5936 skb_reset_mac_header(skb); 5937 skb_reset_mac_len(skb); 5938 5939 return 0; 5940 } 5941 EXPORT_SYMBOL(skb_eth_pop); 5942 5943 /** 5944 * skb_eth_push() - Add a new Ethernet header at the head of a packet 5945 * 5946 * @skb: Socket buffer to modify 5947 * @dst: Destination MAC address of the new header 5948 * @src: Source MAC address of the new header 5949 * 5950 * Prepend @skb with a new Ethernet header. 5951 * 5952 * Expects that skb->data points to the mac header, which must be empty. 5953 * 5954 * Returns 0 on success, -errno otherwise. 5955 */ 5956 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst, 5957 const unsigned char *src) 5958 { 5959 struct ethhdr *eth; 5960 int err; 5961 5962 if (skb_network_offset(skb) || skb_vlan_tag_present(skb)) 5963 return -EPROTO; 5964 5965 err = skb_cow_head(skb, sizeof(*eth)); 5966 if (err < 0) 5967 return err; 5968 5969 skb_push(skb, sizeof(*eth)); 5970 skb_reset_mac_header(skb); 5971 skb_reset_mac_len(skb); 5972 5973 eth = eth_hdr(skb); 5974 ether_addr_copy(eth->h_dest, dst); 5975 ether_addr_copy(eth->h_source, src); 5976 eth->h_proto = skb->protocol; 5977 5978 skb_postpush_rcsum(skb, eth, sizeof(*eth)); 5979 5980 return 0; 5981 } 5982 EXPORT_SYMBOL(skb_eth_push); 5983 5984 /* Update the ethertype of hdr and the skb csum value if required. */ 5985 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr, 5986 __be16 ethertype) 5987 { 5988 if (skb->ip_summed == CHECKSUM_COMPLETE) { 5989 __be16 diff[] = { ~hdr->h_proto, ethertype }; 5990 5991 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 5992 } 5993 5994 hdr->h_proto = ethertype; 5995 } 5996 5997 /** 5998 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of 5999 * the packet 6000 * 6001 * @skb: buffer 6002 * @mpls_lse: MPLS label stack entry to push 6003 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848) 6004 * @mac_len: length of the MAC header 6005 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is 6006 * ethernet 6007 * 6008 * Expects skb->data at mac header. 6009 * 6010 * Returns 0 on success, -errno otherwise. 6011 */ 6012 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto, 6013 int mac_len, bool ethernet) 6014 { 6015 struct mpls_shim_hdr *lse; 6016 int err; 6017 6018 if (unlikely(!eth_p_mpls(mpls_proto))) 6019 return -EINVAL; 6020 6021 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */ 6022 if (skb->encapsulation) 6023 return -EINVAL; 6024 6025 err = skb_cow_head(skb, MPLS_HLEN); 6026 if (unlikely(err)) 6027 return err; 6028 6029 if (!skb->inner_protocol) { 6030 skb_set_inner_network_header(skb, skb_network_offset(skb)); 6031 skb_set_inner_protocol(skb, skb->protocol); 6032 } 6033 6034 skb_push(skb, MPLS_HLEN); 6035 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb), 6036 mac_len); 6037 skb_reset_mac_header(skb); 6038 skb_set_network_header(skb, mac_len); 6039 skb_reset_mac_len(skb); 6040 6041 lse = mpls_hdr(skb); 6042 lse->label_stack_entry = mpls_lse; 6043 skb_postpush_rcsum(skb, lse, MPLS_HLEN); 6044 6045 if (ethernet && mac_len >= ETH_HLEN) 6046 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto); 6047 skb->protocol = mpls_proto; 6048 6049 return 0; 6050 } 6051 EXPORT_SYMBOL_GPL(skb_mpls_push); 6052 6053 /** 6054 * skb_mpls_pop() - pop the outermost MPLS header 6055 * 6056 * @skb: buffer 6057 * @next_proto: ethertype of header after popped MPLS header 6058 * @mac_len: length of the MAC header 6059 * @ethernet: flag to indicate if the packet is ethernet 6060 * 6061 * Expects skb->data at mac header. 6062 * 6063 * Returns 0 on success, -errno otherwise. 6064 */ 6065 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, 6066 bool ethernet) 6067 { 6068 int err; 6069 6070 if (unlikely(!eth_p_mpls(skb->protocol))) 6071 return 0; 6072 6073 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN); 6074 if (unlikely(err)) 6075 return err; 6076 6077 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN); 6078 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb), 6079 mac_len); 6080 6081 __skb_pull(skb, MPLS_HLEN); 6082 skb_reset_mac_header(skb); 6083 skb_set_network_header(skb, mac_len); 6084 6085 if (ethernet && mac_len >= ETH_HLEN) { 6086 struct ethhdr *hdr; 6087 6088 /* use mpls_hdr() to get ethertype to account for VLANs. */ 6089 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN); 6090 skb_mod_eth_type(skb, hdr, next_proto); 6091 } 6092 skb->protocol = next_proto; 6093 6094 return 0; 6095 } 6096 EXPORT_SYMBOL_GPL(skb_mpls_pop); 6097 6098 /** 6099 * skb_mpls_update_lse() - modify outermost MPLS header and update csum 6100 * 6101 * @skb: buffer 6102 * @mpls_lse: new MPLS label stack entry to update to 6103 * 6104 * Expects skb->data at mac header. 6105 * 6106 * Returns 0 on success, -errno otherwise. 6107 */ 6108 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse) 6109 { 6110 int err; 6111 6112 if (unlikely(!eth_p_mpls(skb->protocol))) 6113 return -EINVAL; 6114 6115 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN); 6116 if (unlikely(err)) 6117 return err; 6118 6119 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6120 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse }; 6121 6122 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6123 } 6124 6125 mpls_hdr(skb)->label_stack_entry = mpls_lse; 6126 6127 return 0; 6128 } 6129 EXPORT_SYMBOL_GPL(skb_mpls_update_lse); 6130 6131 /** 6132 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header 6133 * 6134 * @skb: buffer 6135 * 6136 * Expects skb->data at mac header. 6137 * 6138 * Returns 0 on success, -errno otherwise. 6139 */ 6140 int skb_mpls_dec_ttl(struct sk_buff *skb) 6141 { 6142 u32 lse; 6143 u8 ttl; 6144 6145 if (unlikely(!eth_p_mpls(skb->protocol))) 6146 return -EINVAL; 6147 6148 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN)) 6149 return -ENOMEM; 6150 6151 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry); 6152 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT; 6153 if (!--ttl) 6154 return -EINVAL; 6155 6156 lse &= ~MPLS_LS_TTL_MASK; 6157 lse |= ttl << MPLS_LS_TTL_SHIFT; 6158 6159 return skb_mpls_update_lse(skb, cpu_to_be32(lse)); 6160 } 6161 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); 6162 6163 /** 6164 * alloc_skb_with_frags - allocate skb with page frags 6165 * 6166 * @header_len: size of linear part 6167 * @data_len: needed length in frags 6168 * @max_page_order: max page order desired. 6169 * @errcode: pointer to error code if any 6170 * @gfp_mask: allocation mask 6171 * 6172 * This can be used to allocate a paged skb, given a maximal order for frags. 6173 */ 6174 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 6175 unsigned long data_len, 6176 int max_page_order, 6177 int *errcode, 6178 gfp_t gfp_mask) 6179 { 6180 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 6181 unsigned long chunk; 6182 struct sk_buff *skb; 6183 struct page *page; 6184 int i; 6185 6186 *errcode = -EMSGSIZE; 6187 /* Note this test could be relaxed, if we succeed to allocate 6188 * high order pages... 6189 */ 6190 if (npages > MAX_SKB_FRAGS) 6191 return NULL; 6192 6193 *errcode = -ENOBUFS; 6194 skb = alloc_skb(header_len, gfp_mask); 6195 if (!skb) 6196 return NULL; 6197 6198 skb->truesize += npages << PAGE_SHIFT; 6199 6200 for (i = 0; npages > 0; i++) { 6201 int order = max_page_order; 6202 6203 while (order) { 6204 if (npages >= 1 << order) { 6205 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 6206 __GFP_COMP | 6207 __GFP_NOWARN, 6208 order); 6209 if (page) 6210 goto fill_page; 6211 /* Do not retry other high order allocations */ 6212 order = 1; 6213 max_page_order = 0; 6214 } 6215 order--; 6216 } 6217 page = alloc_page(gfp_mask); 6218 if (!page) 6219 goto failure; 6220 fill_page: 6221 chunk = min_t(unsigned long, data_len, 6222 PAGE_SIZE << order); 6223 skb_fill_page_desc(skb, i, page, 0, chunk); 6224 data_len -= chunk; 6225 npages -= 1 << order; 6226 } 6227 return skb; 6228 6229 failure: 6230 kfree_skb(skb); 6231 return NULL; 6232 } 6233 EXPORT_SYMBOL(alloc_skb_with_frags); 6234 6235 /* carve out the first off bytes from skb when off < headlen */ 6236 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 6237 const int headlen, gfp_t gfp_mask) 6238 { 6239 int i; 6240 unsigned int size = skb_end_offset(skb); 6241 int new_hlen = headlen - off; 6242 u8 *data; 6243 6244 if (skb_pfmemalloc(skb)) 6245 gfp_mask |= __GFP_MEMALLOC; 6246 6247 size = SKB_DATA_ALIGN(size); 6248 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 6249 size = kmalloc_size_roundup(size); 6250 data = kmalloc_reserve(size, gfp_mask, NUMA_NO_NODE, NULL); 6251 if (!data) 6252 return -ENOMEM; 6253 size = SKB_WITH_OVERHEAD(size); 6254 6255 /* Copy real data, and all frags */ 6256 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 6257 skb->len -= off; 6258 6259 memcpy((struct skb_shared_info *)(data + size), 6260 skb_shinfo(skb), 6261 offsetof(struct skb_shared_info, 6262 frags[skb_shinfo(skb)->nr_frags])); 6263 if (skb_cloned(skb)) { 6264 /* drop the old head gracefully */ 6265 if (skb_orphan_frags(skb, gfp_mask)) { 6266 kfree(data); 6267 return -ENOMEM; 6268 } 6269 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 6270 skb_frag_ref(skb, i); 6271 if (skb_has_frag_list(skb)) 6272 skb_clone_fraglist(skb); 6273 skb_release_data(skb, SKB_CONSUMED); 6274 } else { 6275 /* we can reuse existing recount- all we did was 6276 * relocate values 6277 */ 6278 skb_free_head(skb); 6279 } 6280 6281 skb->head = data; 6282 skb->data = data; 6283 skb->head_frag = 0; 6284 skb_set_end_offset(skb, size); 6285 skb_set_tail_pointer(skb, skb_headlen(skb)); 6286 skb_headers_offset_update(skb, 0); 6287 skb->cloned = 0; 6288 skb->hdr_len = 0; 6289 skb->nohdr = 0; 6290 atomic_set(&skb_shinfo(skb)->dataref, 1); 6291 6292 return 0; 6293 } 6294 6295 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 6296 6297 /* carve out the first eat bytes from skb's frag_list. May recurse into 6298 * pskb_carve() 6299 */ 6300 static int pskb_carve_frag_list(struct sk_buff *skb, 6301 struct skb_shared_info *shinfo, int eat, 6302 gfp_t gfp_mask) 6303 { 6304 struct sk_buff *list = shinfo->frag_list; 6305 struct sk_buff *clone = NULL; 6306 struct sk_buff *insp = NULL; 6307 6308 do { 6309 if (!list) { 6310 pr_err("Not enough bytes to eat. Want %d\n", eat); 6311 return -EFAULT; 6312 } 6313 if (list->len <= eat) { 6314 /* Eaten as whole. */ 6315 eat -= list->len; 6316 list = list->next; 6317 insp = list; 6318 } else { 6319 /* Eaten partially. */ 6320 if (skb_shared(list)) { 6321 clone = skb_clone(list, gfp_mask); 6322 if (!clone) 6323 return -ENOMEM; 6324 insp = list->next; 6325 list = clone; 6326 } else { 6327 /* This may be pulled without problems. */ 6328 insp = list; 6329 } 6330 if (pskb_carve(list, eat, gfp_mask) < 0) { 6331 kfree_skb(clone); 6332 return -ENOMEM; 6333 } 6334 break; 6335 } 6336 } while (eat); 6337 6338 /* Free pulled out fragments. */ 6339 while ((list = shinfo->frag_list) != insp) { 6340 shinfo->frag_list = list->next; 6341 consume_skb(list); 6342 } 6343 /* And insert new clone at head. */ 6344 if (clone) { 6345 clone->next = list; 6346 shinfo->frag_list = clone; 6347 } 6348 return 0; 6349 } 6350 6351 /* carve off first len bytes from skb. Split line (off) is in the 6352 * non-linear part of skb 6353 */ 6354 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 6355 int pos, gfp_t gfp_mask) 6356 { 6357 int i, k = 0; 6358 unsigned int size = skb_end_offset(skb); 6359 u8 *data; 6360 const int nfrags = skb_shinfo(skb)->nr_frags; 6361 struct skb_shared_info *shinfo; 6362 6363 if (skb_pfmemalloc(skb)) 6364 gfp_mask |= __GFP_MEMALLOC; 6365 6366 size = SKB_DATA_ALIGN(size); 6367 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 6368 size = kmalloc_size_roundup(size); 6369 data = kmalloc_reserve(size, gfp_mask, NUMA_NO_NODE, NULL); 6370 if (!data) 6371 return -ENOMEM; 6372 size = SKB_WITH_OVERHEAD(size); 6373 6374 memcpy((struct skb_shared_info *)(data + size), 6375 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0])); 6376 if (skb_orphan_frags(skb, gfp_mask)) { 6377 kfree(data); 6378 return -ENOMEM; 6379 } 6380 shinfo = (struct skb_shared_info *)(data + size); 6381 for (i = 0; i < nfrags; i++) { 6382 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 6383 6384 if (pos + fsize > off) { 6385 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 6386 6387 if (pos < off) { 6388 /* Split frag. 6389 * We have two variants in this case: 6390 * 1. Move all the frag to the second 6391 * part, if it is possible. F.e. 6392 * this approach is mandatory for TUX, 6393 * where splitting is expensive. 6394 * 2. Split is accurately. We make this. 6395 */ 6396 skb_frag_off_add(&shinfo->frags[0], off - pos); 6397 skb_frag_size_sub(&shinfo->frags[0], off - pos); 6398 } 6399 skb_frag_ref(skb, i); 6400 k++; 6401 } 6402 pos += fsize; 6403 } 6404 shinfo->nr_frags = k; 6405 if (skb_has_frag_list(skb)) 6406 skb_clone_fraglist(skb); 6407 6408 /* split line is in frag list */ 6409 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) { 6410 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */ 6411 if (skb_has_frag_list(skb)) 6412 kfree_skb_list(skb_shinfo(skb)->frag_list); 6413 kfree(data); 6414 return -ENOMEM; 6415 } 6416 skb_release_data(skb, SKB_CONSUMED); 6417 6418 skb->head = data; 6419 skb->head_frag = 0; 6420 skb->data = data; 6421 skb_set_end_offset(skb, size); 6422 skb_reset_tail_pointer(skb); 6423 skb_headers_offset_update(skb, 0); 6424 skb->cloned = 0; 6425 skb->hdr_len = 0; 6426 skb->nohdr = 0; 6427 skb->len -= off; 6428 skb->data_len = skb->len; 6429 atomic_set(&skb_shinfo(skb)->dataref, 1); 6430 return 0; 6431 } 6432 6433 /* remove len bytes from the beginning of the skb */ 6434 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 6435 { 6436 int headlen = skb_headlen(skb); 6437 6438 if (len < headlen) 6439 return pskb_carve_inside_header(skb, len, headlen, gfp); 6440 else 6441 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 6442 } 6443 6444 /* Extract to_copy bytes starting at off from skb, and return this in 6445 * a new skb 6446 */ 6447 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 6448 int to_copy, gfp_t gfp) 6449 { 6450 struct sk_buff *clone = skb_clone(skb, gfp); 6451 6452 if (!clone) 6453 return NULL; 6454 6455 if (pskb_carve(clone, off, gfp) < 0 || 6456 pskb_trim(clone, to_copy)) { 6457 kfree_skb(clone); 6458 return NULL; 6459 } 6460 return clone; 6461 } 6462 EXPORT_SYMBOL(pskb_extract); 6463 6464 /** 6465 * skb_condense - try to get rid of fragments/frag_list if possible 6466 * @skb: buffer 6467 * 6468 * Can be used to save memory before skb is added to a busy queue. 6469 * If packet has bytes in frags and enough tail room in skb->head, 6470 * pull all of them, so that we can free the frags right now and adjust 6471 * truesize. 6472 * Notes: 6473 * We do not reallocate skb->head thus can not fail. 6474 * Caller must re-evaluate skb->truesize if needed. 6475 */ 6476 void skb_condense(struct sk_buff *skb) 6477 { 6478 if (skb->data_len) { 6479 if (skb->data_len > skb->end - skb->tail || 6480 skb_cloned(skb)) 6481 return; 6482 6483 /* Nice, we can free page frag(s) right now */ 6484 __pskb_pull_tail(skb, skb->data_len); 6485 } 6486 /* At this point, skb->truesize might be over estimated, 6487 * because skb had a fragment, and fragments do not tell 6488 * their truesize. 6489 * When we pulled its content into skb->head, fragment 6490 * was freed, but __pskb_pull_tail() could not possibly 6491 * adjust skb->truesize, not knowing the frag truesize. 6492 */ 6493 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 6494 } 6495 EXPORT_SYMBOL(skb_condense); 6496 6497 #ifdef CONFIG_SKB_EXTENSIONS 6498 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id) 6499 { 6500 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE); 6501 } 6502 6503 /** 6504 * __skb_ext_alloc - allocate a new skb extensions storage 6505 * 6506 * @flags: See kmalloc(). 6507 * 6508 * Returns the newly allocated pointer. The pointer can later attached to a 6509 * skb via __skb_ext_set(). 6510 * Note: caller must handle the skb_ext as an opaque data. 6511 */ 6512 struct skb_ext *__skb_ext_alloc(gfp_t flags) 6513 { 6514 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags); 6515 6516 if (new) { 6517 memset(new->offset, 0, sizeof(new->offset)); 6518 refcount_set(&new->refcnt, 1); 6519 } 6520 6521 return new; 6522 } 6523 6524 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old, 6525 unsigned int old_active) 6526 { 6527 struct skb_ext *new; 6528 6529 if (refcount_read(&old->refcnt) == 1) 6530 return old; 6531 6532 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 6533 if (!new) 6534 return NULL; 6535 6536 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE); 6537 refcount_set(&new->refcnt, 1); 6538 6539 #ifdef CONFIG_XFRM 6540 if (old_active & (1 << SKB_EXT_SEC_PATH)) { 6541 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH); 6542 unsigned int i; 6543 6544 for (i = 0; i < sp->len; i++) 6545 xfrm_state_hold(sp->xvec[i]); 6546 } 6547 #endif 6548 __skb_ext_put(old); 6549 return new; 6550 } 6551 6552 /** 6553 * __skb_ext_set - attach the specified extension storage to this skb 6554 * @skb: buffer 6555 * @id: extension id 6556 * @ext: extension storage previously allocated via __skb_ext_alloc() 6557 * 6558 * Existing extensions, if any, are cleared. 6559 * 6560 * Returns the pointer to the extension. 6561 */ 6562 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id, 6563 struct skb_ext *ext) 6564 { 6565 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext); 6566 6567 skb_ext_put(skb); 6568 newlen = newoff + skb_ext_type_len[id]; 6569 ext->chunks = newlen; 6570 ext->offset[id] = newoff; 6571 skb->extensions = ext; 6572 skb->active_extensions = 1 << id; 6573 return skb_ext_get_ptr(ext, id); 6574 } 6575 6576 /** 6577 * skb_ext_add - allocate space for given extension, COW if needed 6578 * @skb: buffer 6579 * @id: extension to allocate space for 6580 * 6581 * Allocates enough space for the given extension. 6582 * If the extension is already present, a pointer to that extension 6583 * is returned. 6584 * 6585 * If the skb was cloned, COW applies and the returned memory can be 6586 * modified without changing the extension space of clones buffers. 6587 * 6588 * Returns pointer to the extension or NULL on allocation failure. 6589 */ 6590 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id) 6591 { 6592 struct skb_ext *new, *old = NULL; 6593 unsigned int newlen, newoff; 6594 6595 if (skb->active_extensions) { 6596 old = skb->extensions; 6597 6598 new = skb_ext_maybe_cow(old, skb->active_extensions); 6599 if (!new) 6600 return NULL; 6601 6602 if (__skb_ext_exist(new, id)) 6603 goto set_active; 6604 6605 newoff = new->chunks; 6606 } else { 6607 newoff = SKB_EXT_CHUNKSIZEOF(*new); 6608 6609 new = __skb_ext_alloc(GFP_ATOMIC); 6610 if (!new) 6611 return NULL; 6612 } 6613 6614 newlen = newoff + skb_ext_type_len[id]; 6615 new->chunks = newlen; 6616 new->offset[id] = newoff; 6617 set_active: 6618 skb->slow_gro = 1; 6619 skb->extensions = new; 6620 skb->active_extensions |= 1 << id; 6621 return skb_ext_get_ptr(new, id); 6622 } 6623 EXPORT_SYMBOL(skb_ext_add); 6624 6625 #ifdef CONFIG_XFRM 6626 static void skb_ext_put_sp(struct sec_path *sp) 6627 { 6628 unsigned int i; 6629 6630 for (i = 0; i < sp->len; i++) 6631 xfrm_state_put(sp->xvec[i]); 6632 } 6633 #endif 6634 6635 #ifdef CONFIG_MCTP_FLOWS 6636 static void skb_ext_put_mctp(struct mctp_flow *flow) 6637 { 6638 if (flow->key) 6639 mctp_key_unref(flow->key); 6640 } 6641 #endif 6642 6643 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 6644 { 6645 struct skb_ext *ext = skb->extensions; 6646 6647 skb->active_extensions &= ~(1 << id); 6648 if (skb->active_extensions == 0) { 6649 skb->extensions = NULL; 6650 __skb_ext_put(ext); 6651 #ifdef CONFIG_XFRM 6652 } else if (id == SKB_EXT_SEC_PATH && 6653 refcount_read(&ext->refcnt) == 1) { 6654 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH); 6655 6656 skb_ext_put_sp(sp); 6657 sp->len = 0; 6658 #endif 6659 } 6660 } 6661 EXPORT_SYMBOL(__skb_ext_del); 6662 6663 void __skb_ext_put(struct skb_ext *ext) 6664 { 6665 /* If this is last clone, nothing can increment 6666 * it after check passes. Avoids one atomic op. 6667 */ 6668 if (refcount_read(&ext->refcnt) == 1) 6669 goto free_now; 6670 6671 if (!refcount_dec_and_test(&ext->refcnt)) 6672 return; 6673 free_now: 6674 #ifdef CONFIG_XFRM 6675 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH)) 6676 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 6677 #endif 6678 #ifdef CONFIG_MCTP_FLOWS 6679 if (__skb_ext_exist(ext, SKB_EXT_MCTP)) 6680 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP)); 6681 #endif 6682 6683 kmem_cache_free(skbuff_ext_cache, ext); 6684 } 6685 EXPORT_SYMBOL(__skb_ext_put); 6686 #endif /* CONFIG_SKB_EXTENSIONS */ 6687 6688 /** 6689 * skb_attempt_defer_free - queue skb for remote freeing 6690 * @skb: buffer 6691 * 6692 * Put @skb in a per-cpu list, using the cpu which 6693 * allocated the skb/pages to reduce false sharing 6694 * and memory zone spinlock contention. 6695 */ 6696 void skb_attempt_defer_free(struct sk_buff *skb) 6697 { 6698 int cpu = skb->alloc_cpu; 6699 struct softnet_data *sd; 6700 unsigned long flags; 6701 unsigned int defer_max; 6702 bool kick; 6703 6704 if (WARN_ON_ONCE(cpu >= nr_cpu_ids) || 6705 !cpu_online(cpu) || 6706 cpu == raw_smp_processor_id()) { 6707 nodefer: __kfree_skb(skb); 6708 return; 6709 } 6710 6711 sd = &per_cpu(softnet_data, cpu); 6712 defer_max = READ_ONCE(sysctl_skb_defer_max); 6713 if (READ_ONCE(sd->defer_count) >= defer_max) 6714 goto nodefer; 6715 6716 spin_lock_irqsave(&sd->defer_lock, flags); 6717 /* Send an IPI every time queue reaches half capacity. */ 6718 kick = sd->defer_count == (defer_max >> 1); 6719 /* Paired with the READ_ONCE() few lines above */ 6720 WRITE_ONCE(sd->defer_count, sd->defer_count + 1); 6721 6722 skb->next = sd->defer_list; 6723 /* Paired with READ_ONCE() in skb_defer_free_flush() */ 6724 WRITE_ONCE(sd->defer_list, skb); 6725 spin_unlock_irqrestore(&sd->defer_lock, flags); 6726 6727 /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU 6728 * if we are unlucky enough (this seems very unlikely). 6729 */ 6730 if (unlikely(kick) && !cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) 6731 smp_call_function_single_async(cpu, &sd->defer_csd); 6732 } 6733