1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Routines having to do with the 'struct sk_buff' memory handlers. 4 * 5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 6 * Florian La Roche <rzsfl@rz.uni-sb.de> 7 * 8 * Fixes: 9 * Alan Cox : Fixed the worst of the load 10 * balancer bugs. 11 * Dave Platt : Interrupt stacking fix. 12 * Richard Kooijman : Timestamp fixes. 13 * Alan Cox : Changed buffer format. 14 * Alan Cox : destructor hook for AF_UNIX etc. 15 * Linus Torvalds : Better skb_clone. 16 * Alan Cox : Added skb_copy. 17 * Alan Cox : Added all the changed routines Linus 18 * only put in the headers 19 * Ray VanTassle : Fixed --skb->lock in free 20 * Alan Cox : skb_copy copy arp field 21 * Andi Kleen : slabified it. 22 * Robert Olsson : Removed skb_head_pool 23 * 24 * NOTE: 25 * The __skb_ routines should be called with interrupts 26 * disabled, or you better be *real* sure that the operation is atomic 27 * with respect to whatever list is being frobbed (e.g. via lock_sock() 28 * or via disabling bottom half handlers, etc). 29 */ 30 31 /* 32 * The functions in this file will not compile correctly with gcc 2.4.x 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/module.h> 38 #include <linux/types.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/interrupt.h> 42 #include <linux/in.h> 43 #include <linux/inet.h> 44 #include <linux/slab.h> 45 #include <linux/tcp.h> 46 #include <linux/udp.h> 47 #include <linux/sctp.h> 48 #include <linux/netdevice.h> 49 #ifdef CONFIG_NET_CLS_ACT 50 #include <net/pkt_sched.h> 51 #endif 52 #include <linux/string.h> 53 #include <linux/skbuff.h> 54 #include <linux/splice.h> 55 #include <linux/cache.h> 56 #include <linux/rtnetlink.h> 57 #include <linux/init.h> 58 #include <linux/scatterlist.h> 59 #include <linux/errqueue.h> 60 #include <linux/prefetch.h> 61 #include <linux/bitfield.h> 62 #include <linux/if_vlan.h> 63 #include <linux/mpls.h> 64 #include <linux/kcov.h> 65 66 #include <net/protocol.h> 67 #include <net/dst.h> 68 #include <net/sock.h> 69 #include <net/checksum.h> 70 #include <net/ip6_checksum.h> 71 #include <net/xfrm.h> 72 #include <net/mpls.h> 73 #include <net/mptcp.h> 74 #include <net/mctp.h> 75 #include <net/page_pool.h> 76 #include <net/dropreason.h> 77 78 #include <linux/uaccess.h> 79 #include <trace/events/skb.h> 80 #include <linux/highmem.h> 81 #include <linux/capability.h> 82 #include <linux/user_namespace.h> 83 #include <linux/indirect_call_wrapper.h> 84 #include <linux/textsearch.h> 85 86 #include "dev.h" 87 #include "sock_destructor.h" 88 89 struct kmem_cache *skbuff_cache __ro_after_init; 90 static struct kmem_cache *skbuff_fclone_cache __ro_after_init; 91 #ifdef CONFIG_SKB_EXTENSIONS 92 static struct kmem_cache *skbuff_ext_cache __ro_after_init; 93 #endif 94 95 /* skb_small_head_cache and related code is only supported 96 * for CONFIG_SLAB and CONFIG_SLUB. 97 * As soon as SLOB is removed from the kernel, we can clean up this. 98 */ 99 #if !defined(CONFIG_SLOB) 100 # define HAVE_SKB_SMALL_HEAD_CACHE 1 101 #endif 102 103 #ifdef HAVE_SKB_SMALL_HEAD_CACHE 104 static struct kmem_cache *skb_small_head_cache __ro_after_init; 105 106 #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER) 107 108 /* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two. 109 * This should ensure that SKB_SMALL_HEAD_HEADROOM is a unique 110 * size, and we can differentiate heads from skb_small_head_cache 111 * vs system slabs by looking at their size (skb_end_offset()). 112 */ 113 #define SKB_SMALL_HEAD_CACHE_SIZE \ 114 (is_power_of_2(SKB_SMALL_HEAD_SIZE) ? \ 115 (SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) : \ 116 SKB_SMALL_HEAD_SIZE) 117 118 #define SKB_SMALL_HEAD_HEADROOM \ 119 SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) 120 #endif /* HAVE_SKB_SMALL_HEAD_CACHE */ 121 122 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; 123 EXPORT_SYMBOL(sysctl_max_skb_frags); 124 125 #undef FN 126 #define FN(reason) [SKB_DROP_REASON_##reason] = #reason, 127 static const char * const drop_reasons[] = { 128 [SKB_CONSUMED] = "CONSUMED", 129 DEFINE_DROP_REASON(FN, FN) 130 }; 131 132 static const struct drop_reason_list drop_reasons_core = { 133 .reasons = drop_reasons, 134 .n_reasons = ARRAY_SIZE(drop_reasons), 135 }; 136 137 const struct drop_reason_list __rcu * 138 drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = { 139 [SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core), 140 }; 141 EXPORT_SYMBOL(drop_reasons_by_subsys); 142 143 /** 144 * drop_reasons_register_subsys - register another drop reason subsystem 145 * @subsys: the subsystem to register, must not be the core 146 * @list: the list of drop reasons within the subsystem, must point to 147 * a statically initialized list 148 */ 149 void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys, 150 const struct drop_reason_list *list) 151 { 152 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 153 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 154 "invalid subsystem %d\n", subsys)) 155 return; 156 157 /* must point to statically allocated memory, so INIT is OK */ 158 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list); 159 } 160 EXPORT_SYMBOL_GPL(drop_reasons_register_subsys); 161 162 /** 163 * drop_reasons_unregister_subsys - unregister a drop reason subsystem 164 * @subsys: the subsystem to remove, must not be the core 165 * 166 * Note: This will synchronize_rcu() to ensure no users when it returns. 167 */ 168 void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys) 169 { 170 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 171 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 172 "invalid subsystem %d\n", subsys)) 173 return; 174 175 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL); 176 177 synchronize_rcu(); 178 } 179 EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys); 180 181 /** 182 * skb_panic - private function for out-of-line support 183 * @skb: buffer 184 * @sz: size 185 * @addr: address 186 * @msg: skb_over_panic or skb_under_panic 187 * 188 * Out-of-line support for skb_put() and skb_push(). 189 * Called via the wrapper skb_over_panic() or skb_under_panic(). 190 * Keep out of line to prevent kernel bloat. 191 * __builtin_return_address is not used because it is not always reliable. 192 */ 193 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 194 const char msg[]) 195 { 196 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n", 197 msg, addr, skb->len, sz, skb->head, skb->data, 198 (unsigned long)skb->tail, (unsigned long)skb->end, 199 skb->dev ? skb->dev->name : "<NULL>"); 200 BUG(); 201 } 202 203 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 204 { 205 skb_panic(skb, sz, addr, __func__); 206 } 207 208 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 209 { 210 skb_panic(skb, sz, addr, __func__); 211 } 212 213 #define NAPI_SKB_CACHE_SIZE 64 214 #define NAPI_SKB_CACHE_BULK 16 215 #define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2) 216 217 #if PAGE_SIZE == SZ_4K 218 219 #define NAPI_HAS_SMALL_PAGE_FRAG 1 220 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) ((nc).pfmemalloc) 221 222 /* specialized page frag allocator using a single order 0 page 223 * and slicing it into 1K sized fragment. Constrained to systems 224 * with a very limited amount of 1K fragments fitting a single 225 * page - to avoid excessive truesize underestimation 226 */ 227 228 struct page_frag_1k { 229 void *va; 230 u16 offset; 231 bool pfmemalloc; 232 }; 233 234 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp) 235 { 236 struct page *page; 237 int offset; 238 239 offset = nc->offset - SZ_1K; 240 if (likely(offset >= 0)) 241 goto use_frag; 242 243 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 244 if (!page) 245 return NULL; 246 247 nc->va = page_address(page); 248 nc->pfmemalloc = page_is_pfmemalloc(page); 249 offset = PAGE_SIZE - SZ_1K; 250 page_ref_add(page, offset / SZ_1K); 251 252 use_frag: 253 nc->offset = offset; 254 return nc->va + offset; 255 } 256 #else 257 258 /* the small page is actually unused in this build; add dummy helpers 259 * to please the compiler and avoid later preprocessor's conditionals 260 */ 261 #define NAPI_HAS_SMALL_PAGE_FRAG 0 262 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) false 263 264 struct page_frag_1k { 265 }; 266 267 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp_mask) 268 { 269 return NULL; 270 } 271 272 #endif 273 274 struct napi_alloc_cache { 275 struct page_frag_cache page; 276 struct page_frag_1k page_small; 277 unsigned int skb_count; 278 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 279 }; 280 281 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 282 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 283 284 /* Double check that napi_get_frags() allocates skbs with 285 * skb->head being backed by slab, not a page fragment. 286 * This is to make sure bug fixed in 3226b158e67c 287 * ("net: avoid 32 x truesize under-estimation for tiny skbs") 288 * does not accidentally come back. 289 */ 290 void napi_get_frags_check(struct napi_struct *napi) 291 { 292 struct sk_buff *skb; 293 294 local_bh_disable(); 295 skb = napi_get_frags(napi); 296 WARN_ON_ONCE(!NAPI_HAS_SMALL_PAGE_FRAG && skb && skb->head_frag); 297 napi_free_frags(napi); 298 local_bh_enable(); 299 } 300 301 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 302 { 303 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 304 305 fragsz = SKB_DATA_ALIGN(fragsz); 306 307 return page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); 308 } 309 EXPORT_SYMBOL(__napi_alloc_frag_align); 310 311 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 312 { 313 void *data; 314 315 fragsz = SKB_DATA_ALIGN(fragsz); 316 if (in_hardirq() || irqs_disabled()) { 317 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache); 318 319 data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask); 320 } else { 321 struct napi_alloc_cache *nc; 322 323 local_bh_disable(); 324 nc = this_cpu_ptr(&napi_alloc_cache); 325 data = page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); 326 local_bh_enable(); 327 } 328 return data; 329 } 330 EXPORT_SYMBOL(__netdev_alloc_frag_align); 331 332 static struct sk_buff *napi_skb_cache_get(void) 333 { 334 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 335 struct sk_buff *skb; 336 337 if (unlikely(!nc->skb_count)) { 338 nc->skb_count = kmem_cache_alloc_bulk(skbuff_cache, 339 GFP_ATOMIC, 340 NAPI_SKB_CACHE_BULK, 341 nc->skb_cache); 342 if (unlikely(!nc->skb_count)) 343 return NULL; 344 } 345 346 skb = nc->skb_cache[--nc->skb_count]; 347 kasan_unpoison_object_data(skbuff_cache, skb); 348 349 return skb; 350 } 351 352 static inline void __finalize_skb_around(struct sk_buff *skb, void *data, 353 unsigned int size) 354 { 355 struct skb_shared_info *shinfo; 356 357 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 358 359 /* Assumes caller memset cleared SKB */ 360 skb->truesize = SKB_TRUESIZE(size); 361 refcount_set(&skb->users, 1); 362 skb->head = data; 363 skb->data = data; 364 skb_reset_tail_pointer(skb); 365 skb_set_end_offset(skb, size); 366 skb->mac_header = (typeof(skb->mac_header))~0U; 367 skb->transport_header = (typeof(skb->transport_header))~0U; 368 skb->alloc_cpu = raw_smp_processor_id(); 369 /* make sure we initialize shinfo sequentially */ 370 shinfo = skb_shinfo(skb); 371 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 372 atomic_set(&shinfo->dataref, 1); 373 374 skb_set_kcov_handle(skb, kcov_common_handle()); 375 } 376 377 static inline void *__slab_build_skb(struct sk_buff *skb, void *data, 378 unsigned int *size) 379 { 380 void *resized; 381 382 /* Must find the allocation size (and grow it to match). */ 383 *size = ksize(data); 384 /* krealloc() will immediately return "data" when 385 * "ksize(data)" is requested: it is the existing upper 386 * bounds. As a result, GFP_ATOMIC will be ignored. Note 387 * that this "new" pointer needs to be passed back to the 388 * caller for use so the __alloc_size hinting will be 389 * tracked correctly. 390 */ 391 resized = krealloc(data, *size, GFP_ATOMIC); 392 WARN_ON_ONCE(resized != data); 393 return resized; 394 } 395 396 /* build_skb() variant which can operate on slab buffers. 397 * Note that this should be used sparingly as slab buffers 398 * cannot be combined efficiently by GRO! 399 */ 400 struct sk_buff *slab_build_skb(void *data) 401 { 402 struct sk_buff *skb; 403 unsigned int size; 404 405 skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); 406 if (unlikely(!skb)) 407 return NULL; 408 409 memset(skb, 0, offsetof(struct sk_buff, tail)); 410 data = __slab_build_skb(skb, data, &size); 411 __finalize_skb_around(skb, data, size); 412 413 return skb; 414 } 415 EXPORT_SYMBOL(slab_build_skb); 416 417 /* Caller must provide SKB that is memset cleared */ 418 static void __build_skb_around(struct sk_buff *skb, void *data, 419 unsigned int frag_size) 420 { 421 unsigned int size = frag_size; 422 423 /* frag_size == 0 is considered deprecated now. Callers 424 * using slab buffer should use slab_build_skb() instead. 425 */ 426 if (WARN_ONCE(size == 0, "Use slab_build_skb() instead")) 427 data = __slab_build_skb(skb, data, &size); 428 429 __finalize_skb_around(skb, data, size); 430 } 431 432 /** 433 * __build_skb - build a network buffer 434 * @data: data buffer provided by caller 435 * @frag_size: size of data (must not be 0) 436 * 437 * Allocate a new &sk_buff. Caller provides space holding head and 438 * skb_shared_info. @data must have been allocated from the page 439 * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc() 440 * allocation is deprecated, and callers should use slab_build_skb() 441 * instead.) 442 * The return is the new skb buffer. 443 * On a failure the return is %NULL, and @data is not freed. 444 * Notes : 445 * Before IO, driver allocates only data buffer where NIC put incoming frame 446 * Driver should add room at head (NET_SKB_PAD) and 447 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 448 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 449 * before giving packet to stack. 450 * RX rings only contains data buffers, not full skbs. 451 */ 452 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 453 { 454 struct sk_buff *skb; 455 456 skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); 457 if (unlikely(!skb)) 458 return NULL; 459 460 memset(skb, 0, offsetof(struct sk_buff, tail)); 461 __build_skb_around(skb, data, frag_size); 462 463 return skb; 464 } 465 466 /* build_skb() is wrapper over __build_skb(), that specifically 467 * takes care of skb->head and skb->pfmemalloc 468 */ 469 struct sk_buff *build_skb(void *data, unsigned int frag_size) 470 { 471 struct sk_buff *skb = __build_skb(data, frag_size); 472 473 if (likely(skb && frag_size)) { 474 skb->head_frag = 1; 475 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 476 } 477 return skb; 478 } 479 EXPORT_SYMBOL(build_skb); 480 481 /** 482 * build_skb_around - build a network buffer around provided skb 483 * @skb: sk_buff provide by caller, must be memset cleared 484 * @data: data buffer provided by caller 485 * @frag_size: size of data 486 */ 487 struct sk_buff *build_skb_around(struct sk_buff *skb, 488 void *data, unsigned int frag_size) 489 { 490 if (unlikely(!skb)) 491 return NULL; 492 493 __build_skb_around(skb, data, frag_size); 494 495 if (frag_size) { 496 skb->head_frag = 1; 497 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 498 } 499 return skb; 500 } 501 EXPORT_SYMBOL(build_skb_around); 502 503 /** 504 * __napi_build_skb - build a network buffer 505 * @data: data buffer provided by caller 506 * @frag_size: size of data 507 * 508 * Version of __build_skb() that uses NAPI percpu caches to obtain 509 * skbuff_head instead of inplace allocation. 510 * 511 * Returns a new &sk_buff on success, %NULL on allocation failure. 512 */ 513 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 514 { 515 struct sk_buff *skb; 516 517 skb = napi_skb_cache_get(); 518 if (unlikely(!skb)) 519 return NULL; 520 521 memset(skb, 0, offsetof(struct sk_buff, tail)); 522 __build_skb_around(skb, data, frag_size); 523 524 return skb; 525 } 526 527 /** 528 * napi_build_skb - build a network buffer 529 * @data: data buffer provided by caller 530 * @frag_size: size of data 531 * 532 * Version of __napi_build_skb() that takes care of skb->head_frag 533 * and skb->pfmemalloc when the data is a page or page fragment. 534 * 535 * Returns a new &sk_buff on success, %NULL on allocation failure. 536 */ 537 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) 538 { 539 struct sk_buff *skb = __napi_build_skb(data, frag_size); 540 541 if (likely(skb) && frag_size) { 542 skb->head_frag = 1; 543 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 544 } 545 546 return skb; 547 } 548 EXPORT_SYMBOL(napi_build_skb); 549 550 /* 551 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 552 * the caller if emergency pfmemalloc reserves are being used. If it is and 553 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 554 * may be used. Otherwise, the packet data may be discarded until enough 555 * memory is free 556 */ 557 static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, 558 bool *pfmemalloc) 559 { 560 bool ret_pfmemalloc = false; 561 unsigned int obj_size; 562 void *obj; 563 564 obj_size = SKB_HEAD_ALIGN(*size); 565 #ifdef HAVE_SKB_SMALL_HEAD_CACHE 566 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE && 567 !(flags & KMALLOC_NOT_NORMAL_BITS)) { 568 obj = kmem_cache_alloc_node(skb_small_head_cache, 569 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 570 node); 571 *size = SKB_SMALL_HEAD_CACHE_SIZE; 572 if (obj || !(gfp_pfmemalloc_allowed(flags))) 573 goto out; 574 /* Try again but now we are using pfmemalloc reserves */ 575 ret_pfmemalloc = true; 576 obj = kmem_cache_alloc_node(skb_small_head_cache, flags, node); 577 goto out; 578 } 579 #endif 580 *size = obj_size = kmalloc_size_roundup(obj_size); 581 /* 582 * Try a regular allocation, when that fails and we're not entitled 583 * to the reserves, fail. 584 */ 585 obj = kmalloc_node_track_caller(obj_size, 586 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 587 node); 588 if (obj || !(gfp_pfmemalloc_allowed(flags))) 589 goto out; 590 591 /* Try again but now we are using pfmemalloc reserves */ 592 ret_pfmemalloc = true; 593 obj = kmalloc_node_track_caller(obj_size, flags, node); 594 595 out: 596 if (pfmemalloc) 597 *pfmemalloc = ret_pfmemalloc; 598 599 return obj; 600 } 601 602 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 603 * 'private' fields and also do memory statistics to find all the 604 * [BEEP] leaks. 605 * 606 */ 607 608 /** 609 * __alloc_skb - allocate a network buffer 610 * @size: size to allocate 611 * @gfp_mask: allocation mask 612 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 613 * instead of head cache and allocate a cloned (child) skb. 614 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 615 * allocations in case the data is required for writeback 616 * @node: numa node to allocate memory on 617 * 618 * Allocate a new &sk_buff. The returned buffer has no headroom and a 619 * tail room of at least size bytes. The object has a reference count 620 * of one. The return is the buffer. On a failure the return is %NULL. 621 * 622 * Buffers may only be allocated from interrupts using a @gfp_mask of 623 * %GFP_ATOMIC. 624 */ 625 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 626 int flags, int node) 627 { 628 struct kmem_cache *cache; 629 struct sk_buff *skb; 630 bool pfmemalloc; 631 u8 *data; 632 633 cache = (flags & SKB_ALLOC_FCLONE) 634 ? skbuff_fclone_cache : skbuff_cache; 635 636 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 637 gfp_mask |= __GFP_MEMALLOC; 638 639 /* Get the HEAD */ 640 if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI && 641 likely(node == NUMA_NO_NODE || node == numa_mem_id())) 642 skb = napi_skb_cache_get(); 643 else 644 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); 645 if (unlikely(!skb)) 646 return NULL; 647 prefetchw(skb); 648 649 /* We do our best to align skb_shared_info on a separate cache 650 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 651 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 652 * Both skb->head and skb_shared_info are cache line aligned. 653 */ 654 data = kmalloc_reserve(&size, gfp_mask, node, &pfmemalloc); 655 if (unlikely(!data)) 656 goto nodata; 657 /* kmalloc_size_roundup() might give us more room than requested. 658 * Put skb_shared_info exactly at the end of allocated zone, 659 * to allow max possible filling before reallocation. 660 */ 661 prefetchw(data + SKB_WITH_OVERHEAD(size)); 662 663 /* 664 * Only clear those fields we need to clear, not those that we will 665 * actually initialise below. Hence, don't put any more fields after 666 * the tail pointer in struct sk_buff! 667 */ 668 memset(skb, 0, offsetof(struct sk_buff, tail)); 669 __build_skb_around(skb, data, size); 670 skb->pfmemalloc = pfmemalloc; 671 672 if (flags & SKB_ALLOC_FCLONE) { 673 struct sk_buff_fclones *fclones; 674 675 fclones = container_of(skb, struct sk_buff_fclones, skb1); 676 677 skb->fclone = SKB_FCLONE_ORIG; 678 refcount_set(&fclones->fclone_ref, 1); 679 } 680 681 return skb; 682 683 nodata: 684 kmem_cache_free(cache, skb); 685 return NULL; 686 } 687 EXPORT_SYMBOL(__alloc_skb); 688 689 /** 690 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 691 * @dev: network device to receive on 692 * @len: length to allocate 693 * @gfp_mask: get_free_pages mask, passed to alloc_skb 694 * 695 * Allocate a new &sk_buff and assign it a usage count of one. The 696 * buffer has NET_SKB_PAD headroom built in. Users should allocate 697 * the headroom they think they need without accounting for the 698 * built in space. The built in space is used for optimisations. 699 * 700 * %NULL is returned if there is no free memory. 701 */ 702 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 703 gfp_t gfp_mask) 704 { 705 struct page_frag_cache *nc; 706 struct sk_buff *skb; 707 bool pfmemalloc; 708 void *data; 709 710 len += NET_SKB_PAD; 711 712 /* If requested length is either too small or too big, 713 * we use kmalloc() for skb->head allocation. 714 */ 715 if (len <= SKB_WITH_OVERHEAD(1024) || 716 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 717 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 718 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 719 if (!skb) 720 goto skb_fail; 721 goto skb_success; 722 } 723 724 len = SKB_HEAD_ALIGN(len); 725 726 if (sk_memalloc_socks()) 727 gfp_mask |= __GFP_MEMALLOC; 728 729 if (in_hardirq() || irqs_disabled()) { 730 nc = this_cpu_ptr(&netdev_alloc_cache); 731 data = page_frag_alloc(nc, len, gfp_mask); 732 pfmemalloc = nc->pfmemalloc; 733 } else { 734 local_bh_disable(); 735 nc = this_cpu_ptr(&napi_alloc_cache.page); 736 data = page_frag_alloc(nc, len, gfp_mask); 737 pfmemalloc = nc->pfmemalloc; 738 local_bh_enable(); 739 } 740 741 if (unlikely(!data)) 742 return NULL; 743 744 skb = __build_skb(data, len); 745 if (unlikely(!skb)) { 746 skb_free_frag(data); 747 return NULL; 748 } 749 750 if (pfmemalloc) 751 skb->pfmemalloc = 1; 752 skb->head_frag = 1; 753 754 skb_success: 755 skb_reserve(skb, NET_SKB_PAD); 756 skb->dev = dev; 757 758 skb_fail: 759 return skb; 760 } 761 EXPORT_SYMBOL(__netdev_alloc_skb); 762 763 /** 764 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 765 * @napi: napi instance this buffer was allocated for 766 * @len: length to allocate 767 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages 768 * 769 * Allocate a new sk_buff for use in NAPI receive. This buffer will 770 * attempt to allocate the head from a special reserved region used 771 * only for NAPI Rx allocation. By doing this we can save several 772 * CPU cycles by avoiding having to disable and re-enable IRQs. 773 * 774 * %NULL is returned if there is no free memory. 775 */ 776 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, 777 gfp_t gfp_mask) 778 { 779 struct napi_alloc_cache *nc; 780 struct sk_buff *skb; 781 bool pfmemalloc; 782 void *data; 783 784 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 785 len += NET_SKB_PAD + NET_IP_ALIGN; 786 787 /* If requested length is either too small or too big, 788 * we use kmalloc() for skb->head allocation. 789 * When the small frag allocator is available, prefer it over kmalloc 790 * for small fragments 791 */ 792 if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) || 793 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 794 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 795 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, 796 NUMA_NO_NODE); 797 if (!skb) 798 goto skb_fail; 799 goto skb_success; 800 } 801 802 nc = this_cpu_ptr(&napi_alloc_cache); 803 804 if (sk_memalloc_socks()) 805 gfp_mask |= __GFP_MEMALLOC; 806 807 if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) { 808 /* we are artificially inflating the allocation size, but 809 * that is not as bad as it may look like, as: 810 * - 'len' less than GRO_MAX_HEAD makes little sense 811 * - On most systems, larger 'len' values lead to fragment 812 * size above 512 bytes 813 * - kmalloc would use the kmalloc-1k slab for such values 814 * - Builds with smaller GRO_MAX_HEAD will very likely do 815 * little networking, as that implies no WiFi and no 816 * tunnels support, and 32 bits arches. 817 */ 818 len = SZ_1K; 819 820 data = page_frag_alloc_1k(&nc->page_small, gfp_mask); 821 pfmemalloc = NAPI_SMALL_PAGE_PFMEMALLOC(nc->page_small); 822 } else { 823 len = SKB_HEAD_ALIGN(len); 824 825 data = page_frag_alloc(&nc->page, len, gfp_mask); 826 pfmemalloc = nc->page.pfmemalloc; 827 } 828 829 if (unlikely(!data)) 830 return NULL; 831 832 skb = __napi_build_skb(data, len); 833 if (unlikely(!skb)) { 834 skb_free_frag(data); 835 return NULL; 836 } 837 838 if (pfmemalloc) 839 skb->pfmemalloc = 1; 840 skb->head_frag = 1; 841 842 skb_success: 843 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 844 skb->dev = napi->dev; 845 846 skb_fail: 847 return skb; 848 } 849 EXPORT_SYMBOL(__napi_alloc_skb); 850 851 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 852 int size, unsigned int truesize) 853 { 854 skb_fill_page_desc(skb, i, page, off, size); 855 skb->len += size; 856 skb->data_len += size; 857 skb->truesize += truesize; 858 } 859 EXPORT_SYMBOL(skb_add_rx_frag); 860 861 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 862 unsigned int truesize) 863 { 864 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 865 866 skb_frag_size_add(frag, size); 867 skb->len += size; 868 skb->data_len += size; 869 skb->truesize += truesize; 870 } 871 EXPORT_SYMBOL(skb_coalesce_rx_frag); 872 873 static void skb_drop_list(struct sk_buff **listp) 874 { 875 kfree_skb_list(*listp); 876 *listp = NULL; 877 } 878 879 static inline void skb_drop_fraglist(struct sk_buff *skb) 880 { 881 skb_drop_list(&skb_shinfo(skb)->frag_list); 882 } 883 884 static void skb_clone_fraglist(struct sk_buff *skb) 885 { 886 struct sk_buff *list; 887 888 skb_walk_frags(skb, list) 889 skb_get(list); 890 } 891 892 static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe) 893 { 894 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) 895 return false; 896 return page_pool_return_skb_page(virt_to_page(data), napi_safe); 897 } 898 899 static void skb_kfree_head(void *head, unsigned int end_offset) 900 { 901 #ifdef HAVE_SKB_SMALL_HEAD_CACHE 902 if (end_offset == SKB_SMALL_HEAD_HEADROOM) 903 kmem_cache_free(skb_small_head_cache, head); 904 else 905 #endif 906 kfree(head); 907 } 908 909 static void skb_free_head(struct sk_buff *skb, bool napi_safe) 910 { 911 unsigned char *head = skb->head; 912 913 if (skb->head_frag) { 914 if (skb_pp_recycle(skb, head, napi_safe)) 915 return; 916 skb_free_frag(head); 917 } else { 918 skb_kfree_head(head, skb_end_offset(skb)); 919 } 920 } 921 922 static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason, 923 bool napi_safe) 924 { 925 struct skb_shared_info *shinfo = skb_shinfo(skb); 926 int i; 927 928 if (skb->cloned && 929 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 930 &shinfo->dataref)) 931 goto exit; 932 933 if (skb_zcopy(skb)) { 934 bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS; 935 936 skb_zcopy_clear(skb, true); 937 if (skip_unref) 938 goto free_head; 939 } 940 941 for (i = 0; i < shinfo->nr_frags; i++) 942 napi_frag_unref(&shinfo->frags[i], skb->pp_recycle, napi_safe); 943 944 free_head: 945 if (shinfo->frag_list) 946 kfree_skb_list_reason(shinfo->frag_list, reason); 947 948 skb_free_head(skb, napi_safe); 949 exit: 950 /* When we clone an SKB we copy the reycling bit. The pp_recycle 951 * bit is only set on the head though, so in order to avoid races 952 * while trying to recycle fragments on __skb_frag_unref() we need 953 * to make one SKB responsible for triggering the recycle path. 954 * So disable the recycling bit if an SKB is cloned and we have 955 * additional references to the fragmented part of the SKB. 956 * Eventually the last SKB will have the recycling bit set and it's 957 * dataref set to 0, which will trigger the recycling 958 */ 959 skb->pp_recycle = 0; 960 } 961 962 /* 963 * Free an skbuff by memory without cleaning the state. 964 */ 965 static void kfree_skbmem(struct sk_buff *skb) 966 { 967 struct sk_buff_fclones *fclones; 968 969 switch (skb->fclone) { 970 case SKB_FCLONE_UNAVAILABLE: 971 kmem_cache_free(skbuff_cache, skb); 972 return; 973 974 case SKB_FCLONE_ORIG: 975 fclones = container_of(skb, struct sk_buff_fclones, skb1); 976 977 /* We usually free the clone (TX completion) before original skb 978 * This test would have no chance to be true for the clone, 979 * while here, branch prediction will be good. 980 */ 981 if (refcount_read(&fclones->fclone_ref) == 1) 982 goto fastpath; 983 break; 984 985 default: /* SKB_FCLONE_CLONE */ 986 fclones = container_of(skb, struct sk_buff_fclones, skb2); 987 break; 988 } 989 if (!refcount_dec_and_test(&fclones->fclone_ref)) 990 return; 991 fastpath: 992 kmem_cache_free(skbuff_fclone_cache, fclones); 993 } 994 995 void skb_release_head_state(struct sk_buff *skb) 996 { 997 skb_dst_drop(skb); 998 if (skb->destructor) { 999 DEBUG_NET_WARN_ON_ONCE(in_hardirq()); 1000 skb->destructor(skb); 1001 } 1002 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 1003 nf_conntrack_put(skb_nfct(skb)); 1004 #endif 1005 skb_ext_put(skb); 1006 } 1007 1008 /* Free everything but the sk_buff shell. */ 1009 static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason, 1010 bool napi_safe) 1011 { 1012 skb_release_head_state(skb); 1013 if (likely(skb->head)) 1014 skb_release_data(skb, reason, napi_safe); 1015 } 1016 1017 /** 1018 * __kfree_skb - private function 1019 * @skb: buffer 1020 * 1021 * Free an sk_buff. Release anything attached to the buffer. 1022 * Clean the state. This is an internal helper function. Users should 1023 * always call kfree_skb 1024 */ 1025 1026 void __kfree_skb(struct sk_buff *skb) 1027 { 1028 skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED, false); 1029 kfree_skbmem(skb); 1030 } 1031 EXPORT_SYMBOL(__kfree_skb); 1032 1033 static __always_inline 1034 bool __kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) 1035 { 1036 if (unlikely(!skb_unref(skb))) 1037 return false; 1038 1039 DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET || 1040 u32_get_bits(reason, 1041 SKB_DROP_REASON_SUBSYS_MASK) >= 1042 SKB_DROP_REASON_SUBSYS_NUM); 1043 1044 if (reason == SKB_CONSUMED) 1045 trace_consume_skb(skb, __builtin_return_address(0)); 1046 else 1047 trace_kfree_skb(skb, __builtin_return_address(0), reason); 1048 return true; 1049 } 1050 1051 /** 1052 * kfree_skb_reason - free an sk_buff with special reason 1053 * @skb: buffer to free 1054 * @reason: reason why this skb is dropped 1055 * 1056 * Drop a reference to the buffer and free it if the usage count has 1057 * hit zero. Meanwhile, pass the drop reason to 'kfree_skb' 1058 * tracepoint. 1059 */ 1060 void __fix_address 1061 kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) 1062 { 1063 if (__kfree_skb_reason(skb, reason)) 1064 __kfree_skb(skb); 1065 } 1066 EXPORT_SYMBOL(kfree_skb_reason); 1067 1068 #define KFREE_SKB_BULK_SIZE 16 1069 1070 struct skb_free_array { 1071 unsigned int skb_count; 1072 void *skb_array[KFREE_SKB_BULK_SIZE]; 1073 }; 1074 1075 static void kfree_skb_add_bulk(struct sk_buff *skb, 1076 struct skb_free_array *sa, 1077 enum skb_drop_reason reason) 1078 { 1079 /* if SKB is a clone, don't handle this case */ 1080 if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) { 1081 __kfree_skb(skb); 1082 return; 1083 } 1084 1085 skb_release_all(skb, reason, false); 1086 sa->skb_array[sa->skb_count++] = skb; 1087 1088 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) { 1089 kmem_cache_free_bulk(skbuff_cache, KFREE_SKB_BULK_SIZE, 1090 sa->skb_array); 1091 sa->skb_count = 0; 1092 } 1093 } 1094 1095 void __fix_address 1096 kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) 1097 { 1098 struct skb_free_array sa; 1099 1100 sa.skb_count = 0; 1101 1102 while (segs) { 1103 struct sk_buff *next = segs->next; 1104 1105 if (__kfree_skb_reason(segs, reason)) { 1106 skb_poison_list(segs); 1107 kfree_skb_add_bulk(segs, &sa, reason); 1108 } 1109 1110 segs = next; 1111 } 1112 1113 if (sa.skb_count) 1114 kmem_cache_free_bulk(skbuff_cache, sa.skb_count, sa.skb_array); 1115 } 1116 EXPORT_SYMBOL(kfree_skb_list_reason); 1117 1118 /* Dump skb information and contents. 1119 * 1120 * Must only be called from net_ratelimit()-ed paths. 1121 * 1122 * Dumps whole packets if full_pkt, only headers otherwise. 1123 */ 1124 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt) 1125 { 1126 struct skb_shared_info *sh = skb_shinfo(skb); 1127 struct net_device *dev = skb->dev; 1128 struct sock *sk = skb->sk; 1129 struct sk_buff *list_skb; 1130 bool has_mac, has_trans; 1131 int headroom, tailroom; 1132 int i, len, seg_len; 1133 1134 if (full_pkt) 1135 len = skb->len; 1136 else 1137 len = min_t(int, skb->len, MAX_HEADER + 128); 1138 1139 headroom = skb_headroom(skb); 1140 tailroom = skb_tailroom(skb); 1141 1142 has_mac = skb_mac_header_was_set(skb); 1143 has_trans = skb_transport_header_was_set(skb); 1144 1145 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n" 1146 "mac=(%d,%d) net=(%d,%d) trans=%d\n" 1147 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" 1148 "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n" 1149 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n", 1150 level, skb->len, headroom, skb_headlen(skb), tailroom, 1151 has_mac ? skb->mac_header : -1, 1152 has_mac ? skb_mac_header_len(skb) : -1, 1153 skb->network_header, 1154 has_trans ? skb_network_header_len(skb) : -1, 1155 has_trans ? skb->transport_header : -1, 1156 sh->tx_flags, sh->nr_frags, 1157 sh->gso_size, sh->gso_type, sh->gso_segs, 1158 skb->csum, skb->ip_summed, skb->csum_complete_sw, 1159 skb->csum_valid, skb->csum_level, 1160 skb->hash, skb->sw_hash, skb->l4_hash, 1161 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif); 1162 1163 if (dev) 1164 printk("%sdev name=%s feat=%pNF\n", 1165 level, dev->name, &dev->features); 1166 if (sk) 1167 printk("%ssk family=%hu type=%u proto=%u\n", 1168 level, sk->sk_family, sk->sk_type, sk->sk_protocol); 1169 1170 if (full_pkt && headroom) 1171 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET, 1172 16, 1, skb->head, headroom, false); 1173 1174 seg_len = min_t(int, skb_headlen(skb), len); 1175 if (seg_len) 1176 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, 1177 16, 1, skb->data, seg_len, false); 1178 len -= seg_len; 1179 1180 if (full_pkt && tailroom) 1181 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET, 1182 16, 1, skb_tail_pointer(skb), tailroom, false); 1183 1184 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { 1185 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1186 u32 p_off, p_len, copied; 1187 struct page *p; 1188 u8 *vaddr; 1189 1190 skb_frag_foreach_page(frag, skb_frag_off(frag), 1191 skb_frag_size(frag), p, p_off, p_len, 1192 copied) { 1193 seg_len = min_t(int, p_len, len); 1194 vaddr = kmap_atomic(p); 1195 print_hex_dump(level, "skb frag: ", 1196 DUMP_PREFIX_OFFSET, 1197 16, 1, vaddr + p_off, seg_len, false); 1198 kunmap_atomic(vaddr); 1199 len -= seg_len; 1200 if (!len) 1201 break; 1202 } 1203 } 1204 1205 if (full_pkt && skb_has_frag_list(skb)) { 1206 printk("skb fraglist:\n"); 1207 skb_walk_frags(skb, list_skb) 1208 skb_dump(level, list_skb, true); 1209 } 1210 } 1211 EXPORT_SYMBOL(skb_dump); 1212 1213 /** 1214 * skb_tx_error - report an sk_buff xmit error 1215 * @skb: buffer that triggered an error 1216 * 1217 * Report xmit error if a device callback is tracking this skb. 1218 * skb must be freed afterwards. 1219 */ 1220 void skb_tx_error(struct sk_buff *skb) 1221 { 1222 if (skb) { 1223 skb_zcopy_downgrade_managed(skb); 1224 skb_zcopy_clear(skb, true); 1225 } 1226 } 1227 EXPORT_SYMBOL(skb_tx_error); 1228 1229 #ifdef CONFIG_TRACEPOINTS 1230 /** 1231 * consume_skb - free an skbuff 1232 * @skb: buffer to free 1233 * 1234 * Drop a ref to the buffer and free it if the usage count has hit zero 1235 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 1236 * is being dropped after a failure and notes that 1237 */ 1238 void consume_skb(struct sk_buff *skb) 1239 { 1240 if (!skb_unref(skb)) 1241 return; 1242 1243 trace_consume_skb(skb, __builtin_return_address(0)); 1244 __kfree_skb(skb); 1245 } 1246 EXPORT_SYMBOL(consume_skb); 1247 #endif 1248 1249 /** 1250 * __consume_stateless_skb - free an skbuff, assuming it is stateless 1251 * @skb: buffer to free 1252 * 1253 * Alike consume_skb(), but this variant assumes that this is the last 1254 * skb reference and all the head states have been already dropped 1255 */ 1256 void __consume_stateless_skb(struct sk_buff *skb) 1257 { 1258 trace_consume_skb(skb, __builtin_return_address(0)); 1259 skb_release_data(skb, SKB_CONSUMED, false); 1260 kfree_skbmem(skb); 1261 } 1262 1263 static void napi_skb_cache_put(struct sk_buff *skb) 1264 { 1265 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 1266 u32 i; 1267 1268 kasan_poison_object_data(skbuff_cache, skb); 1269 nc->skb_cache[nc->skb_count++] = skb; 1270 1271 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 1272 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) 1273 kasan_unpoison_object_data(skbuff_cache, 1274 nc->skb_cache[i]); 1275 1276 kmem_cache_free_bulk(skbuff_cache, NAPI_SKB_CACHE_HALF, 1277 nc->skb_cache + NAPI_SKB_CACHE_HALF); 1278 nc->skb_count = NAPI_SKB_CACHE_HALF; 1279 } 1280 } 1281 1282 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason) 1283 { 1284 skb_release_all(skb, reason, true); 1285 napi_skb_cache_put(skb); 1286 } 1287 1288 void napi_skb_free_stolen_head(struct sk_buff *skb) 1289 { 1290 if (unlikely(skb->slow_gro)) { 1291 nf_reset_ct(skb); 1292 skb_dst_drop(skb); 1293 skb_ext_put(skb); 1294 skb_orphan(skb); 1295 skb->slow_gro = 0; 1296 } 1297 napi_skb_cache_put(skb); 1298 } 1299 1300 void napi_consume_skb(struct sk_buff *skb, int budget) 1301 { 1302 /* Zero budget indicate non-NAPI context called us, like netpoll */ 1303 if (unlikely(!budget)) { 1304 dev_consume_skb_any(skb); 1305 return; 1306 } 1307 1308 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 1309 1310 if (!skb_unref(skb)) 1311 return; 1312 1313 /* if reaching here SKB is ready to free */ 1314 trace_consume_skb(skb, __builtin_return_address(0)); 1315 1316 /* if SKB is a clone, don't handle this case */ 1317 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 1318 __kfree_skb(skb); 1319 return; 1320 } 1321 1322 skb_release_all(skb, SKB_CONSUMED, !!budget); 1323 napi_skb_cache_put(skb); 1324 } 1325 EXPORT_SYMBOL(napi_consume_skb); 1326 1327 /* Make sure a field is contained by headers group */ 1328 #define CHECK_SKB_FIELD(field) \ 1329 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \ 1330 offsetof(struct sk_buff, headers.field)); \ 1331 1332 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 1333 { 1334 new->tstamp = old->tstamp; 1335 /* We do not copy old->sk */ 1336 new->dev = old->dev; 1337 memcpy(new->cb, old->cb, sizeof(old->cb)); 1338 skb_dst_copy(new, old); 1339 __skb_ext_copy(new, old); 1340 __nf_copy(new, old, false); 1341 1342 /* Note : this field could be in the headers group. 1343 * It is not yet because we do not want to have a 16 bit hole 1344 */ 1345 new->queue_mapping = old->queue_mapping; 1346 1347 memcpy(&new->headers, &old->headers, sizeof(new->headers)); 1348 CHECK_SKB_FIELD(protocol); 1349 CHECK_SKB_FIELD(csum); 1350 CHECK_SKB_FIELD(hash); 1351 CHECK_SKB_FIELD(priority); 1352 CHECK_SKB_FIELD(skb_iif); 1353 CHECK_SKB_FIELD(vlan_proto); 1354 CHECK_SKB_FIELD(vlan_tci); 1355 CHECK_SKB_FIELD(transport_header); 1356 CHECK_SKB_FIELD(network_header); 1357 CHECK_SKB_FIELD(mac_header); 1358 CHECK_SKB_FIELD(inner_protocol); 1359 CHECK_SKB_FIELD(inner_transport_header); 1360 CHECK_SKB_FIELD(inner_network_header); 1361 CHECK_SKB_FIELD(inner_mac_header); 1362 CHECK_SKB_FIELD(mark); 1363 #ifdef CONFIG_NETWORK_SECMARK 1364 CHECK_SKB_FIELD(secmark); 1365 #endif 1366 #ifdef CONFIG_NET_RX_BUSY_POLL 1367 CHECK_SKB_FIELD(napi_id); 1368 #endif 1369 CHECK_SKB_FIELD(alloc_cpu); 1370 #ifdef CONFIG_XPS 1371 CHECK_SKB_FIELD(sender_cpu); 1372 #endif 1373 #ifdef CONFIG_NET_SCHED 1374 CHECK_SKB_FIELD(tc_index); 1375 #endif 1376 1377 } 1378 1379 /* 1380 * You should not add any new code to this function. Add it to 1381 * __copy_skb_header above instead. 1382 */ 1383 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 1384 { 1385 #define C(x) n->x = skb->x 1386 1387 n->next = n->prev = NULL; 1388 n->sk = NULL; 1389 __copy_skb_header(n, skb); 1390 1391 C(len); 1392 C(data_len); 1393 C(mac_len); 1394 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 1395 n->cloned = 1; 1396 n->nohdr = 0; 1397 n->peeked = 0; 1398 C(pfmemalloc); 1399 C(pp_recycle); 1400 n->destructor = NULL; 1401 C(tail); 1402 C(end); 1403 C(head); 1404 C(head_frag); 1405 C(data); 1406 C(truesize); 1407 refcount_set(&n->users, 1); 1408 1409 atomic_inc(&(skb_shinfo(skb)->dataref)); 1410 skb->cloned = 1; 1411 1412 return n; 1413 #undef C 1414 } 1415 1416 /** 1417 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg 1418 * @first: first sk_buff of the msg 1419 */ 1420 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first) 1421 { 1422 struct sk_buff *n; 1423 1424 n = alloc_skb(0, GFP_ATOMIC); 1425 if (!n) 1426 return NULL; 1427 1428 n->len = first->len; 1429 n->data_len = first->len; 1430 n->truesize = first->truesize; 1431 1432 skb_shinfo(n)->frag_list = first; 1433 1434 __copy_skb_header(n, first); 1435 n->destructor = NULL; 1436 1437 return n; 1438 } 1439 EXPORT_SYMBOL_GPL(alloc_skb_for_msg); 1440 1441 /** 1442 * skb_morph - morph one skb into another 1443 * @dst: the skb to receive the contents 1444 * @src: the skb to supply the contents 1445 * 1446 * This is identical to skb_clone except that the target skb is 1447 * supplied by the user. 1448 * 1449 * The target skb is returned upon exit. 1450 */ 1451 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 1452 { 1453 skb_release_all(dst, SKB_CONSUMED, false); 1454 return __skb_clone(dst, src); 1455 } 1456 EXPORT_SYMBOL_GPL(skb_morph); 1457 1458 int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 1459 { 1460 unsigned long max_pg, num_pg, new_pg, old_pg, rlim; 1461 struct user_struct *user; 1462 1463 if (capable(CAP_IPC_LOCK) || !size) 1464 return 0; 1465 1466 rlim = rlimit(RLIMIT_MEMLOCK); 1467 if (rlim == RLIM_INFINITY) 1468 return 0; 1469 1470 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 1471 max_pg = rlim >> PAGE_SHIFT; 1472 user = mmp->user ? : current_user(); 1473 1474 old_pg = atomic_long_read(&user->locked_vm); 1475 do { 1476 new_pg = old_pg + num_pg; 1477 if (new_pg > max_pg) 1478 return -ENOBUFS; 1479 } while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg)); 1480 1481 if (!mmp->user) { 1482 mmp->user = get_uid(user); 1483 mmp->num_pg = num_pg; 1484 } else { 1485 mmp->num_pg += num_pg; 1486 } 1487 1488 return 0; 1489 } 1490 EXPORT_SYMBOL_GPL(mm_account_pinned_pages); 1491 1492 void mm_unaccount_pinned_pages(struct mmpin *mmp) 1493 { 1494 if (mmp->user) { 1495 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 1496 free_uid(mmp->user); 1497 } 1498 } 1499 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); 1500 1501 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) 1502 { 1503 struct ubuf_info_msgzc *uarg; 1504 struct sk_buff *skb; 1505 1506 WARN_ON_ONCE(!in_task()); 1507 1508 skb = sock_omalloc(sk, 0, GFP_KERNEL); 1509 if (!skb) 1510 return NULL; 1511 1512 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 1513 uarg = (void *)skb->cb; 1514 uarg->mmp.user = NULL; 1515 1516 if (mm_account_pinned_pages(&uarg->mmp, size)) { 1517 kfree_skb(skb); 1518 return NULL; 1519 } 1520 1521 uarg->ubuf.callback = msg_zerocopy_callback; 1522 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 1523 uarg->len = 1; 1524 uarg->bytelen = size; 1525 uarg->zerocopy = 1; 1526 uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; 1527 refcount_set(&uarg->ubuf.refcnt, 1); 1528 sock_hold(sk); 1529 1530 return &uarg->ubuf; 1531 } 1532 1533 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg) 1534 { 1535 return container_of((void *)uarg, struct sk_buff, cb); 1536 } 1537 1538 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, 1539 struct ubuf_info *uarg) 1540 { 1541 if (uarg) { 1542 struct ubuf_info_msgzc *uarg_zc; 1543 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 1544 u32 bytelen, next; 1545 1546 /* there might be non MSG_ZEROCOPY users */ 1547 if (uarg->callback != msg_zerocopy_callback) 1548 return NULL; 1549 1550 /* realloc only when socket is locked (TCP, UDP cork), 1551 * so uarg->len and sk_zckey access is serialized 1552 */ 1553 if (!sock_owned_by_user(sk)) { 1554 WARN_ON_ONCE(1); 1555 return NULL; 1556 } 1557 1558 uarg_zc = uarg_to_msgzc(uarg); 1559 bytelen = uarg_zc->bytelen + size; 1560 if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) { 1561 /* TCP can create new skb to attach new uarg */ 1562 if (sk->sk_type == SOCK_STREAM) 1563 goto new_alloc; 1564 return NULL; 1565 } 1566 1567 next = (u32)atomic_read(&sk->sk_zckey); 1568 if ((u32)(uarg_zc->id + uarg_zc->len) == next) { 1569 if (mm_account_pinned_pages(&uarg_zc->mmp, size)) 1570 return NULL; 1571 uarg_zc->len++; 1572 uarg_zc->bytelen = bytelen; 1573 atomic_set(&sk->sk_zckey, ++next); 1574 1575 /* no extra ref when appending to datagram (MSG_MORE) */ 1576 if (sk->sk_type == SOCK_STREAM) 1577 net_zcopy_get(uarg); 1578 1579 return uarg; 1580 } 1581 } 1582 1583 new_alloc: 1584 return msg_zerocopy_alloc(sk, size); 1585 } 1586 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); 1587 1588 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1589 { 1590 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1591 u32 old_lo, old_hi; 1592 u64 sum_len; 1593 1594 old_lo = serr->ee.ee_info; 1595 old_hi = serr->ee.ee_data; 1596 sum_len = old_hi - old_lo + 1ULL + len; 1597 1598 if (sum_len >= (1ULL << 32)) 1599 return false; 1600 1601 if (lo != old_hi + 1) 1602 return false; 1603 1604 serr->ee.ee_data += len; 1605 return true; 1606 } 1607 1608 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg) 1609 { 1610 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1611 struct sock_exterr_skb *serr; 1612 struct sock *sk = skb->sk; 1613 struct sk_buff_head *q; 1614 unsigned long flags; 1615 bool is_zerocopy; 1616 u32 lo, hi; 1617 u16 len; 1618 1619 mm_unaccount_pinned_pages(&uarg->mmp); 1620 1621 /* if !len, there was only 1 call, and it was aborted 1622 * so do not queue a completion notification 1623 */ 1624 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1625 goto release; 1626 1627 len = uarg->len; 1628 lo = uarg->id; 1629 hi = uarg->id + len - 1; 1630 is_zerocopy = uarg->zerocopy; 1631 1632 serr = SKB_EXT_ERR(skb); 1633 memset(serr, 0, sizeof(*serr)); 1634 serr->ee.ee_errno = 0; 1635 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1636 serr->ee.ee_data = hi; 1637 serr->ee.ee_info = lo; 1638 if (!is_zerocopy) 1639 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1640 1641 q = &sk->sk_error_queue; 1642 spin_lock_irqsave(&q->lock, flags); 1643 tail = skb_peek_tail(q); 1644 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1645 !skb_zerocopy_notify_extend(tail, lo, len)) { 1646 __skb_queue_tail(q, skb); 1647 skb = NULL; 1648 } 1649 spin_unlock_irqrestore(&q->lock, flags); 1650 1651 sk_error_report(sk); 1652 1653 release: 1654 consume_skb(skb); 1655 sock_put(sk); 1656 } 1657 1658 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, 1659 bool success) 1660 { 1661 struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg); 1662 1663 uarg_zc->zerocopy = uarg_zc->zerocopy & success; 1664 1665 if (refcount_dec_and_test(&uarg->refcnt)) 1666 __msg_zerocopy_callback(uarg_zc); 1667 } 1668 EXPORT_SYMBOL_GPL(msg_zerocopy_callback); 1669 1670 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1671 { 1672 struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk; 1673 1674 atomic_dec(&sk->sk_zckey); 1675 uarg_to_msgzc(uarg)->len--; 1676 1677 if (have_uref) 1678 msg_zerocopy_callback(NULL, uarg, true); 1679 } 1680 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); 1681 1682 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1683 struct msghdr *msg, int len, 1684 struct ubuf_info *uarg) 1685 { 1686 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1687 int err, orig_len = skb->len; 1688 1689 /* An skb can only point to one uarg. This edge case happens when 1690 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc. 1691 */ 1692 if (orig_uarg && uarg != orig_uarg) 1693 return -EEXIST; 1694 1695 err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len); 1696 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1697 struct sock *save_sk = skb->sk; 1698 1699 /* Streams do not free skb on error. Reset to prev state. */ 1700 iov_iter_revert(&msg->msg_iter, skb->len - orig_len); 1701 skb->sk = sk; 1702 ___pskb_trim(skb, orig_len); 1703 skb->sk = save_sk; 1704 return err; 1705 } 1706 1707 skb_zcopy_set(skb, uarg, NULL); 1708 return skb->len - orig_len; 1709 } 1710 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1711 1712 void __skb_zcopy_downgrade_managed(struct sk_buff *skb) 1713 { 1714 int i; 1715 1716 skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS; 1717 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1718 skb_frag_ref(skb, i); 1719 } 1720 EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed); 1721 1722 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1723 gfp_t gfp_mask) 1724 { 1725 if (skb_zcopy(orig)) { 1726 if (skb_zcopy(nskb)) { 1727 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1728 if (!gfp_mask) { 1729 WARN_ON_ONCE(1); 1730 return -ENOMEM; 1731 } 1732 if (skb_uarg(nskb) == skb_uarg(orig)) 1733 return 0; 1734 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1735 return -EIO; 1736 } 1737 skb_zcopy_set(nskb, skb_uarg(orig), NULL); 1738 } 1739 return 0; 1740 } 1741 1742 /** 1743 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1744 * @skb: the skb to modify 1745 * @gfp_mask: allocation priority 1746 * 1747 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. 1748 * It will copy all frags into kernel and drop the reference 1749 * to userspace pages. 1750 * 1751 * If this function is called from an interrupt gfp_mask() must be 1752 * %GFP_ATOMIC. 1753 * 1754 * Returns 0 on success or a negative error code on failure 1755 * to allocate kernel memory to copy to. 1756 */ 1757 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 1758 { 1759 int num_frags = skb_shinfo(skb)->nr_frags; 1760 struct page *page, *head = NULL; 1761 int i, new_frags; 1762 u32 d_off; 1763 1764 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 1765 return -EINVAL; 1766 1767 if (!num_frags) 1768 goto release; 1769 1770 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT; 1771 for (i = 0; i < new_frags; i++) { 1772 page = alloc_page(gfp_mask); 1773 if (!page) { 1774 while (head) { 1775 struct page *next = (struct page *)page_private(head); 1776 put_page(head); 1777 head = next; 1778 } 1779 return -ENOMEM; 1780 } 1781 set_page_private(page, (unsigned long)head); 1782 head = page; 1783 } 1784 1785 page = head; 1786 d_off = 0; 1787 for (i = 0; i < num_frags; i++) { 1788 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1789 u32 p_off, p_len, copied; 1790 struct page *p; 1791 u8 *vaddr; 1792 1793 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f), 1794 p, p_off, p_len, copied) { 1795 u32 copy, done = 0; 1796 vaddr = kmap_atomic(p); 1797 1798 while (done < p_len) { 1799 if (d_off == PAGE_SIZE) { 1800 d_off = 0; 1801 page = (struct page *)page_private(page); 1802 } 1803 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done); 1804 memcpy(page_address(page) + d_off, 1805 vaddr + p_off + done, copy); 1806 done += copy; 1807 d_off += copy; 1808 } 1809 kunmap_atomic(vaddr); 1810 } 1811 } 1812 1813 /* skb frags release userspace buffers */ 1814 for (i = 0; i < num_frags; i++) 1815 skb_frag_unref(skb, i); 1816 1817 /* skb frags point to kernel buffers */ 1818 for (i = 0; i < new_frags - 1; i++) { 1819 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE); 1820 head = (struct page *)page_private(head); 1821 } 1822 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); 1823 skb_shinfo(skb)->nr_frags = new_frags; 1824 1825 release: 1826 skb_zcopy_clear(skb, false); 1827 return 0; 1828 } 1829 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 1830 1831 /** 1832 * skb_clone - duplicate an sk_buff 1833 * @skb: buffer to clone 1834 * @gfp_mask: allocation priority 1835 * 1836 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 1837 * copies share the same packet data but not structure. The new 1838 * buffer has a reference count of 1. If the allocation fails the 1839 * function returns %NULL otherwise the new buffer is returned. 1840 * 1841 * If this function is called from an interrupt gfp_mask() must be 1842 * %GFP_ATOMIC. 1843 */ 1844 1845 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 1846 { 1847 struct sk_buff_fclones *fclones = container_of(skb, 1848 struct sk_buff_fclones, 1849 skb1); 1850 struct sk_buff *n; 1851 1852 if (skb_orphan_frags(skb, gfp_mask)) 1853 return NULL; 1854 1855 if (skb->fclone == SKB_FCLONE_ORIG && 1856 refcount_read(&fclones->fclone_ref) == 1) { 1857 n = &fclones->skb2; 1858 refcount_set(&fclones->fclone_ref, 2); 1859 n->fclone = SKB_FCLONE_CLONE; 1860 } else { 1861 if (skb_pfmemalloc(skb)) 1862 gfp_mask |= __GFP_MEMALLOC; 1863 1864 n = kmem_cache_alloc(skbuff_cache, gfp_mask); 1865 if (!n) 1866 return NULL; 1867 1868 n->fclone = SKB_FCLONE_UNAVAILABLE; 1869 } 1870 1871 return __skb_clone(n, skb); 1872 } 1873 EXPORT_SYMBOL(skb_clone); 1874 1875 void skb_headers_offset_update(struct sk_buff *skb, int off) 1876 { 1877 /* Only adjust this if it actually is csum_start rather than csum */ 1878 if (skb->ip_summed == CHECKSUM_PARTIAL) 1879 skb->csum_start += off; 1880 /* {transport,network,mac}_header and tail are relative to skb->head */ 1881 skb->transport_header += off; 1882 skb->network_header += off; 1883 if (skb_mac_header_was_set(skb)) 1884 skb->mac_header += off; 1885 skb->inner_transport_header += off; 1886 skb->inner_network_header += off; 1887 skb->inner_mac_header += off; 1888 } 1889 EXPORT_SYMBOL(skb_headers_offset_update); 1890 1891 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) 1892 { 1893 __copy_skb_header(new, old); 1894 1895 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 1896 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 1897 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 1898 } 1899 EXPORT_SYMBOL(skb_copy_header); 1900 1901 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 1902 { 1903 if (skb_pfmemalloc(skb)) 1904 return SKB_ALLOC_RX; 1905 return 0; 1906 } 1907 1908 /** 1909 * skb_copy - create private copy of an sk_buff 1910 * @skb: buffer to copy 1911 * @gfp_mask: allocation priority 1912 * 1913 * Make a copy of both an &sk_buff and its data. This is used when the 1914 * caller wishes to modify the data and needs a private copy of the 1915 * data to alter. Returns %NULL on failure or the pointer to the buffer 1916 * on success. The returned buffer has a reference count of 1. 1917 * 1918 * As by-product this function converts non-linear &sk_buff to linear 1919 * one, so that &sk_buff becomes completely private and caller is allowed 1920 * to modify all the data of returned buffer. This means that this 1921 * function is not recommended for use in circumstances when only 1922 * header is going to be modified. Use pskb_copy() instead. 1923 */ 1924 1925 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 1926 { 1927 int headerlen = skb_headroom(skb); 1928 unsigned int size = skb_end_offset(skb) + skb->data_len; 1929 struct sk_buff *n = __alloc_skb(size, gfp_mask, 1930 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 1931 1932 if (!n) 1933 return NULL; 1934 1935 /* Set the data pointer */ 1936 skb_reserve(n, headerlen); 1937 /* Set the tail pointer and length */ 1938 skb_put(n, skb->len); 1939 1940 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); 1941 1942 skb_copy_header(n, skb); 1943 return n; 1944 } 1945 EXPORT_SYMBOL(skb_copy); 1946 1947 /** 1948 * __pskb_copy_fclone - create copy of an sk_buff with private head. 1949 * @skb: buffer to copy 1950 * @headroom: headroom of new skb 1951 * @gfp_mask: allocation priority 1952 * @fclone: if true allocate the copy of the skb from the fclone 1953 * cache instead of the head cache; it is recommended to set this 1954 * to true for the cases where the copy will likely be cloned 1955 * 1956 * Make a copy of both an &sk_buff and part of its data, located 1957 * in header. Fragmented data remain shared. This is used when 1958 * the caller wishes to modify only header of &sk_buff and needs 1959 * private copy of the header to alter. Returns %NULL on failure 1960 * or the pointer to the buffer on success. 1961 * The returned buffer has a reference count of 1. 1962 */ 1963 1964 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 1965 gfp_t gfp_mask, bool fclone) 1966 { 1967 unsigned int size = skb_headlen(skb) + headroom; 1968 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 1969 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 1970 1971 if (!n) 1972 goto out; 1973 1974 /* Set the data pointer */ 1975 skb_reserve(n, headroom); 1976 /* Set the tail pointer and length */ 1977 skb_put(n, skb_headlen(skb)); 1978 /* Copy the bytes */ 1979 skb_copy_from_linear_data(skb, n->data, n->len); 1980 1981 n->truesize += skb->data_len; 1982 n->data_len = skb->data_len; 1983 n->len = skb->len; 1984 1985 if (skb_shinfo(skb)->nr_frags) { 1986 int i; 1987 1988 if (skb_orphan_frags(skb, gfp_mask) || 1989 skb_zerocopy_clone(n, skb, gfp_mask)) { 1990 kfree_skb(n); 1991 n = NULL; 1992 goto out; 1993 } 1994 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1995 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 1996 skb_frag_ref(skb, i); 1997 } 1998 skb_shinfo(n)->nr_frags = i; 1999 } 2000 2001 if (skb_has_frag_list(skb)) { 2002 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 2003 skb_clone_fraglist(n); 2004 } 2005 2006 skb_copy_header(n, skb); 2007 out: 2008 return n; 2009 } 2010 EXPORT_SYMBOL(__pskb_copy_fclone); 2011 2012 /** 2013 * pskb_expand_head - reallocate header of &sk_buff 2014 * @skb: buffer to reallocate 2015 * @nhead: room to add at head 2016 * @ntail: room to add at tail 2017 * @gfp_mask: allocation priority 2018 * 2019 * Expands (or creates identical copy, if @nhead and @ntail are zero) 2020 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 2021 * reference count of 1. Returns zero in the case of success or error, 2022 * if expansion failed. In the last case, &sk_buff is not changed. 2023 * 2024 * All the pointers pointing into skb header may change and must be 2025 * reloaded after call to this function. 2026 */ 2027 2028 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 2029 gfp_t gfp_mask) 2030 { 2031 unsigned int osize = skb_end_offset(skb); 2032 unsigned int size = osize + nhead + ntail; 2033 long off; 2034 u8 *data; 2035 int i; 2036 2037 BUG_ON(nhead < 0); 2038 2039 BUG_ON(skb_shared(skb)); 2040 2041 skb_zcopy_downgrade_managed(skb); 2042 2043 if (skb_pfmemalloc(skb)) 2044 gfp_mask |= __GFP_MEMALLOC; 2045 2046 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 2047 if (!data) 2048 goto nodata; 2049 size = SKB_WITH_OVERHEAD(size); 2050 2051 /* Copy only real data... and, alas, header. This should be 2052 * optimized for the cases when header is void. 2053 */ 2054 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 2055 2056 memcpy((struct skb_shared_info *)(data + size), 2057 skb_shinfo(skb), 2058 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 2059 2060 /* 2061 * if shinfo is shared we must drop the old head gracefully, but if it 2062 * is not we can just drop the old head and let the existing refcount 2063 * be since all we did is relocate the values 2064 */ 2065 if (skb_cloned(skb)) { 2066 if (skb_orphan_frags(skb, gfp_mask)) 2067 goto nofrags; 2068 if (skb_zcopy(skb)) 2069 refcount_inc(&skb_uarg(skb)->refcnt); 2070 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2071 skb_frag_ref(skb, i); 2072 2073 if (skb_has_frag_list(skb)) 2074 skb_clone_fraglist(skb); 2075 2076 skb_release_data(skb, SKB_CONSUMED, false); 2077 } else { 2078 skb_free_head(skb, false); 2079 } 2080 off = (data + nhead) - skb->head; 2081 2082 skb->head = data; 2083 skb->head_frag = 0; 2084 skb->data += off; 2085 2086 skb_set_end_offset(skb, size); 2087 #ifdef NET_SKBUFF_DATA_USES_OFFSET 2088 off = nhead; 2089 #endif 2090 skb->tail += off; 2091 skb_headers_offset_update(skb, nhead); 2092 skb->cloned = 0; 2093 skb->hdr_len = 0; 2094 skb->nohdr = 0; 2095 atomic_set(&skb_shinfo(skb)->dataref, 1); 2096 2097 skb_metadata_clear(skb); 2098 2099 /* It is not generally safe to change skb->truesize. 2100 * For the moment, we really care of rx path, or 2101 * when skb is orphaned (not attached to a socket). 2102 */ 2103 if (!skb->sk || skb->destructor == sock_edemux) 2104 skb->truesize += size - osize; 2105 2106 return 0; 2107 2108 nofrags: 2109 skb_kfree_head(data, size); 2110 nodata: 2111 return -ENOMEM; 2112 } 2113 EXPORT_SYMBOL(pskb_expand_head); 2114 2115 /* Make private copy of skb with writable head and some headroom */ 2116 2117 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 2118 { 2119 struct sk_buff *skb2; 2120 int delta = headroom - skb_headroom(skb); 2121 2122 if (delta <= 0) 2123 skb2 = pskb_copy(skb, GFP_ATOMIC); 2124 else { 2125 skb2 = skb_clone(skb, GFP_ATOMIC); 2126 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 2127 GFP_ATOMIC)) { 2128 kfree_skb(skb2); 2129 skb2 = NULL; 2130 } 2131 } 2132 return skb2; 2133 } 2134 EXPORT_SYMBOL(skb_realloc_headroom); 2135 2136 /* Note: We plan to rework this in linux-6.4 */ 2137 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) 2138 { 2139 unsigned int saved_end_offset, saved_truesize; 2140 struct skb_shared_info *shinfo; 2141 int res; 2142 2143 saved_end_offset = skb_end_offset(skb); 2144 saved_truesize = skb->truesize; 2145 2146 res = pskb_expand_head(skb, 0, 0, pri); 2147 if (res) 2148 return res; 2149 2150 skb->truesize = saved_truesize; 2151 2152 if (likely(skb_end_offset(skb) == saved_end_offset)) 2153 return 0; 2154 2155 #ifdef HAVE_SKB_SMALL_HEAD_CACHE 2156 /* We can not change skb->end if the original or new value 2157 * is SKB_SMALL_HEAD_HEADROOM, as it might break skb_kfree_head(). 2158 */ 2159 if (saved_end_offset == SKB_SMALL_HEAD_HEADROOM || 2160 skb_end_offset(skb) == SKB_SMALL_HEAD_HEADROOM) { 2161 /* We think this path should not be taken. 2162 * Add a temporary trace to warn us just in case. 2163 */ 2164 pr_err_once("__skb_unclone_keeptruesize() skb_end_offset() %u -> %u\n", 2165 saved_end_offset, skb_end_offset(skb)); 2166 WARN_ON_ONCE(1); 2167 return 0; 2168 } 2169 #endif 2170 2171 shinfo = skb_shinfo(skb); 2172 2173 /* We are about to change back skb->end, 2174 * we need to move skb_shinfo() to its new location. 2175 */ 2176 memmove(skb->head + saved_end_offset, 2177 shinfo, 2178 offsetof(struct skb_shared_info, frags[shinfo->nr_frags])); 2179 2180 skb_set_end_offset(skb, saved_end_offset); 2181 2182 return 0; 2183 } 2184 2185 /** 2186 * skb_expand_head - reallocate header of &sk_buff 2187 * @skb: buffer to reallocate 2188 * @headroom: needed headroom 2189 * 2190 * Unlike skb_realloc_headroom, this one does not allocate a new skb 2191 * if possible; copies skb->sk to new skb as needed 2192 * and frees original skb in case of failures. 2193 * 2194 * It expect increased headroom and generates warning otherwise. 2195 */ 2196 2197 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom) 2198 { 2199 int delta = headroom - skb_headroom(skb); 2200 int osize = skb_end_offset(skb); 2201 struct sock *sk = skb->sk; 2202 2203 if (WARN_ONCE(delta <= 0, 2204 "%s is expecting an increase in the headroom", __func__)) 2205 return skb; 2206 2207 delta = SKB_DATA_ALIGN(delta); 2208 /* pskb_expand_head() might crash, if skb is shared. */ 2209 if (skb_shared(skb) || !is_skb_wmem(skb)) { 2210 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); 2211 2212 if (unlikely(!nskb)) 2213 goto fail; 2214 2215 if (sk) 2216 skb_set_owner_w(nskb, sk); 2217 consume_skb(skb); 2218 skb = nskb; 2219 } 2220 if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC)) 2221 goto fail; 2222 2223 if (sk && is_skb_wmem(skb)) { 2224 delta = skb_end_offset(skb) - osize; 2225 refcount_add(delta, &sk->sk_wmem_alloc); 2226 skb->truesize += delta; 2227 } 2228 return skb; 2229 2230 fail: 2231 kfree_skb(skb); 2232 return NULL; 2233 } 2234 EXPORT_SYMBOL(skb_expand_head); 2235 2236 /** 2237 * skb_copy_expand - copy and expand sk_buff 2238 * @skb: buffer to copy 2239 * @newheadroom: new free bytes at head 2240 * @newtailroom: new free bytes at tail 2241 * @gfp_mask: allocation priority 2242 * 2243 * Make a copy of both an &sk_buff and its data and while doing so 2244 * allocate additional space. 2245 * 2246 * This is used when the caller wishes to modify the data and needs a 2247 * private copy of the data to alter as well as more space for new fields. 2248 * Returns %NULL on failure or the pointer to the buffer 2249 * on success. The returned buffer has a reference count of 1. 2250 * 2251 * You must pass %GFP_ATOMIC as the allocation priority if this function 2252 * is called from an interrupt. 2253 */ 2254 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 2255 int newheadroom, int newtailroom, 2256 gfp_t gfp_mask) 2257 { 2258 /* 2259 * Allocate the copy buffer 2260 */ 2261 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 2262 gfp_mask, skb_alloc_rx_flag(skb), 2263 NUMA_NO_NODE); 2264 int oldheadroom = skb_headroom(skb); 2265 int head_copy_len, head_copy_off; 2266 2267 if (!n) 2268 return NULL; 2269 2270 skb_reserve(n, newheadroom); 2271 2272 /* Set the tail pointer and length */ 2273 skb_put(n, skb->len); 2274 2275 head_copy_len = oldheadroom; 2276 head_copy_off = 0; 2277 if (newheadroom <= head_copy_len) 2278 head_copy_len = newheadroom; 2279 else 2280 head_copy_off = newheadroom - head_copy_len; 2281 2282 /* Copy the linear header and data. */ 2283 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 2284 skb->len + head_copy_len)); 2285 2286 skb_copy_header(n, skb); 2287 2288 skb_headers_offset_update(n, newheadroom - oldheadroom); 2289 2290 return n; 2291 } 2292 EXPORT_SYMBOL(skb_copy_expand); 2293 2294 /** 2295 * __skb_pad - zero pad the tail of an skb 2296 * @skb: buffer to pad 2297 * @pad: space to pad 2298 * @free_on_error: free buffer on error 2299 * 2300 * Ensure that a buffer is followed by a padding area that is zero 2301 * filled. Used by network drivers which may DMA or transfer data 2302 * beyond the buffer end onto the wire. 2303 * 2304 * May return error in out of memory cases. The skb is freed on error 2305 * if @free_on_error is true. 2306 */ 2307 2308 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 2309 { 2310 int err; 2311 int ntail; 2312 2313 /* If the skbuff is non linear tailroom is always zero.. */ 2314 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 2315 memset(skb->data+skb->len, 0, pad); 2316 return 0; 2317 } 2318 2319 ntail = skb->data_len + pad - (skb->end - skb->tail); 2320 if (likely(skb_cloned(skb) || ntail > 0)) { 2321 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 2322 if (unlikely(err)) 2323 goto free_skb; 2324 } 2325 2326 /* FIXME: The use of this function with non-linear skb's really needs 2327 * to be audited. 2328 */ 2329 err = skb_linearize(skb); 2330 if (unlikely(err)) 2331 goto free_skb; 2332 2333 memset(skb->data + skb->len, 0, pad); 2334 return 0; 2335 2336 free_skb: 2337 if (free_on_error) 2338 kfree_skb(skb); 2339 return err; 2340 } 2341 EXPORT_SYMBOL(__skb_pad); 2342 2343 /** 2344 * pskb_put - add data to the tail of a potentially fragmented buffer 2345 * @skb: start of the buffer to use 2346 * @tail: tail fragment of the buffer to use 2347 * @len: amount of data to add 2348 * 2349 * This function extends the used data area of the potentially 2350 * fragmented buffer. @tail must be the last fragment of @skb -- or 2351 * @skb itself. If this would exceed the total buffer size the kernel 2352 * will panic. A pointer to the first byte of the extra data is 2353 * returned. 2354 */ 2355 2356 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 2357 { 2358 if (tail != skb) { 2359 skb->data_len += len; 2360 skb->len += len; 2361 } 2362 return skb_put(tail, len); 2363 } 2364 EXPORT_SYMBOL_GPL(pskb_put); 2365 2366 /** 2367 * skb_put - add data to a buffer 2368 * @skb: buffer to use 2369 * @len: amount of data to add 2370 * 2371 * This function extends the used data area of the buffer. If this would 2372 * exceed the total buffer size the kernel will panic. A pointer to the 2373 * first byte of the extra data is returned. 2374 */ 2375 void *skb_put(struct sk_buff *skb, unsigned int len) 2376 { 2377 void *tmp = skb_tail_pointer(skb); 2378 SKB_LINEAR_ASSERT(skb); 2379 skb->tail += len; 2380 skb->len += len; 2381 if (unlikely(skb->tail > skb->end)) 2382 skb_over_panic(skb, len, __builtin_return_address(0)); 2383 return tmp; 2384 } 2385 EXPORT_SYMBOL(skb_put); 2386 2387 /** 2388 * skb_push - add data to the start of a buffer 2389 * @skb: buffer to use 2390 * @len: amount of data to add 2391 * 2392 * This function extends the used data area of the buffer at the buffer 2393 * start. If this would exceed the total buffer headroom the kernel will 2394 * panic. A pointer to the first byte of the extra data is returned. 2395 */ 2396 void *skb_push(struct sk_buff *skb, unsigned int len) 2397 { 2398 skb->data -= len; 2399 skb->len += len; 2400 if (unlikely(skb->data < skb->head)) 2401 skb_under_panic(skb, len, __builtin_return_address(0)); 2402 return skb->data; 2403 } 2404 EXPORT_SYMBOL(skb_push); 2405 2406 /** 2407 * skb_pull - remove data from the start of a buffer 2408 * @skb: buffer to use 2409 * @len: amount of data to remove 2410 * 2411 * This function removes data from the start of a buffer, returning 2412 * the memory to the headroom. A pointer to the next data in the buffer 2413 * is returned. Once the data has been pulled future pushes will overwrite 2414 * the old data. 2415 */ 2416 void *skb_pull(struct sk_buff *skb, unsigned int len) 2417 { 2418 return skb_pull_inline(skb, len); 2419 } 2420 EXPORT_SYMBOL(skb_pull); 2421 2422 /** 2423 * skb_pull_data - remove data from the start of a buffer returning its 2424 * original position. 2425 * @skb: buffer to use 2426 * @len: amount of data to remove 2427 * 2428 * This function removes data from the start of a buffer, returning 2429 * the memory to the headroom. A pointer to the original data in the buffer 2430 * is returned after checking if there is enough data to pull. Once the 2431 * data has been pulled future pushes will overwrite the old data. 2432 */ 2433 void *skb_pull_data(struct sk_buff *skb, size_t len) 2434 { 2435 void *data = skb->data; 2436 2437 if (skb->len < len) 2438 return NULL; 2439 2440 skb_pull(skb, len); 2441 2442 return data; 2443 } 2444 EXPORT_SYMBOL(skb_pull_data); 2445 2446 /** 2447 * skb_trim - remove end from a buffer 2448 * @skb: buffer to alter 2449 * @len: new length 2450 * 2451 * Cut the length of a buffer down by removing data from the tail. If 2452 * the buffer is already under the length specified it is not modified. 2453 * The skb must be linear. 2454 */ 2455 void skb_trim(struct sk_buff *skb, unsigned int len) 2456 { 2457 if (skb->len > len) 2458 __skb_trim(skb, len); 2459 } 2460 EXPORT_SYMBOL(skb_trim); 2461 2462 /* Trims skb to length len. It can change skb pointers. 2463 */ 2464 2465 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 2466 { 2467 struct sk_buff **fragp; 2468 struct sk_buff *frag; 2469 int offset = skb_headlen(skb); 2470 int nfrags = skb_shinfo(skb)->nr_frags; 2471 int i; 2472 int err; 2473 2474 if (skb_cloned(skb) && 2475 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 2476 return err; 2477 2478 i = 0; 2479 if (offset >= len) 2480 goto drop_pages; 2481 2482 for (; i < nfrags; i++) { 2483 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2484 2485 if (end < len) { 2486 offset = end; 2487 continue; 2488 } 2489 2490 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 2491 2492 drop_pages: 2493 skb_shinfo(skb)->nr_frags = i; 2494 2495 for (; i < nfrags; i++) 2496 skb_frag_unref(skb, i); 2497 2498 if (skb_has_frag_list(skb)) 2499 skb_drop_fraglist(skb); 2500 goto done; 2501 } 2502 2503 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 2504 fragp = &frag->next) { 2505 int end = offset + frag->len; 2506 2507 if (skb_shared(frag)) { 2508 struct sk_buff *nfrag; 2509 2510 nfrag = skb_clone(frag, GFP_ATOMIC); 2511 if (unlikely(!nfrag)) 2512 return -ENOMEM; 2513 2514 nfrag->next = frag->next; 2515 consume_skb(frag); 2516 frag = nfrag; 2517 *fragp = frag; 2518 } 2519 2520 if (end < len) { 2521 offset = end; 2522 continue; 2523 } 2524 2525 if (end > len && 2526 unlikely((err = pskb_trim(frag, len - offset)))) 2527 return err; 2528 2529 if (frag->next) 2530 skb_drop_list(&frag->next); 2531 break; 2532 } 2533 2534 done: 2535 if (len > skb_headlen(skb)) { 2536 skb->data_len -= skb->len - len; 2537 skb->len = len; 2538 } else { 2539 skb->len = len; 2540 skb->data_len = 0; 2541 skb_set_tail_pointer(skb, len); 2542 } 2543 2544 if (!skb->sk || skb->destructor == sock_edemux) 2545 skb_condense(skb); 2546 return 0; 2547 } 2548 EXPORT_SYMBOL(___pskb_trim); 2549 2550 /* Note : use pskb_trim_rcsum() instead of calling this directly 2551 */ 2552 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) 2553 { 2554 if (skb->ip_summed == CHECKSUM_COMPLETE) { 2555 int delta = skb->len - len; 2556 2557 skb->csum = csum_block_sub(skb->csum, 2558 skb_checksum(skb, len, delta, 0), 2559 len); 2560 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 2561 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; 2562 int offset = skb_checksum_start_offset(skb) + skb->csum_offset; 2563 2564 if (offset + sizeof(__sum16) > hdlen) 2565 return -EINVAL; 2566 } 2567 return __pskb_trim(skb, len); 2568 } 2569 EXPORT_SYMBOL(pskb_trim_rcsum_slow); 2570 2571 /** 2572 * __pskb_pull_tail - advance tail of skb header 2573 * @skb: buffer to reallocate 2574 * @delta: number of bytes to advance tail 2575 * 2576 * The function makes a sense only on a fragmented &sk_buff, 2577 * it expands header moving its tail forward and copying necessary 2578 * data from fragmented part. 2579 * 2580 * &sk_buff MUST have reference count of 1. 2581 * 2582 * Returns %NULL (and &sk_buff does not change) if pull failed 2583 * or value of new tail of skb in the case of success. 2584 * 2585 * All the pointers pointing into skb header may change and must be 2586 * reloaded after call to this function. 2587 */ 2588 2589 /* Moves tail of skb head forward, copying data from fragmented part, 2590 * when it is necessary. 2591 * 1. It may fail due to malloc failure. 2592 * 2. It may change skb pointers. 2593 * 2594 * It is pretty complicated. Luckily, it is called only in exceptional cases. 2595 */ 2596 void *__pskb_pull_tail(struct sk_buff *skb, int delta) 2597 { 2598 /* If skb has not enough free space at tail, get new one 2599 * plus 128 bytes for future expansions. If we have enough 2600 * room at tail, reallocate without expansion only if skb is cloned. 2601 */ 2602 int i, k, eat = (skb->tail + delta) - skb->end; 2603 2604 if (eat > 0 || skb_cloned(skb)) { 2605 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 2606 GFP_ATOMIC)) 2607 return NULL; 2608 } 2609 2610 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 2611 skb_tail_pointer(skb), delta)); 2612 2613 /* Optimization: no fragments, no reasons to preestimate 2614 * size of pulled pages. Superb. 2615 */ 2616 if (!skb_has_frag_list(skb)) 2617 goto pull_pages; 2618 2619 /* Estimate size of pulled pages. */ 2620 eat = delta; 2621 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2622 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2623 2624 if (size >= eat) 2625 goto pull_pages; 2626 eat -= size; 2627 } 2628 2629 /* If we need update frag list, we are in troubles. 2630 * Certainly, it is possible to add an offset to skb data, 2631 * but taking into account that pulling is expected to 2632 * be very rare operation, it is worth to fight against 2633 * further bloating skb head and crucify ourselves here instead. 2634 * Pure masohism, indeed. 8)8) 2635 */ 2636 if (eat) { 2637 struct sk_buff *list = skb_shinfo(skb)->frag_list; 2638 struct sk_buff *clone = NULL; 2639 struct sk_buff *insp = NULL; 2640 2641 do { 2642 if (list->len <= eat) { 2643 /* Eaten as whole. */ 2644 eat -= list->len; 2645 list = list->next; 2646 insp = list; 2647 } else { 2648 /* Eaten partially. */ 2649 if (skb_is_gso(skb) && !list->head_frag && 2650 skb_headlen(list)) 2651 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 2652 2653 if (skb_shared(list)) { 2654 /* Sucks! We need to fork list. :-( */ 2655 clone = skb_clone(list, GFP_ATOMIC); 2656 if (!clone) 2657 return NULL; 2658 insp = list->next; 2659 list = clone; 2660 } else { 2661 /* This may be pulled without 2662 * problems. */ 2663 insp = list; 2664 } 2665 if (!pskb_pull(list, eat)) { 2666 kfree_skb(clone); 2667 return NULL; 2668 } 2669 break; 2670 } 2671 } while (eat); 2672 2673 /* Free pulled out fragments. */ 2674 while ((list = skb_shinfo(skb)->frag_list) != insp) { 2675 skb_shinfo(skb)->frag_list = list->next; 2676 consume_skb(list); 2677 } 2678 /* And insert new clone at head. */ 2679 if (clone) { 2680 clone->next = list; 2681 skb_shinfo(skb)->frag_list = clone; 2682 } 2683 } 2684 /* Success! Now we may commit changes to skb data. */ 2685 2686 pull_pages: 2687 eat = delta; 2688 k = 0; 2689 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2690 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2691 2692 if (size <= eat) { 2693 skb_frag_unref(skb, i); 2694 eat -= size; 2695 } else { 2696 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2697 2698 *frag = skb_shinfo(skb)->frags[i]; 2699 if (eat) { 2700 skb_frag_off_add(frag, eat); 2701 skb_frag_size_sub(frag, eat); 2702 if (!i) 2703 goto end; 2704 eat = 0; 2705 } 2706 k++; 2707 } 2708 } 2709 skb_shinfo(skb)->nr_frags = k; 2710 2711 end: 2712 skb->tail += delta; 2713 skb->data_len -= delta; 2714 2715 if (!skb->data_len) 2716 skb_zcopy_clear(skb, false); 2717 2718 return skb_tail_pointer(skb); 2719 } 2720 EXPORT_SYMBOL(__pskb_pull_tail); 2721 2722 /** 2723 * skb_copy_bits - copy bits from skb to kernel buffer 2724 * @skb: source skb 2725 * @offset: offset in source 2726 * @to: destination buffer 2727 * @len: number of bytes to copy 2728 * 2729 * Copy the specified number of bytes from the source skb to the 2730 * destination buffer. 2731 * 2732 * CAUTION ! : 2733 * If its prototype is ever changed, 2734 * check arch/{*}/net/{*}.S files, 2735 * since it is called from BPF assembly code. 2736 */ 2737 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 2738 { 2739 int start = skb_headlen(skb); 2740 struct sk_buff *frag_iter; 2741 int i, copy; 2742 2743 if (offset > (int)skb->len - len) 2744 goto fault; 2745 2746 /* Copy header. */ 2747 if ((copy = start - offset) > 0) { 2748 if (copy > len) 2749 copy = len; 2750 skb_copy_from_linear_data_offset(skb, offset, to, copy); 2751 if ((len -= copy) == 0) 2752 return 0; 2753 offset += copy; 2754 to += copy; 2755 } 2756 2757 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2758 int end; 2759 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2760 2761 WARN_ON(start > offset + len); 2762 2763 end = start + skb_frag_size(f); 2764 if ((copy = end - offset) > 0) { 2765 u32 p_off, p_len, copied; 2766 struct page *p; 2767 u8 *vaddr; 2768 2769 if (copy > len) 2770 copy = len; 2771 2772 skb_frag_foreach_page(f, 2773 skb_frag_off(f) + offset - start, 2774 copy, p, p_off, p_len, copied) { 2775 vaddr = kmap_atomic(p); 2776 memcpy(to + copied, vaddr + p_off, p_len); 2777 kunmap_atomic(vaddr); 2778 } 2779 2780 if ((len -= copy) == 0) 2781 return 0; 2782 offset += copy; 2783 to += copy; 2784 } 2785 start = end; 2786 } 2787 2788 skb_walk_frags(skb, frag_iter) { 2789 int end; 2790 2791 WARN_ON(start > offset + len); 2792 2793 end = start + frag_iter->len; 2794 if ((copy = end - offset) > 0) { 2795 if (copy > len) 2796 copy = len; 2797 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 2798 goto fault; 2799 if ((len -= copy) == 0) 2800 return 0; 2801 offset += copy; 2802 to += copy; 2803 } 2804 start = end; 2805 } 2806 2807 if (!len) 2808 return 0; 2809 2810 fault: 2811 return -EFAULT; 2812 } 2813 EXPORT_SYMBOL(skb_copy_bits); 2814 2815 /* 2816 * Callback from splice_to_pipe(), if we need to release some pages 2817 * at the end of the spd in case we error'ed out in filling the pipe. 2818 */ 2819 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 2820 { 2821 put_page(spd->pages[i]); 2822 } 2823 2824 static struct page *linear_to_page(struct page *page, unsigned int *len, 2825 unsigned int *offset, 2826 struct sock *sk) 2827 { 2828 struct page_frag *pfrag = sk_page_frag(sk); 2829 2830 if (!sk_page_frag_refill(sk, pfrag)) 2831 return NULL; 2832 2833 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 2834 2835 memcpy(page_address(pfrag->page) + pfrag->offset, 2836 page_address(page) + *offset, *len); 2837 *offset = pfrag->offset; 2838 pfrag->offset += *len; 2839 2840 return pfrag->page; 2841 } 2842 2843 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 2844 struct page *page, 2845 unsigned int offset) 2846 { 2847 return spd->nr_pages && 2848 spd->pages[spd->nr_pages - 1] == page && 2849 (spd->partial[spd->nr_pages - 1].offset + 2850 spd->partial[spd->nr_pages - 1].len == offset); 2851 } 2852 2853 /* 2854 * Fill page/offset/length into spd, if it can hold more pages. 2855 */ 2856 static bool spd_fill_page(struct splice_pipe_desc *spd, 2857 struct pipe_inode_info *pipe, struct page *page, 2858 unsigned int *len, unsigned int offset, 2859 bool linear, 2860 struct sock *sk) 2861 { 2862 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 2863 return true; 2864 2865 if (linear) { 2866 page = linear_to_page(page, len, &offset, sk); 2867 if (!page) 2868 return true; 2869 } 2870 if (spd_can_coalesce(spd, page, offset)) { 2871 spd->partial[spd->nr_pages - 1].len += *len; 2872 return false; 2873 } 2874 get_page(page); 2875 spd->pages[spd->nr_pages] = page; 2876 spd->partial[spd->nr_pages].len = *len; 2877 spd->partial[spd->nr_pages].offset = offset; 2878 spd->nr_pages++; 2879 2880 return false; 2881 } 2882 2883 static bool __splice_segment(struct page *page, unsigned int poff, 2884 unsigned int plen, unsigned int *off, 2885 unsigned int *len, 2886 struct splice_pipe_desc *spd, bool linear, 2887 struct sock *sk, 2888 struct pipe_inode_info *pipe) 2889 { 2890 if (!*len) 2891 return true; 2892 2893 /* skip this segment if already processed */ 2894 if (*off >= plen) { 2895 *off -= plen; 2896 return false; 2897 } 2898 2899 /* ignore any bits we already processed */ 2900 poff += *off; 2901 plen -= *off; 2902 *off = 0; 2903 2904 do { 2905 unsigned int flen = min(*len, plen); 2906 2907 if (spd_fill_page(spd, pipe, page, &flen, poff, 2908 linear, sk)) 2909 return true; 2910 poff += flen; 2911 plen -= flen; 2912 *len -= flen; 2913 } while (*len && plen); 2914 2915 return false; 2916 } 2917 2918 /* 2919 * Map linear and fragment data from the skb to spd. It reports true if the 2920 * pipe is full or if we already spliced the requested length. 2921 */ 2922 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 2923 unsigned int *offset, unsigned int *len, 2924 struct splice_pipe_desc *spd, struct sock *sk) 2925 { 2926 int seg; 2927 struct sk_buff *iter; 2928 2929 /* map the linear part : 2930 * If skb->head_frag is set, this 'linear' part is backed by a 2931 * fragment, and if the head is not shared with any clones then 2932 * we can avoid a copy since we own the head portion of this page. 2933 */ 2934 if (__splice_segment(virt_to_page(skb->data), 2935 (unsigned long) skb->data & (PAGE_SIZE - 1), 2936 skb_headlen(skb), 2937 offset, len, spd, 2938 skb_head_is_locked(skb), 2939 sk, pipe)) 2940 return true; 2941 2942 /* 2943 * then map the fragments 2944 */ 2945 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 2946 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 2947 2948 if (__splice_segment(skb_frag_page(f), 2949 skb_frag_off(f), skb_frag_size(f), 2950 offset, len, spd, false, sk, pipe)) 2951 return true; 2952 } 2953 2954 skb_walk_frags(skb, iter) { 2955 if (*offset >= iter->len) { 2956 *offset -= iter->len; 2957 continue; 2958 } 2959 /* __skb_splice_bits() only fails if the output has no room 2960 * left, so no point in going over the frag_list for the error 2961 * case. 2962 */ 2963 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 2964 return true; 2965 } 2966 2967 return false; 2968 } 2969 2970 /* 2971 * Map data from the skb to a pipe. Should handle both the linear part, 2972 * the fragments, and the frag list. 2973 */ 2974 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 2975 struct pipe_inode_info *pipe, unsigned int tlen, 2976 unsigned int flags) 2977 { 2978 struct partial_page partial[MAX_SKB_FRAGS]; 2979 struct page *pages[MAX_SKB_FRAGS]; 2980 struct splice_pipe_desc spd = { 2981 .pages = pages, 2982 .partial = partial, 2983 .nr_pages_max = MAX_SKB_FRAGS, 2984 .ops = &nosteal_pipe_buf_ops, 2985 .spd_release = sock_spd_release, 2986 }; 2987 int ret = 0; 2988 2989 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 2990 2991 if (spd.nr_pages) 2992 ret = splice_to_pipe(pipe, &spd); 2993 2994 return ret; 2995 } 2996 EXPORT_SYMBOL_GPL(skb_splice_bits); 2997 2998 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg, 2999 struct kvec *vec, size_t num, size_t size) 3000 { 3001 struct socket *sock = sk->sk_socket; 3002 3003 if (!sock) 3004 return -EINVAL; 3005 return kernel_sendmsg(sock, msg, vec, num, size); 3006 } 3007 3008 static int sendpage_unlocked(struct sock *sk, struct page *page, int offset, 3009 size_t size, int flags) 3010 { 3011 struct socket *sock = sk->sk_socket; 3012 3013 if (!sock) 3014 return -EINVAL; 3015 return kernel_sendpage(sock, page, offset, size, flags); 3016 } 3017 3018 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg, 3019 struct kvec *vec, size_t num, size_t size); 3020 typedef int (*sendpage_func)(struct sock *sk, struct page *page, int offset, 3021 size_t size, int flags); 3022 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, 3023 int len, sendmsg_func sendmsg, sendpage_func sendpage) 3024 { 3025 unsigned int orig_len = len; 3026 struct sk_buff *head = skb; 3027 unsigned short fragidx; 3028 int slen, ret; 3029 3030 do_frag_list: 3031 3032 /* Deal with head data */ 3033 while (offset < skb_headlen(skb) && len) { 3034 struct kvec kv; 3035 struct msghdr msg; 3036 3037 slen = min_t(int, len, skb_headlen(skb) - offset); 3038 kv.iov_base = skb->data + offset; 3039 kv.iov_len = slen; 3040 memset(&msg, 0, sizeof(msg)); 3041 msg.msg_flags = MSG_DONTWAIT; 3042 3043 ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked, 3044 sendmsg_unlocked, sk, &msg, &kv, 1, slen); 3045 if (ret <= 0) 3046 goto error; 3047 3048 offset += ret; 3049 len -= ret; 3050 } 3051 3052 /* All the data was skb head? */ 3053 if (!len) 3054 goto out; 3055 3056 /* Make offset relative to start of frags */ 3057 offset -= skb_headlen(skb); 3058 3059 /* Find where we are in frag list */ 3060 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3061 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3062 3063 if (offset < skb_frag_size(frag)) 3064 break; 3065 3066 offset -= skb_frag_size(frag); 3067 } 3068 3069 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3070 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3071 3072 slen = min_t(size_t, len, skb_frag_size(frag) - offset); 3073 3074 while (slen) { 3075 ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked, 3076 sendpage_unlocked, sk, 3077 skb_frag_page(frag), 3078 skb_frag_off(frag) + offset, 3079 slen, MSG_DONTWAIT); 3080 if (ret <= 0) 3081 goto error; 3082 3083 len -= ret; 3084 offset += ret; 3085 slen -= ret; 3086 } 3087 3088 offset = 0; 3089 } 3090 3091 if (len) { 3092 /* Process any frag lists */ 3093 3094 if (skb == head) { 3095 if (skb_has_frag_list(skb)) { 3096 skb = skb_shinfo(skb)->frag_list; 3097 goto do_frag_list; 3098 } 3099 } else if (skb->next) { 3100 skb = skb->next; 3101 goto do_frag_list; 3102 } 3103 } 3104 3105 out: 3106 return orig_len - len; 3107 3108 error: 3109 return orig_len == len ? ret : orig_len - len; 3110 } 3111 3112 /* Send skb data on a socket. Socket must be locked. */ 3113 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 3114 int len) 3115 { 3116 return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked, 3117 kernel_sendpage_locked); 3118 } 3119 EXPORT_SYMBOL_GPL(skb_send_sock_locked); 3120 3121 /* Send skb data on a socket. Socket must be unlocked. */ 3122 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) 3123 { 3124 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked, 3125 sendpage_unlocked); 3126 } 3127 3128 /** 3129 * skb_store_bits - store bits from kernel buffer to skb 3130 * @skb: destination buffer 3131 * @offset: offset in destination 3132 * @from: source buffer 3133 * @len: number of bytes to copy 3134 * 3135 * Copy the specified number of bytes from the source buffer to the 3136 * destination skb. This function handles all the messy bits of 3137 * traversing fragment lists and such. 3138 */ 3139 3140 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 3141 { 3142 int start = skb_headlen(skb); 3143 struct sk_buff *frag_iter; 3144 int i, copy; 3145 3146 if (offset > (int)skb->len - len) 3147 goto fault; 3148 3149 if ((copy = start - offset) > 0) { 3150 if (copy > len) 3151 copy = len; 3152 skb_copy_to_linear_data_offset(skb, offset, from, copy); 3153 if ((len -= copy) == 0) 3154 return 0; 3155 offset += copy; 3156 from += copy; 3157 } 3158 3159 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3160 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3161 int end; 3162 3163 WARN_ON(start > offset + len); 3164 3165 end = start + skb_frag_size(frag); 3166 if ((copy = end - offset) > 0) { 3167 u32 p_off, p_len, copied; 3168 struct page *p; 3169 u8 *vaddr; 3170 3171 if (copy > len) 3172 copy = len; 3173 3174 skb_frag_foreach_page(frag, 3175 skb_frag_off(frag) + offset - start, 3176 copy, p, p_off, p_len, copied) { 3177 vaddr = kmap_atomic(p); 3178 memcpy(vaddr + p_off, from + copied, p_len); 3179 kunmap_atomic(vaddr); 3180 } 3181 3182 if ((len -= copy) == 0) 3183 return 0; 3184 offset += copy; 3185 from += copy; 3186 } 3187 start = end; 3188 } 3189 3190 skb_walk_frags(skb, frag_iter) { 3191 int end; 3192 3193 WARN_ON(start > offset + len); 3194 3195 end = start + frag_iter->len; 3196 if ((copy = end - offset) > 0) { 3197 if (copy > len) 3198 copy = len; 3199 if (skb_store_bits(frag_iter, offset - start, 3200 from, copy)) 3201 goto fault; 3202 if ((len -= copy) == 0) 3203 return 0; 3204 offset += copy; 3205 from += copy; 3206 } 3207 start = end; 3208 } 3209 if (!len) 3210 return 0; 3211 3212 fault: 3213 return -EFAULT; 3214 } 3215 EXPORT_SYMBOL(skb_store_bits); 3216 3217 /* Checksum skb data. */ 3218 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 3219 __wsum csum, const struct skb_checksum_ops *ops) 3220 { 3221 int start = skb_headlen(skb); 3222 int i, copy = start - offset; 3223 struct sk_buff *frag_iter; 3224 int pos = 0; 3225 3226 /* Checksum header. */ 3227 if (copy > 0) { 3228 if (copy > len) 3229 copy = len; 3230 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext, 3231 skb->data + offset, copy, csum); 3232 if ((len -= copy) == 0) 3233 return csum; 3234 offset += copy; 3235 pos = copy; 3236 } 3237 3238 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3239 int end; 3240 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3241 3242 WARN_ON(start > offset + len); 3243 3244 end = start + skb_frag_size(frag); 3245 if ((copy = end - offset) > 0) { 3246 u32 p_off, p_len, copied; 3247 struct page *p; 3248 __wsum csum2; 3249 u8 *vaddr; 3250 3251 if (copy > len) 3252 copy = len; 3253 3254 skb_frag_foreach_page(frag, 3255 skb_frag_off(frag) + offset - start, 3256 copy, p, p_off, p_len, copied) { 3257 vaddr = kmap_atomic(p); 3258 csum2 = INDIRECT_CALL_1(ops->update, 3259 csum_partial_ext, 3260 vaddr + p_off, p_len, 0); 3261 kunmap_atomic(vaddr); 3262 csum = INDIRECT_CALL_1(ops->combine, 3263 csum_block_add_ext, csum, 3264 csum2, pos, p_len); 3265 pos += p_len; 3266 } 3267 3268 if (!(len -= copy)) 3269 return csum; 3270 offset += copy; 3271 } 3272 start = end; 3273 } 3274 3275 skb_walk_frags(skb, frag_iter) { 3276 int end; 3277 3278 WARN_ON(start > offset + len); 3279 3280 end = start + frag_iter->len; 3281 if ((copy = end - offset) > 0) { 3282 __wsum csum2; 3283 if (copy > len) 3284 copy = len; 3285 csum2 = __skb_checksum(frag_iter, offset - start, 3286 copy, 0, ops); 3287 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext, 3288 csum, csum2, pos, copy); 3289 if ((len -= copy) == 0) 3290 return csum; 3291 offset += copy; 3292 pos += copy; 3293 } 3294 start = end; 3295 } 3296 BUG_ON(len); 3297 3298 return csum; 3299 } 3300 EXPORT_SYMBOL(__skb_checksum); 3301 3302 __wsum skb_checksum(const struct sk_buff *skb, int offset, 3303 int len, __wsum csum) 3304 { 3305 const struct skb_checksum_ops ops = { 3306 .update = csum_partial_ext, 3307 .combine = csum_block_add_ext, 3308 }; 3309 3310 return __skb_checksum(skb, offset, len, csum, &ops); 3311 } 3312 EXPORT_SYMBOL(skb_checksum); 3313 3314 /* Both of above in one bottle. */ 3315 3316 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 3317 u8 *to, int len) 3318 { 3319 int start = skb_headlen(skb); 3320 int i, copy = start - offset; 3321 struct sk_buff *frag_iter; 3322 int pos = 0; 3323 __wsum csum = 0; 3324 3325 /* Copy header. */ 3326 if (copy > 0) { 3327 if (copy > len) 3328 copy = len; 3329 csum = csum_partial_copy_nocheck(skb->data + offset, to, 3330 copy); 3331 if ((len -= copy) == 0) 3332 return csum; 3333 offset += copy; 3334 to += copy; 3335 pos = copy; 3336 } 3337 3338 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3339 int end; 3340 3341 WARN_ON(start > offset + len); 3342 3343 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3344 if ((copy = end - offset) > 0) { 3345 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3346 u32 p_off, p_len, copied; 3347 struct page *p; 3348 __wsum csum2; 3349 u8 *vaddr; 3350 3351 if (copy > len) 3352 copy = len; 3353 3354 skb_frag_foreach_page(frag, 3355 skb_frag_off(frag) + offset - start, 3356 copy, p, p_off, p_len, copied) { 3357 vaddr = kmap_atomic(p); 3358 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 3359 to + copied, 3360 p_len); 3361 kunmap_atomic(vaddr); 3362 csum = csum_block_add(csum, csum2, pos); 3363 pos += p_len; 3364 } 3365 3366 if (!(len -= copy)) 3367 return csum; 3368 offset += copy; 3369 to += copy; 3370 } 3371 start = end; 3372 } 3373 3374 skb_walk_frags(skb, frag_iter) { 3375 __wsum csum2; 3376 int end; 3377 3378 WARN_ON(start > offset + len); 3379 3380 end = start + frag_iter->len; 3381 if ((copy = end - offset) > 0) { 3382 if (copy > len) 3383 copy = len; 3384 csum2 = skb_copy_and_csum_bits(frag_iter, 3385 offset - start, 3386 to, copy); 3387 csum = csum_block_add(csum, csum2, pos); 3388 if ((len -= copy) == 0) 3389 return csum; 3390 offset += copy; 3391 to += copy; 3392 pos += copy; 3393 } 3394 start = end; 3395 } 3396 BUG_ON(len); 3397 return csum; 3398 } 3399 EXPORT_SYMBOL(skb_copy_and_csum_bits); 3400 3401 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 3402 { 3403 __sum16 sum; 3404 3405 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 3406 /* See comments in __skb_checksum_complete(). */ 3407 if (likely(!sum)) { 3408 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3409 !skb->csum_complete_sw) 3410 netdev_rx_csum_fault(skb->dev, skb); 3411 } 3412 if (!skb_shared(skb)) 3413 skb->csum_valid = !sum; 3414 return sum; 3415 } 3416 EXPORT_SYMBOL(__skb_checksum_complete_head); 3417 3418 /* This function assumes skb->csum already holds pseudo header's checksum, 3419 * which has been changed from the hardware checksum, for example, by 3420 * __skb_checksum_validate_complete(). And, the original skb->csum must 3421 * have been validated unsuccessfully for CHECKSUM_COMPLETE case. 3422 * 3423 * It returns non-zero if the recomputed checksum is still invalid, otherwise 3424 * zero. The new checksum is stored back into skb->csum unless the skb is 3425 * shared. 3426 */ 3427 __sum16 __skb_checksum_complete(struct sk_buff *skb) 3428 { 3429 __wsum csum; 3430 __sum16 sum; 3431 3432 csum = skb_checksum(skb, 0, skb->len, 0); 3433 3434 sum = csum_fold(csum_add(skb->csum, csum)); 3435 /* This check is inverted, because we already knew the hardware 3436 * checksum is invalid before calling this function. So, if the 3437 * re-computed checksum is valid instead, then we have a mismatch 3438 * between the original skb->csum and skb_checksum(). This means either 3439 * the original hardware checksum is incorrect or we screw up skb->csum 3440 * when moving skb->data around. 3441 */ 3442 if (likely(!sum)) { 3443 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3444 !skb->csum_complete_sw) 3445 netdev_rx_csum_fault(skb->dev, skb); 3446 } 3447 3448 if (!skb_shared(skb)) { 3449 /* Save full packet checksum */ 3450 skb->csum = csum; 3451 skb->ip_summed = CHECKSUM_COMPLETE; 3452 skb->csum_complete_sw = 1; 3453 skb->csum_valid = !sum; 3454 } 3455 3456 return sum; 3457 } 3458 EXPORT_SYMBOL(__skb_checksum_complete); 3459 3460 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum) 3461 { 3462 net_warn_ratelimited( 3463 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3464 __func__); 3465 return 0; 3466 } 3467 3468 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2, 3469 int offset, int len) 3470 { 3471 net_warn_ratelimited( 3472 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3473 __func__); 3474 return 0; 3475 } 3476 3477 static const struct skb_checksum_ops default_crc32c_ops = { 3478 .update = warn_crc32c_csum_update, 3479 .combine = warn_crc32c_csum_combine, 3480 }; 3481 3482 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly = 3483 &default_crc32c_ops; 3484 EXPORT_SYMBOL(crc32c_csum_stub); 3485 3486 /** 3487 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 3488 * @from: source buffer 3489 * 3490 * Calculates the amount of linear headroom needed in the 'to' skb passed 3491 * into skb_zerocopy(). 3492 */ 3493 unsigned int 3494 skb_zerocopy_headlen(const struct sk_buff *from) 3495 { 3496 unsigned int hlen = 0; 3497 3498 if (!from->head_frag || 3499 skb_headlen(from) < L1_CACHE_BYTES || 3500 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) { 3501 hlen = skb_headlen(from); 3502 if (!hlen) 3503 hlen = from->len; 3504 } 3505 3506 if (skb_has_frag_list(from)) 3507 hlen = from->len; 3508 3509 return hlen; 3510 } 3511 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 3512 3513 /** 3514 * skb_zerocopy - Zero copy skb to skb 3515 * @to: destination buffer 3516 * @from: source buffer 3517 * @len: number of bytes to copy from source buffer 3518 * @hlen: size of linear headroom in destination buffer 3519 * 3520 * Copies up to `len` bytes from `from` to `to` by creating references 3521 * to the frags in the source buffer. 3522 * 3523 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 3524 * headroom in the `to` buffer. 3525 * 3526 * Return value: 3527 * 0: everything is OK 3528 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 3529 * -EFAULT: skb_copy_bits() found some problem with skb geometry 3530 */ 3531 int 3532 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 3533 { 3534 int i, j = 0; 3535 int plen = 0; /* length of skb->head fragment */ 3536 int ret; 3537 struct page *page; 3538 unsigned int offset; 3539 3540 BUG_ON(!from->head_frag && !hlen); 3541 3542 /* dont bother with small payloads */ 3543 if (len <= skb_tailroom(to)) 3544 return skb_copy_bits(from, 0, skb_put(to, len), len); 3545 3546 if (hlen) { 3547 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 3548 if (unlikely(ret)) 3549 return ret; 3550 len -= hlen; 3551 } else { 3552 plen = min_t(int, skb_headlen(from), len); 3553 if (plen) { 3554 page = virt_to_head_page(from->head); 3555 offset = from->data - (unsigned char *)page_address(page); 3556 __skb_fill_page_desc(to, 0, page, offset, plen); 3557 get_page(page); 3558 j = 1; 3559 len -= plen; 3560 } 3561 } 3562 3563 skb_len_add(to, len + plen); 3564 3565 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 3566 skb_tx_error(from); 3567 return -ENOMEM; 3568 } 3569 skb_zerocopy_clone(to, from, GFP_ATOMIC); 3570 3571 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 3572 int size; 3573 3574 if (!len) 3575 break; 3576 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 3577 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]), 3578 len); 3579 skb_frag_size_set(&skb_shinfo(to)->frags[j], size); 3580 len -= size; 3581 skb_frag_ref(to, j); 3582 j++; 3583 } 3584 skb_shinfo(to)->nr_frags = j; 3585 3586 return 0; 3587 } 3588 EXPORT_SYMBOL_GPL(skb_zerocopy); 3589 3590 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 3591 { 3592 __wsum csum; 3593 long csstart; 3594 3595 if (skb->ip_summed == CHECKSUM_PARTIAL) 3596 csstart = skb_checksum_start_offset(skb); 3597 else 3598 csstart = skb_headlen(skb); 3599 3600 BUG_ON(csstart > skb_headlen(skb)); 3601 3602 skb_copy_from_linear_data(skb, to, csstart); 3603 3604 csum = 0; 3605 if (csstart != skb->len) 3606 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 3607 skb->len - csstart); 3608 3609 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3610 long csstuff = csstart + skb->csum_offset; 3611 3612 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 3613 } 3614 } 3615 EXPORT_SYMBOL(skb_copy_and_csum_dev); 3616 3617 /** 3618 * skb_dequeue - remove from the head of the queue 3619 * @list: list to dequeue from 3620 * 3621 * Remove the head of the list. The list lock is taken so the function 3622 * may be used safely with other locking list functions. The head item is 3623 * returned or %NULL if the list is empty. 3624 */ 3625 3626 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 3627 { 3628 unsigned long flags; 3629 struct sk_buff *result; 3630 3631 spin_lock_irqsave(&list->lock, flags); 3632 result = __skb_dequeue(list); 3633 spin_unlock_irqrestore(&list->lock, flags); 3634 return result; 3635 } 3636 EXPORT_SYMBOL(skb_dequeue); 3637 3638 /** 3639 * skb_dequeue_tail - remove from the tail of the queue 3640 * @list: list to dequeue from 3641 * 3642 * Remove the tail of the list. The list lock is taken so the function 3643 * may be used safely with other locking list functions. The tail item is 3644 * returned or %NULL if the list is empty. 3645 */ 3646 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 3647 { 3648 unsigned long flags; 3649 struct sk_buff *result; 3650 3651 spin_lock_irqsave(&list->lock, flags); 3652 result = __skb_dequeue_tail(list); 3653 spin_unlock_irqrestore(&list->lock, flags); 3654 return result; 3655 } 3656 EXPORT_SYMBOL(skb_dequeue_tail); 3657 3658 /** 3659 * skb_queue_purge - empty a list 3660 * @list: list to empty 3661 * 3662 * Delete all buffers on an &sk_buff list. Each buffer is removed from 3663 * the list and one reference dropped. This function takes the list 3664 * lock and is atomic with respect to other list locking functions. 3665 */ 3666 void skb_queue_purge(struct sk_buff_head *list) 3667 { 3668 struct sk_buff *skb; 3669 while ((skb = skb_dequeue(list)) != NULL) 3670 kfree_skb(skb); 3671 } 3672 EXPORT_SYMBOL(skb_queue_purge); 3673 3674 /** 3675 * skb_rbtree_purge - empty a skb rbtree 3676 * @root: root of the rbtree to empty 3677 * Return value: the sum of truesizes of all purged skbs. 3678 * 3679 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 3680 * the list and one reference dropped. This function does not take 3681 * any lock. Synchronization should be handled by the caller (e.g., TCP 3682 * out-of-order queue is protected by the socket lock). 3683 */ 3684 unsigned int skb_rbtree_purge(struct rb_root *root) 3685 { 3686 struct rb_node *p = rb_first(root); 3687 unsigned int sum = 0; 3688 3689 while (p) { 3690 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 3691 3692 p = rb_next(p); 3693 rb_erase(&skb->rbnode, root); 3694 sum += skb->truesize; 3695 kfree_skb(skb); 3696 } 3697 return sum; 3698 } 3699 3700 /** 3701 * skb_queue_head - queue a buffer at the list head 3702 * @list: list to use 3703 * @newsk: buffer to queue 3704 * 3705 * Queue a buffer at the start of the list. This function takes the 3706 * list lock and can be used safely with other locking &sk_buff functions 3707 * safely. 3708 * 3709 * A buffer cannot be placed on two lists at the same time. 3710 */ 3711 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 3712 { 3713 unsigned long flags; 3714 3715 spin_lock_irqsave(&list->lock, flags); 3716 __skb_queue_head(list, newsk); 3717 spin_unlock_irqrestore(&list->lock, flags); 3718 } 3719 EXPORT_SYMBOL(skb_queue_head); 3720 3721 /** 3722 * skb_queue_tail - queue a buffer at the list tail 3723 * @list: list to use 3724 * @newsk: buffer to queue 3725 * 3726 * Queue a buffer at the tail of the list. This function takes the 3727 * list lock and can be used safely with other locking &sk_buff functions 3728 * safely. 3729 * 3730 * A buffer cannot be placed on two lists at the same time. 3731 */ 3732 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 3733 { 3734 unsigned long flags; 3735 3736 spin_lock_irqsave(&list->lock, flags); 3737 __skb_queue_tail(list, newsk); 3738 spin_unlock_irqrestore(&list->lock, flags); 3739 } 3740 EXPORT_SYMBOL(skb_queue_tail); 3741 3742 /** 3743 * skb_unlink - remove a buffer from a list 3744 * @skb: buffer to remove 3745 * @list: list to use 3746 * 3747 * Remove a packet from a list. The list locks are taken and this 3748 * function is atomic with respect to other list locked calls 3749 * 3750 * You must know what list the SKB is on. 3751 */ 3752 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 3753 { 3754 unsigned long flags; 3755 3756 spin_lock_irqsave(&list->lock, flags); 3757 __skb_unlink(skb, list); 3758 spin_unlock_irqrestore(&list->lock, flags); 3759 } 3760 EXPORT_SYMBOL(skb_unlink); 3761 3762 /** 3763 * skb_append - append a buffer 3764 * @old: buffer to insert after 3765 * @newsk: buffer to insert 3766 * @list: list to use 3767 * 3768 * Place a packet after a given packet in a list. The list locks are taken 3769 * and this function is atomic with respect to other list locked calls. 3770 * A buffer cannot be placed on two lists at the same time. 3771 */ 3772 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 3773 { 3774 unsigned long flags; 3775 3776 spin_lock_irqsave(&list->lock, flags); 3777 __skb_queue_after(list, old, newsk); 3778 spin_unlock_irqrestore(&list->lock, flags); 3779 } 3780 EXPORT_SYMBOL(skb_append); 3781 3782 static inline void skb_split_inside_header(struct sk_buff *skb, 3783 struct sk_buff* skb1, 3784 const u32 len, const int pos) 3785 { 3786 int i; 3787 3788 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 3789 pos - len); 3790 /* And move data appendix as is. */ 3791 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 3792 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 3793 3794 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 3795 skb_shinfo(skb)->nr_frags = 0; 3796 skb1->data_len = skb->data_len; 3797 skb1->len += skb1->data_len; 3798 skb->data_len = 0; 3799 skb->len = len; 3800 skb_set_tail_pointer(skb, len); 3801 } 3802 3803 static inline void skb_split_no_header(struct sk_buff *skb, 3804 struct sk_buff* skb1, 3805 const u32 len, int pos) 3806 { 3807 int i, k = 0; 3808 const int nfrags = skb_shinfo(skb)->nr_frags; 3809 3810 skb_shinfo(skb)->nr_frags = 0; 3811 skb1->len = skb1->data_len = skb->len - len; 3812 skb->len = len; 3813 skb->data_len = len - pos; 3814 3815 for (i = 0; i < nfrags; i++) { 3816 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 3817 3818 if (pos + size > len) { 3819 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 3820 3821 if (pos < len) { 3822 /* Split frag. 3823 * We have two variants in this case: 3824 * 1. Move all the frag to the second 3825 * part, if it is possible. F.e. 3826 * this approach is mandatory for TUX, 3827 * where splitting is expensive. 3828 * 2. Split is accurately. We make this. 3829 */ 3830 skb_frag_ref(skb, i); 3831 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos); 3832 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 3833 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 3834 skb_shinfo(skb)->nr_frags++; 3835 } 3836 k++; 3837 } else 3838 skb_shinfo(skb)->nr_frags++; 3839 pos += size; 3840 } 3841 skb_shinfo(skb1)->nr_frags = k; 3842 } 3843 3844 /** 3845 * skb_split - Split fragmented skb to two parts at length len. 3846 * @skb: the buffer to split 3847 * @skb1: the buffer to receive the second part 3848 * @len: new length for skb 3849 */ 3850 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 3851 { 3852 int pos = skb_headlen(skb); 3853 const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY; 3854 3855 skb_zcopy_downgrade_managed(skb); 3856 3857 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags; 3858 skb_zerocopy_clone(skb1, skb, 0); 3859 if (len < pos) /* Split line is inside header. */ 3860 skb_split_inside_header(skb, skb1, len, pos); 3861 else /* Second chunk has no header, nothing to copy. */ 3862 skb_split_no_header(skb, skb1, len, pos); 3863 } 3864 EXPORT_SYMBOL(skb_split); 3865 3866 /* Shifting from/to a cloned skb is a no-go. 3867 * 3868 * Caller cannot keep skb_shinfo related pointers past calling here! 3869 */ 3870 static int skb_prepare_for_shift(struct sk_buff *skb) 3871 { 3872 return skb_unclone_keeptruesize(skb, GFP_ATOMIC); 3873 } 3874 3875 /** 3876 * skb_shift - Shifts paged data partially from skb to another 3877 * @tgt: buffer into which tail data gets added 3878 * @skb: buffer from which the paged data comes from 3879 * @shiftlen: shift up to this many bytes 3880 * 3881 * Attempts to shift up to shiftlen worth of bytes, which may be less than 3882 * the length of the skb, from skb to tgt. Returns number bytes shifted. 3883 * It's up to caller to free skb if everything was shifted. 3884 * 3885 * If @tgt runs out of frags, the whole operation is aborted. 3886 * 3887 * Skb cannot include anything else but paged data while tgt is allowed 3888 * to have non-paged data as well. 3889 * 3890 * TODO: full sized shift could be optimized but that would need 3891 * specialized skb free'er to handle frags without up-to-date nr_frags. 3892 */ 3893 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 3894 { 3895 int from, to, merge, todo; 3896 skb_frag_t *fragfrom, *fragto; 3897 3898 BUG_ON(shiftlen > skb->len); 3899 3900 if (skb_headlen(skb)) 3901 return 0; 3902 if (skb_zcopy(tgt) || skb_zcopy(skb)) 3903 return 0; 3904 3905 todo = shiftlen; 3906 from = 0; 3907 to = skb_shinfo(tgt)->nr_frags; 3908 fragfrom = &skb_shinfo(skb)->frags[from]; 3909 3910 /* Actual merge is delayed until the point when we know we can 3911 * commit all, so that we don't have to undo partial changes 3912 */ 3913 if (!to || 3914 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 3915 skb_frag_off(fragfrom))) { 3916 merge = -1; 3917 } else { 3918 merge = to - 1; 3919 3920 todo -= skb_frag_size(fragfrom); 3921 if (todo < 0) { 3922 if (skb_prepare_for_shift(skb) || 3923 skb_prepare_for_shift(tgt)) 3924 return 0; 3925 3926 /* All previous frag pointers might be stale! */ 3927 fragfrom = &skb_shinfo(skb)->frags[from]; 3928 fragto = &skb_shinfo(tgt)->frags[merge]; 3929 3930 skb_frag_size_add(fragto, shiftlen); 3931 skb_frag_size_sub(fragfrom, shiftlen); 3932 skb_frag_off_add(fragfrom, shiftlen); 3933 3934 goto onlymerged; 3935 } 3936 3937 from++; 3938 } 3939 3940 /* Skip full, not-fitting skb to avoid expensive operations */ 3941 if ((shiftlen == skb->len) && 3942 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 3943 return 0; 3944 3945 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 3946 return 0; 3947 3948 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 3949 if (to == MAX_SKB_FRAGS) 3950 return 0; 3951 3952 fragfrom = &skb_shinfo(skb)->frags[from]; 3953 fragto = &skb_shinfo(tgt)->frags[to]; 3954 3955 if (todo >= skb_frag_size(fragfrom)) { 3956 *fragto = *fragfrom; 3957 todo -= skb_frag_size(fragfrom); 3958 from++; 3959 to++; 3960 3961 } else { 3962 __skb_frag_ref(fragfrom); 3963 skb_frag_page_copy(fragto, fragfrom); 3964 skb_frag_off_copy(fragto, fragfrom); 3965 skb_frag_size_set(fragto, todo); 3966 3967 skb_frag_off_add(fragfrom, todo); 3968 skb_frag_size_sub(fragfrom, todo); 3969 todo = 0; 3970 3971 to++; 3972 break; 3973 } 3974 } 3975 3976 /* Ready to "commit" this state change to tgt */ 3977 skb_shinfo(tgt)->nr_frags = to; 3978 3979 if (merge >= 0) { 3980 fragfrom = &skb_shinfo(skb)->frags[0]; 3981 fragto = &skb_shinfo(tgt)->frags[merge]; 3982 3983 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 3984 __skb_frag_unref(fragfrom, skb->pp_recycle); 3985 } 3986 3987 /* Reposition in the original skb */ 3988 to = 0; 3989 while (from < skb_shinfo(skb)->nr_frags) 3990 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 3991 skb_shinfo(skb)->nr_frags = to; 3992 3993 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 3994 3995 onlymerged: 3996 /* Most likely the tgt won't ever need its checksum anymore, skb on 3997 * the other hand might need it if it needs to be resent 3998 */ 3999 tgt->ip_summed = CHECKSUM_PARTIAL; 4000 skb->ip_summed = CHECKSUM_PARTIAL; 4001 4002 skb_len_add(skb, -shiftlen); 4003 skb_len_add(tgt, shiftlen); 4004 4005 return shiftlen; 4006 } 4007 4008 /** 4009 * skb_prepare_seq_read - Prepare a sequential read of skb data 4010 * @skb: the buffer to read 4011 * @from: lower offset of data to be read 4012 * @to: upper offset of data to be read 4013 * @st: state variable 4014 * 4015 * Initializes the specified state variable. Must be called before 4016 * invoking skb_seq_read() for the first time. 4017 */ 4018 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 4019 unsigned int to, struct skb_seq_state *st) 4020 { 4021 st->lower_offset = from; 4022 st->upper_offset = to; 4023 st->root_skb = st->cur_skb = skb; 4024 st->frag_idx = st->stepped_offset = 0; 4025 st->frag_data = NULL; 4026 st->frag_off = 0; 4027 } 4028 EXPORT_SYMBOL(skb_prepare_seq_read); 4029 4030 /** 4031 * skb_seq_read - Sequentially read skb data 4032 * @consumed: number of bytes consumed by the caller so far 4033 * @data: destination pointer for data to be returned 4034 * @st: state variable 4035 * 4036 * Reads a block of skb data at @consumed relative to the 4037 * lower offset specified to skb_prepare_seq_read(). Assigns 4038 * the head of the data block to @data and returns the length 4039 * of the block or 0 if the end of the skb data or the upper 4040 * offset has been reached. 4041 * 4042 * The caller is not required to consume all of the data 4043 * returned, i.e. @consumed is typically set to the number 4044 * of bytes already consumed and the next call to 4045 * skb_seq_read() will return the remaining part of the block. 4046 * 4047 * Note 1: The size of each block of data returned can be arbitrary, 4048 * this limitation is the cost for zerocopy sequential 4049 * reads of potentially non linear data. 4050 * 4051 * Note 2: Fragment lists within fragments are not implemented 4052 * at the moment, state->root_skb could be replaced with 4053 * a stack for this purpose. 4054 */ 4055 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 4056 struct skb_seq_state *st) 4057 { 4058 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 4059 skb_frag_t *frag; 4060 4061 if (unlikely(abs_offset >= st->upper_offset)) { 4062 if (st->frag_data) { 4063 kunmap_atomic(st->frag_data); 4064 st->frag_data = NULL; 4065 } 4066 return 0; 4067 } 4068 4069 next_skb: 4070 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 4071 4072 if (abs_offset < block_limit && !st->frag_data) { 4073 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 4074 return block_limit - abs_offset; 4075 } 4076 4077 if (st->frag_idx == 0 && !st->frag_data) 4078 st->stepped_offset += skb_headlen(st->cur_skb); 4079 4080 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 4081 unsigned int pg_idx, pg_off, pg_sz; 4082 4083 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 4084 4085 pg_idx = 0; 4086 pg_off = skb_frag_off(frag); 4087 pg_sz = skb_frag_size(frag); 4088 4089 if (skb_frag_must_loop(skb_frag_page(frag))) { 4090 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT; 4091 pg_off = offset_in_page(pg_off + st->frag_off); 4092 pg_sz = min_t(unsigned int, pg_sz - st->frag_off, 4093 PAGE_SIZE - pg_off); 4094 } 4095 4096 block_limit = pg_sz + st->stepped_offset; 4097 if (abs_offset < block_limit) { 4098 if (!st->frag_data) 4099 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx); 4100 4101 *data = (u8 *)st->frag_data + pg_off + 4102 (abs_offset - st->stepped_offset); 4103 4104 return block_limit - abs_offset; 4105 } 4106 4107 if (st->frag_data) { 4108 kunmap_atomic(st->frag_data); 4109 st->frag_data = NULL; 4110 } 4111 4112 st->stepped_offset += pg_sz; 4113 st->frag_off += pg_sz; 4114 if (st->frag_off == skb_frag_size(frag)) { 4115 st->frag_off = 0; 4116 st->frag_idx++; 4117 } 4118 } 4119 4120 if (st->frag_data) { 4121 kunmap_atomic(st->frag_data); 4122 st->frag_data = NULL; 4123 } 4124 4125 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 4126 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 4127 st->frag_idx = 0; 4128 goto next_skb; 4129 } else if (st->cur_skb->next) { 4130 st->cur_skb = st->cur_skb->next; 4131 st->frag_idx = 0; 4132 goto next_skb; 4133 } 4134 4135 return 0; 4136 } 4137 EXPORT_SYMBOL(skb_seq_read); 4138 4139 /** 4140 * skb_abort_seq_read - Abort a sequential read of skb data 4141 * @st: state variable 4142 * 4143 * Must be called if skb_seq_read() was not called until it 4144 * returned 0. 4145 */ 4146 void skb_abort_seq_read(struct skb_seq_state *st) 4147 { 4148 if (st->frag_data) 4149 kunmap_atomic(st->frag_data); 4150 } 4151 EXPORT_SYMBOL(skb_abort_seq_read); 4152 4153 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 4154 4155 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 4156 struct ts_config *conf, 4157 struct ts_state *state) 4158 { 4159 return skb_seq_read(offset, text, TS_SKB_CB(state)); 4160 } 4161 4162 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 4163 { 4164 skb_abort_seq_read(TS_SKB_CB(state)); 4165 } 4166 4167 /** 4168 * skb_find_text - Find a text pattern in skb data 4169 * @skb: the buffer to look in 4170 * @from: search offset 4171 * @to: search limit 4172 * @config: textsearch configuration 4173 * 4174 * Finds a pattern in the skb data according to the specified 4175 * textsearch configuration. Use textsearch_next() to retrieve 4176 * subsequent occurrences of the pattern. Returns the offset 4177 * to the first occurrence or UINT_MAX if no match was found. 4178 */ 4179 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 4180 unsigned int to, struct ts_config *config) 4181 { 4182 struct ts_state state; 4183 unsigned int ret; 4184 4185 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb)); 4186 4187 config->get_next_block = skb_ts_get_next_block; 4188 config->finish = skb_ts_finish; 4189 4190 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 4191 4192 ret = textsearch_find(config, &state); 4193 return (ret <= to - from ? ret : UINT_MAX); 4194 } 4195 EXPORT_SYMBOL(skb_find_text); 4196 4197 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 4198 int offset, size_t size) 4199 { 4200 int i = skb_shinfo(skb)->nr_frags; 4201 4202 if (skb_can_coalesce(skb, i, page, offset)) { 4203 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 4204 } else if (i < MAX_SKB_FRAGS) { 4205 skb_zcopy_downgrade_managed(skb); 4206 get_page(page); 4207 skb_fill_page_desc_noacc(skb, i, page, offset, size); 4208 } else { 4209 return -EMSGSIZE; 4210 } 4211 4212 return 0; 4213 } 4214 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 4215 4216 /** 4217 * skb_pull_rcsum - pull skb and update receive checksum 4218 * @skb: buffer to update 4219 * @len: length of data pulled 4220 * 4221 * This function performs an skb_pull on the packet and updates 4222 * the CHECKSUM_COMPLETE checksum. It should be used on 4223 * receive path processing instead of skb_pull unless you know 4224 * that the checksum difference is zero (e.g., a valid IP header) 4225 * or you are setting ip_summed to CHECKSUM_NONE. 4226 */ 4227 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 4228 { 4229 unsigned char *data = skb->data; 4230 4231 BUG_ON(len > skb->len); 4232 __skb_pull(skb, len); 4233 skb_postpull_rcsum(skb, data, len); 4234 return skb->data; 4235 } 4236 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 4237 4238 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb) 4239 { 4240 skb_frag_t head_frag; 4241 struct page *page; 4242 4243 page = virt_to_head_page(frag_skb->head); 4244 __skb_frag_set_page(&head_frag, page); 4245 skb_frag_off_set(&head_frag, frag_skb->data - 4246 (unsigned char *)page_address(page)); 4247 skb_frag_size_set(&head_frag, skb_headlen(frag_skb)); 4248 return head_frag; 4249 } 4250 4251 struct sk_buff *skb_segment_list(struct sk_buff *skb, 4252 netdev_features_t features, 4253 unsigned int offset) 4254 { 4255 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list; 4256 unsigned int tnl_hlen = skb_tnl_header_len(skb); 4257 unsigned int delta_truesize = 0; 4258 unsigned int delta_len = 0; 4259 struct sk_buff *tail = NULL; 4260 struct sk_buff *nskb, *tmp; 4261 int len_diff, err; 4262 4263 skb_push(skb, -skb_network_offset(skb) + offset); 4264 4265 skb_shinfo(skb)->frag_list = NULL; 4266 4267 while (list_skb) { 4268 nskb = list_skb; 4269 list_skb = list_skb->next; 4270 4271 err = 0; 4272 delta_truesize += nskb->truesize; 4273 if (skb_shared(nskb)) { 4274 tmp = skb_clone(nskb, GFP_ATOMIC); 4275 if (tmp) { 4276 consume_skb(nskb); 4277 nskb = tmp; 4278 err = skb_unclone(nskb, GFP_ATOMIC); 4279 } else { 4280 err = -ENOMEM; 4281 } 4282 } 4283 4284 if (!tail) 4285 skb->next = nskb; 4286 else 4287 tail->next = nskb; 4288 4289 if (unlikely(err)) { 4290 nskb->next = list_skb; 4291 goto err_linearize; 4292 } 4293 4294 tail = nskb; 4295 4296 delta_len += nskb->len; 4297 4298 skb_push(nskb, -skb_network_offset(nskb) + offset); 4299 4300 skb_release_head_state(nskb); 4301 len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb); 4302 __copy_skb_header(nskb, skb); 4303 4304 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb)); 4305 nskb->transport_header += len_diff; 4306 skb_copy_from_linear_data_offset(skb, -tnl_hlen, 4307 nskb->data - tnl_hlen, 4308 offset + tnl_hlen); 4309 4310 if (skb_needs_linearize(nskb, features) && 4311 __skb_linearize(nskb)) 4312 goto err_linearize; 4313 } 4314 4315 skb->truesize = skb->truesize - delta_truesize; 4316 skb->data_len = skb->data_len - delta_len; 4317 skb->len = skb->len - delta_len; 4318 4319 skb_gso_reset(skb); 4320 4321 skb->prev = tail; 4322 4323 if (skb_needs_linearize(skb, features) && 4324 __skb_linearize(skb)) 4325 goto err_linearize; 4326 4327 skb_get(skb); 4328 4329 return skb; 4330 4331 err_linearize: 4332 kfree_skb_list(skb->next); 4333 skb->next = NULL; 4334 return ERR_PTR(-ENOMEM); 4335 } 4336 EXPORT_SYMBOL_GPL(skb_segment_list); 4337 4338 /** 4339 * skb_segment - Perform protocol segmentation on skb. 4340 * @head_skb: buffer to segment 4341 * @features: features for the output path (see dev->features) 4342 * 4343 * This function performs segmentation on the given skb. It returns 4344 * a pointer to the first in a list of new skbs for the segments. 4345 * In case of error it returns ERR_PTR(err). 4346 */ 4347 struct sk_buff *skb_segment(struct sk_buff *head_skb, 4348 netdev_features_t features) 4349 { 4350 struct sk_buff *segs = NULL; 4351 struct sk_buff *tail = NULL; 4352 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 4353 skb_frag_t *frag = skb_shinfo(head_skb)->frags; 4354 unsigned int mss = skb_shinfo(head_skb)->gso_size; 4355 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 4356 struct sk_buff *frag_skb = head_skb; 4357 unsigned int offset = doffset; 4358 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 4359 unsigned int partial_segs = 0; 4360 unsigned int headroom; 4361 unsigned int len = head_skb->len; 4362 __be16 proto; 4363 bool csum, sg; 4364 int nfrags = skb_shinfo(head_skb)->nr_frags; 4365 int err = -ENOMEM; 4366 int i = 0; 4367 int pos; 4368 4369 if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) && 4370 mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) { 4371 struct sk_buff *check_skb; 4372 4373 for (check_skb = list_skb; check_skb; check_skb = check_skb->next) { 4374 if (skb_headlen(check_skb) && !check_skb->head_frag) { 4375 /* gso_size is untrusted, and we have a frag_list with 4376 * a linear non head_frag item. 4377 * 4378 * If head_skb's headlen does not fit requested gso_size, 4379 * it means that the frag_list members do NOT terminate 4380 * on exact gso_size boundaries. Hence we cannot perform 4381 * skb_frag_t page sharing. Therefore we must fallback to 4382 * copying the frag_list skbs; we do so by disabling SG. 4383 */ 4384 features &= ~NETIF_F_SG; 4385 break; 4386 } 4387 } 4388 } 4389 4390 __skb_push(head_skb, doffset); 4391 proto = skb_network_protocol(head_skb, NULL); 4392 if (unlikely(!proto)) 4393 return ERR_PTR(-EINVAL); 4394 4395 sg = !!(features & NETIF_F_SG); 4396 csum = !!can_checksum_protocol(features, proto); 4397 4398 if (sg && csum && (mss != GSO_BY_FRAGS)) { 4399 if (!(features & NETIF_F_GSO_PARTIAL)) { 4400 struct sk_buff *iter; 4401 unsigned int frag_len; 4402 4403 if (!list_skb || 4404 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 4405 goto normal; 4406 4407 /* If we get here then all the required 4408 * GSO features except frag_list are supported. 4409 * Try to split the SKB to multiple GSO SKBs 4410 * with no frag_list. 4411 * Currently we can do that only when the buffers don't 4412 * have a linear part and all the buffers except 4413 * the last are of the same length. 4414 */ 4415 frag_len = list_skb->len; 4416 skb_walk_frags(head_skb, iter) { 4417 if (frag_len != iter->len && iter->next) 4418 goto normal; 4419 if (skb_headlen(iter) && !iter->head_frag) 4420 goto normal; 4421 4422 len -= iter->len; 4423 } 4424 4425 if (len != frag_len) 4426 goto normal; 4427 } 4428 4429 /* GSO partial only requires that we trim off any excess that 4430 * doesn't fit into an MSS sized block, so take care of that 4431 * now. 4432 */ 4433 partial_segs = len / mss; 4434 if (partial_segs > 1) 4435 mss *= partial_segs; 4436 else 4437 partial_segs = 0; 4438 } 4439 4440 normal: 4441 headroom = skb_headroom(head_skb); 4442 pos = skb_headlen(head_skb); 4443 4444 do { 4445 struct sk_buff *nskb; 4446 skb_frag_t *nskb_frag; 4447 int hsize; 4448 int size; 4449 4450 if (unlikely(mss == GSO_BY_FRAGS)) { 4451 len = list_skb->len; 4452 } else { 4453 len = head_skb->len - offset; 4454 if (len > mss) 4455 len = mss; 4456 } 4457 4458 hsize = skb_headlen(head_skb) - offset; 4459 4460 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) && 4461 (skb_headlen(list_skb) == len || sg)) { 4462 BUG_ON(skb_headlen(list_skb) > len); 4463 4464 i = 0; 4465 nfrags = skb_shinfo(list_skb)->nr_frags; 4466 frag = skb_shinfo(list_skb)->frags; 4467 frag_skb = list_skb; 4468 pos += skb_headlen(list_skb); 4469 4470 while (pos < offset + len) { 4471 BUG_ON(i >= nfrags); 4472 4473 size = skb_frag_size(frag); 4474 if (pos + size > offset + len) 4475 break; 4476 4477 i++; 4478 pos += size; 4479 frag++; 4480 } 4481 4482 nskb = skb_clone(list_skb, GFP_ATOMIC); 4483 list_skb = list_skb->next; 4484 4485 if (unlikely(!nskb)) 4486 goto err; 4487 4488 if (unlikely(pskb_trim(nskb, len))) { 4489 kfree_skb(nskb); 4490 goto err; 4491 } 4492 4493 hsize = skb_end_offset(nskb); 4494 if (skb_cow_head(nskb, doffset + headroom)) { 4495 kfree_skb(nskb); 4496 goto err; 4497 } 4498 4499 nskb->truesize += skb_end_offset(nskb) - hsize; 4500 skb_release_head_state(nskb); 4501 __skb_push(nskb, doffset); 4502 } else { 4503 if (hsize < 0) 4504 hsize = 0; 4505 if (hsize > len || !sg) 4506 hsize = len; 4507 4508 nskb = __alloc_skb(hsize + doffset + headroom, 4509 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 4510 NUMA_NO_NODE); 4511 4512 if (unlikely(!nskb)) 4513 goto err; 4514 4515 skb_reserve(nskb, headroom); 4516 __skb_put(nskb, doffset); 4517 } 4518 4519 if (segs) 4520 tail->next = nskb; 4521 else 4522 segs = nskb; 4523 tail = nskb; 4524 4525 __copy_skb_header(nskb, head_skb); 4526 4527 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 4528 skb_reset_mac_len(nskb); 4529 4530 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 4531 nskb->data - tnl_hlen, 4532 doffset + tnl_hlen); 4533 4534 if (nskb->len == len + doffset) 4535 goto perform_csum_check; 4536 4537 if (!sg) { 4538 if (!csum) { 4539 if (!nskb->remcsum_offload) 4540 nskb->ip_summed = CHECKSUM_NONE; 4541 SKB_GSO_CB(nskb)->csum = 4542 skb_copy_and_csum_bits(head_skb, offset, 4543 skb_put(nskb, 4544 len), 4545 len); 4546 SKB_GSO_CB(nskb)->csum_start = 4547 skb_headroom(nskb) + doffset; 4548 } else { 4549 if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len)) 4550 goto err; 4551 } 4552 continue; 4553 } 4554 4555 nskb_frag = skb_shinfo(nskb)->frags; 4556 4557 skb_copy_from_linear_data_offset(head_skb, offset, 4558 skb_put(nskb, hsize), hsize); 4559 4560 skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags & 4561 SKBFL_SHARED_FRAG; 4562 4563 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4564 skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 4565 goto err; 4566 4567 while (pos < offset + len) { 4568 if (i >= nfrags) { 4569 i = 0; 4570 nfrags = skb_shinfo(list_skb)->nr_frags; 4571 frag = skb_shinfo(list_skb)->frags; 4572 frag_skb = list_skb; 4573 if (!skb_headlen(list_skb)) { 4574 BUG_ON(!nfrags); 4575 } else { 4576 BUG_ON(!list_skb->head_frag); 4577 4578 /* to make room for head_frag. */ 4579 i--; 4580 frag--; 4581 } 4582 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4583 skb_zerocopy_clone(nskb, frag_skb, 4584 GFP_ATOMIC)) 4585 goto err; 4586 4587 list_skb = list_skb->next; 4588 } 4589 4590 if (unlikely(skb_shinfo(nskb)->nr_frags >= 4591 MAX_SKB_FRAGS)) { 4592 net_warn_ratelimited( 4593 "skb_segment: too many frags: %u %u\n", 4594 pos, mss); 4595 err = -EINVAL; 4596 goto err; 4597 } 4598 4599 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; 4600 __skb_frag_ref(nskb_frag); 4601 size = skb_frag_size(nskb_frag); 4602 4603 if (pos < offset) { 4604 skb_frag_off_add(nskb_frag, offset - pos); 4605 skb_frag_size_sub(nskb_frag, offset - pos); 4606 } 4607 4608 skb_shinfo(nskb)->nr_frags++; 4609 4610 if (pos + size <= offset + len) { 4611 i++; 4612 frag++; 4613 pos += size; 4614 } else { 4615 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 4616 goto skip_fraglist; 4617 } 4618 4619 nskb_frag++; 4620 } 4621 4622 skip_fraglist: 4623 nskb->data_len = len - hsize; 4624 nskb->len += nskb->data_len; 4625 nskb->truesize += nskb->data_len; 4626 4627 perform_csum_check: 4628 if (!csum) { 4629 if (skb_has_shared_frag(nskb) && 4630 __skb_linearize(nskb)) 4631 goto err; 4632 4633 if (!nskb->remcsum_offload) 4634 nskb->ip_summed = CHECKSUM_NONE; 4635 SKB_GSO_CB(nskb)->csum = 4636 skb_checksum(nskb, doffset, 4637 nskb->len - doffset, 0); 4638 SKB_GSO_CB(nskb)->csum_start = 4639 skb_headroom(nskb) + doffset; 4640 } 4641 } while ((offset += len) < head_skb->len); 4642 4643 /* Some callers want to get the end of the list. 4644 * Put it in segs->prev to avoid walking the list. 4645 * (see validate_xmit_skb_list() for example) 4646 */ 4647 segs->prev = tail; 4648 4649 if (partial_segs) { 4650 struct sk_buff *iter; 4651 int type = skb_shinfo(head_skb)->gso_type; 4652 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 4653 4654 /* Update type to add partial and then remove dodgy if set */ 4655 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 4656 type &= ~SKB_GSO_DODGY; 4657 4658 /* Update GSO info and prepare to start updating headers on 4659 * our way back down the stack of protocols. 4660 */ 4661 for (iter = segs; iter; iter = iter->next) { 4662 skb_shinfo(iter)->gso_size = gso_size; 4663 skb_shinfo(iter)->gso_segs = partial_segs; 4664 skb_shinfo(iter)->gso_type = type; 4665 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 4666 } 4667 4668 if (tail->len - doffset <= gso_size) 4669 skb_shinfo(tail)->gso_size = 0; 4670 else if (tail != segs) 4671 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 4672 } 4673 4674 /* Following permits correct backpressure, for protocols 4675 * using skb_set_owner_w(). 4676 * Idea is to tranfert ownership from head_skb to last segment. 4677 */ 4678 if (head_skb->destructor == sock_wfree) { 4679 swap(tail->truesize, head_skb->truesize); 4680 swap(tail->destructor, head_skb->destructor); 4681 swap(tail->sk, head_skb->sk); 4682 } 4683 return segs; 4684 4685 err: 4686 kfree_skb_list(segs); 4687 return ERR_PTR(err); 4688 } 4689 EXPORT_SYMBOL_GPL(skb_segment); 4690 4691 #ifdef CONFIG_SKB_EXTENSIONS 4692 #define SKB_EXT_ALIGN_VALUE 8 4693 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) 4694 4695 static const u8 skb_ext_type_len[] = { 4696 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4697 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info), 4698 #endif 4699 #ifdef CONFIG_XFRM 4700 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path), 4701 #endif 4702 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4703 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext), 4704 #endif 4705 #if IS_ENABLED(CONFIG_MPTCP) 4706 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext), 4707 #endif 4708 #if IS_ENABLED(CONFIG_MCTP_FLOWS) 4709 [SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow), 4710 #endif 4711 }; 4712 4713 static __always_inline unsigned int skb_ext_total_length(void) 4714 { 4715 return SKB_EXT_CHUNKSIZEOF(struct skb_ext) + 4716 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4717 skb_ext_type_len[SKB_EXT_BRIDGE_NF] + 4718 #endif 4719 #ifdef CONFIG_XFRM 4720 skb_ext_type_len[SKB_EXT_SEC_PATH] + 4721 #endif 4722 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4723 skb_ext_type_len[TC_SKB_EXT] + 4724 #endif 4725 #if IS_ENABLED(CONFIG_MPTCP) 4726 skb_ext_type_len[SKB_EXT_MPTCP] + 4727 #endif 4728 #if IS_ENABLED(CONFIG_MCTP_FLOWS) 4729 skb_ext_type_len[SKB_EXT_MCTP] + 4730 #endif 4731 0; 4732 } 4733 4734 static void skb_extensions_init(void) 4735 { 4736 BUILD_BUG_ON(SKB_EXT_NUM >= 8); 4737 BUILD_BUG_ON(skb_ext_total_length() > 255); 4738 4739 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", 4740 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), 4741 0, 4742 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4743 NULL); 4744 } 4745 #else 4746 static void skb_extensions_init(void) {} 4747 #endif 4748 4749 void __init skb_init(void) 4750 { 4751 skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache", 4752 sizeof(struct sk_buff), 4753 0, 4754 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4755 offsetof(struct sk_buff, cb), 4756 sizeof_field(struct sk_buff, cb), 4757 NULL); 4758 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 4759 sizeof(struct sk_buff_fclones), 4760 0, 4761 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4762 NULL); 4763 #ifdef HAVE_SKB_SMALL_HEAD_CACHE 4764 /* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes. 4765 * struct skb_shared_info is located at the end of skb->head, 4766 * and should not be copied to/from user. 4767 */ 4768 skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head", 4769 SKB_SMALL_HEAD_CACHE_SIZE, 4770 0, 4771 SLAB_HWCACHE_ALIGN | SLAB_PANIC, 4772 0, 4773 SKB_SMALL_HEAD_HEADROOM, 4774 NULL); 4775 #endif 4776 skb_extensions_init(); 4777 } 4778 4779 static int 4780 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 4781 unsigned int recursion_level) 4782 { 4783 int start = skb_headlen(skb); 4784 int i, copy = start - offset; 4785 struct sk_buff *frag_iter; 4786 int elt = 0; 4787 4788 if (unlikely(recursion_level >= 24)) 4789 return -EMSGSIZE; 4790 4791 if (copy > 0) { 4792 if (copy > len) 4793 copy = len; 4794 sg_set_buf(sg, skb->data + offset, copy); 4795 elt++; 4796 if ((len -= copy) == 0) 4797 return elt; 4798 offset += copy; 4799 } 4800 4801 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4802 int end; 4803 4804 WARN_ON(start > offset + len); 4805 4806 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 4807 if ((copy = end - offset) > 0) { 4808 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4809 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4810 return -EMSGSIZE; 4811 4812 if (copy > len) 4813 copy = len; 4814 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 4815 skb_frag_off(frag) + offset - start); 4816 elt++; 4817 if (!(len -= copy)) 4818 return elt; 4819 offset += copy; 4820 } 4821 start = end; 4822 } 4823 4824 skb_walk_frags(skb, frag_iter) { 4825 int end, ret; 4826 4827 WARN_ON(start > offset + len); 4828 4829 end = start + frag_iter->len; 4830 if ((copy = end - offset) > 0) { 4831 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4832 return -EMSGSIZE; 4833 4834 if (copy > len) 4835 copy = len; 4836 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 4837 copy, recursion_level + 1); 4838 if (unlikely(ret < 0)) 4839 return ret; 4840 elt += ret; 4841 if ((len -= copy) == 0) 4842 return elt; 4843 offset += copy; 4844 } 4845 start = end; 4846 } 4847 BUG_ON(len); 4848 return elt; 4849 } 4850 4851 /** 4852 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 4853 * @skb: Socket buffer containing the buffers to be mapped 4854 * @sg: The scatter-gather list to map into 4855 * @offset: The offset into the buffer's contents to start mapping 4856 * @len: Length of buffer space to be mapped 4857 * 4858 * Fill the specified scatter-gather list with mappings/pointers into a 4859 * region of the buffer space attached to a socket buffer. Returns either 4860 * the number of scatterlist items used, or -EMSGSIZE if the contents 4861 * could not fit. 4862 */ 4863 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 4864 { 4865 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 4866 4867 if (nsg <= 0) 4868 return nsg; 4869 4870 sg_mark_end(&sg[nsg - 1]); 4871 4872 return nsg; 4873 } 4874 EXPORT_SYMBOL_GPL(skb_to_sgvec); 4875 4876 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 4877 * sglist without mark the sg which contain last skb data as the end. 4878 * So the caller can mannipulate sg list as will when padding new data after 4879 * the first call without calling sg_unmark_end to expend sg list. 4880 * 4881 * Scenario to use skb_to_sgvec_nomark: 4882 * 1. sg_init_table 4883 * 2. skb_to_sgvec_nomark(payload1) 4884 * 3. skb_to_sgvec_nomark(payload2) 4885 * 4886 * This is equivalent to: 4887 * 1. sg_init_table 4888 * 2. skb_to_sgvec(payload1) 4889 * 3. sg_unmark_end 4890 * 4. skb_to_sgvec(payload2) 4891 * 4892 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 4893 * is more preferable. 4894 */ 4895 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 4896 int offset, int len) 4897 { 4898 return __skb_to_sgvec(skb, sg, offset, len, 0); 4899 } 4900 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 4901 4902 4903 4904 /** 4905 * skb_cow_data - Check that a socket buffer's data buffers are writable 4906 * @skb: The socket buffer to check. 4907 * @tailbits: Amount of trailing space to be added 4908 * @trailer: Returned pointer to the skb where the @tailbits space begins 4909 * 4910 * Make sure that the data buffers attached to a socket buffer are 4911 * writable. If they are not, private copies are made of the data buffers 4912 * and the socket buffer is set to use these instead. 4913 * 4914 * If @tailbits is given, make sure that there is space to write @tailbits 4915 * bytes of data beyond current end of socket buffer. @trailer will be 4916 * set to point to the skb in which this space begins. 4917 * 4918 * The number of scatterlist elements required to completely map the 4919 * COW'd and extended socket buffer will be returned. 4920 */ 4921 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 4922 { 4923 int copyflag; 4924 int elt; 4925 struct sk_buff *skb1, **skb_p; 4926 4927 /* If skb is cloned or its head is paged, reallocate 4928 * head pulling out all the pages (pages are considered not writable 4929 * at the moment even if they are anonymous). 4930 */ 4931 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 4932 !__pskb_pull_tail(skb, __skb_pagelen(skb))) 4933 return -ENOMEM; 4934 4935 /* Easy case. Most of packets will go this way. */ 4936 if (!skb_has_frag_list(skb)) { 4937 /* A little of trouble, not enough of space for trailer. 4938 * This should not happen, when stack is tuned to generate 4939 * good frames. OK, on miss we reallocate and reserve even more 4940 * space, 128 bytes is fair. */ 4941 4942 if (skb_tailroom(skb) < tailbits && 4943 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 4944 return -ENOMEM; 4945 4946 /* Voila! */ 4947 *trailer = skb; 4948 return 1; 4949 } 4950 4951 /* Misery. We are in troubles, going to mincer fragments... */ 4952 4953 elt = 1; 4954 skb_p = &skb_shinfo(skb)->frag_list; 4955 copyflag = 0; 4956 4957 while ((skb1 = *skb_p) != NULL) { 4958 int ntail = 0; 4959 4960 /* The fragment is partially pulled by someone, 4961 * this can happen on input. Copy it and everything 4962 * after it. */ 4963 4964 if (skb_shared(skb1)) 4965 copyflag = 1; 4966 4967 /* If the skb is the last, worry about trailer. */ 4968 4969 if (skb1->next == NULL && tailbits) { 4970 if (skb_shinfo(skb1)->nr_frags || 4971 skb_has_frag_list(skb1) || 4972 skb_tailroom(skb1) < tailbits) 4973 ntail = tailbits + 128; 4974 } 4975 4976 if (copyflag || 4977 skb_cloned(skb1) || 4978 ntail || 4979 skb_shinfo(skb1)->nr_frags || 4980 skb_has_frag_list(skb1)) { 4981 struct sk_buff *skb2; 4982 4983 /* Fuck, we are miserable poor guys... */ 4984 if (ntail == 0) 4985 skb2 = skb_copy(skb1, GFP_ATOMIC); 4986 else 4987 skb2 = skb_copy_expand(skb1, 4988 skb_headroom(skb1), 4989 ntail, 4990 GFP_ATOMIC); 4991 if (unlikely(skb2 == NULL)) 4992 return -ENOMEM; 4993 4994 if (skb1->sk) 4995 skb_set_owner_w(skb2, skb1->sk); 4996 4997 /* Looking around. Are we still alive? 4998 * OK, link new skb, drop old one */ 4999 5000 skb2->next = skb1->next; 5001 *skb_p = skb2; 5002 kfree_skb(skb1); 5003 skb1 = skb2; 5004 } 5005 elt++; 5006 *trailer = skb1; 5007 skb_p = &skb1->next; 5008 } 5009 5010 return elt; 5011 } 5012 EXPORT_SYMBOL_GPL(skb_cow_data); 5013 5014 static void sock_rmem_free(struct sk_buff *skb) 5015 { 5016 struct sock *sk = skb->sk; 5017 5018 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 5019 } 5020 5021 static void skb_set_err_queue(struct sk_buff *skb) 5022 { 5023 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING. 5024 * So, it is safe to (mis)use it to mark skbs on the error queue. 5025 */ 5026 skb->pkt_type = PACKET_OUTGOING; 5027 BUILD_BUG_ON(PACKET_OUTGOING == 0); 5028 } 5029 5030 /* 5031 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 5032 */ 5033 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 5034 { 5035 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 5036 (unsigned int)READ_ONCE(sk->sk_rcvbuf)) 5037 return -ENOMEM; 5038 5039 skb_orphan(skb); 5040 skb->sk = sk; 5041 skb->destructor = sock_rmem_free; 5042 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 5043 skb_set_err_queue(skb); 5044 5045 /* before exiting rcu section, make sure dst is refcounted */ 5046 skb_dst_force(skb); 5047 5048 skb_queue_tail(&sk->sk_error_queue, skb); 5049 if (!sock_flag(sk, SOCK_DEAD)) 5050 sk_error_report(sk); 5051 return 0; 5052 } 5053 EXPORT_SYMBOL(sock_queue_err_skb); 5054 5055 static bool is_icmp_err_skb(const struct sk_buff *skb) 5056 { 5057 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 5058 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 5059 } 5060 5061 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 5062 { 5063 struct sk_buff_head *q = &sk->sk_error_queue; 5064 struct sk_buff *skb, *skb_next = NULL; 5065 bool icmp_next = false; 5066 unsigned long flags; 5067 5068 spin_lock_irqsave(&q->lock, flags); 5069 skb = __skb_dequeue(q); 5070 if (skb && (skb_next = skb_peek(q))) { 5071 icmp_next = is_icmp_err_skb(skb_next); 5072 if (icmp_next) 5073 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 5074 } 5075 spin_unlock_irqrestore(&q->lock, flags); 5076 5077 if (is_icmp_err_skb(skb) && !icmp_next) 5078 sk->sk_err = 0; 5079 5080 if (skb_next) 5081 sk_error_report(sk); 5082 5083 return skb; 5084 } 5085 EXPORT_SYMBOL(sock_dequeue_err_skb); 5086 5087 /** 5088 * skb_clone_sk - create clone of skb, and take reference to socket 5089 * @skb: the skb to clone 5090 * 5091 * This function creates a clone of a buffer that holds a reference on 5092 * sk_refcnt. Buffers created via this function are meant to be 5093 * returned using sock_queue_err_skb, or free via kfree_skb. 5094 * 5095 * When passing buffers allocated with this function to sock_queue_err_skb 5096 * it is necessary to wrap the call with sock_hold/sock_put in order to 5097 * prevent the socket from being released prior to being enqueued on 5098 * the sk_error_queue. 5099 */ 5100 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 5101 { 5102 struct sock *sk = skb->sk; 5103 struct sk_buff *clone; 5104 5105 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 5106 return NULL; 5107 5108 clone = skb_clone(skb, GFP_ATOMIC); 5109 if (!clone) { 5110 sock_put(sk); 5111 return NULL; 5112 } 5113 5114 clone->sk = sk; 5115 clone->destructor = sock_efree; 5116 5117 return clone; 5118 } 5119 EXPORT_SYMBOL(skb_clone_sk); 5120 5121 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 5122 struct sock *sk, 5123 int tstype, 5124 bool opt_stats) 5125 { 5126 struct sock_exterr_skb *serr; 5127 int err; 5128 5129 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 5130 5131 serr = SKB_EXT_ERR(skb); 5132 memset(serr, 0, sizeof(*serr)); 5133 serr->ee.ee_errno = ENOMSG; 5134 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 5135 serr->ee.ee_info = tstype; 5136 serr->opt_stats = opt_stats; 5137 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 5138 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { 5139 serr->ee.ee_data = skb_shinfo(skb)->tskey; 5140 if (sk_is_tcp(sk)) 5141 serr->ee.ee_data -= atomic_read(&sk->sk_tskey); 5142 } 5143 5144 err = sock_queue_err_skb(sk, skb); 5145 5146 if (err) 5147 kfree_skb(skb); 5148 } 5149 5150 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 5151 { 5152 bool ret; 5153 5154 if (likely(READ_ONCE(sysctl_tstamp_allow_data) || tsonly)) 5155 return true; 5156 5157 read_lock_bh(&sk->sk_callback_lock); 5158 ret = sk->sk_socket && sk->sk_socket->file && 5159 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); 5160 read_unlock_bh(&sk->sk_callback_lock); 5161 return ret; 5162 } 5163 5164 void skb_complete_tx_timestamp(struct sk_buff *skb, 5165 struct skb_shared_hwtstamps *hwtstamps) 5166 { 5167 struct sock *sk = skb->sk; 5168 5169 if (!skb_may_tx_timestamp(sk, false)) 5170 goto err; 5171 5172 /* Take a reference to prevent skb_orphan() from freeing the socket, 5173 * but only if the socket refcount is not zero. 5174 */ 5175 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5176 *skb_hwtstamps(skb) = *hwtstamps; 5177 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 5178 sock_put(sk); 5179 return; 5180 } 5181 5182 err: 5183 kfree_skb(skb); 5184 } 5185 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 5186 5187 void __skb_tstamp_tx(struct sk_buff *orig_skb, 5188 const struct sk_buff *ack_skb, 5189 struct skb_shared_hwtstamps *hwtstamps, 5190 struct sock *sk, int tstype) 5191 { 5192 struct sk_buff *skb; 5193 bool tsonly, opt_stats = false; 5194 5195 if (!sk) 5196 return; 5197 5198 if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 5199 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 5200 return; 5201 5202 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 5203 if (!skb_may_tx_timestamp(sk, tsonly)) 5204 return; 5205 5206 if (tsonly) { 5207 #ifdef CONFIG_INET 5208 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && 5209 sk_is_tcp(sk)) { 5210 skb = tcp_get_timestamping_opt_stats(sk, orig_skb, 5211 ack_skb); 5212 opt_stats = true; 5213 } else 5214 #endif 5215 skb = alloc_skb(0, GFP_ATOMIC); 5216 } else { 5217 skb = skb_clone(orig_skb, GFP_ATOMIC); 5218 } 5219 if (!skb) 5220 return; 5221 5222 if (tsonly) { 5223 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 5224 SKBTX_ANY_TSTAMP; 5225 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 5226 } 5227 5228 if (hwtstamps) 5229 *skb_hwtstamps(skb) = *hwtstamps; 5230 else 5231 __net_timestamp(skb); 5232 5233 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 5234 } 5235 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 5236 5237 void skb_tstamp_tx(struct sk_buff *orig_skb, 5238 struct skb_shared_hwtstamps *hwtstamps) 5239 { 5240 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk, 5241 SCM_TSTAMP_SND); 5242 } 5243 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 5244 5245 #ifdef CONFIG_WIRELESS 5246 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 5247 { 5248 struct sock *sk = skb->sk; 5249 struct sock_exterr_skb *serr; 5250 int err = 1; 5251 5252 skb->wifi_acked_valid = 1; 5253 skb->wifi_acked = acked; 5254 5255 serr = SKB_EXT_ERR(skb); 5256 memset(serr, 0, sizeof(*serr)); 5257 serr->ee.ee_errno = ENOMSG; 5258 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 5259 5260 /* Take a reference to prevent skb_orphan() from freeing the socket, 5261 * but only if the socket refcount is not zero. 5262 */ 5263 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5264 err = sock_queue_err_skb(sk, skb); 5265 sock_put(sk); 5266 } 5267 if (err) 5268 kfree_skb(skb); 5269 } 5270 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 5271 #endif /* CONFIG_WIRELESS */ 5272 5273 /** 5274 * skb_partial_csum_set - set up and verify partial csum values for packet 5275 * @skb: the skb to set 5276 * @start: the number of bytes after skb->data to start checksumming. 5277 * @off: the offset from start to place the checksum. 5278 * 5279 * For untrusted partially-checksummed packets, we need to make sure the values 5280 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 5281 * 5282 * This function checks and sets those values and skb->ip_summed: if this 5283 * returns false you should drop the packet. 5284 */ 5285 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 5286 { 5287 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16); 5288 u32 csum_start = skb_headroom(skb) + (u32)start; 5289 5290 if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) { 5291 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n", 5292 start, off, skb_headroom(skb), skb_headlen(skb)); 5293 return false; 5294 } 5295 skb->ip_summed = CHECKSUM_PARTIAL; 5296 skb->csum_start = csum_start; 5297 skb->csum_offset = off; 5298 skb_set_transport_header(skb, start); 5299 return true; 5300 } 5301 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 5302 5303 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 5304 unsigned int max) 5305 { 5306 if (skb_headlen(skb) >= len) 5307 return 0; 5308 5309 /* If we need to pullup then pullup to the max, so we 5310 * won't need to do it again. 5311 */ 5312 if (max > skb->len) 5313 max = skb->len; 5314 5315 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 5316 return -ENOMEM; 5317 5318 if (skb_headlen(skb) < len) 5319 return -EPROTO; 5320 5321 return 0; 5322 } 5323 5324 #define MAX_TCP_HDR_LEN (15 * 4) 5325 5326 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 5327 typeof(IPPROTO_IP) proto, 5328 unsigned int off) 5329 { 5330 int err; 5331 5332 switch (proto) { 5333 case IPPROTO_TCP: 5334 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 5335 off + MAX_TCP_HDR_LEN); 5336 if (!err && !skb_partial_csum_set(skb, off, 5337 offsetof(struct tcphdr, 5338 check))) 5339 err = -EPROTO; 5340 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 5341 5342 case IPPROTO_UDP: 5343 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 5344 off + sizeof(struct udphdr)); 5345 if (!err && !skb_partial_csum_set(skb, off, 5346 offsetof(struct udphdr, 5347 check))) 5348 err = -EPROTO; 5349 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 5350 } 5351 5352 return ERR_PTR(-EPROTO); 5353 } 5354 5355 /* This value should be large enough to cover a tagged ethernet header plus 5356 * maximally sized IP and TCP or UDP headers. 5357 */ 5358 #define MAX_IP_HDR_LEN 128 5359 5360 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 5361 { 5362 unsigned int off; 5363 bool fragment; 5364 __sum16 *csum; 5365 int err; 5366 5367 fragment = false; 5368 5369 err = skb_maybe_pull_tail(skb, 5370 sizeof(struct iphdr), 5371 MAX_IP_HDR_LEN); 5372 if (err < 0) 5373 goto out; 5374 5375 if (ip_is_fragment(ip_hdr(skb))) 5376 fragment = true; 5377 5378 off = ip_hdrlen(skb); 5379 5380 err = -EPROTO; 5381 5382 if (fragment) 5383 goto out; 5384 5385 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 5386 if (IS_ERR(csum)) 5387 return PTR_ERR(csum); 5388 5389 if (recalculate) 5390 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 5391 ip_hdr(skb)->daddr, 5392 skb->len - off, 5393 ip_hdr(skb)->protocol, 0); 5394 err = 0; 5395 5396 out: 5397 return err; 5398 } 5399 5400 /* This value should be large enough to cover a tagged ethernet header plus 5401 * an IPv6 header, all options, and a maximal TCP or UDP header. 5402 */ 5403 #define MAX_IPV6_HDR_LEN 256 5404 5405 #define OPT_HDR(type, skb, off) \ 5406 (type *)(skb_network_header(skb) + (off)) 5407 5408 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 5409 { 5410 int err; 5411 u8 nexthdr; 5412 unsigned int off; 5413 unsigned int len; 5414 bool fragment; 5415 bool done; 5416 __sum16 *csum; 5417 5418 fragment = false; 5419 done = false; 5420 5421 off = sizeof(struct ipv6hdr); 5422 5423 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 5424 if (err < 0) 5425 goto out; 5426 5427 nexthdr = ipv6_hdr(skb)->nexthdr; 5428 5429 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 5430 while (off <= len && !done) { 5431 switch (nexthdr) { 5432 case IPPROTO_DSTOPTS: 5433 case IPPROTO_HOPOPTS: 5434 case IPPROTO_ROUTING: { 5435 struct ipv6_opt_hdr *hp; 5436 5437 err = skb_maybe_pull_tail(skb, 5438 off + 5439 sizeof(struct ipv6_opt_hdr), 5440 MAX_IPV6_HDR_LEN); 5441 if (err < 0) 5442 goto out; 5443 5444 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 5445 nexthdr = hp->nexthdr; 5446 off += ipv6_optlen(hp); 5447 break; 5448 } 5449 case IPPROTO_AH: { 5450 struct ip_auth_hdr *hp; 5451 5452 err = skb_maybe_pull_tail(skb, 5453 off + 5454 sizeof(struct ip_auth_hdr), 5455 MAX_IPV6_HDR_LEN); 5456 if (err < 0) 5457 goto out; 5458 5459 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 5460 nexthdr = hp->nexthdr; 5461 off += ipv6_authlen(hp); 5462 break; 5463 } 5464 case IPPROTO_FRAGMENT: { 5465 struct frag_hdr *hp; 5466 5467 err = skb_maybe_pull_tail(skb, 5468 off + 5469 sizeof(struct frag_hdr), 5470 MAX_IPV6_HDR_LEN); 5471 if (err < 0) 5472 goto out; 5473 5474 hp = OPT_HDR(struct frag_hdr, skb, off); 5475 5476 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 5477 fragment = true; 5478 5479 nexthdr = hp->nexthdr; 5480 off += sizeof(struct frag_hdr); 5481 break; 5482 } 5483 default: 5484 done = true; 5485 break; 5486 } 5487 } 5488 5489 err = -EPROTO; 5490 5491 if (!done || fragment) 5492 goto out; 5493 5494 csum = skb_checksum_setup_ip(skb, nexthdr, off); 5495 if (IS_ERR(csum)) 5496 return PTR_ERR(csum); 5497 5498 if (recalculate) 5499 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 5500 &ipv6_hdr(skb)->daddr, 5501 skb->len - off, nexthdr, 0); 5502 err = 0; 5503 5504 out: 5505 return err; 5506 } 5507 5508 /** 5509 * skb_checksum_setup - set up partial checksum offset 5510 * @skb: the skb to set up 5511 * @recalculate: if true the pseudo-header checksum will be recalculated 5512 */ 5513 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 5514 { 5515 int err; 5516 5517 switch (skb->protocol) { 5518 case htons(ETH_P_IP): 5519 err = skb_checksum_setup_ipv4(skb, recalculate); 5520 break; 5521 5522 case htons(ETH_P_IPV6): 5523 err = skb_checksum_setup_ipv6(skb, recalculate); 5524 break; 5525 5526 default: 5527 err = -EPROTO; 5528 break; 5529 } 5530 5531 return err; 5532 } 5533 EXPORT_SYMBOL(skb_checksum_setup); 5534 5535 /** 5536 * skb_checksum_maybe_trim - maybe trims the given skb 5537 * @skb: the skb to check 5538 * @transport_len: the data length beyond the network header 5539 * 5540 * Checks whether the given skb has data beyond the given transport length. 5541 * If so, returns a cloned skb trimmed to this transport length. 5542 * Otherwise returns the provided skb. Returns NULL in error cases 5543 * (e.g. transport_len exceeds skb length or out-of-memory). 5544 * 5545 * Caller needs to set the skb transport header and free any returned skb if it 5546 * differs from the provided skb. 5547 */ 5548 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 5549 unsigned int transport_len) 5550 { 5551 struct sk_buff *skb_chk; 5552 unsigned int len = skb_transport_offset(skb) + transport_len; 5553 int ret; 5554 5555 if (skb->len < len) 5556 return NULL; 5557 else if (skb->len == len) 5558 return skb; 5559 5560 skb_chk = skb_clone(skb, GFP_ATOMIC); 5561 if (!skb_chk) 5562 return NULL; 5563 5564 ret = pskb_trim_rcsum(skb_chk, len); 5565 if (ret) { 5566 kfree_skb(skb_chk); 5567 return NULL; 5568 } 5569 5570 return skb_chk; 5571 } 5572 5573 /** 5574 * skb_checksum_trimmed - validate checksum of an skb 5575 * @skb: the skb to check 5576 * @transport_len: the data length beyond the network header 5577 * @skb_chkf: checksum function to use 5578 * 5579 * Applies the given checksum function skb_chkf to the provided skb. 5580 * Returns a checked and maybe trimmed skb. Returns NULL on error. 5581 * 5582 * If the skb has data beyond the given transport length, then a 5583 * trimmed & cloned skb is checked and returned. 5584 * 5585 * Caller needs to set the skb transport header and free any returned skb if it 5586 * differs from the provided skb. 5587 */ 5588 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 5589 unsigned int transport_len, 5590 __sum16(*skb_chkf)(struct sk_buff *skb)) 5591 { 5592 struct sk_buff *skb_chk; 5593 unsigned int offset = skb_transport_offset(skb); 5594 __sum16 ret; 5595 5596 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 5597 if (!skb_chk) 5598 goto err; 5599 5600 if (!pskb_may_pull(skb_chk, offset)) 5601 goto err; 5602 5603 skb_pull_rcsum(skb_chk, offset); 5604 ret = skb_chkf(skb_chk); 5605 skb_push_rcsum(skb_chk, offset); 5606 5607 if (ret) 5608 goto err; 5609 5610 return skb_chk; 5611 5612 err: 5613 if (skb_chk && skb_chk != skb) 5614 kfree_skb(skb_chk); 5615 5616 return NULL; 5617 5618 } 5619 EXPORT_SYMBOL(skb_checksum_trimmed); 5620 5621 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 5622 { 5623 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 5624 skb->dev->name); 5625 } 5626 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 5627 5628 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 5629 { 5630 if (head_stolen) { 5631 skb_release_head_state(skb); 5632 kmem_cache_free(skbuff_cache, skb); 5633 } else { 5634 __kfree_skb(skb); 5635 } 5636 } 5637 EXPORT_SYMBOL(kfree_skb_partial); 5638 5639 /** 5640 * skb_try_coalesce - try to merge skb to prior one 5641 * @to: prior buffer 5642 * @from: buffer to add 5643 * @fragstolen: pointer to boolean 5644 * @delta_truesize: how much more was allocated than was requested 5645 */ 5646 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 5647 bool *fragstolen, int *delta_truesize) 5648 { 5649 struct skb_shared_info *to_shinfo, *from_shinfo; 5650 int i, delta, len = from->len; 5651 5652 *fragstolen = false; 5653 5654 if (skb_cloned(to)) 5655 return false; 5656 5657 /* In general, avoid mixing page_pool and non-page_pool allocated 5658 * pages within the same SKB. Additionally avoid dealing with clones 5659 * with page_pool pages, in case the SKB is using page_pool fragment 5660 * references (PP_FLAG_PAGE_FRAG). Since we only take full page 5661 * references for cloned SKBs at the moment that would result in 5662 * inconsistent reference counts. 5663 * In theory we could take full references if @from is cloned and 5664 * !@to->pp_recycle but its tricky (due to potential race with 5665 * the clone disappearing) and rare, so not worth dealing with. 5666 */ 5667 if (to->pp_recycle != from->pp_recycle || 5668 (from->pp_recycle && skb_cloned(from))) 5669 return false; 5670 5671 if (len <= skb_tailroom(to)) { 5672 if (len) 5673 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 5674 *delta_truesize = 0; 5675 return true; 5676 } 5677 5678 to_shinfo = skb_shinfo(to); 5679 from_shinfo = skb_shinfo(from); 5680 if (to_shinfo->frag_list || from_shinfo->frag_list) 5681 return false; 5682 if (skb_zcopy(to) || skb_zcopy(from)) 5683 return false; 5684 5685 if (skb_headlen(from) != 0) { 5686 struct page *page; 5687 unsigned int offset; 5688 5689 if (to_shinfo->nr_frags + 5690 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 5691 return false; 5692 5693 if (skb_head_is_locked(from)) 5694 return false; 5695 5696 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 5697 5698 page = virt_to_head_page(from->head); 5699 offset = from->data - (unsigned char *)page_address(page); 5700 5701 skb_fill_page_desc(to, to_shinfo->nr_frags, 5702 page, offset, skb_headlen(from)); 5703 *fragstolen = true; 5704 } else { 5705 if (to_shinfo->nr_frags + 5706 from_shinfo->nr_frags > MAX_SKB_FRAGS) 5707 return false; 5708 5709 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 5710 } 5711 5712 WARN_ON_ONCE(delta < len); 5713 5714 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 5715 from_shinfo->frags, 5716 from_shinfo->nr_frags * sizeof(skb_frag_t)); 5717 to_shinfo->nr_frags += from_shinfo->nr_frags; 5718 5719 if (!skb_cloned(from)) 5720 from_shinfo->nr_frags = 0; 5721 5722 /* if the skb is not cloned this does nothing 5723 * since we set nr_frags to 0. 5724 */ 5725 for (i = 0; i < from_shinfo->nr_frags; i++) 5726 __skb_frag_ref(&from_shinfo->frags[i]); 5727 5728 to->truesize += delta; 5729 to->len += len; 5730 to->data_len += len; 5731 5732 *delta_truesize = delta; 5733 return true; 5734 } 5735 EXPORT_SYMBOL(skb_try_coalesce); 5736 5737 /** 5738 * skb_scrub_packet - scrub an skb 5739 * 5740 * @skb: buffer to clean 5741 * @xnet: packet is crossing netns 5742 * 5743 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 5744 * into/from a tunnel. Some information have to be cleared during these 5745 * operations. 5746 * skb_scrub_packet can also be used to clean a skb before injecting it in 5747 * another namespace (@xnet == true). We have to clear all information in the 5748 * skb that could impact namespace isolation. 5749 */ 5750 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 5751 { 5752 skb->pkt_type = PACKET_HOST; 5753 skb->skb_iif = 0; 5754 skb->ignore_df = 0; 5755 skb_dst_drop(skb); 5756 skb_ext_reset(skb); 5757 nf_reset_ct(skb); 5758 nf_reset_trace(skb); 5759 5760 #ifdef CONFIG_NET_SWITCHDEV 5761 skb->offload_fwd_mark = 0; 5762 skb->offload_l3_fwd_mark = 0; 5763 #endif 5764 5765 if (!xnet) 5766 return; 5767 5768 ipvs_reset(skb); 5769 skb->mark = 0; 5770 skb_clear_tstamp(skb); 5771 } 5772 EXPORT_SYMBOL_GPL(skb_scrub_packet); 5773 5774 /** 5775 * skb_gso_transport_seglen - Return length of individual segments of a gso packet 5776 * 5777 * @skb: GSO skb 5778 * 5779 * skb_gso_transport_seglen is used to determine the real size of the 5780 * individual segments, including Layer4 headers (TCP/UDP). 5781 * 5782 * The MAC/L2 or network (IP, IPv6) headers are not accounted for. 5783 */ 5784 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb) 5785 { 5786 const struct skb_shared_info *shinfo = skb_shinfo(skb); 5787 unsigned int thlen = 0; 5788 5789 if (skb->encapsulation) { 5790 thlen = skb_inner_transport_header(skb) - 5791 skb_transport_header(skb); 5792 5793 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 5794 thlen += inner_tcp_hdrlen(skb); 5795 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 5796 thlen = tcp_hdrlen(skb); 5797 } else if (unlikely(skb_is_gso_sctp(skb))) { 5798 thlen = sizeof(struct sctphdr); 5799 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { 5800 thlen = sizeof(struct udphdr); 5801 } 5802 /* UFO sets gso_size to the size of the fragmentation 5803 * payload, i.e. the size of the L4 (UDP) header is already 5804 * accounted for. 5805 */ 5806 return thlen + shinfo->gso_size; 5807 } 5808 5809 /** 5810 * skb_gso_network_seglen - Return length of individual segments of a gso packet 5811 * 5812 * @skb: GSO skb 5813 * 5814 * skb_gso_network_seglen is used to determine the real size of the 5815 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP). 5816 * 5817 * The MAC/L2 header is not accounted for. 5818 */ 5819 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb) 5820 { 5821 unsigned int hdr_len = skb_transport_header(skb) - 5822 skb_network_header(skb); 5823 5824 return hdr_len + skb_gso_transport_seglen(skb); 5825 } 5826 5827 /** 5828 * skb_gso_mac_seglen - Return length of individual segments of a gso packet 5829 * 5830 * @skb: GSO skb 5831 * 5832 * skb_gso_mac_seglen is used to determine the real size of the 5833 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4 5834 * headers (TCP/UDP). 5835 */ 5836 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb) 5837 { 5838 unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 5839 5840 return hdr_len + skb_gso_transport_seglen(skb); 5841 } 5842 5843 /** 5844 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS 5845 * 5846 * There are a couple of instances where we have a GSO skb, and we 5847 * want to determine what size it would be after it is segmented. 5848 * 5849 * We might want to check: 5850 * - L3+L4+payload size (e.g. IP forwarding) 5851 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver) 5852 * 5853 * This is a helper to do that correctly considering GSO_BY_FRAGS. 5854 * 5855 * @skb: GSO skb 5856 * 5857 * @seg_len: The segmented length (from skb_gso_*_seglen). In the 5858 * GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS]. 5859 * 5860 * @max_len: The maximum permissible length. 5861 * 5862 * Returns true if the segmented length <= max length. 5863 */ 5864 static inline bool skb_gso_size_check(const struct sk_buff *skb, 5865 unsigned int seg_len, 5866 unsigned int max_len) { 5867 const struct skb_shared_info *shinfo = skb_shinfo(skb); 5868 const struct sk_buff *iter; 5869 5870 if (shinfo->gso_size != GSO_BY_FRAGS) 5871 return seg_len <= max_len; 5872 5873 /* Undo this so we can re-use header sizes */ 5874 seg_len -= GSO_BY_FRAGS; 5875 5876 skb_walk_frags(skb, iter) { 5877 if (seg_len + skb_headlen(iter) > max_len) 5878 return false; 5879 } 5880 5881 return true; 5882 } 5883 5884 /** 5885 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU? 5886 * 5887 * @skb: GSO skb 5888 * @mtu: MTU to validate against 5889 * 5890 * skb_gso_validate_network_len validates if a given skb will fit a 5891 * wanted MTU once split. It considers L3 headers, L4 headers, and the 5892 * payload. 5893 */ 5894 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu) 5895 { 5896 return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu); 5897 } 5898 EXPORT_SYMBOL_GPL(skb_gso_validate_network_len); 5899 5900 /** 5901 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length? 5902 * 5903 * @skb: GSO skb 5904 * @len: length to validate against 5905 * 5906 * skb_gso_validate_mac_len validates if a given skb will fit a wanted 5907 * length once split, including L2, L3 and L4 headers and the payload. 5908 */ 5909 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len) 5910 { 5911 return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len); 5912 } 5913 EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len); 5914 5915 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 5916 { 5917 int mac_len, meta_len; 5918 void *meta; 5919 5920 if (skb_cow(skb, skb_headroom(skb)) < 0) { 5921 kfree_skb(skb); 5922 return NULL; 5923 } 5924 5925 mac_len = skb->data - skb_mac_header(skb); 5926 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) { 5927 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), 5928 mac_len - VLAN_HLEN - ETH_TLEN); 5929 } 5930 5931 meta_len = skb_metadata_len(skb); 5932 if (meta_len) { 5933 meta = skb_metadata_end(skb) - meta_len; 5934 memmove(meta + VLAN_HLEN, meta, meta_len); 5935 } 5936 5937 skb->mac_header += VLAN_HLEN; 5938 return skb; 5939 } 5940 5941 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 5942 { 5943 struct vlan_hdr *vhdr; 5944 u16 vlan_tci; 5945 5946 if (unlikely(skb_vlan_tag_present(skb))) { 5947 /* vlan_tci is already set-up so leave this for another time */ 5948 return skb; 5949 } 5950 5951 skb = skb_share_check(skb, GFP_ATOMIC); 5952 if (unlikely(!skb)) 5953 goto err_free; 5954 /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */ 5955 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short)))) 5956 goto err_free; 5957 5958 vhdr = (struct vlan_hdr *)skb->data; 5959 vlan_tci = ntohs(vhdr->h_vlan_TCI); 5960 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 5961 5962 skb_pull_rcsum(skb, VLAN_HLEN); 5963 vlan_set_encap_proto(skb, vhdr); 5964 5965 skb = skb_reorder_vlan_header(skb); 5966 if (unlikely(!skb)) 5967 goto err_free; 5968 5969 skb_reset_network_header(skb); 5970 if (!skb_transport_header_was_set(skb)) 5971 skb_reset_transport_header(skb); 5972 skb_reset_mac_len(skb); 5973 5974 return skb; 5975 5976 err_free: 5977 kfree_skb(skb); 5978 return NULL; 5979 } 5980 EXPORT_SYMBOL(skb_vlan_untag); 5981 5982 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len) 5983 { 5984 if (!pskb_may_pull(skb, write_len)) 5985 return -ENOMEM; 5986 5987 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 5988 return 0; 5989 5990 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 5991 } 5992 EXPORT_SYMBOL(skb_ensure_writable); 5993 5994 /* remove VLAN header from packet and update csum accordingly. 5995 * expects a non skb_vlan_tag_present skb with a vlan tag payload 5996 */ 5997 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 5998 { 5999 struct vlan_hdr *vhdr; 6000 int offset = skb->data - skb_mac_header(skb); 6001 int err; 6002 6003 if (WARN_ONCE(offset, 6004 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 6005 offset)) { 6006 return -EINVAL; 6007 } 6008 6009 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 6010 if (unlikely(err)) 6011 return err; 6012 6013 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 6014 6015 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN); 6016 *vlan_tci = ntohs(vhdr->h_vlan_TCI); 6017 6018 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN); 6019 __skb_pull(skb, VLAN_HLEN); 6020 6021 vlan_set_encap_proto(skb, vhdr); 6022 skb->mac_header += VLAN_HLEN; 6023 6024 if (skb_network_offset(skb) < ETH_HLEN) 6025 skb_set_network_header(skb, ETH_HLEN); 6026 6027 skb_reset_mac_len(skb); 6028 6029 return err; 6030 } 6031 EXPORT_SYMBOL(__skb_vlan_pop); 6032 6033 /* Pop a vlan tag either from hwaccel or from payload. 6034 * Expects skb->data at mac header. 6035 */ 6036 int skb_vlan_pop(struct sk_buff *skb) 6037 { 6038 u16 vlan_tci; 6039 __be16 vlan_proto; 6040 int err; 6041 6042 if (likely(skb_vlan_tag_present(skb))) { 6043 __vlan_hwaccel_clear_tag(skb); 6044 } else { 6045 if (unlikely(!eth_type_vlan(skb->protocol))) 6046 return 0; 6047 6048 err = __skb_vlan_pop(skb, &vlan_tci); 6049 if (err) 6050 return err; 6051 } 6052 /* move next vlan tag to hw accel tag */ 6053 if (likely(!eth_type_vlan(skb->protocol))) 6054 return 0; 6055 6056 vlan_proto = skb->protocol; 6057 err = __skb_vlan_pop(skb, &vlan_tci); 6058 if (unlikely(err)) 6059 return err; 6060 6061 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 6062 return 0; 6063 } 6064 EXPORT_SYMBOL(skb_vlan_pop); 6065 6066 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 6067 * Expects skb->data at mac header. 6068 */ 6069 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 6070 { 6071 if (skb_vlan_tag_present(skb)) { 6072 int offset = skb->data - skb_mac_header(skb); 6073 int err; 6074 6075 if (WARN_ONCE(offset, 6076 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 6077 offset)) { 6078 return -EINVAL; 6079 } 6080 6081 err = __vlan_insert_tag(skb, skb->vlan_proto, 6082 skb_vlan_tag_get(skb)); 6083 if (err) 6084 return err; 6085 6086 skb->protocol = skb->vlan_proto; 6087 skb->mac_len += VLAN_HLEN; 6088 6089 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 6090 } 6091 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 6092 return 0; 6093 } 6094 EXPORT_SYMBOL(skb_vlan_push); 6095 6096 /** 6097 * skb_eth_pop() - Drop the Ethernet header at the head of a packet 6098 * 6099 * @skb: Socket buffer to modify 6100 * 6101 * Drop the Ethernet header of @skb. 6102 * 6103 * Expects that skb->data points to the mac header and that no VLAN tags are 6104 * present. 6105 * 6106 * Returns 0 on success, -errno otherwise. 6107 */ 6108 int skb_eth_pop(struct sk_buff *skb) 6109 { 6110 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) || 6111 skb_network_offset(skb) < ETH_HLEN) 6112 return -EPROTO; 6113 6114 skb_pull_rcsum(skb, ETH_HLEN); 6115 skb_reset_mac_header(skb); 6116 skb_reset_mac_len(skb); 6117 6118 return 0; 6119 } 6120 EXPORT_SYMBOL(skb_eth_pop); 6121 6122 /** 6123 * skb_eth_push() - Add a new Ethernet header at the head of a packet 6124 * 6125 * @skb: Socket buffer to modify 6126 * @dst: Destination MAC address of the new header 6127 * @src: Source MAC address of the new header 6128 * 6129 * Prepend @skb with a new Ethernet header. 6130 * 6131 * Expects that skb->data points to the mac header, which must be empty. 6132 * 6133 * Returns 0 on success, -errno otherwise. 6134 */ 6135 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst, 6136 const unsigned char *src) 6137 { 6138 struct ethhdr *eth; 6139 int err; 6140 6141 if (skb_network_offset(skb) || skb_vlan_tag_present(skb)) 6142 return -EPROTO; 6143 6144 err = skb_cow_head(skb, sizeof(*eth)); 6145 if (err < 0) 6146 return err; 6147 6148 skb_push(skb, sizeof(*eth)); 6149 skb_reset_mac_header(skb); 6150 skb_reset_mac_len(skb); 6151 6152 eth = eth_hdr(skb); 6153 ether_addr_copy(eth->h_dest, dst); 6154 ether_addr_copy(eth->h_source, src); 6155 eth->h_proto = skb->protocol; 6156 6157 skb_postpush_rcsum(skb, eth, sizeof(*eth)); 6158 6159 return 0; 6160 } 6161 EXPORT_SYMBOL(skb_eth_push); 6162 6163 /* Update the ethertype of hdr and the skb csum value if required. */ 6164 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr, 6165 __be16 ethertype) 6166 { 6167 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6168 __be16 diff[] = { ~hdr->h_proto, ethertype }; 6169 6170 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6171 } 6172 6173 hdr->h_proto = ethertype; 6174 } 6175 6176 /** 6177 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of 6178 * the packet 6179 * 6180 * @skb: buffer 6181 * @mpls_lse: MPLS label stack entry to push 6182 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848) 6183 * @mac_len: length of the MAC header 6184 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is 6185 * ethernet 6186 * 6187 * Expects skb->data at mac header. 6188 * 6189 * Returns 0 on success, -errno otherwise. 6190 */ 6191 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto, 6192 int mac_len, bool ethernet) 6193 { 6194 struct mpls_shim_hdr *lse; 6195 int err; 6196 6197 if (unlikely(!eth_p_mpls(mpls_proto))) 6198 return -EINVAL; 6199 6200 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */ 6201 if (skb->encapsulation) 6202 return -EINVAL; 6203 6204 err = skb_cow_head(skb, MPLS_HLEN); 6205 if (unlikely(err)) 6206 return err; 6207 6208 if (!skb->inner_protocol) { 6209 skb_set_inner_network_header(skb, skb_network_offset(skb)); 6210 skb_set_inner_protocol(skb, skb->protocol); 6211 } 6212 6213 skb_push(skb, MPLS_HLEN); 6214 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb), 6215 mac_len); 6216 skb_reset_mac_header(skb); 6217 skb_set_network_header(skb, mac_len); 6218 skb_reset_mac_len(skb); 6219 6220 lse = mpls_hdr(skb); 6221 lse->label_stack_entry = mpls_lse; 6222 skb_postpush_rcsum(skb, lse, MPLS_HLEN); 6223 6224 if (ethernet && mac_len >= ETH_HLEN) 6225 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto); 6226 skb->protocol = mpls_proto; 6227 6228 return 0; 6229 } 6230 EXPORT_SYMBOL_GPL(skb_mpls_push); 6231 6232 /** 6233 * skb_mpls_pop() - pop the outermost MPLS header 6234 * 6235 * @skb: buffer 6236 * @next_proto: ethertype of header after popped MPLS header 6237 * @mac_len: length of the MAC header 6238 * @ethernet: flag to indicate if the packet is ethernet 6239 * 6240 * Expects skb->data at mac header. 6241 * 6242 * Returns 0 on success, -errno otherwise. 6243 */ 6244 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, 6245 bool ethernet) 6246 { 6247 int err; 6248 6249 if (unlikely(!eth_p_mpls(skb->protocol))) 6250 return 0; 6251 6252 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN); 6253 if (unlikely(err)) 6254 return err; 6255 6256 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN); 6257 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb), 6258 mac_len); 6259 6260 __skb_pull(skb, MPLS_HLEN); 6261 skb_reset_mac_header(skb); 6262 skb_set_network_header(skb, mac_len); 6263 6264 if (ethernet && mac_len >= ETH_HLEN) { 6265 struct ethhdr *hdr; 6266 6267 /* use mpls_hdr() to get ethertype to account for VLANs. */ 6268 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN); 6269 skb_mod_eth_type(skb, hdr, next_proto); 6270 } 6271 skb->protocol = next_proto; 6272 6273 return 0; 6274 } 6275 EXPORT_SYMBOL_GPL(skb_mpls_pop); 6276 6277 /** 6278 * skb_mpls_update_lse() - modify outermost MPLS header and update csum 6279 * 6280 * @skb: buffer 6281 * @mpls_lse: new MPLS label stack entry to update to 6282 * 6283 * Expects skb->data at mac header. 6284 * 6285 * Returns 0 on success, -errno otherwise. 6286 */ 6287 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse) 6288 { 6289 int err; 6290 6291 if (unlikely(!eth_p_mpls(skb->protocol))) 6292 return -EINVAL; 6293 6294 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN); 6295 if (unlikely(err)) 6296 return err; 6297 6298 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6299 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse }; 6300 6301 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6302 } 6303 6304 mpls_hdr(skb)->label_stack_entry = mpls_lse; 6305 6306 return 0; 6307 } 6308 EXPORT_SYMBOL_GPL(skb_mpls_update_lse); 6309 6310 /** 6311 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header 6312 * 6313 * @skb: buffer 6314 * 6315 * Expects skb->data at mac header. 6316 * 6317 * Returns 0 on success, -errno otherwise. 6318 */ 6319 int skb_mpls_dec_ttl(struct sk_buff *skb) 6320 { 6321 u32 lse; 6322 u8 ttl; 6323 6324 if (unlikely(!eth_p_mpls(skb->protocol))) 6325 return -EINVAL; 6326 6327 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN)) 6328 return -ENOMEM; 6329 6330 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry); 6331 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT; 6332 if (!--ttl) 6333 return -EINVAL; 6334 6335 lse &= ~MPLS_LS_TTL_MASK; 6336 lse |= ttl << MPLS_LS_TTL_SHIFT; 6337 6338 return skb_mpls_update_lse(skb, cpu_to_be32(lse)); 6339 } 6340 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); 6341 6342 /** 6343 * alloc_skb_with_frags - allocate skb with page frags 6344 * 6345 * @header_len: size of linear part 6346 * @data_len: needed length in frags 6347 * @max_page_order: max page order desired. 6348 * @errcode: pointer to error code if any 6349 * @gfp_mask: allocation mask 6350 * 6351 * This can be used to allocate a paged skb, given a maximal order for frags. 6352 */ 6353 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 6354 unsigned long data_len, 6355 int max_page_order, 6356 int *errcode, 6357 gfp_t gfp_mask) 6358 { 6359 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 6360 unsigned long chunk; 6361 struct sk_buff *skb; 6362 struct page *page; 6363 int i; 6364 6365 *errcode = -EMSGSIZE; 6366 /* Note this test could be relaxed, if we succeed to allocate 6367 * high order pages... 6368 */ 6369 if (npages > MAX_SKB_FRAGS) 6370 return NULL; 6371 6372 *errcode = -ENOBUFS; 6373 skb = alloc_skb(header_len, gfp_mask); 6374 if (!skb) 6375 return NULL; 6376 6377 skb->truesize += npages << PAGE_SHIFT; 6378 6379 for (i = 0; npages > 0; i++) { 6380 int order = max_page_order; 6381 6382 while (order) { 6383 if (npages >= 1 << order) { 6384 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 6385 __GFP_COMP | 6386 __GFP_NOWARN, 6387 order); 6388 if (page) 6389 goto fill_page; 6390 /* Do not retry other high order allocations */ 6391 order = 1; 6392 max_page_order = 0; 6393 } 6394 order--; 6395 } 6396 page = alloc_page(gfp_mask); 6397 if (!page) 6398 goto failure; 6399 fill_page: 6400 chunk = min_t(unsigned long, data_len, 6401 PAGE_SIZE << order); 6402 skb_fill_page_desc(skb, i, page, 0, chunk); 6403 data_len -= chunk; 6404 npages -= 1 << order; 6405 } 6406 return skb; 6407 6408 failure: 6409 kfree_skb(skb); 6410 return NULL; 6411 } 6412 EXPORT_SYMBOL(alloc_skb_with_frags); 6413 6414 /* carve out the first off bytes from skb when off < headlen */ 6415 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 6416 const int headlen, gfp_t gfp_mask) 6417 { 6418 int i; 6419 unsigned int size = skb_end_offset(skb); 6420 int new_hlen = headlen - off; 6421 u8 *data; 6422 6423 if (skb_pfmemalloc(skb)) 6424 gfp_mask |= __GFP_MEMALLOC; 6425 6426 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 6427 if (!data) 6428 return -ENOMEM; 6429 size = SKB_WITH_OVERHEAD(size); 6430 6431 /* Copy real data, and all frags */ 6432 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 6433 skb->len -= off; 6434 6435 memcpy((struct skb_shared_info *)(data + size), 6436 skb_shinfo(skb), 6437 offsetof(struct skb_shared_info, 6438 frags[skb_shinfo(skb)->nr_frags])); 6439 if (skb_cloned(skb)) { 6440 /* drop the old head gracefully */ 6441 if (skb_orphan_frags(skb, gfp_mask)) { 6442 skb_kfree_head(data, size); 6443 return -ENOMEM; 6444 } 6445 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 6446 skb_frag_ref(skb, i); 6447 if (skb_has_frag_list(skb)) 6448 skb_clone_fraglist(skb); 6449 skb_release_data(skb, SKB_CONSUMED, false); 6450 } else { 6451 /* we can reuse existing recount- all we did was 6452 * relocate values 6453 */ 6454 skb_free_head(skb, false); 6455 } 6456 6457 skb->head = data; 6458 skb->data = data; 6459 skb->head_frag = 0; 6460 skb_set_end_offset(skb, size); 6461 skb_set_tail_pointer(skb, skb_headlen(skb)); 6462 skb_headers_offset_update(skb, 0); 6463 skb->cloned = 0; 6464 skb->hdr_len = 0; 6465 skb->nohdr = 0; 6466 atomic_set(&skb_shinfo(skb)->dataref, 1); 6467 6468 return 0; 6469 } 6470 6471 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 6472 6473 /* carve out the first eat bytes from skb's frag_list. May recurse into 6474 * pskb_carve() 6475 */ 6476 static int pskb_carve_frag_list(struct sk_buff *skb, 6477 struct skb_shared_info *shinfo, int eat, 6478 gfp_t gfp_mask) 6479 { 6480 struct sk_buff *list = shinfo->frag_list; 6481 struct sk_buff *clone = NULL; 6482 struct sk_buff *insp = NULL; 6483 6484 do { 6485 if (!list) { 6486 pr_err("Not enough bytes to eat. Want %d\n", eat); 6487 return -EFAULT; 6488 } 6489 if (list->len <= eat) { 6490 /* Eaten as whole. */ 6491 eat -= list->len; 6492 list = list->next; 6493 insp = list; 6494 } else { 6495 /* Eaten partially. */ 6496 if (skb_shared(list)) { 6497 clone = skb_clone(list, gfp_mask); 6498 if (!clone) 6499 return -ENOMEM; 6500 insp = list->next; 6501 list = clone; 6502 } else { 6503 /* This may be pulled without problems. */ 6504 insp = list; 6505 } 6506 if (pskb_carve(list, eat, gfp_mask) < 0) { 6507 kfree_skb(clone); 6508 return -ENOMEM; 6509 } 6510 break; 6511 } 6512 } while (eat); 6513 6514 /* Free pulled out fragments. */ 6515 while ((list = shinfo->frag_list) != insp) { 6516 shinfo->frag_list = list->next; 6517 consume_skb(list); 6518 } 6519 /* And insert new clone at head. */ 6520 if (clone) { 6521 clone->next = list; 6522 shinfo->frag_list = clone; 6523 } 6524 return 0; 6525 } 6526 6527 /* carve off first len bytes from skb. Split line (off) is in the 6528 * non-linear part of skb 6529 */ 6530 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 6531 int pos, gfp_t gfp_mask) 6532 { 6533 int i, k = 0; 6534 unsigned int size = skb_end_offset(skb); 6535 u8 *data; 6536 const int nfrags = skb_shinfo(skb)->nr_frags; 6537 struct skb_shared_info *shinfo; 6538 6539 if (skb_pfmemalloc(skb)) 6540 gfp_mask |= __GFP_MEMALLOC; 6541 6542 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 6543 if (!data) 6544 return -ENOMEM; 6545 size = SKB_WITH_OVERHEAD(size); 6546 6547 memcpy((struct skb_shared_info *)(data + size), 6548 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0])); 6549 if (skb_orphan_frags(skb, gfp_mask)) { 6550 skb_kfree_head(data, size); 6551 return -ENOMEM; 6552 } 6553 shinfo = (struct skb_shared_info *)(data + size); 6554 for (i = 0; i < nfrags; i++) { 6555 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 6556 6557 if (pos + fsize > off) { 6558 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 6559 6560 if (pos < off) { 6561 /* Split frag. 6562 * We have two variants in this case: 6563 * 1. Move all the frag to the second 6564 * part, if it is possible. F.e. 6565 * this approach is mandatory for TUX, 6566 * where splitting is expensive. 6567 * 2. Split is accurately. We make this. 6568 */ 6569 skb_frag_off_add(&shinfo->frags[0], off - pos); 6570 skb_frag_size_sub(&shinfo->frags[0], off - pos); 6571 } 6572 skb_frag_ref(skb, i); 6573 k++; 6574 } 6575 pos += fsize; 6576 } 6577 shinfo->nr_frags = k; 6578 if (skb_has_frag_list(skb)) 6579 skb_clone_fraglist(skb); 6580 6581 /* split line is in frag list */ 6582 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) { 6583 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */ 6584 if (skb_has_frag_list(skb)) 6585 kfree_skb_list(skb_shinfo(skb)->frag_list); 6586 skb_kfree_head(data, size); 6587 return -ENOMEM; 6588 } 6589 skb_release_data(skb, SKB_CONSUMED, false); 6590 6591 skb->head = data; 6592 skb->head_frag = 0; 6593 skb->data = data; 6594 skb_set_end_offset(skb, size); 6595 skb_reset_tail_pointer(skb); 6596 skb_headers_offset_update(skb, 0); 6597 skb->cloned = 0; 6598 skb->hdr_len = 0; 6599 skb->nohdr = 0; 6600 skb->len -= off; 6601 skb->data_len = skb->len; 6602 atomic_set(&skb_shinfo(skb)->dataref, 1); 6603 return 0; 6604 } 6605 6606 /* remove len bytes from the beginning of the skb */ 6607 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 6608 { 6609 int headlen = skb_headlen(skb); 6610 6611 if (len < headlen) 6612 return pskb_carve_inside_header(skb, len, headlen, gfp); 6613 else 6614 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 6615 } 6616 6617 /* Extract to_copy bytes starting at off from skb, and return this in 6618 * a new skb 6619 */ 6620 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 6621 int to_copy, gfp_t gfp) 6622 { 6623 struct sk_buff *clone = skb_clone(skb, gfp); 6624 6625 if (!clone) 6626 return NULL; 6627 6628 if (pskb_carve(clone, off, gfp) < 0 || 6629 pskb_trim(clone, to_copy)) { 6630 kfree_skb(clone); 6631 return NULL; 6632 } 6633 return clone; 6634 } 6635 EXPORT_SYMBOL(pskb_extract); 6636 6637 /** 6638 * skb_condense - try to get rid of fragments/frag_list if possible 6639 * @skb: buffer 6640 * 6641 * Can be used to save memory before skb is added to a busy queue. 6642 * If packet has bytes in frags and enough tail room in skb->head, 6643 * pull all of them, so that we can free the frags right now and adjust 6644 * truesize. 6645 * Notes: 6646 * We do not reallocate skb->head thus can not fail. 6647 * Caller must re-evaluate skb->truesize if needed. 6648 */ 6649 void skb_condense(struct sk_buff *skb) 6650 { 6651 if (skb->data_len) { 6652 if (skb->data_len > skb->end - skb->tail || 6653 skb_cloned(skb)) 6654 return; 6655 6656 /* Nice, we can free page frag(s) right now */ 6657 __pskb_pull_tail(skb, skb->data_len); 6658 } 6659 /* At this point, skb->truesize might be over estimated, 6660 * because skb had a fragment, and fragments do not tell 6661 * their truesize. 6662 * When we pulled its content into skb->head, fragment 6663 * was freed, but __pskb_pull_tail() could not possibly 6664 * adjust skb->truesize, not knowing the frag truesize. 6665 */ 6666 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 6667 } 6668 EXPORT_SYMBOL(skb_condense); 6669 6670 #ifdef CONFIG_SKB_EXTENSIONS 6671 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id) 6672 { 6673 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE); 6674 } 6675 6676 /** 6677 * __skb_ext_alloc - allocate a new skb extensions storage 6678 * 6679 * @flags: See kmalloc(). 6680 * 6681 * Returns the newly allocated pointer. The pointer can later attached to a 6682 * skb via __skb_ext_set(). 6683 * Note: caller must handle the skb_ext as an opaque data. 6684 */ 6685 struct skb_ext *__skb_ext_alloc(gfp_t flags) 6686 { 6687 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags); 6688 6689 if (new) { 6690 memset(new->offset, 0, sizeof(new->offset)); 6691 refcount_set(&new->refcnt, 1); 6692 } 6693 6694 return new; 6695 } 6696 6697 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old, 6698 unsigned int old_active) 6699 { 6700 struct skb_ext *new; 6701 6702 if (refcount_read(&old->refcnt) == 1) 6703 return old; 6704 6705 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 6706 if (!new) 6707 return NULL; 6708 6709 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE); 6710 refcount_set(&new->refcnt, 1); 6711 6712 #ifdef CONFIG_XFRM 6713 if (old_active & (1 << SKB_EXT_SEC_PATH)) { 6714 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH); 6715 unsigned int i; 6716 6717 for (i = 0; i < sp->len; i++) 6718 xfrm_state_hold(sp->xvec[i]); 6719 } 6720 #endif 6721 __skb_ext_put(old); 6722 return new; 6723 } 6724 6725 /** 6726 * __skb_ext_set - attach the specified extension storage to this skb 6727 * @skb: buffer 6728 * @id: extension id 6729 * @ext: extension storage previously allocated via __skb_ext_alloc() 6730 * 6731 * Existing extensions, if any, are cleared. 6732 * 6733 * Returns the pointer to the extension. 6734 */ 6735 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id, 6736 struct skb_ext *ext) 6737 { 6738 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext); 6739 6740 skb_ext_put(skb); 6741 newlen = newoff + skb_ext_type_len[id]; 6742 ext->chunks = newlen; 6743 ext->offset[id] = newoff; 6744 skb->extensions = ext; 6745 skb->active_extensions = 1 << id; 6746 return skb_ext_get_ptr(ext, id); 6747 } 6748 6749 /** 6750 * skb_ext_add - allocate space for given extension, COW if needed 6751 * @skb: buffer 6752 * @id: extension to allocate space for 6753 * 6754 * Allocates enough space for the given extension. 6755 * If the extension is already present, a pointer to that extension 6756 * is returned. 6757 * 6758 * If the skb was cloned, COW applies and the returned memory can be 6759 * modified without changing the extension space of clones buffers. 6760 * 6761 * Returns pointer to the extension or NULL on allocation failure. 6762 */ 6763 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id) 6764 { 6765 struct skb_ext *new, *old = NULL; 6766 unsigned int newlen, newoff; 6767 6768 if (skb->active_extensions) { 6769 old = skb->extensions; 6770 6771 new = skb_ext_maybe_cow(old, skb->active_extensions); 6772 if (!new) 6773 return NULL; 6774 6775 if (__skb_ext_exist(new, id)) 6776 goto set_active; 6777 6778 newoff = new->chunks; 6779 } else { 6780 newoff = SKB_EXT_CHUNKSIZEOF(*new); 6781 6782 new = __skb_ext_alloc(GFP_ATOMIC); 6783 if (!new) 6784 return NULL; 6785 } 6786 6787 newlen = newoff + skb_ext_type_len[id]; 6788 new->chunks = newlen; 6789 new->offset[id] = newoff; 6790 set_active: 6791 skb->slow_gro = 1; 6792 skb->extensions = new; 6793 skb->active_extensions |= 1 << id; 6794 return skb_ext_get_ptr(new, id); 6795 } 6796 EXPORT_SYMBOL(skb_ext_add); 6797 6798 #ifdef CONFIG_XFRM 6799 static void skb_ext_put_sp(struct sec_path *sp) 6800 { 6801 unsigned int i; 6802 6803 for (i = 0; i < sp->len; i++) 6804 xfrm_state_put(sp->xvec[i]); 6805 } 6806 #endif 6807 6808 #ifdef CONFIG_MCTP_FLOWS 6809 static void skb_ext_put_mctp(struct mctp_flow *flow) 6810 { 6811 if (flow->key) 6812 mctp_key_unref(flow->key); 6813 } 6814 #endif 6815 6816 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 6817 { 6818 struct skb_ext *ext = skb->extensions; 6819 6820 skb->active_extensions &= ~(1 << id); 6821 if (skb->active_extensions == 0) { 6822 skb->extensions = NULL; 6823 __skb_ext_put(ext); 6824 #ifdef CONFIG_XFRM 6825 } else if (id == SKB_EXT_SEC_PATH && 6826 refcount_read(&ext->refcnt) == 1) { 6827 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH); 6828 6829 skb_ext_put_sp(sp); 6830 sp->len = 0; 6831 #endif 6832 } 6833 } 6834 EXPORT_SYMBOL(__skb_ext_del); 6835 6836 void __skb_ext_put(struct skb_ext *ext) 6837 { 6838 /* If this is last clone, nothing can increment 6839 * it after check passes. Avoids one atomic op. 6840 */ 6841 if (refcount_read(&ext->refcnt) == 1) 6842 goto free_now; 6843 6844 if (!refcount_dec_and_test(&ext->refcnt)) 6845 return; 6846 free_now: 6847 #ifdef CONFIG_XFRM 6848 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH)) 6849 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 6850 #endif 6851 #ifdef CONFIG_MCTP_FLOWS 6852 if (__skb_ext_exist(ext, SKB_EXT_MCTP)) 6853 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP)); 6854 #endif 6855 6856 kmem_cache_free(skbuff_ext_cache, ext); 6857 } 6858 EXPORT_SYMBOL(__skb_ext_put); 6859 #endif /* CONFIG_SKB_EXTENSIONS */ 6860 6861 /** 6862 * skb_attempt_defer_free - queue skb for remote freeing 6863 * @skb: buffer 6864 * 6865 * Put @skb in a per-cpu list, using the cpu which 6866 * allocated the skb/pages to reduce false sharing 6867 * and memory zone spinlock contention. 6868 */ 6869 void skb_attempt_defer_free(struct sk_buff *skb) 6870 { 6871 int cpu = skb->alloc_cpu; 6872 struct softnet_data *sd; 6873 unsigned long flags; 6874 unsigned int defer_max; 6875 bool kick; 6876 6877 if (WARN_ON_ONCE(cpu >= nr_cpu_ids) || 6878 !cpu_online(cpu) || 6879 cpu == raw_smp_processor_id()) { 6880 nodefer: __kfree_skb(skb); 6881 return; 6882 } 6883 6884 sd = &per_cpu(softnet_data, cpu); 6885 defer_max = READ_ONCE(sysctl_skb_defer_max); 6886 if (READ_ONCE(sd->defer_count) >= defer_max) 6887 goto nodefer; 6888 6889 spin_lock_irqsave(&sd->defer_lock, flags); 6890 /* Send an IPI every time queue reaches half capacity. */ 6891 kick = sd->defer_count == (defer_max >> 1); 6892 /* Paired with the READ_ONCE() few lines above */ 6893 WRITE_ONCE(sd->defer_count, sd->defer_count + 1); 6894 6895 skb->next = sd->defer_list; 6896 /* Paired with READ_ONCE() in skb_defer_free_flush() */ 6897 WRITE_ONCE(sd->defer_list, skb); 6898 spin_unlock_irqrestore(&sd->defer_lock, flags); 6899 6900 /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU 6901 * if we are unlucky enough (this seems very unlikely). 6902 */ 6903 if (unlikely(kick) && !cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) 6904 smp_call_function_single_async(cpu, &sd->defer_csd); 6905 } 6906