1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Routines having to do with the 'struct sk_buff' memory handlers. 4 * 5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 6 * Florian La Roche <rzsfl@rz.uni-sb.de> 7 * 8 * Fixes: 9 * Alan Cox : Fixed the worst of the load 10 * balancer bugs. 11 * Dave Platt : Interrupt stacking fix. 12 * Richard Kooijman : Timestamp fixes. 13 * Alan Cox : Changed buffer format. 14 * Alan Cox : destructor hook for AF_UNIX etc. 15 * Linus Torvalds : Better skb_clone. 16 * Alan Cox : Added skb_copy. 17 * Alan Cox : Added all the changed routines Linus 18 * only put in the headers 19 * Ray VanTassle : Fixed --skb->lock in free 20 * Alan Cox : skb_copy copy arp field 21 * Andi Kleen : slabified it. 22 * Robert Olsson : Removed skb_head_pool 23 * 24 * NOTE: 25 * The __skb_ routines should be called with interrupts 26 * disabled, or you better be *real* sure that the operation is atomic 27 * with respect to whatever list is being frobbed (e.g. via lock_sock() 28 * or via disabling bottom half handlers, etc). 29 */ 30 31 /* 32 * The functions in this file will not compile correctly with gcc 2.4.x 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/module.h> 38 #include <linux/types.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/interrupt.h> 42 #include <linux/in.h> 43 #include <linux/inet.h> 44 #include <linux/slab.h> 45 #include <linux/tcp.h> 46 #include <linux/udp.h> 47 #include <linux/sctp.h> 48 #include <linux/netdevice.h> 49 #ifdef CONFIG_NET_CLS_ACT 50 #include <net/pkt_sched.h> 51 #endif 52 #include <linux/string.h> 53 #include <linux/skbuff.h> 54 #include <linux/splice.h> 55 #include <linux/cache.h> 56 #include <linux/rtnetlink.h> 57 #include <linux/init.h> 58 #include <linux/scatterlist.h> 59 #include <linux/errqueue.h> 60 #include <linux/prefetch.h> 61 #include <linux/bitfield.h> 62 #include <linux/if_vlan.h> 63 #include <linux/mpls.h> 64 #include <linux/kcov.h> 65 66 #include <net/protocol.h> 67 #include <net/dst.h> 68 #include <net/sock.h> 69 #include <net/checksum.h> 70 #include <net/gso.h> 71 #include <net/ip6_checksum.h> 72 #include <net/xfrm.h> 73 #include <net/mpls.h> 74 #include <net/mptcp.h> 75 #include <net/mctp.h> 76 #include <net/page_pool/helpers.h> 77 #include <net/dropreason.h> 78 79 #include <linux/uaccess.h> 80 #include <trace/events/skb.h> 81 #include <linux/highmem.h> 82 #include <linux/capability.h> 83 #include <linux/user_namespace.h> 84 #include <linux/indirect_call_wrapper.h> 85 #include <linux/textsearch.h> 86 87 #include "dev.h" 88 #include "sock_destructor.h" 89 90 struct kmem_cache *skbuff_cache __ro_after_init; 91 static struct kmem_cache *skbuff_fclone_cache __ro_after_init; 92 #ifdef CONFIG_SKB_EXTENSIONS 93 static struct kmem_cache *skbuff_ext_cache __ro_after_init; 94 #endif 95 96 97 static struct kmem_cache *skb_small_head_cache __ro_after_init; 98 99 #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER) 100 101 /* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two. 102 * This should ensure that SKB_SMALL_HEAD_HEADROOM is a unique 103 * size, and we can differentiate heads from skb_small_head_cache 104 * vs system slabs by looking at their size (skb_end_offset()). 105 */ 106 #define SKB_SMALL_HEAD_CACHE_SIZE \ 107 (is_power_of_2(SKB_SMALL_HEAD_SIZE) ? \ 108 (SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) : \ 109 SKB_SMALL_HEAD_SIZE) 110 111 #define SKB_SMALL_HEAD_HEADROOM \ 112 SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) 113 114 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; 115 EXPORT_SYMBOL(sysctl_max_skb_frags); 116 117 #undef FN 118 #define FN(reason) [SKB_DROP_REASON_##reason] = #reason, 119 static const char * const drop_reasons[] = { 120 [SKB_CONSUMED] = "CONSUMED", 121 DEFINE_DROP_REASON(FN, FN) 122 }; 123 124 static const struct drop_reason_list drop_reasons_core = { 125 .reasons = drop_reasons, 126 .n_reasons = ARRAY_SIZE(drop_reasons), 127 }; 128 129 const struct drop_reason_list __rcu * 130 drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = { 131 [SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core), 132 }; 133 EXPORT_SYMBOL(drop_reasons_by_subsys); 134 135 /** 136 * drop_reasons_register_subsys - register another drop reason subsystem 137 * @subsys: the subsystem to register, must not be the core 138 * @list: the list of drop reasons within the subsystem, must point to 139 * a statically initialized list 140 */ 141 void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys, 142 const struct drop_reason_list *list) 143 { 144 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 145 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 146 "invalid subsystem %d\n", subsys)) 147 return; 148 149 /* must point to statically allocated memory, so INIT is OK */ 150 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list); 151 } 152 EXPORT_SYMBOL_GPL(drop_reasons_register_subsys); 153 154 /** 155 * drop_reasons_unregister_subsys - unregister a drop reason subsystem 156 * @subsys: the subsystem to remove, must not be the core 157 * 158 * Note: This will synchronize_rcu() to ensure no users when it returns. 159 */ 160 void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys) 161 { 162 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 163 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 164 "invalid subsystem %d\n", subsys)) 165 return; 166 167 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL); 168 169 synchronize_rcu(); 170 } 171 EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys); 172 173 /** 174 * skb_panic - private function for out-of-line support 175 * @skb: buffer 176 * @sz: size 177 * @addr: address 178 * @msg: skb_over_panic or skb_under_panic 179 * 180 * Out-of-line support for skb_put() and skb_push(). 181 * Called via the wrapper skb_over_panic() or skb_under_panic(). 182 * Keep out of line to prevent kernel bloat. 183 * __builtin_return_address is not used because it is not always reliable. 184 */ 185 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 186 const char msg[]) 187 { 188 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n", 189 msg, addr, skb->len, sz, skb->head, skb->data, 190 (unsigned long)skb->tail, (unsigned long)skb->end, 191 skb->dev ? skb->dev->name : "<NULL>"); 192 BUG(); 193 } 194 195 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 196 { 197 skb_panic(skb, sz, addr, __func__); 198 } 199 200 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 201 { 202 skb_panic(skb, sz, addr, __func__); 203 } 204 205 #define NAPI_SKB_CACHE_SIZE 64 206 #define NAPI_SKB_CACHE_BULK 16 207 #define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2) 208 209 #if PAGE_SIZE == SZ_4K 210 211 #define NAPI_HAS_SMALL_PAGE_FRAG 1 212 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) ((nc).pfmemalloc) 213 214 /* specialized page frag allocator using a single order 0 page 215 * and slicing it into 1K sized fragment. Constrained to systems 216 * with a very limited amount of 1K fragments fitting a single 217 * page - to avoid excessive truesize underestimation 218 */ 219 220 struct page_frag_1k { 221 void *va; 222 u16 offset; 223 bool pfmemalloc; 224 }; 225 226 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp) 227 { 228 struct page *page; 229 int offset; 230 231 offset = nc->offset - SZ_1K; 232 if (likely(offset >= 0)) 233 goto use_frag; 234 235 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 236 if (!page) 237 return NULL; 238 239 nc->va = page_address(page); 240 nc->pfmemalloc = page_is_pfmemalloc(page); 241 offset = PAGE_SIZE - SZ_1K; 242 page_ref_add(page, offset / SZ_1K); 243 244 use_frag: 245 nc->offset = offset; 246 return nc->va + offset; 247 } 248 #else 249 250 /* the small page is actually unused in this build; add dummy helpers 251 * to please the compiler and avoid later preprocessor's conditionals 252 */ 253 #define NAPI_HAS_SMALL_PAGE_FRAG 0 254 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) false 255 256 struct page_frag_1k { 257 }; 258 259 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp_mask) 260 { 261 return NULL; 262 } 263 264 #endif 265 266 struct napi_alloc_cache { 267 struct page_frag_cache page; 268 struct page_frag_1k page_small; 269 unsigned int skb_count; 270 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 271 }; 272 273 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 274 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 275 276 /* Double check that napi_get_frags() allocates skbs with 277 * skb->head being backed by slab, not a page fragment. 278 * This is to make sure bug fixed in 3226b158e67c 279 * ("net: avoid 32 x truesize under-estimation for tiny skbs") 280 * does not accidentally come back. 281 */ 282 void napi_get_frags_check(struct napi_struct *napi) 283 { 284 struct sk_buff *skb; 285 286 local_bh_disable(); 287 skb = napi_get_frags(napi); 288 WARN_ON_ONCE(!NAPI_HAS_SMALL_PAGE_FRAG && skb && skb->head_frag); 289 napi_free_frags(napi); 290 local_bh_enable(); 291 } 292 293 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 294 { 295 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 296 297 fragsz = SKB_DATA_ALIGN(fragsz); 298 299 return page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); 300 } 301 EXPORT_SYMBOL(__napi_alloc_frag_align); 302 303 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 304 { 305 void *data; 306 307 fragsz = SKB_DATA_ALIGN(fragsz); 308 if (in_hardirq() || irqs_disabled()) { 309 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache); 310 311 data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask); 312 } else { 313 struct napi_alloc_cache *nc; 314 315 local_bh_disable(); 316 nc = this_cpu_ptr(&napi_alloc_cache); 317 data = page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); 318 local_bh_enable(); 319 } 320 return data; 321 } 322 EXPORT_SYMBOL(__netdev_alloc_frag_align); 323 324 static struct sk_buff *napi_skb_cache_get(void) 325 { 326 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 327 struct sk_buff *skb; 328 329 if (unlikely(!nc->skb_count)) { 330 nc->skb_count = kmem_cache_alloc_bulk(skbuff_cache, 331 GFP_ATOMIC, 332 NAPI_SKB_CACHE_BULK, 333 nc->skb_cache); 334 if (unlikely(!nc->skb_count)) 335 return NULL; 336 } 337 338 skb = nc->skb_cache[--nc->skb_count]; 339 kasan_unpoison_object_data(skbuff_cache, skb); 340 341 return skb; 342 } 343 344 static inline void __finalize_skb_around(struct sk_buff *skb, void *data, 345 unsigned int size) 346 { 347 struct skb_shared_info *shinfo; 348 349 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 350 351 /* Assumes caller memset cleared SKB */ 352 skb->truesize = SKB_TRUESIZE(size); 353 refcount_set(&skb->users, 1); 354 skb->head = data; 355 skb->data = data; 356 skb_reset_tail_pointer(skb); 357 skb_set_end_offset(skb, size); 358 skb->mac_header = (typeof(skb->mac_header))~0U; 359 skb->transport_header = (typeof(skb->transport_header))~0U; 360 skb->alloc_cpu = raw_smp_processor_id(); 361 /* make sure we initialize shinfo sequentially */ 362 shinfo = skb_shinfo(skb); 363 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 364 atomic_set(&shinfo->dataref, 1); 365 366 skb_set_kcov_handle(skb, kcov_common_handle()); 367 } 368 369 static inline void *__slab_build_skb(struct sk_buff *skb, void *data, 370 unsigned int *size) 371 { 372 void *resized; 373 374 /* Must find the allocation size (and grow it to match). */ 375 *size = ksize(data); 376 /* krealloc() will immediately return "data" when 377 * "ksize(data)" is requested: it is the existing upper 378 * bounds. As a result, GFP_ATOMIC will be ignored. Note 379 * that this "new" pointer needs to be passed back to the 380 * caller for use so the __alloc_size hinting will be 381 * tracked correctly. 382 */ 383 resized = krealloc(data, *size, GFP_ATOMIC); 384 WARN_ON_ONCE(resized != data); 385 return resized; 386 } 387 388 /* build_skb() variant which can operate on slab buffers. 389 * Note that this should be used sparingly as slab buffers 390 * cannot be combined efficiently by GRO! 391 */ 392 struct sk_buff *slab_build_skb(void *data) 393 { 394 struct sk_buff *skb; 395 unsigned int size; 396 397 skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); 398 if (unlikely(!skb)) 399 return NULL; 400 401 memset(skb, 0, offsetof(struct sk_buff, tail)); 402 data = __slab_build_skb(skb, data, &size); 403 __finalize_skb_around(skb, data, size); 404 405 return skb; 406 } 407 EXPORT_SYMBOL(slab_build_skb); 408 409 /* Caller must provide SKB that is memset cleared */ 410 static void __build_skb_around(struct sk_buff *skb, void *data, 411 unsigned int frag_size) 412 { 413 unsigned int size = frag_size; 414 415 /* frag_size == 0 is considered deprecated now. Callers 416 * using slab buffer should use slab_build_skb() instead. 417 */ 418 if (WARN_ONCE(size == 0, "Use slab_build_skb() instead")) 419 data = __slab_build_skb(skb, data, &size); 420 421 __finalize_skb_around(skb, data, size); 422 } 423 424 /** 425 * __build_skb - build a network buffer 426 * @data: data buffer provided by caller 427 * @frag_size: size of data (must not be 0) 428 * 429 * Allocate a new &sk_buff. Caller provides space holding head and 430 * skb_shared_info. @data must have been allocated from the page 431 * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc() 432 * allocation is deprecated, and callers should use slab_build_skb() 433 * instead.) 434 * The return is the new skb buffer. 435 * On a failure the return is %NULL, and @data is not freed. 436 * Notes : 437 * Before IO, driver allocates only data buffer where NIC put incoming frame 438 * Driver should add room at head (NET_SKB_PAD) and 439 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 440 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 441 * before giving packet to stack. 442 * RX rings only contains data buffers, not full skbs. 443 */ 444 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 445 { 446 struct sk_buff *skb; 447 448 skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); 449 if (unlikely(!skb)) 450 return NULL; 451 452 memset(skb, 0, offsetof(struct sk_buff, tail)); 453 __build_skb_around(skb, data, frag_size); 454 455 return skb; 456 } 457 458 /* build_skb() is wrapper over __build_skb(), that specifically 459 * takes care of skb->head and skb->pfmemalloc 460 */ 461 struct sk_buff *build_skb(void *data, unsigned int frag_size) 462 { 463 struct sk_buff *skb = __build_skb(data, frag_size); 464 465 if (likely(skb && frag_size)) { 466 skb->head_frag = 1; 467 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 468 } 469 return skb; 470 } 471 EXPORT_SYMBOL(build_skb); 472 473 /** 474 * build_skb_around - build a network buffer around provided skb 475 * @skb: sk_buff provide by caller, must be memset cleared 476 * @data: data buffer provided by caller 477 * @frag_size: size of data 478 */ 479 struct sk_buff *build_skb_around(struct sk_buff *skb, 480 void *data, unsigned int frag_size) 481 { 482 if (unlikely(!skb)) 483 return NULL; 484 485 __build_skb_around(skb, data, frag_size); 486 487 if (frag_size) { 488 skb->head_frag = 1; 489 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 490 } 491 return skb; 492 } 493 EXPORT_SYMBOL(build_skb_around); 494 495 /** 496 * __napi_build_skb - build a network buffer 497 * @data: data buffer provided by caller 498 * @frag_size: size of data 499 * 500 * Version of __build_skb() that uses NAPI percpu caches to obtain 501 * skbuff_head instead of inplace allocation. 502 * 503 * Returns a new &sk_buff on success, %NULL on allocation failure. 504 */ 505 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 506 { 507 struct sk_buff *skb; 508 509 skb = napi_skb_cache_get(); 510 if (unlikely(!skb)) 511 return NULL; 512 513 memset(skb, 0, offsetof(struct sk_buff, tail)); 514 __build_skb_around(skb, data, frag_size); 515 516 return skb; 517 } 518 519 /** 520 * napi_build_skb - build a network buffer 521 * @data: data buffer provided by caller 522 * @frag_size: size of data 523 * 524 * Version of __napi_build_skb() that takes care of skb->head_frag 525 * and skb->pfmemalloc when the data is a page or page fragment. 526 * 527 * Returns a new &sk_buff on success, %NULL on allocation failure. 528 */ 529 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) 530 { 531 struct sk_buff *skb = __napi_build_skb(data, frag_size); 532 533 if (likely(skb) && frag_size) { 534 skb->head_frag = 1; 535 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 536 } 537 538 return skb; 539 } 540 EXPORT_SYMBOL(napi_build_skb); 541 542 /* 543 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 544 * the caller if emergency pfmemalloc reserves are being used. If it is and 545 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 546 * may be used. Otherwise, the packet data may be discarded until enough 547 * memory is free 548 */ 549 static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, 550 bool *pfmemalloc) 551 { 552 bool ret_pfmemalloc = false; 553 unsigned int obj_size; 554 void *obj; 555 556 obj_size = SKB_HEAD_ALIGN(*size); 557 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE && 558 !(flags & KMALLOC_NOT_NORMAL_BITS)) { 559 obj = kmem_cache_alloc_node(skb_small_head_cache, 560 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 561 node); 562 *size = SKB_SMALL_HEAD_CACHE_SIZE; 563 if (obj || !(gfp_pfmemalloc_allowed(flags))) 564 goto out; 565 /* Try again but now we are using pfmemalloc reserves */ 566 ret_pfmemalloc = true; 567 obj = kmem_cache_alloc_node(skb_small_head_cache, flags, node); 568 goto out; 569 } 570 *size = obj_size = kmalloc_size_roundup(obj_size); 571 /* 572 * Try a regular allocation, when that fails and we're not entitled 573 * to the reserves, fail. 574 */ 575 obj = kmalloc_node_track_caller(obj_size, 576 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 577 node); 578 if (obj || !(gfp_pfmemalloc_allowed(flags))) 579 goto out; 580 581 /* Try again but now we are using pfmemalloc reserves */ 582 ret_pfmemalloc = true; 583 obj = kmalloc_node_track_caller(obj_size, flags, node); 584 585 out: 586 if (pfmemalloc) 587 *pfmemalloc = ret_pfmemalloc; 588 589 return obj; 590 } 591 592 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 593 * 'private' fields and also do memory statistics to find all the 594 * [BEEP] leaks. 595 * 596 */ 597 598 /** 599 * __alloc_skb - allocate a network buffer 600 * @size: size to allocate 601 * @gfp_mask: allocation mask 602 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 603 * instead of head cache and allocate a cloned (child) skb. 604 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 605 * allocations in case the data is required for writeback 606 * @node: numa node to allocate memory on 607 * 608 * Allocate a new &sk_buff. The returned buffer has no headroom and a 609 * tail room of at least size bytes. The object has a reference count 610 * of one. The return is the buffer. On a failure the return is %NULL. 611 * 612 * Buffers may only be allocated from interrupts using a @gfp_mask of 613 * %GFP_ATOMIC. 614 */ 615 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 616 int flags, int node) 617 { 618 struct kmem_cache *cache; 619 struct sk_buff *skb; 620 bool pfmemalloc; 621 u8 *data; 622 623 cache = (flags & SKB_ALLOC_FCLONE) 624 ? skbuff_fclone_cache : skbuff_cache; 625 626 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 627 gfp_mask |= __GFP_MEMALLOC; 628 629 /* Get the HEAD */ 630 if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI && 631 likely(node == NUMA_NO_NODE || node == numa_mem_id())) 632 skb = napi_skb_cache_get(); 633 else 634 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); 635 if (unlikely(!skb)) 636 return NULL; 637 prefetchw(skb); 638 639 /* We do our best to align skb_shared_info on a separate cache 640 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 641 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 642 * Both skb->head and skb_shared_info are cache line aligned. 643 */ 644 data = kmalloc_reserve(&size, gfp_mask, node, &pfmemalloc); 645 if (unlikely(!data)) 646 goto nodata; 647 /* kmalloc_size_roundup() might give us more room than requested. 648 * Put skb_shared_info exactly at the end of allocated zone, 649 * to allow max possible filling before reallocation. 650 */ 651 prefetchw(data + SKB_WITH_OVERHEAD(size)); 652 653 /* 654 * Only clear those fields we need to clear, not those that we will 655 * actually initialise below. Hence, don't put any more fields after 656 * the tail pointer in struct sk_buff! 657 */ 658 memset(skb, 0, offsetof(struct sk_buff, tail)); 659 __build_skb_around(skb, data, size); 660 skb->pfmemalloc = pfmemalloc; 661 662 if (flags & SKB_ALLOC_FCLONE) { 663 struct sk_buff_fclones *fclones; 664 665 fclones = container_of(skb, struct sk_buff_fclones, skb1); 666 667 skb->fclone = SKB_FCLONE_ORIG; 668 refcount_set(&fclones->fclone_ref, 1); 669 } 670 671 return skb; 672 673 nodata: 674 kmem_cache_free(cache, skb); 675 return NULL; 676 } 677 EXPORT_SYMBOL(__alloc_skb); 678 679 /** 680 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 681 * @dev: network device to receive on 682 * @len: length to allocate 683 * @gfp_mask: get_free_pages mask, passed to alloc_skb 684 * 685 * Allocate a new &sk_buff and assign it a usage count of one. The 686 * buffer has NET_SKB_PAD headroom built in. Users should allocate 687 * the headroom they think they need without accounting for the 688 * built in space. The built in space is used for optimisations. 689 * 690 * %NULL is returned if there is no free memory. 691 */ 692 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 693 gfp_t gfp_mask) 694 { 695 struct page_frag_cache *nc; 696 struct sk_buff *skb; 697 bool pfmemalloc; 698 void *data; 699 700 len += NET_SKB_PAD; 701 702 /* If requested length is either too small or too big, 703 * we use kmalloc() for skb->head allocation. 704 */ 705 if (len <= SKB_WITH_OVERHEAD(1024) || 706 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 707 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 708 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 709 if (!skb) 710 goto skb_fail; 711 goto skb_success; 712 } 713 714 len = SKB_HEAD_ALIGN(len); 715 716 if (sk_memalloc_socks()) 717 gfp_mask |= __GFP_MEMALLOC; 718 719 if (in_hardirq() || irqs_disabled()) { 720 nc = this_cpu_ptr(&netdev_alloc_cache); 721 data = page_frag_alloc(nc, len, gfp_mask); 722 pfmemalloc = nc->pfmemalloc; 723 } else { 724 local_bh_disable(); 725 nc = this_cpu_ptr(&napi_alloc_cache.page); 726 data = page_frag_alloc(nc, len, gfp_mask); 727 pfmemalloc = nc->pfmemalloc; 728 local_bh_enable(); 729 } 730 731 if (unlikely(!data)) 732 return NULL; 733 734 skb = __build_skb(data, len); 735 if (unlikely(!skb)) { 736 skb_free_frag(data); 737 return NULL; 738 } 739 740 if (pfmemalloc) 741 skb->pfmemalloc = 1; 742 skb->head_frag = 1; 743 744 skb_success: 745 skb_reserve(skb, NET_SKB_PAD); 746 skb->dev = dev; 747 748 skb_fail: 749 return skb; 750 } 751 EXPORT_SYMBOL(__netdev_alloc_skb); 752 753 /** 754 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 755 * @napi: napi instance this buffer was allocated for 756 * @len: length to allocate 757 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages 758 * 759 * Allocate a new sk_buff for use in NAPI receive. This buffer will 760 * attempt to allocate the head from a special reserved region used 761 * only for NAPI Rx allocation. By doing this we can save several 762 * CPU cycles by avoiding having to disable and re-enable IRQs. 763 * 764 * %NULL is returned if there is no free memory. 765 */ 766 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, 767 gfp_t gfp_mask) 768 { 769 struct napi_alloc_cache *nc; 770 struct sk_buff *skb; 771 bool pfmemalloc; 772 void *data; 773 774 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 775 len += NET_SKB_PAD + NET_IP_ALIGN; 776 777 /* If requested length is either too small or too big, 778 * we use kmalloc() for skb->head allocation. 779 * When the small frag allocator is available, prefer it over kmalloc 780 * for small fragments 781 */ 782 if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) || 783 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 784 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 785 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, 786 NUMA_NO_NODE); 787 if (!skb) 788 goto skb_fail; 789 goto skb_success; 790 } 791 792 nc = this_cpu_ptr(&napi_alloc_cache); 793 794 if (sk_memalloc_socks()) 795 gfp_mask |= __GFP_MEMALLOC; 796 797 if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) { 798 /* we are artificially inflating the allocation size, but 799 * that is not as bad as it may look like, as: 800 * - 'len' less than GRO_MAX_HEAD makes little sense 801 * - On most systems, larger 'len' values lead to fragment 802 * size above 512 bytes 803 * - kmalloc would use the kmalloc-1k slab for such values 804 * - Builds with smaller GRO_MAX_HEAD will very likely do 805 * little networking, as that implies no WiFi and no 806 * tunnels support, and 32 bits arches. 807 */ 808 len = SZ_1K; 809 810 data = page_frag_alloc_1k(&nc->page_small, gfp_mask); 811 pfmemalloc = NAPI_SMALL_PAGE_PFMEMALLOC(nc->page_small); 812 } else { 813 len = SKB_HEAD_ALIGN(len); 814 815 data = page_frag_alloc(&nc->page, len, gfp_mask); 816 pfmemalloc = nc->page.pfmemalloc; 817 } 818 819 if (unlikely(!data)) 820 return NULL; 821 822 skb = __napi_build_skb(data, len); 823 if (unlikely(!skb)) { 824 skb_free_frag(data); 825 return NULL; 826 } 827 828 if (pfmemalloc) 829 skb->pfmemalloc = 1; 830 skb->head_frag = 1; 831 832 skb_success: 833 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 834 skb->dev = napi->dev; 835 836 skb_fail: 837 return skb; 838 } 839 EXPORT_SYMBOL(__napi_alloc_skb); 840 841 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 842 int size, unsigned int truesize) 843 { 844 skb_fill_page_desc(skb, i, page, off, size); 845 skb->len += size; 846 skb->data_len += size; 847 skb->truesize += truesize; 848 } 849 EXPORT_SYMBOL(skb_add_rx_frag); 850 851 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 852 unsigned int truesize) 853 { 854 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 855 856 skb_frag_size_add(frag, size); 857 skb->len += size; 858 skb->data_len += size; 859 skb->truesize += truesize; 860 } 861 EXPORT_SYMBOL(skb_coalesce_rx_frag); 862 863 static void skb_drop_list(struct sk_buff **listp) 864 { 865 kfree_skb_list(*listp); 866 *listp = NULL; 867 } 868 869 static inline void skb_drop_fraglist(struct sk_buff *skb) 870 { 871 skb_drop_list(&skb_shinfo(skb)->frag_list); 872 } 873 874 static void skb_clone_fraglist(struct sk_buff *skb) 875 { 876 struct sk_buff *list; 877 878 skb_walk_frags(skb, list) 879 skb_get(list); 880 } 881 882 #if IS_ENABLED(CONFIG_PAGE_POOL) 883 bool napi_pp_put_page(struct page *page, bool napi_safe) 884 { 885 bool allow_direct = false; 886 struct page_pool *pp; 887 888 page = compound_head(page); 889 890 /* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation 891 * in order to preserve any existing bits, such as bit 0 for the 892 * head page of compound page and bit 1 for pfmemalloc page, so 893 * mask those bits for freeing side when doing below checking, 894 * and page_is_pfmemalloc() is checked in __page_pool_put_page() 895 * to avoid recycling the pfmemalloc page. 896 */ 897 if (unlikely((page->pp_magic & ~0x3UL) != PP_SIGNATURE)) 898 return false; 899 900 pp = page->pp; 901 902 /* Allow direct recycle if we have reasons to believe that we are 903 * in the same context as the consumer would run, so there's 904 * no possible race. 905 * __page_pool_put_page() makes sure we're not in hardirq context 906 * and interrupts are enabled prior to accessing the cache. 907 */ 908 if (napi_safe || in_softirq()) { 909 const struct napi_struct *napi = READ_ONCE(pp->p.napi); 910 911 allow_direct = napi && 912 READ_ONCE(napi->list_owner) == smp_processor_id(); 913 } 914 915 /* Driver set this to memory recycling info. Reset it on recycle. 916 * This will *not* work for NIC using a split-page memory model. 917 * The page will be returned to the pool here regardless of the 918 * 'flipped' fragment being in use or not. 919 */ 920 page_pool_put_full_page(pp, page, allow_direct); 921 922 return true; 923 } 924 EXPORT_SYMBOL(napi_pp_put_page); 925 #endif 926 927 static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe) 928 { 929 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) 930 return false; 931 return napi_pp_put_page(virt_to_page(data), napi_safe); 932 } 933 934 static void skb_kfree_head(void *head, unsigned int end_offset) 935 { 936 if (end_offset == SKB_SMALL_HEAD_HEADROOM) 937 kmem_cache_free(skb_small_head_cache, head); 938 else 939 kfree(head); 940 } 941 942 static void skb_free_head(struct sk_buff *skb, bool napi_safe) 943 { 944 unsigned char *head = skb->head; 945 946 if (skb->head_frag) { 947 if (skb_pp_recycle(skb, head, napi_safe)) 948 return; 949 skb_free_frag(head); 950 } else { 951 skb_kfree_head(head, skb_end_offset(skb)); 952 } 953 } 954 955 static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason, 956 bool napi_safe) 957 { 958 struct skb_shared_info *shinfo = skb_shinfo(skb); 959 int i; 960 961 if (skb->cloned && 962 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 963 &shinfo->dataref)) 964 goto exit; 965 966 if (skb_zcopy(skb)) { 967 bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS; 968 969 skb_zcopy_clear(skb, true); 970 if (skip_unref) 971 goto free_head; 972 } 973 974 for (i = 0; i < shinfo->nr_frags; i++) 975 napi_frag_unref(&shinfo->frags[i], skb->pp_recycle, napi_safe); 976 977 free_head: 978 if (shinfo->frag_list) 979 kfree_skb_list_reason(shinfo->frag_list, reason); 980 981 skb_free_head(skb, napi_safe); 982 exit: 983 /* When we clone an SKB we copy the reycling bit. The pp_recycle 984 * bit is only set on the head though, so in order to avoid races 985 * while trying to recycle fragments on __skb_frag_unref() we need 986 * to make one SKB responsible for triggering the recycle path. 987 * So disable the recycling bit if an SKB is cloned and we have 988 * additional references to the fragmented part of the SKB. 989 * Eventually the last SKB will have the recycling bit set and it's 990 * dataref set to 0, which will trigger the recycling 991 */ 992 skb->pp_recycle = 0; 993 } 994 995 /* 996 * Free an skbuff by memory without cleaning the state. 997 */ 998 static void kfree_skbmem(struct sk_buff *skb) 999 { 1000 struct sk_buff_fclones *fclones; 1001 1002 switch (skb->fclone) { 1003 case SKB_FCLONE_UNAVAILABLE: 1004 kmem_cache_free(skbuff_cache, skb); 1005 return; 1006 1007 case SKB_FCLONE_ORIG: 1008 fclones = container_of(skb, struct sk_buff_fclones, skb1); 1009 1010 /* We usually free the clone (TX completion) before original skb 1011 * This test would have no chance to be true for the clone, 1012 * while here, branch prediction will be good. 1013 */ 1014 if (refcount_read(&fclones->fclone_ref) == 1) 1015 goto fastpath; 1016 break; 1017 1018 default: /* SKB_FCLONE_CLONE */ 1019 fclones = container_of(skb, struct sk_buff_fclones, skb2); 1020 break; 1021 } 1022 if (!refcount_dec_and_test(&fclones->fclone_ref)) 1023 return; 1024 fastpath: 1025 kmem_cache_free(skbuff_fclone_cache, fclones); 1026 } 1027 1028 void skb_release_head_state(struct sk_buff *skb) 1029 { 1030 skb_dst_drop(skb); 1031 if (skb->destructor) { 1032 DEBUG_NET_WARN_ON_ONCE(in_hardirq()); 1033 skb->destructor(skb); 1034 } 1035 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 1036 nf_conntrack_put(skb_nfct(skb)); 1037 #endif 1038 skb_ext_put(skb); 1039 } 1040 1041 /* Free everything but the sk_buff shell. */ 1042 static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason, 1043 bool napi_safe) 1044 { 1045 skb_release_head_state(skb); 1046 if (likely(skb->head)) 1047 skb_release_data(skb, reason, napi_safe); 1048 } 1049 1050 /** 1051 * __kfree_skb - private function 1052 * @skb: buffer 1053 * 1054 * Free an sk_buff. Release anything attached to the buffer. 1055 * Clean the state. This is an internal helper function. Users should 1056 * always call kfree_skb 1057 */ 1058 1059 void __kfree_skb(struct sk_buff *skb) 1060 { 1061 skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED, false); 1062 kfree_skbmem(skb); 1063 } 1064 EXPORT_SYMBOL(__kfree_skb); 1065 1066 static __always_inline 1067 bool __kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) 1068 { 1069 if (unlikely(!skb_unref(skb))) 1070 return false; 1071 1072 DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET || 1073 u32_get_bits(reason, 1074 SKB_DROP_REASON_SUBSYS_MASK) >= 1075 SKB_DROP_REASON_SUBSYS_NUM); 1076 1077 if (reason == SKB_CONSUMED) 1078 trace_consume_skb(skb, __builtin_return_address(0)); 1079 else 1080 trace_kfree_skb(skb, __builtin_return_address(0), reason); 1081 return true; 1082 } 1083 1084 /** 1085 * kfree_skb_reason - free an sk_buff with special reason 1086 * @skb: buffer to free 1087 * @reason: reason why this skb is dropped 1088 * 1089 * Drop a reference to the buffer and free it if the usage count has 1090 * hit zero. Meanwhile, pass the drop reason to 'kfree_skb' 1091 * tracepoint. 1092 */ 1093 void __fix_address 1094 kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) 1095 { 1096 if (__kfree_skb_reason(skb, reason)) 1097 __kfree_skb(skb); 1098 } 1099 EXPORT_SYMBOL(kfree_skb_reason); 1100 1101 #define KFREE_SKB_BULK_SIZE 16 1102 1103 struct skb_free_array { 1104 unsigned int skb_count; 1105 void *skb_array[KFREE_SKB_BULK_SIZE]; 1106 }; 1107 1108 static void kfree_skb_add_bulk(struct sk_buff *skb, 1109 struct skb_free_array *sa, 1110 enum skb_drop_reason reason) 1111 { 1112 /* if SKB is a clone, don't handle this case */ 1113 if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) { 1114 __kfree_skb(skb); 1115 return; 1116 } 1117 1118 skb_release_all(skb, reason, false); 1119 sa->skb_array[sa->skb_count++] = skb; 1120 1121 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) { 1122 kmem_cache_free_bulk(skbuff_cache, KFREE_SKB_BULK_SIZE, 1123 sa->skb_array); 1124 sa->skb_count = 0; 1125 } 1126 } 1127 1128 void __fix_address 1129 kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) 1130 { 1131 struct skb_free_array sa; 1132 1133 sa.skb_count = 0; 1134 1135 while (segs) { 1136 struct sk_buff *next = segs->next; 1137 1138 if (__kfree_skb_reason(segs, reason)) { 1139 skb_poison_list(segs); 1140 kfree_skb_add_bulk(segs, &sa, reason); 1141 } 1142 1143 segs = next; 1144 } 1145 1146 if (sa.skb_count) 1147 kmem_cache_free_bulk(skbuff_cache, sa.skb_count, sa.skb_array); 1148 } 1149 EXPORT_SYMBOL(kfree_skb_list_reason); 1150 1151 /* Dump skb information and contents. 1152 * 1153 * Must only be called from net_ratelimit()-ed paths. 1154 * 1155 * Dumps whole packets if full_pkt, only headers otherwise. 1156 */ 1157 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt) 1158 { 1159 struct skb_shared_info *sh = skb_shinfo(skb); 1160 struct net_device *dev = skb->dev; 1161 struct sock *sk = skb->sk; 1162 struct sk_buff *list_skb; 1163 bool has_mac, has_trans; 1164 int headroom, tailroom; 1165 int i, len, seg_len; 1166 1167 if (full_pkt) 1168 len = skb->len; 1169 else 1170 len = min_t(int, skb->len, MAX_HEADER + 128); 1171 1172 headroom = skb_headroom(skb); 1173 tailroom = skb_tailroom(skb); 1174 1175 has_mac = skb_mac_header_was_set(skb); 1176 has_trans = skb_transport_header_was_set(skb); 1177 1178 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n" 1179 "mac=(%d,%d) net=(%d,%d) trans=%d\n" 1180 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" 1181 "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n" 1182 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n", 1183 level, skb->len, headroom, skb_headlen(skb), tailroom, 1184 has_mac ? skb->mac_header : -1, 1185 has_mac ? skb_mac_header_len(skb) : -1, 1186 skb->network_header, 1187 has_trans ? skb_network_header_len(skb) : -1, 1188 has_trans ? skb->transport_header : -1, 1189 sh->tx_flags, sh->nr_frags, 1190 sh->gso_size, sh->gso_type, sh->gso_segs, 1191 skb->csum, skb->ip_summed, skb->csum_complete_sw, 1192 skb->csum_valid, skb->csum_level, 1193 skb->hash, skb->sw_hash, skb->l4_hash, 1194 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif); 1195 1196 if (dev) 1197 printk("%sdev name=%s feat=%pNF\n", 1198 level, dev->name, &dev->features); 1199 if (sk) 1200 printk("%ssk family=%hu type=%u proto=%u\n", 1201 level, sk->sk_family, sk->sk_type, sk->sk_protocol); 1202 1203 if (full_pkt && headroom) 1204 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET, 1205 16, 1, skb->head, headroom, false); 1206 1207 seg_len = min_t(int, skb_headlen(skb), len); 1208 if (seg_len) 1209 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, 1210 16, 1, skb->data, seg_len, false); 1211 len -= seg_len; 1212 1213 if (full_pkt && tailroom) 1214 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET, 1215 16, 1, skb_tail_pointer(skb), tailroom, false); 1216 1217 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { 1218 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1219 u32 p_off, p_len, copied; 1220 struct page *p; 1221 u8 *vaddr; 1222 1223 skb_frag_foreach_page(frag, skb_frag_off(frag), 1224 skb_frag_size(frag), p, p_off, p_len, 1225 copied) { 1226 seg_len = min_t(int, p_len, len); 1227 vaddr = kmap_atomic(p); 1228 print_hex_dump(level, "skb frag: ", 1229 DUMP_PREFIX_OFFSET, 1230 16, 1, vaddr + p_off, seg_len, false); 1231 kunmap_atomic(vaddr); 1232 len -= seg_len; 1233 if (!len) 1234 break; 1235 } 1236 } 1237 1238 if (full_pkt && skb_has_frag_list(skb)) { 1239 printk("skb fraglist:\n"); 1240 skb_walk_frags(skb, list_skb) 1241 skb_dump(level, list_skb, true); 1242 } 1243 } 1244 EXPORT_SYMBOL(skb_dump); 1245 1246 /** 1247 * skb_tx_error - report an sk_buff xmit error 1248 * @skb: buffer that triggered an error 1249 * 1250 * Report xmit error if a device callback is tracking this skb. 1251 * skb must be freed afterwards. 1252 */ 1253 void skb_tx_error(struct sk_buff *skb) 1254 { 1255 if (skb) { 1256 skb_zcopy_downgrade_managed(skb); 1257 skb_zcopy_clear(skb, true); 1258 } 1259 } 1260 EXPORT_SYMBOL(skb_tx_error); 1261 1262 #ifdef CONFIG_TRACEPOINTS 1263 /** 1264 * consume_skb - free an skbuff 1265 * @skb: buffer to free 1266 * 1267 * Drop a ref to the buffer and free it if the usage count has hit zero 1268 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 1269 * is being dropped after a failure and notes that 1270 */ 1271 void consume_skb(struct sk_buff *skb) 1272 { 1273 if (!skb_unref(skb)) 1274 return; 1275 1276 trace_consume_skb(skb, __builtin_return_address(0)); 1277 __kfree_skb(skb); 1278 } 1279 EXPORT_SYMBOL(consume_skb); 1280 #endif 1281 1282 /** 1283 * __consume_stateless_skb - free an skbuff, assuming it is stateless 1284 * @skb: buffer to free 1285 * 1286 * Alike consume_skb(), but this variant assumes that this is the last 1287 * skb reference and all the head states have been already dropped 1288 */ 1289 void __consume_stateless_skb(struct sk_buff *skb) 1290 { 1291 trace_consume_skb(skb, __builtin_return_address(0)); 1292 skb_release_data(skb, SKB_CONSUMED, false); 1293 kfree_skbmem(skb); 1294 } 1295 1296 static void napi_skb_cache_put(struct sk_buff *skb) 1297 { 1298 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 1299 u32 i; 1300 1301 kasan_poison_object_data(skbuff_cache, skb); 1302 nc->skb_cache[nc->skb_count++] = skb; 1303 1304 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 1305 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) 1306 kasan_unpoison_object_data(skbuff_cache, 1307 nc->skb_cache[i]); 1308 1309 kmem_cache_free_bulk(skbuff_cache, NAPI_SKB_CACHE_HALF, 1310 nc->skb_cache + NAPI_SKB_CACHE_HALF); 1311 nc->skb_count = NAPI_SKB_CACHE_HALF; 1312 } 1313 } 1314 1315 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason) 1316 { 1317 skb_release_all(skb, reason, true); 1318 napi_skb_cache_put(skb); 1319 } 1320 1321 void napi_skb_free_stolen_head(struct sk_buff *skb) 1322 { 1323 if (unlikely(skb->slow_gro)) { 1324 nf_reset_ct(skb); 1325 skb_dst_drop(skb); 1326 skb_ext_put(skb); 1327 skb_orphan(skb); 1328 skb->slow_gro = 0; 1329 } 1330 napi_skb_cache_put(skb); 1331 } 1332 1333 void napi_consume_skb(struct sk_buff *skb, int budget) 1334 { 1335 /* Zero budget indicate non-NAPI context called us, like netpoll */ 1336 if (unlikely(!budget)) { 1337 dev_consume_skb_any(skb); 1338 return; 1339 } 1340 1341 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 1342 1343 if (!skb_unref(skb)) 1344 return; 1345 1346 /* if reaching here SKB is ready to free */ 1347 trace_consume_skb(skb, __builtin_return_address(0)); 1348 1349 /* if SKB is a clone, don't handle this case */ 1350 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 1351 __kfree_skb(skb); 1352 return; 1353 } 1354 1355 skb_release_all(skb, SKB_CONSUMED, !!budget); 1356 napi_skb_cache_put(skb); 1357 } 1358 EXPORT_SYMBOL(napi_consume_skb); 1359 1360 /* Make sure a field is contained by headers group */ 1361 #define CHECK_SKB_FIELD(field) \ 1362 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \ 1363 offsetof(struct sk_buff, headers.field)); \ 1364 1365 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 1366 { 1367 new->tstamp = old->tstamp; 1368 /* We do not copy old->sk */ 1369 new->dev = old->dev; 1370 memcpy(new->cb, old->cb, sizeof(old->cb)); 1371 skb_dst_copy(new, old); 1372 __skb_ext_copy(new, old); 1373 __nf_copy(new, old, false); 1374 1375 /* Note : this field could be in the headers group. 1376 * It is not yet because we do not want to have a 16 bit hole 1377 */ 1378 new->queue_mapping = old->queue_mapping; 1379 1380 memcpy(&new->headers, &old->headers, sizeof(new->headers)); 1381 CHECK_SKB_FIELD(protocol); 1382 CHECK_SKB_FIELD(csum); 1383 CHECK_SKB_FIELD(hash); 1384 CHECK_SKB_FIELD(priority); 1385 CHECK_SKB_FIELD(skb_iif); 1386 CHECK_SKB_FIELD(vlan_proto); 1387 CHECK_SKB_FIELD(vlan_tci); 1388 CHECK_SKB_FIELD(transport_header); 1389 CHECK_SKB_FIELD(network_header); 1390 CHECK_SKB_FIELD(mac_header); 1391 CHECK_SKB_FIELD(inner_protocol); 1392 CHECK_SKB_FIELD(inner_transport_header); 1393 CHECK_SKB_FIELD(inner_network_header); 1394 CHECK_SKB_FIELD(inner_mac_header); 1395 CHECK_SKB_FIELD(mark); 1396 #ifdef CONFIG_NETWORK_SECMARK 1397 CHECK_SKB_FIELD(secmark); 1398 #endif 1399 #ifdef CONFIG_NET_RX_BUSY_POLL 1400 CHECK_SKB_FIELD(napi_id); 1401 #endif 1402 CHECK_SKB_FIELD(alloc_cpu); 1403 #ifdef CONFIG_XPS 1404 CHECK_SKB_FIELD(sender_cpu); 1405 #endif 1406 #ifdef CONFIG_NET_SCHED 1407 CHECK_SKB_FIELD(tc_index); 1408 #endif 1409 1410 } 1411 1412 /* 1413 * You should not add any new code to this function. Add it to 1414 * __copy_skb_header above instead. 1415 */ 1416 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 1417 { 1418 #define C(x) n->x = skb->x 1419 1420 n->next = n->prev = NULL; 1421 n->sk = NULL; 1422 __copy_skb_header(n, skb); 1423 1424 C(len); 1425 C(data_len); 1426 C(mac_len); 1427 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 1428 n->cloned = 1; 1429 n->nohdr = 0; 1430 n->peeked = 0; 1431 C(pfmemalloc); 1432 C(pp_recycle); 1433 n->destructor = NULL; 1434 C(tail); 1435 C(end); 1436 C(head); 1437 C(head_frag); 1438 C(data); 1439 C(truesize); 1440 refcount_set(&n->users, 1); 1441 1442 atomic_inc(&(skb_shinfo(skb)->dataref)); 1443 skb->cloned = 1; 1444 1445 return n; 1446 #undef C 1447 } 1448 1449 /** 1450 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg 1451 * @first: first sk_buff of the msg 1452 */ 1453 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first) 1454 { 1455 struct sk_buff *n; 1456 1457 n = alloc_skb(0, GFP_ATOMIC); 1458 if (!n) 1459 return NULL; 1460 1461 n->len = first->len; 1462 n->data_len = first->len; 1463 n->truesize = first->truesize; 1464 1465 skb_shinfo(n)->frag_list = first; 1466 1467 __copy_skb_header(n, first); 1468 n->destructor = NULL; 1469 1470 return n; 1471 } 1472 EXPORT_SYMBOL_GPL(alloc_skb_for_msg); 1473 1474 /** 1475 * skb_morph - morph one skb into another 1476 * @dst: the skb to receive the contents 1477 * @src: the skb to supply the contents 1478 * 1479 * This is identical to skb_clone except that the target skb is 1480 * supplied by the user. 1481 * 1482 * The target skb is returned upon exit. 1483 */ 1484 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 1485 { 1486 skb_release_all(dst, SKB_CONSUMED, false); 1487 return __skb_clone(dst, src); 1488 } 1489 EXPORT_SYMBOL_GPL(skb_morph); 1490 1491 int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 1492 { 1493 unsigned long max_pg, num_pg, new_pg, old_pg, rlim; 1494 struct user_struct *user; 1495 1496 if (capable(CAP_IPC_LOCK) || !size) 1497 return 0; 1498 1499 rlim = rlimit(RLIMIT_MEMLOCK); 1500 if (rlim == RLIM_INFINITY) 1501 return 0; 1502 1503 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 1504 max_pg = rlim >> PAGE_SHIFT; 1505 user = mmp->user ? : current_user(); 1506 1507 old_pg = atomic_long_read(&user->locked_vm); 1508 do { 1509 new_pg = old_pg + num_pg; 1510 if (new_pg > max_pg) 1511 return -ENOBUFS; 1512 } while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg)); 1513 1514 if (!mmp->user) { 1515 mmp->user = get_uid(user); 1516 mmp->num_pg = num_pg; 1517 } else { 1518 mmp->num_pg += num_pg; 1519 } 1520 1521 return 0; 1522 } 1523 EXPORT_SYMBOL_GPL(mm_account_pinned_pages); 1524 1525 void mm_unaccount_pinned_pages(struct mmpin *mmp) 1526 { 1527 if (mmp->user) { 1528 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 1529 free_uid(mmp->user); 1530 } 1531 } 1532 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); 1533 1534 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) 1535 { 1536 struct ubuf_info_msgzc *uarg; 1537 struct sk_buff *skb; 1538 1539 WARN_ON_ONCE(!in_task()); 1540 1541 skb = sock_omalloc(sk, 0, GFP_KERNEL); 1542 if (!skb) 1543 return NULL; 1544 1545 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 1546 uarg = (void *)skb->cb; 1547 uarg->mmp.user = NULL; 1548 1549 if (mm_account_pinned_pages(&uarg->mmp, size)) { 1550 kfree_skb(skb); 1551 return NULL; 1552 } 1553 1554 uarg->ubuf.callback = msg_zerocopy_callback; 1555 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 1556 uarg->len = 1; 1557 uarg->bytelen = size; 1558 uarg->zerocopy = 1; 1559 uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; 1560 refcount_set(&uarg->ubuf.refcnt, 1); 1561 sock_hold(sk); 1562 1563 return &uarg->ubuf; 1564 } 1565 1566 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg) 1567 { 1568 return container_of((void *)uarg, struct sk_buff, cb); 1569 } 1570 1571 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, 1572 struct ubuf_info *uarg) 1573 { 1574 if (uarg) { 1575 struct ubuf_info_msgzc *uarg_zc; 1576 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 1577 u32 bytelen, next; 1578 1579 /* there might be non MSG_ZEROCOPY users */ 1580 if (uarg->callback != msg_zerocopy_callback) 1581 return NULL; 1582 1583 /* realloc only when socket is locked (TCP, UDP cork), 1584 * so uarg->len and sk_zckey access is serialized 1585 */ 1586 if (!sock_owned_by_user(sk)) { 1587 WARN_ON_ONCE(1); 1588 return NULL; 1589 } 1590 1591 uarg_zc = uarg_to_msgzc(uarg); 1592 bytelen = uarg_zc->bytelen + size; 1593 if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) { 1594 /* TCP can create new skb to attach new uarg */ 1595 if (sk->sk_type == SOCK_STREAM) 1596 goto new_alloc; 1597 return NULL; 1598 } 1599 1600 next = (u32)atomic_read(&sk->sk_zckey); 1601 if ((u32)(uarg_zc->id + uarg_zc->len) == next) { 1602 if (mm_account_pinned_pages(&uarg_zc->mmp, size)) 1603 return NULL; 1604 uarg_zc->len++; 1605 uarg_zc->bytelen = bytelen; 1606 atomic_set(&sk->sk_zckey, ++next); 1607 1608 /* no extra ref when appending to datagram (MSG_MORE) */ 1609 if (sk->sk_type == SOCK_STREAM) 1610 net_zcopy_get(uarg); 1611 1612 return uarg; 1613 } 1614 } 1615 1616 new_alloc: 1617 return msg_zerocopy_alloc(sk, size); 1618 } 1619 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); 1620 1621 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1622 { 1623 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1624 u32 old_lo, old_hi; 1625 u64 sum_len; 1626 1627 old_lo = serr->ee.ee_info; 1628 old_hi = serr->ee.ee_data; 1629 sum_len = old_hi - old_lo + 1ULL + len; 1630 1631 if (sum_len >= (1ULL << 32)) 1632 return false; 1633 1634 if (lo != old_hi + 1) 1635 return false; 1636 1637 serr->ee.ee_data += len; 1638 return true; 1639 } 1640 1641 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg) 1642 { 1643 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1644 struct sock_exterr_skb *serr; 1645 struct sock *sk = skb->sk; 1646 struct sk_buff_head *q; 1647 unsigned long flags; 1648 bool is_zerocopy; 1649 u32 lo, hi; 1650 u16 len; 1651 1652 mm_unaccount_pinned_pages(&uarg->mmp); 1653 1654 /* if !len, there was only 1 call, and it was aborted 1655 * so do not queue a completion notification 1656 */ 1657 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1658 goto release; 1659 1660 len = uarg->len; 1661 lo = uarg->id; 1662 hi = uarg->id + len - 1; 1663 is_zerocopy = uarg->zerocopy; 1664 1665 serr = SKB_EXT_ERR(skb); 1666 memset(serr, 0, sizeof(*serr)); 1667 serr->ee.ee_errno = 0; 1668 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1669 serr->ee.ee_data = hi; 1670 serr->ee.ee_info = lo; 1671 if (!is_zerocopy) 1672 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1673 1674 q = &sk->sk_error_queue; 1675 spin_lock_irqsave(&q->lock, flags); 1676 tail = skb_peek_tail(q); 1677 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1678 !skb_zerocopy_notify_extend(tail, lo, len)) { 1679 __skb_queue_tail(q, skb); 1680 skb = NULL; 1681 } 1682 spin_unlock_irqrestore(&q->lock, flags); 1683 1684 sk_error_report(sk); 1685 1686 release: 1687 consume_skb(skb); 1688 sock_put(sk); 1689 } 1690 1691 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, 1692 bool success) 1693 { 1694 struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg); 1695 1696 uarg_zc->zerocopy = uarg_zc->zerocopy & success; 1697 1698 if (refcount_dec_and_test(&uarg->refcnt)) 1699 __msg_zerocopy_callback(uarg_zc); 1700 } 1701 EXPORT_SYMBOL_GPL(msg_zerocopy_callback); 1702 1703 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1704 { 1705 struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk; 1706 1707 atomic_dec(&sk->sk_zckey); 1708 uarg_to_msgzc(uarg)->len--; 1709 1710 if (have_uref) 1711 msg_zerocopy_callback(NULL, uarg, true); 1712 } 1713 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); 1714 1715 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1716 struct msghdr *msg, int len, 1717 struct ubuf_info *uarg) 1718 { 1719 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1720 int err, orig_len = skb->len; 1721 1722 /* An skb can only point to one uarg. This edge case happens when 1723 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc. 1724 */ 1725 if (orig_uarg && uarg != orig_uarg) 1726 return -EEXIST; 1727 1728 err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len); 1729 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1730 struct sock *save_sk = skb->sk; 1731 1732 /* Streams do not free skb on error. Reset to prev state. */ 1733 iov_iter_revert(&msg->msg_iter, skb->len - orig_len); 1734 skb->sk = sk; 1735 ___pskb_trim(skb, orig_len); 1736 skb->sk = save_sk; 1737 return err; 1738 } 1739 1740 skb_zcopy_set(skb, uarg, NULL); 1741 return skb->len - orig_len; 1742 } 1743 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1744 1745 void __skb_zcopy_downgrade_managed(struct sk_buff *skb) 1746 { 1747 int i; 1748 1749 skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS; 1750 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1751 skb_frag_ref(skb, i); 1752 } 1753 EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed); 1754 1755 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1756 gfp_t gfp_mask) 1757 { 1758 if (skb_zcopy(orig)) { 1759 if (skb_zcopy(nskb)) { 1760 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1761 if (!gfp_mask) { 1762 WARN_ON_ONCE(1); 1763 return -ENOMEM; 1764 } 1765 if (skb_uarg(nskb) == skb_uarg(orig)) 1766 return 0; 1767 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1768 return -EIO; 1769 } 1770 skb_zcopy_set(nskb, skb_uarg(orig), NULL); 1771 } 1772 return 0; 1773 } 1774 1775 /** 1776 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1777 * @skb: the skb to modify 1778 * @gfp_mask: allocation priority 1779 * 1780 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. 1781 * It will copy all frags into kernel and drop the reference 1782 * to userspace pages. 1783 * 1784 * If this function is called from an interrupt gfp_mask() must be 1785 * %GFP_ATOMIC. 1786 * 1787 * Returns 0 on success or a negative error code on failure 1788 * to allocate kernel memory to copy to. 1789 */ 1790 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 1791 { 1792 int num_frags = skb_shinfo(skb)->nr_frags; 1793 struct page *page, *head = NULL; 1794 int i, order, psize, new_frags; 1795 u32 d_off; 1796 1797 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 1798 return -EINVAL; 1799 1800 if (!num_frags) 1801 goto release; 1802 1803 /* We might have to allocate high order pages, so compute what minimum 1804 * page order is needed. 1805 */ 1806 order = 0; 1807 while ((PAGE_SIZE << order) * MAX_SKB_FRAGS < __skb_pagelen(skb)) 1808 order++; 1809 psize = (PAGE_SIZE << order); 1810 1811 new_frags = (__skb_pagelen(skb) + psize - 1) >> (PAGE_SHIFT + order); 1812 for (i = 0; i < new_frags; i++) { 1813 page = alloc_pages(gfp_mask | __GFP_COMP, order); 1814 if (!page) { 1815 while (head) { 1816 struct page *next = (struct page *)page_private(head); 1817 put_page(head); 1818 head = next; 1819 } 1820 return -ENOMEM; 1821 } 1822 set_page_private(page, (unsigned long)head); 1823 head = page; 1824 } 1825 1826 page = head; 1827 d_off = 0; 1828 for (i = 0; i < num_frags; i++) { 1829 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1830 u32 p_off, p_len, copied; 1831 struct page *p; 1832 u8 *vaddr; 1833 1834 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f), 1835 p, p_off, p_len, copied) { 1836 u32 copy, done = 0; 1837 vaddr = kmap_atomic(p); 1838 1839 while (done < p_len) { 1840 if (d_off == psize) { 1841 d_off = 0; 1842 page = (struct page *)page_private(page); 1843 } 1844 copy = min_t(u32, psize - d_off, p_len - done); 1845 memcpy(page_address(page) + d_off, 1846 vaddr + p_off + done, copy); 1847 done += copy; 1848 d_off += copy; 1849 } 1850 kunmap_atomic(vaddr); 1851 } 1852 } 1853 1854 /* skb frags release userspace buffers */ 1855 for (i = 0; i < num_frags; i++) 1856 skb_frag_unref(skb, i); 1857 1858 /* skb frags point to kernel buffers */ 1859 for (i = 0; i < new_frags - 1; i++) { 1860 __skb_fill_page_desc(skb, i, head, 0, psize); 1861 head = (struct page *)page_private(head); 1862 } 1863 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); 1864 skb_shinfo(skb)->nr_frags = new_frags; 1865 1866 release: 1867 skb_zcopy_clear(skb, false); 1868 return 0; 1869 } 1870 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 1871 1872 /** 1873 * skb_clone - duplicate an sk_buff 1874 * @skb: buffer to clone 1875 * @gfp_mask: allocation priority 1876 * 1877 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 1878 * copies share the same packet data but not structure. The new 1879 * buffer has a reference count of 1. If the allocation fails the 1880 * function returns %NULL otherwise the new buffer is returned. 1881 * 1882 * If this function is called from an interrupt gfp_mask() must be 1883 * %GFP_ATOMIC. 1884 */ 1885 1886 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 1887 { 1888 struct sk_buff_fclones *fclones = container_of(skb, 1889 struct sk_buff_fclones, 1890 skb1); 1891 struct sk_buff *n; 1892 1893 if (skb_orphan_frags(skb, gfp_mask)) 1894 return NULL; 1895 1896 if (skb->fclone == SKB_FCLONE_ORIG && 1897 refcount_read(&fclones->fclone_ref) == 1) { 1898 n = &fclones->skb2; 1899 refcount_set(&fclones->fclone_ref, 2); 1900 n->fclone = SKB_FCLONE_CLONE; 1901 } else { 1902 if (skb_pfmemalloc(skb)) 1903 gfp_mask |= __GFP_MEMALLOC; 1904 1905 n = kmem_cache_alloc(skbuff_cache, gfp_mask); 1906 if (!n) 1907 return NULL; 1908 1909 n->fclone = SKB_FCLONE_UNAVAILABLE; 1910 } 1911 1912 return __skb_clone(n, skb); 1913 } 1914 EXPORT_SYMBOL(skb_clone); 1915 1916 void skb_headers_offset_update(struct sk_buff *skb, int off) 1917 { 1918 /* Only adjust this if it actually is csum_start rather than csum */ 1919 if (skb->ip_summed == CHECKSUM_PARTIAL) 1920 skb->csum_start += off; 1921 /* {transport,network,mac}_header and tail are relative to skb->head */ 1922 skb->transport_header += off; 1923 skb->network_header += off; 1924 if (skb_mac_header_was_set(skb)) 1925 skb->mac_header += off; 1926 skb->inner_transport_header += off; 1927 skb->inner_network_header += off; 1928 skb->inner_mac_header += off; 1929 } 1930 EXPORT_SYMBOL(skb_headers_offset_update); 1931 1932 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) 1933 { 1934 __copy_skb_header(new, old); 1935 1936 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 1937 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 1938 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 1939 } 1940 EXPORT_SYMBOL(skb_copy_header); 1941 1942 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 1943 { 1944 if (skb_pfmemalloc(skb)) 1945 return SKB_ALLOC_RX; 1946 return 0; 1947 } 1948 1949 /** 1950 * skb_copy - create private copy of an sk_buff 1951 * @skb: buffer to copy 1952 * @gfp_mask: allocation priority 1953 * 1954 * Make a copy of both an &sk_buff and its data. This is used when the 1955 * caller wishes to modify the data and needs a private copy of the 1956 * data to alter. Returns %NULL on failure or the pointer to the buffer 1957 * on success. The returned buffer has a reference count of 1. 1958 * 1959 * As by-product this function converts non-linear &sk_buff to linear 1960 * one, so that &sk_buff becomes completely private and caller is allowed 1961 * to modify all the data of returned buffer. This means that this 1962 * function is not recommended for use in circumstances when only 1963 * header is going to be modified. Use pskb_copy() instead. 1964 */ 1965 1966 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 1967 { 1968 int headerlen = skb_headroom(skb); 1969 unsigned int size = skb_end_offset(skb) + skb->data_len; 1970 struct sk_buff *n = __alloc_skb(size, gfp_mask, 1971 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 1972 1973 if (!n) 1974 return NULL; 1975 1976 /* Set the data pointer */ 1977 skb_reserve(n, headerlen); 1978 /* Set the tail pointer and length */ 1979 skb_put(n, skb->len); 1980 1981 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); 1982 1983 skb_copy_header(n, skb); 1984 return n; 1985 } 1986 EXPORT_SYMBOL(skb_copy); 1987 1988 /** 1989 * __pskb_copy_fclone - create copy of an sk_buff with private head. 1990 * @skb: buffer to copy 1991 * @headroom: headroom of new skb 1992 * @gfp_mask: allocation priority 1993 * @fclone: if true allocate the copy of the skb from the fclone 1994 * cache instead of the head cache; it is recommended to set this 1995 * to true for the cases where the copy will likely be cloned 1996 * 1997 * Make a copy of both an &sk_buff and part of its data, located 1998 * in header. Fragmented data remain shared. This is used when 1999 * the caller wishes to modify only header of &sk_buff and needs 2000 * private copy of the header to alter. Returns %NULL on failure 2001 * or the pointer to the buffer on success. 2002 * The returned buffer has a reference count of 1. 2003 */ 2004 2005 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 2006 gfp_t gfp_mask, bool fclone) 2007 { 2008 unsigned int size = skb_headlen(skb) + headroom; 2009 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 2010 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 2011 2012 if (!n) 2013 goto out; 2014 2015 /* Set the data pointer */ 2016 skb_reserve(n, headroom); 2017 /* Set the tail pointer and length */ 2018 skb_put(n, skb_headlen(skb)); 2019 /* Copy the bytes */ 2020 skb_copy_from_linear_data(skb, n->data, n->len); 2021 2022 n->truesize += skb->data_len; 2023 n->data_len = skb->data_len; 2024 n->len = skb->len; 2025 2026 if (skb_shinfo(skb)->nr_frags) { 2027 int i; 2028 2029 if (skb_orphan_frags(skb, gfp_mask) || 2030 skb_zerocopy_clone(n, skb, gfp_mask)) { 2031 kfree_skb(n); 2032 n = NULL; 2033 goto out; 2034 } 2035 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2036 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 2037 skb_frag_ref(skb, i); 2038 } 2039 skb_shinfo(n)->nr_frags = i; 2040 } 2041 2042 if (skb_has_frag_list(skb)) { 2043 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 2044 skb_clone_fraglist(n); 2045 } 2046 2047 skb_copy_header(n, skb); 2048 out: 2049 return n; 2050 } 2051 EXPORT_SYMBOL(__pskb_copy_fclone); 2052 2053 /** 2054 * pskb_expand_head - reallocate header of &sk_buff 2055 * @skb: buffer to reallocate 2056 * @nhead: room to add at head 2057 * @ntail: room to add at tail 2058 * @gfp_mask: allocation priority 2059 * 2060 * Expands (or creates identical copy, if @nhead and @ntail are zero) 2061 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 2062 * reference count of 1. Returns zero in the case of success or error, 2063 * if expansion failed. In the last case, &sk_buff is not changed. 2064 * 2065 * All the pointers pointing into skb header may change and must be 2066 * reloaded after call to this function. 2067 */ 2068 2069 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 2070 gfp_t gfp_mask) 2071 { 2072 unsigned int osize = skb_end_offset(skb); 2073 unsigned int size = osize + nhead + ntail; 2074 long off; 2075 u8 *data; 2076 int i; 2077 2078 BUG_ON(nhead < 0); 2079 2080 BUG_ON(skb_shared(skb)); 2081 2082 skb_zcopy_downgrade_managed(skb); 2083 2084 if (skb_pfmemalloc(skb)) 2085 gfp_mask |= __GFP_MEMALLOC; 2086 2087 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 2088 if (!data) 2089 goto nodata; 2090 size = SKB_WITH_OVERHEAD(size); 2091 2092 /* Copy only real data... and, alas, header. This should be 2093 * optimized for the cases when header is void. 2094 */ 2095 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 2096 2097 memcpy((struct skb_shared_info *)(data + size), 2098 skb_shinfo(skb), 2099 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 2100 2101 /* 2102 * if shinfo is shared we must drop the old head gracefully, but if it 2103 * is not we can just drop the old head and let the existing refcount 2104 * be since all we did is relocate the values 2105 */ 2106 if (skb_cloned(skb)) { 2107 if (skb_orphan_frags(skb, gfp_mask)) 2108 goto nofrags; 2109 if (skb_zcopy(skb)) 2110 refcount_inc(&skb_uarg(skb)->refcnt); 2111 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2112 skb_frag_ref(skb, i); 2113 2114 if (skb_has_frag_list(skb)) 2115 skb_clone_fraglist(skb); 2116 2117 skb_release_data(skb, SKB_CONSUMED, false); 2118 } else { 2119 skb_free_head(skb, false); 2120 } 2121 off = (data + nhead) - skb->head; 2122 2123 skb->head = data; 2124 skb->head_frag = 0; 2125 skb->data += off; 2126 2127 skb_set_end_offset(skb, size); 2128 #ifdef NET_SKBUFF_DATA_USES_OFFSET 2129 off = nhead; 2130 #endif 2131 skb->tail += off; 2132 skb_headers_offset_update(skb, nhead); 2133 skb->cloned = 0; 2134 skb->hdr_len = 0; 2135 skb->nohdr = 0; 2136 atomic_set(&skb_shinfo(skb)->dataref, 1); 2137 2138 skb_metadata_clear(skb); 2139 2140 /* It is not generally safe to change skb->truesize. 2141 * For the moment, we really care of rx path, or 2142 * when skb is orphaned (not attached to a socket). 2143 */ 2144 if (!skb->sk || skb->destructor == sock_edemux) 2145 skb->truesize += size - osize; 2146 2147 return 0; 2148 2149 nofrags: 2150 skb_kfree_head(data, size); 2151 nodata: 2152 return -ENOMEM; 2153 } 2154 EXPORT_SYMBOL(pskb_expand_head); 2155 2156 /* Make private copy of skb with writable head and some headroom */ 2157 2158 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 2159 { 2160 struct sk_buff *skb2; 2161 int delta = headroom - skb_headroom(skb); 2162 2163 if (delta <= 0) 2164 skb2 = pskb_copy(skb, GFP_ATOMIC); 2165 else { 2166 skb2 = skb_clone(skb, GFP_ATOMIC); 2167 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 2168 GFP_ATOMIC)) { 2169 kfree_skb(skb2); 2170 skb2 = NULL; 2171 } 2172 } 2173 return skb2; 2174 } 2175 EXPORT_SYMBOL(skb_realloc_headroom); 2176 2177 /* Note: We plan to rework this in linux-6.4 */ 2178 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) 2179 { 2180 unsigned int saved_end_offset, saved_truesize; 2181 struct skb_shared_info *shinfo; 2182 int res; 2183 2184 saved_end_offset = skb_end_offset(skb); 2185 saved_truesize = skb->truesize; 2186 2187 res = pskb_expand_head(skb, 0, 0, pri); 2188 if (res) 2189 return res; 2190 2191 skb->truesize = saved_truesize; 2192 2193 if (likely(skb_end_offset(skb) == saved_end_offset)) 2194 return 0; 2195 2196 /* We can not change skb->end if the original or new value 2197 * is SKB_SMALL_HEAD_HEADROOM, as it might break skb_kfree_head(). 2198 */ 2199 if (saved_end_offset == SKB_SMALL_HEAD_HEADROOM || 2200 skb_end_offset(skb) == SKB_SMALL_HEAD_HEADROOM) { 2201 /* We think this path should not be taken. 2202 * Add a temporary trace to warn us just in case. 2203 */ 2204 pr_err_once("__skb_unclone_keeptruesize() skb_end_offset() %u -> %u\n", 2205 saved_end_offset, skb_end_offset(skb)); 2206 WARN_ON_ONCE(1); 2207 return 0; 2208 } 2209 2210 shinfo = skb_shinfo(skb); 2211 2212 /* We are about to change back skb->end, 2213 * we need to move skb_shinfo() to its new location. 2214 */ 2215 memmove(skb->head + saved_end_offset, 2216 shinfo, 2217 offsetof(struct skb_shared_info, frags[shinfo->nr_frags])); 2218 2219 skb_set_end_offset(skb, saved_end_offset); 2220 2221 return 0; 2222 } 2223 2224 /** 2225 * skb_expand_head - reallocate header of &sk_buff 2226 * @skb: buffer to reallocate 2227 * @headroom: needed headroom 2228 * 2229 * Unlike skb_realloc_headroom, this one does not allocate a new skb 2230 * if possible; copies skb->sk to new skb as needed 2231 * and frees original skb in case of failures. 2232 * 2233 * It expect increased headroom and generates warning otherwise. 2234 */ 2235 2236 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom) 2237 { 2238 int delta = headroom - skb_headroom(skb); 2239 int osize = skb_end_offset(skb); 2240 struct sock *sk = skb->sk; 2241 2242 if (WARN_ONCE(delta <= 0, 2243 "%s is expecting an increase in the headroom", __func__)) 2244 return skb; 2245 2246 delta = SKB_DATA_ALIGN(delta); 2247 /* pskb_expand_head() might crash, if skb is shared. */ 2248 if (skb_shared(skb) || !is_skb_wmem(skb)) { 2249 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); 2250 2251 if (unlikely(!nskb)) 2252 goto fail; 2253 2254 if (sk) 2255 skb_set_owner_w(nskb, sk); 2256 consume_skb(skb); 2257 skb = nskb; 2258 } 2259 if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC)) 2260 goto fail; 2261 2262 if (sk && is_skb_wmem(skb)) { 2263 delta = skb_end_offset(skb) - osize; 2264 refcount_add(delta, &sk->sk_wmem_alloc); 2265 skb->truesize += delta; 2266 } 2267 return skb; 2268 2269 fail: 2270 kfree_skb(skb); 2271 return NULL; 2272 } 2273 EXPORT_SYMBOL(skb_expand_head); 2274 2275 /** 2276 * skb_copy_expand - copy and expand sk_buff 2277 * @skb: buffer to copy 2278 * @newheadroom: new free bytes at head 2279 * @newtailroom: new free bytes at tail 2280 * @gfp_mask: allocation priority 2281 * 2282 * Make a copy of both an &sk_buff and its data and while doing so 2283 * allocate additional space. 2284 * 2285 * This is used when the caller wishes to modify the data and needs a 2286 * private copy of the data to alter as well as more space for new fields. 2287 * Returns %NULL on failure or the pointer to the buffer 2288 * on success. The returned buffer has a reference count of 1. 2289 * 2290 * You must pass %GFP_ATOMIC as the allocation priority if this function 2291 * is called from an interrupt. 2292 */ 2293 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 2294 int newheadroom, int newtailroom, 2295 gfp_t gfp_mask) 2296 { 2297 /* 2298 * Allocate the copy buffer 2299 */ 2300 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 2301 gfp_mask, skb_alloc_rx_flag(skb), 2302 NUMA_NO_NODE); 2303 int oldheadroom = skb_headroom(skb); 2304 int head_copy_len, head_copy_off; 2305 2306 if (!n) 2307 return NULL; 2308 2309 skb_reserve(n, newheadroom); 2310 2311 /* Set the tail pointer and length */ 2312 skb_put(n, skb->len); 2313 2314 head_copy_len = oldheadroom; 2315 head_copy_off = 0; 2316 if (newheadroom <= head_copy_len) 2317 head_copy_len = newheadroom; 2318 else 2319 head_copy_off = newheadroom - head_copy_len; 2320 2321 /* Copy the linear header and data. */ 2322 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 2323 skb->len + head_copy_len)); 2324 2325 skb_copy_header(n, skb); 2326 2327 skb_headers_offset_update(n, newheadroom - oldheadroom); 2328 2329 return n; 2330 } 2331 EXPORT_SYMBOL(skb_copy_expand); 2332 2333 /** 2334 * __skb_pad - zero pad the tail of an skb 2335 * @skb: buffer to pad 2336 * @pad: space to pad 2337 * @free_on_error: free buffer on error 2338 * 2339 * Ensure that a buffer is followed by a padding area that is zero 2340 * filled. Used by network drivers which may DMA or transfer data 2341 * beyond the buffer end onto the wire. 2342 * 2343 * May return error in out of memory cases. The skb is freed on error 2344 * if @free_on_error is true. 2345 */ 2346 2347 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 2348 { 2349 int err; 2350 int ntail; 2351 2352 /* If the skbuff is non linear tailroom is always zero.. */ 2353 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 2354 memset(skb->data+skb->len, 0, pad); 2355 return 0; 2356 } 2357 2358 ntail = skb->data_len + pad - (skb->end - skb->tail); 2359 if (likely(skb_cloned(skb) || ntail > 0)) { 2360 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 2361 if (unlikely(err)) 2362 goto free_skb; 2363 } 2364 2365 /* FIXME: The use of this function with non-linear skb's really needs 2366 * to be audited. 2367 */ 2368 err = skb_linearize(skb); 2369 if (unlikely(err)) 2370 goto free_skb; 2371 2372 memset(skb->data + skb->len, 0, pad); 2373 return 0; 2374 2375 free_skb: 2376 if (free_on_error) 2377 kfree_skb(skb); 2378 return err; 2379 } 2380 EXPORT_SYMBOL(__skb_pad); 2381 2382 /** 2383 * pskb_put - add data to the tail of a potentially fragmented buffer 2384 * @skb: start of the buffer to use 2385 * @tail: tail fragment of the buffer to use 2386 * @len: amount of data to add 2387 * 2388 * This function extends the used data area of the potentially 2389 * fragmented buffer. @tail must be the last fragment of @skb -- or 2390 * @skb itself. If this would exceed the total buffer size the kernel 2391 * will panic. A pointer to the first byte of the extra data is 2392 * returned. 2393 */ 2394 2395 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 2396 { 2397 if (tail != skb) { 2398 skb->data_len += len; 2399 skb->len += len; 2400 } 2401 return skb_put(tail, len); 2402 } 2403 EXPORT_SYMBOL_GPL(pskb_put); 2404 2405 /** 2406 * skb_put - add data to a buffer 2407 * @skb: buffer to use 2408 * @len: amount of data to add 2409 * 2410 * This function extends the used data area of the buffer. If this would 2411 * exceed the total buffer size the kernel will panic. A pointer to the 2412 * first byte of the extra data is returned. 2413 */ 2414 void *skb_put(struct sk_buff *skb, unsigned int len) 2415 { 2416 void *tmp = skb_tail_pointer(skb); 2417 SKB_LINEAR_ASSERT(skb); 2418 skb->tail += len; 2419 skb->len += len; 2420 if (unlikely(skb->tail > skb->end)) 2421 skb_over_panic(skb, len, __builtin_return_address(0)); 2422 return tmp; 2423 } 2424 EXPORT_SYMBOL(skb_put); 2425 2426 /** 2427 * skb_push - add data to the start of a buffer 2428 * @skb: buffer to use 2429 * @len: amount of data to add 2430 * 2431 * This function extends the used data area of the buffer at the buffer 2432 * start. If this would exceed the total buffer headroom the kernel will 2433 * panic. A pointer to the first byte of the extra data is returned. 2434 */ 2435 void *skb_push(struct sk_buff *skb, unsigned int len) 2436 { 2437 skb->data -= len; 2438 skb->len += len; 2439 if (unlikely(skb->data < skb->head)) 2440 skb_under_panic(skb, len, __builtin_return_address(0)); 2441 return skb->data; 2442 } 2443 EXPORT_SYMBOL(skb_push); 2444 2445 /** 2446 * skb_pull - remove data from the start of a buffer 2447 * @skb: buffer to use 2448 * @len: amount of data to remove 2449 * 2450 * This function removes data from the start of a buffer, returning 2451 * the memory to the headroom. A pointer to the next data in the buffer 2452 * is returned. Once the data has been pulled future pushes will overwrite 2453 * the old data. 2454 */ 2455 void *skb_pull(struct sk_buff *skb, unsigned int len) 2456 { 2457 return skb_pull_inline(skb, len); 2458 } 2459 EXPORT_SYMBOL(skb_pull); 2460 2461 /** 2462 * skb_pull_data - remove data from the start of a buffer returning its 2463 * original position. 2464 * @skb: buffer to use 2465 * @len: amount of data to remove 2466 * 2467 * This function removes data from the start of a buffer, returning 2468 * the memory to the headroom. A pointer to the original data in the buffer 2469 * is returned after checking if there is enough data to pull. Once the 2470 * data has been pulled future pushes will overwrite the old data. 2471 */ 2472 void *skb_pull_data(struct sk_buff *skb, size_t len) 2473 { 2474 void *data = skb->data; 2475 2476 if (skb->len < len) 2477 return NULL; 2478 2479 skb_pull(skb, len); 2480 2481 return data; 2482 } 2483 EXPORT_SYMBOL(skb_pull_data); 2484 2485 /** 2486 * skb_trim - remove end from a buffer 2487 * @skb: buffer to alter 2488 * @len: new length 2489 * 2490 * Cut the length of a buffer down by removing data from the tail. If 2491 * the buffer is already under the length specified it is not modified. 2492 * The skb must be linear. 2493 */ 2494 void skb_trim(struct sk_buff *skb, unsigned int len) 2495 { 2496 if (skb->len > len) 2497 __skb_trim(skb, len); 2498 } 2499 EXPORT_SYMBOL(skb_trim); 2500 2501 /* Trims skb to length len. It can change skb pointers. 2502 */ 2503 2504 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 2505 { 2506 struct sk_buff **fragp; 2507 struct sk_buff *frag; 2508 int offset = skb_headlen(skb); 2509 int nfrags = skb_shinfo(skb)->nr_frags; 2510 int i; 2511 int err; 2512 2513 if (skb_cloned(skb) && 2514 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 2515 return err; 2516 2517 i = 0; 2518 if (offset >= len) 2519 goto drop_pages; 2520 2521 for (; i < nfrags; i++) { 2522 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2523 2524 if (end < len) { 2525 offset = end; 2526 continue; 2527 } 2528 2529 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 2530 2531 drop_pages: 2532 skb_shinfo(skb)->nr_frags = i; 2533 2534 for (; i < nfrags; i++) 2535 skb_frag_unref(skb, i); 2536 2537 if (skb_has_frag_list(skb)) 2538 skb_drop_fraglist(skb); 2539 goto done; 2540 } 2541 2542 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 2543 fragp = &frag->next) { 2544 int end = offset + frag->len; 2545 2546 if (skb_shared(frag)) { 2547 struct sk_buff *nfrag; 2548 2549 nfrag = skb_clone(frag, GFP_ATOMIC); 2550 if (unlikely(!nfrag)) 2551 return -ENOMEM; 2552 2553 nfrag->next = frag->next; 2554 consume_skb(frag); 2555 frag = nfrag; 2556 *fragp = frag; 2557 } 2558 2559 if (end < len) { 2560 offset = end; 2561 continue; 2562 } 2563 2564 if (end > len && 2565 unlikely((err = pskb_trim(frag, len - offset)))) 2566 return err; 2567 2568 if (frag->next) 2569 skb_drop_list(&frag->next); 2570 break; 2571 } 2572 2573 done: 2574 if (len > skb_headlen(skb)) { 2575 skb->data_len -= skb->len - len; 2576 skb->len = len; 2577 } else { 2578 skb->len = len; 2579 skb->data_len = 0; 2580 skb_set_tail_pointer(skb, len); 2581 } 2582 2583 if (!skb->sk || skb->destructor == sock_edemux) 2584 skb_condense(skb); 2585 return 0; 2586 } 2587 EXPORT_SYMBOL(___pskb_trim); 2588 2589 /* Note : use pskb_trim_rcsum() instead of calling this directly 2590 */ 2591 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) 2592 { 2593 if (skb->ip_summed == CHECKSUM_COMPLETE) { 2594 int delta = skb->len - len; 2595 2596 skb->csum = csum_block_sub(skb->csum, 2597 skb_checksum(skb, len, delta, 0), 2598 len); 2599 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 2600 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; 2601 int offset = skb_checksum_start_offset(skb) + skb->csum_offset; 2602 2603 if (offset + sizeof(__sum16) > hdlen) 2604 return -EINVAL; 2605 } 2606 return __pskb_trim(skb, len); 2607 } 2608 EXPORT_SYMBOL(pskb_trim_rcsum_slow); 2609 2610 /** 2611 * __pskb_pull_tail - advance tail of skb header 2612 * @skb: buffer to reallocate 2613 * @delta: number of bytes to advance tail 2614 * 2615 * The function makes a sense only on a fragmented &sk_buff, 2616 * it expands header moving its tail forward and copying necessary 2617 * data from fragmented part. 2618 * 2619 * &sk_buff MUST have reference count of 1. 2620 * 2621 * Returns %NULL (and &sk_buff does not change) if pull failed 2622 * or value of new tail of skb in the case of success. 2623 * 2624 * All the pointers pointing into skb header may change and must be 2625 * reloaded after call to this function. 2626 */ 2627 2628 /* Moves tail of skb head forward, copying data from fragmented part, 2629 * when it is necessary. 2630 * 1. It may fail due to malloc failure. 2631 * 2. It may change skb pointers. 2632 * 2633 * It is pretty complicated. Luckily, it is called only in exceptional cases. 2634 */ 2635 void *__pskb_pull_tail(struct sk_buff *skb, int delta) 2636 { 2637 /* If skb has not enough free space at tail, get new one 2638 * plus 128 bytes for future expansions. If we have enough 2639 * room at tail, reallocate without expansion only if skb is cloned. 2640 */ 2641 int i, k, eat = (skb->tail + delta) - skb->end; 2642 2643 if (eat > 0 || skb_cloned(skb)) { 2644 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 2645 GFP_ATOMIC)) 2646 return NULL; 2647 } 2648 2649 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 2650 skb_tail_pointer(skb), delta)); 2651 2652 /* Optimization: no fragments, no reasons to preestimate 2653 * size of pulled pages. Superb. 2654 */ 2655 if (!skb_has_frag_list(skb)) 2656 goto pull_pages; 2657 2658 /* Estimate size of pulled pages. */ 2659 eat = delta; 2660 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2661 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2662 2663 if (size >= eat) 2664 goto pull_pages; 2665 eat -= size; 2666 } 2667 2668 /* If we need update frag list, we are in troubles. 2669 * Certainly, it is possible to add an offset to skb data, 2670 * but taking into account that pulling is expected to 2671 * be very rare operation, it is worth to fight against 2672 * further bloating skb head and crucify ourselves here instead. 2673 * Pure masohism, indeed. 8)8) 2674 */ 2675 if (eat) { 2676 struct sk_buff *list = skb_shinfo(skb)->frag_list; 2677 struct sk_buff *clone = NULL; 2678 struct sk_buff *insp = NULL; 2679 2680 do { 2681 if (list->len <= eat) { 2682 /* Eaten as whole. */ 2683 eat -= list->len; 2684 list = list->next; 2685 insp = list; 2686 } else { 2687 /* Eaten partially. */ 2688 if (skb_is_gso(skb) && !list->head_frag && 2689 skb_headlen(list)) 2690 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 2691 2692 if (skb_shared(list)) { 2693 /* Sucks! We need to fork list. :-( */ 2694 clone = skb_clone(list, GFP_ATOMIC); 2695 if (!clone) 2696 return NULL; 2697 insp = list->next; 2698 list = clone; 2699 } else { 2700 /* This may be pulled without 2701 * problems. */ 2702 insp = list; 2703 } 2704 if (!pskb_pull(list, eat)) { 2705 kfree_skb(clone); 2706 return NULL; 2707 } 2708 break; 2709 } 2710 } while (eat); 2711 2712 /* Free pulled out fragments. */ 2713 while ((list = skb_shinfo(skb)->frag_list) != insp) { 2714 skb_shinfo(skb)->frag_list = list->next; 2715 consume_skb(list); 2716 } 2717 /* And insert new clone at head. */ 2718 if (clone) { 2719 clone->next = list; 2720 skb_shinfo(skb)->frag_list = clone; 2721 } 2722 } 2723 /* Success! Now we may commit changes to skb data. */ 2724 2725 pull_pages: 2726 eat = delta; 2727 k = 0; 2728 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2729 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2730 2731 if (size <= eat) { 2732 skb_frag_unref(skb, i); 2733 eat -= size; 2734 } else { 2735 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2736 2737 *frag = skb_shinfo(skb)->frags[i]; 2738 if (eat) { 2739 skb_frag_off_add(frag, eat); 2740 skb_frag_size_sub(frag, eat); 2741 if (!i) 2742 goto end; 2743 eat = 0; 2744 } 2745 k++; 2746 } 2747 } 2748 skb_shinfo(skb)->nr_frags = k; 2749 2750 end: 2751 skb->tail += delta; 2752 skb->data_len -= delta; 2753 2754 if (!skb->data_len) 2755 skb_zcopy_clear(skb, false); 2756 2757 return skb_tail_pointer(skb); 2758 } 2759 EXPORT_SYMBOL(__pskb_pull_tail); 2760 2761 /** 2762 * skb_copy_bits - copy bits from skb to kernel buffer 2763 * @skb: source skb 2764 * @offset: offset in source 2765 * @to: destination buffer 2766 * @len: number of bytes to copy 2767 * 2768 * Copy the specified number of bytes from the source skb to the 2769 * destination buffer. 2770 * 2771 * CAUTION ! : 2772 * If its prototype is ever changed, 2773 * check arch/{*}/net/{*}.S files, 2774 * since it is called from BPF assembly code. 2775 */ 2776 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 2777 { 2778 int start = skb_headlen(skb); 2779 struct sk_buff *frag_iter; 2780 int i, copy; 2781 2782 if (offset > (int)skb->len - len) 2783 goto fault; 2784 2785 /* Copy header. */ 2786 if ((copy = start - offset) > 0) { 2787 if (copy > len) 2788 copy = len; 2789 skb_copy_from_linear_data_offset(skb, offset, to, copy); 2790 if ((len -= copy) == 0) 2791 return 0; 2792 offset += copy; 2793 to += copy; 2794 } 2795 2796 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2797 int end; 2798 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2799 2800 WARN_ON(start > offset + len); 2801 2802 end = start + skb_frag_size(f); 2803 if ((copy = end - offset) > 0) { 2804 u32 p_off, p_len, copied; 2805 struct page *p; 2806 u8 *vaddr; 2807 2808 if (copy > len) 2809 copy = len; 2810 2811 skb_frag_foreach_page(f, 2812 skb_frag_off(f) + offset - start, 2813 copy, p, p_off, p_len, copied) { 2814 vaddr = kmap_atomic(p); 2815 memcpy(to + copied, vaddr + p_off, p_len); 2816 kunmap_atomic(vaddr); 2817 } 2818 2819 if ((len -= copy) == 0) 2820 return 0; 2821 offset += copy; 2822 to += copy; 2823 } 2824 start = end; 2825 } 2826 2827 skb_walk_frags(skb, frag_iter) { 2828 int end; 2829 2830 WARN_ON(start > offset + len); 2831 2832 end = start + frag_iter->len; 2833 if ((copy = end - offset) > 0) { 2834 if (copy > len) 2835 copy = len; 2836 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 2837 goto fault; 2838 if ((len -= copy) == 0) 2839 return 0; 2840 offset += copy; 2841 to += copy; 2842 } 2843 start = end; 2844 } 2845 2846 if (!len) 2847 return 0; 2848 2849 fault: 2850 return -EFAULT; 2851 } 2852 EXPORT_SYMBOL(skb_copy_bits); 2853 2854 /* 2855 * Callback from splice_to_pipe(), if we need to release some pages 2856 * at the end of the spd in case we error'ed out in filling the pipe. 2857 */ 2858 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 2859 { 2860 put_page(spd->pages[i]); 2861 } 2862 2863 static struct page *linear_to_page(struct page *page, unsigned int *len, 2864 unsigned int *offset, 2865 struct sock *sk) 2866 { 2867 struct page_frag *pfrag = sk_page_frag(sk); 2868 2869 if (!sk_page_frag_refill(sk, pfrag)) 2870 return NULL; 2871 2872 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 2873 2874 memcpy(page_address(pfrag->page) + pfrag->offset, 2875 page_address(page) + *offset, *len); 2876 *offset = pfrag->offset; 2877 pfrag->offset += *len; 2878 2879 return pfrag->page; 2880 } 2881 2882 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 2883 struct page *page, 2884 unsigned int offset) 2885 { 2886 return spd->nr_pages && 2887 spd->pages[spd->nr_pages - 1] == page && 2888 (spd->partial[spd->nr_pages - 1].offset + 2889 spd->partial[spd->nr_pages - 1].len == offset); 2890 } 2891 2892 /* 2893 * Fill page/offset/length into spd, if it can hold more pages. 2894 */ 2895 static bool spd_fill_page(struct splice_pipe_desc *spd, 2896 struct pipe_inode_info *pipe, struct page *page, 2897 unsigned int *len, unsigned int offset, 2898 bool linear, 2899 struct sock *sk) 2900 { 2901 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 2902 return true; 2903 2904 if (linear) { 2905 page = linear_to_page(page, len, &offset, sk); 2906 if (!page) 2907 return true; 2908 } 2909 if (spd_can_coalesce(spd, page, offset)) { 2910 spd->partial[spd->nr_pages - 1].len += *len; 2911 return false; 2912 } 2913 get_page(page); 2914 spd->pages[spd->nr_pages] = page; 2915 spd->partial[spd->nr_pages].len = *len; 2916 spd->partial[spd->nr_pages].offset = offset; 2917 spd->nr_pages++; 2918 2919 return false; 2920 } 2921 2922 static bool __splice_segment(struct page *page, unsigned int poff, 2923 unsigned int plen, unsigned int *off, 2924 unsigned int *len, 2925 struct splice_pipe_desc *spd, bool linear, 2926 struct sock *sk, 2927 struct pipe_inode_info *pipe) 2928 { 2929 if (!*len) 2930 return true; 2931 2932 /* skip this segment if already processed */ 2933 if (*off >= plen) { 2934 *off -= plen; 2935 return false; 2936 } 2937 2938 /* ignore any bits we already processed */ 2939 poff += *off; 2940 plen -= *off; 2941 *off = 0; 2942 2943 do { 2944 unsigned int flen = min(*len, plen); 2945 2946 if (spd_fill_page(spd, pipe, page, &flen, poff, 2947 linear, sk)) 2948 return true; 2949 poff += flen; 2950 plen -= flen; 2951 *len -= flen; 2952 } while (*len && plen); 2953 2954 return false; 2955 } 2956 2957 /* 2958 * Map linear and fragment data from the skb to spd. It reports true if the 2959 * pipe is full or if we already spliced the requested length. 2960 */ 2961 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 2962 unsigned int *offset, unsigned int *len, 2963 struct splice_pipe_desc *spd, struct sock *sk) 2964 { 2965 int seg; 2966 struct sk_buff *iter; 2967 2968 /* map the linear part : 2969 * If skb->head_frag is set, this 'linear' part is backed by a 2970 * fragment, and if the head is not shared with any clones then 2971 * we can avoid a copy since we own the head portion of this page. 2972 */ 2973 if (__splice_segment(virt_to_page(skb->data), 2974 (unsigned long) skb->data & (PAGE_SIZE - 1), 2975 skb_headlen(skb), 2976 offset, len, spd, 2977 skb_head_is_locked(skb), 2978 sk, pipe)) 2979 return true; 2980 2981 /* 2982 * then map the fragments 2983 */ 2984 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 2985 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 2986 2987 if (__splice_segment(skb_frag_page(f), 2988 skb_frag_off(f), skb_frag_size(f), 2989 offset, len, spd, false, sk, pipe)) 2990 return true; 2991 } 2992 2993 skb_walk_frags(skb, iter) { 2994 if (*offset >= iter->len) { 2995 *offset -= iter->len; 2996 continue; 2997 } 2998 /* __skb_splice_bits() only fails if the output has no room 2999 * left, so no point in going over the frag_list for the error 3000 * case. 3001 */ 3002 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 3003 return true; 3004 } 3005 3006 return false; 3007 } 3008 3009 /* 3010 * Map data from the skb to a pipe. Should handle both the linear part, 3011 * the fragments, and the frag list. 3012 */ 3013 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 3014 struct pipe_inode_info *pipe, unsigned int tlen, 3015 unsigned int flags) 3016 { 3017 struct partial_page partial[MAX_SKB_FRAGS]; 3018 struct page *pages[MAX_SKB_FRAGS]; 3019 struct splice_pipe_desc spd = { 3020 .pages = pages, 3021 .partial = partial, 3022 .nr_pages_max = MAX_SKB_FRAGS, 3023 .ops = &nosteal_pipe_buf_ops, 3024 .spd_release = sock_spd_release, 3025 }; 3026 int ret = 0; 3027 3028 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 3029 3030 if (spd.nr_pages) 3031 ret = splice_to_pipe(pipe, &spd); 3032 3033 return ret; 3034 } 3035 EXPORT_SYMBOL_GPL(skb_splice_bits); 3036 3037 static int sendmsg_locked(struct sock *sk, struct msghdr *msg) 3038 { 3039 struct socket *sock = sk->sk_socket; 3040 size_t size = msg_data_left(msg); 3041 3042 if (!sock) 3043 return -EINVAL; 3044 3045 if (!sock->ops->sendmsg_locked) 3046 return sock_no_sendmsg_locked(sk, msg, size); 3047 3048 return sock->ops->sendmsg_locked(sk, msg, size); 3049 } 3050 3051 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg) 3052 { 3053 struct socket *sock = sk->sk_socket; 3054 3055 if (!sock) 3056 return -EINVAL; 3057 return sock_sendmsg(sock, msg); 3058 } 3059 3060 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg); 3061 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, 3062 int len, sendmsg_func sendmsg) 3063 { 3064 unsigned int orig_len = len; 3065 struct sk_buff *head = skb; 3066 unsigned short fragidx; 3067 int slen, ret; 3068 3069 do_frag_list: 3070 3071 /* Deal with head data */ 3072 while (offset < skb_headlen(skb) && len) { 3073 struct kvec kv; 3074 struct msghdr msg; 3075 3076 slen = min_t(int, len, skb_headlen(skb) - offset); 3077 kv.iov_base = skb->data + offset; 3078 kv.iov_len = slen; 3079 memset(&msg, 0, sizeof(msg)); 3080 msg.msg_flags = MSG_DONTWAIT; 3081 3082 iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen); 3083 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked, 3084 sendmsg_unlocked, sk, &msg); 3085 if (ret <= 0) 3086 goto error; 3087 3088 offset += ret; 3089 len -= ret; 3090 } 3091 3092 /* All the data was skb head? */ 3093 if (!len) 3094 goto out; 3095 3096 /* Make offset relative to start of frags */ 3097 offset -= skb_headlen(skb); 3098 3099 /* Find where we are in frag list */ 3100 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3101 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3102 3103 if (offset < skb_frag_size(frag)) 3104 break; 3105 3106 offset -= skb_frag_size(frag); 3107 } 3108 3109 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3110 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3111 3112 slen = min_t(size_t, len, skb_frag_size(frag) - offset); 3113 3114 while (slen) { 3115 struct bio_vec bvec; 3116 struct msghdr msg = { 3117 .msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT, 3118 }; 3119 3120 bvec_set_page(&bvec, skb_frag_page(frag), slen, 3121 skb_frag_off(frag) + offset); 3122 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, 3123 slen); 3124 3125 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked, 3126 sendmsg_unlocked, sk, &msg); 3127 if (ret <= 0) 3128 goto error; 3129 3130 len -= ret; 3131 offset += ret; 3132 slen -= ret; 3133 } 3134 3135 offset = 0; 3136 } 3137 3138 if (len) { 3139 /* Process any frag lists */ 3140 3141 if (skb == head) { 3142 if (skb_has_frag_list(skb)) { 3143 skb = skb_shinfo(skb)->frag_list; 3144 goto do_frag_list; 3145 } 3146 } else if (skb->next) { 3147 skb = skb->next; 3148 goto do_frag_list; 3149 } 3150 } 3151 3152 out: 3153 return orig_len - len; 3154 3155 error: 3156 return orig_len == len ? ret : orig_len - len; 3157 } 3158 3159 /* Send skb data on a socket. Socket must be locked. */ 3160 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 3161 int len) 3162 { 3163 return __skb_send_sock(sk, skb, offset, len, sendmsg_locked); 3164 } 3165 EXPORT_SYMBOL_GPL(skb_send_sock_locked); 3166 3167 /* Send skb data on a socket. Socket must be unlocked. */ 3168 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) 3169 { 3170 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked); 3171 } 3172 3173 /** 3174 * skb_store_bits - store bits from kernel buffer to skb 3175 * @skb: destination buffer 3176 * @offset: offset in destination 3177 * @from: source buffer 3178 * @len: number of bytes to copy 3179 * 3180 * Copy the specified number of bytes from the source buffer to the 3181 * destination skb. This function handles all the messy bits of 3182 * traversing fragment lists and such. 3183 */ 3184 3185 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 3186 { 3187 int start = skb_headlen(skb); 3188 struct sk_buff *frag_iter; 3189 int i, copy; 3190 3191 if (offset > (int)skb->len - len) 3192 goto fault; 3193 3194 if ((copy = start - offset) > 0) { 3195 if (copy > len) 3196 copy = len; 3197 skb_copy_to_linear_data_offset(skb, offset, from, copy); 3198 if ((len -= copy) == 0) 3199 return 0; 3200 offset += copy; 3201 from += copy; 3202 } 3203 3204 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3205 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3206 int end; 3207 3208 WARN_ON(start > offset + len); 3209 3210 end = start + skb_frag_size(frag); 3211 if ((copy = end - offset) > 0) { 3212 u32 p_off, p_len, copied; 3213 struct page *p; 3214 u8 *vaddr; 3215 3216 if (copy > len) 3217 copy = len; 3218 3219 skb_frag_foreach_page(frag, 3220 skb_frag_off(frag) + offset - start, 3221 copy, p, p_off, p_len, copied) { 3222 vaddr = kmap_atomic(p); 3223 memcpy(vaddr + p_off, from + copied, p_len); 3224 kunmap_atomic(vaddr); 3225 } 3226 3227 if ((len -= copy) == 0) 3228 return 0; 3229 offset += copy; 3230 from += copy; 3231 } 3232 start = end; 3233 } 3234 3235 skb_walk_frags(skb, frag_iter) { 3236 int end; 3237 3238 WARN_ON(start > offset + len); 3239 3240 end = start + frag_iter->len; 3241 if ((copy = end - offset) > 0) { 3242 if (copy > len) 3243 copy = len; 3244 if (skb_store_bits(frag_iter, offset - start, 3245 from, copy)) 3246 goto fault; 3247 if ((len -= copy) == 0) 3248 return 0; 3249 offset += copy; 3250 from += copy; 3251 } 3252 start = end; 3253 } 3254 if (!len) 3255 return 0; 3256 3257 fault: 3258 return -EFAULT; 3259 } 3260 EXPORT_SYMBOL(skb_store_bits); 3261 3262 /* Checksum skb data. */ 3263 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 3264 __wsum csum, const struct skb_checksum_ops *ops) 3265 { 3266 int start = skb_headlen(skb); 3267 int i, copy = start - offset; 3268 struct sk_buff *frag_iter; 3269 int pos = 0; 3270 3271 /* Checksum header. */ 3272 if (copy > 0) { 3273 if (copy > len) 3274 copy = len; 3275 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext, 3276 skb->data + offset, copy, csum); 3277 if ((len -= copy) == 0) 3278 return csum; 3279 offset += copy; 3280 pos = copy; 3281 } 3282 3283 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3284 int end; 3285 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3286 3287 WARN_ON(start > offset + len); 3288 3289 end = start + skb_frag_size(frag); 3290 if ((copy = end - offset) > 0) { 3291 u32 p_off, p_len, copied; 3292 struct page *p; 3293 __wsum csum2; 3294 u8 *vaddr; 3295 3296 if (copy > len) 3297 copy = len; 3298 3299 skb_frag_foreach_page(frag, 3300 skb_frag_off(frag) + offset - start, 3301 copy, p, p_off, p_len, copied) { 3302 vaddr = kmap_atomic(p); 3303 csum2 = INDIRECT_CALL_1(ops->update, 3304 csum_partial_ext, 3305 vaddr + p_off, p_len, 0); 3306 kunmap_atomic(vaddr); 3307 csum = INDIRECT_CALL_1(ops->combine, 3308 csum_block_add_ext, csum, 3309 csum2, pos, p_len); 3310 pos += p_len; 3311 } 3312 3313 if (!(len -= copy)) 3314 return csum; 3315 offset += copy; 3316 } 3317 start = end; 3318 } 3319 3320 skb_walk_frags(skb, frag_iter) { 3321 int end; 3322 3323 WARN_ON(start > offset + len); 3324 3325 end = start + frag_iter->len; 3326 if ((copy = end - offset) > 0) { 3327 __wsum csum2; 3328 if (copy > len) 3329 copy = len; 3330 csum2 = __skb_checksum(frag_iter, offset - start, 3331 copy, 0, ops); 3332 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext, 3333 csum, csum2, pos, copy); 3334 if ((len -= copy) == 0) 3335 return csum; 3336 offset += copy; 3337 pos += copy; 3338 } 3339 start = end; 3340 } 3341 BUG_ON(len); 3342 3343 return csum; 3344 } 3345 EXPORT_SYMBOL(__skb_checksum); 3346 3347 __wsum skb_checksum(const struct sk_buff *skb, int offset, 3348 int len, __wsum csum) 3349 { 3350 const struct skb_checksum_ops ops = { 3351 .update = csum_partial_ext, 3352 .combine = csum_block_add_ext, 3353 }; 3354 3355 return __skb_checksum(skb, offset, len, csum, &ops); 3356 } 3357 EXPORT_SYMBOL(skb_checksum); 3358 3359 /* Both of above in one bottle. */ 3360 3361 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 3362 u8 *to, int len) 3363 { 3364 int start = skb_headlen(skb); 3365 int i, copy = start - offset; 3366 struct sk_buff *frag_iter; 3367 int pos = 0; 3368 __wsum csum = 0; 3369 3370 /* Copy header. */ 3371 if (copy > 0) { 3372 if (copy > len) 3373 copy = len; 3374 csum = csum_partial_copy_nocheck(skb->data + offset, to, 3375 copy); 3376 if ((len -= copy) == 0) 3377 return csum; 3378 offset += copy; 3379 to += copy; 3380 pos = copy; 3381 } 3382 3383 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3384 int end; 3385 3386 WARN_ON(start > offset + len); 3387 3388 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3389 if ((copy = end - offset) > 0) { 3390 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3391 u32 p_off, p_len, copied; 3392 struct page *p; 3393 __wsum csum2; 3394 u8 *vaddr; 3395 3396 if (copy > len) 3397 copy = len; 3398 3399 skb_frag_foreach_page(frag, 3400 skb_frag_off(frag) + offset - start, 3401 copy, p, p_off, p_len, copied) { 3402 vaddr = kmap_atomic(p); 3403 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 3404 to + copied, 3405 p_len); 3406 kunmap_atomic(vaddr); 3407 csum = csum_block_add(csum, csum2, pos); 3408 pos += p_len; 3409 } 3410 3411 if (!(len -= copy)) 3412 return csum; 3413 offset += copy; 3414 to += copy; 3415 } 3416 start = end; 3417 } 3418 3419 skb_walk_frags(skb, frag_iter) { 3420 __wsum csum2; 3421 int end; 3422 3423 WARN_ON(start > offset + len); 3424 3425 end = start + frag_iter->len; 3426 if ((copy = end - offset) > 0) { 3427 if (copy > len) 3428 copy = len; 3429 csum2 = skb_copy_and_csum_bits(frag_iter, 3430 offset - start, 3431 to, copy); 3432 csum = csum_block_add(csum, csum2, pos); 3433 if ((len -= copy) == 0) 3434 return csum; 3435 offset += copy; 3436 to += copy; 3437 pos += copy; 3438 } 3439 start = end; 3440 } 3441 BUG_ON(len); 3442 return csum; 3443 } 3444 EXPORT_SYMBOL(skb_copy_and_csum_bits); 3445 3446 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 3447 { 3448 __sum16 sum; 3449 3450 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 3451 /* See comments in __skb_checksum_complete(). */ 3452 if (likely(!sum)) { 3453 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3454 !skb->csum_complete_sw) 3455 netdev_rx_csum_fault(skb->dev, skb); 3456 } 3457 if (!skb_shared(skb)) 3458 skb->csum_valid = !sum; 3459 return sum; 3460 } 3461 EXPORT_SYMBOL(__skb_checksum_complete_head); 3462 3463 /* This function assumes skb->csum already holds pseudo header's checksum, 3464 * which has been changed from the hardware checksum, for example, by 3465 * __skb_checksum_validate_complete(). And, the original skb->csum must 3466 * have been validated unsuccessfully for CHECKSUM_COMPLETE case. 3467 * 3468 * It returns non-zero if the recomputed checksum is still invalid, otherwise 3469 * zero. The new checksum is stored back into skb->csum unless the skb is 3470 * shared. 3471 */ 3472 __sum16 __skb_checksum_complete(struct sk_buff *skb) 3473 { 3474 __wsum csum; 3475 __sum16 sum; 3476 3477 csum = skb_checksum(skb, 0, skb->len, 0); 3478 3479 sum = csum_fold(csum_add(skb->csum, csum)); 3480 /* This check is inverted, because we already knew the hardware 3481 * checksum is invalid before calling this function. So, if the 3482 * re-computed checksum is valid instead, then we have a mismatch 3483 * between the original skb->csum and skb_checksum(). This means either 3484 * the original hardware checksum is incorrect or we screw up skb->csum 3485 * when moving skb->data around. 3486 */ 3487 if (likely(!sum)) { 3488 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3489 !skb->csum_complete_sw) 3490 netdev_rx_csum_fault(skb->dev, skb); 3491 } 3492 3493 if (!skb_shared(skb)) { 3494 /* Save full packet checksum */ 3495 skb->csum = csum; 3496 skb->ip_summed = CHECKSUM_COMPLETE; 3497 skb->csum_complete_sw = 1; 3498 skb->csum_valid = !sum; 3499 } 3500 3501 return sum; 3502 } 3503 EXPORT_SYMBOL(__skb_checksum_complete); 3504 3505 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum) 3506 { 3507 net_warn_ratelimited( 3508 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3509 __func__); 3510 return 0; 3511 } 3512 3513 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2, 3514 int offset, int len) 3515 { 3516 net_warn_ratelimited( 3517 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3518 __func__); 3519 return 0; 3520 } 3521 3522 static const struct skb_checksum_ops default_crc32c_ops = { 3523 .update = warn_crc32c_csum_update, 3524 .combine = warn_crc32c_csum_combine, 3525 }; 3526 3527 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly = 3528 &default_crc32c_ops; 3529 EXPORT_SYMBOL(crc32c_csum_stub); 3530 3531 /** 3532 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 3533 * @from: source buffer 3534 * 3535 * Calculates the amount of linear headroom needed in the 'to' skb passed 3536 * into skb_zerocopy(). 3537 */ 3538 unsigned int 3539 skb_zerocopy_headlen(const struct sk_buff *from) 3540 { 3541 unsigned int hlen = 0; 3542 3543 if (!from->head_frag || 3544 skb_headlen(from) < L1_CACHE_BYTES || 3545 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) { 3546 hlen = skb_headlen(from); 3547 if (!hlen) 3548 hlen = from->len; 3549 } 3550 3551 if (skb_has_frag_list(from)) 3552 hlen = from->len; 3553 3554 return hlen; 3555 } 3556 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 3557 3558 /** 3559 * skb_zerocopy - Zero copy skb to skb 3560 * @to: destination buffer 3561 * @from: source buffer 3562 * @len: number of bytes to copy from source buffer 3563 * @hlen: size of linear headroom in destination buffer 3564 * 3565 * Copies up to `len` bytes from `from` to `to` by creating references 3566 * to the frags in the source buffer. 3567 * 3568 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 3569 * headroom in the `to` buffer. 3570 * 3571 * Return value: 3572 * 0: everything is OK 3573 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 3574 * -EFAULT: skb_copy_bits() found some problem with skb geometry 3575 */ 3576 int 3577 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 3578 { 3579 int i, j = 0; 3580 int plen = 0; /* length of skb->head fragment */ 3581 int ret; 3582 struct page *page; 3583 unsigned int offset; 3584 3585 BUG_ON(!from->head_frag && !hlen); 3586 3587 /* dont bother with small payloads */ 3588 if (len <= skb_tailroom(to)) 3589 return skb_copy_bits(from, 0, skb_put(to, len), len); 3590 3591 if (hlen) { 3592 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 3593 if (unlikely(ret)) 3594 return ret; 3595 len -= hlen; 3596 } else { 3597 plen = min_t(int, skb_headlen(from), len); 3598 if (plen) { 3599 page = virt_to_head_page(from->head); 3600 offset = from->data - (unsigned char *)page_address(page); 3601 __skb_fill_page_desc(to, 0, page, offset, plen); 3602 get_page(page); 3603 j = 1; 3604 len -= plen; 3605 } 3606 } 3607 3608 skb_len_add(to, len + plen); 3609 3610 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 3611 skb_tx_error(from); 3612 return -ENOMEM; 3613 } 3614 skb_zerocopy_clone(to, from, GFP_ATOMIC); 3615 3616 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 3617 int size; 3618 3619 if (!len) 3620 break; 3621 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 3622 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]), 3623 len); 3624 skb_frag_size_set(&skb_shinfo(to)->frags[j], size); 3625 len -= size; 3626 skb_frag_ref(to, j); 3627 j++; 3628 } 3629 skb_shinfo(to)->nr_frags = j; 3630 3631 return 0; 3632 } 3633 EXPORT_SYMBOL_GPL(skb_zerocopy); 3634 3635 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 3636 { 3637 __wsum csum; 3638 long csstart; 3639 3640 if (skb->ip_summed == CHECKSUM_PARTIAL) 3641 csstart = skb_checksum_start_offset(skb); 3642 else 3643 csstart = skb_headlen(skb); 3644 3645 BUG_ON(csstart > skb_headlen(skb)); 3646 3647 skb_copy_from_linear_data(skb, to, csstart); 3648 3649 csum = 0; 3650 if (csstart != skb->len) 3651 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 3652 skb->len - csstart); 3653 3654 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3655 long csstuff = csstart + skb->csum_offset; 3656 3657 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 3658 } 3659 } 3660 EXPORT_SYMBOL(skb_copy_and_csum_dev); 3661 3662 /** 3663 * skb_dequeue - remove from the head of the queue 3664 * @list: list to dequeue from 3665 * 3666 * Remove the head of the list. The list lock is taken so the function 3667 * may be used safely with other locking list functions. The head item is 3668 * returned or %NULL if the list is empty. 3669 */ 3670 3671 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 3672 { 3673 unsigned long flags; 3674 struct sk_buff *result; 3675 3676 spin_lock_irqsave(&list->lock, flags); 3677 result = __skb_dequeue(list); 3678 spin_unlock_irqrestore(&list->lock, flags); 3679 return result; 3680 } 3681 EXPORT_SYMBOL(skb_dequeue); 3682 3683 /** 3684 * skb_dequeue_tail - remove from the tail of the queue 3685 * @list: list to dequeue from 3686 * 3687 * Remove the tail of the list. The list lock is taken so the function 3688 * may be used safely with other locking list functions. The tail item is 3689 * returned or %NULL if the list is empty. 3690 */ 3691 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 3692 { 3693 unsigned long flags; 3694 struct sk_buff *result; 3695 3696 spin_lock_irqsave(&list->lock, flags); 3697 result = __skb_dequeue_tail(list); 3698 spin_unlock_irqrestore(&list->lock, flags); 3699 return result; 3700 } 3701 EXPORT_SYMBOL(skb_dequeue_tail); 3702 3703 /** 3704 * skb_queue_purge_reason - empty a list 3705 * @list: list to empty 3706 * @reason: drop reason 3707 * 3708 * Delete all buffers on an &sk_buff list. Each buffer is removed from 3709 * the list and one reference dropped. This function takes the list 3710 * lock and is atomic with respect to other list locking functions. 3711 */ 3712 void skb_queue_purge_reason(struct sk_buff_head *list, 3713 enum skb_drop_reason reason) 3714 { 3715 struct sk_buff *skb; 3716 3717 while ((skb = skb_dequeue(list)) != NULL) 3718 kfree_skb_reason(skb, reason); 3719 } 3720 EXPORT_SYMBOL(skb_queue_purge_reason); 3721 3722 /** 3723 * skb_rbtree_purge - empty a skb rbtree 3724 * @root: root of the rbtree to empty 3725 * Return value: the sum of truesizes of all purged skbs. 3726 * 3727 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 3728 * the list and one reference dropped. This function does not take 3729 * any lock. Synchronization should be handled by the caller (e.g., TCP 3730 * out-of-order queue is protected by the socket lock). 3731 */ 3732 unsigned int skb_rbtree_purge(struct rb_root *root) 3733 { 3734 struct rb_node *p = rb_first(root); 3735 unsigned int sum = 0; 3736 3737 while (p) { 3738 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 3739 3740 p = rb_next(p); 3741 rb_erase(&skb->rbnode, root); 3742 sum += skb->truesize; 3743 kfree_skb(skb); 3744 } 3745 return sum; 3746 } 3747 3748 /** 3749 * skb_queue_head - queue a buffer at the list head 3750 * @list: list to use 3751 * @newsk: buffer to queue 3752 * 3753 * Queue a buffer at the start of the list. This function takes the 3754 * list lock and can be used safely with other locking &sk_buff functions 3755 * safely. 3756 * 3757 * A buffer cannot be placed on two lists at the same time. 3758 */ 3759 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 3760 { 3761 unsigned long flags; 3762 3763 spin_lock_irqsave(&list->lock, flags); 3764 __skb_queue_head(list, newsk); 3765 spin_unlock_irqrestore(&list->lock, flags); 3766 } 3767 EXPORT_SYMBOL(skb_queue_head); 3768 3769 /** 3770 * skb_queue_tail - queue a buffer at the list tail 3771 * @list: list to use 3772 * @newsk: buffer to queue 3773 * 3774 * Queue a buffer at the tail of the list. This function takes the 3775 * list lock and can be used safely with other locking &sk_buff functions 3776 * safely. 3777 * 3778 * A buffer cannot be placed on two lists at the same time. 3779 */ 3780 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 3781 { 3782 unsigned long flags; 3783 3784 spin_lock_irqsave(&list->lock, flags); 3785 __skb_queue_tail(list, newsk); 3786 spin_unlock_irqrestore(&list->lock, flags); 3787 } 3788 EXPORT_SYMBOL(skb_queue_tail); 3789 3790 /** 3791 * skb_unlink - remove a buffer from a list 3792 * @skb: buffer to remove 3793 * @list: list to use 3794 * 3795 * Remove a packet from a list. The list locks are taken and this 3796 * function is atomic with respect to other list locked calls 3797 * 3798 * You must know what list the SKB is on. 3799 */ 3800 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 3801 { 3802 unsigned long flags; 3803 3804 spin_lock_irqsave(&list->lock, flags); 3805 __skb_unlink(skb, list); 3806 spin_unlock_irqrestore(&list->lock, flags); 3807 } 3808 EXPORT_SYMBOL(skb_unlink); 3809 3810 /** 3811 * skb_append - append a buffer 3812 * @old: buffer to insert after 3813 * @newsk: buffer to insert 3814 * @list: list to use 3815 * 3816 * Place a packet after a given packet in a list. The list locks are taken 3817 * and this function is atomic with respect to other list locked calls. 3818 * A buffer cannot be placed on two lists at the same time. 3819 */ 3820 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 3821 { 3822 unsigned long flags; 3823 3824 spin_lock_irqsave(&list->lock, flags); 3825 __skb_queue_after(list, old, newsk); 3826 spin_unlock_irqrestore(&list->lock, flags); 3827 } 3828 EXPORT_SYMBOL(skb_append); 3829 3830 static inline void skb_split_inside_header(struct sk_buff *skb, 3831 struct sk_buff* skb1, 3832 const u32 len, const int pos) 3833 { 3834 int i; 3835 3836 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 3837 pos - len); 3838 /* And move data appendix as is. */ 3839 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 3840 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 3841 3842 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 3843 skb_shinfo(skb)->nr_frags = 0; 3844 skb1->data_len = skb->data_len; 3845 skb1->len += skb1->data_len; 3846 skb->data_len = 0; 3847 skb->len = len; 3848 skb_set_tail_pointer(skb, len); 3849 } 3850 3851 static inline void skb_split_no_header(struct sk_buff *skb, 3852 struct sk_buff* skb1, 3853 const u32 len, int pos) 3854 { 3855 int i, k = 0; 3856 const int nfrags = skb_shinfo(skb)->nr_frags; 3857 3858 skb_shinfo(skb)->nr_frags = 0; 3859 skb1->len = skb1->data_len = skb->len - len; 3860 skb->len = len; 3861 skb->data_len = len - pos; 3862 3863 for (i = 0; i < nfrags; i++) { 3864 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 3865 3866 if (pos + size > len) { 3867 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 3868 3869 if (pos < len) { 3870 /* Split frag. 3871 * We have two variants in this case: 3872 * 1. Move all the frag to the second 3873 * part, if it is possible. F.e. 3874 * this approach is mandatory for TUX, 3875 * where splitting is expensive. 3876 * 2. Split is accurately. We make this. 3877 */ 3878 skb_frag_ref(skb, i); 3879 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos); 3880 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 3881 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 3882 skb_shinfo(skb)->nr_frags++; 3883 } 3884 k++; 3885 } else 3886 skb_shinfo(skb)->nr_frags++; 3887 pos += size; 3888 } 3889 skb_shinfo(skb1)->nr_frags = k; 3890 } 3891 3892 /** 3893 * skb_split - Split fragmented skb to two parts at length len. 3894 * @skb: the buffer to split 3895 * @skb1: the buffer to receive the second part 3896 * @len: new length for skb 3897 */ 3898 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 3899 { 3900 int pos = skb_headlen(skb); 3901 const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY; 3902 3903 skb_zcopy_downgrade_managed(skb); 3904 3905 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags; 3906 skb_zerocopy_clone(skb1, skb, 0); 3907 if (len < pos) /* Split line is inside header. */ 3908 skb_split_inside_header(skb, skb1, len, pos); 3909 else /* Second chunk has no header, nothing to copy. */ 3910 skb_split_no_header(skb, skb1, len, pos); 3911 } 3912 EXPORT_SYMBOL(skb_split); 3913 3914 /* Shifting from/to a cloned skb is a no-go. 3915 * 3916 * Caller cannot keep skb_shinfo related pointers past calling here! 3917 */ 3918 static int skb_prepare_for_shift(struct sk_buff *skb) 3919 { 3920 return skb_unclone_keeptruesize(skb, GFP_ATOMIC); 3921 } 3922 3923 /** 3924 * skb_shift - Shifts paged data partially from skb to another 3925 * @tgt: buffer into which tail data gets added 3926 * @skb: buffer from which the paged data comes from 3927 * @shiftlen: shift up to this many bytes 3928 * 3929 * Attempts to shift up to shiftlen worth of bytes, which may be less than 3930 * the length of the skb, from skb to tgt. Returns number bytes shifted. 3931 * It's up to caller to free skb if everything was shifted. 3932 * 3933 * If @tgt runs out of frags, the whole operation is aborted. 3934 * 3935 * Skb cannot include anything else but paged data while tgt is allowed 3936 * to have non-paged data as well. 3937 * 3938 * TODO: full sized shift could be optimized but that would need 3939 * specialized skb free'er to handle frags without up-to-date nr_frags. 3940 */ 3941 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 3942 { 3943 int from, to, merge, todo; 3944 skb_frag_t *fragfrom, *fragto; 3945 3946 BUG_ON(shiftlen > skb->len); 3947 3948 if (skb_headlen(skb)) 3949 return 0; 3950 if (skb_zcopy(tgt) || skb_zcopy(skb)) 3951 return 0; 3952 3953 todo = shiftlen; 3954 from = 0; 3955 to = skb_shinfo(tgt)->nr_frags; 3956 fragfrom = &skb_shinfo(skb)->frags[from]; 3957 3958 /* Actual merge is delayed until the point when we know we can 3959 * commit all, so that we don't have to undo partial changes 3960 */ 3961 if (!to || 3962 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 3963 skb_frag_off(fragfrom))) { 3964 merge = -1; 3965 } else { 3966 merge = to - 1; 3967 3968 todo -= skb_frag_size(fragfrom); 3969 if (todo < 0) { 3970 if (skb_prepare_for_shift(skb) || 3971 skb_prepare_for_shift(tgt)) 3972 return 0; 3973 3974 /* All previous frag pointers might be stale! */ 3975 fragfrom = &skb_shinfo(skb)->frags[from]; 3976 fragto = &skb_shinfo(tgt)->frags[merge]; 3977 3978 skb_frag_size_add(fragto, shiftlen); 3979 skb_frag_size_sub(fragfrom, shiftlen); 3980 skb_frag_off_add(fragfrom, shiftlen); 3981 3982 goto onlymerged; 3983 } 3984 3985 from++; 3986 } 3987 3988 /* Skip full, not-fitting skb to avoid expensive operations */ 3989 if ((shiftlen == skb->len) && 3990 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 3991 return 0; 3992 3993 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 3994 return 0; 3995 3996 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 3997 if (to == MAX_SKB_FRAGS) 3998 return 0; 3999 4000 fragfrom = &skb_shinfo(skb)->frags[from]; 4001 fragto = &skb_shinfo(tgt)->frags[to]; 4002 4003 if (todo >= skb_frag_size(fragfrom)) { 4004 *fragto = *fragfrom; 4005 todo -= skb_frag_size(fragfrom); 4006 from++; 4007 to++; 4008 4009 } else { 4010 __skb_frag_ref(fragfrom); 4011 skb_frag_page_copy(fragto, fragfrom); 4012 skb_frag_off_copy(fragto, fragfrom); 4013 skb_frag_size_set(fragto, todo); 4014 4015 skb_frag_off_add(fragfrom, todo); 4016 skb_frag_size_sub(fragfrom, todo); 4017 todo = 0; 4018 4019 to++; 4020 break; 4021 } 4022 } 4023 4024 /* Ready to "commit" this state change to tgt */ 4025 skb_shinfo(tgt)->nr_frags = to; 4026 4027 if (merge >= 0) { 4028 fragfrom = &skb_shinfo(skb)->frags[0]; 4029 fragto = &skb_shinfo(tgt)->frags[merge]; 4030 4031 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 4032 __skb_frag_unref(fragfrom, skb->pp_recycle); 4033 } 4034 4035 /* Reposition in the original skb */ 4036 to = 0; 4037 while (from < skb_shinfo(skb)->nr_frags) 4038 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 4039 skb_shinfo(skb)->nr_frags = to; 4040 4041 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 4042 4043 onlymerged: 4044 /* Most likely the tgt won't ever need its checksum anymore, skb on 4045 * the other hand might need it if it needs to be resent 4046 */ 4047 tgt->ip_summed = CHECKSUM_PARTIAL; 4048 skb->ip_summed = CHECKSUM_PARTIAL; 4049 4050 skb_len_add(skb, -shiftlen); 4051 skb_len_add(tgt, shiftlen); 4052 4053 return shiftlen; 4054 } 4055 4056 /** 4057 * skb_prepare_seq_read - Prepare a sequential read of skb data 4058 * @skb: the buffer to read 4059 * @from: lower offset of data to be read 4060 * @to: upper offset of data to be read 4061 * @st: state variable 4062 * 4063 * Initializes the specified state variable. Must be called before 4064 * invoking skb_seq_read() for the first time. 4065 */ 4066 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 4067 unsigned int to, struct skb_seq_state *st) 4068 { 4069 st->lower_offset = from; 4070 st->upper_offset = to; 4071 st->root_skb = st->cur_skb = skb; 4072 st->frag_idx = st->stepped_offset = 0; 4073 st->frag_data = NULL; 4074 st->frag_off = 0; 4075 } 4076 EXPORT_SYMBOL(skb_prepare_seq_read); 4077 4078 /** 4079 * skb_seq_read - Sequentially read skb data 4080 * @consumed: number of bytes consumed by the caller so far 4081 * @data: destination pointer for data to be returned 4082 * @st: state variable 4083 * 4084 * Reads a block of skb data at @consumed relative to the 4085 * lower offset specified to skb_prepare_seq_read(). Assigns 4086 * the head of the data block to @data and returns the length 4087 * of the block or 0 if the end of the skb data or the upper 4088 * offset has been reached. 4089 * 4090 * The caller is not required to consume all of the data 4091 * returned, i.e. @consumed is typically set to the number 4092 * of bytes already consumed and the next call to 4093 * skb_seq_read() will return the remaining part of the block. 4094 * 4095 * Note 1: The size of each block of data returned can be arbitrary, 4096 * this limitation is the cost for zerocopy sequential 4097 * reads of potentially non linear data. 4098 * 4099 * Note 2: Fragment lists within fragments are not implemented 4100 * at the moment, state->root_skb could be replaced with 4101 * a stack for this purpose. 4102 */ 4103 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 4104 struct skb_seq_state *st) 4105 { 4106 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 4107 skb_frag_t *frag; 4108 4109 if (unlikely(abs_offset >= st->upper_offset)) { 4110 if (st->frag_data) { 4111 kunmap_atomic(st->frag_data); 4112 st->frag_data = NULL; 4113 } 4114 return 0; 4115 } 4116 4117 next_skb: 4118 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 4119 4120 if (abs_offset < block_limit && !st->frag_data) { 4121 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 4122 return block_limit - abs_offset; 4123 } 4124 4125 if (st->frag_idx == 0 && !st->frag_data) 4126 st->stepped_offset += skb_headlen(st->cur_skb); 4127 4128 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 4129 unsigned int pg_idx, pg_off, pg_sz; 4130 4131 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 4132 4133 pg_idx = 0; 4134 pg_off = skb_frag_off(frag); 4135 pg_sz = skb_frag_size(frag); 4136 4137 if (skb_frag_must_loop(skb_frag_page(frag))) { 4138 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT; 4139 pg_off = offset_in_page(pg_off + st->frag_off); 4140 pg_sz = min_t(unsigned int, pg_sz - st->frag_off, 4141 PAGE_SIZE - pg_off); 4142 } 4143 4144 block_limit = pg_sz + st->stepped_offset; 4145 if (abs_offset < block_limit) { 4146 if (!st->frag_data) 4147 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx); 4148 4149 *data = (u8 *)st->frag_data + pg_off + 4150 (abs_offset - st->stepped_offset); 4151 4152 return block_limit - abs_offset; 4153 } 4154 4155 if (st->frag_data) { 4156 kunmap_atomic(st->frag_data); 4157 st->frag_data = NULL; 4158 } 4159 4160 st->stepped_offset += pg_sz; 4161 st->frag_off += pg_sz; 4162 if (st->frag_off == skb_frag_size(frag)) { 4163 st->frag_off = 0; 4164 st->frag_idx++; 4165 } 4166 } 4167 4168 if (st->frag_data) { 4169 kunmap_atomic(st->frag_data); 4170 st->frag_data = NULL; 4171 } 4172 4173 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 4174 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 4175 st->frag_idx = 0; 4176 goto next_skb; 4177 } else if (st->cur_skb->next) { 4178 st->cur_skb = st->cur_skb->next; 4179 st->frag_idx = 0; 4180 goto next_skb; 4181 } 4182 4183 return 0; 4184 } 4185 EXPORT_SYMBOL(skb_seq_read); 4186 4187 /** 4188 * skb_abort_seq_read - Abort a sequential read of skb data 4189 * @st: state variable 4190 * 4191 * Must be called if skb_seq_read() was not called until it 4192 * returned 0. 4193 */ 4194 void skb_abort_seq_read(struct skb_seq_state *st) 4195 { 4196 if (st->frag_data) 4197 kunmap_atomic(st->frag_data); 4198 } 4199 EXPORT_SYMBOL(skb_abort_seq_read); 4200 4201 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 4202 4203 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 4204 struct ts_config *conf, 4205 struct ts_state *state) 4206 { 4207 return skb_seq_read(offset, text, TS_SKB_CB(state)); 4208 } 4209 4210 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 4211 { 4212 skb_abort_seq_read(TS_SKB_CB(state)); 4213 } 4214 4215 /** 4216 * skb_find_text - Find a text pattern in skb data 4217 * @skb: the buffer to look in 4218 * @from: search offset 4219 * @to: search limit 4220 * @config: textsearch configuration 4221 * 4222 * Finds a pattern in the skb data according to the specified 4223 * textsearch configuration. Use textsearch_next() to retrieve 4224 * subsequent occurrences of the pattern. Returns the offset 4225 * to the first occurrence or UINT_MAX if no match was found. 4226 */ 4227 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 4228 unsigned int to, struct ts_config *config) 4229 { 4230 struct ts_state state; 4231 unsigned int ret; 4232 4233 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb)); 4234 4235 config->get_next_block = skb_ts_get_next_block; 4236 config->finish = skb_ts_finish; 4237 4238 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 4239 4240 ret = textsearch_find(config, &state); 4241 return (ret <= to - from ? ret : UINT_MAX); 4242 } 4243 EXPORT_SYMBOL(skb_find_text); 4244 4245 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 4246 int offset, size_t size, size_t max_frags) 4247 { 4248 int i = skb_shinfo(skb)->nr_frags; 4249 4250 if (skb_can_coalesce(skb, i, page, offset)) { 4251 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 4252 } else if (i < max_frags) { 4253 skb_zcopy_downgrade_managed(skb); 4254 get_page(page); 4255 skb_fill_page_desc_noacc(skb, i, page, offset, size); 4256 } else { 4257 return -EMSGSIZE; 4258 } 4259 4260 return 0; 4261 } 4262 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 4263 4264 /** 4265 * skb_pull_rcsum - pull skb and update receive checksum 4266 * @skb: buffer to update 4267 * @len: length of data pulled 4268 * 4269 * This function performs an skb_pull on the packet and updates 4270 * the CHECKSUM_COMPLETE checksum. It should be used on 4271 * receive path processing instead of skb_pull unless you know 4272 * that the checksum difference is zero (e.g., a valid IP header) 4273 * or you are setting ip_summed to CHECKSUM_NONE. 4274 */ 4275 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 4276 { 4277 unsigned char *data = skb->data; 4278 4279 BUG_ON(len > skb->len); 4280 __skb_pull(skb, len); 4281 skb_postpull_rcsum(skb, data, len); 4282 return skb->data; 4283 } 4284 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 4285 4286 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb) 4287 { 4288 skb_frag_t head_frag; 4289 struct page *page; 4290 4291 page = virt_to_head_page(frag_skb->head); 4292 skb_frag_fill_page_desc(&head_frag, page, frag_skb->data - 4293 (unsigned char *)page_address(page), 4294 skb_headlen(frag_skb)); 4295 return head_frag; 4296 } 4297 4298 struct sk_buff *skb_segment_list(struct sk_buff *skb, 4299 netdev_features_t features, 4300 unsigned int offset) 4301 { 4302 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list; 4303 unsigned int tnl_hlen = skb_tnl_header_len(skb); 4304 unsigned int delta_truesize = 0; 4305 unsigned int delta_len = 0; 4306 struct sk_buff *tail = NULL; 4307 struct sk_buff *nskb, *tmp; 4308 int len_diff, err; 4309 4310 skb_push(skb, -skb_network_offset(skb) + offset); 4311 4312 /* Ensure the head is writeable before touching the shared info */ 4313 err = skb_unclone(skb, GFP_ATOMIC); 4314 if (err) 4315 goto err_linearize; 4316 4317 skb_shinfo(skb)->frag_list = NULL; 4318 4319 while (list_skb) { 4320 nskb = list_skb; 4321 list_skb = list_skb->next; 4322 4323 err = 0; 4324 delta_truesize += nskb->truesize; 4325 if (skb_shared(nskb)) { 4326 tmp = skb_clone(nskb, GFP_ATOMIC); 4327 if (tmp) { 4328 consume_skb(nskb); 4329 nskb = tmp; 4330 err = skb_unclone(nskb, GFP_ATOMIC); 4331 } else { 4332 err = -ENOMEM; 4333 } 4334 } 4335 4336 if (!tail) 4337 skb->next = nskb; 4338 else 4339 tail->next = nskb; 4340 4341 if (unlikely(err)) { 4342 nskb->next = list_skb; 4343 goto err_linearize; 4344 } 4345 4346 tail = nskb; 4347 4348 delta_len += nskb->len; 4349 4350 skb_push(nskb, -skb_network_offset(nskb) + offset); 4351 4352 skb_release_head_state(nskb); 4353 len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb); 4354 __copy_skb_header(nskb, skb); 4355 4356 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb)); 4357 nskb->transport_header += len_diff; 4358 skb_copy_from_linear_data_offset(skb, -tnl_hlen, 4359 nskb->data - tnl_hlen, 4360 offset + tnl_hlen); 4361 4362 if (skb_needs_linearize(nskb, features) && 4363 __skb_linearize(nskb)) 4364 goto err_linearize; 4365 } 4366 4367 skb->truesize = skb->truesize - delta_truesize; 4368 skb->data_len = skb->data_len - delta_len; 4369 skb->len = skb->len - delta_len; 4370 4371 skb_gso_reset(skb); 4372 4373 skb->prev = tail; 4374 4375 if (skb_needs_linearize(skb, features) && 4376 __skb_linearize(skb)) 4377 goto err_linearize; 4378 4379 skb_get(skb); 4380 4381 return skb; 4382 4383 err_linearize: 4384 kfree_skb_list(skb->next); 4385 skb->next = NULL; 4386 return ERR_PTR(-ENOMEM); 4387 } 4388 EXPORT_SYMBOL_GPL(skb_segment_list); 4389 4390 /** 4391 * skb_segment - Perform protocol segmentation on skb. 4392 * @head_skb: buffer to segment 4393 * @features: features for the output path (see dev->features) 4394 * 4395 * This function performs segmentation on the given skb. It returns 4396 * a pointer to the first in a list of new skbs for the segments. 4397 * In case of error it returns ERR_PTR(err). 4398 */ 4399 struct sk_buff *skb_segment(struct sk_buff *head_skb, 4400 netdev_features_t features) 4401 { 4402 struct sk_buff *segs = NULL; 4403 struct sk_buff *tail = NULL; 4404 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 4405 skb_frag_t *frag = skb_shinfo(head_skb)->frags; 4406 unsigned int mss = skb_shinfo(head_skb)->gso_size; 4407 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 4408 struct sk_buff *frag_skb = head_skb; 4409 unsigned int offset = doffset; 4410 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 4411 unsigned int partial_segs = 0; 4412 unsigned int headroom; 4413 unsigned int len = head_skb->len; 4414 __be16 proto; 4415 bool csum, sg; 4416 int nfrags = skb_shinfo(head_skb)->nr_frags; 4417 int err = -ENOMEM; 4418 int i = 0; 4419 int pos; 4420 4421 if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) && 4422 mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) { 4423 struct sk_buff *check_skb; 4424 4425 for (check_skb = list_skb; check_skb; check_skb = check_skb->next) { 4426 if (skb_headlen(check_skb) && !check_skb->head_frag) { 4427 /* gso_size is untrusted, and we have a frag_list with 4428 * a linear non head_frag item. 4429 * 4430 * If head_skb's headlen does not fit requested gso_size, 4431 * it means that the frag_list members do NOT terminate 4432 * on exact gso_size boundaries. Hence we cannot perform 4433 * skb_frag_t page sharing. Therefore we must fallback to 4434 * copying the frag_list skbs; we do so by disabling SG. 4435 */ 4436 features &= ~NETIF_F_SG; 4437 break; 4438 } 4439 } 4440 } 4441 4442 __skb_push(head_skb, doffset); 4443 proto = skb_network_protocol(head_skb, NULL); 4444 if (unlikely(!proto)) 4445 return ERR_PTR(-EINVAL); 4446 4447 sg = !!(features & NETIF_F_SG); 4448 csum = !!can_checksum_protocol(features, proto); 4449 4450 if (sg && csum && (mss != GSO_BY_FRAGS)) { 4451 if (!(features & NETIF_F_GSO_PARTIAL)) { 4452 struct sk_buff *iter; 4453 unsigned int frag_len; 4454 4455 if (!list_skb || 4456 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 4457 goto normal; 4458 4459 /* If we get here then all the required 4460 * GSO features except frag_list are supported. 4461 * Try to split the SKB to multiple GSO SKBs 4462 * with no frag_list. 4463 * Currently we can do that only when the buffers don't 4464 * have a linear part and all the buffers except 4465 * the last are of the same length. 4466 */ 4467 frag_len = list_skb->len; 4468 skb_walk_frags(head_skb, iter) { 4469 if (frag_len != iter->len && iter->next) 4470 goto normal; 4471 if (skb_headlen(iter) && !iter->head_frag) 4472 goto normal; 4473 4474 len -= iter->len; 4475 } 4476 4477 if (len != frag_len) 4478 goto normal; 4479 } 4480 4481 /* GSO partial only requires that we trim off any excess that 4482 * doesn't fit into an MSS sized block, so take care of that 4483 * now. 4484 */ 4485 partial_segs = len / mss; 4486 if (partial_segs > 1) 4487 mss *= partial_segs; 4488 else 4489 partial_segs = 0; 4490 } 4491 4492 normal: 4493 headroom = skb_headroom(head_skb); 4494 pos = skb_headlen(head_skb); 4495 4496 do { 4497 struct sk_buff *nskb; 4498 skb_frag_t *nskb_frag; 4499 int hsize; 4500 int size; 4501 4502 if (unlikely(mss == GSO_BY_FRAGS)) { 4503 len = list_skb->len; 4504 } else { 4505 len = head_skb->len - offset; 4506 if (len > mss) 4507 len = mss; 4508 } 4509 4510 hsize = skb_headlen(head_skb) - offset; 4511 4512 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) && 4513 (skb_headlen(list_skb) == len || sg)) { 4514 BUG_ON(skb_headlen(list_skb) > len); 4515 4516 i = 0; 4517 nfrags = skb_shinfo(list_skb)->nr_frags; 4518 frag = skb_shinfo(list_skb)->frags; 4519 frag_skb = list_skb; 4520 pos += skb_headlen(list_skb); 4521 4522 while (pos < offset + len) { 4523 BUG_ON(i >= nfrags); 4524 4525 size = skb_frag_size(frag); 4526 if (pos + size > offset + len) 4527 break; 4528 4529 i++; 4530 pos += size; 4531 frag++; 4532 } 4533 4534 nskb = skb_clone(list_skb, GFP_ATOMIC); 4535 list_skb = list_skb->next; 4536 4537 if (unlikely(!nskb)) 4538 goto err; 4539 4540 if (unlikely(pskb_trim(nskb, len))) { 4541 kfree_skb(nskb); 4542 goto err; 4543 } 4544 4545 hsize = skb_end_offset(nskb); 4546 if (skb_cow_head(nskb, doffset + headroom)) { 4547 kfree_skb(nskb); 4548 goto err; 4549 } 4550 4551 nskb->truesize += skb_end_offset(nskb) - hsize; 4552 skb_release_head_state(nskb); 4553 __skb_push(nskb, doffset); 4554 } else { 4555 if (hsize < 0) 4556 hsize = 0; 4557 if (hsize > len || !sg) 4558 hsize = len; 4559 4560 nskb = __alloc_skb(hsize + doffset + headroom, 4561 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 4562 NUMA_NO_NODE); 4563 4564 if (unlikely(!nskb)) 4565 goto err; 4566 4567 skb_reserve(nskb, headroom); 4568 __skb_put(nskb, doffset); 4569 } 4570 4571 if (segs) 4572 tail->next = nskb; 4573 else 4574 segs = nskb; 4575 tail = nskb; 4576 4577 __copy_skb_header(nskb, head_skb); 4578 4579 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 4580 skb_reset_mac_len(nskb); 4581 4582 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 4583 nskb->data - tnl_hlen, 4584 doffset + tnl_hlen); 4585 4586 if (nskb->len == len + doffset) 4587 goto perform_csum_check; 4588 4589 if (!sg) { 4590 if (!csum) { 4591 if (!nskb->remcsum_offload) 4592 nskb->ip_summed = CHECKSUM_NONE; 4593 SKB_GSO_CB(nskb)->csum = 4594 skb_copy_and_csum_bits(head_skb, offset, 4595 skb_put(nskb, 4596 len), 4597 len); 4598 SKB_GSO_CB(nskb)->csum_start = 4599 skb_headroom(nskb) + doffset; 4600 } else { 4601 if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len)) 4602 goto err; 4603 } 4604 continue; 4605 } 4606 4607 nskb_frag = skb_shinfo(nskb)->frags; 4608 4609 skb_copy_from_linear_data_offset(head_skb, offset, 4610 skb_put(nskb, hsize), hsize); 4611 4612 skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags & 4613 SKBFL_SHARED_FRAG; 4614 4615 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4616 skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 4617 goto err; 4618 4619 while (pos < offset + len) { 4620 if (i >= nfrags) { 4621 i = 0; 4622 nfrags = skb_shinfo(list_skb)->nr_frags; 4623 frag = skb_shinfo(list_skb)->frags; 4624 frag_skb = list_skb; 4625 if (!skb_headlen(list_skb)) { 4626 BUG_ON(!nfrags); 4627 } else { 4628 BUG_ON(!list_skb->head_frag); 4629 4630 /* to make room for head_frag. */ 4631 i--; 4632 frag--; 4633 } 4634 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4635 skb_zerocopy_clone(nskb, frag_skb, 4636 GFP_ATOMIC)) 4637 goto err; 4638 4639 list_skb = list_skb->next; 4640 } 4641 4642 if (unlikely(skb_shinfo(nskb)->nr_frags >= 4643 MAX_SKB_FRAGS)) { 4644 net_warn_ratelimited( 4645 "skb_segment: too many frags: %u %u\n", 4646 pos, mss); 4647 err = -EINVAL; 4648 goto err; 4649 } 4650 4651 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; 4652 __skb_frag_ref(nskb_frag); 4653 size = skb_frag_size(nskb_frag); 4654 4655 if (pos < offset) { 4656 skb_frag_off_add(nskb_frag, offset - pos); 4657 skb_frag_size_sub(nskb_frag, offset - pos); 4658 } 4659 4660 skb_shinfo(nskb)->nr_frags++; 4661 4662 if (pos + size <= offset + len) { 4663 i++; 4664 frag++; 4665 pos += size; 4666 } else { 4667 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 4668 goto skip_fraglist; 4669 } 4670 4671 nskb_frag++; 4672 } 4673 4674 skip_fraglist: 4675 nskb->data_len = len - hsize; 4676 nskb->len += nskb->data_len; 4677 nskb->truesize += nskb->data_len; 4678 4679 perform_csum_check: 4680 if (!csum) { 4681 if (skb_has_shared_frag(nskb) && 4682 __skb_linearize(nskb)) 4683 goto err; 4684 4685 if (!nskb->remcsum_offload) 4686 nskb->ip_summed = CHECKSUM_NONE; 4687 SKB_GSO_CB(nskb)->csum = 4688 skb_checksum(nskb, doffset, 4689 nskb->len - doffset, 0); 4690 SKB_GSO_CB(nskb)->csum_start = 4691 skb_headroom(nskb) + doffset; 4692 } 4693 } while ((offset += len) < head_skb->len); 4694 4695 /* Some callers want to get the end of the list. 4696 * Put it in segs->prev to avoid walking the list. 4697 * (see validate_xmit_skb_list() for example) 4698 */ 4699 segs->prev = tail; 4700 4701 if (partial_segs) { 4702 struct sk_buff *iter; 4703 int type = skb_shinfo(head_skb)->gso_type; 4704 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 4705 4706 /* Update type to add partial and then remove dodgy if set */ 4707 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 4708 type &= ~SKB_GSO_DODGY; 4709 4710 /* Update GSO info and prepare to start updating headers on 4711 * our way back down the stack of protocols. 4712 */ 4713 for (iter = segs; iter; iter = iter->next) { 4714 skb_shinfo(iter)->gso_size = gso_size; 4715 skb_shinfo(iter)->gso_segs = partial_segs; 4716 skb_shinfo(iter)->gso_type = type; 4717 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 4718 } 4719 4720 if (tail->len - doffset <= gso_size) 4721 skb_shinfo(tail)->gso_size = 0; 4722 else if (tail != segs) 4723 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 4724 } 4725 4726 /* Following permits correct backpressure, for protocols 4727 * using skb_set_owner_w(). 4728 * Idea is to tranfert ownership from head_skb to last segment. 4729 */ 4730 if (head_skb->destructor == sock_wfree) { 4731 swap(tail->truesize, head_skb->truesize); 4732 swap(tail->destructor, head_skb->destructor); 4733 swap(tail->sk, head_skb->sk); 4734 } 4735 return segs; 4736 4737 err: 4738 kfree_skb_list(segs); 4739 return ERR_PTR(err); 4740 } 4741 EXPORT_SYMBOL_GPL(skb_segment); 4742 4743 #ifdef CONFIG_SKB_EXTENSIONS 4744 #define SKB_EXT_ALIGN_VALUE 8 4745 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) 4746 4747 static const u8 skb_ext_type_len[] = { 4748 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4749 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info), 4750 #endif 4751 #ifdef CONFIG_XFRM 4752 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path), 4753 #endif 4754 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4755 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext), 4756 #endif 4757 #if IS_ENABLED(CONFIG_MPTCP) 4758 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext), 4759 #endif 4760 #if IS_ENABLED(CONFIG_MCTP_FLOWS) 4761 [SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow), 4762 #endif 4763 }; 4764 4765 static __always_inline unsigned int skb_ext_total_length(void) 4766 { 4767 return SKB_EXT_CHUNKSIZEOF(struct skb_ext) + 4768 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4769 skb_ext_type_len[SKB_EXT_BRIDGE_NF] + 4770 #endif 4771 #ifdef CONFIG_XFRM 4772 skb_ext_type_len[SKB_EXT_SEC_PATH] + 4773 #endif 4774 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4775 skb_ext_type_len[TC_SKB_EXT] + 4776 #endif 4777 #if IS_ENABLED(CONFIG_MPTCP) 4778 skb_ext_type_len[SKB_EXT_MPTCP] + 4779 #endif 4780 #if IS_ENABLED(CONFIG_MCTP_FLOWS) 4781 skb_ext_type_len[SKB_EXT_MCTP] + 4782 #endif 4783 0; 4784 } 4785 4786 static void skb_extensions_init(void) 4787 { 4788 BUILD_BUG_ON(SKB_EXT_NUM >= 8); 4789 BUILD_BUG_ON(skb_ext_total_length() > 255); 4790 4791 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", 4792 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), 4793 0, 4794 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4795 NULL); 4796 } 4797 #else 4798 static void skb_extensions_init(void) {} 4799 #endif 4800 4801 /* The SKB kmem_cache slab is critical for network performance. Never 4802 * merge/alias the slab with similar sized objects. This avoids fragmentation 4803 * that hurts performance of kmem_cache_{alloc,free}_bulk APIs. 4804 */ 4805 #ifndef CONFIG_SLUB_TINY 4806 #define FLAG_SKB_NO_MERGE SLAB_NO_MERGE 4807 #else /* CONFIG_SLUB_TINY - simple loop in kmem_cache_alloc_bulk */ 4808 #define FLAG_SKB_NO_MERGE 0 4809 #endif 4810 4811 void __init skb_init(void) 4812 { 4813 skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache", 4814 sizeof(struct sk_buff), 4815 0, 4816 SLAB_HWCACHE_ALIGN|SLAB_PANIC| 4817 FLAG_SKB_NO_MERGE, 4818 offsetof(struct sk_buff, cb), 4819 sizeof_field(struct sk_buff, cb), 4820 NULL); 4821 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 4822 sizeof(struct sk_buff_fclones), 4823 0, 4824 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4825 NULL); 4826 /* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes. 4827 * struct skb_shared_info is located at the end of skb->head, 4828 * and should not be copied to/from user. 4829 */ 4830 skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head", 4831 SKB_SMALL_HEAD_CACHE_SIZE, 4832 0, 4833 SLAB_HWCACHE_ALIGN | SLAB_PANIC, 4834 0, 4835 SKB_SMALL_HEAD_HEADROOM, 4836 NULL); 4837 skb_extensions_init(); 4838 } 4839 4840 static int 4841 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 4842 unsigned int recursion_level) 4843 { 4844 int start = skb_headlen(skb); 4845 int i, copy = start - offset; 4846 struct sk_buff *frag_iter; 4847 int elt = 0; 4848 4849 if (unlikely(recursion_level >= 24)) 4850 return -EMSGSIZE; 4851 4852 if (copy > 0) { 4853 if (copy > len) 4854 copy = len; 4855 sg_set_buf(sg, skb->data + offset, copy); 4856 elt++; 4857 if ((len -= copy) == 0) 4858 return elt; 4859 offset += copy; 4860 } 4861 4862 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4863 int end; 4864 4865 WARN_ON(start > offset + len); 4866 4867 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 4868 if ((copy = end - offset) > 0) { 4869 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4870 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4871 return -EMSGSIZE; 4872 4873 if (copy > len) 4874 copy = len; 4875 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 4876 skb_frag_off(frag) + offset - start); 4877 elt++; 4878 if (!(len -= copy)) 4879 return elt; 4880 offset += copy; 4881 } 4882 start = end; 4883 } 4884 4885 skb_walk_frags(skb, frag_iter) { 4886 int end, ret; 4887 4888 WARN_ON(start > offset + len); 4889 4890 end = start + frag_iter->len; 4891 if ((copy = end - offset) > 0) { 4892 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4893 return -EMSGSIZE; 4894 4895 if (copy > len) 4896 copy = len; 4897 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 4898 copy, recursion_level + 1); 4899 if (unlikely(ret < 0)) 4900 return ret; 4901 elt += ret; 4902 if ((len -= copy) == 0) 4903 return elt; 4904 offset += copy; 4905 } 4906 start = end; 4907 } 4908 BUG_ON(len); 4909 return elt; 4910 } 4911 4912 /** 4913 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 4914 * @skb: Socket buffer containing the buffers to be mapped 4915 * @sg: The scatter-gather list to map into 4916 * @offset: The offset into the buffer's contents to start mapping 4917 * @len: Length of buffer space to be mapped 4918 * 4919 * Fill the specified scatter-gather list with mappings/pointers into a 4920 * region of the buffer space attached to a socket buffer. Returns either 4921 * the number of scatterlist items used, or -EMSGSIZE if the contents 4922 * could not fit. 4923 */ 4924 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 4925 { 4926 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 4927 4928 if (nsg <= 0) 4929 return nsg; 4930 4931 sg_mark_end(&sg[nsg - 1]); 4932 4933 return nsg; 4934 } 4935 EXPORT_SYMBOL_GPL(skb_to_sgvec); 4936 4937 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 4938 * sglist without mark the sg which contain last skb data as the end. 4939 * So the caller can mannipulate sg list as will when padding new data after 4940 * the first call without calling sg_unmark_end to expend sg list. 4941 * 4942 * Scenario to use skb_to_sgvec_nomark: 4943 * 1. sg_init_table 4944 * 2. skb_to_sgvec_nomark(payload1) 4945 * 3. skb_to_sgvec_nomark(payload2) 4946 * 4947 * This is equivalent to: 4948 * 1. sg_init_table 4949 * 2. skb_to_sgvec(payload1) 4950 * 3. sg_unmark_end 4951 * 4. skb_to_sgvec(payload2) 4952 * 4953 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 4954 * is more preferable. 4955 */ 4956 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 4957 int offset, int len) 4958 { 4959 return __skb_to_sgvec(skb, sg, offset, len, 0); 4960 } 4961 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 4962 4963 4964 4965 /** 4966 * skb_cow_data - Check that a socket buffer's data buffers are writable 4967 * @skb: The socket buffer to check. 4968 * @tailbits: Amount of trailing space to be added 4969 * @trailer: Returned pointer to the skb where the @tailbits space begins 4970 * 4971 * Make sure that the data buffers attached to a socket buffer are 4972 * writable. If they are not, private copies are made of the data buffers 4973 * and the socket buffer is set to use these instead. 4974 * 4975 * If @tailbits is given, make sure that there is space to write @tailbits 4976 * bytes of data beyond current end of socket buffer. @trailer will be 4977 * set to point to the skb in which this space begins. 4978 * 4979 * The number of scatterlist elements required to completely map the 4980 * COW'd and extended socket buffer will be returned. 4981 */ 4982 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 4983 { 4984 int copyflag; 4985 int elt; 4986 struct sk_buff *skb1, **skb_p; 4987 4988 /* If skb is cloned or its head is paged, reallocate 4989 * head pulling out all the pages (pages are considered not writable 4990 * at the moment even if they are anonymous). 4991 */ 4992 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 4993 !__pskb_pull_tail(skb, __skb_pagelen(skb))) 4994 return -ENOMEM; 4995 4996 /* Easy case. Most of packets will go this way. */ 4997 if (!skb_has_frag_list(skb)) { 4998 /* A little of trouble, not enough of space for trailer. 4999 * This should not happen, when stack is tuned to generate 5000 * good frames. OK, on miss we reallocate and reserve even more 5001 * space, 128 bytes is fair. */ 5002 5003 if (skb_tailroom(skb) < tailbits && 5004 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 5005 return -ENOMEM; 5006 5007 /* Voila! */ 5008 *trailer = skb; 5009 return 1; 5010 } 5011 5012 /* Misery. We are in troubles, going to mincer fragments... */ 5013 5014 elt = 1; 5015 skb_p = &skb_shinfo(skb)->frag_list; 5016 copyflag = 0; 5017 5018 while ((skb1 = *skb_p) != NULL) { 5019 int ntail = 0; 5020 5021 /* The fragment is partially pulled by someone, 5022 * this can happen on input. Copy it and everything 5023 * after it. */ 5024 5025 if (skb_shared(skb1)) 5026 copyflag = 1; 5027 5028 /* If the skb is the last, worry about trailer. */ 5029 5030 if (skb1->next == NULL && tailbits) { 5031 if (skb_shinfo(skb1)->nr_frags || 5032 skb_has_frag_list(skb1) || 5033 skb_tailroom(skb1) < tailbits) 5034 ntail = tailbits + 128; 5035 } 5036 5037 if (copyflag || 5038 skb_cloned(skb1) || 5039 ntail || 5040 skb_shinfo(skb1)->nr_frags || 5041 skb_has_frag_list(skb1)) { 5042 struct sk_buff *skb2; 5043 5044 /* Fuck, we are miserable poor guys... */ 5045 if (ntail == 0) 5046 skb2 = skb_copy(skb1, GFP_ATOMIC); 5047 else 5048 skb2 = skb_copy_expand(skb1, 5049 skb_headroom(skb1), 5050 ntail, 5051 GFP_ATOMIC); 5052 if (unlikely(skb2 == NULL)) 5053 return -ENOMEM; 5054 5055 if (skb1->sk) 5056 skb_set_owner_w(skb2, skb1->sk); 5057 5058 /* Looking around. Are we still alive? 5059 * OK, link new skb, drop old one */ 5060 5061 skb2->next = skb1->next; 5062 *skb_p = skb2; 5063 kfree_skb(skb1); 5064 skb1 = skb2; 5065 } 5066 elt++; 5067 *trailer = skb1; 5068 skb_p = &skb1->next; 5069 } 5070 5071 return elt; 5072 } 5073 EXPORT_SYMBOL_GPL(skb_cow_data); 5074 5075 static void sock_rmem_free(struct sk_buff *skb) 5076 { 5077 struct sock *sk = skb->sk; 5078 5079 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 5080 } 5081 5082 static void skb_set_err_queue(struct sk_buff *skb) 5083 { 5084 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING. 5085 * So, it is safe to (mis)use it to mark skbs on the error queue. 5086 */ 5087 skb->pkt_type = PACKET_OUTGOING; 5088 BUILD_BUG_ON(PACKET_OUTGOING == 0); 5089 } 5090 5091 /* 5092 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 5093 */ 5094 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 5095 { 5096 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 5097 (unsigned int)READ_ONCE(sk->sk_rcvbuf)) 5098 return -ENOMEM; 5099 5100 skb_orphan(skb); 5101 skb->sk = sk; 5102 skb->destructor = sock_rmem_free; 5103 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 5104 skb_set_err_queue(skb); 5105 5106 /* before exiting rcu section, make sure dst is refcounted */ 5107 skb_dst_force(skb); 5108 5109 skb_queue_tail(&sk->sk_error_queue, skb); 5110 if (!sock_flag(sk, SOCK_DEAD)) 5111 sk_error_report(sk); 5112 return 0; 5113 } 5114 EXPORT_SYMBOL(sock_queue_err_skb); 5115 5116 static bool is_icmp_err_skb(const struct sk_buff *skb) 5117 { 5118 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 5119 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 5120 } 5121 5122 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 5123 { 5124 struct sk_buff_head *q = &sk->sk_error_queue; 5125 struct sk_buff *skb, *skb_next = NULL; 5126 bool icmp_next = false; 5127 unsigned long flags; 5128 5129 spin_lock_irqsave(&q->lock, flags); 5130 skb = __skb_dequeue(q); 5131 if (skb && (skb_next = skb_peek(q))) { 5132 icmp_next = is_icmp_err_skb(skb_next); 5133 if (icmp_next) 5134 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 5135 } 5136 spin_unlock_irqrestore(&q->lock, flags); 5137 5138 if (is_icmp_err_skb(skb) && !icmp_next) 5139 sk->sk_err = 0; 5140 5141 if (skb_next) 5142 sk_error_report(sk); 5143 5144 return skb; 5145 } 5146 EXPORT_SYMBOL(sock_dequeue_err_skb); 5147 5148 /** 5149 * skb_clone_sk - create clone of skb, and take reference to socket 5150 * @skb: the skb to clone 5151 * 5152 * This function creates a clone of a buffer that holds a reference on 5153 * sk_refcnt. Buffers created via this function are meant to be 5154 * returned using sock_queue_err_skb, or free via kfree_skb. 5155 * 5156 * When passing buffers allocated with this function to sock_queue_err_skb 5157 * it is necessary to wrap the call with sock_hold/sock_put in order to 5158 * prevent the socket from being released prior to being enqueued on 5159 * the sk_error_queue. 5160 */ 5161 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 5162 { 5163 struct sock *sk = skb->sk; 5164 struct sk_buff *clone; 5165 5166 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 5167 return NULL; 5168 5169 clone = skb_clone(skb, GFP_ATOMIC); 5170 if (!clone) { 5171 sock_put(sk); 5172 return NULL; 5173 } 5174 5175 clone->sk = sk; 5176 clone->destructor = sock_efree; 5177 5178 return clone; 5179 } 5180 EXPORT_SYMBOL(skb_clone_sk); 5181 5182 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 5183 struct sock *sk, 5184 int tstype, 5185 bool opt_stats) 5186 { 5187 struct sock_exterr_skb *serr; 5188 int err; 5189 5190 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 5191 5192 serr = SKB_EXT_ERR(skb); 5193 memset(serr, 0, sizeof(*serr)); 5194 serr->ee.ee_errno = ENOMSG; 5195 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 5196 serr->ee.ee_info = tstype; 5197 serr->opt_stats = opt_stats; 5198 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 5199 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { 5200 serr->ee.ee_data = skb_shinfo(skb)->tskey; 5201 if (sk_is_tcp(sk)) 5202 serr->ee.ee_data -= atomic_read(&sk->sk_tskey); 5203 } 5204 5205 err = sock_queue_err_skb(sk, skb); 5206 5207 if (err) 5208 kfree_skb(skb); 5209 } 5210 5211 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 5212 { 5213 bool ret; 5214 5215 if (likely(READ_ONCE(sysctl_tstamp_allow_data) || tsonly)) 5216 return true; 5217 5218 read_lock_bh(&sk->sk_callback_lock); 5219 ret = sk->sk_socket && sk->sk_socket->file && 5220 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); 5221 read_unlock_bh(&sk->sk_callback_lock); 5222 return ret; 5223 } 5224 5225 void skb_complete_tx_timestamp(struct sk_buff *skb, 5226 struct skb_shared_hwtstamps *hwtstamps) 5227 { 5228 struct sock *sk = skb->sk; 5229 5230 if (!skb_may_tx_timestamp(sk, false)) 5231 goto err; 5232 5233 /* Take a reference to prevent skb_orphan() from freeing the socket, 5234 * but only if the socket refcount is not zero. 5235 */ 5236 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5237 *skb_hwtstamps(skb) = *hwtstamps; 5238 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 5239 sock_put(sk); 5240 return; 5241 } 5242 5243 err: 5244 kfree_skb(skb); 5245 } 5246 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 5247 5248 void __skb_tstamp_tx(struct sk_buff *orig_skb, 5249 const struct sk_buff *ack_skb, 5250 struct skb_shared_hwtstamps *hwtstamps, 5251 struct sock *sk, int tstype) 5252 { 5253 struct sk_buff *skb; 5254 bool tsonly, opt_stats = false; 5255 5256 if (!sk) 5257 return; 5258 5259 if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 5260 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 5261 return; 5262 5263 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 5264 if (!skb_may_tx_timestamp(sk, tsonly)) 5265 return; 5266 5267 if (tsonly) { 5268 #ifdef CONFIG_INET 5269 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && 5270 sk_is_tcp(sk)) { 5271 skb = tcp_get_timestamping_opt_stats(sk, orig_skb, 5272 ack_skb); 5273 opt_stats = true; 5274 } else 5275 #endif 5276 skb = alloc_skb(0, GFP_ATOMIC); 5277 } else { 5278 skb = skb_clone(orig_skb, GFP_ATOMIC); 5279 5280 if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) { 5281 kfree_skb(skb); 5282 return; 5283 } 5284 } 5285 if (!skb) 5286 return; 5287 5288 if (tsonly) { 5289 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 5290 SKBTX_ANY_TSTAMP; 5291 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 5292 } 5293 5294 if (hwtstamps) 5295 *skb_hwtstamps(skb) = *hwtstamps; 5296 else 5297 __net_timestamp(skb); 5298 5299 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 5300 } 5301 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 5302 5303 void skb_tstamp_tx(struct sk_buff *orig_skb, 5304 struct skb_shared_hwtstamps *hwtstamps) 5305 { 5306 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk, 5307 SCM_TSTAMP_SND); 5308 } 5309 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 5310 5311 #ifdef CONFIG_WIRELESS 5312 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 5313 { 5314 struct sock *sk = skb->sk; 5315 struct sock_exterr_skb *serr; 5316 int err = 1; 5317 5318 skb->wifi_acked_valid = 1; 5319 skb->wifi_acked = acked; 5320 5321 serr = SKB_EXT_ERR(skb); 5322 memset(serr, 0, sizeof(*serr)); 5323 serr->ee.ee_errno = ENOMSG; 5324 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 5325 5326 /* Take a reference to prevent skb_orphan() from freeing the socket, 5327 * but only if the socket refcount is not zero. 5328 */ 5329 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5330 err = sock_queue_err_skb(sk, skb); 5331 sock_put(sk); 5332 } 5333 if (err) 5334 kfree_skb(skb); 5335 } 5336 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 5337 #endif /* CONFIG_WIRELESS */ 5338 5339 /** 5340 * skb_partial_csum_set - set up and verify partial csum values for packet 5341 * @skb: the skb to set 5342 * @start: the number of bytes after skb->data to start checksumming. 5343 * @off: the offset from start to place the checksum. 5344 * 5345 * For untrusted partially-checksummed packets, we need to make sure the values 5346 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 5347 * 5348 * This function checks and sets those values and skb->ip_summed: if this 5349 * returns false you should drop the packet. 5350 */ 5351 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 5352 { 5353 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16); 5354 u32 csum_start = skb_headroom(skb) + (u32)start; 5355 5356 if (unlikely(csum_start >= U16_MAX || csum_end > skb_headlen(skb))) { 5357 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n", 5358 start, off, skb_headroom(skb), skb_headlen(skb)); 5359 return false; 5360 } 5361 skb->ip_summed = CHECKSUM_PARTIAL; 5362 skb->csum_start = csum_start; 5363 skb->csum_offset = off; 5364 skb->transport_header = csum_start; 5365 return true; 5366 } 5367 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 5368 5369 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 5370 unsigned int max) 5371 { 5372 if (skb_headlen(skb) >= len) 5373 return 0; 5374 5375 /* If we need to pullup then pullup to the max, so we 5376 * won't need to do it again. 5377 */ 5378 if (max > skb->len) 5379 max = skb->len; 5380 5381 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 5382 return -ENOMEM; 5383 5384 if (skb_headlen(skb) < len) 5385 return -EPROTO; 5386 5387 return 0; 5388 } 5389 5390 #define MAX_TCP_HDR_LEN (15 * 4) 5391 5392 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 5393 typeof(IPPROTO_IP) proto, 5394 unsigned int off) 5395 { 5396 int err; 5397 5398 switch (proto) { 5399 case IPPROTO_TCP: 5400 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 5401 off + MAX_TCP_HDR_LEN); 5402 if (!err && !skb_partial_csum_set(skb, off, 5403 offsetof(struct tcphdr, 5404 check))) 5405 err = -EPROTO; 5406 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 5407 5408 case IPPROTO_UDP: 5409 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 5410 off + sizeof(struct udphdr)); 5411 if (!err && !skb_partial_csum_set(skb, off, 5412 offsetof(struct udphdr, 5413 check))) 5414 err = -EPROTO; 5415 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 5416 } 5417 5418 return ERR_PTR(-EPROTO); 5419 } 5420 5421 /* This value should be large enough to cover a tagged ethernet header plus 5422 * maximally sized IP and TCP or UDP headers. 5423 */ 5424 #define MAX_IP_HDR_LEN 128 5425 5426 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 5427 { 5428 unsigned int off; 5429 bool fragment; 5430 __sum16 *csum; 5431 int err; 5432 5433 fragment = false; 5434 5435 err = skb_maybe_pull_tail(skb, 5436 sizeof(struct iphdr), 5437 MAX_IP_HDR_LEN); 5438 if (err < 0) 5439 goto out; 5440 5441 if (ip_is_fragment(ip_hdr(skb))) 5442 fragment = true; 5443 5444 off = ip_hdrlen(skb); 5445 5446 err = -EPROTO; 5447 5448 if (fragment) 5449 goto out; 5450 5451 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 5452 if (IS_ERR(csum)) 5453 return PTR_ERR(csum); 5454 5455 if (recalculate) 5456 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 5457 ip_hdr(skb)->daddr, 5458 skb->len - off, 5459 ip_hdr(skb)->protocol, 0); 5460 err = 0; 5461 5462 out: 5463 return err; 5464 } 5465 5466 /* This value should be large enough to cover a tagged ethernet header plus 5467 * an IPv6 header, all options, and a maximal TCP or UDP header. 5468 */ 5469 #define MAX_IPV6_HDR_LEN 256 5470 5471 #define OPT_HDR(type, skb, off) \ 5472 (type *)(skb_network_header(skb) + (off)) 5473 5474 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 5475 { 5476 int err; 5477 u8 nexthdr; 5478 unsigned int off; 5479 unsigned int len; 5480 bool fragment; 5481 bool done; 5482 __sum16 *csum; 5483 5484 fragment = false; 5485 done = false; 5486 5487 off = sizeof(struct ipv6hdr); 5488 5489 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 5490 if (err < 0) 5491 goto out; 5492 5493 nexthdr = ipv6_hdr(skb)->nexthdr; 5494 5495 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 5496 while (off <= len && !done) { 5497 switch (nexthdr) { 5498 case IPPROTO_DSTOPTS: 5499 case IPPROTO_HOPOPTS: 5500 case IPPROTO_ROUTING: { 5501 struct ipv6_opt_hdr *hp; 5502 5503 err = skb_maybe_pull_tail(skb, 5504 off + 5505 sizeof(struct ipv6_opt_hdr), 5506 MAX_IPV6_HDR_LEN); 5507 if (err < 0) 5508 goto out; 5509 5510 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 5511 nexthdr = hp->nexthdr; 5512 off += ipv6_optlen(hp); 5513 break; 5514 } 5515 case IPPROTO_AH: { 5516 struct ip_auth_hdr *hp; 5517 5518 err = skb_maybe_pull_tail(skb, 5519 off + 5520 sizeof(struct ip_auth_hdr), 5521 MAX_IPV6_HDR_LEN); 5522 if (err < 0) 5523 goto out; 5524 5525 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 5526 nexthdr = hp->nexthdr; 5527 off += ipv6_authlen(hp); 5528 break; 5529 } 5530 case IPPROTO_FRAGMENT: { 5531 struct frag_hdr *hp; 5532 5533 err = skb_maybe_pull_tail(skb, 5534 off + 5535 sizeof(struct frag_hdr), 5536 MAX_IPV6_HDR_LEN); 5537 if (err < 0) 5538 goto out; 5539 5540 hp = OPT_HDR(struct frag_hdr, skb, off); 5541 5542 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 5543 fragment = true; 5544 5545 nexthdr = hp->nexthdr; 5546 off += sizeof(struct frag_hdr); 5547 break; 5548 } 5549 default: 5550 done = true; 5551 break; 5552 } 5553 } 5554 5555 err = -EPROTO; 5556 5557 if (!done || fragment) 5558 goto out; 5559 5560 csum = skb_checksum_setup_ip(skb, nexthdr, off); 5561 if (IS_ERR(csum)) 5562 return PTR_ERR(csum); 5563 5564 if (recalculate) 5565 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 5566 &ipv6_hdr(skb)->daddr, 5567 skb->len - off, nexthdr, 0); 5568 err = 0; 5569 5570 out: 5571 return err; 5572 } 5573 5574 /** 5575 * skb_checksum_setup - set up partial checksum offset 5576 * @skb: the skb to set up 5577 * @recalculate: if true the pseudo-header checksum will be recalculated 5578 */ 5579 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 5580 { 5581 int err; 5582 5583 switch (skb->protocol) { 5584 case htons(ETH_P_IP): 5585 err = skb_checksum_setup_ipv4(skb, recalculate); 5586 break; 5587 5588 case htons(ETH_P_IPV6): 5589 err = skb_checksum_setup_ipv6(skb, recalculate); 5590 break; 5591 5592 default: 5593 err = -EPROTO; 5594 break; 5595 } 5596 5597 return err; 5598 } 5599 EXPORT_SYMBOL(skb_checksum_setup); 5600 5601 /** 5602 * skb_checksum_maybe_trim - maybe trims the given skb 5603 * @skb: the skb to check 5604 * @transport_len: the data length beyond the network header 5605 * 5606 * Checks whether the given skb has data beyond the given transport length. 5607 * If so, returns a cloned skb trimmed to this transport length. 5608 * Otherwise returns the provided skb. Returns NULL in error cases 5609 * (e.g. transport_len exceeds skb length or out-of-memory). 5610 * 5611 * Caller needs to set the skb transport header and free any returned skb if it 5612 * differs from the provided skb. 5613 */ 5614 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 5615 unsigned int transport_len) 5616 { 5617 struct sk_buff *skb_chk; 5618 unsigned int len = skb_transport_offset(skb) + transport_len; 5619 int ret; 5620 5621 if (skb->len < len) 5622 return NULL; 5623 else if (skb->len == len) 5624 return skb; 5625 5626 skb_chk = skb_clone(skb, GFP_ATOMIC); 5627 if (!skb_chk) 5628 return NULL; 5629 5630 ret = pskb_trim_rcsum(skb_chk, len); 5631 if (ret) { 5632 kfree_skb(skb_chk); 5633 return NULL; 5634 } 5635 5636 return skb_chk; 5637 } 5638 5639 /** 5640 * skb_checksum_trimmed - validate checksum of an skb 5641 * @skb: the skb to check 5642 * @transport_len: the data length beyond the network header 5643 * @skb_chkf: checksum function to use 5644 * 5645 * Applies the given checksum function skb_chkf to the provided skb. 5646 * Returns a checked and maybe trimmed skb. Returns NULL on error. 5647 * 5648 * If the skb has data beyond the given transport length, then a 5649 * trimmed & cloned skb is checked and returned. 5650 * 5651 * Caller needs to set the skb transport header and free any returned skb if it 5652 * differs from the provided skb. 5653 */ 5654 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 5655 unsigned int transport_len, 5656 __sum16(*skb_chkf)(struct sk_buff *skb)) 5657 { 5658 struct sk_buff *skb_chk; 5659 unsigned int offset = skb_transport_offset(skb); 5660 __sum16 ret; 5661 5662 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 5663 if (!skb_chk) 5664 goto err; 5665 5666 if (!pskb_may_pull(skb_chk, offset)) 5667 goto err; 5668 5669 skb_pull_rcsum(skb_chk, offset); 5670 ret = skb_chkf(skb_chk); 5671 skb_push_rcsum(skb_chk, offset); 5672 5673 if (ret) 5674 goto err; 5675 5676 return skb_chk; 5677 5678 err: 5679 if (skb_chk && skb_chk != skb) 5680 kfree_skb(skb_chk); 5681 5682 return NULL; 5683 5684 } 5685 EXPORT_SYMBOL(skb_checksum_trimmed); 5686 5687 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 5688 { 5689 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 5690 skb->dev->name); 5691 } 5692 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 5693 5694 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 5695 { 5696 if (head_stolen) { 5697 skb_release_head_state(skb); 5698 kmem_cache_free(skbuff_cache, skb); 5699 } else { 5700 __kfree_skb(skb); 5701 } 5702 } 5703 EXPORT_SYMBOL(kfree_skb_partial); 5704 5705 /** 5706 * skb_try_coalesce - try to merge skb to prior one 5707 * @to: prior buffer 5708 * @from: buffer to add 5709 * @fragstolen: pointer to boolean 5710 * @delta_truesize: how much more was allocated than was requested 5711 */ 5712 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 5713 bool *fragstolen, int *delta_truesize) 5714 { 5715 struct skb_shared_info *to_shinfo, *from_shinfo; 5716 int i, delta, len = from->len; 5717 5718 *fragstolen = false; 5719 5720 if (skb_cloned(to)) 5721 return false; 5722 5723 /* In general, avoid mixing page_pool and non-page_pool allocated 5724 * pages within the same SKB. Additionally avoid dealing with clones 5725 * with page_pool pages, in case the SKB is using page_pool fragment 5726 * references (PP_FLAG_PAGE_FRAG). Since we only take full page 5727 * references for cloned SKBs at the moment that would result in 5728 * inconsistent reference counts. 5729 * In theory we could take full references if @from is cloned and 5730 * !@to->pp_recycle but its tricky (due to potential race with 5731 * the clone disappearing) and rare, so not worth dealing with. 5732 */ 5733 if (to->pp_recycle != from->pp_recycle || 5734 (from->pp_recycle && skb_cloned(from))) 5735 return false; 5736 5737 if (len <= skb_tailroom(to)) { 5738 if (len) 5739 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 5740 *delta_truesize = 0; 5741 return true; 5742 } 5743 5744 to_shinfo = skb_shinfo(to); 5745 from_shinfo = skb_shinfo(from); 5746 if (to_shinfo->frag_list || from_shinfo->frag_list) 5747 return false; 5748 if (skb_zcopy(to) || skb_zcopy(from)) 5749 return false; 5750 5751 if (skb_headlen(from) != 0) { 5752 struct page *page; 5753 unsigned int offset; 5754 5755 if (to_shinfo->nr_frags + 5756 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 5757 return false; 5758 5759 if (skb_head_is_locked(from)) 5760 return false; 5761 5762 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 5763 5764 page = virt_to_head_page(from->head); 5765 offset = from->data - (unsigned char *)page_address(page); 5766 5767 skb_fill_page_desc(to, to_shinfo->nr_frags, 5768 page, offset, skb_headlen(from)); 5769 *fragstolen = true; 5770 } else { 5771 if (to_shinfo->nr_frags + 5772 from_shinfo->nr_frags > MAX_SKB_FRAGS) 5773 return false; 5774 5775 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 5776 } 5777 5778 WARN_ON_ONCE(delta < len); 5779 5780 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 5781 from_shinfo->frags, 5782 from_shinfo->nr_frags * sizeof(skb_frag_t)); 5783 to_shinfo->nr_frags += from_shinfo->nr_frags; 5784 5785 if (!skb_cloned(from)) 5786 from_shinfo->nr_frags = 0; 5787 5788 /* if the skb is not cloned this does nothing 5789 * since we set nr_frags to 0. 5790 */ 5791 for (i = 0; i < from_shinfo->nr_frags; i++) 5792 __skb_frag_ref(&from_shinfo->frags[i]); 5793 5794 to->truesize += delta; 5795 to->len += len; 5796 to->data_len += len; 5797 5798 *delta_truesize = delta; 5799 return true; 5800 } 5801 EXPORT_SYMBOL(skb_try_coalesce); 5802 5803 /** 5804 * skb_scrub_packet - scrub an skb 5805 * 5806 * @skb: buffer to clean 5807 * @xnet: packet is crossing netns 5808 * 5809 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 5810 * into/from a tunnel. Some information have to be cleared during these 5811 * operations. 5812 * skb_scrub_packet can also be used to clean a skb before injecting it in 5813 * another namespace (@xnet == true). We have to clear all information in the 5814 * skb that could impact namespace isolation. 5815 */ 5816 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 5817 { 5818 skb->pkt_type = PACKET_HOST; 5819 skb->skb_iif = 0; 5820 skb->ignore_df = 0; 5821 skb_dst_drop(skb); 5822 skb_ext_reset(skb); 5823 nf_reset_ct(skb); 5824 nf_reset_trace(skb); 5825 5826 #ifdef CONFIG_NET_SWITCHDEV 5827 skb->offload_fwd_mark = 0; 5828 skb->offload_l3_fwd_mark = 0; 5829 #endif 5830 5831 if (!xnet) 5832 return; 5833 5834 ipvs_reset(skb); 5835 skb->mark = 0; 5836 skb_clear_tstamp(skb); 5837 } 5838 EXPORT_SYMBOL_GPL(skb_scrub_packet); 5839 5840 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 5841 { 5842 int mac_len, meta_len; 5843 void *meta; 5844 5845 if (skb_cow(skb, skb_headroom(skb)) < 0) { 5846 kfree_skb(skb); 5847 return NULL; 5848 } 5849 5850 mac_len = skb->data - skb_mac_header(skb); 5851 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) { 5852 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), 5853 mac_len - VLAN_HLEN - ETH_TLEN); 5854 } 5855 5856 meta_len = skb_metadata_len(skb); 5857 if (meta_len) { 5858 meta = skb_metadata_end(skb) - meta_len; 5859 memmove(meta + VLAN_HLEN, meta, meta_len); 5860 } 5861 5862 skb->mac_header += VLAN_HLEN; 5863 return skb; 5864 } 5865 5866 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 5867 { 5868 struct vlan_hdr *vhdr; 5869 u16 vlan_tci; 5870 5871 if (unlikely(skb_vlan_tag_present(skb))) { 5872 /* vlan_tci is already set-up so leave this for another time */ 5873 return skb; 5874 } 5875 5876 skb = skb_share_check(skb, GFP_ATOMIC); 5877 if (unlikely(!skb)) 5878 goto err_free; 5879 /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */ 5880 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short)))) 5881 goto err_free; 5882 5883 vhdr = (struct vlan_hdr *)skb->data; 5884 vlan_tci = ntohs(vhdr->h_vlan_TCI); 5885 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 5886 5887 skb_pull_rcsum(skb, VLAN_HLEN); 5888 vlan_set_encap_proto(skb, vhdr); 5889 5890 skb = skb_reorder_vlan_header(skb); 5891 if (unlikely(!skb)) 5892 goto err_free; 5893 5894 skb_reset_network_header(skb); 5895 if (!skb_transport_header_was_set(skb)) 5896 skb_reset_transport_header(skb); 5897 skb_reset_mac_len(skb); 5898 5899 return skb; 5900 5901 err_free: 5902 kfree_skb(skb); 5903 return NULL; 5904 } 5905 EXPORT_SYMBOL(skb_vlan_untag); 5906 5907 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len) 5908 { 5909 if (!pskb_may_pull(skb, write_len)) 5910 return -ENOMEM; 5911 5912 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 5913 return 0; 5914 5915 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 5916 } 5917 EXPORT_SYMBOL(skb_ensure_writable); 5918 5919 /* remove VLAN header from packet and update csum accordingly. 5920 * expects a non skb_vlan_tag_present skb with a vlan tag payload 5921 */ 5922 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 5923 { 5924 int offset = skb->data - skb_mac_header(skb); 5925 int err; 5926 5927 if (WARN_ONCE(offset, 5928 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 5929 offset)) { 5930 return -EINVAL; 5931 } 5932 5933 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 5934 if (unlikely(err)) 5935 return err; 5936 5937 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5938 5939 vlan_remove_tag(skb, vlan_tci); 5940 5941 skb->mac_header += VLAN_HLEN; 5942 5943 if (skb_network_offset(skb) < ETH_HLEN) 5944 skb_set_network_header(skb, ETH_HLEN); 5945 5946 skb_reset_mac_len(skb); 5947 5948 return err; 5949 } 5950 EXPORT_SYMBOL(__skb_vlan_pop); 5951 5952 /* Pop a vlan tag either from hwaccel or from payload. 5953 * Expects skb->data at mac header. 5954 */ 5955 int skb_vlan_pop(struct sk_buff *skb) 5956 { 5957 u16 vlan_tci; 5958 __be16 vlan_proto; 5959 int err; 5960 5961 if (likely(skb_vlan_tag_present(skb))) { 5962 __vlan_hwaccel_clear_tag(skb); 5963 } else { 5964 if (unlikely(!eth_type_vlan(skb->protocol))) 5965 return 0; 5966 5967 err = __skb_vlan_pop(skb, &vlan_tci); 5968 if (err) 5969 return err; 5970 } 5971 /* move next vlan tag to hw accel tag */ 5972 if (likely(!eth_type_vlan(skb->protocol))) 5973 return 0; 5974 5975 vlan_proto = skb->protocol; 5976 err = __skb_vlan_pop(skb, &vlan_tci); 5977 if (unlikely(err)) 5978 return err; 5979 5980 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5981 return 0; 5982 } 5983 EXPORT_SYMBOL(skb_vlan_pop); 5984 5985 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 5986 * Expects skb->data at mac header. 5987 */ 5988 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 5989 { 5990 if (skb_vlan_tag_present(skb)) { 5991 int offset = skb->data - skb_mac_header(skb); 5992 int err; 5993 5994 if (WARN_ONCE(offset, 5995 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 5996 offset)) { 5997 return -EINVAL; 5998 } 5999 6000 err = __vlan_insert_tag(skb, skb->vlan_proto, 6001 skb_vlan_tag_get(skb)); 6002 if (err) 6003 return err; 6004 6005 skb->protocol = skb->vlan_proto; 6006 skb->mac_len += VLAN_HLEN; 6007 6008 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 6009 } 6010 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 6011 return 0; 6012 } 6013 EXPORT_SYMBOL(skb_vlan_push); 6014 6015 /** 6016 * skb_eth_pop() - Drop the Ethernet header at the head of a packet 6017 * 6018 * @skb: Socket buffer to modify 6019 * 6020 * Drop the Ethernet header of @skb. 6021 * 6022 * Expects that skb->data points to the mac header and that no VLAN tags are 6023 * present. 6024 * 6025 * Returns 0 on success, -errno otherwise. 6026 */ 6027 int skb_eth_pop(struct sk_buff *skb) 6028 { 6029 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) || 6030 skb_network_offset(skb) < ETH_HLEN) 6031 return -EPROTO; 6032 6033 skb_pull_rcsum(skb, ETH_HLEN); 6034 skb_reset_mac_header(skb); 6035 skb_reset_mac_len(skb); 6036 6037 return 0; 6038 } 6039 EXPORT_SYMBOL(skb_eth_pop); 6040 6041 /** 6042 * skb_eth_push() - Add a new Ethernet header at the head of a packet 6043 * 6044 * @skb: Socket buffer to modify 6045 * @dst: Destination MAC address of the new header 6046 * @src: Source MAC address of the new header 6047 * 6048 * Prepend @skb with a new Ethernet header. 6049 * 6050 * Expects that skb->data points to the mac header, which must be empty. 6051 * 6052 * Returns 0 on success, -errno otherwise. 6053 */ 6054 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst, 6055 const unsigned char *src) 6056 { 6057 struct ethhdr *eth; 6058 int err; 6059 6060 if (skb_network_offset(skb) || skb_vlan_tag_present(skb)) 6061 return -EPROTO; 6062 6063 err = skb_cow_head(skb, sizeof(*eth)); 6064 if (err < 0) 6065 return err; 6066 6067 skb_push(skb, sizeof(*eth)); 6068 skb_reset_mac_header(skb); 6069 skb_reset_mac_len(skb); 6070 6071 eth = eth_hdr(skb); 6072 ether_addr_copy(eth->h_dest, dst); 6073 ether_addr_copy(eth->h_source, src); 6074 eth->h_proto = skb->protocol; 6075 6076 skb_postpush_rcsum(skb, eth, sizeof(*eth)); 6077 6078 return 0; 6079 } 6080 EXPORT_SYMBOL(skb_eth_push); 6081 6082 /* Update the ethertype of hdr and the skb csum value if required. */ 6083 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr, 6084 __be16 ethertype) 6085 { 6086 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6087 __be16 diff[] = { ~hdr->h_proto, ethertype }; 6088 6089 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6090 } 6091 6092 hdr->h_proto = ethertype; 6093 } 6094 6095 /** 6096 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of 6097 * the packet 6098 * 6099 * @skb: buffer 6100 * @mpls_lse: MPLS label stack entry to push 6101 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848) 6102 * @mac_len: length of the MAC header 6103 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is 6104 * ethernet 6105 * 6106 * Expects skb->data at mac header. 6107 * 6108 * Returns 0 on success, -errno otherwise. 6109 */ 6110 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto, 6111 int mac_len, bool ethernet) 6112 { 6113 struct mpls_shim_hdr *lse; 6114 int err; 6115 6116 if (unlikely(!eth_p_mpls(mpls_proto))) 6117 return -EINVAL; 6118 6119 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */ 6120 if (skb->encapsulation) 6121 return -EINVAL; 6122 6123 err = skb_cow_head(skb, MPLS_HLEN); 6124 if (unlikely(err)) 6125 return err; 6126 6127 if (!skb->inner_protocol) { 6128 skb_set_inner_network_header(skb, skb_network_offset(skb)); 6129 skb_set_inner_protocol(skb, skb->protocol); 6130 } 6131 6132 skb_push(skb, MPLS_HLEN); 6133 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb), 6134 mac_len); 6135 skb_reset_mac_header(skb); 6136 skb_set_network_header(skb, mac_len); 6137 skb_reset_mac_len(skb); 6138 6139 lse = mpls_hdr(skb); 6140 lse->label_stack_entry = mpls_lse; 6141 skb_postpush_rcsum(skb, lse, MPLS_HLEN); 6142 6143 if (ethernet && mac_len >= ETH_HLEN) 6144 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto); 6145 skb->protocol = mpls_proto; 6146 6147 return 0; 6148 } 6149 EXPORT_SYMBOL_GPL(skb_mpls_push); 6150 6151 /** 6152 * skb_mpls_pop() - pop the outermost MPLS header 6153 * 6154 * @skb: buffer 6155 * @next_proto: ethertype of header after popped MPLS header 6156 * @mac_len: length of the MAC header 6157 * @ethernet: flag to indicate if the packet is ethernet 6158 * 6159 * Expects skb->data at mac header. 6160 * 6161 * Returns 0 on success, -errno otherwise. 6162 */ 6163 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, 6164 bool ethernet) 6165 { 6166 int err; 6167 6168 if (unlikely(!eth_p_mpls(skb->protocol))) 6169 return 0; 6170 6171 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN); 6172 if (unlikely(err)) 6173 return err; 6174 6175 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN); 6176 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb), 6177 mac_len); 6178 6179 __skb_pull(skb, MPLS_HLEN); 6180 skb_reset_mac_header(skb); 6181 skb_set_network_header(skb, mac_len); 6182 6183 if (ethernet && mac_len >= ETH_HLEN) { 6184 struct ethhdr *hdr; 6185 6186 /* use mpls_hdr() to get ethertype to account for VLANs. */ 6187 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN); 6188 skb_mod_eth_type(skb, hdr, next_proto); 6189 } 6190 skb->protocol = next_proto; 6191 6192 return 0; 6193 } 6194 EXPORT_SYMBOL_GPL(skb_mpls_pop); 6195 6196 /** 6197 * skb_mpls_update_lse() - modify outermost MPLS header and update csum 6198 * 6199 * @skb: buffer 6200 * @mpls_lse: new MPLS label stack entry to update to 6201 * 6202 * Expects skb->data at mac header. 6203 * 6204 * Returns 0 on success, -errno otherwise. 6205 */ 6206 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse) 6207 { 6208 int err; 6209 6210 if (unlikely(!eth_p_mpls(skb->protocol))) 6211 return -EINVAL; 6212 6213 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN); 6214 if (unlikely(err)) 6215 return err; 6216 6217 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6218 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse }; 6219 6220 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6221 } 6222 6223 mpls_hdr(skb)->label_stack_entry = mpls_lse; 6224 6225 return 0; 6226 } 6227 EXPORT_SYMBOL_GPL(skb_mpls_update_lse); 6228 6229 /** 6230 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header 6231 * 6232 * @skb: buffer 6233 * 6234 * Expects skb->data at mac header. 6235 * 6236 * Returns 0 on success, -errno otherwise. 6237 */ 6238 int skb_mpls_dec_ttl(struct sk_buff *skb) 6239 { 6240 u32 lse; 6241 u8 ttl; 6242 6243 if (unlikely(!eth_p_mpls(skb->protocol))) 6244 return -EINVAL; 6245 6246 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN)) 6247 return -ENOMEM; 6248 6249 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry); 6250 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT; 6251 if (!--ttl) 6252 return -EINVAL; 6253 6254 lse &= ~MPLS_LS_TTL_MASK; 6255 lse |= ttl << MPLS_LS_TTL_SHIFT; 6256 6257 return skb_mpls_update_lse(skb, cpu_to_be32(lse)); 6258 } 6259 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); 6260 6261 /** 6262 * alloc_skb_with_frags - allocate skb with page frags 6263 * 6264 * @header_len: size of linear part 6265 * @data_len: needed length in frags 6266 * @order: max page order desired. 6267 * @errcode: pointer to error code if any 6268 * @gfp_mask: allocation mask 6269 * 6270 * This can be used to allocate a paged skb, given a maximal order for frags. 6271 */ 6272 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 6273 unsigned long data_len, 6274 int order, 6275 int *errcode, 6276 gfp_t gfp_mask) 6277 { 6278 unsigned long chunk; 6279 struct sk_buff *skb; 6280 struct page *page; 6281 int nr_frags = 0; 6282 6283 *errcode = -EMSGSIZE; 6284 if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order))) 6285 return NULL; 6286 6287 *errcode = -ENOBUFS; 6288 skb = alloc_skb(header_len, gfp_mask); 6289 if (!skb) 6290 return NULL; 6291 6292 while (data_len) { 6293 if (nr_frags == MAX_SKB_FRAGS - 1) 6294 goto failure; 6295 while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order)) 6296 order--; 6297 6298 if (order) { 6299 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 6300 __GFP_COMP | 6301 __GFP_NOWARN, 6302 order); 6303 if (!page) { 6304 order--; 6305 continue; 6306 } 6307 } else { 6308 page = alloc_page(gfp_mask); 6309 if (!page) 6310 goto failure; 6311 } 6312 chunk = min_t(unsigned long, data_len, 6313 PAGE_SIZE << order); 6314 skb_fill_page_desc(skb, nr_frags, page, 0, chunk); 6315 nr_frags++; 6316 skb->truesize += (PAGE_SIZE << order); 6317 data_len -= chunk; 6318 } 6319 return skb; 6320 6321 failure: 6322 kfree_skb(skb); 6323 return NULL; 6324 } 6325 EXPORT_SYMBOL(alloc_skb_with_frags); 6326 6327 /* carve out the first off bytes from skb when off < headlen */ 6328 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 6329 const int headlen, gfp_t gfp_mask) 6330 { 6331 int i; 6332 unsigned int size = skb_end_offset(skb); 6333 int new_hlen = headlen - off; 6334 u8 *data; 6335 6336 if (skb_pfmemalloc(skb)) 6337 gfp_mask |= __GFP_MEMALLOC; 6338 6339 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 6340 if (!data) 6341 return -ENOMEM; 6342 size = SKB_WITH_OVERHEAD(size); 6343 6344 /* Copy real data, and all frags */ 6345 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 6346 skb->len -= off; 6347 6348 memcpy((struct skb_shared_info *)(data + size), 6349 skb_shinfo(skb), 6350 offsetof(struct skb_shared_info, 6351 frags[skb_shinfo(skb)->nr_frags])); 6352 if (skb_cloned(skb)) { 6353 /* drop the old head gracefully */ 6354 if (skb_orphan_frags(skb, gfp_mask)) { 6355 skb_kfree_head(data, size); 6356 return -ENOMEM; 6357 } 6358 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 6359 skb_frag_ref(skb, i); 6360 if (skb_has_frag_list(skb)) 6361 skb_clone_fraglist(skb); 6362 skb_release_data(skb, SKB_CONSUMED, false); 6363 } else { 6364 /* we can reuse existing recount- all we did was 6365 * relocate values 6366 */ 6367 skb_free_head(skb, false); 6368 } 6369 6370 skb->head = data; 6371 skb->data = data; 6372 skb->head_frag = 0; 6373 skb_set_end_offset(skb, size); 6374 skb_set_tail_pointer(skb, skb_headlen(skb)); 6375 skb_headers_offset_update(skb, 0); 6376 skb->cloned = 0; 6377 skb->hdr_len = 0; 6378 skb->nohdr = 0; 6379 atomic_set(&skb_shinfo(skb)->dataref, 1); 6380 6381 return 0; 6382 } 6383 6384 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 6385 6386 /* carve out the first eat bytes from skb's frag_list. May recurse into 6387 * pskb_carve() 6388 */ 6389 static int pskb_carve_frag_list(struct sk_buff *skb, 6390 struct skb_shared_info *shinfo, int eat, 6391 gfp_t gfp_mask) 6392 { 6393 struct sk_buff *list = shinfo->frag_list; 6394 struct sk_buff *clone = NULL; 6395 struct sk_buff *insp = NULL; 6396 6397 do { 6398 if (!list) { 6399 pr_err("Not enough bytes to eat. Want %d\n", eat); 6400 return -EFAULT; 6401 } 6402 if (list->len <= eat) { 6403 /* Eaten as whole. */ 6404 eat -= list->len; 6405 list = list->next; 6406 insp = list; 6407 } else { 6408 /* Eaten partially. */ 6409 if (skb_shared(list)) { 6410 clone = skb_clone(list, gfp_mask); 6411 if (!clone) 6412 return -ENOMEM; 6413 insp = list->next; 6414 list = clone; 6415 } else { 6416 /* This may be pulled without problems. */ 6417 insp = list; 6418 } 6419 if (pskb_carve(list, eat, gfp_mask) < 0) { 6420 kfree_skb(clone); 6421 return -ENOMEM; 6422 } 6423 break; 6424 } 6425 } while (eat); 6426 6427 /* Free pulled out fragments. */ 6428 while ((list = shinfo->frag_list) != insp) { 6429 shinfo->frag_list = list->next; 6430 consume_skb(list); 6431 } 6432 /* And insert new clone at head. */ 6433 if (clone) { 6434 clone->next = list; 6435 shinfo->frag_list = clone; 6436 } 6437 return 0; 6438 } 6439 6440 /* carve off first len bytes from skb. Split line (off) is in the 6441 * non-linear part of skb 6442 */ 6443 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 6444 int pos, gfp_t gfp_mask) 6445 { 6446 int i, k = 0; 6447 unsigned int size = skb_end_offset(skb); 6448 u8 *data; 6449 const int nfrags = skb_shinfo(skb)->nr_frags; 6450 struct skb_shared_info *shinfo; 6451 6452 if (skb_pfmemalloc(skb)) 6453 gfp_mask |= __GFP_MEMALLOC; 6454 6455 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 6456 if (!data) 6457 return -ENOMEM; 6458 size = SKB_WITH_OVERHEAD(size); 6459 6460 memcpy((struct skb_shared_info *)(data + size), 6461 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0])); 6462 if (skb_orphan_frags(skb, gfp_mask)) { 6463 skb_kfree_head(data, size); 6464 return -ENOMEM; 6465 } 6466 shinfo = (struct skb_shared_info *)(data + size); 6467 for (i = 0; i < nfrags; i++) { 6468 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 6469 6470 if (pos + fsize > off) { 6471 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 6472 6473 if (pos < off) { 6474 /* Split frag. 6475 * We have two variants in this case: 6476 * 1. Move all the frag to the second 6477 * part, if it is possible. F.e. 6478 * this approach is mandatory for TUX, 6479 * where splitting is expensive. 6480 * 2. Split is accurately. We make this. 6481 */ 6482 skb_frag_off_add(&shinfo->frags[0], off - pos); 6483 skb_frag_size_sub(&shinfo->frags[0], off - pos); 6484 } 6485 skb_frag_ref(skb, i); 6486 k++; 6487 } 6488 pos += fsize; 6489 } 6490 shinfo->nr_frags = k; 6491 if (skb_has_frag_list(skb)) 6492 skb_clone_fraglist(skb); 6493 6494 /* split line is in frag list */ 6495 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) { 6496 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */ 6497 if (skb_has_frag_list(skb)) 6498 kfree_skb_list(skb_shinfo(skb)->frag_list); 6499 skb_kfree_head(data, size); 6500 return -ENOMEM; 6501 } 6502 skb_release_data(skb, SKB_CONSUMED, false); 6503 6504 skb->head = data; 6505 skb->head_frag = 0; 6506 skb->data = data; 6507 skb_set_end_offset(skb, size); 6508 skb_reset_tail_pointer(skb); 6509 skb_headers_offset_update(skb, 0); 6510 skb->cloned = 0; 6511 skb->hdr_len = 0; 6512 skb->nohdr = 0; 6513 skb->len -= off; 6514 skb->data_len = skb->len; 6515 atomic_set(&skb_shinfo(skb)->dataref, 1); 6516 return 0; 6517 } 6518 6519 /* remove len bytes from the beginning of the skb */ 6520 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 6521 { 6522 int headlen = skb_headlen(skb); 6523 6524 if (len < headlen) 6525 return pskb_carve_inside_header(skb, len, headlen, gfp); 6526 else 6527 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 6528 } 6529 6530 /* Extract to_copy bytes starting at off from skb, and return this in 6531 * a new skb 6532 */ 6533 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 6534 int to_copy, gfp_t gfp) 6535 { 6536 struct sk_buff *clone = skb_clone(skb, gfp); 6537 6538 if (!clone) 6539 return NULL; 6540 6541 if (pskb_carve(clone, off, gfp) < 0 || 6542 pskb_trim(clone, to_copy)) { 6543 kfree_skb(clone); 6544 return NULL; 6545 } 6546 return clone; 6547 } 6548 EXPORT_SYMBOL(pskb_extract); 6549 6550 /** 6551 * skb_condense - try to get rid of fragments/frag_list if possible 6552 * @skb: buffer 6553 * 6554 * Can be used to save memory before skb is added to a busy queue. 6555 * If packet has bytes in frags and enough tail room in skb->head, 6556 * pull all of them, so that we can free the frags right now and adjust 6557 * truesize. 6558 * Notes: 6559 * We do not reallocate skb->head thus can not fail. 6560 * Caller must re-evaluate skb->truesize if needed. 6561 */ 6562 void skb_condense(struct sk_buff *skb) 6563 { 6564 if (skb->data_len) { 6565 if (skb->data_len > skb->end - skb->tail || 6566 skb_cloned(skb)) 6567 return; 6568 6569 /* Nice, we can free page frag(s) right now */ 6570 __pskb_pull_tail(skb, skb->data_len); 6571 } 6572 /* At this point, skb->truesize might be over estimated, 6573 * because skb had a fragment, and fragments do not tell 6574 * their truesize. 6575 * When we pulled its content into skb->head, fragment 6576 * was freed, but __pskb_pull_tail() could not possibly 6577 * adjust skb->truesize, not knowing the frag truesize. 6578 */ 6579 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 6580 } 6581 EXPORT_SYMBOL(skb_condense); 6582 6583 #ifdef CONFIG_SKB_EXTENSIONS 6584 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id) 6585 { 6586 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE); 6587 } 6588 6589 /** 6590 * __skb_ext_alloc - allocate a new skb extensions storage 6591 * 6592 * @flags: See kmalloc(). 6593 * 6594 * Returns the newly allocated pointer. The pointer can later attached to a 6595 * skb via __skb_ext_set(). 6596 * Note: caller must handle the skb_ext as an opaque data. 6597 */ 6598 struct skb_ext *__skb_ext_alloc(gfp_t flags) 6599 { 6600 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags); 6601 6602 if (new) { 6603 memset(new->offset, 0, sizeof(new->offset)); 6604 refcount_set(&new->refcnt, 1); 6605 } 6606 6607 return new; 6608 } 6609 6610 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old, 6611 unsigned int old_active) 6612 { 6613 struct skb_ext *new; 6614 6615 if (refcount_read(&old->refcnt) == 1) 6616 return old; 6617 6618 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 6619 if (!new) 6620 return NULL; 6621 6622 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE); 6623 refcount_set(&new->refcnt, 1); 6624 6625 #ifdef CONFIG_XFRM 6626 if (old_active & (1 << SKB_EXT_SEC_PATH)) { 6627 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH); 6628 unsigned int i; 6629 6630 for (i = 0; i < sp->len; i++) 6631 xfrm_state_hold(sp->xvec[i]); 6632 } 6633 #endif 6634 __skb_ext_put(old); 6635 return new; 6636 } 6637 6638 /** 6639 * __skb_ext_set - attach the specified extension storage to this skb 6640 * @skb: buffer 6641 * @id: extension id 6642 * @ext: extension storage previously allocated via __skb_ext_alloc() 6643 * 6644 * Existing extensions, if any, are cleared. 6645 * 6646 * Returns the pointer to the extension. 6647 */ 6648 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id, 6649 struct skb_ext *ext) 6650 { 6651 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext); 6652 6653 skb_ext_put(skb); 6654 newlen = newoff + skb_ext_type_len[id]; 6655 ext->chunks = newlen; 6656 ext->offset[id] = newoff; 6657 skb->extensions = ext; 6658 skb->active_extensions = 1 << id; 6659 return skb_ext_get_ptr(ext, id); 6660 } 6661 6662 /** 6663 * skb_ext_add - allocate space for given extension, COW if needed 6664 * @skb: buffer 6665 * @id: extension to allocate space for 6666 * 6667 * Allocates enough space for the given extension. 6668 * If the extension is already present, a pointer to that extension 6669 * is returned. 6670 * 6671 * If the skb was cloned, COW applies and the returned memory can be 6672 * modified without changing the extension space of clones buffers. 6673 * 6674 * Returns pointer to the extension or NULL on allocation failure. 6675 */ 6676 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id) 6677 { 6678 struct skb_ext *new, *old = NULL; 6679 unsigned int newlen, newoff; 6680 6681 if (skb->active_extensions) { 6682 old = skb->extensions; 6683 6684 new = skb_ext_maybe_cow(old, skb->active_extensions); 6685 if (!new) 6686 return NULL; 6687 6688 if (__skb_ext_exist(new, id)) 6689 goto set_active; 6690 6691 newoff = new->chunks; 6692 } else { 6693 newoff = SKB_EXT_CHUNKSIZEOF(*new); 6694 6695 new = __skb_ext_alloc(GFP_ATOMIC); 6696 if (!new) 6697 return NULL; 6698 } 6699 6700 newlen = newoff + skb_ext_type_len[id]; 6701 new->chunks = newlen; 6702 new->offset[id] = newoff; 6703 set_active: 6704 skb->slow_gro = 1; 6705 skb->extensions = new; 6706 skb->active_extensions |= 1 << id; 6707 return skb_ext_get_ptr(new, id); 6708 } 6709 EXPORT_SYMBOL(skb_ext_add); 6710 6711 #ifdef CONFIG_XFRM 6712 static void skb_ext_put_sp(struct sec_path *sp) 6713 { 6714 unsigned int i; 6715 6716 for (i = 0; i < sp->len; i++) 6717 xfrm_state_put(sp->xvec[i]); 6718 } 6719 #endif 6720 6721 #ifdef CONFIG_MCTP_FLOWS 6722 static void skb_ext_put_mctp(struct mctp_flow *flow) 6723 { 6724 if (flow->key) 6725 mctp_key_unref(flow->key); 6726 } 6727 #endif 6728 6729 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 6730 { 6731 struct skb_ext *ext = skb->extensions; 6732 6733 skb->active_extensions &= ~(1 << id); 6734 if (skb->active_extensions == 0) { 6735 skb->extensions = NULL; 6736 __skb_ext_put(ext); 6737 #ifdef CONFIG_XFRM 6738 } else if (id == SKB_EXT_SEC_PATH && 6739 refcount_read(&ext->refcnt) == 1) { 6740 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH); 6741 6742 skb_ext_put_sp(sp); 6743 sp->len = 0; 6744 #endif 6745 } 6746 } 6747 EXPORT_SYMBOL(__skb_ext_del); 6748 6749 void __skb_ext_put(struct skb_ext *ext) 6750 { 6751 /* If this is last clone, nothing can increment 6752 * it after check passes. Avoids one atomic op. 6753 */ 6754 if (refcount_read(&ext->refcnt) == 1) 6755 goto free_now; 6756 6757 if (!refcount_dec_and_test(&ext->refcnt)) 6758 return; 6759 free_now: 6760 #ifdef CONFIG_XFRM 6761 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH)) 6762 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 6763 #endif 6764 #ifdef CONFIG_MCTP_FLOWS 6765 if (__skb_ext_exist(ext, SKB_EXT_MCTP)) 6766 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP)); 6767 #endif 6768 6769 kmem_cache_free(skbuff_ext_cache, ext); 6770 } 6771 EXPORT_SYMBOL(__skb_ext_put); 6772 #endif /* CONFIG_SKB_EXTENSIONS */ 6773 6774 /** 6775 * skb_attempt_defer_free - queue skb for remote freeing 6776 * @skb: buffer 6777 * 6778 * Put @skb in a per-cpu list, using the cpu which 6779 * allocated the skb/pages to reduce false sharing 6780 * and memory zone spinlock contention. 6781 */ 6782 void skb_attempt_defer_free(struct sk_buff *skb) 6783 { 6784 int cpu = skb->alloc_cpu; 6785 struct softnet_data *sd; 6786 unsigned int defer_max; 6787 bool kick; 6788 6789 if (WARN_ON_ONCE(cpu >= nr_cpu_ids) || 6790 !cpu_online(cpu) || 6791 cpu == raw_smp_processor_id()) { 6792 nodefer: __kfree_skb(skb); 6793 return; 6794 } 6795 6796 DEBUG_NET_WARN_ON_ONCE(skb_dst(skb)); 6797 DEBUG_NET_WARN_ON_ONCE(skb->destructor); 6798 6799 sd = &per_cpu(softnet_data, cpu); 6800 defer_max = READ_ONCE(sysctl_skb_defer_max); 6801 if (READ_ONCE(sd->defer_count) >= defer_max) 6802 goto nodefer; 6803 6804 spin_lock_bh(&sd->defer_lock); 6805 /* Send an IPI every time queue reaches half capacity. */ 6806 kick = sd->defer_count == (defer_max >> 1); 6807 /* Paired with the READ_ONCE() few lines above */ 6808 WRITE_ONCE(sd->defer_count, sd->defer_count + 1); 6809 6810 skb->next = sd->defer_list; 6811 /* Paired with READ_ONCE() in skb_defer_free_flush() */ 6812 WRITE_ONCE(sd->defer_list, skb); 6813 spin_unlock_bh(&sd->defer_lock); 6814 6815 /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU 6816 * if we are unlucky enough (this seems very unlikely). 6817 */ 6818 if (unlikely(kick) && !cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) 6819 smp_call_function_single_async(cpu, &sd->defer_csd); 6820 } 6821 6822 static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, 6823 size_t offset, size_t len) 6824 { 6825 const char *kaddr; 6826 __wsum csum; 6827 6828 kaddr = kmap_local_page(page); 6829 csum = csum_partial(kaddr + offset, len, 0); 6830 kunmap_local(kaddr); 6831 skb->csum = csum_block_add(skb->csum, csum, skb->len); 6832 } 6833 6834 /** 6835 * skb_splice_from_iter - Splice (or copy) pages to skbuff 6836 * @skb: The buffer to add pages to 6837 * @iter: Iterator representing the pages to be added 6838 * @maxsize: Maximum amount of pages to be added 6839 * @gfp: Allocation flags 6840 * 6841 * This is a common helper function for supporting MSG_SPLICE_PAGES. It 6842 * extracts pages from an iterator and adds them to the socket buffer if 6843 * possible, copying them to fragments if not possible (such as if they're slab 6844 * pages). 6845 * 6846 * Returns the amount of data spliced/copied or -EMSGSIZE if there's 6847 * insufficient space in the buffer to transfer anything. 6848 */ 6849 ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, 6850 ssize_t maxsize, gfp_t gfp) 6851 { 6852 size_t frag_limit = READ_ONCE(sysctl_max_skb_frags); 6853 struct page *pages[8], **ppages = pages; 6854 ssize_t spliced = 0, ret = 0; 6855 unsigned int i; 6856 6857 while (iter->count > 0) { 6858 ssize_t space, nr, len; 6859 size_t off; 6860 6861 ret = -EMSGSIZE; 6862 space = frag_limit - skb_shinfo(skb)->nr_frags; 6863 if (space < 0) 6864 break; 6865 6866 /* We might be able to coalesce without increasing nr_frags */ 6867 nr = clamp_t(size_t, space, 1, ARRAY_SIZE(pages)); 6868 6869 len = iov_iter_extract_pages(iter, &ppages, maxsize, nr, 0, &off); 6870 if (len <= 0) { 6871 ret = len ?: -EIO; 6872 break; 6873 } 6874 6875 i = 0; 6876 do { 6877 struct page *page = pages[i++]; 6878 size_t part = min_t(size_t, PAGE_SIZE - off, len); 6879 6880 ret = -EIO; 6881 if (WARN_ON_ONCE(!sendpage_ok(page))) 6882 goto out; 6883 6884 ret = skb_append_pagefrags(skb, page, off, part, 6885 frag_limit); 6886 if (ret < 0) { 6887 iov_iter_revert(iter, len); 6888 goto out; 6889 } 6890 6891 if (skb->ip_summed == CHECKSUM_NONE) 6892 skb_splice_csum_page(skb, page, off, part); 6893 6894 off = 0; 6895 spliced += part; 6896 maxsize -= part; 6897 len -= part; 6898 } while (len > 0); 6899 6900 if (maxsize <= 0) 6901 break; 6902 } 6903 6904 out: 6905 skb_len_add(skb, spliced); 6906 return spliced ?: ret; 6907 } 6908 EXPORT_SYMBOL(skb_splice_from_iter); 6909