1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Routines having to do with the 'struct sk_buff' memory handlers. 4 * 5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 6 * Florian La Roche <rzsfl@rz.uni-sb.de> 7 * 8 * Fixes: 9 * Alan Cox : Fixed the worst of the load 10 * balancer bugs. 11 * Dave Platt : Interrupt stacking fix. 12 * Richard Kooijman : Timestamp fixes. 13 * Alan Cox : Changed buffer format. 14 * Alan Cox : destructor hook for AF_UNIX etc. 15 * Linus Torvalds : Better skb_clone. 16 * Alan Cox : Added skb_copy. 17 * Alan Cox : Added all the changed routines Linus 18 * only put in the headers 19 * Ray VanTassle : Fixed --skb->lock in free 20 * Alan Cox : skb_copy copy arp field 21 * Andi Kleen : slabified it. 22 * Robert Olsson : Removed skb_head_pool 23 * 24 * NOTE: 25 * The __skb_ routines should be called with interrupts 26 * disabled, or you better be *real* sure that the operation is atomic 27 * with respect to whatever list is being frobbed (e.g. via lock_sock() 28 * or via disabling bottom half handlers, etc). 29 */ 30 31 /* 32 * The functions in this file will not compile correctly with gcc 2.4.x 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/module.h> 38 #include <linux/types.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/interrupt.h> 42 #include <linux/in.h> 43 #include <linux/inet.h> 44 #include <linux/slab.h> 45 #include <linux/tcp.h> 46 #include <linux/udp.h> 47 #include <linux/sctp.h> 48 #include <linux/netdevice.h> 49 #ifdef CONFIG_NET_CLS_ACT 50 #include <net/pkt_sched.h> 51 #endif 52 #include <linux/string.h> 53 #include <linux/skbuff.h> 54 #include <linux/splice.h> 55 #include <linux/cache.h> 56 #include <linux/rtnetlink.h> 57 #include <linux/init.h> 58 #include <linux/scatterlist.h> 59 #include <linux/errqueue.h> 60 #include <linux/prefetch.h> 61 #include <linux/if_vlan.h> 62 #include <linux/mpls.h> 63 #include <linux/kcov.h> 64 65 #include <net/protocol.h> 66 #include <net/dst.h> 67 #include <net/sock.h> 68 #include <net/checksum.h> 69 #include <net/ip6_checksum.h> 70 #include <net/xfrm.h> 71 #include <net/mpls.h> 72 #include <net/mptcp.h> 73 #include <net/page_pool.h> 74 75 #include <linux/uaccess.h> 76 #include <trace/events/skb.h> 77 #include <linux/highmem.h> 78 #include <linux/capability.h> 79 #include <linux/user_namespace.h> 80 #include <linux/indirect_call_wrapper.h> 81 82 #include "datagram.h" 83 84 struct kmem_cache *skbuff_head_cache __ro_after_init; 85 static struct kmem_cache *skbuff_fclone_cache __ro_after_init; 86 #ifdef CONFIG_SKB_EXTENSIONS 87 static struct kmem_cache *skbuff_ext_cache __ro_after_init; 88 #endif 89 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; 90 EXPORT_SYMBOL(sysctl_max_skb_frags); 91 92 /** 93 * skb_panic - private function for out-of-line support 94 * @skb: buffer 95 * @sz: size 96 * @addr: address 97 * @msg: skb_over_panic or skb_under_panic 98 * 99 * Out-of-line support for skb_put() and skb_push(). 100 * Called via the wrapper skb_over_panic() or skb_under_panic(). 101 * Keep out of line to prevent kernel bloat. 102 * __builtin_return_address is not used because it is not always reliable. 103 */ 104 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 105 const char msg[]) 106 { 107 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n", 108 msg, addr, skb->len, sz, skb->head, skb->data, 109 (unsigned long)skb->tail, (unsigned long)skb->end, 110 skb->dev ? skb->dev->name : "<NULL>"); 111 BUG(); 112 } 113 114 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 115 { 116 skb_panic(skb, sz, addr, __func__); 117 } 118 119 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 120 { 121 skb_panic(skb, sz, addr, __func__); 122 } 123 124 #define NAPI_SKB_CACHE_SIZE 64 125 #define NAPI_SKB_CACHE_BULK 16 126 #define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2) 127 128 struct napi_alloc_cache { 129 struct page_frag_cache page; 130 unsigned int skb_count; 131 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 132 }; 133 134 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 135 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 136 137 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 138 { 139 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 140 141 fragsz = SKB_DATA_ALIGN(fragsz); 142 143 return page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); 144 } 145 EXPORT_SYMBOL(__napi_alloc_frag_align); 146 147 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 148 { 149 void *data; 150 151 fragsz = SKB_DATA_ALIGN(fragsz); 152 if (in_hardirq() || irqs_disabled()) { 153 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache); 154 155 data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask); 156 } else { 157 struct napi_alloc_cache *nc; 158 159 local_bh_disable(); 160 nc = this_cpu_ptr(&napi_alloc_cache); 161 data = page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); 162 local_bh_enable(); 163 } 164 return data; 165 } 166 EXPORT_SYMBOL(__netdev_alloc_frag_align); 167 168 static struct sk_buff *napi_skb_cache_get(void) 169 { 170 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 171 struct sk_buff *skb; 172 173 if (unlikely(!nc->skb_count)) 174 nc->skb_count = kmem_cache_alloc_bulk(skbuff_head_cache, 175 GFP_ATOMIC, 176 NAPI_SKB_CACHE_BULK, 177 nc->skb_cache); 178 if (unlikely(!nc->skb_count)) 179 return NULL; 180 181 skb = nc->skb_cache[--nc->skb_count]; 182 kasan_unpoison_object_data(skbuff_head_cache, skb); 183 184 return skb; 185 } 186 187 /* Caller must provide SKB that is memset cleared */ 188 static void __build_skb_around(struct sk_buff *skb, void *data, 189 unsigned int frag_size) 190 { 191 struct skb_shared_info *shinfo; 192 unsigned int size = frag_size ? : ksize(data); 193 194 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 195 196 /* Assumes caller memset cleared SKB */ 197 skb->truesize = SKB_TRUESIZE(size); 198 refcount_set(&skb->users, 1); 199 skb->head = data; 200 skb->data = data; 201 skb_reset_tail_pointer(skb); 202 skb->end = skb->tail + size; 203 skb->mac_header = (typeof(skb->mac_header))~0U; 204 skb->transport_header = (typeof(skb->transport_header))~0U; 205 206 /* make sure we initialize shinfo sequentially */ 207 shinfo = skb_shinfo(skb); 208 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 209 atomic_set(&shinfo->dataref, 1); 210 211 skb_set_kcov_handle(skb, kcov_common_handle()); 212 } 213 214 /** 215 * __build_skb - build a network buffer 216 * @data: data buffer provided by caller 217 * @frag_size: size of data, or 0 if head was kmalloced 218 * 219 * Allocate a new &sk_buff. Caller provides space holding head and 220 * skb_shared_info. @data must have been allocated by kmalloc() only if 221 * @frag_size is 0, otherwise data should come from the page allocator 222 * or vmalloc() 223 * The return is the new skb buffer. 224 * On a failure the return is %NULL, and @data is not freed. 225 * Notes : 226 * Before IO, driver allocates only data buffer where NIC put incoming frame 227 * Driver should add room at head (NET_SKB_PAD) and 228 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 229 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 230 * before giving packet to stack. 231 * RX rings only contains data buffers, not full skbs. 232 */ 233 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 234 { 235 struct sk_buff *skb; 236 237 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); 238 if (unlikely(!skb)) 239 return NULL; 240 241 memset(skb, 0, offsetof(struct sk_buff, tail)); 242 __build_skb_around(skb, data, frag_size); 243 244 return skb; 245 } 246 247 /* build_skb() is wrapper over __build_skb(), that specifically 248 * takes care of skb->head and skb->pfmemalloc 249 * This means that if @frag_size is not zero, then @data must be backed 250 * by a page fragment, not kmalloc() or vmalloc() 251 */ 252 struct sk_buff *build_skb(void *data, unsigned int frag_size) 253 { 254 struct sk_buff *skb = __build_skb(data, frag_size); 255 256 if (skb && frag_size) { 257 skb->head_frag = 1; 258 if (page_is_pfmemalloc(virt_to_head_page(data))) 259 skb->pfmemalloc = 1; 260 } 261 return skb; 262 } 263 EXPORT_SYMBOL(build_skb); 264 265 /** 266 * build_skb_around - build a network buffer around provided skb 267 * @skb: sk_buff provide by caller, must be memset cleared 268 * @data: data buffer provided by caller 269 * @frag_size: size of data, or 0 if head was kmalloced 270 */ 271 struct sk_buff *build_skb_around(struct sk_buff *skb, 272 void *data, unsigned int frag_size) 273 { 274 if (unlikely(!skb)) 275 return NULL; 276 277 __build_skb_around(skb, data, frag_size); 278 279 if (frag_size) { 280 skb->head_frag = 1; 281 if (page_is_pfmemalloc(virt_to_head_page(data))) 282 skb->pfmemalloc = 1; 283 } 284 return skb; 285 } 286 EXPORT_SYMBOL(build_skb_around); 287 288 /** 289 * __napi_build_skb - build a network buffer 290 * @data: data buffer provided by caller 291 * @frag_size: size of data, or 0 if head was kmalloced 292 * 293 * Version of __build_skb() that uses NAPI percpu caches to obtain 294 * skbuff_head instead of inplace allocation. 295 * 296 * Returns a new &sk_buff on success, %NULL on allocation failure. 297 */ 298 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 299 { 300 struct sk_buff *skb; 301 302 skb = napi_skb_cache_get(); 303 if (unlikely(!skb)) 304 return NULL; 305 306 memset(skb, 0, offsetof(struct sk_buff, tail)); 307 __build_skb_around(skb, data, frag_size); 308 309 return skb; 310 } 311 312 /** 313 * napi_build_skb - build a network buffer 314 * @data: data buffer provided by caller 315 * @frag_size: size of data, or 0 if head was kmalloced 316 * 317 * Version of __napi_build_skb() that takes care of skb->head_frag 318 * and skb->pfmemalloc when the data is a page or page fragment. 319 * 320 * Returns a new &sk_buff on success, %NULL on allocation failure. 321 */ 322 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) 323 { 324 struct sk_buff *skb = __napi_build_skb(data, frag_size); 325 326 if (likely(skb) && frag_size) { 327 skb->head_frag = 1; 328 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 329 } 330 331 return skb; 332 } 333 EXPORT_SYMBOL(napi_build_skb); 334 335 /* 336 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 337 * the caller if emergency pfmemalloc reserves are being used. If it is and 338 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 339 * may be used. Otherwise, the packet data may be discarded until enough 340 * memory is free 341 */ 342 static void *kmalloc_reserve(size_t size, gfp_t flags, int node, 343 bool *pfmemalloc) 344 { 345 void *obj; 346 bool ret_pfmemalloc = false; 347 348 /* 349 * Try a regular allocation, when that fails and we're not entitled 350 * to the reserves, fail. 351 */ 352 obj = kmalloc_node_track_caller(size, 353 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 354 node); 355 if (obj || !(gfp_pfmemalloc_allowed(flags))) 356 goto out; 357 358 /* Try again but now we are using pfmemalloc reserves */ 359 ret_pfmemalloc = true; 360 obj = kmalloc_node_track_caller(size, flags, node); 361 362 out: 363 if (pfmemalloc) 364 *pfmemalloc = ret_pfmemalloc; 365 366 return obj; 367 } 368 369 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 370 * 'private' fields and also do memory statistics to find all the 371 * [BEEP] leaks. 372 * 373 */ 374 375 /** 376 * __alloc_skb - allocate a network buffer 377 * @size: size to allocate 378 * @gfp_mask: allocation mask 379 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 380 * instead of head cache and allocate a cloned (child) skb. 381 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 382 * allocations in case the data is required for writeback 383 * @node: numa node to allocate memory on 384 * 385 * Allocate a new &sk_buff. The returned buffer has no headroom and a 386 * tail room of at least size bytes. The object has a reference count 387 * of one. The return is the buffer. On a failure the return is %NULL. 388 * 389 * Buffers may only be allocated from interrupts using a @gfp_mask of 390 * %GFP_ATOMIC. 391 */ 392 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 393 int flags, int node) 394 { 395 struct kmem_cache *cache; 396 struct sk_buff *skb; 397 u8 *data; 398 bool pfmemalloc; 399 400 cache = (flags & SKB_ALLOC_FCLONE) 401 ? skbuff_fclone_cache : skbuff_head_cache; 402 403 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 404 gfp_mask |= __GFP_MEMALLOC; 405 406 /* Get the HEAD */ 407 if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI && 408 likely(node == NUMA_NO_NODE || node == numa_mem_id())) 409 skb = napi_skb_cache_get(); 410 else 411 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); 412 if (unlikely(!skb)) 413 return NULL; 414 prefetchw(skb); 415 416 /* We do our best to align skb_shared_info on a separate cache 417 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 418 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 419 * Both skb->head and skb_shared_info are cache line aligned. 420 */ 421 size = SKB_DATA_ALIGN(size); 422 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 423 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc); 424 if (unlikely(!data)) 425 goto nodata; 426 /* kmalloc(size) might give us more room than requested. 427 * Put skb_shared_info exactly at the end of allocated zone, 428 * to allow max possible filling before reallocation. 429 */ 430 size = SKB_WITH_OVERHEAD(ksize(data)); 431 prefetchw(data + size); 432 433 /* 434 * Only clear those fields we need to clear, not those that we will 435 * actually initialise below. Hence, don't put any more fields after 436 * the tail pointer in struct sk_buff! 437 */ 438 memset(skb, 0, offsetof(struct sk_buff, tail)); 439 __build_skb_around(skb, data, 0); 440 skb->pfmemalloc = pfmemalloc; 441 442 if (flags & SKB_ALLOC_FCLONE) { 443 struct sk_buff_fclones *fclones; 444 445 fclones = container_of(skb, struct sk_buff_fclones, skb1); 446 447 skb->fclone = SKB_FCLONE_ORIG; 448 refcount_set(&fclones->fclone_ref, 1); 449 450 fclones->skb2.fclone = SKB_FCLONE_CLONE; 451 } 452 453 return skb; 454 455 nodata: 456 kmem_cache_free(cache, skb); 457 return NULL; 458 } 459 EXPORT_SYMBOL(__alloc_skb); 460 461 /** 462 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 463 * @dev: network device to receive on 464 * @len: length to allocate 465 * @gfp_mask: get_free_pages mask, passed to alloc_skb 466 * 467 * Allocate a new &sk_buff and assign it a usage count of one. The 468 * buffer has NET_SKB_PAD headroom built in. Users should allocate 469 * the headroom they think they need without accounting for the 470 * built in space. The built in space is used for optimisations. 471 * 472 * %NULL is returned if there is no free memory. 473 */ 474 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 475 gfp_t gfp_mask) 476 { 477 struct page_frag_cache *nc; 478 struct sk_buff *skb; 479 bool pfmemalloc; 480 void *data; 481 482 len += NET_SKB_PAD; 483 484 /* If requested length is either too small or too big, 485 * we use kmalloc() for skb->head allocation. 486 */ 487 if (len <= SKB_WITH_OVERHEAD(1024) || 488 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 489 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 490 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 491 if (!skb) 492 goto skb_fail; 493 goto skb_success; 494 } 495 496 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 497 len = SKB_DATA_ALIGN(len); 498 499 if (sk_memalloc_socks()) 500 gfp_mask |= __GFP_MEMALLOC; 501 502 if (in_hardirq() || irqs_disabled()) { 503 nc = this_cpu_ptr(&netdev_alloc_cache); 504 data = page_frag_alloc(nc, len, gfp_mask); 505 pfmemalloc = nc->pfmemalloc; 506 } else { 507 local_bh_disable(); 508 nc = this_cpu_ptr(&napi_alloc_cache.page); 509 data = page_frag_alloc(nc, len, gfp_mask); 510 pfmemalloc = nc->pfmemalloc; 511 local_bh_enable(); 512 } 513 514 if (unlikely(!data)) 515 return NULL; 516 517 skb = __build_skb(data, len); 518 if (unlikely(!skb)) { 519 skb_free_frag(data); 520 return NULL; 521 } 522 523 if (pfmemalloc) 524 skb->pfmemalloc = 1; 525 skb->head_frag = 1; 526 527 skb_success: 528 skb_reserve(skb, NET_SKB_PAD); 529 skb->dev = dev; 530 531 skb_fail: 532 return skb; 533 } 534 EXPORT_SYMBOL(__netdev_alloc_skb); 535 536 /** 537 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 538 * @napi: napi instance this buffer was allocated for 539 * @len: length to allocate 540 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages 541 * 542 * Allocate a new sk_buff for use in NAPI receive. This buffer will 543 * attempt to allocate the head from a special reserved region used 544 * only for NAPI Rx allocation. By doing this we can save several 545 * CPU cycles by avoiding having to disable and re-enable IRQs. 546 * 547 * %NULL is returned if there is no free memory. 548 */ 549 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, 550 gfp_t gfp_mask) 551 { 552 struct napi_alloc_cache *nc; 553 struct sk_buff *skb; 554 void *data; 555 556 len += NET_SKB_PAD + NET_IP_ALIGN; 557 558 /* If requested length is either too small or too big, 559 * we use kmalloc() for skb->head allocation. 560 */ 561 if (len <= SKB_WITH_OVERHEAD(1024) || 562 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 563 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 564 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, 565 NUMA_NO_NODE); 566 if (!skb) 567 goto skb_fail; 568 goto skb_success; 569 } 570 571 nc = this_cpu_ptr(&napi_alloc_cache); 572 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 573 len = SKB_DATA_ALIGN(len); 574 575 if (sk_memalloc_socks()) 576 gfp_mask |= __GFP_MEMALLOC; 577 578 data = page_frag_alloc(&nc->page, len, gfp_mask); 579 if (unlikely(!data)) 580 return NULL; 581 582 skb = __napi_build_skb(data, len); 583 if (unlikely(!skb)) { 584 skb_free_frag(data); 585 return NULL; 586 } 587 588 if (nc->page.pfmemalloc) 589 skb->pfmemalloc = 1; 590 skb->head_frag = 1; 591 592 skb_success: 593 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 594 skb->dev = napi->dev; 595 596 skb_fail: 597 return skb; 598 } 599 EXPORT_SYMBOL(__napi_alloc_skb); 600 601 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 602 int size, unsigned int truesize) 603 { 604 skb_fill_page_desc(skb, i, page, off, size); 605 skb->len += size; 606 skb->data_len += size; 607 skb->truesize += truesize; 608 } 609 EXPORT_SYMBOL(skb_add_rx_frag); 610 611 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 612 unsigned int truesize) 613 { 614 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 615 616 skb_frag_size_add(frag, size); 617 skb->len += size; 618 skb->data_len += size; 619 skb->truesize += truesize; 620 } 621 EXPORT_SYMBOL(skb_coalesce_rx_frag); 622 623 static void skb_drop_list(struct sk_buff **listp) 624 { 625 kfree_skb_list(*listp); 626 *listp = NULL; 627 } 628 629 static inline void skb_drop_fraglist(struct sk_buff *skb) 630 { 631 skb_drop_list(&skb_shinfo(skb)->frag_list); 632 } 633 634 static void skb_clone_fraglist(struct sk_buff *skb) 635 { 636 struct sk_buff *list; 637 638 skb_walk_frags(skb, list) 639 skb_get(list); 640 } 641 642 static void skb_free_head(struct sk_buff *skb) 643 { 644 unsigned char *head = skb->head; 645 646 if (skb->head_frag) { 647 if (skb_pp_recycle(skb, head)) 648 return; 649 skb_free_frag(head); 650 } else { 651 kfree(head); 652 } 653 } 654 655 static void skb_release_data(struct sk_buff *skb) 656 { 657 struct skb_shared_info *shinfo = skb_shinfo(skb); 658 int i; 659 660 if (skb->cloned && 661 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 662 &shinfo->dataref)) 663 goto exit; 664 665 skb_zcopy_clear(skb, true); 666 667 for (i = 0; i < shinfo->nr_frags; i++) 668 __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle); 669 670 if (shinfo->frag_list) 671 kfree_skb_list(shinfo->frag_list); 672 673 skb_free_head(skb); 674 exit: 675 /* When we clone an SKB we copy the reycling bit. The pp_recycle 676 * bit is only set on the head though, so in order to avoid races 677 * while trying to recycle fragments on __skb_frag_unref() we need 678 * to make one SKB responsible for triggering the recycle path. 679 * So disable the recycling bit if an SKB is cloned and we have 680 * additional references to to the fragmented part of the SKB. 681 * Eventually the last SKB will have the recycling bit set and it's 682 * dataref set to 0, which will trigger the recycling 683 */ 684 skb->pp_recycle = 0; 685 } 686 687 /* 688 * Free an skbuff by memory without cleaning the state. 689 */ 690 static void kfree_skbmem(struct sk_buff *skb) 691 { 692 struct sk_buff_fclones *fclones; 693 694 switch (skb->fclone) { 695 case SKB_FCLONE_UNAVAILABLE: 696 kmem_cache_free(skbuff_head_cache, skb); 697 return; 698 699 case SKB_FCLONE_ORIG: 700 fclones = container_of(skb, struct sk_buff_fclones, skb1); 701 702 /* We usually free the clone (TX completion) before original skb 703 * This test would have no chance to be true for the clone, 704 * while here, branch prediction will be good. 705 */ 706 if (refcount_read(&fclones->fclone_ref) == 1) 707 goto fastpath; 708 break; 709 710 default: /* SKB_FCLONE_CLONE */ 711 fclones = container_of(skb, struct sk_buff_fclones, skb2); 712 break; 713 } 714 if (!refcount_dec_and_test(&fclones->fclone_ref)) 715 return; 716 fastpath: 717 kmem_cache_free(skbuff_fclone_cache, fclones); 718 } 719 720 void skb_release_head_state(struct sk_buff *skb) 721 { 722 skb_dst_drop(skb); 723 if (skb->destructor) { 724 WARN_ON(in_hardirq()); 725 skb->destructor(skb); 726 } 727 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 728 nf_conntrack_put(skb_nfct(skb)); 729 #endif 730 skb_ext_put(skb); 731 } 732 733 /* Free everything but the sk_buff shell. */ 734 static void skb_release_all(struct sk_buff *skb) 735 { 736 skb_release_head_state(skb); 737 if (likely(skb->head)) 738 skb_release_data(skb); 739 } 740 741 /** 742 * __kfree_skb - private function 743 * @skb: buffer 744 * 745 * Free an sk_buff. Release anything attached to the buffer. 746 * Clean the state. This is an internal helper function. Users should 747 * always call kfree_skb 748 */ 749 750 void __kfree_skb(struct sk_buff *skb) 751 { 752 skb_release_all(skb); 753 kfree_skbmem(skb); 754 } 755 EXPORT_SYMBOL(__kfree_skb); 756 757 /** 758 * kfree_skb - free an sk_buff 759 * @skb: buffer to free 760 * 761 * Drop a reference to the buffer and free it if the usage count has 762 * hit zero. 763 */ 764 void kfree_skb(struct sk_buff *skb) 765 { 766 if (!skb_unref(skb)) 767 return; 768 769 trace_kfree_skb(skb, __builtin_return_address(0)); 770 __kfree_skb(skb); 771 } 772 EXPORT_SYMBOL(kfree_skb); 773 774 void kfree_skb_list(struct sk_buff *segs) 775 { 776 while (segs) { 777 struct sk_buff *next = segs->next; 778 779 kfree_skb(segs); 780 segs = next; 781 } 782 } 783 EXPORT_SYMBOL(kfree_skb_list); 784 785 /* Dump skb information and contents. 786 * 787 * Must only be called from net_ratelimit()-ed paths. 788 * 789 * Dumps whole packets if full_pkt, only headers otherwise. 790 */ 791 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt) 792 { 793 struct skb_shared_info *sh = skb_shinfo(skb); 794 struct net_device *dev = skb->dev; 795 struct sock *sk = skb->sk; 796 struct sk_buff *list_skb; 797 bool has_mac, has_trans; 798 int headroom, tailroom; 799 int i, len, seg_len; 800 801 if (full_pkt) 802 len = skb->len; 803 else 804 len = min_t(int, skb->len, MAX_HEADER + 128); 805 806 headroom = skb_headroom(skb); 807 tailroom = skb_tailroom(skb); 808 809 has_mac = skb_mac_header_was_set(skb); 810 has_trans = skb_transport_header_was_set(skb); 811 812 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n" 813 "mac=(%d,%d) net=(%d,%d) trans=%d\n" 814 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" 815 "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n" 816 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n", 817 level, skb->len, headroom, skb_headlen(skb), tailroom, 818 has_mac ? skb->mac_header : -1, 819 has_mac ? skb_mac_header_len(skb) : -1, 820 skb->network_header, 821 has_trans ? skb_network_header_len(skb) : -1, 822 has_trans ? skb->transport_header : -1, 823 sh->tx_flags, sh->nr_frags, 824 sh->gso_size, sh->gso_type, sh->gso_segs, 825 skb->csum, skb->ip_summed, skb->csum_complete_sw, 826 skb->csum_valid, skb->csum_level, 827 skb->hash, skb->sw_hash, skb->l4_hash, 828 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif); 829 830 if (dev) 831 printk("%sdev name=%s feat=0x%pNF\n", 832 level, dev->name, &dev->features); 833 if (sk) 834 printk("%ssk family=%hu type=%u proto=%u\n", 835 level, sk->sk_family, sk->sk_type, sk->sk_protocol); 836 837 if (full_pkt && headroom) 838 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET, 839 16, 1, skb->head, headroom, false); 840 841 seg_len = min_t(int, skb_headlen(skb), len); 842 if (seg_len) 843 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, 844 16, 1, skb->data, seg_len, false); 845 len -= seg_len; 846 847 if (full_pkt && tailroom) 848 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET, 849 16, 1, skb_tail_pointer(skb), tailroom, false); 850 851 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { 852 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 853 u32 p_off, p_len, copied; 854 struct page *p; 855 u8 *vaddr; 856 857 skb_frag_foreach_page(frag, skb_frag_off(frag), 858 skb_frag_size(frag), p, p_off, p_len, 859 copied) { 860 seg_len = min_t(int, p_len, len); 861 vaddr = kmap_atomic(p); 862 print_hex_dump(level, "skb frag: ", 863 DUMP_PREFIX_OFFSET, 864 16, 1, vaddr + p_off, seg_len, false); 865 kunmap_atomic(vaddr); 866 len -= seg_len; 867 if (!len) 868 break; 869 } 870 } 871 872 if (full_pkt && skb_has_frag_list(skb)) { 873 printk("skb fraglist:\n"); 874 skb_walk_frags(skb, list_skb) 875 skb_dump(level, list_skb, true); 876 } 877 } 878 EXPORT_SYMBOL(skb_dump); 879 880 /** 881 * skb_tx_error - report an sk_buff xmit error 882 * @skb: buffer that triggered an error 883 * 884 * Report xmit error if a device callback is tracking this skb. 885 * skb must be freed afterwards. 886 */ 887 void skb_tx_error(struct sk_buff *skb) 888 { 889 skb_zcopy_clear(skb, true); 890 } 891 EXPORT_SYMBOL(skb_tx_error); 892 893 #ifdef CONFIG_TRACEPOINTS 894 /** 895 * consume_skb - free an skbuff 896 * @skb: buffer to free 897 * 898 * Drop a ref to the buffer and free it if the usage count has hit zero 899 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 900 * is being dropped after a failure and notes that 901 */ 902 void consume_skb(struct sk_buff *skb) 903 { 904 if (!skb_unref(skb)) 905 return; 906 907 trace_consume_skb(skb); 908 __kfree_skb(skb); 909 } 910 EXPORT_SYMBOL(consume_skb); 911 #endif 912 913 /** 914 * __consume_stateless_skb - free an skbuff, assuming it is stateless 915 * @skb: buffer to free 916 * 917 * Alike consume_skb(), but this variant assumes that this is the last 918 * skb reference and all the head states have been already dropped 919 */ 920 void __consume_stateless_skb(struct sk_buff *skb) 921 { 922 trace_consume_skb(skb); 923 skb_release_data(skb); 924 kfree_skbmem(skb); 925 } 926 927 static void napi_skb_cache_put(struct sk_buff *skb) 928 { 929 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 930 u32 i; 931 932 kasan_poison_object_data(skbuff_head_cache, skb); 933 nc->skb_cache[nc->skb_count++] = skb; 934 935 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 936 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) 937 kasan_unpoison_object_data(skbuff_head_cache, 938 nc->skb_cache[i]); 939 940 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_HALF, 941 nc->skb_cache + NAPI_SKB_CACHE_HALF); 942 nc->skb_count = NAPI_SKB_CACHE_HALF; 943 } 944 } 945 946 void __kfree_skb_defer(struct sk_buff *skb) 947 { 948 skb_release_all(skb); 949 napi_skb_cache_put(skb); 950 } 951 952 void napi_skb_free_stolen_head(struct sk_buff *skb) 953 { 954 if (unlikely(skb->slow_gro)) { 955 nf_reset_ct(skb); 956 skb_dst_drop(skb); 957 skb_ext_put(skb); 958 skb_orphan(skb); 959 skb->slow_gro = 0; 960 } 961 napi_skb_cache_put(skb); 962 } 963 964 void napi_consume_skb(struct sk_buff *skb, int budget) 965 { 966 /* Zero budget indicate non-NAPI context called us, like netpoll */ 967 if (unlikely(!budget)) { 968 dev_consume_skb_any(skb); 969 return; 970 } 971 972 lockdep_assert_in_softirq(); 973 974 if (!skb_unref(skb)) 975 return; 976 977 /* if reaching here SKB is ready to free */ 978 trace_consume_skb(skb); 979 980 /* if SKB is a clone, don't handle this case */ 981 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 982 __kfree_skb(skb); 983 return; 984 } 985 986 skb_release_all(skb); 987 napi_skb_cache_put(skb); 988 } 989 EXPORT_SYMBOL(napi_consume_skb); 990 991 /* Make sure a field is enclosed inside headers_start/headers_end section */ 992 #define CHECK_SKB_FIELD(field) \ 993 BUILD_BUG_ON(offsetof(struct sk_buff, field) < \ 994 offsetof(struct sk_buff, headers_start)); \ 995 BUILD_BUG_ON(offsetof(struct sk_buff, field) > \ 996 offsetof(struct sk_buff, headers_end)); \ 997 998 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 999 { 1000 new->tstamp = old->tstamp; 1001 /* We do not copy old->sk */ 1002 new->dev = old->dev; 1003 memcpy(new->cb, old->cb, sizeof(old->cb)); 1004 skb_dst_copy(new, old); 1005 __skb_ext_copy(new, old); 1006 __nf_copy(new, old, false); 1007 1008 /* Note : this field could be in headers_start/headers_end section 1009 * It is not yet because we do not want to have a 16 bit hole 1010 */ 1011 new->queue_mapping = old->queue_mapping; 1012 1013 memcpy(&new->headers_start, &old->headers_start, 1014 offsetof(struct sk_buff, headers_end) - 1015 offsetof(struct sk_buff, headers_start)); 1016 CHECK_SKB_FIELD(protocol); 1017 CHECK_SKB_FIELD(csum); 1018 CHECK_SKB_FIELD(hash); 1019 CHECK_SKB_FIELD(priority); 1020 CHECK_SKB_FIELD(skb_iif); 1021 CHECK_SKB_FIELD(vlan_proto); 1022 CHECK_SKB_FIELD(vlan_tci); 1023 CHECK_SKB_FIELD(transport_header); 1024 CHECK_SKB_FIELD(network_header); 1025 CHECK_SKB_FIELD(mac_header); 1026 CHECK_SKB_FIELD(inner_protocol); 1027 CHECK_SKB_FIELD(inner_transport_header); 1028 CHECK_SKB_FIELD(inner_network_header); 1029 CHECK_SKB_FIELD(inner_mac_header); 1030 CHECK_SKB_FIELD(mark); 1031 #ifdef CONFIG_NETWORK_SECMARK 1032 CHECK_SKB_FIELD(secmark); 1033 #endif 1034 #ifdef CONFIG_NET_RX_BUSY_POLL 1035 CHECK_SKB_FIELD(napi_id); 1036 #endif 1037 #ifdef CONFIG_XPS 1038 CHECK_SKB_FIELD(sender_cpu); 1039 #endif 1040 #ifdef CONFIG_NET_SCHED 1041 CHECK_SKB_FIELD(tc_index); 1042 #endif 1043 1044 } 1045 1046 /* 1047 * You should not add any new code to this function. Add it to 1048 * __copy_skb_header above instead. 1049 */ 1050 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 1051 { 1052 #define C(x) n->x = skb->x 1053 1054 n->next = n->prev = NULL; 1055 n->sk = NULL; 1056 __copy_skb_header(n, skb); 1057 1058 C(len); 1059 C(data_len); 1060 C(mac_len); 1061 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 1062 n->cloned = 1; 1063 n->nohdr = 0; 1064 n->peeked = 0; 1065 C(pfmemalloc); 1066 C(pp_recycle); 1067 n->destructor = NULL; 1068 C(tail); 1069 C(end); 1070 C(head); 1071 C(head_frag); 1072 C(data); 1073 C(truesize); 1074 refcount_set(&n->users, 1); 1075 1076 atomic_inc(&(skb_shinfo(skb)->dataref)); 1077 skb->cloned = 1; 1078 1079 return n; 1080 #undef C 1081 } 1082 1083 /** 1084 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg 1085 * @first: first sk_buff of the msg 1086 */ 1087 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first) 1088 { 1089 struct sk_buff *n; 1090 1091 n = alloc_skb(0, GFP_ATOMIC); 1092 if (!n) 1093 return NULL; 1094 1095 n->len = first->len; 1096 n->data_len = first->len; 1097 n->truesize = first->truesize; 1098 1099 skb_shinfo(n)->frag_list = first; 1100 1101 __copy_skb_header(n, first); 1102 n->destructor = NULL; 1103 1104 return n; 1105 } 1106 EXPORT_SYMBOL_GPL(alloc_skb_for_msg); 1107 1108 /** 1109 * skb_morph - morph one skb into another 1110 * @dst: the skb to receive the contents 1111 * @src: the skb to supply the contents 1112 * 1113 * This is identical to skb_clone except that the target skb is 1114 * supplied by the user. 1115 * 1116 * The target skb is returned upon exit. 1117 */ 1118 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 1119 { 1120 skb_release_all(dst); 1121 return __skb_clone(dst, src); 1122 } 1123 EXPORT_SYMBOL_GPL(skb_morph); 1124 1125 int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 1126 { 1127 unsigned long max_pg, num_pg, new_pg, old_pg; 1128 struct user_struct *user; 1129 1130 if (capable(CAP_IPC_LOCK) || !size) 1131 return 0; 1132 1133 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 1134 max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1135 user = mmp->user ? : current_user(); 1136 1137 do { 1138 old_pg = atomic_long_read(&user->locked_vm); 1139 new_pg = old_pg + num_pg; 1140 if (new_pg > max_pg) 1141 return -ENOBUFS; 1142 } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) != 1143 old_pg); 1144 1145 if (!mmp->user) { 1146 mmp->user = get_uid(user); 1147 mmp->num_pg = num_pg; 1148 } else { 1149 mmp->num_pg += num_pg; 1150 } 1151 1152 return 0; 1153 } 1154 EXPORT_SYMBOL_GPL(mm_account_pinned_pages); 1155 1156 void mm_unaccount_pinned_pages(struct mmpin *mmp) 1157 { 1158 if (mmp->user) { 1159 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 1160 free_uid(mmp->user); 1161 } 1162 } 1163 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); 1164 1165 struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) 1166 { 1167 struct ubuf_info *uarg; 1168 struct sk_buff *skb; 1169 1170 WARN_ON_ONCE(!in_task()); 1171 1172 skb = sock_omalloc(sk, 0, GFP_KERNEL); 1173 if (!skb) 1174 return NULL; 1175 1176 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 1177 uarg = (void *)skb->cb; 1178 uarg->mmp.user = NULL; 1179 1180 if (mm_account_pinned_pages(&uarg->mmp, size)) { 1181 kfree_skb(skb); 1182 return NULL; 1183 } 1184 1185 uarg->callback = msg_zerocopy_callback; 1186 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 1187 uarg->len = 1; 1188 uarg->bytelen = size; 1189 uarg->zerocopy = 1; 1190 uarg->flags = SKBFL_ZEROCOPY_FRAG; 1191 refcount_set(&uarg->refcnt, 1); 1192 sock_hold(sk); 1193 1194 return uarg; 1195 } 1196 EXPORT_SYMBOL_GPL(msg_zerocopy_alloc); 1197 1198 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg) 1199 { 1200 return container_of((void *)uarg, struct sk_buff, cb); 1201 } 1202 1203 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, 1204 struct ubuf_info *uarg) 1205 { 1206 if (uarg) { 1207 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 1208 u32 bytelen, next; 1209 1210 /* realloc only when socket is locked (TCP, UDP cork), 1211 * so uarg->len and sk_zckey access is serialized 1212 */ 1213 if (!sock_owned_by_user(sk)) { 1214 WARN_ON_ONCE(1); 1215 return NULL; 1216 } 1217 1218 bytelen = uarg->bytelen + size; 1219 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) { 1220 /* TCP can create new skb to attach new uarg */ 1221 if (sk->sk_type == SOCK_STREAM) 1222 goto new_alloc; 1223 return NULL; 1224 } 1225 1226 next = (u32)atomic_read(&sk->sk_zckey); 1227 if ((u32)(uarg->id + uarg->len) == next) { 1228 if (mm_account_pinned_pages(&uarg->mmp, size)) 1229 return NULL; 1230 uarg->len++; 1231 uarg->bytelen = bytelen; 1232 atomic_set(&sk->sk_zckey, ++next); 1233 1234 /* no extra ref when appending to datagram (MSG_MORE) */ 1235 if (sk->sk_type == SOCK_STREAM) 1236 net_zcopy_get(uarg); 1237 1238 return uarg; 1239 } 1240 } 1241 1242 new_alloc: 1243 return msg_zerocopy_alloc(sk, size); 1244 } 1245 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); 1246 1247 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1248 { 1249 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1250 u32 old_lo, old_hi; 1251 u64 sum_len; 1252 1253 old_lo = serr->ee.ee_info; 1254 old_hi = serr->ee.ee_data; 1255 sum_len = old_hi - old_lo + 1ULL + len; 1256 1257 if (sum_len >= (1ULL << 32)) 1258 return false; 1259 1260 if (lo != old_hi + 1) 1261 return false; 1262 1263 serr->ee.ee_data += len; 1264 return true; 1265 } 1266 1267 static void __msg_zerocopy_callback(struct ubuf_info *uarg) 1268 { 1269 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1270 struct sock_exterr_skb *serr; 1271 struct sock *sk = skb->sk; 1272 struct sk_buff_head *q; 1273 unsigned long flags; 1274 bool is_zerocopy; 1275 u32 lo, hi; 1276 u16 len; 1277 1278 mm_unaccount_pinned_pages(&uarg->mmp); 1279 1280 /* if !len, there was only 1 call, and it was aborted 1281 * so do not queue a completion notification 1282 */ 1283 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1284 goto release; 1285 1286 len = uarg->len; 1287 lo = uarg->id; 1288 hi = uarg->id + len - 1; 1289 is_zerocopy = uarg->zerocopy; 1290 1291 serr = SKB_EXT_ERR(skb); 1292 memset(serr, 0, sizeof(*serr)); 1293 serr->ee.ee_errno = 0; 1294 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1295 serr->ee.ee_data = hi; 1296 serr->ee.ee_info = lo; 1297 if (!is_zerocopy) 1298 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1299 1300 q = &sk->sk_error_queue; 1301 spin_lock_irqsave(&q->lock, flags); 1302 tail = skb_peek_tail(q); 1303 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1304 !skb_zerocopy_notify_extend(tail, lo, len)) { 1305 __skb_queue_tail(q, skb); 1306 skb = NULL; 1307 } 1308 spin_unlock_irqrestore(&q->lock, flags); 1309 1310 sk_error_report(sk); 1311 1312 release: 1313 consume_skb(skb); 1314 sock_put(sk); 1315 } 1316 1317 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, 1318 bool success) 1319 { 1320 uarg->zerocopy = uarg->zerocopy & success; 1321 1322 if (refcount_dec_and_test(&uarg->refcnt)) 1323 __msg_zerocopy_callback(uarg); 1324 } 1325 EXPORT_SYMBOL_GPL(msg_zerocopy_callback); 1326 1327 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1328 { 1329 struct sock *sk = skb_from_uarg(uarg)->sk; 1330 1331 atomic_dec(&sk->sk_zckey); 1332 uarg->len--; 1333 1334 if (have_uref) 1335 msg_zerocopy_callback(NULL, uarg, true); 1336 } 1337 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); 1338 1339 int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len) 1340 { 1341 return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len); 1342 } 1343 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram); 1344 1345 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1346 struct msghdr *msg, int len, 1347 struct ubuf_info *uarg) 1348 { 1349 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1350 struct iov_iter orig_iter = msg->msg_iter; 1351 int err, orig_len = skb->len; 1352 1353 /* An skb can only point to one uarg. This edge case happens when 1354 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc. 1355 */ 1356 if (orig_uarg && uarg != orig_uarg) 1357 return -EEXIST; 1358 1359 err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len); 1360 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1361 struct sock *save_sk = skb->sk; 1362 1363 /* Streams do not free skb on error. Reset to prev state. */ 1364 msg->msg_iter = orig_iter; 1365 skb->sk = sk; 1366 ___pskb_trim(skb, orig_len); 1367 skb->sk = save_sk; 1368 return err; 1369 } 1370 1371 skb_zcopy_set(skb, uarg, NULL); 1372 return skb->len - orig_len; 1373 } 1374 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1375 1376 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1377 gfp_t gfp_mask) 1378 { 1379 if (skb_zcopy(orig)) { 1380 if (skb_zcopy(nskb)) { 1381 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1382 if (!gfp_mask) { 1383 WARN_ON_ONCE(1); 1384 return -ENOMEM; 1385 } 1386 if (skb_uarg(nskb) == skb_uarg(orig)) 1387 return 0; 1388 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1389 return -EIO; 1390 } 1391 skb_zcopy_set(nskb, skb_uarg(orig), NULL); 1392 } 1393 return 0; 1394 } 1395 1396 /** 1397 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1398 * @skb: the skb to modify 1399 * @gfp_mask: allocation priority 1400 * 1401 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. 1402 * It will copy all frags into kernel and drop the reference 1403 * to userspace pages. 1404 * 1405 * If this function is called from an interrupt gfp_mask() must be 1406 * %GFP_ATOMIC. 1407 * 1408 * Returns 0 on success or a negative error code on failure 1409 * to allocate kernel memory to copy to. 1410 */ 1411 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 1412 { 1413 int num_frags = skb_shinfo(skb)->nr_frags; 1414 struct page *page, *head = NULL; 1415 int i, new_frags; 1416 u32 d_off; 1417 1418 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 1419 return -EINVAL; 1420 1421 if (!num_frags) 1422 goto release; 1423 1424 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT; 1425 for (i = 0; i < new_frags; i++) { 1426 page = alloc_page(gfp_mask); 1427 if (!page) { 1428 while (head) { 1429 struct page *next = (struct page *)page_private(head); 1430 put_page(head); 1431 head = next; 1432 } 1433 return -ENOMEM; 1434 } 1435 set_page_private(page, (unsigned long)head); 1436 head = page; 1437 } 1438 1439 page = head; 1440 d_off = 0; 1441 for (i = 0; i < num_frags; i++) { 1442 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1443 u32 p_off, p_len, copied; 1444 struct page *p; 1445 u8 *vaddr; 1446 1447 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f), 1448 p, p_off, p_len, copied) { 1449 u32 copy, done = 0; 1450 vaddr = kmap_atomic(p); 1451 1452 while (done < p_len) { 1453 if (d_off == PAGE_SIZE) { 1454 d_off = 0; 1455 page = (struct page *)page_private(page); 1456 } 1457 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done); 1458 memcpy(page_address(page) + d_off, 1459 vaddr + p_off + done, copy); 1460 done += copy; 1461 d_off += copy; 1462 } 1463 kunmap_atomic(vaddr); 1464 } 1465 } 1466 1467 /* skb frags release userspace buffers */ 1468 for (i = 0; i < num_frags; i++) 1469 skb_frag_unref(skb, i); 1470 1471 /* skb frags point to kernel buffers */ 1472 for (i = 0; i < new_frags - 1; i++) { 1473 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE); 1474 head = (struct page *)page_private(head); 1475 } 1476 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); 1477 skb_shinfo(skb)->nr_frags = new_frags; 1478 1479 release: 1480 skb_zcopy_clear(skb, false); 1481 return 0; 1482 } 1483 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 1484 1485 /** 1486 * skb_clone - duplicate an sk_buff 1487 * @skb: buffer to clone 1488 * @gfp_mask: allocation priority 1489 * 1490 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 1491 * copies share the same packet data but not structure. The new 1492 * buffer has a reference count of 1. If the allocation fails the 1493 * function returns %NULL otherwise the new buffer is returned. 1494 * 1495 * If this function is called from an interrupt gfp_mask() must be 1496 * %GFP_ATOMIC. 1497 */ 1498 1499 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 1500 { 1501 struct sk_buff_fclones *fclones = container_of(skb, 1502 struct sk_buff_fclones, 1503 skb1); 1504 struct sk_buff *n; 1505 1506 if (skb_orphan_frags(skb, gfp_mask)) 1507 return NULL; 1508 1509 if (skb->fclone == SKB_FCLONE_ORIG && 1510 refcount_read(&fclones->fclone_ref) == 1) { 1511 n = &fclones->skb2; 1512 refcount_set(&fclones->fclone_ref, 2); 1513 } else { 1514 if (skb_pfmemalloc(skb)) 1515 gfp_mask |= __GFP_MEMALLOC; 1516 1517 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); 1518 if (!n) 1519 return NULL; 1520 1521 n->fclone = SKB_FCLONE_UNAVAILABLE; 1522 } 1523 1524 return __skb_clone(n, skb); 1525 } 1526 EXPORT_SYMBOL(skb_clone); 1527 1528 void skb_headers_offset_update(struct sk_buff *skb, int off) 1529 { 1530 /* Only adjust this if it actually is csum_start rather than csum */ 1531 if (skb->ip_summed == CHECKSUM_PARTIAL) 1532 skb->csum_start += off; 1533 /* {transport,network,mac}_header and tail are relative to skb->head */ 1534 skb->transport_header += off; 1535 skb->network_header += off; 1536 if (skb_mac_header_was_set(skb)) 1537 skb->mac_header += off; 1538 skb->inner_transport_header += off; 1539 skb->inner_network_header += off; 1540 skb->inner_mac_header += off; 1541 } 1542 EXPORT_SYMBOL(skb_headers_offset_update); 1543 1544 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) 1545 { 1546 __copy_skb_header(new, old); 1547 1548 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 1549 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 1550 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 1551 } 1552 EXPORT_SYMBOL(skb_copy_header); 1553 1554 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 1555 { 1556 if (skb_pfmemalloc(skb)) 1557 return SKB_ALLOC_RX; 1558 return 0; 1559 } 1560 1561 /** 1562 * skb_copy - create private copy of an sk_buff 1563 * @skb: buffer to copy 1564 * @gfp_mask: allocation priority 1565 * 1566 * Make a copy of both an &sk_buff and its data. This is used when the 1567 * caller wishes to modify the data and needs a private copy of the 1568 * data to alter. Returns %NULL on failure or the pointer to the buffer 1569 * on success. The returned buffer has a reference count of 1. 1570 * 1571 * As by-product this function converts non-linear &sk_buff to linear 1572 * one, so that &sk_buff becomes completely private and caller is allowed 1573 * to modify all the data of returned buffer. This means that this 1574 * function is not recommended for use in circumstances when only 1575 * header is going to be modified. Use pskb_copy() instead. 1576 */ 1577 1578 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 1579 { 1580 int headerlen = skb_headroom(skb); 1581 unsigned int size = skb_end_offset(skb) + skb->data_len; 1582 struct sk_buff *n = __alloc_skb(size, gfp_mask, 1583 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 1584 1585 if (!n) 1586 return NULL; 1587 1588 /* Set the data pointer */ 1589 skb_reserve(n, headerlen); 1590 /* Set the tail pointer and length */ 1591 skb_put(n, skb->len); 1592 1593 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); 1594 1595 skb_copy_header(n, skb); 1596 return n; 1597 } 1598 EXPORT_SYMBOL(skb_copy); 1599 1600 /** 1601 * __pskb_copy_fclone - create copy of an sk_buff with private head. 1602 * @skb: buffer to copy 1603 * @headroom: headroom of new skb 1604 * @gfp_mask: allocation priority 1605 * @fclone: if true allocate the copy of the skb from the fclone 1606 * cache instead of the head cache; it is recommended to set this 1607 * to true for the cases where the copy will likely be cloned 1608 * 1609 * Make a copy of both an &sk_buff and part of its data, located 1610 * in header. Fragmented data remain shared. This is used when 1611 * the caller wishes to modify only header of &sk_buff and needs 1612 * private copy of the header to alter. Returns %NULL on failure 1613 * or the pointer to the buffer on success. 1614 * The returned buffer has a reference count of 1. 1615 */ 1616 1617 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 1618 gfp_t gfp_mask, bool fclone) 1619 { 1620 unsigned int size = skb_headlen(skb) + headroom; 1621 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 1622 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 1623 1624 if (!n) 1625 goto out; 1626 1627 /* Set the data pointer */ 1628 skb_reserve(n, headroom); 1629 /* Set the tail pointer and length */ 1630 skb_put(n, skb_headlen(skb)); 1631 /* Copy the bytes */ 1632 skb_copy_from_linear_data(skb, n->data, n->len); 1633 1634 n->truesize += skb->data_len; 1635 n->data_len = skb->data_len; 1636 n->len = skb->len; 1637 1638 if (skb_shinfo(skb)->nr_frags) { 1639 int i; 1640 1641 if (skb_orphan_frags(skb, gfp_mask) || 1642 skb_zerocopy_clone(n, skb, gfp_mask)) { 1643 kfree_skb(n); 1644 n = NULL; 1645 goto out; 1646 } 1647 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1648 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 1649 skb_frag_ref(skb, i); 1650 } 1651 skb_shinfo(n)->nr_frags = i; 1652 } 1653 1654 if (skb_has_frag_list(skb)) { 1655 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 1656 skb_clone_fraglist(n); 1657 } 1658 1659 skb_copy_header(n, skb); 1660 out: 1661 return n; 1662 } 1663 EXPORT_SYMBOL(__pskb_copy_fclone); 1664 1665 /** 1666 * pskb_expand_head - reallocate header of &sk_buff 1667 * @skb: buffer to reallocate 1668 * @nhead: room to add at head 1669 * @ntail: room to add at tail 1670 * @gfp_mask: allocation priority 1671 * 1672 * Expands (or creates identical copy, if @nhead and @ntail are zero) 1673 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 1674 * reference count of 1. Returns zero in the case of success or error, 1675 * if expansion failed. In the last case, &sk_buff is not changed. 1676 * 1677 * All the pointers pointing into skb header may change and must be 1678 * reloaded after call to this function. 1679 */ 1680 1681 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 1682 gfp_t gfp_mask) 1683 { 1684 int i, osize = skb_end_offset(skb); 1685 int size = osize + nhead + ntail; 1686 long off; 1687 u8 *data; 1688 1689 BUG_ON(nhead < 0); 1690 1691 BUG_ON(skb_shared(skb)); 1692 1693 size = SKB_DATA_ALIGN(size); 1694 1695 if (skb_pfmemalloc(skb)) 1696 gfp_mask |= __GFP_MEMALLOC; 1697 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 1698 gfp_mask, NUMA_NO_NODE, NULL); 1699 if (!data) 1700 goto nodata; 1701 size = SKB_WITH_OVERHEAD(ksize(data)); 1702 1703 /* Copy only real data... and, alas, header. This should be 1704 * optimized for the cases when header is void. 1705 */ 1706 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 1707 1708 memcpy((struct skb_shared_info *)(data + size), 1709 skb_shinfo(skb), 1710 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 1711 1712 /* 1713 * if shinfo is shared we must drop the old head gracefully, but if it 1714 * is not we can just drop the old head and let the existing refcount 1715 * be since all we did is relocate the values 1716 */ 1717 if (skb_cloned(skb)) { 1718 if (skb_orphan_frags(skb, gfp_mask)) 1719 goto nofrags; 1720 if (skb_zcopy(skb)) 1721 refcount_inc(&skb_uarg(skb)->refcnt); 1722 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1723 skb_frag_ref(skb, i); 1724 1725 if (skb_has_frag_list(skb)) 1726 skb_clone_fraglist(skb); 1727 1728 skb_release_data(skb); 1729 } else { 1730 skb_free_head(skb); 1731 } 1732 off = (data + nhead) - skb->head; 1733 1734 skb->head = data; 1735 skb->head_frag = 0; 1736 skb->data += off; 1737 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1738 skb->end = size; 1739 off = nhead; 1740 #else 1741 skb->end = skb->head + size; 1742 #endif 1743 skb->tail += off; 1744 skb_headers_offset_update(skb, nhead); 1745 skb->cloned = 0; 1746 skb->hdr_len = 0; 1747 skb->nohdr = 0; 1748 atomic_set(&skb_shinfo(skb)->dataref, 1); 1749 1750 skb_metadata_clear(skb); 1751 1752 /* It is not generally safe to change skb->truesize. 1753 * For the moment, we really care of rx path, or 1754 * when skb is orphaned (not attached to a socket). 1755 */ 1756 if (!skb->sk || skb->destructor == sock_edemux) 1757 skb->truesize += size - osize; 1758 1759 return 0; 1760 1761 nofrags: 1762 kfree(data); 1763 nodata: 1764 return -ENOMEM; 1765 } 1766 EXPORT_SYMBOL(pskb_expand_head); 1767 1768 /* Make private copy of skb with writable head and some headroom */ 1769 1770 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 1771 { 1772 struct sk_buff *skb2; 1773 int delta = headroom - skb_headroom(skb); 1774 1775 if (delta <= 0) 1776 skb2 = pskb_copy(skb, GFP_ATOMIC); 1777 else { 1778 skb2 = skb_clone(skb, GFP_ATOMIC); 1779 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 1780 GFP_ATOMIC)) { 1781 kfree_skb(skb2); 1782 skb2 = NULL; 1783 } 1784 } 1785 return skb2; 1786 } 1787 EXPORT_SYMBOL(skb_realloc_headroom); 1788 1789 /** 1790 * skb_expand_head - reallocate header of &sk_buff 1791 * @skb: buffer to reallocate 1792 * @headroom: needed headroom 1793 * 1794 * Unlike skb_realloc_headroom, this one does not allocate a new skb 1795 * if possible; copies skb->sk to new skb as needed 1796 * and frees original skb in case of failures. 1797 * 1798 * It expect increased headroom and generates warning otherwise. 1799 */ 1800 1801 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom) 1802 { 1803 int delta = headroom - skb_headroom(skb); 1804 1805 if (WARN_ONCE(delta <= 0, 1806 "%s is expecting an increase in the headroom", __func__)) 1807 return skb; 1808 1809 /* pskb_expand_head() might crash, if skb is shared */ 1810 if (skb_shared(skb)) { 1811 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); 1812 1813 if (likely(nskb)) { 1814 if (skb->sk) 1815 skb_set_owner_w(nskb, skb->sk); 1816 consume_skb(skb); 1817 } else { 1818 kfree_skb(skb); 1819 } 1820 skb = nskb; 1821 } 1822 if (skb && 1823 pskb_expand_head(skb, SKB_DATA_ALIGN(delta), 0, GFP_ATOMIC)) { 1824 kfree_skb(skb); 1825 skb = NULL; 1826 } 1827 return skb; 1828 } 1829 EXPORT_SYMBOL(skb_expand_head); 1830 1831 /** 1832 * skb_copy_expand - copy and expand sk_buff 1833 * @skb: buffer to copy 1834 * @newheadroom: new free bytes at head 1835 * @newtailroom: new free bytes at tail 1836 * @gfp_mask: allocation priority 1837 * 1838 * Make a copy of both an &sk_buff and its data and while doing so 1839 * allocate additional space. 1840 * 1841 * This is used when the caller wishes to modify the data and needs a 1842 * private copy of the data to alter as well as more space for new fields. 1843 * Returns %NULL on failure or the pointer to the buffer 1844 * on success. The returned buffer has a reference count of 1. 1845 * 1846 * You must pass %GFP_ATOMIC as the allocation priority if this function 1847 * is called from an interrupt. 1848 */ 1849 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 1850 int newheadroom, int newtailroom, 1851 gfp_t gfp_mask) 1852 { 1853 /* 1854 * Allocate the copy buffer 1855 */ 1856 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 1857 gfp_mask, skb_alloc_rx_flag(skb), 1858 NUMA_NO_NODE); 1859 int oldheadroom = skb_headroom(skb); 1860 int head_copy_len, head_copy_off; 1861 1862 if (!n) 1863 return NULL; 1864 1865 skb_reserve(n, newheadroom); 1866 1867 /* Set the tail pointer and length */ 1868 skb_put(n, skb->len); 1869 1870 head_copy_len = oldheadroom; 1871 head_copy_off = 0; 1872 if (newheadroom <= head_copy_len) 1873 head_copy_len = newheadroom; 1874 else 1875 head_copy_off = newheadroom - head_copy_len; 1876 1877 /* Copy the linear header and data. */ 1878 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 1879 skb->len + head_copy_len)); 1880 1881 skb_copy_header(n, skb); 1882 1883 skb_headers_offset_update(n, newheadroom - oldheadroom); 1884 1885 return n; 1886 } 1887 EXPORT_SYMBOL(skb_copy_expand); 1888 1889 /** 1890 * __skb_pad - zero pad the tail of an skb 1891 * @skb: buffer to pad 1892 * @pad: space to pad 1893 * @free_on_error: free buffer on error 1894 * 1895 * Ensure that a buffer is followed by a padding area that is zero 1896 * filled. Used by network drivers which may DMA or transfer data 1897 * beyond the buffer end onto the wire. 1898 * 1899 * May return error in out of memory cases. The skb is freed on error 1900 * if @free_on_error is true. 1901 */ 1902 1903 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 1904 { 1905 int err; 1906 int ntail; 1907 1908 /* If the skbuff is non linear tailroom is always zero.. */ 1909 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 1910 memset(skb->data+skb->len, 0, pad); 1911 return 0; 1912 } 1913 1914 ntail = skb->data_len + pad - (skb->end - skb->tail); 1915 if (likely(skb_cloned(skb) || ntail > 0)) { 1916 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 1917 if (unlikely(err)) 1918 goto free_skb; 1919 } 1920 1921 /* FIXME: The use of this function with non-linear skb's really needs 1922 * to be audited. 1923 */ 1924 err = skb_linearize(skb); 1925 if (unlikely(err)) 1926 goto free_skb; 1927 1928 memset(skb->data + skb->len, 0, pad); 1929 return 0; 1930 1931 free_skb: 1932 if (free_on_error) 1933 kfree_skb(skb); 1934 return err; 1935 } 1936 EXPORT_SYMBOL(__skb_pad); 1937 1938 /** 1939 * pskb_put - add data to the tail of a potentially fragmented buffer 1940 * @skb: start of the buffer to use 1941 * @tail: tail fragment of the buffer to use 1942 * @len: amount of data to add 1943 * 1944 * This function extends the used data area of the potentially 1945 * fragmented buffer. @tail must be the last fragment of @skb -- or 1946 * @skb itself. If this would exceed the total buffer size the kernel 1947 * will panic. A pointer to the first byte of the extra data is 1948 * returned. 1949 */ 1950 1951 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 1952 { 1953 if (tail != skb) { 1954 skb->data_len += len; 1955 skb->len += len; 1956 } 1957 return skb_put(tail, len); 1958 } 1959 EXPORT_SYMBOL_GPL(pskb_put); 1960 1961 /** 1962 * skb_put - add data to a buffer 1963 * @skb: buffer to use 1964 * @len: amount of data to add 1965 * 1966 * This function extends the used data area of the buffer. If this would 1967 * exceed the total buffer size the kernel will panic. A pointer to the 1968 * first byte of the extra data is returned. 1969 */ 1970 void *skb_put(struct sk_buff *skb, unsigned int len) 1971 { 1972 void *tmp = skb_tail_pointer(skb); 1973 SKB_LINEAR_ASSERT(skb); 1974 skb->tail += len; 1975 skb->len += len; 1976 if (unlikely(skb->tail > skb->end)) 1977 skb_over_panic(skb, len, __builtin_return_address(0)); 1978 return tmp; 1979 } 1980 EXPORT_SYMBOL(skb_put); 1981 1982 /** 1983 * skb_push - add data to the start of a buffer 1984 * @skb: buffer to use 1985 * @len: amount of data to add 1986 * 1987 * This function extends the used data area of the buffer at the buffer 1988 * start. If this would exceed the total buffer headroom the kernel will 1989 * panic. A pointer to the first byte of the extra data is returned. 1990 */ 1991 void *skb_push(struct sk_buff *skb, unsigned int len) 1992 { 1993 skb->data -= len; 1994 skb->len += len; 1995 if (unlikely(skb->data < skb->head)) 1996 skb_under_panic(skb, len, __builtin_return_address(0)); 1997 return skb->data; 1998 } 1999 EXPORT_SYMBOL(skb_push); 2000 2001 /** 2002 * skb_pull - remove data from the start of a buffer 2003 * @skb: buffer to use 2004 * @len: amount of data to remove 2005 * 2006 * This function removes data from the start of a buffer, returning 2007 * the memory to the headroom. A pointer to the next data in the buffer 2008 * is returned. Once the data has been pulled future pushes will overwrite 2009 * the old data. 2010 */ 2011 void *skb_pull(struct sk_buff *skb, unsigned int len) 2012 { 2013 return skb_pull_inline(skb, len); 2014 } 2015 EXPORT_SYMBOL(skb_pull); 2016 2017 /** 2018 * skb_trim - remove end from a buffer 2019 * @skb: buffer to alter 2020 * @len: new length 2021 * 2022 * Cut the length of a buffer down by removing data from the tail. If 2023 * the buffer is already under the length specified it is not modified. 2024 * The skb must be linear. 2025 */ 2026 void skb_trim(struct sk_buff *skb, unsigned int len) 2027 { 2028 if (skb->len > len) 2029 __skb_trim(skb, len); 2030 } 2031 EXPORT_SYMBOL(skb_trim); 2032 2033 /* Trims skb to length len. It can change skb pointers. 2034 */ 2035 2036 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 2037 { 2038 struct sk_buff **fragp; 2039 struct sk_buff *frag; 2040 int offset = skb_headlen(skb); 2041 int nfrags = skb_shinfo(skb)->nr_frags; 2042 int i; 2043 int err; 2044 2045 if (skb_cloned(skb) && 2046 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 2047 return err; 2048 2049 i = 0; 2050 if (offset >= len) 2051 goto drop_pages; 2052 2053 for (; i < nfrags; i++) { 2054 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2055 2056 if (end < len) { 2057 offset = end; 2058 continue; 2059 } 2060 2061 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 2062 2063 drop_pages: 2064 skb_shinfo(skb)->nr_frags = i; 2065 2066 for (; i < nfrags; i++) 2067 skb_frag_unref(skb, i); 2068 2069 if (skb_has_frag_list(skb)) 2070 skb_drop_fraglist(skb); 2071 goto done; 2072 } 2073 2074 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 2075 fragp = &frag->next) { 2076 int end = offset + frag->len; 2077 2078 if (skb_shared(frag)) { 2079 struct sk_buff *nfrag; 2080 2081 nfrag = skb_clone(frag, GFP_ATOMIC); 2082 if (unlikely(!nfrag)) 2083 return -ENOMEM; 2084 2085 nfrag->next = frag->next; 2086 consume_skb(frag); 2087 frag = nfrag; 2088 *fragp = frag; 2089 } 2090 2091 if (end < len) { 2092 offset = end; 2093 continue; 2094 } 2095 2096 if (end > len && 2097 unlikely((err = pskb_trim(frag, len - offset)))) 2098 return err; 2099 2100 if (frag->next) 2101 skb_drop_list(&frag->next); 2102 break; 2103 } 2104 2105 done: 2106 if (len > skb_headlen(skb)) { 2107 skb->data_len -= skb->len - len; 2108 skb->len = len; 2109 } else { 2110 skb->len = len; 2111 skb->data_len = 0; 2112 skb_set_tail_pointer(skb, len); 2113 } 2114 2115 if (!skb->sk || skb->destructor == sock_edemux) 2116 skb_condense(skb); 2117 return 0; 2118 } 2119 EXPORT_SYMBOL(___pskb_trim); 2120 2121 /* Note : use pskb_trim_rcsum() instead of calling this directly 2122 */ 2123 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) 2124 { 2125 if (skb->ip_summed == CHECKSUM_COMPLETE) { 2126 int delta = skb->len - len; 2127 2128 skb->csum = csum_block_sub(skb->csum, 2129 skb_checksum(skb, len, delta, 0), 2130 len); 2131 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 2132 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; 2133 int offset = skb_checksum_start_offset(skb) + skb->csum_offset; 2134 2135 if (offset + sizeof(__sum16) > hdlen) 2136 return -EINVAL; 2137 } 2138 return __pskb_trim(skb, len); 2139 } 2140 EXPORT_SYMBOL(pskb_trim_rcsum_slow); 2141 2142 /** 2143 * __pskb_pull_tail - advance tail of skb header 2144 * @skb: buffer to reallocate 2145 * @delta: number of bytes to advance tail 2146 * 2147 * The function makes a sense only on a fragmented &sk_buff, 2148 * it expands header moving its tail forward and copying necessary 2149 * data from fragmented part. 2150 * 2151 * &sk_buff MUST have reference count of 1. 2152 * 2153 * Returns %NULL (and &sk_buff does not change) if pull failed 2154 * or value of new tail of skb in the case of success. 2155 * 2156 * All the pointers pointing into skb header may change and must be 2157 * reloaded after call to this function. 2158 */ 2159 2160 /* Moves tail of skb head forward, copying data from fragmented part, 2161 * when it is necessary. 2162 * 1. It may fail due to malloc failure. 2163 * 2. It may change skb pointers. 2164 * 2165 * It is pretty complicated. Luckily, it is called only in exceptional cases. 2166 */ 2167 void *__pskb_pull_tail(struct sk_buff *skb, int delta) 2168 { 2169 /* If skb has not enough free space at tail, get new one 2170 * plus 128 bytes for future expansions. If we have enough 2171 * room at tail, reallocate without expansion only if skb is cloned. 2172 */ 2173 int i, k, eat = (skb->tail + delta) - skb->end; 2174 2175 if (eat > 0 || skb_cloned(skb)) { 2176 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 2177 GFP_ATOMIC)) 2178 return NULL; 2179 } 2180 2181 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 2182 skb_tail_pointer(skb), delta)); 2183 2184 /* Optimization: no fragments, no reasons to preestimate 2185 * size of pulled pages. Superb. 2186 */ 2187 if (!skb_has_frag_list(skb)) 2188 goto pull_pages; 2189 2190 /* Estimate size of pulled pages. */ 2191 eat = delta; 2192 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2193 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2194 2195 if (size >= eat) 2196 goto pull_pages; 2197 eat -= size; 2198 } 2199 2200 /* If we need update frag list, we are in troubles. 2201 * Certainly, it is possible to add an offset to skb data, 2202 * but taking into account that pulling is expected to 2203 * be very rare operation, it is worth to fight against 2204 * further bloating skb head and crucify ourselves here instead. 2205 * Pure masohism, indeed. 8)8) 2206 */ 2207 if (eat) { 2208 struct sk_buff *list = skb_shinfo(skb)->frag_list; 2209 struct sk_buff *clone = NULL; 2210 struct sk_buff *insp = NULL; 2211 2212 do { 2213 if (list->len <= eat) { 2214 /* Eaten as whole. */ 2215 eat -= list->len; 2216 list = list->next; 2217 insp = list; 2218 } else { 2219 /* Eaten partially. */ 2220 2221 if (skb_shared(list)) { 2222 /* Sucks! We need to fork list. :-( */ 2223 clone = skb_clone(list, GFP_ATOMIC); 2224 if (!clone) 2225 return NULL; 2226 insp = list->next; 2227 list = clone; 2228 } else { 2229 /* This may be pulled without 2230 * problems. */ 2231 insp = list; 2232 } 2233 if (!pskb_pull(list, eat)) { 2234 kfree_skb(clone); 2235 return NULL; 2236 } 2237 break; 2238 } 2239 } while (eat); 2240 2241 /* Free pulled out fragments. */ 2242 while ((list = skb_shinfo(skb)->frag_list) != insp) { 2243 skb_shinfo(skb)->frag_list = list->next; 2244 kfree_skb(list); 2245 } 2246 /* And insert new clone at head. */ 2247 if (clone) { 2248 clone->next = list; 2249 skb_shinfo(skb)->frag_list = clone; 2250 } 2251 } 2252 /* Success! Now we may commit changes to skb data. */ 2253 2254 pull_pages: 2255 eat = delta; 2256 k = 0; 2257 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2258 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2259 2260 if (size <= eat) { 2261 skb_frag_unref(skb, i); 2262 eat -= size; 2263 } else { 2264 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2265 2266 *frag = skb_shinfo(skb)->frags[i]; 2267 if (eat) { 2268 skb_frag_off_add(frag, eat); 2269 skb_frag_size_sub(frag, eat); 2270 if (!i) 2271 goto end; 2272 eat = 0; 2273 } 2274 k++; 2275 } 2276 } 2277 skb_shinfo(skb)->nr_frags = k; 2278 2279 end: 2280 skb->tail += delta; 2281 skb->data_len -= delta; 2282 2283 if (!skb->data_len) 2284 skb_zcopy_clear(skb, false); 2285 2286 return skb_tail_pointer(skb); 2287 } 2288 EXPORT_SYMBOL(__pskb_pull_tail); 2289 2290 /** 2291 * skb_copy_bits - copy bits from skb to kernel buffer 2292 * @skb: source skb 2293 * @offset: offset in source 2294 * @to: destination buffer 2295 * @len: number of bytes to copy 2296 * 2297 * Copy the specified number of bytes from the source skb to the 2298 * destination buffer. 2299 * 2300 * CAUTION ! : 2301 * If its prototype is ever changed, 2302 * check arch/{*}/net/{*}.S files, 2303 * since it is called from BPF assembly code. 2304 */ 2305 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 2306 { 2307 int start = skb_headlen(skb); 2308 struct sk_buff *frag_iter; 2309 int i, copy; 2310 2311 if (offset > (int)skb->len - len) 2312 goto fault; 2313 2314 /* Copy header. */ 2315 if ((copy = start - offset) > 0) { 2316 if (copy > len) 2317 copy = len; 2318 skb_copy_from_linear_data_offset(skb, offset, to, copy); 2319 if ((len -= copy) == 0) 2320 return 0; 2321 offset += copy; 2322 to += copy; 2323 } 2324 2325 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2326 int end; 2327 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2328 2329 WARN_ON(start > offset + len); 2330 2331 end = start + skb_frag_size(f); 2332 if ((copy = end - offset) > 0) { 2333 u32 p_off, p_len, copied; 2334 struct page *p; 2335 u8 *vaddr; 2336 2337 if (copy > len) 2338 copy = len; 2339 2340 skb_frag_foreach_page(f, 2341 skb_frag_off(f) + offset - start, 2342 copy, p, p_off, p_len, copied) { 2343 vaddr = kmap_atomic(p); 2344 memcpy(to + copied, vaddr + p_off, p_len); 2345 kunmap_atomic(vaddr); 2346 } 2347 2348 if ((len -= copy) == 0) 2349 return 0; 2350 offset += copy; 2351 to += copy; 2352 } 2353 start = end; 2354 } 2355 2356 skb_walk_frags(skb, frag_iter) { 2357 int end; 2358 2359 WARN_ON(start > offset + len); 2360 2361 end = start + frag_iter->len; 2362 if ((copy = end - offset) > 0) { 2363 if (copy > len) 2364 copy = len; 2365 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 2366 goto fault; 2367 if ((len -= copy) == 0) 2368 return 0; 2369 offset += copy; 2370 to += copy; 2371 } 2372 start = end; 2373 } 2374 2375 if (!len) 2376 return 0; 2377 2378 fault: 2379 return -EFAULT; 2380 } 2381 EXPORT_SYMBOL(skb_copy_bits); 2382 2383 /* 2384 * Callback from splice_to_pipe(), if we need to release some pages 2385 * at the end of the spd in case we error'ed out in filling the pipe. 2386 */ 2387 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 2388 { 2389 put_page(spd->pages[i]); 2390 } 2391 2392 static struct page *linear_to_page(struct page *page, unsigned int *len, 2393 unsigned int *offset, 2394 struct sock *sk) 2395 { 2396 struct page_frag *pfrag = sk_page_frag(sk); 2397 2398 if (!sk_page_frag_refill(sk, pfrag)) 2399 return NULL; 2400 2401 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 2402 2403 memcpy(page_address(pfrag->page) + pfrag->offset, 2404 page_address(page) + *offset, *len); 2405 *offset = pfrag->offset; 2406 pfrag->offset += *len; 2407 2408 return pfrag->page; 2409 } 2410 2411 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 2412 struct page *page, 2413 unsigned int offset) 2414 { 2415 return spd->nr_pages && 2416 spd->pages[spd->nr_pages - 1] == page && 2417 (spd->partial[spd->nr_pages - 1].offset + 2418 spd->partial[spd->nr_pages - 1].len == offset); 2419 } 2420 2421 /* 2422 * Fill page/offset/length into spd, if it can hold more pages. 2423 */ 2424 static bool spd_fill_page(struct splice_pipe_desc *spd, 2425 struct pipe_inode_info *pipe, struct page *page, 2426 unsigned int *len, unsigned int offset, 2427 bool linear, 2428 struct sock *sk) 2429 { 2430 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 2431 return true; 2432 2433 if (linear) { 2434 page = linear_to_page(page, len, &offset, sk); 2435 if (!page) 2436 return true; 2437 } 2438 if (spd_can_coalesce(spd, page, offset)) { 2439 spd->partial[spd->nr_pages - 1].len += *len; 2440 return false; 2441 } 2442 get_page(page); 2443 spd->pages[spd->nr_pages] = page; 2444 spd->partial[spd->nr_pages].len = *len; 2445 spd->partial[spd->nr_pages].offset = offset; 2446 spd->nr_pages++; 2447 2448 return false; 2449 } 2450 2451 static bool __splice_segment(struct page *page, unsigned int poff, 2452 unsigned int plen, unsigned int *off, 2453 unsigned int *len, 2454 struct splice_pipe_desc *spd, bool linear, 2455 struct sock *sk, 2456 struct pipe_inode_info *pipe) 2457 { 2458 if (!*len) 2459 return true; 2460 2461 /* skip this segment if already processed */ 2462 if (*off >= plen) { 2463 *off -= plen; 2464 return false; 2465 } 2466 2467 /* ignore any bits we already processed */ 2468 poff += *off; 2469 plen -= *off; 2470 *off = 0; 2471 2472 do { 2473 unsigned int flen = min(*len, plen); 2474 2475 if (spd_fill_page(spd, pipe, page, &flen, poff, 2476 linear, sk)) 2477 return true; 2478 poff += flen; 2479 plen -= flen; 2480 *len -= flen; 2481 } while (*len && plen); 2482 2483 return false; 2484 } 2485 2486 /* 2487 * Map linear and fragment data from the skb to spd. It reports true if the 2488 * pipe is full or if we already spliced the requested length. 2489 */ 2490 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 2491 unsigned int *offset, unsigned int *len, 2492 struct splice_pipe_desc *spd, struct sock *sk) 2493 { 2494 int seg; 2495 struct sk_buff *iter; 2496 2497 /* map the linear part : 2498 * If skb->head_frag is set, this 'linear' part is backed by a 2499 * fragment, and if the head is not shared with any clones then 2500 * we can avoid a copy since we own the head portion of this page. 2501 */ 2502 if (__splice_segment(virt_to_page(skb->data), 2503 (unsigned long) skb->data & (PAGE_SIZE - 1), 2504 skb_headlen(skb), 2505 offset, len, spd, 2506 skb_head_is_locked(skb), 2507 sk, pipe)) 2508 return true; 2509 2510 /* 2511 * then map the fragments 2512 */ 2513 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 2514 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 2515 2516 if (__splice_segment(skb_frag_page(f), 2517 skb_frag_off(f), skb_frag_size(f), 2518 offset, len, spd, false, sk, pipe)) 2519 return true; 2520 } 2521 2522 skb_walk_frags(skb, iter) { 2523 if (*offset >= iter->len) { 2524 *offset -= iter->len; 2525 continue; 2526 } 2527 /* __skb_splice_bits() only fails if the output has no room 2528 * left, so no point in going over the frag_list for the error 2529 * case. 2530 */ 2531 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 2532 return true; 2533 } 2534 2535 return false; 2536 } 2537 2538 /* 2539 * Map data from the skb to a pipe. Should handle both the linear part, 2540 * the fragments, and the frag list. 2541 */ 2542 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 2543 struct pipe_inode_info *pipe, unsigned int tlen, 2544 unsigned int flags) 2545 { 2546 struct partial_page partial[MAX_SKB_FRAGS]; 2547 struct page *pages[MAX_SKB_FRAGS]; 2548 struct splice_pipe_desc spd = { 2549 .pages = pages, 2550 .partial = partial, 2551 .nr_pages_max = MAX_SKB_FRAGS, 2552 .ops = &nosteal_pipe_buf_ops, 2553 .spd_release = sock_spd_release, 2554 }; 2555 int ret = 0; 2556 2557 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 2558 2559 if (spd.nr_pages) 2560 ret = splice_to_pipe(pipe, &spd); 2561 2562 return ret; 2563 } 2564 EXPORT_SYMBOL_GPL(skb_splice_bits); 2565 2566 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg, 2567 struct kvec *vec, size_t num, size_t size) 2568 { 2569 struct socket *sock = sk->sk_socket; 2570 2571 if (!sock) 2572 return -EINVAL; 2573 return kernel_sendmsg(sock, msg, vec, num, size); 2574 } 2575 2576 static int sendpage_unlocked(struct sock *sk, struct page *page, int offset, 2577 size_t size, int flags) 2578 { 2579 struct socket *sock = sk->sk_socket; 2580 2581 if (!sock) 2582 return -EINVAL; 2583 return kernel_sendpage(sock, page, offset, size, flags); 2584 } 2585 2586 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg, 2587 struct kvec *vec, size_t num, size_t size); 2588 typedef int (*sendpage_func)(struct sock *sk, struct page *page, int offset, 2589 size_t size, int flags); 2590 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, 2591 int len, sendmsg_func sendmsg, sendpage_func sendpage) 2592 { 2593 unsigned int orig_len = len; 2594 struct sk_buff *head = skb; 2595 unsigned short fragidx; 2596 int slen, ret; 2597 2598 do_frag_list: 2599 2600 /* Deal with head data */ 2601 while (offset < skb_headlen(skb) && len) { 2602 struct kvec kv; 2603 struct msghdr msg; 2604 2605 slen = min_t(int, len, skb_headlen(skb) - offset); 2606 kv.iov_base = skb->data + offset; 2607 kv.iov_len = slen; 2608 memset(&msg, 0, sizeof(msg)); 2609 msg.msg_flags = MSG_DONTWAIT; 2610 2611 ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked, 2612 sendmsg_unlocked, sk, &msg, &kv, 1, slen); 2613 if (ret <= 0) 2614 goto error; 2615 2616 offset += ret; 2617 len -= ret; 2618 } 2619 2620 /* All the data was skb head? */ 2621 if (!len) 2622 goto out; 2623 2624 /* Make offset relative to start of frags */ 2625 offset -= skb_headlen(skb); 2626 2627 /* Find where we are in frag list */ 2628 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2629 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2630 2631 if (offset < skb_frag_size(frag)) 2632 break; 2633 2634 offset -= skb_frag_size(frag); 2635 } 2636 2637 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2638 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2639 2640 slen = min_t(size_t, len, skb_frag_size(frag) - offset); 2641 2642 while (slen) { 2643 ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked, 2644 sendpage_unlocked, sk, 2645 skb_frag_page(frag), 2646 skb_frag_off(frag) + offset, 2647 slen, MSG_DONTWAIT); 2648 if (ret <= 0) 2649 goto error; 2650 2651 len -= ret; 2652 offset += ret; 2653 slen -= ret; 2654 } 2655 2656 offset = 0; 2657 } 2658 2659 if (len) { 2660 /* Process any frag lists */ 2661 2662 if (skb == head) { 2663 if (skb_has_frag_list(skb)) { 2664 skb = skb_shinfo(skb)->frag_list; 2665 goto do_frag_list; 2666 } 2667 } else if (skb->next) { 2668 skb = skb->next; 2669 goto do_frag_list; 2670 } 2671 } 2672 2673 out: 2674 return orig_len - len; 2675 2676 error: 2677 return orig_len == len ? ret : orig_len - len; 2678 } 2679 2680 /* Send skb data on a socket. Socket must be locked. */ 2681 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 2682 int len) 2683 { 2684 return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked, 2685 kernel_sendpage_locked); 2686 } 2687 EXPORT_SYMBOL_GPL(skb_send_sock_locked); 2688 2689 /* Send skb data on a socket. Socket must be unlocked. */ 2690 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) 2691 { 2692 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked, 2693 sendpage_unlocked); 2694 } 2695 2696 /** 2697 * skb_store_bits - store bits from kernel buffer to skb 2698 * @skb: destination buffer 2699 * @offset: offset in destination 2700 * @from: source buffer 2701 * @len: number of bytes to copy 2702 * 2703 * Copy the specified number of bytes from the source buffer to the 2704 * destination skb. This function handles all the messy bits of 2705 * traversing fragment lists and such. 2706 */ 2707 2708 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 2709 { 2710 int start = skb_headlen(skb); 2711 struct sk_buff *frag_iter; 2712 int i, copy; 2713 2714 if (offset > (int)skb->len - len) 2715 goto fault; 2716 2717 if ((copy = start - offset) > 0) { 2718 if (copy > len) 2719 copy = len; 2720 skb_copy_to_linear_data_offset(skb, offset, from, copy); 2721 if ((len -= copy) == 0) 2722 return 0; 2723 offset += copy; 2724 from += copy; 2725 } 2726 2727 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2728 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2729 int end; 2730 2731 WARN_ON(start > offset + len); 2732 2733 end = start + skb_frag_size(frag); 2734 if ((copy = end - offset) > 0) { 2735 u32 p_off, p_len, copied; 2736 struct page *p; 2737 u8 *vaddr; 2738 2739 if (copy > len) 2740 copy = len; 2741 2742 skb_frag_foreach_page(frag, 2743 skb_frag_off(frag) + offset - start, 2744 copy, p, p_off, p_len, copied) { 2745 vaddr = kmap_atomic(p); 2746 memcpy(vaddr + p_off, from + copied, p_len); 2747 kunmap_atomic(vaddr); 2748 } 2749 2750 if ((len -= copy) == 0) 2751 return 0; 2752 offset += copy; 2753 from += copy; 2754 } 2755 start = end; 2756 } 2757 2758 skb_walk_frags(skb, frag_iter) { 2759 int end; 2760 2761 WARN_ON(start > offset + len); 2762 2763 end = start + frag_iter->len; 2764 if ((copy = end - offset) > 0) { 2765 if (copy > len) 2766 copy = len; 2767 if (skb_store_bits(frag_iter, offset - start, 2768 from, copy)) 2769 goto fault; 2770 if ((len -= copy) == 0) 2771 return 0; 2772 offset += copy; 2773 from += copy; 2774 } 2775 start = end; 2776 } 2777 if (!len) 2778 return 0; 2779 2780 fault: 2781 return -EFAULT; 2782 } 2783 EXPORT_SYMBOL(skb_store_bits); 2784 2785 /* Checksum skb data. */ 2786 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 2787 __wsum csum, const struct skb_checksum_ops *ops) 2788 { 2789 int start = skb_headlen(skb); 2790 int i, copy = start - offset; 2791 struct sk_buff *frag_iter; 2792 int pos = 0; 2793 2794 /* Checksum header. */ 2795 if (copy > 0) { 2796 if (copy > len) 2797 copy = len; 2798 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext, 2799 skb->data + offset, copy, csum); 2800 if ((len -= copy) == 0) 2801 return csum; 2802 offset += copy; 2803 pos = copy; 2804 } 2805 2806 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2807 int end; 2808 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2809 2810 WARN_ON(start > offset + len); 2811 2812 end = start + skb_frag_size(frag); 2813 if ((copy = end - offset) > 0) { 2814 u32 p_off, p_len, copied; 2815 struct page *p; 2816 __wsum csum2; 2817 u8 *vaddr; 2818 2819 if (copy > len) 2820 copy = len; 2821 2822 skb_frag_foreach_page(frag, 2823 skb_frag_off(frag) + offset - start, 2824 copy, p, p_off, p_len, copied) { 2825 vaddr = kmap_atomic(p); 2826 csum2 = INDIRECT_CALL_1(ops->update, 2827 csum_partial_ext, 2828 vaddr + p_off, p_len, 0); 2829 kunmap_atomic(vaddr); 2830 csum = INDIRECT_CALL_1(ops->combine, 2831 csum_block_add_ext, csum, 2832 csum2, pos, p_len); 2833 pos += p_len; 2834 } 2835 2836 if (!(len -= copy)) 2837 return csum; 2838 offset += copy; 2839 } 2840 start = end; 2841 } 2842 2843 skb_walk_frags(skb, frag_iter) { 2844 int end; 2845 2846 WARN_ON(start > offset + len); 2847 2848 end = start + frag_iter->len; 2849 if ((copy = end - offset) > 0) { 2850 __wsum csum2; 2851 if (copy > len) 2852 copy = len; 2853 csum2 = __skb_checksum(frag_iter, offset - start, 2854 copy, 0, ops); 2855 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext, 2856 csum, csum2, pos, copy); 2857 if ((len -= copy) == 0) 2858 return csum; 2859 offset += copy; 2860 pos += copy; 2861 } 2862 start = end; 2863 } 2864 BUG_ON(len); 2865 2866 return csum; 2867 } 2868 EXPORT_SYMBOL(__skb_checksum); 2869 2870 __wsum skb_checksum(const struct sk_buff *skb, int offset, 2871 int len, __wsum csum) 2872 { 2873 const struct skb_checksum_ops ops = { 2874 .update = csum_partial_ext, 2875 .combine = csum_block_add_ext, 2876 }; 2877 2878 return __skb_checksum(skb, offset, len, csum, &ops); 2879 } 2880 EXPORT_SYMBOL(skb_checksum); 2881 2882 /* Both of above in one bottle. */ 2883 2884 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 2885 u8 *to, int len) 2886 { 2887 int start = skb_headlen(skb); 2888 int i, copy = start - offset; 2889 struct sk_buff *frag_iter; 2890 int pos = 0; 2891 __wsum csum = 0; 2892 2893 /* Copy header. */ 2894 if (copy > 0) { 2895 if (copy > len) 2896 copy = len; 2897 csum = csum_partial_copy_nocheck(skb->data + offset, to, 2898 copy); 2899 if ((len -= copy) == 0) 2900 return csum; 2901 offset += copy; 2902 to += copy; 2903 pos = copy; 2904 } 2905 2906 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2907 int end; 2908 2909 WARN_ON(start > offset + len); 2910 2911 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2912 if ((copy = end - offset) > 0) { 2913 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2914 u32 p_off, p_len, copied; 2915 struct page *p; 2916 __wsum csum2; 2917 u8 *vaddr; 2918 2919 if (copy > len) 2920 copy = len; 2921 2922 skb_frag_foreach_page(frag, 2923 skb_frag_off(frag) + offset - start, 2924 copy, p, p_off, p_len, copied) { 2925 vaddr = kmap_atomic(p); 2926 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 2927 to + copied, 2928 p_len); 2929 kunmap_atomic(vaddr); 2930 csum = csum_block_add(csum, csum2, pos); 2931 pos += p_len; 2932 } 2933 2934 if (!(len -= copy)) 2935 return csum; 2936 offset += copy; 2937 to += copy; 2938 } 2939 start = end; 2940 } 2941 2942 skb_walk_frags(skb, frag_iter) { 2943 __wsum csum2; 2944 int end; 2945 2946 WARN_ON(start > offset + len); 2947 2948 end = start + frag_iter->len; 2949 if ((copy = end - offset) > 0) { 2950 if (copy > len) 2951 copy = len; 2952 csum2 = skb_copy_and_csum_bits(frag_iter, 2953 offset - start, 2954 to, copy); 2955 csum = csum_block_add(csum, csum2, pos); 2956 if ((len -= copy) == 0) 2957 return csum; 2958 offset += copy; 2959 to += copy; 2960 pos += copy; 2961 } 2962 start = end; 2963 } 2964 BUG_ON(len); 2965 return csum; 2966 } 2967 EXPORT_SYMBOL(skb_copy_and_csum_bits); 2968 2969 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 2970 { 2971 __sum16 sum; 2972 2973 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 2974 /* See comments in __skb_checksum_complete(). */ 2975 if (likely(!sum)) { 2976 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 2977 !skb->csum_complete_sw) 2978 netdev_rx_csum_fault(skb->dev, skb); 2979 } 2980 if (!skb_shared(skb)) 2981 skb->csum_valid = !sum; 2982 return sum; 2983 } 2984 EXPORT_SYMBOL(__skb_checksum_complete_head); 2985 2986 /* This function assumes skb->csum already holds pseudo header's checksum, 2987 * which has been changed from the hardware checksum, for example, by 2988 * __skb_checksum_validate_complete(). And, the original skb->csum must 2989 * have been validated unsuccessfully for CHECKSUM_COMPLETE case. 2990 * 2991 * It returns non-zero if the recomputed checksum is still invalid, otherwise 2992 * zero. The new checksum is stored back into skb->csum unless the skb is 2993 * shared. 2994 */ 2995 __sum16 __skb_checksum_complete(struct sk_buff *skb) 2996 { 2997 __wsum csum; 2998 __sum16 sum; 2999 3000 csum = skb_checksum(skb, 0, skb->len, 0); 3001 3002 sum = csum_fold(csum_add(skb->csum, csum)); 3003 /* This check is inverted, because we already knew the hardware 3004 * checksum is invalid before calling this function. So, if the 3005 * re-computed checksum is valid instead, then we have a mismatch 3006 * between the original skb->csum and skb_checksum(). This means either 3007 * the original hardware checksum is incorrect or we screw up skb->csum 3008 * when moving skb->data around. 3009 */ 3010 if (likely(!sum)) { 3011 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3012 !skb->csum_complete_sw) 3013 netdev_rx_csum_fault(skb->dev, skb); 3014 } 3015 3016 if (!skb_shared(skb)) { 3017 /* Save full packet checksum */ 3018 skb->csum = csum; 3019 skb->ip_summed = CHECKSUM_COMPLETE; 3020 skb->csum_complete_sw = 1; 3021 skb->csum_valid = !sum; 3022 } 3023 3024 return sum; 3025 } 3026 EXPORT_SYMBOL(__skb_checksum_complete); 3027 3028 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum) 3029 { 3030 net_warn_ratelimited( 3031 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3032 __func__); 3033 return 0; 3034 } 3035 3036 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2, 3037 int offset, int len) 3038 { 3039 net_warn_ratelimited( 3040 "%s: attempt to compute crc32c without libcrc32c.ko\n", 3041 __func__); 3042 return 0; 3043 } 3044 3045 static const struct skb_checksum_ops default_crc32c_ops = { 3046 .update = warn_crc32c_csum_update, 3047 .combine = warn_crc32c_csum_combine, 3048 }; 3049 3050 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly = 3051 &default_crc32c_ops; 3052 EXPORT_SYMBOL(crc32c_csum_stub); 3053 3054 /** 3055 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 3056 * @from: source buffer 3057 * 3058 * Calculates the amount of linear headroom needed in the 'to' skb passed 3059 * into skb_zerocopy(). 3060 */ 3061 unsigned int 3062 skb_zerocopy_headlen(const struct sk_buff *from) 3063 { 3064 unsigned int hlen = 0; 3065 3066 if (!from->head_frag || 3067 skb_headlen(from) < L1_CACHE_BYTES || 3068 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) { 3069 hlen = skb_headlen(from); 3070 if (!hlen) 3071 hlen = from->len; 3072 } 3073 3074 if (skb_has_frag_list(from)) 3075 hlen = from->len; 3076 3077 return hlen; 3078 } 3079 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 3080 3081 /** 3082 * skb_zerocopy - Zero copy skb to skb 3083 * @to: destination buffer 3084 * @from: source buffer 3085 * @len: number of bytes to copy from source buffer 3086 * @hlen: size of linear headroom in destination buffer 3087 * 3088 * Copies up to `len` bytes from `from` to `to` by creating references 3089 * to the frags in the source buffer. 3090 * 3091 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 3092 * headroom in the `to` buffer. 3093 * 3094 * Return value: 3095 * 0: everything is OK 3096 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 3097 * -EFAULT: skb_copy_bits() found some problem with skb geometry 3098 */ 3099 int 3100 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 3101 { 3102 int i, j = 0; 3103 int plen = 0; /* length of skb->head fragment */ 3104 int ret; 3105 struct page *page; 3106 unsigned int offset; 3107 3108 BUG_ON(!from->head_frag && !hlen); 3109 3110 /* dont bother with small payloads */ 3111 if (len <= skb_tailroom(to)) 3112 return skb_copy_bits(from, 0, skb_put(to, len), len); 3113 3114 if (hlen) { 3115 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 3116 if (unlikely(ret)) 3117 return ret; 3118 len -= hlen; 3119 } else { 3120 plen = min_t(int, skb_headlen(from), len); 3121 if (plen) { 3122 page = virt_to_head_page(from->head); 3123 offset = from->data - (unsigned char *)page_address(page); 3124 __skb_fill_page_desc(to, 0, page, offset, plen); 3125 get_page(page); 3126 j = 1; 3127 len -= plen; 3128 } 3129 } 3130 3131 to->truesize += len + plen; 3132 to->len += len + plen; 3133 to->data_len += len + plen; 3134 3135 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 3136 skb_tx_error(from); 3137 return -ENOMEM; 3138 } 3139 skb_zerocopy_clone(to, from, GFP_ATOMIC); 3140 3141 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 3142 int size; 3143 3144 if (!len) 3145 break; 3146 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 3147 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]), 3148 len); 3149 skb_frag_size_set(&skb_shinfo(to)->frags[j], size); 3150 len -= size; 3151 skb_frag_ref(to, j); 3152 j++; 3153 } 3154 skb_shinfo(to)->nr_frags = j; 3155 3156 return 0; 3157 } 3158 EXPORT_SYMBOL_GPL(skb_zerocopy); 3159 3160 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 3161 { 3162 __wsum csum; 3163 long csstart; 3164 3165 if (skb->ip_summed == CHECKSUM_PARTIAL) 3166 csstart = skb_checksum_start_offset(skb); 3167 else 3168 csstart = skb_headlen(skb); 3169 3170 BUG_ON(csstart > skb_headlen(skb)); 3171 3172 skb_copy_from_linear_data(skb, to, csstart); 3173 3174 csum = 0; 3175 if (csstart != skb->len) 3176 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 3177 skb->len - csstart); 3178 3179 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3180 long csstuff = csstart + skb->csum_offset; 3181 3182 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 3183 } 3184 } 3185 EXPORT_SYMBOL(skb_copy_and_csum_dev); 3186 3187 /** 3188 * skb_dequeue - remove from the head of the queue 3189 * @list: list to dequeue from 3190 * 3191 * Remove the head of the list. The list lock is taken so the function 3192 * may be used safely with other locking list functions. The head item is 3193 * returned or %NULL if the list is empty. 3194 */ 3195 3196 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 3197 { 3198 unsigned long flags; 3199 struct sk_buff *result; 3200 3201 spin_lock_irqsave(&list->lock, flags); 3202 result = __skb_dequeue(list); 3203 spin_unlock_irqrestore(&list->lock, flags); 3204 return result; 3205 } 3206 EXPORT_SYMBOL(skb_dequeue); 3207 3208 /** 3209 * skb_dequeue_tail - remove from the tail of the queue 3210 * @list: list to dequeue from 3211 * 3212 * Remove the tail of the list. The list lock is taken so the function 3213 * may be used safely with other locking list functions. The tail item is 3214 * returned or %NULL if the list is empty. 3215 */ 3216 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 3217 { 3218 unsigned long flags; 3219 struct sk_buff *result; 3220 3221 spin_lock_irqsave(&list->lock, flags); 3222 result = __skb_dequeue_tail(list); 3223 spin_unlock_irqrestore(&list->lock, flags); 3224 return result; 3225 } 3226 EXPORT_SYMBOL(skb_dequeue_tail); 3227 3228 /** 3229 * skb_queue_purge - empty a list 3230 * @list: list to empty 3231 * 3232 * Delete all buffers on an &sk_buff list. Each buffer is removed from 3233 * the list and one reference dropped. This function takes the list 3234 * lock and is atomic with respect to other list locking functions. 3235 */ 3236 void skb_queue_purge(struct sk_buff_head *list) 3237 { 3238 struct sk_buff *skb; 3239 while ((skb = skb_dequeue(list)) != NULL) 3240 kfree_skb(skb); 3241 } 3242 EXPORT_SYMBOL(skb_queue_purge); 3243 3244 /** 3245 * skb_rbtree_purge - empty a skb rbtree 3246 * @root: root of the rbtree to empty 3247 * Return value: the sum of truesizes of all purged skbs. 3248 * 3249 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 3250 * the list and one reference dropped. This function does not take 3251 * any lock. Synchronization should be handled by the caller (e.g., TCP 3252 * out-of-order queue is protected by the socket lock). 3253 */ 3254 unsigned int skb_rbtree_purge(struct rb_root *root) 3255 { 3256 struct rb_node *p = rb_first(root); 3257 unsigned int sum = 0; 3258 3259 while (p) { 3260 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 3261 3262 p = rb_next(p); 3263 rb_erase(&skb->rbnode, root); 3264 sum += skb->truesize; 3265 kfree_skb(skb); 3266 } 3267 return sum; 3268 } 3269 3270 /** 3271 * skb_queue_head - queue a buffer at the list head 3272 * @list: list to use 3273 * @newsk: buffer to queue 3274 * 3275 * Queue a buffer at the start of the list. This function takes the 3276 * list lock and can be used safely with other locking &sk_buff functions 3277 * safely. 3278 * 3279 * A buffer cannot be placed on two lists at the same time. 3280 */ 3281 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 3282 { 3283 unsigned long flags; 3284 3285 spin_lock_irqsave(&list->lock, flags); 3286 __skb_queue_head(list, newsk); 3287 spin_unlock_irqrestore(&list->lock, flags); 3288 } 3289 EXPORT_SYMBOL(skb_queue_head); 3290 3291 /** 3292 * skb_queue_tail - queue a buffer at the list tail 3293 * @list: list to use 3294 * @newsk: buffer to queue 3295 * 3296 * Queue a buffer at the tail of the list. This function takes the 3297 * list lock and can be used safely with other locking &sk_buff functions 3298 * safely. 3299 * 3300 * A buffer cannot be placed on two lists at the same time. 3301 */ 3302 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 3303 { 3304 unsigned long flags; 3305 3306 spin_lock_irqsave(&list->lock, flags); 3307 __skb_queue_tail(list, newsk); 3308 spin_unlock_irqrestore(&list->lock, flags); 3309 } 3310 EXPORT_SYMBOL(skb_queue_tail); 3311 3312 /** 3313 * skb_unlink - remove a buffer from a list 3314 * @skb: buffer to remove 3315 * @list: list to use 3316 * 3317 * Remove a packet from a list. The list locks are taken and this 3318 * function is atomic with respect to other list locked calls 3319 * 3320 * You must know what list the SKB is on. 3321 */ 3322 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 3323 { 3324 unsigned long flags; 3325 3326 spin_lock_irqsave(&list->lock, flags); 3327 __skb_unlink(skb, list); 3328 spin_unlock_irqrestore(&list->lock, flags); 3329 } 3330 EXPORT_SYMBOL(skb_unlink); 3331 3332 /** 3333 * skb_append - append a buffer 3334 * @old: buffer to insert after 3335 * @newsk: buffer to insert 3336 * @list: list to use 3337 * 3338 * Place a packet after a given packet in a list. The list locks are taken 3339 * and this function is atomic with respect to other list locked calls. 3340 * A buffer cannot be placed on two lists at the same time. 3341 */ 3342 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 3343 { 3344 unsigned long flags; 3345 3346 spin_lock_irqsave(&list->lock, flags); 3347 __skb_queue_after(list, old, newsk); 3348 spin_unlock_irqrestore(&list->lock, flags); 3349 } 3350 EXPORT_SYMBOL(skb_append); 3351 3352 static inline void skb_split_inside_header(struct sk_buff *skb, 3353 struct sk_buff* skb1, 3354 const u32 len, const int pos) 3355 { 3356 int i; 3357 3358 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 3359 pos - len); 3360 /* And move data appendix as is. */ 3361 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 3362 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 3363 3364 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 3365 skb_shinfo(skb)->nr_frags = 0; 3366 skb1->data_len = skb->data_len; 3367 skb1->len += skb1->data_len; 3368 skb->data_len = 0; 3369 skb->len = len; 3370 skb_set_tail_pointer(skb, len); 3371 } 3372 3373 static inline void skb_split_no_header(struct sk_buff *skb, 3374 struct sk_buff* skb1, 3375 const u32 len, int pos) 3376 { 3377 int i, k = 0; 3378 const int nfrags = skb_shinfo(skb)->nr_frags; 3379 3380 skb_shinfo(skb)->nr_frags = 0; 3381 skb1->len = skb1->data_len = skb->len - len; 3382 skb->len = len; 3383 skb->data_len = len - pos; 3384 3385 for (i = 0; i < nfrags; i++) { 3386 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 3387 3388 if (pos + size > len) { 3389 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 3390 3391 if (pos < len) { 3392 /* Split frag. 3393 * We have two variants in this case: 3394 * 1. Move all the frag to the second 3395 * part, if it is possible. F.e. 3396 * this approach is mandatory for TUX, 3397 * where splitting is expensive. 3398 * 2. Split is accurately. We make this. 3399 */ 3400 skb_frag_ref(skb, i); 3401 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos); 3402 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 3403 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 3404 skb_shinfo(skb)->nr_frags++; 3405 } 3406 k++; 3407 } else 3408 skb_shinfo(skb)->nr_frags++; 3409 pos += size; 3410 } 3411 skb_shinfo(skb1)->nr_frags = k; 3412 } 3413 3414 /** 3415 * skb_split - Split fragmented skb to two parts at length len. 3416 * @skb: the buffer to split 3417 * @skb1: the buffer to receive the second part 3418 * @len: new length for skb 3419 */ 3420 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 3421 { 3422 int pos = skb_headlen(skb); 3423 3424 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; 3425 skb_zerocopy_clone(skb1, skb, 0); 3426 if (len < pos) /* Split line is inside header. */ 3427 skb_split_inside_header(skb, skb1, len, pos); 3428 else /* Second chunk has no header, nothing to copy. */ 3429 skb_split_no_header(skb, skb1, len, pos); 3430 } 3431 EXPORT_SYMBOL(skb_split); 3432 3433 /* Shifting from/to a cloned skb is a no-go. 3434 * 3435 * Caller cannot keep skb_shinfo related pointers past calling here! 3436 */ 3437 static int skb_prepare_for_shift(struct sk_buff *skb) 3438 { 3439 int ret = 0; 3440 3441 if (skb_cloned(skb)) { 3442 /* Save and restore truesize: pskb_expand_head() may reallocate 3443 * memory where ksize(kmalloc(S)) != ksize(kmalloc(S)), but we 3444 * cannot change truesize at this point. 3445 */ 3446 unsigned int save_truesize = skb->truesize; 3447 3448 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 3449 skb->truesize = save_truesize; 3450 } 3451 return ret; 3452 } 3453 3454 /** 3455 * skb_shift - Shifts paged data partially from skb to another 3456 * @tgt: buffer into which tail data gets added 3457 * @skb: buffer from which the paged data comes from 3458 * @shiftlen: shift up to this many bytes 3459 * 3460 * Attempts to shift up to shiftlen worth of bytes, which may be less than 3461 * the length of the skb, from skb to tgt. Returns number bytes shifted. 3462 * It's up to caller to free skb if everything was shifted. 3463 * 3464 * If @tgt runs out of frags, the whole operation is aborted. 3465 * 3466 * Skb cannot include anything else but paged data while tgt is allowed 3467 * to have non-paged data as well. 3468 * 3469 * TODO: full sized shift could be optimized but that would need 3470 * specialized skb free'er to handle frags without up-to-date nr_frags. 3471 */ 3472 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 3473 { 3474 int from, to, merge, todo; 3475 skb_frag_t *fragfrom, *fragto; 3476 3477 BUG_ON(shiftlen > skb->len); 3478 3479 if (skb_headlen(skb)) 3480 return 0; 3481 if (skb_zcopy(tgt) || skb_zcopy(skb)) 3482 return 0; 3483 3484 todo = shiftlen; 3485 from = 0; 3486 to = skb_shinfo(tgt)->nr_frags; 3487 fragfrom = &skb_shinfo(skb)->frags[from]; 3488 3489 /* Actual merge is delayed until the point when we know we can 3490 * commit all, so that we don't have to undo partial changes 3491 */ 3492 if (!to || 3493 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 3494 skb_frag_off(fragfrom))) { 3495 merge = -1; 3496 } else { 3497 merge = to - 1; 3498 3499 todo -= skb_frag_size(fragfrom); 3500 if (todo < 0) { 3501 if (skb_prepare_for_shift(skb) || 3502 skb_prepare_for_shift(tgt)) 3503 return 0; 3504 3505 /* All previous frag pointers might be stale! */ 3506 fragfrom = &skb_shinfo(skb)->frags[from]; 3507 fragto = &skb_shinfo(tgt)->frags[merge]; 3508 3509 skb_frag_size_add(fragto, shiftlen); 3510 skb_frag_size_sub(fragfrom, shiftlen); 3511 skb_frag_off_add(fragfrom, shiftlen); 3512 3513 goto onlymerged; 3514 } 3515 3516 from++; 3517 } 3518 3519 /* Skip full, not-fitting skb to avoid expensive operations */ 3520 if ((shiftlen == skb->len) && 3521 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 3522 return 0; 3523 3524 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 3525 return 0; 3526 3527 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 3528 if (to == MAX_SKB_FRAGS) 3529 return 0; 3530 3531 fragfrom = &skb_shinfo(skb)->frags[from]; 3532 fragto = &skb_shinfo(tgt)->frags[to]; 3533 3534 if (todo >= skb_frag_size(fragfrom)) { 3535 *fragto = *fragfrom; 3536 todo -= skb_frag_size(fragfrom); 3537 from++; 3538 to++; 3539 3540 } else { 3541 __skb_frag_ref(fragfrom); 3542 skb_frag_page_copy(fragto, fragfrom); 3543 skb_frag_off_copy(fragto, fragfrom); 3544 skb_frag_size_set(fragto, todo); 3545 3546 skb_frag_off_add(fragfrom, todo); 3547 skb_frag_size_sub(fragfrom, todo); 3548 todo = 0; 3549 3550 to++; 3551 break; 3552 } 3553 } 3554 3555 /* Ready to "commit" this state change to tgt */ 3556 skb_shinfo(tgt)->nr_frags = to; 3557 3558 if (merge >= 0) { 3559 fragfrom = &skb_shinfo(skb)->frags[0]; 3560 fragto = &skb_shinfo(tgt)->frags[merge]; 3561 3562 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 3563 __skb_frag_unref(fragfrom, skb->pp_recycle); 3564 } 3565 3566 /* Reposition in the original skb */ 3567 to = 0; 3568 while (from < skb_shinfo(skb)->nr_frags) 3569 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 3570 skb_shinfo(skb)->nr_frags = to; 3571 3572 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 3573 3574 onlymerged: 3575 /* Most likely the tgt won't ever need its checksum anymore, skb on 3576 * the other hand might need it if it needs to be resent 3577 */ 3578 tgt->ip_summed = CHECKSUM_PARTIAL; 3579 skb->ip_summed = CHECKSUM_PARTIAL; 3580 3581 /* Yak, is it really working this way? Some helper please? */ 3582 skb->len -= shiftlen; 3583 skb->data_len -= shiftlen; 3584 skb->truesize -= shiftlen; 3585 tgt->len += shiftlen; 3586 tgt->data_len += shiftlen; 3587 tgt->truesize += shiftlen; 3588 3589 return shiftlen; 3590 } 3591 3592 /** 3593 * skb_prepare_seq_read - Prepare a sequential read of skb data 3594 * @skb: the buffer to read 3595 * @from: lower offset of data to be read 3596 * @to: upper offset of data to be read 3597 * @st: state variable 3598 * 3599 * Initializes the specified state variable. Must be called before 3600 * invoking skb_seq_read() for the first time. 3601 */ 3602 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 3603 unsigned int to, struct skb_seq_state *st) 3604 { 3605 st->lower_offset = from; 3606 st->upper_offset = to; 3607 st->root_skb = st->cur_skb = skb; 3608 st->frag_idx = st->stepped_offset = 0; 3609 st->frag_data = NULL; 3610 st->frag_off = 0; 3611 } 3612 EXPORT_SYMBOL(skb_prepare_seq_read); 3613 3614 /** 3615 * skb_seq_read - Sequentially read skb data 3616 * @consumed: number of bytes consumed by the caller so far 3617 * @data: destination pointer for data to be returned 3618 * @st: state variable 3619 * 3620 * Reads a block of skb data at @consumed relative to the 3621 * lower offset specified to skb_prepare_seq_read(). Assigns 3622 * the head of the data block to @data and returns the length 3623 * of the block or 0 if the end of the skb data or the upper 3624 * offset has been reached. 3625 * 3626 * The caller is not required to consume all of the data 3627 * returned, i.e. @consumed is typically set to the number 3628 * of bytes already consumed and the next call to 3629 * skb_seq_read() will return the remaining part of the block. 3630 * 3631 * Note 1: The size of each block of data returned can be arbitrary, 3632 * this limitation is the cost for zerocopy sequential 3633 * reads of potentially non linear data. 3634 * 3635 * Note 2: Fragment lists within fragments are not implemented 3636 * at the moment, state->root_skb could be replaced with 3637 * a stack for this purpose. 3638 */ 3639 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 3640 struct skb_seq_state *st) 3641 { 3642 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 3643 skb_frag_t *frag; 3644 3645 if (unlikely(abs_offset >= st->upper_offset)) { 3646 if (st->frag_data) { 3647 kunmap_atomic(st->frag_data); 3648 st->frag_data = NULL; 3649 } 3650 return 0; 3651 } 3652 3653 next_skb: 3654 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 3655 3656 if (abs_offset < block_limit && !st->frag_data) { 3657 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 3658 return block_limit - abs_offset; 3659 } 3660 3661 if (st->frag_idx == 0 && !st->frag_data) 3662 st->stepped_offset += skb_headlen(st->cur_skb); 3663 3664 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 3665 unsigned int pg_idx, pg_off, pg_sz; 3666 3667 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 3668 3669 pg_idx = 0; 3670 pg_off = skb_frag_off(frag); 3671 pg_sz = skb_frag_size(frag); 3672 3673 if (skb_frag_must_loop(skb_frag_page(frag))) { 3674 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT; 3675 pg_off = offset_in_page(pg_off + st->frag_off); 3676 pg_sz = min_t(unsigned int, pg_sz - st->frag_off, 3677 PAGE_SIZE - pg_off); 3678 } 3679 3680 block_limit = pg_sz + st->stepped_offset; 3681 if (abs_offset < block_limit) { 3682 if (!st->frag_data) 3683 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx); 3684 3685 *data = (u8 *)st->frag_data + pg_off + 3686 (abs_offset - st->stepped_offset); 3687 3688 return block_limit - abs_offset; 3689 } 3690 3691 if (st->frag_data) { 3692 kunmap_atomic(st->frag_data); 3693 st->frag_data = NULL; 3694 } 3695 3696 st->stepped_offset += pg_sz; 3697 st->frag_off += pg_sz; 3698 if (st->frag_off == skb_frag_size(frag)) { 3699 st->frag_off = 0; 3700 st->frag_idx++; 3701 } 3702 } 3703 3704 if (st->frag_data) { 3705 kunmap_atomic(st->frag_data); 3706 st->frag_data = NULL; 3707 } 3708 3709 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 3710 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 3711 st->frag_idx = 0; 3712 goto next_skb; 3713 } else if (st->cur_skb->next) { 3714 st->cur_skb = st->cur_skb->next; 3715 st->frag_idx = 0; 3716 goto next_skb; 3717 } 3718 3719 return 0; 3720 } 3721 EXPORT_SYMBOL(skb_seq_read); 3722 3723 /** 3724 * skb_abort_seq_read - Abort a sequential read of skb data 3725 * @st: state variable 3726 * 3727 * Must be called if skb_seq_read() was not called until it 3728 * returned 0. 3729 */ 3730 void skb_abort_seq_read(struct skb_seq_state *st) 3731 { 3732 if (st->frag_data) 3733 kunmap_atomic(st->frag_data); 3734 } 3735 EXPORT_SYMBOL(skb_abort_seq_read); 3736 3737 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 3738 3739 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 3740 struct ts_config *conf, 3741 struct ts_state *state) 3742 { 3743 return skb_seq_read(offset, text, TS_SKB_CB(state)); 3744 } 3745 3746 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 3747 { 3748 skb_abort_seq_read(TS_SKB_CB(state)); 3749 } 3750 3751 /** 3752 * skb_find_text - Find a text pattern in skb data 3753 * @skb: the buffer to look in 3754 * @from: search offset 3755 * @to: search limit 3756 * @config: textsearch configuration 3757 * 3758 * Finds a pattern in the skb data according to the specified 3759 * textsearch configuration. Use textsearch_next() to retrieve 3760 * subsequent occurrences of the pattern. Returns the offset 3761 * to the first occurrence or UINT_MAX if no match was found. 3762 */ 3763 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 3764 unsigned int to, struct ts_config *config) 3765 { 3766 struct ts_state state; 3767 unsigned int ret; 3768 3769 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb)); 3770 3771 config->get_next_block = skb_ts_get_next_block; 3772 config->finish = skb_ts_finish; 3773 3774 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 3775 3776 ret = textsearch_find(config, &state); 3777 return (ret <= to - from ? ret : UINT_MAX); 3778 } 3779 EXPORT_SYMBOL(skb_find_text); 3780 3781 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 3782 int offset, size_t size) 3783 { 3784 int i = skb_shinfo(skb)->nr_frags; 3785 3786 if (skb_can_coalesce(skb, i, page, offset)) { 3787 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 3788 } else if (i < MAX_SKB_FRAGS) { 3789 get_page(page); 3790 skb_fill_page_desc(skb, i, page, offset, size); 3791 } else { 3792 return -EMSGSIZE; 3793 } 3794 3795 return 0; 3796 } 3797 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 3798 3799 /** 3800 * skb_pull_rcsum - pull skb and update receive checksum 3801 * @skb: buffer to update 3802 * @len: length of data pulled 3803 * 3804 * This function performs an skb_pull on the packet and updates 3805 * the CHECKSUM_COMPLETE checksum. It should be used on 3806 * receive path processing instead of skb_pull unless you know 3807 * that the checksum difference is zero (e.g., a valid IP header) 3808 * or you are setting ip_summed to CHECKSUM_NONE. 3809 */ 3810 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 3811 { 3812 unsigned char *data = skb->data; 3813 3814 BUG_ON(len > skb->len); 3815 __skb_pull(skb, len); 3816 skb_postpull_rcsum(skb, data, len); 3817 return skb->data; 3818 } 3819 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 3820 3821 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb) 3822 { 3823 skb_frag_t head_frag; 3824 struct page *page; 3825 3826 page = virt_to_head_page(frag_skb->head); 3827 __skb_frag_set_page(&head_frag, page); 3828 skb_frag_off_set(&head_frag, frag_skb->data - 3829 (unsigned char *)page_address(page)); 3830 skb_frag_size_set(&head_frag, skb_headlen(frag_skb)); 3831 return head_frag; 3832 } 3833 3834 struct sk_buff *skb_segment_list(struct sk_buff *skb, 3835 netdev_features_t features, 3836 unsigned int offset) 3837 { 3838 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list; 3839 unsigned int tnl_hlen = skb_tnl_header_len(skb); 3840 unsigned int delta_truesize = 0; 3841 unsigned int delta_len = 0; 3842 struct sk_buff *tail = NULL; 3843 struct sk_buff *nskb, *tmp; 3844 int err; 3845 3846 skb_push(skb, -skb_network_offset(skb) + offset); 3847 3848 skb_shinfo(skb)->frag_list = NULL; 3849 3850 do { 3851 nskb = list_skb; 3852 list_skb = list_skb->next; 3853 3854 err = 0; 3855 if (skb_shared(nskb)) { 3856 tmp = skb_clone(nskb, GFP_ATOMIC); 3857 if (tmp) { 3858 consume_skb(nskb); 3859 nskb = tmp; 3860 err = skb_unclone(nskb, GFP_ATOMIC); 3861 } else { 3862 err = -ENOMEM; 3863 } 3864 } 3865 3866 if (!tail) 3867 skb->next = nskb; 3868 else 3869 tail->next = nskb; 3870 3871 if (unlikely(err)) { 3872 nskb->next = list_skb; 3873 goto err_linearize; 3874 } 3875 3876 tail = nskb; 3877 3878 delta_len += nskb->len; 3879 delta_truesize += nskb->truesize; 3880 3881 skb_push(nskb, -skb_network_offset(nskb) + offset); 3882 3883 skb_release_head_state(nskb); 3884 __copy_skb_header(nskb, skb); 3885 3886 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb)); 3887 skb_copy_from_linear_data_offset(skb, -tnl_hlen, 3888 nskb->data - tnl_hlen, 3889 offset + tnl_hlen); 3890 3891 if (skb_needs_linearize(nskb, features) && 3892 __skb_linearize(nskb)) 3893 goto err_linearize; 3894 3895 } while (list_skb); 3896 3897 skb->truesize = skb->truesize - delta_truesize; 3898 skb->data_len = skb->data_len - delta_len; 3899 skb->len = skb->len - delta_len; 3900 3901 skb_gso_reset(skb); 3902 3903 skb->prev = tail; 3904 3905 if (skb_needs_linearize(skb, features) && 3906 __skb_linearize(skb)) 3907 goto err_linearize; 3908 3909 skb_get(skb); 3910 3911 return skb; 3912 3913 err_linearize: 3914 kfree_skb_list(skb->next); 3915 skb->next = NULL; 3916 return ERR_PTR(-ENOMEM); 3917 } 3918 EXPORT_SYMBOL_GPL(skb_segment_list); 3919 3920 int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) 3921 { 3922 if (unlikely(p->len + skb->len >= 65536)) 3923 return -E2BIG; 3924 3925 if (NAPI_GRO_CB(p)->last == p) 3926 skb_shinfo(p)->frag_list = skb; 3927 else 3928 NAPI_GRO_CB(p)->last->next = skb; 3929 3930 skb_pull(skb, skb_gro_offset(skb)); 3931 3932 NAPI_GRO_CB(p)->last = skb; 3933 NAPI_GRO_CB(p)->count++; 3934 p->data_len += skb->len; 3935 3936 /* sk owenrship - if any - completely transferred to the aggregated packet */ 3937 skb->destructor = NULL; 3938 p->truesize += skb->truesize; 3939 p->len += skb->len; 3940 3941 NAPI_GRO_CB(skb)->same_flow = 1; 3942 3943 return 0; 3944 } 3945 3946 /** 3947 * skb_segment - Perform protocol segmentation on skb. 3948 * @head_skb: buffer to segment 3949 * @features: features for the output path (see dev->features) 3950 * 3951 * This function performs segmentation on the given skb. It returns 3952 * a pointer to the first in a list of new skbs for the segments. 3953 * In case of error it returns ERR_PTR(err). 3954 */ 3955 struct sk_buff *skb_segment(struct sk_buff *head_skb, 3956 netdev_features_t features) 3957 { 3958 struct sk_buff *segs = NULL; 3959 struct sk_buff *tail = NULL; 3960 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 3961 skb_frag_t *frag = skb_shinfo(head_skb)->frags; 3962 unsigned int mss = skb_shinfo(head_skb)->gso_size; 3963 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 3964 struct sk_buff *frag_skb = head_skb; 3965 unsigned int offset = doffset; 3966 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 3967 unsigned int partial_segs = 0; 3968 unsigned int headroom; 3969 unsigned int len = head_skb->len; 3970 __be16 proto; 3971 bool csum, sg; 3972 int nfrags = skb_shinfo(head_skb)->nr_frags; 3973 int err = -ENOMEM; 3974 int i = 0; 3975 int pos; 3976 3977 if (list_skb && !list_skb->head_frag && skb_headlen(list_skb) && 3978 (skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY)) { 3979 /* gso_size is untrusted, and we have a frag_list with a linear 3980 * non head_frag head. 3981 * 3982 * (we assume checking the first list_skb member suffices; 3983 * i.e if either of the list_skb members have non head_frag 3984 * head, then the first one has too). 3985 * 3986 * If head_skb's headlen does not fit requested gso_size, it 3987 * means that the frag_list members do NOT terminate on exact 3988 * gso_size boundaries. Hence we cannot perform skb_frag_t page 3989 * sharing. Therefore we must fallback to copying the frag_list 3990 * skbs; we do so by disabling SG. 3991 */ 3992 if (mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) 3993 features &= ~NETIF_F_SG; 3994 } 3995 3996 __skb_push(head_skb, doffset); 3997 proto = skb_network_protocol(head_skb, NULL); 3998 if (unlikely(!proto)) 3999 return ERR_PTR(-EINVAL); 4000 4001 sg = !!(features & NETIF_F_SG); 4002 csum = !!can_checksum_protocol(features, proto); 4003 4004 if (sg && csum && (mss != GSO_BY_FRAGS)) { 4005 if (!(features & NETIF_F_GSO_PARTIAL)) { 4006 struct sk_buff *iter; 4007 unsigned int frag_len; 4008 4009 if (!list_skb || 4010 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 4011 goto normal; 4012 4013 /* If we get here then all the required 4014 * GSO features except frag_list are supported. 4015 * Try to split the SKB to multiple GSO SKBs 4016 * with no frag_list. 4017 * Currently we can do that only when the buffers don't 4018 * have a linear part and all the buffers except 4019 * the last are of the same length. 4020 */ 4021 frag_len = list_skb->len; 4022 skb_walk_frags(head_skb, iter) { 4023 if (frag_len != iter->len && iter->next) 4024 goto normal; 4025 if (skb_headlen(iter) && !iter->head_frag) 4026 goto normal; 4027 4028 len -= iter->len; 4029 } 4030 4031 if (len != frag_len) 4032 goto normal; 4033 } 4034 4035 /* GSO partial only requires that we trim off any excess that 4036 * doesn't fit into an MSS sized block, so take care of that 4037 * now. 4038 */ 4039 partial_segs = len / mss; 4040 if (partial_segs > 1) 4041 mss *= partial_segs; 4042 else 4043 partial_segs = 0; 4044 } 4045 4046 normal: 4047 headroom = skb_headroom(head_skb); 4048 pos = skb_headlen(head_skb); 4049 4050 do { 4051 struct sk_buff *nskb; 4052 skb_frag_t *nskb_frag; 4053 int hsize; 4054 int size; 4055 4056 if (unlikely(mss == GSO_BY_FRAGS)) { 4057 len = list_skb->len; 4058 } else { 4059 len = head_skb->len - offset; 4060 if (len > mss) 4061 len = mss; 4062 } 4063 4064 hsize = skb_headlen(head_skb) - offset; 4065 4066 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) && 4067 (skb_headlen(list_skb) == len || sg)) { 4068 BUG_ON(skb_headlen(list_skb) > len); 4069 4070 i = 0; 4071 nfrags = skb_shinfo(list_skb)->nr_frags; 4072 frag = skb_shinfo(list_skb)->frags; 4073 frag_skb = list_skb; 4074 pos += skb_headlen(list_skb); 4075 4076 while (pos < offset + len) { 4077 BUG_ON(i >= nfrags); 4078 4079 size = skb_frag_size(frag); 4080 if (pos + size > offset + len) 4081 break; 4082 4083 i++; 4084 pos += size; 4085 frag++; 4086 } 4087 4088 nskb = skb_clone(list_skb, GFP_ATOMIC); 4089 list_skb = list_skb->next; 4090 4091 if (unlikely(!nskb)) 4092 goto err; 4093 4094 if (unlikely(pskb_trim(nskb, len))) { 4095 kfree_skb(nskb); 4096 goto err; 4097 } 4098 4099 hsize = skb_end_offset(nskb); 4100 if (skb_cow_head(nskb, doffset + headroom)) { 4101 kfree_skb(nskb); 4102 goto err; 4103 } 4104 4105 nskb->truesize += skb_end_offset(nskb) - hsize; 4106 skb_release_head_state(nskb); 4107 __skb_push(nskb, doffset); 4108 } else { 4109 if (hsize < 0) 4110 hsize = 0; 4111 if (hsize > len || !sg) 4112 hsize = len; 4113 4114 nskb = __alloc_skb(hsize + doffset + headroom, 4115 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 4116 NUMA_NO_NODE); 4117 4118 if (unlikely(!nskb)) 4119 goto err; 4120 4121 skb_reserve(nskb, headroom); 4122 __skb_put(nskb, doffset); 4123 } 4124 4125 if (segs) 4126 tail->next = nskb; 4127 else 4128 segs = nskb; 4129 tail = nskb; 4130 4131 __copy_skb_header(nskb, head_skb); 4132 4133 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 4134 skb_reset_mac_len(nskb); 4135 4136 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 4137 nskb->data - tnl_hlen, 4138 doffset + tnl_hlen); 4139 4140 if (nskb->len == len + doffset) 4141 goto perform_csum_check; 4142 4143 if (!sg) { 4144 if (!csum) { 4145 if (!nskb->remcsum_offload) 4146 nskb->ip_summed = CHECKSUM_NONE; 4147 SKB_GSO_CB(nskb)->csum = 4148 skb_copy_and_csum_bits(head_skb, offset, 4149 skb_put(nskb, 4150 len), 4151 len); 4152 SKB_GSO_CB(nskb)->csum_start = 4153 skb_headroom(nskb) + doffset; 4154 } else { 4155 skb_copy_bits(head_skb, offset, 4156 skb_put(nskb, len), 4157 len); 4158 } 4159 continue; 4160 } 4161 4162 nskb_frag = skb_shinfo(nskb)->frags; 4163 4164 skb_copy_from_linear_data_offset(head_skb, offset, 4165 skb_put(nskb, hsize), hsize); 4166 4167 skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags & 4168 SKBFL_SHARED_FRAG; 4169 4170 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4171 skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 4172 goto err; 4173 4174 while (pos < offset + len) { 4175 if (i >= nfrags) { 4176 i = 0; 4177 nfrags = skb_shinfo(list_skb)->nr_frags; 4178 frag = skb_shinfo(list_skb)->frags; 4179 frag_skb = list_skb; 4180 if (!skb_headlen(list_skb)) { 4181 BUG_ON(!nfrags); 4182 } else { 4183 BUG_ON(!list_skb->head_frag); 4184 4185 /* to make room for head_frag. */ 4186 i--; 4187 frag--; 4188 } 4189 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4190 skb_zerocopy_clone(nskb, frag_skb, 4191 GFP_ATOMIC)) 4192 goto err; 4193 4194 list_skb = list_skb->next; 4195 } 4196 4197 if (unlikely(skb_shinfo(nskb)->nr_frags >= 4198 MAX_SKB_FRAGS)) { 4199 net_warn_ratelimited( 4200 "skb_segment: too many frags: %u %u\n", 4201 pos, mss); 4202 err = -EINVAL; 4203 goto err; 4204 } 4205 4206 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; 4207 __skb_frag_ref(nskb_frag); 4208 size = skb_frag_size(nskb_frag); 4209 4210 if (pos < offset) { 4211 skb_frag_off_add(nskb_frag, offset - pos); 4212 skb_frag_size_sub(nskb_frag, offset - pos); 4213 } 4214 4215 skb_shinfo(nskb)->nr_frags++; 4216 4217 if (pos + size <= offset + len) { 4218 i++; 4219 frag++; 4220 pos += size; 4221 } else { 4222 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 4223 goto skip_fraglist; 4224 } 4225 4226 nskb_frag++; 4227 } 4228 4229 skip_fraglist: 4230 nskb->data_len = len - hsize; 4231 nskb->len += nskb->data_len; 4232 nskb->truesize += nskb->data_len; 4233 4234 perform_csum_check: 4235 if (!csum) { 4236 if (skb_has_shared_frag(nskb) && 4237 __skb_linearize(nskb)) 4238 goto err; 4239 4240 if (!nskb->remcsum_offload) 4241 nskb->ip_summed = CHECKSUM_NONE; 4242 SKB_GSO_CB(nskb)->csum = 4243 skb_checksum(nskb, doffset, 4244 nskb->len - doffset, 0); 4245 SKB_GSO_CB(nskb)->csum_start = 4246 skb_headroom(nskb) + doffset; 4247 } 4248 } while ((offset += len) < head_skb->len); 4249 4250 /* Some callers want to get the end of the list. 4251 * Put it in segs->prev to avoid walking the list. 4252 * (see validate_xmit_skb_list() for example) 4253 */ 4254 segs->prev = tail; 4255 4256 if (partial_segs) { 4257 struct sk_buff *iter; 4258 int type = skb_shinfo(head_skb)->gso_type; 4259 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 4260 4261 /* Update type to add partial and then remove dodgy if set */ 4262 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 4263 type &= ~SKB_GSO_DODGY; 4264 4265 /* Update GSO info and prepare to start updating headers on 4266 * our way back down the stack of protocols. 4267 */ 4268 for (iter = segs; iter; iter = iter->next) { 4269 skb_shinfo(iter)->gso_size = gso_size; 4270 skb_shinfo(iter)->gso_segs = partial_segs; 4271 skb_shinfo(iter)->gso_type = type; 4272 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 4273 } 4274 4275 if (tail->len - doffset <= gso_size) 4276 skb_shinfo(tail)->gso_size = 0; 4277 else if (tail != segs) 4278 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 4279 } 4280 4281 /* Following permits correct backpressure, for protocols 4282 * using skb_set_owner_w(). 4283 * Idea is to tranfert ownership from head_skb to last segment. 4284 */ 4285 if (head_skb->destructor == sock_wfree) { 4286 swap(tail->truesize, head_skb->truesize); 4287 swap(tail->destructor, head_skb->destructor); 4288 swap(tail->sk, head_skb->sk); 4289 } 4290 return segs; 4291 4292 err: 4293 kfree_skb_list(segs); 4294 return ERR_PTR(err); 4295 } 4296 EXPORT_SYMBOL_GPL(skb_segment); 4297 4298 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) 4299 { 4300 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); 4301 unsigned int offset = skb_gro_offset(skb); 4302 unsigned int headlen = skb_headlen(skb); 4303 unsigned int len = skb_gro_len(skb); 4304 unsigned int delta_truesize; 4305 unsigned int new_truesize; 4306 struct sk_buff *lp; 4307 4308 if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush)) 4309 return -E2BIG; 4310 4311 lp = NAPI_GRO_CB(p)->last; 4312 pinfo = skb_shinfo(lp); 4313 4314 if (headlen <= offset) { 4315 skb_frag_t *frag; 4316 skb_frag_t *frag2; 4317 int i = skbinfo->nr_frags; 4318 int nr_frags = pinfo->nr_frags + i; 4319 4320 if (nr_frags > MAX_SKB_FRAGS) 4321 goto merge; 4322 4323 offset -= headlen; 4324 pinfo->nr_frags = nr_frags; 4325 skbinfo->nr_frags = 0; 4326 4327 frag = pinfo->frags + nr_frags; 4328 frag2 = skbinfo->frags + i; 4329 do { 4330 *--frag = *--frag2; 4331 } while (--i); 4332 4333 skb_frag_off_add(frag, offset); 4334 skb_frag_size_sub(frag, offset); 4335 4336 /* all fragments truesize : remove (head size + sk_buff) */ 4337 new_truesize = SKB_TRUESIZE(skb_end_offset(skb)); 4338 delta_truesize = skb->truesize - new_truesize; 4339 4340 skb->truesize = new_truesize; 4341 skb->len -= skb->data_len; 4342 skb->data_len = 0; 4343 4344 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; 4345 goto done; 4346 } else if (skb->head_frag) { 4347 int nr_frags = pinfo->nr_frags; 4348 skb_frag_t *frag = pinfo->frags + nr_frags; 4349 struct page *page = virt_to_head_page(skb->head); 4350 unsigned int first_size = headlen - offset; 4351 unsigned int first_offset; 4352 4353 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) 4354 goto merge; 4355 4356 first_offset = skb->data - 4357 (unsigned char *)page_address(page) + 4358 offset; 4359 4360 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; 4361 4362 __skb_frag_set_page(frag, page); 4363 skb_frag_off_set(frag, first_offset); 4364 skb_frag_size_set(frag, first_size); 4365 4366 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); 4367 /* We dont need to clear skbinfo->nr_frags here */ 4368 4369 new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff)); 4370 delta_truesize = skb->truesize - new_truesize; 4371 skb->truesize = new_truesize; 4372 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; 4373 goto done; 4374 } 4375 4376 merge: 4377 /* sk owenrship - if any - completely transferred to the aggregated packet */ 4378 skb->destructor = NULL; 4379 delta_truesize = skb->truesize; 4380 if (offset > headlen) { 4381 unsigned int eat = offset - headlen; 4382 4383 skb_frag_off_add(&skbinfo->frags[0], eat); 4384 skb_frag_size_sub(&skbinfo->frags[0], eat); 4385 skb->data_len -= eat; 4386 skb->len -= eat; 4387 offset = headlen; 4388 } 4389 4390 __skb_pull(skb, offset); 4391 4392 if (NAPI_GRO_CB(p)->last == p) 4393 skb_shinfo(p)->frag_list = skb; 4394 else 4395 NAPI_GRO_CB(p)->last->next = skb; 4396 NAPI_GRO_CB(p)->last = skb; 4397 __skb_header_release(skb); 4398 lp = p; 4399 4400 done: 4401 NAPI_GRO_CB(p)->count++; 4402 p->data_len += len; 4403 p->truesize += delta_truesize; 4404 p->len += len; 4405 if (lp != p) { 4406 lp->data_len += len; 4407 lp->truesize += delta_truesize; 4408 lp->len += len; 4409 } 4410 NAPI_GRO_CB(skb)->same_flow = 1; 4411 return 0; 4412 } 4413 4414 #ifdef CONFIG_SKB_EXTENSIONS 4415 #define SKB_EXT_ALIGN_VALUE 8 4416 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) 4417 4418 static const u8 skb_ext_type_len[] = { 4419 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4420 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info), 4421 #endif 4422 #ifdef CONFIG_XFRM 4423 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path), 4424 #endif 4425 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4426 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext), 4427 #endif 4428 #if IS_ENABLED(CONFIG_MPTCP) 4429 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext), 4430 #endif 4431 }; 4432 4433 static __always_inline unsigned int skb_ext_total_length(void) 4434 { 4435 return SKB_EXT_CHUNKSIZEOF(struct skb_ext) + 4436 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4437 skb_ext_type_len[SKB_EXT_BRIDGE_NF] + 4438 #endif 4439 #ifdef CONFIG_XFRM 4440 skb_ext_type_len[SKB_EXT_SEC_PATH] + 4441 #endif 4442 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4443 skb_ext_type_len[TC_SKB_EXT] + 4444 #endif 4445 #if IS_ENABLED(CONFIG_MPTCP) 4446 skb_ext_type_len[SKB_EXT_MPTCP] + 4447 #endif 4448 0; 4449 } 4450 4451 static void skb_extensions_init(void) 4452 { 4453 BUILD_BUG_ON(SKB_EXT_NUM >= 8); 4454 BUILD_BUG_ON(skb_ext_total_length() > 255); 4455 4456 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", 4457 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), 4458 0, 4459 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4460 NULL); 4461 } 4462 #else 4463 static void skb_extensions_init(void) {} 4464 #endif 4465 4466 void __init skb_init(void) 4467 { 4468 skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache", 4469 sizeof(struct sk_buff), 4470 0, 4471 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4472 offsetof(struct sk_buff, cb), 4473 sizeof_field(struct sk_buff, cb), 4474 NULL); 4475 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 4476 sizeof(struct sk_buff_fclones), 4477 0, 4478 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4479 NULL); 4480 skb_extensions_init(); 4481 } 4482 4483 static int 4484 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 4485 unsigned int recursion_level) 4486 { 4487 int start = skb_headlen(skb); 4488 int i, copy = start - offset; 4489 struct sk_buff *frag_iter; 4490 int elt = 0; 4491 4492 if (unlikely(recursion_level >= 24)) 4493 return -EMSGSIZE; 4494 4495 if (copy > 0) { 4496 if (copy > len) 4497 copy = len; 4498 sg_set_buf(sg, skb->data + offset, copy); 4499 elt++; 4500 if ((len -= copy) == 0) 4501 return elt; 4502 offset += copy; 4503 } 4504 4505 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4506 int end; 4507 4508 WARN_ON(start > offset + len); 4509 4510 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 4511 if ((copy = end - offset) > 0) { 4512 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4513 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4514 return -EMSGSIZE; 4515 4516 if (copy > len) 4517 copy = len; 4518 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 4519 skb_frag_off(frag) + offset - start); 4520 elt++; 4521 if (!(len -= copy)) 4522 return elt; 4523 offset += copy; 4524 } 4525 start = end; 4526 } 4527 4528 skb_walk_frags(skb, frag_iter) { 4529 int end, ret; 4530 4531 WARN_ON(start > offset + len); 4532 4533 end = start + frag_iter->len; 4534 if ((copy = end - offset) > 0) { 4535 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4536 return -EMSGSIZE; 4537 4538 if (copy > len) 4539 copy = len; 4540 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 4541 copy, recursion_level + 1); 4542 if (unlikely(ret < 0)) 4543 return ret; 4544 elt += ret; 4545 if ((len -= copy) == 0) 4546 return elt; 4547 offset += copy; 4548 } 4549 start = end; 4550 } 4551 BUG_ON(len); 4552 return elt; 4553 } 4554 4555 /** 4556 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 4557 * @skb: Socket buffer containing the buffers to be mapped 4558 * @sg: The scatter-gather list to map into 4559 * @offset: The offset into the buffer's contents to start mapping 4560 * @len: Length of buffer space to be mapped 4561 * 4562 * Fill the specified scatter-gather list with mappings/pointers into a 4563 * region of the buffer space attached to a socket buffer. Returns either 4564 * the number of scatterlist items used, or -EMSGSIZE if the contents 4565 * could not fit. 4566 */ 4567 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 4568 { 4569 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 4570 4571 if (nsg <= 0) 4572 return nsg; 4573 4574 sg_mark_end(&sg[nsg - 1]); 4575 4576 return nsg; 4577 } 4578 EXPORT_SYMBOL_GPL(skb_to_sgvec); 4579 4580 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 4581 * sglist without mark the sg which contain last skb data as the end. 4582 * So the caller can mannipulate sg list as will when padding new data after 4583 * the first call without calling sg_unmark_end to expend sg list. 4584 * 4585 * Scenario to use skb_to_sgvec_nomark: 4586 * 1. sg_init_table 4587 * 2. skb_to_sgvec_nomark(payload1) 4588 * 3. skb_to_sgvec_nomark(payload2) 4589 * 4590 * This is equivalent to: 4591 * 1. sg_init_table 4592 * 2. skb_to_sgvec(payload1) 4593 * 3. sg_unmark_end 4594 * 4. skb_to_sgvec(payload2) 4595 * 4596 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 4597 * is more preferable. 4598 */ 4599 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 4600 int offset, int len) 4601 { 4602 return __skb_to_sgvec(skb, sg, offset, len, 0); 4603 } 4604 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 4605 4606 4607 4608 /** 4609 * skb_cow_data - Check that a socket buffer's data buffers are writable 4610 * @skb: The socket buffer to check. 4611 * @tailbits: Amount of trailing space to be added 4612 * @trailer: Returned pointer to the skb where the @tailbits space begins 4613 * 4614 * Make sure that the data buffers attached to a socket buffer are 4615 * writable. If they are not, private copies are made of the data buffers 4616 * and the socket buffer is set to use these instead. 4617 * 4618 * If @tailbits is given, make sure that there is space to write @tailbits 4619 * bytes of data beyond current end of socket buffer. @trailer will be 4620 * set to point to the skb in which this space begins. 4621 * 4622 * The number of scatterlist elements required to completely map the 4623 * COW'd and extended socket buffer will be returned. 4624 */ 4625 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 4626 { 4627 int copyflag; 4628 int elt; 4629 struct sk_buff *skb1, **skb_p; 4630 4631 /* If skb is cloned or its head is paged, reallocate 4632 * head pulling out all the pages (pages are considered not writable 4633 * at the moment even if they are anonymous). 4634 */ 4635 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 4636 !__pskb_pull_tail(skb, __skb_pagelen(skb))) 4637 return -ENOMEM; 4638 4639 /* Easy case. Most of packets will go this way. */ 4640 if (!skb_has_frag_list(skb)) { 4641 /* A little of trouble, not enough of space for trailer. 4642 * This should not happen, when stack is tuned to generate 4643 * good frames. OK, on miss we reallocate and reserve even more 4644 * space, 128 bytes is fair. */ 4645 4646 if (skb_tailroom(skb) < tailbits && 4647 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 4648 return -ENOMEM; 4649 4650 /* Voila! */ 4651 *trailer = skb; 4652 return 1; 4653 } 4654 4655 /* Misery. We are in troubles, going to mincer fragments... */ 4656 4657 elt = 1; 4658 skb_p = &skb_shinfo(skb)->frag_list; 4659 copyflag = 0; 4660 4661 while ((skb1 = *skb_p) != NULL) { 4662 int ntail = 0; 4663 4664 /* The fragment is partially pulled by someone, 4665 * this can happen on input. Copy it and everything 4666 * after it. */ 4667 4668 if (skb_shared(skb1)) 4669 copyflag = 1; 4670 4671 /* If the skb is the last, worry about trailer. */ 4672 4673 if (skb1->next == NULL && tailbits) { 4674 if (skb_shinfo(skb1)->nr_frags || 4675 skb_has_frag_list(skb1) || 4676 skb_tailroom(skb1) < tailbits) 4677 ntail = tailbits + 128; 4678 } 4679 4680 if (copyflag || 4681 skb_cloned(skb1) || 4682 ntail || 4683 skb_shinfo(skb1)->nr_frags || 4684 skb_has_frag_list(skb1)) { 4685 struct sk_buff *skb2; 4686 4687 /* Fuck, we are miserable poor guys... */ 4688 if (ntail == 0) 4689 skb2 = skb_copy(skb1, GFP_ATOMIC); 4690 else 4691 skb2 = skb_copy_expand(skb1, 4692 skb_headroom(skb1), 4693 ntail, 4694 GFP_ATOMIC); 4695 if (unlikely(skb2 == NULL)) 4696 return -ENOMEM; 4697 4698 if (skb1->sk) 4699 skb_set_owner_w(skb2, skb1->sk); 4700 4701 /* Looking around. Are we still alive? 4702 * OK, link new skb, drop old one */ 4703 4704 skb2->next = skb1->next; 4705 *skb_p = skb2; 4706 kfree_skb(skb1); 4707 skb1 = skb2; 4708 } 4709 elt++; 4710 *trailer = skb1; 4711 skb_p = &skb1->next; 4712 } 4713 4714 return elt; 4715 } 4716 EXPORT_SYMBOL_GPL(skb_cow_data); 4717 4718 static void sock_rmem_free(struct sk_buff *skb) 4719 { 4720 struct sock *sk = skb->sk; 4721 4722 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 4723 } 4724 4725 static void skb_set_err_queue(struct sk_buff *skb) 4726 { 4727 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING. 4728 * So, it is safe to (mis)use it to mark skbs on the error queue. 4729 */ 4730 skb->pkt_type = PACKET_OUTGOING; 4731 BUILD_BUG_ON(PACKET_OUTGOING == 0); 4732 } 4733 4734 /* 4735 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 4736 */ 4737 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 4738 { 4739 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 4740 (unsigned int)READ_ONCE(sk->sk_rcvbuf)) 4741 return -ENOMEM; 4742 4743 skb_orphan(skb); 4744 skb->sk = sk; 4745 skb->destructor = sock_rmem_free; 4746 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 4747 skb_set_err_queue(skb); 4748 4749 /* before exiting rcu section, make sure dst is refcounted */ 4750 skb_dst_force(skb); 4751 4752 skb_queue_tail(&sk->sk_error_queue, skb); 4753 if (!sock_flag(sk, SOCK_DEAD)) 4754 sk_error_report(sk); 4755 return 0; 4756 } 4757 EXPORT_SYMBOL(sock_queue_err_skb); 4758 4759 static bool is_icmp_err_skb(const struct sk_buff *skb) 4760 { 4761 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 4762 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 4763 } 4764 4765 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 4766 { 4767 struct sk_buff_head *q = &sk->sk_error_queue; 4768 struct sk_buff *skb, *skb_next = NULL; 4769 bool icmp_next = false; 4770 unsigned long flags; 4771 4772 spin_lock_irqsave(&q->lock, flags); 4773 skb = __skb_dequeue(q); 4774 if (skb && (skb_next = skb_peek(q))) { 4775 icmp_next = is_icmp_err_skb(skb_next); 4776 if (icmp_next) 4777 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 4778 } 4779 spin_unlock_irqrestore(&q->lock, flags); 4780 4781 if (is_icmp_err_skb(skb) && !icmp_next) 4782 sk->sk_err = 0; 4783 4784 if (skb_next) 4785 sk_error_report(sk); 4786 4787 return skb; 4788 } 4789 EXPORT_SYMBOL(sock_dequeue_err_skb); 4790 4791 /** 4792 * skb_clone_sk - create clone of skb, and take reference to socket 4793 * @skb: the skb to clone 4794 * 4795 * This function creates a clone of a buffer that holds a reference on 4796 * sk_refcnt. Buffers created via this function are meant to be 4797 * returned using sock_queue_err_skb, or free via kfree_skb. 4798 * 4799 * When passing buffers allocated with this function to sock_queue_err_skb 4800 * it is necessary to wrap the call with sock_hold/sock_put in order to 4801 * prevent the socket from being released prior to being enqueued on 4802 * the sk_error_queue. 4803 */ 4804 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 4805 { 4806 struct sock *sk = skb->sk; 4807 struct sk_buff *clone; 4808 4809 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 4810 return NULL; 4811 4812 clone = skb_clone(skb, GFP_ATOMIC); 4813 if (!clone) { 4814 sock_put(sk); 4815 return NULL; 4816 } 4817 4818 clone->sk = sk; 4819 clone->destructor = sock_efree; 4820 4821 return clone; 4822 } 4823 EXPORT_SYMBOL(skb_clone_sk); 4824 4825 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 4826 struct sock *sk, 4827 int tstype, 4828 bool opt_stats) 4829 { 4830 struct sock_exterr_skb *serr; 4831 int err; 4832 4833 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 4834 4835 serr = SKB_EXT_ERR(skb); 4836 memset(serr, 0, sizeof(*serr)); 4837 serr->ee.ee_errno = ENOMSG; 4838 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 4839 serr->ee.ee_info = tstype; 4840 serr->opt_stats = opt_stats; 4841 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 4842 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { 4843 serr->ee.ee_data = skb_shinfo(skb)->tskey; 4844 if (sk->sk_protocol == IPPROTO_TCP && 4845 sk->sk_type == SOCK_STREAM) 4846 serr->ee.ee_data -= sk->sk_tskey; 4847 } 4848 4849 err = sock_queue_err_skb(sk, skb); 4850 4851 if (err) 4852 kfree_skb(skb); 4853 } 4854 4855 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 4856 { 4857 bool ret; 4858 4859 if (likely(sysctl_tstamp_allow_data || tsonly)) 4860 return true; 4861 4862 read_lock_bh(&sk->sk_callback_lock); 4863 ret = sk->sk_socket && sk->sk_socket->file && 4864 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); 4865 read_unlock_bh(&sk->sk_callback_lock); 4866 return ret; 4867 } 4868 4869 void skb_complete_tx_timestamp(struct sk_buff *skb, 4870 struct skb_shared_hwtstamps *hwtstamps) 4871 { 4872 struct sock *sk = skb->sk; 4873 4874 if (!skb_may_tx_timestamp(sk, false)) 4875 goto err; 4876 4877 /* Take a reference to prevent skb_orphan() from freeing the socket, 4878 * but only if the socket refcount is not zero. 4879 */ 4880 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 4881 *skb_hwtstamps(skb) = *hwtstamps; 4882 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 4883 sock_put(sk); 4884 return; 4885 } 4886 4887 err: 4888 kfree_skb(skb); 4889 } 4890 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 4891 4892 void __skb_tstamp_tx(struct sk_buff *orig_skb, 4893 const struct sk_buff *ack_skb, 4894 struct skb_shared_hwtstamps *hwtstamps, 4895 struct sock *sk, int tstype) 4896 { 4897 struct sk_buff *skb; 4898 bool tsonly, opt_stats = false; 4899 4900 if (!sk) 4901 return; 4902 4903 if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 4904 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 4905 return; 4906 4907 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 4908 if (!skb_may_tx_timestamp(sk, tsonly)) 4909 return; 4910 4911 if (tsonly) { 4912 #ifdef CONFIG_INET 4913 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && 4914 sk->sk_protocol == IPPROTO_TCP && 4915 sk->sk_type == SOCK_STREAM) { 4916 skb = tcp_get_timestamping_opt_stats(sk, orig_skb, 4917 ack_skb); 4918 opt_stats = true; 4919 } else 4920 #endif 4921 skb = alloc_skb(0, GFP_ATOMIC); 4922 } else { 4923 skb = skb_clone(orig_skb, GFP_ATOMIC); 4924 } 4925 if (!skb) 4926 return; 4927 4928 if (tsonly) { 4929 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 4930 SKBTX_ANY_TSTAMP; 4931 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 4932 } 4933 4934 if (hwtstamps) 4935 *skb_hwtstamps(skb) = *hwtstamps; 4936 else 4937 skb->tstamp = ktime_get_real(); 4938 4939 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 4940 } 4941 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 4942 4943 void skb_tstamp_tx(struct sk_buff *orig_skb, 4944 struct skb_shared_hwtstamps *hwtstamps) 4945 { 4946 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk, 4947 SCM_TSTAMP_SND); 4948 } 4949 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 4950 4951 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 4952 { 4953 struct sock *sk = skb->sk; 4954 struct sock_exterr_skb *serr; 4955 int err = 1; 4956 4957 skb->wifi_acked_valid = 1; 4958 skb->wifi_acked = acked; 4959 4960 serr = SKB_EXT_ERR(skb); 4961 memset(serr, 0, sizeof(*serr)); 4962 serr->ee.ee_errno = ENOMSG; 4963 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 4964 4965 /* Take a reference to prevent skb_orphan() from freeing the socket, 4966 * but only if the socket refcount is not zero. 4967 */ 4968 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 4969 err = sock_queue_err_skb(sk, skb); 4970 sock_put(sk); 4971 } 4972 if (err) 4973 kfree_skb(skb); 4974 } 4975 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 4976 4977 /** 4978 * skb_partial_csum_set - set up and verify partial csum values for packet 4979 * @skb: the skb to set 4980 * @start: the number of bytes after skb->data to start checksumming. 4981 * @off: the offset from start to place the checksum. 4982 * 4983 * For untrusted partially-checksummed packets, we need to make sure the values 4984 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 4985 * 4986 * This function checks and sets those values and skb->ip_summed: if this 4987 * returns false you should drop the packet. 4988 */ 4989 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 4990 { 4991 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16); 4992 u32 csum_start = skb_headroom(skb) + (u32)start; 4993 4994 if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) { 4995 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n", 4996 start, off, skb_headroom(skb), skb_headlen(skb)); 4997 return false; 4998 } 4999 skb->ip_summed = CHECKSUM_PARTIAL; 5000 skb->csum_start = csum_start; 5001 skb->csum_offset = off; 5002 skb_set_transport_header(skb, start); 5003 return true; 5004 } 5005 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 5006 5007 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 5008 unsigned int max) 5009 { 5010 if (skb_headlen(skb) >= len) 5011 return 0; 5012 5013 /* If we need to pullup then pullup to the max, so we 5014 * won't need to do it again. 5015 */ 5016 if (max > skb->len) 5017 max = skb->len; 5018 5019 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 5020 return -ENOMEM; 5021 5022 if (skb_headlen(skb) < len) 5023 return -EPROTO; 5024 5025 return 0; 5026 } 5027 5028 #define MAX_TCP_HDR_LEN (15 * 4) 5029 5030 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 5031 typeof(IPPROTO_IP) proto, 5032 unsigned int off) 5033 { 5034 int err; 5035 5036 switch (proto) { 5037 case IPPROTO_TCP: 5038 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 5039 off + MAX_TCP_HDR_LEN); 5040 if (!err && !skb_partial_csum_set(skb, off, 5041 offsetof(struct tcphdr, 5042 check))) 5043 err = -EPROTO; 5044 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 5045 5046 case IPPROTO_UDP: 5047 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 5048 off + sizeof(struct udphdr)); 5049 if (!err && !skb_partial_csum_set(skb, off, 5050 offsetof(struct udphdr, 5051 check))) 5052 err = -EPROTO; 5053 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 5054 } 5055 5056 return ERR_PTR(-EPROTO); 5057 } 5058 5059 /* This value should be large enough to cover a tagged ethernet header plus 5060 * maximally sized IP and TCP or UDP headers. 5061 */ 5062 #define MAX_IP_HDR_LEN 128 5063 5064 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 5065 { 5066 unsigned int off; 5067 bool fragment; 5068 __sum16 *csum; 5069 int err; 5070 5071 fragment = false; 5072 5073 err = skb_maybe_pull_tail(skb, 5074 sizeof(struct iphdr), 5075 MAX_IP_HDR_LEN); 5076 if (err < 0) 5077 goto out; 5078 5079 if (ip_is_fragment(ip_hdr(skb))) 5080 fragment = true; 5081 5082 off = ip_hdrlen(skb); 5083 5084 err = -EPROTO; 5085 5086 if (fragment) 5087 goto out; 5088 5089 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 5090 if (IS_ERR(csum)) 5091 return PTR_ERR(csum); 5092 5093 if (recalculate) 5094 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 5095 ip_hdr(skb)->daddr, 5096 skb->len - off, 5097 ip_hdr(skb)->protocol, 0); 5098 err = 0; 5099 5100 out: 5101 return err; 5102 } 5103 5104 /* This value should be large enough to cover a tagged ethernet header plus 5105 * an IPv6 header, all options, and a maximal TCP or UDP header. 5106 */ 5107 #define MAX_IPV6_HDR_LEN 256 5108 5109 #define OPT_HDR(type, skb, off) \ 5110 (type *)(skb_network_header(skb) + (off)) 5111 5112 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 5113 { 5114 int err; 5115 u8 nexthdr; 5116 unsigned int off; 5117 unsigned int len; 5118 bool fragment; 5119 bool done; 5120 __sum16 *csum; 5121 5122 fragment = false; 5123 done = false; 5124 5125 off = sizeof(struct ipv6hdr); 5126 5127 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 5128 if (err < 0) 5129 goto out; 5130 5131 nexthdr = ipv6_hdr(skb)->nexthdr; 5132 5133 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 5134 while (off <= len && !done) { 5135 switch (nexthdr) { 5136 case IPPROTO_DSTOPTS: 5137 case IPPROTO_HOPOPTS: 5138 case IPPROTO_ROUTING: { 5139 struct ipv6_opt_hdr *hp; 5140 5141 err = skb_maybe_pull_tail(skb, 5142 off + 5143 sizeof(struct ipv6_opt_hdr), 5144 MAX_IPV6_HDR_LEN); 5145 if (err < 0) 5146 goto out; 5147 5148 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 5149 nexthdr = hp->nexthdr; 5150 off += ipv6_optlen(hp); 5151 break; 5152 } 5153 case IPPROTO_AH: { 5154 struct ip_auth_hdr *hp; 5155 5156 err = skb_maybe_pull_tail(skb, 5157 off + 5158 sizeof(struct ip_auth_hdr), 5159 MAX_IPV6_HDR_LEN); 5160 if (err < 0) 5161 goto out; 5162 5163 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 5164 nexthdr = hp->nexthdr; 5165 off += ipv6_authlen(hp); 5166 break; 5167 } 5168 case IPPROTO_FRAGMENT: { 5169 struct frag_hdr *hp; 5170 5171 err = skb_maybe_pull_tail(skb, 5172 off + 5173 sizeof(struct frag_hdr), 5174 MAX_IPV6_HDR_LEN); 5175 if (err < 0) 5176 goto out; 5177 5178 hp = OPT_HDR(struct frag_hdr, skb, off); 5179 5180 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 5181 fragment = true; 5182 5183 nexthdr = hp->nexthdr; 5184 off += sizeof(struct frag_hdr); 5185 break; 5186 } 5187 default: 5188 done = true; 5189 break; 5190 } 5191 } 5192 5193 err = -EPROTO; 5194 5195 if (!done || fragment) 5196 goto out; 5197 5198 csum = skb_checksum_setup_ip(skb, nexthdr, off); 5199 if (IS_ERR(csum)) 5200 return PTR_ERR(csum); 5201 5202 if (recalculate) 5203 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 5204 &ipv6_hdr(skb)->daddr, 5205 skb->len - off, nexthdr, 0); 5206 err = 0; 5207 5208 out: 5209 return err; 5210 } 5211 5212 /** 5213 * skb_checksum_setup - set up partial checksum offset 5214 * @skb: the skb to set up 5215 * @recalculate: if true the pseudo-header checksum will be recalculated 5216 */ 5217 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 5218 { 5219 int err; 5220 5221 switch (skb->protocol) { 5222 case htons(ETH_P_IP): 5223 err = skb_checksum_setup_ipv4(skb, recalculate); 5224 break; 5225 5226 case htons(ETH_P_IPV6): 5227 err = skb_checksum_setup_ipv6(skb, recalculate); 5228 break; 5229 5230 default: 5231 err = -EPROTO; 5232 break; 5233 } 5234 5235 return err; 5236 } 5237 EXPORT_SYMBOL(skb_checksum_setup); 5238 5239 /** 5240 * skb_checksum_maybe_trim - maybe trims the given skb 5241 * @skb: the skb to check 5242 * @transport_len: the data length beyond the network header 5243 * 5244 * Checks whether the given skb has data beyond the given transport length. 5245 * If so, returns a cloned skb trimmed to this transport length. 5246 * Otherwise returns the provided skb. Returns NULL in error cases 5247 * (e.g. transport_len exceeds skb length or out-of-memory). 5248 * 5249 * Caller needs to set the skb transport header and free any returned skb if it 5250 * differs from the provided skb. 5251 */ 5252 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 5253 unsigned int transport_len) 5254 { 5255 struct sk_buff *skb_chk; 5256 unsigned int len = skb_transport_offset(skb) + transport_len; 5257 int ret; 5258 5259 if (skb->len < len) 5260 return NULL; 5261 else if (skb->len == len) 5262 return skb; 5263 5264 skb_chk = skb_clone(skb, GFP_ATOMIC); 5265 if (!skb_chk) 5266 return NULL; 5267 5268 ret = pskb_trim_rcsum(skb_chk, len); 5269 if (ret) { 5270 kfree_skb(skb_chk); 5271 return NULL; 5272 } 5273 5274 return skb_chk; 5275 } 5276 5277 /** 5278 * skb_checksum_trimmed - validate checksum of an skb 5279 * @skb: the skb to check 5280 * @transport_len: the data length beyond the network header 5281 * @skb_chkf: checksum function to use 5282 * 5283 * Applies the given checksum function skb_chkf to the provided skb. 5284 * Returns a checked and maybe trimmed skb. Returns NULL on error. 5285 * 5286 * If the skb has data beyond the given transport length, then a 5287 * trimmed & cloned skb is checked and returned. 5288 * 5289 * Caller needs to set the skb transport header and free any returned skb if it 5290 * differs from the provided skb. 5291 */ 5292 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 5293 unsigned int transport_len, 5294 __sum16(*skb_chkf)(struct sk_buff *skb)) 5295 { 5296 struct sk_buff *skb_chk; 5297 unsigned int offset = skb_transport_offset(skb); 5298 __sum16 ret; 5299 5300 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 5301 if (!skb_chk) 5302 goto err; 5303 5304 if (!pskb_may_pull(skb_chk, offset)) 5305 goto err; 5306 5307 skb_pull_rcsum(skb_chk, offset); 5308 ret = skb_chkf(skb_chk); 5309 skb_push_rcsum(skb_chk, offset); 5310 5311 if (ret) 5312 goto err; 5313 5314 return skb_chk; 5315 5316 err: 5317 if (skb_chk && skb_chk != skb) 5318 kfree_skb(skb_chk); 5319 5320 return NULL; 5321 5322 } 5323 EXPORT_SYMBOL(skb_checksum_trimmed); 5324 5325 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 5326 { 5327 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 5328 skb->dev->name); 5329 } 5330 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 5331 5332 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 5333 { 5334 if (head_stolen) { 5335 skb_release_head_state(skb); 5336 kmem_cache_free(skbuff_head_cache, skb); 5337 } else { 5338 __kfree_skb(skb); 5339 } 5340 } 5341 EXPORT_SYMBOL(kfree_skb_partial); 5342 5343 /** 5344 * skb_try_coalesce - try to merge skb to prior one 5345 * @to: prior buffer 5346 * @from: buffer to add 5347 * @fragstolen: pointer to boolean 5348 * @delta_truesize: how much more was allocated than was requested 5349 */ 5350 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 5351 bool *fragstolen, int *delta_truesize) 5352 { 5353 struct skb_shared_info *to_shinfo, *from_shinfo; 5354 int i, delta, len = from->len; 5355 5356 *fragstolen = false; 5357 5358 if (skb_cloned(to)) 5359 return false; 5360 5361 /* The page pool signature of struct page will eventually figure out 5362 * which pages can be recycled or not but for now let's prohibit slab 5363 * allocated and page_pool allocated SKBs from being coalesced. 5364 */ 5365 if (to->pp_recycle != from->pp_recycle) 5366 return false; 5367 5368 if (len <= skb_tailroom(to)) { 5369 if (len) 5370 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 5371 *delta_truesize = 0; 5372 return true; 5373 } 5374 5375 to_shinfo = skb_shinfo(to); 5376 from_shinfo = skb_shinfo(from); 5377 if (to_shinfo->frag_list || from_shinfo->frag_list) 5378 return false; 5379 if (skb_zcopy(to) || skb_zcopy(from)) 5380 return false; 5381 5382 if (skb_headlen(from) != 0) { 5383 struct page *page; 5384 unsigned int offset; 5385 5386 if (to_shinfo->nr_frags + 5387 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 5388 return false; 5389 5390 if (skb_head_is_locked(from)) 5391 return false; 5392 5393 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 5394 5395 page = virt_to_head_page(from->head); 5396 offset = from->data - (unsigned char *)page_address(page); 5397 5398 skb_fill_page_desc(to, to_shinfo->nr_frags, 5399 page, offset, skb_headlen(from)); 5400 *fragstolen = true; 5401 } else { 5402 if (to_shinfo->nr_frags + 5403 from_shinfo->nr_frags > MAX_SKB_FRAGS) 5404 return false; 5405 5406 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 5407 } 5408 5409 WARN_ON_ONCE(delta < len); 5410 5411 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 5412 from_shinfo->frags, 5413 from_shinfo->nr_frags * sizeof(skb_frag_t)); 5414 to_shinfo->nr_frags += from_shinfo->nr_frags; 5415 5416 if (!skb_cloned(from)) 5417 from_shinfo->nr_frags = 0; 5418 5419 /* if the skb is not cloned this does nothing 5420 * since we set nr_frags to 0. 5421 */ 5422 for (i = 0; i < from_shinfo->nr_frags; i++) 5423 __skb_frag_ref(&from_shinfo->frags[i]); 5424 5425 to->truesize += delta; 5426 to->len += len; 5427 to->data_len += len; 5428 5429 *delta_truesize = delta; 5430 return true; 5431 } 5432 EXPORT_SYMBOL(skb_try_coalesce); 5433 5434 /** 5435 * skb_scrub_packet - scrub an skb 5436 * 5437 * @skb: buffer to clean 5438 * @xnet: packet is crossing netns 5439 * 5440 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 5441 * into/from a tunnel. Some information have to be cleared during these 5442 * operations. 5443 * skb_scrub_packet can also be used to clean a skb before injecting it in 5444 * another namespace (@xnet == true). We have to clear all information in the 5445 * skb that could impact namespace isolation. 5446 */ 5447 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 5448 { 5449 skb->pkt_type = PACKET_HOST; 5450 skb->skb_iif = 0; 5451 skb->ignore_df = 0; 5452 skb_dst_drop(skb); 5453 skb_ext_reset(skb); 5454 nf_reset_ct(skb); 5455 nf_reset_trace(skb); 5456 5457 #ifdef CONFIG_NET_SWITCHDEV 5458 skb->offload_fwd_mark = 0; 5459 skb->offload_l3_fwd_mark = 0; 5460 #endif 5461 5462 if (!xnet) 5463 return; 5464 5465 ipvs_reset(skb); 5466 skb->mark = 0; 5467 skb->tstamp = 0; 5468 } 5469 EXPORT_SYMBOL_GPL(skb_scrub_packet); 5470 5471 /** 5472 * skb_gso_transport_seglen - Return length of individual segments of a gso packet 5473 * 5474 * @skb: GSO skb 5475 * 5476 * skb_gso_transport_seglen is used to determine the real size of the 5477 * individual segments, including Layer4 headers (TCP/UDP). 5478 * 5479 * The MAC/L2 or network (IP, IPv6) headers are not accounted for. 5480 */ 5481 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb) 5482 { 5483 const struct skb_shared_info *shinfo = skb_shinfo(skb); 5484 unsigned int thlen = 0; 5485 5486 if (skb->encapsulation) { 5487 thlen = skb_inner_transport_header(skb) - 5488 skb_transport_header(skb); 5489 5490 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 5491 thlen += inner_tcp_hdrlen(skb); 5492 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 5493 thlen = tcp_hdrlen(skb); 5494 } else if (unlikely(skb_is_gso_sctp(skb))) { 5495 thlen = sizeof(struct sctphdr); 5496 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { 5497 thlen = sizeof(struct udphdr); 5498 } 5499 /* UFO sets gso_size to the size of the fragmentation 5500 * payload, i.e. the size of the L4 (UDP) header is already 5501 * accounted for. 5502 */ 5503 return thlen + shinfo->gso_size; 5504 } 5505 5506 /** 5507 * skb_gso_network_seglen - Return length of individual segments of a gso packet 5508 * 5509 * @skb: GSO skb 5510 * 5511 * skb_gso_network_seglen is used to determine the real size of the 5512 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP). 5513 * 5514 * The MAC/L2 header is not accounted for. 5515 */ 5516 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb) 5517 { 5518 unsigned int hdr_len = skb_transport_header(skb) - 5519 skb_network_header(skb); 5520 5521 return hdr_len + skb_gso_transport_seglen(skb); 5522 } 5523 5524 /** 5525 * skb_gso_mac_seglen - Return length of individual segments of a gso packet 5526 * 5527 * @skb: GSO skb 5528 * 5529 * skb_gso_mac_seglen is used to determine the real size of the 5530 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4 5531 * headers (TCP/UDP). 5532 */ 5533 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb) 5534 { 5535 unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 5536 5537 return hdr_len + skb_gso_transport_seglen(skb); 5538 } 5539 5540 /** 5541 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS 5542 * 5543 * There are a couple of instances where we have a GSO skb, and we 5544 * want to determine what size it would be after it is segmented. 5545 * 5546 * We might want to check: 5547 * - L3+L4+payload size (e.g. IP forwarding) 5548 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver) 5549 * 5550 * This is a helper to do that correctly considering GSO_BY_FRAGS. 5551 * 5552 * @skb: GSO skb 5553 * 5554 * @seg_len: The segmented length (from skb_gso_*_seglen). In the 5555 * GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS]. 5556 * 5557 * @max_len: The maximum permissible length. 5558 * 5559 * Returns true if the segmented length <= max length. 5560 */ 5561 static inline bool skb_gso_size_check(const struct sk_buff *skb, 5562 unsigned int seg_len, 5563 unsigned int max_len) { 5564 const struct skb_shared_info *shinfo = skb_shinfo(skb); 5565 const struct sk_buff *iter; 5566 5567 if (shinfo->gso_size != GSO_BY_FRAGS) 5568 return seg_len <= max_len; 5569 5570 /* Undo this so we can re-use header sizes */ 5571 seg_len -= GSO_BY_FRAGS; 5572 5573 skb_walk_frags(skb, iter) { 5574 if (seg_len + skb_headlen(iter) > max_len) 5575 return false; 5576 } 5577 5578 return true; 5579 } 5580 5581 /** 5582 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU? 5583 * 5584 * @skb: GSO skb 5585 * @mtu: MTU to validate against 5586 * 5587 * skb_gso_validate_network_len validates if a given skb will fit a 5588 * wanted MTU once split. It considers L3 headers, L4 headers, and the 5589 * payload. 5590 */ 5591 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu) 5592 { 5593 return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu); 5594 } 5595 EXPORT_SYMBOL_GPL(skb_gso_validate_network_len); 5596 5597 /** 5598 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length? 5599 * 5600 * @skb: GSO skb 5601 * @len: length to validate against 5602 * 5603 * skb_gso_validate_mac_len validates if a given skb will fit a wanted 5604 * length once split, including L2, L3 and L4 headers and the payload. 5605 */ 5606 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len) 5607 { 5608 return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len); 5609 } 5610 EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len); 5611 5612 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 5613 { 5614 int mac_len, meta_len; 5615 void *meta; 5616 5617 if (skb_cow(skb, skb_headroom(skb)) < 0) { 5618 kfree_skb(skb); 5619 return NULL; 5620 } 5621 5622 mac_len = skb->data - skb_mac_header(skb); 5623 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) { 5624 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), 5625 mac_len - VLAN_HLEN - ETH_TLEN); 5626 } 5627 5628 meta_len = skb_metadata_len(skb); 5629 if (meta_len) { 5630 meta = skb_metadata_end(skb) - meta_len; 5631 memmove(meta + VLAN_HLEN, meta, meta_len); 5632 } 5633 5634 skb->mac_header += VLAN_HLEN; 5635 return skb; 5636 } 5637 5638 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 5639 { 5640 struct vlan_hdr *vhdr; 5641 u16 vlan_tci; 5642 5643 if (unlikely(skb_vlan_tag_present(skb))) { 5644 /* vlan_tci is already set-up so leave this for another time */ 5645 return skb; 5646 } 5647 5648 skb = skb_share_check(skb, GFP_ATOMIC); 5649 if (unlikely(!skb)) 5650 goto err_free; 5651 /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */ 5652 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short)))) 5653 goto err_free; 5654 5655 vhdr = (struct vlan_hdr *)skb->data; 5656 vlan_tci = ntohs(vhdr->h_vlan_TCI); 5657 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 5658 5659 skb_pull_rcsum(skb, VLAN_HLEN); 5660 vlan_set_encap_proto(skb, vhdr); 5661 5662 skb = skb_reorder_vlan_header(skb); 5663 if (unlikely(!skb)) 5664 goto err_free; 5665 5666 skb_reset_network_header(skb); 5667 if (!skb_transport_header_was_set(skb)) 5668 skb_reset_transport_header(skb); 5669 skb_reset_mac_len(skb); 5670 5671 return skb; 5672 5673 err_free: 5674 kfree_skb(skb); 5675 return NULL; 5676 } 5677 EXPORT_SYMBOL(skb_vlan_untag); 5678 5679 int skb_ensure_writable(struct sk_buff *skb, int write_len) 5680 { 5681 if (!pskb_may_pull(skb, write_len)) 5682 return -ENOMEM; 5683 5684 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 5685 return 0; 5686 5687 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 5688 } 5689 EXPORT_SYMBOL(skb_ensure_writable); 5690 5691 /* remove VLAN header from packet and update csum accordingly. 5692 * expects a non skb_vlan_tag_present skb with a vlan tag payload 5693 */ 5694 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 5695 { 5696 struct vlan_hdr *vhdr; 5697 int offset = skb->data - skb_mac_header(skb); 5698 int err; 5699 5700 if (WARN_ONCE(offset, 5701 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 5702 offset)) { 5703 return -EINVAL; 5704 } 5705 5706 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 5707 if (unlikely(err)) 5708 return err; 5709 5710 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5711 5712 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN); 5713 *vlan_tci = ntohs(vhdr->h_vlan_TCI); 5714 5715 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN); 5716 __skb_pull(skb, VLAN_HLEN); 5717 5718 vlan_set_encap_proto(skb, vhdr); 5719 skb->mac_header += VLAN_HLEN; 5720 5721 if (skb_network_offset(skb) < ETH_HLEN) 5722 skb_set_network_header(skb, ETH_HLEN); 5723 5724 skb_reset_mac_len(skb); 5725 5726 return err; 5727 } 5728 EXPORT_SYMBOL(__skb_vlan_pop); 5729 5730 /* Pop a vlan tag either from hwaccel or from payload. 5731 * Expects skb->data at mac header. 5732 */ 5733 int skb_vlan_pop(struct sk_buff *skb) 5734 { 5735 u16 vlan_tci; 5736 __be16 vlan_proto; 5737 int err; 5738 5739 if (likely(skb_vlan_tag_present(skb))) { 5740 __vlan_hwaccel_clear_tag(skb); 5741 } else { 5742 if (unlikely(!eth_type_vlan(skb->protocol))) 5743 return 0; 5744 5745 err = __skb_vlan_pop(skb, &vlan_tci); 5746 if (err) 5747 return err; 5748 } 5749 /* move next vlan tag to hw accel tag */ 5750 if (likely(!eth_type_vlan(skb->protocol))) 5751 return 0; 5752 5753 vlan_proto = skb->protocol; 5754 err = __skb_vlan_pop(skb, &vlan_tci); 5755 if (unlikely(err)) 5756 return err; 5757 5758 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5759 return 0; 5760 } 5761 EXPORT_SYMBOL(skb_vlan_pop); 5762 5763 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 5764 * Expects skb->data at mac header. 5765 */ 5766 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 5767 { 5768 if (skb_vlan_tag_present(skb)) { 5769 int offset = skb->data - skb_mac_header(skb); 5770 int err; 5771 5772 if (WARN_ONCE(offset, 5773 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 5774 offset)) { 5775 return -EINVAL; 5776 } 5777 5778 err = __vlan_insert_tag(skb, skb->vlan_proto, 5779 skb_vlan_tag_get(skb)); 5780 if (err) 5781 return err; 5782 5783 skb->protocol = skb->vlan_proto; 5784 skb->mac_len += VLAN_HLEN; 5785 5786 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5787 } 5788 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5789 return 0; 5790 } 5791 EXPORT_SYMBOL(skb_vlan_push); 5792 5793 /** 5794 * skb_eth_pop() - Drop the Ethernet header at the head of a packet 5795 * 5796 * @skb: Socket buffer to modify 5797 * 5798 * Drop the Ethernet header of @skb. 5799 * 5800 * Expects that skb->data points to the mac header and that no VLAN tags are 5801 * present. 5802 * 5803 * Returns 0 on success, -errno otherwise. 5804 */ 5805 int skb_eth_pop(struct sk_buff *skb) 5806 { 5807 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) || 5808 skb_network_offset(skb) < ETH_HLEN) 5809 return -EPROTO; 5810 5811 skb_pull_rcsum(skb, ETH_HLEN); 5812 skb_reset_mac_header(skb); 5813 skb_reset_mac_len(skb); 5814 5815 return 0; 5816 } 5817 EXPORT_SYMBOL(skb_eth_pop); 5818 5819 /** 5820 * skb_eth_push() - Add a new Ethernet header at the head of a packet 5821 * 5822 * @skb: Socket buffer to modify 5823 * @dst: Destination MAC address of the new header 5824 * @src: Source MAC address of the new header 5825 * 5826 * Prepend @skb with a new Ethernet header. 5827 * 5828 * Expects that skb->data points to the mac header, which must be empty. 5829 * 5830 * Returns 0 on success, -errno otherwise. 5831 */ 5832 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst, 5833 const unsigned char *src) 5834 { 5835 struct ethhdr *eth; 5836 int err; 5837 5838 if (skb_network_offset(skb) || skb_vlan_tag_present(skb)) 5839 return -EPROTO; 5840 5841 err = skb_cow_head(skb, sizeof(*eth)); 5842 if (err < 0) 5843 return err; 5844 5845 skb_push(skb, sizeof(*eth)); 5846 skb_reset_mac_header(skb); 5847 skb_reset_mac_len(skb); 5848 5849 eth = eth_hdr(skb); 5850 ether_addr_copy(eth->h_dest, dst); 5851 ether_addr_copy(eth->h_source, src); 5852 eth->h_proto = skb->protocol; 5853 5854 skb_postpush_rcsum(skb, eth, sizeof(*eth)); 5855 5856 return 0; 5857 } 5858 EXPORT_SYMBOL(skb_eth_push); 5859 5860 /* Update the ethertype of hdr and the skb csum value if required. */ 5861 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr, 5862 __be16 ethertype) 5863 { 5864 if (skb->ip_summed == CHECKSUM_COMPLETE) { 5865 __be16 diff[] = { ~hdr->h_proto, ethertype }; 5866 5867 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 5868 } 5869 5870 hdr->h_proto = ethertype; 5871 } 5872 5873 /** 5874 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of 5875 * the packet 5876 * 5877 * @skb: buffer 5878 * @mpls_lse: MPLS label stack entry to push 5879 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848) 5880 * @mac_len: length of the MAC header 5881 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is 5882 * ethernet 5883 * 5884 * Expects skb->data at mac header. 5885 * 5886 * Returns 0 on success, -errno otherwise. 5887 */ 5888 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto, 5889 int mac_len, bool ethernet) 5890 { 5891 struct mpls_shim_hdr *lse; 5892 int err; 5893 5894 if (unlikely(!eth_p_mpls(mpls_proto))) 5895 return -EINVAL; 5896 5897 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */ 5898 if (skb->encapsulation) 5899 return -EINVAL; 5900 5901 err = skb_cow_head(skb, MPLS_HLEN); 5902 if (unlikely(err)) 5903 return err; 5904 5905 if (!skb->inner_protocol) { 5906 skb_set_inner_network_header(skb, skb_network_offset(skb)); 5907 skb_set_inner_protocol(skb, skb->protocol); 5908 } 5909 5910 skb_push(skb, MPLS_HLEN); 5911 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb), 5912 mac_len); 5913 skb_reset_mac_header(skb); 5914 skb_set_network_header(skb, mac_len); 5915 skb_reset_mac_len(skb); 5916 5917 lse = mpls_hdr(skb); 5918 lse->label_stack_entry = mpls_lse; 5919 skb_postpush_rcsum(skb, lse, MPLS_HLEN); 5920 5921 if (ethernet && mac_len >= ETH_HLEN) 5922 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto); 5923 skb->protocol = mpls_proto; 5924 5925 return 0; 5926 } 5927 EXPORT_SYMBOL_GPL(skb_mpls_push); 5928 5929 /** 5930 * skb_mpls_pop() - pop the outermost MPLS header 5931 * 5932 * @skb: buffer 5933 * @next_proto: ethertype of header after popped MPLS header 5934 * @mac_len: length of the MAC header 5935 * @ethernet: flag to indicate if the packet is ethernet 5936 * 5937 * Expects skb->data at mac header. 5938 * 5939 * Returns 0 on success, -errno otherwise. 5940 */ 5941 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, 5942 bool ethernet) 5943 { 5944 int err; 5945 5946 if (unlikely(!eth_p_mpls(skb->protocol))) 5947 return 0; 5948 5949 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN); 5950 if (unlikely(err)) 5951 return err; 5952 5953 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN); 5954 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb), 5955 mac_len); 5956 5957 __skb_pull(skb, MPLS_HLEN); 5958 skb_reset_mac_header(skb); 5959 skb_set_network_header(skb, mac_len); 5960 5961 if (ethernet && mac_len >= ETH_HLEN) { 5962 struct ethhdr *hdr; 5963 5964 /* use mpls_hdr() to get ethertype to account for VLANs. */ 5965 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN); 5966 skb_mod_eth_type(skb, hdr, next_proto); 5967 } 5968 skb->protocol = next_proto; 5969 5970 return 0; 5971 } 5972 EXPORT_SYMBOL_GPL(skb_mpls_pop); 5973 5974 /** 5975 * skb_mpls_update_lse() - modify outermost MPLS header and update csum 5976 * 5977 * @skb: buffer 5978 * @mpls_lse: new MPLS label stack entry to update to 5979 * 5980 * Expects skb->data at mac header. 5981 * 5982 * Returns 0 on success, -errno otherwise. 5983 */ 5984 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse) 5985 { 5986 int err; 5987 5988 if (unlikely(!eth_p_mpls(skb->protocol))) 5989 return -EINVAL; 5990 5991 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN); 5992 if (unlikely(err)) 5993 return err; 5994 5995 if (skb->ip_summed == CHECKSUM_COMPLETE) { 5996 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse }; 5997 5998 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 5999 } 6000 6001 mpls_hdr(skb)->label_stack_entry = mpls_lse; 6002 6003 return 0; 6004 } 6005 EXPORT_SYMBOL_GPL(skb_mpls_update_lse); 6006 6007 /** 6008 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header 6009 * 6010 * @skb: buffer 6011 * 6012 * Expects skb->data at mac header. 6013 * 6014 * Returns 0 on success, -errno otherwise. 6015 */ 6016 int skb_mpls_dec_ttl(struct sk_buff *skb) 6017 { 6018 u32 lse; 6019 u8 ttl; 6020 6021 if (unlikely(!eth_p_mpls(skb->protocol))) 6022 return -EINVAL; 6023 6024 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN)) 6025 return -ENOMEM; 6026 6027 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry); 6028 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT; 6029 if (!--ttl) 6030 return -EINVAL; 6031 6032 lse &= ~MPLS_LS_TTL_MASK; 6033 lse |= ttl << MPLS_LS_TTL_SHIFT; 6034 6035 return skb_mpls_update_lse(skb, cpu_to_be32(lse)); 6036 } 6037 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); 6038 6039 /** 6040 * alloc_skb_with_frags - allocate skb with page frags 6041 * 6042 * @header_len: size of linear part 6043 * @data_len: needed length in frags 6044 * @max_page_order: max page order desired. 6045 * @errcode: pointer to error code if any 6046 * @gfp_mask: allocation mask 6047 * 6048 * This can be used to allocate a paged skb, given a maximal order for frags. 6049 */ 6050 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 6051 unsigned long data_len, 6052 int max_page_order, 6053 int *errcode, 6054 gfp_t gfp_mask) 6055 { 6056 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 6057 unsigned long chunk; 6058 struct sk_buff *skb; 6059 struct page *page; 6060 int i; 6061 6062 *errcode = -EMSGSIZE; 6063 /* Note this test could be relaxed, if we succeed to allocate 6064 * high order pages... 6065 */ 6066 if (npages > MAX_SKB_FRAGS) 6067 return NULL; 6068 6069 *errcode = -ENOBUFS; 6070 skb = alloc_skb(header_len, gfp_mask); 6071 if (!skb) 6072 return NULL; 6073 6074 skb->truesize += npages << PAGE_SHIFT; 6075 6076 for (i = 0; npages > 0; i++) { 6077 int order = max_page_order; 6078 6079 while (order) { 6080 if (npages >= 1 << order) { 6081 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 6082 __GFP_COMP | 6083 __GFP_NOWARN, 6084 order); 6085 if (page) 6086 goto fill_page; 6087 /* Do not retry other high order allocations */ 6088 order = 1; 6089 max_page_order = 0; 6090 } 6091 order--; 6092 } 6093 page = alloc_page(gfp_mask); 6094 if (!page) 6095 goto failure; 6096 fill_page: 6097 chunk = min_t(unsigned long, data_len, 6098 PAGE_SIZE << order); 6099 skb_fill_page_desc(skb, i, page, 0, chunk); 6100 data_len -= chunk; 6101 npages -= 1 << order; 6102 } 6103 return skb; 6104 6105 failure: 6106 kfree_skb(skb); 6107 return NULL; 6108 } 6109 EXPORT_SYMBOL(alloc_skb_with_frags); 6110 6111 /* carve out the first off bytes from skb when off < headlen */ 6112 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 6113 const int headlen, gfp_t gfp_mask) 6114 { 6115 int i; 6116 int size = skb_end_offset(skb); 6117 int new_hlen = headlen - off; 6118 u8 *data; 6119 6120 size = SKB_DATA_ALIGN(size); 6121 6122 if (skb_pfmemalloc(skb)) 6123 gfp_mask |= __GFP_MEMALLOC; 6124 data = kmalloc_reserve(size + 6125 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 6126 gfp_mask, NUMA_NO_NODE, NULL); 6127 if (!data) 6128 return -ENOMEM; 6129 6130 size = SKB_WITH_OVERHEAD(ksize(data)); 6131 6132 /* Copy real data, and all frags */ 6133 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 6134 skb->len -= off; 6135 6136 memcpy((struct skb_shared_info *)(data + size), 6137 skb_shinfo(skb), 6138 offsetof(struct skb_shared_info, 6139 frags[skb_shinfo(skb)->nr_frags])); 6140 if (skb_cloned(skb)) { 6141 /* drop the old head gracefully */ 6142 if (skb_orphan_frags(skb, gfp_mask)) { 6143 kfree(data); 6144 return -ENOMEM; 6145 } 6146 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 6147 skb_frag_ref(skb, i); 6148 if (skb_has_frag_list(skb)) 6149 skb_clone_fraglist(skb); 6150 skb_release_data(skb); 6151 } else { 6152 /* we can reuse existing recount- all we did was 6153 * relocate values 6154 */ 6155 skb_free_head(skb); 6156 } 6157 6158 skb->head = data; 6159 skb->data = data; 6160 skb->head_frag = 0; 6161 #ifdef NET_SKBUFF_DATA_USES_OFFSET 6162 skb->end = size; 6163 #else 6164 skb->end = skb->head + size; 6165 #endif 6166 skb_set_tail_pointer(skb, skb_headlen(skb)); 6167 skb_headers_offset_update(skb, 0); 6168 skb->cloned = 0; 6169 skb->hdr_len = 0; 6170 skb->nohdr = 0; 6171 atomic_set(&skb_shinfo(skb)->dataref, 1); 6172 6173 return 0; 6174 } 6175 6176 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 6177 6178 /* carve out the first eat bytes from skb's frag_list. May recurse into 6179 * pskb_carve() 6180 */ 6181 static int pskb_carve_frag_list(struct sk_buff *skb, 6182 struct skb_shared_info *shinfo, int eat, 6183 gfp_t gfp_mask) 6184 { 6185 struct sk_buff *list = shinfo->frag_list; 6186 struct sk_buff *clone = NULL; 6187 struct sk_buff *insp = NULL; 6188 6189 do { 6190 if (!list) { 6191 pr_err("Not enough bytes to eat. Want %d\n", eat); 6192 return -EFAULT; 6193 } 6194 if (list->len <= eat) { 6195 /* Eaten as whole. */ 6196 eat -= list->len; 6197 list = list->next; 6198 insp = list; 6199 } else { 6200 /* Eaten partially. */ 6201 if (skb_shared(list)) { 6202 clone = skb_clone(list, gfp_mask); 6203 if (!clone) 6204 return -ENOMEM; 6205 insp = list->next; 6206 list = clone; 6207 } else { 6208 /* This may be pulled without problems. */ 6209 insp = list; 6210 } 6211 if (pskb_carve(list, eat, gfp_mask) < 0) { 6212 kfree_skb(clone); 6213 return -ENOMEM; 6214 } 6215 break; 6216 } 6217 } while (eat); 6218 6219 /* Free pulled out fragments. */ 6220 while ((list = shinfo->frag_list) != insp) { 6221 shinfo->frag_list = list->next; 6222 kfree_skb(list); 6223 } 6224 /* And insert new clone at head. */ 6225 if (clone) { 6226 clone->next = list; 6227 shinfo->frag_list = clone; 6228 } 6229 return 0; 6230 } 6231 6232 /* carve off first len bytes from skb. Split line (off) is in the 6233 * non-linear part of skb 6234 */ 6235 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 6236 int pos, gfp_t gfp_mask) 6237 { 6238 int i, k = 0; 6239 int size = skb_end_offset(skb); 6240 u8 *data; 6241 const int nfrags = skb_shinfo(skb)->nr_frags; 6242 struct skb_shared_info *shinfo; 6243 6244 size = SKB_DATA_ALIGN(size); 6245 6246 if (skb_pfmemalloc(skb)) 6247 gfp_mask |= __GFP_MEMALLOC; 6248 data = kmalloc_reserve(size + 6249 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 6250 gfp_mask, NUMA_NO_NODE, NULL); 6251 if (!data) 6252 return -ENOMEM; 6253 6254 size = SKB_WITH_OVERHEAD(ksize(data)); 6255 6256 memcpy((struct skb_shared_info *)(data + size), 6257 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0])); 6258 if (skb_orphan_frags(skb, gfp_mask)) { 6259 kfree(data); 6260 return -ENOMEM; 6261 } 6262 shinfo = (struct skb_shared_info *)(data + size); 6263 for (i = 0; i < nfrags; i++) { 6264 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 6265 6266 if (pos + fsize > off) { 6267 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 6268 6269 if (pos < off) { 6270 /* Split frag. 6271 * We have two variants in this case: 6272 * 1. Move all the frag to the second 6273 * part, if it is possible. F.e. 6274 * this approach is mandatory for TUX, 6275 * where splitting is expensive. 6276 * 2. Split is accurately. We make this. 6277 */ 6278 skb_frag_off_add(&shinfo->frags[0], off - pos); 6279 skb_frag_size_sub(&shinfo->frags[0], off - pos); 6280 } 6281 skb_frag_ref(skb, i); 6282 k++; 6283 } 6284 pos += fsize; 6285 } 6286 shinfo->nr_frags = k; 6287 if (skb_has_frag_list(skb)) 6288 skb_clone_fraglist(skb); 6289 6290 /* split line is in frag list */ 6291 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) { 6292 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */ 6293 if (skb_has_frag_list(skb)) 6294 kfree_skb_list(skb_shinfo(skb)->frag_list); 6295 kfree(data); 6296 return -ENOMEM; 6297 } 6298 skb_release_data(skb); 6299 6300 skb->head = data; 6301 skb->head_frag = 0; 6302 skb->data = data; 6303 #ifdef NET_SKBUFF_DATA_USES_OFFSET 6304 skb->end = size; 6305 #else 6306 skb->end = skb->head + size; 6307 #endif 6308 skb_reset_tail_pointer(skb); 6309 skb_headers_offset_update(skb, 0); 6310 skb->cloned = 0; 6311 skb->hdr_len = 0; 6312 skb->nohdr = 0; 6313 skb->len -= off; 6314 skb->data_len = skb->len; 6315 atomic_set(&skb_shinfo(skb)->dataref, 1); 6316 return 0; 6317 } 6318 6319 /* remove len bytes from the beginning of the skb */ 6320 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 6321 { 6322 int headlen = skb_headlen(skb); 6323 6324 if (len < headlen) 6325 return pskb_carve_inside_header(skb, len, headlen, gfp); 6326 else 6327 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 6328 } 6329 6330 /* Extract to_copy bytes starting at off from skb, and return this in 6331 * a new skb 6332 */ 6333 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 6334 int to_copy, gfp_t gfp) 6335 { 6336 struct sk_buff *clone = skb_clone(skb, gfp); 6337 6338 if (!clone) 6339 return NULL; 6340 6341 if (pskb_carve(clone, off, gfp) < 0 || 6342 pskb_trim(clone, to_copy)) { 6343 kfree_skb(clone); 6344 return NULL; 6345 } 6346 return clone; 6347 } 6348 EXPORT_SYMBOL(pskb_extract); 6349 6350 /** 6351 * skb_condense - try to get rid of fragments/frag_list if possible 6352 * @skb: buffer 6353 * 6354 * Can be used to save memory before skb is added to a busy queue. 6355 * If packet has bytes in frags and enough tail room in skb->head, 6356 * pull all of them, so that we can free the frags right now and adjust 6357 * truesize. 6358 * Notes: 6359 * We do not reallocate skb->head thus can not fail. 6360 * Caller must re-evaluate skb->truesize if needed. 6361 */ 6362 void skb_condense(struct sk_buff *skb) 6363 { 6364 if (skb->data_len) { 6365 if (skb->data_len > skb->end - skb->tail || 6366 skb_cloned(skb)) 6367 return; 6368 6369 /* Nice, we can free page frag(s) right now */ 6370 __pskb_pull_tail(skb, skb->data_len); 6371 } 6372 /* At this point, skb->truesize might be over estimated, 6373 * because skb had a fragment, and fragments do not tell 6374 * their truesize. 6375 * When we pulled its content into skb->head, fragment 6376 * was freed, but __pskb_pull_tail() could not possibly 6377 * adjust skb->truesize, not knowing the frag truesize. 6378 */ 6379 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 6380 } 6381 6382 #ifdef CONFIG_SKB_EXTENSIONS 6383 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id) 6384 { 6385 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE); 6386 } 6387 6388 /** 6389 * __skb_ext_alloc - allocate a new skb extensions storage 6390 * 6391 * @flags: See kmalloc(). 6392 * 6393 * Returns the newly allocated pointer. The pointer can later attached to a 6394 * skb via __skb_ext_set(). 6395 * Note: caller must handle the skb_ext as an opaque data. 6396 */ 6397 struct skb_ext *__skb_ext_alloc(gfp_t flags) 6398 { 6399 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags); 6400 6401 if (new) { 6402 memset(new->offset, 0, sizeof(new->offset)); 6403 refcount_set(&new->refcnt, 1); 6404 } 6405 6406 return new; 6407 } 6408 6409 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old, 6410 unsigned int old_active) 6411 { 6412 struct skb_ext *new; 6413 6414 if (refcount_read(&old->refcnt) == 1) 6415 return old; 6416 6417 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 6418 if (!new) 6419 return NULL; 6420 6421 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE); 6422 refcount_set(&new->refcnt, 1); 6423 6424 #ifdef CONFIG_XFRM 6425 if (old_active & (1 << SKB_EXT_SEC_PATH)) { 6426 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH); 6427 unsigned int i; 6428 6429 for (i = 0; i < sp->len; i++) 6430 xfrm_state_hold(sp->xvec[i]); 6431 } 6432 #endif 6433 __skb_ext_put(old); 6434 return new; 6435 } 6436 6437 /** 6438 * __skb_ext_set - attach the specified extension storage to this skb 6439 * @skb: buffer 6440 * @id: extension id 6441 * @ext: extension storage previously allocated via __skb_ext_alloc() 6442 * 6443 * Existing extensions, if any, are cleared. 6444 * 6445 * Returns the pointer to the extension. 6446 */ 6447 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id, 6448 struct skb_ext *ext) 6449 { 6450 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext); 6451 6452 skb_ext_put(skb); 6453 newlen = newoff + skb_ext_type_len[id]; 6454 ext->chunks = newlen; 6455 ext->offset[id] = newoff; 6456 skb->extensions = ext; 6457 skb->active_extensions = 1 << id; 6458 return skb_ext_get_ptr(ext, id); 6459 } 6460 6461 /** 6462 * skb_ext_add - allocate space for given extension, COW if needed 6463 * @skb: buffer 6464 * @id: extension to allocate space for 6465 * 6466 * Allocates enough space for the given extension. 6467 * If the extension is already present, a pointer to that extension 6468 * is returned. 6469 * 6470 * If the skb was cloned, COW applies and the returned memory can be 6471 * modified without changing the extension space of clones buffers. 6472 * 6473 * Returns pointer to the extension or NULL on allocation failure. 6474 */ 6475 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id) 6476 { 6477 struct skb_ext *new, *old = NULL; 6478 unsigned int newlen, newoff; 6479 6480 if (skb->active_extensions) { 6481 old = skb->extensions; 6482 6483 new = skb_ext_maybe_cow(old, skb->active_extensions); 6484 if (!new) 6485 return NULL; 6486 6487 if (__skb_ext_exist(new, id)) 6488 goto set_active; 6489 6490 newoff = new->chunks; 6491 } else { 6492 newoff = SKB_EXT_CHUNKSIZEOF(*new); 6493 6494 new = __skb_ext_alloc(GFP_ATOMIC); 6495 if (!new) 6496 return NULL; 6497 } 6498 6499 newlen = newoff + skb_ext_type_len[id]; 6500 new->chunks = newlen; 6501 new->offset[id] = newoff; 6502 set_active: 6503 skb->slow_gro = 1; 6504 skb->extensions = new; 6505 skb->active_extensions |= 1 << id; 6506 return skb_ext_get_ptr(new, id); 6507 } 6508 EXPORT_SYMBOL(skb_ext_add); 6509 6510 #ifdef CONFIG_XFRM 6511 static void skb_ext_put_sp(struct sec_path *sp) 6512 { 6513 unsigned int i; 6514 6515 for (i = 0; i < sp->len; i++) 6516 xfrm_state_put(sp->xvec[i]); 6517 } 6518 #endif 6519 6520 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 6521 { 6522 struct skb_ext *ext = skb->extensions; 6523 6524 skb->active_extensions &= ~(1 << id); 6525 if (skb->active_extensions == 0) { 6526 skb->extensions = NULL; 6527 __skb_ext_put(ext); 6528 #ifdef CONFIG_XFRM 6529 } else if (id == SKB_EXT_SEC_PATH && 6530 refcount_read(&ext->refcnt) == 1) { 6531 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH); 6532 6533 skb_ext_put_sp(sp); 6534 sp->len = 0; 6535 #endif 6536 } 6537 } 6538 EXPORT_SYMBOL(__skb_ext_del); 6539 6540 void __skb_ext_put(struct skb_ext *ext) 6541 { 6542 /* If this is last clone, nothing can increment 6543 * it after check passes. Avoids one atomic op. 6544 */ 6545 if (refcount_read(&ext->refcnt) == 1) 6546 goto free_now; 6547 6548 if (!refcount_dec_and_test(&ext->refcnt)) 6549 return; 6550 free_now: 6551 #ifdef CONFIG_XFRM 6552 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH)) 6553 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 6554 #endif 6555 6556 kmem_cache_free(skbuff_ext_cache, ext); 6557 } 6558 EXPORT_SYMBOL(__skb_ext_put); 6559 #endif /* CONFIG_SKB_EXTENSIONS */ 6560