xref: /openbmc/linux/net/core/skbuff.c (revision f17f06a0)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *	Routines having to do with the 'struct sk_buff' memory handlers.
4  *
5  *	Authors:	Alan Cox <alan@lxorguk.ukuu.org.uk>
6  *			Florian La Roche <rzsfl@rz.uni-sb.de>
7  *
8  *	Fixes:
9  *		Alan Cox	:	Fixed the worst of the load
10  *					balancer bugs.
11  *		Dave Platt	:	Interrupt stacking fix.
12  *	Richard Kooijman	:	Timestamp fixes.
13  *		Alan Cox	:	Changed buffer format.
14  *		Alan Cox	:	destructor hook for AF_UNIX etc.
15  *		Linus Torvalds	:	Better skb_clone.
16  *		Alan Cox	:	Added skb_copy.
17  *		Alan Cox	:	Added all the changed routines Linus
18  *					only put in the headers
19  *		Ray VanTassle	:	Fixed --skb->lock in free
20  *		Alan Cox	:	skb_copy copy arp field
21  *		Andi Kleen	:	slabified it.
22  *		Robert Olsson	:	Removed skb_head_pool
23  *
24  *	NOTE:
25  *		The __skb_ routines should be called with interrupts
26  *	disabled, or you better be *real* sure that the operation is atomic
27  *	with respect to whatever list is being frobbed (e.g. via lock_sock()
28  *	or via disabling bottom half handlers, etc).
29  */
30 
31 /*
32  *	The functions in this file will not compile correctly with gcc 2.4.x
33  */
34 
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36 
37 #include <linux/module.h>
38 #include <linux/types.h>
39 #include <linux/kernel.h>
40 #include <linux/mm.h>
41 #include <linux/interrupt.h>
42 #include <linux/in.h>
43 #include <linux/inet.h>
44 #include <linux/slab.h>
45 #include <linux/tcp.h>
46 #include <linux/udp.h>
47 #include <linux/sctp.h>
48 #include <linux/netdevice.h>
49 #ifdef CONFIG_NET_CLS_ACT
50 #include <net/pkt_sched.h>
51 #endif
52 #include <linux/string.h>
53 #include <linux/skbuff.h>
54 #include <linux/splice.h>
55 #include <linux/cache.h>
56 #include <linux/rtnetlink.h>
57 #include <linux/init.h>
58 #include <linux/scatterlist.h>
59 #include <linux/errqueue.h>
60 #include <linux/prefetch.h>
61 #include <linux/if_vlan.h>
62 #include <linux/mpls.h>
63 
64 #include <net/protocol.h>
65 #include <net/dst.h>
66 #include <net/sock.h>
67 #include <net/checksum.h>
68 #include <net/ip6_checksum.h>
69 #include <net/xfrm.h>
70 #include <net/mpls.h>
71 #include <net/mptcp.h>
72 
73 #include <linux/uaccess.h>
74 #include <trace/events/skb.h>
75 #include <linux/highmem.h>
76 #include <linux/capability.h>
77 #include <linux/user_namespace.h>
78 #include <linux/indirect_call_wrapper.h>
79 
80 #include "datagram.h"
81 
82 struct kmem_cache *skbuff_head_cache __ro_after_init;
83 static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
84 #ifdef CONFIG_SKB_EXTENSIONS
85 static struct kmem_cache *skbuff_ext_cache __ro_after_init;
86 #endif
87 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
88 EXPORT_SYMBOL(sysctl_max_skb_frags);
89 
90 /**
91  *	skb_panic - private function for out-of-line support
92  *	@skb:	buffer
93  *	@sz:	size
94  *	@addr:	address
95  *	@msg:	skb_over_panic or skb_under_panic
96  *
97  *	Out-of-line support for skb_put() and skb_push().
98  *	Called via the wrapper skb_over_panic() or skb_under_panic().
99  *	Keep out of line to prevent kernel bloat.
100  *	__builtin_return_address is not used because it is not always reliable.
101  */
102 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
103 		      const char msg[])
104 {
105 	pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
106 		 msg, addr, skb->len, sz, skb->head, skb->data,
107 		 (unsigned long)skb->tail, (unsigned long)skb->end,
108 		 skb->dev ? skb->dev->name : "<NULL>");
109 	BUG();
110 }
111 
112 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
113 {
114 	skb_panic(skb, sz, addr, __func__);
115 }
116 
117 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
118 {
119 	skb_panic(skb, sz, addr, __func__);
120 }
121 
122 /*
123  * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
124  * the caller if emergency pfmemalloc reserves are being used. If it is and
125  * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
126  * may be used. Otherwise, the packet data may be discarded until enough
127  * memory is free
128  */
129 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \
130 	 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
131 
132 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
133 			       unsigned long ip, bool *pfmemalloc)
134 {
135 	void *obj;
136 	bool ret_pfmemalloc = false;
137 
138 	/*
139 	 * Try a regular allocation, when that fails and we're not entitled
140 	 * to the reserves, fail.
141 	 */
142 	obj = kmalloc_node_track_caller(size,
143 					flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
144 					node);
145 	if (obj || !(gfp_pfmemalloc_allowed(flags)))
146 		goto out;
147 
148 	/* Try again but now we are using pfmemalloc reserves */
149 	ret_pfmemalloc = true;
150 	obj = kmalloc_node_track_caller(size, flags, node);
151 
152 out:
153 	if (pfmemalloc)
154 		*pfmemalloc = ret_pfmemalloc;
155 
156 	return obj;
157 }
158 
159 /* 	Allocate a new skbuff. We do this ourselves so we can fill in a few
160  *	'private' fields and also do memory statistics to find all the
161  *	[BEEP] leaks.
162  *
163  */
164 
165 /**
166  *	__alloc_skb	-	allocate a network buffer
167  *	@size: size to allocate
168  *	@gfp_mask: allocation mask
169  *	@flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
170  *		instead of head cache and allocate a cloned (child) skb.
171  *		If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
172  *		allocations in case the data is required for writeback
173  *	@node: numa node to allocate memory on
174  *
175  *	Allocate a new &sk_buff. The returned buffer has no headroom and a
176  *	tail room of at least size bytes. The object has a reference count
177  *	of one. The return is the buffer. On a failure the return is %NULL.
178  *
179  *	Buffers may only be allocated from interrupts using a @gfp_mask of
180  *	%GFP_ATOMIC.
181  */
182 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
183 			    int flags, int node)
184 {
185 	struct kmem_cache *cache;
186 	struct skb_shared_info *shinfo;
187 	struct sk_buff *skb;
188 	u8 *data;
189 	bool pfmemalloc;
190 
191 	cache = (flags & SKB_ALLOC_FCLONE)
192 		? skbuff_fclone_cache : skbuff_head_cache;
193 
194 	if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
195 		gfp_mask |= __GFP_MEMALLOC;
196 
197 	/* Get the HEAD */
198 	skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
199 	if (!skb)
200 		goto out;
201 	prefetchw(skb);
202 
203 	/* We do our best to align skb_shared_info on a separate cache
204 	 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
205 	 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
206 	 * Both skb->head and skb_shared_info are cache line aligned.
207 	 */
208 	size = SKB_DATA_ALIGN(size);
209 	size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
210 	data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
211 	if (!data)
212 		goto nodata;
213 	/* kmalloc(size) might give us more room than requested.
214 	 * Put skb_shared_info exactly at the end of allocated zone,
215 	 * to allow max possible filling before reallocation.
216 	 */
217 	size = SKB_WITH_OVERHEAD(ksize(data));
218 	prefetchw(data + size);
219 
220 	/*
221 	 * Only clear those fields we need to clear, not those that we will
222 	 * actually initialise below. Hence, don't put any more fields after
223 	 * the tail pointer in struct sk_buff!
224 	 */
225 	memset(skb, 0, offsetof(struct sk_buff, tail));
226 	/* Account for allocated memory : skb + skb->head */
227 	skb->truesize = SKB_TRUESIZE(size);
228 	skb->pfmemalloc = pfmemalloc;
229 	refcount_set(&skb->users, 1);
230 	skb->head = data;
231 	skb->data = data;
232 	skb_reset_tail_pointer(skb);
233 	skb->end = skb->tail + size;
234 	skb->mac_header = (typeof(skb->mac_header))~0U;
235 	skb->transport_header = (typeof(skb->transport_header))~0U;
236 
237 	/* make sure we initialize shinfo sequentially */
238 	shinfo = skb_shinfo(skb);
239 	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
240 	atomic_set(&shinfo->dataref, 1);
241 
242 	if (flags & SKB_ALLOC_FCLONE) {
243 		struct sk_buff_fclones *fclones;
244 
245 		fclones = container_of(skb, struct sk_buff_fclones, skb1);
246 
247 		skb->fclone = SKB_FCLONE_ORIG;
248 		refcount_set(&fclones->fclone_ref, 1);
249 
250 		fclones->skb2.fclone = SKB_FCLONE_CLONE;
251 	}
252 out:
253 	return skb;
254 nodata:
255 	kmem_cache_free(cache, skb);
256 	skb = NULL;
257 	goto out;
258 }
259 EXPORT_SYMBOL(__alloc_skb);
260 
261 /* Caller must provide SKB that is memset cleared */
262 static struct sk_buff *__build_skb_around(struct sk_buff *skb,
263 					  void *data, unsigned int frag_size)
264 {
265 	struct skb_shared_info *shinfo;
266 	unsigned int size = frag_size ? : ksize(data);
267 
268 	size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
269 
270 	/* Assumes caller memset cleared SKB */
271 	skb->truesize = SKB_TRUESIZE(size);
272 	refcount_set(&skb->users, 1);
273 	skb->head = data;
274 	skb->data = data;
275 	skb_reset_tail_pointer(skb);
276 	skb->end = skb->tail + size;
277 	skb->mac_header = (typeof(skb->mac_header))~0U;
278 	skb->transport_header = (typeof(skb->transport_header))~0U;
279 
280 	/* make sure we initialize shinfo sequentially */
281 	shinfo = skb_shinfo(skb);
282 	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
283 	atomic_set(&shinfo->dataref, 1);
284 
285 	return skb;
286 }
287 
288 /**
289  * __build_skb - build a network buffer
290  * @data: data buffer provided by caller
291  * @frag_size: size of data, or 0 if head was kmalloced
292  *
293  * Allocate a new &sk_buff. Caller provides space holding head and
294  * skb_shared_info. @data must have been allocated by kmalloc() only if
295  * @frag_size is 0, otherwise data should come from the page allocator
296  *  or vmalloc()
297  * The return is the new skb buffer.
298  * On a failure the return is %NULL, and @data is not freed.
299  * Notes :
300  *  Before IO, driver allocates only data buffer where NIC put incoming frame
301  *  Driver should add room at head (NET_SKB_PAD) and
302  *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
303  *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
304  *  before giving packet to stack.
305  *  RX rings only contains data buffers, not full skbs.
306  */
307 struct sk_buff *__build_skb(void *data, unsigned int frag_size)
308 {
309 	struct sk_buff *skb;
310 
311 	skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
312 	if (unlikely(!skb))
313 		return NULL;
314 
315 	memset(skb, 0, offsetof(struct sk_buff, tail));
316 
317 	return __build_skb_around(skb, data, frag_size);
318 }
319 
320 /* build_skb() is wrapper over __build_skb(), that specifically
321  * takes care of skb->head and skb->pfmemalloc
322  * This means that if @frag_size is not zero, then @data must be backed
323  * by a page fragment, not kmalloc() or vmalloc()
324  */
325 struct sk_buff *build_skb(void *data, unsigned int frag_size)
326 {
327 	struct sk_buff *skb = __build_skb(data, frag_size);
328 
329 	if (skb && frag_size) {
330 		skb->head_frag = 1;
331 		if (page_is_pfmemalloc(virt_to_head_page(data)))
332 			skb->pfmemalloc = 1;
333 	}
334 	return skb;
335 }
336 EXPORT_SYMBOL(build_skb);
337 
338 /**
339  * build_skb_around - build a network buffer around provided skb
340  * @skb: sk_buff provide by caller, must be memset cleared
341  * @data: data buffer provided by caller
342  * @frag_size: size of data, or 0 if head was kmalloced
343  */
344 struct sk_buff *build_skb_around(struct sk_buff *skb,
345 				 void *data, unsigned int frag_size)
346 {
347 	if (unlikely(!skb))
348 		return NULL;
349 
350 	skb = __build_skb_around(skb, data, frag_size);
351 
352 	if (skb && frag_size) {
353 		skb->head_frag = 1;
354 		if (page_is_pfmemalloc(virt_to_head_page(data)))
355 			skb->pfmemalloc = 1;
356 	}
357 	return skb;
358 }
359 EXPORT_SYMBOL(build_skb_around);
360 
361 #define NAPI_SKB_CACHE_SIZE	64
362 
363 struct napi_alloc_cache {
364 	struct page_frag_cache page;
365 	unsigned int skb_count;
366 	void *skb_cache[NAPI_SKB_CACHE_SIZE];
367 };
368 
369 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
370 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
371 
372 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
373 {
374 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
375 
376 	return page_frag_alloc(&nc->page, fragsz, gfp_mask);
377 }
378 
379 void *napi_alloc_frag(unsigned int fragsz)
380 {
381 	fragsz = SKB_DATA_ALIGN(fragsz);
382 
383 	return __napi_alloc_frag(fragsz, GFP_ATOMIC);
384 }
385 EXPORT_SYMBOL(napi_alloc_frag);
386 
387 /**
388  * netdev_alloc_frag - allocate a page fragment
389  * @fragsz: fragment size
390  *
391  * Allocates a frag from a page for receive buffer.
392  * Uses GFP_ATOMIC allocations.
393  */
394 void *netdev_alloc_frag(unsigned int fragsz)
395 {
396 	struct page_frag_cache *nc;
397 	void *data;
398 
399 	fragsz = SKB_DATA_ALIGN(fragsz);
400 	if (in_irq() || irqs_disabled()) {
401 		nc = this_cpu_ptr(&netdev_alloc_cache);
402 		data = page_frag_alloc(nc, fragsz, GFP_ATOMIC);
403 	} else {
404 		local_bh_disable();
405 		data = __napi_alloc_frag(fragsz, GFP_ATOMIC);
406 		local_bh_enable();
407 	}
408 	return data;
409 }
410 EXPORT_SYMBOL(netdev_alloc_frag);
411 
412 /**
413  *	__netdev_alloc_skb - allocate an skbuff for rx on a specific device
414  *	@dev: network device to receive on
415  *	@len: length to allocate
416  *	@gfp_mask: get_free_pages mask, passed to alloc_skb
417  *
418  *	Allocate a new &sk_buff and assign it a usage count of one. The
419  *	buffer has NET_SKB_PAD headroom built in. Users should allocate
420  *	the headroom they think they need without accounting for the
421  *	built in space. The built in space is used for optimisations.
422  *
423  *	%NULL is returned if there is no free memory.
424  */
425 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
426 				   gfp_t gfp_mask)
427 {
428 	struct page_frag_cache *nc;
429 	struct sk_buff *skb;
430 	bool pfmemalloc;
431 	void *data;
432 
433 	len += NET_SKB_PAD;
434 
435 	if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
436 	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
437 		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
438 		if (!skb)
439 			goto skb_fail;
440 		goto skb_success;
441 	}
442 
443 	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
444 	len = SKB_DATA_ALIGN(len);
445 
446 	if (sk_memalloc_socks())
447 		gfp_mask |= __GFP_MEMALLOC;
448 
449 	if (in_irq() || irqs_disabled()) {
450 		nc = this_cpu_ptr(&netdev_alloc_cache);
451 		data = page_frag_alloc(nc, len, gfp_mask);
452 		pfmemalloc = nc->pfmemalloc;
453 	} else {
454 		local_bh_disable();
455 		nc = this_cpu_ptr(&napi_alloc_cache.page);
456 		data = page_frag_alloc(nc, len, gfp_mask);
457 		pfmemalloc = nc->pfmemalloc;
458 		local_bh_enable();
459 	}
460 
461 	if (unlikely(!data))
462 		return NULL;
463 
464 	skb = __build_skb(data, len);
465 	if (unlikely(!skb)) {
466 		skb_free_frag(data);
467 		return NULL;
468 	}
469 
470 	if (pfmemalloc)
471 		skb->pfmemalloc = 1;
472 	skb->head_frag = 1;
473 
474 skb_success:
475 	skb_reserve(skb, NET_SKB_PAD);
476 	skb->dev = dev;
477 
478 skb_fail:
479 	return skb;
480 }
481 EXPORT_SYMBOL(__netdev_alloc_skb);
482 
483 /**
484  *	__napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
485  *	@napi: napi instance this buffer was allocated for
486  *	@len: length to allocate
487  *	@gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
488  *
489  *	Allocate a new sk_buff for use in NAPI receive.  This buffer will
490  *	attempt to allocate the head from a special reserved region used
491  *	only for NAPI Rx allocation.  By doing this we can save several
492  *	CPU cycles by avoiding having to disable and re-enable IRQs.
493  *
494  *	%NULL is returned if there is no free memory.
495  */
496 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
497 				 gfp_t gfp_mask)
498 {
499 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
500 	struct sk_buff *skb;
501 	void *data;
502 
503 	len += NET_SKB_PAD + NET_IP_ALIGN;
504 
505 	if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
506 	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
507 		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
508 		if (!skb)
509 			goto skb_fail;
510 		goto skb_success;
511 	}
512 
513 	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
514 	len = SKB_DATA_ALIGN(len);
515 
516 	if (sk_memalloc_socks())
517 		gfp_mask |= __GFP_MEMALLOC;
518 
519 	data = page_frag_alloc(&nc->page, len, gfp_mask);
520 	if (unlikely(!data))
521 		return NULL;
522 
523 	skb = __build_skb(data, len);
524 	if (unlikely(!skb)) {
525 		skb_free_frag(data);
526 		return NULL;
527 	}
528 
529 	if (nc->page.pfmemalloc)
530 		skb->pfmemalloc = 1;
531 	skb->head_frag = 1;
532 
533 skb_success:
534 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
535 	skb->dev = napi->dev;
536 
537 skb_fail:
538 	return skb;
539 }
540 EXPORT_SYMBOL(__napi_alloc_skb);
541 
542 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
543 		     int size, unsigned int truesize)
544 {
545 	skb_fill_page_desc(skb, i, page, off, size);
546 	skb->len += size;
547 	skb->data_len += size;
548 	skb->truesize += truesize;
549 }
550 EXPORT_SYMBOL(skb_add_rx_frag);
551 
552 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
553 			  unsigned int truesize)
554 {
555 	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
556 
557 	skb_frag_size_add(frag, size);
558 	skb->len += size;
559 	skb->data_len += size;
560 	skb->truesize += truesize;
561 }
562 EXPORT_SYMBOL(skb_coalesce_rx_frag);
563 
564 static void skb_drop_list(struct sk_buff **listp)
565 {
566 	kfree_skb_list(*listp);
567 	*listp = NULL;
568 }
569 
570 static inline void skb_drop_fraglist(struct sk_buff *skb)
571 {
572 	skb_drop_list(&skb_shinfo(skb)->frag_list);
573 }
574 
575 static void skb_clone_fraglist(struct sk_buff *skb)
576 {
577 	struct sk_buff *list;
578 
579 	skb_walk_frags(skb, list)
580 		skb_get(list);
581 }
582 
583 static void skb_free_head(struct sk_buff *skb)
584 {
585 	unsigned char *head = skb->head;
586 
587 	if (skb->head_frag)
588 		skb_free_frag(head);
589 	else
590 		kfree(head);
591 }
592 
593 static void skb_release_data(struct sk_buff *skb)
594 {
595 	struct skb_shared_info *shinfo = skb_shinfo(skb);
596 	int i;
597 
598 	if (skb->cloned &&
599 	    atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
600 			      &shinfo->dataref))
601 		return;
602 
603 	for (i = 0; i < shinfo->nr_frags; i++)
604 		__skb_frag_unref(&shinfo->frags[i]);
605 
606 	if (shinfo->frag_list)
607 		kfree_skb_list(shinfo->frag_list);
608 
609 	skb_zcopy_clear(skb, true);
610 	skb_free_head(skb);
611 }
612 
613 /*
614  *	Free an skbuff by memory without cleaning the state.
615  */
616 static void kfree_skbmem(struct sk_buff *skb)
617 {
618 	struct sk_buff_fclones *fclones;
619 
620 	switch (skb->fclone) {
621 	case SKB_FCLONE_UNAVAILABLE:
622 		kmem_cache_free(skbuff_head_cache, skb);
623 		return;
624 
625 	case SKB_FCLONE_ORIG:
626 		fclones = container_of(skb, struct sk_buff_fclones, skb1);
627 
628 		/* We usually free the clone (TX completion) before original skb
629 		 * This test would have no chance to be true for the clone,
630 		 * while here, branch prediction will be good.
631 		 */
632 		if (refcount_read(&fclones->fclone_ref) == 1)
633 			goto fastpath;
634 		break;
635 
636 	default: /* SKB_FCLONE_CLONE */
637 		fclones = container_of(skb, struct sk_buff_fclones, skb2);
638 		break;
639 	}
640 	if (!refcount_dec_and_test(&fclones->fclone_ref))
641 		return;
642 fastpath:
643 	kmem_cache_free(skbuff_fclone_cache, fclones);
644 }
645 
646 void skb_release_head_state(struct sk_buff *skb)
647 {
648 	skb_dst_drop(skb);
649 	if (skb->destructor) {
650 		WARN_ON(in_irq());
651 		skb->destructor(skb);
652 	}
653 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
654 	nf_conntrack_put(skb_nfct(skb));
655 #endif
656 	skb_ext_put(skb);
657 }
658 
659 /* Free everything but the sk_buff shell. */
660 static void skb_release_all(struct sk_buff *skb)
661 {
662 	skb_release_head_state(skb);
663 	if (likely(skb->head))
664 		skb_release_data(skb);
665 }
666 
667 /**
668  *	__kfree_skb - private function
669  *	@skb: buffer
670  *
671  *	Free an sk_buff. Release anything attached to the buffer.
672  *	Clean the state. This is an internal helper function. Users should
673  *	always call kfree_skb
674  */
675 
676 void __kfree_skb(struct sk_buff *skb)
677 {
678 	skb_release_all(skb);
679 	kfree_skbmem(skb);
680 }
681 EXPORT_SYMBOL(__kfree_skb);
682 
683 /**
684  *	kfree_skb - free an sk_buff
685  *	@skb: buffer to free
686  *
687  *	Drop a reference to the buffer and free it if the usage count has
688  *	hit zero.
689  */
690 void kfree_skb(struct sk_buff *skb)
691 {
692 	if (!skb_unref(skb))
693 		return;
694 
695 	trace_kfree_skb(skb, __builtin_return_address(0));
696 	__kfree_skb(skb);
697 }
698 EXPORT_SYMBOL(kfree_skb);
699 
700 void kfree_skb_list(struct sk_buff *segs)
701 {
702 	while (segs) {
703 		struct sk_buff *next = segs->next;
704 
705 		kfree_skb(segs);
706 		segs = next;
707 	}
708 }
709 EXPORT_SYMBOL(kfree_skb_list);
710 
711 /* Dump skb information and contents.
712  *
713  * Must only be called from net_ratelimit()-ed paths.
714  *
715  * Dumps up to can_dump_full whole packets if full_pkt, headers otherwise.
716  */
717 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
718 {
719 	static atomic_t can_dump_full = ATOMIC_INIT(5);
720 	struct skb_shared_info *sh = skb_shinfo(skb);
721 	struct net_device *dev = skb->dev;
722 	struct sock *sk = skb->sk;
723 	struct sk_buff *list_skb;
724 	bool has_mac, has_trans;
725 	int headroom, tailroom;
726 	int i, len, seg_len;
727 
728 	if (full_pkt)
729 		full_pkt = atomic_dec_if_positive(&can_dump_full) >= 0;
730 
731 	if (full_pkt)
732 		len = skb->len;
733 	else
734 		len = min_t(int, skb->len, MAX_HEADER + 128);
735 
736 	headroom = skb_headroom(skb);
737 	tailroom = skb_tailroom(skb);
738 
739 	has_mac = skb_mac_header_was_set(skb);
740 	has_trans = skb_transport_header_was_set(skb);
741 
742 	printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
743 	       "mac=(%d,%d) net=(%d,%d) trans=%d\n"
744 	       "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
745 	       "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
746 	       "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n",
747 	       level, skb->len, headroom, skb_headlen(skb), tailroom,
748 	       has_mac ? skb->mac_header : -1,
749 	       has_mac ? skb_mac_header_len(skb) : -1,
750 	       skb->network_header,
751 	       has_trans ? skb_network_header_len(skb) : -1,
752 	       has_trans ? skb->transport_header : -1,
753 	       sh->tx_flags, sh->nr_frags,
754 	       sh->gso_size, sh->gso_type, sh->gso_segs,
755 	       skb->csum, skb->ip_summed, skb->csum_complete_sw,
756 	       skb->csum_valid, skb->csum_level,
757 	       skb->hash, skb->sw_hash, skb->l4_hash,
758 	       ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
759 
760 	if (dev)
761 		printk("%sdev name=%s feat=0x%pNF\n",
762 		       level, dev->name, &dev->features);
763 	if (sk)
764 		printk("%ssk family=%hu type=%u proto=%u\n",
765 		       level, sk->sk_family, sk->sk_type, sk->sk_protocol);
766 
767 	if (full_pkt && headroom)
768 		print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
769 			       16, 1, skb->head, headroom, false);
770 
771 	seg_len = min_t(int, skb_headlen(skb), len);
772 	if (seg_len)
773 		print_hex_dump(level, "skb linear:   ", DUMP_PREFIX_OFFSET,
774 			       16, 1, skb->data, seg_len, false);
775 	len -= seg_len;
776 
777 	if (full_pkt && tailroom)
778 		print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
779 			       16, 1, skb_tail_pointer(skb), tailroom, false);
780 
781 	for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
782 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
783 		u32 p_off, p_len, copied;
784 		struct page *p;
785 		u8 *vaddr;
786 
787 		skb_frag_foreach_page(frag, skb_frag_off(frag),
788 				      skb_frag_size(frag), p, p_off, p_len,
789 				      copied) {
790 			seg_len = min_t(int, p_len, len);
791 			vaddr = kmap_atomic(p);
792 			print_hex_dump(level, "skb frag:     ",
793 				       DUMP_PREFIX_OFFSET,
794 				       16, 1, vaddr + p_off, seg_len, false);
795 			kunmap_atomic(vaddr);
796 			len -= seg_len;
797 			if (!len)
798 				break;
799 		}
800 	}
801 
802 	if (full_pkt && skb_has_frag_list(skb)) {
803 		printk("skb fraglist:\n");
804 		skb_walk_frags(skb, list_skb)
805 			skb_dump(level, list_skb, true);
806 	}
807 }
808 EXPORT_SYMBOL(skb_dump);
809 
810 /**
811  *	skb_tx_error - report an sk_buff xmit error
812  *	@skb: buffer that triggered an error
813  *
814  *	Report xmit error if a device callback is tracking this skb.
815  *	skb must be freed afterwards.
816  */
817 void skb_tx_error(struct sk_buff *skb)
818 {
819 	skb_zcopy_clear(skb, true);
820 }
821 EXPORT_SYMBOL(skb_tx_error);
822 
823 /**
824  *	consume_skb - free an skbuff
825  *	@skb: buffer to free
826  *
827  *	Drop a ref to the buffer and free it if the usage count has hit zero
828  *	Functions identically to kfree_skb, but kfree_skb assumes that the frame
829  *	is being dropped after a failure and notes that
830  */
831 void consume_skb(struct sk_buff *skb)
832 {
833 	if (!skb_unref(skb))
834 		return;
835 
836 	trace_consume_skb(skb);
837 	__kfree_skb(skb);
838 }
839 EXPORT_SYMBOL(consume_skb);
840 
841 /**
842  *	consume_stateless_skb - free an skbuff, assuming it is stateless
843  *	@skb: buffer to free
844  *
845  *	Alike consume_skb(), but this variant assumes that this is the last
846  *	skb reference and all the head states have been already dropped
847  */
848 void __consume_stateless_skb(struct sk_buff *skb)
849 {
850 	trace_consume_skb(skb);
851 	skb_release_data(skb);
852 	kfree_skbmem(skb);
853 }
854 
855 void __kfree_skb_flush(void)
856 {
857 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
858 
859 	/* flush skb_cache if containing objects */
860 	if (nc->skb_count) {
861 		kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
862 				     nc->skb_cache);
863 		nc->skb_count = 0;
864 	}
865 }
866 
867 static inline void _kfree_skb_defer(struct sk_buff *skb)
868 {
869 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
870 
871 	/* drop skb->head and call any destructors for packet */
872 	skb_release_all(skb);
873 
874 	/* record skb to CPU local list */
875 	nc->skb_cache[nc->skb_count++] = skb;
876 
877 #ifdef CONFIG_SLUB
878 	/* SLUB writes into objects when freeing */
879 	prefetchw(skb);
880 #endif
881 
882 	/* flush skb_cache if it is filled */
883 	if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
884 		kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE,
885 				     nc->skb_cache);
886 		nc->skb_count = 0;
887 	}
888 }
889 void __kfree_skb_defer(struct sk_buff *skb)
890 {
891 	_kfree_skb_defer(skb);
892 }
893 
894 void napi_consume_skb(struct sk_buff *skb, int budget)
895 {
896 	if (unlikely(!skb))
897 		return;
898 
899 	/* Zero budget indicate non-NAPI context called us, like netpoll */
900 	if (unlikely(!budget)) {
901 		dev_consume_skb_any(skb);
902 		return;
903 	}
904 
905 	if (!skb_unref(skb))
906 		return;
907 
908 	/* if reaching here SKB is ready to free */
909 	trace_consume_skb(skb);
910 
911 	/* if SKB is a clone, don't handle this case */
912 	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
913 		__kfree_skb(skb);
914 		return;
915 	}
916 
917 	_kfree_skb_defer(skb);
918 }
919 EXPORT_SYMBOL(napi_consume_skb);
920 
921 /* Make sure a field is enclosed inside headers_start/headers_end section */
922 #define CHECK_SKB_FIELD(field) \
923 	BUILD_BUG_ON(offsetof(struct sk_buff, field) <		\
924 		     offsetof(struct sk_buff, headers_start));	\
925 	BUILD_BUG_ON(offsetof(struct sk_buff, field) >		\
926 		     offsetof(struct sk_buff, headers_end));	\
927 
928 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
929 {
930 	new->tstamp		= old->tstamp;
931 	/* We do not copy old->sk */
932 	new->dev		= old->dev;
933 	memcpy(new->cb, old->cb, sizeof(old->cb));
934 	skb_dst_copy(new, old);
935 	__skb_ext_copy(new, old);
936 	__nf_copy(new, old, false);
937 
938 	/* Note : this field could be in headers_start/headers_end section
939 	 * It is not yet because we do not want to have a 16 bit hole
940 	 */
941 	new->queue_mapping = old->queue_mapping;
942 
943 	memcpy(&new->headers_start, &old->headers_start,
944 	       offsetof(struct sk_buff, headers_end) -
945 	       offsetof(struct sk_buff, headers_start));
946 	CHECK_SKB_FIELD(protocol);
947 	CHECK_SKB_FIELD(csum);
948 	CHECK_SKB_FIELD(hash);
949 	CHECK_SKB_FIELD(priority);
950 	CHECK_SKB_FIELD(skb_iif);
951 	CHECK_SKB_FIELD(vlan_proto);
952 	CHECK_SKB_FIELD(vlan_tci);
953 	CHECK_SKB_FIELD(transport_header);
954 	CHECK_SKB_FIELD(network_header);
955 	CHECK_SKB_FIELD(mac_header);
956 	CHECK_SKB_FIELD(inner_protocol);
957 	CHECK_SKB_FIELD(inner_transport_header);
958 	CHECK_SKB_FIELD(inner_network_header);
959 	CHECK_SKB_FIELD(inner_mac_header);
960 	CHECK_SKB_FIELD(mark);
961 #ifdef CONFIG_NETWORK_SECMARK
962 	CHECK_SKB_FIELD(secmark);
963 #endif
964 #ifdef CONFIG_NET_RX_BUSY_POLL
965 	CHECK_SKB_FIELD(napi_id);
966 #endif
967 #ifdef CONFIG_XPS
968 	CHECK_SKB_FIELD(sender_cpu);
969 #endif
970 #ifdef CONFIG_NET_SCHED
971 	CHECK_SKB_FIELD(tc_index);
972 #endif
973 
974 }
975 
976 /*
977  * You should not add any new code to this function.  Add it to
978  * __copy_skb_header above instead.
979  */
980 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
981 {
982 #define C(x) n->x = skb->x
983 
984 	n->next = n->prev = NULL;
985 	n->sk = NULL;
986 	__copy_skb_header(n, skb);
987 
988 	C(len);
989 	C(data_len);
990 	C(mac_len);
991 	n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
992 	n->cloned = 1;
993 	n->nohdr = 0;
994 	n->peeked = 0;
995 	C(pfmemalloc);
996 	n->destructor = NULL;
997 	C(tail);
998 	C(end);
999 	C(head);
1000 	C(head_frag);
1001 	C(data);
1002 	C(truesize);
1003 	refcount_set(&n->users, 1);
1004 
1005 	atomic_inc(&(skb_shinfo(skb)->dataref));
1006 	skb->cloned = 1;
1007 
1008 	return n;
1009 #undef C
1010 }
1011 
1012 /**
1013  * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
1014  * @first: first sk_buff of the msg
1015  */
1016 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
1017 {
1018 	struct sk_buff *n;
1019 
1020 	n = alloc_skb(0, GFP_ATOMIC);
1021 	if (!n)
1022 		return NULL;
1023 
1024 	n->len = first->len;
1025 	n->data_len = first->len;
1026 	n->truesize = first->truesize;
1027 
1028 	skb_shinfo(n)->frag_list = first;
1029 
1030 	__copy_skb_header(n, first);
1031 	n->destructor = NULL;
1032 
1033 	return n;
1034 }
1035 EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
1036 
1037 /**
1038  *	skb_morph	-	morph one skb into another
1039  *	@dst: the skb to receive the contents
1040  *	@src: the skb to supply the contents
1041  *
1042  *	This is identical to skb_clone except that the target skb is
1043  *	supplied by the user.
1044  *
1045  *	The target skb is returned upon exit.
1046  */
1047 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
1048 {
1049 	skb_release_all(dst);
1050 	return __skb_clone(dst, src);
1051 }
1052 EXPORT_SYMBOL_GPL(skb_morph);
1053 
1054 int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
1055 {
1056 	unsigned long max_pg, num_pg, new_pg, old_pg;
1057 	struct user_struct *user;
1058 
1059 	if (capable(CAP_IPC_LOCK) || !size)
1060 		return 0;
1061 
1062 	num_pg = (size >> PAGE_SHIFT) + 2;	/* worst case */
1063 	max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1064 	user = mmp->user ? : current_user();
1065 
1066 	do {
1067 		old_pg = atomic_long_read(&user->locked_vm);
1068 		new_pg = old_pg + num_pg;
1069 		if (new_pg > max_pg)
1070 			return -ENOBUFS;
1071 	} while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
1072 		 old_pg);
1073 
1074 	if (!mmp->user) {
1075 		mmp->user = get_uid(user);
1076 		mmp->num_pg = num_pg;
1077 	} else {
1078 		mmp->num_pg += num_pg;
1079 	}
1080 
1081 	return 0;
1082 }
1083 EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
1084 
1085 void mm_unaccount_pinned_pages(struct mmpin *mmp)
1086 {
1087 	if (mmp->user) {
1088 		atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
1089 		free_uid(mmp->user);
1090 	}
1091 }
1092 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
1093 
1094 struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size)
1095 {
1096 	struct ubuf_info *uarg;
1097 	struct sk_buff *skb;
1098 
1099 	WARN_ON_ONCE(!in_task());
1100 
1101 	skb = sock_omalloc(sk, 0, GFP_KERNEL);
1102 	if (!skb)
1103 		return NULL;
1104 
1105 	BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1106 	uarg = (void *)skb->cb;
1107 	uarg->mmp.user = NULL;
1108 
1109 	if (mm_account_pinned_pages(&uarg->mmp, size)) {
1110 		kfree_skb(skb);
1111 		return NULL;
1112 	}
1113 
1114 	uarg->callback = sock_zerocopy_callback;
1115 	uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
1116 	uarg->len = 1;
1117 	uarg->bytelen = size;
1118 	uarg->zerocopy = 1;
1119 	refcount_set(&uarg->refcnt, 1);
1120 	sock_hold(sk);
1121 
1122 	return uarg;
1123 }
1124 EXPORT_SYMBOL_GPL(sock_zerocopy_alloc);
1125 
1126 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
1127 {
1128 	return container_of((void *)uarg, struct sk_buff, cb);
1129 }
1130 
1131 struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
1132 					struct ubuf_info *uarg)
1133 {
1134 	if (uarg) {
1135 		const u32 byte_limit = 1 << 19;		/* limit to a few TSO */
1136 		u32 bytelen, next;
1137 
1138 		/* realloc only when socket is locked (TCP, UDP cork),
1139 		 * so uarg->len and sk_zckey access is serialized
1140 		 */
1141 		if (!sock_owned_by_user(sk)) {
1142 			WARN_ON_ONCE(1);
1143 			return NULL;
1144 		}
1145 
1146 		bytelen = uarg->bytelen + size;
1147 		if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
1148 			/* TCP can create new skb to attach new uarg */
1149 			if (sk->sk_type == SOCK_STREAM)
1150 				goto new_alloc;
1151 			return NULL;
1152 		}
1153 
1154 		next = (u32)atomic_read(&sk->sk_zckey);
1155 		if ((u32)(uarg->id + uarg->len) == next) {
1156 			if (mm_account_pinned_pages(&uarg->mmp, size))
1157 				return NULL;
1158 			uarg->len++;
1159 			uarg->bytelen = bytelen;
1160 			atomic_set(&sk->sk_zckey, ++next);
1161 
1162 			/* no extra ref when appending to datagram (MSG_MORE) */
1163 			if (sk->sk_type == SOCK_STREAM)
1164 				sock_zerocopy_get(uarg);
1165 
1166 			return uarg;
1167 		}
1168 	}
1169 
1170 new_alloc:
1171 	return sock_zerocopy_alloc(sk, size);
1172 }
1173 EXPORT_SYMBOL_GPL(sock_zerocopy_realloc);
1174 
1175 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1176 {
1177 	struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1178 	u32 old_lo, old_hi;
1179 	u64 sum_len;
1180 
1181 	old_lo = serr->ee.ee_info;
1182 	old_hi = serr->ee.ee_data;
1183 	sum_len = old_hi - old_lo + 1ULL + len;
1184 
1185 	if (sum_len >= (1ULL << 32))
1186 		return false;
1187 
1188 	if (lo != old_hi + 1)
1189 		return false;
1190 
1191 	serr->ee.ee_data += len;
1192 	return true;
1193 }
1194 
1195 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
1196 {
1197 	struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1198 	struct sock_exterr_skb *serr;
1199 	struct sock *sk = skb->sk;
1200 	struct sk_buff_head *q;
1201 	unsigned long flags;
1202 	u32 lo, hi;
1203 	u16 len;
1204 
1205 	mm_unaccount_pinned_pages(&uarg->mmp);
1206 
1207 	/* if !len, there was only 1 call, and it was aborted
1208 	 * so do not queue a completion notification
1209 	 */
1210 	if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1211 		goto release;
1212 
1213 	len = uarg->len;
1214 	lo = uarg->id;
1215 	hi = uarg->id + len - 1;
1216 
1217 	serr = SKB_EXT_ERR(skb);
1218 	memset(serr, 0, sizeof(*serr));
1219 	serr->ee.ee_errno = 0;
1220 	serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1221 	serr->ee.ee_data = hi;
1222 	serr->ee.ee_info = lo;
1223 	if (!success)
1224 		serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1225 
1226 	q = &sk->sk_error_queue;
1227 	spin_lock_irqsave(&q->lock, flags);
1228 	tail = skb_peek_tail(q);
1229 	if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1230 	    !skb_zerocopy_notify_extend(tail, lo, len)) {
1231 		__skb_queue_tail(q, skb);
1232 		skb = NULL;
1233 	}
1234 	spin_unlock_irqrestore(&q->lock, flags);
1235 
1236 	sk->sk_error_report(sk);
1237 
1238 release:
1239 	consume_skb(skb);
1240 	sock_put(sk);
1241 }
1242 EXPORT_SYMBOL_GPL(sock_zerocopy_callback);
1243 
1244 void sock_zerocopy_put(struct ubuf_info *uarg)
1245 {
1246 	if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
1247 		if (uarg->callback)
1248 			uarg->callback(uarg, uarg->zerocopy);
1249 		else
1250 			consume_skb(skb_from_uarg(uarg));
1251 	}
1252 }
1253 EXPORT_SYMBOL_GPL(sock_zerocopy_put);
1254 
1255 void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1256 {
1257 	if (uarg) {
1258 		struct sock *sk = skb_from_uarg(uarg)->sk;
1259 
1260 		atomic_dec(&sk->sk_zckey);
1261 		uarg->len--;
1262 
1263 		if (have_uref)
1264 			sock_zerocopy_put(uarg);
1265 	}
1266 }
1267 EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);
1268 
1269 int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len)
1270 {
1271 	return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
1272 }
1273 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);
1274 
1275 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1276 			     struct msghdr *msg, int len,
1277 			     struct ubuf_info *uarg)
1278 {
1279 	struct ubuf_info *orig_uarg = skb_zcopy(skb);
1280 	struct iov_iter orig_iter = msg->msg_iter;
1281 	int err, orig_len = skb->len;
1282 
1283 	/* An skb can only point to one uarg. This edge case happens when
1284 	 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1285 	 */
1286 	if (orig_uarg && uarg != orig_uarg)
1287 		return -EEXIST;
1288 
1289 	err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
1290 	if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1291 		struct sock *save_sk = skb->sk;
1292 
1293 		/* Streams do not free skb on error. Reset to prev state. */
1294 		msg->msg_iter = orig_iter;
1295 		skb->sk = sk;
1296 		___pskb_trim(skb, orig_len);
1297 		skb->sk = save_sk;
1298 		return err;
1299 	}
1300 
1301 	skb_zcopy_set(skb, uarg, NULL);
1302 	return skb->len - orig_len;
1303 }
1304 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1305 
1306 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1307 			      gfp_t gfp_mask)
1308 {
1309 	if (skb_zcopy(orig)) {
1310 		if (skb_zcopy(nskb)) {
1311 			/* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1312 			if (!gfp_mask) {
1313 				WARN_ON_ONCE(1);
1314 				return -ENOMEM;
1315 			}
1316 			if (skb_uarg(nskb) == skb_uarg(orig))
1317 				return 0;
1318 			if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1319 				return -EIO;
1320 		}
1321 		skb_zcopy_set(nskb, skb_uarg(orig), NULL);
1322 	}
1323 	return 0;
1324 }
1325 
1326 /**
1327  *	skb_copy_ubufs	-	copy userspace skb frags buffers to kernel
1328  *	@skb: the skb to modify
1329  *	@gfp_mask: allocation priority
1330  *
1331  *	This must be called on SKBTX_DEV_ZEROCOPY skb.
1332  *	It will copy all frags into kernel and drop the reference
1333  *	to userspace pages.
1334  *
1335  *	If this function is called from an interrupt gfp_mask() must be
1336  *	%GFP_ATOMIC.
1337  *
1338  *	Returns 0 on success or a negative error code on failure
1339  *	to allocate kernel memory to copy to.
1340  */
1341 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1342 {
1343 	int num_frags = skb_shinfo(skb)->nr_frags;
1344 	struct page *page, *head = NULL;
1345 	int i, new_frags;
1346 	u32 d_off;
1347 
1348 	if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1349 		return -EINVAL;
1350 
1351 	if (!num_frags)
1352 		goto release;
1353 
1354 	new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1355 	for (i = 0; i < new_frags; i++) {
1356 		page = alloc_page(gfp_mask);
1357 		if (!page) {
1358 			while (head) {
1359 				struct page *next = (struct page *)page_private(head);
1360 				put_page(head);
1361 				head = next;
1362 			}
1363 			return -ENOMEM;
1364 		}
1365 		set_page_private(page, (unsigned long)head);
1366 		head = page;
1367 	}
1368 
1369 	page = head;
1370 	d_off = 0;
1371 	for (i = 0; i < num_frags; i++) {
1372 		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1373 		u32 p_off, p_len, copied;
1374 		struct page *p;
1375 		u8 *vaddr;
1376 
1377 		skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
1378 				      p, p_off, p_len, copied) {
1379 			u32 copy, done = 0;
1380 			vaddr = kmap_atomic(p);
1381 
1382 			while (done < p_len) {
1383 				if (d_off == PAGE_SIZE) {
1384 					d_off = 0;
1385 					page = (struct page *)page_private(page);
1386 				}
1387 				copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
1388 				memcpy(page_address(page) + d_off,
1389 				       vaddr + p_off + done, copy);
1390 				done += copy;
1391 				d_off += copy;
1392 			}
1393 			kunmap_atomic(vaddr);
1394 		}
1395 	}
1396 
1397 	/* skb frags release userspace buffers */
1398 	for (i = 0; i < num_frags; i++)
1399 		skb_frag_unref(skb, i);
1400 
1401 	/* skb frags point to kernel buffers */
1402 	for (i = 0; i < new_frags - 1; i++) {
1403 		__skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
1404 		head = (struct page *)page_private(head);
1405 	}
1406 	__skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
1407 	skb_shinfo(skb)->nr_frags = new_frags;
1408 
1409 release:
1410 	skb_zcopy_clear(skb, false);
1411 	return 0;
1412 }
1413 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
1414 
1415 /**
1416  *	skb_clone	-	duplicate an sk_buff
1417  *	@skb: buffer to clone
1418  *	@gfp_mask: allocation priority
1419  *
1420  *	Duplicate an &sk_buff. The new one is not owned by a socket. Both
1421  *	copies share the same packet data but not structure. The new
1422  *	buffer has a reference count of 1. If the allocation fails the
1423  *	function returns %NULL otherwise the new buffer is returned.
1424  *
1425  *	If this function is called from an interrupt gfp_mask() must be
1426  *	%GFP_ATOMIC.
1427  */
1428 
1429 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
1430 {
1431 	struct sk_buff_fclones *fclones = container_of(skb,
1432 						       struct sk_buff_fclones,
1433 						       skb1);
1434 	struct sk_buff *n;
1435 
1436 	if (skb_orphan_frags(skb, gfp_mask))
1437 		return NULL;
1438 
1439 	if (skb->fclone == SKB_FCLONE_ORIG &&
1440 	    refcount_read(&fclones->fclone_ref) == 1) {
1441 		n = &fclones->skb2;
1442 		refcount_set(&fclones->fclone_ref, 2);
1443 	} else {
1444 		if (skb_pfmemalloc(skb))
1445 			gfp_mask |= __GFP_MEMALLOC;
1446 
1447 		n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
1448 		if (!n)
1449 			return NULL;
1450 
1451 		n->fclone = SKB_FCLONE_UNAVAILABLE;
1452 	}
1453 
1454 	return __skb_clone(n, skb);
1455 }
1456 EXPORT_SYMBOL(skb_clone);
1457 
1458 void skb_headers_offset_update(struct sk_buff *skb, int off)
1459 {
1460 	/* Only adjust this if it actually is csum_start rather than csum */
1461 	if (skb->ip_summed == CHECKSUM_PARTIAL)
1462 		skb->csum_start += off;
1463 	/* {transport,network,mac}_header and tail are relative to skb->head */
1464 	skb->transport_header += off;
1465 	skb->network_header   += off;
1466 	if (skb_mac_header_was_set(skb))
1467 		skb->mac_header += off;
1468 	skb->inner_transport_header += off;
1469 	skb->inner_network_header += off;
1470 	skb->inner_mac_header += off;
1471 }
1472 EXPORT_SYMBOL(skb_headers_offset_update);
1473 
1474 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
1475 {
1476 	__copy_skb_header(new, old);
1477 
1478 	skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
1479 	skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
1480 	skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
1481 }
1482 EXPORT_SYMBOL(skb_copy_header);
1483 
1484 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
1485 {
1486 	if (skb_pfmemalloc(skb))
1487 		return SKB_ALLOC_RX;
1488 	return 0;
1489 }
1490 
1491 /**
1492  *	skb_copy	-	create private copy of an sk_buff
1493  *	@skb: buffer to copy
1494  *	@gfp_mask: allocation priority
1495  *
1496  *	Make a copy of both an &sk_buff and its data. This is used when the
1497  *	caller wishes to modify the data and needs a private copy of the
1498  *	data to alter. Returns %NULL on failure or the pointer to the buffer
1499  *	on success. The returned buffer has a reference count of 1.
1500  *
1501  *	As by-product this function converts non-linear &sk_buff to linear
1502  *	one, so that &sk_buff becomes completely private and caller is allowed
1503  *	to modify all the data of returned buffer. This means that this
1504  *	function is not recommended for use in circumstances when only
1505  *	header is going to be modified. Use pskb_copy() instead.
1506  */
1507 
1508 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
1509 {
1510 	int headerlen = skb_headroom(skb);
1511 	unsigned int size = skb_end_offset(skb) + skb->data_len;
1512 	struct sk_buff *n = __alloc_skb(size, gfp_mask,
1513 					skb_alloc_rx_flag(skb), NUMA_NO_NODE);
1514 
1515 	if (!n)
1516 		return NULL;
1517 
1518 	/* Set the data pointer */
1519 	skb_reserve(n, headerlen);
1520 	/* Set the tail pointer and length */
1521 	skb_put(n, skb->len);
1522 
1523 	BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
1524 
1525 	skb_copy_header(n, skb);
1526 	return n;
1527 }
1528 EXPORT_SYMBOL(skb_copy);
1529 
1530 /**
1531  *	__pskb_copy_fclone	-  create copy of an sk_buff with private head.
1532  *	@skb: buffer to copy
1533  *	@headroom: headroom of new skb
1534  *	@gfp_mask: allocation priority
1535  *	@fclone: if true allocate the copy of the skb from the fclone
1536  *	cache instead of the head cache; it is recommended to set this
1537  *	to true for the cases where the copy will likely be cloned
1538  *
1539  *	Make a copy of both an &sk_buff and part of its data, located
1540  *	in header. Fragmented data remain shared. This is used when
1541  *	the caller wishes to modify only header of &sk_buff and needs
1542  *	private copy of the header to alter. Returns %NULL on failure
1543  *	or the pointer to the buffer on success.
1544  *	The returned buffer has a reference count of 1.
1545  */
1546 
1547 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1548 				   gfp_t gfp_mask, bool fclone)
1549 {
1550 	unsigned int size = skb_headlen(skb) + headroom;
1551 	int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
1552 	struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
1553 
1554 	if (!n)
1555 		goto out;
1556 
1557 	/* Set the data pointer */
1558 	skb_reserve(n, headroom);
1559 	/* Set the tail pointer and length */
1560 	skb_put(n, skb_headlen(skb));
1561 	/* Copy the bytes */
1562 	skb_copy_from_linear_data(skb, n->data, n->len);
1563 
1564 	n->truesize += skb->data_len;
1565 	n->data_len  = skb->data_len;
1566 	n->len	     = skb->len;
1567 
1568 	if (skb_shinfo(skb)->nr_frags) {
1569 		int i;
1570 
1571 		if (skb_orphan_frags(skb, gfp_mask) ||
1572 		    skb_zerocopy_clone(n, skb, gfp_mask)) {
1573 			kfree_skb(n);
1574 			n = NULL;
1575 			goto out;
1576 		}
1577 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1578 			skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1579 			skb_frag_ref(skb, i);
1580 		}
1581 		skb_shinfo(n)->nr_frags = i;
1582 	}
1583 
1584 	if (skb_has_frag_list(skb)) {
1585 		skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1586 		skb_clone_fraglist(n);
1587 	}
1588 
1589 	skb_copy_header(n, skb);
1590 out:
1591 	return n;
1592 }
1593 EXPORT_SYMBOL(__pskb_copy_fclone);
1594 
1595 /**
1596  *	pskb_expand_head - reallocate header of &sk_buff
1597  *	@skb: buffer to reallocate
1598  *	@nhead: room to add at head
1599  *	@ntail: room to add at tail
1600  *	@gfp_mask: allocation priority
1601  *
1602  *	Expands (or creates identical copy, if @nhead and @ntail are zero)
1603  *	header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
1604  *	reference count of 1. Returns zero in the case of success or error,
1605  *	if expansion failed. In the last case, &sk_buff is not changed.
1606  *
1607  *	All the pointers pointing into skb header may change and must be
1608  *	reloaded after call to this function.
1609  */
1610 
1611 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1612 		     gfp_t gfp_mask)
1613 {
1614 	int i, osize = skb_end_offset(skb);
1615 	int size = osize + nhead + ntail;
1616 	long off;
1617 	u8 *data;
1618 
1619 	BUG_ON(nhead < 0);
1620 
1621 	BUG_ON(skb_shared(skb));
1622 
1623 	size = SKB_DATA_ALIGN(size);
1624 
1625 	if (skb_pfmemalloc(skb))
1626 		gfp_mask |= __GFP_MEMALLOC;
1627 	data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1628 			       gfp_mask, NUMA_NO_NODE, NULL);
1629 	if (!data)
1630 		goto nodata;
1631 	size = SKB_WITH_OVERHEAD(ksize(data));
1632 
1633 	/* Copy only real data... and, alas, header. This should be
1634 	 * optimized for the cases when header is void.
1635 	 */
1636 	memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1637 
1638 	memcpy((struct skb_shared_info *)(data + size),
1639 	       skb_shinfo(skb),
1640 	       offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1641 
1642 	/*
1643 	 * if shinfo is shared we must drop the old head gracefully, but if it
1644 	 * is not we can just drop the old head and let the existing refcount
1645 	 * be since all we did is relocate the values
1646 	 */
1647 	if (skb_cloned(skb)) {
1648 		if (skb_orphan_frags(skb, gfp_mask))
1649 			goto nofrags;
1650 		if (skb_zcopy(skb))
1651 			refcount_inc(&skb_uarg(skb)->refcnt);
1652 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1653 			skb_frag_ref(skb, i);
1654 
1655 		if (skb_has_frag_list(skb))
1656 			skb_clone_fraglist(skb);
1657 
1658 		skb_release_data(skb);
1659 	} else {
1660 		skb_free_head(skb);
1661 	}
1662 	off = (data + nhead) - skb->head;
1663 
1664 	skb->head     = data;
1665 	skb->head_frag = 0;
1666 	skb->data    += off;
1667 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1668 	skb->end      = size;
1669 	off           = nhead;
1670 #else
1671 	skb->end      = skb->head + size;
1672 #endif
1673 	skb->tail	      += off;
1674 	skb_headers_offset_update(skb, nhead);
1675 	skb->cloned   = 0;
1676 	skb->hdr_len  = 0;
1677 	skb->nohdr    = 0;
1678 	atomic_set(&skb_shinfo(skb)->dataref, 1);
1679 
1680 	skb_metadata_clear(skb);
1681 
1682 	/* It is not generally safe to change skb->truesize.
1683 	 * For the moment, we really care of rx path, or
1684 	 * when skb is orphaned (not attached to a socket).
1685 	 */
1686 	if (!skb->sk || skb->destructor == sock_edemux)
1687 		skb->truesize += size - osize;
1688 
1689 	return 0;
1690 
1691 nofrags:
1692 	kfree(data);
1693 nodata:
1694 	return -ENOMEM;
1695 }
1696 EXPORT_SYMBOL(pskb_expand_head);
1697 
1698 /* Make private copy of skb with writable head and some headroom */
1699 
1700 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1701 {
1702 	struct sk_buff *skb2;
1703 	int delta = headroom - skb_headroom(skb);
1704 
1705 	if (delta <= 0)
1706 		skb2 = pskb_copy(skb, GFP_ATOMIC);
1707 	else {
1708 		skb2 = skb_clone(skb, GFP_ATOMIC);
1709 		if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1710 					     GFP_ATOMIC)) {
1711 			kfree_skb(skb2);
1712 			skb2 = NULL;
1713 		}
1714 	}
1715 	return skb2;
1716 }
1717 EXPORT_SYMBOL(skb_realloc_headroom);
1718 
1719 /**
1720  *	skb_copy_expand	-	copy and expand sk_buff
1721  *	@skb: buffer to copy
1722  *	@newheadroom: new free bytes at head
1723  *	@newtailroom: new free bytes at tail
1724  *	@gfp_mask: allocation priority
1725  *
1726  *	Make a copy of both an &sk_buff and its data and while doing so
1727  *	allocate additional space.
1728  *
1729  *	This is used when the caller wishes to modify the data and needs a
1730  *	private copy of the data to alter as well as more space for new fields.
1731  *	Returns %NULL on failure or the pointer to the buffer
1732  *	on success. The returned buffer has a reference count of 1.
1733  *
1734  *	You must pass %GFP_ATOMIC as the allocation priority if this function
1735  *	is called from an interrupt.
1736  */
1737 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1738 				int newheadroom, int newtailroom,
1739 				gfp_t gfp_mask)
1740 {
1741 	/*
1742 	 *	Allocate the copy buffer
1743 	 */
1744 	struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1745 					gfp_mask, skb_alloc_rx_flag(skb),
1746 					NUMA_NO_NODE);
1747 	int oldheadroom = skb_headroom(skb);
1748 	int head_copy_len, head_copy_off;
1749 
1750 	if (!n)
1751 		return NULL;
1752 
1753 	skb_reserve(n, newheadroom);
1754 
1755 	/* Set the tail pointer and length */
1756 	skb_put(n, skb->len);
1757 
1758 	head_copy_len = oldheadroom;
1759 	head_copy_off = 0;
1760 	if (newheadroom <= head_copy_len)
1761 		head_copy_len = newheadroom;
1762 	else
1763 		head_copy_off = newheadroom - head_copy_len;
1764 
1765 	/* Copy the linear header and data. */
1766 	BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1767 			     skb->len + head_copy_len));
1768 
1769 	skb_copy_header(n, skb);
1770 
1771 	skb_headers_offset_update(n, newheadroom - oldheadroom);
1772 
1773 	return n;
1774 }
1775 EXPORT_SYMBOL(skb_copy_expand);
1776 
1777 /**
1778  *	__skb_pad		-	zero pad the tail of an skb
1779  *	@skb: buffer to pad
1780  *	@pad: space to pad
1781  *	@free_on_error: free buffer on error
1782  *
1783  *	Ensure that a buffer is followed by a padding area that is zero
1784  *	filled. Used by network drivers which may DMA or transfer data
1785  *	beyond the buffer end onto the wire.
1786  *
1787  *	May return error in out of memory cases. The skb is freed on error
1788  *	if @free_on_error is true.
1789  */
1790 
1791 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
1792 {
1793 	int err;
1794 	int ntail;
1795 
1796 	/* If the skbuff is non linear tailroom is always zero.. */
1797 	if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1798 		memset(skb->data+skb->len, 0, pad);
1799 		return 0;
1800 	}
1801 
1802 	ntail = skb->data_len + pad - (skb->end - skb->tail);
1803 	if (likely(skb_cloned(skb) || ntail > 0)) {
1804 		err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1805 		if (unlikely(err))
1806 			goto free_skb;
1807 	}
1808 
1809 	/* FIXME: The use of this function with non-linear skb's really needs
1810 	 * to be audited.
1811 	 */
1812 	err = skb_linearize(skb);
1813 	if (unlikely(err))
1814 		goto free_skb;
1815 
1816 	memset(skb->data + skb->len, 0, pad);
1817 	return 0;
1818 
1819 free_skb:
1820 	if (free_on_error)
1821 		kfree_skb(skb);
1822 	return err;
1823 }
1824 EXPORT_SYMBOL(__skb_pad);
1825 
1826 /**
1827  *	pskb_put - add data to the tail of a potentially fragmented buffer
1828  *	@skb: start of the buffer to use
1829  *	@tail: tail fragment of the buffer to use
1830  *	@len: amount of data to add
1831  *
1832  *	This function extends the used data area of the potentially
1833  *	fragmented buffer. @tail must be the last fragment of @skb -- or
1834  *	@skb itself. If this would exceed the total buffer size the kernel
1835  *	will panic. A pointer to the first byte of the extra data is
1836  *	returned.
1837  */
1838 
1839 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1840 {
1841 	if (tail != skb) {
1842 		skb->data_len += len;
1843 		skb->len += len;
1844 	}
1845 	return skb_put(tail, len);
1846 }
1847 EXPORT_SYMBOL_GPL(pskb_put);
1848 
1849 /**
1850  *	skb_put - add data to a buffer
1851  *	@skb: buffer to use
1852  *	@len: amount of data to add
1853  *
1854  *	This function extends the used data area of the buffer. If this would
1855  *	exceed the total buffer size the kernel will panic. A pointer to the
1856  *	first byte of the extra data is returned.
1857  */
1858 void *skb_put(struct sk_buff *skb, unsigned int len)
1859 {
1860 	void *tmp = skb_tail_pointer(skb);
1861 	SKB_LINEAR_ASSERT(skb);
1862 	skb->tail += len;
1863 	skb->len  += len;
1864 	if (unlikely(skb->tail > skb->end))
1865 		skb_over_panic(skb, len, __builtin_return_address(0));
1866 	return tmp;
1867 }
1868 EXPORT_SYMBOL(skb_put);
1869 
1870 /**
1871  *	skb_push - add data to the start of a buffer
1872  *	@skb: buffer to use
1873  *	@len: amount of data to add
1874  *
1875  *	This function extends the used data area of the buffer at the buffer
1876  *	start. If this would exceed the total buffer headroom the kernel will
1877  *	panic. A pointer to the first byte of the extra data is returned.
1878  */
1879 void *skb_push(struct sk_buff *skb, unsigned int len)
1880 {
1881 	skb->data -= len;
1882 	skb->len  += len;
1883 	if (unlikely(skb->data < skb->head))
1884 		skb_under_panic(skb, len, __builtin_return_address(0));
1885 	return skb->data;
1886 }
1887 EXPORT_SYMBOL(skb_push);
1888 
1889 /**
1890  *	skb_pull - remove data from the start of a buffer
1891  *	@skb: buffer to use
1892  *	@len: amount of data to remove
1893  *
1894  *	This function removes data from the start of a buffer, returning
1895  *	the memory to the headroom. A pointer to the next data in the buffer
1896  *	is returned. Once the data has been pulled future pushes will overwrite
1897  *	the old data.
1898  */
1899 void *skb_pull(struct sk_buff *skb, unsigned int len)
1900 {
1901 	return skb_pull_inline(skb, len);
1902 }
1903 EXPORT_SYMBOL(skb_pull);
1904 
1905 /**
1906  *	skb_trim - remove end from a buffer
1907  *	@skb: buffer to alter
1908  *	@len: new length
1909  *
1910  *	Cut the length of a buffer down by removing data from the tail. If
1911  *	the buffer is already under the length specified it is not modified.
1912  *	The skb must be linear.
1913  */
1914 void skb_trim(struct sk_buff *skb, unsigned int len)
1915 {
1916 	if (skb->len > len)
1917 		__skb_trim(skb, len);
1918 }
1919 EXPORT_SYMBOL(skb_trim);
1920 
1921 /* Trims skb to length len. It can change skb pointers.
1922  */
1923 
1924 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1925 {
1926 	struct sk_buff **fragp;
1927 	struct sk_buff *frag;
1928 	int offset = skb_headlen(skb);
1929 	int nfrags = skb_shinfo(skb)->nr_frags;
1930 	int i;
1931 	int err;
1932 
1933 	if (skb_cloned(skb) &&
1934 	    unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1935 		return err;
1936 
1937 	i = 0;
1938 	if (offset >= len)
1939 		goto drop_pages;
1940 
1941 	for (; i < nfrags; i++) {
1942 		int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1943 
1944 		if (end < len) {
1945 			offset = end;
1946 			continue;
1947 		}
1948 
1949 		skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1950 
1951 drop_pages:
1952 		skb_shinfo(skb)->nr_frags = i;
1953 
1954 		for (; i < nfrags; i++)
1955 			skb_frag_unref(skb, i);
1956 
1957 		if (skb_has_frag_list(skb))
1958 			skb_drop_fraglist(skb);
1959 		goto done;
1960 	}
1961 
1962 	for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1963 	     fragp = &frag->next) {
1964 		int end = offset + frag->len;
1965 
1966 		if (skb_shared(frag)) {
1967 			struct sk_buff *nfrag;
1968 
1969 			nfrag = skb_clone(frag, GFP_ATOMIC);
1970 			if (unlikely(!nfrag))
1971 				return -ENOMEM;
1972 
1973 			nfrag->next = frag->next;
1974 			consume_skb(frag);
1975 			frag = nfrag;
1976 			*fragp = frag;
1977 		}
1978 
1979 		if (end < len) {
1980 			offset = end;
1981 			continue;
1982 		}
1983 
1984 		if (end > len &&
1985 		    unlikely((err = pskb_trim(frag, len - offset))))
1986 			return err;
1987 
1988 		if (frag->next)
1989 			skb_drop_list(&frag->next);
1990 		break;
1991 	}
1992 
1993 done:
1994 	if (len > skb_headlen(skb)) {
1995 		skb->data_len -= skb->len - len;
1996 		skb->len       = len;
1997 	} else {
1998 		skb->len       = len;
1999 		skb->data_len  = 0;
2000 		skb_set_tail_pointer(skb, len);
2001 	}
2002 
2003 	if (!skb->sk || skb->destructor == sock_edemux)
2004 		skb_condense(skb);
2005 	return 0;
2006 }
2007 EXPORT_SYMBOL(___pskb_trim);
2008 
2009 /* Note : use pskb_trim_rcsum() instead of calling this directly
2010  */
2011 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
2012 {
2013 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
2014 		int delta = skb->len - len;
2015 
2016 		skb->csum = csum_block_sub(skb->csum,
2017 					   skb_checksum(skb, len, delta, 0),
2018 					   len);
2019 	}
2020 	return __pskb_trim(skb, len);
2021 }
2022 EXPORT_SYMBOL(pskb_trim_rcsum_slow);
2023 
2024 /**
2025  *	__pskb_pull_tail - advance tail of skb header
2026  *	@skb: buffer to reallocate
2027  *	@delta: number of bytes to advance tail
2028  *
2029  *	The function makes a sense only on a fragmented &sk_buff,
2030  *	it expands header moving its tail forward and copying necessary
2031  *	data from fragmented part.
2032  *
2033  *	&sk_buff MUST have reference count of 1.
2034  *
2035  *	Returns %NULL (and &sk_buff does not change) if pull failed
2036  *	or value of new tail of skb in the case of success.
2037  *
2038  *	All the pointers pointing into skb header may change and must be
2039  *	reloaded after call to this function.
2040  */
2041 
2042 /* Moves tail of skb head forward, copying data from fragmented part,
2043  * when it is necessary.
2044  * 1. It may fail due to malloc failure.
2045  * 2. It may change skb pointers.
2046  *
2047  * It is pretty complicated. Luckily, it is called only in exceptional cases.
2048  */
2049 void *__pskb_pull_tail(struct sk_buff *skb, int delta)
2050 {
2051 	/* If skb has not enough free space at tail, get new one
2052 	 * plus 128 bytes for future expansions. If we have enough
2053 	 * room at tail, reallocate without expansion only if skb is cloned.
2054 	 */
2055 	int i, k, eat = (skb->tail + delta) - skb->end;
2056 
2057 	if (eat > 0 || skb_cloned(skb)) {
2058 		if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
2059 				     GFP_ATOMIC))
2060 			return NULL;
2061 	}
2062 
2063 	BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
2064 			     skb_tail_pointer(skb), delta));
2065 
2066 	/* Optimization: no fragments, no reasons to preestimate
2067 	 * size of pulled pages. Superb.
2068 	 */
2069 	if (!skb_has_frag_list(skb))
2070 		goto pull_pages;
2071 
2072 	/* Estimate size of pulled pages. */
2073 	eat = delta;
2074 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2075 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2076 
2077 		if (size >= eat)
2078 			goto pull_pages;
2079 		eat -= size;
2080 	}
2081 
2082 	/* If we need update frag list, we are in troubles.
2083 	 * Certainly, it is possible to add an offset to skb data,
2084 	 * but taking into account that pulling is expected to
2085 	 * be very rare operation, it is worth to fight against
2086 	 * further bloating skb head and crucify ourselves here instead.
2087 	 * Pure masohism, indeed. 8)8)
2088 	 */
2089 	if (eat) {
2090 		struct sk_buff *list = skb_shinfo(skb)->frag_list;
2091 		struct sk_buff *clone = NULL;
2092 		struct sk_buff *insp = NULL;
2093 
2094 		do {
2095 			if (list->len <= eat) {
2096 				/* Eaten as whole. */
2097 				eat -= list->len;
2098 				list = list->next;
2099 				insp = list;
2100 			} else {
2101 				/* Eaten partially. */
2102 
2103 				if (skb_shared(list)) {
2104 					/* Sucks! We need to fork list. :-( */
2105 					clone = skb_clone(list, GFP_ATOMIC);
2106 					if (!clone)
2107 						return NULL;
2108 					insp = list->next;
2109 					list = clone;
2110 				} else {
2111 					/* This may be pulled without
2112 					 * problems. */
2113 					insp = list;
2114 				}
2115 				if (!pskb_pull(list, eat)) {
2116 					kfree_skb(clone);
2117 					return NULL;
2118 				}
2119 				break;
2120 			}
2121 		} while (eat);
2122 
2123 		/* Free pulled out fragments. */
2124 		while ((list = skb_shinfo(skb)->frag_list) != insp) {
2125 			skb_shinfo(skb)->frag_list = list->next;
2126 			kfree_skb(list);
2127 		}
2128 		/* And insert new clone at head. */
2129 		if (clone) {
2130 			clone->next = list;
2131 			skb_shinfo(skb)->frag_list = clone;
2132 		}
2133 	}
2134 	/* Success! Now we may commit changes to skb data. */
2135 
2136 pull_pages:
2137 	eat = delta;
2138 	k = 0;
2139 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2140 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2141 
2142 		if (size <= eat) {
2143 			skb_frag_unref(skb, i);
2144 			eat -= size;
2145 		} else {
2146 			skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2147 
2148 			*frag = skb_shinfo(skb)->frags[i];
2149 			if (eat) {
2150 				skb_frag_off_add(frag, eat);
2151 				skb_frag_size_sub(frag, eat);
2152 				if (!i)
2153 					goto end;
2154 				eat = 0;
2155 			}
2156 			k++;
2157 		}
2158 	}
2159 	skb_shinfo(skb)->nr_frags = k;
2160 
2161 end:
2162 	skb->tail     += delta;
2163 	skb->data_len -= delta;
2164 
2165 	if (!skb->data_len)
2166 		skb_zcopy_clear(skb, false);
2167 
2168 	return skb_tail_pointer(skb);
2169 }
2170 EXPORT_SYMBOL(__pskb_pull_tail);
2171 
2172 /**
2173  *	skb_copy_bits - copy bits from skb to kernel buffer
2174  *	@skb: source skb
2175  *	@offset: offset in source
2176  *	@to: destination buffer
2177  *	@len: number of bytes to copy
2178  *
2179  *	Copy the specified number of bytes from the source skb to the
2180  *	destination buffer.
2181  *
2182  *	CAUTION ! :
2183  *		If its prototype is ever changed,
2184  *		check arch/{*}/net/{*}.S files,
2185  *		since it is called from BPF assembly code.
2186  */
2187 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2188 {
2189 	int start = skb_headlen(skb);
2190 	struct sk_buff *frag_iter;
2191 	int i, copy;
2192 
2193 	if (offset > (int)skb->len - len)
2194 		goto fault;
2195 
2196 	/* Copy header. */
2197 	if ((copy = start - offset) > 0) {
2198 		if (copy > len)
2199 			copy = len;
2200 		skb_copy_from_linear_data_offset(skb, offset, to, copy);
2201 		if ((len -= copy) == 0)
2202 			return 0;
2203 		offset += copy;
2204 		to     += copy;
2205 	}
2206 
2207 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2208 		int end;
2209 		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2210 
2211 		WARN_ON(start > offset + len);
2212 
2213 		end = start + skb_frag_size(f);
2214 		if ((copy = end - offset) > 0) {
2215 			u32 p_off, p_len, copied;
2216 			struct page *p;
2217 			u8 *vaddr;
2218 
2219 			if (copy > len)
2220 				copy = len;
2221 
2222 			skb_frag_foreach_page(f,
2223 					      skb_frag_off(f) + offset - start,
2224 					      copy, p, p_off, p_len, copied) {
2225 				vaddr = kmap_atomic(p);
2226 				memcpy(to + copied, vaddr + p_off, p_len);
2227 				kunmap_atomic(vaddr);
2228 			}
2229 
2230 			if ((len -= copy) == 0)
2231 				return 0;
2232 			offset += copy;
2233 			to     += copy;
2234 		}
2235 		start = end;
2236 	}
2237 
2238 	skb_walk_frags(skb, frag_iter) {
2239 		int end;
2240 
2241 		WARN_ON(start > offset + len);
2242 
2243 		end = start + frag_iter->len;
2244 		if ((copy = end - offset) > 0) {
2245 			if (copy > len)
2246 				copy = len;
2247 			if (skb_copy_bits(frag_iter, offset - start, to, copy))
2248 				goto fault;
2249 			if ((len -= copy) == 0)
2250 				return 0;
2251 			offset += copy;
2252 			to     += copy;
2253 		}
2254 		start = end;
2255 	}
2256 
2257 	if (!len)
2258 		return 0;
2259 
2260 fault:
2261 	return -EFAULT;
2262 }
2263 EXPORT_SYMBOL(skb_copy_bits);
2264 
2265 /*
2266  * Callback from splice_to_pipe(), if we need to release some pages
2267  * at the end of the spd in case we error'ed out in filling the pipe.
2268  */
2269 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
2270 {
2271 	put_page(spd->pages[i]);
2272 }
2273 
2274 static struct page *linear_to_page(struct page *page, unsigned int *len,
2275 				   unsigned int *offset,
2276 				   struct sock *sk)
2277 {
2278 	struct page_frag *pfrag = sk_page_frag(sk);
2279 
2280 	if (!sk_page_frag_refill(sk, pfrag))
2281 		return NULL;
2282 
2283 	*len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
2284 
2285 	memcpy(page_address(pfrag->page) + pfrag->offset,
2286 	       page_address(page) + *offset, *len);
2287 	*offset = pfrag->offset;
2288 	pfrag->offset += *len;
2289 
2290 	return pfrag->page;
2291 }
2292 
2293 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
2294 			     struct page *page,
2295 			     unsigned int offset)
2296 {
2297 	return	spd->nr_pages &&
2298 		spd->pages[spd->nr_pages - 1] == page &&
2299 		(spd->partial[spd->nr_pages - 1].offset +
2300 		 spd->partial[spd->nr_pages - 1].len == offset);
2301 }
2302 
2303 /*
2304  * Fill page/offset/length into spd, if it can hold more pages.
2305  */
2306 static bool spd_fill_page(struct splice_pipe_desc *spd,
2307 			  struct pipe_inode_info *pipe, struct page *page,
2308 			  unsigned int *len, unsigned int offset,
2309 			  bool linear,
2310 			  struct sock *sk)
2311 {
2312 	if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
2313 		return true;
2314 
2315 	if (linear) {
2316 		page = linear_to_page(page, len, &offset, sk);
2317 		if (!page)
2318 			return true;
2319 	}
2320 	if (spd_can_coalesce(spd, page, offset)) {
2321 		spd->partial[spd->nr_pages - 1].len += *len;
2322 		return false;
2323 	}
2324 	get_page(page);
2325 	spd->pages[spd->nr_pages] = page;
2326 	spd->partial[spd->nr_pages].len = *len;
2327 	spd->partial[spd->nr_pages].offset = offset;
2328 	spd->nr_pages++;
2329 
2330 	return false;
2331 }
2332 
2333 static bool __splice_segment(struct page *page, unsigned int poff,
2334 			     unsigned int plen, unsigned int *off,
2335 			     unsigned int *len,
2336 			     struct splice_pipe_desc *spd, bool linear,
2337 			     struct sock *sk,
2338 			     struct pipe_inode_info *pipe)
2339 {
2340 	if (!*len)
2341 		return true;
2342 
2343 	/* skip this segment if already processed */
2344 	if (*off >= plen) {
2345 		*off -= plen;
2346 		return false;
2347 	}
2348 
2349 	/* ignore any bits we already processed */
2350 	poff += *off;
2351 	plen -= *off;
2352 	*off = 0;
2353 
2354 	do {
2355 		unsigned int flen = min(*len, plen);
2356 
2357 		if (spd_fill_page(spd, pipe, page, &flen, poff,
2358 				  linear, sk))
2359 			return true;
2360 		poff += flen;
2361 		plen -= flen;
2362 		*len -= flen;
2363 	} while (*len && plen);
2364 
2365 	return false;
2366 }
2367 
2368 /*
2369  * Map linear and fragment data from the skb to spd. It reports true if the
2370  * pipe is full or if we already spliced the requested length.
2371  */
2372 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
2373 			      unsigned int *offset, unsigned int *len,
2374 			      struct splice_pipe_desc *spd, struct sock *sk)
2375 {
2376 	int seg;
2377 	struct sk_buff *iter;
2378 
2379 	/* map the linear part :
2380 	 * If skb->head_frag is set, this 'linear' part is backed by a
2381 	 * fragment, and if the head is not shared with any clones then
2382 	 * we can avoid a copy since we own the head portion of this page.
2383 	 */
2384 	if (__splice_segment(virt_to_page(skb->data),
2385 			     (unsigned long) skb->data & (PAGE_SIZE - 1),
2386 			     skb_headlen(skb),
2387 			     offset, len, spd,
2388 			     skb_head_is_locked(skb),
2389 			     sk, pipe))
2390 		return true;
2391 
2392 	/*
2393 	 * then map the fragments
2394 	 */
2395 	for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
2396 		const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2397 
2398 		if (__splice_segment(skb_frag_page(f),
2399 				     skb_frag_off(f), skb_frag_size(f),
2400 				     offset, len, spd, false, sk, pipe))
2401 			return true;
2402 	}
2403 
2404 	skb_walk_frags(skb, iter) {
2405 		if (*offset >= iter->len) {
2406 			*offset -= iter->len;
2407 			continue;
2408 		}
2409 		/* __skb_splice_bits() only fails if the output has no room
2410 		 * left, so no point in going over the frag_list for the error
2411 		 * case.
2412 		 */
2413 		if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
2414 			return true;
2415 	}
2416 
2417 	return false;
2418 }
2419 
2420 /*
2421  * Map data from the skb to a pipe. Should handle both the linear part,
2422  * the fragments, and the frag list.
2423  */
2424 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
2425 		    struct pipe_inode_info *pipe, unsigned int tlen,
2426 		    unsigned int flags)
2427 {
2428 	struct partial_page partial[MAX_SKB_FRAGS];
2429 	struct page *pages[MAX_SKB_FRAGS];
2430 	struct splice_pipe_desc spd = {
2431 		.pages = pages,
2432 		.partial = partial,
2433 		.nr_pages_max = MAX_SKB_FRAGS,
2434 		.ops = &nosteal_pipe_buf_ops,
2435 		.spd_release = sock_spd_release,
2436 	};
2437 	int ret = 0;
2438 
2439 	__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
2440 
2441 	if (spd.nr_pages)
2442 		ret = splice_to_pipe(pipe, &spd);
2443 
2444 	return ret;
2445 }
2446 EXPORT_SYMBOL_GPL(skb_splice_bits);
2447 
2448 /* Send skb data on a socket. Socket must be locked. */
2449 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2450 			 int len)
2451 {
2452 	unsigned int orig_len = len;
2453 	struct sk_buff *head = skb;
2454 	unsigned short fragidx;
2455 	int slen, ret;
2456 
2457 do_frag_list:
2458 
2459 	/* Deal with head data */
2460 	while (offset < skb_headlen(skb) && len) {
2461 		struct kvec kv;
2462 		struct msghdr msg;
2463 
2464 		slen = min_t(int, len, skb_headlen(skb) - offset);
2465 		kv.iov_base = skb->data + offset;
2466 		kv.iov_len = slen;
2467 		memset(&msg, 0, sizeof(msg));
2468 		msg.msg_flags = MSG_DONTWAIT;
2469 
2470 		ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
2471 		if (ret <= 0)
2472 			goto error;
2473 
2474 		offset += ret;
2475 		len -= ret;
2476 	}
2477 
2478 	/* All the data was skb head? */
2479 	if (!len)
2480 		goto out;
2481 
2482 	/* Make offset relative to start of frags */
2483 	offset -= skb_headlen(skb);
2484 
2485 	/* Find where we are in frag list */
2486 	for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2487 		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
2488 
2489 		if (offset < skb_frag_size(frag))
2490 			break;
2491 
2492 		offset -= skb_frag_size(frag);
2493 	}
2494 
2495 	for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2496 		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
2497 
2498 		slen = min_t(size_t, len, skb_frag_size(frag) - offset);
2499 
2500 		while (slen) {
2501 			ret = kernel_sendpage_locked(sk, skb_frag_page(frag),
2502 						     skb_frag_off(frag) + offset,
2503 						     slen, MSG_DONTWAIT);
2504 			if (ret <= 0)
2505 				goto error;
2506 
2507 			len -= ret;
2508 			offset += ret;
2509 			slen -= ret;
2510 		}
2511 
2512 		offset = 0;
2513 	}
2514 
2515 	if (len) {
2516 		/* Process any frag lists */
2517 
2518 		if (skb == head) {
2519 			if (skb_has_frag_list(skb)) {
2520 				skb = skb_shinfo(skb)->frag_list;
2521 				goto do_frag_list;
2522 			}
2523 		} else if (skb->next) {
2524 			skb = skb->next;
2525 			goto do_frag_list;
2526 		}
2527 	}
2528 
2529 out:
2530 	return orig_len - len;
2531 
2532 error:
2533 	return orig_len == len ? ret : orig_len - len;
2534 }
2535 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2536 
2537 /**
2538  *	skb_store_bits - store bits from kernel buffer to skb
2539  *	@skb: destination buffer
2540  *	@offset: offset in destination
2541  *	@from: source buffer
2542  *	@len: number of bytes to copy
2543  *
2544  *	Copy the specified number of bytes from the source buffer to the
2545  *	destination skb.  This function handles all the messy bits of
2546  *	traversing fragment lists and such.
2547  */
2548 
2549 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
2550 {
2551 	int start = skb_headlen(skb);
2552 	struct sk_buff *frag_iter;
2553 	int i, copy;
2554 
2555 	if (offset > (int)skb->len - len)
2556 		goto fault;
2557 
2558 	if ((copy = start - offset) > 0) {
2559 		if (copy > len)
2560 			copy = len;
2561 		skb_copy_to_linear_data_offset(skb, offset, from, copy);
2562 		if ((len -= copy) == 0)
2563 			return 0;
2564 		offset += copy;
2565 		from += copy;
2566 	}
2567 
2568 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2569 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2570 		int end;
2571 
2572 		WARN_ON(start > offset + len);
2573 
2574 		end = start + skb_frag_size(frag);
2575 		if ((copy = end - offset) > 0) {
2576 			u32 p_off, p_len, copied;
2577 			struct page *p;
2578 			u8 *vaddr;
2579 
2580 			if (copy > len)
2581 				copy = len;
2582 
2583 			skb_frag_foreach_page(frag,
2584 					      skb_frag_off(frag) + offset - start,
2585 					      copy, p, p_off, p_len, copied) {
2586 				vaddr = kmap_atomic(p);
2587 				memcpy(vaddr + p_off, from + copied, p_len);
2588 				kunmap_atomic(vaddr);
2589 			}
2590 
2591 			if ((len -= copy) == 0)
2592 				return 0;
2593 			offset += copy;
2594 			from += copy;
2595 		}
2596 		start = end;
2597 	}
2598 
2599 	skb_walk_frags(skb, frag_iter) {
2600 		int end;
2601 
2602 		WARN_ON(start > offset + len);
2603 
2604 		end = start + frag_iter->len;
2605 		if ((copy = end - offset) > 0) {
2606 			if (copy > len)
2607 				copy = len;
2608 			if (skb_store_bits(frag_iter, offset - start,
2609 					   from, copy))
2610 				goto fault;
2611 			if ((len -= copy) == 0)
2612 				return 0;
2613 			offset += copy;
2614 			from += copy;
2615 		}
2616 		start = end;
2617 	}
2618 	if (!len)
2619 		return 0;
2620 
2621 fault:
2622 	return -EFAULT;
2623 }
2624 EXPORT_SYMBOL(skb_store_bits);
2625 
2626 /* Checksum skb data. */
2627 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2628 		      __wsum csum, const struct skb_checksum_ops *ops)
2629 {
2630 	int start = skb_headlen(skb);
2631 	int i, copy = start - offset;
2632 	struct sk_buff *frag_iter;
2633 	int pos = 0;
2634 
2635 	/* Checksum header. */
2636 	if (copy > 0) {
2637 		if (copy > len)
2638 			copy = len;
2639 		csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
2640 				       skb->data + offset, copy, csum);
2641 		if ((len -= copy) == 0)
2642 			return csum;
2643 		offset += copy;
2644 		pos	= copy;
2645 	}
2646 
2647 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2648 		int end;
2649 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2650 
2651 		WARN_ON(start > offset + len);
2652 
2653 		end = start + skb_frag_size(frag);
2654 		if ((copy = end - offset) > 0) {
2655 			u32 p_off, p_len, copied;
2656 			struct page *p;
2657 			__wsum csum2;
2658 			u8 *vaddr;
2659 
2660 			if (copy > len)
2661 				copy = len;
2662 
2663 			skb_frag_foreach_page(frag,
2664 					      skb_frag_off(frag) + offset - start,
2665 					      copy, p, p_off, p_len, copied) {
2666 				vaddr = kmap_atomic(p);
2667 				csum2 = INDIRECT_CALL_1(ops->update,
2668 							csum_partial_ext,
2669 							vaddr + p_off, p_len, 0);
2670 				kunmap_atomic(vaddr);
2671 				csum = INDIRECT_CALL_1(ops->combine,
2672 						       csum_block_add_ext, csum,
2673 						       csum2, pos, p_len);
2674 				pos += p_len;
2675 			}
2676 
2677 			if (!(len -= copy))
2678 				return csum;
2679 			offset += copy;
2680 		}
2681 		start = end;
2682 	}
2683 
2684 	skb_walk_frags(skb, frag_iter) {
2685 		int end;
2686 
2687 		WARN_ON(start > offset + len);
2688 
2689 		end = start + frag_iter->len;
2690 		if ((copy = end - offset) > 0) {
2691 			__wsum csum2;
2692 			if (copy > len)
2693 				copy = len;
2694 			csum2 = __skb_checksum(frag_iter, offset - start,
2695 					       copy, 0, ops);
2696 			csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
2697 					       csum, csum2, pos, copy);
2698 			if ((len -= copy) == 0)
2699 				return csum;
2700 			offset += copy;
2701 			pos    += copy;
2702 		}
2703 		start = end;
2704 	}
2705 	BUG_ON(len);
2706 
2707 	return csum;
2708 }
2709 EXPORT_SYMBOL(__skb_checksum);
2710 
2711 __wsum skb_checksum(const struct sk_buff *skb, int offset,
2712 		    int len, __wsum csum)
2713 {
2714 	const struct skb_checksum_ops ops = {
2715 		.update  = csum_partial_ext,
2716 		.combine = csum_block_add_ext,
2717 	};
2718 
2719 	return __skb_checksum(skb, offset, len, csum, &ops);
2720 }
2721 EXPORT_SYMBOL(skb_checksum);
2722 
2723 /* Both of above in one bottle. */
2724 
2725 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2726 				    u8 *to, int len, __wsum csum)
2727 {
2728 	int start = skb_headlen(skb);
2729 	int i, copy = start - offset;
2730 	struct sk_buff *frag_iter;
2731 	int pos = 0;
2732 
2733 	/* Copy header. */
2734 	if (copy > 0) {
2735 		if (copy > len)
2736 			copy = len;
2737 		csum = csum_partial_copy_nocheck(skb->data + offset, to,
2738 						 copy, csum);
2739 		if ((len -= copy) == 0)
2740 			return csum;
2741 		offset += copy;
2742 		to     += copy;
2743 		pos	= copy;
2744 	}
2745 
2746 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2747 		int end;
2748 
2749 		WARN_ON(start > offset + len);
2750 
2751 		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2752 		if ((copy = end - offset) > 0) {
2753 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2754 			u32 p_off, p_len, copied;
2755 			struct page *p;
2756 			__wsum csum2;
2757 			u8 *vaddr;
2758 
2759 			if (copy > len)
2760 				copy = len;
2761 
2762 			skb_frag_foreach_page(frag,
2763 					      skb_frag_off(frag) + offset - start,
2764 					      copy, p, p_off, p_len, copied) {
2765 				vaddr = kmap_atomic(p);
2766 				csum2 = csum_partial_copy_nocheck(vaddr + p_off,
2767 								  to + copied,
2768 								  p_len, 0);
2769 				kunmap_atomic(vaddr);
2770 				csum = csum_block_add(csum, csum2, pos);
2771 				pos += p_len;
2772 			}
2773 
2774 			if (!(len -= copy))
2775 				return csum;
2776 			offset += copy;
2777 			to     += copy;
2778 		}
2779 		start = end;
2780 	}
2781 
2782 	skb_walk_frags(skb, frag_iter) {
2783 		__wsum csum2;
2784 		int end;
2785 
2786 		WARN_ON(start > offset + len);
2787 
2788 		end = start + frag_iter->len;
2789 		if ((copy = end - offset) > 0) {
2790 			if (copy > len)
2791 				copy = len;
2792 			csum2 = skb_copy_and_csum_bits(frag_iter,
2793 						       offset - start,
2794 						       to, copy, 0);
2795 			csum = csum_block_add(csum, csum2, pos);
2796 			if ((len -= copy) == 0)
2797 				return csum;
2798 			offset += copy;
2799 			to     += copy;
2800 			pos    += copy;
2801 		}
2802 		start = end;
2803 	}
2804 	BUG_ON(len);
2805 	return csum;
2806 }
2807 EXPORT_SYMBOL(skb_copy_and_csum_bits);
2808 
2809 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
2810 {
2811 	__sum16 sum;
2812 
2813 	sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
2814 	/* See comments in __skb_checksum_complete(). */
2815 	if (likely(!sum)) {
2816 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
2817 		    !skb->csum_complete_sw)
2818 			netdev_rx_csum_fault(skb->dev, skb);
2819 	}
2820 	if (!skb_shared(skb))
2821 		skb->csum_valid = !sum;
2822 	return sum;
2823 }
2824 EXPORT_SYMBOL(__skb_checksum_complete_head);
2825 
2826 /* This function assumes skb->csum already holds pseudo header's checksum,
2827  * which has been changed from the hardware checksum, for example, by
2828  * __skb_checksum_validate_complete(). And, the original skb->csum must
2829  * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
2830  *
2831  * It returns non-zero if the recomputed checksum is still invalid, otherwise
2832  * zero. The new checksum is stored back into skb->csum unless the skb is
2833  * shared.
2834  */
2835 __sum16 __skb_checksum_complete(struct sk_buff *skb)
2836 {
2837 	__wsum csum;
2838 	__sum16 sum;
2839 
2840 	csum = skb_checksum(skb, 0, skb->len, 0);
2841 
2842 	sum = csum_fold(csum_add(skb->csum, csum));
2843 	/* This check is inverted, because we already knew the hardware
2844 	 * checksum is invalid before calling this function. So, if the
2845 	 * re-computed checksum is valid instead, then we have a mismatch
2846 	 * between the original skb->csum and skb_checksum(). This means either
2847 	 * the original hardware checksum is incorrect or we screw up skb->csum
2848 	 * when moving skb->data around.
2849 	 */
2850 	if (likely(!sum)) {
2851 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
2852 		    !skb->csum_complete_sw)
2853 			netdev_rx_csum_fault(skb->dev, skb);
2854 	}
2855 
2856 	if (!skb_shared(skb)) {
2857 		/* Save full packet checksum */
2858 		skb->csum = csum;
2859 		skb->ip_summed = CHECKSUM_COMPLETE;
2860 		skb->csum_complete_sw = 1;
2861 		skb->csum_valid = !sum;
2862 	}
2863 
2864 	return sum;
2865 }
2866 EXPORT_SYMBOL(__skb_checksum_complete);
2867 
2868 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
2869 {
2870 	net_warn_ratelimited(
2871 		"%s: attempt to compute crc32c without libcrc32c.ko\n",
2872 		__func__);
2873 	return 0;
2874 }
2875 
2876 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
2877 				       int offset, int len)
2878 {
2879 	net_warn_ratelimited(
2880 		"%s: attempt to compute crc32c without libcrc32c.ko\n",
2881 		__func__);
2882 	return 0;
2883 }
2884 
2885 static const struct skb_checksum_ops default_crc32c_ops = {
2886 	.update  = warn_crc32c_csum_update,
2887 	.combine = warn_crc32c_csum_combine,
2888 };
2889 
2890 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
2891 	&default_crc32c_ops;
2892 EXPORT_SYMBOL(crc32c_csum_stub);
2893 
2894  /**
2895  *	skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
2896  *	@from: source buffer
2897  *
2898  *	Calculates the amount of linear headroom needed in the 'to' skb passed
2899  *	into skb_zerocopy().
2900  */
2901 unsigned int
2902 skb_zerocopy_headlen(const struct sk_buff *from)
2903 {
2904 	unsigned int hlen = 0;
2905 
2906 	if (!from->head_frag ||
2907 	    skb_headlen(from) < L1_CACHE_BYTES ||
2908 	    skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
2909 		hlen = skb_headlen(from);
2910 
2911 	if (skb_has_frag_list(from))
2912 		hlen = from->len;
2913 
2914 	return hlen;
2915 }
2916 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
2917 
2918 /**
2919  *	skb_zerocopy - Zero copy skb to skb
2920  *	@to: destination buffer
2921  *	@from: source buffer
2922  *	@len: number of bytes to copy from source buffer
2923  *	@hlen: size of linear headroom in destination buffer
2924  *
2925  *	Copies up to `len` bytes from `from` to `to` by creating references
2926  *	to the frags in the source buffer.
2927  *
2928  *	The `hlen` as calculated by skb_zerocopy_headlen() specifies the
2929  *	headroom in the `to` buffer.
2930  *
2931  *	Return value:
2932  *	0: everything is OK
2933  *	-ENOMEM: couldn't orphan frags of @from due to lack of memory
2934  *	-EFAULT: skb_copy_bits() found some problem with skb geometry
2935  */
2936 int
2937 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
2938 {
2939 	int i, j = 0;
2940 	int plen = 0; /* length of skb->head fragment */
2941 	int ret;
2942 	struct page *page;
2943 	unsigned int offset;
2944 
2945 	BUG_ON(!from->head_frag && !hlen);
2946 
2947 	/* dont bother with small payloads */
2948 	if (len <= skb_tailroom(to))
2949 		return skb_copy_bits(from, 0, skb_put(to, len), len);
2950 
2951 	if (hlen) {
2952 		ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
2953 		if (unlikely(ret))
2954 			return ret;
2955 		len -= hlen;
2956 	} else {
2957 		plen = min_t(int, skb_headlen(from), len);
2958 		if (plen) {
2959 			page = virt_to_head_page(from->head);
2960 			offset = from->data - (unsigned char *)page_address(page);
2961 			__skb_fill_page_desc(to, 0, page, offset, plen);
2962 			get_page(page);
2963 			j = 1;
2964 			len -= plen;
2965 		}
2966 	}
2967 
2968 	to->truesize += len + plen;
2969 	to->len += len + plen;
2970 	to->data_len += len + plen;
2971 
2972 	if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
2973 		skb_tx_error(from);
2974 		return -ENOMEM;
2975 	}
2976 	skb_zerocopy_clone(to, from, GFP_ATOMIC);
2977 
2978 	for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
2979 		int size;
2980 
2981 		if (!len)
2982 			break;
2983 		skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2984 		size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
2985 					len);
2986 		skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
2987 		len -= size;
2988 		skb_frag_ref(to, j);
2989 		j++;
2990 	}
2991 	skb_shinfo(to)->nr_frags = j;
2992 
2993 	return 0;
2994 }
2995 EXPORT_SYMBOL_GPL(skb_zerocopy);
2996 
2997 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2998 {
2999 	__wsum csum;
3000 	long csstart;
3001 
3002 	if (skb->ip_summed == CHECKSUM_PARTIAL)
3003 		csstart = skb_checksum_start_offset(skb);
3004 	else
3005 		csstart = skb_headlen(skb);
3006 
3007 	BUG_ON(csstart > skb_headlen(skb));
3008 
3009 	skb_copy_from_linear_data(skb, to, csstart);
3010 
3011 	csum = 0;
3012 	if (csstart != skb->len)
3013 		csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
3014 					      skb->len - csstart, 0);
3015 
3016 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
3017 		long csstuff = csstart + skb->csum_offset;
3018 
3019 		*((__sum16 *)(to + csstuff)) = csum_fold(csum);
3020 	}
3021 }
3022 EXPORT_SYMBOL(skb_copy_and_csum_dev);
3023 
3024 /**
3025  *	skb_dequeue - remove from the head of the queue
3026  *	@list: list to dequeue from
3027  *
3028  *	Remove the head of the list. The list lock is taken so the function
3029  *	may be used safely with other locking list functions. The head item is
3030  *	returned or %NULL if the list is empty.
3031  */
3032 
3033 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
3034 {
3035 	unsigned long flags;
3036 	struct sk_buff *result;
3037 
3038 	spin_lock_irqsave(&list->lock, flags);
3039 	result = __skb_dequeue(list);
3040 	spin_unlock_irqrestore(&list->lock, flags);
3041 	return result;
3042 }
3043 EXPORT_SYMBOL(skb_dequeue);
3044 
3045 /**
3046  *	skb_dequeue_tail - remove from the tail of the queue
3047  *	@list: list to dequeue from
3048  *
3049  *	Remove the tail of the list. The list lock is taken so the function
3050  *	may be used safely with other locking list functions. The tail item is
3051  *	returned or %NULL if the list is empty.
3052  */
3053 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
3054 {
3055 	unsigned long flags;
3056 	struct sk_buff *result;
3057 
3058 	spin_lock_irqsave(&list->lock, flags);
3059 	result = __skb_dequeue_tail(list);
3060 	spin_unlock_irqrestore(&list->lock, flags);
3061 	return result;
3062 }
3063 EXPORT_SYMBOL(skb_dequeue_tail);
3064 
3065 /**
3066  *	skb_queue_purge - empty a list
3067  *	@list: list to empty
3068  *
3069  *	Delete all buffers on an &sk_buff list. Each buffer is removed from
3070  *	the list and one reference dropped. This function takes the list
3071  *	lock and is atomic with respect to other list locking functions.
3072  */
3073 void skb_queue_purge(struct sk_buff_head *list)
3074 {
3075 	struct sk_buff *skb;
3076 	while ((skb = skb_dequeue(list)) != NULL)
3077 		kfree_skb(skb);
3078 }
3079 EXPORT_SYMBOL(skb_queue_purge);
3080 
3081 /**
3082  *	skb_rbtree_purge - empty a skb rbtree
3083  *	@root: root of the rbtree to empty
3084  *	Return value: the sum of truesizes of all purged skbs.
3085  *
3086  *	Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
3087  *	the list and one reference dropped. This function does not take
3088  *	any lock. Synchronization should be handled by the caller (e.g., TCP
3089  *	out-of-order queue is protected by the socket lock).
3090  */
3091 unsigned int skb_rbtree_purge(struct rb_root *root)
3092 {
3093 	struct rb_node *p = rb_first(root);
3094 	unsigned int sum = 0;
3095 
3096 	while (p) {
3097 		struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
3098 
3099 		p = rb_next(p);
3100 		rb_erase(&skb->rbnode, root);
3101 		sum += skb->truesize;
3102 		kfree_skb(skb);
3103 	}
3104 	return sum;
3105 }
3106 
3107 /**
3108  *	skb_queue_head - queue a buffer at the list head
3109  *	@list: list to use
3110  *	@newsk: buffer to queue
3111  *
3112  *	Queue a buffer at the start of the list. This function takes the
3113  *	list lock and can be used safely with other locking &sk_buff functions
3114  *	safely.
3115  *
3116  *	A buffer cannot be placed on two lists at the same time.
3117  */
3118 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
3119 {
3120 	unsigned long flags;
3121 
3122 	spin_lock_irqsave(&list->lock, flags);
3123 	__skb_queue_head(list, newsk);
3124 	spin_unlock_irqrestore(&list->lock, flags);
3125 }
3126 EXPORT_SYMBOL(skb_queue_head);
3127 
3128 /**
3129  *	skb_queue_tail - queue a buffer at the list tail
3130  *	@list: list to use
3131  *	@newsk: buffer to queue
3132  *
3133  *	Queue a buffer at the tail of the list. This function takes the
3134  *	list lock and can be used safely with other locking &sk_buff functions
3135  *	safely.
3136  *
3137  *	A buffer cannot be placed on two lists at the same time.
3138  */
3139 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
3140 {
3141 	unsigned long flags;
3142 
3143 	spin_lock_irqsave(&list->lock, flags);
3144 	__skb_queue_tail(list, newsk);
3145 	spin_unlock_irqrestore(&list->lock, flags);
3146 }
3147 EXPORT_SYMBOL(skb_queue_tail);
3148 
3149 /**
3150  *	skb_unlink	-	remove a buffer from a list
3151  *	@skb: buffer to remove
3152  *	@list: list to use
3153  *
3154  *	Remove a packet from a list. The list locks are taken and this
3155  *	function is atomic with respect to other list locked calls
3156  *
3157  *	You must know what list the SKB is on.
3158  */
3159 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
3160 {
3161 	unsigned long flags;
3162 
3163 	spin_lock_irqsave(&list->lock, flags);
3164 	__skb_unlink(skb, list);
3165 	spin_unlock_irqrestore(&list->lock, flags);
3166 }
3167 EXPORT_SYMBOL(skb_unlink);
3168 
3169 /**
3170  *	skb_append	-	append a buffer
3171  *	@old: buffer to insert after
3172  *	@newsk: buffer to insert
3173  *	@list: list to use
3174  *
3175  *	Place a packet after a given packet in a list. The list locks are taken
3176  *	and this function is atomic with respect to other list locked calls.
3177  *	A buffer cannot be placed on two lists at the same time.
3178  */
3179 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
3180 {
3181 	unsigned long flags;
3182 
3183 	spin_lock_irqsave(&list->lock, flags);
3184 	__skb_queue_after(list, old, newsk);
3185 	spin_unlock_irqrestore(&list->lock, flags);
3186 }
3187 EXPORT_SYMBOL(skb_append);
3188 
3189 static inline void skb_split_inside_header(struct sk_buff *skb,
3190 					   struct sk_buff* skb1,
3191 					   const u32 len, const int pos)
3192 {
3193 	int i;
3194 
3195 	skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
3196 					 pos - len);
3197 	/* And move data appendix as is. */
3198 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
3199 		skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
3200 
3201 	skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
3202 	skb_shinfo(skb)->nr_frags  = 0;
3203 	skb1->data_len		   = skb->data_len;
3204 	skb1->len		   += skb1->data_len;
3205 	skb->data_len		   = 0;
3206 	skb->len		   = len;
3207 	skb_set_tail_pointer(skb, len);
3208 }
3209 
3210 static inline void skb_split_no_header(struct sk_buff *skb,
3211 				       struct sk_buff* skb1,
3212 				       const u32 len, int pos)
3213 {
3214 	int i, k = 0;
3215 	const int nfrags = skb_shinfo(skb)->nr_frags;
3216 
3217 	skb_shinfo(skb)->nr_frags = 0;
3218 	skb1->len		  = skb1->data_len = skb->len - len;
3219 	skb->len		  = len;
3220 	skb->data_len		  = len - pos;
3221 
3222 	for (i = 0; i < nfrags; i++) {
3223 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
3224 
3225 		if (pos + size > len) {
3226 			skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3227 
3228 			if (pos < len) {
3229 				/* Split frag.
3230 				 * We have two variants in this case:
3231 				 * 1. Move all the frag to the second
3232 				 *    part, if it is possible. F.e.
3233 				 *    this approach is mandatory for TUX,
3234 				 *    where splitting is expensive.
3235 				 * 2. Split is accurately. We make this.
3236 				 */
3237 				skb_frag_ref(skb, i);
3238 				skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
3239 				skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3240 				skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3241 				skb_shinfo(skb)->nr_frags++;
3242 			}
3243 			k++;
3244 		} else
3245 			skb_shinfo(skb)->nr_frags++;
3246 		pos += size;
3247 	}
3248 	skb_shinfo(skb1)->nr_frags = k;
3249 }
3250 
3251 /**
3252  * skb_split - Split fragmented skb to two parts at length len.
3253  * @skb: the buffer to split
3254  * @skb1: the buffer to receive the second part
3255  * @len: new length for skb
3256  */
3257 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3258 {
3259 	int pos = skb_headlen(skb);
3260 
3261 	skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
3262 				      SKBTX_SHARED_FRAG;
3263 	skb_zerocopy_clone(skb1, skb, 0);
3264 	if (len < pos)	/* Split line is inside header. */
3265 		skb_split_inside_header(skb, skb1, len, pos);
3266 	else		/* Second chunk has no header, nothing to copy. */
3267 		skb_split_no_header(skb, skb1, len, pos);
3268 }
3269 EXPORT_SYMBOL(skb_split);
3270 
3271 /* Shifting from/to a cloned skb is a no-go.
3272  *
3273  * Caller cannot keep skb_shinfo related pointers past calling here!
3274  */
3275 static int skb_prepare_for_shift(struct sk_buff *skb)
3276 {
3277 	return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3278 }
3279 
3280 /**
3281  * skb_shift - Shifts paged data partially from skb to another
3282  * @tgt: buffer into which tail data gets added
3283  * @skb: buffer from which the paged data comes from
3284  * @shiftlen: shift up to this many bytes
3285  *
3286  * Attempts to shift up to shiftlen worth of bytes, which may be less than
3287  * the length of the skb, from skb to tgt. Returns number bytes shifted.
3288  * It's up to caller to free skb if everything was shifted.
3289  *
3290  * If @tgt runs out of frags, the whole operation is aborted.
3291  *
3292  * Skb cannot include anything else but paged data while tgt is allowed
3293  * to have non-paged data as well.
3294  *
3295  * TODO: full sized shift could be optimized but that would need
3296  * specialized skb free'er to handle frags without up-to-date nr_frags.
3297  */
3298 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
3299 {
3300 	int from, to, merge, todo;
3301 	skb_frag_t *fragfrom, *fragto;
3302 
3303 	BUG_ON(shiftlen > skb->len);
3304 
3305 	if (skb_headlen(skb))
3306 		return 0;
3307 	if (skb_zcopy(tgt) || skb_zcopy(skb))
3308 		return 0;
3309 
3310 	todo = shiftlen;
3311 	from = 0;
3312 	to = skb_shinfo(tgt)->nr_frags;
3313 	fragfrom = &skb_shinfo(skb)->frags[from];
3314 
3315 	/* Actual merge is delayed until the point when we know we can
3316 	 * commit all, so that we don't have to undo partial changes
3317 	 */
3318 	if (!to ||
3319 	    !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3320 			      skb_frag_off(fragfrom))) {
3321 		merge = -1;
3322 	} else {
3323 		merge = to - 1;
3324 
3325 		todo -= skb_frag_size(fragfrom);
3326 		if (todo < 0) {
3327 			if (skb_prepare_for_shift(skb) ||
3328 			    skb_prepare_for_shift(tgt))
3329 				return 0;
3330 
3331 			/* All previous frag pointers might be stale! */
3332 			fragfrom = &skb_shinfo(skb)->frags[from];
3333 			fragto = &skb_shinfo(tgt)->frags[merge];
3334 
3335 			skb_frag_size_add(fragto, shiftlen);
3336 			skb_frag_size_sub(fragfrom, shiftlen);
3337 			skb_frag_off_add(fragfrom, shiftlen);
3338 
3339 			goto onlymerged;
3340 		}
3341 
3342 		from++;
3343 	}
3344 
3345 	/* Skip full, not-fitting skb to avoid expensive operations */
3346 	if ((shiftlen == skb->len) &&
3347 	    (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
3348 		return 0;
3349 
3350 	if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
3351 		return 0;
3352 
3353 	while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
3354 		if (to == MAX_SKB_FRAGS)
3355 			return 0;
3356 
3357 		fragfrom = &skb_shinfo(skb)->frags[from];
3358 		fragto = &skb_shinfo(tgt)->frags[to];
3359 
3360 		if (todo >= skb_frag_size(fragfrom)) {
3361 			*fragto = *fragfrom;
3362 			todo -= skb_frag_size(fragfrom);
3363 			from++;
3364 			to++;
3365 
3366 		} else {
3367 			__skb_frag_ref(fragfrom);
3368 			skb_frag_page_copy(fragto, fragfrom);
3369 			skb_frag_off_copy(fragto, fragfrom);
3370 			skb_frag_size_set(fragto, todo);
3371 
3372 			skb_frag_off_add(fragfrom, todo);
3373 			skb_frag_size_sub(fragfrom, todo);
3374 			todo = 0;
3375 
3376 			to++;
3377 			break;
3378 		}
3379 	}
3380 
3381 	/* Ready to "commit" this state change to tgt */
3382 	skb_shinfo(tgt)->nr_frags = to;
3383 
3384 	if (merge >= 0) {
3385 		fragfrom = &skb_shinfo(skb)->frags[0];
3386 		fragto = &skb_shinfo(tgt)->frags[merge];
3387 
3388 		skb_frag_size_add(fragto, skb_frag_size(fragfrom));
3389 		__skb_frag_unref(fragfrom);
3390 	}
3391 
3392 	/* Reposition in the original skb */
3393 	to = 0;
3394 	while (from < skb_shinfo(skb)->nr_frags)
3395 		skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
3396 	skb_shinfo(skb)->nr_frags = to;
3397 
3398 	BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
3399 
3400 onlymerged:
3401 	/* Most likely the tgt won't ever need its checksum anymore, skb on
3402 	 * the other hand might need it if it needs to be resent
3403 	 */
3404 	tgt->ip_summed = CHECKSUM_PARTIAL;
3405 	skb->ip_summed = CHECKSUM_PARTIAL;
3406 
3407 	/* Yak, is it really working this way? Some helper please? */
3408 	skb->len -= shiftlen;
3409 	skb->data_len -= shiftlen;
3410 	skb->truesize -= shiftlen;
3411 	tgt->len += shiftlen;
3412 	tgt->data_len += shiftlen;
3413 	tgt->truesize += shiftlen;
3414 
3415 	return shiftlen;
3416 }
3417 
3418 /**
3419  * skb_prepare_seq_read - Prepare a sequential read of skb data
3420  * @skb: the buffer to read
3421  * @from: lower offset of data to be read
3422  * @to: upper offset of data to be read
3423  * @st: state variable
3424  *
3425  * Initializes the specified state variable. Must be called before
3426  * invoking skb_seq_read() for the first time.
3427  */
3428 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
3429 			  unsigned int to, struct skb_seq_state *st)
3430 {
3431 	st->lower_offset = from;
3432 	st->upper_offset = to;
3433 	st->root_skb = st->cur_skb = skb;
3434 	st->frag_idx = st->stepped_offset = 0;
3435 	st->frag_data = NULL;
3436 }
3437 EXPORT_SYMBOL(skb_prepare_seq_read);
3438 
3439 /**
3440  * skb_seq_read - Sequentially read skb data
3441  * @consumed: number of bytes consumed by the caller so far
3442  * @data: destination pointer for data to be returned
3443  * @st: state variable
3444  *
3445  * Reads a block of skb data at @consumed relative to the
3446  * lower offset specified to skb_prepare_seq_read(). Assigns
3447  * the head of the data block to @data and returns the length
3448  * of the block or 0 if the end of the skb data or the upper
3449  * offset has been reached.
3450  *
3451  * The caller is not required to consume all of the data
3452  * returned, i.e. @consumed is typically set to the number
3453  * of bytes already consumed and the next call to
3454  * skb_seq_read() will return the remaining part of the block.
3455  *
3456  * Note 1: The size of each block of data returned can be arbitrary,
3457  *       this limitation is the cost for zerocopy sequential
3458  *       reads of potentially non linear data.
3459  *
3460  * Note 2: Fragment lists within fragments are not implemented
3461  *       at the moment, state->root_skb could be replaced with
3462  *       a stack for this purpose.
3463  */
3464 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
3465 			  struct skb_seq_state *st)
3466 {
3467 	unsigned int block_limit, abs_offset = consumed + st->lower_offset;
3468 	skb_frag_t *frag;
3469 
3470 	if (unlikely(abs_offset >= st->upper_offset)) {
3471 		if (st->frag_data) {
3472 			kunmap_atomic(st->frag_data);
3473 			st->frag_data = NULL;
3474 		}
3475 		return 0;
3476 	}
3477 
3478 next_skb:
3479 	block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
3480 
3481 	if (abs_offset < block_limit && !st->frag_data) {
3482 		*data = st->cur_skb->data + (abs_offset - st->stepped_offset);
3483 		return block_limit - abs_offset;
3484 	}
3485 
3486 	if (st->frag_idx == 0 && !st->frag_data)
3487 		st->stepped_offset += skb_headlen(st->cur_skb);
3488 
3489 	while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
3490 		frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
3491 		block_limit = skb_frag_size(frag) + st->stepped_offset;
3492 
3493 		if (abs_offset < block_limit) {
3494 			if (!st->frag_data)
3495 				st->frag_data = kmap_atomic(skb_frag_page(frag));
3496 
3497 			*data = (u8 *) st->frag_data + skb_frag_off(frag) +
3498 				(abs_offset - st->stepped_offset);
3499 
3500 			return block_limit - abs_offset;
3501 		}
3502 
3503 		if (st->frag_data) {
3504 			kunmap_atomic(st->frag_data);
3505 			st->frag_data = NULL;
3506 		}
3507 
3508 		st->frag_idx++;
3509 		st->stepped_offset += skb_frag_size(frag);
3510 	}
3511 
3512 	if (st->frag_data) {
3513 		kunmap_atomic(st->frag_data);
3514 		st->frag_data = NULL;
3515 	}
3516 
3517 	if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
3518 		st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
3519 		st->frag_idx = 0;
3520 		goto next_skb;
3521 	} else if (st->cur_skb->next) {
3522 		st->cur_skb = st->cur_skb->next;
3523 		st->frag_idx = 0;
3524 		goto next_skb;
3525 	}
3526 
3527 	return 0;
3528 }
3529 EXPORT_SYMBOL(skb_seq_read);
3530 
3531 /**
3532  * skb_abort_seq_read - Abort a sequential read of skb data
3533  * @st: state variable
3534  *
3535  * Must be called if skb_seq_read() was not called until it
3536  * returned 0.
3537  */
3538 void skb_abort_seq_read(struct skb_seq_state *st)
3539 {
3540 	if (st->frag_data)
3541 		kunmap_atomic(st->frag_data);
3542 }
3543 EXPORT_SYMBOL(skb_abort_seq_read);
3544 
3545 #define TS_SKB_CB(state)	((struct skb_seq_state *) &((state)->cb))
3546 
3547 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
3548 					  struct ts_config *conf,
3549 					  struct ts_state *state)
3550 {
3551 	return skb_seq_read(offset, text, TS_SKB_CB(state));
3552 }
3553 
3554 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
3555 {
3556 	skb_abort_seq_read(TS_SKB_CB(state));
3557 }
3558 
3559 /**
3560  * skb_find_text - Find a text pattern in skb data
3561  * @skb: the buffer to look in
3562  * @from: search offset
3563  * @to: search limit
3564  * @config: textsearch configuration
3565  *
3566  * Finds a pattern in the skb data according to the specified
3567  * textsearch configuration. Use textsearch_next() to retrieve
3568  * subsequent occurrences of the pattern. Returns the offset
3569  * to the first occurrence or UINT_MAX if no match was found.
3570  */
3571 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
3572 			   unsigned int to, struct ts_config *config)
3573 {
3574 	struct ts_state state;
3575 	unsigned int ret;
3576 
3577 	config->get_next_block = skb_ts_get_next_block;
3578 	config->finish = skb_ts_finish;
3579 
3580 	skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
3581 
3582 	ret = textsearch_find(config, &state);
3583 	return (ret <= to - from ? ret : UINT_MAX);
3584 }
3585 EXPORT_SYMBOL(skb_find_text);
3586 
3587 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
3588 			 int offset, size_t size)
3589 {
3590 	int i = skb_shinfo(skb)->nr_frags;
3591 
3592 	if (skb_can_coalesce(skb, i, page, offset)) {
3593 		skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
3594 	} else if (i < MAX_SKB_FRAGS) {
3595 		get_page(page);
3596 		skb_fill_page_desc(skb, i, page, offset, size);
3597 	} else {
3598 		return -EMSGSIZE;
3599 	}
3600 
3601 	return 0;
3602 }
3603 EXPORT_SYMBOL_GPL(skb_append_pagefrags);
3604 
3605 /**
3606  *	skb_pull_rcsum - pull skb and update receive checksum
3607  *	@skb: buffer to update
3608  *	@len: length of data pulled
3609  *
3610  *	This function performs an skb_pull on the packet and updates
3611  *	the CHECKSUM_COMPLETE checksum.  It should be used on
3612  *	receive path processing instead of skb_pull unless you know
3613  *	that the checksum difference is zero (e.g., a valid IP header)
3614  *	or you are setting ip_summed to CHECKSUM_NONE.
3615  */
3616 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
3617 {
3618 	unsigned char *data = skb->data;
3619 
3620 	BUG_ON(len > skb->len);
3621 	__skb_pull(skb, len);
3622 	skb_postpull_rcsum(skb, data, len);
3623 	return skb->data;
3624 }
3625 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
3626 
3627 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
3628 {
3629 	skb_frag_t head_frag;
3630 	struct page *page;
3631 
3632 	page = virt_to_head_page(frag_skb->head);
3633 	__skb_frag_set_page(&head_frag, page);
3634 	skb_frag_off_set(&head_frag, frag_skb->data -
3635 			 (unsigned char *)page_address(page));
3636 	skb_frag_size_set(&head_frag, skb_headlen(frag_skb));
3637 	return head_frag;
3638 }
3639 
3640 struct sk_buff *skb_segment_list(struct sk_buff *skb,
3641 				 netdev_features_t features,
3642 				 unsigned int offset)
3643 {
3644 	struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
3645 	unsigned int tnl_hlen = skb_tnl_header_len(skb);
3646 	unsigned int delta_truesize = 0;
3647 	unsigned int delta_len = 0;
3648 	struct sk_buff *tail = NULL;
3649 	struct sk_buff *nskb;
3650 
3651 	skb_push(skb, -skb_network_offset(skb) + offset);
3652 
3653 	skb_shinfo(skb)->frag_list = NULL;
3654 
3655 	do {
3656 		nskb = list_skb;
3657 		list_skb = list_skb->next;
3658 
3659 		if (!tail)
3660 			skb->next = nskb;
3661 		else
3662 			tail->next = nskb;
3663 
3664 		tail = nskb;
3665 
3666 		delta_len += nskb->len;
3667 		delta_truesize += nskb->truesize;
3668 
3669 		skb_push(nskb, -skb_network_offset(nskb) + offset);
3670 
3671 		 __copy_skb_header(nskb, skb);
3672 
3673 		skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
3674 		skb_copy_from_linear_data_offset(skb, -tnl_hlen,
3675 						 nskb->data - tnl_hlen,
3676 						 offset + tnl_hlen);
3677 
3678 		if (skb_needs_linearize(nskb, features) &&
3679 		    __skb_linearize(nskb))
3680 			goto err_linearize;
3681 
3682 	} while (list_skb);
3683 
3684 	skb->truesize = skb->truesize - delta_truesize;
3685 	skb->data_len = skb->data_len - delta_len;
3686 	skb->len = skb->len - delta_len;
3687 
3688 	skb_gso_reset(skb);
3689 
3690 	skb->prev = tail;
3691 
3692 	if (skb_needs_linearize(skb, features) &&
3693 	    __skb_linearize(skb))
3694 		goto err_linearize;
3695 
3696 	skb_get(skb);
3697 
3698 	return skb;
3699 
3700 err_linearize:
3701 	kfree_skb_list(skb->next);
3702 	skb->next = NULL;
3703 	return ERR_PTR(-ENOMEM);
3704 }
3705 EXPORT_SYMBOL_GPL(skb_segment_list);
3706 
3707 int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
3708 {
3709 	if (unlikely(p->len + skb->len >= 65536))
3710 		return -E2BIG;
3711 
3712 	if (NAPI_GRO_CB(p)->last == p)
3713 		skb_shinfo(p)->frag_list = skb;
3714 	else
3715 		NAPI_GRO_CB(p)->last->next = skb;
3716 
3717 	skb_pull(skb, skb_gro_offset(skb));
3718 
3719 	NAPI_GRO_CB(p)->last = skb;
3720 	NAPI_GRO_CB(p)->count++;
3721 	p->data_len += skb->len;
3722 	p->truesize += skb->truesize;
3723 	p->len += skb->len;
3724 
3725 	NAPI_GRO_CB(skb)->same_flow = 1;
3726 
3727 	return 0;
3728 }
3729 EXPORT_SYMBOL_GPL(skb_gro_receive_list);
3730 
3731 /**
3732  *	skb_segment - Perform protocol segmentation on skb.
3733  *	@head_skb: buffer to segment
3734  *	@features: features for the output path (see dev->features)
3735  *
3736  *	This function performs segmentation on the given skb.  It returns
3737  *	a pointer to the first in a list of new skbs for the segments.
3738  *	In case of error it returns ERR_PTR(err).
3739  */
3740 struct sk_buff *skb_segment(struct sk_buff *head_skb,
3741 			    netdev_features_t features)
3742 {
3743 	struct sk_buff *segs = NULL;
3744 	struct sk_buff *tail = NULL;
3745 	struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
3746 	skb_frag_t *frag = skb_shinfo(head_skb)->frags;
3747 	unsigned int mss = skb_shinfo(head_skb)->gso_size;
3748 	unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
3749 	struct sk_buff *frag_skb = head_skb;
3750 	unsigned int offset = doffset;
3751 	unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
3752 	unsigned int partial_segs = 0;
3753 	unsigned int headroom;
3754 	unsigned int len = head_skb->len;
3755 	__be16 proto;
3756 	bool csum, sg;
3757 	int nfrags = skb_shinfo(head_skb)->nr_frags;
3758 	int err = -ENOMEM;
3759 	int i = 0;
3760 	int pos;
3761 	int dummy;
3762 
3763 	if (list_skb && !list_skb->head_frag && skb_headlen(list_skb) &&
3764 	    (skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY)) {
3765 		/* gso_size is untrusted, and we have a frag_list with a linear
3766 		 * non head_frag head.
3767 		 *
3768 		 * (we assume checking the first list_skb member suffices;
3769 		 * i.e if either of the list_skb members have non head_frag
3770 		 * head, then the first one has too).
3771 		 *
3772 		 * If head_skb's headlen does not fit requested gso_size, it
3773 		 * means that the frag_list members do NOT terminate on exact
3774 		 * gso_size boundaries. Hence we cannot perform skb_frag_t page
3775 		 * sharing. Therefore we must fallback to copying the frag_list
3776 		 * skbs; we do so by disabling SG.
3777 		 */
3778 		if (mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb))
3779 			features &= ~NETIF_F_SG;
3780 	}
3781 
3782 	__skb_push(head_skb, doffset);
3783 	proto = skb_network_protocol(head_skb, &dummy);
3784 	if (unlikely(!proto))
3785 		return ERR_PTR(-EINVAL);
3786 
3787 	sg = !!(features & NETIF_F_SG);
3788 	csum = !!can_checksum_protocol(features, proto);
3789 
3790 	if (sg && csum && (mss != GSO_BY_FRAGS))  {
3791 		if (!(features & NETIF_F_GSO_PARTIAL)) {
3792 			struct sk_buff *iter;
3793 			unsigned int frag_len;
3794 
3795 			if (!list_skb ||
3796 			    !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
3797 				goto normal;
3798 
3799 			/* If we get here then all the required
3800 			 * GSO features except frag_list are supported.
3801 			 * Try to split the SKB to multiple GSO SKBs
3802 			 * with no frag_list.
3803 			 * Currently we can do that only when the buffers don't
3804 			 * have a linear part and all the buffers except
3805 			 * the last are of the same length.
3806 			 */
3807 			frag_len = list_skb->len;
3808 			skb_walk_frags(head_skb, iter) {
3809 				if (frag_len != iter->len && iter->next)
3810 					goto normal;
3811 				if (skb_headlen(iter) && !iter->head_frag)
3812 					goto normal;
3813 
3814 				len -= iter->len;
3815 			}
3816 
3817 			if (len != frag_len)
3818 				goto normal;
3819 		}
3820 
3821 		/* GSO partial only requires that we trim off any excess that
3822 		 * doesn't fit into an MSS sized block, so take care of that
3823 		 * now.
3824 		 */
3825 		partial_segs = len / mss;
3826 		if (partial_segs > 1)
3827 			mss *= partial_segs;
3828 		else
3829 			partial_segs = 0;
3830 	}
3831 
3832 normal:
3833 	headroom = skb_headroom(head_skb);
3834 	pos = skb_headlen(head_skb);
3835 
3836 	do {
3837 		struct sk_buff *nskb;
3838 		skb_frag_t *nskb_frag;
3839 		int hsize;
3840 		int size;
3841 
3842 		if (unlikely(mss == GSO_BY_FRAGS)) {
3843 			len = list_skb->len;
3844 		} else {
3845 			len = head_skb->len - offset;
3846 			if (len > mss)
3847 				len = mss;
3848 		}
3849 
3850 		hsize = skb_headlen(head_skb) - offset;
3851 		if (hsize < 0)
3852 			hsize = 0;
3853 		if (hsize > len || !sg)
3854 			hsize = len;
3855 
3856 		if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
3857 		    (skb_headlen(list_skb) == len || sg)) {
3858 			BUG_ON(skb_headlen(list_skb) > len);
3859 
3860 			i = 0;
3861 			nfrags = skb_shinfo(list_skb)->nr_frags;
3862 			frag = skb_shinfo(list_skb)->frags;
3863 			frag_skb = list_skb;
3864 			pos += skb_headlen(list_skb);
3865 
3866 			while (pos < offset + len) {
3867 				BUG_ON(i >= nfrags);
3868 
3869 				size = skb_frag_size(frag);
3870 				if (pos + size > offset + len)
3871 					break;
3872 
3873 				i++;
3874 				pos += size;
3875 				frag++;
3876 			}
3877 
3878 			nskb = skb_clone(list_skb, GFP_ATOMIC);
3879 			list_skb = list_skb->next;
3880 
3881 			if (unlikely(!nskb))
3882 				goto err;
3883 
3884 			if (unlikely(pskb_trim(nskb, len))) {
3885 				kfree_skb(nskb);
3886 				goto err;
3887 			}
3888 
3889 			hsize = skb_end_offset(nskb);
3890 			if (skb_cow_head(nskb, doffset + headroom)) {
3891 				kfree_skb(nskb);
3892 				goto err;
3893 			}
3894 
3895 			nskb->truesize += skb_end_offset(nskb) - hsize;
3896 			skb_release_head_state(nskb);
3897 			__skb_push(nskb, doffset);
3898 		} else {
3899 			nskb = __alloc_skb(hsize + doffset + headroom,
3900 					   GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
3901 					   NUMA_NO_NODE);
3902 
3903 			if (unlikely(!nskb))
3904 				goto err;
3905 
3906 			skb_reserve(nskb, headroom);
3907 			__skb_put(nskb, doffset);
3908 		}
3909 
3910 		if (segs)
3911 			tail->next = nskb;
3912 		else
3913 			segs = nskb;
3914 		tail = nskb;
3915 
3916 		__copy_skb_header(nskb, head_skb);
3917 
3918 		skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
3919 		skb_reset_mac_len(nskb);
3920 
3921 		skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
3922 						 nskb->data - tnl_hlen,
3923 						 doffset + tnl_hlen);
3924 
3925 		if (nskb->len == len + doffset)
3926 			goto perform_csum_check;
3927 
3928 		if (!sg) {
3929 			if (!nskb->remcsum_offload)
3930 				nskb->ip_summed = CHECKSUM_NONE;
3931 			SKB_GSO_CB(nskb)->csum =
3932 				skb_copy_and_csum_bits(head_skb, offset,
3933 						       skb_put(nskb, len),
3934 						       len, 0);
3935 			SKB_GSO_CB(nskb)->csum_start =
3936 				skb_headroom(nskb) + doffset;
3937 			continue;
3938 		}
3939 
3940 		nskb_frag = skb_shinfo(nskb)->frags;
3941 
3942 		skb_copy_from_linear_data_offset(head_skb, offset,
3943 						 skb_put(nskb, hsize), hsize);
3944 
3945 		skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags &
3946 					      SKBTX_SHARED_FRAG;
3947 
3948 		if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
3949 		    skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
3950 			goto err;
3951 
3952 		while (pos < offset + len) {
3953 			if (i >= nfrags) {
3954 				i = 0;
3955 				nfrags = skb_shinfo(list_skb)->nr_frags;
3956 				frag = skb_shinfo(list_skb)->frags;
3957 				frag_skb = list_skb;
3958 				if (!skb_headlen(list_skb)) {
3959 					BUG_ON(!nfrags);
3960 				} else {
3961 					BUG_ON(!list_skb->head_frag);
3962 
3963 					/* to make room for head_frag. */
3964 					i--;
3965 					frag--;
3966 				}
3967 				if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
3968 				    skb_zerocopy_clone(nskb, frag_skb,
3969 						       GFP_ATOMIC))
3970 					goto err;
3971 
3972 				list_skb = list_skb->next;
3973 			}
3974 
3975 			if (unlikely(skb_shinfo(nskb)->nr_frags >=
3976 				     MAX_SKB_FRAGS)) {
3977 				net_warn_ratelimited(
3978 					"skb_segment: too many frags: %u %u\n",
3979 					pos, mss);
3980 				err = -EINVAL;
3981 				goto err;
3982 			}
3983 
3984 			*nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
3985 			__skb_frag_ref(nskb_frag);
3986 			size = skb_frag_size(nskb_frag);
3987 
3988 			if (pos < offset) {
3989 				skb_frag_off_add(nskb_frag, offset - pos);
3990 				skb_frag_size_sub(nskb_frag, offset - pos);
3991 			}
3992 
3993 			skb_shinfo(nskb)->nr_frags++;
3994 
3995 			if (pos + size <= offset + len) {
3996 				i++;
3997 				frag++;
3998 				pos += size;
3999 			} else {
4000 				skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
4001 				goto skip_fraglist;
4002 			}
4003 
4004 			nskb_frag++;
4005 		}
4006 
4007 skip_fraglist:
4008 		nskb->data_len = len - hsize;
4009 		nskb->len += nskb->data_len;
4010 		nskb->truesize += nskb->data_len;
4011 
4012 perform_csum_check:
4013 		if (!csum) {
4014 			if (skb_has_shared_frag(nskb) &&
4015 			    __skb_linearize(nskb))
4016 				goto err;
4017 
4018 			if (!nskb->remcsum_offload)
4019 				nskb->ip_summed = CHECKSUM_NONE;
4020 			SKB_GSO_CB(nskb)->csum =
4021 				skb_checksum(nskb, doffset,
4022 					     nskb->len - doffset, 0);
4023 			SKB_GSO_CB(nskb)->csum_start =
4024 				skb_headroom(nskb) + doffset;
4025 		}
4026 	} while ((offset += len) < head_skb->len);
4027 
4028 	/* Some callers want to get the end of the list.
4029 	 * Put it in segs->prev to avoid walking the list.
4030 	 * (see validate_xmit_skb_list() for example)
4031 	 */
4032 	segs->prev = tail;
4033 
4034 	if (partial_segs) {
4035 		struct sk_buff *iter;
4036 		int type = skb_shinfo(head_skb)->gso_type;
4037 		unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
4038 
4039 		/* Update type to add partial and then remove dodgy if set */
4040 		type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
4041 		type &= ~SKB_GSO_DODGY;
4042 
4043 		/* Update GSO info and prepare to start updating headers on
4044 		 * our way back down the stack of protocols.
4045 		 */
4046 		for (iter = segs; iter; iter = iter->next) {
4047 			skb_shinfo(iter)->gso_size = gso_size;
4048 			skb_shinfo(iter)->gso_segs = partial_segs;
4049 			skb_shinfo(iter)->gso_type = type;
4050 			SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
4051 		}
4052 
4053 		if (tail->len - doffset <= gso_size)
4054 			skb_shinfo(tail)->gso_size = 0;
4055 		else if (tail != segs)
4056 			skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
4057 	}
4058 
4059 	/* Following permits correct backpressure, for protocols
4060 	 * using skb_set_owner_w().
4061 	 * Idea is to tranfert ownership from head_skb to last segment.
4062 	 */
4063 	if (head_skb->destructor == sock_wfree) {
4064 		swap(tail->truesize, head_skb->truesize);
4065 		swap(tail->destructor, head_skb->destructor);
4066 		swap(tail->sk, head_skb->sk);
4067 	}
4068 	return segs;
4069 
4070 err:
4071 	kfree_skb_list(segs);
4072 	return ERR_PTR(err);
4073 }
4074 EXPORT_SYMBOL_GPL(skb_segment);
4075 
4076 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
4077 {
4078 	struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
4079 	unsigned int offset = skb_gro_offset(skb);
4080 	unsigned int headlen = skb_headlen(skb);
4081 	unsigned int len = skb_gro_len(skb);
4082 	unsigned int delta_truesize;
4083 	struct sk_buff *lp;
4084 
4085 	if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush))
4086 		return -E2BIG;
4087 
4088 	lp = NAPI_GRO_CB(p)->last;
4089 	pinfo = skb_shinfo(lp);
4090 
4091 	if (headlen <= offset) {
4092 		skb_frag_t *frag;
4093 		skb_frag_t *frag2;
4094 		int i = skbinfo->nr_frags;
4095 		int nr_frags = pinfo->nr_frags + i;
4096 
4097 		if (nr_frags > MAX_SKB_FRAGS)
4098 			goto merge;
4099 
4100 		offset -= headlen;
4101 		pinfo->nr_frags = nr_frags;
4102 		skbinfo->nr_frags = 0;
4103 
4104 		frag = pinfo->frags + nr_frags;
4105 		frag2 = skbinfo->frags + i;
4106 		do {
4107 			*--frag = *--frag2;
4108 		} while (--i);
4109 
4110 		skb_frag_off_add(frag, offset);
4111 		skb_frag_size_sub(frag, offset);
4112 
4113 		/* all fragments truesize : remove (head size + sk_buff) */
4114 		delta_truesize = skb->truesize -
4115 				 SKB_TRUESIZE(skb_end_offset(skb));
4116 
4117 		skb->truesize -= skb->data_len;
4118 		skb->len -= skb->data_len;
4119 		skb->data_len = 0;
4120 
4121 		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
4122 		goto done;
4123 	} else if (skb->head_frag) {
4124 		int nr_frags = pinfo->nr_frags;
4125 		skb_frag_t *frag = pinfo->frags + nr_frags;
4126 		struct page *page = virt_to_head_page(skb->head);
4127 		unsigned int first_size = headlen - offset;
4128 		unsigned int first_offset;
4129 
4130 		if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
4131 			goto merge;
4132 
4133 		first_offset = skb->data -
4134 			       (unsigned char *)page_address(page) +
4135 			       offset;
4136 
4137 		pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
4138 
4139 		__skb_frag_set_page(frag, page);
4140 		skb_frag_off_set(frag, first_offset);
4141 		skb_frag_size_set(frag, first_size);
4142 
4143 		memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
4144 		/* We dont need to clear skbinfo->nr_frags here */
4145 
4146 		delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
4147 		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
4148 		goto done;
4149 	}
4150 
4151 merge:
4152 	delta_truesize = skb->truesize;
4153 	if (offset > headlen) {
4154 		unsigned int eat = offset - headlen;
4155 
4156 		skb_frag_off_add(&skbinfo->frags[0], eat);
4157 		skb_frag_size_sub(&skbinfo->frags[0], eat);
4158 		skb->data_len -= eat;
4159 		skb->len -= eat;
4160 		offset = headlen;
4161 	}
4162 
4163 	__skb_pull(skb, offset);
4164 
4165 	if (NAPI_GRO_CB(p)->last == p)
4166 		skb_shinfo(p)->frag_list = skb;
4167 	else
4168 		NAPI_GRO_CB(p)->last->next = skb;
4169 	NAPI_GRO_CB(p)->last = skb;
4170 	__skb_header_release(skb);
4171 	lp = p;
4172 
4173 done:
4174 	NAPI_GRO_CB(p)->count++;
4175 	p->data_len += len;
4176 	p->truesize += delta_truesize;
4177 	p->len += len;
4178 	if (lp != p) {
4179 		lp->data_len += len;
4180 		lp->truesize += delta_truesize;
4181 		lp->len += len;
4182 	}
4183 	NAPI_GRO_CB(skb)->same_flow = 1;
4184 	return 0;
4185 }
4186 EXPORT_SYMBOL_GPL(skb_gro_receive);
4187 
4188 #ifdef CONFIG_SKB_EXTENSIONS
4189 #define SKB_EXT_ALIGN_VALUE	8
4190 #define SKB_EXT_CHUNKSIZEOF(x)	(ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
4191 
4192 static const u8 skb_ext_type_len[] = {
4193 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4194 	[SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
4195 #endif
4196 #ifdef CONFIG_XFRM
4197 	[SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
4198 #endif
4199 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
4200 	[TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
4201 #endif
4202 #if IS_ENABLED(CONFIG_MPTCP)
4203 	[SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
4204 #endif
4205 };
4206 
4207 static __always_inline unsigned int skb_ext_total_length(void)
4208 {
4209 	return SKB_EXT_CHUNKSIZEOF(struct skb_ext) +
4210 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4211 		skb_ext_type_len[SKB_EXT_BRIDGE_NF] +
4212 #endif
4213 #ifdef CONFIG_XFRM
4214 		skb_ext_type_len[SKB_EXT_SEC_PATH] +
4215 #endif
4216 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
4217 		skb_ext_type_len[TC_SKB_EXT] +
4218 #endif
4219 #if IS_ENABLED(CONFIG_MPTCP)
4220 		skb_ext_type_len[SKB_EXT_MPTCP] +
4221 #endif
4222 		0;
4223 }
4224 
4225 static void skb_extensions_init(void)
4226 {
4227 	BUILD_BUG_ON(SKB_EXT_NUM >= 8);
4228 	BUILD_BUG_ON(skb_ext_total_length() > 255);
4229 
4230 	skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
4231 					     SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
4232 					     0,
4233 					     SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4234 					     NULL);
4235 }
4236 #else
4237 static void skb_extensions_init(void) {}
4238 #endif
4239 
4240 void __init skb_init(void)
4241 {
4242 	skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache",
4243 					      sizeof(struct sk_buff),
4244 					      0,
4245 					      SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4246 					      offsetof(struct sk_buff, cb),
4247 					      sizeof_field(struct sk_buff, cb),
4248 					      NULL);
4249 	skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
4250 						sizeof(struct sk_buff_fclones),
4251 						0,
4252 						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4253 						NULL);
4254 	skb_extensions_init();
4255 }
4256 
4257 static int
4258 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
4259 	       unsigned int recursion_level)
4260 {
4261 	int start = skb_headlen(skb);
4262 	int i, copy = start - offset;
4263 	struct sk_buff *frag_iter;
4264 	int elt = 0;
4265 
4266 	if (unlikely(recursion_level >= 24))
4267 		return -EMSGSIZE;
4268 
4269 	if (copy > 0) {
4270 		if (copy > len)
4271 			copy = len;
4272 		sg_set_buf(sg, skb->data + offset, copy);
4273 		elt++;
4274 		if ((len -= copy) == 0)
4275 			return elt;
4276 		offset += copy;
4277 	}
4278 
4279 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4280 		int end;
4281 
4282 		WARN_ON(start > offset + len);
4283 
4284 		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
4285 		if ((copy = end - offset) > 0) {
4286 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4287 			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
4288 				return -EMSGSIZE;
4289 
4290 			if (copy > len)
4291 				copy = len;
4292 			sg_set_page(&sg[elt], skb_frag_page(frag), copy,
4293 				    skb_frag_off(frag) + offset - start);
4294 			elt++;
4295 			if (!(len -= copy))
4296 				return elt;
4297 			offset += copy;
4298 		}
4299 		start = end;
4300 	}
4301 
4302 	skb_walk_frags(skb, frag_iter) {
4303 		int end, ret;
4304 
4305 		WARN_ON(start > offset + len);
4306 
4307 		end = start + frag_iter->len;
4308 		if ((copy = end - offset) > 0) {
4309 			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
4310 				return -EMSGSIZE;
4311 
4312 			if (copy > len)
4313 				copy = len;
4314 			ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
4315 					      copy, recursion_level + 1);
4316 			if (unlikely(ret < 0))
4317 				return ret;
4318 			elt += ret;
4319 			if ((len -= copy) == 0)
4320 				return elt;
4321 			offset += copy;
4322 		}
4323 		start = end;
4324 	}
4325 	BUG_ON(len);
4326 	return elt;
4327 }
4328 
4329 /**
4330  *	skb_to_sgvec - Fill a scatter-gather list from a socket buffer
4331  *	@skb: Socket buffer containing the buffers to be mapped
4332  *	@sg: The scatter-gather list to map into
4333  *	@offset: The offset into the buffer's contents to start mapping
4334  *	@len: Length of buffer space to be mapped
4335  *
4336  *	Fill the specified scatter-gather list with mappings/pointers into a
4337  *	region of the buffer space attached to a socket buffer. Returns either
4338  *	the number of scatterlist items used, or -EMSGSIZE if the contents
4339  *	could not fit.
4340  */
4341 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
4342 {
4343 	int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
4344 
4345 	if (nsg <= 0)
4346 		return nsg;
4347 
4348 	sg_mark_end(&sg[nsg - 1]);
4349 
4350 	return nsg;
4351 }
4352 EXPORT_SYMBOL_GPL(skb_to_sgvec);
4353 
4354 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
4355  * sglist without mark the sg which contain last skb data as the end.
4356  * So the caller can mannipulate sg list as will when padding new data after
4357  * the first call without calling sg_unmark_end to expend sg list.
4358  *
4359  * Scenario to use skb_to_sgvec_nomark:
4360  * 1. sg_init_table
4361  * 2. skb_to_sgvec_nomark(payload1)
4362  * 3. skb_to_sgvec_nomark(payload2)
4363  *
4364  * This is equivalent to:
4365  * 1. sg_init_table
4366  * 2. skb_to_sgvec(payload1)
4367  * 3. sg_unmark_end
4368  * 4. skb_to_sgvec(payload2)
4369  *
4370  * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
4371  * is more preferable.
4372  */
4373 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
4374 			int offset, int len)
4375 {
4376 	return __skb_to_sgvec(skb, sg, offset, len, 0);
4377 }
4378 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
4379 
4380 
4381 
4382 /**
4383  *	skb_cow_data - Check that a socket buffer's data buffers are writable
4384  *	@skb: The socket buffer to check.
4385  *	@tailbits: Amount of trailing space to be added
4386  *	@trailer: Returned pointer to the skb where the @tailbits space begins
4387  *
4388  *	Make sure that the data buffers attached to a socket buffer are
4389  *	writable. If they are not, private copies are made of the data buffers
4390  *	and the socket buffer is set to use these instead.
4391  *
4392  *	If @tailbits is given, make sure that there is space to write @tailbits
4393  *	bytes of data beyond current end of socket buffer.  @trailer will be
4394  *	set to point to the skb in which this space begins.
4395  *
4396  *	The number of scatterlist elements required to completely map the
4397  *	COW'd and extended socket buffer will be returned.
4398  */
4399 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
4400 {
4401 	int copyflag;
4402 	int elt;
4403 	struct sk_buff *skb1, **skb_p;
4404 
4405 	/* If skb is cloned or its head is paged, reallocate
4406 	 * head pulling out all the pages (pages are considered not writable
4407 	 * at the moment even if they are anonymous).
4408 	 */
4409 	if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
4410 	    __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
4411 		return -ENOMEM;
4412 
4413 	/* Easy case. Most of packets will go this way. */
4414 	if (!skb_has_frag_list(skb)) {
4415 		/* A little of trouble, not enough of space for trailer.
4416 		 * This should not happen, when stack is tuned to generate
4417 		 * good frames. OK, on miss we reallocate and reserve even more
4418 		 * space, 128 bytes is fair. */
4419 
4420 		if (skb_tailroom(skb) < tailbits &&
4421 		    pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
4422 			return -ENOMEM;
4423 
4424 		/* Voila! */
4425 		*trailer = skb;
4426 		return 1;
4427 	}
4428 
4429 	/* Misery. We are in troubles, going to mincer fragments... */
4430 
4431 	elt = 1;
4432 	skb_p = &skb_shinfo(skb)->frag_list;
4433 	copyflag = 0;
4434 
4435 	while ((skb1 = *skb_p) != NULL) {
4436 		int ntail = 0;
4437 
4438 		/* The fragment is partially pulled by someone,
4439 		 * this can happen on input. Copy it and everything
4440 		 * after it. */
4441 
4442 		if (skb_shared(skb1))
4443 			copyflag = 1;
4444 
4445 		/* If the skb is the last, worry about trailer. */
4446 
4447 		if (skb1->next == NULL && tailbits) {
4448 			if (skb_shinfo(skb1)->nr_frags ||
4449 			    skb_has_frag_list(skb1) ||
4450 			    skb_tailroom(skb1) < tailbits)
4451 				ntail = tailbits + 128;
4452 		}
4453 
4454 		if (copyflag ||
4455 		    skb_cloned(skb1) ||
4456 		    ntail ||
4457 		    skb_shinfo(skb1)->nr_frags ||
4458 		    skb_has_frag_list(skb1)) {
4459 			struct sk_buff *skb2;
4460 
4461 			/* Fuck, we are miserable poor guys... */
4462 			if (ntail == 0)
4463 				skb2 = skb_copy(skb1, GFP_ATOMIC);
4464 			else
4465 				skb2 = skb_copy_expand(skb1,
4466 						       skb_headroom(skb1),
4467 						       ntail,
4468 						       GFP_ATOMIC);
4469 			if (unlikely(skb2 == NULL))
4470 				return -ENOMEM;
4471 
4472 			if (skb1->sk)
4473 				skb_set_owner_w(skb2, skb1->sk);
4474 
4475 			/* Looking around. Are we still alive?
4476 			 * OK, link new skb, drop old one */
4477 
4478 			skb2->next = skb1->next;
4479 			*skb_p = skb2;
4480 			kfree_skb(skb1);
4481 			skb1 = skb2;
4482 		}
4483 		elt++;
4484 		*trailer = skb1;
4485 		skb_p = &skb1->next;
4486 	}
4487 
4488 	return elt;
4489 }
4490 EXPORT_SYMBOL_GPL(skb_cow_data);
4491 
4492 static void sock_rmem_free(struct sk_buff *skb)
4493 {
4494 	struct sock *sk = skb->sk;
4495 
4496 	atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
4497 }
4498 
4499 static void skb_set_err_queue(struct sk_buff *skb)
4500 {
4501 	/* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
4502 	 * So, it is safe to (mis)use it to mark skbs on the error queue.
4503 	 */
4504 	skb->pkt_type = PACKET_OUTGOING;
4505 	BUILD_BUG_ON(PACKET_OUTGOING == 0);
4506 }
4507 
4508 /*
4509  * Note: We dont mem charge error packets (no sk_forward_alloc changes)
4510  */
4511 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
4512 {
4513 	if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4514 	    (unsigned int)READ_ONCE(sk->sk_rcvbuf))
4515 		return -ENOMEM;
4516 
4517 	skb_orphan(skb);
4518 	skb->sk = sk;
4519 	skb->destructor = sock_rmem_free;
4520 	atomic_add(skb->truesize, &sk->sk_rmem_alloc);
4521 	skb_set_err_queue(skb);
4522 
4523 	/* before exiting rcu section, make sure dst is refcounted */
4524 	skb_dst_force(skb);
4525 
4526 	skb_queue_tail(&sk->sk_error_queue, skb);
4527 	if (!sock_flag(sk, SOCK_DEAD))
4528 		sk->sk_error_report(sk);
4529 	return 0;
4530 }
4531 EXPORT_SYMBOL(sock_queue_err_skb);
4532 
4533 static bool is_icmp_err_skb(const struct sk_buff *skb)
4534 {
4535 	return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
4536 		       SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
4537 }
4538 
4539 struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
4540 {
4541 	struct sk_buff_head *q = &sk->sk_error_queue;
4542 	struct sk_buff *skb, *skb_next = NULL;
4543 	bool icmp_next = false;
4544 	unsigned long flags;
4545 
4546 	spin_lock_irqsave(&q->lock, flags);
4547 	skb = __skb_dequeue(q);
4548 	if (skb && (skb_next = skb_peek(q))) {
4549 		icmp_next = is_icmp_err_skb(skb_next);
4550 		if (icmp_next)
4551 			sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_origin;
4552 	}
4553 	spin_unlock_irqrestore(&q->lock, flags);
4554 
4555 	if (is_icmp_err_skb(skb) && !icmp_next)
4556 		sk->sk_err = 0;
4557 
4558 	if (skb_next)
4559 		sk->sk_error_report(sk);
4560 
4561 	return skb;
4562 }
4563 EXPORT_SYMBOL(sock_dequeue_err_skb);
4564 
4565 /**
4566  * skb_clone_sk - create clone of skb, and take reference to socket
4567  * @skb: the skb to clone
4568  *
4569  * This function creates a clone of a buffer that holds a reference on
4570  * sk_refcnt.  Buffers created via this function are meant to be
4571  * returned using sock_queue_err_skb, or free via kfree_skb.
4572  *
4573  * When passing buffers allocated with this function to sock_queue_err_skb
4574  * it is necessary to wrap the call with sock_hold/sock_put in order to
4575  * prevent the socket from being released prior to being enqueued on
4576  * the sk_error_queue.
4577  */
4578 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
4579 {
4580 	struct sock *sk = skb->sk;
4581 	struct sk_buff *clone;
4582 
4583 	if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
4584 		return NULL;
4585 
4586 	clone = skb_clone(skb, GFP_ATOMIC);
4587 	if (!clone) {
4588 		sock_put(sk);
4589 		return NULL;
4590 	}
4591 
4592 	clone->sk = sk;
4593 	clone->destructor = sock_efree;
4594 
4595 	return clone;
4596 }
4597 EXPORT_SYMBOL(skb_clone_sk);
4598 
4599 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
4600 					struct sock *sk,
4601 					int tstype,
4602 					bool opt_stats)
4603 {
4604 	struct sock_exterr_skb *serr;
4605 	int err;
4606 
4607 	BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
4608 
4609 	serr = SKB_EXT_ERR(skb);
4610 	memset(serr, 0, sizeof(*serr));
4611 	serr->ee.ee_errno = ENOMSG;
4612 	serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
4613 	serr->ee.ee_info = tstype;
4614 	serr->opt_stats = opt_stats;
4615 	serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
4616 	if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
4617 		serr->ee.ee_data = skb_shinfo(skb)->tskey;
4618 		if (sk->sk_protocol == IPPROTO_TCP &&
4619 		    sk->sk_type == SOCK_STREAM)
4620 			serr->ee.ee_data -= sk->sk_tskey;
4621 	}
4622 
4623 	err = sock_queue_err_skb(sk, skb);
4624 
4625 	if (err)
4626 		kfree_skb(skb);
4627 }
4628 
4629 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
4630 {
4631 	bool ret;
4632 
4633 	if (likely(sysctl_tstamp_allow_data || tsonly))
4634 		return true;
4635 
4636 	read_lock_bh(&sk->sk_callback_lock);
4637 	ret = sk->sk_socket && sk->sk_socket->file &&
4638 	      file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
4639 	read_unlock_bh(&sk->sk_callback_lock);
4640 	return ret;
4641 }
4642 
4643 void skb_complete_tx_timestamp(struct sk_buff *skb,
4644 			       struct skb_shared_hwtstamps *hwtstamps)
4645 {
4646 	struct sock *sk = skb->sk;
4647 
4648 	if (!skb_may_tx_timestamp(sk, false))
4649 		goto err;
4650 
4651 	/* Take a reference to prevent skb_orphan() from freeing the socket,
4652 	 * but only if the socket refcount is not zero.
4653 	 */
4654 	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4655 		*skb_hwtstamps(skb) = *hwtstamps;
4656 		__skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
4657 		sock_put(sk);
4658 		return;
4659 	}
4660 
4661 err:
4662 	kfree_skb(skb);
4663 }
4664 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
4665 
4666 void __skb_tstamp_tx(struct sk_buff *orig_skb,
4667 		     struct skb_shared_hwtstamps *hwtstamps,
4668 		     struct sock *sk, int tstype)
4669 {
4670 	struct sk_buff *skb;
4671 	bool tsonly, opt_stats = false;
4672 
4673 	if (!sk)
4674 		return;
4675 
4676 	if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
4677 	    skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
4678 		return;
4679 
4680 	tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
4681 	if (!skb_may_tx_timestamp(sk, tsonly))
4682 		return;
4683 
4684 	if (tsonly) {
4685 #ifdef CONFIG_INET
4686 		if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
4687 		    sk->sk_protocol == IPPROTO_TCP &&
4688 		    sk->sk_type == SOCK_STREAM) {
4689 			skb = tcp_get_timestamping_opt_stats(sk);
4690 			opt_stats = true;
4691 		} else
4692 #endif
4693 			skb = alloc_skb(0, GFP_ATOMIC);
4694 	} else {
4695 		skb = skb_clone(orig_skb, GFP_ATOMIC);
4696 	}
4697 	if (!skb)
4698 		return;
4699 
4700 	if (tsonly) {
4701 		skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
4702 					     SKBTX_ANY_TSTAMP;
4703 		skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
4704 	}
4705 
4706 	if (hwtstamps)
4707 		*skb_hwtstamps(skb) = *hwtstamps;
4708 	else
4709 		skb->tstamp = ktime_get_real();
4710 
4711 	__skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
4712 }
4713 EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
4714 
4715 void skb_tstamp_tx(struct sk_buff *orig_skb,
4716 		   struct skb_shared_hwtstamps *hwtstamps)
4717 {
4718 	return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
4719 			       SCM_TSTAMP_SND);
4720 }
4721 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
4722 
4723 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
4724 {
4725 	struct sock *sk = skb->sk;
4726 	struct sock_exterr_skb *serr;
4727 	int err = 1;
4728 
4729 	skb->wifi_acked_valid = 1;
4730 	skb->wifi_acked = acked;
4731 
4732 	serr = SKB_EXT_ERR(skb);
4733 	memset(serr, 0, sizeof(*serr));
4734 	serr->ee.ee_errno = ENOMSG;
4735 	serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
4736 
4737 	/* Take a reference to prevent skb_orphan() from freeing the socket,
4738 	 * but only if the socket refcount is not zero.
4739 	 */
4740 	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4741 		err = sock_queue_err_skb(sk, skb);
4742 		sock_put(sk);
4743 	}
4744 	if (err)
4745 		kfree_skb(skb);
4746 }
4747 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
4748 
4749 /**
4750  * skb_partial_csum_set - set up and verify partial csum values for packet
4751  * @skb: the skb to set
4752  * @start: the number of bytes after skb->data to start checksumming.
4753  * @off: the offset from start to place the checksum.
4754  *
4755  * For untrusted partially-checksummed packets, we need to make sure the values
4756  * for skb->csum_start and skb->csum_offset are valid so we don't oops.
4757  *
4758  * This function checks and sets those values and skb->ip_summed: if this
4759  * returns false you should drop the packet.
4760  */
4761 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
4762 {
4763 	u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
4764 	u32 csum_start = skb_headroom(skb) + (u32)start;
4765 
4766 	if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) {
4767 		net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
4768 				     start, off, skb_headroom(skb), skb_headlen(skb));
4769 		return false;
4770 	}
4771 	skb->ip_summed = CHECKSUM_PARTIAL;
4772 	skb->csum_start = csum_start;
4773 	skb->csum_offset = off;
4774 	skb_set_transport_header(skb, start);
4775 	return true;
4776 }
4777 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
4778 
4779 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
4780 			       unsigned int max)
4781 {
4782 	if (skb_headlen(skb) >= len)
4783 		return 0;
4784 
4785 	/* If we need to pullup then pullup to the max, so we
4786 	 * won't need to do it again.
4787 	 */
4788 	if (max > skb->len)
4789 		max = skb->len;
4790 
4791 	if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
4792 		return -ENOMEM;
4793 
4794 	if (skb_headlen(skb) < len)
4795 		return -EPROTO;
4796 
4797 	return 0;
4798 }
4799 
4800 #define MAX_TCP_HDR_LEN (15 * 4)
4801 
4802 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
4803 				      typeof(IPPROTO_IP) proto,
4804 				      unsigned int off)
4805 {
4806 	int err;
4807 
4808 	switch (proto) {
4809 	case IPPROTO_TCP:
4810 		err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
4811 					  off + MAX_TCP_HDR_LEN);
4812 		if (!err && !skb_partial_csum_set(skb, off,
4813 						  offsetof(struct tcphdr,
4814 							   check)))
4815 			err = -EPROTO;
4816 		return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
4817 
4818 	case IPPROTO_UDP:
4819 		err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
4820 					  off + sizeof(struct udphdr));
4821 		if (!err && !skb_partial_csum_set(skb, off,
4822 						  offsetof(struct udphdr,
4823 							   check)))
4824 			err = -EPROTO;
4825 		return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
4826 	}
4827 
4828 	return ERR_PTR(-EPROTO);
4829 }
4830 
4831 /* This value should be large enough to cover a tagged ethernet header plus
4832  * maximally sized IP and TCP or UDP headers.
4833  */
4834 #define MAX_IP_HDR_LEN 128
4835 
4836 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
4837 {
4838 	unsigned int off;
4839 	bool fragment;
4840 	__sum16 *csum;
4841 	int err;
4842 
4843 	fragment = false;
4844 
4845 	err = skb_maybe_pull_tail(skb,
4846 				  sizeof(struct iphdr),
4847 				  MAX_IP_HDR_LEN);
4848 	if (err < 0)
4849 		goto out;
4850 
4851 	if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF))
4852 		fragment = true;
4853 
4854 	off = ip_hdrlen(skb);
4855 
4856 	err = -EPROTO;
4857 
4858 	if (fragment)
4859 		goto out;
4860 
4861 	csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
4862 	if (IS_ERR(csum))
4863 		return PTR_ERR(csum);
4864 
4865 	if (recalculate)
4866 		*csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
4867 					   ip_hdr(skb)->daddr,
4868 					   skb->len - off,
4869 					   ip_hdr(skb)->protocol, 0);
4870 	err = 0;
4871 
4872 out:
4873 	return err;
4874 }
4875 
4876 /* This value should be large enough to cover a tagged ethernet header plus
4877  * an IPv6 header, all options, and a maximal TCP or UDP header.
4878  */
4879 #define MAX_IPV6_HDR_LEN 256
4880 
4881 #define OPT_HDR(type, skb, off) \
4882 	(type *)(skb_network_header(skb) + (off))
4883 
4884 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
4885 {
4886 	int err;
4887 	u8 nexthdr;
4888 	unsigned int off;
4889 	unsigned int len;
4890 	bool fragment;
4891 	bool done;
4892 	__sum16 *csum;
4893 
4894 	fragment = false;
4895 	done = false;
4896 
4897 	off = sizeof(struct ipv6hdr);
4898 
4899 	err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
4900 	if (err < 0)
4901 		goto out;
4902 
4903 	nexthdr = ipv6_hdr(skb)->nexthdr;
4904 
4905 	len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
4906 	while (off <= len && !done) {
4907 		switch (nexthdr) {
4908 		case IPPROTO_DSTOPTS:
4909 		case IPPROTO_HOPOPTS:
4910 		case IPPROTO_ROUTING: {
4911 			struct ipv6_opt_hdr *hp;
4912 
4913 			err = skb_maybe_pull_tail(skb,
4914 						  off +
4915 						  sizeof(struct ipv6_opt_hdr),
4916 						  MAX_IPV6_HDR_LEN);
4917 			if (err < 0)
4918 				goto out;
4919 
4920 			hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
4921 			nexthdr = hp->nexthdr;
4922 			off += ipv6_optlen(hp);
4923 			break;
4924 		}
4925 		case IPPROTO_AH: {
4926 			struct ip_auth_hdr *hp;
4927 
4928 			err = skb_maybe_pull_tail(skb,
4929 						  off +
4930 						  sizeof(struct ip_auth_hdr),
4931 						  MAX_IPV6_HDR_LEN);
4932 			if (err < 0)
4933 				goto out;
4934 
4935 			hp = OPT_HDR(struct ip_auth_hdr, skb, off);
4936 			nexthdr = hp->nexthdr;
4937 			off += ipv6_authlen(hp);
4938 			break;
4939 		}
4940 		case IPPROTO_FRAGMENT: {
4941 			struct frag_hdr *hp;
4942 
4943 			err = skb_maybe_pull_tail(skb,
4944 						  off +
4945 						  sizeof(struct frag_hdr),
4946 						  MAX_IPV6_HDR_LEN);
4947 			if (err < 0)
4948 				goto out;
4949 
4950 			hp = OPT_HDR(struct frag_hdr, skb, off);
4951 
4952 			if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
4953 				fragment = true;
4954 
4955 			nexthdr = hp->nexthdr;
4956 			off += sizeof(struct frag_hdr);
4957 			break;
4958 		}
4959 		default:
4960 			done = true;
4961 			break;
4962 		}
4963 	}
4964 
4965 	err = -EPROTO;
4966 
4967 	if (!done || fragment)
4968 		goto out;
4969 
4970 	csum = skb_checksum_setup_ip(skb, nexthdr, off);
4971 	if (IS_ERR(csum))
4972 		return PTR_ERR(csum);
4973 
4974 	if (recalculate)
4975 		*csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4976 					 &ipv6_hdr(skb)->daddr,
4977 					 skb->len - off, nexthdr, 0);
4978 	err = 0;
4979 
4980 out:
4981 	return err;
4982 }
4983 
4984 /**
4985  * skb_checksum_setup - set up partial checksum offset
4986  * @skb: the skb to set up
4987  * @recalculate: if true the pseudo-header checksum will be recalculated
4988  */
4989 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
4990 {
4991 	int err;
4992 
4993 	switch (skb->protocol) {
4994 	case htons(ETH_P_IP):
4995 		err = skb_checksum_setup_ipv4(skb, recalculate);
4996 		break;
4997 
4998 	case htons(ETH_P_IPV6):
4999 		err = skb_checksum_setup_ipv6(skb, recalculate);
5000 		break;
5001 
5002 	default:
5003 		err = -EPROTO;
5004 		break;
5005 	}
5006 
5007 	return err;
5008 }
5009 EXPORT_SYMBOL(skb_checksum_setup);
5010 
5011 /**
5012  * skb_checksum_maybe_trim - maybe trims the given skb
5013  * @skb: the skb to check
5014  * @transport_len: the data length beyond the network header
5015  *
5016  * Checks whether the given skb has data beyond the given transport length.
5017  * If so, returns a cloned skb trimmed to this transport length.
5018  * Otherwise returns the provided skb. Returns NULL in error cases
5019  * (e.g. transport_len exceeds skb length or out-of-memory).
5020  *
5021  * Caller needs to set the skb transport header and free any returned skb if it
5022  * differs from the provided skb.
5023  */
5024 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
5025 					       unsigned int transport_len)
5026 {
5027 	struct sk_buff *skb_chk;
5028 	unsigned int len = skb_transport_offset(skb) + transport_len;
5029 	int ret;
5030 
5031 	if (skb->len < len)
5032 		return NULL;
5033 	else if (skb->len == len)
5034 		return skb;
5035 
5036 	skb_chk = skb_clone(skb, GFP_ATOMIC);
5037 	if (!skb_chk)
5038 		return NULL;
5039 
5040 	ret = pskb_trim_rcsum(skb_chk, len);
5041 	if (ret) {
5042 		kfree_skb(skb_chk);
5043 		return NULL;
5044 	}
5045 
5046 	return skb_chk;
5047 }
5048 
5049 /**
5050  * skb_checksum_trimmed - validate checksum of an skb
5051  * @skb: the skb to check
5052  * @transport_len: the data length beyond the network header
5053  * @skb_chkf: checksum function to use
5054  *
5055  * Applies the given checksum function skb_chkf to the provided skb.
5056  * Returns a checked and maybe trimmed skb. Returns NULL on error.
5057  *
5058  * If the skb has data beyond the given transport length, then a
5059  * trimmed & cloned skb is checked and returned.
5060  *
5061  * Caller needs to set the skb transport header and free any returned skb if it
5062  * differs from the provided skb.
5063  */
5064 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
5065 				     unsigned int transport_len,
5066 				     __sum16(*skb_chkf)(struct sk_buff *skb))
5067 {
5068 	struct sk_buff *skb_chk;
5069 	unsigned int offset = skb_transport_offset(skb);
5070 	__sum16 ret;
5071 
5072 	skb_chk = skb_checksum_maybe_trim(skb, transport_len);
5073 	if (!skb_chk)
5074 		goto err;
5075 
5076 	if (!pskb_may_pull(skb_chk, offset))
5077 		goto err;
5078 
5079 	skb_pull_rcsum(skb_chk, offset);
5080 	ret = skb_chkf(skb_chk);
5081 	skb_push_rcsum(skb_chk, offset);
5082 
5083 	if (ret)
5084 		goto err;
5085 
5086 	return skb_chk;
5087 
5088 err:
5089 	if (skb_chk && skb_chk != skb)
5090 		kfree_skb(skb_chk);
5091 
5092 	return NULL;
5093 
5094 }
5095 EXPORT_SYMBOL(skb_checksum_trimmed);
5096 
5097 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
5098 {
5099 	net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
5100 			     skb->dev->name);
5101 }
5102 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
5103 
5104 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
5105 {
5106 	if (head_stolen) {
5107 		skb_release_head_state(skb);
5108 		kmem_cache_free(skbuff_head_cache, skb);
5109 	} else {
5110 		__kfree_skb(skb);
5111 	}
5112 }
5113 EXPORT_SYMBOL(kfree_skb_partial);
5114 
5115 /**
5116  * skb_try_coalesce - try to merge skb to prior one
5117  * @to: prior buffer
5118  * @from: buffer to add
5119  * @fragstolen: pointer to boolean
5120  * @delta_truesize: how much more was allocated than was requested
5121  */
5122 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
5123 		      bool *fragstolen, int *delta_truesize)
5124 {
5125 	struct skb_shared_info *to_shinfo, *from_shinfo;
5126 	int i, delta, len = from->len;
5127 
5128 	*fragstolen = false;
5129 
5130 	if (skb_cloned(to))
5131 		return false;
5132 
5133 	if (len <= skb_tailroom(to)) {
5134 		if (len)
5135 			BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
5136 		*delta_truesize = 0;
5137 		return true;
5138 	}
5139 
5140 	to_shinfo = skb_shinfo(to);
5141 	from_shinfo = skb_shinfo(from);
5142 	if (to_shinfo->frag_list || from_shinfo->frag_list)
5143 		return false;
5144 	if (skb_zcopy(to) || skb_zcopy(from))
5145 		return false;
5146 
5147 	if (skb_headlen(from) != 0) {
5148 		struct page *page;
5149 		unsigned int offset;
5150 
5151 		if (to_shinfo->nr_frags +
5152 		    from_shinfo->nr_frags >= MAX_SKB_FRAGS)
5153 			return false;
5154 
5155 		if (skb_head_is_locked(from))
5156 			return false;
5157 
5158 		delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
5159 
5160 		page = virt_to_head_page(from->head);
5161 		offset = from->data - (unsigned char *)page_address(page);
5162 
5163 		skb_fill_page_desc(to, to_shinfo->nr_frags,
5164 				   page, offset, skb_headlen(from));
5165 		*fragstolen = true;
5166 	} else {
5167 		if (to_shinfo->nr_frags +
5168 		    from_shinfo->nr_frags > MAX_SKB_FRAGS)
5169 			return false;
5170 
5171 		delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
5172 	}
5173 
5174 	WARN_ON_ONCE(delta < len);
5175 
5176 	memcpy(to_shinfo->frags + to_shinfo->nr_frags,
5177 	       from_shinfo->frags,
5178 	       from_shinfo->nr_frags * sizeof(skb_frag_t));
5179 	to_shinfo->nr_frags += from_shinfo->nr_frags;
5180 
5181 	if (!skb_cloned(from))
5182 		from_shinfo->nr_frags = 0;
5183 
5184 	/* if the skb is not cloned this does nothing
5185 	 * since we set nr_frags to 0.
5186 	 */
5187 	for (i = 0; i < from_shinfo->nr_frags; i++)
5188 		__skb_frag_ref(&from_shinfo->frags[i]);
5189 
5190 	to->truesize += delta;
5191 	to->len += len;
5192 	to->data_len += len;
5193 
5194 	*delta_truesize = delta;
5195 	return true;
5196 }
5197 EXPORT_SYMBOL(skb_try_coalesce);
5198 
5199 /**
5200  * skb_scrub_packet - scrub an skb
5201  *
5202  * @skb: buffer to clean
5203  * @xnet: packet is crossing netns
5204  *
5205  * skb_scrub_packet can be used after encapsulating or decapsulting a packet
5206  * into/from a tunnel. Some information have to be cleared during these
5207  * operations.
5208  * skb_scrub_packet can also be used to clean a skb before injecting it in
5209  * another namespace (@xnet == true). We have to clear all information in the
5210  * skb that could impact namespace isolation.
5211  */
5212 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
5213 {
5214 	skb->pkt_type = PACKET_HOST;
5215 	skb->skb_iif = 0;
5216 	skb->ignore_df = 0;
5217 	skb_dst_drop(skb);
5218 	skb_ext_reset(skb);
5219 	nf_reset_ct(skb);
5220 	nf_reset_trace(skb);
5221 
5222 #ifdef CONFIG_NET_SWITCHDEV
5223 	skb->offload_fwd_mark = 0;
5224 	skb->offload_l3_fwd_mark = 0;
5225 #endif
5226 
5227 	if (!xnet)
5228 		return;
5229 
5230 	ipvs_reset(skb);
5231 	skb->mark = 0;
5232 	skb->tstamp = 0;
5233 }
5234 EXPORT_SYMBOL_GPL(skb_scrub_packet);
5235 
5236 /**
5237  * skb_gso_transport_seglen - Return length of individual segments of a gso packet
5238  *
5239  * @skb: GSO skb
5240  *
5241  * skb_gso_transport_seglen is used to determine the real size of the
5242  * individual segments, including Layer4 headers (TCP/UDP).
5243  *
5244  * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
5245  */
5246 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
5247 {
5248 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
5249 	unsigned int thlen = 0;
5250 
5251 	if (skb->encapsulation) {
5252 		thlen = skb_inner_transport_header(skb) -
5253 			skb_transport_header(skb);
5254 
5255 		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
5256 			thlen += inner_tcp_hdrlen(skb);
5257 	} else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
5258 		thlen = tcp_hdrlen(skb);
5259 	} else if (unlikely(skb_is_gso_sctp(skb))) {
5260 		thlen = sizeof(struct sctphdr);
5261 	} else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
5262 		thlen = sizeof(struct udphdr);
5263 	}
5264 	/* UFO sets gso_size to the size of the fragmentation
5265 	 * payload, i.e. the size of the L4 (UDP) header is already
5266 	 * accounted for.
5267 	 */
5268 	return thlen + shinfo->gso_size;
5269 }
5270 
5271 /**
5272  * skb_gso_network_seglen - Return length of individual segments of a gso packet
5273  *
5274  * @skb: GSO skb
5275  *
5276  * skb_gso_network_seglen is used to determine the real size of the
5277  * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
5278  *
5279  * The MAC/L2 header is not accounted for.
5280  */
5281 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
5282 {
5283 	unsigned int hdr_len = skb_transport_header(skb) -
5284 			       skb_network_header(skb);
5285 
5286 	return hdr_len + skb_gso_transport_seglen(skb);
5287 }
5288 
5289 /**
5290  * skb_gso_mac_seglen - Return length of individual segments of a gso packet
5291  *
5292  * @skb: GSO skb
5293  *
5294  * skb_gso_mac_seglen is used to determine the real size of the
5295  * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4
5296  * headers (TCP/UDP).
5297  */
5298 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
5299 {
5300 	unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
5301 
5302 	return hdr_len + skb_gso_transport_seglen(skb);
5303 }
5304 
5305 /**
5306  * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
5307  *
5308  * There are a couple of instances where we have a GSO skb, and we
5309  * want to determine what size it would be after it is segmented.
5310  *
5311  * We might want to check:
5312  * -    L3+L4+payload size (e.g. IP forwarding)
5313  * - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
5314  *
5315  * This is a helper to do that correctly considering GSO_BY_FRAGS.
5316  *
5317  * @skb: GSO skb
5318  *
5319  * @seg_len: The segmented length (from skb_gso_*_seglen). In the
5320  *           GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
5321  *
5322  * @max_len: The maximum permissible length.
5323  *
5324  * Returns true if the segmented length <= max length.
5325  */
5326 static inline bool skb_gso_size_check(const struct sk_buff *skb,
5327 				      unsigned int seg_len,
5328 				      unsigned int max_len) {
5329 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
5330 	const struct sk_buff *iter;
5331 
5332 	if (shinfo->gso_size != GSO_BY_FRAGS)
5333 		return seg_len <= max_len;
5334 
5335 	/* Undo this so we can re-use header sizes */
5336 	seg_len -= GSO_BY_FRAGS;
5337 
5338 	skb_walk_frags(skb, iter) {
5339 		if (seg_len + skb_headlen(iter) > max_len)
5340 			return false;
5341 	}
5342 
5343 	return true;
5344 }
5345 
5346 /**
5347  * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
5348  *
5349  * @skb: GSO skb
5350  * @mtu: MTU to validate against
5351  *
5352  * skb_gso_validate_network_len validates if a given skb will fit a
5353  * wanted MTU once split. It considers L3 headers, L4 headers, and the
5354  * payload.
5355  */
5356 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
5357 {
5358 	return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
5359 }
5360 EXPORT_SYMBOL_GPL(skb_gso_validate_network_len);
5361 
5362 /**
5363  * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
5364  *
5365  * @skb: GSO skb
5366  * @len: length to validate against
5367  *
5368  * skb_gso_validate_mac_len validates if a given skb will fit a wanted
5369  * length once split, including L2, L3 and L4 headers and the payload.
5370  */
5371 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
5372 {
5373 	return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
5374 }
5375 EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len);
5376 
5377 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
5378 {
5379 	int mac_len, meta_len;
5380 	void *meta;
5381 
5382 	if (skb_cow(skb, skb_headroom(skb)) < 0) {
5383 		kfree_skb(skb);
5384 		return NULL;
5385 	}
5386 
5387 	mac_len = skb->data - skb_mac_header(skb);
5388 	if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
5389 		memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
5390 			mac_len - VLAN_HLEN - ETH_TLEN);
5391 	}
5392 
5393 	meta_len = skb_metadata_len(skb);
5394 	if (meta_len) {
5395 		meta = skb_metadata_end(skb) - meta_len;
5396 		memmove(meta + VLAN_HLEN, meta, meta_len);
5397 	}
5398 
5399 	skb->mac_header += VLAN_HLEN;
5400 	return skb;
5401 }
5402 
5403 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
5404 {
5405 	struct vlan_hdr *vhdr;
5406 	u16 vlan_tci;
5407 
5408 	if (unlikely(skb_vlan_tag_present(skb))) {
5409 		/* vlan_tci is already set-up so leave this for another time */
5410 		return skb;
5411 	}
5412 
5413 	skb = skb_share_check(skb, GFP_ATOMIC);
5414 	if (unlikely(!skb))
5415 		goto err_free;
5416 
5417 	if (unlikely(!pskb_may_pull(skb, VLAN_HLEN)))
5418 		goto err_free;
5419 
5420 	vhdr = (struct vlan_hdr *)skb->data;
5421 	vlan_tci = ntohs(vhdr->h_vlan_TCI);
5422 	__vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
5423 
5424 	skb_pull_rcsum(skb, VLAN_HLEN);
5425 	vlan_set_encap_proto(skb, vhdr);
5426 
5427 	skb = skb_reorder_vlan_header(skb);
5428 	if (unlikely(!skb))
5429 		goto err_free;
5430 
5431 	skb_reset_network_header(skb);
5432 	skb_reset_transport_header(skb);
5433 	skb_reset_mac_len(skb);
5434 
5435 	return skb;
5436 
5437 err_free:
5438 	kfree_skb(skb);
5439 	return NULL;
5440 }
5441 EXPORT_SYMBOL(skb_vlan_untag);
5442 
5443 int skb_ensure_writable(struct sk_buff *skb, int write_len)
5444 {
5445 	if (!pskb_may_pull(skb, write_len))
5446 		return -ENOMEM;
5447 
5448 	if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
5449 		return 0;
5450 
5451 	return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5452 }
5453 EXPORT_SYMBOL(skb_ensure_writable);
5454 
5455 /* remove VLAN header from packet and update csum accordingly.
5456  * expects a non skb_vlan_tag_present skb with a vlan tag payload
5457  */
5458 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
5459 {
5460 	struct vlan_hdr *vhdr;
5461 	int offset = skb->data - skb_mac_header(skb);
5462 	int err;
5463 
5464 	if (WARN_ONCE(offset,
5465 		      "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
5466 		      offset)) {
5467 		return -EINVAL;
5468 	}
5469 
5470 	err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
5471 	if (unlikely(err))
5472 		return err;
5473 
5474 	skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5475 
5476 	vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
5477 	*vlan_tci = ntohs(vhdr->h_vlan_TCI);
5478 
5479 	memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
5480 	__skb_pull(skb, VLAN_HLEN);
5481 
5482 	vlan_set_encap_proto(skb, vhdr);
5483 	skb->mac_header += VLAN_HLEN;
5484 
5485 	if (skb_network_offset(skb) < ETH_HLEN)
5486 		skb_set_network_header(skb, ETH_HLEN);
5487 
5488 	skb_reset_mac_len(skb);
5489 
5490 	return err;
5491 }
5492 EXPORT_SYMBOL(__skb_vlan_pop);
5493 
5494 /* Pop a vlan tag either from hwaccel or from payload.
5495  * Expects skb->data at mac header.
5496  */
5497 int skb_vlan_pop(struct sk_buff *skb)
5498 {
5499 	u16 vlan_tci;
5500 	__be16 vlan_proto;
5501 	int err;
5502 
5503 	if (likely(skb_vlan_tag_present(skb))) {
5504 		__vlan_hwaccel_clear_tag(skb);
5505 	} else {
5506 		if (unlikely(!eth_type_vlan(skb->protocol)))
5507 			return 0;
5508 
5509 		err = __skb_vlan_pop(skb, &vlan_tci);
5510 		if (err)
5511 			return err;
5512 	}
5513 	/* move next vlan tag to hw accel tag */
5514 	if (likely(!eth_type_vlan(skb->protocol)))
5515 		return 0;
5516 
5517 	vlan_proto = skb->protocol;
5518 	err = __skb_vlan_pop(skb, &vlan_tci);
5519 	if (unlikely(err))
5520 		return err;
5521 
5522 	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5523 	return 0;
5524 }
5525 EXPORT_SYMBOL(skb_vlan_pop);
5526 
5527 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
5528  * Expects skb->data at mac header.
5529  */
5530 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
5531 {
5532 	if (skb_vlan_tag_present(skb)) {
5533 		int offset = skb->data - skb_mac_header(skb);
5534 		int err;
5535 
5536 		if (WARN_ONCE(offset,
5537 			      "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
5538 			      offset)) {
5539 			return -EINVAL;
5540 		}
5541 
5542 		err = __vlan_insert_tag(skb, skb->vlan_proto,
5543 					skb_vlan_tag_get(skb));
5544 		if (err)
5545 			return err;
5546 
5547 		skb->protocol = skb->vlan_proto;
5548 		skb->mac_len += VLAN_HLEN;
5549 
5550 		skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5551 	}
5552 	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5553 	return 0;
5554 }
5555 EXPORT_SYMBOL(skb_vlan_push);
5556 
5557 /* Update the ethertype of hdr and the skb csum value if required. */
5558 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
5559 			     __be16 ethertype)
5560 {
5561 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
5562 		__be16 diff[] = { ~hdr->h_proto, ethertype };
5563 
5564 		skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
5565 	}
5566 
5567 	hdr->h_proto = ethertype;
5568 }
5569 
5570 /**
5571  * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
5572  *                   the packet
5573  *
5574  * @skb: buffer
5575  * @mpls_lse: MPLS label stack entry to push
5576  * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
5577  * @mac_len: length of the MAC header
5578  * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
5579  *            ethernet
5580  *
5581  * Expects skb->data at mac header.
5582  *
5583  * Returns 0 on success, -errno otherwise.
5584  */
5585 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
5586 		  int mac_len, bool ethernet)
5587 {
5588 	struct mpls_shim_hdr *lse;
5589 	int err;
5590 
5591 	if (unlikely(!eth_p_mpls(mpls_proto)))
5592 		return -EINVAL;
5593 
5594 	/* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
5595 	if (skb->encapsulation)
5596 		return -EINVAL;
5597 
5598 	err = skb_cow_head(skb, MPLS_HLEN);
5599 	if (unlikely(err))
5600 		return err;
5601 
5602 	if (!skb->inner_protocol) {
5603 		skb_set_inner_network_header(skb, skb_network_offset(skb));
5604 		skb_set_inner_protocol(skb, skb->protocol);
5605 	}
5606 
5607 	skb_push(skb, MPLS_HLEN);
5608 	memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
5609 		mac_len);
5610 	skb_reset_mac_header(skb);
5611 	skb_set_network_header(skb, mac_len);
5612 	skb_reset_mac_len(skb);
5613 
5614 	lse = mpls_hdr(skb);
5615 	lse->label_stack_entry = mpls_lse;
5616 	skb_postpush_rcsum(skb, lse, MPLS_HLEN);
5617 
5618 	if (ethernet)
5619 		skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
5620 	skb->protocol = mpls_proto;
5621 
5622 	return 0;
5623 }
5624 EXPORT_SYMBOL_GPL(skb_mpls_push);
5625 
5626 /**
5627  * skb_mpls_pop() - pop the outermost MPLS header
5628  *
5629  * @skb: buffer
5630  * @next_proto: ethertype of header after popped MPLS header
5631  * @mac_len: length of the MAC header
5632  * @ethernet: flag to indicate if the packet is ethernet
5633  *
5634  * Expects skb->data at mac header.
5635  *
5636  * Returns 0 on success, -errno otherwise.
5637  */
5638 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
5639 		 bool ethernet)
5640 {
5641 	int err;
5642 
5643 	if (unlikely(!eth_p_mpls(skb->protocol)))
5644 		return 0;
5645 
5646 	err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
5647 	if (unlikely(err))
5648 		return err;
5649 
5650 	skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
5651 	memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
5652 		mac_len);
5653 
5654 	__skb_pull(skb, MPLS_HLEN);
5655 	skb_reset_mac_header(skb);
5656 	skb_set_network_header(skb, mac_len);
5657 
5658 	if (ethernet) {
5659 		struct ethhdr *hdr;
5660 
5661 		/* use mpls_hdr() to get ethertype to account for VLANs. */
5662 		hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
5663 		skb_mod_eth_type(skb, hdr, next_proto);
5664 	}
5665 	skb->protocol = next_proto;
5666 
5667 	return 0;
5668 }
5669 EXPORT_SYMBOL_GPL(skb_mpls_pop);
5670 
5671 /**
5672  * skb_mpls_update_lse() - modify outermost MPLS header and update csum
5673  *
5674  * @skb: buffer
5675  * @mpls_lse: new MPLS label stack entry to update to
5676  *
5677  * Expects skb->data at mac header.
5678  *
5679  * Returns 0 on success, -errno otherwise.
5680  */
5681 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
5682 {
5683 	int err;
5684 
5685 	if (unlikely(!eth_p_mpls(skb->protocol)))
5686 		return -EINVAL;
5687 
5688 	err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
5689 	if (unlikely(err))
5690 		return err;
5691 
5692 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
5693 		__be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
5694 
5695 		skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
5696 	}
5697 
5698 	mpls_hdr(skb)->label_stack_entry = mpls_lse;
5699 
5700 	return 0;
5701 }
5702 EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
5703 
5704 /**
5705  * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
5706  *
5707  * @skb: buffer
5708  *
5709  * Expects skb->data at mac header.
5710  *
5711  * Returns 0 on success, -errno otherwise.
5712  */
5713 int skb_mpls_dec_ttl(struct sk_buff *skb)
5714 {
5715 	u32 lse;
5716 	u8 ttl;
5717 
5718 	if (unlikely(!eth_p_mpls(skb->protocol)))
5719 		return -EINVAL;
5720 
5721 	lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
5722 	ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
5723 	if (!--ttl)
5724 		return -EINVAL;
5725 
5726 	lse &= ~MPLS_LS_TTL_MASK;
5727 	lse |= ttl << MPLS_LS_TTL_SHIFT;
5728 
5729 	return skb_mpls_update_lse(skb, cpu_to_be32(lse));
5730 }
5731 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
5732 
5733 /**
5734  * alloc_skb_with_frags - allocate skb with page frags
5735  *
5736  * @header_len: size of linear part
5737  * @data_len: needed length in frags
5738  * @max_page_order: max page order desired.
5739  * @errcode: pointer to error code if any
5740  * @gfp_mask: allocation mask
5741  *
5742  * This can be used to allocate a paged skb, given a maximal order for frags.
5743  */
5744 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
5745 				     unsigned long data_len,
5746 				     int max_page_order,
5747 				     int *errcode,
5748 				     gfp_t gfp_mask)
5749 {
5750 	int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
5751 	unsigned long chunk;
5752 	struct sk_buff *skb;
5753 	struct page *page;
5754 	int i;
5755 
5756 	*errcode = -EMSGSIZE;
5757 	/* Note this test could be relaxed, if we succeed to allocate
5758 	 * high order pages...
5759 	 */
5760 	if (npages > MAX_SKB_FRAGS)
5761 		return NULL;
5762 
5763 	*errcode = -ENOBUFS;
5764 	skb = alloc_skb(header_len, gfp_mask);
5765 	if (!skb)
5766 		return NULL;
5767 
5768 	skb->truesize += npages << PAGE_SHIFT;
5769 
5770 	for (i = 0; npages > 0; i++) {
5771 		int order = max_page_order;
5772 
5773 		while (order) {
5774 			if (npages >= 1 << order) {
5775 				page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
5776 						   __GFP_COMP |
5777 						   __GFP_NOWARN,
5778 						   order);
5779 				if (page)
5780 					goto fill_page;
5781 				/* Do not retry other high order allocations */
5782 				order = 1;
5783 				max_page_order = 0;
5784 			}
5785 			order--;
5786 		}
5787 		page = alloc_page(gfp_mask);
5788 		if (!page)
5789 			goto failure;
5790 fill_page:
5791 		chunk = min_t(unsigned long, data_len,
5792 			      PAGE_SIZE << order);
5793 		skb_fill_page_desc(skb, i, page, 0, chunk);
5794 		data_len -= chunk;
5795 		npages -= 1 << order;
5796 	}
5797 	return skb;
5798 
5799 failure:
5800 	kfree_skb(skb);
5801 	return NULL;
5802 }
5803 EXPORT_SYMBOL(alloc_skb_with_frags);
5804 
5805 /* carve out the first off bytes from skb when off < headlen */
5806 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
5807 				    const int headlen, gfp_t gfp_mask)
5808 {
5809 	int i;
5810 	int size = skb_end_offset(skb);
5811 	int new_hlen = headlen - off;
5812 	u8 *data;
5813 
5814 	size = SKB_DATA_ALIGN(size);
5815 
5816 	if (skb_pfmemalloc(skb))
5817 		gfp_mask |= __GFP_MEMALLOC;
5818 	data = kmalloc_reserve(size +
5819 			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5820 			       gfp_mask, NUMA_NO_NODE, NULL);
5821 	if (!data)
5822 		return -ENOMEM;
5823 
5824 	size = SKB_WITH_OVERHEAD(ksize(data));
5825 
5826 	/* Copy real data, and all frags */
5827 	skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
5828 	skb->len -= off;
5829 
5830 	memcpy((struct skb_shared_info *)(data + size),
5831 	       skb_shinfo(skb),
5832 	       offsetof(struct skb_shared_info,
5833 			frags[skb_shinfo(skb)->nr_frags]));
5834 	if (skb_cloned(skb)) {
5835 		/* drop the old head gracefully */
5836 		if (skb_orphan_frags(skb, gfp_mask)) {
5837 			kfree(data);
5838 			return -ENOMEM;
5839 		}
5840 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
5841 			skb_frag_ref(skb, i);
5842 		if (skb_has_frag_list(skb))
5843 			skb_clone_fraglist(skb);
5844 		skb_release_data(skb);
5845 	} else {
5846 		/* we can reuse existing recount- all we did was
5847 		 * relocate values
5848 		 */
5849 		skb_free_head(skb);
5850 	}
5851 
5852 	skb->head = data;
5853 	skb->data = data;
5854 	skb->head_frag = 0;
5855 #ifdef NET_SKBUFF_DATA_USES_OFFSET
5856 	skb->end = size;
5857 #else
5858 	skb->end = skb->head + size;
5859 #endif
5860 	skb_set_tail_pointer(skb, skb_headlen(skb));
5861 	skb_headers_offset_update(skb, 0);
5862 	skb->cloned = 0;
5863 	skb->hdr_len = 0;
5864 	skb->nohdr = 0;
5865 	atomic_set(&skb_shinfo(skb)->dataref, 1);
5866 
5867 	return 0;
5868 }
5869 
5870 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
5871 
5872 /* carve out the first eat bytes from skb's frag_list. May recurse into
5873  * pskb_carve()
5874  */
5875 static int pskb_carve_frag_list(struct sk_buff *skb,
5876 				struct skb_shared_info *shinfo, int eat,
5877 				gfp_t gfp_mask)
5878 {
5879 	struct sk_buff *list = shinfo->frag_list;
5880 	struct sk_buff *clone = NULL;
5881 	struct sk_buff *insp = NULL;
5882 
5883 	do {
5884 		if (!list) {
5885 			pr_err("Not enough bytes to eat. Want %d\n", eat);
5886 			return -EFAULT;
5887 		}
5888 		if (list->len <= eat) {
5889 			/* Eaten as whole. */
5890 			eat -= list->len;
5891 			list = list->next;
5892 			insp = list;
5893 		} else {
5894 			/* Eaten partially. */
5895 			if (skb_shared(list)) {
5896 				clone = skb_clone(list, gfp_mask);
5897 				if (!clone)
5898 					return -ENOMEM;
5899 				insp = list->next;
5900 				list = clone;
5901 			} else {
5902 				/* This may be pulled without problems. */
5903 				insp = list;
5904 			}
5905 			if (pskb_carve(list, eat, gfp_mask) < 0) {
5906 				kfree_skb(clone);
5907 				return -ENOMEM;
5908 			}
5909 			break;
5910 		}
5911 	} while (eat);
5912 
5913 	/* Free pulled out fragments. */
5914 	while ((list = shinfo->frag_list) != insp) {
5915 		shinfo->frag_list = list->next;
5916 		kfree_skb(list);
5917 	}
5918 	/* And insert new clone at head. */
5919 	if (clone) {
5920 		clone->next = list;
5921 		shinfo->frag_list = clone;
5922 	}
5923 	return 0;
5924 }
5925 
5926 /* carve off first len bytes from skb. Split line (off) is in the
5927  * non-linear part of skb
5928  */
5929 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
5930 				       int pos, gfp_t gfp_mask)
5931 {
5932 	int i, k = 0;
5933 	int size = skb_end_offset(skb);
5934 	u8 *data;
5935 	const int nfrags = skb_shinfo(skb)->nr_frags;
5936 	struct skb_shared_info *shinfo;
5937 
5938 	size = SKB_DATA_ALIGN(size);
5939 
5940 	if (skb_pfmemalloc(skb))
5941 		gfp_mask |= __GFP_MEMALLOC;
5942 	data = kmalloc_reserve(size +
5943 			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5944 			       gfp_mask, NUMA_NO_NODE, NULL);
5945 	if (!data)
5946 		return -ENOMEM;
5947 
5948 	size = SKB_WITH_OVERHEAD(ksize(data));
5949 
5950 	memcpy((struct skb_shared_info *)(data + size),
5951 	       skb_shinfo(skb), offsetof(struct skb_shared_info,
5952 					 frags[skb_shinfo(skb)->nr_frags]));
5953 	if (skb_orphan_frags(skb, gfp_mask)) {
5954 		kfree(data);
5955 		return -ENOMEM;
5956 	}
5957 	shinfo = (struct skb_shared_info *)(data + size);
5958 	for (i = 0; i < nfrags; i++) {
5959 		int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
5960 
5961 		if (pos + fsize > off) {
5962 			shinfo->frags[k] = skb_shinfo(skb)->frags[i];
5963 
5964 			if (pos < off) {
5965 				/* Split frag.
5966 				 * We have two variants in this case:
5967 				 * 1. Move all the frag to the second
5968 				 *    part, if it is possible. F.e.
5969 				 *    this approach is mandatory for TUX,
5970 				 *    where splitting is expensive.
5971 				 * 2. Split is accurately. We make this.
5972 				 */
5973 				skb_frag_off_add(&shinfo->frags[0], off - pos);
5974 				skb_frag_size_sub(&shinfo->frags[0], off - pos);
5975 			}
5976 			skb_frag_ref(skb, i);
5977 			k++;
5978 		}
5979 		pos += fsize;
5980 	}
5981 	shinfo->nr_frags = k;
5982 	if (skb_has_frag_list(skb))
5983 		skb_clone_fraglist(skb);
5984 
5985 	if (k == 0) {
5986 		/* split line is in frag list */
5987 		pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask);
5988 	}
5989 	skb_release_data(skb);
5990 
5991 	skb->head = data;
5992 	skb->head_frag = 0;
5993 	skb->data = data;
5994 #ifdef NET_SKBUFF_DATA_USES_OFFSET
5995 	skb->end = size;
5996 #else
5997 	skb->end = skb->head + size;
5998 #endif
5999 	skb_reset_tail_pointer(skb);
6000 	skb_headers_offset_update(skb, 0);
6001 	skb->cloned   = 0;
6002 	skb->hdr_len  = 0;
6003 	skb->nohdr    = 0;
6004 	skb->len -= off;
6005 	skb->data_len = skb->len;
6006 	atomic_set(&skb_shinfo(skb)->dataref, 1);
6007 	return 0;
6008 }
6009 
6010 /* remove len bytes from the beginning of the skb */
6011 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
6012 {
6013 	int headlen = skb_headlen(skb);
6014 
6015 	if (len < headlen)
6016 		return pskb_carve_inside_header(skb, len, headlen, gfp);
6017 	else
6018 		return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
6019 }
6020 
6021 /* Extract to_copy bytes starting at off from skb, and return this in
6022  * a new skb
6023  */
6024 struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
6025 			     int to_copy, gfp_t gfp)
6026 {
6027 	struct sk_buff  *clone = skb_clone(skb, gfp);
6028 
6029 	if (!clone)
6030 		return NULL;
6031 
6032 	if (pskb_carve(clone, off, gfp) < 0 ||
6033 	    pskb_trim(clone, to_copy)) {
6034 		kfree_skb(clone);
6035 		return NULL;
6036 	}
6037 	return clone;
6038 }
6039 EXPORT_SYMBOL(pskb_extract);
6040 
6041 /**
6042  * skb_condense - try to get rid of fragments/frag_list if possible
6043  * @skb: buffer
6044  *
6045  * Can be used to save memory before skb is added to a busy queue.
6046  * If packet has bytes in frags and enough tail room in skb->head,
6047  * pull all of them, so that we can free the frags right now and adjust
6048  * truesize.
6049  * Notes:
6050  *	We do not reallocate skb->head thus can not fail.
6051  *	Caller must re-evaluate skb->truesize if needed.
6052  */
6053 void skb_condense(struct sk_buff *skb)
6054 {
6055 	if (skb->data_len) {
6056 		if (skb->data_len > skb->end - skb->tail ||
6057 		    skb_cloned(skb))
6058 			return;
6059 
6060 		/* Nice, we can free page frag(s) right now */
6061 		__pskb_pull_tail(skb, skb->data_len);
6062 	}
6063 	/* At this point, skb->truesize might be over estimated,
6064 	 * because skb had a fragment, and fragments do not tell
6065 	 * their truesize.
6066 	 * When we pulled its content into skb->head, fragment
6067 	 * was freed, but __pskb_pull_tail() could not possibly
6068 	 * adjust skb->truesize, not knowing the frag truesize.
6069 	 */
6070 	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
6071 }
6072 
6073 #ifdef CONFIG_SKB_EXTENSIONS
6074 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
6075 {
6076 	return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
6077 }
6078 
6079 /**
6080  * __skb_ext_alloc - allocate a new skb extensions storage
6081  *
6082  * Returns the newly allocated pointer. The pointer can later attached to a
6083  * skb via __skb_ext_set().
6084  * Note: caller must handle the skb_ext as an opaque data.
6085  */
6086 struct skb_ext *__skb_ext_alloc(void)
6087 {
6088 	struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
6089 
6090 	if (new) {
6091 		memset(new->offset, 0, sizeof(new->offset));
6092 		refcount_set(&new->refcnt, 1);
6093 	}
6094 
6095 	return new;
6096 }
6097 
6098 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
6099 					 unsigned int old_active)
6100 {
6101 	struct skb_ext *new;
6102 
6103 	if (refcount_read(&old->refcnt) == 1)
6104 		return old;
6105 
6106 	new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
6107 	if (!new)
6108 		return NULL;
6109 
6110 	memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
6111 	refcount_set(&new->refcnt, 1);
6112 
6113 #ifdef CONFIG_XFRM
6114 	if (old_active & (1 << SKB_EXT_SEC_PATH)) {
6115 		struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
6116 		unsigned int i;
6117 
6118 		for (i = 0; i < sp->len; i++)
6119 			xfrm_state_hold(sp->xvec[i]);
6120 	}
6121 #endif
6122 	__skb_ext_put(old);
6123 	return new;
6124 }
6125 
6126 /**
6127  * __skb_ext_set - attach the specified extension storage to this skb
6128  * @skb: buffer
6129  * @id: extension id
6130  * @ext: extension storage previously allocated via __skb_ext_alloc()
6131  *
6132  * Existing extensions, if any, are cleared.
6133  *
6134  * Returns the pointer to the extension.
6135  */
6136 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
6137 		    struct skb_ext *ext)
6138 {
6139 	unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
6140 
6141 	skb_ext_put(skb);
6142 	newlen = newoff + skb_ext_type_len[id];
6143 	ext->chunks = newlen;
6144 	ext->offset[id] = newoff;
6145 	skb->extensions = ext;
6146 	skb->active_extensions = 1 << id;
6147 	return skb_ext_get_ptr(ext, id);
6148 }
6149 
6150 /**
6151  * skb_ext_add - allocate space for given extension, COW if needed
6152  * @skb: buffer
6153  * @id: extension to allocate space for
6154  *
6155  * Allocates enough space for the given extension.
6156  * If the extension is already present, a pointer to that extension
6157  * is returned.
6158  *
6159  * If the skb was cloned, COW applies and the returned memory can be
6160  * modified without changing the extension space of clones buffers.
6161  *
6162  * Returns pointer to the extension or NULL on allocation failure.
6163  */
6164 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
6165 {
6166 	struct skb_ext *new, *old = NULL;
6167 	unsigned int newlen, newoff;
6168 
6169 	if (skb->active_extensions) {
6170 		old = skb->extensions;
6171 
6172 		new = skb_ext_maybe_cow(old, skb->active_extensions);
6173 		if (!new)
6174 			return NULL;
6175 
6176 		if (__skb_ext_exist(new, id))
6177 			goto set_active;
6178 
6179 		newoff = new->chunks;
6180 	} else {
6181 		newoff = SKB_EXT_CHUNKSIZEOF(*new);
6182 
6183 		new = __skb_ext_alloc();
6184 		if (!new)
6185 			return NULL;
6186 	}
6187 
6188 	newlen = newoff + skb_ext_type_len[id];
6189 	new->chunks = newlen;
6190 	new->offset[id] = newoff;
6191 set_active:
6192 	skb->extensions = new;
6193 	skb->active_extensions |= 1 << id;
6194 	return skb_ext_get_ptr(new, id);
6195 }
6196 EXPORT_SYMBOL(skb_ext_add);
6197 
6198 #ifdef CONFIG_XFRM
6199 static void skb_ext_put_sp(struct sec_path *sp)
6200 {
6201 	unsigned int i;
6202 
6203 	for (i = 0; i < sp->len; i++)
6204 		xfrm_state_put(sp->xvec[i]);
6205 }
6206 #endif
6207 
6208 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
6209 {
6210 	struct skb_ext *ext = skb->extensions;
6211 
6212 	skb->active_extensions &= ~(1 << id);
6213 	if (skb->active_extensions == 0) {
6214 		skb->extensions = NULL;
6215 		__skb_ext_put(ext);
6216 #ifdef CONFIG_XFRM
6217 	} else if (id == SKB_EXT_SEC_PATH &&
6218 		   refcount_read(&ext->refcnt) == 1) {
6219 		struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
6220 
6221 		skb_ext_put_sp(sp);
6222 		sp->len = 0;
6223 #endif
6224 	}
6225 }
6226 EXPORT_SYMBOL(__skb_ext_del);
6227 
6228 void __skb_ext_put(struct skb_ext *ext)
6229 {
6230 	/* If this is last clone, nothing can increment
6231 	 * it after check passes.  Avoids one atomic op.
6232 	 */
6233 	if (refcount_read(&ext->refcnt) == 1)
6234 		goto free_now;
6235 
6236 	if (!refcount_dec_and_test(&ext->refcnt))
6237 		return;
6238 free_now:
6239 #ifdef CONFIG_XFRM
6240 	if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
6241 		skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
6242 #endif
6243 
6244 	kmem_cache_free(skbuff_ext_cache, ext);
6245 }
6246 EXPORT_SYMBOL(__skb_ext_put);
6247 #endif /* CONFIG_SKB_EXTENSIONS */
6248