xref: /openbmc/linux/net/core/skbuff.c (revision 6f52b16c5b29b89d92c0e7236f4655dc8491ad70)
1 /*
2  *	Routines having to do with the 'struct sk_buff' memory handlers.
3  *
4  *	Authors:	Alan Cox <alan@lxorguk.ukuu.org.uk>
5  *			Florian La Roche <rzsfl@rz.uni-sb.de>
6  *
7  *	Fixes:
8  *		Alan Cox	:	Fixed the worst of the load
9  *					balancer bugs.
10  *		Dave Platt	:	Interrupt stacking fix.
11  *	Richard Kooijman	:	Timestamp fixes.
12  *		Alan Cox	:	Changed buffer format.
13  *		Alan Cox	:	destructor hook for AF_UNIX etc.
14  *		Linus Torvalds	:	Better skb_clone.
15  *		Alan Cox	:	Added skb_copy.
16  *		Alan Cox	:	Added all the changed routines Linus
17  *					only put in the headers
18  *		Ray VanTassle	:	Fixed --skb->lock in free
19  *		Alan Cox	:	skb_copy copy arp field
20  *		Andi Kleen	:	slabified it.
21  *		Robert Olsson	:	Removed skb_head_pool
22  *
23  *	NOTE:
24  *		The __skb_ routines should be called with interrupts
25  *	disabled, or you better be *real* sure that the operation is atomic
26  *	with respect to whatever list is being frobbed (e.g. via lock_sock()
27  *	or via disabling bottom half handlers, etc).
28  *
29  *	This program is free software; you can redistribute it and/or
30  *	modify it under the terms of the GNU General Public License
31  *	as published by the Free Software Foundation; either version
32  *	2 of the License, or (at your option) any later version.
33  */
34 
35 /*
36  *	The functions in this file will not compile correctly with gcc 2.4.x
37  */
38 
39 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
40 
41 #include <linux/module.h>
42 #include <linux/types.h>
43 #include <linux/kernel.h>
44 #include <linux/kmemcheck.h>
45 #include <linux/mm.h>
46 #include <linux/interrupt.h>
47 #include <linux/in.h>
48 #include <linux/inet.h>
49 #include <linux/slab.h>
50 #include <linux/tcp.h>
51 #include <linux/udp.h>
52 #include <linux/sctp.h>
53 #include <linux/netdevice.h>
54 #ifdef CONFIG_NET_CLS_ACT
55 #include <net/pkt_sched.h>
56 #endif
57 #include <linux/string.h>
58 #include <linux/skbuff.h>
59 #include <linux/splice.h>
60 #include <linux/cache.h>
61 #include <linux/rtnetlink.h>
62 #include <linux/init.h>
63 #include <linux/scatterlist.h>
64 #include <linux/errqueue.h>
65 #include <linux/prefetch.h>
66 #include <linux/if_vlan.h>
67 
68 #include <net/protocol.h>
69 #include <net/dst.h>
70 #include <net/sock.h>
71 #include <net/checksum.h>
72 #include <net/ip6_checksum.h>
73 #include <net/xfrm.h>
74 
75 #include <linux/uaccess.h>
76 #include <trace/events/skb.h>
77 #include <linux/highmem.h>
78 #include <linux/capability.h>
79 #include <linux/user_namespace.h>
80 
81 struct kmem_cache *skbuff_head_cache __read_mostly;
82 static struct kmem_cache *skbuff_fclone_cache __read_mostly;
83 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
84 EXPORT_SYMBOL(sysctl_max_skb_frags);
85 
86 /**
87  *	skb_panic - private function for out-of-line support
88  *	@skb:	buffer
89  *	@sz:	size
90  *	@addr:	address
91  *	@msg:	skb_over_panic or skb_under_panic
92  *
93  *	Out-of-line support for skb_put() and skb_push().
94  *	Called via the wrapper skb_over_panic() or skb_under_panic().
95  *	Keep out of line to prevent kernel bloat.
96  *	__builtin_return_address is not used because it is not always reliable.
97  */
98 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
99 		      const char msg[])
100 {
101 	pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
102 		 msg, addr, skb->len, sz, skb->head, skb->data,
103 		 (unsigned long)skb->tail, (unsigned long)skb->end,
104 		 skb->dev ? skb->dev->name : "<NULL>");
105 	BUG();
106 }
107 
108 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
109 {
110 	skb_panic(skb, sz, addr, __func__);
111 }
112 
113 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
114 {
115 	skb_panic(skb, sz, addr, __func__);
116 }
117 
118 /*
119  * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
120  * the caller if emergency pfmemalloc reserves are being used. If it is and
121  * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
122  * may be used. Otherwise, the packet data may be discarded until enough
123  * memory is free
124  */
125 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \
126 	 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
127 
128 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
129 			       unsigned long ip, bool *pfmemalloc)
130 {
131 	void *obj;
132 	bool ret_pfmemalloc = false;
133 
134 	/*
135 	 * Try a regular allocation, when that fails and we're not entitled
136 	 * to the reserves, fail.
137 	 */
138 	obj = kmalloc_node_track_caller(size,
139 					flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
140 					node);
141 	if (obj || !(gfp_pfmemalloc_allowed(flags)))
142 		goto out;
143 
144 	/* Try again but now we are using pfmemalloc reserves */
145 	ret_pfmemalloc = true;
146 	obj = kmalloc_node_track_caller(size, flags, node);
147 
148 out:
149 	if (pfmemalloc)
150 		*pfmemalloc = ret_pfmemalloc;
151 
152 	return obj;
153 }
154 
155 /* 	Allocate a new skbuff. We do this ourselves so we can fill in a few
156  *	'private' fields and also do memory statistics to find all the
157  *	[BEEP] leaks.
158  *
159  */
160 
161 /**
162  *	__alloc_skb	-	allocate a network buffer
163  *	@size: size to allocate
164  *	@gfp_mask: allocation mask
165  *	@flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
166  *		instead of head cache and allocate a cloned (child) skb.
167  *		If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
168  *		allocations in case the data is required for writeback
169  *	@node: numa node to allocate memory on
170  *
171  *	Allocate a new &sk_buff. The returned buffer has no headroom and a
172  *	tail room of at least size bytes. The object has a reference count
173  *	of one. The return is the buffer. On a failure the return is %NULL.
174  *
175  *	Buffers may only be allocated from interrupts using a @gfp_mask of
176  *	%GFP_ATOMIC.
177  */
178 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
179 			    int flags, int node)
180 {
181 	struct kmem_cache *cache;
182 	struct skb_shared_info *shinfo;
183 	struct sk_buff *skb;
184 	u8 *data;
185 	bool pfmemalloc;
186 
187 	cache = (flags & SKB_ALLOC_FCLONE)
188 		? skbuff_fclone_cache : skbuff_head_cache;
189 
190 	if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
191 		gfp_mask |= __GFP_MEMALLOC;
192 
193 	/* Get the HEAD */
194 	skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
195 	if (!skb)
196 		goto out;
197 	prefetchw(skb);
198 
199 	/* We do our best to align skb_shared_info on a separate cache
200 	 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
201 	 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
202 	 * Both skb->head and skb_shared_info are cache line aligned.
203 	 */
204 	size = SKB_DATA_ALIGN(size);
205 	size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
206 	data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
207 	if (!data)
208 		goto nodata;
209 	/* kmalloc(size) might give us more room than requested.
210 	 * Put skb_shared_info exactly at the end of allocated zone,
211 	 * to allow max possible filling before reallocation.
212 	 */
213 	size = SKB_WITH_OVERHEAD(ksize(data));
214 	prefetchw(data + size);
215 
216 	/*
217 	 * Only clear those fields we need to clear, not those that we will
218 	 * actually initialise below. Hence, don't put any more fields after
219 	 * the tail pointer in struct sk_buff!
220 	 */
221 	memset(skb, 0, offsetof(struct sk_buff, tail));
222 	/* Account for allocated memory : skb + skb->head */
223 	skb->truesize = SKB_TRUESIZE(size);
224 	skb->pfmemalloc = pfmemalloc;
225 	refcount_set(&skb->users, 1);
226 	skb->head = data;
227 	skb->data = data;
228 	skb_reset_tail_pointer(skb);
229 	skb->end = skb->tail + size;
230 	skb->mac_header = (typeof(skb->mac_header))~0U;
231 	skb->transport_header = (typeof(skb->transport_header))~0U;
232 
233 	/* make sure we initialize shinfo sequentially */
234 	shinfo = skb_shinfo(skb);
235 	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
236 	atomic_set(&shinfo->dataref, 1);
237 	kmemcheck_annotate_variable(shinfo->destructor_arg);
238 
239 	if (flags & SKB_ALLOC_FCLONE) {
240 		struct sk_buff_fclones *fclones;
241 
242 		fclones = container_of(skb, struct sk_buff_fclones, skb1);
243 
244 		kmemcheck_annotate_bitfield(&fclones->skb2, flags1);
245 		skb->fclone = SKB_FCLONE_ORIG;
246 		refcount_set(&fclones->fclone_ref, 1);
247 
248 		fclones->skb2.fclone = SKB_FCLONE_CLONE;
249 	}
250 out:
251 	return skb;
252 nodata:
253 	kmem_cache_free(cache, skb);
254 	skb = NULL;
255 	goto out;
256 }
257 EXPORT_SYMBOL(__alloc_skb);
258 
259 /**
260  * __build_skb - build a network buffer
261  * @data: data buffer provided by caller
262  * @frag_size: size of data, or 0 if head was kmalloced
263  *
264  * Allocate a new &sk_buff. Caller provides space holding head and
265  * skb_shared_info. @data must have been allocated by kmalloc() only if
266  * @frag_size is 0, otherwise data should come from the page allocator
267  *  or vmalloc()
268  * The return is the new skb buffer.
269  * On a failure the return is %NULL, and @data is not freed.
270  * Notes :
271  *  Before IO, driver allocates only data buffer where NIC put incoming frame
272  *  Driver should add room at head (NET_SKB_PAD) and
273  *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
274  *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
275  *  before giving packet to stack.
276  *  RX rings only contains data buffers, not full skbs.
277  */
278 struct sk_buff *__build_skb(void *data, unsigned int frag_size)
279 {
280 	struct skb_shared_info *shinfo;
281 	struct sk_buff *skb;
282 	unsigned int size = frag_size ? : ksize(data);
283 
284 	skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
285 	if (!skb)
286 		return NULL;
287 
288 	size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
289 
290 	memset(skb, 0, offsetof(struct sk_buff, tail));
291 	skb->truesize = SKB_TRUESIZE(size);
292 	refcount_set(&skb->users, 1);
293 	skb->head = data;
294 	skb->data = data;
295 	skb_reset_tail_pointer(skb);
296 	skb->end = skb->tail + size;
297 	skb->mac_header = (typeof(skb->mac_header))~0U;
298 	skb->transport_header = (typeof(skb->transport_header))~0U;
299 
300 	/* make sure we initialize shinfo sequentially */
301 	shinfo = skb_shinfo(skb);
302 	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
303 	atomic_set(&shinfo->dataref, 1);
304 	kmemcheck_annotate_variable(shinfo->destructor_arg);
305 
306 	return skb;
307 }
308 
309 /* build_skb() is wrapper over __build_skb(), that specifically
310  * takes care of skb->head and skb->pfmemalloc
311  * This means that if @frag_size is not zero, then @data must be backed
312  * by a page fragment, not kmalloc() or vmalloc()
313  */
314 struct sk_buff *build_skb(void *data, unsigned int frag_size)
315 {
316 	struct sk_buff *skb = __build_skb(data, frag_size);
317 
318 	if (skb && frag_size) {
319 		skb->head_frag = 1;
320 		if (page_is_pfmemalloc(virt_to_head_page(data)))
321 			skb->pfmemalloc = 1;
322 	}
323 	return skb;
324 }
325 EXPORT_SYMBOL(build_skb);
326 
327 #define NAPI_SKB_CACHE_SIZE	64
328 
329 struct napi_alloc_cache {
330 	struct page_frag_cache page;
331 	unsigned int skb_count;
332 	void *skb_cache[NAPI_SKB_CACHE_SIZE];
333 };
334 
335 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
336 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
337 
338 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
339 {
340 	struct page_frag_cache *nc;
341 	unsigned long flags;
342 	void *data;
343 
344 	local_irq_save(flags);
345 	nc = this_cpu_ptr(&netdev_alloc_cache);
346 	data = page_frag_alloc(nc, fragsz, gfp_mask);
347 	local_irq_restore(flags);
348 	return data;
349 }
350 
351 /**
352  * netdev_alloc_frag - allocate a page fragment
353  * @fragsz: fragment size
354  *
355  * Allocates a frag from a page for receive buffer.
356  * Uses GFP_ATOMIC allocations.
357  */
358 void *netdev_alloc_frag(unsigned int fragsz)
359 {
360 	return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
361 }
362 EXPORT_SYMBOL(netdev_alloc_frag);
363 
364 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
365 {
366 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
367 
368 	return page_frag_alloc(&nc->page, fragsz, gfp_mask);
369 }
370 
371 void *napi_alloc_frag(unsigned int fragsz)
372 {
373 	return __napi_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
374 }
375 EXPORT_SYMBOL(napi_alloc_frag);
376 
377 /**
378  *	__netdev_alloc_skb - allocate an skbuff for rx on a specific device
379  *	@dev: network device to receive on
380  *	@len: length to allocate
381  *	@gfp_mask: get_free_pages mask, passed to alloc_skb
382  *
383  *	Allocate a new &sk_buff and assign it a usage count of one. The
384  *	buffer has NET_SKB_PAD headroom built in. Users should allocate
385  *	the headroom they think they need without accounting for the
386  *	built in space. The built in space is used for optimisations.
387  *
388  *	%NULL is returned if there is no free memory.
389  */
390 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
391 				   gfp_t gfp_mask)
392 {
393 	struct page_frag_cache *nc;
394 	unsigned long flags;
395 	struct sk_buff *skb;
396 	bool pfmemalloc;
397 	void *data;
398 
399 	len += NET_SKB_PAD;
400 
401 	if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
402 	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
403 		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
404 		if (!skb)
405 			goto skb_fail;
406 		goto skb_success;
407 	}
408 
409 	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
410 	len = SKB_DATA_ALIGN(len);
411 
412 	if (sk_memalloc_socks())
413 		gfp_mask |= __GFP_MEMALLOC;
414 
415 	local_irq_save(flags);
416 
417 	nc = this_cpu_ptr(&netdev_alloc_cache);
418 	data = page_frag_alloc(nc, len, gfp_mask);
419 	pfmemalloc = nc->pfmemalloc;
420 
421 	local_irq_restore(flags);
422 
423 	if (unlikely(!data))
424 		return NULL;
425 
426 	skb = __build_skb(data, len);
427 	if (unlikely(!skb)) {
428 		skb_free_frag(data);
429 		return NULL;
430 	}
431 
432 	/* use OR instead of assignment to avoid clearing of bits in mask */
433 	if (pfmemalloc)
434 		skb->pfmemalloc = 1;
435 	skb->head_frag = 1;
436 
437 skb_success:
438 	skb_reserve(skb, NET_SKB_PAD);
439 	skb->dev = dev;
440 
441 skb_fail:
442 	return skb;
443 }
444 EXPORT_SYMBOL(__netdev_alloc_skb);
445 
446 /**
447  *	__napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
448  *	@napi: napi instance this buffer was allocated for
449  *	@len: length to allocate
450  *	@gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
451  *
452  *	Allocate a new sk_buff for use in NAPI receive.  This buffer will
453  *	attempt to allocate the head from a special reserved region used
454  *	only for NAPI Rx allocation.  By doing this we can save several
455  *	CPU cycles by avoiding having to disable and re-enable IRQs.
456  *
457  *	%NULL is returned if there is no free memory.
458  */
459 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
460 				 gfp_t gfp_mask)
461 {
462 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
463 	struct sk_buff *skb;
464 	void *data;
465 
466 	len += NET_SKB_PAD + NET_IP_ALIGN;
467 
468 	if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
469 	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
470 		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
471 		if (!skb)
472 			goto skb_fail;
473 		goto skb_success;
474 	}
475 
476 	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
477 	len = SKB_DATA_ALIGN(len);
478 
479 	if (sk_memalloc_socks())
480 		gfp_mask |= __GFP_MEMALLOC;
481 
482 	data = page_frag_alloc(&nc->page, len, gfp_mask);
483 	if (unlikely(!data))
484 		return NULL;
485 
486 	skb = __build_skb(data, len);
487 	if (unlikely(!skb)) {
488 		skb_free_frag(data);
489 		return NULL;
490 	}
491 
492 	/* use OR instead of assignment to avoid clearing of bits in mask */
493 	if (nc->page.pfmemalloc)
494 		skb->pfmemalloc = 1;
495 	skb->head_frag = 1;
496 
497 skb_success:
498 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
499 	skb->dev = napi->dev;
500 
501 skb_fail:
502 	return skb;
503 }
504 EXPORT_SYMBOL(__napi_alloc_skb);
505 
506 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
507 		     int size, unsigned int truesize)
508 {
509 	skb_fill_page_desc(skb, i, page, off, size);
510 	skb->len += size;
511 	skb->data_len += size;
512 	skb->truesize += truesize;
513 }
514 EXPORT_SYMBOL(skb_add_rx_frag);
515 
516 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
517 			  unsigned int truesize)
518 {
519 	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
520 
521 	skb_frag_size_add(frag, size);
522 	skb->len += size;
523 	skb->data_len += size;
524 	skb->truesize += truesize;
525 }
526 EXPORT_SYMBOL(skb_coalesce_rx_frag);
527 
528 static void skb_drop_list(struct sk_buff **listp)
529 {
530 	kfree_skb_list(*listp);
531 	*listp = NULL;
532 }
533 
534 static inline void skb_drop_fraglist(struct sk_buff *skb)
535 {
536 	skb_drop_list(&skb_shinfo(skb)->frag_list);
537 }
538 
539 static void skb_clone_fraglist(struct sk_buff *skb)
540 {
541 	struct sk_buff *list;
542 
543 	skb_walk_frags(skb, list)
544 		skb_get(list);
545 }
546 
547 static void skb_free_head(struct sk_buff *skb)
548 {
549 	unsigned char *head = skb->head;
550 
551 	if (skb->head_frag)
552 		skb_free_frag(head);
553 	else
554 		kfree(head);
555 }
556 
557 static void skb_release_data(struct sk_buff *skb)
558 {
559 	struct skb_shared_info *shinfo = skb_shinfo(skb);
560 	int i;
561 
562 	if (skb->cloned &&
563 	    atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
564 			      &shinfo->dataref))
565 		return;
566 
567 	for (i = 0; i < shinfo->nr_frags; i++)
568 		__skb_frag_unref(&shinfo->frags[i]);
569 
570 	if (shinfo->frag_list)
571 		kfree_skb_list(shinfo->frag_list);
572 
573 	skb_zcopy_clear(skb, true);
574 	skb_free_head(skb);
575 }
576 
577 /*
578  *	Free an skbuff by memory without cleaning the state.
579  */
580 static void kfree_skbmem(struct sk_buff *skb)
581 {
582 	struct sk_buff_fclones *fclones;
583 
584 	switch (skb->fclone) {
585 	case SKB_FCLONE_UNAVAILABLE:
586 		kmem_cache_free(skbuff_head_cache, skb);
587 		return;
588 
589 	case SKB_FCLONE_ORIG:
590 		fclones = container_of(skb, struct sk_buff_fclones, skb1);
591 
592 		/* We usually free the clone (TX completion) before original skb
593 		 * This test would have no chance to be true for the clone,
594 		 * while here, branch prediction will be good.
595 		 */
596 		if (refcount_read(&fclones->fclone_ref) == 1)
597 			goto fastpath;
598 		break;
599 
600 	default: /* SKB_FCLONE_CLONE */
601 		fclones = container_of(skb, struct sk_buff_fclones, skb2);
602 		break;
603 	}
604 	if (!refcount_dec_and_test(&fclones->fclone_ref))
605 		return;
606 fastpath:
607 	kmem_cache_free(skbuff_fclone_cache, fclones);
608 }
609 
610 void skb_release_head_state(struct sk_buff *skb)
611 {
612 	skb_dst_drop(skb);
613 	secpath_reset(skb);
614 	if (skb->destructor) {
615 		WARN_ON(in_irq());
616 		skb->destructor(skb);
617 	}
618 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
619 	nf_conntrack_put(skb_nfct(skb));
620 #endif
621 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
622 	nf_bridge_put(skb->nf_bridge);
623 #endif
624 }
625 
626 /* Free everything but the sk_buff shell. */
627 static void skb_release_all(struct sk_buff *skb)
628 {
629 	skb_release_head_state(skb);
630 	if (likely(skb->head))
631 		skb_release_data(skb);
632 }
633 
634 /**
635  *	__kfree_skb - private function
636  *	@skb: buffer
637  *
638  *	Free an sk_buff. Release anything attached to the buffer.
639  *	Clean the state. This is an internal helper function. Users should
640  *	always call kfree_skb
641  */
642 
643 void __kfree_skb(struct sk_buff *skb)
644 {
645 	skb_release_all(skb);
646 	kfree_skbmem(skb);
647 }
648 EXPORT_SYMBOL(__kfree_skb);
649 
650 /**
651  *	kfree_skb - free an sk_buff
652  *	@skb: buffer to free
653  *
654  *	Drop a reference to the buffer and free it if the usage count has
655  *	hit zero.
656  */
657 void kfree_skb(struct sk_buff *skb)
658 {
659 	if (!skb_unref(skb))
660 		return;
661 
662 	trace_kfree_skb(skb, __builtin_return_address(0));
663 	__kfree_skb(skb);
664 }
665 EXPORT_SYMBOL(kfree_skb);
666 
667 void kfree_skb_list(struct sk_buff *segs)
668 {
669 	while (segs) {
670 		struct sk_buff *next = segs->next;
671 
672 		kfree_skb(segs);
673 		segs = next;
674 	}
675 }
676 EXPORT_SYMBOL(kfree_skb_list);
677 
678 /**
679  *	skb_tx_error - report an sk_buff xmit error
680  *	@skb: buffer that triggered an error
681  *
682  *	Report xmit error if a device callback is tracking this skb.
683  *	skb must be freed afterwards.
684  */
685 void skb_tx_error(struct sk_buff *skb)
686 {
687 	skb_zcopy_clear(skb, true);
688 }
689 EXPORT_SYMBOL(skb_tx_error);
690 
691 /**
692  *	consume_skb - free an skbuff
693  *	@skb: buffer to free
694  *
695  *	Drop a ref to the buffer and free it if the usage count has hit zero
696  *	Functions identically to kfree_skb, but kfree_skb assumes that the frame
697  *	is being dropped after a failure and notes that
698  */
699 void consume_skb(struct sk_buff *skb)
700 {
701 	if (!skb_unref(skb))
702 		return;
703 
704 	trace_consume_skb(skb);
705 	__kfree_skb(skb);
706 }
707 EXPORT_SYMBOL(consume_skb);
708 
709 /**
710  *	consume_stateless_skb - free an skbuff, assuming it is stateless
711  *	@skb: buffer to free
712  *
713  *	Alike consume_skb(), but this variant assumes that this is the last
714  *	skb reference and all the head states have been already dropped
715  */
716 void __consume_stateless_skb(struct sk_buff *skb)
717 {
718 	trace_consume_skb(skb);
719 	skb_release_data(skb);
720 	kfree_skbmem(skb);
721 }
722 
723 void __kfree_skb_flush(void)
724 {
725 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
726 
727 	/* flush skb_cache if containing objects */
728 	if (nc->skb_count) {
729 		kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
730 				     nc->skb_cache);
731 		nc->skb_count = 0;
732 	}
733 }
734 
735 static inline void _kfree_skb_defer(struct sk_buff *skb)
736 {
737 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
738 
739 	/* drop skb->head and call any destructors for packet */
740 	skb_release_all(skb);
741 
742 	/* record skb to CPU local list */
743 	nc->skb_cache[nc->skb_count++] = skb;
744 
745 #ifdef CONFIG_SLUB
746 	/* SLUB writes into objects when freeing */
747 	prefetchw(skb);
748 #endif
749 
750 	/* flush skb_cache if it is filled */
751 	if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
752 		kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE,
753 				     nc->skb_cache);
754 		nc->skb_count = 0;
755 	}
756 }
757 void __kfree_skb_defer(struct sk_buff *skb)
758 {
759 	_kfree_skb_defer(skb);
760 }
761 
762 void napi_consume_skb(struct sk_buff *skb, int budget)
763 {
764 	if (unlikely(!skb))
765 		return;
766 
767 	/* Zero budget indicate non-NAPI context called us, like netpoll */
768 	if (unlikely(!budget)) {
769 		dev_consume_skb_any(skb);
770 		return;
771 	}
772 
773 	if (!skb_unref(skb))
774 		return;
775 
776 	/* if reaching here SKB is ready to free */
777 	trace_consume_skb(skb);
778 
779 	/* if SKB is a clone, don't handle this case */
780 	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
781 		__kfree_skb(skb);
782 		return;
783 	}
784 
785 	_kfree_skb_defer(skb);
786 }
787 EXPORT_SYMBOL(napi_consume_skb);
788 
789 /* Make sure a field is enclosed inside headers_start/headers_end section */
790 #define CHECK_SKB_FIELD(field) \
791 	BUILD_BUG_ON(offsetof(struct sk_buff, field) <		\
792 		     offsetof(struct sk_buff, headers_start));	\
793 	BUILD_BUG_ON(offsetof(struct sk_buff, field) >		\
794 		     offsetof(struct sk_buff, headers_end));	\
795 
796 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
797 {
798 	new->tstamp		= old->tstamp;
799 	/* We do not copy old->sk */
800 	new->dev		= old->dev;
801 	memcpy(new->cb, old->cb, sizeof(old->cb));
802 	skb_dst_copy(new, old);
803 #ifdef CONFIG_XFRM
804 	new->sp			= secpath_get(old->sp);
805 #endif
806 	__nf_copy(new, old, false);
807 
808 	/* Note : this field could be in headers_start/headers_end section
809 	 * It is not yet because we do not want to have a 16 bit hole
810 	 */
811 	new->queue_mapping = old->queue_mapping;
812 
813 	memcpy(&new->headers_start, &old->headers_start,
814 	       offsetof(struct sk_buff, headers_end) -
815 	       offsetof(struct sk_buff, headers_start));
816 	CHECK_SKB_FIELD(protocol);
817 	CHECK_SKB_FIELD(csum);
818 	CHECK_SKB_FIELD(hash);
819 	CHECK_SKB_FIELD(priority);
820 	CHECK_SKB_FIELD(skb_iif);
821 	CHECK_SKB_FIELD(vlan_proto);
822 	CHECK_SKB_FIELD(vlan_tci);
823 	CHECK_SKB_FIELD(transport_header);
824 	CHECK_SKB_FIELD(network_header);
825 	CHECK_SKB_FIELD(mac_header);
826 	CHECK_SKB_FIELD(inner_protocol);
827 	CHECK_SKB_FIELD(inner_transport_header);
828 	CHECK_SKB_FIELD(inner_network_header);
829 	CHECK_SKB_FIELD(inner_mac_header);
830 	CHECK_SKB_FIELD(mark);
831 #ifdef CONFIG_NETWORK_SECMARK
832 	CHECK_SKB_FIELD(secmark);
833 #endif
834 #ifdef CONFIG_NET_RX_BUSY_POLL
835 	CHECK_SKB_FIELD(napi_id);
836 #endif
837 #ifdef CONFIG_XPS
838 	CHECK_SKB_FIELD(sender_cpu);
839 #endif
840 #ifdef CONFIG_NET_SCHED
841 	CHECK_SKB_FIELD(tc_index);
842 #endif
843 
844 }
845 
846 /*
847  * You should not add any new code to this function.  Add it to
848  * __copy_skb_header above instead.
849  */
850 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
851 {
852 #define C(x) n->x = skb->x
853 
854 	n->next = n->prev = NULL;
855 	n->sk = NULL;
856 	__copy_skb_header(n, skb);
857 
858 	C(len);
859 	C(data_len);
860 	C(mac_len);
861 	n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
862 	n->cloned = 1;
863 	n->nohdr = 0;
864 	n->destructor = NULL;
865 	C(tail);
866 	C(end);
867 	C(head);
868 	C(head_frag);
869 	C(data);
870 	C(truesize);
871 	refcount_set(&n->users, 1);
872 
873 	atomic_inc(&(skb_shinfo(skb)->dataref));
874 	skb->cloned = 1;
875 
876 	return n;
877 #undef C
878 }
879 
880 /**
881  *	skb_morph	-	morph one skb into another
882  *	@dst: the skb to receive the contents
883  *	@src: the skb to supply the contents
884  *
885  *	This is identical to skb_clone except that the target skb is
886  *	supplied by the user.
887  *
888  *	The target skb is returned upon exit.
889  */
890 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
891 {
892 	skb_release_all(dst);
893 	return __skb_clone(dst, src);
894 }
895 EXPORT_SYMBOL_GPL(skb_morph);
896 
897 static int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
898 {
899 	unsigned long max_pg, num_pg, new_pg, old_pg;
900 	struct user_struct *user;
901 
902 	if (capable(CAP_IPC_LOCK) || !size)
903 		return 0;
904 
905 	num_pg = (size >> PAGE_SHIFT) + 2;	/* worst case */
906 	max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
907 	user = mmp->user ? : current_user();
908 
909 	do {
910 		old_pg = atomic_long_read(&user->locked_vm);
911 		new_pg = old_pg + num_pg;
912 		if (new_pg > max_pg)
913 			return -ENOBUFS;
914 	} while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
915 		 old_pg);
916 
917 	if (!mmp->user) {
918 		mmp->user = get_uid(user);
919 		mmp->num_pg = num_pg;
920 	} else {
921 		mmp->num_pg += num_pg;
922 	}
923 
924 	return 0;
925 }
926 
927 static void mm_unaccount_pinned_pages(struct mmpin *mmp)
928 {
929 	if (mmp->user) {
930 		atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
931 		free_uid(mmp->user);
932 	}
933 }
934 
935 struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size)
936 {
937 	struct ubuf_info *uarg;
938 	struct sk_buff *skb;
939 
940 	WARN_ON_ONCE(!in_task());
941 
942 	if (!sock_flag(sk, SOCK_ZEROCOPY))
943 		return NULL;
944 
945 	skb = sock_omalloc(sk, 0, GFP_KERNEL);
946 	if (!skb)
947 		return NULL;
948 
949 	BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
950 	uarg = (void *)skb->cb;
951 	uarg->mmp.user = NULL;
952 
953 	if (mm_account_pinned_pages(&uarg->mmp, size)) {
954 		kfree_skb(skb);
955 		return NULL;
956 	}
957 
958 	uarg->callback = sock_zerocopy_callback;
959 	uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
960 	uarg->len = 1;
961 	uarg->bytelen = size;
962 	uarg->zerocopy = 1;
963 	refcount_set(&uarg->refcnt, 1);
964 	sock_hold(sk);
965 
966 	return uarg;
967 }
968 EXPORT_SYMBOL_GPL(sock_zerocopy_alloc);
969 
970 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
971 {
972 	return container_of((void *)uarg, struct sk_buff, cb);
973 }
974 
975 struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
976 					struct ubuf_info *uarg)
977 {
978 	if (uarg) {
979 		const u32 byte_limit = 1 << 19;		/* limit to a few TSO */
980 		u32 bytelen, next;
981 
982 		/* realloc only when socket is locked (TCP, UDP cork),
983 		 * so uarg->len and sk_zckey access is serialized
984 		 */
985 		if (!sock_owned_by_user(sk)) {
986 			WARN_ON_ONCE(1);
987 			return NULL;
988 		}
989 
990 		bytelen = uarg->bytelen + size;
991 		if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
992 			/* TCP can create new skb to attach new uarg */
993 			if (sk->sk_type == SOCK_STREAM)
994 				goto new_alloc;
995 			return NULL;
996 		}
997 
998 		next = (u32)atomic_read(&sk->sk_zckey);
999 		if ((u32)(uarg->id + uarg->len) == next) {
1000 			if (mm_account_pinned_pages(&uarg->mmp, size))
1001 				return NULL;
1002 			uarg->len++;
1003 			uarg->bytelen = bytelen;
1004 			atomic_set(&sk->sk_zckey, ++next);
1005 			sock_zerocopy_get(uarg);
1006 			return uarg;
1007 		}
1008 	}
1009 
1010 new_alloc:
1011 	return sock_zerocopy_alloc(sk, size);
1012 }
1013 EXPORT_SYMBOL_GPL(sock_zerocopy_realloc);
1014 
1015 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1016 {
1017 	struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1018 	u32 old_lo, old_hi;
1019 	u64 sum_len;
1020 
1021 	old_lo = serr->ee.ee_info;
1022 	old_hi = serr->ee.ee_data;
1023 	sum_len = old_hi - old_lo + 1ULL + len;
1024 
1025 	if (sum_len >= (1ULL << 32))
1026 		return false;
1027 
1028 	if (lo != old_hi + 1)
1029 		return false;
1030 
1031 	serr->ee.ee_data += len;
1032 	return true;
1033 }
1034 
1035 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
1036 {
1037 	struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1038 	struct sock_exterr_skb *serr;
1039 	struct sock *sk = skb->sk;
1040 	struct sk_buff_head *q;
1041 	unsigned long flags;
1042 	u32 lo, hi;
1043 	u16 len;
1044 
1045 	mm_unaccount_pinned_pages(&uarg->mmp);
1046 
1047 	/* if !len, there was only 1 call, and it was aborted
1048 	 * so do not queue a completion notification
1049 	 */
1050 	if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1051 		goto release;
1052 
1053 	len = uarg->len;
1054 	lo = uarg->id;
1055 	hi = uarg->id + len - 1;
1056 
1057 	serr = SKB_EXT_ERR(skb);
1058 	memset(serr, 0, sizeof(*serr));
1059 	serr->ee.ee_errno = 0;
1060 	serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1061 	serr->ee.ee_data = hi;
1062 	serr->ee.ee_info = lo;
1063 	if (!success)
1064 		serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1065 
1066 	q = &sk->sk_error_queue;
1067 	spin_lock_irqsave(&q->lock, flags);
1068 	tail = skb_peek_tail(q);
1069 	if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1070 	    !skb_zerocopy_notify_extend(tail, lo, len)) {
1071 		__skb_queue_tail(q, skb);
1072 		skb = NULL;
1073 	}
1074 	spin_unlock_irqrestore(&q->lock, flags);
1075 
1076 	sk->sk_error_report(sk);
1077 
1078 release:
1079 	consume_skb(skb);
1080 	sock_put(sk);
1081 }
1082 EXPORT_SYMBOL_GPL(sock_zerocopy_callback);
1083 
1084 void sock_zerocopy_put(struct ubuf_info *uarg)
1085 {
1086 	if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
1087 		if (uarg->callback)
1088 			uarg->callback(uarg, uarg->zerocopy);
1089 		else
1090 			consume_skb(skb_from_uarg(uarg));
1091 	}
1092 }
1093 EXPORT_SYMBOL_GPL(sock_zerocopy_put);
1094 
1095 void sock_zerocopy_put_abort(struct ubuf_info *uarg)
1096 {
1097 	if (uarg) {
1098 		struct sock *sk = skb_from_uarg(uarg)->sk;
1099 
1100 		atomic_dec(&sk->sk_zckey);
1101 		uarg->len--;
1102 
1103 		sock_zerocopy_put(uarg);
1104 	}
1105 }
1106 EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);
1107 
1108 extern int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb,
1109 				   struct iov_iter *from, size_t length);
1110 
1111 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1112 			     struct msghdr *msg, int len,
1113 			     struct ubuf_info *uarg)
1114 {
1115 	struct ubuf_info *orig_uarg = skb_zcopy(skb);
1116 	struct iov_iter orig_iter = msg->msg_iter;
1117 	int err, orig_len = skb->len;
1118 
1119 	/* An skb can only point to one uarg. This edge case happens when
1120 	 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1121 	 */
1122 	if (orig_uarg && uarg != orig_uarg)
1123 		return -EEXIST;
1124 
1125 	err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
1126 	if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1127 		struct sock *save_sk = skb->sk;
1128 
1129 		/* Streams do not free skb on error. Reset to prev state. */
1130 		msg->msg_iter = orig_iter;
1131 		skb->sk = sk;
1132 		___pskb_trim(skb, orig_len);
1133 		skb->sk = save_sk;
1134 		return err;
1135 	}
1136 
1137 	skb_zcopy_set(skb, uarg);
1138 	return skb->len - orig_len;
1139 }
1140 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1141 
1142 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1143 			      gfp_t gfp_mask)
1144 {
1145 	if (skb_zcopy(orig)) {
1146 		if (skb_zcopy(nskb)) {
1147 			/* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1148 			if (!gfp_mask) {
1149 				WARN_ON_ONCE(1);
1150 				return -ENOMEM;
1151 			}
1152 			if (skb_uarg(nskb) == skb_uarg(orig))
1153 				return 0;
1154 			if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1155 				return -EIO;
1156 		}
1157 		skb_zcopy_set(nskb, skb_uarg(orig));
1158 	}
1159 	return 0;
1160 }
1161 
1162 /**
1163  *	skb_copy_ubufs	-	copy userspace skb frags buffers to kernel
1164  *	@skb: the skb to modify
1165  *	@gfp_mask: allocation priority
1166  *
1167  *	This must be called on SKBTX_DEV_ZEROCOPY skb.
1168  *	It will copy all frags into kernel and drop the reference
1169  *	to userspace pages.
1170  *
1171  *	If this function is called from an interrupt gfp_mask() must be
1172  *	%GFP_ATOMIC.
1173  *
1174  *	Returns 0 on success or a negative error code on failure
1175  *	to allocate kernel memory to copy to.
1176  */
1177 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1178 {
1179 	int num_frags = skb_shinfo(skb)->nr_frags;
1180 	struct page *page, *head = NULL;
1181 	int i, new_frags;
1182 	u32 d_off;
1183 
1184 	if (!num_frags)
1185 		return 0;
1186 
1187 	if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1188 		return -EINVAL;
1189 
1190 	new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1191 	for (i = 0; i < new_frags; i++) {
1192 		page = alloc_page(gfp_mask);
1193 		if (!page) {
1194 			while (head) {
1195 				struct page *next = (struct page *)page_private(head);
1196 				put_page(head);
1197 				head = next;
1198 			}
1199 			return -ENOMEM;
1200 		}
1201 		set_page_private(page, (unsigned long)head);
1202 		head = page;
1203 	}
1204 
1205 	page = head;
1206 	d_off = 0;
1207 	for (i = 0; i < num_frags; i++) {
1208 		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1209 		u32 p_off, p_len, copied;
1210 		struct page *p;
1211 		u8 *vaddr;
1212 
1213 		skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f),
1214 				      p, p_off, p_len, copied) {
1215 			u32 copy, done = 0;
1216 			vaddr = kmap_atomic(p);
1217 
1218 			while (done < p_len) {
1219 				if (d_off == PAGE_SIZE) {
1220 					d_off = 0;
1221 					page = (struct page *)page_private(page);
1222 				}
1223 				copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
1224 				memcpy(page_address(page) + d_off,
1225 				       vaddr + p_off + done, copy);
1226 				done += copy;
1227 				d_off += copy;
1228 			}
1229 			kunmap_atomic(vaddr);
1230 		}
1231 	}
1232 
1233 	/* skb frags release userspace buffers */
1234 	for (i = 0; i < num_frags; i++)
1235 		skb_frag_unref(skb, i);
1236 
1237 	/* skb frags point to kernel buffers */
1238 	for (i = 0; i < new_frags - 1; i++) {
1239 		__skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
1240 		head = (struct page *)page_private(head);
1241 	}
1242 	__skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
1243 	skb_shinfo(skb)->nr_frags = new_frags;
1244 
1245 	skb_zcopy_clear(skb, false);
1246 	return 0;
1247 }
1248 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
1249 
1250 /**
1251  *	skb_clone	-	duplicate an sk_buff
1252  *	@skb: buffer to clone
1253  *	@gfp_mask: allocation priority
1254  *
1255  *	Duplicate an &sk_buff. The new one is not owned by a socket. Both
1256  *	copies share the same packet data but not structure. The new
1257  *	buffer has a reference count of 1. If the allocation fails the
1258  *	function returns %NULL otherwise the new buffer is returned.
1259  *
1260  *	If this function is called from an interrupt gfp_mask() must be
1261  *	%GFP_ATOMIC.
1262  */
1263 
1264 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
1265 {
1266 	struct sk_buff_fclones *fclones = container_of(skb,
1267 						       struct sk_buff_fclones,
1268 						       skb1);
1269 	struct sk_buff *n;
1270 
1271 	if (skb_orphan_frags(skb, gfp_mask))
1272 		return NULL;
1273 
1274 	if (skb->fclone == SKB_FCLONE_ORIG &&
1275 	    refcount_read(&fclones->fclone_ref) == 1) {
1276 		n = &fclones->skb2;
1277 		refcount_set(&fclones->fclone_ref, 2);
1278 	} else {
1279 		if (skb_pfmemalloc(skb))
1280 			gfp_mask |= __GFP_MEMALLOC;
1281 
1282 		n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
1283 		if (!n)
1284 			return NULL;
1285 
1286 		kmemcheck_annotate_bitfield(n, flags1);
1287 		n->fclone = SKB_FCLONE_UNAVAILABLE;
1288 	}
1289 
1290 	return __skb_clone(n, skb);
1291 }
1292 EXPORT_SYMBOL(skb_clone);
1293 
1294 static void skb_headers_offset_update(struct sk_buff *skb, int off)
1295 {
1296 	/* Only adjust this if it actually is csum_start rather than csum */
1297 	if (skb->ip_summed == CHECKSUM_PARTIAL)
1298 		skb->csum_start += off;
1299 	/* {transport,network,mac}_header and tail are relative to skb->head */
1300 	skb->transport_header += off;
1301 	skb->network_header   += off;
1302 	if (skb_mac_header_was_set(skb))
1303 		skb->mac_header += off;
1304 	skb->inner_transport_header += off;
1305 	skb->inner_network_header += off;
1306 	skb->inner_mac_header += off;
1307 }
1308 
1309 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1310 {
1311 	__copy_skb_header(new, old);
1312 
1313 	skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
1314 	skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
1315 	skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
1316 }
1317 
1318 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
1319 {
1320 	if (skb_pfmemalloc(skb))
1321 		return SKB_ALLOC_RX;
1322 	return 0;
1323 }
1324 
1325 /**
1326  *	skb_copy	-	create private copy of an sk_buff
1327  *	@skb: buffer to copy
1328  *	@gfp_mask: allocation priority
1329  *
1330  *	Make a copy of both an &sk_buff and its data. This is used when the
1331  *	caller wishes to modify the data and needs a private copy of the
1332  *	data to alter. Returns %NULL on failure or the pointer to the buffer
1333  *	on success. The returned buffer has a reference count of 1.
1334  *
1335  *	As by-product this function converts non-linear &sk_buff to linear
1336  *	one, so that &sk_buff becomes completely private and caller is allowed
1337  *	to modify all the data of returned buffer. This means that this
1338  *	function is not recommended for use in circumstances when only
1339  *	header is going to be modified. Use pskb_copy() instead.
1340  */
1341 
1342 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
1343 {
1344 	int headerlen = skb_headroom(skb);
1345 	unsigned int size = skb_end_offset(skb) + skb->data_len;
1346 	struct sk_buff *n = __alloc_skb(size, gfp_mask,
1347 					skb_alloc_rx_flag(skb), NUMA_NO_NODE);
1348 
1349 	if (!n)
1350 		return NULL;
1351 
1352 	/* Set the data pointer */
1353 	skb_reserve(n, headerlen);
1354 	/* Set the tail pointer and length */
1355 	skb_put(n, skb->len);
1356 
1357 	if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
1358 		BUG();
1359 
1360 	copy_skb_header(n, skb);
1361 	return n;
1362 }
1363 EXPORT_SYMBOL(skb_copy);
1364 
1365 /**
1366  *	__pskb_copy_fclone	-  create copy of an sk_buff with private head.
1367  *	@skb: buffer to copy
1368  *	@headroom: headroom of new skb
1369  *	@gfp_mask: allocation priority
1370  *	@fclone: if true allocate the copy of the skb from the fclone
1371  *	cache instead of the head cache; it is recommended to set this
1372  *	to true for the cases where the copy will likely be cloned
1373  *
1374  *	Make a copy of both an &sk_buff and part of its data, located
1375  *	in header. Fragmented data remain shared. This is used when
1376  *	the caller wishes to modify only header of &sk_buff and needs
1377  *	private copy of the header to alter. Returns %NULL on failure
1378  *	or the pointer to the buffer on success.
1379  *	The returned buffer has a reference count of 1.
1380  */
1381 
1382 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1383 				   gfp_t gfp_mask, bool fclone)
1384 {
1385 	unsigned int size = skb_headlen(skb) + headroom;
1386 	int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
1387 	struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
1388 
1389 	if (!n)
1390 		goto out;
1391 
1392 	/* Set the data pointer */
1393 	skb_reserve(n, headroom);
1394 	/* Set the tail pointer and length */
1395 	skb_put(n, skb_headlen(skb));
1396 	/* Copy the bytes */
1397 	skb_copy_from_linear_data(skb, n->data, n->len);
1398 
1399 	n->truesize += skb->data_len;
1400 	n->data_len  = skb->data_len;
1401 	n->len	     = skb->len;
1402 
1403 	if (skb_shinfo(skb)->nr_frags) {
1404 		int i;
1405 
1406 		if (skb_orphan_frags(skb, gfp_mask) ||
1407 		    skb_zerocopy_clone(n, skb, gfp_mask)) {
1408 			kfree_skb(n);
1409 			n = NULL;
1410 			goto out;
1411 		}
1412 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1413 			skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1414 			skb_frag_ref(skb, i);
1415 		}
1416 		skb_shinfo(n)->nr_frags = i;
1417 	}
1418 
1419 	if (skb_has_frag_list(skb)) {
1420 		skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1421 		skb_clone_fraglist(n);
1422 	}
1423 
1424 	copy_skb_header(n, skb);
1425 out:
1426 	return n;
1427 }
1428 EXPORT_SYMBOL(__pskb_copy_fclone);
1429 
1430 /**
1431  *	pskb_expand_head - reallocate header of &sk_buff
1432  *	@skb: buffer to reallocate
1433  *	@nhead: room to add at head
1434  *	@ntail: room to add at tail
1435  *	@gfp_mask: allocation priority
1436  *
1437  *	Expands (or creates identical copy, if @nhead and @ntail are zero)
1438  *	header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
1439  *	reference count of 1. Returns zero in the case of success or error,
1440  *	if expansion failed. In the last case, &sk_buff is not changed.
1441  *
1442  *	All the pointers pointing into skb header may change and must be
1443  *	reloaded after call to this function.
1444  */
1445 
1446 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1447 		     gfp_t gfp_mask)
1448 {
1449 	int i, osize = skb_end_offset(skb);
1450 	int size = osize + nhead + ntail;
1451 	long off;
1452 	u8 *data;
1453 
1454 	BUG_ON(nhead < 0);
1455 
1456 	if (skb_shared(skb))
1457 		BUG();
1458 
1459 	size = SKB_DATA_ALIGN(size);
1460 
1461 	if (skb_pfmemalloc(skb))
1462 		gfp_mask |= __GFP_MEMALLOC;
1463 	data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1464 			       gfp_mask, NUMA_NO_NODE, NULL);
1465 	if (!data)
1466 		goto nodata;
1467 	size = SKB_WITH_OVERHEAD(ksize(data));
1468 
1469 	/* Copy only real data... and, alas, header. This should be
1470 	 * optimized for the cases when header is void.
1471 	 */
1472 	memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1473 
1474 	memcpy((struct skb_shared_info *)(data + size),
1475 	       skb_shinfo(skb),
1476 	       offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1477 
1478 	/*
1479 	 * if shinfo is shared we must drop the old head gracefully, but if it
1480 	 * is not we can just drop the old head and let the existing refcount
1481 	 * be since all we did is relocate the values
1482 	 */
1483 	if (skb_cloned(skb)) {
1484 		if (skb_orphan_frags(skb, gfp_mask))
1485 			goto nofrags;
1486 		if (skb_zcopy(skb))
1487 			refcount_inc(&skb_uarg(skb)->refcnt);
1488 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1489 			skb_frag_ref(skb, i);
1490 
1491 		if (skb_has_frag_list(skb))
1492 			skb_clone_fraglist(skb);
1493 
1494 		skb_release_data(skb);
1495 	} else {
1496 		skb_free_head(skb);
1497 	}
1498 	off = (data + nhead) - skb->head;
1499 
1500 	skb->head     = data;
1501 	skb->head_frag = 0;
1502 	skb->data    += off;
1503 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1504 	skb->end      = size;
1505 	off           = nhead;
1506 #else
1507 	skb->end      = skb->head + size;
1508 #endif
1509 	skb->tail	      += off;
1510 	skb_headers_offset_update(skb, nhead);
1511 	skb->cloned   = 0;
1512 	skb->hdr_len  = 0;
1513 	skb->nohdr    = 0;
1514 	atomic_set(&skb_shinfo(skb)->dataref, 1);
1515 
1516 	/* It is not generally safe to change skb->truesize.
1517 	 * For the moment, we really care of rx path, or
1518 	 * when skb is orphaned (not attached to a socket).
1519 	 */
1520 	if (!skb->sk || skb->destructor == sock_edemux)
1521 		skb->truesize += size - osize;
1522 
1523 	return 0;
1524 
1525 nofrags:
1526 	kfree(data);
1527 nodata:
1528 	return -ENOMEM;
1529 }
1530 EXPORT_SYMBOL(pskb_expand_head);
1531 
1532 /* Make private copy of skb with writable head and some headroom */
1533 
1534 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1535 {
1536 	struct sk_buff *skb2;
1537 	int delta = headroom - skb_headroom(skb);
1538 
1539 	if (delta <= 0)
1540 		skb2 = pskb_copy(skb, GFP_ATOMIC);
1541 	else {
1542 		skb2 = skb_clone(skb, GFP_ATOMIC);
1543 		if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1544 					     GFP_ATOMIC)) {
1545 			kfree_skb(skb2);
1546 			skb2 = NULL;
1547 		}
1548 	}
1549 	return skb2;
1550 }
1551 EXPORT_SYMBOL(skb_realloc_headroom);
1552 
1553 /**
1554  *	skb_copy_expand	-	copy and expand sk_buff
1555  *	@skb: buffer to copy
1556  *	@newheadroom: new free bytes at head
1557  *	@newtailroom: new free bytes at tail
1558  *	@gfp_mask: allocation priority
1559  *
1560  *	Make a copy of both an &sk_buff and its data and while doing so
1561  *	allocate additional space.
1562  *
1563  *	This is used when the caller wishes to modify the data and needs a
1564  *	private copy of the data to alter as well as more space for new fields.
1565  *	Returns %NULL on failure or the pointer to the buffer
1566  *	on success. The returned buffer has a reference count of 1.
1567  *
1568  *	You must pass %GFP_ATOMIC as the allocation priority if this function
1569  *	is called from an interrupt.
1570  */
1571 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1572 				int newheadroom, int newtailroom,
1573 				gfp_t gfp_mask)
1574 {
1575 	/*
1576 	 *	Allocate the copy buffer
1577 	 */
1578 	struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1579 					gfp_mask, skb_alloc_rx_flag(skb),
1580 					NUMA_NO_NODE);
1581 	int oldheadroom = skb_headroom(skb);
1582 	int head_copy_len, head_copy_off;
1583 
1584 	if (!n)
1585 		return NULL;
1586 
1587 	skb_reserve(n, newheadroom);
1588 
1589 	/* Set the tail pointer and length */
1590 	skb_put(n, skb->len);
1591 
1592 	head_copy_len = oldheadroom;
1593 	head_copy_off = 0;
1594 	if (newheadroom <= head_copy_len)
1595 		head_copy_len = newheadroom;
1596 	else
1597 		head_copy_off = newheadroom - head_copy_len;
1598 
1599 	/* Copy the linear header and data. */
1600 	if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1601 			  skb->len + head_copy_len))
1602 		BUG();
1603 
1604 	copy_skb_header(n, skb);
1605 
1606 	skb_headers_offset_update(n, newheadroom - oldheadroom);
1607 
1608 	return n;
1609 }
1610 EXPORT_SYMBOL(skb_copy_expand);
1611 
1612 /**
1613  *	__skb_pad		-	zero pad the tail of an skb
1614  *	@skb: buffer to pad
1615  *	@pad: space to pad
1616  *	@free_on_error: free buffer on error
1617  *
1618  *	Ensure that a buffer is followed by a padding area that is zero
1619  *	filled. Used by network drivers which may DMA or transfer data
1620  *	beyond the buffer end onto the wire.
1621  *
1622  *	May return error in out of memory cases. The skb is freed on error
1623  *	if @free_on_error is true.
1624  */
1625 
1626 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
1627 {
1628 	int err;
1629 	int ntail;
1630 
1631 	/* If the skbuff is non linear tailroom is always zero.. */
1632 	if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1633 		memset(skb->data+skb->len, 0, pad);
1634 		return 0;
1635 	}
1636 
1637 	ntail = skb->data_len + pad - (skb->end - skb->tail);
1638 	if (likely(skb_cloned(skb) || ntail > 0)) {
1639 		err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1640 		if (unlikely(err))
1641 			goto free_skb;
1642 	}
1643 
1644 	/* FIXME: The use of this function with non-linear skb's really needs
1645 	 * to be audited.
1646 	 */
1647 	err = skb_linearize(skb);
1648 	if (unlikely(err))
1649 		goto free_skb;
1650 
1651 	memset(skb->data + skb->len, 0, pad);
1652 	return 0;
1653 
1654 free_skb:
1655 	if (free_on_error)
1656 		kfree_skb(skb);
1657 	return err;
1658 }
1659 EXPORT_SYMBOL(__skb_pad);
1660 
1661 /**
1662  *	pskb_put - add data to the tail of a potentially fragmented buffer
1663  *	@skb: start of the buffer to use
1664  *	@tail: tail fragment of the buffer to use
1665  *	@len: amount of data to add
1666  *
1667  *	This function extends the used data area of the potentially
1668  *	fragmented buffer. @tail must be the last fragment of @skb -- or
1669  *	@skb itself. If this would exceed the total buffer size the kernel
1670  *	will panic. A pointer to the first byte of the extra data is
1671  *	returned.
1672  */
1673 
1674 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1675 {
1676 	if (tail != skb) {
1677 		skb->data_len += len;
1678 		skb->len += len;
1679 	}
1680 	return skb_put(tail, len);
1681 }
1682 EXPORT_SYMBOL_GPL(pskb_put);
1683 
1684 /**
1685  *	skb_put - add data to a buffer
1686  *	@skb: buffer to use
1687  *	@len: amount of data to add
1688  *
1689  *	This function extends the used data area of the buffer. If this would
1690  *	exceed the total buffer size the kernel will panic. A pointer to the
1691  *	first byte of the extra data is returned.
1692  */
1693 void *skb_put(struct sk_buff *skb, unsigned int len)
1694 {
1695 	void *tmp = skb_tail_pointer(skb);
1696 	SKB_LINEAR_ASSERT(skb);
1697 	skb->tail += len;
1698 	skb->len  += len;
1699 	if (unlikely(skb->tail > skb->end))
1700 		skb_over_panic(skb, len, __builtin_return_address(0));
1701 	return tmp;
1702 }
1703 EXPORT_SYMBOL(skb_put);
1704 
1705 /**
1706  *	skb_push - add data to the start of a buffer
1707  *	@skb: buffer to use
1708  *	@len: amount of data to add
1709  *
1710  *	This function extends the used data area of the buffer at the buffer
1711  *	start. If this would exceed the total buffer headroom the kernel will
1712  *	panic. A pointer to the first byte of the extra data is returned.
1713  */
1714 void *skb_push(struct sk_buff *skb, unsigned int len)
1715 {
1716 	skb->data -= len;
1717 	skb->len  += len;
1718 	if (unlikely(skb->data<skb->head))
1719 		skb_under_panic(skb, len, __builtin_return_address(0));
1720 	return skb->data;
1721 }
1722 EXPORT_SYMBOL(skb_push);
1723 
1724 /**
1725  *	skb_pull - remove data from the start of a buffer
1726  *	@skb: buffer to use
1727  *	@len: amount of data to remove
1728  *
1729  *	This function removes data from the start of a buffer, returning
1730  *	the memory to the headroom. A pointer to the next data in the buffer
1731  *	is returned. Once the data has been pulled future pushes will overwrite
1732  *	the old data.
1733  */
1734 void *skb_pull(struct sk_buff *skb, unsigned int len)
1735 {
1736 	return skb_pull_inline(skb, len);
1737 }
1738 EXPORT_SYMBOL(skb_pull);
1739 
1740 /**
1741  *	skb_trim - remove end from a buffer
1742  *	@skb: buffer to alter
1743  *	@len: new length
1744  *
1745  *	Cut the length of a buffer down by removing data from the tail. If
1746  *	the buffer is already under the length specified it is not modified.
1747  *	The skb must be linear.
1748  */
1749 void skb_trim(struct sk_buff *skb, unsigned int len)
1750 {
1751 	if (skb->len > len)
1752 		__skb_trim(skb, len);
1753 }
1754 EXPORT_SYMBOL(skb_trim);
1755 
1756 /* Trims skb to length len. It can change skb pointers.
1757  */
1758 
1759 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1760 {
1761 	struct sk_buff **fragp;
1762 	struct sk_buff *frag;
1763 	int offset = skb_headlen(skb);
1764 	int nfrags = skb_shinfo(skb)->nr_frags;
1765 	int i;
1766 	int err;
1767 
1768 	if (skb_cloned(skb) &&
1769 	    unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1770 		return err;
1771 
1772 	i = 0;
1773 	if (offset >= len)
1774 		goto drop_pages;
1775 
1776 	for (; i < nfrags; i++) {
1777 		int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1778 
1779 		if (end < len) {
1780 			offset = end;
1781 			continue;
1782 		}
1783 
1784 		skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1785 
1786 drop_pages:
1787 		skb_shinfo(skb)->nr_frags = i;
1788 
1789 		for (; i < nfrags; i++)
1790 			skb_frag_unref(skb, i);
1791 
1792 		if (skb_has_frag_list(skb))
1793 			skb_drop_fraglist(skb);
1794 		goto done;
1795 	}
1796 
1797 	for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1798 	     fragp = &frag->next) {
1799 		int end = offset + frag->len;
1800 
1801 		if (skb_shared(frag)) {
1802 			struct sk_buff *nfrag;
1803 
1804 			nfrag = skb_clone(frag, GFP_ATOMIC);
1805 			if (unlikely(!nfrag))
1806 				return -ENOMEM;
1807 
1808 			nfrag->next = frag->next;
1809 			consume_skb(frag);
1810 			frag = nfrag;
1811 			*fragp = frag;
1812 		}
1813 
1814 		if (end < len) {
1815 			offset = end;
1816 			continue;
1817 		}
1818 
1819 		if (end > len &&
1820 		    unlikely((err = pskb_trim(frag, len - offset))))
1821 			return err;
1822 
1823 		if (frag->next)
1824 			skb_drop_list(&frag->next);
1825 		break;
1826 	}
1827 
1828 done:
1829 	if (len > skb_headlen(skb)) {
1830 		skb->data_len -= skb->len - len;
1831 		skb->len       = len;
1832 	} else {
1833 		skb->len       = len;
1834 		skb->data_len  = 0;
1835 		skb_set_tail_pointer(skb, len);
1836 	}
1837 
1838 	if (!skb->sk || skb->destructor == sock_edemux)
1839 		skb_condense(skb);
1840 	return 0;
1841 }
1842 EXPORT_SYMBOL(___pskb_trim);
1843 
1844 /**
1845  *	__pskb_pull_tail - advance tail of skb header
1846  *	@skb: buffer to reallocate
1847  *	@delta: number of bytes to advance tail
1848  *
1849  *	The function makes a sense only on a fragmented &sk_buff,
1850  *	it expands header moving its tail forward and copying necessary
1851  *	data from fragmented part.
1852  *
1853  *	&sk_buff MUST have reference count of 1.
1854  *
1855  *	Returns %NULL (and &sk_buff does not change) if pull failed
1856  *	or value of new tail of skb in the case of success.
1857  *
1858  *	All the pointers pointing into skb header may change and must be
1859  *	reloaded after call to this function.
1860  */
1861 
1862 /* Moves tail of skb head forward, copying data from fragmented part,
1863  * when it is necessary.
1864  * 1. It may fail due to malloc failure.
1865  * 2. It may change skb pointers.
1866  *
1867  * It is pretty complicated. Luckily, it is called only in exceptional cases.
1868  */
1869 void *__pskb_pull_tail(struct sk_buff *skb, int delta)
1870 {
1871 	/* If skb has not enough free space at tail, get new one
1872 	 * plus 128 bytes for future expansions. If we have enough
1873 	 * room at tail, reallocate without expansion only if skb is cloned.
1874 	 */
1875 	int i, k, eat = (skb->tail + delta) - skb->end;
1876 
1877 	if (eat > 0 || skb_cloned(skb)) {
1878 		if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1879 				     GFP_ATOMIC))
1880 			return NULL;
1881 	}
1882 
1883 	if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
1884 		BUG();
1885 
1886 	/* Optimization: no fragments, no reasons to preestimate
1887 	 * size of pulled pages. Superb.
1888 	 */
1889 	if (!skb_has_frag_list(skb))
1890 		goto pull_pages;
1891 
1892 	/* Estimate size of pulled pages. */
1893 	eat = delta;
1894 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1895 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1896 
1897 		if (size >= eat)
1898 			goto pull_pages;
1899 		eat -= size;
1900 	}
1901 
1902 	/* If we need update frag list, we are in troubles.
1903 	 * Certainly, it is possible to add an offset to skb data,
1904 	 * but taking into account that pulling is expected to
1905 	 * be very rare operation, it is worth to fight against
1906 	 * further bloating skb head and crucify ourselves here instead.
1907 	 * Pure masohism, indeed. 8)8)
1908 	 */
1909 	if (eat) {
1910 		struct sk_buff *list = skb_shinfo(skb)->frag_list;
1911 		struct sk_buff *clone = NULL;
1912 		struct sk_buff *insp = NULL;
1913 
1914 		do {
1915 			BUG_ON(!list);
1916 
1917 			if (list->len <= eat) {
1918 				/* Eaten as whole. */
1919 				eat -= list->len;
1920 				list = list->next;
1921 				insp = list;
1922 			} else {
1923 				/* Eaten partially. */
1924 
1925 				if (skb_shared(list)) {
1926 					/* Sucks! We need to fork list. :-( */
1927 					clone = skb_clone(list, GFP_ATOMIC);
1928 					if (!clone)
1929 						return NULL;
1930 					insp = list->next;
1931 					list = clone;
1932 				} else {
1933 					/* This may be pulled without
1934 					 * problems. */
1935 					insp = list;
1936 				}
1937 				if (!pskb_pull(list, eat)) {
1938 					kfree_skb(clone);
1939 					return NULL;
1940 				}
1941 				break;
1942 			}
1943 		} while (eat);
1944 
1945 		/* Free pulled out fragments. */
1946 		while ((list = skb_shinfo(skb)->frag_list) != insp) {
1947 			skb_shinfo(skb)->frag_list = list->next;
1948 			kfree_skb(list);
1949 		}
1950 		/* And insert new clone at head. */
1951 		if (clone) {
1952 			clone->next = list;
1953 			skb_shinfo(skb)->frag_list = clone;
1954 		}
1955 	}
1956 	/* Success! Now we may commit changes to skb data. */
1957 
1958 pull_pages:
1959 	eat = delta;
1960 	k = 0;
1961 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1962 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1963 
1964 		if (size <= eat) {
1965 			skb_frag_unref(skb, i);
1966 			eat -= size;
1967 		} else {
1968 			skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1969 			if (eat) {
1970 				skb_shinfo(skb)->frags[k].page_offset += eat;
1971 				skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1972 				if (!i)
1973 					goto end;
1974 				eat = 0;
1975 			}
1976 			k++;
1977 		}
1978 	}
1979 	skb_shinfo(skb)->nr_frags = k;
1980 
1981 end:
1982 	skb->tail     += delta;
1983 	skb->data_len -= delta;
1984 
1985 	if (!skb->data_len)
1986 		skb_zcopy_clear(skb, false);
1987 
1988 	return skb_tail_pointer(skb);
1989 }
1990 EXPORT_SYMBOL(__pskb_pull_tail);
1991 
1992 /**
1993  *	skb_copy_bits - copy bits from skb to kernel buffer
1994  *	@skb: source skb
1995  *	@offset: offset in source
1996  *	@to: destination buffer
1997  *	@len: number of bytes to copy
1998  *
1999  *	Copy the specified number of bytes from the source skb to the
2000  *	destination buffer.
2001  *
2002  *	CAUTION ! :
2003  *		If its prototype is ever changed,
2004  *		check arch/{*}/net/{*}.S files,
2005  *		since it is called from BPF assembly code.
2006  */
2007 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2008 {
2009 	int start = skb_headlen(skb);
2010 	struct sk_buff *frag_iter;
2011 	int i, copy;
2012 
2013 	if (offset > (int)skb->len - len)
2014 		goto fault;
2015 
2016 	/* Copy header. */
2017 	if ((copy = start - offset) > 0) {
2018 		if (copy > len)
2019 			copy = len;
2020 		skb_copy_from_linear_data_offset(skb, offset, to, copy);
2021 		if ((len -= copy) == 0)
2022 			return 0;
2023 		offset += copy;
2024 		to     += copy;
2025 	}
2026 
2027 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2028 		int end;
2029 		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2030 
2031 		WARN_ON(start > offset + len);
2032 
2033 		end = start + skb_frag_size(f);
2034 		if ((copy = end - offset) > 0) {
2035 			u32 p_off, p_len, copied;
2036 			struct page *p;
2037 			u8 *vaddr;
2038 
2039 			if (copy > len)
2040 				copy = len;
2041 
2042 			skb_frag_foreach_page(f,
2043 					      f->page_offset + offset - start,
2044 					      copy, p, p_off, p_len, copied) {
2045 				vaddr = kmap_atomic(p);
2046 				memcpy(to + copied, vaddr + p_off, p_len);
2047 				kunmap_atomic(vaddr);
2048 			}
2049 
2050 			if ((len -= copy) == 0)
2051 				return 0;
2052 			offset += copy;
2053 			to     += copy;
2054 		}
2055 		start = end;
2056 	}
2057 
2058 	skb_walk_frags(skb, frag_iter) {
2059 		int end;
2060 
2061 		WARN_ON(start > offset + len);
2062 
2063 		end = start + frag_iter->len;
2064 		if ((copy = end - offset) > 0) {
2065 			if (copy > len)
2066 				copy = len;
2067 			if (skb_copy_bits(frag_iter, offset - start, to, copy))
2068 				goto fault;
2069 			if ((len -= copy) == 0)
2070 				return 0;
2071 			offset += copy;
2072 			to     += copy;
2073 		}
2074 		start = end;
2075 	}
2076 
2077 	if (!len)
2078 		return 0;
2079 
2080 fault:
2081 	return -EFAULT;
2082 }
2083 EXPORT_SYMBOL(skb_copy_bits);
2084 
2085 /*
2086  * Callback from splice_to_pipe(), if we need to release some pages
2087  * at the end of the spd in case we error'ed out in filling the pipe.
2088  */
2089 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
2090 {
2091 	put_page(spd->pages[i]);
2092 }
2093 
2094 static struct page *linear_to_page(struct page *page, unsigned int *len,
2095 				   unsigned int *offset,
2096 				   struct sock *sk)
2097 {
2098 	struct page_frag *pfrag = sk_page_frag(sk);
2099 
2100 	if (!sk_page_frag_refill(sk, pfrag))
2101 		return NULL;
2102 
2103 	*len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
2104 
2105 	memcpy(page_address(pfrag->page) + pfrag->offset,
2106 	       page_address(page) + *offset, *len);
2107 	*offset = pfrag->offset;
2108 	pfrag->offset += *len;
2109 
2110 	return pfrag->page;
2111 }
2112 
2113 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
2114 			     struct page *page,
2115 			     unsigned int offset)
2116 {
2117 	return	spd->nr_pages &&
2118 		spd->pages[spd->nr_pages - 1] == page &&
2119 		(spd->partial[spd->nr_pages - 1].offset +
2120 		 spd->partial[spd->nr_pages - 1].len == offset);
2121 }
2122 
2123 /*
2124  * Fill page/offset/length into spd, if it can hold more pages.
2125  */
2126 static bool spd_fill_page(struct splice_pipe_desc *spd,
2127 			  struct pipe_inode_info *pipe, struct page *page,
2128 			  unsigned int *len, unsigned int offset,
2129 			  bool linear,
2130 			  struct sock *sk)
2131 {
2132 	if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
2133 		return true;
2134 
2135 	if (linear) {
2136 		page = linear_to_page(page, len, &offset, sk);
2137 		if (!page)
2138 			return true;
2139 	}
2140 	if (spd_can_coalesce(spd, page, offset)) {
2141 		spd->partial[spd->nr_pages - 1].len += *len;
2142 		return false;
2143 	}
2144 	get_page(page);
2145 	spd->pages[spd->nr_pages] = page;
2146 	spd->partial[spd->nr_pages].len = *len;
2147 	spd->partial[spd->nr_pages].offset = offset;
2148 	spd->nr_pages++;
2149 
2150 	return false;
2151 }
2152 
2153 static bool __splice_segment(struct page *page, unsigned int poff,
2154 			     unsigned int plen, unsigned int *off,
2155 			     unsigned int *len,
2156 			     struct splice_pipe_desc *spd, bool linear,
2157 			     struct sock *sk,
2158 			     struct pipe_inode_info *pipe)
2159 {
2160 	if (!*len)
2161 		return true;
2162 
2163 	/* skip this segment if already processed */
2164 	if (*off >= plen) {
2165 		*off -= plen;
2166 		return false;
2167 	}
2168 
2169 	/* ignore any bits we already processed */
2170 	poff += *off;
2171 	plen -= *off;
2172 	*off = 0;
2173 
2174 	do {
2175 		unsigned int flen = min(*len, plen);
2176 
2177 		if (spd_fill_page(spd, pipe, page, &flen, poff,
2178 				  linear, sk))
2179 			return true;
2180 		poff += flen;
2181 		plen -= flen;
2182 		*len -= flen;
2183 	} while (*len && plen);
2184 
2185 	return false;
2186 }
2187 
2188 /*
2189  * Map linear and fragment data from the skb to spd. It reports true if the
2190  * pipe is full or if we already spliced the requested length.
2191  */
2192 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
2193 			      unsigned int *offset, unsigned int *len,
2194 			      struct splice_pipe_desc *spd, struct sock *sk)
2195 {
2196 	int seg;
2197 	struct sk_buff *iter;
2198 
2199 	/* map the linear part :
2200 	 * If skb->head_frag is set, this 'linear' part is backed by a
2201 	 * fragment, and if the head is not shared with any clones then
2202 	 * we can avoid a copy since we own the head portion of this page.
2203 	 */
2204 	if (__splice_segment(virt_to_page(skb->data),
2205 			     (unsigned long) skb->data & (PAGE_SIZE - 1),
2206 			     skb_headlen(skb),
2207 			     offset, len, spd,
2208 			     skb_head_is_locked(skb),
2209 			     sk, pipe))
2210 		return true;
2211 
2212 	/*
2213 	 * then map the fragments
2214 	 */
2215 	for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
2216 		const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2217 
2218 		if (__splice_segment(skb_frag_page(f),
2219 				     f->page_offset, skb_frag_size(f),
2220 				     offset, len, spd, false, sk, pipe))
2221 			return true;
2222 	}
2223 
2224 	skb_walk_frags(skb, iter) {
2225 		if (*offset >= iter->len) {
2226 			*offset -= iter->len;
2227 			continue;
2228 		}
2229 		/* __skb_splice_bits() only fails if the output has no room
2230 		 * left, so no point in going over the frag_list for the error
2231 		 * case.
2232 		 */
2233 		if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
2234 			return true;
2235 	}
2236 
2237 	return false;
2238 }
2239 
2240 /*
2241  * Map data from the skb to a pipe. Should handle both the linear part,
2242  * the fragments, and the frag list.
2243  */
2244 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
2245 		    struct pipe_inode_info *pipe, unsigned int tlen,
2246 		    unsigned int flags)
2247 {
2248 	struct partial_page partial[MAX_SKB_FRAGS];
2249 	struct page *pages[MAX_SKB_FRAGS];
2250 	struct splice_pipe_desc spd = {
2251 		.pages = pages,
2252 		.partial = partial,
2253 		.nr_pages_max = MAX_SKB_FRAGS,
2254 		.ops = &nosteal_pipe_buf_ops,
2255 		.spd_release = sock_spd_release,
2256 	};
2257 	int ret = 0;
2258 
2259 	__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
2260 
2261 	if (spd.nr_pages)
2262 		ret = splice_to_pipe(pipe, &spd);
2263 
2264 	return ret;
2265 }
2266 EXPORT_SYMBOL_GPL(skb_splice_bits);
2267 
2268 /* Send skb data on a socket. Socket must be locked. */
2269 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2270 			 int len)
2271 {
2272 	unsigned int orig_len = len;
2273 	struct sk_buff *head = skb;
2274 	unsigned short fragidx;
2275 	int slen, ret;
2276 
2277 do_frag_list:
2278 
2279 	/* Deal with head data */
2280 	while (offset < skb_headlen(skb) && len) {
2281 		struct kvec kv;
2282 		struct msghdr msg;
2283 
2284 		slen = min_t(int, len, skb_headlen(skb) - offset);
2285 		kv.iov_base = skb->data + offset;
2286 		kv.iov_len = slen;
2287 		memset(&msg, 0, sizeof(msg));
2288 
2289 		ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
2290 		if (ret <= 0)
2291 			goto error;
2292 
2293 		offset += ret;
2294 		len -= ret;
2295 	}
2296 
2297 	/* All the data was skb head? */
2298 	if (!len)
2299 		goto out;
2300 
2301 	/* Make offset relative to start of frags */
2302 	offset -= skb_headlen(skb);
2303 
2304 	/* Find where we are in frag list */
2305 	for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2306 		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
2307 
2308 		if (offset < frag->size)
2309 			break;
2310 
2311 		offset -= frag->size;
2312 	}
2313 
2314 	for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2315 		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
2316 
2317 		slen = min_t(size_t, len, frag->size - offset);
2318 
2319 		while (slen) {
2320 			ret = kernel_sendpage_locked(sk, frag->page.p,
2321 						     frag->page_offset + offset,
2322 						     slen, MSG_DONTWAIT);
2323 			if (ret <= 0)
2324 				goto error;
2325 
2326 			len -= ret;
2327 			offset += ret;
2328 			slen -= ret;
2329 		}
2330 
2331 		offset = 0;
2332 	}
2333 
2334 	if (len) {
2335 		/* Process any frag lists */
2336 
2337 		if (skb == head) {
2338 			if (skb_has_frag_list(skb)) {
2339 				skb = skb_shinfo(skb)->frag_list;
2340 				goto do_frag_list;
2341 			}
2342 		} else if (skb->next) {
2343 			skb = skb->next;
2344 			goto do_frag_list;
2345 		}
2346 	}
2347 
2348 out:
2349 	return orig_len - len;
2350 
2351 error:
2352 	return orig_len == len ? ret : orig_len - len;
2353 }
2354 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2355 
2356 /* Send skb data on a socket. */
2357 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
2358 {
2359 	int ret = 0;
2360 
2361 	lock_sock(sk);
2362 	ret = skb_send_sock_locked(sk, skb, offset, len);
2363 	release_sock(sk);
2364 
2365 	return ret;
2366 }
2367 EXPORT_SYMBOL_GPL(skb_send_sock);
2368 
2369 /**
2370  *	skb_store_bits - store bits from kernel buffer to skb
2371  *	@skb: destination buffer
2372  *	@offset: offset in destination
2373  *	@from: source buffer
2374  *	@len: number of bytes to copy
2375  *
2376  *	Copy the specified number of bytes from the source buffer to the
2377  *	destination skb.  This function handles all the messy bits of
2378  *	traversing fragment lists and such.
2379  */
2380 
2381 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
2382 {
2383 	int start = skb_headlen(skb);
2384 	struct sk_buff *frag_iter;
2385 	int i, copy;
2386 
2387 	if (offset > (int)skb->len - len)
2388 		goto fault;
2389 
2390 	if ((copy = start - offset) > 0) {
2391 		if (copy > len)
2392 			copy = len;
2393 		skb_copy_to_linear_data_offset(skb, offset, from, copy);
2394 		if ((len -= copy) == 0)
2395 			return 0;
2396 		offset += copy;
2397 		from += copy;
2398 	}
2399 
2400 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2401 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2402 		int end;
2403 
2404 		WARN_ON(start > offset + len);
2405 
2406 		end = start + skb_frag_size(frag);
2407 		if ((copy = end - offset) > 0) {
2408 			u32 p_off, p_len, copied;
2409 			struct page *p;
2410 			u8 *vaddr;
2411 
2412 			if (copy > len)
2413 				copy = len;
2414 
2415 			skb_frag_foreach_page(frag,
2416 					      frag->page_offset + offset - start,
2417 					      copy, p, p_off, p_len, copied) {
2418 				vaddr = kmap_atomic(p);
2419 				memcpy(vaddr + p_off, from + copied, p_len);
2420 				kunmap_atomic(vaddr);
2421 			}
2422 
2423 			if ((len -= copy) == 0)
2424 				return 0;
2425 			offset += copy;
2426 			from += copy;
2427 		}
2428 		start = end;
2429 	}
2430 
2431 	skb_walk_frags(skb, frag_iter) {
2432 		int end;
2433 
2434 		WARN_ON(start > offset + len);
2435 
2436 		end = start + frag_iter->len;
2437 		if ((copy = end - offset) > 0) {
2438 			if (copy > len)
2439 				copy = len;
2440 			if (skb_store_bits(frag_iter, offset - start,
2441 					   from, copy))
2442 				goto fault;
2443 			if ((len -= copy) == 0)
2444 				return 0;
2445 			offset += copy;
2446 			from += copy;
2447 		}
2448 		start = end;
2449 	}
2450 	if (!len)
2451 		return 0;
2452 
2453 fault:
2454 	return -EFAULT;
2455 }
2456 EXPORT_SYMBOL(skb_store_bits);
2457 
2458 /* Checksum skb data. */
2459 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2460 		      __wsum csum, const struct skb_checksum_ops *ops)
2461 {
2462 	int start = skb_headlen(skb);
2463 	int i, copy = start - offset;
2464 	struct sk_buff *frag_iter;
2465 	int pos = 0;
2466 
2467 	/* Checksum header. */
2468 	if (copy > 0) {
2469 		if (copy > len)
2470 			copy = len;
2471 		csum = ops->update(skb->data + offset, copy, csum);
2472 		if ((len -= copy) == 0)
2473 			return csum;
2474 		offset += copy;
2475 		pos	= copy;
2476 	}
2477 
2478 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2479 		int end;
2480 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2481 
2482 		WARN_ON(start > offset + len);
2483 
2484 		end = start + skb_frag_size(frag);
2485 		if ((copy = end - offset) > 0) {
2486 			u32 p_off, p_len, copied;
2487 			struct page *p;
2488 			__wsum csum2;
2489 			u8 *vaddr;
2490 
2491 			if (copy > len)
2492 				copy = len;
2493 
2494 			skb_frag_foreach_page(frag,
2495 					      frag->page_offset + offset - start,
2496 					      copy, p, p_off, p_len, copied) {
2497 				vaddr = kmap_atomic(p);
2498 				csum2 = ops->update(vaddr + p_off, p_len, 0);
2499 				kunmap_atomic(vaddr);
2500 				csum = ops->combine(csum, csum2, pos, p_len);
2501 				pos += p_len;
2502 			}
2503 
2504 			if (!(len -= copy))
2505 				return csum;
2506 			offset += copy;
2507 		}
2508 		start = end;
2509 	}
2510 
2511 	skb_walk_frags(skb, frag_iter) {
2512 		int end;
2513 
2514 		WARN_ON(start > offset + len);
2515 
2516 		end = start + frag_iter->len;
2517 		if ((copy = end - offset) > 0) {
2518 			__wsum csum2;
2519 			if (copy > len)
2520 				copy = len;
2521 			csum2 = __skb_checksum(frag_iter, offset - start,
2522 					       copy, 0, ops);
2523 			csum = ops->combine(csum, csum2, pos, copy);
2524 			if ((len -= copy) == 0)
2525 				return csum;
2526 			offset += copy;
2527 			pos    += copy;
2528 		}
2529 		start = end;
2530 	}
2531 	BUG_ON(len);
2532 
2533 	return csum;
2534 }
2535 EXPORT_SYMBOL(__skb_checksum);
2536 
2537 __wsum skb_checksum(const struct sk_buff *skb, int offset,
2538 		    int len, __wsum csum)
2539 {
2540 	const struct skb_checksum_ops ops = {
2541 		.update  = csum_partial_ext,
2542 		.combine = csum_block_add_ext,
2543 	};
2544 
2545 	return __skb_checksum(skb, offset, len, csum, &ops);
2546 }
2547 EXPORT_SYMBOL(skb_checksum);
2548 
2549 /* Both of above in one bottle. */
2550 
2551 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2552 				    u8 *to, int len, __wsum csum)
2553 {
2554 	int start = skb_headlen(skb);
2555 	int i, copy = start - offset;
2556 	struct sk_buff *frag_iter;
2557 	int pos = 0;
2558 
2559 	/* Copy header. */
2560 	if (copy > 0) {
2561 		if (copy > len)
2562 			copy = len;
2563 		csum = csum_partial_copy_nocheck(skb->data + offset, to,
2564 						 copy, csum);
2565 		if ((len -= copy) == 0)
2566 			return csum;
2567 		offset += copy;
2568 		to     += copy;
2569 		pos	= copy;
2570 	}
2571 
2572 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2573 		int end;
2574 
2575 		WARN_ON(start > offset + len);
2576 
2577 		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2578 		if ((copy = end - offset) > 0) {
2579 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2580 			u32 p_off, p_len, copied;
2581 			struct page *p;
2582 			__wsum csum2;
2583 			u8 *vaddr;
2584 
2585 			if (copy > len)
2586 				copy = len;
2587 
2588 			skb_frag_foreach_page(frag,
2589 					      frag->page_offset + offset - start,
2590 					      copy, p, p_off, p_len, copied) {
2591 				vaddr = kmap_atomic(p);
2592 				csum2 = csum_partial_copy_nocheck(vaddr + p_off,
2593 								  to + copied,
2594 								  p_len, 0);
2595 				kunmap_atomic(vaddr);
2596 				csum = csum_block_add(csum, csum2, pos);
2597 				pos += p_len;
2598 			}
2599 
2600 			if (!(len -= copy))
2601 				return csum;
2602 			offset += copy;
2603 			to     += copy;
2604 		}
2605 		start = end;
2606 	}
2607 
2608 	skb_walk_frags(skb, frag_iter) {
2609 		__wsum csum2;
2610 		int end;
2611 
2612 		WARN_ON(start > offset + len);
2613 
2614 		end = start + frag_iter->len;
2615 		if ((copy = end - offset) > 0) {
2616 			if (copy > len)
2617 				copy = len;
2618 			csum2 = skb_copy_and_csum_bits(frag_iter,
2619 						       offset - start,
2620 						       to, copy, 0);
2621 			csum = csum_block_add(csum, csum2, pos);
2622 			if ((len -= copy) == 0)
2623 				return csum;
2624 			offset += copy;
2625 			to     += copy;
2626 			pos    += copy;
2627 		}
2628 		start = end;
2629 	}
2630 	BUG_ON(len);
2631 	return csum;
2632 }
2633 EXPORT_SYMBOL(skb_copy_and_csum_bits);
2634 
2635 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
2636 {
2637 	net_warn_ratelimited(
2638 		"%s: attempt to compute crc32c without libcrc32c.ko\n",
2639 		__func__);
2640 	return 0;
2641 }
2642 
2643 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
2644 				       int offset, int len)
2645 {
2646 	net_warn_ratelimited(
2647 		"%s: attempt to compute crc32c without libcrc32c.ko\n",
2648 		__func__);
2649 	return 0;
2650 }
2651 
2652 static const struct skb_checksum_ops default_crc32c_ops = {
2653 	.update  = warn_crc32c_csum_update,
2654 	.combine = warn_crc32c_csum_combine,
2655 };
2656 
2657 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
2658 	&default_crc32c_ops;
2659 EXPORT_SYMBOL(crc32c_csum_stub);
2660 
2661  /**
2662  *	skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
2663  *	@from: source buffer
2664  *
2665  *	Calculates the amount of linear headroom needed in the 'to' skb passed
2666  *	into skb_zerocopy().
2667  */
2668 unsigned int
2669 skb_zerocopy_headlen(const struct sk_buff *from)
2670 {
2671 	unsigned int hlen = 0;
2672 
2673 	if (!from->head_frag ||
2674 	    skb_headlen(from) < L1_CACHE_BYTES ||
2675 	    skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
2676 		hlen = skb_headlen(from);
2677 
2678 	if (skb_has_frag_list(from))
2679 		hlen = from->len;
2680 
2681 	return hlen;
2682 }
2683 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
2684 
2685 /**
2686  *	skb_zerocopy - Zero copy skb to skb
2687  *	@to: destination buffer
2688  *	@from: source buffer
2689  *	@len: number of bytes to copy from source buffer
2690  *	@hlen: size of linear headroom in destination buffer
2691  *
2692  *	Copies up to `len` bytes from `from` to `to` by creating references
2693  *	to the frags in the source buffer.
2694  *
2695  *	The `hlen` as calculated by skb_zerocopy_headlen() specifies the
2696  *	headroom in the `to` buffer.
2697  *
2698  *	Return value:
2699  *	0: everything is OK
2700  *	-ENOMEM: couldn't orphan frags of @from due to lack of memory
2701  *	-EFAULT: skb_copy_bits() found some problem with skb geometry
2702  */
2703 int
2704 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
2705 {
2706 	int i, j = 0;
2707 	int plen = 0; /* length of skb->head fragment */
2708 	int ret;
2709 	struct page *page;
2710 	unsigned int offset;
2711 
2712 	BUG_ON(!from->head_frag && !hlen);
2713 
2714 	/* dont bother with small payloads */
2715 	if (len <= skb_tailroom(to))
2716 		return skb_copy_bits(from, 0, skb_put(to, len), len);
2717 
2718 	if (hlen) {
2719 		ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
2720 		if (unlikely(ret))
2721 			return ret;
2722 		len -= hlen;
2723 	} else {
2724 		plen = min_t(int, skb_headlen(from), len);
2725 		if (plen) {
2726 			page = virt_to_head_page(from->head);
2727 			offset = from->data - (unsigned char *)page_address(page);
2728 			__skb_fill_page_desc(to, 0, page, offset, plen);
2729 			get_page(page);
2730 			j = 1;
2731 			len -= plen;
2732 		}
2733 	}
2734 
2735 	to->truesize += len + plen;
2736 	to->len += len + plen;
2737 	to->data_len += len + plen;
2738 
2739 	if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
2740 		skb_tx_error(from);
2741 		return -ENOMEM;
2742 	}
2743 	skb_zerocopy_clone(to, from, GFP_ATOMIC);
2744 
2745 	for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
2746 		if (!len)
2747 			break;
2748 		skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2749 		skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
2750 		len -= skb_shinfo(to)->frags[j].size;
2751 		skb_frag_ref(to, j);
2752 		j++;
2753 	}
2754 	skb_shinfo(to)->nr_frags = j;
2755 
2756 	return 0;
2757 }
2758 EXPORT_SYMBOL_GPL(skb_zerocopy);
2759 
2760 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2761 {
2762 	__wsum csum;
2763 	long csstart;
2764 
2765 	if (skb->ip_summed == CHECKSUM_PARTIAL)
2766 		csstart = skb_checksum_start_offset(skb);
2767 	else
2768 		csstart = skb_headlen(skb);
2769 
2770 	BUG_ON(csstart > skb_headlen(skb));
2771 
2772 	skb_copy_from_linear_data(skb, to, csstart);
2773 
2774 	csum = 0;
2775 	if (csstart != skb->len)
2776 		csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2777 					      skb->len - csstart, 0);
2778 
2779 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
2780 		long csstuff = csstart + skb->csum_offset;
2781 
2782 		*((__sum16 *)(to + csstuff)) = csum_fold(csum);
2783 	}
2784 }
2785 EXPORT_SYMBOL(skb_copy_and_csum_dev);
2786 
2787 /**
2788  *	skb_dequeue - remove from the head of the queue
2789  *	@list: list to dequeue from
2790  *
2791  *	Remove the head of the list. The list lock is taken so the function
2792  *	may be used safely with other locking list functions. The head item is
2793  *	returned or %NULL if the list is empty.
2794  */
2795 
2796 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
2797 {
2798 	unsigned long flags;
2799 	struct sk_buff *result;
2800 
2801 	spin_lock_irqsave(&list->lock, flags);
2802 	result = __skb_dequeue(list);
2803 	spin_unlock_irqrestore(&list->lock, flags);
2804 	return result;
2805 }
2806 EXPORT_SYMBOL(skb_dequeue);
2807 
2808 /**
2809  *	skb_dequeue_tail - remove from the tail of the queue
2810  *	@list: list to dequeue from
2811  *
2812  *	Remove the tail of the list. The list lock is taken so the function
2813  *	may be used safely with other locking list functions. The tail item is
2814  *	returned or %NULL if the list is empty.
2815  */
2816 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
2817 {
2818 	unsigned long flags;
2819 	struct sk_buff *result;
2820 
2821 	spin_lock_irqsave(&list->lock, flags);
2822 	result = __skb_dequeue_tail(list);
2823 	spin_unlock_irqrestore(&list->lock, flags);
2824 	return result;
2825 }
2826 EXPORT_SYMBOL(skb_dequeue_tail);
2827 
2828 /**
2829  *	skb_queue_purge - empty a list
2830  *	@list: list to empty
2831  *
2832  *	Delete all buffers on an &sk_buff list. Each buffer is removed from
2833  *	the list and one reference dropped. This function takes the list
2834  *	lock and is atomic with respect to other list locking functions.
2835  */
2836 void skb_queue_purge(struct sk_buff_head *list)
2837 {
2838 	struct sk_buff *skb;
2839 	while ((skb = skb_dequeue(list)) != NULL)
2840 		kfree_skb(skb);
2841 }
2842 EXPORT_SYMBOL(skb_queue_purge);
2843 
2844 /**
2845  *	skb_rbtree_purge - empty a skb rbtree
2846  *	@root: root of the rbtree to empty
2847  *
2848  *	Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
2849  *	the list and one reference dropped. This function does not take
2850  *	any lock. Synchronization should be handled by the caller (e.g., TCP
2851  *	out-of-order queue is protected by the socket lock).
2852  */
2853 void skb_rbtree_purge(struct rb_root *root)
2854 {
2855 	struct sk_buff *skb, *next;
2856 
2857 	rbtree_postorder_for_each_entry_safe(skb, next, root, rbnode)
2858 		kfree_skb(skb);
2859 
2860 	*root = RB_ROOT;
2861 }
2862 
2863 /**
2864  *	skb_queue_head - queue a buffer at the list head
2865  *	@list: list to use
2866  *	@newsk: buffer to queue
2867  *
2868  *	Queue a buffer at the start of the list. This function takes the
2869  *	list lock and can be used safely with other locking &sk_buff functions
2870  *	safely.
2871  *
2872  *	A buffer cannot be placed on two lists at the same time.
2873  */
2874 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
2875 {
2876 	unsigned long flags;
2877 
2878 	spin_lock_irqsave(&list->lock, flags);
2879 	__skb_queue_head(list, newsk);
2880 	spin_unlock_irqrestore(&list->lock, flags);
2881 }
2882 EXPORT_SYMBOL(skb_queue_head);
2883 
2884 /**
2885  *	skb_queue_tail - queue a buffer at the list tail
2886  *	@list: list to use
2887  *	@newsk: buffer to queue
2888  *
2889  *	Queue a buffer at the tail of the list. This function takes the
2890  *	list lock and can be used safely with other locking &sk_buff functions
2891  *	safely.
2892  *
2893  *	A buffer cannot be placed on two lists at the same time.
2894  */
2895 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
2896 {
2897 	unsigned long flags;
2898 
2899 	spin_lock_irqsave(&list->lock, flags);
2900 	__skb_queue_tail(list, newsk);
2901 	spin_unlock_irqrestore(&list->lock, flags);
2902 }
2903 EXPORT_SYMBOL(skb_queue_tail);
2904 
2905 /**
2906  *	skb_unlink	-	remove a buffer from a list
2907  *	@skb: buffer to remove
2908  *	@list: list to use
2909  *
2910  *	Remove a packet from a list. The list locks are taken and this
2911  *	function is atomic with respect to other list locked calls
2912  *
2913  *	You must know what list the SKB is on.
2914  */
2915 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2916 {
2917 	unsigned long flags;
2918 
2919 	spin_lock_irqsave(&list->lock, flags);
2920 	__skb_unlink(skb, list);
2921 	spin_unlock_irqrestore(&list->lock, flags);
2922 }
2923 EXPORT_SYMBOL(skb_unlink);
2924 
2925 /**
2926  *	skb_append	-	append a buffer
2927  *	@old: buffer to insert after
2928  *	@newsk: buffer to insert
2929  *	@list: list to use
2930  *
2931  *	Place a packet after a given packet in a list. The list locks are taken
2932  *	and this function is atomic with respect to other list locked calls.
2933  *	A buffer cannot be placed on two lists at the same time.
2934  */
2935 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2936 {
2937 	unsigned long flags;
2938 
2939 	spin_lock_irqsave(&list->lock, flags);
2940 	__skb_queue_after(list, old, newsk);
2941 	spin_unlock_irqrestore(&list->lock, flags);
2942 }
2943 EXPORT_SYMBOL(skb_append);
2944 
2945 /**
2946  *	skb_insert	-	insert a buffer
2947  *	@old: buffer to insert before
2948  *	@newsk: buffer to insert
2949  *	@list: list to use
2950  *
2951  *	Place a packet before a given packet in a list. The list locks are
2952  * 	taken and this function is atomic with respect to other list locked
2953  *	calls.
2954  *
2955  *	A buffer cannot be placed on two lists at the same time.
2956  */
2957 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2958 {
2959 	unsigned long flags;
2960 
2961 	spin_lock_irqsave(&list->lock, flags);
2962 	__skb_insert(newsk, old->prev, old, list);
2963 	spin_unlock_irqrestore(&list->lock, flags);
2964 }
2965 EXPORT_SYMBOL(skb_insert);
2966 
2967 static inline void skb_split_inside_header(struct sk_buff *skb,
2968 					   struct sk_buff* skb1,
2969 					   const u32 len, const int pos)
2970 {
2971 	int i;
2972 
2973 	skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2974 					 pos - len);
2975 	/* And move data appendix as is. */
2976 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2977 		skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2978 
2979 	skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2980 	skb_shinfo(skb)->nr_frags  = 0;
2981 	skb1->data_len		   = skb->data_len;
2982 	skb1->len		   += skb1->data_len;
2983 	skb->data_len		   = 0;
2984 	skb->len		   = len;
2985 	skb_set_tail_pointer(skb, len);
2986 }
2987 
2988 static inline void skb_split_no_header(struct sk_buff *skb,
2989 				       struct sk_buff* skb1,
2990 				       const u32 len, int pos)
2991 {
2992 	int i, k = 0;
2993 	const int nfrags = skb_shinfo(skb)->nr_frags;
2994 
2995 	skb_shinfo(skb)->nr_frags = 0;
2996 	skb1->len		  = skb1->data_len = skb->len - len;
2997 	skb->len		  = len;
2998 	skb->data_len		  = len - pos;
2999 
3000 	for (i = 0; i < nfrags; i++) {
3001 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
3002 
3003 		if (pos + size > len) {
3004 			skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3005 
3006 			if (pos < len) {
3007 				/* Split frag.
3008 				 * We have two variants in this case:
3009 				 * 1. Move all the frag to the second
3010 				 *    part, if it is possible. F.e.
3011 				 *    this approach is mandatory for TUX,
3012 				 *    where splitting is expensive.
3013 				 * 2. Split is accurately. We make this.
3014 				 */
3015 				skb_frag_ref(skb, i);
3016 				skb_shinfo(skb1)->frags[0].page_offset += len - pos;
3017 				skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3018 				skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3019 				skb_shinfo(skb)->nr_frags++;
3020 			}
3021 			k++;
3022 		} else
3023 			skb_shinfo(skb)->nr_frags++;
3024 		pos += size;
3025 	}
3026 	skb_shinfo(skb1)->nr_frags = k;
3027 }
3028 
3029 /**
3030  * skb_split - Split fragmented skb to two parts at length len.
3031  * @skb: the buffer to split
3032  * @skb1: the buffer to receive the second part
3033  * @len: new length for skb
3034  */
3035 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3036 {
3037 	int pos = skb_headlen(skb);
3038 
3039 	skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
3040 				      SKBTX_SHARED_FRAG;
3041 	skb_zerocopy_clone(skb1, skb, 0);
3042 	if (len < pos)	/* Split line is inside header. */
3043 		skb_split_inside_header(skb, skb1, len, pos);
3044 	else		/* Second chunk has no header, nothing to copy. */
3045 		skb_split_no_header(skb, skb1, len, pos);
3046 }
3047 EXPORT_SYMBOL(skb_split);
3048 
3049 /* Shifting from/to a cloned skb is a no-go.
3050  *
3051  * Caller cannot keep skb_shinfo related pointers past calling here!
3052  */
3053 static int skb_prepare_for_shift(struct sk_buff *skb)
3054 {
3055 	return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3056 }
3057 
3058 /**
3059  * skb_shift - Shifts paged data partially from skb to another
3060  * @tgt: buffer into which tail data gets added
3061  * @skb: buffer from which the paged data comes from
3062  * @shiftlen: shift up to this many bytes
3063  *
3064  * Attempts to shift up to shiftlen worth of bytes, which may be less than
3065  * the length of the skb, from skb to tgt. Returns number bytes shifted.
3066  * It's up to caller to free skb if everything was shifted.
3067  *
3068  * If @tgt runs out of frags, the whole operation is aborted.
3069  *
3070  * Skb cannot include anything else but paged data while tgt is allowed
3071  * to have non-paged data as well.
3072  *
3073  * TODO: full sized shift could be optimized but that would need
3074  * specialized skb free'er to handle frags without up-to-date nr_frags.
3075  */
3076 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
3077 {
3078 	int from, to, merge, todo;
3079 	struct skb_frag_struct *fragfrom, *fragto;
3080 
3081 	BUG_ON(shiftlen > skb->len);
3082 
3083 	if (skb_headlen(skb))
3084 		return 0;
3085 	if (skb_zcopy(tgt) || skb_zcopy(skb))
3086 		return 0;
3087 
3088 	todo = shiftlen;
3089 	from = 0;
3090 	to = skb_shinfo(tgt)->nr_frags;
3091 	fragfrom = &skb_shinfo(skb)->frags[from];
3092 
3093 	/* Actual merge is delayed until the point when we know we can
3094 	 * commit all, so that we don't have to undo partial changes
3095 	 */
3096 	if (!to ||
3097 	    !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3098 			      fragfrom->page_offset)) {
3099 		merge = -1;
3100 	} else {
3101 		merge = to - 1;
3102 
3103 		todo -= skb_frag_size(fragfrom);
3104 		if (todo < 0) {
3105 			if (skb_prepare_for_shift(skb) ||
3106 			    skb_prepare_for_shift(tgt))
3107 				return 0;
3108 
3109 			/* All previous frag pointers might be stale! */
3110 			fragfrom = &skb_shinfo(skb)->frags[from];
3111 			fragto = &skb_shinfo(tgt)->frags[merge];
3112 
3113 			skb_frag_size_add(fragto, shiftlen);
3114 			skb_frag_size_sub(fragfrom, shiftlen);
3115 			fragfrom->page_offset += shiftlen;
3116 
3117 			goto onlymerged;
3118 		}
3119 
3120 		from++;
3121 	}
3122 
3123 	/* Skip full, not-fitting skb to avoid expensive operations */
3124 	if ((shiftlen == skb->len) &&
3125 	    (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
3126 		return 0;
3127 
3128 	if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
3129 		return 0;
3130 
3131 	while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
3132 		if (to == MAX_SKB_FRAGS)
3133 			return 0;
3134 
3135 		fragfrom = &skb_shinfo(skb)->frags[from];
3136 		fragto = &skb_shinfo(tgt)->frags[to];
3137 
3138 		if (todo >= skb_frag_size(fragfrom)) {
3139 			*fragto = *fragfrom;
3140 			todo -= skb_frag_size(fragfrom);
3141 			from++;
3142 			to++;
3143 
3144 		} else {
3145 			__skb_frag_ref(fragfrom);
3146 			fragto->page = fragfrom->page;
3147 			fragto->page_offset = fragfrom->page_offset;
3148 			skb_frag_size_set(fragto, todo);
3149 
3150 			fragfrom->page_offset += todo;
3151 			skb_frag_size_sub(fragfrom, todo);
3152 			todo = 0;
3153 
3154 			to++;
3155 			break;
3156 		}
3157 	}
3158 
3159 	/* Ready to "commit" this state change to tgt */
3160 	skb_shinfo(tgt)->nr_frags = to;
3161 
3162 	if (merge >= 0) {
3163 		fragfrom = &skb_shinfo(skb)->frags[0];
3164 		fragto = &skb_shinfo(tgt)->frags[merge];
3165 
3166 		skb_frag_size_add(fragto, skb_frag_size(fragfrom));
3167 		__skb_frag_unref(fragfrom);
3168 	}
3169 
3170 	/* Reposition in the original skb */
3171 	to = 0;
3172 	while (from < skb_shinfo(skb)->nr_frags)
3173 		skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
3174 	skb_shinfo(skb)->nr_frags = to;
3175 
3176 	BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
3177 
3178 onlymerged:
3179 	/* Most likely the tgt won't ever need its checksum anymore, skb on
3180 	 * the other hand might need it if it needs to be resent
3181 	 */
3182 	tgt->ip_summed = CHECKSUM_PARTIAL;
3183 	skb->ip_summed = CHECKSUM_PARTIAL;
3184 
3185 	/* Yak, is it really working this way? Some helper please? */
3186 	skb->len -= shiftlen;
3187 	skb->data_len -= shiftlen;
3188 	skb->truesize -= shiftlen;
3189 	tgt->len += shiftlen;
3190 	tgt->data_len += shiftlen;
3191 	tgt->truesize += shiftlen;
3192 
3193 	return shiftlen;
3194 }
3195 
3196 /**
3197  * skb_prepare_seq_read - Prepare a sequential read of skb data
3198  * @skb: the buffer to read
3199  * @from: lower offset of data to be read
3200  * @to: upper offset of data to be read
3201  * @st: state variable
3202  *
3203  * Initializes the specified state variable. Must be called before
3204  * invoking skb_seq_read() for the first time.
3205  */
3206 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
3207 			  unsigned int to, struct skb_seq_state *st)
3208 {
3209 	st->lower_offset = from;
3210 	st->upper_offset = to;
3211 	st->root_skb = st->cur_skb = skb;
3212 	st->frag_idx = st->stepped_offset = 0;
3213 	st->frag_data = NULL;
3214 }
3215 EXPORT_SYMBOL(skb_prepare_seq_read);
3216 
3217 /**
3218  * skb_seq_read - Sequentially read skb data
3219  * @consumed: number of bytes consumed by the caller so far
3220  * @data: destination pointer for data to be returned
3221  * @st: state variable
3222  *
3223  * Reads a block of skb data at @consumed relative to the
3224  * lower offset specified to skb_prepare_seq_read(). Assigns
3225  * the head of the data block to @data and returns the length
3226  * of the block or 0 if the end of the skb data or the upper
3227  * offset has been reached.
3228  *
3229  * The caller is not required to consume all of the data
3230  * returned, i.e. @consumed is typically set to the number
3231  * of bytes already consumed and the next call to
3232  * skb_seq_read() will return the remaining part of the block.
3233  *
3234  * Note 1: The size of each block of data returned can be arbitrary,
3235  *       this limitation is the cost for zerocopy sequential
3236  *       reads of potentially non linear data.
3237  *
3238  * Note 2: Fragment lists within fragments are not implemented
3239  *       at the moment, state->root_skb could be replaced with
3240  *       a stack for this purpose.
3241  */
3242 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
3243 			  struct skb_seq_state *st)
3244 {
3245 	unsigned int block_limit, abs_offset = consumed + st->lower_offset;
3246 	skb_frag_t *frag;
3247 
3248 	if (unlikely(abs_offset >= st->upper_offset)) {
3249 		if (st->frag_data) {
3250 			kunmap_atomic(st->frag_data);
3251 			st->frag_data = NULL;
3252 		}
3253 		return 0;
3254 	}
3255 
3256 next_skb:
3257 	block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
3258 
3259 	if (abs_offset < block_limit && !st->frag_data) {
3260 		*data = st->cur_skb->data + (abs_offset - st->stepped_offset);
3261 		return block_limit - abs_offset;
3262 	}
3263 
3264 	if (st->frag_idx == 0 && !st->frag_data)
3265 		st->stepped_offset += skb_headlen(st->cur_skb);
3266 
3267 	while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
3268 		frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
3269 		block_limit = skb_frag_size(frag) + st->stepped_offset;
3270 
3271 		if (abs_offset < block_limit) {
3272 			if (!st->frag_data)
3273 				st->frag_data = kmap_atomic(skb_frag_page(frag));
3274 
3275 			*data = (u8 *) st->frag_data + frag->page_offset +
3276 				(abs_offset - st->stepped_offset);
3277 
3278 			return block_limit - abs_offset;
3279 		}
3280 
3281 		if (st->frag_data) {
3282 			kunmap_atomic(st->frag_data);
3283 			st->frag_data = NULL;
3284 		}
3285 
3286 		st->frag_idx++;
3287 		st->stepped_offset += skb_frag_size(frag);
3288 	}
3289 
3290 	if (st->frag_data) {
3291 		kunmap_atomic(st->frag_data);
3292 		st->frag_data = NULL;
3293 	}
3294 
3295 	if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
3296 		st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
3297 		st->frag_idx = 0;
3298 		goto next_skb;
3299 	} else if (st->cur_skb->next) {
3300 		st->cur_skb = st->cur_skb->next;
3301 		st->frag_idx = 0;
3302 		goto next_skb;
3303 	}
3304 
3305 	return 0;
3306 }
3307 EXPORT_SYMBOL(skb_seq_read);
3308 
3309 /**
3310  * skb_abort_seq_read - Abort a sequential read of skb data
3311  * @st: state variable
3312  *
3313  * Must be called if skb_seq_read() was not called until it
3314  * returned 0.
3315  */
3316 void skb_abort_seq_read(struct skb_seq_state *st)
3317 {
3318 	if (st->frag_data)
3319 		kunmap_atomic(st->frag_data);
3320 }
3321 EXPORT_SYMBOL(skb_abort_seq_read);
3322 
3323 #define TS_SKB_CB(state)	((struct skb_seq_state *) &((state)->cb))
3324 
3325 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
3326 					  struct ts_config *conf,
3327 					  struct ts_state *state)
3328 {
3329 	return skb_seq_read(offset, text, TS_SKB_CB(state));
3330 }
3331 
3332 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
3333 {
3334 	skb_abort_seq_read(TS_SKB_CB(state));
3335 }
3336 
3337 /**
3338  * skb_find_text - Find a text pattern in skb data
3339  * @skb: the buffer to look in
3340  * @from: search offset
3341  * @to: search limit
3342  * @config: textsearch configuration
3343  *
3344  * Finds a pattern in the skb data according to the specified
3345  * textsearch configuration. Use textsearch_next() to retrieve
3346  * subsequent occurrences of the pattern. Returns the offset
3347  * to the first occurrence or UINT_MAX if no match was found.
3348  */
3349 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
3350 			   unsigned int to, struct ts_config *config)
3351 {
3352 	struct ts_state state;
3353 	unsigned int ret;
3354 
3355 	config->get_next_block = skb_ts_get_next_block;
3356 	config->finish = skb_ts_finish;
3357 
3358 	skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
3359 
3360 	ret = textsearch_find(config, &state);
3361 	return (ret <= to - from ? ret : UINT_MAX);
3362 }
3363 EXPORT_SYMBOL(skb_find_text);
3364 
3365 /**
3366  * skb_append_datato_frags - append the user data to a skb
3367  * @sk: sock  structure
3368  * @skb: skb structure to be appended with user data.
3369  * @getfrag: call back function to be used for getting the user data
3370  * @from: pointer to user message iov
3371  * @length: length of the iov message
3372  *
3373  * Description: This procedure append the user data in the fragment part
3374  * of the skb if any page alloc fails user this procedure returns  -ENOMEM
3375  */
3376 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
3377 			int (*getfrag)(void *from, char *to, int offset,
3378 					int len, int odd, struct sk_buff *skb),
3379 			void *from, int length)
3380 {
3381 	int frg_cnt = skb_shinfo(skb)->nr_frags;
3382 	int copy;
3383 	int offset = 0;
3384 	int ret;
3385 	struct page_frag *pfrag = &current->task_frag;
3386 
3387 	do {
3388 		/* Return error if we don't have space for new frag */
3389 		if (frg_cnt >= MAX_SKB_FRAGS)
3390 			return -EMSGSIZE;
3391 
3392 		if (!sk_page_frag_refill(sk, pfrag))
3393 			return -ENOMEM;
3394 
3395 		/* copy the user data to page */
3396 		copy = min_t(int, length, pfrag->size - pfrag->offset);
3397 
3398 		ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
3399 			      offset, copy, 0, skb);
3400 		if (ret < 0)
3401 			return -EFAULT;
3402 
3403 		/* copy was successful so update the size parameters */
3404 		skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
3405 				   copy);
3406 		frg_cnt++;
3407 		pfrag->offset += copy;
3408 		get_page(pfrag->page);
3409 
3410 		skb->truesize += copy;
3411 		refcount_add(copy, &sk->sk_wmem_alloc);
3412 		skb->len += copy;
3413 		skb->data_len += copy;
3414 		offset += copy;
3415 		length -= copy;
3416 
3417 	} while (length > 0);
3418 
3419 	return 0;
3420 }
3421 EXPORT_SYMBOL(skb_append_datato_frags);
3422 
3423 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
3424 			 int offset, size_t size)
3425 {
3426 	int i = skb_shinfo(skb)->nr_frags;
3427 
3428 	if (skb_can_coalesce(skb, i, page, offset)) {
3429 		skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
3430 	} else if (i < MAX_SKB_FRAGS) {
3431 		get_page(page);
3432 		skb_fill_page_desc(skb, i, page, offset, size);
3433 	} else {
3434 		return -EMSGSIZE;
3435 	}
3436 
3437 	return 0;
3438 }
3439 EXPORT_SYMBOL_GPL(skb_append_pagefrags);
3440 
3441 /**
3442  *	skb_pull_rcsum - pull skb and update receive checksum
3443  *	@skb: buffer to update
3444  *	@len: length of data pulled
3445  *
3446  *	This function performs an skb_pull on the packet and updates
3447  *	the CHECKSUM_COMPLETE checksum.  It should be used on
3448  *	receive path processing instead of skb_pull unless you know
3449  *	that the checksum difference is zero (e.g., a valid IP header)
3450  *	or you are setting ip_summed to CHECKSUM_NONE.
3451  */
3452 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
3453 {
3454 	unsigned char *data = skb->data;
3455 
3456 	BUG_ON(len > skb->len);
3457 	__skb_pull(skb, len);
3458 	skb_postpull_rcsum(skb, data, len);
3459 	return skb->data;
3460 }
3461 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
3462 
3463 /**
3464  *	skb_segment - Perform protocol segmentation on skb.
3465  *	@head_skb: buffer to segment
3466  *	@features: features for the output path (see dev->features)
3467  *
3468  *	This function performs segmentation on the given skb.  It returns
3469  *	a pointer to the first in a list of new skbs for the segments.
3470  *	In case of error it returns ERR_PTR(err).
3471  */
3472 struct sk_buff *skb_segment(struct sk_buff *head_skb,
3473 			    netdev_features_t features)
3474 {
3475 	struct sk_buff *segs = NULL;
3476 	struct sk_buff *tail = NULL;
3477 	struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
3478 	skb_frag_t *frag = skb_shinfo(head_skb)->frags;
3479 	unsigned int mss = skb_shinfo(head_skb)->gso_size;
3480 	unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
3481 	struct sk_buff *frag_skb = head_skb;
3482 	unsigned int offset = doffset;
3483 	unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
3484 	unsigned int partial_segs = 0;
3485 	unsigned int headroom;
3486 	unsigned int len = head_skb->len;
3487 	__be16 proto;
3488 	bool csum, sg;
3489 	int nfrags = skb_shinfo(head_skb)->nr_frags;
3490 	int err = -ENOMEM;
3491 	int i = 0;
3492 	int pos;
3493 	int dummy;
3494 
3495 	__skb_push(head_skb, doffset);
3496 	proto = skb_network_protocol(head_skb, &dummy);
3497 	if (unlikely(!proto))
3498 		return ERR_PTR(-EINVAL);
3499 
3500 	sg = !!(features & NETIF_F_SG);
3501 	csum = !!can_checksum_protocol(features, proto);
3502 
3503 	if (sg && csum && (mss != GSO_BY_FRAGS))  {
3504 		if (!(features & NETIF_F_GSO_PARTIAL)) {
3505 			struct sk_buff *iter;
3506 			unsigned int frag_len;
3507 
3508 			if (!list_skb ||
3509 			    !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
3510 				goto normal;
3511 
3512 			/* If we get here then all the required
3513 			 * GSO features except frag_list are supported.
3514 			 * Try to split the SKB to multiple GSO SKBs
3515 			 * with no frag_list.
3516 			 * Currently we can do that only when the buffers don't
3517 			 * have a linear part and all the buffers except
3518 			 * the last are of the same length.
3519 			 */
3520 			frag_len = list_skb->len;
3521 			skb_walk_frags(head_skb, iter) {
3522 				if (frag_len != iter->len && iter->next)
3523 					goto normal;
3524 				if (skb_headlen(iter) && !iter->head_frag)
3525 					goto normal;
3526 
3527 				len -= iter->len;
3528 			}
3529 
3530 			if (len != frag_len)
3531 				goto normal;
3532 		}
3533 
3534 		/* GSO partial only requires that we trim off any excess that
3535 		 * doesn't fit into an MSS sized block, so take care of that
3536 		 * now.
3537 		 */
3538 		partial_segs = len / mss;
3539 		if (partial_segs > 1)
3540 			mss *= partial_segs;
3541 		else
3542 			partial_segs = 0;
3543 	}
3544 
3545 normal:
3546 	headroom = skb_headroom(head_skb);
3547 	pos = skb_headlen(head_skb);
3548 
3549 	do {
3550 		struct sk_buff *nskb;
3551 		skb_frag_t *nskb_frag;
3552 		int hsize;
3553 		int size;
3554 
3555 		if (unlikely(mss == GSO_BY_FRAGS)) {
3556 			len = list_skb->len;
3557 		} else {
3558 			len = head_skb->len - offset;
3559 			if (len > mss)
3560 				len = mss;
3561 		}
3562 
3563 		hsize = skb_headlen(head_skb) - offset;
3564 		if (hsize < 0)
3565 			hsize = 0;
3566 		if (hsize > len || !sg)
3567 			hsize = len;
3568 
3569 		if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
3570 		    (skb_headlen(list_skb) == len || sg)) {
3571 			BUG_ON(skb_headlen(list_skb) > len);
3572 
3573 			i = 0;
3574 			nfrags = skb_shinfo(list_skb)->nr_frags;
3575 			frag = skb_shinfo(list_skb)->frags;
3576 			frag_skb = list_skb;
3577 			pos += skb_headlen(list_skb);
3578 
3579 			while (pos < offset + len) {
3580 				BUG_ON(i >= nfrags);
3581 
3582 				size = skb_frag_size(frag);
3583 				if (pos + size > offset + len)
3584 					break;
3585 
3586 				i++;
3587 				pos += size;
3588 				frag++;
3589 			}
3590 
3591 			nskb = skb_clone(list_skb, GFP_ATOMIC);
3592 			list_skb = list_skb->next;
3593 
3594 			if (unlikely(!nskb))
3595 				goto err;
3596 
3597 			if (unlikely(pskb_trim(nskb, len))) {
3598 				kfree_skb(nskb);
3599 				goto err;
3600 			}
3601 
3602 			hsize = skb_end_offset(nskb);
3603 			if (skb_cow_head(nskb, doffset + headroom)) {
3604 				kfree_skb(nskb);
3605 				goto err;
3606 			}
3607 
3608 			nskb->truesize += skb_end_offset(nskb) - hsize;
3609 			skb_release_head_state(nskb);
3610 			__skb_push(nskb, doffset);
3611 		} else {
3612 			nskb = __alloc_skb(hsize + doffset + headroom,
3613 					   GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
3614 					   NUMA_NO_NODE);
3615 
3616 			if (unlikely(!nskb))
3617 				goto err;
3618 
3619 			skb_reserve(nskb, headroom);
3620 			__skb_put(nskb, doffset);
3621 		}
3622 
3623 		if (segs)
3624 			tail->next = nskb;
3625 		else
3626 			segs = nskb;
3627 		tail = nskb;
3628 
3629 		__copy_skb_header(nskb, head_skb);
3630 
3631 		skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
3632 		skb_reset_mac_len(nskb);
3633 
3634 		skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
3635 						 nskb->data - tnl_hlen,
3636 						 doffset + tnl_hlen);
3637 
3638 		if (nskb->len == len + doffset)
3639 			goto perform_csum_check;
3640 
3641 		if (!sg) {
3642 			if (!nskb->remcsum_offload)
3643 				nskb->ip_summed = CHECKSUM_NONE;
3644 			SKB_GSO_CB(nskb)->csum =
3645 				skb_copy_and_csum_bits(head_skb, offset,
3646 						       skb_put(nskb, len),
3647 						       len, 0);
3648 			SKB_GSO_CB(nskb)->csum_start =
3649 				skb_headroom(nskb) + doffset;
3650 			continue;
3651 		}
3652 
3653 		nskb_frag = skb_shinfo(nskb)->frags;
3654 
3655 		skb_copy_from_linear_data_offset(head_skb, offset,
3656 						 skb_put(nskb, hsize), hsize);
3657 
3658 		skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags &
3659 					      SKBTX_SHARED_FRAG;
3660 		if (skb_zerocopy_clone(nskb, head_skb, GFP_ATOMIC))
3661 			goto err;
3662 
3663 		while (pos < offset + len) {
3664 			if (i >= nfrags) {
3665 				BUG_ON(skb_headlen(list_skb));
3666 
3667 				i = 0;
3668 				nfrags = skb_shinfo(list_skb)->nr_frags;
3669 				frag = skb_shinfo(list_skb)->frags;
3670 				frag_skb = list_skb;
3671 
3672 				BUG_ON(!nfrags);
3673 
3674 				list_skb = list_skb->next;
3675 			}
3676 
3677 			if (unlikely(skb_shinfo(nskb)->nr_frags >=
3678 				     MAX_SKB_FRAGS)) {
3679 				net_warn_ratelimited(
3680 					"skb_segment: too many frags: %u %u\n",
3681 					pos, mss);
3682 				goto err;
3683 			}
3684 
3685 			if (unlikely(skb_orphan_frags(frag_skb, GFP_ATOMIC)))
3686 				goto err;
3687 
3688 			*nskb_frag = *frag;
3689 			__skb_frag_ref(nskb_frag);
3690 			size = skb_frag_size(nskb_frag);
3691 
3692 			if (pos < offset) {
3693 				nskb_frag->page_offset += offset - pos;
3694 				skb_frag_size_sub(nskb_frag, offset - pos);
3695 			}
3696 
3697 			skb_shinfo(nskb)->nr_frags++;
3698 
3699 			if (pos + size <= offset + len) {
3700 				i++;
3701 				frag++;
3702 				pos += size;
3703 			} else {
3704 				skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
3705 				goto skip_fraglist;
3706 			}
3707 
3708 			nskb_frag++;
3709 		}
3710 
3711 skip_fraglist:
3712 		nskb->data_len = len - hsize;
3713 		nskb->len += nskb->data_len;
3714 		nskb->truesize += nskb->data_len;
3715 
3716 perform_csum_check:
3717 		if (!csum) {
3718 			if (skb_has_shared_frag(nskb)) {
3719 				err = __skb_linearize(nskb);
3720 				if (err)
3721 					goto err;
3722 			}
3723 			if (!nskb->remcsum_offload)
3724 				nskb->ip_summed = CHECKSUM_NONE;
3725 			SKB_GSO_CB(nskb)->csum =
3726 				skb_checksum(nskb, doffset,
3727 					     nskb->len - doffset, 0);
3728 			SKB_GSO_CB(nskb)->csum_start =
3729 				skb_headroom(nskb) + doffset;
3730 		}
3731 	} while ((offset += len) < head_skb->len);
3732 
3733 	/* Some callers want to get the end of the list.
3734 	 * Put it in segs->prev to avoid walking the list.
3735 	 * (see validate_xmit_skb_list() for example)
3736 	 */
3737 	segs->prev = tail;
3738 
3739 	if (partial_segs) {
3740 		struct sk_buff *iter;
3741 		int type = skb_shinfo(head_skb)->gso_type;
3742 		unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
3743 
3744 		/* Update type to add partial and then remove dodgy if set */
3745 		type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
3746 		type &= ~SKB_GSO_DODGY;
3747 
3748 		/* Update GSO info and prepare to start updating headers on
3749 		 * our way back down the stack of protocols.
3750 		 */
3751 		for (iter = segs; iter; iter = iter->next) {
3752 			skb_shinfo(iter)->gso_size = gso_size;
3753 			skb_shinfo(iter)->gso_segs = partial_segs;
3754 			skb_shinfo(iter)->gso_type = type;
3755 			SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
3756 		}
3757 
3758 		if (tail->len - doffset <= gso_size)
3759 			skb_shinfo(tail)->gso_size = 0;
3760 		else if (tail != segs)
3761 			skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
3762 	}
3763 
3764 	/* Following permits correct backpressure, for protocols
3765 	 * using skb_set_owner_w().
3766 	 * Idea is to tranfert ownership from head_skb to last segment.
3767 	 */
3768 	if (head_skb->destructor == sock_wfree) {
3769 		swap(tail->truesize, head_skb->truesize);
3770 		swap(tail->destructor, head_skb->destructor);
3771 		swap(tail->sk, head_skb->sk);
3772 	}
3773 	return segs;
3774 
3775 err:
3776 	kfree_skb_list(segs);
3777 	return ERR_PTR(err);
3778 }
3779 EXPORT_SYMBOL_GPL(skb_segment);
3780 
3781 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
3782 {
3783 	struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
3784 	unsigned int offset = skb_gro_offset(skb);
3785 	unsigned int headlen = skb_headlen(skb);
3786 	unsigned int len = skb_gro_len(skb);
3787 	struct sk_buff *lp, *p = *head;
3788 	unsigned int delta_truesize;
3789 
3790 	if (unlikely(p->len + len >= 65536))
3791 		return -E2BIG;
3792 
3793 	lp = NAPI_GRO_CB(p)->last;
3794 	pinfo = skb_shinfo(lp);
3795 
3796 	if (headlen <= offset) {
3797 		skb_frag_t *frag;
3798 		skb_frag_t *frag2;
3799 		int i = skbinfo->nr_frags;
3800 		int nr_frags = pinfo->nr_frags + i;
3801 
3802 		if (nr_frags > MAX_SKB_FRAGS)
3803 			goto merge;
3804 
3805 		offset -= headlen;
3806 		pinfo->nr_frags = nr_frags;
3807 		skbinfo->nr_frags = 0;
3808 
3809 		frag = pinfo->frags + nr_frags;
3810 		frag2 = skbinfo->frags + i;
3811 		do {
3812 			*--frag = *--frag2;
3813 		} while (--i);
3814 
3815 		frag->page_offset += offset;
3816 		skb_frag_size_sub(frag, offset);
3817 
3818 		/* all fragments truesize : remove (head size + sk_buff) */
3819 		delta_truesize = skb->truesize -
3820 				 SKB_TRUESIZE(skb_end_offset(skb));
3821 
3822 		skb->truesize -= skb->data_len;
3823 		skb->len -= skb->data_len;
3824 		skb->data_len = 0;
3825 
3826 		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
3827 		goto done;
3828 	} else if (skb->head_frag) {
3829 		int nr_frags = pinfo->nr_frags;
3830 		skb_frag_t *frag = pinfo->frags + nr_frags;
3831 		struct page *page = virt_to_head_page(skb->head);
3832 		unsigned int first_size = headlen - offset;
3833 		unsigned int first_offset;
3834 
3835 		if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
3836 			goto merge;
3837 
3838 		first_offset = skb->data -
3839 			       (unsigned char *)page_address(page) +
3840 			       offset;
3841 
3842 		pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3843 
3844 		frag->page.p	  = page;
3845 		frag->page_offset = first_offset;
3846 		skb_frag_size_set(frag, first_size);
3847 
3848 		memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
3849 		/* We dont need to clear skbinfo->nr_frags here */
3850 
3851 		delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3852 		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
3853 		goto done;
3854 	}
3855 
3856 merge:
3857 	delta_truesize = skb->truesize;
3858 	if (offset > headlen) {
3859 		unsigned int eat = offset - headlen;
3860 
3861 		skbinfo->frags[0].page_offset += eat;
3862 		skb_frag_size_sub(&skbinfo->frags[0], eat);
3863 		skb->data_len -= eat;
3864 		skb->len -= eat;
3865 		offset = headlen;
3866 	}
3867 
3868 	__skb_pull(skb, offset);
3869 
3870 	if (NAPI_GRO_CB(p)->last == p)
3871 		skb_shinfo(p)->frag_list = skb;
3872 	else
3873 		NAPI_GRO_CB(p)->last->next = skb;
3874 	NAPI_GRO_CB(p)->last = skb;
3875 	__skb_header_release(skb);
3876 	lp = p;
3877 
3878 done:
3879 	NAPI_GRO_CB(p)->count++;
3880 	p->data_len += len;
3881 	p->truesize += delta_truesize;
3882 	p->len += len;
3883 	if (lp != p) {
3884 		lp->data_len += len;
3885 		lp->truesize += delta_truesize;
3886 		lp->len += len;
3887 	}
3888 	NAPI_GRO_CB(skb)->same_flow = 1;
3889 	return 0;
3890 }
3891 EXPORT_SYMBOL_GPL(skb_gro_receive);
3892 
3893 void __init skb_init(void)
3894 {
3895 	skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
3896 					      sizeof(struct sk_buff),
3897 					      0,
3898 					      SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3899 					      NULL);
3900 	skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
3901 						sizeof(struct sk_buff_fclones),
3902 						0,
3903 						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3904 						NULL);
3905 }
3906 
3907 static int
3908 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
3909 	       unsigned int recursion_level)
3910 {
3911 	int start = skb_headlen(skb);
3912 	int i, copy = start - offset;
3913 	struct sk_buff *frag_iter;
3914 	int elt = 0;
3915 
3916 	if (unlikely(recursion_level >= 24))
3917 		return -EMSGSIZE;
3918 
3919 	if (copy > 0) {
3920 		if (copy > len)
3921 			copy = len;
3922 		sg_set_buf(sg, skb->data + offset, copy);
3923 		elt++;
3924 		if ((len -= copy) == 0)
3925 			return elt;
3926 		offset += copy;
3927 	}
3928 
3929 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3930 		int end;
3931 
3932 		WARN_ON(start > offset + len);
3933 
3934 		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3935 		if ((copy = end - offset) > 0) {
3936 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3937 			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
3938 				return -EMSGSIZE;
3939 
3940 			if (copy > len)
3941 				copy = len;
3942 			sg_set_page(&sg[elt], skb_frag_page(frag), copy,
3943 					frag->page_offset+offset-start);
3944 			elt++;
3945 			if (!(len -= copy))
3946 				return elt;
3947 			offset += copy;
3948 		}
3949 		start = end;
3950 	}
3951 
3952 	skb_walk_frags(skb, frag_iter) {
3953 		int end, ret;
3954 
3955 		WARN_ON(start > offset + len);
3956 
3957 		end = start + frag_iter->len;
3958 		if ((copy = end - offset) > 0) {
3959 			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
3960 				return -EMSGSIZE;
3961 
3962 			if (copy > len)
3963 				copy = len;
3964 			ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
3965 					      copy, recursion_level + 1);
3966 			if (unlikely(ret < 0))
3967 				return ret;
3968 			elt += ret;
3969 			if ((len -= copy) == 0)
3970 				return elt;
3971 			offset += copy;
3972 		}
3973 		start = end;
3974 	}
3975 	BUG_ON(len);
3976 	return elt;
3977 }
3978 
3979 /**
3980  *	skb_to_sgvec - Fill a scatter-gather list from a socket buffer
3981  *	@skb: Socket buffer containing the buffers to be mapped
3982  *	@sg: The scatter-gather list to map into
3983  *	@offset: The offset into the buffer's contents to start mapping
3984  *	@len: Length of buffer space to be mapped
3985  *
3986  *	Fill the specified scatter-gather list with mappings/pointers into a
3987  *	region of the buffer space attached to a socket buffer. Returns either
3988  *	the number of scatterlist items used, or -EMSGSIZE if the contents
3989  *	could not fit.
3990  */
3991 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3992 {
3993 	int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
3994 
3995 	if (nsg <= 0)
3996 		return nsg;
3997 
3998 	sg_mark_end(&sg[nsg - 1]);
3999 
4000 	return nsg;
4001 }
4002 EXPORT_SYMBOL_GPL(skb_to_sgvec);
4003 
4004 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
4005  * sglist without mark the sg which contain last skb data as the end.
4006  * So the caller can mannipulate sg list as will when padding new data after
4007  * the first call without calling sg_unmark_end to expend sg list.
4008  *
4009  * Scenario to use skb_to_sgvec_nomark:
4010  * 1. sg_init_table
4011  * 2. skb_to_sgvec_nomark(payload1)
4012  * 3. skb_to_sgvec_nomark(payload2)
4013  *
4014  * This is equivalent to:
4015  * 1. sg_init_table
4016  * 2. skb_to_sgvec(payload1)
4017  * 3. sg_unmark_end
4018  * 4. skb_to_sgvec(payload2)
4019  *
4020  * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
4021  * is more preferable.
4022  */
4023 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
4024 			int offset, int len)
4025 {
4026 	return __skb_to_sgvec(skb, sg, offset, len, 0);
4027 }
4028 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
4029 
4030 
4031 
4032 /**
4033  *	skb_cow_data - Check that a socket buffer's data buffers are writable
4034  *	@skb: The socket buffer to check.
4035  *	@tailbits: Amount of trailing space to be added
4036  *	@trailer: Returned pointer to the skb where the @tailbits space begins
4037  *
4038  *	Make sure that the data buffers attached to a socket buffer are
4039  *	writable. If they are not, private copies are made of the data buffers
4040  *	and the socket buffer is set to use these instead.
4041  *
4042  *	If @tailbits is given, make sure that there is space to write @tailbits
4043  *	bytes of data beyond current end of socket buffer.  @trailer will be
4044  *	set to point to the skb in which this space begins.
4045  *
4046  *	The number of scatterlist elements required to completely map the
4047  *	COW'd and extended socket buffer will be returned.
4048  */
4049 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
4050 {
4051 	int copyflag;
4052 	int elt;
4053 	struct sk_buff *skb1, **skb_p;
4054 
4055 	/* If skb is cloned or its head is paged, reallocate
4056 	 * head pulling out all the pages (pages are considered not writable
4057 	 * at the moment even if they are anonymous).
4058 	 */
4059 	if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
4060 	    __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
4061 		return -ENOMEM;
4062 
4063 	/* Easy case. Most of packets will go this way. */
4064 	if (!skb_has_frag_list(skb)) {
4065 		/* A little of trouble, not enough of space for trailer.
4066 		 * This should not happen, when stack is tuned to generate
4067 		 * good frames. OK, on miss we reallocate and reserve even more
4068 		 * space, 128 bytes is fair. */
4069 
4070 		if (skb_tailroom(skb) < tailbits &&
4071 		    pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
4072 			return -ENOMEM;
4073 
4074 		/* Voila! */
4075 		*trailer = skb;
4076 		return 1;
4077 	}
4078 
4079 	/* Misery. We are in troubles, going to mincer fragments... */
4080 
4081 	elt = 1;
4082 	skb_p = &skb_shinfo(skb)->frag_list;
4083 	copyflag = 0;
4084 
4085 	while ((skb1 = *skb_p) != NULL) {
4086 		int ntail = 0;
4087 
4088 		/* The fragment is partially pulled by someone,
4089 		 * this can happen on input. Copy it and everything
4090 		 * after it. */
4091 
4092 		if (skb_shared(skb1))
4093 			copyflag = 1;
4094 
4095 		/* If the skb is the last, worry about trailer. */
4096 
4097 		if (skb1->next == NULL && tailbits) {
4098 			if (skb_shinfo(skb1)->nr_frags ||
4099 			    skb_has_frag_list(skb1) ||
4100 			    skb_tailroom(skb1) < tailbits)
4101 				ntail = tailbits + 128;
4102 		}
4103 
4104 		if (copyflag ||
4105 		    skb_cloned(skb1) ||
4106 		    ntail ||
4107 		    skb_shinfo(skb1)->nr_frags ||
4108 		    skb_has_frag_list(skb1)) {
4109 			struct sk_buff *skb2;
4110 
4111 			/* Fuck, we are miserable poor guys... */
4112 			if (ntail == 0)
4113 				skb2 = skb_copy(skb1, GFP_ATOMIC);
4114 			else
4115 				skb2 = skb_copy_expand(skb1,
4116 						       skb_headroom(skb1),
4117 						       ntail,
4118 						       GFP_ATOMIC);
4119 			if (unlikely(skb2 == NULL))
4120 				return -ENOMEM;
4121 
4122 			if (skb1->sk)
4123 				skb_set_owner_w(skb2, skb1->sk);
4124 
4125 			/* Looking around. Are we still alive?
4126 			 * OK, link new skb, drop old one */
4127 
4128 			skb2->next = skb1->next;
4129 			*skb_p = skb2;
4130 			kfree_skb(skb1);
4131 			skb1 = skb2;
4132 		}
4133 		elt++;
4134 		*trailer = skb1;
4135 		skb_p = &skb1->next;
4136 	}
4137 
4138 	return elt;
4139 }
4140 EXPORT_SYMBOL_GPL(skb_cow_data);
4141 
4142 static void sock_rmem_free(struct sk_buff *skb)
4143 {
4144 	struct sock *sk = skb->sk;
4145 
4146 	atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
4147 }
4148 
4149 static void skb_set_err_queue(struct sk_buff *skb)
4150 {
4151 	/* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
4152 	 * So, it is safe to (mis)use it to mark skbs on the error queue.
4153 	 */
4154 	skb->pkt_type = PACKET_OUTGOING;
4155 	BUILD_BUG_ON(PACKET_OUTGOING == 0);
4156 }
4157 
4158 /*
4159  * Note: We dont mem charge error packets (no sk_forward_alloc changes)
4160  */
4161 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
4162 {
4163 	if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4164 	    (unsigned int)sk->sk_rcvbuf)
4165 		return -ENOMEM;
4166 
4167 	skb_orphan(skb);
4168 	skb->sk = sk;
4169 	skb->destructor = sock_rmem_free;
4170 	atomic_add(skb->truesize, &sk->sk_rmem_alloc);
4171 	skb_set_err_queue(skb);
4172 
4173 	/* before exiting rcu section, make sure dst is refcounted */
4174 	skb_dst_force(skb);
4175 
4176 	skb_queue_tail(&sk->sk_error_queue, skb);
4177 	if (!sock_flag(sk, SOCK_DEAD))
4178 		sk->sk_data_ready(sk);
4179 	return 0;
4180 }
4181 EXPORT_SYMBOL(sock_queue_err_skb);
4182 
4183 static bool is_icmp_err_skb(const struct sk_buff *skb)
4184 {
4185 	return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
4186 		       SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
4187 }
4188 
4189 struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
4190 {
4191 	struct sk_buff_head *q = &sk->sk_error_queue;
4192 	struct sk_buff *skb, *skb_next = NULL;
4193 	bool icmp_next = false;
4194 	unsigned long flags;
4195 
4196 	spin_lock_irqsave(&q->lock, flags);
4197 	skb = __skb_dequeue(q);
4198 	if (skb && (skb_next = skb_peek(q))) {
4199 		icmp_next = is_icmp_err_skb(skb_next);
4200 		if (icmp_next)
4201 			sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_origin;
4202 	}
4203 	spin_unlock_irqrestore(&q->lock, flags);
4204 
4205 	if (is_icmp_err_skb(skb) && !icmp_next)
4206 		sk->sk_err = 0;
4207 
4208 	if (skb_next)
4209 		sk->sk_error_report(sk);
4210 
4211 	return skb;
4212 }
4213 EXPORT_SYMBOL(sock_dequeue_err_skb);
4214 
4215 /**
4216  * skb_clone_sk - create clone of skb, and take reference to socket
4217  * @skb: the skb to clone
4218  *
4219  * This function creates a clone of a buffer that holds a reference on
4220  * sk_refcnt.  Buffers created via this function are meant to be
4221  * returned using sock_queue_err_skb, or free via kfree_skb.
4222  *
4223  * When passing buffers allocated with this function to sock_queue_err_skb
4224  * it is necessary to wrap the call with sock_hold/sock_put in order to
4225  * prevent the socket from being released prior to being enqueued on
4226  * the sk_error_queue.
4227  */
4228 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
4229 {
4230 	struct sock *sk = skb->sk;
4231 	struct sk_buff *clone;
4232 
4233 	if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
4234 		return NULL;
4235 
4236 	clone = skb_clone(skb, GFP_ATOMIC);
4237 	if (!clone) {
4238 		sock_put(sk);
4239 		return NULL;
4240 	}
4241 
4242 	clone->sk = sk;
4243 	clone->destructor = sock_efree;
4244 
4245 	return clone;
4246 }
4247 EXPORT_SYMBOL(skb_clone_sk);
4248 
4249 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
4250 					struct sock *sk,
4251 					int tstype,
4252 					bool opt_stats)
4253 {
4254 	struct sock_exterr_skb *serr;
4255 	int err;
4256 
4257 	BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
4258 
4259 	serr = SKB_EXT_ERR(skb);
4260 	memset(serr, 0, sizeof(*serr));
4261 	serr->ee.ee_errno = ENOMSG;
4262 	serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
4263 	serr->ee.ee_info = tstype;
4264 	serr->opt_stats = opt_stats;
4265 	serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
4266 	if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
4267 		serr->ee.ee_data = skb_shinfo(skb)->tskey;
4268 		if (sk->sk_protocol == IPPROTO_TCP &&
4269 		    sk->sk_type == SOCK_STREAM)
4270 			serr->ee.ee_data -= sk->sk_tskey;
4271 	}
4272 
4273 	err = sock_queue_err_skb(sk, skb);
4274 
4275 	if (err)
4276 		kfree_skb(skb);
4277 }
4278 
4279 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
4280 {
4281 	bool ret;
4282 
4283 	if (likely(sysctl_tstamp_allow_data || tsonly))
4284 		return true;
4285 
4286 	read_lock_bh(&sk->sk_callback_lock);
4287 	ret = sk->sk_socket && sk->sk_socket->file &&
4288 	      file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
4289 	read_unlock_bh(&sk->sk_callback_lock);
4290 	return ret;
4291 }
4292 
4293 void skb_complete_tx_timestamp(struct sk_buff *skb,
4294 			       struct skb_shared_hwtstamps *hwtstamps)
4295 {
4296 	struct sock *sk = skb->sk;
4297 
4298 	if (!skb_may_tx_timestamp(sk, false))
4299 		return;
4300 
4301 	/* Take a reference to prevent skb_orphan() from freeing the socket,
4302 	 * but only if the socket refcount is not zero.
4303 	 */
4304 	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4305 		*skb_hwtstamps(skb) = *hwtstamps;
4306 		__skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
4307 		sock_put(sk);
4308 	}
4309 }
4310 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
4311 
4312 void __skb_tstamp_tx(struct sk_buff *orig_skb,
4313 		     struct skb_shared_hwtstamps *hwtstamps,
4314 		     struct sock *sk, int tstype)
4315 {
4316 	struct sk_buff *skb;
4317 	bool tsonly, opt_stats = false;
4318 
4319 	if (!sk)
4320 		return;
4321 
4322 	if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
4323 	    skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
4324 		return;
4325 
4326 	tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
4327 	if (!skb_may_tx_timestamp(sk, tsonly))
4328 		return;
4329 
4330 	if (tsonly) {
4331 #ifdef CONFIG_INET
4332 		if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
4333 		    sk->sk_protocol == IPPROTO_TCP &&
4334 		    sk->sk_type == SOCK_STREAM) {
4335 			skb = tcp_get_timestamping_opt_stats(sk);
4336 			opt_stats = true;
4337 		} else
4338 #endif
4339 			skb = alloc_skb(0, GFP_ATOMIC);
4340 	} else {
4341 		skb = skb_clone(orig_skb, GFP_ATOMIC);
4342 	}
4343 	if (!skb)
4344 		return;
4345 
4346 	if (tsonly) {
4347 		skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
4348 					     SKBTX_ANY_TSTAMP;
4349 		skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
4350 	}
4351 
4352 	if (hwtstamps)
4353 		*skb_hwtstamps(skb) = *hwtstamps;
4354 	else
4355 		skb->tstamp = ktime_get_real();
4356 
4357 	__skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
4358 }
4359 EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
4360 
4361 void skb_tstamp_tx(struct sk_buff *orig_skb,
4362 		   struct skb_shared_hwtstamps *hwtstamps)
4363 {
4364 	return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
4365 			       SCM_TSTAMP_SND);
4366 }
4367 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
4368 
4369 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
4370 {
4371 	struct sock *sk = skb->sk;
4372 	struct sock_exterr_skb *serr;
4373 	int err = 1;
4374 
4375 	skb->wifi_acked_valid = 1;
4376 	skb->wifi_acked = acked;
4377 
4378 	serr = SKB_EXT_ERR(skb);
4379 	memset(serr, 0, sizeof(*serr));
4380 	serr->ee.ee_errno = ENOMSG;
4381 	serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
4382 
4383 	/* Take a reference to prevent skb_orphan() from freeing the socket,
4384 	 * but only if the socket refcount is not zero.
4385 	 */
4386 	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4387 		err = sock_queue_err_skb(sk, skb);
4388 		sock_put(sk);
4389 	}
4390 	if (err)
4391 		kfree_skb(skb);
4392 }
4393 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
4394 
4395 /**
4396  * skb_partial_csum_set - set up and verify partial csum values for packet
4397  * @skb: the skb to set
4398  * @start: the number of bytes after skb->data to start checksumming.
4399  * @off: the offset from start to place the checksum.
4400  *
4401  * For untrusted partially-checksummed packets, we need to make sure the values
4402  * for skb->csum_start and skb->csum_offset are valid so we don't oops.
4403  *
4404  * This function checks and sets those values and skb->ip_summed: if this
4405  * returns false you should drop the packet.
4406  */
4407 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
4408 {
4409 	if (unlikely(start > skb_headlen(skb)) ||
4410 	    unlikely((int)start + off > skb_headlen(skb) - 2)) {
4411 		net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n",
4412 				     start, off, skb_headlen(skb));
4413 		return false;
4414 	}
4415 	skb->ip_summed = CHECKSUM_PARTIAL;
4416 	skb->csum_start = skb_headroom(skb) + start;
4417 	skb->csum_offset = off;
4418 	skb_set_transport_header(skb, start);
4419 	return true;
4420 }
4421 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
4422 
4423 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
4424 			       unsigned int max)
4425 {
4426 	if (skb_headlen(skb) >= len)
4427 		return 0;
4428 
4429 	/* If we need to pullup then pullup to the max, so we
4430 	 * won't need to do it again.
4431 	 */
4432 	if (max > skb->len)
4433 		max = skb->len;
4434 
4435 	if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
4436 		return -ENOMEM;
4437 
4438 	if (skb_headlen(skb) < len)
4439 		return -EPROTO;
4440 
4441 	return 0;
4442 }
4443 
4444 #define MAX_TCP_HDR_LEN (15 * 4)
4445 
4446 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
4447 				      typeof(IPPROTO_IP) proto,
4448 				      unsigned int off)
4449 {
4450 	switch (proto) {
4451 		int err;
4452 
4453 	case IPPROTO_TCP:
4454 		err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
4455 					  off + MAX_TCP_HDR_LEN);
4456 		if (!err && !skb_partial_csum_set(skb, off,
4457 						  offsetof(struct tcphdr,
4458 							   check)))
4459 			err = -EPROTO;
4460 		return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
4461 
4462 	case IPPROTO_UDP:
4463 		err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
4464 					  off + sizeof(struct udphdr));
4465 		if (!err && !skb_partial_csum_set(skb, off,
4466 						  offsetof(struct udphdr,
4467 							   check)))
4468 			err = -EPROTO;
4469 		return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
4470 	}
4471 
4472 	return ERR_PTR(-EPROTO);
4473 }
4474 
4475 /* This value should be large enough to cover a tagged ethernet header plus
4476  * maximally sized IP and TCP or UDP headers.
4477  */
4478 #define MAX_IP_HDR_LEN 128
4479 
4480 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
4481 {
4482 	unsigned int off;
4483 	bool fragment;
4484 	__sum16 *csum;
4485 	int err;
4486 
4487 	fragment = false;
4488 
4489 	err = skb_maybe_pull_tail(skb,
4490 				  sizeof(struct iphdr),
4491 				  MAX_IP_HDR_LEN);
4492 	if (err < 0)
4493 		goto out;
4494 
4495 	if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF))
4496 		fragment = true;
4497 
4498 	off = ip_hdrlen(skb);
4499 
4500 	err = -EPROTO;
4501 
4502 	if (fragment)
4503 		goto out;
4504 
4505 	csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
4506 	if (IS_ERR(csum))
4507 		return PTR_ERR(csum);
4508 
4509 	if (recalculate)
4510 		*csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
4511 					   ip_hdr(skb)->daddr,
4512 					   skb->len - off,
4513 					   ip_hdr(skb)->protocol, 0);
4514 	err = 0;
4515 
4516 out:
4517 	return err;
4518 }
4519 
4520 /* This value should be large enough to cover a tagged ethernet header plus
4521  * an IPv6 header, all options, and a maximal TCP or UDP header.
4522  */
4523 #define MAX_IPV6_HDR_LEN 256
4524 
4525 #define OPT_HDR(type, skb, off) \
4526 	(type *)(skb_network_header(skb) + (off))
4527 
4528 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
4529 {
4530 	int err;
4531 	u8 nexthdr;
4532 	unsigned int off;
4533 	unsigned int len;
4534 	bool fragment;
4535 	bool done;
4536 	__sum16 *csum;
4537 
4538 	fragment = false;
4539 	done = false;
4540 
4541 	off = sizeof(struct ipv6hdr);
4542 
4543 	err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
4544 	if (err < 0)
4545 		goto out;
4546 
4547 	nexthdr = ipv6_hdr(skb)->nexthdr;
4548 
4549 	len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
4550 	while (off <= len && !done) {
4551 		switch (nexthdr) {
4552 		case IPPROTO_DSTOPTS:
4553 		case IPPROTO_HOPOPTS:
4554 		case IPPROTO_ROUTING: {
4555 			struct ipv6_opt_hdr *hp;
4556 
4557 			err = skb_maybe_pull_tail(skb,
4558 						  off +
4559 						  sizeof(struct ipv6_opt_hdr),
4560 						  MAX_IPV6_HDR_LEN);
4561 			if (err < 0)
4562 				goto out;
4563 
4564 			hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
4565 			nexthdr = hp->nexthdr;
4566 			off += ipv6_optlen(hp);
4567 			break;
4568 		}
4569 		case IPPROTO_AH: {
4570 			struct ip_auth_hdr *hp;
4571 
4572 			err = skb_maybe_pull_tail(skb,
4573 						  off +
4574 						  sizeof(struct ip_auth_hdr),
4575 						  MAX_IPV6_HDR_LEN);
4576 			if (err < 0)
4577 				goto out;
4578 
4579 			hp = OPT_HDR(struct ip_auth_hdr, skb, off);
4580 			nexthdr = hp->nexthdr;
4581 			off += ipv6_authlen(hp);
4582 			break;
4583 		}
4584 		case IPPROTO_FRAGMENT: {
4585 			struct frag_hdr *hp;
4586 
4587 			err = skb_maybe_pull_tail(skb,
4588 						  off +
4589 						  sizeof(struct frag_hdr),
4590 						  MAX_IPV6_HDR_LEN);
4591 			if (err < 0)
4592 				goto out;
4593 
4594 			hp = OPT_HDR(struct frag_hdr, skb, off);
4595 
4596 			if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
4597 				fragment = true;
4598 
4599 			nexthdr = hp->nexthdr;
4600 			off += sizeof(struct frag_hdr);
4601 			break;
4602 		}
4603 		default:
4604 			done = true;
4605 			break;
4606 		}
4607 	}
4608 
4609 	err = -EPROTO;
4610 
4611 	if (!done || fragment)
4612 		goto out;
4613 
4614 	csum = skb_checksum_setup_ip(skb, nexthdr, off);
4615 	if (IS_ERR(csum))
4616 		return PTR_ERR(csum);
4617 
4618 	if (recalculate)
4619 		*csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4620 					 &ipv6_hdr(skb)->daddr,
4621 					 skb->len - off, nexthdr, 0);
4622 	err = 0;
4623 
4624 out:
4625 	return err;
4626 }
4627 
4628 /**
4629  * skb_checksum_setup - set up partial checksum offset
4630  * @skb: the skb to set up
4631  * @recalculate: if true the pseudo-header checksum will be recalculated
4632  */
4633 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
4634 {
4635 	int err;
4636 
4637 	switch (skb->protocol) {
4638 	case htons(ETH_P_IP):
4639 		err = skb_checksum_setup_ipv4(skb, recalculate);
4640 		break;
4641 
4642 	case htons(ETH_P_IPV6):
4643 		err = skb_checksum_setup_ipv6(skb, recalculate);
4644 		break;
4645 
4646 	default:
4647 		err = -EPROTO;
4648 		break;
4649 	}
4650 
4651 	return err;
4652 }
4653 EXPORT_SYMBOL(skb_checksum_setup);
4654 
4655 /**
4656  * skb_checksum_maybe_trim - maybe trims the given skb
4657  * @skb: the skb to check
4658  * @transport_len: the data length beyond the network header
4659  *
4660  * Checks whether the given skb has data beyond the given transport length.
4661  * If so, returns a cloned skb trimmed to this transport length.
4662  * Otherwise returns the provided skb. Returns NULL in error cases
4663  * (e.g. transport_len exceeds skb length or out-of-memory).
4664  *
4665  * Caller needs to set the skb transport header and free any returned skb if it
4666  * differs from the provided skb.
4667  */
4668 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
4669 					       unsigned int transport_len)
4670 {
4671 	struct sk_buff *skb_chk;
4672 	unsigned int len = skb_transport_offset(skb) + transport_len;
4673 	int ret;
4674 
4675 	if (skb->len < len)
4676 		return NULL;
4677 	else if (skb->len == len)
4678 		return skb;
4679 
4680 	skb_chk = skb_clone(skb, GFP_ATOMIC);
4681 	if (!skb_chk)
4682 		return NULL;
4683 
4684 	ret = pskb_trim_rcsum(skb_chk, len);
4685 	if (ret) {
4686 		kfree_skb(skb_chk);
4687 		return NULL;
4688 	}
4689 
4690 	return skb_chk;
4691 }
4692 
4693 /**
4694  * skb_checksum_trimmed - validate checksum of an skb
4695  * @skb: the skb to check
4696  * @transport_len: the data length beyond the network header
4697  * @skb_chkf: checksum function to use
4698  *
4699  * Applies the given checksum function skb_chkf to the provided skb.
4700  * Returns a checked and maybe trimmed skb. Returns NULL on error.
4701  *
4702  * If the skb has data beyond the given transport length, then a
4703  * trimmed & cloned skb is checked and returned.
4704  *
4705  * Caller needs to set the skb transport header and free any returned skb if it
4706  * differs from the provided skb.
4707  */
4708 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
4709 				     unsigned int transport_len,
4710 				     __sum16(*skb_chkf)(struct sk_buff *skb))
4711 {
4712 	struct sk_buff *skb_chk;
4713 	unsigned int offset = skb_transport_offset(skb);
4714 	__sum16 ret;
4715 
4716 	skb_chk = skb_checksum_maybe_trim(skb, transport_len);
4717 	if (!skb_chk)
4718 		goto err;
4719 
4720 	if (!pskb_may_pull(skb_chk, offset))
4721 		goto err;
4722 
4723 	skb_pull_rcsum(skb_chk, offset);
4724 	ret = skb_chkf(skb_chk);
4725 	skb_push_rcsum(skb_chk, offset);
4726 
4727 	if (ret)
4728 		goto err;
4729 
4730 	return skb_chk;
4731 
4732 err:
4733 	if (skb_chk && skb_chk != skb)
4734 		kfree_skb(skb_chk);
4735 
4736 	return NULL;
4737 
4738 }
4739 EXPORT_SYMBOL(skb_checksum_trimmed);
4740 
4741 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
4742 {
4743 	net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
4744 			     skb->dev->name);
4745 }
4746 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
4747 
4748 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
4749 {
4750 	if (head_stolen) {
4751 		skb_release_head_state(skb);
4752 		kmem_cache_free(skbuff_head_cache, skb);
4753 	} else {
4754 		__kfree_skb(skb);
4755 	}
4756 }
4757 EXPORT_SYMBOL(kfree_skb_partial);
4758 
4759 /**
4760  * skb_try_coalesce - try to merge skb to prior one
4761  * @to: prior buffer
4762  * @from: buffer to add
4763  * @fragstolen: pointer to boolean
4764  * @delta_truesize: how much more was allocated than was requested
4765  */
4766 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
4767 		      bool *fragstolen, int *delta_truesize)
4768 {
4769 	int i, delta, len = from->len;
4770 
4771 	*fragstolen = false;
4772 
4773 	if (skb_cloned(to))
4774 		return false;
4775 
4776 	if (len <= skb_tailroom(to)) {
4777 		if (len)
4778 			BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
4779 		*delta_truesize = 0;
4780 		return true;
4781 	}
4782 
4783 	if (skb_has_frag_list(to) || skb_has_frag_list(from))
4784 		return false;
4785 	if (skb_zcopy(to) || skb_zcopy(from))
4786 		return false;
4787 
4788 	if (skb_headlen(from) != 0) {
4789 		struct page *page;
4790 		unsigned int offset;
4791 
4792 		if (skb_shinfo(to)->nr_frags +
4793 		    skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
4794 			return false;
4795 
4796 		if (skb_head_is_locked(from))
4797 			return false;
4798 
4799 		delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
4800 
4801 		page = virt_to_head_page(from->head);
4802 		offset = from->data - (unsigned char *)page_address(page);
4803 
4804 		skb_fill_page_desc(to, skb_shinfo(to)->nr_frags,
4805 				   page, offset, skb_headlen(from));
4806 		*fragstolen = true;
4807 	} else {
4808 		if (skb_shinfo(to)->nr_frags +
4809 		    skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS)
4810 			return false;
4811 
4812 		delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
4813 	}
4814 
4815 	WARN_ON_ONCE(delta < len);
4816 
4817 	memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags,
4818 	       skb_shinfo(from)->frags,
4819 	       skb_shinfo(from)->nr_frags * sizeof(skb_frag_t));
4820 	skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags;
4821 
4822 	if (!skb_cloned(from))
4823 		skb_shinfo(from)->nr_frags = 0;
4824 
4825 	/* if the skb is not cloned this does nothing
4826 	 * since we set nr_frags to 0.
4827 	 */
4828 	for (i = 0; i < skb_shinfo(from)->nr_frags; i++)
4829 		skb_frag_ref(from, i);
4830 
4831 	to->truesize += delta;
4832 	to->len += len;
4833 	to->data_len += len;
4834 
4835 	*delta_truesize = delta;
4836 	return true;
4837 }
4838 EXPORT_SYMBOL(skb_try_coalesce);
4839 
4840 /**
4841  * skb_scrub_packet - scrub an skb
4842  *
4843  * @skb: buffer to clean
4844  * @xnet: packet is crossing netns
4845  *
4846  * skb_scrub_packet can be used after encapsulating or decapsulting a packet
4847  * into/from a tunnel. Some information have to be cleared during these
4848  * operations.
4849  * skb_scrub_packet can also be used to clean a skb before injecting it in
4850  * another namespace (@xnet == true). We have to clear all information in the
4851  * skb that could impact namespace isolation.
4852  */
4853 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
4854 {
4855 	skb->tstamp = 0;
4856 	skb->pkt_type = PACKET_HOST;
4857 	skb->skb_iif = 0;
4858 	skb->ignore_df = 0;
4859 	skb_dst_drop(skb);
4860 	secpath_reset(skb);
4861 	nf_reset(skb);
4862 	nf_reset_trace(skb);
4863 
4864 	if (!xnet)
4865 		return;
4866 
4867 	skb_orphan(skb);
4868 	skb->mark = 0;
4869 }
4870 EXPORT_SYMBOL_GPL(skb_scrub_packet);
4871 
4872 /**
4873  * skb_gso_transport_seglen - Return length of individual segments of a gso packet
4874  *
4875  * @skb: GSO skb
4876  *
4877  * skb_gso_transport_seglen is used to determine the real size of the
4878  * individual segments, including Layer4 headers (TCP/UDP).
4879  *
4880  * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
4881  */
4882 unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
4883 {
4884 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
4885 	unsigned int thlen = 0;
4886 
4887 	if (skb->encapsulation) {
4888 		thlen = skb_inner_transport_header(skb) -
4889 			skb_transport_header(skb);
4890 
4891 		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
4892 			thlen += inner_tcp_hdrlen(skb);
4893 	} else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
4894 		thlen = tcp_hdrlen(skb);
4895 	} else if (unlikely(shinfo->gso_type & SKB_GSO_SCTP)) {
4896 		thlen = sizeof(struct sctphdr);
4897 	}
4898 	/* UFO sets gso_size to the size of the fragmentation
4899 	 * payload, i.e. the size of the L4 (UDP) header is already
4900 	 * accounted for.
4901 	 */
4902 	return thlen + shinfo->gso_size;
4903 }
4904 EXPORT_SYMBOL_GPL(skb_gso_transport_seglen);
4905 
4906 /**
4907  * skb_gso_validate_mtu - Return in case such skb fits a given MTU
4908  *
4909  * @skb: GSO skb
4910  * @mtu: MTU to validate against
4911  *
4912  * skb_gso_validate_mtu validates if a given skb will fit a wanted MTU
4913  * once split.
4914  */
4915 bool skb_gso_validate_mtu(const struct sk_buff *skb, unsigned int mtu)
4916 {
4917 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
4918 	const struct sk_buff *iter;
4919 	unsigned int hlen;
4920 
4921 	hlen = skb_gso_network_seglen(skb);
4922 
4923 	if (shinfo->gso_size != GSO_BY_FRAGS)
4924 		return hlen <= mtu;
4925 
4926 	/* Undo this so we can re-use header sizes */
4927 	hlen -= GSO_BY_FRAGS;
4928 
4929 	skb_walk_frags(skb, iter) {
4930 		if (hlen + skb_headlen(iter) > mtu)
4931 			return false;
4932 	}
4933 
4934 	return true;
4935 }
4936 EXPORT_SYMBOL_GPL(skb_gso_validate_mtu);
4937 
4938 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
4939 {
4940 	if (skb_cow(skb, skb_headroom(skb)) < 0) {
4941 		kfree_skb(skb);
4942 		return NULL;
4943 	}
4944 
4945 	memmove(skb->data - ETH_HLEN, skb->data - skb->mac_len - VLAN_HLEN,
4946 		2 * ETH_ALEN);
4947 	skb->mac_header += VLAN_HLEN;
4948 	return skb;
4949 }
4950 
4951 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
4952 {
4953 	struct vlan_hdr *vhdr;
4954 	u16 vlan_tci;
4955 
4956 	if (unlikely(skb_vlan_tag_present(skb))) {
4957 		/* vlan_tci is already set-up so leave this for another time */
4958 		return skb;
4959 	}
4960 
4961 	skb = skb_share_check(skb, GFP_ATOMIC);
4962 	if (unlikely(!skb))
4963 		goto err_free;
4964 
4965 	if (unlikely(!pskb_may_pull(skb, VLAN_HLEN)))
4966 		goto err_free;
4967 
4968 	vhdr = (struct vlan_hdr *)skb->data;
4969 	vlan_tci = ntohs(vhdr->h_vlan_TCI);
4970 	__vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
4971 
4972 	skb_pull_rcsum(skb, VLAN_HLEN);
4973 	vlan_set_encap_proto(skb, vhdr);
4974 
4975 	skb = skb_reorder_vlan_header(skb);
4976 	if (unlikely(!skb))
4977 		goto err_free;
4978 
4979 	skb_reset_network_header(skb);
4980 	skb_reset_transport_header(skb);
4981 	skb_reset_mac_len(skb);
4982 
4983 	return skb;
4984 
4985 err_free:
4986 	kfree_skb(skb);
4987 	return NULL;
4988 }
4989 EXPORT_SYMBOL(skb_vlan_untag);
4990 
4991 int skb_ensure_writable(struct sk_buff *skb, int write_len)
4992 {
4993 	if (!pskb_may_pull(skb, write_len))
4994 		return -ENOMEM;
4995 
4996 	if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
4997 		return 0;
4998 
4999 	return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5000 }
5001 EXPORT_SYMBOL(skb_ensure_writable);
5002 
5003 /* remove VLAN header from packet and update csum accordingly.
5004  * expects a non skb_vlan_tag_present skb with a vlan tag payload
5005  */
5006 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
5007 {
5008 	struct vlan_hdr *vhdr;
5009 	int offset = skb->data - skb_mac_header(skb);
5010 	int err;
5011 
5012 	if (WARN_ONCE(offset,
5013 		      "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
5014 		      offset)) {
5015 		return -EINVAL;
5016 	}
5017 
5018 	err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
5019 	if (unlikely(err))
5020 		return err;
5021 
5022 	skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5023 
5024 	vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
5025 	*vlan_tci = ntohs(vhdr->h_vlan_TCI);
5026 
5027 	memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
5028 	__skb_pull(skb, VLAN_HLEN);
5029 
5030 	vlan_set_encap_proto(skb, vhdr);
5031 	skb->mac_header += VLAN_HLEN;
5032 
5033 	if (skb_network_offset(skb) < ETH_HLEN)
5034 		skb_set_network_header(skb, ETH_HLEN);
5035 
5036 	skb_reset_mac_len(skb);
5037 
5038 	return err;
5039 }
5040 EXPORT_SYMBOL(__skb_vlan_pop);
5041 
5042 /* Pop a vlan tag either from hwaccel or from payload.
5043  * Expects skb->data at mac header.
5044  */
5045 int skb_vlan_pop(struct sk_buff *skb)
5046 {
5047 	u16 vlan_tci;
5048 	__be16 vlan_proto;
5049 	int err;
5050 
5051 	if (likely(skb_vlan_tag_present(skb))) {
5052 		skb->vlan_tci = 0;
5053 	} else {
5054 		if (unlikely(!eth_type_vlan(skb->protocol)))
5055 			return 0;
5056 
5057 		err = __skb_vlan_pop(skb, &vlan_tci);
5058 		if (err)
5059 			return err;
5060 	}
5061 	/* move next vlan tag to hw accel tag */
5062 	if (likely(!eth_type_vlan(skb->protocol)))
5063 		return 0;
5064 
5065 	vlan_proto = skb->protocol;
5066 	err = __skb_vlan_pop(skb, &vlan_tci);
5067 	if (unlikely(err))
5068 		return err;
5069 
5070 	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5071 	return 0;
5072 }
5073 EXPORT_SYMBOL(skb_vlan_pop);
5074 
5075 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
5076  * Expects skb->data at mac header.
5077  */
5078 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
5079 {
5080 	if (skb_vlan_tag_present(skb)) {
5081 		int offset = skb->data - skb_mac_header(skb);
5082 		int err;
5083 
5084 		if (WARN_ONCE(offset,
5085 			      "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
5086 			      offset)) {
5087 			return -EINVAL;
5088 		}
5089 
5090 		err = __vlan_insert_tag(skb, skb->vlan_proto,
5091 					skb_vlan_tag_get(skb));
5092 		if (err)
5093 			return err;
5094 
5095 		skb->protocol = skb->vlan_proto;
5096 		skb->mac_len += VLAN_HLEN;
5097 
5098 		skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5099 	}
5100 	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5101 	return 0;
5102 }
5103 EXPORT_SYMBOL(skb_vlan_push);
5104 
5105 /**
5106  * alloc_skb_with_frags - allocate skb with page frags
5107  *
5108  * @header_len: size of linear part
5109  * @data_len: needed length in frags
5110  * @max_page_order: max page order desired.
5111  * @errcode: pointer to error code if any
5112  * @gfp_mask: allocation mask
5113  *
5114  * This can be used to allocate a paged skb, given a maximal order for frags.
5115  */
5116 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
5117 				     unsigned long data_len,
5118 				     int max_page_order,
5119 				     int *errcode,
5120 				     gfp_t gfp_mask)
5121 {
5122 	int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
5123 	unsigned long chunk;
5124 	struct sk_buff *skb;
5125 	struct page *page;
5126 	gfp_t gfp_head;
5127 	int i;
5128 
5129 	*errcode = -EMSGSIZE;
5130 	/* Note this test could be relaxed, if we succeed to allocate
5131 	 * high order pages...
5132 	 */
5133 	if (npages > MAX_SKB_FRAGS)
5134 		return NULL;
5135 
5136 	gfp_head = gfp_mask;
5137 	if (gfp_head & __GFP_DIRECT_RECLAIM)
5138 		gfp_head |= __GFP_RETRY_MAYFAIL;
5139 
5140 	*errcode = -ENOBUFS;
5141 	skb = alloc_skb(header_len, gfp_head);
5142 	if (!skb)
5143 		return NULL;
5144 
5145 	skb->truesize += npages << PAGE_SHIFT;
5146 
5147 	for (i = 0; npages > 0; i++) {
5148 		int order = max_page_order;
5149 
5150 		while (order) {
5151 			if (npages >= 1 << order) {
5152 				page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
5153 						   __GFP_COMP |
5154 						   __GFP_NOWARN |
5155 						   __GFP_NORETRY,
5156 						   order);
5157 				if (page)
5158 					goto fill_page;
5159 				/* Do not retry other high order allocations */
5160 				order = 1;
5161 				max_page_order = 0;
5162 			}
5163 			order--;
5164 		}
5165 		page = alloc_page(gfp_mask);
5166 		if (!page)
5167 			goto failure;
5168 fill_page:
5169 		chunk = min_t(unsigned long, data_len,
5170 			      PAGE_SIZE << order);
5171 		skb_fill_page_desc(skb, i, page, 0, chunk);
5172 		data_len -= chunk;
5173 		npages -= 1 << order;
5174 	}
5175 	return skb;
5176 
5177 failure:
5178 	kfree_skb(skb);
5179 	return NULL;
5180 }
5181 EXPORT_SYMBOL(alloc_skb_with_frags);
5182 
5183 /* carve out the first off bytes from skb when off < headlen */
5184 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
5185 				    const int headlen, gfp_t gfp_mask)
5186 {
5187 	int i;
5188 	int size = skb_end_offset(skb);
5189 	int new_hlen = headlen - off;
5190 	u8 *data;
5191 
5192 	size = SKB_DATA_ALIGN(size);
5193 
5194 	if (skb_pfmemalloc(skb))
5195 		gfp_mask |= __GFP_MEMALLOC;
5196 	data = kmalloc_reserve(size +
5197 			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5198 			       gfp_mask, NUMA_NO_NODE, NULL);
5199 	if (!data)
5200 		return -ENOMEM;
5201 
5202 	size = SKB_WITH_OVERHEAD(ksize(data));
5203 
5204 	/* Copy real data, and all frags */
5205 	skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
5206 	skb->len -= off;
5207 
5208 	memcpy((struct skb_shared_info *)(data + size),
5209 	       skb_shinfo(skb),
5210 	       offsetof(struct skb_shared_info,
5211 			frags[skb_shinfo(skb)->nr_frags]));
5212 	if (skb_cloned(skb)) {
5213 		/* drop the old head gracefully */
5214 		if (skb_orphan_frags(skb, gfp_mask)) {
5215 			kfree(data);
5216 			return -ENOMEM;
5217 		}
5218 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
5219 			skb_frag_ref(skb, i);
5220 		if (skb_has_frag_list(skb))
5221 			skb_clone_fraglist(skb);
5222 		skb_release_data(skb);
5223 	} else {
5224 		/* we can reuse existing recount- all we did was
5225 		 * relocate values
5226 		 */
5227 		skb_free_head(skb);
5228 	}
5229 
5230 	skb->head = data;
5231 	skb->data = data;
5232 	skb->head_frag = 0;
5233 #ifdef NET_SKBUFF_DATA_USES_OFFSET
5234 	skb->end = size;
5235 #else
5236 	skb->end = skb->head + size;
5237 #endif
5238 	skb_set_tail_pointer(skb, skb_headlen(skb));
5239 	skb_headers_offset_update(skb, 0);
5240 	skb->cloned = 0;
5241 	skb->hdr_len = 0;
5242 	skb->nohdr = 0;
5243 	atomic_set(&skb_shinfo(skb)->dataref, 1);
5244 
5245 	return 0;
5246 }
5247 
5248 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
5249 
5250 /* carve out the first eat bytes from skb's frag_list. May recurse into
5251  * pskb_carve()
5252  */
5253 static int pskb_carve_frag_list(struct sk_buff *skb,
5254 				struct skb_shared_info *shinfo, int eat,
5255 				gfp_t gfp_mask)
5256 {
5257 	struct sk_buff *list = shinfo->frag_list;
5258 	struct sk_buff *clone = NULL;
5259 	struct sk_buff *insp = NULL;
5260 
5261 	do {
5262 		if (!list) {
5263 			pr_err("Not enough bytes to eat. Want %d\n", eat);
5264 			return -EFAULT;
5265 		}
5266 		if (list->len <= eat) {
5267 			/* Eaten as whole. */
5268 			eat -= list->len;
5269 			list = list->next;
5270 			insp = list;
5271 		} else {
5272 			/* Eaten partially. */
5273 			if (skb_shared(list)) {
5274 				clone = skb_clone(list, gfp_mask);
5275 				if (!clone)
5276 					return -ENOMEM;
5277 				insp = list->next;
5278 				list = clone;
5279 			} else {
5280 				/* This may be pulled without problems. */
5281 				insp = list;
5282 			}
5283 			if (pskb_carve(list, eat, gfp_mask) < 0) {
5284 				kfree_skb(clone);
5285 				return -ENOMEM;
5286 			}
5287 			break;
5288 		}
5289 	} while (eat);
5290 
5291 	/* Free pulled out fragments. */
5292 	while ((list = shinfo->frag_list) != insp) {
5293 		shinfo->frag_list = list->next;
5294 		kfree_skb(list);
5295 	}
5296 	/* And insert new clone at head. */
5297 	if (clone) {
5298 		clone->next = list;
5299 		shinfo->frag_list = clone;
5300 	}
5301 	return 0;
5302 }
5303 
5304 /* carve off first len bytes from skb. Split line (off) is in the
5305  * non-linear part of skb
5306  */
5307 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
5308 				       int pos, gfp_t gfp_mask)
5309 {
5310 	int i, k = 0;
5311 	int size = skb_end_offset(skb);
5312 	u8 *data;
5313 	const int nfrags = skb_shinfo(skb)->nr_frags;
5314 	struct skb_shared_info *shinfo;
5315 
5316 	size = SKB_DATA_ALIGN(size);
5317 
5318 	if (skb_pfmemalloc(skb))
5319 		gfp_mask |= __GFP_MEMALLOC;
5320 	data = kmalloc_reserve(size +
5321 			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5322 			       gfp_mask, NUMA_NO_NODE, NULL);
5323 	if (!data)
5324 		return -ENOMEM;
5325 
5326 	size = SKB_WITH_OVERHEAD(ksize(data));
5327 
5328 	memcpy((struct skb_shared_info *)(data + size),
5329 	       skb_shinfo(skb), offsetof(struct skb_shared_info,
5330 					 frags[skb_shinfo(skb)->nr_frags]));
5331 	if (skb_orphan_frags(skb, gfp_mask)) {
5332 		kfree(data);
5333 		return -ENOMEM;
5334 	}
5335 	shinfo = (struct skb_shared_info *)(data + size);
5336 	for (i = 0; i < nfrags; i++) {
5337 		int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
5338 
5339 		if (pos + fsize > off) {
5340 			shinfo->frags[k] = skb_shinfo(skb)->frags[i];
5341 
5342 			if (pos < off) {
5343 				/* Split frag.
5344 				 * We have two variants in this case:
5345 				 * 1. Move all the frag to the second
5346 				 *    part, if it is possible. F.e.
5347 				 *    this approach is mandatory for TUX,
5348 				 *    where splitting is expensive.
5349 				 * 2. Split is accurately. We make this.
5350 				 */
5351 				shinfo->frags[0].page_offset += off - pos;
5352 				skb_frag_size_sub(&shinfo->frags[0], off - pos);
5353 			}
5354 			skb_frag_ref(skb, i);
5355 			k++;
5356 		}
5357 		pos += fsize;
5358 	}
5359 	shinfo->nr_frags = k;
5360 	if (skb_has_frag_list(skb))
5361 		skb_clone_fraglist(skb);
5362 
5363 	if (k == 0) {
5364 		/* split line is in frag list */
5365 		pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask);
5366 	}
5367 	skb_release_data(skb);
5368 
5369 	skb->head = data;
5370 	skb->head_frag = 0;
5371 	skb->data = data;
5372 #ifdef NET_SKBUFF_DATA_USES_OFFSET
5373 	skb->end = size;
5374 #else
5375 	skb->end = skb->head + size;
5376 #endif
5377 	skb_reset_tail_pointer(skb);
5378 	skb_headers_offset_update(skb, 0);
5379 	skb->cloned   = 0;
5380 	skb->hdr_len  = 0;
5381 	skb->nohdr    = 0;
5382 	skb->len -= off;
5383 	skb->data_len = skb->len;
5384 	atomic_set(&skb_shinfo(skb)->dataref, 1);
5385 	return 0;
5386 }
5387 
5388 /* remove len bytes from the beginning of the skb */
5389 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
5390 {
5391 	int headlen = skb_headlen(skb);
5392 
5393 	if (len < headlen)
5394 		return pskb_carve_inside_header(skb, len, headlen, gfp);
5395 	else
5396 		return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
5397 }
5398 
5399 /* Extract to_copy bytes starting at off from skb, and return this in
5400  * a new skb
5401  */
5402 struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
5403 			     int to_copy, gfp_t gfp)
5404 {
5405 	struct sk_buff  *clone = skb_clone(skb, gfp);
5406 
5407 	if (!clone)
5408 		return NULL;
5409 
5410 	if (pskb_carve(clone, off, gfp) < 0 ||
5411 	    pskb_trim(clone, to_copy)) {
5412 		kfree_skb(clone);
5413 		return NULL;
5414 	}
5415 	return clone;
5416 }
5417 EXPORT_SYMBOL(pskb_extract);
5418 
5419 /**
5420  * skb_condense - try to get rid of fragments/frag_list if possible
5421  * @skb: buffer
5422  *
5423  * Can be used to save memory before skb is added to a busy queue.
5424  * If packet has bytes in frags and enough tail room in skb->head,
5425  * pull all of them, so that we can free the frags right now and adjust
5426  * truesize.
5427  * Notes:
5428  *	We do not reallocate skb->head thus can not fail.
5429  *	Caller must re-evaluate skb->truesize if needed.
5430  */
5431 void skb_condense(struct sk_buff *skb)
5432 {
5433 	if (skb->data_len) {
5434 		if (skb->data_len > skb->end - skb->tail ||
5435 		    skb_cloned(skb))
5436 			return;
5437 
5438 		/* Nice, we can free page frag(s) right now */
5439 		__pskb_pull_tail(skb, skb->data_len);
5440 	}
5441 	/* At this point, skb->truesize might be over estimated,
5442 	 * because skb had a fragment, and fragments do not tell
5443 	 * their truesize.
5444 	 * When we pulled its content into skb->head, fragment
5445 	 * was freed, but __pskb_pull_tail() could not possibly
5446 	 * adjust skb->truesize, not knowing the frag truesize.
5447 	 */
5448 	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5449 }
5450