skbuff.c (0422fe2666aea4c0986f4c89dc107731aa6a7a81) skbuff.c (c504e5c2f9648a1e5c2be01e8c3f59d394192bd3)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Routines having to do with the 'struct sk_buff' memory handlers.
4 *
5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
6 * Florian La Roche <rzsfl@rz.uni-sb.de>
7 *
8 * Fixes:

--- 745 unchanged lines hidden (view full) ---

754void __kfree_skb(struct sk_buff *skb)
755{
756 skb_release_all(skb);
757 kfree_skbmem(skb);
758}
759EXPORT_SYMBOL(__kfree_skb);
760
761/**
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Routines having to do with the 'struct sk_buff' memory handlers.
4 *
5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
6 * Florian La Roche <rzsfl@rz.uni-sb.de>
7 *
8 * Fixes:

--- 745 unchanged lines hidden (view full) ---

754void __kfree_skb(struct sk_buff *skb)
755{
756 skb_release_all(skb);
757 kfree_skbmem(skb);
758}
759EXPORT_SYMBOL(__kfree_skb);
760
761/**
762 * kfree_skb - free an sk_buff
762 * kfree_skb_reason - free an sk_buff with special reason
763 * @skb: buffer to free
763 * @skb: buffer to free
764 * @reason: reason why this skb is dropped
764 *
765 * Drop a reference to the buffer and free it if the usage count has
765 *
766 * Drop a reference to the buffer and free it if the usage count has
766 * hit zero.
767 * hit zero. Meanwhile, pass the drop reason to 'kfree_skb'
768 * tracepoint.
767 */
769 */
768void kfree_skb(struct sk_buff *skb)
770void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
769{
770 if (!skb_unref(skb))
771 return;
772
771{
772 if (!skb_unref(skb))
773 return;
774
773 trace_kfree_skb(skb, __builtin_return_address(0));
775 trace_kfree_skb(skb, __builtin_return_address(0), reason);
774 __kfree_skb(skb);
775}
776 __kfree_skb(skb);
777}
776EXPORT_SYMBOL(kfree_skb);
778EXPORT_SYMBOL(kfree_skb_reason);
777
778void kfree_skb_list(struct sk_buff *segs)
779{
780 while (segs) {
781 struct sk_buff *next = segs->next;
782
783 kfree_skb(segs);
784 segs = next;

--- 202 unchanged lines hidden (view full) ---

987 return;
988 }
989
990 skb_release_all(skb);
991 napi_skb_cache_put(skb);
992}
993EXPORT_SYMBOL(napi_consume_skb);
994
779
780void kfree_skb_list(struct sk_buff *segs)
781{
782 while (segs) {
783 struct sk_buff *next = segs->next;
784
785 kfree_skb(segs);
786 segs = next;

--- 202 unchanged lines hidden (view full) ---

989 return;
990 }
991
992 skb_release_all(skb);
993 napi_skb_cache_put(skb);
994}
995EXPORT_SYMBOL(napi_consume_skb);
996
995/* Make sure a field is enclosed inside headers_start/headers_end section */
997/* Make sure a field is contained by headers group */
996#define CHECK_SKB_FIELD(field) \
998#define CHECK_SKB_FIELD(field) \
997 BUILD_BUG_ON(offsetof(struct sk_buff, field) < \
998 offsetof(struct sk_buff, headers_start)); \
999 BUILD_BUG_ON(offsetof(struct sk_buff, field) > \
1000 offsetof(struct sk_buff, headers_end)); \
999 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \
1000 offsetof(struct sk_buff, headers.field)); \
1001
1002static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1003{
1004 new->tstamp = old->tstamp;
1005 /* We do not copy old->sk */
1006 new->dev = old->dev;
1007 memcpy(new->cb, old->cb, sizeof(old->cb));
1008 skb_dst_copy(new, old);
1009 __skb_ext_copy(new, old);
1010 __nf_copy(new, old, false);
1011
1001
1002static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1003{
1004 new->tstamp = old->tstamp;
1005 /* We do not copy old->sk */
1006 new->dev = old->dev;
1007 memcpy(new->cb, old->cb, sizeof(old->cb));
1008 skb_dst_copy(new, old);
1009 __skb_ext_copy(new, old);
1010 __nf_copy(new, old, false);
1011
1012 /* Note : this field could be in headers_start/headers_end section
1012 /* Note : this field could be in the headers group.
1013 * It is not yet because we do not want to have a 16 bit hole
1014 */
1015 new->queue_mapping = old->queue_mapping;
1016
1013 * It is not yet because we do not want to have a 16 bit hole
1014 */
1015 new->queue_mapping = old->queue_mapping;
1016
1017 memcpy(&new->headers_start, &old->headers_start,
1018 offsetof(struct sk_buff, headers_end) -
1019 offsetof(struct sk_buff, headers_start));
1017 memcpy(&new->headers, &old->headers, sizeof(new->headers));
1020 CHECK_SKB_FIELD(protocol);
1021 CHECK_SKB_FIELD(csum);
1022 CHECK_SKB_FIELD(hash);
1023 CHECK_SKB_FIELD(priority);
1024 CHECK_SKB_FIELD(skb_iif);
1025 CHECK_SKB_FIELD(vlan_proto);
1026 CHECK_SKB_FIELD(vlan_tci);
1027 CHECK_SKB_FIELD(transport_header);

--- 995 unchanged lines hidden (view full) ---

2023 */
2024void *skb_pull(struct sk_buff *skb, unsigned int len)
2025{
2026 return skb_pull_inline(skb, len);
2027}
2028EXPORT_SYMBOL(skb_pull);
2029
2030/**
1018 CHECK_SKB_FIELD(protocol);
1019 CHECK_SKB_FIELD(csum);
1020 CHECK_SKB_FIELD(hash);
1021 CHECK_SKB_FIELD(priority);
1022 CHECK_SKB_FIELD(skb_iif);
1023 CHECK_SKB_FIELD(vlan_proto);
1024 CHECK_SKB_FIELD(vlan_tci);
1025 CHECK_SKB_FIELD(transport_header);

--- 995 unchanged lines hidden (view full) ---

2021 */
2022void *skb_pull(struct sk_buff *skb, unsigned int len)
2023{
2024 return skb_pull_inline(skb, len);
2025}
2026EXPORT_SYMBOL(skb_pull);
2027
2028/**
2029 * skb_pull_data - remove data from the start of a buffer returning its
2030 * original position.
2031 * @skb: buffer to use
2032 * @len: amount of data to remove
2033 *
2034 * This function removes data from the start of a buffer, returning
2035 * the memory to the headroom. A pointer to the original data in the buffer
2036 * is returned after checking if there is enough data to pull. Once the
2037 * data has been pulled future pushes will overwrite the old data.
2038 */
2039void *skb_pull_data(struct sk_buff *skb, size_t len)
2040{
2041 void *data = skb->data;
2042
2043 if (skb->len < len)
2044 return NULL;
2045
2046 skb_pull(skb, len);
2047
2048 return data;
2049}
2050EXPORT_SYMBOL(skb_pull_data);
2051
2052/**
2031 * skb_trim - remove end from a buffer
2032 * @skb: buffer to alter
2033 * @len: new length
2034 *
2035 * Cut the length of a buffer down by removing data from the tail. If
2036 * the buffer is already under the length specified it is not modified.
2037 * The skb must be linear.
2038 */

--- 1875 unchanged lines hidden (view full) ---

3914
3915err_linearize:
3916 kfree_skb_list(skb->next);
3917 skb->next = NULL;
3918 return ERR_PTR(-ENOMEM);
3919}
3920EXPORT_SYMBOL_GPL(skb_segment_list);
3921
2053 * skb_trim - remove end from a buffer
2054 * @skb: buffer to alter
2055 * @len: new length
2056 *
2057 * Cut the length of a buffer down by removing data from the tail. If
2058 * the buffer is already under the length specified it is not modified.
2059 * The skb must be linear.
2060 */

--- 1875 unchanged lines hidden (view full) ---

3936
3937err_linearize:
3938 kfree_skb_list(skb->next);
3939 skb->next = NULL;
3940 return ERR_PTR(-ENOMEM);
3941}
3942EXPORT_SYMBOL_GPL(skb_segment_list);
3943
3922int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
3923{
3924 if (unlikely(p->len + skb->len >= 65536))
3925 return -E2BIG;
3926
3927 if (NAPI_GRO_CB(p)->last == p)
3928 skb_shinfo(p)->frag_list = skb;
3929 else
3930 NAPI_GRO_CB(p)->last->next = skb;
3931
3932 skb_pull(skb, skb_gro_offset(skb));
3933
3934 NAPI_GRO_CB(p)->last = skb;
3935 NAPI_GRO_CB(p)->count++;
3936 p->data_len += skb->len;
3937
3938 /* sk owenrship - if any - completely transferred to the aggregated packet */
3939 skb->destructor = NULL;
3940 p->truesize += skb->truesize;
3941 p->len += skb->len;
3942
3943 NAPI_GRO_CB(skb)->same_flow = 1;
3944
3945 return 0;
3946}
3947
3948/**
3949 * skb_segment - Perform protocol segmentation on skb.
3950 * @head_skb: buffer to segment
3951 * @features: features for the output path (see dev->features)
3952 *
3953 * This function performs segmentation on the given skb. It returns
3954 * a pointer to the first in a list of new skbs for the segments.
3955 * In case of error it returns ERR_PTR(err).

--- 336 unchanged lines hidden (view full) ---

4292 return segs;
4293
4294err:
4295 kfree_skb_list(segs);
4296 return ERR_PTR(err);
4297}
4298EXPORT_SYMBOL_GPL(skb_segment);
4299
3944/**
3945 * skb_segment - Perform protocol segmentation on skb.
3946 * @head_skb: buffer to segment
3947 * @features: features for the output path (see dev->features)
3948 *
3949 * This function performs segmentation on the given skb. It returns
3950 * a pointer to the first in a list of new skbs for the segments.
3951 * In case of error it returns ERR_PTR(err).

--- 336 unchanged lines hidden (view full) ---

4288 return segs;
4289
4290err:
4291 kfree_skb_list(segs);
4292 return ERR_PTR(err);
4293}
4294EXPORT_SYMBOL_GPL(skb_segment);
4295
4300int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
4301{
4302 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
4303 unsigned int offset = skb_gro_offset(skb);
4304 unsigned int headlen = skb_headlen(skb);
4305 unsigned int len = skb_gro_len(skb);
4306 unsigned int delta_truesize;
4307 unsigned int new_truesize;
4308 struct sk_buff *lp;
4309
4310 if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush))
4311 return -E2BIG;
4312
4313 lp = NAPI_GRO_CB(p)->last;
4314 pinfo = skb_shinfo(lp);
4315
4316 if (headlen <= offset) {
4317 skb_frag_t *frag;
4318 skb_frag_t *frag2;
4319 int i = skbinfo->nr_frags;
4320 int nr_frags = pinfo->nr_frags + i;
4321
4322 if (nr_frags > MAX_SKB_FRAGS)
4323 goto merge;
4324
4325 offset -= headlen;
4326 pinfo->nr_frags = nr_frags;
4327 skbinfo->nr_frags = 0;
4328
4329 frag = pinfo->frags + nr_frags;
4330 frag2 = skbinfo->frags + i;
4331 do {
4332 *--frag = *--frag2;
4333 } while (--i);
4334
4335 skb_frag_off_add(frag, offset);
4336 skb_frag_size_sub(frag, offset);
4337
4338 /* all fragments truesize : remove (head size + sk_buff) */
4339 new_truesize = SKB_TRUESIZE(skb_end_offset(skb));
4340 delta_truesize = skb->truesize - new_truesize;
4341
4342 skb->truesize = new_truesize;
4343 skb->len -= skb->data_len;
4344 skb->data_len = 0;
4345
4346 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
4347 goto done;
4348 } else if (skb->head_frag) {
4349 int nr_frags = pinfo->nr_frags;
4350 skb_frag_t *frag = pinfo->frags + nr_frags;
4351 struct page *page = virt_to_head_page(skb->head);
4352 unsigned int first_size = headlen - offset;
4353 unsigned int first_offset;
4354
4355 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
4356 goto merge;
4357
4358 first_offset = skb->data -
4359 (unsigned char *)page_address(page) +
4360 offset;
4361
4362 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
4363
4364 __skb_frag_set_page(frag, page);
4365 skb_frag_off_set(frag, first_offset);
4366 skb_frag_size_set(frag, first_size);
4367
4368 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
4369 /* We dont need to clear skbinfo->nr_frags here */
4370
4371 new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff));
4372 delta_truesize = skb->truesize - new_truesize;
4373 skb->truesize = new_truesize;
4374 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
4375 goto done;
4376 }
4377
4378merge:
4379 /* sk owenrship - if any - completely transferred to the aggregated packet */
4380 skb->destructor = NULL;
4381 delta_truesize = skb->truesize;
4382 if (offset > headlen) {
4383 unsigned int eat = offset - headlen;
4384
4385 skb_frag_off_add(&skbinfo->frags[0], eat);
4386 skb_frag_size_sub(&skbinfo->frags[0], eat);
4387 skb->data_len -= eat;
4388 skb->len -= eat;
4389 offset = headlen;
4390 }
4391
4392 __skb_pull(skb, offset);
4393
4394 if (NAPI_GRO_CB(p)->last == p)
4395 skb_shinfo(p)->frag_list = skb;
4396 else
4397 NAPI_GRO_CB(p)->last->next = skb;
4398 NAPI_GRO_CB(p)->last = skb;
4399 __skb_header_release(skb);
4400 lp = p;
4401
4402done:
4403 NAPI_GRO_CB(p)->count++;
4404 p->data_len += len;
4405 p->truesize += delta_truesize;
4406 p->len += len;
4407 if (lp != p) {
4408 lp->data_len += len;
4409 lp->truesize += delta_truesize;
4410 lp->len += len;
4411 }
4412 NAPI_GRO_CB(skb)->same_flow = 1;
4413 return 0;
4414}
4415
4416#ifdef CONFIG_SKB_EXTENSIONS
4417#define SKB_EXT_ALIGN_VALUE 8
4418#define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
4419
4420static const u8 skb_ext_type_len[] = {
4421#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4422 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
4423#endif

--- 420 unchanged lines hidden (view full) ---

4844 memset(serr, 0, sizeof(*serr));
4845 serr->ee.ee_errno = ENOMSG;
4846 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
4847 serr->ee.ee_info = tstype;
4848 serr->opt_stats = opt_stats;
4849 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
4850 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
4851 serr->ee.ee_data = skb_shinfo(skb)->tskey;
4296#ifdef CONFIG_SKB_EXTENSIONS
4297#define SKB_EXT_ALIGN_VALUE 8
4298#define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
4299
4300static const u8 skb_ext_type_len[] = {
4301#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4302 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
4303#endif

--- 420 unchanged lines hidden (view full) ---

4724 memset(serr, 0, sizeof(*serr));
4725 serr->ee.ee_errno = ENOMSG;
4726 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
4727 serr->ee.ee_info = tstype;
4728 serr->opt_stats = opt_stats;
4729 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
4730 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
4731 serr->ee.ee_data = skb_shinfo(skb)->tskey;
4852 if (sk->sk_protocol == IPPROTO_TCP &&
4853 sk->sk_type == SOCK_STREAM)
4732 if (sk_is_tcp(sk))
4854 serr->ee.ee_data -= sk->sk_tskey;
4855 }
4856
4857 err = sock_queue_err_skb(sk, skb);
4858
4859 if (err)
4860 kfree_skb(skb);
4861}

--- 52 unchanged lines hidden (view full) ---

4914
4915 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
4916 if (!skb_may_tx_timestamp(sk, tsonly))
4917 return;
4918
4919 if (tsonly) {
4920#ifdef CONFIG_INET
4921 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
4733 serr->ee.ee_data -= sk->sk_tskey;
4734 }
4735
4736 err = sock_queue_err_skb(sk, skb);
4737
4738 if (err)
4739 kfree_skb(skb);
4740}

--- 52 unchanged lines hidden (view full) ---

4793
4794 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
4795 if (!skb_may_tx_timestamp(sk, tsonly))
4796 return;
4797
4798 if (tsonly) {
4799#ifdef CONFIG_INET
4800 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
4922 sk->sk_protocol == IPPROTO_TCP &&
4923 sk->sk_type == SOCK_STREAM) {
4801 sk_is_tcp(sk)) {
4924 skb = tcp_get_timestamping_opt_stats(sk, orig_skb,
4925 ack_skb);
4926 opt_stats = true;
4927 } else
4928#endif
4929 skb = alloc_skb(0, GFP_ATOMIC);
4930 } else {
4931 skb = skb_clone(orig_skb, GFP_ATOMIC);

--- 1648 unchanged lines hidden --- 		4802 skb = tcp_get_timestamping_opt_stats(sk, orig_skb,
4803 ack_skb);
4804 opt_stats = true;
4805 } else
4806#endif
4807 skb = alloc_skb(0, GFP_ATOMIC);
4808 } else {
4809 skb = skb_clone(orig_skb, GFP_ATOMIC);

--- 1648 unchanged lines hidden ---