skbuff.h (6f022c2ddbcefaee79502ce5386dfe351d457070) skbuff.h (c504e5c2f9648a1e5c2be01e8c3f59d394192bd3)
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Definitions for the 'struct sk_buff' memory handlers.
4 *
5 * Authors:
6 * Alan Cox, <gw4pts@gw4pts.ampr.org>
7 * Florian La Roche, <rzsfl@rz.uni-sb.de>
8 */

--- 22 unchanged lines hidden (view full) ---

31#include <linux/dma-mapping.h>
32#include <linux/netdev_features.h>
33#include <linux/sched.h>
34#include <linux/sched/clock.h>
35#include <net/flow_dissector.h>
36#include <linux/splice.h>
37#include <linux/in6.h>
38#include <linux/if_packet.h>
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Definitions for the 'struct sk_buff' memory handlers.
4 *
5 * Authors:
6 * Alan Cox, <gw4pts@gw4pts.ampr.org>
7 * Florian La Roche, <rzsfl@rz.uni-sb.de>
8 */

--- 22 unchanged lines hidden (view full) ---

31#include <linux/dma-mapping.h>
32#include <linux/netdev_features.h>
33#include <linux/sched.h>
34#include <linux/sched/clock.h>
35#include <net/flow_dissector.h>
36#include <linux/splice.h>
37#include <linux/in6.h>
38#include <linux/if_packet.h>
39#include <linux/llist.h>
39#include <net/flow.h>
40#include <net/page_pool.h>
41#if IS_ENABLED(CONFIG_NF_CONNTRACK)
42#include <linux/netfilter/nf_conntrack_common.h>
43#endif
44
45/* The interface for checksum offload between the stack and networking drivers
46 * is as follows...

--- 235 unchanged lines hidden (view full) ---

282/* Chain in tc_skb_ext will be used to share the tc chain with
283 * ovs recirc_id. It will be set to the current chain by tc
284 * and read by ovs to recirc_id.
285 */
286struct tc_skb_ext {
287 __u32 chain;
288 __u16 mru;
289 __u16 zone;
40#include <net/flow.h>
41#include <net/page_pool.h>
42#if IS_ENABLED(CONFIG_NF_CONNTRACK)
43#include <linux/netfilter/nf_conntrack_common.h>
44#endif
45
46/* The interface for checksum offload between the stack and networking drivers
47 * is as follows...

--- 235 unchanged lines hidden (view full) ---

283/* Chain in tc_skb_ext will be used to share the tc chain with
284 * ovs recirc_id. It will be set to the current chain by tc
285 * and read by ovs to recirc_id.
286 */
287struct tc_skb_ext {
288 __u32 chain;
289 __u16 mru;
290 __u16 zone;
290 u8 post_ct:1;
291 u8 post_ct_snat:1;
292 u8 post_ct_dnat:1;
291 bool post_ct;
293};
294#endif
295
296struct sk_buff_head {
292};
293#endif
294
295struct sk_buff_head {
297 /* These two members must be first. */
298 struct sk_buff *next;
299 struct sk_buff *prev;
296 /* These two members must be first to match sk_buff. */
297 struct_group_tagged(sk_buff_list, list,
298 struct sk_buff *next;
299 struct sk_buff *prev;
300 );
300
301 __u32 qlen;
302 spinlock_t lock;
303};
304
305struct sk_buff;
306
301
302 __u32 qlen;
303 spinlock_t lock;
304};
305
306struct sk_buff;
307
308/* The reason of skb drop, which is used in kfree_skb_reason().
309 * en...maybe they should be splited by group?
310 *
311 * Each item here should also be in 'TRACE_SKB_DROP_REASON', which is
312 * used to translate the reason to string.
313 */
314enum skb_drop_reason {
315 SKB_DROP_REASON_NOT_SPECIFIED,
316 SKB_DROP_REASON_MAX,
317};
318
307/* To allow 64K frame to be packed as single skb without frag_list we
308 * require 64K/PAGE_SIZE pages plus 1 additional page to allow for
309 * buffers which do not start on a page boundary.
310 *
311 * Since GRO uses frags we allocate at least 16 regardless of page
312 * size.
313 */
314#if (65536/PAGE_SIZE + 1) < 16

--- 309 unchanged lines hidden (view full) ---

624 * struct sk_buff - socket buffer
625 * @next: Next buffer in list
626 * @prev: Previous buffer in list
627 * @tstamp: Time we arrived/left
628 * @skb_mstamp_ns: (aka @tstamp) earliest departure time; start point
629 * for retransmit timer
630 * @rbnode: RB tree node, alternative to next/prev for netem/tcp
631 * @list: queue head
319/* To allow 64K frame to be packed as single skb without frag_list we
320 * require 64K/PAGE_SIZE pages plus 1 additional page to allow for
321 * buffers which do not start on a page boundary.
322 *
323 * Since GRO uses frags we allocate at least 16 regardless of page
324 * size.
325 */
326#if (65536/PAGE_SIZE + 1) < 16

--- 309 unchanged lines hidden (view full) ---

636 * struct sk_buff - socket buffer
637 * @next: Next buffer in list
638 * @prev: Previous buffer in list
639 * @tstamp: Time we arrived/left
640 * @skb_mstamp_ns: (aka @tstamp) earliest departure time; start point
641 * for retransmit timer
642 * @rbnode: RB tree node, alternative to next/prev for netem/tcp
643 * @list: queue head
644 * @ll_node: anchor in an llist (eg socket defer_list)
632 * @sk: Socket we are owned by
633 * @ip_defrag_offset: (aka @sk) alternate use of @sk, used in
634 * fragmentation management
635 * @dev: Device we arrived on/are leaving by
636 * @dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL
637 * @cb: Control buffer. Free for use by every layer. Put private vars here
638 * @_skb_refdst: destination entry (with norefcount bit)
639 * @sp: the security path, used for xfrm

--- 86 unchanged lines hidden (view full) ---

726 * @truesize: Buffer size
727 * @users: User count - see {datagram,tcp}.c
728 * @extensions: allocated extensions, valid if active_extensions is nonzero
729 */
730
731struct sk_buff {
732 union {
733 struct {
645 * @sk: Socket we are owned by
646 * @ip_defrag_offset: (aka @sk) alternate use of @sk, used in
647 * fragmentation management
648 * @dev: Device we arrived on/are leaving by
649 * @dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL
650 * @cb: Control buffer. Free for use by every layer. Put private vars here
651 * @_skb_refdst: destination entry (with norefcount bit)
652 * @sp: the security path, used for xfrm

--- 86 unchanged lines hidden (view full) ---

739 * @truesize: Buffer size
740 * @users: User count - see {datagram,tcp}.c
741 * @extensions: allocated extensions, valid if active_extensions is nonzero
742 */
743
744struct sk_buff {
745 union {
746 struct {
734 /* These two members must be first. */
747 /* These two members must be first to match sk_buff_head. */
735 struct sk_buff *next;
736 struct sk_buff *prev;
737
738 union {
739 struct net_device *dev;
740 /* Some protocols might use this space to store information,
741 * while device pointer would be NULL.
742 * UDP receive path is one user.
743 */
744 unsigned long dev_scratch;
745 };
746 };
747 struct rb_node rbnode; /* used in netem, ip4 defrag, and tcp stack */
748 struct list_head list;
748 struct sk_buff *next;
749 struct sk_buff *prev;
750
751 union {
752 struct net_device *dev;
753 /* Some protocols might use this space to store information,
754 * while device pointer would be NULL.
755 * UDP receive path is one user.
756 */
757 unsigned long dev_scratch;
758 };
759 };
760 struct rb_node rbnode; /* used in netem, ip4 defrag, and tcp stack */
761 struct list_head list;
762 struct llist_node ll_node;
749 };
750
751 union {
752 struct sock *sk;
753 int ip_defrag_offset;
754 };
755
756 union {

--- 33 unchanged lines hidden (view full) ---

790 __u16 queue_mapping;
791
792/* if you move cloned around you also must adapt those constants */
793#ifdef __BIG_ENDIAN_BITFIELD
794#define CLONED_MASK (1 << 7)
795#else
796#define CLONED_MASK 1
797#endif
763 };
764
765 union {
766 struct sock *sk;
767 int ip_defrag_offset;
768 };
769
770 union {

--- 33 unchanged lines hidden (view full) ---

804 __u16 queue_mapping;
805
806/* if you move cloned around you also must adapt those constants */
807#ifdef __BIG_ENDIAN_BITFIELD
808#define CLONED_MASK (1 << 7)
809#else
810#define CLONED_MASK 1
811#endif
798#define CLONED_OFFSET() offsetof(struct sk_buff, __cloned_offset)
812#define CLONED_OFFSET offsetof(struct sk_buff, __cloned_offset)
799
800 /* private: */
801 __u8 __cloned_offset[0];
802 /* public: */
803 __u8 cloned:1,
804 nohdr:1,
805 fclone:2,
806 peeked:1,
807 head_frag:1,
808 pfmemalloc:1,
809 pp_recycle:1; /* page_pool recycle indicator */
810#ifdef CONFIG_SKB_EXTENSIONS
811 __u8 active_extensions;
812#endif
813
813
814 /* private: */
815 __u8 __cloned_offset[0];
816 /* public: */
817 __u8 cloned:1,
818 nohdr:1,
819 fclone:2,
820 peeked:1,
821 head_frag:1,
822 pfmemalloc:1,
823 pp_recycle:1; /* page_pool recycle indicator */
824#ifdef CONFIG_SKB_EXTENSIONS
825 __u8 active_extensions;
826#endif
827
814 /* fields enclosed in headers_start/headers_end are copied
828 /* Fields enclosed in headers group are copied
815 * using a single memcpy() in __copy_skb_header()
816 */
829 * using a single memcpy() in __copy_skb_header()
830 */
817 /* private: */
818 __u32 headers_start[0];
819 /* public: */
831 struct_group(headers,
820
832
821/* if you move pkt_type around you also must adapt those constants */
822#ifdef __BIG_ENDIAN_BITFIELD
823#define PKT_TYPE_MAX (7 << 5)
824#else
825#define PKT_TYPE_MAX 7
826#endif
827#define PKT_TYPE_OFFSET() offsetof(struct sk_buff, __pkt_type_offset)
828
829 /* private: */
830 __u8 __pkt_type_offset[0];
831 /* public: */
833 /* private: */
834 __u8 __pkt_type_offset[0];
835 /* public: */
832 __u8 pkt_type:3;
836 __u8 pkt_type:3; /* see PKT_TYPE_MAX */
833 __u8 ignore_df:1;
834 __u8 nf_trace:1;
835 __u8 ip_summed:2;
836 __u8 ooo_okay:1;
837
838 __u8 l4_hash:1;
839 __u8 sw_hash:1;
840 __u8 wifi_acked_valid:1;
841 __u8 wifi_acked:1;
842 __u8 no_fcs:1;
843 /* Indicates the inner headers are valid in the skbuff. */
844 __u8 encapsulation:1;
845 __u8 encap_hdr_csum:1;
846 __u8 csum_valid:1;
847
837 __u8 ignore_df:1;
838 __u8 nf_trace:1;
839 __u8 ip_summed:2;
840 __u8 ooo_okay:1;
841
842 __u8 l4_hash:1;
843 __u8 sw_hash:1;
844 __u8 wifi_acked_valid:1;
845 __u8 wifi_acked:1;
846 __u8 no_fcs:1;
847 /* Indicates the inner headers are valid in the skbuff. */
848 __u8 encapsulation:1;
849 __u8 encap_hdr_csum:1;
850 __u8 csum_valid:1;
851
848#ifdef __BIG_ENDIAN_BITFIELD
849#define PKT_VLAN_PRESENT_BIT 7
850#else
851#define PKT_VLAN_PRESENT_BIT 0
852#endif
853#define PKT_VLAN_PRESENT_OFFSET() offsetof(struct sk_buff, __pkt_vlan_present_offset)
854 /* private: */
855 __u8 __pkt_vlan_present_offset[0];
856 /* public: */
852 /* private: */
853 __u8 __pkt_vlan_present_offset[0];
854 /* public: */
857 __u8 vlan_present:1;
855 __u8 vlan_present:1; /* See PKT_VLAN_PRESENT_BIT */
858 __u8 csum_complete_sw:1;
859 __u8 csum_level:2;
860 __u8 csum_not_inet:1;
861 __u8 dst_pending_confirm:1;
862#ifdef CONFIG_IPV6_NDISC_NODETYPE
863 __u8 ndisc_nodetype:2;
864#endif
865

--- 64 unchanged lines hidden (view full) ---

930 __u16 transport_header;
931 __u16 network_header;
932 __u16 mac_header;
933
934#ifdef CONFIG_KCOV
935 u64 kcov_handle;
936#endif
937
856 __u8 csum_complete_sw:1;
857 __u8 csum_level:2;
858 __u8 csum_not_inet:1;
859 __u8 dst_pending_confirm:1;
860#ifdef CONFIG_IPV6_NDISC_NODETYPE
861 __u8 ndisc_nodetype:2;
862#endif
863

--- 64 unchanged lines hidden (view full) ---

928 __u16 transport_header;
929 __u16 network_header;
930 __u16 mac_header;
931
932#ifdef CONFIG_KCOV
933 u64 kcov_handle;
934#endif
935
938 /* private: */
939 __u32 headers_end[0];
940 /* public: */
936 ); /* end headers group */
941
942 /* These elements must be at the end, see alloc_skb() for details. */
943 sk_buff_data_t tail;
944 sk_buff_data_t end;
945 unsigned char *head,
946 *data;
947 unsigned int truesize;
948 refcount_t users;
949
950#ifdef CONFIG_SKB_EXTENSIONS
951 /* only useable after checking ->active_extensions != 0 */
952 struct skb_ext *extensions;
953#endif
954};
955
937
938 /* These elements must be at the end, see alloc_skb() for details. */
939 sk_buff_data_t tail;
940 sk_buff_data_t end;
941 unsigned char *head,
942 *data;
943 unsigned int truesize;
944 refcount_t users;
945
946#ifdef CONFIG_SKB_EXTENSIONS
947 /* only useable after checking ->active_extensions != 0 */
948 struct skb_ext *extensions;
949#endif
950};
951
952/* if you move pkt_type around you also must adapt those constants */
953#ifdef __BIG_ENDIAN_BITFIELD
954#define PKT_TYPE_MAX (7 << 5)
955#else
956#define PKT_TYPE_MAX 7
957#endif
958#define PKT_TYPE_OFFSET offsetof(struct sk_buff, __pkt_type_offset)
959
960/* if you move pkt_vlan_present around you also must adapt these constants */
961#ifdef __BIG_ENDIAN_BITFIELD
962#define PKT_VLAN_PRESENT_BIT 7
963#else
964#define PKT_VLAN_PRESENT_BIT 0
965#endif
966#define PKT_VLAN_PRESENT_OFFSET offsetof(struct sk_buff, __pkt_vlan_present_offset)
967
956#ifdef __KERNEL__
957/*
958 * Handling routines are only of interest to the kernel
959 */
960
961#define SKB_ALLOC_FCLONE 0x01
962#define SKB_ALLOC_RX 0x02
963#define SKB_ALLOC_NAPI 0x04

--- 115 unchanged lines hidden (view full) ---

1079 if (likely(refcount_read(&skb->users) == 1))
1080 smp_rmb();
1081 else if (likely(!refcount_dec_and_test(&skb->users)))
1082 return false;
1083
1084 return true;
1085}
1086
968#ifdef __KERNEL__
969/*
970 * Handling routines are only of interest to the kernel
971 */
972
973#define SKB_ALLOC_FCLONE 0x01
974#define SKB_ALLOC_RX 0x02
975#define SKB_ALLOC_NAPI 0x04

--- 115 unchanged lines hidden (view full) ---

1091 if (likely(refcount_read(&skb->users) == 1))
1092 smp_rmb();
1093 else if (likely(!refcount_dec_and_test(&skb->users)))
1094 return false;
1095
1096 return true;
1097}
1098
1099void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason);
1100
1101/**
1102 * kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason
1103 * @skb: buffer to free
1104 */
1105static inline void kfree_skb(struct sk_buff *skb)
1106{
1107 kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
1108}
1109
1087void skb_release_head_state(struct sk_buff *skb);
1110void skb_release_head_state(struct sk_buff *skb);
1088void kfree_skb(struct sk_buff *skb);
1089void kfree_skb_list(struct sk_buff *segs);
1090void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
1091void skb_tx_error(struct sk_buff *skb);
1092
1093#ifdef CONFIG_TRACEPOINTS
1094void consume_skb(struct sk_buff *skb);
1095#else
1096static inline void consume_skb(struct sk_buff *skb)

--- 876 unchanged lines hidden (view full) ---

1973 struct sk_buff *prev, struct sk_buff *next,
1974 struct sk_buff_head *list)
1975{
1976 /* See skb_queue_empty_lockless() and skb_peek_tail()
1977 * for the opposite READ_ONCE()
1978 */
1979 WRITE_ONCE(newsk->next, next);
1980 WRITE_ONCE(newsk->prev, prev);
1111void kfree_skb_list(struct sk_buff *segs);
1112void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
1113void skb_tx_error(struct sk_buff *skb);
1114
1115#ifdef CONFIG_TRACEPOINTS
1116void consume_skb(struct sk_buff *skb);
1117#else
1118static inline void consume_skb(struct sk_buff *skb)

--- 876 unchanged lines hidden (view full) ---

1995 struct sk_buff *prev, struct sk_buff *next,
1996 struct sk_buff_head *list)
1997{
1998 /* See skb_queue_empty_lockless() and skb_peek_tail()
1999 * for the opposite READ_ONCE()
2000 */
2001 WRITE_ONCE(newsk->next, next);
2002 WRITE_ONCE(newsk->prev, prev);
1981 WRITE_ONCE(next->prev, newsk);
1982 WRITE_ONCE(prev->next, newsk);
2003 WRITE_ONCE(((struct sk_buff_list *)next)->prev, newsk);
2004 WRITE_ONCE(((struct sk_buff_list *)prev)->next, newsk);
1983 WRITE_ONCE(list->qlen, list->qlen + 1);
1984}
1985
1986static inline void __skb_queue_splice(const struct sk_buff_head *list,
1987 struct sk_buff *prev,
1988 struct sk_buff *next)
1989{
1990 struct sk_buff *first = list->next;

--- 79 unchanged lines hidden (view full) ---

2070 * and you must therefore hold required locks before calling it.
2071 *
2072 * A buffer cannot be placed on two lists at the same time.
2073 */
2074static inline void __skb_queue_after(struct sk_buff_head *list,
2075 struct sk_buff *prev,
2076 struct sk_buff *newsk)
2077{
2005 WRITE_ONCE(list->qlen, list->qlen + 1);
2006}
2007
2008static inline void __skb_queue_splice(const struct sk_buff_head *list,
2009 struct sk_buff *prev,
2010 struct sk_buff *next)
2011{
2012 struct sk_buff *first = list->next;

--- 79 unchanged lines hidden (view full) ---

2092 * and you must therefore hold required locks before calling it.
2093 *
2094 * A buffer cannot be placed on two lists at the same time.
2095 */
2096static inline void __skb_queue_after(struct sk_buff_head *list,
2097 struct sk_buff *prev,
2098 struct sk_buff *newsk)
2099{
2078 __skb_insert(newsk, prev, prev->next, list);
2100 __skb_insert(newsk, prev, ((struct sk_buff_list *)prev)->next, list);
2079}
2080
2081void skb_append(struct sk_buff *old, struct sk_buff *newsk,
2082 struct sk_buff_head *list);
2083
2084static inline void __skb_queue_before(struct sk_buff_head *list,
2085 struct sk_buff *next,
2086 struct sk_buff *newsk)
2087{
2101}
2102
2103void skb_append(struct sk_buff *old, struct sk_buff *newsk,
2104 struct sk_buff_head *list);
2105
2106static inline void __skb_queue_before(struct sk_buff_head *list,
2107 struct sk_buff *next,
2108 struct sk_buff *newsk)
2109{
2088 __skb_insert(newsk, next->prev, next, list);
2110 __skb_insert(newsk, ((struct sk_buff_list *)next)->prev, next, list);
2089}
2090
2091/**
2092 * __skb_queue_head - queue a buffer at the list head
2093 * @list: list to use
2094 * @newsk: buffer to queue
2095 *
2096 * Queue a buffer at the start of a list. This function takes no locks

--- 273 unchanged lines hidden (view full) ---

2370 return skb->data += len;
2371}
2372
2373static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len)
2374{
2375 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
2376}
2377
2111}
2112
2113/**
2114 * __skb_queue_head - queue a buffer at the list head
2115 * @list: list to use
2116 * @newsk: buffer to queue
2117 *
2118 * Queue a buffer at the start of a list. This function takes no locks

--- 273 unchanged lines hidden (view full) ---

2392 return skb->data += len;
2393}
2394
2395static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len)
2396{
2397 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
2398}
2399
2400void *skb_pull_data(struct sk_buff *skb, size_t len);
2401
2378void *__pskb_pull_tail(struct sk_buff *skb, int delta);
2379
2380static inline void *__pskb_pull(struct sk_buff *skb, unsigned int len)
2381{
2382 if (len > skb_headlen(skb) &&
2383 !__pskb_pull_tail(skb, len - skb_headlen(skb)))
2384 return NULL;
2385 skb->len -= len;

--- 1096 unchanged lines hidden (view full) ---

3482 *
3483 * After doing a pull on a received packet, you need to call this to
3484 * update the CHECKSUM_COMPLETE checksum, or set ip_summed to
3485 * CHECKSUM_NONE so that it can be recomputed from scratch.
3486 */
3487static inline void skb_postpull_rcsum(struct sk_buff *skb,
3488 const void *start, unsigned int len)
3489{
2402void *__pskb_pull_tail(struct sk_buff *skb, int delta);
2403
2404static inline void *__pskb_pull(struct sk_buff *skb, unsigned int len)
2405{
2406 if (len > skb_headlen(skb) &&
2407 !__pskb_pull_tail(skb, len - skb_headlen(skb)))
2408 return NULL;
2409 skb->len -= len;

--- 1096 unchanged lines hidden (view full) ---

3506 *
3507 * After doing a pull on a received packet, you need to call this to
3508 * update the CHECKSUM_COMPLETE checksum, or set ip_summed to
3509 * CHECKSUM_NONE so that it can be recomputed from scratch.
3510 */
3511static inline void skb_postpull_rcsum(struct sk_buff *skb,
3512 const void *start, unsigned int len)
3513{
3490 __skb_postpull_rcsum(skb, start, len, 0);
3514 if (skb->ip_summed == CHECKSUM_COMPLETE)
3515 skb->csum = wsum_negate(csum_partial(start, len,
3516 wsum_negate(skb->csum)));
3517 else if (skb->ip_summed == CHECKSUM_PARTIAL &&
3518 skb_checksum_start_offset(skb) < 0)
3519 skb->ip_summed = CHECKSUM_NONE;
3491}
3492
3493static __always_inline void
3494__skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3495 unsigned int off)
3496{
3497 if (skb->ip_summed == CHECKSUM_COMPLETE)
3498 skb->csum = csum_block_add(skb->csum,

--- 1274 unchanged lines hidden --- 		3520}
3521
3522static __always_inline void
3523__skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3524 unsigned int off)
3525{
3526 if (skb->ip_summed == CHECKSUM_COMPLETE)
3527 skb->csum = csum_block_add(skb->csum,

--- 1274 unchanged lines hidden ---