xref: /openbmc/linux/include/net/gro.h (revision 8e74a48d) 
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 
3 #ifndef _NET_IPV6_GRO_H
4 #define _NET_IPV6_GRO_H
5 
6 #include <linux/indirect_call_wrapper.h>
7 #include <linux/ip.h>
8 #include <linux/ipv6.h>
9 #include <net/ip6_checksum.h>
10 #include <linux/skbuff.h>
11 #include <net/udp.h>
12 
13 struct napi_gro_cb {
14 	/* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
15 	void	*frag0;
16 
17 	/* Length of frag0. */
18 	unsigned int frag0_len;
19 
20 	/* This indicates where we are processing relative to skb->data. */
21 	int	data_offset;
22 
23 	/* This is non-zero if the packet cannot be merged with the new skb. */
24 	u16	flush;
25 
26 	/* Save the IP ID here and check when we get to the transport layer */
27 	u16	flush_id;
28 
29 	/* Number of segments aggregated. */
30 	u16	count;
31 
32 	/* Start offset for remote checksum offload */
33 	u16	gro_remcsum_start;
34 
35 	/* jiffies when first packet was created/queued */
36 	unsigned long age;
37 
38 	/* Used in ipv6_gro_receive() and foo-over-udp */
39 	u16	proto;
40 
41 	/* This is non-zero if the packet may be of the same flow. */
42 	u8	same_flow:1;
43 
44 	/* Used in tunnel GRO receive */
45 	u8	encap_mark:1;
46 
47 	/* GRO checksum is valid */
48 	u8	csum_valid:1;
49 
50 	/* Number of checksums via CHECKSUM_UNNECESSARY */
51 	u8	csum_cnt:3;
52 
53 	/* Free the skb? */
54 	u8	free:2;
55 #define NAPI_GRO_FREE		  1
56 #define NAPI_GRO_FREE_STOLEN_HEAD 2
57 
58 	/* Used in foo-over-udp, set in udp[46]_gro_receive */
59 	u8	is_ipv6:1;
60 
61 	/* Used in GRE, set in fou/gue_gro_receive */
62 	u8	is_fou:1;
63 
64 	/* Used to determine if flush_id can be ignored */
65 	u8	is_atomic:1;
66 
67 	/* Number of gro_receive callbacks this packet already went through */
68 	u8 recursion_counter:4;
69 
70 	/* GRO is done by frag_list pointer chaining. */
71 	u8	is_flist:1;
72 
73 	/* used to support CHECKSUM_COMPLETE for tunneling protocols */
74 	__wsum	csum;
75 
76 	/* used in skb_gro_receive() slow path */
77 	struct sk_buff *last;
78 };
79 
80 #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
81 
82 #define GRO_RECURSION_LIMIT 15
83 static inline int gro_recursion_inc_test(struct sk_buff *skb)
84 {
85 	return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
86 }
87 
88 typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);
89 static inline struct sk_buff *call_gro_receive(gro_receive_t cb,
90 					       struct list_head *head,
91 					       struct sk_buff *skb)
92 {
93 	if (unlikely(gro_recursion_inc_test(skb))) {
94 		NAPI_GRO_CB(skb)->flush |= 1;
95 		return NULL;
96 	}
97 
98 	return cb(head, skb);
99 }
100 
101 typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,
102 					    struct sk_buff *);
103 static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,
104 						  struct sock *sk,
105 						  struct list_head *head,
106 						  struct sk_buff *skb)
107 {
108 	if (unlikely(gro_recursion_inc_test(skb))) {
109 		NAPI_GRO_CB(skb)->flush |= 1;
110 		return NULL;
111 	}
112 
113 	return cb(sk, head, skb);
114 }
115 
116 static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
117 {
118 	return NAPI_GRO_CB(skb)->data_offset;
119 }
120 
121 static inline unsigned int skb_gro_len(const struct sk_buff *skb)
122 {
123 	return skb->len - NAPI_GRO_CB(skb)->data_offset;
124 }
125 
126 static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
127 {
128 	NAPI_GRO_CB(skb)->data_offset += len;
129 }
130 
131 static inline void *skb_gro_header_fast(struct sk_buff *skb,
132 					unsigned int offset)
133 {
134 	return NAPI_GRO_CB(skb)->frag0 + offset;
135 }
136 
137 static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
138 {
139 	return NAPI_GRO_CB(skb)->frag0_len < hlen;
140 }
141 
142 static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
143 {
144 	NAPI_GRO_CB(skb)->frag0 = NULL;
145 	NAPI_GRO_CB(skb)->frag0_len = 0;
146 }
147 
148 static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
149 					unsigned int offset)
150 {
151 	if (!pskb_may_pull(skb, hlen))
152 		return NULL;
153 
154 	skb_gro_frag0_invalidate(skb);
155 	return skb->data + offset;
156 }
157 
158 static inline void *skb_gro_network_header(struct sk_buff *skb)
159 {
160 	return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
161 	       skb_network_offset(skb);
162 }
163 
164 static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto)
165 {
166 	const struct iphdr *iph = skb_gro_network_header(skb);
167 
168 	return csum_tcpudp_nofold(iph->saddr, iph->daddr,
169 				  skb_gro_len(skb), proto, 0);
170 }
171 
172 static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
173 					const void *start, unsigned int len)
174 {
175 	if (NAPI_GRO_CB(skb)->csum_valid)
176 		NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len,
177 						wsum_negate(NAPI_GRO_CB(skb)->csum)));
178 }
179 
180 /* GRO checksum functions. These are logical equivalents of the normal
181  * checksum functions (in skbuff.h) except that they operate on the GRO
182  * offsets and fields in sk_buff.
183  */
184 
185 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
186 
187 static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
188 {
189 	return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
190 }
191 
192 static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
193 						      bool zero_okay,
194 						      __sum16 check)
195 {
196 	return ((skb->ip_summed != CHECKSUM_PARTIAL ||
197 		skb_checksum_start_offset(skb) <
198 		 skb_gro_offset(skb)) &&
199 		!skb_at_gro_remcsum_start(skb) &&
200 		NAPI_GRO_CB(skb)->csum_cnt == 0 &&
201 		(!zero_okay || check));
202 }
203 
204 static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
205 							   __wsum psum)
206 {
207 	if (NAPI_GRO_CB(skb)->csum_valid &&
208 	    !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
209 		return 0;
210 
211 	NAPI_GRO_CB(skb)->csum = psum;
212 
213 	return __skb_gro_checksum_complete(skb);
214 }
215 
216 static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
217 {
218 	if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
219 		/* Consume a checksum from CHECKSUM_UNNECESSARY */
220 		NAPI_GRO_CB(skb)->csum_cnt--;
221 	} else {
222 		/* Update skb for CHECKSUM_UNNECESSARY and csum_level when we
223 		 * verified a new top level checksum or an encapsulated one
224 		 * during GRO. This saves work if we fallback to normal path.
225 		 */
226 		__skb_incr_checksum_unnecessary(skb);
227 	}
228 }
229 
230 #define __skb_gro_checksum_validate(skb, proto, zero_okay, check,	\
231 				    compute_pseudo)			\
232 ({									\
233 	__sum16 __ret = 0;						\
234 	if (__skb_gro_checksum_validate_needed(skb, zero_okay, check))	\
235 		__ret = __skb_gro_checksum_validate_complete(skb,	\
236 				compute_pseudo(skb, proto));		\
237 	if (!__ret)							\
238 		skb_gro_incr_csum_unnecessary(skb);			\
239 	__ret;								\
240 })
241 
242 #define skb_gro_checksum_validate(skb, proto, compute_pseudo)		\
243 	__skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
244 
245 #define skb_gro_checksum_validate_zero_check(skb, proto, check,		\
246 					     compute_pseudo)		\
247 	__skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
248 
249 #define skb_gro_checksum_simple_validate(skb)				\
250 	__skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
251 
252 static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
253 {
254 	return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
255 		!NAPI_GRO_CB(skb)->csum_valid);
256 }
257 
258 static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
259 					      __wsum pseudo)
260 {
261 	NAPI_GRO_CB(skb)->csum = ~pseudo;
262 	NAPI_GRO_CB(skb)->csum_valid = 1;
263 }
264 
265 #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo)	\
266 do {									\
267 	if (__skb_gro_checksum_convert_check(skb))			\
268 		__skb_gro_checksum_convert(skb, 			\
269 					   compute_pseudo(skb, proto));	\
270 } while (0)
271 
272 struct gro_remcsum {
273 	int offset;
274 	__wsum delta;
275 };
276 
277 static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
278 {
279 	grc->offset = 0;
280 	grc->delta = 0;
281 }
282 
283 static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
284 					    unsigned int off, size_t hdrlen,
285 					    int start, int offset,
286 					    struct gro_remcsum *grc,
287 					    bool nopartial)
288 {
289 	__wsum delta;
290 	size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
291 
292 	BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
293 
294 	if (!nopartial) {
295 		NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
296 		return ptr;
297 	}
298 
299 	ptr = skb_gro_header_fast(skb, off);
300 	if (skb_gro_header_hard(skb, off + plen)) {
301 		ptr = skb_gro_header_slow(skb, off + plen, off);
302 		if (!ptr)
303 			return NULL;
304 	}
305 
306 	delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
307 			       start, offset);
308 
309 	/* Adjust skb->csum since we changed the packet */
310 	NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
311 
312 	grc->offset = off + hdrlen + offset;
313 	grc->delta = delta;
314 
315 	return ptr;
316 }
317 
318 static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
319 					   struct gro_remcsum *grc)
320 {
321 	void *ptr;
322 	size_t plen = grc->offset + sizeof(u16);
323 
324 	if (!grc->delta)
325 		return;
326 
327 	ptr = skb_gro_header_fast(skb, grc->offset);
328 	if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) {
329 		ptr = skb_gro_header_slow(skb, plen, grc->offset);
330 		if (!ptr)
331 			return;
332 	}
333 
334 	remcsum_unadjust((__sum16 *)ptr, grc->delta);
335 }
336 
337 #ifdef CONFIG_XFRM_OFFLOAD
338 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
339 {
340 	if (PTR_ERR(pp) != -EINPROGRESS)
341 		NAPI_GRO_CB(skb)->flush |= flush;
342 }
343 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
344 					       struct sk_buff *pp,
345 					       int flush,
346 					       struct gro_remcsum *grc)
347 {
348 	if (PTR_ERR(pp) != -EINPROGRESS) {
349 		NAPI_GRO_CB(skb)->flush |= flush;
350 		skb_gro_remcsum_cleanup(skb, grc);
351 		skb->remcsum_offload = 0;
352 	}
353 }
354 #else
355 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
356 {
357 	NAPI_GRO_CB(skb)->flush |= flush;
358 }
359 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
360 					       struct sk_buff *pp,
361 					       int flush,
362 					       struct gro_remcsum *grc)
363 {
364 	NAPI_GRO_CB(skb)->flush |= flush;
365 	skb_gro_remcsum_cleanup(skb, grc);
366 	skb->remcsum_offload = 0;
367 }
368 #endif
369 
370 INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,
371 							   struct sk_buff *));
372 INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
373 INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
374 							   struct sk_buff *));
375 INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
376 
377 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *,
378 							   struct sk_buff *));
379 INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int));
380 
381 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *,
382 							   struct sk_buff *));
383 INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int));
384 
385 #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb)	\
386 ({								\
387 	unlikely(gro_recursion_inc_test(skb)) ?			\
388 		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
389 		INDIRECT_CALL_INET(cb, f2, f1, head, skb);	\
390 })
391 
392 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
393 				struct udphdr *uh, struct sock *sk);
394 int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
395 
396 static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
397 {
398 	struct udphdr *uh;
399 	unsigned int hlen, off;
400 
401 	off  = skb_gro_offset(skb);
402 	hlen = off + sizeof(*uh);
403 	uh   = skb_gro_header_fast(skb, off);
404 	if (skb_gro_header_hard(skb, hlen))
405 		uh = skb_gro_header_slow(skb, hlen, off);
406 
407 	return uh;
408 }
409 
410 static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto)
411 {
412 	const struct ipv6hdr *iph = skb_gro_network_header(skb);
413 
414 	return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr,
415 					    skb_gro_len(skb), proto, 0));
416 }
417 
418 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
419 
420 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
421 static inline void gro_normal_list(struct napi_struct *napi)
422 {
423 	if (!napi->rx_count)
424 		return;
425 	netif_receive_skb_list_internal(&napi->rx_list);
426 	INIT_LIST_HEAD(&napi->rx_list);
427 	napi->rx_count = 0;
428 }
429 
430 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
431  * pass the whole batch up to the stack.
432  */
433 static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
434 {
435 	list_add_tail(&skb->list, &napi->rx_list);
436 	napi->rx_count += segs;
437 	if (napi->rx_count >= gro_normal_batch)
438 		gro_normal_list(napi);
439 }
440 
441 
442 #endif /* _NET_IPV6_GRO_H */
443