1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 3 #ifndef _NET_IPV6_GRO_H 4 #define _NET_IPV6_GRO_H 5 6 #include <linux/indirect_call_wrapper.h> 7 #include <linux/ip.h> 8 #include <linux/ipv6.h> 9 #include <net/ip6_checksum.h> 10 #include <linux/skbuff.h> 11 #include <net/udp.h> 12 13 struct napi_gro_cb { 14 /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */ 15 void *frag0; 16 17 /* Length of frag0. */ 18 unsigned int frag0_len; 19 20 /* This indicates where we are processing relative to skb->data. */ 21 int data_offset; 22 23 /* This is non-zero if the packet cannot be merged with the new skb. */ 24 u16 flush; 25 26 /* Save the IP ID here and check when we get to the transport layer */ 27 u16 flush_id; 28 29 /* Number of segments aggregated. */ 30 u16 count; 31 32 /* Used in ipv6_gro_receive() and foo-over-udp */ 33 u16 proto; 34 35 /* jiffies when first packet was created/queued */ 36 unsigned long age; 37 38 /* Used in napi_gro_cb::free */ 39 #define NAPI_GRO_FREE 1 40 #define NAPI_GRO_FREE_STOLEN_HEAD 2 41 /* portion of the cb set to zero at every gro iteration */ 42 struct_group(zeroed, 43 44 /* Start offset for remote checksum offload */ 45 u16 gro_remcsum_start; 46 47 /* This is non-zero if the packet may be of the same flow. */ 48 u8 same_flow:1; 49 50 /* Used in tunnel GRO receive */ 51 u8 encap_mark:1; 52 53 /* GRO checksum is valid */ 54 u8 csum_valid:1; 55 56 /* Number of checksums via CHECKSUM_UNNECESSARY */ 57 u8 csum_cnt:3; 58 59 /* Free the skb? */ 60 u8 free:2; 61 62 /* Used in foo-over-udp, set in udp[46]_gro_receive */ 63 u8 is_ipv6:1; 64 65 /* Used in GRE, set in fou/gue_gro_receive */ 66 u8 is_fou:1; 67 68 /* Used to determine if flush_id can be ignored */ 69 u8 is_atomic:1; 70 71 /* Number of gro_receive callbacks this packet already went through */ 72 u8 recursion_counter:4; 73 74 /* GRO is done by frag_list pointer chaining. */ 75 u8 is_flist:1; 76 ); 77 78 /* used to support CHECKSUM_COMPLETE for tunneling protocols */ 79 __wsum csum; 80 81 /* used in skb_gro_receive() slow path */ 82 struct sk_buff *last; 83 }; 84 85 #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) 86 87 #define GRO_RECURSION_LIMIT 15 88 static inline int gro_recursion_inc_test(struct sk_buff *skb) 89 { 90 return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; 91 } 92 93 typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *); 94 static inline struct sk_buff *call_gro_receive(gro_receive_t cb, 95 struct list_head *head, 96 struct sk_buff *skb) 97 { 98 if (unlikely(gro_recursion_inc_test(skb))) { 99 NAPI_GRO_CB(skb)->flush |= 1; 100 return NULL; 101 } 102 103 return cb(head, skb); 104 } 105 106 typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *, 107 struct sk_buff *); 108 static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb, 109 struct sock *sk, 110 struct list_head *head, 111 struct sk_buff *skb) 112 { 113 if (unlikely(gro_recursion_inc_test(skb))) { 114 NAPI_GRO_CB(skb)->flush |= 1; 115 return NULL; 116 } 117 118 return cb(sk, head, skb); 119 } 120 121 static inline unsigned int skb_gro_offset(const struct sk_buff *skb) 122 { 123 return NAPI_GRO_CB(skb)->data_offset; 124 } 125 126 static inline unsigned int skb_gro_len(const struct sk_buff *skb) 127 { 128 return skb->len - NAPI_GRO_CB(skb)->data_offset; 129 } 130 131 static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) 132 { 133 NAPI_GRO_CB(skb)->data_offset += len; 134 } 135 136 static inline void *skb_gro_header_fast(struct sk_buff *skb, 137 unsigned int offset) 138 { 139 return NAPI_GRO_CB(skb)->frag0 + offset; 140 } 141 142 static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen) 143 { 144 return NAPI_GRO_CB(skb)->frag0_len < hlen; 145 } 146 147 static inline void skb_gro_frag0_invalidate(struct sk_buff *skb) 148 { 149 NAPI_GRO_CB(skb)->frag0 = NULL; 150 NAPI_GRO_CB(skb)->frag0_len = 0; 151 } 152 153 static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, 154 unsigned int offset) 155 { 156 if (!pskb_may_pull(skb, hlen)) 157 return NULL; 158 159 skb_gro_frag0_invalidate(skb); 160 return skb->data + offset; 161 } 162 163 static inline void *skb_gro_network_header(struct sk_buff *skb) 164 { 165 return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + 166 skb_network_offset(skb); 167 } 168 169 static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto) 170 { 171 const struct iphdr *iph = skb_gro_network_header(skb); 172 173 return csum_tcpudp_nofold(iph->saddr, iph->daddr, 174 skb_gro_len(skb), proto, 0); 175 } 176 177 static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, 178 const void *start, unsigned int len) 179 { 180 if (NAPI_GRO_CB(skb)->csum_valid) 181 NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len, 182 wsum_negate(NAPI_GRO_CB(skb)->csum))); 183 } 184 185 /* GRO checksum functions. These are logical equivalents of the normal 186 * checksum functions (in skbuff.h) except that they operate on the GRO 187 * offsets and fields in sk_buff. 188 */ 189 190 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb); 191 192 static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb) 193 { 194 return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb)); 195 } 196 197 static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb, 198 bool zero_okay, 199 __sum16 check) 200 { 201 return ((skb->ip_summed != CHECKSUM_PARTIAL || 202 skb_checksum_start_offset(skb) < 203 skb_gro_offset(skb)) && 204 !skb_at_gro_remcsum_start(skb) && 205 NAPI_GRO_CB(skb)->csum_cnt == 0 && 206 (!zero_okay || check)); 207 } 208 209 static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb, 210 __wsum psum) 211 { 212 if (NAPI_GRO_CB(skb)->csum_valid && 213 !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum))) 214 return 0; 215 216 NAPI_GRO_CB(skb)->csum = psum; 217 218 return __skb_gro_checksum_complete(skb); 219 } 220 221 static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb) 222 { 223 if (NAPI_GRO_CB(skb)->csum_cnt > 0) { 224 /* Consume a checksum from CHECKSUM_UNNECESSARY */ 225 NAPI_GRO_CB(skb)->csum_cnt--; 226 } else { 227 /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we 228 * verified a new top level checksum or an encapsulated one 229 * during GRO. This saves work if we fallback to normal path. 230 */ 231 __skb_incr_checksum_unnecessary(skb); 232 } 233 } 234 235 #define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \ 236 compute_pseudo) \ 237 ({ \ 238 __sum16 __ret = 0; \ 239 if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \ 240 __ret = __skb_gro_checksum_validate_complete(skb, \ 241 compute_pseudo(skb, proto)); \ 242 if (!__ret) \ 243 skb_gro_incr_csum_unnecessary(skb); \ 244 __ret; \ 245 }) 246 247 #define skb_gro_checksum_validate(skb, proto, compute_pseudo) \ 248 __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo) 249 250 #define skb_gro_checksum_validate_zero_check(skb, proto, check, \ 251 compute_pseudo) \ 252 __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo) 253 254 #define skb_gro_checksum_simple_validate(skb) \ 255 __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo) 256 257 static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb) 258 { 259 return (NAPI_GRO_CB(skb)->csum_cnt == 0 && 260 !NAPI_GRO_CB(skb)->csum_valid); 261 } 262 263 static inline void __skb_gro_checksum_convert(struct sk_buff *skb, 264 __wsum pseudo) 265 { 266 NAPI_GRO_CB(skb)->csum = ~pseudo; 267 NAPI_GRO_CB(skb)->csum_valid = 1; 268 } 269 270 #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \ 271 do { \ 272 if (__skb_gro_checksum_convert_check(skb)) \ 273 __skb_gro_checksum_convert(skb, \ 274 compute_pseudo(skb, proto)); \ 275 } while (0) 276 277 struct gro_remcsum { 278 int offset; 279 __wsum delta; 280 }; 281 282 static inline void skb_gro_remcsum_init(struct gro_remcsum *grc) 283 { 284 grc->offset = 0; 285 grc->delta = 0; 286 } 287 288 static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, 289 unsigned int off, size_t hdrlen, 290 int start, int offset, 291 struct gro_remcsum *grc, 292 bool nopartial) 293 { 294 __wsum delta; 295 size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); 296 297 BUG_ON(!NAPI_GRO_CB(skb)->csum_valid); 298 299 if (!nopartial) { 300 NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start; 301 return ptr; 302 } 303 304 ptr = skb_gro_header_fast(skb, off); 305 if (skb_gro_header_hard(skb, off + plen)) { 306 ptr = skb_gro_header_slow(skb, off + plen, off); 307 if (!ptr) 308 return NULL; 309 } 310 311 delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum, 312 start, offset); 313 314 /* Adjust skb->csum since we changed the packet */ 315 NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta); 316 317 grc->offset = off + hdrlen + offset; 318 grc->delta = delta; 319 320 return ptr; 321 } 322 323 static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, 324 struct gro_remcsum *grc) 325 { 326 void *ptr; 327 size_t plen = grc->offset + sizeof(u16); 328 329 if (!grc->delta) 330 return; 331 332 ptr = skb_gro_header_fast(skb, grc->offset); 333 if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) { 334 ptr = skb_gro_header_slow(skb, plen, grc->offset); 335 if (!ptr) 336 return; 337 } 338 339 remcsum_unadjust((__sum16 *)ptr, grc->delta); 340 } 341 342 #ifdef CONFIG_XFRM_OFFLOAD 343 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) 344 { 345 if (PTR_ERR(pp) != -EINPROGRESS) 346 NAPI_GRO_CB(skb)->flush |= flush; 347 } 348 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, 349 struct sk_buff *pp, 350 int flush, 351 struct gro_remcsum *grc) 352 { 353 if (PTR_ERR(pp) != -EINPROGRESS) { 354 NAPI_GRO_CB(skb)->flush |= flush; 355 skb_gro_remcsum_cleanup(skb, grc); 356 skb->remcsum_offload = 0; 357 } 358 } 359 #else 360 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) 361 { 362 NAPI_GRO_CB(skb)->flush |= flush; 363 } 364 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, 365 struct sk_buff *pp, 366 int flush, 367 struct gro_remcsum *grc) 368 { 369 NAPI_GRO_CB(skb)->flush |= flush; 370 skb_gro_remcsum_cleanup(skb, grc); 371 skb->remcsum_offload = 0; 372 } 373 #endif 374 375 INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, 376 struct sk_buff *)); 377 INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int)); 378 INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, 379 struct sk_buff *)); 380 INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); 381 382 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, 383 struct sk_buff *)); 384 INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int)); 385 386 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *, 387 struct sk_buff *)); 388 INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int)); 389 390 #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \ 391 ({ \ 392 unlikely(gro_recursion_inc_test(skb)) ? \ 393 NAPI_GRO_CB(skb)->flush |= 1, NULL : \ 394 INDIRECT_CALL_INET(cb, f2, f1, head, skb); \ 395 }) 396 397 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, 398 struct udphdr *uh, struct sock *sk); 399 int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); 400 401 static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) 402 { 403 struct udphdr *uh; 404 unsigned int hlen, off; 405 406 off = skb_gro_offset(skb); 407 hlen = off + sizeof(*uh); 408 uh = skb_gro_header_fast(skb, off); 409 if (skb_gro_header_hard(skb, hlen)) 410 uh = skb_gro_header_slow(skb, hlen, off); 411 412 return uh; 413 } 414 415 static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) 416 { 417 const struct ipv6hdr *iph = skb_gro_network_header(skb); 418 419 return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr, 420 skb_gro_len(skb), proto, 0)); 421 } 422 423 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); 424 425 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ 426 static inline void gro_normal_list(struct napi_struct *napi) 427 { 428 if (!napi->rx_count) 429 return; 430 netif_receive_skb_list_internal(&napi->rx_list); 431 INIT_LIST_HEAD(&napi->rx_list); 432 napi->rx_count = 0; 433 } 434 435 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, 436 * pass the whole batch up to the stack. 437 */ 438 static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) 439 { 440 list_add_tail(&skb->list, &napi->rx_list); 441 napi->rx_count += segs; 442 if (napi->rx_count >= READ_ONCE(gro_normal_batch)) 443 gro_normal_list(napi); 444 } 445 446 447 #endif /* _NET_IPV6_GRO_H */ 448