1 // SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3 
4 #include "vmlinux.h"
5 
6 #include <bpf/bpf_helpers.h>
7 #include <bpf/bpf_endian.h>
8 #include <asm/errno.h>
9 
10 #define TC_ACT_OK 0
11 #define TC_ACT_SHOT 2
12 
13 #define NSEC_PER_SEC 1000000000L
14 
15 #define ETH_ALEN 6
16 #define ETH_P_IP 0x0800
17 #define ETH_P_IPV6 0x86DD
18 
19 #define tcp_flag_word(tp) (((union tcp_word_hdr *)(tp))->words[3])
20 
21 #define IP_DF 0x4000
22 #define IP_MF 0x2000
23 #define IP_OFFSET 0x1fff
24 
25 #define NEXTHDR_TCP 6
26 
27 #define TCPOPT_NOP 1
28 #define TCPOPT_EOL 0
29 #define TCPOPT_MSS 2
30 #define TCPOPT_WINDOW 3
31 #define TCPOPT_SACK_PERM 4
32 #define TCPOPT_TIMESTAMP 8
33 
34 #define TCPOLEN_MSS 4
35 #define TCPOLEN_WINDOW 3
36 #define TCPOLEN_SACK_PERM 2
37 #define TCPOLEN_TIMESTAMP 10
38 
39 #define TCP_TS_HZ 1000
40 #define TS_OPT_WSCALE_MASK 0xf
41 #define TS_OPT_SACK (1 << 4)
42 #define TS_OPT_ECN (1 << 5)
43 #define TSBITS 6
44 #define TSMASK (((__u32)1 << TSBITS) - 1)
45 #define TCP_MAX_WSCALE 14U
46 
47 #define IPV4_MAXLEN 60
48 #define TCP_MAXLEN 60
49 
50 #define DEFAULT_MSS4 1460
51 #define DEFAULT_MSS6 1440
52 #define DEFAULT_WSCALE 7
53 #define DEFAULT_TTL 64
54 #define MAX_ALLOWED_PORTS 8
55 
56 #define swap(a, b) \
57 	do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0)
58 
59 #define __get_unaligned_t(type, ptr) ({						\
60 	const struct { type x; } __attribute__((__packed__)) *__pptr = (typeof(__pptr))(ptr); \
61 	__pptr->x;								\
62 })
63 
64 #define get_unaligned(ptr) __get_unaligned_t(typeof(*(ptr)), (ptr))
65 
66 struct {
67 	__uint(type, BPF_MAP_TYPE_ARRAY);
68 	__type(key, __u32);
69 	__type(value, __u64);
70 	__uint(max_entries, 2);
71 } values SEC(".maps");
72 
73 struct {
74 	__uint(type, BPF_MAP_TYPE_ARRAY);
75 	__type(key, __u32);
76 	__type(value, __u16);
77 	__uint(max_entries, MAX_ALLOWED_PORTS);
78 } allowed_ports SEC(".maps");
79 
80 /* Some symbols defined in net/netfilter/nf_conntrack_bpf.c are unavailable in
81  * vmlinux.h if CONFIG_NF_CONNTRACK=m, so they are redefined locally.
82  */
83 
84 struct bpf_ct_opts___local {
85 	s32 netns_id;
86 	s32 error;
87 	u8 l4proto;
88 	u8 dir;
89 	u8 reserved[2];
90 } __attribute__((preserve_access_index));
91 
92 #define BPF_F_CURRENT_NETNS (-1)
93 
94 extern struct nf_conn *bpf_xdp_ct_lookup(struct xdp_md *xdp_ctx,
95 					 struct bpf_sock_tuple *bpf_tuple,
96 					 __u32 len_tuple,
97 					 struct bpf_ct_opts___local *opts,
98 					 __u32 len_opts) __ksym;
99 
100 extern struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *skb_ctx,
101 					 struct bpf_sock_tuple *bpf_tuple,
102 					 u32 len_tuple,
103 					 struct bpf_ct_opts___local *opts,
104 					 u32 len_opts) __ksym;
105 
106 extern void bpf_ct_release(struct nf_conn *ct) __ksym;
107 
108 static __always_inline void swap_eth_addr(__u8 *a, __u8 *b)
109 {
110 	__u8 tmp[ETH_ALEN];
111 
112 	__builtin_memcpy(tmp, a, ETH_ALEN);
113 	__builtin_memcpy(a, b, ETH_ALEN);
114 	__builtin_memcpy(b, tmp, ETH_ALEN);
115 }
116 
117 static __always_inline __u16 csum_fold(__u32 csum)
118 {
119 	csum = (csum & 0xffff) + (csum >> 16);
120 	csum = (csum & 0xffff) + (csum >> 16);
121 	return (__u16)~csum;
122 }
123 
124 static __always_inline __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
125 					       __u32 len, __u8 proto,
126 					       __u32 csum)
127 {
128 	__u64 s = csum;
129 
130 	s += (__u32)saddr;
131 	s += (__u32)daddr;
132 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
133 	s += proto + len;
134 #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
135 	s += (proto + len) << 8;
136 #else
137 #error Unknown endian
138 #endif
139 	s = (s & 0xffffffff) + (s >> 32);
140 	s = (s & 0xffffffff) + (s >> 32);
141 
142 	return csum_fold((__u32)s);
143 }
144 
145 static __always_inline __u16 csum_ipv6_magic(const struct in6_addr *saddr,
146 					     const struct in6_addr *daddr,
147 					     __u32 len, __u8 proto, __u32 csum)
148 {
149 	__u64 sum = csum;
150 	int i;
151 
152 #pragma unroll
153 	for (i = 0; i < 4; i++)
154 		sum += (__u32)saddr->in6_u.u6_addr32[i];
155 
156 #pragma unroll
157 	for (i = 0; i < 4; i++)
158 		sum += (__u32)daddr->in6_u.u6_addr32[i];
159 
160 	/* Don't combine additions to avoid 32-bit overflow. */
161 	sum += bpf_htonl(len);
162 	sum += bpf_htonl(proto);
163 
164 	sum = (sum & 0xffffffff) + (sum >> 32);
165 	sum = (sum & 0xffffffff) + (sum >> 32);
166 
167 	return csum_fold((__u32)sum);
168 }
169 
170 static __always_inline __u64 tcp_clock_ns(void)
171 {
172 	return bpf_ktime_get_ns();
173 }
174 
175 static __always_inline __u32 tcp_ns_to_ts(__u64 ns)
176 {
177 	return ns / (NSEC_PER_SEC / TCP_TS_HZ);
178 }
179 
180 static __always_inline __u32 tcp_time_stamp_raw(void)
181 {
182 	return tcp_ns_to_ts(tcp_clock_ns());
183 }
184 
185 struct tcpopt_context {
186 	__u8 *ptr;
187 	__u8 *end;
188 	void *data_end;
189 	__be32 *tsecr;
190 	__u8 wscale;
191 	bool option_timestamp;
192 	bool option_sack;
193 };
194 
195 static int tscookie_tcpopt_parse(struct tcpopt_context *ctx)
196 {
197 	__u8 opcode, opsize;
198 
199 	if (ctx->ptr >= ctx->end)
200 		return 1;
201 	if (ctx->ptr >= ctx->data_end)
202 		return 1;
203 
204 	opcode = ctx->ptr[0];
205 
206 	if (opcode == TCPOPT_EOL)
207 		return 1;
208 	if (opcode == TCPOPT_NOP) {
209 		++ctx->ptr;
210 		return 0;
211 	}
212 
213 	if (ctx->ptr + 1 >= ctx->end)
214 		return 1;
215 	if (ctx->ptr + 1 >= ctx->data_end)
216 		return 1;
217 	opsize = ctx->ptr[1];
218 	if (opsize < 2)
219 		return 1;
220 
221 	if (ctx->ptr + opsize > ctx->end)
222 		return 1;
223 
224 	switch (opcode) {
225 	case TCPOPT_WINDOW:
226 		if (opsize == TCPOLEN_WINDOW && ctx->ptr + TCPOLEN_WINDOW <= ctx->data_end)
227 			ctx->wscale = ctx->ptr[2] < TCP_MAX_WSCALE ? ctx->ptr[2] : TCP_MAX_WSCALE;
228 		break;
229 	case TCPOPT_TIMESTAMP:
230 		if (opsize == TCPOLEN_TIMESTAMP && ctx->ptr + TCPOLEN_TIMESTAMP <= ctx->data_end) {
231 			ctx->option_timestamp = true;
232 			/* Client's tsval becomes our tsecr. */
233 			*ctx->tsecr = get_unaligned((__be32 *)(ctx->ptr + 2));
234 		}
235 		break;
236 	case TCPOPT_SACK_PERM:
237 		if (opsize == TCPOLEN_SACK_PERM)
238 			ctx->option_sack = true;
239 		break;
240 	}
241 
242 	ctx->ptr += opsize;
243 
244 	return 0;
245 }
246 
247 static int tscookie_tcpopt_parse_batch(__u32 index, void *context)
248 {
249 	int i;
250 
251 	for (i = 0; i < 7; i++)
252 		if (tscookie_tcpopt_parse(context))
253 			return 1;
254 	return 0;
255 }
256 
257 static __always_inline bool tscookie_init(struct tcphdr *tcp_header,
258 					  __u16 tcp_len, __be32 *tsval,
259 					  __be32 *tsecr, void *data_end)
260 {
261 	struct tcpopt_context loop_ctx = {
262 		.ptr = (__u8 *)(tcp_header + 1),
263 		.end = (__u8 *)tcp_header + tcp_len,
264 		.data_end = data_end,
265 		.tsecr = tsecr,
266 		.wscale = TS_OPT_WSCALE_MASK,
267 		.option_timestamp = false,
268 		.option_sack = false,
269 	};
270 	u32 cookie;
271 
272 	bpf_loop(6, tscookie_tcpopt_parse_batch, &loop_ctx, 0);
273 
274 	if (!loop_ctx.option_timestamp)
275 		return false;
276 
277 	cookie = tcp_time_stamp_raw() & ~TSMASK;
278 	cookie |= loop_ctx.wscale & TS_OPT_WSCALE_MASK;
279 	if (loop_ctx.option_sack)
280 		cookie |= TS_OPT_SACK;
281 	if (tcp_header->ece && tcp_header->cwr)
282 		cookie |= TS_OPT_ECN;
283 	*tsval = bpf_htonl(cookie);
284 
285 	return true;
286 }
287 
288 static __always_inline void values_get_tcpipopts(__u16 *mss, __u8 *wscale,
289 						 __u8 *ttl, bool ipv6)
290 {
291 	__u32 key = 0;
292 	__u64 *value;
293 
294 	value = bpf_map_lookup_elem(&values, &key);
295 	if (value && *value != 0) {
296 		if (ipv6)
297 			*mss = (*value >> 32) & 0xffff;
298 		else
299 			*mss = *value & 0xffff;
300 		*wscale = (*value >> 16) & 0xf;
301 		*ttl = (*value >> 24) & 0xff;
302 		return;
303 	}
304 
305 	*mss = ipv6 ? DEFAULT_MSS6 : DEFAULT_MSS4;
306 	*wscale = DEFAULT_WSCALE;
307 	*ttl = DEFAULT_TTL;
308 }
309 
310 static __always_inline void values_inc_synacks(void)
311 {
312 	__u32 key = 1;
313 	__u32 *value;
314 
315 	value = bpf_map_lookup_elem(&values, &key);
316 	if (value)
317 		__sync_fetch_and_add(value, 1);
318 }
319 
320 static __always_inline bool check_port_allowed(__u16 port)
321 {
322 	__u32 i;
323 
324 	for (i = 0; i < MAX_ALLOWED_PORTS; i++) {
325 		__u32 key = i;
326 		__u16 *value;
327 
328 		value = bpf_map_lookup_elem(&allowed_ports, &key);
329 
330 		if (!value)
331 			break;
332 		/* 0 is a terminator value. Check it first to avoid matching on
333 		 * a forbidden port == 0 and returning true.
334 		 */
335 		if (*value == 0)
336 			break;
337 
338 		if (*value == port)
339 			return true;
340 	}
341 
342 	return false;
343 }
344 
345 struct header_pointers {
346 	struct ethhdr *eth;
347 	struct iphdr *ipv4;
348 	struct ipv6hdr *ipv6;
349 	struct tcphdr *tcp;
350 	__u16 tcp_len;
351 };
352 
353 static __always_inline int tcp_dissect(void *data, void *data_end,
354 				       struct header_pointers *hdr)
355 {
356 	hdr->eth = data;
357 	if (hdr->eth + 1 > data_end)
358 		return XDP_DROP;
359 
360 	switch (bpf_ntohs(hdr->eth->h_proto)) {
361 	case ETH_P_IP:
362 		hdr->ipv6 = NULL;
363 
364 		hdr->ipv4 = (void *)hdr->eth + sizeof(*hdr->eth);
365 		if (hdr->ipv4 + 1 > data_end)
366 			return XDP_DROP;
367 		if (hdr->ipv4->ihl * 4 < sizeof(*hdr->ipv4))
368 			return XDP_DROP;
369 		if (hdr->ipv4->version != 4)
370 			return XDP_DROP;
371 
372 		if (hdr->ipv4->protocol != IPPROTO_TCP)
373 			return XDP_PASS;
374 
375 		hdr->tcp = (void *)hdr->ipv4 + hdr->ipv4->ihl * 4;
376 		break;
377 	case ETH_P_IPV6:
378 		hdr->ipv4 = NULL;
379 
380 		hdr->ipv6 = (void *)hdr->eth + sizeof(*hdr->eth);
381 		if (hdr->ipv6 + 1 > data_end)
382 			return XDP_DROP;
383 		if (hdr->ipv6->version != 6)
384 			return XDP_DROP;
385 
386 		/* XXX: Extension headers are not supported and could circumvent
387 		 * XDP SYN flood protection.
388 		 */
389 		if (hdr->ipv6->nexthdr != NEXTHDR_TCP)
390 			return XDP_PASS;
391 
392 		hdr->tcp = (void *)hdr->ipv6 + sizeof(*hdr->ipv6);
393 		break;
394 	default:
395 		/* XXX: VLANs will circumvent XDP SYN flood protection. */
396 		return XDP_PASS;
397 	}
398 
399 	if (hdr->tcp + 1 > data_end)
400 		return XDP_DROP;
401 	hdr->tcp_len = hdr->tcp->doff * 4;
402 	if (hdr->tcp_len < sizeof(*hdr->tcp))
403 		return XDP_DROP;
404 
405 	return XDP_TX;
406 }
407 
408 static __always_inline int tcp_lookup(void *ctx, struct header_pointers *hdr, bool xdp)
409 {
410 	struct bpf_ct_opts___local ct_lookup_opts = {
411 		.netns_id = BPF_F_CURRENT_NETNS,
412 		.l4proto = IPPROTO_TCP,
413 	};
414 	struct bpf_sock_tuple tup = {};
415 	struct nf_conn *ct;
416 	__u32 tup_size;
417 
418 	if (hdr->ipv4) {
419 		/* TCP doesn't normally use fragments, and XDP can't reassemble
420 		 * them.
421 		 */
422 		if ((hdr->ipv4->frag_off & bpf_htons(IP_DF | IP_MF | IP_OFFSET)) != bpf_htons(IP_DF))
423 			return XDP_DROP;
424 
425 		tup.ipv4.saddr = hdr->ipv4->saddr;
426 		tup.ipv4.daddr = hdr->ipv4->daddr;
427 		tup.ipv4.sport = hdr->tcp->source;
428 		tup.ipv4.dport = hdr->tcp->dest;
429 		tup_size = sizeof(tup.ipv4);
430 	} else if (hdr->ipv6) {
431 		__builtin_memcpy(tup.ipv6.saddr, &hdr->ipv6->saddr, sizeof(tup.ipv6.saddr));
432 		__builtin_memcpy(tup.ipv6.daddr, &hdr->ipv6->daddr, sizeof(tup.ipv6.daddr));
433 		tup.ipv6.sport = hdr->tcp->source;
434 		tup.ipv6.dport = hdr->tcp->dest;
435 		tup_size = sizeof(tup.ipv6);
436 	} else {
437 		/* The verifier can't track that either ipv4 or ipv6 is not
438 		 * NULL.
439 		 */
440 		return XDP_ABORTED;
441 	}
442 	if (xdp)
443 		ct = bpf_xdp_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));
444 	else
445 		ct = bpf_skb_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));
446 	if (ct) {
447 		unsigned long status = ct->status;
448 
449 		bpf_ct_release(ct);
450 		if (status & IPS_CONFIRMED_BIT)
451 			return XDP_PASS;
452 	} else if (ct_lookup_opts.error != -ENOENT) {
453 		return XDP_ABORTED;
454 	}
455 
456 	/* error == -ENOENT || !(status & IPS_CONFIRMED_BIT) */
457 	return XDP_TX;
458 }
459 
460 static __always_inline __u8 tcp_mkoptions(__be32 *buf, __be32 *tsopt, __u16 mss,
461 					  __u8 wscale)
462 {
463 	__be32 *start = buf;
464 
465 	*buf++ = bpf_htonl((TCPOPT_MSS << 24) | (TCPOLEN_MSS << 16) | mss);
466 
467 	if (!tsopt)
468 		return buf - start;
469 
470 	if (tsopt[0] & bpf_htonl(1 << 4))
471 		*buf++ = bpf_htonl((TCPOPT_SACK_PERM << 24) |
472 				   (TCPOLEN_SACK_PERM << 16) |
473 				   (TCPOPT_TIMESTAMP << 8) |
474 				   TCPOLEN_TIMESTAMP);
475 	else
476 		*buf++ = bpf_htonl((TCPOPT_NOP << 24) |
477 				   (TCPOPT_NOP << 16) |
478 				   (TCPOPT_TIMESTAMP << 8) |
479 				   TCPOLEN_TIMESTAMP);
480 	*buf++ = tsopt[0];
481 	*buf++ = tsopt[1];
482 
483 	if ((tsopt[0] & bpf_htonl(0xf)) != bpf_htonl(0xf))
484 		*buf++ = bpf_htonl((TCPOPT_NOP << 24) |
485 				   (TCPOPT_WINDOW << 16) |
486 				   (TCPOLEN_WINDOW << 8) |
487 				   wscale);
488 
489 	return buf - start;
490 }
491 
492 static __always_inline void tcp_gen_synack(struct tcphdr *tcp_header,
493 					   __u32 cookie, __be32 *tsopt,
494 					   __u16 mss, __u8 wscale)
495 {
496 	void *tcp_options;
497 
498 	tcp_flag_word(tcp_header) = TCP_FLAG_SYN | TCP_FLAG_ACK;
499 	if (tsopt && (tsopt[0] & bpf_htonl(1 << 5)))
500 		tcp_flag_word(tcp_header) |= TCP_FLAG_ECE;
501 	tcp_header->doff = 5; /* doff is part of tcp_flag_word. */
502 	swap(tcp_header->source, tcp_header->dest);
503 	tcp_header->ack_seq = bpf_htonl(bpf_ntohl(tcp_header->seq) + 1);
504 	tcp_header->seq = bpf_htonl(cookie);
505 	tcp_header->window = 0;
506 	tcp_header->urg_ptr = 0;
507 	tcp_header->check = 0; /* Calculate checksum later. */
508 
509 	tcp_options = (void *)(tcp_header + 1);
510 	tcp_header->doff += tcp_mkoptions(tcp_options, tsopt, mss, wscale);
511 }
512 
513 static __always_inline void tcpv4_gen_synack(struct header_pointers *hdr,
514 					     __u32 cookie, __be32 *tsopt)
515 {
516 	__u8 wscale;
517 	__u16 mss;
518 	__u8 ttl;
519 
520 	values_get_tcpipopts(&mss, &wscale, &ttl, false);
521 
522 	swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);
523 
524 	swap(hdr->ipv4->saddr, hdr->ipv4->daddr);
525 	hdr->ipv4->check = 0; /* Calculate checksum later. */
526 	hdr->ipv4->tos = 0;
527 	hdr->ipv4->id = 0;
528 	hdr->ipv4->ttl = ttl;
529 
530 	tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);
531 
532 	hdr->tcp_len = hdr->tcp->doff * 4;
533 	hdr->ipv4->tot_len = bpf_htons(sizeof(*hdr->ipv4) + hdr->tcp_len);
534 }
535 
536 static __always_inline void tcpv6_gen_synack(struct header_pointers *hdr,
537 					     __u32 cookie, __be32 *tsopt)
538 {
539 	__u8 wscale;
540 	__u16 mss;
541 	__u8 ttl;
542 
543 	values_get_tcpipopts(&mss, &wscale, &ttl, true);
544 
545 	swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);
546 
547 	swap(hdr->ipv6->saddr, hdr->ipv6->daddr);
548 	*(__be32 *)hdr->ipv6 = bpf_htonl(0x60000000);
549 	hdr->ipv6->hop_limit = ttl;
550 
551 	tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);
552 
553 	hdr->tcp_len = hdr->tcp->doff * 4;
554 	hdr->ipv6->payload_len = bpf_htons(hdr->tcp_len);
555 }
556 
557 static __always_inline int syncookie_handle_syn(struct header_pointers *hdr,
558 						void *ctx,
559 						void *data, void *data_end,
560 						bool xdp)
561 {
562 	__u32 old_pkt_size, new_pkt_size;
563 	/* Unlike clang 10, clang 11 and 12 generate code that doesn't pass the
564 	 * BPF verifier if tsopt is not volatile. Volatile forces it to store
565 	 * the pointer value and use it directly, otherwise tcp_mkoptions is
566 	 * (mis)compiled like this:
567 	 *   if (!tsopt)
568 	 *       return buf - start;
569 	 *   reg = stored_return_value_of_tscookie_init;
570 	 *   if (reg)
571 	 *       tsopt = tsopt_buf;
572 	 *   else
573 	 *       tsopt = NULL;
574 	 *   ...
575 	 *   *buf++ = tsopt[1];
576 	 * It creates a dead branch where tsopt is assigned NULL, but the
577 	 * verifier can't prove it's dead and blocks the program.
578 	 */
579 	__be32 * volatile tsopt = NULL;
580 	__be32 tsopt_buf[2] = {};
581 	__u16 ip_len;
582 	__u32 cookie;
583 	__s64 value;
584 
585 	/* Checksum is not yet verified, but both checksum failure and TCP
586 	 * header checks return XDP_DROP, so the order doesn't matter.
587 	 */
588 	if (hdr->tcp->fin || hdr->tcp->rst)
589 		return XDP_DROP;
590 
591 	/* Issue SYN cookies on allowed ports, drop SYN packets on blocked
592 	 * ports.
593 	 */
594 	if (!check_port_allowed(bpf_ntohs(hdr->tcp->dest)))
595 		return XDP_DROP;
596 
597 	if (hdr->ipv4) {
598 		/* Check the IPv4 and TCP checksums before creating a SYNACK. */
599 		value = bpf_csum_diff(0, 0, (void *)hdr->ipv4, hdr->ipv4->ihl * 4, 0);
600 		if (value < 0)
601 			return XDP_ABORTED;
602 		if (csum_fold(value) != 0)
603 			return XDP_DROP; /* Bad IPv4 checksum. */
604 
605 		value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
606 		if (value < 0)
607 			return XDP_ABORTED;
608 		if (csum_tcpudp_magic(hdr->ipv4->saddr, hdr->ipv4->daddr,
609 				      hdr->tcp_len, IPPROTO_TCP, value) != 0)
610 			return XDP_DROP; /* Bad TCP checksum. */
611 
612 		ip_len = sizeof(*hdr->ipv4);
613 
614 		value = bpf_tcp_raw_gen_syncookie_ipv4(hdr->ipv4, hdr->tcp,
615 						       hdr->tcp_len);
616 	} else if (hdr->ipv6) {
617 		/* Check the TCP checksum before creating a SYNACK. */
618 		value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
619 		if (value < 0)
620 			return XDP_ABORTED;
621 		if (csum_ipv6_magic(&hdr->ipv6->saddr, &hdr->ipv6->daddr,
622 				    hdr->tcp_len, IPPROTO_TCP, value) != 0)
623 			return XDP_DROP; /* Bad TCP checksum. */
624 
625 		ip_len = sizeof(*hdr->ipv6);
626 
627 		value = bpf_tcp_raw_gen_syncookie_ipv6(hdr->ipv6, hdr->tcp,
628 						       hdr->tcp_len);
629 	} else {
630 		return XDP_ABORTED;
631 	}
632 
633 	if (value < 0)
634 		return XDP_ABORTED;
635 	cookie = (__u32)value;
636 
637 	if (tscookie_init((void *)hdr->tcp, hdr->tcp_len,
638 			  &tsopt_buf[0], &tsopt_buf[1], data_end))
639 		tsopt = tsopt_buf;
640 
641 	/* Check that there is enough space for a SYNACK. It also covers
642 	 * the check that the destination of the __builtin_memmove below
643 	 * doesn't overflow.
644 	 */
645 	if (data + sizeof(*hdr->eth) + ip_len + TCP_MAXLEN > data_end)
646 		return XDP_ABORTED;
647 
648 	if (hdr->ipv4) {
649 		if (hdr->ipv4->ihl * 4 > sizeof(*hdr->ipv4)) {
650 			struct tcphdr *new_tcp_header;
651 
652 			new_tcp_header = data + sizeof(*hdr->eth) + sizeof(*hdr->ipv4);
653 			__builtin_memmove(new_tcp_header, hdr->tcp, sizeof(*hdr->tcp));
654 			hdr->tcp = new_tcp_header;
655 
656 			hdr->ipv4->ihl = sizeof(*hdr->ipv4) / 4;
657 		}
658 
659 		tcpv4_gen_synack(hdr, cookie, tsopt);
660 	} else if (hdr->ipv6) {
661 		tcpv6_gen_synack(hdr, cookie, tsopt);
662 	} else {
663 		return XDP_ABORTED;
664 	}
665 
666 	/* Recalculate checksums. */
667 	hdr->tcp->check = 0;
668 	value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
669 	if (value < 0)
670 		return XDP_ABORTED;
671 	if (hdr->ipv4) {
672 		hdr->tcp->check = csum_tcpudp_magic(hdr->ipv4->saddr,
673 						    hdr->ipv4->daddr,
674 						    hdr->tcp_len,
675 						    IPPROTO_TCP,
676 						    value);
677 
678 		hdr->ipv4->check = 0;
679 		value = bpf_csum_diff(0, 0, (void *)hdr->ipv4, sizeof(*hdr->ipv4), 0);
680 		if (value < 0)
681 			return XDP_ABORTED;
682 		hdr->ipv4->check = csum_fold(value);
683 	} else if (hdr->ipv6) {
684 		hdr->tcp->check = csum_ipv6_magic(&hdr->ipv6->saddr,
685 						  &hdr->ipv6->daddr,
686 						  hdr->tcp_len,
687 						  IPPROTO_TCP,
688 						  value);
689 	} else {
690 		return XDP_ABORTED;
691 	}
692 
693 	/* Set the new packet size. */
694 	old_pkt_size = data_end - data;
695 	new_pkt_size = sizeof(*hdr->eth) + ip_len + hdr->tcp->doff * 4;
696 	if (xdp) {
697 		if (bpf_xdp_adjust_tail(ctx, new_pkt_size - old_pkt_size))
698 			return XDP_ABORTED;
699 	} else {
700 		if (bpf_skb_change_tail(ctx, new_pkt_size, 0))
701 			return XDP_ABORTED;
702 	}
703 
704 	values_inc_synacks();
705 
706 	return XDP_TX;
707 }
708 
709 static __always_inline int syncookie_handle_ack(struct header_pointers *hdr)
710 {
711 	int err;
712 
713 	if (hdr->tcp->rst)
714 		return XDP_DROP;
715 
716 	if (hdr->ipv4)
717 		err = bpf_tcp_raw_check_syncookie_ipv4(hdr->ipv4, hdr->tcp);
718 	else if (hdr->ipv6)
719 		err = bpf_tcp_raw_check_syncookie_ipv6(hdr->ipv6, hdr->tcp);
720 	else
721 		return XDP_ABORTED;
722 	if (err)
723 		return XDP_DROP;
724 
725 	return XDP_PASS;
726 }
727 
728 static __always_inline int syncookie_part1(void *ctx, void *data, void *data_end,
729 					   struct header_pointers *hdr, bool xdp)
730 {
731 	int ret;
732 
733 	ret = tcp_dissect(data, data_end, hdr);
734 	if (ret != XDP_TX)
735 		return ret;
736 
737 	ret = tcp_lookup(ctx, hdr, xdp);
738 	if (ret != XDP_TX)
739 		return ret;
740 
741 	/* Packet is TCP and doesn't belong to an established connection. */
742 
743 	if ((hdr->tcp->syn ^ hdr->tcp->ack) != 1)
744 		return XDP_DROP;
745 
746 	/* Grow the TCP header to TCP_MAXLEN to be able to pass any hdr->tcp_len
747 	 * to bpf_tcp_raw_gen_syncookie_ipv{4,6} and pass the verifier.
748 	 */
749 	if (xdp) {
750 		if (bpf_xdp_adjust_tail(ctx, TCP_MAXLEN - hdr->tcp_len))
751 			return XDP_ABORTED;
752 	} else {
753 		/* Without volatile the verifier throws this error:
754 		 * R9 32-bit pointer arithmetic prohibited
755 		 */
756 		volatile u64 old_len = data_end - data;
757 
758 		if (bpf_skb_change_tail(ctx, old_len + TCP_MAXLEN - hdr->tcp_len, 0))
759 			return XDP_ABORTED;
760 	}
761 
762 	return XDP_TX;
763 }
764 
765 static __always_inline int syncookie_part2(void *ctx, void *data, void *data_end,
766 					   struct header_pointers *hdr, bool xdp)
767 {
768 	if (hdr->ipv4) {
769 		hdr->eth = data;
770 		hdr->ipv4 = (void *)hdr->eth + sizeof(*hdr->eth);
771 		/* IPV4_MAXLEN is needed when calculating checksum.
772 		 * At least sizeof(struct iphdr) is needed here to access ihl.
773 		 */
774 		if ((void *)hdr->ipv4 + IPV4_MAXLEN > data_end)
775 			return XDP_ABORTED;
776 		hdr->tcp = (void *)hdr->ipv4 + hdr->ipv4->ihl * 4;
777 	} else if (hdr->ipv6) {
778 		hdr->eth = data;
779 		hdr->ipv6 = (void *)hdr->eth + sizeof(*hdr->eth);
780 		hdr->tcp = (void *)hdr->ipv6 + sizeof(*hdr->ipv6);
781 	} else {
782 		return XDP_ABORTED;
783 	}
784 
785 	if ((void *)hdr->tcp + TCP_MAXLEN > data_end)
786 		return XDP_ABORTED;
787 
788 	/* We run out of registers, tcp_len gets spilled to the stack, and the
789 	 * verifier forgets its min and max values checked above in tcp_dissect.
790 	 */
791 	hdr->tcp_len = hdr->tcp->doff * 4;
792 	if (hdr->tcp_len < sizeof(*hdr->tcp))
793 		return XDP_ABORTED;
794 
795 	return hdr->tcp->syn ? syncookie_handle_syn(hdr, ctx, data, data_end, xdp) :
796 			       syncookie_handle_ack(hdr);
797 }
798 
799 SEC("xdp")
800 int syncookie_xdp(struct xdp_md *ctx)
801 {
802 	void *data_end = (void *)(long)ctx->data_end;
803 	void *data = (void *)(long)ctx->data;
804 	struct header_pointers hdr;
805 	int ret;
806 
807 	ret = syncookie_part1(ctx, data, data_end, &hdr, true);
808 	if (ret != XDP_TX)
809 		return ret;
810 
811 	data_end = (void *)(long)ctx->data_end;
812 	data = (void *)(long)ctx->data;
813 
814 	return syncookie_part2(ctx, data, data_end, &hdr, true);
815 }
816 
817 SEC("tc")
818 int syncookie_tc(struct __sk_buff *skb)
819 {
820 	void *data_end = (void *)(long)skb->data_end;
821 	void *data = (void *)(long)skb->data;
822 	struct header_pointers hdr;
823 	int ret;
824 
825 	ret = syncookie_part1(skb, data, data_end, &hdr, false);
826 	if (ret != XDP_TX)
827 		return ret == XDP_PASS ? TC_ACT_OK : TC_ACT_SHOT;
828 
829 	data_end = (void *)(long)skb->data_end;
830 	data = (void *)(long)skb->data;
831 
832 	ret = syncookie_part2(skb, data, data_end, &hdr, false);
833 	switch (ret) {
834 	case XDP_PASS:
835 		return TC_ACT_OK;
836 	case XDP_TX:
837 		return bpf_redirect(skb->ifindex, 0);
838 	default:
839 		return TC_ACT_SHOT;
840 	}
841 }
842 
843 char _license[] SEC("license") = "GPL";
844