1*4dba3e78SLorenzo Bianconi // SPDX-License-Identifier: GPL-2.0
2*4dba3e78SLorenzo Bianconi
3*4dba3e78SLorenzo Bianconi #include <stdbool.h>
4*4dba3e78SLorenzo Bianconi #include <linux/bpf.h>
5*4dba3e78SLorenzo Bianconi #include <linux/netdev.h>
6*4dba3e78SLorenzo Bianconi #include <bpf/bpf_helpers.h>
7*4dba3e78SLorenzo Bianconi #include <bpf/bpf_endian.h>
8*4dba3e78SLorenzo Bianconi #include <bpf/bpf_tracing.h>
9*4dba3e78SLorenzo Bianconi #include <linux/if_ether.h>
10*4dba3e78SLorenzo Bianconi #include <linux/ip.h>
11*4dba3e78SLorenzo Bianconi #include <linux/ipv6.h>
12*4dba3e78SLorenzo Bianconi #include <linux/in.h>
13*4dba3e78SLorenzo Bianconi #include <linux/in6.h>
14*4dba3e78SLorenzo Bianconi #include <linux/udp.h>
15*4dba3e78SLorenzo Bianconi #include <asm-generic/errno-base.h>
16*4dba3e78SLorenzo Bianconi
17*4dba3e78SLorenzo Bianconi #include "xdp_features.h"
18*4dba3e78SLorenzo Bianconi
19*4dba3e78SLorenzo Bianconi #define ipv6_addr_equal(a, b) ((a).s6_addr32[0] == (b).s6_addr32[0] && \
20*4dba3e78SLorenzo Bianconi (a).s6_addr32[1] == (b).s6_addr32[1] && \
21*4dba3e78SLorenzo Bianconi (a).s6_addr32[2] == (b).s6_addr32[2] && \
22*4dba3e78SLorenzo Bianconi (a).s6_addr32[3] == (b).s6_addr32[3])
23*4dba3e78SLorenzo Bianconi
24*4dba3e78SLorenzo Bianconi struct net_device;
25*4dba3e78SLorenzo Bianconi struct bpf_prog;
26*4dba3e78SLorenzo Bianconi
27*4dba3e78SLorenzo Bianconi struct xdp_cpumap_stats {
28*4dba3e78SLorenzo Bianconi unsigned int redirect;
29*4dba3e78SLorenzo Bianconi unsigned int pass;
30*4dba3e78SLorenzo Bianconi unsigned int drop;
31*4dba3e78SLorenzo Bianconi };
32*4dba3e78SLorenzo Bianconi
33*4dba3e78SLorenzo Bianconi struct {
34*4dba3e78SLorenzo Bianconi __uint(type, BPF_MAP_TYPE_ARRAY);
35*4dba3e78SLorenzo Bianconi __type(key, __u32);
36*4dba3e78SLorenzo Bianconi __type(value, __u32);
37*4dba3e78SLorenzo Bianconi __uint(max_entries, 1);
38*4dba3e78SLorenzo Bianconi } stats SEC(".maps");
39*4dba3e78SLorenzo Bianconi
40*4dba3e78SLorenzo Bianconi struct {
41*4dba3e78SLorenzo Bianconi __uint(type, BPF_MAP_TYPE_ARRAY);
42*4dba3e78SLorenzo Bianconi __type(key, __u32);
43*4dba3e78SLorenzo Bianconi __type(value, __u32);
44*4dba3e78SLorenzo Bianconi __uint(max_entries, 1);
45*4dba3e78SLorenzo Bianconi } dut_stats SEC(".maps");
46*4dba3e78SLorenzo Bianconi
47*4dba3e78SLorenzo Bianconi struct {
48*4dba3e78SLorenzo Bianconi __uint(type, BPF_MAP_TYPE_CPUMAP);
49*4dba3e78SLorenzo Bianconi __uint(key_size, sizeof(__u32));
50*4dba3e78SLorenzo Bianconi __uint(value_size, sizeof(struct bpf_cpumap_val));
51*4dba3e78SLorenzo Bianconi __uint(max_entries, 1);
52*4dba3e78SLorenzo Bianconi } cpu_map SEC(".maps");
53*4dba3e78SLorenzo Bianconi
54*4dba3e78SLorenzo Bianconi struct {
55*4dba3e78SLorenzo Bianconi __uint(type, BPF_MAP_TYPE_DEVMAP);
56*4dba3e78SLorenzo Bianconi __uint(key_size, sizeof(__u32));
57*4dba3e78SLorenzo Bianconi __uint(value_size, sizeof(struct bpf_devmap_val));
58*4dba3e78SLorenzo Bianconi __uint(max_entries, 1);
59*4dba3e78SLorenzo Bianconi } dev_map SEC(".maps");
60*4dba3e78SLorenzo Bianconi
61*4dba3e78SLorenzo Bianconi const volatile struct in6_addr tester_addr;
62*4dba3e78SLorenzo Bianconi const volatile struct in6_addr dut_addr;
63*4dba3e78SLorenzo Bianconi
64*4dba3e78SLorenzo Bianconi static __always_inline int
xdp_process_echo_packet(struct xdp_md * xdp,bool dut)65*4dba3e78SLorenzo Bianconi xdp_process_echo_packet(struct xdp_md *xdp, bool dut)
66*4dba3e78SLorenzo Bianconi {
67*4dba3e78SLorenzo Bianconi void *data_end = (void *)(long)xdp->data_end;
68*4dba3e78SLorenzo Bianconi void *data = (void *)(long)xdp->data;
69*4dba3e78SLorenzo Bianconi struct ethhdr *eh = data;
70*4dba3e78SLorenzo Bianconi struct tlv_hdr *tlv;
71*4dba3e78SLorenzo Bianconi struct udphdr *uh;
72*4dba3e78SLorenzo Bianconi __be16 port;
73*4dba3e78SLorenzo Bianconi
74*4dba3e78SLorenzo Bianconi if (eh + 1 > (struct ethhdr *)data_end)
75*4dba3e78SLorenzo Bianconi return -EINVAL;
76*4dba3e78SLorenzo Bianconi
77*4dba3e78SLorenzo Bianconi if (eh->h_proto == bpf_htons(ETH_P_IP)) {
78*4dba3e78SLorenzo Bianconi struct iphdr *ih = (struct iphdr *)(eh + 1);
79*4dba3e78SLorenzo Bianconi __be32 saddr = dut ? tester_addr.s6_addr32[3]
80*4dba3e78SLorenzo Bianconi : dut_addr.s6_addr32[3];
81*4dba3e78SLorenzo Bianconi __be32 daddr = dut ? dut_addr.s6_addr32[3]
82*4dba3e78SLorenzo Bianconi : tester_addr.s6_addr32[3];
83*4dba3e78SLorenzo Bianconi
84*4dba3e78SLorenzo Bianconi ih = (struct iphdr *)(eh + 1);
85*4dba3e78SLorenzo Bianconi if (ih + 1 > (struct iphdr *)data_end)
86*4dba3e78SLorenzo Bianconi return -EINVAL;
87*4dba3e78SLorenzo Bianconi
88*4dba3e78SLorenzo Bianconi if (saddr != ih->saddr)
89*4dba3e78SLorenzo Bianconi return -EINVAL;
90*4dba3e78SLorenzo Bianconi
91*4dba3e78SLorenzo Bianconi if (daddr != ih->daddr)
92*4dba3e78SLorenzo Bianconi return -EINVAL;
93*4dba3e78SLorenzo Bianconi
94*4dba3e78SLorenzo Bianconi if (ih->protocol != IPPROTO_UDP)
95*4dba3e78SLorenzo Bianconi return -EINVAL;
96*4dba3e78SLorenzo Bianconi
97*4dba3e78SLorenzo Bianconi uh = (struct udphdr *)(ih + 1);
98*4dba3e78SLorenzo Bianconi } else if (eh->h_proto == bpf_htons(ETH_P_IPV6)) {
99*4dba3e78SLorenzo Bianconi struct in6_addr saddr = dut ? tester_addr : dut_addr;
100*4dba3e78SLorenzo Bianconi struct in6_addr daddr = dut ? dut_addr : tester_addr;
101*4dba3e78SLorenzo Bianconi struct ipv6hdr *ih6 = (struct ipv6hdr *)(eh + 1);
102*4dba3e78SLorenzo Bianconi
103*4dba3e78SLorenzo Bianconi if (ih6 + 1 > (struct ipv6hdr *)data_end)
104*4dba3e78SLorenzo Bianconi return -EINVAL;
105*4dba3e78SLorenzo Bianconi
106*4dba3e78SLorenzo Bianconi if (!ipv6_addr_equal(saddr, ih6->saddr))
107*4dba3e78SLorenzo Bianconi return -EINVAL;
108*4dba3e78SLorenzo Bianconi
109*4dba3e78SLorenzo Bianconi if (!ipv6_addr_equal(daddr, ih6->daddr))
110*4dba3e78SLorenzo Bianconi return -EINVAL;
111*4dba3e78SLorenzo Bianconi
112*4dba3e78SLorenzo Bianconi if (ih6->nexthdr != IPPROTO_UDP)
113*4dba3e78SLorenzo Bianconi return -EINVAL;
114*4dba3e78SLorenzo Bianconi
115*4dba3e78SLorenzo Bianconi uh = (struct udphdr *)(ih6 + 1);
116*4dba3e78SLorenzo Bianconi } else {
117*4dba3e78SLorenzo Bianconi return -EINVAL;
118*4dba3e78SLorenzo Bianconi }
119*4dba3e78SLorenzo Bianconi
120*4dba3e78SLorenzo Bianconi if (uh + 1 > (struct udphdr *)data_end)
121*4dba3e78SLorenzo Bianconi return -EINVAL;
122*4dba3e78SLorenzo Bianconi
123*4dba3e78SLorenzo Bianconi port = dut ? uh->dest : uh->source;
124*4dba3e78SLorenzo Bianconi if (port != bpf_htons(DUT_ECHO_PORT))
125*4dba3e78SLorenzo Bianconi return -EINVAL;
126*4dba3e78SLorenzo Bianconi
127*4dba3e78SLorenzo Bianconi tlv = (struct tlv_hdr *)(uh + 1);
128*4dba3e78SLorenzo Bianconi if (tlv + 1 > data_end)
129*4dba3e78SLorenzo Bianconi return -EINVAL;
130*4dba3e78SLorenzo Bianconi
131*4dba3e78SLorenzo Bianconi return bpf_htons(tlv->type) == CMD_ECHO ? 0 : -EINVAL;
132*4dba3e78SLorenzo Bianconi }
133*4dba3e78SLorenzo Bianconi
134*4dba3e78SLorenzo Bianconi static __always_inline int
xdp_update_stats(struct xdp_md * xdp,bool tx,bool dut)135*4dba3e78SLorenzo Bianconi xdp_update_stats(struct xdp_md *xdp, bool tx, bool dut)
136*4dba3e78SLorenzo Bianconi {
137*4dba3e78SLorenzo Bianconi __u32 *val, key = 0;
138*4dba3e78SLorenzo Bianconi
139*4dba3e78SLorenzo Bianconi if (xdp_process_echo_packet(xdp, tx))
140*4dba3e78SLorenzo Bianconi return -EINVAL;
141*4dba3e78SLorenzo Bianconi
142*4dba3e78SLorenzo Bianconi if (dut)
143*4dba3e78SLorenzo Bianconi val = bpf_map_lookup_elem(&dut_stats, &key);
144*4dba3e78SLorenzo Bianconi else
145*4dba3e78SLorenzo Bianconi val = bpf_map_lookup_elem(&stats, &key);
146*4dba3e78SLorenzo Bianconi
147*4dba3e78SLorenzo Bianconi if (val)
148*4dba3e78SLorenzo Bianconi __sync_add_and_fetch(val, 1);
149*4dba3e78SLorenzo Bianconi
150*4dba3e78SLorenzo Bianconi return 0;
151*4dba3e78SLorenzo Bianconi }
152*4dba3e78SLorenzo Bianconi
153*4dba3e78SLorenzo Bianconi /* Tester */
154*4dba3e78SLorenzo Bianconi
155*4dba3e78SLorenzo Bianconi SEC("xdp")
xdp_tester_check_tx(struct xdp_md * xdp)156*4dba3e78SLorenzo Bianconi int xdp_tester_check_tx(struct xdp_md *xdp)
157*4dba3e78SLorenzo Bianconi {
158*4dba3e78SLorenzo Bianconi xdp_update_stats(xdp, true, false);
159*4dba3e78SLorenzo Bianconi
160*4dba3e78SLorenzo Bianconi return XDP_PASS;
161*4dba3e78SLorenzo Bianconi }
162*4dba3e78SLorenzo Bianconi
163*4dba3e78SLorenzo Bianconi SEC("xdp")
xdp_tester_check_rx(struct xdp_md * xdp)164*4dba3e78SLorenzo Bianconi int xdp_tester_check_rx(struct xdp_md *xdp)
165*4dba3e78SLorenzo Bianconi {
166*4dba3e78SLorenzo Bianconi xdp_update_stats(xdp, false, false);
167*4dba3e78SLorenzo Bianconi
168*4dba3e78SLorenzo Bianconi return XDP_PASS;
169*4dba3e78SLorenzo Bianconi }
170*4dba3e78SLorenzo Bianconi
171*4dba3e78SLorenzo Bianconi /* DUT */
172*4dba3e78SLorenzo Bianconi
173*4dba3e78SLorenzo Bianconi SEC("xdp")
xdp_do_pass(struct xdp_md * xdp)174*4dba3e78SLorenzo Bianconi int xdp_do_pass(struct xdp_md *xdp)
175*4dba3e78SLorenzo Bianconi {
176*4dba3e78SLorenzo Bianconi xdp_update_stats(xdp, true, true);
177*4dba3e78SLorenzo Bianconi
178*4dba3e78SLorenzo Bianconi return XDP_PASS;
179*4dba3e78SLorenzo Bianconi }
180*4dba3e78SLorenzo Bianconi
181*4dba3e78SLorenzo Bianconi SEC("xdp")
xdp_do_drop(struct xdp_md * xdp)182*4dba3e78SLorenzo Bianconi int xdp_do_drop(struct xdp_md *xdp)
183*4dba3e78SLorenzo Bianconi {
184*4dba3e78SLorenzo Bianconi if (xdp_update_stats(xdp, true, true))
185*4dba3e78SLorenzo Bianconi return XDP_PASS;
186*4dba3e78SLorenzo Bianconi
187*4dba3e78SLorenzo Bianconi return XDP_DROP;
188*4dba3e78SLorenzo Bianconi }
189*4dba3e78SLorenzo Bianconi
190*4dba3e78SLorenzo Bianconi SEC("xdp")
xdp_do_aborted(struct xdp_md * xdp)191*4dba3e78SLorenzo Bianconi int xdp_do_aborted(struct xdp_md *xdp)
192*4dba3e78SLorenzo Bianconi {
193*4dba3e78SLorenzo Bianconi if (xdp_process_echo_packet(xdp, true))
194*4dba3e78SLorenzo Bianconi return XDP_PASS;
195*4dba3e78SLorenzo Bianconi
196*4dba3e78SLorenzo Bianconi return XDP_ABORTED;
197*4dba3e78SLorenzo Bianconi }
198*4dba3e78SLorenzo Bianconi
199*4dba3e78SLorenzo Bianconi SEC("xdp")
xdp_do_tx(struct xdp_md * xdp)200*4dba3e78SLorenzo Bianconi int xdp_do_tx(struct xdp_md *xdp)
201*4dba3e78SLorenzo Bianconi {
202*4dba3e78SLorenzo Bianconi void *data = (void *)(long)xdp->data;
203*4dba3e78SLorenzo Bianconi struct ethhdr *eh = data;
204*4dba3e78SLorenzo Bianconi __u8 tmp_mac[ETH_ALEN];
205*4dba3e78SLorenzo Bianconi
206*4dba3e78SLorenzo Bianconi if (xdp_update_stats(xdp, true, true))
207*4dba3e78SLorenzo Bianconi return XDP_PASS;
208*4dba3e78SLorenzo Bianconi
209*4dba3e78SLorenzo Bianconi __builtin_memcpy(tmp_mac, eh->h_source, ETH_ALEN);
210*4dba3e78SLorenzo Bianconi __builtin_memcpy(eh->h_source, eh->h_dest, ETH_ALEN);
211*4dba3e78SLorenzo Bianconi __builtin_memcpy(eh->h_dest, tmp_mac, ETH_ALEN);
212*4dba3e78SLorenzo Bianconi
213*4dba3e78SLorenzo Bianconi return XDP_TX;
214*4dba3e78SLorenzo Bianconi }
215*4dba3e78SLorenzo Bianconi
216*4dba3e78SLorenzo Bianconi SEC("xdp")
xdp_do_redirect(struct xdp_md * xdp)217*4dba3e78SLorenzo Bianconi int xdp_do_redirect(struct xdp_md *xdp)
218*4dba3e78SLorenzo Bianconi {
219*4dba3e78SLorenzo Bianconi if (xdp_process_echo_packet(xdp, true))
220*4dba3e78SLorenzo Bianconi return XDP_PASS;
221*4dba3e78SLorenzo Bianconi
222*4dba3e78SLorenzo Bianconi return bpf_redirect_map(&cpu_map, 0, 0);
223*4dba3e78SLorenzo Bianconi }
224*4dba3e78SLorenzo Bianconi
225*4dba3e78SLorenzo Bianconi SEC("tp_btf/xdp_exception")
BPF_PROG(xdp_exception,const struct net_device * dev,const struct bpf_prog * xdp,__u32 act)226*4dba3e78SLorenzo Bianconi int BPF_PROG(xdp_exception, const struct net_device *dev,
227*4dba3e78SLorenzo Bianconi const struct bpf_prog *xdp, __u32 act)
228*4dba3e78SLorenzo Bianconi {
229*4dba3e78SLorenzo Bianconi __u32 *val, key = 0;
230*4dba3e78SLorenzo Bianconi
231*4dba3e78SLorenzo Bianconi val = bpf_map_lookup_elem(&dut_stats, &key);
232*4dba3e78SLorenzo Bianconi if (val)
233*4dba3e78SLorenzo Bianconi __sync_add_and_fetch(val, 1);
234*4dba3e78SLorenzo Bianconi
235*4dba3e78SLorenzo Bianconi return 0;
236*4dba3e78SLorenzo Bianconi }
237*4dba3e78SLorenzo Bianconi
238*4dba3e78SLorenzo Bianconi SEC("tp_btf/xdp_cpumap_kthread")
BPF_PROG(tp_xdp_cpumap_kthread,int map_id,unsigned int processed,unsigned int drops,int sched,struct xdp_cpumap_stats * xdp_stats)239*4dba3e78SLorenzo Bianconi int BPF_PROG(tp_xdp_cpumap_kthread, int map_id, unsigned int processed,
240*4dba3e78SLorenzo Bianconi unsigned int drops, int sched, struct xdp_cpumap_stats *xdp_stats)
241*4dba3e78SLorenzo Bianconi {
242*4dba3e78SLorenzo Bianconi __u32 *val, key = 0;
243*4dba3e78SLorenzo Bianconi
244*4dba3e78SLorenzo Bianconi val = bpf_map_lookup_elem(&dut_stats, &key);
245*4dba3e78SLorenzo Bianconi if (val)
246*4dba3e78SLorenzo Bianconi __sync_add_and_fetch(val, 1);
247*4dba3e78SLorenzo Bianconi
248*4dba3e78SLorenzo Bianconi return 0;
249*4dba3e78SLorenzo Bianconi }
250*4dba3e78SLorenzo Bianconi
251*4dba3e78SLorenzo Bianconi SEC("xdp/cpumap")
xdp_do_redirect_cpumap(struct xdp_md * xdp)252*4dba3e78SLorenzo Bianconi int xdp_do_redirect_cpumap(struct xdp_md *xdp)
253*4dba3e78SLorenzo Bianconi {
254*4dba3e78SLorenzo Bianconi void *data = (void *)(long)xdp->data;
255*4dba3e78SLorenzo Bianconi struct ethhdr *eh = data;
256*4dba3e78SLorenzo Bianconi __u8 tmp_mac[ETH_ALEN];
257*4dba3e78SLorenzo Bianconi
258*4dba3e78SLorenzo Bianconi if (xdp_process_echo_packet(xdp, true))
259*4dba3e78SLorenzo Bianconi return XDP_PASS;
260*4dba3e78SLorenzo Bianconi
261*4dba3e78SLorenzo Bianconi __builtin_memcpy(tmp_mac, eh->h_source, ETH_ALEN);
262*4dba3e78SLorenzo Bianconi __builtin_memcpy(eh->h_source, eh->h_dest, ETH_ALEN);
263*4dba3e78SLorenzo Bianconi __builtin_memcpy(eh->h_dest, tmp_mac, ETH_ALEN);
264*4dba3e78SLorenzo Bianconi
265*4dba3e78SLorenzo Bianconi return bpf_redirect_map(&dev_map, 0, 0);
266*4dba3e78SLorenzo Bianconi }
267*4dba3e78SLorenzo Bianconi
268*4dba3e78SLorenzo Bianconi char _license[] SEC("license") = "GPL";
269