1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2020 Cloudflare
3 #include <error.h>
4 #include <netinet/tcp.h>
5 #include <sys/epoll.h>
6 
7 #include "test_progs.h"
8 #include "test_skmsg_load_helpers.skel.h"
9 #include "test_sockmap_update.skel.h"
10 #include "test_sockmap_invalid_update.skel.h"
11 #include "test_sockmap_skb_verdict_attach.skel.h"
12 #include "test_sockmap_progs_query.skel.h"
13 #include "test_sockmap_pass_prog.skel.h"
14 #include "test_sockmap_drop_prog.skel.h"
15 #include "bpf_iter_sockmap.skel.h"
16 
17 #include "sockmap_helpers.h"
18 
19 #define TCP_REPAIR		19	/* TCP sock is under repair right now */
20 
21 #define TCP_REPAIR_ON		1
22 #define TCP_REPAIR_OFF_NO_WP	-1	/* Turn off without window probes */
23 
24 static int connected_socket_v4(void)
25 {
26 	struct sockaddr_in addr = {
27 		.sin_family = AF_INET,
28 		.sin_port = htons(80),
29 		.sin_addr = { inet_addr("127.0.0.1") },
30 	};
31 	socklen_t len = sizeof(addr);
32 	int s, repair, err;
33 
34 	s = socket(AF_INET, SOCK_STREAM, 0);
35 	if (!ASSERT_GE(s, 0, "socket"))
36 		goto error;
37 
38 	repair = TCP_REPAIR_ON;
39 	err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair));
40 	if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)"))
41 		goto error;
42 
43 	err = connect(s, (struct sockaddr *)&addr, len);
44 	if (!ASSERT_OK(err, "connect"))
45 		goto error;
46 
47 	repair = TCP_REPAIR_OFF_NO_WP;
48 	err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair));
49 	if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)"))
50 		goto error;
51 
52 	return s;
53 error:
54 	perror(__func__);
55 	close(s);
56 	return -1;
57 }
58 
59 static void compare_cookies(struct bpf_map *src, struct bpf_map *dst)
60 {
61 	__u32 i, max_entries = bpf_map__max_entries(src);
62 	int err, src_fd, dst_fd;
63 
64 	src_fd = bpf_map__fd(src);
65 	dst_fd = bpf_map__fd(dst);
66 
67 	for (i = 0; i < max_entries; i++) {
68 		__u64 src_cookie, dst_cookie;
69 
70 		err = bpf_map_lookup_elem(src_fd, &i, &src_cookie);
71 		if (err && errno == ENOENT) {
72 			err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie);
73 			ASSERT_ERR(err, "map_lookup_elem(dst)");
74 			ASSERT_EQ(errno, ENOENT, "map_lookup_elem(dst)");
75 			continue;
76 		}
77 		if (!ASSERT_OK(err, "lookup_elem(src)"))
78 			continue;
79 
80 		err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie);
81 		if (!ASSERT_OK(err, "lookup_elem(dst)"))
82 			continue;
83 
84 		ASSERT_EQ(dst_cookie, src_cookie, "cookie mismatch");
85 	}
86 }
87 
88 /* Create a map, populate it with one socket, and free the map. */
89 static void test_sockmap_create_update_free(enum bpf_map_type map_type)
90 {
91 	const int zero = 0;
92 	int s, map, err;
93 
94 	s = connected_socket_v4();
95 	if (!ASSERT_GE(s, 0, "connected_socket_v4"))
96 		return;
97 
98 	map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL);
99 	if (!ASSERT_GE(map, 0, "bpf_map_create"))
100 		goto out;
101 
102 	err = bpf_map_update_elem(map, &zero, &s, BPF_NOEXIST);
103 	if (!ASSERT_OK(err, "bpf_map_update"))
104 		goto out;
105 
106 out:
107 	close(map);
108 	close(s);
109 }
110 
111 static void test_skmsg_helpers(enum bpf_map_type map_type)
112 {
113 	struct test_skmsg_load_helpers *skel;
114 	int err, map, verdict;
115 
116 	skel = test_skmsg_load_helpers__open_and_load();
117 	if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load"))
118 		return;
119 
120 	verdict = bpf_program__fd(skel->progs.prog_msg_verdict);
121 	map = bpf_map__fd(skel->maps.sock_map);
122 
123 	err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0);
124 	if (!ASSERT_OK(err, "bpf_prog_attach"))
125 		goto out;
126 
127 	err = bpf_prog_detach2(verdict, map, BPF_SK_MSG_VERDICT);
128 	if (!ASSERT_OK(err, "bpf_prog_detach2"))
129 		goto out;
130 out:
131 	test_skmsg_load_helpers__destroy(skel);
132 }
133 
134 static void test_sockmap_update(enum bpf_map_type map_type)
135 {
136 	int err, prog, src;
137 	struct test_sockmap_update *skel;
138 	struct bpf_map *dst_map;
139 	const __u32 zero = 0;
140 	char dummy[14] = {0};
141 	LIBBPF_OPTS(bpf_test_run_opts, topts,
142 		.data_in = dummy,
143 		.data_size_in = sizeof(dummy),
144 		.repeat = 1,
145 	);
146 	__s64 sk;
147 
148 	sk = connected_socket_v4();
149 	if (!ASSERT_NEQ(sk, -1, "connected_socket_v4"))
150 		return;
151 
152 	skel = test_sockmap_update__open_and_load();
153 	if (!ASSERT_OK_PTR(skel, "open_and_load"))
154 		goto close_sk;
155 
156 	prog = bpf_program__fd(skel->progs.copy_sock_map);
157 	src = bpf_map__fd(skel->maps.src);
158 	if (map_type == BPF_MAP_TYPE_SOCKMAP)
159 		dst_map = skel->maps.dst_sock_map;
160 	else
161 		dst_map = skel->maps.dst_sock_hash;
162 
163 	err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST);
164 	if (!ASSERT_OK(err, "update_elem(src)"))
165 		goto out;
166 
167 	err = bpf_prog_test_run_opts(prog, &topts);
168 	if (!ASSERT_OK(err, "test_run"))
169 		goto out;
170 	if (!ASSERT_NEQ(topts.retval, 0, "test_run retval"))
171 		goto out;
172 
173 	compare_cookies(skel->maps.src, dst_map);
174 
175 out:
176 	test_sockmap_update__destroy(skel);
177 close_sk:
178 	close(sk);
179 }
180 
181 static void test_sockmap_invalid_update(void)
182 {
183 	struct test_sockmap_invalid_update *skel;
184 
185 	skel = test_sockmap_invalid_update__open_and_load();
186 	if (!ASSERT_NULL(skel, "open_and_load"))
187 		test_sockmap_invalid_update__destroy(skel);
188 }
189 
190 static void test_sockmap_copy(enum bpf_map_type map_type)
191 {
192 	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
193 	int err, len, src_fd, iter_fd;
194 	union bpf_iter_link_info linfo = {};
195 	__u32 i, num_sockets, num_elems;
196 	struct bpf_iter_sockmap *skel;
197 	__s64 *sock_fd = NULL;
198 	struct bpf_link *link;
199 	struct bpf_map *src;
200 	char buf[64];
201 
202 	skel = bpf_iter_sockmap__open_and_load();
203 	if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
204 		return;
205 
206 	if (map_type == BPF_MAP_TYPE_SOCKMAP) {
207 		src = skel->maps.sockmap;
208 		num_elems = bpf_map__max_entries(src);
209 		num_sockets = num_elems - 1;
210 	} else {
211 		src = skel->maps.sockhash;
212 		num_elems = bpf_map__max_entries(src) - 1;
213 		num_sockets = num_elems;
214 	}
215 
216 	sock_fd = calloc(num_sockets, sizeof(*sock_fd));
217 	if (!ASSERT_OK_PTR(sock_fd, "calloc(sock_fd)"))
218 		goto out;
219 
220 	for (i = 0; i < num_sockets; i++)
221 		sock_fd[i] = -1;
222 
223 	src_fd = bpf_map__fd(src);
224 
225 	for (i = 0; i < num_sockets; i++) {
226 		sock_fd[i] = connected_socket_v4();
227 		if (!ASSERT_NEQ(sock_fd[i], -1, "connected_socket_v4"))
228 			goto out;
229 
230 		err = bpf_map_update_elem(src_fd, &i, &sock_fd[i], BPF_NOEXIST);
231 		if (!ASSERT_OK(err, "map_update"))
232 			goto out;
233 	}
234 
235 	linfo.map.map_fd = src_fd;
236 	opts.link_info = &linfo;
237 	opts.link_info_len = sizeof(linfo);
238 	link = bpf_program__attach_iter(skel->progs.copy, &opts);
239 	if (!ASSERT_OK_PTR(link, "attach_iter"))
240 		goto out;
241 
242 	iter_fd = bpf_iter_create(bpf_link__fd(link));
243 	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
244 		goto free_link;
245 
246 	/* do some tests */
247 	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
248 		;
249 	if (!ASSERT_GE(len, 0, "read"))
250 		goto close_iter;
251 
252 	/* test results */
253 	if (!ASSERT_EQ(skel->bss->elems, num_elems, "elems"))
254 		goto close_iter;
255 
256 	if (!ASSERT_EQ(skel->bss->socks, num_sockets, "socks"))
257 		goto close_iter;
258 
259 	compare_cookies(src, skel->maps.dst);
260 
261 close_iter:
262 	close(iter_fd);
263 free_link:
264 	bpf_link__destroy(link);
265 out:
266 	for (i = 0; sock_fd && i < num_sockets; i++)
267 		if (sock_fd[i] >= 0)
268 			close(sock_fd[i]);
269 	if (sock_fd)
270 		free(sock_fd);
271 	bpf_iter_sockmap__destroy(skel);
272 }
273 
274 static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first,
275 					    enum bpf_attach_type second)
276 {
277 	struct test_sockmap_skb_verdict_attach *skel;
278 	int err, map, verdict;
279 
280 	skel = test_sockmap_skb_verdict_attach__open_and_load();
281 	if (!ASSERT_OK_PTR(skel, "open_and_load"))
282 		return;
283 
284 	verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
285 	map = bpf_map__fd(skel->maps.sock_map);
286 
287 	err = bpf_prog_attach(verdict, map, first, 0);
288 	if (!ASSERT_OK(err, "bpf_prog_attach"))
289 		goto out;
290 
291 	err = bpf_prog_attach(verdict, map, second, 0);
292 	ASSERT_EQ(err, -EBUSY, "prog_attach_fail");
293 
294 	err = bpf_prog_detach2(verdict, map, first);
295 	if (!ASSERT_OK(err, "bpf_prog_detach2"))
296 		goto out;
297 out:
298 	test_sockmap_skb_verdict_attach__destroy(skel);
299 }
300 
301 static __u32 query_prog_id(int prog_fd)
302 {
303 	struct bpf_prog_info info = {};
304 	__u32 info_len = sizeof(info);
305 	int err;
306 
307 	err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len);
308 	if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd") ||
309 	    !ASSERT_EQ(info_len, sizeof(info), "bpf_prog_get_info_by_fd"))
310 		return 0;
311 
312 	return info.id;
313 }
314 
315 static void test_sockmap_progs_query(enum bpf_attach_type attach_type)
316 {
317 	struct test_sockmap_progs_query *skel;
318 	int err, map_fd, verdict_fd;
319 	__u32 attach_flags = 0;
320 	__u32 prog_ids[3] = {};
321 	__u32 prog_cnt = 3;
322 
323 	skel = test_sockmap_progs_query__open_and_load();
324 	if (!ASSERT_OK_PTR(skel, "test_sockmap_progs_query__open_and_load"))
325 		return;
326 
327 	map_fd = bpf_map__fd(skel->maps.sock_map);
328 
329 	if (attach_type == BPF_SK_MSG_VERDICT)
330 		verdict_fd = bpf_program__fd(skel->progs.prog_skmsg_verdict);
331 	else
332 		verdict_fd = bpf_program__fd(skel->progs.prog_skb_verdict);
333 
334 	err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */,
335 			     &attach_flags, prog_ids, &prog_cnt);
336 	ASSERT_OK(err, "bpf_prog_query failed");
337 	ASSERT_EQ(attach_flags,  0, "wrong attach_flags on query");
338 	ASSERT_EQ(prog_cnt, 0, "wrong program count on query");
339 
340 	err = bpf_prog_attach(verdict_fd, map_fd, attach_type, 0);
341 	if (!ASSERT_OK(err, "bpf_prog_attach failed"))
342 		goto out;
343 
344 	prog_cnt = 1;
345 	err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */,
346 			     &attach_flags, prog_ids, &prog_cnt);
347 	ASSERT_OK(err, "bpf_prog_query failed");
348 	ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query");
349 	ASSERT_EQ(prog_cnt, 1, "wrong program count on query");
350 	ASSERT_EQ(prog_ids[0], query_prog_id(verdict_fd),
351 		  "wrong prog_ids on query");
352 
353 	bpf_prog_detach2(verdict_fd, map_fd, attach_type);
354 out:
355 	test_sockmap_progs_query__destroy(skel);
356 }
357 
358 #define MAX_EVENTS 10
359 static void test_sockmap_skb_verdict_shutdown(void)
360 {
361 	struct epoll_event ev, events[MAX_EVENTS];
362 	int n, err, map, verdict, s, c1, p1;
363 	struct test_sockmap_pass_prog *skel;
364 	int epollfd;
365 	int zero = 0;
366 	char b;
367 
368 	skel = test_sockmap_pass_prog__open_and_load();
369 	if (!ASSERT_OK_PTR(skel, "open_and_load"))
370 		return;
371 
372 	verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
373 	map = bpf_map__fd(skel->maps.sock_map_rx);
374 
375 	err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0);
376 	if (!ASSERT_OK(err, "bpf_prog_attach"))
377 		goto out;
378 
379 	s = socket_loopback(AF_INET, SOCK_STREAM);
380 	if (s < 0)
381 		goto out;
382 	err = create_pair(s, AF_INET, SOCK_STREAM, &c1, &p1);
383 	if (err < 0)
384 		goto out;
385 
386 	err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST);
387 	if (err < 0)
388 		goto out_close;
389 
390 	shutdown(p1, SHUT_WR);
391 
392 	ev.events = EPOLLIN;
393 	ev.data.fd = c1;
394 
395 	epollfd = epoll_create1(0);
396 	if (!ASSERT_GT(epollfd, -1, "epoll_create(0)"))
397 		goto out_close;
398 	err = epoll_ctl(epollfd, EPOLL_CTL_ADD, c1, &ev);
399 	if (!ASSERT_OK(err, "epoll_ctl(EPOLL_CTL_ADD)"))
400 		goto out_close;
401 	err = epoll_wait(epollfd, events, MAX_EVENTS, -1);
402 	if (!ASSERT_EQ(err, 1, "epoll_wait(fd)"))
403 		goto out_close;
404 
405 	n = recv(c1, &b, 1, SOCK_NONBLOCK);
406 	ASSERT_EQ(n, 0, "recv_timeout(fin)");
407 out_close:
408 	close(c1);
409 	close(p1);
410 out:
411 	test_sockmap_pass_prog__destroy(skel);
412 }
413 
414 static void test_sockmap_skb_verdict_fionread(bool pass_prog)
415 {
416 	int expected, zero = 0, sent, recvd, avail;
417 	int err, map, verdict, s, c0, c1, p0, p1;
418 	struct test_sockmap_pass_prog *pass;
419 	struct test_sockmap_drop_prog *drop;
420 	char buf[256] = "0123456789";
421 
422 	if (pass_prog) {
423 		pass = test_sockmap_pass_prog__open_and_load();
424 		if (!ASSERT_OK_PTR(pass, "open_and_load"))
425 			return;
426 		verdict = bpf_program__fd(pass->progs.prog_skb_verdict);
427 		map = bpf_map__fd(pass->maps.sock_map_rx);
428 		expected = sizeof(buf);
429 	} else {
430 		drop = test_sockmap_drop_prog__open_and_load();
431 		if (!ASSERT_OK_PTR(drop, "open_and_load"))
432 			return;
433 		verdict = bpf_program__fd(drop->progs.prog_skb_verdict);
434 		map = bpf_map__fd(drop->maps.sock_map_rx);
435 		/* On drop data is consumed immediately and copied_seq inc'd */
436 		expected = 0;
437 	}
438 
439 
440 	err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0);
441 	if (!ASSERT_OK(err, "bpf_prog_attach"))
442 		goto out;
443 
444 	s = socket_loopback(AF_INET, SOCK_STREAM);
445 	if (!ASSERT_GT(s, -1, "socket_loopback(s)"))
446 		goto out;
447 	err = create_socket_pairs(s, AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1);
448 	if (!ASSERT_OK(err, "create_socket_pairs(s)"))
449 		goto out;
450 
451 	err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST);
452 	if (!ASSERT_OK(err, "bpf_map_update_elem(c1)"))
453 		goto out_close;
454 
455 	sent = xsend(p1, &buf, sizeof(buf), 0);
456 	ASSERT_EQ(sent, sizeof(buf), "xsend(p0)");
457 	err = ioctl(c1, FIONREAD, &avail);
458 	ASSERT_OK(err, "ioctl(FIONREAD) error");
459 	ASSERT_EQ(avail, expected, "ioctl(FIONREAD)");
460 	/* On DROP test there will be no data to read */
461 	if (pass_prog) {
462 		recvd = recv_timeout(c1, &buf, sizeof(buf), SOCK_NONBLOCK, IO_TIMEOUT_SEC);
463 		ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(c0)");
464 	}
465 
466 out_close:
467 	close(c0);
468 	close(p0);
469 	close(c1);
470 	close(p1);
471 out:
472 	if (pass_prog)
473 		test_sockmap_pass_prog__destroy(pass);
474 	else
475 		test_sockmap_drop_prog__destroy(drop);
476 }
477 
478 void test_sockmap_basic(void)
479 {
480 	if (test__start_subtest("sockmap create_update_free"))
481 		test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKMAP);
482 	if (test__start_subtest("sockhash create_update_free"))
483 		test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKHASH);
484 	if (test__start_subtest("sockmap sk_msg load helpers"))
485 		test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP);
486 	if (test__start_subtest("sockhash sk_msg load helpers"))
487 		test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH);
488 	if (test__start_subtest("sockmap update"))
489 		test_sockmap_update(BPF_MAP_TYPE_SOCKMAP);
490 	if (test__start_subtest("sockhash update"))
491 		test_sockmap_update(BPF_MAP_TYPE_SOCKHASH);
492 	if (test__start_subtest("sockmap update in unsafe context"))
493 		test_sockmap_invalid_update();
494 	if (test__start_subtest("sockmap copy"))
495 		test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP);
496 	if (test__start_subtest("sockhash copy"))
497 		test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH);
498 	if (test__start_subtest("sockmap skb_verdict attach")) {
499 		test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT,
500 						BPF_SK_SKB_STREAM_VERDICT);
501 		test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT,
502 						BPF_SK_SKB_VERDICT);
503 	}
504 	if (test__start_subtest("sockmap msg_verdict progs query"))
505 		test_sockmap_progs_query(BPF_SK_MSG_VERDICT);
506 	if (test__start_subtest("sockmap stream_parser progs query"))
507 		test_sockmap_progs_query(BPF_SK_SKB_STREAM_PARSER);
508 	if (test__start_subtest("sockmap stream_verdict progs query"))
509 		test_sockmap_progs_query(BPF_SK_SKB_STREAM_VERDICT);
510 	if (test__start_subtest("sockmap skb_verdict progs query"))
511 		test_sockmap_progs_query(BPF_SK_SKB_VERDICT);
512 	if (test__start_subtest("sockmap skb_verdict shutdown"))
513 		test_sockmap_skb_verdict_shutdown();
514 	if (test__start_subtest("sockmap skb_verdict fionread"))
515 		test_sockmap_skb_verdict_fionread(true);
516 	if (test__start_subtest("sockmap skb_verdict fionread on drop"))
517 		test_sockmap_skb_verdict_fionread(false);
518 }
519