151a0e301SMartin KaFai Lau // SPDX-License-Identifier: GPL-2.0
251a0e301SMartin KaFai Lau /* Copyright (c) 2019 Facebook */
351a0e301SMartin KaFai Lau #include <linux/compiler.h>
451a0e301SMartin KaFai Lau #include <linux/err.h>
551a0e301SMartin KaFai Lau
651a0e301SMartin KaFai Lau #include <sys/resource.h>
751a0e301SMartin KaFai Lau #include <sys/socket.h>
851a0e301SMartin KaFai Lau #include <sys/types.h>
951a0e301SMartin KaFai Lau #include <linux/btf.h>
1051a0e301SMartin KaFai Lau #include <unistd.h>
1151a0e301SMartin KaFai Lau #include <signal.h>
1251a0e301SMartin KaFai Lau #include <errno.h>
1351a0e301SMartin KaFai Lau #include <string.h>
1451a0e301SMartin KaFai Lau #include <pthread.h>
1551a0e301SMartin KaFai Lau
1651a0e301SMartin KaFai Lau #include <bpf/bpf.h>
1751a0e301SMartin KaFai Lau #include <bpf/libbpf.h>
1851a0e301SMartin KaFai Lau
1951a0e301SMartin KaFai Lau #include <test_btf.h>
2051a0e301SMartin KaFai Lau #include <test_maps.h>
2151a0e301SMartin KaFai Lau
222fe256a4SAndrii Nakryiko static struct bpf_map_create_opts map_opts = {
232fe256a4SAndrii Nakryiko .sz = sizeof(map_opts),
2451a0e301SMartin KaFai Lau .btf_key_type_id = 1,
2551a0e301SMartin KaFai Lau .btf_value_type_id = 3,
2651a0e301SMartin KaFai Lau .btf_fd = -1,
272fe256a4SAndrii Nakryiko .map_flags = BPF_F_NO_PREALLOC,
2851a0e301SMartin KaFai Lau };
2951a0e301SMartin KaFai Lau
3051a0e301SMartin KaFai Lau static unsigned int nr_sk_threads_done;
3151a0e301SMartin KaFai Lau static unsigned int nr_sk_threads_err;
3251a0e301SMartin KaFai Lau static unsigned int nr_sk_per_thread = 4096;
3351a0e301SMartin KaFai Lau static unsigned int nr_sk_threads = 4;
3451a0e301SMartin KaFai Lau static int sk_storage_map = -1;
3551a0e301SMartin KaFai Lau static unsigned int stop;
3651a0e301SMartin KaFai Lau static int runtime_s = 5;
3751a0e301SMartin KaFai Lau
is_stopped(void)3851a0e301SMartin KaFai Lau static bool is_stopped(void)
3951a0e301SMartin KaFai Lau {
4051a0e301SMartin KaFai Lau return READ_ONCE(stop);
4151a0e301SMartin KaFai Lau }
4251a0e301SMartin KaFai Lau
threads_err(void)4351a0e301SMartin KaFai Lau static unsigned int threads_err(void)
4451a0e301SMartin KaFai Lau {
4551a0e301SMartin KaFai Lau return READ_ONCE(nr_sk_threads_err);
4651a0e301SMartin KaFai Lau }
4751a0e301SMartin KaFai Lau
notify_thread_err(void)4851a0e301SMartin KaFai Lau static void notify_thread_err(void)
4951a0e301SMartin KaFai Lau {
5051a0e301SMartin KaFai Lau __sync_add_and_fetch(&nr_sk_threads_err, 1);
5151a0e301SMartin KaFai Lau }
5251a0e301SMartin KaFai Lau
wait_for_threads_err(void)5351a0e301SMartin KaFai Lau static bool wait_for_threads_err(void)
5451a0e301SMartin KaFai Lau {
5551a0e301SMartin KaFai Lau while (!is_stopped() && !threads_err())
5651a0e301SMartin KaFai Lau usleep(500);
5751a0e301SMartin KaFai Lau
5851a0e301SMartin KaFai Lau return !is_stopped();
5951a0e301SMartin KaFai Lau }
6051a0e301SMartin KaFai Lau
threads_done(void)6151a0e301SMartin KaFai Lau static unsigned int threads_done(void)
6251a0e301SMartin KaFai Lau {
6351a0e301SMartin KaFai Lau return READ_ONCE(nr_sk_threads_done);
6451a0e301SMartin KaFai Lau }
6551a0e301SMartin KaFai Lau
notify_thread_done(void)6651a0e301SMartin KaFai Lau static void notify_thread_done(void)
6751a0e301SMartin KaFai Lau {
6851a0e301SMartin KaFai Lau __sync_add_and_fetch(&nr_sk_threads_done, 1);
6951a0e301SMartin KaFai Lau }
7051a0e301SMartin KaFai Lau
notify_thread_redo(void)7151a0e301SMartin KaFai Lau static void notify_thread_redo(void)
7251a0e301SMartin KaFai Lau {
7351a0e301SMartin KaFai Lau __sync_sub_and_fetch(&nr_sk_threads_done, 1);
7451a0e301SMartin KaFai Lau }
7551a0e301SMartin KaFai Lau
wait_for_threads_done(void)7651a0e301SMartin KaFai Lau static bool wait_for_threads_done(void)
7751a0e301SMartin KaFai Lau {
7851a0e301SMartin KaFai Lau while (threads_done() != nr_sk_threads && !is_stopped() &&
7951a0e301SMartin KaFai Lau !threads_err())
8051a0e301SMartin KaFai Lau usleep(50);
8151a0e301SMartin KaFai Lau
8251a0e301SMartin KaFai Lau return !is_stopped() && !threads_err();
8351a0e301SMartin KaFai Lau }
8451a0e301SMartin KaFai Lau
wait_for_threads_redo(void)8551a0e301SMartin KaFai Lau static bool wait_for_threads_redo(void)
8651a0e301SMartin KaFai Lau {
8751a0e301SMartin KaFai Lau while (threads_done() && !is_stopped() && !threads_err())
8851a0e301SMartin KaFai Lau usleep(50);
8951a0e301SMartin KaFai Lau
9051a0e301SMartin KaFai Lau return !is_stopped() && !threads_err();
9151a0e301SMartin KaFai Lau }
9251a0e301SMartin KaFai Lau
wait_for_map(void)9351a0e301SMartin KaFai Lau static bool wait_for_map(void)
9451a0e301SMartin KaFai Lau {
9551a0e301SMartin KaFai Lau while (READ_ONCE(sk_storage_map) == -1 && !is_stopped())
9651a0e301SMartin KaFai Lau usleep(50);
9751a0e301SMartin KaFai Lau
9851a0e301SMartin KaFai Lau return !is_stopped();
9951a0e301SMartin KaFai Lau }
10051a0e301SMartin KaFai Lau
wait_for_map_close(void)10151a0e301SMartin KaFai Lau static bool wait_for_map_close(void)
10251a0e301SMartin KaFai Lau {
10351a0e301SMartin KaFai Lau while (READ_ONCE(sk_storage_map) != -1 && !is_stopped())
10451a0e301SMartin KaFai Lau ;
10551a0e301SMartin KaFai Lau
10651a0e301SMartin KaFai Lau return !is_stopped();
10751a0e301SMartin KaFai Lau }
10851a0e301SMartin KaFai Lau
load_btf(void)10951a0e301SMartin KaFai Lau static int load_btf(void)
11051a0e301SMartin KaFai Lau {
11151a0e301SMartin KaFai Lau const char btf_str_sec[] = "\0bpf_spin_lock\0val\0cnt\0l";
11251a0e301SMartin KaFai Lau __u32 btf_raw_types[] = {
11351a0e301SMartin KaFai Lau /* int */
11451a0e301SMartin KaFai Lau BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
11551a0e301SMartin KaFai Lau /* struct bpf_spin_lock */ /* [2] */
11651a0e301SMartin KaFai Lau BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 4),
11751a0e301SMartin KaFai Lau BTF_MEMBER_ENC(15, 1, 0), /* int val; */
11851a0e301SMartin KaFai Lau /* struct val */ /* [3] */
11951a0e301SMartin KaFai Lau BTF_TYPE_ENC(15, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
12051a0e301SMartin KaFai Lau BTF_MEMBER_ENC(19, 1, 0), /* int cnt; */
12151a0e301SMartin KaFai Lau BTF_MEMBER_ENC(23, 2, 32),/* struct bpf_spin_lock l; */
12251a0e301SMartin KaFai Lau };
12351a0e301SMartin KaFai Lau struct btf_header btf_hdr = {
12451a0e301SMartin KaFai Lau .magic = BTF_MAGIC,
12551a0e301SMartin KaFai Lau .version = BTF_VERSION,
12651a0e301SMartin KaFai Lau .hdr_len = sizeof(struct btf_header),
12751a0e301SMartin KaFai Lau .type_len = sizeof(btf_raw_types),
12851a0e301SMartin KaFai Lau .str_off = sizeof(btf_raw_types),
12951a0e301SMartin KaFai Lau .str_len = sizeof(btf_str_sec),
13051a0e301SMartin KaFai Lau };
13151a0e301SMartin KaFai Lau __u8 raw_btf[sizeof(struct btf_header) + sizeof(btf_raw_types) +
13251a0e301SMartin KaFai Lau sizeof(btf_str_sec)];
13351a0e301SMartin KaFai Lau
13451a0e301SMartin KaFai Lau memcpy(raw_btf, &btf_hdr, sizeof(btf_hdr));
13551a0e301SMartin KaFai Lau memcpy(raw_btf + sizeof(btf_hdr), btf_raw_types, sizeof(btf_raw_types));
13651a0e301SMartin KaFai Lau memcpy(raw_btf + sizeof(btf_hdr) + sizeof(btf_raw_types),
13751a0e301SMartin KaFai Lau btf_str_sec, sizeof(btf_str_sec));
13851a0e301SMartin KaFai Lau
139dc94121bSAndrii Nakryiko return bpf_btf_load(raw_btf, sizeof(raw_btf), NULL);
14051a0e301SMartin KaFai Lau }
14151a0e301SMartin KaFai Lau
create_sk_storage_map(void)14251a0e301SMartin KaFai Lau static int create_sk_storage_map(void)
14351a0e301SMartin KaFai Lau {
14451a0e301SMartin KaFai Lau int btf_fd, map_fd;
14551a0e301SMartin KaFai Lau
14651a0e301SMartin KaFai Lau btf_fd = load_btf();
14751a0e301SMartin KaFai Lau CHECK(btf_fd == -1, "bpf_load_btf", "btf_fd:%d errno:%d\n",
14851a0e301SMartin KaFai Lau btf_fd, errno);
1492fe256a4SAndrii Nakryiko map_opts.btf_fd = btf_fd;
15051a0e301SMartin KaFai Lau
1512fe256a4SAndrii Nakryiko map_fd = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &map_opts);
1522fe256a4SAndrii Nakryiko map_opts.btf_fd = -1;
15351a0e301SMartin KaFai Lau close(btf_fd);
15451a0e301SMartin KaFai Lau CHECK(map_fd == -1,
1552fe256a4SAndrii Nakryiko "bpf_map_create()", "errno:%d\n", errno);
15651a0e301SMartin KaFai Lau
15751a0e301SMartin KaFai Lau return map_fd;
15851a0e301SMartin KaFai Lau }
15951a0e301SMartin KaFai Lau
insert_close_thread(void * arg)16051a0e301SMartin KaFai Lau static void *insert_close_thread(void *arg)
16151a0e301SMartin KaFai Lau {
16251a0e301SMartin KaFai Lau struct {
16351a0e301SMartin KaFai Lau int cnt;
16451a0e301SMartin KaFai Lau int lock;
16551a0e301SMartin KaFai Lau } value = { .cnt = 0xeB9F, .lock = 0, };
16651a0e301SMartin KaFai Lau int i, map_fd, err, *sk_fds;
16751a0e301SMartin KaFai Lau
16851a0e301SMartin KaFai Lau sk_fds = malloc(sizeof(*sk_fds) * nr_sk_per_thread);
16951a0e301SMartin KaFai Lau if (!sk_fds) {
17051a0e301SMartin KaFai Lau notify_thread_err();
17151a0e301SMartin KaFai Lau return ERR_PTR(-ENOMEM);
17251a0e301SMartin KaFai Lau }
17351a0e301SMartin KaFai Lau
17451a0e301SMartin KaFai Lau for (i = 0; i < nr_sk_per_thread; i++)
17551a0e301SMartin KaFai Lau sk_fds[i] = -1;
17651a0e301SMartin KaFai Lau
17751a0e301SMartin KaFai Lau while (!is_stopped()) {
17851a0e301SMartin KaFai Lau if (!wait_for_map())
17951a0e301SMartin KaFai Lau goto close_all;
18051a0e301SMartin KaFai Lau
18151a0e301SMartin KaFai Lau map_fd = READ_ONCE(sk_storage_map);
18251a0e301SMartin KaFai Lau for (i = 0; i < nr_sk_per_thread && !is_stopped(); i++) {
18351a0e301SMartin KaFai Lau sk_fds[i] = socket(AF_INET6, SOCK_STREAM, 0);
18451a0e301SMartin KaFai Lau if (sk_fds[i] == -1) {
18551a0e301SMartin KaFai Lau err = -errno;
18651a0e301SMartin KaFai Lau fprintf(stderr, "socket(): errno:%d\n", errno);
18751a0e301SMartin KaFai Lau goto errout;
18851a0e301SMartin KaFai Lau }
18951a0e301SMartin KaFai Lau err = bpf_map_update_elem(map_fd, &sk_fds[i], &value,
19051a0e301SMartin KaFai Lau BPF_NOEXIST);
19151a0e301SMartin KaFai Lau if (err) {
19251a0e301SMartin KaFai Lau err = -errno;
19351a0e301SMartin KaFai Lau fprintf(stderr,
19451a0e301SMartin KaFai Lau "bpf_map_update_elem(): errno:%d\n",
19551a0e301SMartin KaFai Lau errno);
19651a0e301SMartin KaFai Lau goto errout;
19751a0e301SMartin KaFai Lau }
19851a0e301SMartin KaFai Lau }
19951a0e301SMartin KaFai Lau
20051a0e301SMartin KaFai Lau notify_thread_done();
20151a0e301SMartin KaFai Lau wait_for_map_close();
20251a0e301SMartin KaFai Lau
20351a0e301SMartin KaFai Lau close_all:
20451a0e301SMartin KaFai Lau for (i = 0; i < nr_sk_per_thread; i++) {
20551a0e301SMartin KaFai Lau close(sk_fds[i]);
20651a0e301SMartin KaFai Lau sk_fds[i] = -1;
20751a0e301SMartin KaFai Lau }
20851a0e301SMartin KaFai Lau
20951a0e301SMartin KaFai Lau notify_thread_redo();
21051a0e301SMartin KaFai Lau }
21151a0e301SMartin KaFai Lau
21251a0e301SMartin KaFai Lau free(sk_fds);
21351a0e301SMartin KaFai Lau return NULL;
21451a0e301SMartin KaFai Lau
21551a0e301SMartin KaFai Lau errout:
21651a0e301SMartin KaFai Lau for (i = 0; i < nr_sk_per_thread && sk_fds[i] != -1; i++)
21751a0e301SMartin KaFai Lau close(sk_fds[i]);
21851a0e301SMartin KaFai Lau free(sk_fds);
21951a0e301SMartin KaFai Lau notify_thread_err();
22051a0e301SMartin KaFai Lau return ERR_PTR(err);
22151a0e301SMartin KaFai Lau }
22251a0e301SMartin KaFai Lau
do_sk_storage_map_stress_free(void)22351a0e301SMartin KaFai Lau static int do_sk_storage_map_stress_free(void)
22451a0e301SMartin KaFai Lau {
22551a0e301SMartin KaFai Lau int i, map_fd = -1, err = 0, nr_threads_created = 0;
22651a0e301SMartin KaFai Lau pthread_t *sk_thread_ids;
22751a0e301SMartin KaFai Lau void *thread_ret;
22851a0e301SMartin KaFai Lau
22951a0e301SMartin KaFai Lau sk_thread_ids = malloc(sizeof(pthread_t) * nr_sk_threads);
23051a0e301SMartin KaFai Lau if (!sk_thread_ids) {
23151a0e301SMartin KaFai Lau fprintf(stderr, "malloc(sk_threads): NULL\n");
23251a0e301SMartin KaFai Lau return -ENOMEM;
23351a0e301SMartin KaFai Lau }
23451a0e301SMartin KaFai Lau
23551a0e301SMartin KaFai Lau for (i = 0; i < nr_sk_threads; i++) {
23651a0e301SMartin KaFai Lau err = pthread_create(&sk_thread_ids[i], NULL,
23751a0e301SMartin KaFai Lau insert_close_thread, NULL);
23851a0e301SMartin KaFai Lau if (err) {
23951a0e301SMartin KaFai Lau err = -errno;
24051a0e301SMartin KaFai Lau goto done;
24151a0e301SMartin KaFai Lau }
24251a0e301SMartin KaFai Lau nr_threads_created++;
24351a0e301SMartin KaFai Lau }
24451a0e301SMartin KaFai Lau
24551a0e301SMartin KaFai Lau while (!is_stopped()) {
24651a0e301SMartin KaFai Lau map_fd = create_sk_storage_map();
24751a0e301SMartin KaFai Lau WRITE_ONCE(sk_storage_map, map_fd);
24851a0e301SMartin KaFai Lau
24951a0e301SMartin KaFai Lau if (!wait_for_threads_done())
25051a0e301SMartin KaFai Lau break;
25151a0e301SMartin KaFai Lau
25251a0e301SMartin KaFai Lau WRITE_ONCE(sk_storage_map, -1);
25351a0e301SMartin KaFai Lau close(map_fd);
25451a0e301SMartin KaFai Lau map_fd = -1;
25551a0e301SMartin KaFai Lau
25651a0e301SMartin KaFai Lau if (!wait_for_threads_redo())
25751a0e301SMartin KaFai Lau break;
25851a0e301SMartin KaFai Lau }
25951a0e301SMartin KaFai Lau
26051a0e301SMartin KaFai Lau done:
26151a0e301SMartin KaFai Lau WRITE_ONCE(stop, 1);
26251a0e301SMartin KaFai Lau for (i = 0; i < nr_threads_created; i++) {
26351a0e301SMartin KaFai Lau pthread_join(sk_thread_ids[i], &thread_ret);
26451a0e301SMartin KaFai Lau if (IS_ERR(thread_ret) && !err) {
26551a0e301SMartin KaFai Lau err = PTR_ERR(thread_ret);
26651a0e301SMartin KaFai Lau fprintf(stderr, "threads#%u: err:%d\n", i, err);
26751a0e301SMartin KaFai Lau }
26851a0e301SMartin KaFai Lau }
26951a0e301SMartin KaFai Lau free(sk_thread_ids);
27051a0e301SMartin KaFai Lau
27151a0e301SMartin KaFai Lau if (map_fd != -1)
27251a0e301SMartin KaFai Lau close(map_fd);
27351a0e301SMartin KaFai Lau
27451a0e301SMartin KaFai Lau return err;
27551a0e301SMartin KaFai Lau }
27651a0e301SMartin KaFai Lau
update_thread(void * arg)27751a0e301SMartin KaFai Lau static void *update_thread(void *arg)
27851a0e301SMartin KaFai Lau {
27951a0e301SMartin KaFai Lau struct {
28051a0e301SMartin KaFai Lau int cnt;
28151a0e301SMartin KaFai Lau int lock;
28251a0e301SMartin KaFai Lau } value = { .cnt = 0xeB9F, .lock = 0, };
28351a0e301SMartin KaFai Lau int map_fd = READ_ONCE(sk_storage_map);
28451a0e301SMartin KaFai Lau int sk_fd = *(int *)arg;
28551a0e301SMartin KaFai Lau int err = 0; /* Suppress compiler false alarm */
28651a0e301SMartin KaFai Lau
28751a0e301SMartin KaFai Lau while (!is_stopped()) {
28851a0e301SMartin KaFai Lau err = bpf_map_update_elem(map_fd, &sk_fd, &value, 0);
28951a0e301SMartin KaFai Lau if (err && errno != EAGAIN) {
29051a0e301SMartin KaFai Lau err = -errno;
29151a0e301SMartin KaFai Lau fprintf(stderr, "bpf_map_update_elem: %d %d\n",
29251a0e301SMartin KaFai Lau err, errno);
29351a0e301SMartin KaFai Lau break;
29451a0e301SMartin KaFai Lau }
29551a0e301SMartin KaFai Lau }
29651a0e301SMartin KaFai Lau
29751a0e301SMartin KaFai Lau if (!is_stopped()) {
29851a0e301SMartin KaFai Lau notify_thread_err();
29951a0e301SMartin KaFai Lau return ERR_PTR(err);
30051a0e301SMartin KaFai Lau }
30151a0e301SMartin KaFai Lau
30251a0e301SMartin KaFai Lau return NULL;
30351a0e301SMartin KaFai Lau }
30451a0e301SMartin KaFai Lau
delete_thread(void * arg)30551a0e301SMartin KaFai Lau static void *delete_thread(void *arg)
30651a0e301SMartin KaFai Lau {
30751a0e301SMartin KaFai Lau int map_fd = READ_ONCE(sk_storage_map);
30851a0e301SMartin KaFai Lau int sk_fd = *(int *)arg;
30951a0e301SMartin KaFai Lau int err = 0; /* Suppress compiler false alarm */
31051a0e301SMartin KaFai Lau
31151a0e301SMartin KaFai Lau while (!is_stopped()) {
31251a0e301SMartin KaFai Lau err = bpf_map_delete_elem(map_fd, &sk_fd);
31351a0e301SMartin KaFai Lau if (err && errno != ENOENT) {
31451a0e301SMartin KaFai Lau err = -errno;
31551a0e301SMartin KaFai Lau fprintf(stderr, "bpf_map_delete_elem: %d %d\n",
31651a0e301SMartin KaFai Lau err, errno);
31751a0e301SMartin KaFai Lau break;
31851a0e301SMartin KaFai Lau }
31951a0e301SMartin KaFai Lau }
32051a0e301SMartin KaFai Lau
32151a0e301SMartin KaFai Lau if (!is_stopped()) {
32251a0e301SMartin KaFai Lau notify_thread_err();
32351a0e301SMartin KaFai Lau return ERR_PTR(err);
32451a0e301SMartin KaFai Lau }
32551a0e301SMartin KaFai Lau
32651a0e301SMartin KaFai Lau return NULL;
32751a0e301SMartin KaFai Lau }
32851a0e301SMartin KaFai Lau
do_sk_storage_map_stress_change(void)32951a0e301SMartin KaFai Lau static int do_sk_storage_map_stress_change(void)
33051a0e301SMartin KaFai Lau {
33151a0e301SMartin KaFai Lau int i, sk_fd, map_fd = -1, err = 0, nr_threads_created = 0;
33251a0e301SMartin KaFai Lau pthread_t *sk_thread_ids;
33351a0e301SMartin KaFai Lau void *thread_ret;
33451a0e301SMartin KaFai Lau
33551a0e301SMartin KaFai Lau sk_thread_ids = malloc(sizeof(pthread_t) * nr_sk_threads);
33651a0e301SMartin KaFai Lau if (!sk_thread_ids) {
33751a0e301SMartin KaFai Lau fprintf(stderr, "malloc(sk_threads): NULL\n");
33851a0e301SMartin KaFai Lau return -ENOMEM;
33951a0e301SMartin KaFai Lau }
34051a0e301SMartin KaFai Lau
34151a0e301SMartin KaFai Lau sk_fd = socket(AF_INET6, SOCK_STREAM, 0);
34251a0e301SMartin KaFai Lau if (sk_fd == -1) {
34351a0e301SMartin KaFai Lau err = -errno;
34451a0e301SMartin KaFai Lau goto done;
34551a0e301SMartin KaFai Lau }
34651a0e301SMartin KaFai Lau
34751a0e301SMartin KaFai Lau map_fd = create_sk_storage_map();
34851a0e301SMartin KaFai Lau WRITE_ONCE(sk_storage_map, map_fd);
34951a0e301SMartin KaFai Lau
35051a0e301SMartin KaFai Lau for (i = 0; i < nr_sk_threads; i++) {
35151a0e301SMartin KaFai Lau if (i & 0x1)
35251a0e301SMartin KaFai Lau err = pthread_create(&sk_thread_ids[i], NULL,
35351a0e301SMartin KaFai Lau update_thread, &sk_fd);
35451a0e301SMartin KaFai Lau else
35551a0e301SMartin KaFai Lau err = pthread_create(&sk_thread_ids[i], NULL,
35651a0e301SMartin KaFai Lau delete_thread, &sk_fd);
35751a0e301SMartin KaFai Lau if (err) {
35851a0e301SMartin KaFai Lau err = -errno;
35951a0e301SMartin KaFai Lau goto done;
36051a0e301SMartin KaFai Lau }
36151a0e301SMartin KaFai Lau nr_threads_created++;
36251a0e301SMartin KaFai Lau }
36351a0e301SMartin KaFai Lau
36451a0e301SMartin KaFai Lau wait_for_threads_err();
36551a0e301SMartin KaFai Lau
36651a0e301SMartin KaFai Lau done:
36751a0e301SMartin KaFai Lau WRITE_ONCE(stop, 1);
36851a0e301SMartin KaFai Lau for (i = 0; i < nr_threads_created; i++) {
36951a0e301SMartin KaFai Lau pthread_join(sk_thread_ids[i], &thread_ret);
37051a0e301SMartin KaFai Lau if (IS_ERR(thread_ret) && !err) {
37151a0e301SMartin KaFai Lau err = PTR_ERR(thread_ret);
37251a0e301SMartin KaFai Lau fprintf(stderr, "threads#%u: err:%d\n", i, err);
37351a0e301SMartin KaFai Lau }
37451a0e301SMartin KaFai Lau }
37551a0e301SMartin KaFai Lau free(sk_thread_ids);
37651a0e301SMartin KaFai Lau
37751a0e301SMartin KaFai Lau if (sk_fd != -1)
37851a0e301SMartin KaFai Lau close(sk_fd);
37951a0e301SMartin KaFai Lau close(map_fd);
38051a0e301SMartin KaFai Lau
38151a0e301SMartin KaFai Lau return err;
38251a0e301SMartin KaFai Lau }
38351a0e301SMartin KaFai Lau
stop_handler(int signum)38451a0e301SMartin KaFai Lau static void stop_handler(int signum)
38551a0e301SMartin KaFai Lau {
38651a0e301SMartin KaFai Lau if (signum != SIGALRM)
38751a0e301SMartin KaFai Lau printf("stopping...\n");
38851a0e301SMartin KaFai Lau WRITE_ONCE(stop, 1);
38951a0e301SMartin KaFai Lau }
39051a0e301SMartin KaFai Lau
39151a0e301SMartin KaFai Lau #define BPF_SK_STORAGE_MAP_TEST_NR_THREADS "BPF_SK_STORAGE_MAP_TEST_NR_THREADS"
39251a0e301SMartin KaFai Lau #define BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD "BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD"
39351a0e301SMartin KaFai Lau #define BPF_SK_STORAGE_MAP_TEST_RUNTIME_S "BPF_SK_STORAGE_MAP_TEST_RUNTIME_S"
39451a0e301SMartin KaFai Lau #define BPF_SK_STORAGE_MAP_TEST_NAME "BPF_SK_STORAGE_MAP_TEST_NAME"
39551a0e301SMartin KaFai Lau
test_sk_storage_map_stress_free(void)39651a0e301SMartin KaFai Lau static void test_sk_storage_map_stress_free(void)
39751a0e301SMartin KaFai Lau {
39851a0e301SMartin KaFai Lau struct rlimit rlim_old, rlim_new = {};
39951a0e301SMartin KaFai Lau int err;
40051a0e301SMartin KaFai Lau
40151a0e301SMartin KaFai Lau getrlimit(RLIMIT_NOFILE, &rlim_old);
40251a0e301SMartin KaFai Lau
40351a0e301SMartin KaFai Lau signal(SIGTERM, stop_handler);
40451a0e301SMartin KaFai Lau signal(SIGINT, stop_handler);
40551a0e301SMartin KaFai Lau if (runtime_s > 0) {
40651a0e301SMartin KaFai Lau signal(SIGALRM, stop_handler);
40751a0e301SMartin KaFai Lau alarm(runtime_s);
40851a0e301SMartin KaFai Lau }
40951a0e301SMartin KaFai Lau
41051a0e301SMartin KaFai Lau if (rlim_old.rlim_cur < nr_sk_threads * nr_sk_per_thread) {
41151a0e301SMartin KaFai Lau rlim_new.rlim_cur = nr_sk_threads * nr_sk_per_thread + 128;
41251a0e301SMartin KaFai Lau rlim_new.rlim_max = rlim_new.rlim_cur + 128;
41351a0e301SMartin KaFai Lau err = setrlimit(RLIMIT_NOFILE, &rlim_new);
41451a0e301SMartin KaFai Lau CHECK(err, "setrlimit(RLIMIT_NOFILE)", "rlim_new:%lu errno:%d",
41551a0e301SMartin KaFai Lau rlim_new.rlim_cur, errno);
41651a0e301SMartin KaFai Lau }
41751a0e301SMartin KaFai Lau
41851a0e301SMartin KaFai Lau err = do_sk_storage_map_stress_free();
41951a0e301SMartin KaFai Lau
42051a0e301SMartin KaFai Lau signal(SIGTERM, SIG_DFL);
42151a0e301SMartin KaFai Lau signal(SIGINT, SIG_DFL);
42251a0e301SMartin KaFai Lau if (runtime_s > 0) {
42351a0e301SMartin KaFai Lau signal(SIGALRM, SIG_DFL);
42451a0e301SMartin KaFai Lau alarm(0);
42551a0e301SMartin KaFai Lau }
42651a0e301SMartin KaFai Lau
42751a0e301SMartin KaFai Lau if (rlim_new.rlim_cur)
42851a0e301SMartin KaFai Lau setrlimit(RLIMIT_NOFILE, &rlim_old);
42951a0e301SMartin KaFai Lau
43051a0e301SMartin KaFai Lau CHECK(err, "test_sk_storage_map_stress_free", "err:%d\n", err);
43151a0e301SMartin KaFai Lau }
43251a0e301SMartin KaFai Lau
test_sk_storage_map_stress_change(void)43351a0e301SMartin KaFai Lau static void test_sk_storage_map_stress_change(void)
43451a0e301SMartin KaFai Lau {
43551a0e301SMartin KaFai Lau int err;
43651a0e301SMartin KaFai Lau
43751a0e301SMartin KaFai Lau signal(SIGTERM, stop_handler);
43851a0e301SMartin KaFai Lau signal(SIGINT, stop_handler);
43951a0e301SMartin KaFai Lau if (runtime_s > 0) {
44051a0e301SMartin KaFai Lau signal(SIGALRM, stop_handler);
44151a0e301SMartin KaFai Lau alarm(runtime_s);
44251a0e301SMartin KaFai Lau }
44351a0e301SMartin KaFai Lau
44451a0e301SMartin KaFai Lau err = do_sk_storage_map_stress_change();
44551a0e301SMartin KaFai Lau
44651a0e301SMartin KaFai Lau signal(SIGTERM, SIG_DFL);
44751a0e301SMartin KaFai Lau signal(SIGINT, SIG_DFL);
44851a0e301SMartin KaFai Lau if (runtime_s > 0) {
44951a0e301SMartin KaFai Lau signal(SIGALRM, SIG_DFL);
45051a0e301SMartin KaFai Lau alarm(0);
45151a0e301SMartin KaFai Lau }
45251a0e301SMartin KaFai Lau
45351a0e301SMartin KaFai Lau CHECK(err, "test_sk_storage_map_stress_change", "err:%d\n", err);
45451a0e301SMartin KaFai Lau }
45551a0e301SMartin KaFai Lau
test_sk_storage_map_basic(void)45651a0e301SMartin KaFai Lau static void test_sk_storage_map_basic(void)
45751a0e301SMartin KaFai Lau {
45851a0e301SMartin KaFai Lau struct {
45951a0e301SMartin KaFai Lau int cnt;
46051a0e301SMartin KaFai Lau int lock;
461*d59d3b8aSXu Kuohai } value = { .cnt = 0xeB9f, .lock = 1, }, lookup_value;
4622fe256a4SAndrii Nakryiko struct bpf_map_create_opts bad_xattr;
46351a0e301SMartin KaFai Lau int btf_fd, map_fd, sk_fd, err;
46451a0e301SMartin KaFai Lau
46551a0e301SMartin KaFai Lau btf_fd = load_btf();
46651a0e301SMartin KaFai Lau CHECK(btf_fd == -1, "bpf_load_btf", "btf_fd:%d errno:%d\n",
46751a0e301SMartin KaFai Lau btf_fd, errno);
4682fe256a4SAndrii Nakryiko map_opts.btf_fd = btf_fd;
46951a0e301SMartin KaFai Lau
47051a0e301SMartin KaFai Lau sk_fd = socket(AF_INET6, SOCK_STREAM, 0);
47151a0e301SMartin KaFai Lau CHECK(sk_fd == -1, "socket()", "sk_fd:%d errno:%d\n",
47251a0e301SMartin KaFai Lau sk_fd, errno);
47351a0e301SMartin KaFai Lau
4742fe256a4SAndrii Nakryiko map_fd = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &map_opts);
4752fe256a4SAndrii Nakryiko CHECK(map_fd == -1, "bpf_map_create(good_xattr)",
47651a0e301SMartin KaFai Lau "map_fd:%d errno:%d\n", map_fd, errno);
47751a0e301SMartin KaFai Lau
47851a0e301SMartin KaFai Lau /* Add new elem */
47951a0e301SMartin KaFai Lau memcpy(&lookup_value, &value, sizeof(value));
48051a0e301SMartin KaFai Lau err = bpf_map_update_elem(map_fd, &sk_fd, &value,
48151a0e301SMartin KaFai Lau BPF_NOEXIST | BPF_F_LOCK);
48251a0e301SMartin KaFai Lau CHECK(err, "bpf_map_update_elem(BPF_NOEXIST|BPF_F_LOCK)",
48351a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
48451a0e301SMartin KaFai Lau err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
48551a0e301SMartin KaFai Lau BPF_F_LOCK);
486*d59d3b8aSXu Kuohai CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,
48751a0e301SMartin KaFai Lau "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
488*d59d3b8aSXu Kuohai "err:%d errno:%d lock:%x cnt:%x(%x)\n",
489*d59d3b8aSXu Kuohai err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);
49051a0e301SMartin KaFai Lau
49151a0e301SMartin KaFai Lau /* Bump the cnt and update with BPF_EXIST | BPF_F_LOCK */
49251a0e301SMartin KaFai Lau value.cnt += 1;
493*d59d3b8aSXu Kuohai value.lock = 2;
49451a0e301SMartin KaFai Lau err = bpf_map_update_elem(map_fd, &sk_fd, &value,
49551a0e301SMartin KaFai Lau BPF_EXIST | BPF_F_LOCK);
49651a0e301SMartin KaFai Lau CHECK(err, "bpf_map_update_elem(BPF_EXIST|BPF_F_LOCK)",
49751a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
49851a0e301SMartin KaFai Lau err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
49951a0e301SMartin KaFai Lau BPF_F_LOCK);
500*d59d3b8aSXu Kuohai CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,
50151a0e301SMartin KaFai Lau "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
502*d59d3b8aSXu Kuohai "err:%d errno:%d lock:%x cnt:%x(%x)\n",
503*d59d3b8aSXu Kuohai err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);
50451a0e301SMartin KaFai Lau
50551a0e301SMartin KaFai Lau /* Bump the cnt and update with BPF_EXIST */
50651a0e301SMartin KaFai Lau value.cnt += 1;
507*d59d3b8aSXu Kuohai value.lock = 2;
50851a0e301SMartin KaFai Lau err = bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_EXIST);
50951a0e301SMartin KaFai Lau CHECK(err, "bpf_map_update_elem(BPF_EXIST)",
51051a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
51151a0e301SMartin KaFai Lau err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
51251a0e301SMartin KaFai Lau BPF_F_LOCK);
513*d59d3b8aSXu Kuohai CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,
51451a0e301SMartin KaFai Lau "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
515*d59d3b8aSXu Kuohai "err:%d errno:%d lock:%x cnt:%x(%x)\n",
516*d59d3b8aSXu Kuohai err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);
51751a0e301SMartin KaFai Lau
51851a0e301SMartin KaFai Lau /* Update with BPF_NOEXIST */
51951a0e301SMartin KaFai Lau value.cnt += 1;
520*d59d3b8aSXu Kuohai value.lock = 2;
52151a0e301SMartin KaFai Lau err = bpf_map_update_elem(map_fd, &sk_fd, &value,
52251a0e301SMartin KaFai Lau BPF_NOEXIST | BPF_F_LOCK);
52351a0e301SMartin KaFai Lau CHECK(!err || errno != EEXIST,
52451a0e301SMartin KaFai Lau "bpf_map_update_elem(BPF_NOEXIST|BPF_F_LOCK)",
52551a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
52651a0e301SMartin KaFai Lau err = bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_NOEXIST);
52751a0e301SMartin KaFai Lau CHECK(!err || errno != EEXIST, "bpf_map_update_elem(BPF_NOEXIST)",
52851a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
52951a0e301SMartin KaFai Lau value.cnt -= 1;
53051a0e301SMartin KaFai Lau err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
53151a0e301SMartin KaFai Lau BPF_F_LOCK);
532*d59d3b8aSXu Kuohai CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,
53351a0e301SMartin KaFai Lau "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
534*d59d3b8aSXu Kuohai "err:%d errno:%d lock:%x cnt:%x(%x)\n",
535*d59d3b8aSXu Kuohai err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);
53651a0e301SMartin KaFai Lau
53751a0e301SMartin KaFai Lau /* Bump the cnt again and update with map_flags == 0 */
53851a0e301SMartin KaFai Lau value.cnt += 1;
539*d59d3b8aSXu Kuohai value.lock = 2;
54051a0e301SMartin KaFai Lau err = bpf_map_update_elem(map_fd, &sk_fd, &value, 0);
54151a0e301SMartin KaFai Lau CHECK(err, "bpf_map_update_elem()", "err:%d errno:%d\n",
54251a0e301SMartin KaFai Lau err, errno);
54351a0e301SMartin KaFai Lau err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
54451a0e301SMartin KaFai Lau BPF_F_LOCK);
545*d59d3b8aSXu Kuohai CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,
54651a0e301SMartin KaFai Lau "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
547*d59d3b8aSXu Kuohai "err:%d errno:%d lock:%x cnt:%x(%x)\n",
548*d59d3b8aSXu Kuohai err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);
54951a0e301SMartin KaFai Lau
55051a0e301SMartin KaFai Lau /* Test delete elem */
55151a0e301SMartin KaFai Lau err = bpf_map_delete_elem(map_fd, &sk_fd);
55251a0e301SMartin KaFai Lau CHECK(err, "bpf_map_delete_elem()", "err:%d errno:%d\n",
55351a0e301SMartin KaFai Lau err, errno);
55451a0e301SMartin KaFai Lau err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
55551a0e301SMartin KaFai Lau BPF_F_LOCK);
55651a0e301SMartin KaFai Lau CHECK(!err || errno != ENOENT,
55751a0e301SMartin KaFai Lau "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
55851a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
55951a0e301SMartin KaFai Lau err = bpf_map_delete_elem(map_fd, &sk_fd);
56051a0e301SMartin KaFai Lau CHECK(!err || errno != ENOENT, "bpf_map_delete_elem()",
56151a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
56251a0e301SMartin KaFai Lau
5632fe256a4SAndrii Nakryiko memcpy(&bad_xattr, &map_opts, sizeof(map_opts));
56451a0e301SMartin KaFai Lau bad_xattr.btf_key_type_id = 0;
5652fe256a4SAndrii Nakryiko err = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &bad_xattr);
5662fe256a4SAndrii Nakryiko CHECK(!err || errno != EINVAL, "bpf_map_create(bad_xattr)",
56751a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
56851a0e301SMartin KaFai Lau
5692fe256a4SAndrii Nakryiko memcpy(&bad_xattr, &map_opts, sizeof(map_opts));
57051a0e301SMartin KaFai Lau bad_xattr.btf_key_type_id = 3;
5712fe256a4SAndrii Nakryiko err = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &bad_xattr);
5722fe256a4SAndrii Nakryiko CHECK(!err || errno != EINVAL, "bpf_map_create(bad_xattr)",
57351a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
57451a0e301SMartin KaFai Lau
5752fe256a4SAndrii Nakryiko err = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 1, &map_opts);
5762fe256a4SAndrii Nakryiko CHECK(!err || errno != EINVAL, "bpf_map_create(bad_xattr)",
57751a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
57851a0e301SMartin KaFai Lau
5792fe256a4SAndrii Nakryiko memcpy(&bad_xattr, &map_opts, sizeof(map_opts));
58051a0e301SMartin KaFai Lau bad_xattr.map_flags = 0;
5812fe256a4SAndrii Nakryiko err = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &bad_xattr);
58251a0e301SMartin KaFai Lau CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",
58351a0e301SMartin KaFai Lau "err:%d errno:%d\n", err, errno);
58451a0e301SMartin KaFai Lau
5852fe256a4SAndrii Nakryiko map_opts.btf_fd = -1;
58651a0e301SMartin KaFai Lau close(btf_fd);
58751a0e301SMartin KaFai Lau close(map_fd);
58851a0e301SMartin KaFai Lau close(sk_fd);
58951a0e301SMartin KaFai Lau }
59051a0e301SMartin KaFai Lau
test_sk_storage_map(void)59151a0e301SMartin KaFai Lau void test_sk_storage_map(void)
59251a0e301SMartin KaFai Lau {
59351a0e301SMartin KaFai Lau const char *test_name, *env_opt;
59451a0e301SMartin KaFai Lau bool test_ran = false;
59551a0e301SMartin KaFai Lau
59651a0e301SMartin KaFai Lau test_name = getenv(BPF_SK_STORAGE_MAP_TEST_NAME);
59751a0e301SMartin KaFai Lau
59851a0e301SMartin KaFai Lau env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_NR_THREADS);
59951a0e301SMartin KaFai Lau if (env_opt)
60051a0e301SMartin KaFai Lau nr_sk_threads = atoi(env_opt);
60151a0e301SMartin KaFai Lau
60251a0e301SMartin KaFai Lau env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD);
60351a0e301SMartin KaFai Lau if (env_opt)
60451a0e301SMartin KaFai Lau nr_sk_per_thread = atoi(env_opt);
60551a0e301SMartin KaFai Lau
60651a0e301SMartin KaFai Lau env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_RUNTIME_S);
60751a0e301SMartin KaFai Lau if (env_opt)
60851a0e301SMartin KaFai Lau runtime_s = atoi(env_opt);
60951a0e301SMartin KaFai Lau
61051a0e301SMartin KaFai Lau if (!test_name || !strcmp(test_name, "basic")) {
61151a0e301SMartin KaFai Lau test_sk_storage_map_basic();
61251a0e301SMartin KaFai Lau test_ran = true;
61351a0e301SMartin KaFai Lau }
61451a0e301SMartin KaFai Lau if (!test_name || !strcmp(test_name, "stress_free")) {
61551a0e301SMartin KaFai Lau test_sk_storage_map_stress_free();
61651a0e301SMartin KaFai Lau test_ran = true;
61751a0e301SMartin KaFai Lau }
61851a0e301SMartin KaFai Lau if (!test_name || !strcmp(test_name, "stress_change")) {
61951a0e301SMartin KaFai Lau test_sk_storage_map_stress_change();
62051a0e301SMartin KaFai Lau test_ran = true;
62151a0e301SMartin KaFai Lau }
62251a0e301SMartin KaFai Lau
62351a0e301SMartin KaFai Lau if (test_ran)
62451a0e301SMartin KaFai Lau printf("%s:PASS\n", __func__);
62551a0e301SMartin KaFai Lau else
62651a0e301SMartin KaFai Lau CHECK(1, "Invalid test_name", "%s\n", test_name);
62751a0e301SMartin KaFai Lau }
628