1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Testsuite for eBPF maps 4 * 5 * Copyright (c) 2014 PLUMgrid, http://plumgrid.com 6 * Copyright (c) 2016 Facebook 7 */ 8 9 #include <stdio.h> 10 #include <unistd.h> 11 #include <errno.h> 12 #include <string.h> 13 #include <assert.h> 14 #include <stdlib.h> 15 #include <time.h> 16 17 #include <sys/wait.h> 18 #include <sys/socket.h> 19 #include <netinet/in.h> 20 #include <linux/bpf.h> 21 22 #include <bpf/bpf.h> 23 #include <bpf/libbpf.h> 24 25 #include "bpf_util.h" 26 #include "bpf_rlimit.h" 27 #include "test_maps.h" 28 29 #ifndef ENOTSUPP 30 #define ENOTSUPP 524 31 #endif 32 33 static int skips; 34 35 static int map_flags; 36 37 static void test_hashmap(unsigned int task, void *data) 38 { 39 long long key, next_key, first_key, value; 40 int fd; 41 42 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 43 2, map_flags); 44 if (fd < 0) { 45 printf("Failed to create hashmap '%s'!\n", strerror(errno)); 46 exit(1); 47 } 48 49 key = 1; 50 value = 1234; 51 /* Insert key=1 element. */ 52 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 53 54 value = 0; 55 /* BPF_NOEXIST means add new element if it doesn't exist. */ 56 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 57 /* key=1 already exists. */ 58 errno == EEXIST); 59 60 /* -1 is an invalid flag. */ 61 assert(bpf_map_update_elem(fd, &key, &value, -1) < 0 && 62 errno == EINVAL); 63 64 /* Check that key=1 can be found. */ 65 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234); 66 67 key = 2; 68 value = 1234; 69 /* Insert key=2 element. */ 70 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 71 72 /* Check that key=2 matches the value and delete it */ 73 assert(bpf_map_lookup_and_delete_elem(fd, &key, &value) == 0 && value == 1234); 74 75 /* Check that key=2 is not found. */ 76 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT); 77 78 /* BPF_EXIST means update existing element. */ 79 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) < 0 && 80 /* key=2 is not there. */ 81 errno == ENOENT); 82 83 /* Insert key=2 element. */ 84 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0); 85 86 /* key=1 and key=2 were inserted, check that key=0 cannot be 87 * inserted due to max_entries limit. 88 */ 89 key = 0; 90 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 91 errno == E2BIG); 92 93 /* Update existing element, though the map is full. */ 94 key = 1; 95 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0); 96 key = 2; 97 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 98 key = 3; 99 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 100 errno == E2BIG); 101 102 /* Check that key = 0 doesn't exist. */ 103 key = 0; 104 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT); 105 106 /* Iterate over two elements. */ 107 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 && 108 (first_key == 1 || first_key == 2)); 109 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 && 110 (next_key == first_key)); 111 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 && 112 (next_key == 1 || next_key == 2) && 113 (next_key != first_key)); 114 assert(bpf_map_get_next_key(fd, &next_key, &next_key) < 0 && 115 errno == ENOENT); 116 117 /* Delete both elements. */ 118 key = 1; 119 assert(bpf_map_delete_elem(fd, &key) == 0); 120 key = 2; 121 assert(bpf_map_delete_elem(fd, &key) == 0); 122 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT); 123 124 key = 0; 125 /* Check that map is empty. */ 126 assert(bpf_map_get_next_key(fd, NULL, &next_key) < 0 && 127 errno == ENOENT); 128 assert(bpf_map_get_next_key(fd, &key, &next_key) < 0 && 129 errno == ENOENT); 130 131 close(fd); 132 } 133 134 static void test_hashmap_sizes(unsigned int task, void *data) 135 { 136 int fd, i, j; 137 138 for (i = 1; i <= 512; i <<= 1) 139 for (j = 1; j <= 1 << 18; j <<= 1) { 140 fd = bpf_create_map(BPF_MAP_TYPE_HASH, i, j, 141 2, map_flags); 142 if (fd < 0) { 143 if (errno == ENOMEM) 144 return; 145 printf("Failed to create hashmap key=%d value=%d '%s'\n", 146 i, j, strerror(errno)); 147 exit(1); 148 } 149 close(fd); 150 usleep(10); /* give kernel time to destroy */ 151 } 152 } 153 154 static void test_hashmap_percpu(unsigned int task, void *data) 155 { 156 unsigned int nr_cpus = bpf_num_possible_cpus(); 157 BPF_DECLARE_PERCPU(long, value); 158 long long key, next_key, first_key; 159 int expected_key_mask = 0; 160 int fd, i; 161 162 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_HASH, sizeof(key), 163 sizeof(bpf_percpu(value, 0)), 2, map_flags); 164 if (fd < 0) { 165 printf("Failed to create hashmap '%s'!\n", strerror(errno)); 166 exit(1); 167 } 168 169 for (i = 0; i < nr_cpus; i++) 170 bpf_percpu(value, i) = i + 100; 171 172 key = 1; 173 /* Insert key=1 element. */ 174 assert(!(expected_key_mask & key)); 175 assert(bpf_map_update_elem(fd, &key, value, BPF_ANY) == 0); 176 177 /* Lookup and delete elem key=1 and check value. */ 178 assert(bpf_map_lookup_and_delete_elem(fd, &key, value) == 0 && 179 bpf_percpu(value,0) == 100); 180 181 for (i = 0; i < nr_cpus; i++) 182 bpf_percpu(value,i) = i + 100; 183 184 /* Insert key=1 element which should not exist. */ 185 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == 0); 186 expected_key_mask |= key; 187 188 /* BPF_NOEXIST means add new element if it doesn't exist. */ 189 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) < 0 && 190 /* key=1 already exists. */ 191 errno == EEXIST); 192 193 /* -1 is an invalid flag. */ 194 assert(bpf_map_update_elem(fd, &key, value, -1) < 0 && 195 errno == EINVAL); 196 197 /* Check that key=1 can be found. Value could be 0 if the lookup 198 * was run from a different CPU. 199 */ 200 bpf_percpu(value, 0) = 1; 201 assert(bpf_map_lookup_elem(fd, &key, value) == 0 && 202 bpf_percpu(value, 0) == 100); 203 204 key = 2; 205 /* Check that key=2 is not found. */ 206 assert(bpf_map_lookup_elem(fd, &key, value) < 0 && errno == ENOENT); 207 208 /* BPF_EXIST means update existing element. */ 209 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) < 0 && 210 /* key=2 is not there. */ 211 errno == ENOENT); 212 213 /* Insert key=2 element. */ 214 assert(!(expected_key_mask & key)); 215 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == 0); 216 expected_key_mask |= key; 217 218 /* key=1 and key=2 were inserted, check that key=0 cannot be 219 * inserted due to max_entries limit. 220 */ 221 key = 0; 222 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) < 0 && 223 errno == E2BIG); 224 225 /* Check that key = 0 doesn't exist. */ 226 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT); 227 228 /* Iterate over two elements. */ 229 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 && 230 ((expected_key_mask & first_key) == first_key)); 231 while (!bpf_map_get_next_key(fd, &key, &next_key)) { 232 if (first_key) { 233 assert(next_key == first_key); 234 first_key = 0; 235 } 236 assert((expected_key_mask & next_key) == next_key); 237 expected_key_mask &= ~next_key; 238 239 assert(bpf_map_lookup_elem(fd, &next_key, value) == 0); 240 241 for (i = 0; i < nr_cpus; i++) 242 assert(bpf_percpu(value, i) == i + 100); 243 244 key = next_key; 245 } 246 assert(errno == ENOENT); 247 248 /* Update with BPF_EXIST. */ 249 key = 1; 250 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0); 251 252 /* Delete both elements. */ 253 key = 1; 254 assert(bpf_map_delete_elem(fd, &key) == 0); 255 key = 2; 256 assert(bpf_map_delete_elem(fd, &key) == 0); 257 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT); 258 259 key = 0; 260 /* Check that map is empty. */ 261 assert(bpf_map_get_next_key(fd, NULL, &next_key) < 0 && 262 errno == ENOENT); 263 assert(bpf_map_get_next_key(fd, &key, &next_key) < 0 && 264 errno == ENOENT); 265 266 close(fd); 267 } 268 269 static int helper_fill_hashmap(int max_entries) 270 { 271 int i, fd, ret; 272 long long key, value; 273 274 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 275 max_entries, map_flags); 276 CHECK(fd < 0, 277 "failed to create hashmap", 278 "err: %s, flags: 0x%x\n", strerror(errno), map_flags); 279 280 for (i = 0; i < max_entries; i++) { 281 key = i; value = key; 282 ret = bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST); 283 CHECK(ret != 0, 284 "can't update hashmap", 285 "err: %s\n", strerror(ret)); 286 } 287 288 return fd; 289 } 290 291 static void test_hashmap_walk(unsigned int task, void *data) 292 { 293 int fd, i, max_entries = 1000; 294 long long key, value, next_key; 295 bool next_key_valid = true; 296 297 fd = helper_fill_hashmap(max_entries); 298 299 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key, 300 &next_key) == 0; i++) { 301 key = next_key; 302 assert(bpf_map_lookup_elem(fd, &key, &value) == 0); 303 } 304 305 assert(i == max_entries); 306 307 assert(bpf_map_get_next_key(fd, NULL, &key) == 0); 308 for (i = 0; next_key_valid; i++) { 309 next_key_valid = bpf_map_get_next_key(fd, &key, &next_key) == 0; 310 assert(bpf_map_lookup_elem(fd, &key, &value) == 0); 311 value++; 312 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0); 313 key = next_key; 314 } 315 316 assert(i == max_entries); 317 318 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key, 319 &next_key) == 0; i++) { 320 key = next_key; 321 assert(bpf_map_lookup_elem(fd, &key, &value) == 0); 322 assert(value - 1 == key); 323 } 324 325 assert(i == max_entries); 326 close(fd); 327 } 328 329 static void test_hashmap_zero_seed(void) 330 { 331 int i, first, second, old_flags; 332 long long key, next_first, next_second; 333 334 old_flags = map_flags; 335 map_flags |= BPF_F_ZERO_SEED; 336 337 first = helper_fill_hashmap(3); 338 second = helper_fill_hashmap(3); 339 340 for (i = 0; ; i++) { 341 void *key_ptr = !i ? NULL : &key; 342 343 if (bpf_map_get_next_key(first, key_ptr, &next_first) != 0) 344 break; 345 346 CHECK(bpf_map_get_next_key(second, key_ptr, &next_second) != 0, 347 "next_key for second map must succeed", 348 "key_ptr: %p", key_ptr); 349 CHECK(next_first != next_second, 350 "keys must match", 351 "i: %d first: %lld second: %lld\n", i, 352 next_first, next_second); 353 354 key = next_first; 355 } 356 357 map_flags = old_flags; 358 close(first); 359 close(second); 360 } 361 362 static void test_arraymap(unsigned int task, void *data) 363 { 364 int key, next_key, fd; 365 long long value; 366 367 fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, sizeof(key), sizeof(value), 368 2, 0); 369 if (fd < 0) { 370 printf("Failed to create arraymap '%s'!\n", strerror(errno)); 371 exit(1); 372 } 373 374 key = 1; 375 value = 1234; 376 /* Insert key=1 element. */ 377 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 378 379 value = 0; 380 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 381 errno == EEXIST); 382 383 /* Check that key=1 can be found. */ 384 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234); 385 386 key = 0; 387 /* Check that key=0 is also found and zero initialized. */ 388 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0); 389 390 /* key=0 and key=1 were inserted, check that key=2 cannot be inserted 391 * due to max_entries limit. 392 */ 393 key = 2; 394 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) < 0 && 395 errno == E2BIG); 396 397 /* Check that key = 2 doesn't exist. */ 398 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT); 399 400 /* Iterate over two elements. */ 401 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 && 402 next_key == 0); 403 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 && 404 next_key == 0); 405 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 && 406 next_key == 1); 407 assert(bpf_map_get_next_key(fd, &next_key, &next_key) < 0 && 408 errno == ENOENT); 409 410 /* Delete shouldn't succeed. */ 411 key = 1; 412 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == EINVAL); 413 414 close(fd); 415 } 416 417 static void test_arraymap_percpu(unsigned int task, void *data) 418 { 419 unsigned int nr_cpus = bpf_num_possible_cpus(); 420 BPF_DECLARE_PERCPU(long, values); 421 int key, next_key, fd, i; 422 423 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key), 424 sizeof(bpf_percpu(values, 0)), 2, 0); 425 if (fd < 0) { 426 printf("Failed to create arraymap '%s'!\n", strerror(errno)); 427 exit(1); 428 } 429 430 for (i = 0; i < nr_cpus; i++) 431 bpf_percpu(values, i) = i + 100; 432 433 key = 1; 434 /* Insert key=1 element. */ 435 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0); 436 437 bpf_percpu(values, 0) = 0; 438 assert(bpf_map_update_elem(fd, &key, values, BPF_NOEXIST) < 0 && 439 errno == EEXIST); 440 441 /* Check that key=1 can be found. */ 442 assert(bpf_map_lookup_elem(fd, &key, values) == 0 && 443 bpf_percpu(values, 0) == 100); 444 445 key = 0; 446 /* Check that key=0 is also found and zero initialized. */ 447 assert(bpf_map_lookup_elem(fd, &key, values) == 0 && 448 bpf_percpu(values, 0) == 0 && 449 bpf_percpu(values, nr_cpus - 1) == 0); 450 451 /* Check that key=2 cannot be inserted due to max_entries limit. */ 452 key = 2; 453 assert(bpf_map_update_elem(fd, &key, values, BPF_EXIST) < 0 && 454 errno == E2BIG); 455 456 /* Check that key = 2 doesn't exist. */ 457 assert(bpf_map_lookup_elem(fd, &key, values) < 0 && errno == ENOENT); 458 459 /* Iterate over two elements. */ 460 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 && 461 next_key == 0); 462 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 && 463 next_key == 0); 464 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 && 465 next_key == 1); 466 assert(bpf_map_get_next_key(fd, &next_key, &next_key) < 0 && 467 errno == ENOENT); 468 469 /* Delete shouldn't succeed. */ 470 key = 1; 471 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == EINVAL); 472 473 close(fd); 474 } 475 476 static void test_arraymap_percpu_many_keys(void) 477 { 478 unsigned int nr_cpus = bpf_num_possible_cpus(); 479 BPF_DECLARE_PERCPU(long, values); 480 /* nr_keys is not too large otherwise the test stresses percpu 481 * allocator more than anything else 482 */ 483 unsigned int nr_keys = 2000; 484 int key, fd, i; 485 486 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key), 487 sizeof(bpf_percpu(values, 0)), nr_keys, 0); 488 if (fd < 0) { 489 printf("Failed to create per-cpu arraymap '%s'!\n", 490 strerror(errno)); 491 exit(1); 492 } 493 494 for (i = 0; i < nr_cpus; i++) 495 bpf_percpu(values, i) = i + 10; 496 497 for (key = 0; key < nr_keys; key++) 498 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0); 499 500 for (key = 0; key < nr_keys; key++) { 501 for (i = 0; i < nr_cpus; i++) 502 bpf_percpu(values, i) = 0; 503 504 assert(bpf_map_lookup_elem(fd, &key, values) == 0); 505 506 for (i = 0; i < nr_cpus; i++) 507 assert(bpf_percpu(values, i) == i + 10); 508 } 509 510 close(fd); 511 } 512 513 static void test_devmap(unsigned int task, void *data) 514 { 515 int fd; 516 __u32 key, value; 517 518 fd = bpf_create_map(BPF_MAP_TYPE_DEVMAP, sizeof(key), sizeof(value), 519 2, 0); 520 if (fd < 0) { 521 printf("Failed to create devmap '%s'!\n", strerror(errno)); 522 exit(1); 523 } 524 525 close(fd); 526 } 527 528 static void test_devmap_hash(unsigned int task, void *data) 529 { 530 int fd; 531 __u32 key, value; 532 533 fd = bpf_create_map(BPF_MAP_TYPE_DEVMAP_HASH, sizeof(key), sizeof(value), 534 2, 0); 535 if (fd < 0) { 536 printf("Failed to create devmap_hash '%s'!\n", strerror(errno)); 537 exit(1); 538 } 539 540 close(fd); 541 } 542 543 static void test_queuemap(unsigned int task, void *data) 544 { 545 const int MAP_SIZE = 32; 546 __u32 vals[MAP_SIZE + MAP_SIZE/2], val; 547 int fd, i; 548 549 /* Fill test values to be used */ 550 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++) 551 vals[i] = rand(); 552 553 /* Invalid key size */ 554 fd = bpf_create_map(BPF_MAP_TYPE_QUEUE, 4, sizeof(val), MAP_SIZE, 555 map_flags); 556 assert(fd < 0 && errno == EINVAL); 557 558 fd = bpf_create_map(BPF_MAP_TYPE_QUEUE, 0, sizeof(val), MAP_SIZE, 559 map_flags); 560 /* Queue map does not support BPF_F_NO_PREALLOC */ 561 if (map_flags & BPF_F_NO_PREALLOC) { 562 assert(fd < 0 && errno == EINVAL); 563 return; 564 } 565 if (fd < 0) { 566 printf("Failed to create queuemap '%s'!\n", strerror(errno)); 567 exit(1); 568 } 569 570 /* Push MAP_SIZE elements */ 571 for (i = 0; i < MAP_SIZE; i++) 572 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0); 573 574 /* Check that element cannot be pushed due to max_entries limit */ 575 assert(bpf_map_update_elem(fd, NULL, &val, 0) < 0 && 576 errno == E2BIG); 577 578 /* Peek element */ 579 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[0]); 580 581 /* Replace half elements */ 582 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++) 583 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0); 584 585 /* Pop all elements */ 586 for (i = MAP_SIZE/2; i < MAP_SIZE + MAP_SIZE/2; i++) 587 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 && 588 val == vals[i]); 589 590 /* Check that there are not elements left */ 591 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) < 0 && 592 errno == ENOENT); 593 594 /* Check that non supported functions set errno to EINVAL */ 595 assert(bpf_map_delete_elem(fd, NULL) < 0 && errno == EINVAL); 596 assert(bpf_map_get_next_key(fd, NULL, NULL) < 0 && errno == EINVAL); 597 598 close(fd); 599 } 600 601 static void test_stackmap(unsigned int task, void *data) 602 { 603 const int MAP_SIZE = 32; 604 __u32 vals[MAP_SIZE + MAP_SIZE/2], val; 605 int fd, i; 606 607 /* Fill test values to be used */ 608 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++) 609 vals[i] = rand(); 610 611 /* Invalid key size */ 612 fd = bpf_create_map(BPF_MAP_TYPE_STACK, 4, sizeof(val), MAP_SIZE, 613 map_flags); 614 assert(fd < 0 && errno == EINVAL); 615 616 fd = bpf_create_map(BPF_MAP_TYPE_STACK, 0, sizeof(val), MAP_SIZE, 617 map_flags); 618 /* Stack map does not support BPF_F_NO_PREALLOC */ 619 if (map_flags & BPF_F_NO_PREALLOC) { 620 assert(fd < 0 && errno == EINVAL); 621 return; 622 } 623 if (fd < 0) { 624 printf("Failed to create stackmap '%s'!\n", strerror(errno)); 625 exit(1); 626 } 627 628 /* Push MAP_SIZE elements */ 629 for (i = 0; i < MAP_SIZE; i++) 630 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0); 631 632 /* Check that element cannot be pushed due to max_entries limit */ 633 assert(bpf_map_update_elem(fd, NULL, &val, 0) < 0 && 634 errno == E2BIG); 635 636 /* Peek element */ 637 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[i - 1]); 638 639 /* Replace half elements */ 640 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++) 641 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0); 642 643 /* Pop all elements */ 644 for (i = MAP_SIZE + MAP_SIZE/2 - 1; i >= MAP_SIZE/2; i--) 645 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 && 646 val == vals[i]); 647 648 /* Check that there are not elements left */ 649 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) < 0 && 650 errno == ENOENT); 651 652 /* Check that non supported functions set errno to EINVAL */ 653 assert(bpf_map_delete_elem(fd, NULL) < 0 && errno == EINVAL); 654 assert(bpf_map_get_next_key(fd, NULL, NULL) < 0 && errno == EINVAL); 655 656 close(fd); 657 } 658 659 #include <sys/ioctl.h> 660 #include <arpa/inet.h> 661 #include <sys/select.h> 662 #include <linux/err.h> 663 #define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.o" 664 #define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.o" 665 #define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.o" 666 static void test_sockmap(unsigned int tasks, void *data) 667 { 668 struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_msg, *bpf_map_break; 669 int map_fd_msg = 0, map_fd_rx = 0, map_fd_tx = 0, map_fd_break; 670 int ports[] = {50200, 50201, 50202, 50204}; 671 int err, i, fd, udp, sfd[6] = {0xdeadbeef}; 672 u8 buf[20] = {0x0, 0x5, 0x3, 0x2, 0x1, 0x0}; 673 int parse_prog, verdict_prog, msg_prog; 674 struct sockaddr_in addr; 675 int one = 1, s, sc, rc; 676 struct bpf_object *obj; 677 struct timeval to; 678 __u32 key, value; 679 pid_t pid[tasks]; 680 fd_set w; 681 682 /* Create some sockets to use with sockmap */ 683 for (i = 0; i < 2; i++) { 684 sfd[i] = socket(AF_INET, SOCK_STREAM, 0); 685 if (sfd[i] < 0) 686 goto out; 687 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR, 688 (char *)&one, sizeof(one)); 689 if (err) { 690 printf("failed to setsockopt\n"); 691 goto out; 692 } 693 err = ioctl(sfd[i], FIONBIO, (char *)&one); 694 if (err < 0) { 695 printf("failed to ioctl\n"); 696 goto out; 697 } 698 memset(&addr, 0, sizeof(struct sockaddr_in)); 699 addr.sin_family = AF_INET; 700 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 701 addr.sin_port = htons(ports[i]); 702 err = bind(sfd[i], (struct sockaddr *)&addr, sizeof(addr)); 703 if (err < 0) { 704 printf("failed to bind: err %i: %i:%i\n", 705 err, i, sfd[i]); 706 goto out; 707 } 708 err = listen(sfd[i], 32); 709 if (err < 0) { 710 printf("failed to listen\n"); 711 goto out; 712 } 713 } 714 715 for (i = 2; i < 4; i++) { 716 sfd[i] = socket(AF_INET, SOCK_STREAM, 0); 717 if (sfd[i] < 0) 718 goto out; 719 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR, 720 (char *)&one, sizeof(one)); 721 if (err) { 722 printf("set sock opt\n"); 723 goto out; 724 } 725 memset(&addr, 0, sizeof(struct sockaddr_in)); 726 addr.sin_family = AF_INET; 727 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 728 addr.sin_port = htons(ports[i - 2]); 729 err = connect(sfd[i], (struct sockaddr *)&addr, sizeof(addr)); 730 if (err) { 731 printf("failed to connect\n"); 732 goto out; 733 } 734 } 735 736 737 for (i = 4; i < 6; i++) { 738 sfd[i] = accept(sfd[i - 4], NULL, NULL); 739 if (sfd[i] < 0) { 740 printf("accept failed\n"); 741 goto out; 742 } 743 } 744 745 /* Test sockmap with connected sockets */ 746 fd = bpf_create_map(BPF_MAP_TYPE_SOCKMAP, 747 sizeof(key), sizeof(value), 748 6, 0); 749 if (fd < 0) { 750 if (!bpf_probe_map_type(BPF_MAP_TYPE_SOCKMAP, 0)) { 751 printf("%s SKIP (unsupported map type BPF_MAP_TYPE_SOCKMAP)\n", 752 __func__); 753 skips++; 754 for (i = 0; i < 6; i++) 755 close(sfd[i]); 756 return; 757 } 758 759 printf("Failed to create sockmap %i\n", fd); 760 goto out_sockmap; 761 } 762 763 /* Test update with unsupported UDP socket */ 764 udp = socket(AF_INET, SOCK_DGRAM, 0); 765 i = 0; 766 err = bpf_map_update_elem(fd, &i, &udp, BPF_ANY); 767 if (err) { 768 printf("Failed socket update SOCK_DGRAM '%i:%i'\n", 769 i, udp); 770 goto out_sockmap; 771 } 772 773 /* Test update without programs */ 774 for (i = 0; i < 6; i++) { 775 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY); 776 if (err) { 777 printf("Failed noprog update sockmap '%i:%i'\n", 778 i, sfd[i]); 779 goto out_sockmap; 780 } 781 } 782 783 /* Test attaching/detaching bad fds */ 784 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_PARSER, 0); 785 if (!err) { 786 printf("Failed invalid parser prog attach\n"); 787 goto out_sockmap; 788 } 789 790 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_VERDICT, 0); 791 if (!err) { 792 printf("Failed invalid verdict prog attach\n"); 793 goto out_sockmap; 794 } 795 796 err = bpf_prog_attach(-1, fd, BPF_SK_MSG_VERDICT, 0); 797 if (!err) { 798 printf("Failed invalid msg verdict prog attach\n"); 799 goto out_sockmap; 800 } 801 802 err = bpf_prog_attach(-1, fd, __MAX_BPF_ATTACH_TYPE, 0); 803 if (!err) { 804 printf("Failed unknown prog attach\n"); 805 goto out_sockmap; 806 } 807 808 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_PARSER); 809 if (!err) { 810 printf("Failed empty parser prog detach\n"); 811 goto out_sockmap; 812 } 813 814 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_VERDICT); 815 if (!err) { 816 printf("Failed empty verdict prog detach\n"); 817 goto out_sockmap; 818 } 819 820 err = bpf_prog_detach(fd, BPF_SK_MSG_VERDICT); 821 if (!err) { 822 printf("Failed empty msg verdict prog detach\n"); 823 goto out_sockmap; 824 } 825 826 err = bpf_prog_detach(fd, __MAX_BPF_ATTACH_TYPE); 827 if (!err) { 828 printf("Detach invalid prog successful\n"); 829 goto out_sockmap; 830 } 831 832 /* Load SK_SKB program and Attach */ 833 err = bpf_prog_load(SOCKMAP_PARSE_PROG, 834 BPF_PROG_TYPE_SK_SKB, &obj, &parse_prog); 835 if (err) { 836 printf("Failed to load SK_SKB parse prog\n"); 837 goto out_sockmap; 838 } 839 840 err = bpf_prog_load(SOCKMAP_TCP_MSG_PROG, 841 BPF_PROG_TYPE_SK_MSG, &obj, &msg_prog); 842 if (err) { 843 printf("Failed to load SK_SKB msg prog\n"); 844 goto out_sockmap; 845 } 846 847 err = bpf_prog_load(SOCKMAP_VERDICT_PROG, 848 BPF_PROG_TYPE_SK_SKB, &obj, &verdict_prog); 849 if (err) { 850 printf("Failed to load SK_SKB verdict prog\n"); 851 goto out_sockmap; 852 } 853 854 bpf_map_rx = bpf_object__find_map_by_name(obj, "sock_map_rx"); 855 if (!bpf_map_rx) { 856 printf("Failed to load map rx from verdict prog\n"); 857 goto out_sockmap; 858 } 859 860 map_fd_rx = bpf_map__fd(bpf_map_rx); 861 if (map_fd_rx < 0) { 862 printf("Failed to get map rx fd\n"); 863 goto out_sockmap; 864 } 865 866 bpf_map_tx = bpf_object__find_map_by_name(obj, "sock_map_tx"); 867 if (!bpf_map_tx) { 868 printf("Failed to load map tx from verdict prog\n"); 869 goto out_sockmap; 870 } 871 872 map_fd_tx = bpf_map__fd(bpf_map_tx); 873 if (map_fd_tx < 0) { 874 printf("Failed to get map tx fd\n"); 875 goto out_sockmap; 876 } 877 878 bpf_map_msg = bpf_object__find_map_by_name(obj, "sock_map_msg"); 879 if (!bpf_map_msg) { 880 printf("Failed to load map msg from msg_verdict prog\n"); 881 goto out_sockmap; 882 } 883 884 map_fd_msg = bpf_map__fd(bpf_map_msg); 885 if (map_fd_msg < 0) { 886 printf("Failed to get map msg fd\n"); 887 goto out_sockmap; 888 } 889 890 bpf_map_break = bpf_object__find_map_by_name(obj, "sock_map_break"); 891 if (!bpf_map_break) { 892 printf("Failed to load map tx from verdict prog\n"); 893 goto out_sockmap; 894 } 895 896 map_fd_break = bpf_map__fd(bpf_map_break); 897 if (map_fd_break < 0) { 898 printf("Failed to get map tx fd\n"); 899 goto out_sockmap; 900 } 901 902 err = bpf_prog_attach(parse_prog, map_fd_break, 903 BPF_SK_SKB_STREAM_PARSER, 0); 904 if (!err) { 905 printf("Allowed attaching SK_SKB program to invalid map\n"); 906 goto out_sockmap; 907 } 908 909 err = bpf_prog_attach(parse_prog, map_fd_rx, 910 BPF_SK_SKB_STREAM_PARSER, 0); 911 if (err) { 912 printf("Failed stream parser bpf prog attach\n"); 913 goto out_sockmap; 914 } 915 916 err = bpf_prog_attach(verdict_prog, map_fd_rx, 917 BPF_SK_SKB_STREAM_VERDICT, 0); 918 if (err) { 919 printf("Failed stream verdict bpf prog attach\n"); 920 goto out_sockmap; 921 } 922 923 err = bpf_prog_attach(msg_prog, map_fd_msg, BPF_SK_MSG_VERDICT, 0); 924 if (err) { 925 printf("Failed msg verdict bpf prog attach\n"); 926 goto out_sockmap; 927 } 928 929 err = bpf_prog_attach(verdict_prog, map_fd_rx, 930 __MAX_BPF_ATTACH_TYPE, 0); 931 if (!err) { 932 printf("Attached unknown bpf prog\n"); 933 goto out_sockmap; 934 } 935 936 /* Test map update elem afterwards fd lives in fd and map_fd */ 937 for (i = 2; i < 6; i++) { 938 err = bpf_map_update_elem(map_fd_rx, &i, &sfd[i], BPF_ANY); 939 if (err) { 940 printf("Failed map_fd_rx update sockmap %i '%i:%i'\n", 941 err, i, sfd[i]); 942 goto out_sockmap; 943 } 944 err = bpf_map_update_elem(map_fd_tx, &i, &sfd[i], BPF_ANY); 945 if (err) { 946 printf("Failed map_fd_tx update sockmap %i '%i:%i'\n", 947 err, i, sfd[i]); 948 goto out_sockmap; 949 } 950 } 951 952 /* Test map delete elem and remove send/recv sockets */ 953 for (i = 2; i < 4; i++) { 954 err = bpf_map_delete_elem(map_fd_rx, &i); 955 if (err) { 956 printf("Failed delete sockmap rx %i '%i:%i'\n", 957 err, i, sfd[i]); 958 goto out_sockmap; 959 } 960 err = bpf_map_delete_elem(map_fd_tx, &i); 961 if (err) { 962 printf("Failed delete sockmap tx %i '%i:%i'\n", 963 err, i, sfd[i]); 964 goto out_sockmap; 965 } 966 } 967 968 /* Put sfd[2] (sending fd below) into msg map to test sendmsg bpf */ 969 i = 0; 970 err = bpf_map_update_elem(map_fd_msg, &i, &sfd[2], BPF_ANY); 971 if (err) { 972 printf("Failed map_fd_msg update sockmap %i\n", err); 973 goto out_sockmap; 974 } 975 976 /* Test map send/recv */ 977 for (i = 0; i < 2; i++) { 978 buf[0] = i; 979 buf[1] = 0x5; 980 sc = send(sfd[2], buf, 20, 0); 981 if (sc < 0) { 982 printf("Failed sockmap send\n"); 983 goto out_sockmap; 984 } 985 986 FD_ZERO(&w); 987 FD_SET(sfd[3], &w); 988 to.tv_sec = 30; 989 to.tv_usec = 0; 990 s = select(sfd[3] + 1, &w, NULL, NULL, &to); 991 if (s == -1) { 992 perror("Failed sockmap select()"); 993 goto out_sockmap; 994 } else if (!s) { 995 printf("Failed sockmap unexpected timeout\n"); 996 goto out_sockmap; 997 } 998 999 if (!FD_ISSET(sfd[3], &w)) { 1000 printf("Failed sockmap select/recv\n"); 1001 goto out_sockmap; 1002 } 1003 1004 rc = recv(sfd[3], buf, sizeof(buf), 0); 1005 if (rc < 0) { 1006 printf("Failed sockmap recv\n"); 1007 goto out_sockmap; 1008 } 1009 } 1010 1011 /* Negative null entry lookup from datapath should be dropped */ 1012 buf[0] = 1; 1013 buf[1] = 12; 1014 sc = send(sfd[2], buf, 20, 0); 1015 if (sc < 0) { 1016 printf("Failed sockmap send\n"); 1017 goto out_sockmap; 1018 } 1019 1020 /* Push fd into same slot */ 1021 i = 2; 1022 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST); 1023 if (!err) { 1024 printf("Failed allowed sockmap dup slot BPF_NOEXIST\n"); 1025 goto out_sockmap; 1026 } 1027 1028 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY); 1029 if (err) { 1030 printf("Failed sockmap update new slot BPF_ANY\n"); 1031 goto out_sockmap; 1032 } 1033 1034 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST); 1035 if (err) { 1036 printf("Failed sockmap update new slot BPF_EXIST\n"); 1037 goto out_sockmap; 1038 } 1039 1040 /* Delete the elems without programs */ 1041 for (i = 2; i < 6; i++) { 1042 err = bpf_map_delete_elem(fd, &i); 1043 if (err) { 1044 printf("Failed delete sockmap %i '%i:%i'\n", 1045 err, i, sfd[i]); 1046 } 1047 } 1048 1049 /* Test having multiple maps open and set with programs on same fds */ 1050 err = bpf_prog_attach(parse_prog, fd, 1051 BPF_SK_SKB_STREAM_PARSER, 0); 1052 if (err) { 1053 printf("Failed fd bpf parse prog attach\n"); 1054 goto out_sockmap; 1055 } 1056 err = bpf_prog_attach(verdict_prog, fd, 1057 BPF_SK_SKB_STREAM_VERDICT, 0); 1058 if (err) { 1059 printf("Failed fd bpf verdict prog attach\n"); 1060 goto out_sockmap; 1061 } 1062 1063 for (i = 4; i < 6; i++) { 1064 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY); 1065 if (!err) { 1066 printf("Failed allowed duplicate programs in update ANY sockmap %i '%i:%i'\n", 1067 err, i, sfd[i]); 1068 goto out_sockmap; 1069 } 1070 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST); 1071 if (!err) { 1072 printf("Failed allowed duplicate program in update NOEXIST sockmap %i '%i:%i'\n", 1073 err, i, sfd[i]); 1074 goto out_sockmap; 1075 } 1076 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST); 1077 if (!err) { 1078 printf("Failed allowed duplicate program in update EXIST sockmap %i '%i:%i'\n", 1079 err, i, sfd[i]); 1080 goto out_sockmap; 1081 } 1082 } 1083 1084 /* Test tasks number of forked operations */ 1085 for (i = 0; i < tasks; i++) { 1086 pid[i] = fork(); 1087 if (pid[i] == 0) { 1088 for (i = 0; i < 6; i++) { 1089 bpf_map_delete_elem(map_fd_tx, &i); 1090 bpf_map_delete_elem(map_fd_rx, &i); 1091 bpf_map_update_elem(map_fd_tx, &i, 1092 &sfd[i], BPF_ANY); 1093 bpf_map_update_elem(map_fd_rx, &i, 1094 &sfd[i], BPF_ANY); 1095 } 1096 exit(0); 1097 } else if (pid[i] == -1) { 1098 printf("Couldn't spawn #%d process!\n", i); 1099 exit(1); 1100 } 1101 } 1102 1103 for (i = 0; i < tasks; i++) { 1104 int status; 1105 1106 assert(waitpid(pid[i], &status, 0) == pid[i]); 1107 assert(status == 0); 1108 } 1109 1110 err = bpf_prog_detach2(parse_prog, map_fd_rx, __MAX_BPF_ATTACH_TYPE); 1111 if (!err) { 1112 printf("Detached an invalid prog type.\n"); 1113 goto out_sockmap; 1114 } 1115 1116 err = bpf_prog_detach2(parse_prog, map_fd_rx, BPF_SK_SKB_STREAM_PARSER); 1117 if (err) { 1118 printf("Failed parser prog detach\n"); 1119 goto out_sockmap; 1120 } 1121 1122 err = bpf_prog_detach2(verdict_prog, map_fd_rx, BPF_SK_SKB_STREAM_VERDICT); 1123 if (err) { 1124 printf("Failed parser prog detach\n"); 1125 goto out_sockmap; 1126 } 1127 1128 /* Test map close sockets and empty maps */ 1129 for (i = 0; i < 6; i++) { 1130 bpf_map_delete_elem(map_fd_tx, &i); 1131 bpf_map_delete_elem(map_fd_rx, &i); 1132 close(sfd[i]); 1133 } 1134 close(fd); 1135 close(map_fd_rx); 1136 bpf_object__close(obj); 1137 return; 1138 out: 1139 for (i = 0; i < 6; i++) 1140 close(sfd[i]); 1141 printf("Failed to create sockmap '%i:%s'!\n", i, strerror(errno)); 1142 exit(1); 1143 out_sockmap: 1144 for (i = 0; i < 6; i++) { 1145 if (map_fd_tx) 1146 bpf_map_delete_elem(map_fd_tx, &i); 1147 if (map_fd_rx) 1148 bpf_map_delete_elem(map_fd_rx, &i); 1149 close(sfd[i]); 1150 } 1151 close(fd); 1152 exit(1); 1153 } 1154 1155 #define MAPINMAP_PROG "./test_map_in_map.o" 1156 #define MAPINMAP_INVALID_PROG "./test_map_in_map_invalid.o" 1157 static void test_map_in_map(void) 1158 { 1159 struct bpf_object *obj; 1160 struct bpf_map *map; 1161 int mim_fd, fd, err; 1162 int pos = 0; 1163 struct bpf_map_info info = {}; 1164 __u32 len = sizeof(info); 1165 __u32 id = 0; 1166 libbpf_print_fn_t old_print_fn; 1167 1168 obj = bpf_object__open(MAPINMAP_PROG); 1169 1170 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(int), sizeof(int), 1171 2, 0); 1172 if (fd < 0) { 1173 printf("Failed to create hashmap '%s'!\n", strerror(errno)); 1174 exit(1); 1175 } 1176 1177 map = bpf_object__find_map_by_name(obj, "mim_array"); 1178 if (!map) { 1179 printf("Failed to load array of maps from test prog\n"); 1180 goto out_map_in_map; 1181 } 1182 err = bpf_map__set_inner_map_fd(map, fd); 1183 if (err) { 1184 printf("Failed to set inner_map_fd for array of maps\n"); 1185 goto out_map_in_map; 1186 } 1187 1188 map = bpf_object__find_map_by_name(obj, "mim_hash"); 1189 if (!map) { 1190 printf("Failed to load hash of maps from test prog\n"); 1191 goto out_map_in_map; 1192 } 1193 err = bpf_map__set_inner_map_fd(map, fd); 1194 if (err) { 1195 printf("Failed to set inner_map_fd for hash of maps\n"); 1196 goto out_map_in_map; 1197 } 1198 1199 bpf_object__load(obj); 1200 1201 map = bpf_object__find_map_by_name(obj, "mim_array"); 1202 if (!map) { 1203 printf("Failed to load array of maps from test prog\n"); 1204 goto out_map_in_map; 1205 } 1206 mim_fd = bpf_map__fd(map); 1207 if (mim_fd < 0) { 1208 printf("Failed to get descriptor for array of maps\n"); 1209 goto out_map_in_map; 1210 } 1211 1212 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0); 1213 if (err) { 1214 printf("Failed to update array of maps\n"); 1215 goto out_map_in_map; 1216 } 1217 1218 map = bpf_object__find_map_by_name(obj, "mim_hash"); 1219 if (!map) { 1220 printf("Failed to load hash of maps from test prog\n"); 1221 goto out_map_in_map; 1222 } 1223 mim_fd = bpf_map__fd(map); 1224 if (mim_fd < 0) { 1225 printf("Failed to get descriptor for hash of maps\n"); 1226 goto out_map_in_map; 1227 } 1228 1229 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0); 1230 if (err) { 1231 printf("Failed to update hash of maps\n"); 1232 goto out_map_in_map; 1233 } 1234 1235 close(fd); 1236 fd = -1; 1237 bpf_object__close(obj); 1238 1239 /* Test that failing bpf_object__create_map() destroys the inner map */ 1240 obj = bpf_object__open(MAPINMAP_INVALID_PROG); 1241 err = libbpf_get_error(obj); 1242 if (err) { 1243 printf("Failed to load %s program: %d %d", 1244 MAPINMAP_INVALID_PROG, err, errno); 1245 goto out_map_in_map; 1246 } 1247 1248 map = bpf_object__find_map_by_name(obj, "mim"); 1249 if (!map) { 1250 printf("Failed to load array of maps from test prog\n"); 1251 goto out_map_in_map; 1252 } 1253 1254 old_print_fn = libbpf_set_print(NULL); 1255 1256 err = bpf_object__load(obj); 1257 if (!err) { 1258 printf("Loading obj supposed to fail\n"); 1259 goto out_map_in_map; 1260 } 1261 1262 libbpf_set_print(old_print_fn); 1263 1264 /* Iterate over all maps to check whether the internal map 1265 * ("mim.internal") has been destroyed. 1266 */ 1267 while (true) { 1268 err = bpf_map_get_next_id(id, &id); 1269 if (err) { 1270 if (errno == ENOENT) 1271 break; 1272 printf("Failed to get next map: %d", errno); 1273 goto out_map_in_map; 1274 } 1275 1276 fd = bpf_map_get_fd_by_id(id); 1277 if (fd < 0) { 1278 if (errno == ENOENT) 1279 continue; 1280 printf("Failed to get map by id %u: %d", id, errno); 1281 goto out_map_in_map; 1282 } 1283 1284 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1285 if (err) { 1286 printf("Failed to get map info by fd %d: %d", fd, 1287 errno); 1288 goto out_map_in_map; 1289 } 1290 1291 if (!strcmp(info.name, "mim.inner")) { 1292 printf("Inner map mim.inner was not destroyed\n"); 1293 goto out_map_in_map; 1294 } 1295 } 1296 1297 return; 1298 1299 out_map_in_map: 1300 if (fd >= 0) 1301 close(fd); 1302 exit(1); 1303 } 1304 1305 #define MAP_SIZE (32 * 1024) 1306 1307 static void test_map_large(void) 1308 { 1309 1310 struct bigkey { 1311 int a; 1312 char b[4096]; 1313 long long c; 1314 } key; 1315 int fd, i, value; 1316 1317 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1318 MAP_SIZE, map_flags); 1319 if (fd < 0) { 1320 printf("Failed to create large map '%s'!\n", strerror(errno)); 1321 exit(1); 1322 } 1323 1324 for (i = 0; i < MAP_SIZE; i++) { 1325 key = (struct bigkey) { .c = i }; 1326 value = i; 1327 1328 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0); 1329 } 1330 1331 key.c = -1; 1332 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 1333 errno == E2BIG); 1334 1335 /* Iterate through all elements. */ 1336 assert(bpf_map_get_next_key(fd, NULL, &key) == 0); 1337 key.c = -1; 1338 for (i = 0; i < MAP_SIZE; i++) 1339 assert(bpf_map_get_next_key(fd, &key, &key) == 0); 1340 assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT); 1341 1342 key.c = 0; 1343 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0); 1344 key.a = 1; 1345 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT); 1346 1347 close(fd); 1348 } 1349 1350 #define run_parallel(N, FN, DATA) \ 1351 printf("Fork %u tasks to '" #FN "'\n", N); \ 1352 __run_parallel(N, FN, DATA) 1353 1354 static void __run_parallel(unsigned int tasks, 1355 void (*fn)(unsigned int task, void *data), 1356 void *data) 1357 { 1358 pid_t pid[tasks]; 1359 int i; 1360 1361 fflush(stdout); 1362 1363 for (i = 0; i < tasks; i++) { 1364 pid[i] = fork(); 1365 if (pid[i] == 0) { 1366 fn(i, data); 1367 exit(0); 1368 } else if (pid[i] == -1) { 1369 printf("Couldn't spawn #%d process!\n", i); 1370 exit(1); 1371 } 1372 } 1373 1374 for (i = 0; i < tasks; i++) { 1375 int status; 1376 1377 assert(waitpid(pid[i], &status, 0) == pid[i]); 1378 assert(status == 0); 1379 } 1380 } 1381 1382 static void test_map_stress(void) 1383 { 1384 run_parallel(100, test_hashmap, NULL); 1385 run_parallel(100, test_hashmap_percpu, NULL); 1386 run_parallel(100, test_hashmap_sizes, NULL); 1387 run_parallel(100, test_hashmap_walk, NULL); 1388 1389 run_parallel(100, test_arraymap, NULL); 1390 run_parallel(100, test_arraymap_percpu, NULL); 1391 } 1392 1393 #define TASKS 1024 1394 1395 #define DO_UPDATE 1 1396 #define DO_DELETE 0 1397 1398 #define MAP_RETRIES 20 1399 #define MAX_DELAY_US 50000 1400 #define MIN_DELAY_RANGE_US 5000 1401 1402 static int map_update_retriable(int map_fd, const void *key, const void *value, 1403 int flags, int attempts) 1404 { 1405 int delay = rand() % MIN_DELAY_RANGE_US; 1406 1407 while (bpf_map_update_elem(map_fd, key, value, flags)) { 1408 if (!attempts || (errno != EAGAIN && errno != EBUSY)) 1409 return -errno; 1410 1411 if (delay <= MAX_DELAY_US / 2) 1412 delay *= 2; 1413 1414 usleep(delay); 1415 attempts--; 1416 } 1417 1418 return 0; 1419 } 1420 1421 static int map_delete_retriable(int map_fd, const void *key, int attempts) 1422 { 1423 int delay = rand() % MIN_DELAY_RANGE_US; 1424 1425 while (bpf_map_delete_elem(map_fd, key)) { 1426 if (!attempts || (errno != EAGAIN && errno != EBUSY)) 1427 return -errno; 1428 1429 if (delay <= MAX_DELAY_US / 2) 1430 delay *= 2; 1431 1432 usleep(delay); 1433 attempts--; 1434 } 1435 1436 return 0; 1437 } 1438 1439 static void test_update_delete(unsigned int fn, void *data) 1440 { 1441 int do_update = ((int *)data)[1]; 1442 int fd = ((int *)data)[0]; 1443 int i, key, value, err; 1444 1445 for (i = fn; i < MAP_SIZE; i += TASKS) { 1446 key = value = i; 1447 1448 if (do_update) { 1449 err = map_update_retriable(fd, &key, &value, BPF_NOEXIST, MAP_RETRIES); 1450 if (err) 1451 printf("error %d %d\n", err, errno); 1452 assert(err == 0); 1453 err = map_update_retriable(fd, &key, &value, BPF_EXIST, MAP_RETRIES); 1454 if (err) 1455 printf("error %d %d\n", err, errno); 1456 assert(err == 0); 1457 } else { 1458 err = map_delete_retriable(fd, &key, MAP_RETRIES); 1459 if (err) 1460 printf("error %d %d\n", err, errno); 1461 assert(err == 0); 1462 } 1463 } 1464 } 1465 1466 static void test_map_parallel(void) 1467 { 1468 int i, fd, key = 0, value = 0; 1469 int data[2]; 1470 1471 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1472 MAP_SIZE, map_flags); 1473 if (fd < 0) { 1474 printf("Failed to create map for parallel test '%s'!\n", 1475 strerror(errno)); 1476 exit(1); 1477 } 1478 1479 /* Use the same fd in children to add elements to this map: 1480 * child_0 adds key=0, key=1024, key=2048, ... 1481 * child_1 adds key=1, key=1025, key=2049, ... 1482 * child_1023 adds key=1023, ... 1483 */ 1484 data[0] = fd; 1485 data[1] = DO_UPDATE; 1486 run_parallel(TASKS, test_update_delete, data); 1487 1488 /* Check that key=0 is already there. */ 1489 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 1490 errno == EEXIST); 1491 1492 /* Check that all elements were inserted. */ 1493 assert(bpf_map_get_next_key(fd, NULL, &key) == 0); 1494 key = -1; 1495 for (i = 0; i < MAP_SIZE; i++) 1496 assert(bpf_map_get_next_key(fd, &key, &key) == 0); 1497 assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT); 1498 1499 /* Another check for all elements */ 1500 for (i = 0; i < MAP_SIZE; i++) { 1501 key = MAP_SIZE - i - 1; 1502 1503 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && 1504 value == key); 1505 } 1506 1507 /* Now let's delete all elemenets in parallel. */ 1508 data[1] = DO_DELETE; 1509 run_parallel(TASKS, test_update_delete, data); 1510 1511 /* Nothing should be left. */ 1512 key = -1; 1513 assert(bpf_map_get_next_key(fd, NULL, &key) < 0 && errno == ENOENT); 1514 assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT); 1515 } 1516 1517 static void test_map_rdonly(void) 1518 { 1519 int fd, key = 0, value = 0; 1520 1521 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1522 MAP_SIZE, map_flags | BPF_F_RDONLY); 1523 if (fd < 0) { 1524 printf("Failed to create map for read only test '%s'!\n", 1525 strerror(errno)); 1526 exit(1); 1527 } 1528 1529 key = 1; 1530 value = 1234; 1531 /* Try to insert key=1 element. */ 1532 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) < 0 && 1533 errno == EPERM); 1534 1535 /* Check that key=1 is not found. */ 1536 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT); 1537 assert(bpf_map_get_next_key(fd, &key, &value) < 0 && errno == ENOENT); 1538 1539 close(fd); 1540 } 1541 1542 static void test_map_wronly_hash(void) 1543 { 1544 int fd, key = 0, value = 0; 1545 1546 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1547 MAP_SIZE, map_flags | BPF_F_WRONLY); 1548 if (fd < 0) { 1549 printf("Failed to create map for write only test '%s'!\n", 1550 strerror(errno)); 1551 exit(1); 1552 } 1553 1554 key = 1; 1555 value = 1234; 1556 /* Insert key=1 element. */ 1557 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 1558 1559 /* Check that reading elements and keys from the map is not allowed. */ 1560 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == EPERM); 1561 assert(bpf_map_get_next_key(fd, &key, &value) < 0 && errno == EPERM); 1562 1563 close(fd); 1564 } 1565 1566 static void test_map_wronly_stack_or_queue(enum bpf_map_type map_type) 1567 { 1568 int fd, value = 0; 1569 1570 assert(map_type == BPF_MAP_TYPE_QUEUE || 1571 map_type == BPF_MAP_TYPE_STACK); 1572 fd = bpf_create_map(map_type, 0, sizeof(value), MAP_SIZE, 1573 map_flags | BPF_F_WRONLY); 1574 /* Stack/Queue maps do not support BPF_F_NO_PREALLOC */ 1575 if (map_flags & BPF_F_NO_PREALLOC) { 1576 assert(fd < 0 && errno == EINVAL); 1577 return; 1578 } 1579 if (fd < 0) { 1580 printf("Failed to create map '%s'!\n", strerror(errno)); 1581 exit(1); 1582 } 1583 1584 value = 1234; 1585 assert(bpf_map_update_elem(fd, NULL, &value, BPF_ANY) == 0); 1586 1587 /* Peek element should fail */ 1588 assert(bpf_map_lookup_elem(fd, NULL, &value) < 0 && errno == EPERM); 1589 1590 /* Pop element should fail */ 1591 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &value) < 0 && 1592 errno == EPERM); 1593 1594 close(fd); 1595 } 1596 1597 static void test_map_wronly(void) 1598 { 1599 test_map_wronly_hash(); 1600 test_map_wronly_stack_or_queue(BPF_MAP_TYPE_STACK); 1601 test_map_wronly_stack_or_queue(BPF_MAP_TYPE_QUEUE); 1602 } 1603 1604 static void prepare_reuseport_grp(int type, int map_fd, size_t map_elem_size, 1605 __s64 *fds64, __u64 *sk_cookies, 1606 unsigned int n) 1607 { 1608 socklen_t optlen, addrlen; 1609 struct sockaddr_in6 s6; 1610 const __u32 index0 = 0; 1611 const int optval = 1; 1612 unsigned int i; 1613 u64 sk_cookie; 1614 void *value; 1615 __s32 fd32; 1616 __s64 fd64; 1617 int err; 1618 1619 s6.sin6_family = AF_INET6; 1620 s6.sin6_addr = in6addr_any; 1621 s6.sin6_port = 0; 1622 addrlen = sizeof(s6); 1623 optlen = sizeof(sk_cookie); 1624 1625 for (i = 0; i < n; i++) { 1626 fd64 = socket(AF_INET6, type, 0); 1627 CHECK(fd64 == -1, "socket()", 1628 "sock_type:%d fd64:%lld errno:%d\n", 1629 type, fd64, errno); 1630 1631 err = setsockopt(fd64, SOL_SOCKET, SO_REUSEPORT, 1632 &optval, sizeof(optval)); 1633 CHECK(err == -1, "setsockopt(SO_REUSEPORT)", 1634 "err:%d errno:%d\n", err, errno); 1635 1636 /* reuseport_array does not allow unbound sk */ 1637 if (map_elem_size == sizeof(__u64)) 1638 value = &fd64; 1639 else { 1640 assert(map_elem_size == sizeof(__u32)); 1641 fd32 = (__s32)fd64; 1642 value = &fd32; 1643 } 1644 err = bpf_map_update_elem(map_fd, &index0, value, BPF_ANY); 1645 CHECK(err >= 0 || errno != EINVAL, 1646 "reuseport array update unbound sk", 1647 "sock_type:%d err:%d errno:%d\n", 1648 type, err, errno); 1649 1650 err = bind(fd64, (struct sockaddr *)&s6, sizeof(s6)); 1651 CHECK(err == -1, "bind()", 1652 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1653 1654 if (i == 0) { 1655 err = getsockname(fd64, (struct sockaddr *)&s6, 1656 &addrlen); 1657 CHECK(err == -1, "getsockname()", 1658 "sock_type:%d err:%d errno:%d\n", 1659 type, err, errno); 1660 } 1661 1662 err = getsockopt(fd64, SOL_SOCKET, SO_COOKIE, &sk_cookie, 1663 &optlen); 1664 CHECK(err == -1, "getsockopt(SO_COOKIE)", 1665 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1666 1667 if (type == SOCK_STREAM) { 1668 /* 1669 * reuseport_array does not allow 1670 * non-listening tcp sk. 1671 */ 1672 err = bpf_map_update_elem(map_fd, &index0, value, 1673 BPF_ANY); 1674 CHECK(err >= 0 || errno != EINVAL, 1675 "reuseport array update non-listening sk", 1676 "sock_type:%d err:%d errno:%d\n", 1677 type, err, errno); 1678 err = listen(fd64, 0); 1679 CHECK(err == -1, "listen()", 1680 "sock_type:%d, err:%d errno:%d\n", 1681 type, err, errno); 1682 } 1683 1684 fds64[i] = fd64; 1685 sk_cookies[i] = sk_cookie; 1686 } 1687 } 1688 1689 static void test_reuseport_array(void) 1690 { 1691 #define REUSEPORT_FD_IDX(err, last) ({ (err) ? last : !last; }) 1692 1693 const __u32 array_size = 4, index0 = 0, index3 = 3; 1694 int types[2] = { SOCK_STREAM, SOCK_DGRAM }, type; 1695 __u64 grpa_cookies[2], sk_cookie, map_cookie; 1696 __s64 grpa_fds64[2] = { -1, -1 }, fd64 = -1; 1697 const __u32 bad_index = array_size; 1698 int map_fd, err, t, f; 1699 __u32 fds_idx = 0; 1700 int fd; 1701 1702 map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, 1703 sizeof(__u32), sizeof(__u64), array_size, 0); 1704 CHECK(map_fd < 0, "reuseport array create", 1705 "map_fd:%d, errno:%d\n", map_fd, errno); 1706 1707 /* Test lookup/update/delete with invalid index */ 1708 err = bpf_map_delete_elem(map_fd, &bad_index); 1709 CHECK(err >= 0 || errno != E2BIG, "reuseport array del >=max_entries", 1710 "err:%d errno:%d\n", err, errno); 1711 1712 err = bpf_map_update_elem(map_fd, &bad_index, &fd64, BPF_ANY); 1713 CHECK(err >= 0 || errno != E2BIG, 1714 "reuseport array update >=max_entries", 1715 "err:%d errno:%d\n", err, errno); 1716 1717 err = bpf_map_lookup_elem(map_fd, &bad_index, &map_cookie); 1718 CHECK(err >= 0 || errno != ENOENT, 1719 "reuseport array update >=max_entries", 1720 "err:%d errno:%d\n", err, errno); 1721 1722 /* Test lookup/delete non existence elem */ 1723 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1724 CHECK(err >= 0 || errno != ENOENT, 1725 "reuseport array lookup not-exist elem", 1726 "err:%d errno:%d\n", err, errno); 1727 err = bpf_map_delete_elem(map_fd, &index3); 1728 CHECK(err >= 0 || errno != ENOENT, 1729 "reuseport array del not-exist elem", 1730 "err:%d errno:%d\n", err, errno); 1731 1732 for (t = 0; t < ARRAY_SIZE(types); t++) { 1733 type = types[t]; 1734 1735 prepare_reuseport_grp(type, map_fd, sizeof(__u64), grpa_fds64, 1736 grpa_cookies, ARRAY_SIZE(grpa_fds64)); 1737 1738 /* Test BPF_* update flags */ 1739 /* BPF_EXIST failure case */ 1740 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1741 BPF_EXIST); 1742 CHECK(err >= 0 || errno != ENOENT, 1743 "reuseport array update empty elem BPF_EXIST", 1744 "sock_type:%d err:%d errno:%d\n", 1745 type, err, errno); 1746 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1747 1748 /* BPF_NOEXIST success case */ 1749 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1750 BPF_NOEXIST); 1751 CHECK(err < 0, 1752 "reuseport array update empty elem BPF_NOEXIST", 1753 "sock_type:%d err:%d errno:%d\n", 1754 type, err, errno); 1755 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1756 1757 /* BPF_EXIST success case. */ 1758 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1759 BPF_EXIST); 1760 CHECK(err < 0, 1761 "reuseport array update same elem BPF_EXIST", 1762 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1763 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1764 1765 /* BPF_NOEXIST failure case */ 1766 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1767 BPF_NOEXIST); 1768 CHECK(err >= 0 || errno != EEXIST, 1769 "reuseport array update non-empty elem BPF_NOEXIST", 1770 "sock_type:%d err:%d errno:%d\n", 1771 type, err, errno); 1772 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1773 1774 /* BPF_ANY case (always succeed) */ 1775 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1776 BPF_ANY); 1777 CHECK(err < 0, 1778 "reuseport array update same sk with BPF_ANY", 1779 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1780 1781 fd64 = grpa_fds64[fds_idx]; 1782 sk_cookie = grpa_cookies[fds_idx]; 1783 1784 /* The same sk cannot be added to reuseport_array twice */ 1785 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_ANY); 1786 CHECK(err >= 0 || errno != EBUSY, 1787 "reuseport array update same sk with same index", 1788 "sock_type:%d err:%d errno:%d\n", 1789 type, err, errno); 1790 1791 err = bpf_map_update_elem(map_fd, &index0, &fd64, BPF_ANY); 1792 CHECK(err >= 0 || errno != EBUSY, 1793 "reuseport array update same sk with different index", 1794 "sock_type:%d err:%d errno:%d\n", 1795 type, err, errno); 1796 1797 /* Test delete elem */ 1798 err = bpf_map_delete_elem(map_fd, &index3); 1799 CHECK(err < 0, "reuseport array delete sk", 1800 "sock_type:%d err:%d errno:%d\n", 1801 type, err, errno); 1802 1803 /* Add it back with BPF_NOEXIST */ 1804 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST); 1805 CHECK(err < 0, 1806 "reuseport array re-add with BPF_NOEXIST after del", 1807 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1808 1809 /* Test cookie */ 1810 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1811 CHECK(err < 0 || sk_cookie != map_cookie, 1812 "reuseport array lookup re-added sk", 1813 "sock_type:%d err:%d errno:%d sk_cookie:0x%llx map_cookie:0x%llxn", 1814 type, err, errno, sk_cookie, map_cookie); 1815 1816 /* Test elem removed by close() */ 1817 for (f = 0; f < ARRAY_SIZE(grpa_fds64); f++) 1818 close(grpa_fds64[f]); 1819 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1820 CHECK(err >= 0 || errno != ENOENT, 1821 "reuseport array lookup after close()", 1822 "sock_type:%d err:%d errno:%d\n", 1823 type, err, errno); 1824 } 1825 1826 /* Test SOCK_RAW */ 1827 fd64 = socket(AF_INET6, SOCK_RAW, IPPROTO_UDP); 1828 CHECK(fd64 == -1, "socket(SOCK_RAW)", "err:%d errno:%d\n", 1829 err, errno); 1830 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST); 1831 CHECK(err >= 0 || errno != ENOTSUPP, "reuseport array update SOCK_RAW", 1832 "err:%d errno:%d\n", err, errno); 1833 close(fd64); 1834 1835 /* Close the 64 bit value map */ 1836 close(map_fd); 1837 1838 /* Test 32 bit fd */ 1839 map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, 1840 sizeof(__u32), sizeof(__u32), array_size, 0); 1841 CHECK(map_fd < 0, "reuseport array create", 1842 "map_fd:%d, errno:%d\n", map_fd, errno); 1843 prepare_reuseport_grp(SOCK_STREAM, map_fd, sizeof(__u32), &fd64, 1844 &sk_cookie, 1); 1845 fd = fd64; 1846 err = bpf_map_update_elem(map_fd, &index3, &fd, BPF_NOEXIST); 1847 CHECK(err < 0, "reuseport array update 32 bit fd", 1848 "err:%d errno:%d\n", err, errno); 1849 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1850 CHECK(err >= 0 || errno != ENOSPC, 1851 "reuseport array lookup 32 bit fd", 1852 "err:%d errno:%d\n", err, errno); 1853 close(fd); 1854 close(map_fd); 1855 } 1856 1857 static void run_all_tests(void) 1858 { 1859 test_hashmap(0, NULL); 1860 test_hashmap_percpu(0, NULL); 1861 test_hashmap_walk(0, NULL); 1862 test_hashmap_zero_seed(); 1863 1864 test_arraymap(0, NULL); 1865 test_arraymap_percpu(0, NULL); 1866 1867 test_arraymap_percpu_many_keys(); 1868 1869 test_devmap(0, NULL); 1870 test_devmap_hash(0, NULL); 1871 test_sockmap(0, NULL); 1872 1873 test_map_large(); 1874 test_map_parallel(); 1875 test_map_stress(); 1876 1877 test_map_rdonly(); 1878 test_map_wronly(); 1879 1880 test_reuseport_array(); 1881 1882 test_queuemap(0, NULL); 1883 test_stackmap(0, NULL); 1884 1885 test_map_in_map(); 1886 } 1887 1888 #define DEFINE_TEST(name) extern void test_##name(void); 1889 #include <map_tests/tests.h> 1890 #undef DEFINE_TEST 1891 1892 int main(void) 1893 { 1894 srand(time(NULL)); 1895 1896 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 1897 1898 map_flags = 0; 1899 run_all_tests(); 1900 1901 map_flags = BPF_F_NO_PREALLOC; 1902 run_all_tests(); 1903 1904 #define DEFINE_TEST(name) test_##name(); 1905 #include <map_tests/tests.h> 1906 #undef DEFINE_TEST 1907 1908 printf("test_maps: OK, %d SKIPPED\n", skips); 1909 return 0; 1910 } 1911