1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Testsuite for eBPF maps 4 * 5 * Copyright (c) 2014 PLUMgrid, http://plumgrid.com 6 * Copyright (c) 2016 Facebook 7 */ 8 9 #include <stdio.h> 10 #include <unistd.h> 11 #include <errno.h> 12 #include <string.h> 13 #include <assert.h> 14 #include <stdlib.h> 15 #include <time.h> 16 17 #include <sys/wait.h> 18 #include <sys/socket.h> 19 #include <netinet/in.h> 20 #include <linux/bpf.h> 21 22 #include <bpf/bpf.h> 23 #include <bpf/libbpf.h> 24 25 #include "bpf_util.h" 26 #include "bpf_rlimit.h" 27 #include "test_maps.h" 28 29 #ifndef ENOTSUPP 30 #define ENOTSUPP 524 31 #endif 32 33 static int skips; 34 35 static int map_flags; 36 37 static void test_hashmap(unsigned int task, void *data) 38 { 39 long long key, next_key, first_key, value; 40 int fd; 41 42 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 43 2, map_flags); 44 if (fd < 0) { 45 printf("Failed to create hashmap '%s'!\n", strerror(errno)); 46 exit(1); 47 } 48 49 key = 1; 50 value = 1234; 51 /* Insert key=1 element. */ 52 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 53 54 value = 0; 55 /* BPF_NOEXIST means add new element if it doesn't exist. */ 56 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 && 57 /* key=1 already exists. */ 58 errno == EEXIST); 59 60 /* -1 is an invalid flag. */ 61 assert(bpf_map_update_elem(fd, &key, &value, -1) == -1 && 62 errno == EINVAL); 63 64 /* Check that key=1 can be found. */ 65 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234); 66 67 key = 2; 68 /* Check that key=2 is not found. */ 69 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == ENOENT); 70 71 /* BPF_EXIST means update existing element. */ 72 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == -1 && 73 /* key=2 is not there. */ 74 errno == ENOENT); 75 76 /* Insert key=2 element. */ 77 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0); 78 79 /* key=1 and key=2 were inserted, check that key=0 cannot be 80 * inserted due to max_entries limit. 81 */ 82 key = 0; 83 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 && 84 errno == E2BIG); 85 86 /* Update existing element, though the map is full. */ 87 key = 1; 88 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0); 89 key = 2; 90 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 91 key = 3; 92 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 && 93 errno == E2BIG); 94 95 /* Check that key = 0 doesn't exist. */ 96 key = 0; 97 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == ENOENT); 98 99 /* Iterate over two elements. */ 100 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 && 101 (first_key == 1 || first_key == 2)); 102 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 && 103 (next_key == first_key)); 104 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 && 105 (next_key == 1 || next_key == 2) && 106 (next_key != first_key)); 107 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == -1 && 108 errno == ENOENT); 109 110 /* Delete both elements. */ 111 key = 1; 112 assert(bpf_map_delete_elem(fd, &key) == 0); 113 key = 2; 114 assert(bpf_map_delete_elem(fd, &key) == 0); 115 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == ENOENT); 116 117 key = 0; 118 /* Check that map is empty. */ 119 assert(bpf_map_get_next_key(fd, NULL, &next_key) == -1 && 120 errno == ENOENT); 121 assert(bpf_map_get_next_key(fd, &key, &next_key) == -1 && 122 errno == ENOENT); 123 124 close(fd); 125 } 126 127 static void test_hashmap_sizes(unsigned int task, void *data) 128 { 129 int fd, i, j; 130 131 for (i = 1; i <= 512; i <<= 1) 132 for (j = 1; j <= 1 << 18; j <<= 1) { 133 fd = bpf_create_map(BPF_MAP_TYPE_HASH, i, j, 134 2, map_flags); 135 if (fd < 0) { 136 if (errno == ENOMEM) 137 return; 138 printf("Failed to create hashmap key=%d value=%d '%s'\n", 139 i, j, strerror(errno)); 140 exit(1); 141 } 142 close(fd); 143 usleep(10); /* give kernel time to destroy */ 144 } 145 } 146 147 static void test_hashmap_percpu(unsigned int task, void *data) 148 { 149 unsigned int nr_cpus = bpf_num_possible_cpus(); 150 BPF_DECLARE_PERCPU(long, value); 151 long long key, next_key, first_key; 152 int expected_key_mask = 0; 153 int fd, i; 154 155 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_HASH, sizeof(key), 156 sizeof(bpf_percpu(value, 0)), 2, map_flags); 157 if (fd < 0) { 158 printf("Failed to create hashmap '%s'!\n", strerror(errno)); 159 exit(1); 160 } 161 162 for (i = 0; i < nr_cpus; i++) 163 bpf_percpu(value, i) = i + 100; 164 165 key = 1; 166 /* Insert key=1 element. */ 167 assert(!(expected_key_mask & key)); 168 assert(bpf_map_update_elem(fd, &key, value, BPF_ANY) == 0); 169 expected_key_mask |= key; 170 171 /* BPF_NOEXIST means add new element if it doesn't exist. */ 172 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == -1 && 173 /* key=1 already exists. */ 174 errno == EEXIST); 175 176 /* -1 is an invalid flag. */ 177 assert(bpf_map_update_elem(fd, &key, value, -1) == -1 && 178 errno == EINVAL); 179 180 /* Check that key=1 can be found. Value could be 0 if the lookup 181 * was run from a different CPU. 182 */ 183 bpf_percpu(value, 0) = 1; 184 assert(bpf_map_lookup_elem(fd, &key, value) == 0 && 185 bpf_percpu(value, 0) == 100); 186 187 key = 2; 188 /* Check that key=2 is not found. */ 189 assert(bpf_map_lookup_elem(fd, &key, value) == -1 && errno == ENOENT); 190 191 /* BPF_EXIST means update existing element. */ 192 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == -1 && 193 /* key=2 is not there. */ 194 errno == ENOENT); 195 196 /* Insert key=2 element. */ 197 assert(!(expected_key_mask & key)); 198 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == 0); 199 expected_key_mask |= key; 200 201 /* key=1 and key=2 were inserted, check that key=0 cannot be 202 * inserted due to max_entries limit. 203 */ 204 key = 0; 205 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == -1 && 206 errno == E2BIG); 207 208 /* Check that key = 0 doesn't exist. */ 209 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == ENOENT); 210 211 /* Iterate over two elements. */ 212 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 && 213 ((expected_key_mask & first_key) == first_key)); 214 while (!bpf_map_get_next_key(fd, &key, &next_key)) { 215 if (first_key) { 216 assert(next_key == first_key); 217 first_key = 0; 218 } 219 assert((expected_key_mask & next_key) == next_key); 220 expected_key_mask &= ~next_key; 221 222 assert(bpf_map_lookup_elem(fd, &next_key, value) == 0); 223 224 for (i = 0; i < nr_cpus; i++) 225 assert(bpf_percpu(value, i) == i + 100); 226 227 key = next_key; 228 } 229 assert(errno == ENOENT); 230 231 /* Update with BPF_EXIST. */ 232 key = 1; 233 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0); 234 235 /* Delete both elements. */ 236 key = 1; 237 assert(bpf_map_delete_elem(fd, &key) == 0); 238 key = 2; 239 assert(bpf_map_delete_elem(fd, &key) == 0); 240 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == ENOENT); 241 242 key = 0; 243 /* Check that map is empty. */ 244 assert(bpf_map_get_next_key(fd, NULL, &next_key) == -1 && 245 errno == ENOENT); 246 assert(bpf_map_get_next_key(fd, &key, &next_key) == -1 && 247 errno == ENOENT); 248 249 close(fd); 250 } 251 252 static int helper_fill_hashmap(int max_entries) 253 { 254 int i, fd, ret; 255 long long key, value; 256 257 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 258 max_entries, map_flags); 259 CHECK(fd < 0, 260 "failed to create hashmap", 261 "err: %s, flags: 0x%x\n", strerror(errno), map_flags); 262 263 for (i = 0; i < max_entries; i++) { 264 key = i; value = key; 265 ret = bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST); 266 CHECK(ret != 0, 267 "can't update hashmap", 268 "err: %s\n", strerror(ret)); 269 } 270 271 return fd; 272 } 273 274 static void test_hashmap_walk(unsigned int task, void *data) 275 { 276 int fd, i, max_entries = 1000; 277 long long key, value, next_key; 278 bool next_key_valid = true; 279 280 fd = helper_fill_hashmap(max_entries); 281 282 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key, 283 &next_key) == 0; i++) { 284 key = next_key; 285 assert(bpf_map_lookup_elem(fd, &key, &value) == 0); 286 } 287 288 assert(i == max_entries); 289 290 assert(bpf_map_get_next_key(fd, NULL, &key) == 0); 291 for (i = 0; next_key_valid; i++) { 292 next_key_valid = bpf_map_get_next_key(fd, &key, &next_key) == 0; 293 assert(bpf_map_lookup_elem(fd, &key, &value) == 0); 294 value++; 295 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0); 296 key = next_key; 297 } 298 299 assert(i == max_entries); 300 301 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key, 302 &next_key) == 0; i++) { 303 key = next_key; 304 assert(bpf_map_lookup_elem(fd, &key, &value) == 0); 305 assert(value - 1 == key); 306 } 307 308 assert(i == max_entries); 309 close(fd); 310 } 311 312 static void test_hashmap_zero_seed(void) 313 { 314 int i, first, second, old_flags; 315 long long key, next_first, next_second; 316 317 old_flags = map_flags; 318 map_flags |= BPF_F_ZERO_SEED; 319 320 first = helper_fill_hashmap(3); 321 second = helper_fill_hashmap(3); 322 323 for (i = 0; ; i++) { 324 void *key_ptr = !i ? NULL : &key; 325 326 if (bpf_map_get_next_key(first, key_ptr, &next_first) != 0) 327 break; 328 329 CHECK(bpf_map_get_next_key(second, key_ptr, &next_second) != 0, 330 "next_key for second map must succeed", 331 "key_ptr: %p", key_ptr); 332 CHECK(next_first != next_second, 333 "keys must match", 334 "i: %d first: %lld second: %lld\n", i, 335 next_first, next_second); 336 337 key = next_first; 338 } 339 340 map_flags = old_flags; 341 close(first); 342 close(second); 343 } 344 345 static void test_arraymap(unsigned int task, void *data) 346 { 347 int key, next_key, fd; 348 long long value; 349 350 fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, sizeof(key), sizeof(value), 351 2, 0); 352 if (fd < 0) { 353 printf("Failed to create arraymap '%s'!\n", strerror(errno)); 354 exit(1); 355 } 356 357 key = 1; 358 value = 1234; 359 /* Insert key=1 element. */ 360 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 361 362 value = 0; 363 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 && 364 errno == EEXIST); 365 366 /* Check that key=1 can be found. */ 367 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234); 368 369 key = 0; 370 /* Check that key=0 is also found and zero initialized. */ 371 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0); 372 373 /* key=0 and key=1 were inserted, check that key=2 cannot be inserted 374 * due to max_entries limit. 375 */ 376 key = 2; 377 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == -1 && 378 errno == E2BIG); 379 380 /* Check that key = 2 doesn't exist. */ 381 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == ENOENT); 382 383 /* Iterate over two elements. */ 384 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 && 385 next_key == 0); 386 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 && 387 next_key == 0); 388 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 && 389 next_key == 1); 390 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == -1 && 391 errno == ENOENT); 392 393 /* Delete shouldn't succeed. */ 394 key = 1; 395 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == EINVAL); 396 397 close(fd); 398 } 399 400 static void test_arraymap_percpu(unsigned int task, void *data) 401 { 402 unsigned int nr_cpus = bpf_num_possible_cpus(); 403 BPF_DECLARE_PERCPU(long, values); 404 int key, next_key, fd, i; 405 406 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key), 407 sizeof(bpf_percpu(values, 0)), 2, 0); 408 if (fd < 0) { 409 printf("Failed to create arraymap '%s'!\n", strerror(errno)); 410 exit(1); 411 } 412 413 for (i = 0; i < nr_cpus; i++) 414 bpf_percpu(values, i) = i + 100; 415 416 key = 1; 417 /* Insert key=1 element. */ 418 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0); 419 420 bpf_percpu(values, 0) = 0; 421 assert(bpf_map_update_elem(fd, &key, values, BPF_NOEXIST) == -1 && 422 errno == EEXIST); 423 424 /* Check that key=1 can be found. */ 425 assert(bpf_map_lookup_elem(fd, &key, values) == 0 && 426 bpf_percpu(values, 0) == 100); 427 428 key = 0; 429 /* Check that key=0 is also found and zero initialized. */ 430 assert(bpf_map_lookup_elem(fd, &key, values) == 0 && 431 bpf_percpu(values, 0) == 0 && 432 bpf_percpu(values, nr_cpus - 1) == 0); 433 434 /* Check that key=2 cannot be inserted due to max_entries limit. */ 435 key = 2; 436 assert(bpf_map_update_elem(fd, &key, values, BPF_EXIST) == -1 && 437 errno == E2BIG); 438 439 /* Check that key = 2 doesn't exist. */ 440 assert(bpf_map_lookup_elem(fd, &key, values) == -1 && errno == ENOENT); 441 442 /* Iterate over two elements. */ 443 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 && 444 next_key == 0); 445 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 && 446 next_key == 0); 447 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 && 448 next_key == 1); 449 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == -1 && 450 errno == ENOENT); 451 452 /* Delete shouldn't succeed. */ 453 key = 1; 454 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == EINVAL); 455 456 close(fd); 457 } 458 459 static void test_arraymap_percpu_many_keys(void) 460 { 461 unsigned int nr_cpus = bpf_num_possible_cpus(); 462 BPF_DECLARE_PERCPU(long, values); 463 /* nr_keys is not too large otherwise the test stresses percpu 464 * allocator more than anything else 465 */ 466 unsigned int nr_keys = 2000; 467 int key, fd, i; 468 469 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key), 470 sizeof(bpf_percpu(values, 0)), nr_keys, 0); 471 if (fd < 0) { 472 printf("Failed to create per-cpu arraymap '%s'!\n", 473 strerror(errno)); 474 exit(1); 475 } 476 477 for (i = 0; i < nr_cpus; i++) 478 bpf_percpu(values, i) = i + 10; 479 480 for (key = 0; key < nr_keys; key++) 481 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0); 482 483 for (key = 0; key < nr_keys; key++) { 484 for (i = 0; i < nr_cpus; i++) 485 bpf_percpu(values, i) = 0; 486 487 assert(bpf_map_lookup_elem(fd, &key, values) == 0); 488 489 for (i = 0; i < nr_cpus; i++) 490 assert(bpf_percpu(values, i) == i + 10); 491 } 492 493 close(fd); 494 } 495 496 static void test_devmap(unsigned int task, void *data) 497 { 498 int fd; 499 __u32 key, value; 500 501 fd = bpf_create_map(BPF_MAP_TYPE_DEVMAP, sizeof(key), sizeof(value), 502 2, 0); 503 if (fd < 0) { 504 printf("Failed to create devmap '%s'!\n", strerror(errno)); 505 exit(1); 506 } 507 508 close(fd); 509 } 510 511 static void test_queuemap(unsigned int task, void *data) 512 { 513 const int MAP_SIZE = 32; 514 __u32 vals[MAP_SIZE + MAP_SIZE/2], val; 515 int fd, i; 516 517 /* Fill test values to be used */ 518 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++) 519 vals[i] = rand(); 520 521 /* Invalid key size */ 522 fd = bpf_create_map(BPF_MAP_TYPE_QUEUE, 4, sizeof(val), MAP_SIZE, 523 map_flags); 524 assert(fd < 0 && errno == EINVAL); 525 526 fd = bpf_create_map(BPF_MAP_TYPE_QUEUE, 0, sizeof(val), MAP_SIZE, 527 map_flags); 528 /* Queue map does not support BPF_F_NO_PREALLOC */ 529 if (map_flags & BPF_F_NO_PREALLOC) { 530 assert(fd < 0 && errno == EINVAL); 531 return; 532 } 533 if (fd < 0) { 534 printf("Failed to create queuemap '%s'!\n", strerror(errno)); 535 exit(1); 536 } 537 538 /* Push MAP_SIZE elements */ 539 for (i = 0; i < MAP_SIZE; i++) 540 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0); 541 542 /* Check that element cannot be pushed due to max_entries limit */ 543 assert(bpf_map_update_elem(fd, NULL, &val, 0) == -1 && 544 errno == E2BIG); 545 546 /* Peek element */ 547 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[0]); 548 549 /* Replace half elements */ 550 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++) 551 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0); 552 553 /* Pop all elements */ 554 for (i = MAP_SIZE/2; i < MAP_SIZE + MAP_SIZE/2; i++) 555 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 && 556 val == vals[i]); 557 558 /* Check that there are not elements left */ 559 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == -1 && 560 errno == ENOENT); 561 562 /* Check that non supported functions set errno to EINVAL */ 563 assert(bpf_map_delete_elem(fd, NULL) == -1 && errno == EINVAL); 564 assert(bpf_map_get_next_key(fd, NULL, NULL) == -1 && errno == EINVAL); 565 566 close(fd); 567 } 568 569 static void test_stackmap(unsigned int task, void *data) 570 { 571 const int MAP_SIZE = 32; 572 __u32 vals[MAP_SIZE + MAP_SIZE/2], val; 573 int fd, i; 574 575 /* Fill test values to be used */ 576 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++) 577 vals[i] = rand(); 578 579 /* Invalid key size */ 580 fd = bpf_create_map(BPF_MAP_TYPE_STACK, 4, sizeof(val), MAP_SIZE, 581 map_flags); 582 assert(fd < 0 && errno == EINVAL); 583 584 fd = bpf_create_map(BPF_MAP_TYPE_STACK, 0, sizeof(val), MAP_SIZE, 585 map_flags); 586 /* Stack map does not support BPF_F_NO_PREALLOC */ 587 if (map_flags & BPF_F_NO_PREALLOC) { 588 assert(fd < 0 && errno == EINVAL); 589 return; 590 } 591 if (fd < 0) { 592 printf("Failed to create stackmap '%s'!\n", strerror(errno)); 593 exit(1); 594 } 595 596 /* Push MAP_SIZE elements */ 597 for (i = 0; i < MAP_SIZE; i++) 598 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0); 599 600 /* Check that element cannot be pushed due to max_entries limit */ 601 assert(bpf_map_update_elem(fd, NULL, &val, 0) == -1 && 602 errno == E2BIG); 603 604 /* Peek element */ 605 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[i - 1]); 606 607 /* Replace half elements */ 608 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++) 609 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0); 610 611 /* Pop all elements */ 612 for (i = MAP_SIZE + MAP_SIZE/2 - 1; i >= MAP_SIZE/2; i--) 613 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 && 614 val == vals[i]); 615 616 /* Check that there are not elements left */ 617 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == -1 && 618 errno == ENOENT); 619 620 /* Check that non supported functions set errno to EINVAL */ 621 assert(bpf_map_delete_elem(fd, NULL) == -1 && errno == EINVAL); 622 assert(bpf_map_get_next_key(fd, NULL, NULL) == -1 && errno == EINVAL); 623 624 close(fd); 625 } 626 627 #include <sys/ioctl.h> 628 #include <arpa/inet.h> 629 #include <sys/select.h> 630 #include <linux/err.h> 631 #define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.o" 632 #define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.o" 633 #define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.o" 634 static void test_sockmap(unsigned int tasks, void *data) 635 { 636 struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_msg, *bpf_map_break; 637 int map_fd_msg = 0, map_fd_rx = 0, map_fd_tx = 0, map_fd_break; 638 int ports[] = {50200, 50201, 50202, 50204}; 639 int err, i, fd, udp, sfd[6] = {0xdeadbeef}; 640 u8 buf[20] = {0x0, 0x5, 0x3, 0x2, 0x1, 0x0}; 641 int parse_prog, verdict_prog, msg_prog; 642 struct sockaddr_in addr; 643 int one = 1, s, sc, rc; 644 struct bpf_object *obj; 645 struct timeval to; 646 __u32 key, value; 647 pid_t pid[tasks]; 648 fd_set w; 649 650 /* Create some sockets to use with sockmap */ 651 for (i = 0; i < 2; i++) { 652 sfd[i] = socket(AF_INET, SOCK_STREAM, 0); 653 if (sfd[i] < 0) 654 goto out; 655 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR, 656 (char *)&one, sizeof(one)); 657 if (err) { 658 printf("failed to setsockopt\n"); 659 goto out; 660 } 661 err = ioctl(sfd[i], FIONBIO, (char *)&one); 662 if (err < 0) { 663 printf("failed to ioctl\n"); 664 goto out; 665 } 666 memset(&addr, 0, sizeof(struct sockaddr_in)); 667 addr.sin_family = AF_INET; 668 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 669 addr.sin_port = htons(ports[i]); 670 err = bind(sfd[i], (struct sockaddr *)&addr, sizeof(addr)); 671 if (err < 0) { 672 printf("failed to bind: err %i: %i:%i\n", 673 err, i, sfd[i]); 674 goto out; 675 } 676 err = listen(sfd[i], 32); 677 if (err < 0) { 678 printf("failed to listen\n"); 679 goto out; 680 } 681 } 682 683 for (i = 2; i < 4; i++) { 684 sfd[i] = socket(AF_INET, SOCK_STREAM, 0); 685 if (sfd[i] < 0) 686 goto out; 687 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR, 688 (char *)&one, sizeof(one)); 689 if (err) { 690 printf("set sock opt\n"); 691 goto out; 692 } 693 memset(&addr, 0, sizeof(struct sockaddr_in)); 694 addr.sin_family = AF_INET; 695 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 696 addr.sin_port = htons(ports[i - 2]); 697 err = connect(sfd[i], (struct sockaddr *)&addr, sizeof(addr)); 698 if (err) { 699 printf("failed to connect\n"); 700 goto out; 701 } 702 } 703 704 705 for (i = 4; i < 6; i++) { 706 sfd[i] = accept(sfd[i - 4], NULL, NULL); 707 if (sfd[i] < 0) { 708 printf("accept failed\n"); 709 goto out; 710 } 711 } 712 713 /* Test sockmap with connected sockets */ 714 fd = bpf_create_map(BPF_MAP_TYPE_SOCKMAP, 715 sizeof(key), sizeof(value), 716 6, 0); 717 if (fd < 0) { 718 if (!bpf_probe_map_type(BPF_MAP_TYPE_SOCKMAP, 0)) { 719 printf("%s SKIP (unsupported map type BPF_MAP_TYPE_SOCKMAP)\n", 720 __func__); 721 skips++; 722 for (i = 0; i < 6; i++) 723 close(sfd[i]); 724 return; 725 } 726 727 printf("Failed to create sockmap %i\n", fd); 728 goto out_sockmap; 729 } 730 731 /* Test update with unsupported UDP socket */ 732 udp = socket(AF_INET, SOCK_DGRAM, 0); 733 i = 0; 734 err = bpf_map_update_elem(fd, &i, &udp, BPF_ANY); 735 if (!err) { 736 printf("Failed socket SOCK_DGRAM allowed '%i:%i'\n", 737 i, udp); 738 goto out_sockmap; 739 } 740 741 /* Test update without programs */ 742 for (i = 0; i < 6; i++) { 743 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY); 744 if (i < 2 && !err) { 745 printf("Allowed update sockmap '%i:%i' not in ESTABLISHED\n", 746 i, sfd[i]); 747 goto out_sockmap; 748 } else if (i >= 2 && err) { 749 printf("Failed noprog update sockmap '%i:%i'\n", 750 i, sfd[i]); 751 goto out_sockmap; 752 } 753 } 754 755 /* Test attaching/detaching bad fds */ 756 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_PARSER, 0); 757 if (!err) { 758 printf("Failed invalid parser prog attach\n"); 759 goto out_sockmap; 760 } 761 762 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_VERDICT, 0); 763 if (!err) { 764 printf("Failed invalid verdict prog attach\n"); 765 goto out_sockmap; 766 } 767 768 err = bpf_prog_attach(-1, fd, BPF_SK_MSG_VERDICT, 0); 769 if (!err) { 770 printf("Failed invalid msg verdict prog attach\n"); 771 goto out_sockmap; 772 } 773 774 err = bpf_prog_attach(-1, fd, __MAX_BPF_ATTACH_TYPE, 0); 775 if (!err) { 776 printf("Failed unknown prog attach\n"); 777 goto out_sockmap; 778 } 779 780 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_PARSER); 781 if (err) { 782 printf("Failed empty parser prog detach\n"); 783 goto out_sockmap; 784 } 785 786 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_VERDICT); 787 if (err) { 788 printf("Failed empty verdict prog detach\n"); 789 goto out_sockmap; 790 } 791 792 err = bpf_prog_detach(fd, BPF_SK_MSG_VERDICT); 793 if (err) { 794 printf("Failed empty msg verdict prog detach\n"); 795 goto out_sockmap; 796 } 797 798 err = bpf_prog_detach(fd, __MAX_BPF_ATTACH_TYPE); 799 if (!err) { 800 printf("Detach invalid prog successful\n"); 801 goto out_sockmap; 802 } 803 804 /* Load SK_SKB program and Attach */ 805 err = bpf_prog_load(SOCKMAP_PARSE_PROG, 806 BPF_PROG_TYPE_SK_SKB, &obj, &parse_prog); 807 if (err) { 808 printf("Failed to load SK_SKB parse prog\n"); 809 goto out_sockmap; 810 } 811 812 err = bpf_prog_load(SOCKMAP_TCP_MSG_PROG, 813 BPF_PROG_TYPE_SK_MSG, &obj, &msg_prog); 814 if (err) { 815 printf("Failed to load SK_SKB msg prog\n"); 816 goto out_sockmap; 817 } 818 819 err = bpf_prog_load(SOCKMAP_VERDICT_PROG, 820 BPF_PROG_TYPE_SK_SKB, &obj, &verdict_prog); 821 if (err) { 822 printf("Failed to load SK_SKB verdict prog\n"); 823 goto out_sockmap; 824 } 825 826 bpf_map_rx = bpf_object__find_map_by_name(obj, "sock_map_rx"); 827 if (IS_ERR(bpf_map_rx)) { 828 printf("Failed to load map rx from verdict prog\n"); 829 goto out_sockmap; 830 } 831 832 map_fd_rx = bpf_map__fd(bpf_map_rx); 833 if (map_fd_rx < 0) { 834 printf("Failed to get map rx fd\n"); 835 goto out_sockmap; 836 } 837 838 bpf_map_tx = bpf_object__find_map_by_name(obj, "sock_map_tx"); 839 if (IS_ERR(bpf_map_tx)) { 840 printf("Failed to load map tx from verdict prog\n"); 841 goto out_sockmap; 842 } 843 844 map_fd_tx = bpf_map__fd(bpf_map_tx); 845 if (map_fd_tx < 0) { 846 printf("Failed to get map tx fd\n"); 847 goto out_sockmap; 848 } 849 850 bpf_map_msg = bpf_object__find_map_by_name(obj, "sock_map_msg"); 851 if (IS_ERR(bpf_map_msg)) { 852 printf("Failed to load map msg from msg_verdict prog\n"); 853 goto out_sockmap; 854 } 855 856 map_fd_msg = bpf_map__fd(bpf_map_msg); 857 if (map_fd_msg < 0) { 858 printf("Failed to get map msg fd\n"); 859 goto out_sockmap; 860 } 861 862 bpf_map_break = bpf_object__find_map_by_name(obj, "sock_map_break"); 863 if (IS_ERR(bpf_map_break)) { 864 printf("Failed to load map tx from verdict prog\n"); 865 goto out_sockmap; 866 } 867 868 map_fd_break = bpf_map__fd(bpf_map_break); 869 if (map_fd_break < 0) { 870 printf("Failed to get map tx fd\n"); 871 goto out_sockmap; 872 } 873 874 err = bpf_prog_attach(parse_prog, map_fd_break, 875 BPF_SK_SKB_STREAM_PARSER, 0); 876 if (!err) { 877 printf("Allowed attaching SK_SKB program to invalid map\n"); 878 goto out_sockmap; 879 } 880 881 err = bpf_prog_attach(parse_prog, map_fd_rx, 882 BPF_SK_SKB_STREAM_PARSER, 0); 883 if (err) { 884 printf("Failed stream parser bpf prog attach\n"); 885 goto out_sockmap; 886 } 887 888 err = bpf_prog_attach(verdict_prog, map_fd_rx, 889 BPF_SK_SKB_STREAM_VERDICT, 0); 890 if (err) { 891 printf("Failed stream verdict bpf prog attach\n"); 892 goto out_sockmap; 893 } 894 895 err = bpf_prog_attach(msg_prog, map_fd_msg, BPF_SK_MSG_VERDICT, 0); 896 if (err) { 897 printf("Failed msg verdict bpf prog attach\n"); 898 goto out_sockmap; 899 } 900 901 err = bpf_prog_attach(verdict_prog, map_fd_rx, 902 __MAX_BPF_ATTACH_TYPE, 0); 903 if (!err) { 904 printf("Attached unknown bpf prog\n"); 905 goto out_sockmap; 906 } 907 908 /* Test map update elem afterwards fd lives in fd and map_fd */ 909 for (i = 2; i < 6; i++) { 910 err = bpf_map_update_elem(map_fd_rx, &i, &sfd[i], BPF_ANY); 911 if (err) { 912 printf("Failed map_fd_rx update sockmap %i '%i:%i'\n", 913 err, i, sfd[i]); 914 goto out_sockmap; 915 } 916 err = bpf_map_update_elem(map_fd_tx, &i, &sfd[i], BPF_ANY); 917 if (err) { 918 printf("Failed map_fd_tx update sockmap %i '%i:%i'\n", 919 err, i, sfd[i]); 920 goto out_sockmap; 921 } 922 } 923 924 /* Test map delete elem and remove send/recv sockets */ 925 for (i = 2; i < 4; i++) { 926 err = bpf_map_delete_elem(map_fd_rx, &i); 927 if (err) { 928 printf("Failed delete sockmap rx %i '%i:%i'\n", 929 err, i, sfd[i]); 930 goto out_sockmap; 931 } 932 err = bpf_map_delete_elem(map_fd_tx, &i); 933 if (err) { 934 printf("Failed delete sockmap tx %i '%i:%i'\n", 935 err, i, sfd[i]); 936 goto out_sockmap; 937 } 938 } 939 940 /* Put sfd[2] (sending fd below) into msg map to test sendmsg bpf */ 941 i = 0; 942 err = bpf_map_update_elem(map_fd_msg, &i, &sfd[2], BPF_ANY); 943 if (err) { 944 printf("Failed map_fd_msg update sockmap %i\n", err); 945 goto out_sockmap; 946 } 947 948 /* Test map send/recv */ 949 for (i = 0; i < 2; i++) { 950 buf[0] = i; 951 buf[1] = 0x5; 952 sc = send(sfd[2], buf, 20, 0); 953 if (sc < 0) { 954 printf("Failed sockmap send\n"); 955 goto out_sockmap; 956 } 957 958 FD_ZERO(&w); 959 FD_SET(sfd[3], &w); 960 to.tv_sec = 1; 961 to.tv_usec = 0; 962 s = select(sfd[3] + 1, &w, NULL, NULL, &to); 963 if (s == -1) { 964 perror("Failed sockmap select()"); 965 goto out_sockmap; 966 } else if (!s) { 967 printf("Failed sockmap unexpected timeout\n"); 968 goto out_sockmap; 969 } 970 971 if (!FD_ISSET(sfd[3], &w)) { 972 printf("Failed sockmap select/recv\n"); 973 goto out_sockmap; 974 } 975 976 rc = recv(sfd[3], buf, sizeof(buf), 0); 977 if (rc < 0) { 978 printf("Failed sockmap recv\n"); 979 goto out_sockmap; 980 } 981 } 982 983 /* Negative null entry lookup from datapath should be dropped */ 984 buf[0] = 1; 985 buf[1] = 12; 986 sc = send(sfd[2], buf, 20, 0); 987 if (sc < 0) { 988 printf("Failed sockmap send\n"); 989 goto out_sockmap; 990 } 991 992 /* Push fd into same slot */ 993 i = 2; 994 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST); 995 if (!err) { 996 printf("Failed allowed sockmap dup slot BPF_NOEXIST\n"); 997 goto out_sockmap; 998 } 999 1000 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY); 1001 if (err) { 1002 printf("Failed sockmap update new slot BPF_ANY\n"); 1003 goto out_sockmap; 1004 } 1005 1006 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST); 1007 if (err) { 1008 printf("Failed sockmap update new slot BPF_EXIST\n"); 1009 goto out_sockmap; 1010 } 1011 1012 /* Delete the elems without programs */ 1013 for (i = 2; i < 6; i++) { 1014 err = bpf_map_delete_elem(fd, &i); 1015 if (err) { 1016 printf("Failed delete sockmap %i '%i:%i'\n", 1017 err, i, sfd[i]); 1018 } 1019 } 1020 1021 /* Test having multiple maps open and set with programs on same fds */ 1022 err = bpf_prog_attach(parse_prog, fd, 1023 BPF_SK_SKB_STREAM_PARSER, 0); 1024 if (err) { 1025 printf("Failed fd bpf parse prog attach\n"); 1026 goto out_sockmap; 1027 } 1028 err = bpf_prog_attach(verdict_prog, fd, 1029 BPF_SK_SKB_STREAM_VERDICT, 0); 1030 if (err) { 1031 printf("Failed fd bpf verdict prog attach\n"); 1032 goto out_sockmap; 1033 } 1034 1035 for (i = 4; i < 6; i++) { 1036 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY); 1037 if (!err) { 1038 printf("Failed allowed duplicate programs in update ANY sockmap %i '%i:%i'\n", 1039 err, i, sfd[i]); 1040 goto out_sockmap; 1041 } 1042 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST); 1043 if (!err) { 1044 printf("Failed allowed duplicate program in update NOEXIST sockmap %i '%i:%i'\n", 1045 err, i, sfd[i]); 1046 goto out_sockmap; 1047 } 1048 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST); 1049 if (!err) { 1050 printf("Failed allowed duplicate program in update EXIST sockmap %i '%i:%i'\n", 1051 err, i, sfd[i]); 1052 goto out_sockmap; 1053 } 1054 } 1055 1056 /* Test tasks number of forked operations */ 1057 for (i = 0; i < tasks; i++) { 1058 pid[i] = fork(); 1059 if (pid[i] == 0) { 1060 for (i = 0; i < 6; i++) { 1061 bpf_map_delete_elem(map_fd_tx, &i); 1062 bpf_map_delete_elem(map_fd_rx, &i); 1063 bpf_map_update_elem(map_fd_tx, &i, 1064 &sfd[i], BPF_ANY); 1065 bpf_map_update_elem(map_fd_rx, &i, 1066 &sfd[i], BPF_ANY); 1067 } 1068 exit(0); 1069 } else if (pid[i] == -1) { 1070 printf("Couldn't spawn #%d process!\n", i); 1071 exit(1); 1072 } 1073 } 1074 1075 for (i = 0; i < tasks; i++) { 1076 int status; 1077 1078 assert(waitpid(pid[i], &status, 0) == pid[i]); 1079 assert(status == 0); 1080 } 1081 1082 err = bpf_prog_detach(map_fd_rx, __MAX_BPF_ATTACH_TYPE); 1083 if (!err) { 1084 printf("Detached an invalid prog type.\n"); 1085 goto out_sockmap; 1086 } 1087 1088 err = bpf_prog_detach(map_fd_rx, BPF_SK_SKB_STREAM_PARSER); 1089 if (err) { 1090 printf("Failed parser prog detach\n"); 1091 goto out_sockmap; 1092 } 1093 1094 err = bpf_prog_detach(map_fd_rx, BPF_SK_SKB_STREAM_VERDICT); 1095 if (err) { 1096 printf("Failed parser prog detach\n"); 1097 goto out_sockmap; 1098 } 1099 1100 /* Test map close sockets and empty maps */ 1101 for (i = 0; i < 6; i++) { 1102 bpf_map_delete_elem(map_fd_tx, &i); 1103 bpf_map_delete_elem(map_fd_rx, &i); 1104 close(sfd[i]); 1105 } 1106 close(fd); 1107 close(map_fd_rx); 1108 bpf_object__close(obj); 1109 return; 1110 out: 1111 for (i = 0; i < 6; i++) 1112 close(sfd[i]); 1113 printf("Failed to create sockmap '%i:%s'!\n", i, strerror(errno)); 1114 exit(1); 1115 out_sockmap: 1116 for (i = 0; i < 6; i++) { 1117 if (map_fd_tx) 1118 bpf_map_delete_elem(map_fd_tx, &i); 1119 if (map_fd_rx) 1120 bpf_map_delete_elem(map_fd_rx, &i); 1121 close(sfd[i]); 1122 } 1123 close(fd); 1124 exit(1); 1125 } 1126 1127 #define MAPINMAP_PROG "./test_map_in_map.o" 1128 static void test_map_in_map(void) 1129 { 1130 struct bpf_program *prog; 1131 struct bpf_object *obj; 1132 struct bpf_map *map; 1133 int mim_fd, fd, err; 1134 int pos = 0; 1135 1136 obj = bpf_object__open(MAPINMAP_PROG); 1137 1138 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(int), sizeof(int), 1139 2, 0); 1140 if (fd < 0) { 1141 printf("Failed to create hashmap '%s'!\n", strerror(errno)); 1142 exit(1); 1143 } 1144 1145 map = bpf_object__find_map_by_name(obj, "mim_array"); 1146 if (IS_ERR(map)) { 1147 printf("Failed to load array of maps from test prog\n"); 1148 goto out_map_in_map; 1149 } 1150 err = bpf_map__set_inner_map_fd(map, fd); 1151 if (err) { 1152 printf("Failed to set inner_map_fd for array of maps\n"); 1153 goto out_map_in_map; 1154 } 1155 1156 map = bpf_object__find_map_by_name(obj, "mim_hash"); 1157 if (IS_ERR(map)) { 1158 printf("Failed to load hash of maps from test prog\n"); 1159 goto out_map_in_map; 1160 } 1161 err = bpf_map__set_inner_map_fd(map, fd); 1162 if (err) { 1163 printf("Failed to set inner_map_fd for hash of maps\n"); 1164 goto out_map_in_map; 1165 } 1166 1167 bpf_object__for_each_program(prog, obj) { 1168 bpf_program__set_xdp(prog); 1169 } 1170 bpf_object__load(obj); 1171 1172 map = bpf_object__find_map_by_name(obj, "mim_array"); 1173 if (IS_ERR(map)) { 1174 printf("Failed to load array of maps from test prog\n"); 1175 goto out_map_in_map; 1176 } 1177 mim_fd = bpf_map__fd(map); 1178 if (mim_fd < 0) { 1179 printf("Failed to get descriptor for array of maps\n"); 1180 goto out_map_in_map; 1181 } 1182 1183 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0); 1184 if (err) { 1185 printf("Failed to update array of maps\n"); 1186 goto out_map_in_map; 1187 } 1188 1189 map = bpf_object__find_map_by_name(obj, "mim_hash"); 1190 if (IS_ERR(map)) { 1191 printf("Failed to load hash of maps from test prog\n"); 1192 goto out_map_in_map; 1193 } 1194 mim_fd = bpf_map__fd(map); 1195 if (mim_fd < 0) { 1196 printf("Failed to get descriptor for hash of maps\n"); 1197 goto out_map_in_map; 1198 } 1199 1200 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0); 1201 if (err) { 1202 printf("Failed to update hash of maps\n"); 1203 goto out_map_in_map; 1204 } 1205 1206 close(fd); 1207 bpf_object__close(obj); 1208 return; 1209 1210 out_map_in_map: 1211 close(fd); 1212 exit(1); 1213 } 1214 1215 #define MAP_SIZE (32 * 1024) 1216 1217 static void test_map_large(void) 1218 { 1219 struct bigkey { 1220 int a; 1221 char b[116]; 1222 long long c; 1223 } key; 1224 int fd, i, value; 1225 1226 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1227 MAP_SIZE, map_flags); 1228 if (fd < 0) { 1229 printf("Failed to create large map '%s'!\n", strerror(errno)); 1230 exit(1); 1231 } 1232 1233 for (i = 0; i < MAP_SIZE; i++) { 1234 key = (struct bigkey) { .c = i }; 1235 value = i; 1236 1237 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0); 1238 } 1239 1240 key.c = -1; 1241 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 && 1242 errno == E2BIG); 1243 1244 /* Iterate through all elements. */ 1245 assert(bpf_map_get_next_key(fd, NULL, &key) == 0); 1246 key.c = -1; 1247 for (i = 0; i < MAP_SIZE; i++) 1248 assert(bpf_map_get_next_key(fd, &key, &key) == 0); 1249 assert(bpf_map_get_next_key(fd, &key, &key) == -1 && errno == ENOENT); 1250 1251 key.c = 0; 1252 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0); 1253 key.a = 1; 1254 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == ENOENT); 1255 1256 close(fd); 1257 } 1258 1259 #define run_parallel(N, FN, DATA) \ 1260 printf("Fork %u tasks to '" #FN "'\n", N); \ 1261 __run_parallel(N, FN, DATA) 1262 1263 static void __run_parallel(unsigned int tasks, 1264 void (*fn)(unsigned int task, void *data), 1265 void *data) 1266 { 1267 pid_t pid[tasks]; 1268 int i; 1269 1270 for (i = 0; i < tasks; i++) { 1271 pid[i] = fork(); 1272 if (pid[i] == 0) { 1273 fn(i, data); 1274 exit(0); 1275 } else if (pid[i] == -1) { 1276 printf("Couldn't spawn #%d process!\n", i); 1277 exit(1); 1278 } 1279 } 1280 1281 for (i = 0; i < tasks; i++) { 1282 int status; 1283 1284 assert(waitpid(pid[i], &status, 0) == pid[i]); 1285 assert(status == 0); 1286 } 1287 } 1288 1289 static void test_map_stress(void) 1290 { 1291 run_parallel(100, test_hashmap, NULL); 1292 run_parallel(100, test_hashmap_percpu, NULL); 1293 run_parallel(100, test_hashmap_sizes, NULL); 1294 run_parallel(100, test_hashmap_walk, NULL); 1295 1296 run_parallel(100, test_arraymap, NULL); 1297 run_parallel(100, test_arraymap_percpu, NULL); 1298 } 1299 1300 #define TASKS 1024 1301 1302 #define DO_UPDATE 1 1303 #define DO_DELETE 0 1304 1305 static void test_update_delete(unsigned int fn, void *data) 1306 { 1307 int do_update = ((int *)data)[1]; 1308 int fd = ((int *)data)[0]; 1309 int i, key, value; 1310 1311 for (i = fn; i < MAP_SIZE; i += TASKS) { 1312 key = value = i; 1313 1314 if (do_update) { 1315 assert(bpf_map_update_elem(fd, &key, &value, 1316 BPF_NOEXIST) == 0); 1317 assert(bpf_map_update_elem(fd, &key, &value, 1318 BPF_EXIST) == 0); 1319 } else { 1320 assert(bpf_map_delete_elem(fd, &key) == 0); 1321 } 1322 } 1323 } 1324 1325 static void test_map_parallel(void) 1326 { 1327 int i, fd, key = 0, value = 0; 1328 int data[2]; 1329 1330 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1331 MAP_SIZE, map_flags); 1332 if (fd < 0) { 1333 printf("Failed to create map for parallel test '%s'!\n", 1334 strerror(errno)); 1335 exit(1); 1336 } 1337 1338 /* Use the same fd in children to add elements to this map: 1339 * child_0 adds key=0, key=1024, key=2048, ... 1340 * child_1 adds key=1, key=1025, key=2049, ... 1341 * child_1023 adds key=1023, ... 1342 */ 1343 data[0] = fd; 1344 data[1] = DO_UPDATE; 1345 run_parallel(TASKS, test_update_delete, data); 1346 1347 /* Check that key=0 is already there. */ 1348 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 && 1349 errno == EEXIST); 1350 1351 /* Check that all elements were inserted. */ 1352 assert(bpf_map_get_next_key(fd, NULL, &key) == 0); 1353 key = -1; 1354 for (i = 0; i < MAP_SIZE; i++) 1355 assert(bpf_map_get_next_key(fd, &key, &key) == 0); 1356 assert(bpf_map_get_next_key(fd, &key, &key) == -1 && errno == ENOENT); 1357 1358 /* Another check for all elements */ 1359 for (i = 0; i < MAP_SIZE; i++) { 1360 key = MAP_SIZE - i - 1; 1361 1362 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && 1363 value == key); 1364 } 1365 1366 /* Now let's delete all elemenets in parallel. */ 1367 data[1] = DO_DELETE; 1368 run_parallel(TASKS, test_update_delete, data); 1369 1370 /* Nothing should be left. */ 1371 key = -1; 1372 assert(bpf_map_get_next_key(fd, NULL, &key) == -1 && errno == ENOENT); 1373 assert(bpf_map_get_next_key(fd, &key, &key) == -1 && errno == ENOENT); 1374 } 1375 1376 static void test_map_rdonly(void) 1377 { 1378 int fd, key = 0, value = 0; 1379 1380 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1381 MAP_SIZE, map_flags | BPF_F_RDONLY); 1382 if (fd < 0) { 1383 printf("Failed to create map for read only test '%s'!\n", 1384 strerror(errno)); 1385 exit(1); 1386 } 1387 1388 key = 1; 1389 value = 1234; 1390 /* Insert key=1 element. */ 1391 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == -1 && 1392 errno == EPERM); 1393 1394 /* Check that key=2 is not found. */ 1395 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == ENOENT); 1396 assert(bpf_map_get_next_key(fd, &key, &value) == -1 && errno == ENOENT); 1397 } 1398 1399 static void test_map_wronly(void) 1400 { 1401 int fd, key = 0, value = 0; 1402 1403 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1404 MAP_SIZE, map_flags | BPF_F_WRONLY); 1405 if (fd < 0) { 1406 printf("Failed to create map for read only test '%s'!\n", 1407 strerror(errno)); 1408 exit(1); 1409 } 1410 1411 key = 1; 1412 value = 1234; 1413 /* Insert key=1 element. */ 1414 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 1415 1416 /* Check that key=2 is not found. */ 1417 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == EPERM); 1418 assert(bpf_map_get_next_key(fd, &key, &value) == -1 && errno == EPERM); 1419 } 1420 1421 static void prepare_reuseport_grp(int type, int map_fd, size_t map_elem_size, 1422 __s64 *fds64, __u64 *sk_cookies, 1423 unsigned int n) 1424 { 1425 socklen_t optlen, addrlen; 1426 struct sockaddr_in6 s6; 1427 const __u32 index0 = 0; 1428 const int optval = 1; 1429 unsigned int i; 1430 u64 sk_cookie; 1431 void *value; 1432 __s32 fd32; 1433 __s64 fd64; 1434 int err; 1435 1436 s6.sin6_family = AF_INET6; 1437 s6.sin6_addr = in6addr_any; 1438 s6.sin6_port = 0; 1439 addrlen = sizeof(s6); 1440 optlen = sizeof(sk_cookie); 1441 1442 for (i = 0; i < n; i++) { 1443 fd64 = socket(AF_INET6, type, 0); 1444 CHECK(fd64 == -1, "socket()", 1445 "sock_type:%d fd64:%lld errno:%d\n", 1446 type, fd64, errno); 1447 1448 err = setsockopt(fd64, SOL_SOCKET, SO_REUSEPORT, 1449 &optval, sizeof(optval)); 1450 CHECK(err == -1, "setsockopt(SO_REUSEPORT)", 1451 "err:%d errno:%d\n", err, errno); 1452 1453 /* reuseport_array does not allow unbound sk */ 1454 if (map_elem_size == sizeof(__u64)) 1455 value = &fd64; 1456 else { 1457 assert(map_elem_size == sizeof(__u32)); 1458 fd32 = (__s32)fd64; 1459 value = &fd32; 1460 } 1461 err = bpf_map_update_elem(map_fd, &index0, value, BPF_ANY); 1462 CHECK(err != -1 || errno != EINVAL, 1463 "reuseport array update unbound sk", 1464 "sock_type:%d err:%d errno:%d\n", 1465 type, err, errno); 1466 1467 err = bind(fd64, (struct sockaddr *)&s6, sizeof(s6)); 1468 CHECK(err == -1, "bind()", 1469 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1470 1471 if (i == 0) { 1472 err = getsockname(fd64, (struct sockaddr *)&s6, 1473 &addrlen); 1474 CHECK(err == -1, "getsockname()", 1475 "sock_type:%d err:%d errno:%d\n", 1476 type, err, errno); 1477 } 1478 1479 err = getsockopt(fd64, SOL_SOCKET, SO_COOKIE, &sk_cookie, 1480 &optlen); 1481 CHECK(err == -1, "getsockopt(SO_COOKIE)", 1482 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1483 1484 if (type == SOCK_STREAM) { 1485 /* 1486 * reuseport_array does not allow 1487 * non-listening tcp sk. 1488 */ 1489 err = bpf_map_update_elem(map_fd, &index0, value, 1490 BPF_ANY); 1491 CHECK(err != -1 || errno != EINVAL, 1492 "reuseport array update non-listening sk", 1493 "sock_type:%d err:%d errno:%d\n", 1494 type, err, errno); 1495 err = listen(fd64, 0); 1496 CHECK(err == -1, "listen()", 1497 "sock_type:%d, err:%d errno:%d\n", 1498 type, err, errno); 1499 } 1500 1501 fds64[i] = fd64; 1502 sk_cookies[i] = sk_cookie; 1503 } 1504 } 1505 1506 static void test_reuseport_array(void) 1507 { 1508 #define REUSEPORT_FD_IDX(err, last) ({ (err) ? last : !last; }) 1509 1510 const __u32 array_size = 4, index0 = 0, index3 = 3; 1511 int types[2] = { SOCK_STREAM, SOCK_DGRAM }, type; 1512 __u64 grpa_cookies[2], sk_cookie, map_cookie; 1513 __s64 grpa_fds64[2] = { -1, -1 }, fd64 = -1; 1514 const __u32 bad_index = array_size; 1515 int map_fd, err, t, f; 1516 __u32 fds_idx = 0; 1517 int fd; 1518 1519 map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, 1520 sizeof(__u32), sizeof(__u64), array_size, 0); 1521 CHECK(map_fd == -1, "reuseport array create", 1522 "map_fd:%d, errno:%d\n", map_fd, errno); 1523 1524 /* Test lookup/update/delete with invalid index */ 1525 err = bpf_map_delete_elem(map_fd, &bad_index); 1526 CHECK(err != -1 || errno != E2BIG, "reuseport array del >=max_entries", 1527 "err:%d errno:%d\n", err, errno); 1528 1529 err = bpf_map_update_elem(map_fd, &bad_index, &fd64, BPF_ANY); 1530 CHECK(err != -1 || errno != E2BIG, 1531 "reuseport array update >=max_entries", 1532 "err:%d errno:%d\n", err, errno); 1533 1534 err = bpf_map_lookup_elem(map_fd, &bad_index, &map_cookie); 1535 CHECK(err != -1 || errno != ENOENT, 1536 "reuseport array update >=max_entries", 1537 "err:%d errno:%d\n", err, errno); 1538 1539 /* Test lookup/delete non existence elem */ 1540 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1541 CHECK(err != -1 || errno != ENOENT, 1542 "reuseport array lookup not-exist elem", 1543 "err:%d errno:%d\n", err, errno); 1544 err = bpf_map_delete_elem(map_fd, &index3); 1545 CHECK(err != -1 || errno != ENOENT, 1546 "reuseport array del not-exist elem", 1547 "err:%d errno:%d\n", err, errno); 1548 1549 for (t = 0; t < ARRAY_SIZE(types); t++) { 1550 type = types[t]; 1551 1552 prepare_reuseport_grp(type, map_fd, sizeof(__u64), grpa_fds64, 1553 grpa_cookies, ARRAY_SIZE(grpa_fds64)); 1554 1555 /* Test BPF_* update flags */ 1556 /* BPF_EXIST failure case */ 1557 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1558 BPF_EXIST); 1559 CHECK(err != -1 || errno != ENOENT, 1560 "reuseport array update empty elem BPF_EXIST", 1561 "sock_type:%d err:%d errno:%d\n", 1562 type, err, errno); 1563 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1564 1565 /* BPF_NOEXIST success case */ 1566 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1567 BPF_NOEXIST); 1568 CHECK(err == -1, 1569 "reuseport array update empty elem BPF_NOEXIST", 1570 "sock_type:%d err:%d errno:%d\n", 1571 type, err, errno); 1572 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1573 1574 /* BPF_EXIST success case. */ 1575 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1576 BPF_EXIST); 1577 CHECK(err == -1, 1578 "reuseport array update same elem BPF_EXIST", 1579 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1580 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1581 1582 /* BPF_NOEXIST failure case */ 1583 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1584 BPF_NOEXIST); 1585 CHECK(err != -1 || errno != EEXIST, 1586 "reuseport array update non-empty elem BPF_NOEXIST", 1587 "sock_type:%d err:%d errno:%d\n", 1588 type, err, errno); 1589 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1590 1591 /* BPF_ANY case (always succeed) */ 1592 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1593 BPF_ANY); 1594 CHECK(err == -1, 1595 "reuseport array update same sk with BPF_ANY", 1596 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1597 1598 fd64 = grpa_fds64[fds_idx]; 1599 sk_cookie = grpa_cookies[fds_idx]; 1600 1601 /* The same sk cannot be added to reuseport_array twice */ 1602 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_ANY); 1603 CHECK(err != -1 || errno != EBUSY, 1604 "reuseport array update same sk with same index", 1605 "sock_type:%d err:%d errno:%d\n", 1606 type, err, errno); 1607 1608 err = bpf_map_update_elem(map_fd, &index0, &fd64, BPF_ANY); 1609 CHECK(err != -1 || errno != EBUSY, 1610 "reuseport array update same sk with different index", 1611 "sock_type:%d err:%d errno:%d\n", 1612 type, err, errno); 1613 1614 /* Test delete elem */ 1615 err = bpf_map_delete_elem(map_fd, &index3); 1616 CHECK(err == -1, "reuseport array delete sk", 1617 "sock_type:%d err:%d errno:%d\n", 1618 type, err, errno); 1619 1620 /* Add it back with BPF_NOEXIST */ 1621 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST); 1622 CHECK(err == -1, 1623 "reuseport array re-add with BPF_NOEXIST after del", 1624 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1625 1626 /* Test cookie */ 1627 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1628 CHECK(err == -1 || sk_cookie != map_cookie, 1629 "reuseport array lookup re-added sk", 1630 "sock_type:%d err:%d errno:%d sk_cookie:0x%llx map_cookie:0x%llxn", 1631 type, err, errno, sk_cookie, map_cookie); 1632 1633 /* Test elem removed by close() */ 1634 for (f = 0; f < ARRAY_SIZE(grpa_fds64); f++) 1635 close(grpa_fds64[f]); 1636 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1637 CHECK(err != -1 || errno != ENOENT, 1638 "reuseport array lookup after close()", 1639 "sock_type:%d err:%d errno:%d\n", 1640 type, err, errno); 1641 } 1642 1643 /* Test SOCK_RAW */ 1644 fd64 = socket(AF_INET6, SOCK_RAW, IPPROTO_UDP); 1645 CHECK(fd64 == -1, "socket(SOCK_RAW)", "err:%d errno:%d\n", 1646 err, errno); 1647 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST); 1648 CHECK(err != -1 || errno != ENOTSUPP, "reuseport array update SOCK_RAW", 1649 "err:%d errno:%d\n", err, errno); 1650 close(fd64); 1651 1652 /* Close the 64 bit value map */ 1653 close(map_fd); 1654 1655 /* Test 32 bit fd */ 1656 map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, 1657 sizeof(__u32), sizeof(__u32), array_size, 0); 1658 CHECK(map_fd == -1, "reuseport array create", 1659 "map_fd:%d, errno:%d\n", map_fd, errno); 1660 prepare_reuseport_grp(SOCK_STREAM, map_fd, sizeof(__u32), &fd64, 1661 &sk_cookie, 1); 1662 fd = fd64; 1663 err = bpf_map_update_elem(map_fd, &index3, &fd, BPF_NOEXIST); 1664 CHECK(err == -1, "reuseport array update 32 bit fd", 1665 "err:%d errno:%d\n", err, errno); 1666 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1667 CHECK(err != -1 || errno != ENOSPC, 1668 "reuseport array lookup 32 bit fd", 1669 "err:%d errno:%d\n", err, errno); 1670 close(fd); 1671 close(map_fd); 1672 } 1673 1674 static void run_all_tests(void) 1675 { 1676 test_hashmap(0, NULL); 1677 test_hashmap_percpu(0, NULL); 1678 test_hashmap_walk(0, NULL); 1679 test_hashmap_zero_seed(); 1680 1681 test_arraymap(0, NULL); 1682 test_arraymap_percpu(0, NULL); 1683 1684 test_arraymap_percpu_many_keys(); 1685 1686 test_devmap(0, NULL); 1687 test_sockmap(0, NULL); 1688 1689 test_map_large(); 1690 test_map_parallel(); 1691 test_map_stress(); 1692 1693 test_map_rdonly(); 1694 test_map_wronly(); 1695 1696 test_reuseport_array(); 1697 1698 test_queuemap(0, NULL); 1699 test_stackmap(0, NULL); 1700 1701 test_map_in_map(); 1702 } 1703 1704 #define DECLARE 1705 #include <map_tests/tests.h> 1706 #undef DECLARE 1707 1708 int main(void) 1709 { 1710 srand(time(NULL)); 1711 1712 map_flags = 0; 1713 run_all_tests(); 1714 1715 map_flags = BPF_F_NO_PREALLOC; 1716 run_all_tests(); 1717 1718 #define CALL 1719 #include <map_tests/tests.h> 1720 #undef CALL 1721 1722 printf("test_maps: OK, %d SKIPPED\n", skips); 1723 return 0; 1724 } 1725