1 /* Copyright (c) 2016 Facebook 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 */ 7 #define _GNU_SOURCE 8 #include <sched.h> 9 #include <stdio.h> 10 #include <sys/types.h> 11 #include <asm/unistd.h> 12 #include <unistd.h> 13 #include <assert.h> 14 #include <sys/wait.h> 15 #include <stdlib.h> 16 #include <signal.h> 17 #include <linux/bpf.h> 18 #include <string.h> 19 #include <time.h> 20 #include <sys/resource.h> 21 #include <arpa/inet.h> 22 #include <errno.h> 23 24 #include "libbpf.h" 25 #include "bpf_load.h" 26 27 #define TEST_BIT(t) (1U << (t)) 28 #define MAX_NR_CPUS 1024 29 30 static __u64 time_get_ns(void) 31 { 32 struct timespec ts; 33 34 clock_gettime(CLOCK_MONOTONIC, &ts); 35 return ts.tv_sec * 1000000000ull + ts.tv_nsec; 36 } 37 38 enum test_type { 39 HASH_PREALLOC, 40 PERCPU_HASH_PREALLOC, 41 HASH_KMALLOC, 42 PERCPU_HASH_KMALLOC, 43 LRU_HASH_PREALLOC, 44 NOCOMMON_LRU_HASH_PREALLOC, 45 LPM_KMALLOC, 46 HASH_LOOKUP, 47 ARRAY_LOOKUP, 48 INNER_LRU_HASH_PREALLOC, 49 NR_TESTS, 50 }; 51 52 const char *test_map_names[NR_TESTS] = { 53 [HASH_PREALLOC] = "hash_map", 54 [PERCPU_HASH_PREALLOC] = "percpu_hash_map", 55 [HASH_KMALLOC] = "hash_map_alloc", 56 [PERCPU_HASH_KMALLOC] = "percpu_hash_map_alloc", 57 [LRU_HASH_PREALLOC] = "lru_hash_map", 58 [NOCOMMON_LRU_HASH_PREALLOC] = "nocommon_lru_hash_map", 59 [LPM_KMALLOC] = "lpm_trie_map_alloc", 60 [HASH_LOOKUP] = "hash_map", 61 [ARRAY_LOOKUP] = "array_map", 62 [INNER_LRU_HASH_PREALLOC] = "inner_lru_hash_map", 63 }; 64 65 static int test_flags = ~0; 66 static uint32_t num_map_entries; 67 static uint32_t inner_lru_hash_size; 68 static int inner_lru_hash_idx = -1; 69 static int array_of_lru_hashs_idx = -1; 70 static uint32_t max_cnt = 1000000; 71 72 static int check_test_flags(enum test_type t) 73 { 74 return test_flags & TEST_BIT(t); 75 } 76 77 static void test_hash_prealloc(int cpu) 78 { 79 __u64 start_time; 80 int i; 81 82 start_time = time_get_ns(); 83 for (i = 0; i < max_cnt; i++) 84 syscall(__NR_getuid); 85 printf("%d:hash_map_perf pre-alloc %lld events per sec\n", 86 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 87 } 88 89 static void do_test_lru(enum test_type test, int cpu) 90 { 91 static int inner_lru_map_fds[MAX_NR_CPUS]; 92 93 struct sockaddr_in6 in6 = { .sin6_family = AF_INET6 }; 94 const char *test_name; 95 __u64 start_time; 96 int i, ret; 97 98 if (test == INNER_LRU_HASH_PREALLOC) { 99 int outer_fd = map_fd[array_of_lru_hashs_idx]; 100 101 assert(cpu < MAX_NR_CPUS); 102 103 if (cpu) { 104 inner_lru_map_fds[cpu] = 105 bpf_create_map(BPF_MAP_TYPE_LRU_HASH, 106 sizeof(uint32_t), sizeof(long), 107 inner_lru_hash_size, 0); 108 if (inner_lru_map_fds[cpu] == -1) { 109 printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n", 110 strerror(errno), errno); 111 exit(1); 112 } 113 } else { 114 inner_lru_map_fds[cpu] = map_fd[inner_lru_hash_idx]; 115 } 116 117 ret = bpf_map_update_elem(outer_fd, &cpu, 118 &inner_lru_map_fds[cpu], 119 BPF_ANY); 120 if (ret) { 121 printf("cannot update ARRAY_OF_LRU_HASHS with key:%u. %s(%d)\n", 122 cpu, strerror(errno), errno); 123 exit(1); 124 } 125 } 126 127 in6.sin6_addr.s6_addr16[0] = 0xdead; 128 in6.sin6_addr.s6_addr16[1] = 0xbeef; 129 130 if (test == LRU_HASH_PREALLOC) { 131 test_name = "lru_hash_map_perf"; 132 in6.sin6_addr.s6_addr16[7] = 0; 133 } else if (test == NOCOMMON_LRU_HASH_PREALLOC) { 134 test_name = "nocommon_lru_hash_map_perf"; 135 in6.sin6_addr.s6_addr16[7] = 1; 136 } else if (test == INNER_LRU_HASH_PREALLOC) { 137 test_name = "inner_lru_hash_map_perf"; 138 in6.sin6_addr.s6_addr16[7] = 2; 139 } else { 140 assert(0); 141 } 142 143 start_time = time_get_ns(); 144 for (i = 0; i < max_cnt; i++) { 145 ret = connect(-1, (const struct sockaddr *)&in6, sizeof(in6)); 146 assert(ret == -1 && errno == EBADF); 147 } 148 printf("%d:%s pre-alloc %lld events per sec\n", 149 cpu, test_name, 150 max_cnt * 1000000000ll / (time_get_ns() - start_time)); 151 } 152 153 static void test_lru_hash_prealloc(int cpu) 154 { 155 do_test_lru(LRU_HASH_PREALLOC, cpu); 156 } 157 158 static void test_nocommon_lru_hash_prealloc(int cpu) 159 { 160 do_test_lru(NOCOMMON_LRU_HASH_PREALLOC, cpu); 161 } 162 163 static void test_inner_lru_hash_prealloc(int cpu) 164 { 165 do_test_lru(INNER_LRU_HASH_PREALLOC, cpu); 166 } 167 168 static void test_percpu_hash_prealloc(int cpu) 169 { 170 __u64 start_time; 171 int i; 172 173 start_time = time_get_ns(); 174 for (i = 0; i < max_cnt; i++) 175 syscall(__NR_geteuid); 176 printf("%d:percpu_hash_map_perf pre-alloc %lld events per sec\n", 177 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 178 } 179 180 static void test_hash_kmalloc(int cpu) 181 { 182 __u64 start_time; 183 int i; 184 185 start_time = time_get_ns(); 186 for (i = 0; i < max_cnt; i++) 187 syscall(__NR_getgid); 188 printf("%d:hash_map_perf kmalloc %lld events per sec\n", 189 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 190 } 191 192 static void test_percpu_hash_kmalloc(int cpu) 193 { 194 __u64 start_time; 195 int i; 196 197 start_time = time_get_ns(); 198 for (i = 0; i < max_cnt; i++) 199 syscall(__NR_getegid); 200 printf("%d:percpu_hash_map_perf kmalloc %lld events per sec\n", 201 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 202 } 203 204 static void test_lpm_kmalloc(int cpu) 205 { 206 __u64 start_time; 207 int i; 208 209 start_time = time_get_ns(); 210 for (i = 0; i < max_cnt; i++) 211 syscall(__NR_gettid); 212 printf("%d:lpm_perf kmalloc %lld events per sec\n", 213 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 214 } 215 216 static void test_hash_lookup(int cpu) 217 { 218 __u64 start_time; 219 int i; 220 221 start_time = time_get_ns(); 222 for (i = 0; i < max_cnt; i++) 223 syscall(__NR_getpgid, 0); 224 printf("%d:hash_lookup %lld lookups per sec\n", 225 cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time)); 226 } 227 228 static void test_array_lookup(int cpu) 229 { 230 __u64 start_time; 231 int i; 232 233 start_time = time_get_ns(); 234 for (i = 0; i < max_cnt; i++) 235 syscall(__NR_getpgrp, 0); 236 printf("%d:array_lookup %lld lookups per sec\n", 237 cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time)); 238 } 239 240 typedef void (*test_func)(int cpu); 241 const test_func test_funcs[] = { 242 [HASH_PREALLOC] = test_hash_prealloc, 243 [PERCPU_HASH_PREALLOC] = test_percpu_hash_prealloc, 244 [HASH_KMALLOC] = test_hash_kmalloc, 245 [PERCPU_HASH_KMALLOC] = test_percpu_hash_kmalloc, 246 [LRU_HASH_PREALLOC] = test_lru_hash_prealloc, 247 [NOCOMMON_LRU_HASH_PREALLOC] = test_nocommon_lru_hash_prealloc, 248 [LPM_KMALLOC] = test_lpm_kmalloc, 249 [HASH_LOOKUP] = test_hash_lookup, 250 [ARRAY_LOOKUP] = test_array_lookup, 251 [INNER_LRU_HASH_PREALLOC] = test_inner_lru_hash_prealloc, 252 }; 253 254 static void loop(int cpu) 255 { 256 cpu_set_t cpuset; 257 int i; 258 259 CPU_ZERO(&cpuset); 260 CPU_SET(cpu, &cpuset); 261 sched_setaffinity(0, sizeof(cpuset), &cpuset); 262 263 for (i = 0; i < NR_TESTS; i++) { 264 if (check_test_flags(i)) 265 test_funcs[i](cpu); 266 } 267 } 268 269 static void run_perf_test(int tasks) 270 { 271 pid_t pid[tasks]; 272 int i; 273 274 for (i = 0; i < tasks; i++) { 275 pid[i] = fork(); 276 if (pid[i] == 0) { 277 loop(i); 278 exit(0); 279 } else if (pid[i] == -1) { 280 printf("couldn't spawn #%d process\n", i); 281 exit(1); 282 } 283 } 284 for (i = 0; i < tasks; i++) { 285 int status; 286 287 assert(waitpid(pid[i], &status, 0) == pid[i]); 288 assert(status == 0); 289 } 290 } 291 292 static void fill_lpm_trie(void) 293 { 294 struct bpf_lpm_trie_key *key; 295 unsigned long value = 0; 296 unsigned int i; 297 int r; 298 299 key = alloca(sizeof(*key) + 4); 300 key->prefixlen = 32; 301 302 for (i = 0; i < 512; ++i) { 303 key->prefixlen = rand() % 33; 304 key->data[0] = rand() & 0xff; 305 key->data[1] = rand() & 0xff; 306 key->data[2] = rand() & 0xff; 307 key->data[3] = rand() & 0xff; 308 r = bpf_map_update_elem(map_fd[6], key, &value, 0); 309 assert(!r); 310 } 311 312 key->prefixlen = 32; 313 key->data[0] = 192; 314 key->data[1] = 168; 315 key->data[2] = 0; 316 key->data[3] = 1; 317 value = 128; 318 319 r = bpf_map_update_elem(map_fd[6], key, &value, 0); 320 assert(!r); 321 } 322 323 static void fixup_map(struct bpf_map_data *map, int idx) 324 { 325 int i; 326 327 if (!strcmp("inner_lru_hash_map", map->name)) { 328 inner_lru_hash_idx = idx; 329 inner_lru_hash_size = map->def.max_entries; 330 } 331 332 if (!strcmp("array_of_lru_hashs", map->name)) { 333 if (inner_lru_hash_idx == -1) { 334 printf("inner_lru_hash_map must be defined before array_of_lru_hashs\n"); 335 exit(1); 336 } 337 map->def.inner_map_idx = inner_lru_hash_idx; 338 array_of_lru_hashs_idx = idx; 339 } 340 341 if (num_map_entries <= 0) 342 return; 343 344 inner_lru_hash_size = num_map_entries; 345 346 /* Only change the max_entries for the enabled test(s) */ 347 for (i = 0; i < NR_TESTS; i++) { 348 if (!strcmp(test_map_names[i], map->name) && 349 (check_test_flags(i))) { 350 map->def.max_entries = num_map_entries; 351 } 352 } 353 } 354 355 int main(int argc, char **argv) 356 { 357 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; 358 char filename[256]; 359 int num_cpu = 8; 360 361 snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); 362 setrlimit(RLIMIT_MEMLOCK, &r); 363 364 if (argc > 1) 365 test_flags = atoi(argv[1]) ? : test_flags; 366 367 if (argc > 2) 368 num_cpu = atoi(argv[2]) ? : num_cpu; 369 370 if (argc > 3) 371 num_map_entries = atoi(argv[3]); 372 373 if (argc > 4) 374 max_cnt = atoi(argv[4]); 375 376 if (load_bpf_file_fixup_map(filename, fixup_map)) { 377 printf("%s", bpf_log_buf); 378 return 1; 379 } 380 381 fill_lpm_trie(); 382 383 run_perf_test(num_cpu); 384 385 return 0; 386 } 387