1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * numa.c 4 * 5 * numa: Simulate NUMA-sensitive workload and measure their NUMA performance 6 */ 7 8 #include <inttypes.h> 9 /* For the CLR_() macros */ 10 #include <pthread.h> 11 12 #include <subcmd/parse-options.h> 13 #include "../util/cloexec.h" 14 15 #include "bench.h" 16 17 #include <errno.h> 18 #include <sched.h> 19 #include <stdio.h> 20 #include <assert.h> 21 #include <malloc.h> 22 #include <signal.h> 23 #include <stdlib.h> 24 #include <string.h> 25 #include <unistd.h> 26 #include <sys/mman.h> 27 #include <sys/time.h> 28 #include <sys/resource.h> 29 #include <sys/wait.h> 30 #include <sys/prctl.h> 31 #include <sys/types.h> 32 #include <linux/kernel.h> 33 #include <linux/time64.h> 34 #include <linux/numa.h> 35 #include <linux/zalloc.h> 36 37 #include "../util/header.h" 38 #include <numa.h> 39 #include <numaif.h> 40 41 #ifndef RUSAGE_THREAD 42 # define RUSAGE_THREAD 1 43 #endif 44 45 /* 46 * Regular printout to the terminal, suppressed if -q is specified: 47 */ 48 #define tprintf(x...) do { if (g && g->p.show_details >= 0) printf(x); } while (0) 49 50 /* 51 * Debug printf: 52 */ 53 #undef dprintf 54 #define dprintf(x...) do { if (g && g->p.show_details >= 1) printf(x); } while (0) 55 56 struct thread_data { 57 int curr_cpu; 58 cpu_set_t bind_cpumask; 59 int bind_node; 60 u8 *process_data; 61 int process_nr; 62 int thread_nr; 63 int task_nr; 64 unsigned int loops_done; 65 u64 val; 66 u64 runtime_ns; 67 u64 system_time_ns; 68 u64 user_time_ns; 69 double speed_gbs; 70 pthread_mutex_t *process_lock; 71 }; 72 73 /* Parameters set by options: */ 74 75 struct params { 76 /* Startup synchronization: */ 77 bool serialize_startup; 78 79 /* Task hierarchy: */ 80 int nr_proc; 81 int nr_threads; 82 83 /* Working set sizes: */ 84 const char *mb_global_str; 85 const char *mb_proc_str; 86 const char *mb_proc_locked_str; 87 const char *mb_thread_str; 88 89 double mb_global; 90 double mb_proc; 91 double mb_proc_locked; 92 double mb_thread; 93 94 /* Access patterns to the working set: */ 95 bool data_reads; 96 bool data_writes; 97 bool data_backwards; 98 bool data_zero_memset; 99 bool data_rand_walk; 100 u32 nr_loops; 101 u32 nr_secs; 102 u32 sleep_usecs; 103 104 /* Working set initialization: */ 105 bool init_zero; 106 bool init_random; 107 bool init_cpu0; 108 109 /* Misc options: */ 110 int show_details; 111 int run_all; 112 int thp; 113 114 long bytes_global; 115 long bytes_process; 116 long bytes_process_locked; 117 long bytes_thread; 118 119 int nr_tasks; 120 bool show_quiet; 121 122 bool show_convergence; 123 bool measure_convergence; 124 125 int perturb_secs; 126 int nr_cpus; 127 int nr_nodes; 128 129 /* Affinity options -C and -N: */ 130 char *cpu_list_str; 131 char *node_list_str; 132 }; 133 134 135 /* Global, read-writable area, accessible to all processes and threads: */ 136 137 struct global_info { 138 u8 *data; 139 140 pthread_mutex_t startup_mutex; 141 pthread_cond_t startup_cond; 142 int nr_tasks_started; 143 144 pthread_mutex_t start_work_mutex; 145 pthread_cond_t start_work_cond; 146 int nr_tasks_working; 147 bool start_work; 148 149 pthread_mutex_t stop_work_mutex; 150 u64 bytes_done; 151 152 struct thread_data *threads; 153 154 /* Convergence latency measurement: */ 155 bool all_converged; 156 bool stop_work; 157 158 int print_once; 159 160 struct params p; 161 }; 162 163 static struct global_info *g = NULL; 164 165 static int parse_cpus_opt(const struct option *opt, const char *arg, int unset); 166 static int parse_nodes_opt(const struct option *opt, const char *arg, int unset); 167 168 struct params p0; 169 170 static const struct option options[] = { 171 OPT_INTEGER('p', "nr_proc" , &p0.nr_proc, "number of processes"), 172 OPT_INTEGER('t', "nr_threads" , &p0.nr_threads, "number of threads per process"), 173 174 OPT_STRING('G', "mb_global" , &p0.mb_global_str, "MB", "global memory (MBs)"), 175 OPT_STRING('P', "mb_proc" , &p0.mb_proc_str, "MB", "process memory (MBs)"), 176 OPT_STRING('L', "mb_proc_locked", &p0.mb_proc_locked_str,"MB", "process serialized/locked memory access (MBs), <= process_memory"), 177 OPT_STRING('T', "mb_thread" , &p0.mb_thread_str, "MB", "thread memory (MBs)"), 178 179 OPT_UINTEGER('l', "nr_loops" , &p0.nr_loops, "max number of loops to run (default: unlimited)"), 180 OPT_UINTEGER('s', "nr_secs" , &p0.nr_secs, "max number of seconds to run (default: 5 secs)"), 181 OPT_UINTEGER('u', "usleep" , &p0.sleep_usecs, "usecs to sleep per loop iteration"), 182 183 OPT_BOOLEAN('R', "data_reads" , &p0.data_reads, "access the data via reads (can be mixed with -W)"), 184 OPT_BOOLEAN('W', "data_writes" , &p0.data_writes, "access the data via writes (can be mixed with -R)"), 185 OPT_BOOLEAN('B', "data_backwards", &p0.data_backwards, "access the data backwards as well"), 186 OPT_BOOLEAN('Z', "data_zero_memset", &p0.data_zero_memset,"access the data via glibc bzero only"), 187 OPT_BOOLEAN('r', "data_rand_walk", &p0.data_rand_walk, "access the data with random (32bit LFSR) walk"), 188 189 190 OPT_BOOLEAN('z', "init_zero" , &p0.init_zero, "bzero the initial allocations"), 191 OPT_BOOLEAN('I', "init_random" , &p0.init_random, "randomize the contents of the initial allocations"), 192 OPT_BOOLEAN('0', "init_cpu0" , &p0.init_cpu0, "do the initial allocations on CPU#0"), 193 OPT_INTEGER('x', "perturb_secs", &p0.perturb_secs, "perturb thread 0/0 every X secs, to test convergence stability"), 194 195 OPT_INCR ('d', "show_details" , &p0.show_details, "Show details"), 196 OPT_INCR ('a', "all" , &p0.run_all, "Run all tests in the suite"), 197 OPT_INTEGER('H', "thp" , &p0.thp, "MADV_NOHUGEPAGE < 0 < MADV_HUGEPAGE"), 198 OPT_BOOLEAN('c', "show_convergence", &p0.show_convergence, "show convergence details, " 199 "convergence is reached when each process (all its threads) is running on a single NUMA node."), 200 OPT_BOOLEAN('m', "measure_convergence", &p0.measure_convergence, "measure convergence latency"), 201 OPT_BOOLEAN('q', "quiet" , &p0.show_quiet, "quiet mode"), 202 OPT_BOOLEAN('S', "serialize-startup", &p0.serialize_startup,"serialize thread startup"), 203 204 /* Special option string parsing callbacks: */ 205 OPT_CALLBACK('C', "cpus", NULL, "cpu[,cpu2,...cpuN]", 206 "bind the first N tasks to these specific cpus (the rest is unbound)", 207 parse_cpus_opt), 208 OPT_CALLBACK('M', "memnodes", NULL, "node[,node2,...nodeN]", 209 "bind the first N tasks to these specific memory nodes (the rest is unbound)", 210 parse_nodes_opt), 211 OPT_END() 212 }; 213 214 static const char * const bench_numa_usage[] = { 215 "perf bench numa <options>", 216 NULL 217 }; 218 219 static const char * const numa_usage[] = { 220 "perf bench numa mem [<options>]", 221 NULL 222 }; 223 224 /* 225 * To get number of numa nodes present. 226 */ 227 static int nr_numa_nodes(void) 228 { 229 int i, nr_nodes = 0; 230 231 for (i = 0; i < g->p.nr_nodes; i++) { 232 if (numa_bitmask_isbitset(numa_nodes_ptr, i)) 233 nr_nodes++; 234 } 235 236 return nr_nodes; 237 } 238 239 /* 240 * To check if given numa node is present. 241 */ 242 static int is_node_present(int node) 243 { 244 return numa_bitmask_isbitset(numa_nodes_ptr, node); 245 } 246 247 /* 248 * To check given numa node has cpus. 249 */ 250 static bool node_has_cpus(int node) 251 { 252 struct bitmask *cpumask = numa_allocate_cpumask(); 253 bool ret = false; /* fall back to nocpus */ 254 int cpu; 255 256 BUG_ON(!cpumask); 257 if (!numa_node_to_cpus(node, cpumask)) { 258 for (cpu = 0; cpu < (int)cpumask->size; cpu++) { 259 if (numa_bitmask_isbitset(cpumask, cpu)) { 260 ret = true; 261 break; 262 } 263 } 264 } 265 numa_free_cpumask(cpumask); 266 267 return ret; 268 } 269 270 static cpu_set_t bind_to_cpu(int target_cpu) 271 { 272 cpu_set_t orig_mask, mask; 273 int ret; 274 275 ret = sched_getaffinity(0, sizeof(orig_mask), &orig_mask); 276 BUG_ON(ret); 277 278 CPU_ZERO(&mask); 279 280 if (target_cpu == -1) { 281 int cpu; 282 283 for (cpu = 0; cpu < g->p.nr_cpus; cpu++) 284 CPU_SET(cpu, &mask); 285 } else { 286 BUG_ON(target_cpu < 0 || target_cpu >= g->p.nr_cpus); 287 CPU_SET(target_cpu, &mask); 288 } 289 290 ret = sched_setaffinity(0, sizeof(mask), &mask); 291 BUG_ON(ret); 292 293 return orig_mask; 294 } 295 296 static cpu_set_t bind_to_node(int target_node) 297 { 298 cpu_set_t orig_mask, mask; 299 int cpu; 300 int ret; 301 302 ret = sched_getaffinity(0, sizeof(orig_mask), &orig_mask); 303 BUG_ON(ret); 304 305 CPU_ZERO(&mask); 306 307 if (target_node == NUMA_NO_NODE) { 308 for (cpu = 0; cpu < g->p.nr_cpus; cpu++) 309 CPU_SET(cpu, &mask); 310 } else { 311 struct bitmask *cpumask = numa_allocate_cpumask(); 312 313 BUG_ON(!cpumask); 314 if (!numa_node_to_cpus(target_node, cpumask)) { 315 for (cpu = 0; cpu < (int)cpumask->size; cpu++) { 316 if (numa_bitmask_isbitset(cpumask, cpu)) 317 CPU_SET(cpu, &mask); 318 } 319 } 320 numa_free_cpumask(cpumask); 321 } 322 323 ret = sched_setaffinity(0, sizeof(mask), &mask); 324 BUG_ON(ret); 325 326 return orig_mask; 327 } 328 329 static void bind_to_cpumask(cpu_set_t mask) 330 { 331 int ret; 332 333 ret = sched_setaffinity(0, sizeof(mask), &mask); 334 BUG_ON(ret); 335 } 336 337 static void mempol_restore(void) 338 { 339 int ret; 340 341 ret = set_mempolicy(MPOL_DEFAULT, NULL, g->p.nr_nodes-1); 342 343 BUG_ON(ret); 344 } 345 346 static void bind_to_memnode(int node) 347 { 348 struct bitmask *node_mask; 349 int ret; 350 351 if (node == NUMA_NO_NODE) 352 return; 353 354 node_mask = numa_allocate_nodemask(); 355 BUG_ON(!node_mask); 356 357 numa_bitmask_clearall(node_mask); 358 numa_bitmask_setbit(node_mask, node); 359 360 ret = set_mempolicy(MPOL_BIND, node_mask->maskp, node_mask->size + 1); 361 dprintf("binding to node %d, mask: %016lx => %d\n", node, *node_mask->maskp, ret); 362 363 numa_bitmask_free(node_mask); 364 BUG_ON(ret); 365 } 366 367 #define HPSIZE (2*1024*1024) 368 369 #define set_taskname(fmt...) \ 370 do { \ 371 char name[20]; \ 372 \ 373 snprintf(name, 20, fmt); \ 374 prctl(PR_SET_NAME, name); \ 375 } while (0) 376 377 static u8 *alloc_data(ssize_t bytes0, int map_flags, 378 int init_zero, int init_cpu0, int thp, int init_random) 379 { 380 cpu_set_t orig_mask; 381 ssize_t bytes; 382 u8 *buf; 383 int ret; 384 385 if (!bytes0) 386 return NULL; 387 388 /* Allocate and initialize all memory on CPU#0: */ 389 if (init_cpu0) { 390 int node = numa_node_of_cpu(0); 391 392 orig_mask = bind_to_node(node); 393 bind_to_memnode(node); 394 } 395 396 bytes = bytes0 + HPSIZE; 397 398 buf = (void *)mmap(0, bytes, PROT_READ|PROT_WRITE, MAP_ANON|map_flags, -1, 0); 399 BUG_ON(buf == (void *)-1); 400 401 if (map_flags == MAP_PRIVATE) { 402 if (thp > 0) { 403 ret = madvise(buf, bytes, MADV_HUGEPAGE); 404 if (ret && !g->print_once) { 405 g->print_once = 1; 406 printf("WARNING: Could not enable THP - do: 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled'\n"); 407 } 408 } 409 if (thp < 0) { 410 ret = madvise(buf, bytes, MADV_NOHUGEPAGE); 411 if (ret && !g->print_once) { 412 g->print_once = 1; 413 printf("WARNING: Could not disable THP: run a CONFIG_TRANSPARENT_HUGEPAGE kernel?\n"); 414 } 415 } 416 } 417 418 if (init_zero) { 419 bzero(buf, bytes); 420 } else { 421 /* Initialize random contents, different in each word: */ 422 if (init_random) { 423 u64 *wbuf = (void *)buf; 424 long off = rand(); 425 long i; 426 427 for (i = 0; i < bytes/8; i++) 428 wbuf[i] = i + off; 429 } 430 } 431 432 /* Align to 2MB boundary: */ 433 buf = (void *)(((unsigned long)buf + HPSIZE-1) & ~(HPSIZE-1)); 434 435 /* Restore affinity: */ 436 if (init_cpu0) { 437 bind_to_cpumask(orig_mask); 438 mempol_restore(); 439 } 440 441 return buf; 442 } 443 444 static void free_data(void *data, ssize_t bytes) 445 { 446 int ret; 447 448 if (!data) 449 return; 450 451 ret = munmap(data, bytes); 452 BUG_ON(ret); 453 } 454 455 /* 456 * Create a shared memory buffer that can be shared between processes, zeroed: 457 */ 458 static void * zalloc_shared_data(ssize_t bytes) 459 { 460 return alloc_data(bytes, MAP_SHARED, 1, g->p.init_cpu0, g->p.thp, g->p.init_random); 461 } 462 463 /* 464 * Create a shared memory buffer that can be shared between processes: 465 */ 466 static void * setup_shared_data(ssize_t bytes) 467 { 468 return alloc_data(bytes, MAP_SHARED, 0, g->p.init_cpu0, g->p.thp, g->p.init_random); 469 } 470 471 /* 472 * Allocate process-local memory - this will either be shared between 473 * threads of this process, or only be accessed by this thread: 474 */ 475 static void * setup_private_data(ssize_t bytes) 476 { 477 return alloc_data(bytes, MAP_PRIVATE, 0, g->p.init_cpu0, g->p.thp, g->p.init_random); 478 } 479 480 /* 481 * Return a process-shared (global) mutex: 482 */ 483 static void init_global_mutex(pthread_mutex_t *mutex) 484 { 485 pthread_mutexattr_t attr; 486 487 pthread_mutexattr_init(&attr); 488 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); 489 pthread_mutex_init(mutex, &attr); 490 } 491 492 /* 493 * Return a process-shared (global) condition variable: 494 */ 495 static void init_global_cond(pthread_cond_t *cond) 496 { 497 pthread_condattr_t attr; 498 499 pthread_condattr_init(&attr); 500 pthread_condattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); 501 pthread_cond_init(cond, &attr); 502 } 503 504 static int parse_cpu_list(const char *arg) 505 { 506 p0.cpu_list_str = strdup(arg); 507 508 dprintf("got CPU list: {%s}\n", p0.cpu_list_str); 509 510 return 0; 511 } 512 513 static int parse_setup_cpu_list(void) 514 { 515 struct thread_data *td; 516 char *str0, *str; 517 int t; 518 519 if (!g->p.cpu_list_str) 520 return 0; 521 522 dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks); 523 524 str0 = str = strdup(g->p.cpu_list_str); 525 t = 0; 526 527 BUG_ON(!str); 528 529 tprintf("# binding tasks to CPUs:\n"); 530 tprintf("# "); 531 532 while (true) { 533 int bind_cpu, bind_cpu_0, bind_cpu_1; 534 char *tok, *tok_end, *tok_step, *tok_len, *tok_mul; 535 int bind_len; 536 int step; 537 int mul; 538 539 tok = strsep(&str, ","); 540 if (!tok) 541 break; 542 543 tok_end = strstr(tok, "-"); 544 545 dprintf("\ntoken: {%s}, end: {%s}\n", tok, tok_end); 546 if (!tok_end) { 547 /* Single CPU specified: */ 548 bind_cpu_0 = bind_cpu_1 = atol(tok); 549 } else { 550 /* CPU range specified (for example: "5-11"): */ 551 bind_cpu_0 = atol(tok); 552 bind_cpu_1 = atol(tok_end + 1); 553 } 554 555 step = 1; 556 tok_step = strstr(tok, "#"); 557 if (tok_step) { 558 step = atol(tok_step + 1); 559 BUG_ON(step <= 0 || step >= g->p.nr_cpus); 560 } 561 562 /* 563 * Mask length. 564 * Eg: "--cpus 8_4-16#4" means: '--cpus 8_4,12_4,16_4', 565 * where the _4 means the next 4 CPUs are allowed. 566 */ 567 bind_len = 1; 568 tok_len = strstr(tok, "_"); 569 if (tok_len) { 570 bind_len = atol(tok_len + 1); 571 BUG_ON(bind_len <= 0 || bind_len > g->p.nr_cpus); 572 } 573 574 /* Multiplicator shortcut, "0x8" is a shortcut for: "0,0,0,0,0,0,0,0" */ 575 mul = 1; 576 tok_mul = strstr(tok, "x"); 577 if (tok_mul) { 578 mul = atol(tok_mul + 1); 579 BUG_ON(mul <= 0); 580 } 581 582 dprintf("CPUs: %d_%d-%d#%dx%d\n", bind_cpu_0, bind_len, bind_cpu_1, step, mul); 583 584 if (bind_cpu_0 >= g->p.nr_cpus || bind_cpu_1 >= g->p.nr_cpus) { 585 printf("\nTest not applicable, system has only %d CPUs.\n", g->p.nr_cpus); 586 return -1; 587 } 588 589 if (is_cpu_online(bind_cpu_0) != 1 || is_cpu_online(bind_cpu_1) != 1) { 590 printf("\nTest not applicable, bind_cpu_0 or bind_cpu_1 is offline\n"); 591 return -1; 592 } 593 594 BUG_ON(bind_cpu_0 < 0 || bind_cpu_1 < 0); 595 BUG_ON(bind_cpu_0 > bind_cpu_1); 596 597 for (bind_cpu = bind_cpu_0; bind_cpu <= bind_cpu_1; bind_cpu += step) { 598 int i; 599 600 for (i = 0; i < mul; i++) { 601 int cpu; 602 603 if (t >= g->p.nr_tasks) { 604 printf("\n# NOTE: ignoring bind CPUs starting at CPU#%d\n #", bind_cpu); 605 goto out; 606 } 607 td = g->threads + t; 608 609 if (t) 610 tprintf(","); 611 if (bind_len > 1) { 612 tprintf("%2d/%d", bind_cpu, bind_len); 613 } else { 614 tprintf("%2d", bind_cpu); 615 } 616 617 CPU_ZERO(&td->bind_cpumask); 618 for (cpu = bind_cpu; cpu < bind_cpu+bind_len; cpu++) { 619 BUG_ON(cpu < 0 || cpu >= g->p.nr_cpus); 620 CPU_SET(cpu, &td->bind_cpumask); 621 } 622 t++; 623 } 624 } 625 } 626 out: 627 628 tprintf("\n"); 629 630 if (t < g->p.nr_tasks) 631 printf("# NOTE: %d tasks bound, %d tasks unbound\n", t, g->p.nr_tasks - t); 632 633 free(str0); 634 return 0; 635 } 636 637 static int parse_cpus_opt(const struct option *opt __maybe_unused, 638 const char *arg, int unset __maybe_unused) 639 { 640 if (!arg) 641 return -1; 642 643 return parse_cpu_list(arg); 644 } 645 646 static int parse_node_list(const char *arg) 647 { 648 p0.node_list_str = strdup(arg); 649 650 dprintf("got NODE list: {%s}\n", p0.node_list_str); 651 652 return 0; 653 } 654 655 static int parse_setup_node_list(void) 656 { 657 struct thread_data *td; 658 char *str0, *str; 659 int t; 660 661 if (!g->p.node_list_str) 662 return 0; 663 664 dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks); 665 666 str0 = str = strdup(g->p.node_list_str); 667 t = 0; 668 669 BUG_ON(!str); 670 671 tprintf("# binding tasks to NODEs:\n"); 672 tprintf("# "); 673 674 while (true) { 675 int bind_node, bind_node_0, bind_node_1; 676 char *tok, *tok_end, *tok_step, *tok_mul; 677 int step; 678 int mul; 679 680 tok = strsep(&str, ","); 681 if (!tok) 682 break; 683 684 tok_end = strstr(tok, "-"); 685 686 dprintf("\ntoken: {%s}, end: {%s}\n", tok, tok_end); 687 if (!tok_end) { 688 /* Single NODE specified: */ 689 bind_node_0 = bind_node_1 = atol(tok); 690 } else { 691 /* NODE range specified (for example: "5-11"): */ 692 bind_node_0 = atol(tok); 693 bind_node_1 = atol(tok_end + 1); 694 } 695 696 step = 1; 697 tok_step = strstr(tok, "#"); 698 if (tok_step) { 699 step = atol(tok_step + 1); 700 BUG_ON(step <= 0 || step >= g->p.nr_nodes); 701 } 702 703 /* Multiplicator shortcut, "0x8" is a shortcut for: "0,0,0,0,0,0,0,0" */ 704 mul = 1; 705 tok_mul = strstr(tok, "x"); 706 if (tok_mul) { 707 mul = atol(tok_mul + 1); 708 BUG_ON(mul <= 0); 709 } 710 711 dprintf("NODEs: %d-%d #%d\n", bind_node_0, bind_node_1, step); 712 713 if (bind_node_0 >= g->p.nr_nodes || bind_node_1 >= g->p.nr_nodes) { 714 printf("\nTest not applicable, system has only %d nodes.\n", g->p.nr_nodes); 715 return -1; 716 } 717 718 BUG_ON(bind_node_0 < 0 || bind_node_1 < 0); 719 BUG_ON(bind_node_0 > bind_node_1); 720 721 for (bind_node = bind_node_0; bind_node <= bind_node_1; bind_node += step) { 722 int i; 723 724 for (i = 0; i < mul; i++) { 725 if (t >= g->p.nr_tasks || !node_has_cpus(bind_node)) { 726 printf("\n# NOTE: ignoring bind NODEs starting at NODE#%d\n", bind_node); 727 goto out; 728 } 729 td = g->threads + t; 730 731 if (!t) 732 tprintf(" %2d", bind_node); 733 else 734 tprintf(",%2d", bind_node); 735 736 td->bind_node = bind_node; 737 t++; 738 } 739 } 740 } 741 out: 742 743 tprintf("\n"); 744 745 if (t < g->p.nr_tasks) 746 printf("# NOTE: %d tasks mem-bound, %d tasks unbound\n", t, g->p.nr_tasks - t); 747 748 free(str0); 749 return 0; 750 } 751 752 static int parse_nodes_opt(const struct option *opt __maybe_unused, 753 const char *arg, int unset __maybe_unused) 754 { 755 if (!arg) 756 return -1; 757 758 return parse_node_list(arg); 759 } 760 761 static inline uint32_t lfsr_32(uint32_t lfsr) 762 { 763 const uint32_t taps = BIT(1) | BIT(5) | BIT(6) | BIT(31); 764 return (lfsr>>1) ^ ((0x0u - (lfsr & 0x1u)) & taps); 765 } 766 767 /* 768 * Make sure there's real data dependency to RAM (when read 769 * accesses are enabled), so the compiler, the CPU and the 770 * kernel (KSM, zero page, etc.) cannot optimize away RAM 771 * accesses: 772 */ 773 static inline u64 access_data(u64 *data, u64 val) 774 { 775 if (g->p.data_reads) 776 val += *data; 777 if (g->p.data_writes) 778 *data = val + 1; 779 return val; 780 } 781 782 /* 783 * The worker process does two types of work, a forwards going 784 * loop and a backwards going loop. 785 * 786 * We do this so that on multiprocessor systems we do not create 787 * a 'train' of processing, with highly synchronized processes, 788 * skewing the whole benchmark. 789 */ 790 static u64 do_work(u8 *__data, long bytes, int nr, int nr_max, int loop, u64 val) 791 { 792 long words = bytes/sizeof(u64); 793 u64 *data = (void *)__data; 794 long chunk_0, chunk_1; 795 u64 *d0, *d, *d1; 796 long off; 797 long i; 798 799 BUG_ON(!data && words); 800 BUG_ON(data && !words); 801 802 if (!data) 803 return val; 804 805 /* Very simple memset() work variant: */ 806 if (g->p.data_zero_memset && !g->p.data_rand_walk) { 807 bzero(data, bytes); 808 return val; 809 } 810 811 /* Spread out by PID/TID nr and by loop nr: */ 812 chunk_0 = words/nr_max; 813 chunk_1 = words/g->p.nr_loops; 814 off = nr*chunk_0 + loop*chunk_1; 815 816 while (off >= words) 817 off -= words; 818 819 if (g->p.data_rand_walk) { 820 u32 lfsr = nr + loop + val; 821 int j; 822 823 for (i = 0; i < words/1024; i++) { 824 long start, end; 825 826 lfsr = lfsr_32(lfsr); 827 828 start = lfsr % words; 829 end = min(start + 1024, words-1); 830 831 if (g->p.data_zero_memset) { 832 bzero(data + start, (end-start) * sizeof(u64)); 833 } else { 834 for (j = start; j < end; j++) 835 val = access_data(data + j, val); 836 } 837 } 838 } else if (!g->p.data_backwards || (nr + loop) & 1) { 839 /* Process data forwards: */ 840 841 d0 = data + off; 842 d = data + off + 1; 843 d1 = data + words; 844 845 for (;;) { 846 if (unlikely(d >= d1)) 847 d = data; 848 if (unlikely(d == d0)) 849 break; 850 851 val = access_data(d, val); 852 853 d++; 854 } 855 } else { 856 /* Process data backwards: */ 857 858 d0 = data + off; 859 d = data + off - 1; 860 d1 = data + words; 861 862 for (;;) { 863 if (unlikely(d < data)) 864 d = data + words-1; 865 if (unlikely(d == d0)) 866 break; 867 868 val = access_data(d, val); 869 870 d--; 871 } 872 } 873 874 return val; 875 } 876 877 static void update_curr_cpu(int task_nr, unsigned long bytes_worked) 878 { 879 unsigned int cpu; 880 881 cpu = sched_getcpu(); 882 883 g->threads[task_nr].curr_cpu = cpu; 884 prctl(0, bytes_worked); 885 } 886 887 /* 888 * Count the number of nodes a process's threads 889 * are spread out on. 890 * 891 * A count of 1 means that the process is compressed 892 * to a single node. A count of g->p.nr_nodes means it's 893 * spread out on the whole system. 894 */ 895 static int count_process_nodes(int process_nr) 896 { 897 char *node_present; 898 int nodes; 899 int n, t; 900 901 node_present = (char *)malloc(g->p.nr_nodes * sizeof(char)); 902 BUG_ON(!node_present); 903 for (nodes = 0; nodes < g->p.nr_nodes; nodes++) 904 node_present[nodes] = 0; 905 906 for (t = 0; t < g->p.nr_threads; t++) { 907 struct thread_data *td; 908 int task_nr; 909 int node; 910 911 task_nr = process_nr*g->p.nr_threads + t; 912 td = g->threads + task_nr; 913 914 node = numa_node_of_cpu(td->curr_cpu); 915 if (node < 0) /* curr_cpu was likely still -1 */ { 916 free(node_present); 917 return 0; 918 } 919 920 node_present[node] = 1; 921 } 922 923 nodes = 0; 924 925 for (n = 0; n < g->p.nr_nodes; n++) 926 nodes += node_present[n]; 927 928 free(node_present); 929 return nodes; 930 } 931 932 /* 933 * Count the number of distinct process-threads a node contains. 934 * 935 * A count of 1 means that the node contains only a single 936 * process. If all nodes on the system contain at most one 937 * process then we are well-converged. 938 */ 939 static int count_node_processes(int node) 940 { 941 int processes = 0; 942 int t, p; 943 944 for (p = 0; p < g->p.nr_proc; p++) { 945 for (t = 0; t < g->p.nr_threads; t++) { 946 struct thread_data *td; 947 int task_nr; 948 int n; 949 950 task_nr = p*g->p.nr_threads + t; 951 td = g->threads + task_nr; 952 953 n = numa_node_of_cpu(td->curr_cpu); 954 if (n == node) { 955 processes++; 956 break; 957 } 958 } 959 } 960 961 return processes; 962 } 963 964 static void calc_convergence_compression(int *strong) 965 { 966 unsigned int nodes_min, nodes_max; 967 int p; 968 969 nodes_min = -1; 970 nodes_max = 0; 971 972 for (p = 0; p < g->p.nr_proc; p++) { 973 unsigned int nodes = count_process_nodes(p); 974 975 if (!nodes) { 976 *strong = 0; 977 return; 978 } 979 980 nodes_min = min(nodes, nodes_min); 981 nodes_max = max(nodes, nodes_max); 982 } 983 984 /* Strong convergence: all threads compress on a single node: */ 985 if (nodes_min == 1 && nodes_max == 1) { 986 *strong = 1; 987 } else { 988 *strong = 0; 989 tprintf(" {%d-%d}", nodes_min, nodes_max); 990 } 991 } 992 993 static void calc_convergence(double runtime_ns_max, double *convergence) 994 { 995 unsigned int loops_done_min, loops_done_max; 996 int process_groups; 997 int *nodes; 998 int distance; 999 int nr_min; 1000 int nr_max; 1001 int strong; 1002 int sum; 1003 int nr; 1004 int node; 1005 int cpu; 1006 int t; 1007 1008 if (!g->p.show_convergence && !g->p.measure_convergence) 1009 return; 1010 1011 nodes = (int *)malloc(g->p.nr_nodes * sizeof(int)); 1012 BUG_ON(!nodes); 1013 for (node = 0; node < g->p.nr_nodes; node++) 1014 nodes[node] = 0; 1015 1016 loops_done_min = -1; 1017 loops_done_max = 0; 1018 1019 for (t = 0; t < g->p.nr_tasks; t++) { 1020 struct thread_data *td = g->threads + t; 1021 unsigned int loops_done; 1022 1023 cpu = td->curr_cpu; 1024 1025 /* Not all threads have written it yet: */ 1026 if (cpu < 0) 1027 continue; 1028 1029 node = numa_node_of_cpu(cpu); 1030 1031 nodes[node]++; 1032 1033 loops_done = td->loops_done; 1034 loops_done_min = min(loops_done, loops_done_min); 1035 loops_done_max = max(loops_done, loops_done_max); 1036 } 1037 1038 nr_max = 0; 1039 nr_min = g->p.nr_tasks; 1040 sum = 0; 1041 1042 for (node = 0; node < g->p.nr_nodes; node++) { 1043 if (!is_node_present(node)) 1044 continue; 1045 nr = nodes[node]; 1046 nr_min = min(nr, nr_min); 1047 nr_max = max(nr, nr_max); 1048 sum += nr; 1049 } 1050 BUG_ON(nr_min > nr_max); 1051 1052 BUG_ON(sum > g->p.nr_tasks); 1053 1054 if (0 && (sum < g->p.nr_tasks)) { 1055 free(nodes); 1056 return; 1057 } 1058 1059 /* 1060 * Count the number of distinct process groups present 1061 * on nodes - when we are converged this will decrease 1062 * to g->p.nr_proc: 1063 */ 1064 process_groups = 0; 1065 1066 for (node = 0; node < g->p.nr_nodes; node++) { 1067 int processes; 1068 1069 if (!is_node_present(node)) 1070 continue; 1071 processes = count_node_processes(node); 1072 nr = nodes[node]; 1073 tprintf(" %2d/%-2d", nr, processes); 1074 1075 process_groups += processes; 1076 } 1077 1078 distance = nr_max - nr_min; 1079 1080 tprintf(" [%2d/%-2d]", distance, process_groups); 1081 1082 tprintf(" l:%3d-%-3d (%3d)", 1083 loops_done_min, loops_done_max, loops_done_max-loops_done_min); 1084 1085 if (loops_done_min && loops_done_max) { 1086 double skew = 1.0 - (double)loops_done_min/loops_done_max; 1087 1088 tprintf(" [%4.1f%%]", skew * 100.0); 1089 } 1090 1091 calc_convergence_compression(&strong); 1092 1093 if (strong && process_groups == g->p.nr_proc) { 1094 if (!*convergence) { 1095 *convergence = runtime_ns_max; 1096 tprintf(" (%6.1fs converged)\n", *convergence / NSEC_PER_SEC); 1097 if (g->p.measure_convergence) { 1098 g->all_converged = true; 1099 g->stop_work = true; 1100 } 1101 } 1102 } else { 1103 if (*convergence) { 1104 tprintf(" (%6.1fs de-converged)", runtime_ns_max / NSEC_PER_SEC); 1105 *convergence = 0; 1106 } 1107 tprintf("\n"); 1108 } 1109 1110 free(nodes); 1111 } 1112 1113 static void show_summary(double runtime_ns_max, int l, double *convergence) 1114 { 1115 tprintf("\r # %5.1f%% [%.1f mins]", 1116 (double)(l+1)/g->p.nr_loops*100.0, runtime_ns_max / NSEC_PER_SEC / 60.0); 1117 1118 calc_convergence(runtime_ns_max, convergence); 1119 1120 if (g->p.show_details >= 0) 1121 fflush(stdout); 1122 } 1123 1124 static void *worker_thread(void *__tdata) 1125 { 1126 struct thread_data *td = __tdata; 1127 struct timeval start0, start, stop, diff; 1128 int process_nr = td->process_nr; 1129 int thread_nr = td->thread_nr; 1130 unsigned long last_perturbance; 1131 int task_nr = td->task_nr; 1132 int details = g->p.show_details; 1133 int first_task, last_task; 1134 double convergence = 0; 1135 u64 val = td->val; 1136 double runtime_ns_max; 1137 u8 *global_data; 1138 u8 *process_data; 1139 u8 *thread_data; 1140 u64 bytes_done, secs; 1141 long work_done; 1142 u32 l; 1143 struct rusage rusage; 1144 1145 bind_to_cpumask(td->bind_cpumask); 1146 bind_to_memnode(td->bind_node); 1147 1148 set_taskname("thread %d/%d", process_nr, thread_nr); 1149 1150 global_data = g->data; 1151 process_data = td->process_data; 1152 thread_data = setup_private_data(g->p.bytes_thread); 1153 1154 bytes_done = 0; 1155 1156 last_task = 0; 1157 if (process_nr == g->p.nr_proc-1 && thread_nr == g->p.nr_threads-1) 1158 last_task = 1; 1159 1160 first_task = 0; 1161 if (process_nr == 0 && thread_nr == 0) 1162 first_task = 1; 1163 1164 if (details >= 2) { 1165 printf("# thread %2d / %2d global mem: %p, process mem: %p, thread mem: %p\n", 1166 process_nr, thread_nr, global_data, process_data, thread_data); 1167 } 1168 1169 if (g->p.serialize_startup) { 1170 pthread_mutex_lock(&g->startup_mutex); 1171 g->nr_tasks_started++; 1172 /* The last thread wakes the main process. */ 1173 if (g->nr_tasks_started == g->p.nr_tasks) 1174 pthread_cond_signal(&g->startup_cond); 1175 1176 pthread_mutex_unlock(&g->startup_mutex); 1177 1178 /* Here we will wait for the main process to start us all at once: */ 1179 pthread_mutex_lock(&g->start_work_mutex); 1180 g->start_work = false; 1181 g->nr_tasks_working++; 1182 while (!g->start_work) 1183 pthread_cond_wait(&g->start_work_cond, &g->start_work_mutex); 1184 1185 pthread_mutex_unlock(&g->start_work_mutex); 1186 } 1187 1188 gettimeofday(&start0, NULL); 1189 1190 start = stop = start0; 1191 last_perturbance = start.tv_sec; 1192 1193 for (l = 0; l < g->p.nr_loops; l++) { 1194 start = stop; 1195 1196 if (g->stop_work) 1197 break; 1198 1199 val += do_work(global_data, g->p.bytes_global, process_nr, g->p.nr_proc, l, val); 1200 val += do_work(process_data, g->p.bytes_process, thread_nr, g->p.nr_threads, l, val); 1201 val += do_work(thread_data, g->p.bytes_thread, 0, 1, l, val); 1202 1203 if (g->p.sleep_usecs) { 1204 pthread_mutex_lock(td->process_lock); 1205 usleep(g->p.sleep_usecs); 1206 pthread_mutex_unlock(td->process_lock); 1207 } 1208 /* 1209 * Amount of work to be done under a process-global lock: 1210 */ 1211 if (g->p.bytes_process_locked) { 1212 pthread_mutex_lock(td->process_lock); 1213 val += do_work(process_data, g->p.bytes_process_locked, thread_nr, g->p.nr_threads, l, val); 1214 pthread_mutex_unlock(td->process_lock); 1215 } 1216 1217 work_done = g->p.bytes_global + g->p.bytes_process + 1218 g->p.bytes_process_locked + g->p.bytes_thread; 1219 1220 update_curr_cpu(task_nr, work_done); 1221 bytes_done += work_done; 1222 1223 if (details < 0 && !g->p.perturb_secs && !g->p.measure_convergence && !g->p.nr_secs) 1224 continue; 1225 1226 td->loops_done = l; 1227 1228 gettimeofday(&stop, NULL); 1229 1230 /* Check whether our max runtime timed out: */ 1231 if (g->p.nr_secs) { 1232 timersub(&stop, &start0, &diff); 1233 if ((u32)diff.tv_sec >= g->p.nr_secs) { 1234 g->stop_work = true; 1235 break; 1236 } 1237 } 1238 1239 /* Update the summary at most once per second: */ 1240 if (start.tv_sec == stop.tv_sec) 1241 continue; 1242 1243 /* 1244 * Perturb the first task's equilibrium every g->p.perturb_secs seconds, 1245 * by migrating to CPU#0: 1246 */ 1247 if (first_task && g->p.perturb_secs && (int)(stop.tv_sec - last_perturbance) >= g->p.perturb_secs) { 1248 cpu_set_t orig_mask; 1249 int target_cpu; 1250 int this_cpu; 1251 1252 last_perturbance = stop.tv_sec; 1253 1254 /* 1255 * Depending on where we are running, move into 1256 * the other half of the system, to create some 1257 * real disturbance: 1258 */ 1259 this_cpu = g->threads[task_nr].curr_cpu; 1260 if (this_cpu < g->p.nr_cpus/2) 1261 target_cpu = g->p.nr_cpus-1; 1262 else 1263 target_cpu = 0; 1264 1265 orig_mask = bind_to_cpu(target_cpu); 1266 1267 /* Here we are running on the target CPU already */ 1268 if (details >= 1) 1269 printf(" (injecting perturbalance, moved to CPU#%d)\n", target_cpu); 1270 1271 bind_to_cpumask(orig_mask); 1272 } 1273 1274 if (details >= 3) { 1275 timersub(&stop, &start, &diff); 1276 runtime_ns_max = diff.tv_sec * NSEC_PER_SEC; 1277 runtime_ns_max += diff.tv_usec * NSEC_PER_USEC; 1278 1279 if (details >= 0) { 1280 printf(" #%2d / %2d: %14.2lf nsecs/op [val: %016"PRIx64"]\n", 1281 process_nr, thread_nr, runtime_ns_max / bytes_done, val); 1282 } 1283 fflush(stdout); 1284 } 1285 if (!last_task) 1286 continue; 1287 1288 timersub(&stop, &start0, &diff); 1289 runtime_ns_max = diff.tv_sec * NSEC_PER_SEC; 1290 runtime_ns_max += diff.tv_usec * NSEC_PER_USEC; 1291 1292 show_summary(runtime_ns_max, l, &convergence); 1293 } 1294 1295 gettimeofday(&stop, NULL); 1296 timersub(&stop, &start0, &diff); 1297 td->runtime_ns = diff.tv_sec * NSEC_PER_SEC; 1298 td->runtime_ns += diff.tv_usec * NSEC_PER_USEC; 1299 secs = td->runtime_ns / NSEC_PER_SEC; 1300 td->speed_gbs = secs ? bytes_done / secs / 1e9 : 0; 1301 1302 getrusage(RUSAGE_THREAD, &rusage); 1303 td->system_time_ns = rusage.ru_stime.tv_sec * NSEC_PER_SEC; 1304 td->system_time_ns += rusage.ru_stime.tv_usec * NSEC_PER_USEC; 1305 td->user_time_ns = rusage.ru_utime.tv_sec * NSEC_PER_SEC; 1306 td->user_time_ns += rusage.ru_utime.tv_usec * NSEC_PER_USEC; 1307 1308 free_data(thread_data, g->p.bytes_thread); 1309 1310 pthread_mutex_lock(&g->stop_work_mutex); 1311 g->bytes_done += bytes_done; 1312 pthread_mutex_unlock(&g->stop_work_mutex); 1313 1314 return NULL; 1315 } 1316 1317 /* 1318 * A worker process starts a couple of threads: 1319 */ 1320 static void worker_process(int process_nr) 1321 { 1322 pthread_mutex_t process_lock; 1323 struct thread_data *td; 1324 pthread_t *pthreads; 1325 u8 *process_data; 1326 int task_nr; 1327 int ret; 1328 int t; 1329 1330 pthread_mutex_init(&process_lock, NULL); 1331 set_taskname("process %d", process_nr); 1332 1333 /* 1334 * Pick up the memory policy and the CPU binding of our first thread, 1335 * so that we initialize memory accordingly: 1336 */ 1337 task_nr = process_nr*g->p.nr_threads; 1338 td = g->threads + task_nr; 1339 1340 bind_to_memnode(td->bind_node); 1341 bind_to_cpumask(td->bind_cpumask); 1342 1343 pthreads = zalloc(g->p.nr_threads * sizeof(pthread_t)); 1344 process_data = setup_private_data(g->p.bytes_process); 1345 1346 if (g->p.show_details >= 3) { 1347 printf(" # process %2d global mem: %p, process mem: %p\n", 1348 process_nr, g->data, process_data); 1349 } 1350 1351 for (t = 0; t < g->p.nr_threads; t++) { 1352 task_nr = process_nr*g->p.nr_threads + t; 1353 td = g->threads + task_nr; 1354 1355 td->process_data = process_data; 1356 td->process_nr = process_nr; 1357 td->thread_nr = t; 1358 td->task_nr = task_nr; 1359 td->val = rand(); 1360 td->curr_cpu = -1; 1361 td->process_lock = &process_lock; 1362 1363 ret = pthread_create(pthreads + t, NULL, worker_thread, td); 1364 BUG_ON(ret); 1365 } 1366 1367 for (t = 0; t < g->p.nr_threads; t++) { 1368 ret = pthread_join(pthreads[t], NULL); 1369 BUG_ON(ret); 1370 } 1371 1372 free_data(process_data, g->p.bytes_process); 1373 free(pthreads); 1374 } 1375 1376 static void print_summary(void) 1377 { 1378 if (g->p.show_details < 0) 1379 return; 1380 1381 printf("\n ###\n"); 1382 printf(" # %d %s will execute (on %d nodes, %d CPUs):\n", 1383 g->p.nr_tasks, g->p.nr_tasks == 1 ? "task" : "tasks", nr_numa_nodes(), g->p.nr_cpus); 1384 printf(" # %5dx %5ldMB global shared mem operations\n", 1385 g->p.nr_loops, g->p.bytes_global/1024/1024); 1386 printf(" # %5dx %5ldMB process shared mem operations\n", 1387 g->p.nr_loops, g->p.bytes_process/1024/1024); 1388 printf(" # %5dx %5ldMB thread local mem operations\n", 1389 g->p.nr_loops, g->p.bytes_thread/1024/1024); 1390 1391 printf(" ###\n"); 1392 1393 printf("\n ###\n"); fflush(stdout); 1394 } 1395 1396 static void init_thread_data(void) 1397 { 1398 ssize_t size = sizeof(*g->threads)*g->p.nr_tasks; 1399 int t; 1400 1401 g->threads = zalloc_shared_data(size); 1402 1403 for (t = 0; t < g->p.nr_tasks; t++) { 1404 struct thread_data *td = g->threads + t; 1405 int cpu; 1406 1407 /* Allow all nodes by default: */ 1408 td->bind_node = NUMA_NO_NODE; 1409 1410 /* Allow all CPUs by default: */ 1411 CPU_ZERO(&td->bind_cpumask); 1412 for (cpu = 0; cpu < g->p.nr_cpus; cpu++) 1413 CPU_SET(cpu, &td->bind_cpumask); 1414 } 1415 } 1416 1417 static void deinit_thread_data(void) 1418 { 1419 ssize_t size = sizeof(*g->threads)*g->p.nr_tasks; 1420 1421 free_data(g->threads, size); 1422 } 1423 1424 static int init(void) 1425 { 1426 g = (void *)alloc_data(sizeof(*g), MAP_SHARED, 1, 0, 0 /* THP */, 0); 1427 1428 /* Copy over options: */ 1429 g->p = p0; 1430 1431 g->p.nr_cpus = numa_num_configured_cpus(); 1432 1433 g->p.nr_nodes = numa_max_node() + 1; 1434 1435 /* char array in count_process_nodes(): */ 1436 BUG_ON(g->p.nr_nodes < 0); 1437 1438 if (g->p.show_quiet && !g->p.show_details) 1439 g->p.show_details = -1; 1440 1441 /* Some memory should be specified: */ 1442 if (!g->p.mb_global_str && !g->p.mb_proc_str && !g->p.mb_thread_str) 1443 return -1; 1444 1445 if (g->p.mb_global_str) { 1446 g->p.mb_global = atof(g->p.mb_global_str); 1447 BUG_ON(g->p.mb_global < 0); 1448 } 1449 1450 if (g->p.mb_proc_str) { 1451 g->p.mb_proc = atof(g->p.mb_proc_str); 1452 BUG_ON(g->p.mb_proc < 0); 1453 } 1454 1455 if (g->p.mb_proc_locked_str) { 1456 g->p.mb_proc_locked = atof(g->p.mb_proc_locked_str); 1457 BUG_ON(g->p.mb_proc_locked < 0); 1458 BUG_ON(g->p.mb_proc_locked > g->p.mb_proc); 1459 } 1460 1461 if (g->p.mb_thread_str) { 1462 g->p.mb_thread = atof(g->p.mb_thread_str); 1463 BUG_ON(g->p.mb_thread < 0); 1464 } 1465 1466 BUG_ON(g->p.nr_threads <= 0); 1467 BUG_ON(g->p.nr_proc <= 0); 1468 1469 g->p.nr_tasks = g->p.nr_proc*g->p.nr_threads; 1470 1471 g->p.bytes_global = g->p.mb_global *1024L*1024L; 1472 g->p.bytes_process = g->p.mb_proc *1024L*1024L; 1473 g->p.bytes_process_locked = g->p.mb_proc_locked *1024L*1024L; 1474 g->p.bytes_thread = g->p.mb_thread *1024L*1024L; 1475 1476 g->data = setup_shared_data(g->p.bytes_global); 1477 1478 /* Startup serialization: */ 1479 init_global_mutex(&g->start_work_mutex); 1480 init_global_cond(&g->start_work_cond); 1481 init_global_mutex(&g->startup_mutex); 1482 init_global_cond(&g->startup_cond); 1483 init_global_mutex(&g->stop_work_mutex); 1484 1485 init_thread_data(); 1486 1487 tprintf("#\n"); 1488 if (parse_setup_cpu_list() || parse_setup_node_list()) 1489 return -1; 1490 tprintf("#\n"); 1491 1492 print_summary(); 1493 1494 return 0; 1495 } 1496 1497 static void deinit(void) 1498 { 1499 free_data(g->data, g->p.bytes_global); 1500 g->data = NULL; 1501 1502 deinit_thread_data(); 1503 1504 free_data(g, sizeof(*g)); 1505 g = NULL; 1506 } 1507 1508 /* 1509 * Print a short or long result, depending on the verbosity setting: 1510 */ 1511 static void print_res(const char *name, double val, 1512 const char *txt_unit, const char *txt_short, const char *txt_long) 1513 { 1514 if (!name) 1515 name = "main,"; 1516 1517 if (!g->p.show_quiet) 1518 printf(" %-30s %15.3f, %-15s %s\n", name, val, txt_unit, txt_short); 1519 else 1520 printf(" %14.3f %s\n", val, txt_long); 1521 } 1522 1523 static int __bench_numa(const char *name) 1524 { 1525 struct timeval start, stop, diff; 1526 u64 runtime_ns_min, runtime_ns_sum; 1527 pid_t *pids, pid, wpid; 1528 double delta_runtime; 1529 double runtime_avg; 1530 double runtime_sec_max; 1531 double runtime_sec_min; 1532 int wait_stat; 1533 double bytes; 1534 int i, t, p; 1535 1536 if (init()) 1537 return -1; 1538 1539 pids = zalloc(g->p.nr_proc * sizeof(*pids)); 1540 pid = -1; 1541 1542 if (g->p.serialize_startup) { 1543 tprintf(" #\n"); 1544 tprintf(" # Startup synchronization: ..."); fflush(stdout); 1545 } 1546 1547 gettimeofday(&start, NULL); 1548 1549 for (i = 0; i < g->p.nr_proc; i++) { 1550 pid = fork(); 1551 dprintf(" # process %2d: PID %d\n", i, pid); 1552 1553 BUG_ON(pid < 0); 1554 if (!pid) { 1555 /* Child process: */ 1556 worker_process(i); 1557 1558 exit(0); 1559 } 1560 pids[i] = pid; 1561 1562 } 1563 1564 if (g->p.serialize_startup) { 1565 bool threads_ready = false; 1566 double startup_sec; 1567 1568 /* 1569 * Wait for all the threads to start up. The last thread will 1570 * signal this process. 1571 */ 1572 pthread_mutex_lock(&g->startup_mutex); 1573 while (g->nr_tasks_started != g->p.nr_tasks) 1574 pthread_cond_wait(&g->startup_cond, &g->startup_mutex); 1575 1576 pthread_mutex_unlock(&g->startup_mutex); 1577 1578 /* Wait for all threads to be at the start_work_cond. */ 1579 while (!threads_ready) { 1580 pthread_mutex_lock(&g->start_work_mutex); 1581 threads_ready = (g->nr_tasks_working == g->p.nr_tasks); 1582 pthread_mutex_unlock(&g->start_work_mutex); 1583 if (!threads_ready) 1584 usleep(1); 1585 } 1586 1587 gettimeofday(&stop, NULL); 1588 1589 timersub(&stop, &start, &diff); 1590 1591 startup_sec = diff.tv_sec * NSEC_PER_SEC; 1592 startup_sec += diff.tv_usec * NSEC_PER_USEC; 1593 startup_sec /= NSEC_PER_SEC; 1594 1595 tprintf(" threads initialized in %.6f seconds.\n", startup_sec); 1596 tprintf(" #\n"); 1597 1598 start = stop; 1599 /* Start all threads running. */ 1600 pthread_mutex_lock(&g->start_work_mutex); 1601 g->start_work = true; 1602 pthread_mutex_unlock(&g->start_work_mutex); 1603 pthread_cond_broadcast(&g->start_work_cond); 1604 } else { 1605 gettimeofday(&start, NULL); 1606 } 1607 1608 /* Parent process: */ 1609 1610 1611 for (i = 0; i < g->p.nr_proc; i++) { 1612 wpid = waitpid(pids[i], &wait_stat, 0); 1613 BUG_ON(wpid < 0); 1614 BUG_ON(!WIFEXITED(wait_stat)); 1615 1616 } 1617 1618 runtime_ns_sum = 0; 1619 runtime_ns_min = -1LL; 1620 1621 for (t = 0; t < g->p.nr_tasks; t++) { 1622 u64 thread_runtime_ns = g->threads[t].runtime_ns; 1623 1624 runtime_ns_sum += thread_runtime_ns; 1625 runtime_ns_min = min(thread_runtime_ns, runtime_ns_min); 1626 } 1627 1628 gettimeofday(&stop, NULL); 1629 timersub(&stop, &start, &diff); 1630 1631 BUG_ON(bench_format != BENCH_FORMAT_DEFAULT); 1632 1633 tprintf("\n ###\n"); 1634 tprintf("\n"); 1635 1636 runtime_sec_max = diff.tv_sec * NSEC_PER_SEC; 1637 runtime_sec_max += diff.tv_usec * NSEC_PER_USEC; 1638 runtime_sec_max /= NSEC_PER_SEC; 1639 1640 runtime_sec_min = runtime_ns_min / NSEC_PER_SEC; 1641 1642 bytes = g->bytes_done; 1643 runtime_avg = (double)runtime_ns_sum / g->p.nr_tasks / NSEC_PER_SEC; 1644 1645 if (g->p.measure_convergence) { 1646 print_res(name, runtime_sec_max, 1647 "secs,", "NUMA-convergence-latency", "secs latency to NUMA-converge"); 1648 } 1649 1650 print_res(name, runtime_sec_max, 1651 "secs,", "runtime-max/thread", "secs slowest (max) thread-runtime"); 1652 1653 print_res(name, runtime_sec_min, 1654 "secs,", "runtime-min/thread", "secs fastest (min) thread-runtime"); 1655 1656 print_res(name, runtime_avg, 1657 "secs,", "runtime-avg/thread", "secs average thread-runtime"); 1658 1659 delta_runtime = (runtime_sec_max - runtime_sec_min)/2.0; 1660 print_res(name, delta_runtime / runtime_sec_max * 100.0, 1661 "%,", "spread-runtime/thread", "% difference between max/avg runtime"); 1662 1663 print_res(name, bytes / g->p.nr_tasks / 1e9, 1664 "GB,", "data/thread", "GB data processed, per thread"); 1665 1666 print_res(name, bytes / 1e9, 1667 "GB,", "data-total", "GB data processed, total"); 1668 1669 print_res(name, runtime_sec_max * NSEC_PER_SEC / (bytes / g->p.nr_tasks), 1670 "nsecs,", "runtime/byte/thread","nsecs/byte/thread runtime"); 1671 1672 print_res(name, bytes / g->p.nr_tasks / 1e9 / runtime_sec_max, 1673 "GB/sec,", "thread-speed", "GB/sec/thread speed"); 1674 1675 print_res(name, bytes / runtime_sec_max / 1e9, 1676 "GB/sec,", "total-speed", "GB/sec total speed"); 1677 1678 if (g->p.show_details >= 2) { 1679 char tname[14 + 2 * 10 + 1]; 1680 struct thread_data *td; 1681 for (p = 0; p < g->p.nr_proc; p++) { 1682 for (t = 0; t < g->p.nr_threads; t++) { 1683 memset(tname, 0, sizeof(tname)); 1684 td = g->threads + p*g->p.nr_threads + t; 1685 snprintf(tname, sizeof(tname), "process%d:thread%d", p, t); 1686 print_res(tname, td->speed_gbs, 1687 "GB/sec", "thread-speed", "GB/sec/thread speed"); 1688 print_res(tname, td->system_time_ns / NSEC_PER_SEC, 1689 "secs", "thread-system-time", "system CPU time/thread"); 1690 print_res(tname, td->user_time_ns / NSEC_PER_SEC, 1691 "secs", "thread-user-time", "user CPU time/thread"); 1692 } 1693 } 1694 } 1695 1696 free(pids); 1697 1698 deinit(); 1699 1700 return 0; 1701 } 1702 1703 #define MAX_ARGS 50 1704 1705 static int command_size(const char **argv) 1706 { 1707 int size = 0; 1708 1709 while (*argv) { 1710 size++; 1711 argv++; 1712 } 1713 1714 BUG_ON(size >= MAX_ARGS); 1715 1716 return size; 1717 } 1718 1719 static void init_params(struct params *p, const char *name, int argc, const char **argv) 1720 { 1721 int i; 1722 1723 printf("\n # Running %s \"perf bench numa", name); 1724 1725 for (i = 0; i < argc; i++) 1726 printf(" %s", argv[i]); 1727 1728 printf("\"\n"); 1729 1730 memset(p, 0, sizeof(*p)); 1731 1732 /* Initialize nonzero defaults: */ 1733 1734 p->serialize_startup = 1; 1735 p->data_reads = true; 1736 p->data_writes = true; 1737 p->data_backwards = true; 1738 p->data_rand_walk = true; 1739 p->nr_loops = -1; 1740 p->init_random = true; 1741 p->mb_global_str = "1"; 1742 p->nr_proc = 1; 1743 p->nr_threads = 1; 1744 p->nr_secs = 5; 1745 p->run_all = argc == 1; 1746 } 1747 1748 static int run_bench_numa(const char *name, const char **argv) 1749 { 1750 int argc = command_size(argv); 1751 1752 init_params(&p0, name, argc, argv); 1753 argc = parse_options(argc, argv, options, bench_numa_usage, 0); 1754 if (argc) 1755 goto err; 1756 1757 if (__bench_numa(name)) 1758 goto err; 1759 1760 return 0; 1761 1762 err: 1763 return -1; 1764 } 1765 1766 #define OPT_BW_RAM "-s", "20", "-zZq", "--thp", " 1", "--no-data_rand_walk" 1767 #define OPT_BW_RAM_NOTHP OPT_BW_RAM, "--thp", "-1" 1768 1769 #define OPT_CONV "-s", "100", "-zZ0qcm", "--thp", " 1" 1770 #define OPT_CONV_NOTHP OPT_CONV, "--thp", "-1" 1771 1772 #define OPT_BW "-s", "20", "-zZ0q", "--thp", " 1" 1773 #define OPT_BW_NOTHP OPT_BW, "--thp", "-1" 1774 1775 /* 1776 * The built-in test-suite executed by "perf bench numa -a". 1777 * 1778 * (A minimum of 4 nodes and 16 GB of RAM is recommended.) 1779 */ 1780 static const char *tests[][MAX_ARGS] = { 1781 /* Basic single-stream NUMA bandwidth measurements: */ 1782 { "RAM-bw-local,", "mem", "-p", "1", "-t", "1", "-P", "1024", 1783 "-C" , "0", "-M", "0", OPT_BW_RAM }, 1784 { "RAM-bw-local-NOTHP,", 1785 "mem", "-p", "1", "-t", "1", "-P", "1024", 1786 "-C" , "0", "-M", "0", OPT_BW_RAM_NOTHP }, 1787 { "RAM-bw-remote,", "mem", "-p", "1", "-t", "1", "-P", "1024", 1788 "-C" , "0", "-M", "1", OPT_BW_RAM }, 1789 1790 /* 2-stream NUMA bandwidth measurements: */ 1791 { "RAM-bw-local-2x,", "mem", "-p", "2", "-t", "1", "-P", "1024", 1792 "-C", "0,2", "-M", "0x2", OPT_BW_RAM }, 1793 { "RAM-bw-remote-2x,", "mem", "-p", "2", "-t", "1", "-P", "1024", 1794 "-C", "0,2", "-M", "1x2", OPT_BW_RAM }, 1795 1796 /* Cross-stream NUMA bandwidth measurement: */ 1797 { "RAM-bw-cross,", "mem", "-p", "2", "-t", "1", "-P", "1024", 1798 "-C", "0,8", "-M", "1,0", OPT_BW_RAM }, 1799 1800 /* Convergence latency measurements: */ 1801 { " 1x3-convergence,", "mem", "-p", "1", "-t", "3", "-P", "512", OPT_CONV }, 1802 { " 1x4-convergence,", "mem", "-p", "1", "-t", "4", "-P", "512", OPT_CONV }, 1803 { " 1x6-convergence,", "mem", "-p", "1", "-t", "6", "-P", "1020", OPT_CONV }, 1804 { " 2x3-convergence,", "mem", "-p", "2", "-t", "3", "-P", "1020", OPT_CONV }, 1805 { " 3x3-convergence,", "mem", "-p", "3", "-t", "3", "-P", "1020", OPT_CONV }, 1806 { " 4x4-convergence,", "mem", "-p", "4", "-t", "4", "-P", "512", OPT_CONV }, 1807 { " 4x4-convergence-NOTHP,", 1808 "mem", "-p", "4", "-t", "4", "-P", "512", OPT_CONV_NOTHP }, 1809 { " 4x6-convergence,", "mem", "-p", "4", "-t", "6", "-P", "1020", OPT_CONV }, 1810 { " 4x8-convergence,", "mem", "-p", "4", "-t", "8", "-P", "512", OPT_CONV }, 1811 { " 8x4-convergence,", "mem", "-p", "8", "-t", "4", "-P", "512", OPT_CONV }, 1812 { " 8x4-convergence-NOTHP,", 1813 "mem", "-p", "8", "-t", "4", "-P", "512", OPT_CONV_NOTHP }, 1814 { " 3x1-convergence,", "mem", "-p", "3", "-t", "1", "-P", "512", OPT_CONV }, 1815 { " 4x1-convergence,", "mem", "-p", "4", "-t", "1", "-P", "512", OPT_CONV }, 1816 { " 8x1-convergence,", "mem", "-p", "8", "-t", "1", "-P", "512", OPT_CONV }, 1817 { "16x1-convergence,", "mem", "-p", "16", "-t", "1", "-P", "256", OPT_CONV }, 1818 { "32x1-convergence,", "mem", "-p", "32", "-t", "1", "-P", "128", OPT_CONV }, 1819 1820 /* Various NUMA process/thread layout bandwidth measurements: */ 1821 { " 2x1-bw-process,", "mem", "-p", "2", "-t", "1", "-P", "1024", OPT_BW }, 1822 { " 3x1-bw-process,", "mem", "-p", "3", "-t", "1", "-P", "1024", OPT_BW }, 1823 { " 4x1-bw-process,", "mem", "-p", "4", "-t", "1", "-P", "1024", OPT_BW }, 1824 { " 8x1-bw-process,", "mem", "-p", "8", "-t", "1", "-P", " 512", OPT_BW }, 1825 { " 8x1-bw-process-NOTHP,", 1826 "mem", "-p", "8", "-t", "1", "-P", " 512", OPT_BW_NOTHP }, 1827 { "16x1-bw-process,", "mem", "-p", "16", "-t", "1", "-P", "256", OPT_BW }, 1828 1829 { " 1x4-bw-thread,", "mem", "-p", "1", "-t", "4", "-T", "256", OPT_BW }, 1830 { " 1x8-bw-thread,", "mem", "-p", "1", "-t", "8", "-T", "256", OPT_BW }, 1831 { "1x16-bw-thread,", "mem", "-p", "1", "-t", "16", "-T", "128", OPT_BW }, 1832 { "1x32-bw-thread,", "mem", "-p", "1", "-t", "32", "-T", "64", OPT_BW }, 1833 1834 { " 2x3-bw-process,", "mem", "-p", "2", "-t", "3", "-P", "512", OPT_BW }, 1835 { " 4x4-bw-process,", "mem", "-p", "4", "-t", "4", "-P", "512", OPT_BW }, 1836 { " 4x6-bw-process,", "mem", "-p", "4", "-t", "6", "-P", "512", OPT_BW }, 1837 { " 4x8-bw-process,", "mem", "-p", "4", "-t", "8", "-P", "512", OPT_BW }, 1838 { " 4x8-bw-process-NOTHP,", 1839 "mem", "-p", "4", "-t", "8", "-P", "512", OPT_BW_NOTHP }, 1840 { " 3x3-bw-process,", "mem", "-p", "3", "-t", "3", "-P", "512", OPT_BW }, 1841 { " 5x5-bw-process,", "mem", "-p", "5", "-t", "5", "-P", "512", OPT_BW }, 1842 1843 { "2x16-bw-process,", "mem", "-p", "2", "-t", "16", "-P", "512", OPT_BW }, 1844 { "1x32-bw-process,", "mem", "-p", "1", "-t", "32", "-P", "2048", OPT_BW }, 1845 1846 { "numa02-bw,", "mem", "-p", "1", "-t", "32", "-T", "32", OPT_BW }, 1847 { "numa02-bw-NOTHP,", "mem", "-p", "1", "-t", "32", "-T", "32", OPT_BW_NOTHP }, 1848 { "numa01-bw-thread,", "mem", "-p", "2", "-t", "16", "-T", "192", OPT_BW }, 1849 { "numa01-bw-thread-NOTHP,", 1850 "mem", "-p", "2", "-t", "16", "-T", "192", OPT_BW_NOTHP }, 1851 }; 1852 1853 static int bench_all(void) 1854 { 1855 int nr = ARRAY_SIZE(tests); 1856 int ret; 1857 int i; 1858 1859 ret = system("echo ' #'; echo ' # Running test on: '$(uname -a); echo ' #'"); 1860 BUG_ON(ret < 0); 1861 1862 for (i = 0; i < nr; i++) { 1863 run_bench_numa(tests[i][0], tests[i] + 1); 1864 } 1865 1866 printf("\n"); 1867 1868 return 0; 1869 } 1870 1871 int bench_numa(int argc, const char **argv) 1872 { 1873 init_params(&p0, "main,", argc, argv); 1874 argc = parse_options(argc, argv, options, bench_numa_usage, 0); 1875 if (argc) 1876 goto err; 1877 1878 if (p0.run_all) 1879 return bench_all(); 1880 1881 if (__bench_numa(NULL)) 1882 goto err; 1883 1884 return 0; 1885 1886 err: 1887 usage_with_options(numa_usage, options); 1888 return -1; 1889 } 1890