1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based scalability-test facility 4 * 5 * Copyright (C) IBM Corporation, 2015 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 */ 9 10 #define pr_fmt(fmt) fmt 11 12 #include <linux/types.h> 13 #include <linux/kernel.h> 14 #include <linux/init.h> 15 #include <linux/mm.h> 16 #include <linux/module.h> 17 #include <linux/kthread.h> 18 #include <linux/err.h> 19 #include <linux/spinlock.h> 20 #include <linux/smp.h> 21 #include <linux/rcupdate.h> 22 #include <linux/interrupt.h> 23 #include <linux/sched.h> 24 #include <uapi/linux/sched/types.h> 25 #include <linux/atomic.h> 26 #include <linux/bitops.h> 27 #include <linux/completion.h> 28 #include <linux/moduleparam.h> 29 #include <linux/percpu.h> 30 #include <linux/notifier.h> 31 #include <linux/reboot.h> 32 #include <linux/freezer.h> 33 #include <linux/cpu.h> 34 #include <linux/delay.h> 35 #include <linux/stat.h> 36 #include <linux/srcu.h> 37 #include <linux/slab.h> 38 #include <asm/byteorder.h> 39 #include <linux/torture.h> 40 #include <linux/vmalloc.h> 41 #include <linux/rcupdate_trace.h> 42 43 #include "rcu.h" 44 45 MODULE_LICENSE("GPL"); 46 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 47 48 #define SCALE_FLAG "-scale:" 49 #define SCALEOUT_STRING(s) \ 50 pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s) 51 #define VERBOSE_SCALEOUT_STRING(s) \ 52 do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0) 53 #define SCALEOUT_ERRSTRING(s) \ 54 pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s) 55 56 /* 57 * The intended use cases for the nreaders and nwriters module parameters 58 * are as follows: 59 * 60 * 1. Specify only the nr_cpus kernel boot parameter. This will 61 * set both nreaders and nwriters to the value specified by 62 * nr_cpus for a mixed reader/writer test. 63 * 64 * 2. Specify the nr_cpus kernel boot parameter, but set 65 * rcuscale.nreaders to zero. This will set nwriters to the 66 * value specified by nr_cpus for an update-only test. 67 * 68 * 3. Specify the nr_cpus kernel boot parameter, but set 69 * rcuscale.nwriters to zero. This will set nreaders to the 70 * value specified by nr_cpus for a read-only test. 71 * 72 * Various other use cases may of course be specified. 73 * 74 * Note that this test's readers are intended only as a test load for 75 * the writers. The reader scalability statistics will be overly 76 * pessimistic due to the per-critical-section interrupt disabling, 77 * test-end checks, and the pair of calls through pointers. 78 */ 79 80 #ifdef MODULE 81 # define RCUSCALE_SHUTDOWN 0 82 #else 83 # define RCUSCALE_SHUTDOWN 1 84 #endif 85 86 torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives"); 87 torture_param(int, gp_async_max, 1000, "Max # outstanding waits per reader"); 88 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 89 torture_param(int, holdoff, 10, "Holdoff time before test start (s)"); 90 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 91 torture_param(int, nwriters, -1, "Number of RCU updater threads"); 92 torture_param(bool, shutdown, RCUSCALE_SHUTDOWN, 93 "Shutdown at end of scalability tests."); 94 torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); 95 torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); 96 torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?"); 97 torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate."); 98 torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?"); 99 100 static char *scale_type = "rcu"; 101 module_param(scale_type, charp, 0444); 102 MODULE_PARM_DESC(scale_type, "Type of RCU to scalability-test (rcu, srcu, ...)"); 103 104 static int nrealreaders; 105 static int nrealwriters; 106 static struct task_struct **writer_tasks; 107 static struct task_struct **reader_tasks; 108 static struct task_struct *shutdown_task; 109 110 static u64 **writer_durations; 111 static int *writer_n_durations; 112 static atomic_t n_rcu_scale_reader_started; 113 static atomic_t n_rcu_scale_writer_started; 114 static atomic_t n_rcu_scale_writer_finished; 115 static wait_queue_head_t shutdown_wq; 116 static u64 t_rcu_scale_writer_started; 117 static u64 t_rcu_scale_writer_finished; 118 static unsigned long b_rcu_gp_test_started; 119 static unsigned long b_rcu_gp_test_finished; 120 static DEFINE_PER_CPU(atomic_t, n_async_inflight); 121 122 #define MAX_MEAS 10000 123 #define MIN_MEAS 100 124 125 /* 126 * Operations vector for selecting different types of tests. 127 */ 128 129 struct rcu_scale_ops { 130 int ptype; 131 void (*init)(void); 132 void (*cleanup)(void); 133 int (*readlock)(void); 134 void (*readunlock)(int idx); 135 unsigned long (*get_gp_seq)(void); 136 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 137 unsigned long (*exp_completed)(void); 138 void (*async)(struct rcu_head *head, rcu_callback_t func); 139 void (*gp_barrier)(void); 140 void (*sync)(void); 141 void (*exp_sync)(void); 142 const char *name; 143 }; 144 145 static struct rcu_scale_ops *cur_ops; 146 147 /* 148 * Definitions for rcu scalability testing. 149 */ 150 151 static int rcu_scale_read_lock(void) __acquires(RCU) 152 { 153 rcu_read_lock(); 154 return 0; 155 } 156 157 static void rcu_scale_read_unlock(int idx) __releases(RCU) 158 { 159 rcu_read_unlock(); 160 } 161 162 static unsigned long __maybe_unused rcu_no_completed(void) 163 { 164 return 0; 165 } 166 167 static void rcu_sync_scale_init(void) 168 { 169 } 170 171 static struct rcu_scale_ops rcu_ops = { 172 .ptype = RCU_FLAVOR, 173 .init = rcu_sync_scale_init, 174 .readlock = rcu_scale_read_lock, 175 .readunlock = rcu_scale_read_unlock, 176 .get_gp_seq = rcu_get_gp_seq, 177 .gp_diff = rcu_seq_diff, 178 .exp_completed = rcu_exp_batches_completed, 179 .async = call_rcu_hurry, 180 .gp_barrier = rcu_barrier, 181 .sync = synchronize_rcu, 182 .exp_sync = synchronize_rcu_expedited, 183 .name = "rcu" 184 }; 185 186 /* 187 * Definitions for srcu scalability testing. 188 */ 189 190 DEFINE_STATIC_SRCU(srcu_ctl_scale); 191 static struct srcu_struct *srcu_ctlp = &srcu_ctl_scale; 192 193 static int srcu_scale_read_lock(void) __acquires(srcu_ctlp) 194 { 195 return srcu_read_lock(srcu_ctlp); 196 } 197 198 static void srcu_scale_read_unlock(int idx) __releases(srcu_ctlp) 199 { 200 srcu_read_unlock(srcu_ctlp, idx); 201 } 202 203 static unsigned long srcu_scale_completed(void) 204 { 205 return srcu_batches_completed(srcu_ctlp); 206 } 207 208 static void srcu_call_rcu(struct rcu_head *head, rcu_callback_t func) 209 { 210 call_srcu(srcu_ctlp, head, func); 211 } 212 213 static void srcu_rcu_barrier(void) 214 { 215 srcu_barrier(srcu_ctlp); 216 } 217 218 static void srcu_scale_synchronize(void) 219 { 220 synchronize_srcu(srcu_ctlp); 221 } 222 223 static void srcu_scale_synchronize_expedited(void) 224 { 225 synchronize_srcu_expedited(srcu_ctlp); 226 } 227 228 static struct rcu_scale_ops srcu_ops = { 229 .ptype = SRCU_FLAVOR, 230 .init = rcu_sync_scale_init, 231 .readlock = srcu_scale_read_lock, 232 .readunlock = srcu_scale_read_unlock, 233 .get_gp_seq = srcu_scale_completed, 234 .gp_diff = rcu_seq_diff, 235 .exp_completed = srcu_scale_completed, 236 .async = srcu_call_rcu, 237 .gp_barrier = srcu_rcu_barrier, 238 .sync = srcu_scale_synchronize, 239 .exp_sync = srcu_scale_synchronize_expedited, 240 .name = "srcu" 241 }; 242 243 static struct srcu_struct srcud; 244 245 static void srcu_sync_scale_init(void) 246 { 247 srcu_ctlp = &srcud; 248 init_srcu_struct(srcu_ctlp); 249 } 250 251 static void srcu_sync_scale_cleanup(void) 252 { 253 cleanup_srcu_struct(srcu_ctlp); 254 } 255 256 static struct rcu_scale_ops srcud_ops = { 257 .ptype = SRCU_FLAVOR, 258 .init = srcu_sync_scale_init, 259 .cleanup = srcu_sync_scale_cleanup, 260 .readlock = srcu_scale_read_lock, 261 .readunlock = srcu_scale_read_unlock, 262 .get_gp_seq = srcu_scale_completed, 263 .gp_diff = rcu_seq_diff, 264 .exp_completed = srcu_scale_completed, 265 .async = srcu_call_rcu, 266 .gp_barrier = srcu_rcu_barrier, 267 .sync = srcu_scale_synchronize, 268 .exp_sync = srcu_scale_synchronize_expedited, 269 .name = "srcud" 270 }; 271 272 #ifdef CONFIG_TASKS_RCU 273 274 /* 275 * Definitions for RCU-tasks scalability testing. 276 */ 277 278 static int tasks_scale_read_lock(void) 279 { 280 return 0; 281 } 282 283 static void tasks_scale_read_unlock(int idx) 284 { 285 } 286 287 static struct rcu_scale_ops tasks_ops = { 288 .ptype = RCU_TASKS_FLAVOR, 289 .init = rcu_sync_scale_init, 290 .readlock = tasks_scale_read_lock, 291 .readunlock = tasks_scale_read_unlock, 292 .get_gp_seq = rcu_no_completed, 293 .gp_diff = rcu_seq_diff, 294 .async = call_rcu_tasks, 295 .gp_barrier = rcu_barrier_tasks, 296 .sync = synchronize_rcu_tasks, 297 .exp_sync = synchronize_rcu_tasks, 298 .name = "tasks" 299 }; 300 301 #define TASKS_OPS &tasks_ops, 302 303 #else // #ifdef CONFIG_TASKS_RCU 304 305 #define TASKS_OPS 306 307 #endif // #else // #ifdef CONFIG_TASKS_RCU 308 309 #ifdef CONFIG_TASKS_TRACE_RCU 310 311 /* 312 * Definitions for RCU-tasks-trace scalability testing. 313 */ 314 315 static int tasks_trace_scale_read_lock(void) 316 { 317 rcu_read_lock_trace(); 318 return 0; 319 } 320 321 static void tasks_trace_scale_read_unlock(int idx) 322 { 323 rcu_read_unlock_trace(); 324 } 325 326 static struct rcu_scale_ops tasks_tracing_ops = { 327 .ptype = RCU_TASKS_FLAVOR, 328 .init = rcu_sync_scale_init, 329 .readlock = tasks_trace_scale_read_lock, 330 .readunlock = tasks_trace_scale_read_unlock, 331 .get_gp_seq = rcu_no_completed, 332 .gp_diff = rcu_seq_diff, 333 .async = call_rcu_tasks_trace, 334 .gp_barrier = rcu_barrier_tasks_trace, 335 .sync = synchronize_rcu_tasks_trace, 336 .exp_sync = synchronize_rcu_tasks_trace, 337 .name = "tasks-tracing" 338 }; 339 340 #define TASKS_TRACING_OPS &tasks_tracing_ops, 341 342 #else // #ifdef CONFIG_TASKS_TRACE_RCU 343 344 #define TASKS_TRACING_OPS 345 346 #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU 347 348 static unsigned long rcuscale_seq_diff(unsigned long new, unsigned long old) 349 { 350 if (!cur_ops->gp_diff) 351 return new - old; 352 return cur_ops->gp_diff(new, old); 353 } 354 355 /* 356 * If scalability tests complete, wait for shutdown to commence. 357 */ 358 static void rcu_scale_wait_shutdown(void) 359 { 360 cond_resched_tasks_rcu_qs(); 361 if (atomic_read(&n_rcu_scale_writer_finished) < nrealwriters) 362 return; 363 while (!torture_must_stop()) 364 schedule_timeout_uninterruptible(1); 365 } 366 367 /* 368 * RCU scalability reader kthread. Repeatedly does empty RCU read-side 369 * critical section, minimizing update-side interference. However, the 370 * point of this test is not to evaluate reader scalability, but instead 371 * to serve as a test load for update-side scalability testing. 372 */ 373 static int 374 rcu_scale_reader(void *arg) 375 { 376 unsigned long flags; 377 int idx; 378 long me = (long)arg; 379 380 VERBOSE_SCALEOUT_STRING("rcu_scale_reader task started"); 381 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 382 set_user_nice(current, MAX_NICE); 383 atomic_inc(&n_rcu_scale_reader_started); 384 385 do { 386 local_irq_save(flags); 387 idx = cur_ops->readlock(); 388 cur_ops->readunlock(idx); 389 local_irq_restore(flags); 390 rcu_scale_wait_shutdown(); 391 } while (!torture_must_stop()); 392 torture_kthread_stopping("rcu_scale_reader"); 393 return 0; 394 } 395 396 /* 397 * Callback function for asynchronous grace periods from rcu_scale_writer(). 398 */ 399 static void rcu_scale_async_cb(struct rcu_head *rhp) 400 { 401 atomic_dec(this_cpu_ptr(&n_async_inflight)); 402 kfree(rhp); 403 } 404 405 /* 406 * RCU scale writer kthread. Repeatedly does a grace period. 407 */ 408 static int 409 rcu_scale_writer(void *arg) 410 { 411 int i = 0; 412 int i_max; 413 long me = (long)arg; 414 struct rcu_head *rhp = NULL; 415 bool started = false, done = false, alldone = false; 416 u64 t; 417 u64 *wdp; 418 u64 *wdpp = writer_durations[me]; 419 420 VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started"); 421 WARN_ON(!wdpp); 422 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 423 current->flags |= PF_NO_SETAFFINITY; 424 sched_set_fifo_low(current); 425 426 if (holdoff) 427 schedule_timeout_uninterruptible(holdoff * HZ); 428 429 /* 430 * Wait until rcu_end_inkernel_boot() is called for normal GP tests 431 * so that RCU is not always expedited for normal GP tests. 432 * The system_state test is approximate, but works well in practice. 433 */ 434 while (!gp_exp && system_state != SYSTEM_RUNNING) 435 schedule_timeout_uninterruptible(1); 436 437 t = ktime_get_mono_fast_ns(); 438 if (atomic_inc_return(&n_rcu_scale_writer_started) >= nrealwriters) { 439 t_rcu_scale_writer_started = t; 440 if (gp_exp) { 441 b_rcu_gp_test_started = 442 cur_ops->exp_completed() / 2; 443 } else { 444 b_rcu_gp_test_started = cur_ops->get_gp_seq(); 445 } 446 } 447 448 do { 449 if (writer_holdoff) 450 udelay(writer_holdoff); 451 wdp = &wdpp[i]; 452 *wdp = ktime_get_mono_fast_ns(); 453 if (gp_async) { 454 retry: 455 if (!rhp) 456 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 457 if (rhp && atomic_read(this_cpu_ptr(&n_async_inflight)) < gp_async_max) { 458 atomic_inc(this_cpu_ptr(&n_async_inflight)); 459 cur_ops->async(rhp, rcu_scale_async_cb); 460 rhp = NULL; 461 } else if (!kthread_should_stop()) { 462 cur_ops->gp_barrier(); 463 goto retry; 464 } else { 465 kfree(rhp); /* Because we are stopping. */ 466 } 467 } else if (gp_exp) { 468 cur_ops->exp_sync(); 469 } else { 470 cur_ops->sync(); 471 } 472 t = ktime_get_mono_fast_ns(); 473 *wdp = t - *wdp; 474 i_max = i; 475 if (!started && 476 atomic_read(&n_rcu_scale_writer_started) >= nrealwriters) 477 started = true; 478 if (!done && i >= MIN_MEAS) { 479 done = true; 480 sched_set_normal(current, 0); 481 pr_alert("%s%s rcu_scale_writer %ld has %d measurements\n", 482 scale_type, SCALE_FLAG, me, MIN_MEAS); 483 if (atomic_inc_return(&n_rcu_scale_writer_finished) >= 484 nrealwriters) { 485 schedule_timeout_interruptible(10); 486 rcu_ftrace_dump(DUMP_ALL); 487 SCALEOUT_STRING("Test complete"); 488 t_rcu_scale_writer_finished = t; 489 if (gp_exp) { 490 b_rcu_gp_test_finished = 491 cur_ops->exp_completed() / 2; 492 } else { 493 b_rcu_gp_test_finished = 494 cur_ops->get_gp_seq(); 495 } 496 if (shutdown) { 497 smp_mb(); /* Assign before wake. */ 498 wake_up(&shutdown_wq); 499 } 500 } 501 } 502 if (done && !alldone && 503 atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters) 504 alldone = true; 505 if (started && !alldone && i < MAX_MEAS - 1) 506 i++; 507 rcu_scale_wait_shutdown(); 508 } while (!torture_must_stop()); 509 if (gp_async) { 510 cur_ops->gp_barrier(); 511 } 512 writer_n_durations[me] = i_max + 1; 513 torture_kthread_stopping("rcu_scale_writer"); 514 return 0; 515 } 516 517 static void 518 rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag) 519 { 520 pr_alert("%s" SCALE_FLAG 521 "--- %s: nreaders=%d nwriters=%d verbose=%d shutdown=%d\n", 522 scale_type, tag, nrealreaders, nrealwriters, verbose, shutdown); 523 } 524 525 static void 526 rcu_scale_cleanup(void) 527 { 528 int i; 529 int j; 530 int ngps = 0; 531 u64 *wdp; 532 u64 *wdpp; 533 534 /* 535 * Would like warning at start, but everything is expedited 536 * during the mid-boot phase, so have to wait till the end. 537 */ 538 if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp) 539 SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!"); 540 if (rcu_gp_is_normal() && gp_exp) 541 SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!"); 542 if (gp_exp && gp_async) 543 SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!"); 544 545 if (torture_cleanup_begin()) 546 return; 547 if (!cur_ops) { 548 torture_cleanup_end(); 549 return; 550 } 551 552 if (reader_tasks) { 553 for (i = 0; i < nrealreaders; i++) 554 torture_stop_kthread(rcu_scale_reader, 555 reader_tasks[i]); 556 kfree(reader_tasks); 557 } 558 559 if (writer_tasks) { 560 for (i = 0; i < nrealwriters; i++) { 561 torture_stop_kthread(rcu_scale_writer, 562 writer_tasks[i]); 563 if (!writer_n_durations) 564 continue; 565 j = writer_n_durations[i]; 566 pr_alert("%s%s writer %d gps: %d\n", 567 scale_type, SCALE_FLAG, i, j); 568 ngps += j; 569 } 570 pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n", 571 scale_type, SCALE_FLAG, 572 t_rcu_scale_writer_started, t_rcu_scale_writer_finished, 573 t_rcu_scale_writer_finished - 574 t_rcu_scale_writer_started, 575 ngps, 576 rcuscale_seq_diff(b_rcu_gp_test_finished, 577 b_rcu_gp_test_started)); 578 for (i = 0; i < nrealwriters; i++) { 579 if (!writer_durations) 580 break; 581 if (!writer_n_durations) 582 continue; 583 wdpp = writer_durations[i]; 584 if (!wdpp) 585 continue; 586 for (j = 0; j < writer_n_durations[i]; j++) { 587 wdp = &wdpp[j]; 588 pr_alert("%s%s %4d writer-duration: %5d %llu\n", 589 scale_type, SCALE_FLAG, 590 i, j, *wdp); 591 if (j % 100 == 0) 592 schedule_timeout_uninterruptible(1); 593 } 594 kfree(writer_durations[i]); 595 } 596 kfree(writer_tasks); 597 kfree(writer_durations); 598 kfree(writer_n_durations); 599 } 600 601 /* Do torture-type-specific cleanup operations. */ 602 if (cur_ops->cleanup != NULL) 603 cur_ops->cleanup(); 604 605 torture_cleanup_end(); 606 } 607 608 /* 609 * Return the number if non-negative. If -1, the number of CPUs. 610 * If less than -1, that much less than the number of CPUs, but 611 * at least one. 612 */ 613 static int compute_real(int n) 614 { 615 int nr; 616 617 if (n >= 0) { 618 nr = n; 619 } else { 620 nr = num_online_cpus() + 1 + n; 621 if (nr <= 0) 622 nr = 1; 623 } 624 return nr; 625 } 626 627 /* 628 * RCU scalability shutdown kthread. Just waits to be awakened, then shuts 629 * down system. 630 */ 631 static int 632 rcu_scale_shutdown(void *arg) 633 { 634 wait_event(shutdown_wq, 635 atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters); 636 smp_mb(); /* Wake before output. */ 637 rcu_scale_cleanup(); 638 kernel_power_off(); 639 return -EINVAL; 640 } 641 642 /* 643 * kfree_rcu() scalability tests: Start a kfree_rcu() loop on all CPUs for number 644 * of iterations and measure total time and number of GP for all iterations to complete. 645 */ 646 647 torture_param(int, kfree_nthreads, -1, "Number of threads running loops of kfree_rcu()."); 648 torture_param(int, kfree_alloc_num, 8000, "Number of allocations and frees done in an iteration."); 649 torture_param(int, kfree_loops, 10, "Number of loops doing kfree_alloc_num allocations and frees."); 650 torture_param(bool, kfree_rcu_test_double, false, "Do we run a kfree_rcu() double-argument scale test?"); 651 torture_param(bool, kfree_rcu_test_single, false, "Do we run a kfree_rcu() single-argument scale test?"); 652 653 static struct task_struct **kfree_reader_tasks; 654 static int kfree_nrealthreads; 655 static atomic_t n_kfree_scale_thread_started; 656 static atomic_t n_kfree_scale_thread_ended; 657 658 struct kfree_obj { 659 char kfree_obj[8]; 660 struct rcu_head rh; 661 }; 662 663 /* Used if doing RCU-kfree'ing via call_rcu(). */ 664 static void kfree_call_rcu(struct rcu_head *rh) 665 { 666 struct kfree_obj *obj = container_of(rh, struct kfree_obj, rh); 667 668 kfree(obj); 669 } 670 671 static int 672 kfree_scale_thread(void *arg) 673 { 674 int i, loop = 0; 675 long me = (long)arg; 676 struct kfree_obj *alloc_ptr; 677 u64 start_time, end_time; 678 long long mem_begin, mem_during = 0; 679 bool kfree_rcu_test_both; 680 DEFINE_TORTURE_RANDOM(tr); 681 682 VERBOSE_SCALEOUT_STRING("kfree_scale_thread task started"); 683 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 684 set_user_nice(current, MAX_NICE); 685 kfree_rcu_test_both = (kfree_rcu_test_single == kfree_rcu_test_double); 686 687 start_time = ktime_get_mono_fast_ns(); 688 689 if (atomic_inc_return(&n_kfree_scale_thread_started) >= kfree_nrealthreads) { 690 if (gp_exp) 691 b_rcu_gp_test_started = cur_ops->exp_completed() / 2; 692 else 693 b_rcu_gp_test_started = cur_ops->get_gp_seq(); 694 } 695 696 do { 697 if (!mem_during) { 698 mem_during = mem_begin = si_mem_available(); 699 } else if (loop % (kfree_loops / 4) == 0) { 700 mem_during = (mem_during + si_mem_available()) / 2; 701 } 702 703 for (i = 0; i < kfree_alloc_num; i++) { 704 alloc_ptr = kmalloc(kfree_mult * sizeof(struct kfree_obj), GFP_KERNEL); 705 if (!alloc_ptr) 706 return -ENOMEM; 707 708 if (kfree_by_call_rcu) { 709 call_rcu(&(alloc_ptr->rh), kfree_call_rcu); 710 continue; 711 } 712 713 // By default kfree_rcu_test_single and kfree_rcu_test_double are 714 // initialized to false. If both have the same value (false or true) 715 // both are randomly tested, otherwise only the one with value true 716 // is tested. 717 if ((kfree_rcu_test_single && !kfree_rcu_test_double) || 718 (kfree_rcu_test_both && torture_random(&tr) & 0x800)) 719 kfree_rcu(alloc_ptr); 720 else 721 kfree_rcu(alloc_ptr, rh); 722 } 723 724 cond_resched(); 725 } while (!torture_must_stop() && ++loop < kfree_loops); 726 727 if (atomic_inc_return(&n_kfree_scale_thread_ended) >= kfree_nrealthreads) { 728 end_time = ktime_get_mono_fast_ns(); 729 730 if (gp_exp) 731 b_rcu_gp_test_finished = cur_ops->exp_completed() / 2; 732 else 733 b_rcu_gp_test_finished = cur_ops->get_gp_seq(); 734 735 pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %lldMB\n", 736 (unsigned long long)(end_time - start_time), kfree_loops, 737 rcuscale_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started), 738 (mem_begin - mem_during) >> (20 - PAGE_SHIFT)); 739 740 if (shutdown) { 741 smp_mb(); /* Assign before wake. */ 742 wake_up(&shutdown_wq); 743 } 744 } 745 746 torture_kthread_stopping("kfree_scale_thread"); 747 return 0; 748 } 749 750 static void 751 kfree_scale_cleanup(void) 752 { 753 int i; 754 755 if (torture_cleanup_begin()) 756 return; 757 758 if (kfree_reader_tasks) { 759 for (i = 0; i < kfree_nrealthreads; i++) 760 torture_stop_kthread(kfree_scale_thread, 761 kfree_reader_tasks[i]); 762 kfree(kfree_reader_tasks); 763 } 764 765 torture_cleanup_end(); 766 } 767 768 /* 769 * shutdown kthread. Just waits to be awakened, then shuts down system. 770 */ 771 static int 772 kfree_scale_shutdown(void *arg) 773 { 774 wait_event(shutdown_wq, 775 atomic_read(&n_kfree_scale_thread_ended) >= kfree_nrealthreads); 776 777 smp_mb(); /* Wake before output. */ 778 779 kfree_scale_cleanup(); 780 kernel_power_off(); 781 return -EINVAL; 782 } 783 784 // Used if doing RCU-kfree'ing via call_rcu(). 785 static unsigned long jiffies_at_lazy_cb; 786 static struct rcu_head lazy_test1_rh; 787 static int rcu_lazy_test1_cb_called; 788 static void call_rcu_lazy_test1(struct rcu_head *rh) 789 { 790 jiffies_at_lazy_cb = jiffies; 791 WRITE_ONCE(rcu_lazy_test1_cb_called, 1); 792 } 793 794 static int __init 795 kfree_scale_init(void) 796 { 797 int firsterr = 0; 798 long i; 799 unsigned long jif_start; 800 unsigned long orig_jif; 801 802 // Also, do a quick self-test to ensure laziness is as much as 803 // expected. 804 if (kfree_by_call_rcu && !IS_ENABLED(CONFIG_RCU_LAZY)) { 805 pr_alert("CONFIG_RCU_LAZY is disabled, falling back to kfree_rcu() for delayed RCU kfree'ing\n"); 806 kfree_by_call_rcu = 0; 807 } 808 809 if (kfree_by_call_rcu) { 810 /* do a test to check the timeout. */ 811 orig_jif = rcu_lazy_get_jiffies_till_flush(); 812 813 rcu_lazy_set_jiffies_till_flush(2 * HZ); 814 rcu_barrier(); 815 816 jif_start = jiffies; 817 jiffies_at_lazy_cb = 0; 818 call_rcu(&lazy_test1_rh, call_rcu_lazy_test1); 819 820 smp_cond_load_relaxed(&rcu_lazy_test1_cb_called, VAL == 1); 821 822 rcu_lazy_set_jiffies_till_flush(orig_jif); 823 824 if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start < 2 * HZ)) { 825 pr_alert("ERROR: call_rcu() CBs are not being lazy as expected!\n"); 826 WARN_ON_ONCE(1); 827 return -1; 828 } 829 830 if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start > 3 * HZ)) { 831 pr_alert("ERROR: call_rcu() CBs are being too lazy!\n"); 832 WARN_ON_ONCE(1); 833 return -1; 834 } 835 } 836 837 kfree_nrealthreads = compute_real(kfree_nthreads); 838 /* Start up the kthreads. */ 839 if (shutdown) { 840 init_waitqueue_head(&shutdown_wq); 841 firsterr = torture_create_kthread(kfree_scale_shutdown, NULL, 842 shutdown_task); 843 if (torture_init_error(firsterr)) 844 goto unwind; 845 schedule_timeout_uninterruptible(1); 846 } 847 848 pr_alert("kfree object size=%zu, kfree_by_call_rcu=%d\n", 849 kfree_mult * sizeof(struct kfree_obj), 850 kfree_by_call_rcu); 851 852 kfree_reader_tasks = kcalloc(kfree_nrealthreads, sizeof(kfree_reader_tasks[0]), 853 GFP_KERNEL); 854 if (kfree_reader_tasks == NULL) { 855 firsterr = -ENOMEM; 856 goto unwind; 857 } 858 859 for (i = 0; i < kfree_nrealthreads; i++) { 860 firsterr = torture_create_kthread(kfree_scale_thread, (void *)i, 861 kfree_reader_tasks[i]); 862 if (torture_init_error(firsterr)) 863 goto unwind; 864 } 865 866 while (atomic_read(&n_kfree_scale_thread_started) < kfree_nrealthreads) 867 schedule_timeout_uninterruptible(1); 868 869 torture_init_end(); 870 return 0; 871 872 unwind: 873 torture_init_end(); 874 kfree_scale_cleanup(); 875 return firsterr; 876 } 877 878 static int __init 879 rcu_scale_init(void) 880 { 881 long i; 882 int firsterr = 0; 883 static struct rcu_scale_ops *scale_ops[] = { 884 &rcu_ops, &srcu_ops, &srcud_ops, TASKS_OPS TASKS_TRACING_OPS 885 }; 886 887 if (!torture_init_begin(scale_type, verbose)) 888 return -EBUSY; 889 890 /* Process args and announce that the scalability'er is on the job. */ 891 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { 892 cur_ops = scale_ops[i]; 893 if (strcmp(scale_type, cur_ops->name) == 0) 894 break; 895 } 896 if (i == ARRAY_SIZE(scale_ops)) { 897 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type); 898 pr_alert("rcu-scale types:"); 899 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) 900 pr_cont(" %s", scale_ops[i]->name); 901 pr_cont("\n"); 902 firsterr = -EINVAL; 903 cur_ops = NULL; 904 goto unwind; 905 } 906 if (cur_ops->init) 907 cur_ops->init(); 908 909 if (kfree_rcu_test) 910 return kfree_scale_init(); 911 912 nrealwriters = compute_real(nwriters); 913 nrealreaders = compute_real(nreaders); 914 atomic_set(&n_rcu_scale_reader_started, 0); 915 atomic_set(&n_rcu_scale_writer_started, 0); 916 atomic_set(&n_rcu_scale_writer_finished, 0); 917 rcu_scale_print_module_parms(cur_ops, "Start of test"); 918 919 /* Start up the kthreads. */ 920 921 if (shutdown) { 922 init_waitqueue_head(&shutdown_wq); 923 firsterr = torture_create_kthread(rcu_scale_shutdown, NULL, 924 shutdown_task); 925 if (torture_init_error(firsterr)) 926 goto unwind; 927 schedule_timeout_uninterruptible(1); 928 } 929 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 930 GFP_KERNEL); 931 if (reader_tasks == NULL) { 932 SCALEOUT_ERRSTRING("out of memory"); 933 firsterr = -ENOMEM; 934 goto unwind; 935 } 936 for (i = 0; i < nrealreaders; i++) { 937 firsterr = torture_create_kthread(rcu_scale_reader, (void *)i, 938 reader_tasks[i]); 939 if (torture_init_error(firsterr)) 940 goto unwind; 941 } 942 while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders) 943 schedule_timeout_uninterruptible(1); 944 writer_tasks = kcalloc(nrealwriters, sizeof(reader_tasks[0]), 945 GFP_KERNEL); 946 writer_durations = kcalloc(nrealwriters, sizeof(*writer_durations), 947 GFP_KERNEL); 948 writer_n_durations = 949 kcalloc(nrealwriters, sizeof(*writer_n_durations), 950 GFP_KERNEL); 951 if (!writer_tasks || !writer_durations || !writer_n_durations) { 952 SCALEOUT_ERRSTRING("out of memory"); 953 firsterr = -ENOMEM; 954 goto unwind; 955 } 956 for (i = 0; i < nrealwriters; i++) { 957 writer_durations[i] = 958 kcalloc(MAX_MEAS, sizeof(*writer_durations[i]), 959 GFP_KERNEL); 960 if (!writer_durations[i]) { 961 firsterr = -ENOMEM; 962 goto unwind; 963 } 964 firsterr = torture_create_kthread(rcu_scale_writer, (void *)i, 965 writer_tasks[i]); 966 if (torture_init_error(firsterr)) 967 goto unwind; 968 } 969 torture_init_end(); 970 return 0; 971 972 unwind: 973 torture_init_end(); 974 rcu_scale_cleanup(); 975 if (shutdown) { 976 WARN_ON(!IS_MODULE(CONFIG_RCU_SCALE_TEST)); 977 kernel_power_off(); 978 } 979 return firsterr; 980 } 981 982 module_init(rcu_scale_init); 983 module_exit(rcu_scale_cleanup); 984