1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based scalability-test facility 4 * 5 * Copyright (C) IBM Corporation, 2015 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 */ 9 10 #define pr_fmt(fmt) fmt 11 12 #include <linux/types.h> 13 #include <linux/kernel.h> 14 #include <linux/init.h> 15 #include <linux/mm.h> 16 #include <linux/module.h> 17 #include <linux/kthread.h> 18 #include <linux/err.h> 19 #include <linux/spinlock.h> 20 #include <linux/smp.h> 21 #include <linux/rcupdate.h> 22 #include <linux/interrupt.h> 23 #include <linux/sched.h> 24 #include <uapi/linux/sched/types.h> 25 #include <linux/atomic.h> 26 #include <linux/bitops.h> 27 #include <linux/completion.h> 28 #include <linux/moduleparam.h> 29 #include <linux/percpu.h> 30 #include <linux/notifier.h> 31 #include <linux/reboot.h> 32 #include <linux/freezer.h> 33 #include <linux/cpu.h> 34 #include <linux/delay.h> 35 #include <linux/stat.h> 36 #include <linux/srcu.h> 37 #include <linux/slab.h> 38 #include <asm/byteorder.h> 39 #include <linux/torture.h> 40 #include <linux/vmalloc.h> 41 #include <linux/rcupdate_trace.h> 42 43 #include "rcu.h" 44 45 MODULE_LICENSE("GPL"); 46 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 47 48 #define SCALE_FLAG "-scale:" 49 #define SCALEOUT_STRING(s) \ 50 pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s) 51 #define VERBOSE_SCALEOUT_STRING(s) \ 52 do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0) 53 #define SCALEOUT_ERRSTRING(s) \ 54 pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s) 55 56 /* 57 * The intended use cases for the nreaders and nwriters module parameters 58 * are as follows: 59 * 60 * 1. Specify only the nr_cpus kernel boot parameter. This will 61 * set both nreaders and nwriters to the value specified by 62 * nr_cpus for a mixed reader/writer test. 63 * 64 * 2. Specify the nr_cpus kernel boot parameter, but set 65 * rcuscale.nreaders to zero. This will set nwriters to the 66 * value specified by nr_cpus for an update-only test. 67 * 68 * 3. Specify the nr_cpus kernel boot parameter, but set 69 * rcuscale.nwriters to zero. This will set nreaders to the 70 * value specified by nr_cpus for a read-only test. 71 * 72 * Various other use cases may of course be specified. 73 * 74 * Note that this test's readers are intended only as a test load for 75 * the writers. The reader scalability statistics will be overly 76 * pessimistic due to the per-critical-section interrupt disabling, 77 * test-end checks, and the pair of calls through pointers. 78 */ 79 80 #ifdef MODULE 81 # define RCUSCALE_SHUTDOWN 0 82 #else 83 # define RCUSCALE_SHUTDOWN 1 84 #endif 85 86 torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives"); 87 torture_param(int, gp_async_max, 1000, "Max # outstanding waits per reader"); 88 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 89 torture_param(int, holdoff, 10, "Holdoff time before test start (s)"); 90 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 91 torture_param(int, nwriters, -1, "Number of RCU updater threads"); 92 torture_param(bool, shutdown, RCUSCALE_SHUTDOWN, 93 "Shutdown at end of scalability tests."); 94 torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); 95 torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); 96 torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?"); 97 torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate."); 98 torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?"); 99 100 static char *scale_type = "rcu"; 101 module_param(scale_type, charp, 0444); 102 MODULE_PARM_DESC(scale_type, "Type of RCU to scalability-test (rcu, srcu, ...)"); 103 104 static int nrealreaders; 105 static int nrealwriters; 106 static struct task_struct **writer_tasks; 107 static struct task_struct **reader_tasks; 108 static struct task_struct *shutdown_task; 109 110 static u64 **writer_durations; 111 static int *writer_n_durations; 112 static atomic_t n_rcu_scale_reader_started; 113 static atomic_t n_rcu_scale_writer_started; 114 static atomic_t n_rcu_scale_writer_finished; 115 static wait_queue_head_t shutdown_wq; 116 static u64 t_rcu_scale_writer_started; 117 static u64 t_rcu_scale_writer_finished; 118 static unsigned long b_rcu_gp_test_started; 119 static unsigned long b_rcu_gp_test_finished; 120 static DEFINE_PER_CPU(atomic_t, n_async_inflight); 121 122 #define MAX_MEAS 10000 123 #define MIN_MEAS 100 124 125 /* 126 * Operations vector for selecting different types of tests. 127 */ 128 129 struct rcu_scale_ops { 130 int ptype; 131 void (*init)(void); 132 void (*cleanup)(void); 133 int (*readlock)(void); 134 void (*readunlock)(int idx); 135 unsigned long (*get_gp_seq)(void); 136 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 137 unsigned long (*exp_completed)(void); 138 void (*async)(struct rcu_head *head, rcu_callback_t func); 139 void (*gp_barrier)(void); 140 void (*sync)(void); 141 void (*exp_sync)(void); 142 const char *name; 143 }; 144 145 static struct rcu_scale_ops *cur_ops; 146 147 /* 148 * Definitions for rcu scalability testing. 149 */ 150 151 static int rcu_scale_read_lock(void) __acquires(RCU) 152 { 153 rcu_read_lock(); 154 return 0; 155 } 156 157 static void rcu_scale_read_unlock(int idx) __releases(RCU) 158 { 159 rcu_read_unlock(); 160 } 161 162 static unsigned long __maybe_unused rcu_no_completed(void) 163 { 164 return 0; 165 } 166 167 static void rcu_sync_scale_init(void) 168 { 169 } 170 171 static struct rcu_scale_ops rcu_ops = { 172 .ptype = RCU_FLAVOR, 173 .init = rcu_sync_scale_init, 174 .readlock = rcu_scale_read_lock, 175 .readunlock = rcu_scale_read_unlock, 176 .get_gp_seq = rcu_get_gp_seq, 177 .gp_diff = rcu_seq_diff, 178 .exp_completed = rcu_exp_batches_completed, 179 .async = call_rcu_hurry, 180 .gp_barrier = rcu_barrier, 181 .sync = synchronize_rcu, 182 .exp_sync = synchronize_rcu_expedited, 183 .name = "rcu" 184 }; 185 186 /* 187 * Definitions for srcu scalability testing. 188 */ 189 190 DEFINE_STATIC_SRCU(srcu_ctl_scale); 191 static struct srcu_struct *srcu_ctlp = &srcu_ctl_scale; 192 193 static int srcu_scale_read_lock(void) __acquires(srcu_ctlp) 194 { 195 return srcu_read_lock(srcu_ctlp); 196 } 197 198 static void srcu_scale_read_unlock(int idx) __releases(srcu_ctlp) 199 { 200 srcu_read_unlock(srcu_ctlp, idx); 201 } 202 203 static unsigned long srcu_scale_completed(void) 204 { 205 return srcu_batches_completed(srcu_ctlp); 206 } 207 208 static void srcu_call_rcu(struct rcu_head *head, rcu_callback_t func) 209 { 210 call_srcu(srcu_ctlp, head, func); 211 } 212 213 static void srcu_rcu_barrier(void) 214 { 215 srcu_barrier(srcu_ctlp); 216 } 217 218 static void srcu_scale_synchronize(void) 219 { 220 synchronize_srcu(srcu_ctlp); 221 } 222 223 static void srcu_scale_synchronize_expedited(void) 224 { 225 synchronize_srcu_expedited(srcu_ctlp); 226 } 227 228 static struct rcu_scale_ops srcu_ops = { 229 .ptype = SRCU_FLAVOR, 230 .init = rcu_sync_scale_init, 231 .readlock = srcu_scale_read_lock, 232 .readunlock = srcu_scale_read_unlock, 233 .get_gp_seq = srcu_scale_completed, 234 .gp_diff = rcu_seq_diff, 235 .exp_completed = srcu_scale_completed, 236 .async = srcu_call_rcu, 237 .gp_barrier = srcu_rcu_barrier, 238 .sync = srcu_scale_synchronize, 239 .exp_sync = srcu_scale_synchronize_expedited, 240 .name = "srcu" 241 }; 242 243 static struct srcu_struct srcud; 244 245 static void srcu_sync_scale_init(void) 246 { 247 srcu_ctlp = &srcud; 248 init_srcu_struct(srcu_ctlp); 249 } 250 251 static void srcu_sync_scale_cleanup(void) 252 { 253 cleanup_srcu_struct(srcu_ctlp); 254 } 255 256 static struct rcu_scale_ops srcud_ops = { 257 .ptype = SRCU_FLAVOR, 258 .init = srcu_sync_scale_init, 259 .cleanup = srcu_sync_scale_cleanup, 260 .readlock = srcu_scale_read_lock, 261 .readunlock = srcu_scale_read_unlock, 262 .get_gp_seq = srcu_scale_completed, 263 .gp_diff = rcu_seq_diff, 264 .exp_completed = srcu_scale_completed, 265 .async = srcu_call_rcu, 266 .gp_barrier = srcu_rcu_barrier, 267 .sync = srcu_scale_synchronize, 268 .exp_sync = srcu_scale_synchronize_expedited, 269 .name = "srcud" 270 }; 271 272 #ifdef CONFIG_TASKS_RCU 273 274 /* 275 * Definitions for RCU-tasks scalability testing. 276 */ 277 278 static int tasks_scale_read_lock(void) 279 { 280 return 0; 281 } 282 283 static void tasks_scale_read_unlock(int idx) 284 { 285 } 286 287 static struct rcu_scale_ops tasks_ops = { 288 .ptype = RCU_TASKS_FLAVOR, 289 .init = rcu_sync_scale_init, 290 .readlock = tasks_scale_read_lock, 291 .readunlock = tasks_scale_read_unlock, 292 .get_gp_seq = rcu_no_completed, 293 .gp_diff = rcu_seq_diff, 294 .async = call_rcu_tasks, 295 .gp_barrier = rcu_barrier_tasks, 296 .sync = synchronize_rcu_tasks, 297 .exp_sync = synchronize_rcu_tasks, 298 .name = "tasks" 299 }; 300 301 #define TASKS_OPS &tasks_ops, 302 303 #else // #ifdef CONFIG_TASKS_RCU 304 305 #define TASKS_OPS 306 307 #endif // #else // #ifdef CONFIG_TASKS_RCU 308 309 #ifdef CONFIG_TASKS_TRACE_RCU 310 311 /* 312 * Definitions for RCU-tasks-trace scalability testing. 313 */ 314 315 static int tasks_trace_scale_read_lock(void) 316 { 317 rcu_read_lock_trace(); 318 return 0; 319 } 320 321 static void tasks_trace_scale_read_unlock(int idx) 322 { 323 rcu_read_unlock_trace(); 324 } 325 326 static struct rcu_scale_ops tasks_tracing_ops = { 327 .ptype = RCU_TASKS_FLAVOR, 328 .init = rcu_sync_scale_init, 329 .readlock = tasks_trace_scale_read_lock, 330 .readunlock = tasks_trace_scale_read_unlock, 331 .get_gp_seq = rcu_no_completed, 332 .gp_diff = rcu_seq_diff, 333 .async = call_rcu_tasks_trace, 334 .gp_barrier = rcu_barrier_tasks_trace, 335 .sync = synchronize_rcu_tasks_trace, 336 .exp_sync = synchronize_rcu_tasks_trace, 337 .name = "tasks-tracing" 338 }; 339 340 #define TASKS_TRACING_OPS &tasks_tracing_ops, 341 342 #else // #ifdef CONFIG_TASKS_TRACE_RCU 343 344 #define TASKS_TRACING_OPS 345 346 #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU 347 348 static unsigned long rcuscale_seq_diff(unsigned long new, unsigned long old) 349 { 350 if (!cur_ops->gp_diff) 351 return new - old; 352 return cur_ops->gp_diff(new, old); 353 } 354 355 /* 356 * If scalability tests complete, wait for shutdown to commence. 357 */ 358 static void rcu_scale_wait_shutdown(void) 359 { 360 cond_resched_tasks_rcu_qs(); 361 if (atomic_read(&n_rcu_scale_writer_finished) < nrealwriters) 362 return; 363 while (!torture_must_stop()) 364 schedule_timeout_uninterruptible(1); 365 } 366 367 /* 368 * RCU scalability reader kthread. Repeatedly does empty RCU read-side 369 * critical section, minimizing update-side interference. However, the 370 * point of this test is not to evaluate reader scalability, but instead 371 * to serve as a test load for update-side scalability testing. 372 */ 373 static int 374 rcu_scale_reader(void *arg) 375 { 376 unsigned long flags; 377 int idx; 378 long me = (long)arg; 379 380 VERBOSE_SCALEOUT_STRING("rcu_scale_reader task started"); 381 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 382 set_user_nice(current, MAX_NICE); 383 atomic_inc(&n_rcu_scale_reader_started); 384 385 do { 386 local_irq_save(flags); 387 idx = cur_ops->readlock(); 388 cur_ops->readunlock(idx); 389 local_irq_restore(flags); 390 rcu_scale_wait_shutdown(); 391 } while (!torture_must_stop()); 392 torture_kthread_stopping("rcu_scale_reader"); 393 return 0; 394 } 395 396 /* 397 * Callback function for asynchronous grace periods from rcu_scale_writer(). 398 */ 399 static void rcu_scale_async_cb(struct rcu_head *rhp) 400 { 401 atomic_dec(this_cpu_ptr(&n_async_inflight)); 402 kfree(rhp); 403 } 404 405 /* 406 * RCU scale writer kthread. Repeatedly does a grace period. 407 */ 408 static int 409 rcu_scale_writer(void *arg) 410 { 411 int i = 0; 412 int i_max; 413 long me = (long)arg; 414 struct rcu_head *rhp = NULL; 415 bool started = false, done = false, alldone = false; 416 u64 t; 417 u64 *wdp; 418 u64 *wdpp = writer_durations[me]; 419 420 VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started"); 421 WARN_ON(!wdpp); 422 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 423 current->flags |= PF_NO_SETAFFINITY; 424 sched_set_fifo_low(current); 425 426 if (holdoff) 427 schedule_timeout_uninterruptible(holdoff * HZ); 428 429 /* 430 * Wait until rcu_end_inkernel_boot() is called for normal GP tests 431 * so that RCU is not always expedited for normal GP tests. 432 * The system_state test is approximate, but works well in practice. 433 */ 434 while (!gp_exp && system_state != SYSTEM_RUNNING) 435 schedule_timeout_uninterruptible(1); 436 437 t = ktime_get_mono_fast_ns(); 438 if (atomic_inc_return(&n_rcu_scale_writer_started) >= nrealwriters) { 439 t_rcu_scale_writer_started = t; 440 if (gp_exp) { 441 b_rcu_gp_test_started = 442 cur_ops->exp_completed() / 2; 443 } else { 444 b_rcu_gp_test_started = cur_ops->get_gp_seq(); 445 } 446 } 447 448 do { 449 if (writer_holdoff) 450 udelay(writer_holdoff); 451 wdp = &wdpp[i]; 452 *wdp = ktime_get_mono_fast_ns(); 453 if (gp_async) { 454 retry: 455 if (!rhp) 456 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 457 if (rhp && atomic_read(this_cpu_ptr(&n_async_inflight)) < gp_async_max) { 458 atomic_inc(this_cpu_ptr(&n_async_inflight)); 459 cur_ops->async(rhp, rcu_scale_async_cb); 460 rhp = NULL; 461 } else if (!kthread_should_stop()) { 462 cur_ops->gp_barrier(); 463 goto retry; 464 } else { 465 kfree(rhp); /* Because we are stopping. */ 466 } 467 } else if (gp_exp) { 468 cur_ops->exp_sync(); 469 } else { 470 cur_ops->sync(); 471 } 472 t = ktime_get_mono_fast_ns(); 473 *wdp = t - *wdp; 474 i_max = i; 475 if (!started && 476 atomic_read(&n_rcu_scale_writer_started) >= nrealwriters) 477 started = true; 478 if (!done && i >= MIN_MEAS) { 479 done = true; 480 sched_set_normal(current, 0); 481 pr_alert("%s%s rcu_scale_writer %ld has %d measurements\n", 482 scale_type, SCALE_FLAG, me, MIN_MEAS); 483 if (atomic_inc_return(&n_rcu_scale_writer_finished) >= 484 nrealwriters) { 485 schedule_timeout_interruptible(10); 486 rcu_ftrace_dump(DUMP_ALL); 487 SCALEOUT_STRING("Test complete"); 488 t_rcu_scale_writer_finished = t; 489 if (gp_exp) { 490 b_rcu_gp_test_finished = 491 cur_ops->exp_completed() / 2; 492 } else { 493 b_rcu_gp_test_finished = 494 cur_ops->get_gp_seq(); 495 } 496 if (shutdown) { 497 smp_mb(); /* Assign before wake. */ 498 wake_up(&shutdown_wq); 499 } 500 } 501 } 502 if (done && !alldone && 503 atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters) 504 alldone = true; 505 if (started && !alldone && i < MAX_MEAS - 1) 506 i++; 507 rcu_scale_wait_shutdown(); 508 } while (!torture_must_stop()); 509 if (gp_async) { 510 cur_ops->gp_barrier(); 511 } 512 writer_n_durations[me] = i_max + 1; 513 torture_kthread_stopping("rcu_scale_writer"); 514 return 0; 515 } 516 517 static void 518 rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag) 519 { 520 pr_alert("%s" SCALE_FLAG 521 "--- %s: nreaders=%d nwriters=%d verbose=%d shutdown=%d\n", 522 scale_type, tag, nrealreaders, nrealwriters, verbose, shutdown); 523 } 524 525 static void 526 rcu_scale_cleanup(void) 527 { 528 int i; 529 int j; 530 int ngps = 0; 531 u64 *wdp; 532 u64 *wdpp; 533 534 /* 535 * Would like warning at start, but everything is expedited 536 * during the mid-boot phase, so have to wait till the end. 537 */ 538 if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp) 539 SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!"); 540 if (rcu_gp_is_normal() && gp_exp) 541 SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!"); 542 if (gp_exp && gp_async) 543 SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!"); 544 545 if (torture_cleanup_begin()) 546 return; 547 if (!cur_ops) { 548 torture_cleanup_end(); 549 return; 550 } 551 552 if (reader_tasks) { 553 for (i = 0; i < nrealreaders; i++) 554 torture_stop_kthread(rcu_scale_reader, 555 reader_tasks[i]); 556 kfree(reader_tasks); 557 } 558 559 if (writer_tasks) { 560 for (i = 0; i < nrealwriters; i++) { 561 torture_stop_kthread(rcu_scale_writer, 562 writer_tasks[i]); 563 if (!writer_n_durations) 564 continue; 565 j = writer_n_durations[i]; 566 pr_alert("%s%s writer %d gps: %d\n", 567 scale_type, SCALE_FLAG, i, j); 568 ngps += j; 569 } 570 pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n", 571 scale_type, SCALE_FLAG, 572 t_rcu_scale_writer_started, t_rcu_scale_writer_finished, 573 t_rcu_scale_writer_finished - 574 t_rcu_scale_writer_started, 575 ngps, 576 rcuscale_seq_diff(b_rcu_gp_test_finished, 577 b_rcu_gp_test_started)); 578 for (i = 0; i < nrealwriters; i++) { 579 if (!writer_durations) 580 break; 581 if (!writer_n_durations) 582 continue; 583 wdpp = writer_durations[i]; 584 if (!wdpp) 585 continue; 586 for (j = 0; j < writer_n_durations[i]; j++) { 587 wdp = &wdpp[j]; 588 pr_alert("%s%s %4d writer-duration: %5d %llu\n", 589 scale_type, SCALE_FLAG, 590 i, j, *wdp); 591 if (j % 100 == 0) 592 schedule_timeout_uninterruptible(1); 593 } 594 kfree(writer_durations[i]); 595 } 596 kfree(writer_tasks); 597 kfree(writer_durations); 598 kfree(writer_n_durations); 599 } 600 601 /* Do torture-type-specific cleanup operations. */ 602 if (cur_ops->cleanup != NULL) 603 cur_ops->cleanup(); 604 605 torture_cleanup_end(); 606 } 607 608 /* 609 * Return the number if non-negative. If -1, the number of CPUs. 610 * If less than -1, that much less than the number of CPUs, but 611 * at least one. 612 */ 613 static int compute_real(int n) 614 { 615 int nr; 616 617 if (n >= 0) { 618 nr = n; 619 } else { 620 nr = num_online_cpus() + 1 + n; 621 if (nr <= 0) 622 nr = 1; 623 } 624 return nr; 625 } 626 627 /* 628 * RCU scalability shutdown kthread. Just waits to be awakened, then shuts 629 * down system. 630 */ 631 static int 632 rcu_scale_shutdown(void *arg) 633 { 634 wait_event_idle(shutdown_wq, atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters); 635 smp_mb(); /* Wake before output. */ 636 rcu_scale_cleanup(); 637 kernel_power_off(); 638 return -EINVAL; 639 } 640 641 /* 642 * kfree_rcu() scalability tests: Start a kfree_rcu() loop on all CPUs for number 643 * of iterations and measure total time and number of GP for all iterations to complete. 644 */ 645 646 torture_param(int, kfree_nthreads, -1, "Number of threads running loops of kfree_rcu()."); 647 torture_param(int, kfree_alloc_num, 8000, "Number of allocations and frees done in an iteration."); 648 torture_param(int, kfree_loops, 10, "Number of loops doing kfree_alloc_num allocations and frees."); 649 torture_param(bool, kfree_rcu_test_double, false, "Do we run a kfree_rcu() double-argument scale test?"); 650 torture_param(bool, kfree_rcu_test_single, false, "Do we run a kfree_rcu() single-argument scale test?"); 651 652 static struct task_struct **kfree_reader_tasks; 653 static int kfree_nrealthreads; 654 static atomic_t n_kfree_scale_thread_started; 655 static atomic_t n_kfree_scale_thread_ended; 656 657 struct kfree_obj { 658 char kfree_obj[8]; 659 struct rcu_head rh; 660 }; 661 662 /* Used if doing RCU-kfree'ing via call_rcu(). */ 663 static void kfree_call_rcu(struct rcu_head *rh) 664 { 665 struct kfree_obj *obj = container_of(rh, struct kfree_obj, rh); 666 667 kfree(obj); 668 } 669 670 static int 671 kfree_scale_thread(void *arg) 672 { 673 int i, loop = 0; 674 long me = (long)arg; 675 struct kfree_obj *alloc_ptr; 676 u64 start_time, end_time; 677 long long mem_begin, mem_during = 0; 678 bool kfree_rcu_test_both; 679 DEFINE_TORTURE_RANDOM(tr); 680 681 VERBOSE_SCALEOUT_STRING("kfree_scale_thread task started"); 682 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 683 set_user_nice(current, MAX_NICE); 684 kfree_rcu_test_both = (kfree_rcu_test_single == kfree_rcu_test_double); 685 686 start_time = ktime_get_mono_fast_ns(); 687 688 if (atomic_inc_return(&n_kfree_scale_thread_started) >= kfree_nrealthreads) { 689 if (gp_exp) 690 b_rcu_gp_test_started = cur_ops->exp_completed() / 2; 691 else 692 b_rcu_gp_test_started = cur_ops->get_gp_seq(); 693 } 694 695 do { 696 if (!mem_during) { 697 mem_during = mem_begin = si_mem_available(); 698 } else if (loop % (kfree_loops / 4) == 0) { 699 mem_during = (mem_during + si_mem_available()) / 2; 700 } 701 702 for (i = 0; i < kfree_alloc_num; i++) { 703 alloc_ptr = kmalloc(kfree_mult * sizeof(struct kfree_obj), GFP_KERNEL); 704 if (!alloc_ptr) 705 return -ENOMEM; 706 707 if (kfree_by_call_rcu) { 708 call_rcu(&(alloc_ptr->rh), kfree_call_rcu); 709 continue; 710 } 711 712 // By default kfree_rcu_test_single and kfree_rcu_test_double are 713 // initialized to false. If both have the same value (false or true) 714 // both are randomly tested, otherwise only the one with value true 715 // is tested. 716 if ((kfree_rcu_test_single && !kfree_rcu_test_double) || 717 (kfree_rcu_test_both && torture_random(&tr) & 0x800)) 718 kfree_rcu_mightsleep(alloc_ptr); 719 else 720 kfree_rcu(alloc_ptr, rh); 721 } 722 723 cond_resched(); 724 } while (!torture_must_stop() && ++loop < kfree_loops); 725 726 if (atomic_inc_return(&n_kfree_scale_thread_ended) >= kfree_nrealthreads) { 727 end_time = ktime_get_mono_fast_ns(); 728 729 if (gp_exp) 730 b_rcu_gp_test_finished = cur_ops->exp_completed() / 2; 731 else 732 b_rcu_gp_test_finished = cur_ops->get_gp_seq(); 733 734 pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %lldMB\n", 735 (unsigned long long)(end_time - start_time), kfree_loops, 736 rcuscale_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started), 737 (mem_begin - mem_during) >> (20 - PAGE_SHIFT)); 738 739 if (shutdown) { 740 smp_mb(); /* Assign before wake. */ 741 wake_up(&shutdown_wq); 742 } 743 } 744 745 torture_kthread_stopping("kfree_scale_thread"); 746 return 0; 747 } 748 749 static void 750 kfree_scale_cleanup(void) 751 { 752 int i; 753 754 if (torture_cleanup_begin()) 755 return; 756 757 if (kfree_reader_tasks) { 758 for (i = 0; i < kfree_nrealthreads; i++) 759 torture_stop_kthread(kfree_scale_thread, 760 kfree_reader_tasks[i]); 761 kfree(kfree_reader_tasks); 762 } 763 764 torture_cleanup_end(); 765 } 766 767 /* 768 * shutdown kthread. Just waits to be awakened, then shuts down system. 769 */ 770 static int 771 kfree_scale_shutdown(void *arg) 772 { 773 wait_event_idle(shutdown_wq, 774 atomic_read(&n_kfree_scale_thread_ended) >= kfree_nrealthreads); 775 776 smp_mb(); /* Wake before output. */ 777 778 kfree_scale_cleanup(); 779 kernel_power_off(); 780 return -EINVAL; 781 } 782 783 // Used if doing RCU-kfree'ing via call_rcu(). 784 static unsigned long jiffies_at_lazy_cb; 785 static struct rcu_head lazy_test1_rh; 786 static int rcu_lazy_test1_cb_called; 787 static void call_rcu_lazy_test1(struct rcu_head *rh) 788 { 789 jiffies_at_lazy_cb = jiffies; 790 WRITE_ONCE(rcu_lazy_test1_cb_called, 1); 791 } 792 793 static int __init 794 kfree_scale_init(void) 795 { 796 int firsterr = 0; 797 long i; 798 unsigned long jif_start; 799 unsigned long orig_jif; 800 801 // Also, do a quick self-test to ensure laziness is as much as 802 // expected. 803 if (kfree_by_call_rcu && !IS_ENABLED(CONFIG_RCU_LAZY)) { 804 pr_alert("CONFIG_RCU_LAZY is disabled, falling back to kfree_rcu() for delayed RCU kfree'ing\n"); 805 kfree_by_call_rcu = 0; 806 } 807 808 if (kfree_by_call_rcu) { 809 /* do a test to check the timeout. */ 810 orig_jif = rcu_lazy_get_jiffies_till_flush(); 811 812 rcu_lazy_set_jiffies_till_flush(2 * HZ); 813 rcu_barrier(); 814 815 jif_start = jiffies; 816 jiffies_at_lazy_cb = 0; 817 call_rcu(&lazy_test1_rh, call_rcu_lazy_test1); 818 819 smp_cond_load_relaxed(&rcu_lazy_test1_cb_called, VAL == 1); 820 821 rcu_lazy_set_jiffies_till_flush(orig_jif); 822 823 if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start < 2 * HZ)) { 824 pr_alert("ERROR: call_rcu() CBs are not being lazy as expected!\n"); 825 WARN_ON_ONCE(1); 826 return -1; 827 } 828 829 if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start > 3 * HZ)) { 830 pr_alert("ERROR: call_rcu() CBs are being too lazy!\n"); 831 WARN_ON_ONCE(1); 832 return -1; 833 } 834 } 835 836 kfree_nrealthreads = compute_real(kfree_nthreads); 837 /* Start up the kthreads. */ 838 if (shutdown) { 839 init_waitqueue_head(&shutdown_wq); 840 firsterr = torture_create_kthread(kfree_scale_shutdown, NULL, 841 shutdown_task); 842 if (torture_init_error(firsterr)) 843 goto unwind; 844 schedule_timeout_uninterruptible(1); 845 } 846 847 pr_alert("kfree object size=%zu, kfree_by_call_rcu=%d\n", 848 kfree_mult * sizeof(struct kfree_obj), 849 kfree_by_call_rcu); 850 851 kfree_reader_tasks = kcalloc(kfree_nrealthreads, sizeof(kfree_reader_tasks[0]), 852 GFP_KERNEL); 853 if (kfree_reader_tasks == NULL) { 854 firsterr = -ENOMEM; 855 goto unwind; 856 } 857 858 for (i = 0; i < kfree_nrealthreads; i++) { 859 firsterr = torture_create_kthread(kfree_scale_thread, (void *)i, 860 kfree_reader_tasks[i]); 861 if (torture_init_error(firsterr)) 862 goto unwind; 863 } 864 865 while (atomic_read(&n_kfree_scale_thread_started) < kfree_nrealthreads) 866 schedule_timeout_uninterruptible(1); 867 868 torture_init_end(); 869 return 0; 870 871 unwind: 872 torture_init_end(); 873 kfree_scale_cleanup(); 874 return firsterr; 875 } 876 877 static int __init 878 rcu_scale_init(void) 879 { 880 long i; 881 int firsterr = 0; 882 static struct rcu_scale_ops *scale_ops[] = { 883 &rcu_ops, &srcu_ops, &srcud_ops, TASKS_OPS TASKS_TRACING_OPS 884 }; 885 886 if (!torture_init_begin(scale_type, verbose)) 887 return -EBUSY; 888 889 /* Process args and announce that the scalability'er is on the job. */ 890 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { 891 cur_ops = scale_ops[i]; 892 if (strcmp(scale_type, cur_ops->name) == 0) 893 break; 894 } 895 if (i == ARRAY_SIZE(scale_ops)) { 896 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type); 897 pr_alert("rcu-scale types:"); 898 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) 899 pr_cont(" %s", scale_ops[i]->name); 900 pr_cont("\n"); 901 firsterr = -EINVAL; 902 cur_ops = NULL; 903 goto unwind; 904 } 905 if (cur_ops->init) 906 cur_ops->init(); 907 908 if (kfree_rcu_test) 909 return kfree_scale_init(); 910 911 nrealwriters = compute_real(nwriters); 912 nrealreaders = compute_real(nreaders); 913 atomic_set(&n_rcu_scale_reader_started, 0); 914 atomic_set(&n_rcu_scale_writer_started, 0); 915 atomic_set(&n_rcu_scale_writer_finished, 0); 916 rcu_scale_print_module_parms(cur_ops, "Start of test"); 917 918 /* Start up the kthreads. */ 919 920 if (shutdown) { 921 init_waitqueue_head(&shutdown_wq); 922 firsterr = torture_create_kthread(rcu_scale_shutdown, NULL, 923 shutdown_task); 924 if (torture_init_error(firsterr)) 925 goto unwind; 926 schedule_timeout_uninterruptible(1); 927 } 928 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 929 GFP_KERNEL); 930 if (reader_tasks == NULL) { 931 SCALEOUT_ERRSTRING("out of memory"); 932 firsterr = -ENOMEM; 933 goto unwind; 934 } 935 for (i = 0; i < nrealreaders; i++) { 936 firsterr = torture_create_kthread(rcu_scale_reader, (void *)i, 937 reader_tasks[i]); 938 if (torture_init_error(firsterr)) 939 goto unwind; 940 } 941 while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders) 942 schedule_timeout_uninterruptible(1); 943 writer_tasks = kcalloc(nrealwriters, sizeof(reader_tasks[0]), 944 GFP_KERNEL); 945 writer_durations = kcalloc(nrealwriters, sizeof(*writer_durations), 946 GFP_KERNEL); 947 writer_n_durations = 948 kcalloc(nrealwriters, sizeof(*writer_n_durations), 949 GFP_KERNEL); 950 if (!writer_tasks || !writer_durations || !writer_n_durations) { 951 SCALEOUT_ERRSTRING("out of memory"); 952 firsterr = -ENOMEM; 953 goto unwind; 954 } 955 for (i = 0; i < nrealwriters; i++) { 956 writer_durations[i] = 957 kcalloc(MAX_MEAS, sizeof(*writer_durations[i]), 958 GFP_KERNEL); 959 if (!writer_durations[i]) { 960 firsterr = -ENOMEM; 961 goto unwind; 962 } 963 firsterr = torture_create_kthread(rcu_scale_writer, (void *)i, 964 writer_tasks[i]); 965 if (torture_init_error(firsterr)) 966 goto unwind; 967 } 968 torture_init_end(); 969 return 0; 970 971 unwind: 972 torture_init_end(); 973 rcu_scale_cleanup(); 974 if (shutdown) { 975 WARN_ON(!IS_MODULE(CONFIG_RCU_SCALE_TEST)); 976 kernel_power_off(); 977 } 978 return firsterr; 979 } 980 981 module_init(rcu_scale_init); 982 module_exit(rcu_scale_cleanup); 983