1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched/signal.h> 36 #include <uapi/linux/sched/types.h> 37 #include <linux/atomic.h> 38 #include <linux/bitops.h> 39 #include <linux/completion.h> 40 #include <linux/moduleparam.h> 41 #include <linux/percpu.h> 42 #include <linux/notifier.h> 43 #include <linux/reboot.h> 44 #include <linux/freezer.h> 45 #include <linux/cpu.h> 46 #include <linux/delay.h> 47 #include <linux/stat.h> 48 #include <linux/srcu.h> 49 #include <linux/slab.h> 50 #include <linux/trace_clock.h> 51 #include <asm/byteorder.h> 52 #include <linux/torture.h> 53 #include <linux/vmalloc.h> 54 #include <linux/sched/debug.h> 55 56 #include "rcu.h" 57 58 MODULE_LICENSE("GPL"); 59 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 60 61 62 torture_param(int, cbflood_inter_holdoff, HZ, 63 "Holdoff between floods (jiffies)"); 64 torture_param(int, cbflood_intra_holdoff, 1, 65 "Holdoff between bursts (jiffies)"); 66 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 67 torture_param(int, cbflood_n_per_burst, 20000, 68 "# callbacks per burst in flood"); 69 torture_param(int, fqs_duration, 0, 70 "Duration of fqs bursts (us), 0 to disable"); 71 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 72 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 73 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 74 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 75 torture_param(bool, gp_normal, false, 76 "Use normal (non-expedited) GP wait primitives"); 77 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 78 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 79 torture_param(int, n_barrier_cbs, 0, 80 "# of callbacks/kthreads for barrier testing"); 81 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 82 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 83 torture_param(int, object_debug, 0, 84 "Enable debug-object double call_rcu() testing"); 85 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 86 torture_param(int, onoff_interval, 0, 87 "Time between CPU hotplugs (s), 0=disable"); 88 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 89 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 90 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 91 torture_param(int, stall_cpu_holdoff, 10, 92 "Time to wait before starting stall (s)."); 93 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 94 torture_param(int, stat_interval, 60, 95 "Number of seconds between stats printk()s"); 96 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 97 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 98 torture_param(int, test_boost_duration, 4, 99 "Duration of each boost test, seconds."); 100 torture_param(int, test_boost_interval, 7, 101 "Interval between boost tests, seconds."); 102 torture_param(bool, test_no_idle_hz, true, 103 "Test support for tickless idle CPUs"); 104 torture_param(bool, verbose, true, 105 "Enable verbose debugging printk()s"); 106 107 static char *torture_type = "rcu"; 108 module_param(torture_type, charp, 0444); 109 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 110 111 static int nrealreaders; 112 static int ncbflooders; 113 static struct task_struct *writer_task; 114 static struct task_struct **fakewriter_tasks; 115 static struct task_struct **reader_tasks; 116 static struct task_struct *stats_task; 117 static struct task_struct **cbflood_task; 118 static struct task_struct *fqs_task; 119 static struct task_struct *boost_tasks[NR_CPUS]; 120 static struct task_struct *stall_task; 121 static struct task_struct **barrier_cbs_tasks; 122 static struct task_struct *barrier_task; 123 124 #define RCU_TORTURE_PIPE_LEN 10 125 126 struct rcu_torture { 127 struct rcu_head rtort_rcu; 128 int rtort_pipe_count; 129 struct list_head rtort_free; 130 int rtort_mbtest; 131 }; 132 133 static LIST_HEAD(rcu_torture_freelist); 134 static struct rcu_torture __rcu *rcu_torture_current; 135 static unsigned long rcu_torture_current_version; 136 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 137 static DEFINE_SPINLOCK(rcu_torture_lock); 138 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 139 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 140 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 141 static atomic_t n_rcu_torture_alloc; 142 static atomic_t n_rcu_torture_alloc_fail; 143 static atomic_t n_rcu_torture_free; 144 static atomic_t n_rcu_torture_mberror; 145 static atomic_t n_rcu_torture_error; 146 static long n_rcu_torture_barrier_error; 147 static long n_rcu_torture_boost_ktrerror; 148 static long n_rcu_torture_boost_rterror; 149 static long n_rcu_torture_boost_failure; 150 static long n_rcu_torture_boosts; 151 static long n_rcu_torture_timers; 152 static long n_barrier_attempts; 153 static long n_barrier_successes; 154 static atomic_long_t n_cbfloods; 155 static struct list_head rcu_torture_removed; 156 157 static int rcu_torture_writer_state; 158 #define RTWS_FIXED_DELAY 0 159 #define RTWS_DELAY 1 160 #define RTWS_REPLACE 2 161 #define RTWS_DEF_FREE 3 162 #define RTWS_EXP_SYNC 4 163 #define RTWS_COND_GET 5 164 #define RTWS_COND_SYNC 6 165 #define RTWS_SYNC 7 166 #define RTWS_STUTTER 8 167 #define RTWS_STOPPING 9 168 static const char * const rcu_torture_writer_state_names[] = { 169 "RTWS_FIXED_DELAY", 170 "RTWS_DELAY", 171 "RTWS_REPLACE", 172 "RTWS_DEF_FREE", 173 "RTWS_EXP_SYNC", 174 "RTWS_COND_GET", 175 "RTWS_COND_SYNC", 176 "RTWS_SYNC", 177 "RTWS_STUTTER", 178 "RTWS_STOPPING", 179 }; 180 181 static const char *rcu_torture_writer_state_getname(void) 182 { 183 unsigned int i = READ_ONCE(rcu_torture_writer_state); 184 185 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 186 return "???"; 187 return rcu_torture_writer_state_names[i]; 188 } 189 190 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 191 #define rcu_can_boost() 1 192 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 193 #define rcu_can_boost() 0 194 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 195 196 #ifdef CONFIG_RCU_TRACE 197 static u64 notrace rcu_trace_clock_local(void) 198 { 199 u64 ts = trace_clock_local(); 200 201 (void)do_div(ts, NSEC_PER_USEC); 202 return ts; 203 } 204 #else /* #ifdef CONFIG_RCU_TRACE */ 205 static u64 notrace rcu_trace_clock_local(void) 206 { 207 return 0ULL; 208 } 209 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 210 211 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 212 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 213 /* and boost task create/destroy. */ 214 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 215 static bool barrier_phase; /* Test phase. */ 216 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 217 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 218 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 219 220 /* 221 * Allocate an element from the rcu_tortures pool. 222 */ 223 static struct rcu_torture * 224 rcu_torture_alloc(void) 225 { 226 struct list_head *p; 227 228 spin_lock_bh(&rcu_torture_lock); 229 if (list_empty(&rcu_torture_freelist)) { 230 atomic_inc(&n_rcu_torture_alloc_fail); 231 spin_unlock_bh(&rcu_torture_lock); 232 return NULL; 233 } 234 atomic_inc(&n_rcu_torture_alloc); 235 p = rcu_torture_freelist.next; 236 list_del_init(p); 237 spin_unlock_bh(&rcu_torture_lock); 238 return container_of(p, struct rcu_torture, rtort_free); 239 } 240 241 /* 242 * Free an element to the rcu_tortures pool. 243 */ 244 static void 245 rcu_torture_free(struct rcu_torture *p) 246 { 247 atomic_inc(&n_rcu_torture_free); 248 spin_lock_bh(&rcu_torture_lock); 249 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 250 spin_unlock_bh(&rcu_torture_lock); 251 } 252 253 /* 254 * Operations vector for selecting different types of tests. 255 */ 256 257 struct rcu_torture_ops { 258 int ttype; 259 void (*init)(void); 260 void (*cleanup)(void); 261 int (*readlock)(void); 262 void (*read_delay)(struct torture_random_state *rrsp); 263 void (*readunlock)(int idx); 264 unsigned long (*started)(void); 265 unsigned long (*completed)(void); 266 void (*deferred_free)(struct rcu_torture *p); 267 void (*sync)(void); 268 void (*exp_sync)(void); 269 unsigned long (*get_state)(void); 270 void (*cond_sync)(unsigned long oldstate); 271 call_rcu_func_t call; 272 void (*cb_barrier)(void); 273 void (*fqs)(void); 274 void (*stats)(void); 275 int irq_capable; 276 int can_boost; 277 const char *name; 278 }; 279 280 static struct rcu_torture_ops *cur_ops; 281 282 /* 283 * Definitions for rcu torture testing. 284 */ 285 286 static int rcu_torture_read_lock(void) __acquires(RCU) 287 { 288 rcu_read_lock(); 289 return 0; 290 } 291 292 static void rcu_read_delay(struct torture_random_state *rrsp) 293 { 294 unsigned long started; 295 unsigned long completed; 296 const unsigned long shortdelay_us = 200; 297 const unsigned long longdelay_ms = 50; 298 unsigned long long ts; 299 300 /* We want a short delay sometimes to make a reader delay the grace 301 * period, and we want a long delay occasionally to trigger 302 * force_quiescent_state. */ 303 304 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 305 started = cur_ops->completed(); 306 ts = rcu_trace_clock_local(); 307 mdelay(longdelay_ms); 308 completed = cur_ops->completed(); 309 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 310 started, completed); 311 } 312 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 313 udelay(shortdelay_us); 314 if (!preempt_count() && 315 !(torture_random(rrsp) % (nrealreaders * 500))) 316 torture_preempt_schedule(); /* QS only if preemptible. */ 317 } 318 319 static void rcu_torture_read_unlock(int idx) __releases(RCU) 320 { 321 rcu_read_unlock(); 322 } 323 324 /* 325 * Update callback in the pipe. This should be invoked after a grace period. 326 */ 327 static bool 328 rcu_torture_pipe_update_one(struct rcu_torture *rp) 329 { 330 int i; 331 332 i = rp->rtort_pipe_count; 333 if (i > RCU_TORTURE_PIPE_LEN) 334 i = RCU_TORTURE_PIPE_LEN; 335 atomic_inc(&rcu_torture_wcount[i]); 336 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 337 rp->rtort_mbtest = 0; 338 return true; 339 } 340 return false; 341 } 342 343 /* 344 * Update all callbacks in the pipe. Suitable for synchronous grace-period 345 * primitives. 346 */ 347 static void 348 rcu_torture_pipe_update(struct rcu_torture *old_rp) 349 { 350 struct rcu_torture *rp; 351 struct rcu_torture *rp1; 352 353 if (old_rp) 354 list_add(&old_rp->rtort_free, &rcu_torture_removed); 355 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 356 if (rcu_torture_pipe_update_one(rp)) { 357 list_del(&rp->rtort_free); 358 rcu_torture_free(rp); 359 } 360 } 361 } 362 363 static void 364 rcu_torture_cb(struct rcu_head *p) 365 { 366 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 367 368 if (torture_must_stop_irq()) { 369 /* Test is ending, just drop callbacks on the floor. */ 370 /* The next initialization will pick up the pieces. */ 371 return; 372 } 373 if (rcu_torture_pipe_update_one(rp)) 374 rcu_torture_free(rp); 375 else 376 cur_ops->deferred_free(rp); 377 } 378 379 static unsigned long rcu_no_completed(void) 380 { 381 return 0; 382 } 383 384 static void rcu_torture_deferred_free(struct rcu_torture *p) 385 { 386 call_rcu(&p->rtort_rcu, rcu_torture_cb); 387 } 388 389 static void rcu_sync_torture_init(void) 390 { 391 INIT_LIST_HEAD(&rcu_torture_removed); 392 } 393 394 static struct rcu_torture_ops rcu_ops = { 395 .ttype = RCU_FLAVOR, 396 .init = rcu_sync_torture_init, 397 .readlock = rcu_torture_read_lock, 398 .read_delay = rcu_read_delay, 399 .readunlock = rcu_torture_read_unlock, 400 .started = rcu_batches_started, 401 .completed = rcu_batches_completed, 402 .deferred_free = rcu_torture_deferred_free, 403 .sync = synchronize_rcu, 404 .exp_sync = synchronize_rcu_expedited, 405 .get_state = get_state_synchronize_rcu, 406 .cond_sync = cond_synchronize_rcu, 407 .call = call_rcu, 408 .cb_barrier = rcu_barrier, 409 .fqs = rcu_force_quiescent_state, 410 .stats = NULL, 411 .irq_capable = 1, 412 .can_boost = rcu_can_boost(), 413 .name = "rcu" 414 }; 415 416 /* 417 * Definitions for rcu_bh torture testing. 418 */ 419 420 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 421 { 422 rcu_read_lock_bh(); 423 return 0; 424 } 425 426 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 427 { 428 rcu_read_unlock_bh(); 429 } 430 431 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 432 { 433 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 434 } 435 436 static struct rcu_torture_ops rcu_bh_ops = { 437 .ttype = RCU_BH_FLAVOR, 438 .init = rcu_sync_torture_init, 439 .readlock = rcu_bh_torture_read_lock, 440 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 441 .readunlock = rcu_bh_torture_read_unlock, 442 .started = rcu_batches_started_bh, 443 .completed = rcu_batches_completed_bh, 444 .deferred_free = rcu_bh_torture_deferred_free, 445 .sync = synchronize_rcu_bh, 446 .exp_sync = synchronize_rcu_bh_expedited, 447 .call = call_rcu_bh, 448 .cb_barrier = rcu_barrier_bh, 449 .fqs = rcu_bh_force_quiescent_state, 450 .stats = NULL, 451 .irq_capable = 1, 452 .name = "rcu_bh" 453 }; 454 455 /* 456 * Don't even think about trying any of these in real life!!! 457 * The names includes "busted", and they really means it! 458 * The only purpose of these functions is to provide a buggy RCU 459 * implementation to make sure that rcutorture correctly emits 460 * buggy-RCU error messages. 461 */ 462 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 463 { 464 /* This is a deliberate bug for testing purposes only! */ 465 rcu_torture_cb(&p->rtort_rcu); 466 } 467 468 static void synchronize_rcu_busted(void) 469 { 470 /* This is a deliberate bug for testing purposes only! */ 471 } 472 473 static void 474 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 475 { 476 /* This is a deliberate bug for testing purposes only! */ 477 func(head); 478 } 479 480 static struct rcu_torture_ops rcu_busted_ops = { 481 .ttype = INVALID_RCU_FLAVOR, 482 .init = rcu_sync_torture_init, 483 .readlock = rcu_torture_read_lock, 484 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 485 .readunlock = rcu_torture_read_unlock, 486 .started = rcu_no_completed, 487 .completed = rcu_no_completed, 488 .deferred_free = rcu_busted_torture_deferred_free, 489 .sync = synchronize_rcu_busted, 490 .exp_sync = synchronize_rcu_busted, 491 .call = call_rcu_busted, 492 .cb_barrier = NULL, 493 .fqs = NULL, 494 .stats = NULL, 495 .irq_capable = 1, 496 .name = "busted" 497 }; 498 499 /* 500 * Definitions for srcu torture testing. 501 */ 502 503 DEFINE_STATIC_SRCU(srcu_ctl); 504 static struct srcu_struct srcu_ctld; 505 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 506 507 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 508 { 509 return srcu_read_lock(srcu_ctlp); 510 } 511 512 static void srcu_read_delay(struct torture_random_state *rrsp) 513 { 514 long delay; 515 const long uspertick = 1000000 / HZ; 516 const long longdelay = 10; 517 518 /* We want there to be long-running readers, but not all the time. */ 519 520 delay = torture_random(rrsp) % 521 (nrealreaders * 2 * longdelay * uspertick); 522 if (!delay && in_task()) 523 schedule_timeout_interruptible(longdelay); 524 else 525 rcu_read_delay(rrsp); 526 } 527 528 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 529 { 530 srcu_read_unlock(srcu_ctlp, idx); 531 } 532 533 static unsigned long srcu_torture_completed(void) 534 { 535 return srcu_batches_completed(srcu_ctlp); 536 } 537 538 static void srcu_torture_deferred_free(struct rcu_torture *rp) 539 { 540 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 541 } 542 543 static void srcu_torture_synchronize(void) 544 { 545 synchronize_srcu(srcu_ctlp); 546 } 547 548 static void srcu_torture_call(struct rcu_head *head, 549 rcu_callback_t func) 550 { 551 call_srcu(srcu_ctlp, head, func); 552 } 553 554 static void srcu_torture_barrier(void) 555 { 556 srcu_barrier(srcu_ctlp); 557 } 558 559 static void srcu_torture_stats(void) 560 { 561 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 562 } 563 564 static void srcu_torture_synchronize_expedited(void) 565 { 566 synchronize_srcu_expedited(srcu_ctlp); 567 } 568 569 static struct rcu_torture_ops srcu_ops = { 570 .ttype = SRCU_FLAVOR, 571 .init = rcu_sync_torture_init, 572 .readlock = srcu_torture_read_lock, 573 .read_delay = srcu_read_delay, 574 .readunlock = srcu_torture_read_unlock, 575 .started = NULL, 576 .completed = srcu_torture_completed, 577 .deferred_free = srcu_torture_deferred_free, 578 .sync = srcu_torture_synchronize, 579 .exp_sync = srcu_torture_synchronize_expedited, 580 .call = srcu_torture_call, 581 .cb_barrier = srcu_torture_barrier, 582 .stats = srcu_torture_stats, 583 .irq_capable = 1, 584 .name = "srcu" 585 }; 586 587 static void srcu_torture_init(void) 588 { 589 rcu_sync_torture_init(); 590 WARN_ON(init_srcu_struct(&srcu_ctld)); 591 srcu_ctlp = &srcu_ctld; 592 } 593 594 static void srcu_torture_cleanup(void) 595 { 596 cleanup_srcu_struct(&srcu_ctld); 597 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 598 } 599 600 /* As above, but dynamically allocated. */ 601 static struct rcu_torture_ops srcud_ops = { 602 .ttype = SRCU_FLAVOR, 603 .init = srcu_torture_init, 604 .cleanup = srcu_torture_cleanup, 605 .readlock = srcu_torture_read_lock, 606 .read_delay = srcu_read_delay, 607 .readunlock = srcu_torture_read_unlock, 608 .started = NULL, 609 .completed = srcu_torture_completed, 610 .deferred_free = srcu_torture_deferred_free, 611 .sync = srcu_torture_synchronize, 612 .exp_sync = srcu_torture_synchronize_expedited, 613 .call = srcu_torture_call, 614 .cb_barrier = srcu_torture_barrier, 615 .stats = srcu_torture_stats, 616 .irq_capable = 1, 617 .name = "srcud" 618 }; 619 620 /* 621 * Definitions for sched torture testing. 622 */ 623 624 static int sched_torture_read_lock(void) 625 { 626 preempt_disable(); 627 return 0; 628 } 629 630 static void sched_torture_read_unlock(int idx) 631 { 632 preempt_enable(); 633 } 634 635 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 636 { 637 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 638 } 639 640 static struct rcu_torture_ops sched_ops = { 641 .ttype = RCU_SCHED_FLAVOR, 642 .init = rcu_sync_torture_init, 643 .readlock = sched_torture_read_lock, 644 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 645 .readunlock = sched_torture_read_unlock, 646 .started = rcu_batches_started_sched, 647 .completed = rcu_batches_completed_sched, 648 .deferred_free = rcu_sched_torture_deferred_free, 649 .sync = synchronize_sched, 650 .exp_sync = synchronize_sched_expedited, 651 .get_state = get_state_synchronize_sched, 652 .cond_sync = cond_synchronize_sched, 653 .call = call_rcu_sched, 654 .cb_barrier = rcu_barrier_sched, 655 .fqs = rcu_sched_force_quiescent_state, 656 .stats = NULL, 657 .irq_capable = 1, 658 .name = "sched" 659 }; 660 661 /* 662 * Definitions for RCU-tasks torture testing. 663 */ 664 665 static int tasks_torture_read_lock(void) 666 { 667 return 0; 668 } 669 670 static void tasks_torture_read_unlock(int idx) 671 { 672 } 673 674 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 675 { 676 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 677 } 678 679 static struct rcu_torture_ops tasks_ops = { 680 .ttype = RCU_TASKS_FLAVOR, 681 .init = rcu_sync_torture_init, 682 .readlock = tasks_torture_read_lock, 683 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 684 .readunlock = tasks_torture_read_unlock, 685 .started = rcu_no_completed, 686 .completed = rcu_no_completed, 687 .deferred_free = rcu_tasks_torture_deferred_free, 688 .sync = synchronize_rcu_tasks, 689 .exp_sync = synchronize_rcu_tasks, 690 .call = call_rcu_tasks, 691 .cb_barrier = rcu_barrier_tasks, 692 .fqs = NULL, 693 .stats = NULL, 694 .irq_capable = 1, 695 .name = "tasks" 696 }; 697 698 static bool __maybe_unused torturing_tasks(void) 699 { 700 return cur_ops == &tasks_ops; 701 } 702 703 /* 704 * RCU torture priority-boost testing. Runs one real-time thread per 705 * CPU for moderate bursts, repeatedly registering RCU callbacks and 706 * spinning waiting for them to be invoked. If a given callback takes 707 * too long to be invoked, we assume that priority inversion has occurred. 708 */ 709 710 struct rcu_boost_inflight { 711 struct rcu_head rcu; 712 int inflight; 713 }; 714 715 static void rcu_torture_boost_cb(struct rcu_head *head) 716 { 717 struct rcu_boost_inflight *rbip = 718 container_of(head, struct rcu_boost_inflight, rcu); 719 720 /* Ensure RCU-core accesses precede clearing ->inflight */ 721 smp_store_release(&rbip->inflight, 0); 722 } 723 724 static int rcu_torture_boost(void *arg) 725 { 726 unsigned long call_rcu_time; 727 unsigned long endtime; 728 unsigned long oldstarttime; 729 struct rcu_boost_inflight rbi = { .inflight = 0 }; 730 struct sched_param sp; 731 732 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 733 734 /* Set real-time priority. */ 735 sp.sched_priority = 1; 736 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 737 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 738 n_rcu_torture_boost_rterror++; 739 } 740 741 init_rcu_head_on_stack(&rbi.rcu); 742 /* Each pass through the following loop does one boost-test cycle. */ 743 do { 744 /* Wait for the next test interval. */ 745 oldstarttime = boost_starttime; 746 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 747 schedule_timeout_interruptible(oldstarttime - jiffies); 748 stutter_wait("rcu_torture_boost"); 749 if (torture_must_stop()) 750 goto checkwait; 751 } 752 753 /* Do one boost-test interval. */ 754 endtime = oldstarttime + test_boost_duration * HZ; 755 call_rcu_time = jiffies; 756 while (ULONG_CMP_LT(jiffies, endtime)) { 757 /* If we don't have a callback in flight, post one. */ 758 if (!smp_load_acquire(&rbi.inflight)) { 759 /* RCU core before ->inflight = 1. */ 760 smp_store_release(&rbi.inflight, 1); 761 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 762 if (jiffies - call_rcu_time > 763 test_boost_duration * HZ - HZ / 2) { 764 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 765 n_rcu_torture_boost_failure++; 766 } 767 call_rcu_time = jiffies; 768 } 769 stutter_wait("rcu_torture_boost"); 770 if (torture_must_stop()) 771 goto checkwait; 772 } 773 774 /* 775 * Set the start time of the next test interval. 776 * Yes, this is vulnerable to long delays, but such 777 * delays simply cause a false negative for the next 778 * interval. Besides, we are running at RT priority, 779 * so delays should be relatively rare. 780 */ 781 while (oldstarttime == boost_starttime && 782 !kthread_should_stop()) { 783 if (mutex_trylock(&boost_mutex)) { 784 boost_starttime = jiffies + 785 test_boost_interval * HZ; 786 n_rcu_torture_boosts++; 787 mutex_unlock(&boost_mutex); 788 break; 789 } 790 schedule_timeout_uninterruptible(1); 791 } 792 793 /* Go do the stutter. */ 794 checkwait: stutter_wait("rcu_torture_boost"); 795 } while (!torture_must_stop()); 796 797 /* Clean up and exit. */ 798 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 799 torture_shutdown_absorb("rcu_torture_boost"); 800 schedule_timeout_uninterruptible(1); 801 } 802 destroy_rcu_head_on_stack(&rbi.rcu); 803 torture_kthread_stopping("rcu_torture_boost"); 804 return 0; 805 } 806 807 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 808 { 809 } 810 811 /* 812 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 813 * to call_rcu() or analogous, increasing the probability of occurrence 814 * of callback-overflow corner cases. 815 */ 816 static int 817 rcu_torture_cbflood(void *arg) 818 { 819 int err = 1; 820 int i; 821 int j; 822 struct rcu_head *rhp; 823 824 if (cbflood_n_per_burst > 0 && 825 cbflood_inter_holdoff > 0 && 826 cbflood_intra_holdoff > 0 && 827 cur_ops->call && 828 cur_ops->cb_barrier) { 829 rhp = vmalloc(sizeof(*rhp) * 830 cbflood_n_burst * cbflood_n_per_burst); 831 err = !rhp; 832 } 833 if (err) { 834 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 835 goto wait_for_stop; 836 } 837 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 838 do { 839 schedule_timeout_interruptible(cbflood_inter_holdoff); 840 atomic_long_inc(&n_cbfloods); 841 WARN_ON(signal_pending(current)); 842 for (i = 0; i < cbflood_n_burst; i++) { 843 for (j = 0; j < cbflood_n_per_burst; j++) { 844 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 845 rcu_torture_cbflood_cb); 846 } 847 schedule_timeout_interruptible(cbflood_intra_holdoff); 848 WARN_ON(signal_pending(current)); 849 } 850 cur_ops->cb_barrier(); 851 stutter_wait("rcu_torture_cbflood"); 852 } while (!torture_must_stop()); 853 vfree(rhp); 854 wait_for_stop: 855 torture_kthread_stopping("rcu_torture_cbflood"); 856 return 0; 857 } 858 859 /* 860 * RCU torture force-quiescent-state kthread. Repeatedly induces 861 * bursts of calls to force_quiescent_state(), increasing the probability 862 * of occurrence of some important types of race conditions. 863 */ 864 static int 865 rcu_torture_fqs(void *arg) 866 { 867 unsigned long fqs_resume_time; 868 int fqs_burst_remaining; 869 870 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 871 do { 872 fqs_resume_time = jiffies + fqs_stutter * HZ; 873 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 874 !kthread_should_stop()) { 875 schedule_timeout_interruptible(1); 876 } 877 fqs_burst_remaining = fqs_duration; 878 while (fqs_burst_remaining > 0 && 879 !kthread_should_stop()) { 880 cur_ops->fqs(); 881 udelay(fqs_holdoff); 882 fqs_burst_remaining -= fqs_holdoff; 883 } 884 stutter_wait("rcu_torture_fqs"); 885 } while (!torture_must_stop()); 886 torture_kthread_stopping("rcu_torture_fqs"); 887 return 0; 888 } 889 890 /* 891 * RCU torture writer kthread. Repeatedly substitutes a new structure 892 * for that pointed to by rcu_torture_current, freeing the old structure 893 * after a series of grace periods (the "pipeline"). 894 */ 895 static int 896 rcu_torture_writer(void *arg) 897 { 898 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 899 int expediting = 0; 900 unsigned long gp_snap; 901 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 902 bool gp_sync1 = gp_sync; 903 int i; 904 struct rcu_torture *rp; 905 struct rcu_torture *old_rp; 906 static DEFINE_TORTURE_RANDOM(rand); 907 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 908 RTWS_COND_GET, RTWS_SYNC }; 909 int nsynctypes = 0; 910 911 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 912 if (!can_expedite) { 913 pr_alert("%s" TORTURE_FLAG 914 " GP expediting controlled from boot/sysfs for %s,\n", 915 torture_type, cur_ops->name); 916 pr_alert("%s" TORTURE_FLAG 917 " Disabled dynamic grace-period expediting.\n", 918 torture_type); 919 } 920 921 /* Initialize synctype[] array. If none set, take default. */ 922 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 923 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 924 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) 925 synctype[nsynctypes++] = RTWS_COND_GET; 926 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) 927 pr_alert("rcu_torture_writer: gp_cond without primitives.\n"); 928 if (gp_exp1 && cur_ops->exp_sync) 929 synctype[nsynctypes++] = RTWS_EXP_SYNC; 930 else if (gp_exp && !cur_ops->exp_sync) 931 pr_alert("rcu_torture_writer: gp_exp without primitives.\n"); 932 if (gp_normal1 && cur_ops->deferred_free) 933 synctype[nsynctypes++] = RTWS_DEF_FREE; 934 else if (gp_normal && !cur_ops->deferred_free) 935 pr_alert("rcu_torture_writer: gp_normal without primitives.\n"); 936 if (gp_sync1 && cur_ops->sync) 937 synctype[nsynctypes++] = RTWS_SYNC; 938 else if (gp_sync && !cur_ops->sync) 939 pr_alert("rcu_torture_writer: gp_sync without primitives.\n"); 940 if (WARN_ONCE(nsynctypes == 0, 941 "rcu_torture_writer: No update-side primitives.\n")) { 942 /* 943 * No updates primitives, so don't try updating. 944 * The resulting test won't be testing much, hence the 945 * above WARN_ONCE(). 946 */ 947 rcu_torture_writer_state = RTWS_STOPPING; 948 torture_kthread_stopping("rcu_torture_writer"); 949 } 950 951 do { 952 rcu_torture_writer_state = RTWS_FIXED_DELAY; 953 schedule_timeout_uninterruptible(1); 954 rp = rcu_torture_alloc(); 955 if (rp == NULL) 956 continue; 957 rp->rtort_pipe_count = 0; 958 rcu_torture_writer_state = RTWS_DELAY; 959 udelay(torture_random(&rand) & 0x3ff); 960 rcu_torture_writer_state = RTWS_REPLACE; 961 old_rp = rcu_dereference_check(rcu_torture_current, 962 current == writer_task); 963 rp->rtort_mbtest = 1; 964 rcu_assign_pointer(rcu_torture_current, rp); 965 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 966 if (old_rp) { 967 i = old_rp->rtort_pipe_count; 968 if (i > RCU_TORTURE_PIPE_LEN) 969 i = RCU_TORTURE_PIPE_LEN; 970 atomic_inc(&rcu_torture_wcount[i]); 971 old_rp->rtort_pipe_count++; 972 switch (synctype[torture_random(&rand) % nsynctypes]) { 973 case RTWS_DEF_FREE: 974 rcu_torture_writer_state = RTWS_DEF_FREE; 975 cur_ops->deferred_free(old_rp); 976 break; 977 case RTWS_EXP_SYNC: 978 rcu_torture_writer_state = RTWS_EXP_SYNC; 979 cur_ops->exp_sync(); 980 rcu_torture_pipe_update(old_rp); 981 break; 982 case RTWS_COND_GET: 983 rcu_torture_writer_state = RTWS_COND_GET; 984 gp_snap = cur_ops->get_state(); 985 i = torture_random(&rand) % 16; 986 if (i != 0) 987 schedule_timeout_interruptible(i); 988 udelay(torture_random(&rand) % 1000); 989 rcu_torture_writer_state = RTWS_COND_SYNC; 990 cur_ops->cond_sync(gp_snap); 991 rcu_torture_pipe_update(old_rp); 992 break; 993 case RTWS_SYNC: 994 rcu_torture_writer_state = RTWS_SYNC; 995 cur_ops->sync(); 996 rcu_torture_pipe_update(old_rp); 997 break; 998 default: 999 WARN_ON_ONCE(1); 1000 break; 1001 } 1002 } 1003 rcutorture_record_progress(++rcu_torture_current_version); 1004 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1005 if (can_expedite && 1006 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1007 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1008 if (expediting >= 0) 1009 rcu_expedite_gp(); 1010 else 1011 rcu_unexpedite_gp(); 1012 if (++expediting > 3) 1013 expediting = -expediting; 1014 } 1015 rcu_torture_writer_state = RTWS_STUTTER; 1016 stutter_wait("rcu_torture_writer"); 1017 } while (!torture_must_stop()); 1018 /* Reset expediting back to unexpedited. */ 1019 if (expediting > 0) 1020 expediting = -expediting; 1021 while (can_expedite && expediting++ < 0) 1022 rcu_unexpedite_gp(); 1023 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1024 rcu_torture_writer_state = RTWS_STOPPING; 1025 torture_kthread_stopping("rcu_torture_writer"); 1026 return 0; 1027 } 1028 1029 /* 1030 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1031 * delay between calls. 1032 */ 1033 static int 1034 rcu_torture_fakewriter(void *arg) 1035 { 1036 DEFINE_TORTURE_RANDOM(rand); 1037 1038 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1039 set_user_nice(current, MAX_NICE); 1040 1041 do { 1042 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1043 udelay(torture_random(&rand) & 0x3ff); 1044 if (cur_ops->cb_barrier != NULL && 1045 torture_random(&rand) % (nfakewriters * 8) == 0) { 1046 cur_ops->cb_barrier(); 1047 } else if (gp_normal == gp_exp) { 1048 if (torture_random(&rand) & 0x80) 1049 cur_ops->sync(); 1050 else 1051 cur_ops->exp_sync(); 1052 } else if (gp_normal) { 1053 cur_ops->sync(); 1054 } else { 1055 cur_ops->exp_sync(); 1056 } 1057 stutter_wait("rcu_torture_fakewriter"); 1058 } while (!torture_must_stop()); 1059 1060 torture_kthread_stopping("rcu_torture_fakewriter"); 1061 return 0; 1062 } 1063 1064 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1065 { 1066 kfree(rhp); 1067 } 1068 1069 /* 1070 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1071 * incrementing the corresponding element of the pipeline array. The 1072 * counter in the element should never be greater than 1, otherwise, the 1073 * RCU implementation is broken. 1074 */ 1075 static void rcu_torture_timer(struct timer_list *unused) 1076 { 1077 int idx; 1078 unsigned long started; 1079 unsigned long completed; 1080 static DEFINE_TORTURE_RANDOM(rand); 1081 static DEFINE_SPINLOCK(rand_lock); 1082 struct rcu_torture *p; 1083 int pipe_count; 1084 unsigned long long ts; 1085 1086 idx = cur_ops->readlock(); 1087 if (cur_ops->started) 1088 started = cur_ops->started(); 1089 else 1090 started = cur_ops->completed(); 1091 ts = rcu_trace_clock_local(); 1092 p = rcu_dereference_check(rcu_torture_current, 1093 rcu_read_lock_bh_held() || 1094 rcu_read_lock_sched_held() || 1095 srcu_read_lock_held(srcu_ctlp) || 1096 torturing_tasks()); 1097 if (p == NULL) { 1098 /* Leave because rcu_torture_writer is not yet underway */ 1099 cur_ops->readunlock(idx); 1100 return; 1101 } 1102 if (p->rtort_mbtest == 0) 1103 atomic_inc(&n_rcu_torture_mberror); 1104 spin_lock(&rand_lock); 1105 cur_ops->read_delay(&rand); 1106 n_rcu_torture_timers++; 1107 spin_unlock(&rand_lock); 1108 preempt_disable(); 1109 pipe_count = p->rtort_pipe_count; 1110 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1111 /* Should not happen, but... */ 1112 pipe_count = RCU_TORTURE_PIPE_LEN; 1113 } 1114 completed = cur_ops->completed(); 1115 if (pipe_count > 1) { 1116 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1117 started, completed); 1118 rcu_ftrace_dump(DUMP_ALL); 1119 } 1120 __this_cpu_inc(rcu_torture_count[pipe_count]); 1121 completed = completed - started; 1122 if (cur_ops->started) 1123 completed++; 1124 if (completed > RCU_TORTURE_PIPE_LEN) { 1125 /* Should not happen, but... */ 1126 completed = RCU_TORTURE_PIPE_LEN; 1127 } 1128 __this_cpu_inc(rcu_torture_batch[completed]); 1129 preempt_enable(); 1130 cur_ops->readunlock(idx); 1131 1132 /* Test call_rcu() invocation from interrupt handler. */ 1133 if (cur_ops->call) { 1134 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 1135 1136 if (rhp) 1137 cur_ops->call(rhp, rcu_torture_timer_cb); 1138 } 1139 } 1140 1141 /* 1142 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1143 * incrementing the corresponding element of the pipeline array. The 1144 * counter in the element should never be greater than 1, otherwise, the 1145 * RCU implementation is broken. 1146 */ 1147 static int 1148 rcu_torture_reader(void *arg) 1149 { 1150 unsigned long started; 1151 unsigned long completed; 1152 int idx; 1153 DEFINE_TORTURE_RANDOM(rand); 1154 struct rcu_torture *p; 1155 int pipe_count; 1156 struct timer_list t; 1157 unsigned long long ts; 1158 1159 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1160 set_user_nice(current, MAX_NICE); 1161 if (irqreader && cur_ops->irq_capable) 1162 timer_setup_on_stack(&t, rcu_torture_timer, 0); 1163 1164 do { 1165 if (irqreader && cur_ops->irq_capable) { 1166 if (!timer_pending(&t)) 1167 mod_timer(&t, jiffies + 1); 1168 } 1169 idx = cur_ops->readlock(); 1170 if (cur_ops->started) 1171 started = cur_ops->started(); 1172 else 1173 started = cur_ops->completed(); 1174 ts = rcu_trace_clock_local(); 1175 p = rcu_dereference_check(rcu_torture_current, 1176 rcu_read_lock_bh_held() || 1177 rcu_read_lock_sched_held() || 1178 srcu_read_lock_held(srcu_ctlp) || 1179 torturing_tasks()); 1180 if (p == NULL) { 1181 /* Wait for rcu_torture_writer to get underway */ 1182 cur_ops->readunlock(idx); 1183 schedule_timeout_interruptible(HZ); 1184 continue; 1185 } 1186 if (p->rtort_mbtest == 0) 1187 atomic_inc(&n_rcu_torture_mberror); 1188 cur_ops->read_delay(&rand); 1189 preempt_disable(); 1190 pipe_count = p->rtort_pipe_count; 1191 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1192 /* Should not happen, but... */ 1193 pipe_count = RCU_TORTURE_PIPE_LEN; 1194 } 1195 completed = cur_ops->completed(); 1196 if (pipe_count > 1) { 1197 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1198 ts, started, completed); 1199 rcu_ftrace_dump(DUMP_ALL); 1200 } 1201 __this_cpu_inc(rcu_torture_count[pipe_count]); 1202 completed = completed - started; 1203 if (cur_ops->started) 1204 completed++; 1205 if (completed > RCU_TORTURE_PIPE_LEN) { 1206 /* Should not happen, but... */ 1207 completed = RCU_TORTURE_PIPE_LEN; 1208 } 1209 __this_cpu_inc(rcu_torture_batch[completed]); 1210 preempt_enable(); 1211 cur_ops->readunlock(idx); 1212 stutter_wait("rcu_torture_reader"); 1213 } while (!torture_must_stop()); 1214 if (irqreader && cur_ops->irq_capable) { 1215 del_timer_sync(&t); 1216 destroy_timer_on_stack(&t); 1217 } 1218 torture_kthread_stopping("rcu_torture_reader"); 1219 return 0; 1220 } 1221 1222 /* 1223 * Print torture statistics. Caller must ensure that there is only 1224 * one call to this function at a given time!!! This is normally 1225 * accomplished by relying on the module system to only have one copy 1226 * of the module loaded, and then by giving the rcu_torture_stats 1227 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1228 * thread is not running). 1229 */ 1230 static void 1231 rcu_torture_stats_print(void) 1232 { 1233 int cpu; 1234 int i; 1235 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1236 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1237 static unsigned long rtcv_snap = ULONG_MAX; 1238 static bool splatted; 1239 struct task_struct *wtp; 1240 1241 for_each_possible_cpu(cpu) { 1242 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1243 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1244 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1245 } 1246 } 1247 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1248 if (pipesummary[i] != 0) 1249 break; 1250 } 1251 1252 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1253 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1254 rcu_torture_current, 1255 rcu_torture_current_version, 1256 list_empty(&rcu_torture_freelist), 1257 atomic_read(&n_rcu_torture_alloc), 1258 atomic_read(&n_rcu_torture_alloc_fail), 1259 atomic_read(&n_rcu_torture_free)); 1260 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ", 1261 atomic_read(&n_rcu_torture_mberror), 1262 n_rcu_torture_barrier_error, 1263 n_rcu_torture_boost_ktrerror, 1264 n_rcu_torture_boost_rterror); 1265 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1266 n_rcu_torture_boost_failure, 1267 n_rcu_torture_boosts, 1268 n_rcu_torture_timers); 1269 torture_onoff_stats(); 1270 pr_cont("barrier: %ld/%ld:%ld ", 1271 n_barrier_successes, 1272 n_barrier_attempts, 1273 n_rcu_torture_barrier_error); 1274 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1275 1276 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1277 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1278 n_rcu_torture_barrier_error != 0 || 1279 n_rcu_torture_boost_ktrerror != 0 || 1280 n_rcu_torture_boost_rterror != 0 || 1281 n_rcu_torture_boost_failure != 0 || 1282 i > 1) { 1283 pr_cont("%s", "!!! "); 1284 atomic_inc(&n_rcu_torture_error); 1285 WARN_ON_ONCE(1); 1286 } 1287 pr_cont("Reader Pipe: "); 1288 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1289 pr_cont(" %ld", pipesummary[i]); 1290 pr_cont("\n"); 1291 1292 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1293 pr_cont("Reader Batch: "); 1294 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1295 pr_cont(" %ld", batchsummary[i]); 1296 pr_cont("\n"); 1297 1298 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1299 pr_cont("Free-Block Circulation: "); 1300 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1301 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1302 } 1303 pr_cont("\n"); 1304 1305 if (cur_ops->stats) 1306 cur_ops->stats(); 1307 if (rtcv_snap == rcu_torture_current_version && 1308 rcu_torture_current != NULL) { 1309 int __maybe_unused flags = 0; 1310 unsigned long __maybe_unused gpnum = 0; 1311 unsigned long __maybe_unused completed = 0; 1312 1313 rcutorture_get_gp_data(cur_ops->ttype, 1314 &flags, &gpnum, &completed); 1315 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, 1316 &flags, &gpnum, &completed); 1317 wtp = READ_ONCE(writer_task); 1318 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx cpu %d\n", 1319 rcu_torture_writer_state_getname(), 1320 rcu_torture_writer_state, 1321 gpnum, completed, flags, 1322 wtp == NULL ? ~0UL : wtp->state, 1323 wtp == NULL ? -1 : (int)task_cpu(wtp)); 1324 if (!splatted && wtp) { 1325 sched_show_task(wtp); 1326 splatted = true; 1327 } 1328 show_rcu_gp_kthreads(); 1329 rcu_ftrace_dump(DUMP_ALL); 1330 } 1331 rtcv_snap = rcu_torture_current_version; 1332 } 1333 1334 /* 1335 * Periodically prints torture statistics, if periodic statistics printing 1336 * was specified via the stat_interval module parameter. 1337 */ 1338 static int 1339 rcu_torture_stats(void *arg) 1340 { 1341 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1342 do { 1343 schedule_timeout_interruptible(stat_interval * HZ); 1344 rcu_torture_stats_print(); 1345 torture_shutdown_absorb("rcu_torture_stats"); 1346 } while (!torture_must_stop()); 1347 torture_kthread_stopping("rcu_torture_stats"); 1348 return 0; 1349 } 1350 1351 static inline void 1352 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1353 { 1354 pr_alert("%s" TORTURE_FLAG 1355 "--- %s: nreaders=%d nfakewriters=%d " 1356 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1357 "shuffle_interval=%d stutter=%d irqreader=%d " 1358 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1359 "test_boost=%d/%d test_boost_interval=%d " 1360 "test_boost_duration=%d shutdown_secs=%d " 1361 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 1362 "n_barrier_cbs=%d " 1363 "onoff_interval=%d onoff_holdoff=%d\n", 1364 torture_type, tag, nrealreaders, nfakewriters, 1365 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1366 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1367 test_boost, cur_ops->can_boost, 1368 test_boost_interval, test_boost_duration, shutdown_secs, 1369 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 1370 n_barrier_cbs, 1371 onoff_interval, onoff_holdoff); 1372 } 1373 1374 static int rcutorture_booster_cleanup(unsigned int cpu) 1375 { 1376 struct task_struct *t; 1377 1378 if (boost_tasks[cpu] == NULL) 1379 return 0; 1380 mutex_lock(&boost_mutex); 1381 t = boost_tasks[cpu]; 1382 boost_tasks[cpu] = NULL; 1383 mutex_unlock(&boost_mutex); 1384 1385 /* This must be outside of the mutex, otherwise deadlock! */ 1386 torture_stop_kthread(rcu_torture_boost, t); 1387 return 0; 1388 } 1389 1390 static int rcutorture_booster_init(unsigned int cpu) 1391 { 1392 int retval; 1393 1394 if (boost_tasks[cpu] != NULL) 1395 return 0; /* Already created, nothing more to do. */ 1396 1397 /* Don't allow time recalculation while creating a new task. */ 1398 mutex_lock(&boost_mutex); 1399 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1400 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1401 cpu_to_node(cpu), 1402 "rcu_torture_boost"); 1403 if (IS_ERR(boost_tasks[cpu])) { 1404 retval = PTR_ERR(boost_tasks[cpu]); 1405 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1406 n_rcu_torture_boost_ktrerror++; 1407 boost_tasks[cpu] = NULL; 1408 mutex_unlock(&boost_mutex); 1409 return retval; 1410 } 1411 kthread_bind(boost_tasks[cpu], cpu); 1412 wake_up_process(boost_tasks[cpu]); 1413 mutex_unlock(&boost_mutex); 1414 return 0; 1415 } 1416 1417 /* 1418 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1419 * induces a CPU stall for the time specified by stall_cpu. 1420 */ 1421 static int rcu_torture_stall(void *args) 1422 { 1423 unsigned long stop_at; 1424 1425 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1426 if (stall_cpu_holdoff > 0) { 1427 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1428 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1429 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1430 } 1431 if (!kthread_should_stop()) { 1432 stop_at = get_seconds() + stall_cpu; 1433 /* RCU CPU stall is expected behavior in following code. */ 1434 rcu_read_lock(); 1435 if (stall_cpu_irqsoff) 1436 local_irq_disable(); 1437 else 1438 preempt_disable(); 1439 pr_alert("rcu_torture_stall start on CPU %d.\n", 1440 smp_processor_id()); 1441 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1442 continue; /* Induce RCU CPU stall warning. */ 1443 if (stall_cpu_irqsoff) 1444 local_irq_enable(); 1445 else 1446 preempt_enable(); 1447 rcu_read_unlock(); 1448 pr_alert("rcu_torture_stall end.\n"); 1449 } 1450 torture_shutdown_absorb("rcu_torture_stall"); 1451 while (!kthread_should_stop()) 1452 schedule_timeout_interruptible(10 * HZ); 1453 return 0; 1454 } 1455 1456 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1457 static int __init rcu_torture_stall_init(void) 1458 { 1459 if (stall_cpu <= 0) 1460 return 0; 1461 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1462 } 1463 1464 /* Callback function for RCU barrier testing. */ 1465 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1466 { 1467 atomic_inc(&barrier_cbs_invoked); 1468 } 1469 1470 /* kthread function to register callbacks used to test RCU barriers. */ 1471 static int rcu_torture_barrier_cbs(void *arg) 1472 { 1473 long myid = (long)arg; 1474 bool lastphase = 0; 1475 bool newphase; 1476 struct rcu_head rcu; 1477 1478 init_rcu_head_on_stack(&rcu); 1479 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1480 set_user_nice(current, MAX_NICE); 1481 do { 1482 wait_event(barrier_cbs_wq[myid], 1483 (newphase = 1484 smp_load_acquire(&barrier_phase)) != lastphase || 1485 torture_must_stop()); 1486 lastphase = newphase; 1487 if (torture_must_stop()) 1488 break; 1489 /* 1490 * The above smp_load_acquire() ensures barrier_phase load 1491 * is ordered before the following ->call(). 1492 */ 1493 local_irq_disable(); /* Just to test no-irq call_rcu(). */ 1494 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1495 local_irq_enable(); 1496 if (atomic_dec_and_test(&barrier_cbs_count)) 1497 wake_up(&barrier_wq); 1498 } while (!torture_must_stop()); 1499 if (cur_ops->cb_barrier != NULL) 1500 cur_ops->cb_barrier(); 1501 destroy_rcu_head_on_stack(&rcu); 1502 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1503 return 0; 1504 } 1505 1506 /* kthread function to drive and coordinate RCU barrier testing. */ 1507 static int rcu_torture_barrier(void *arg) 1508 { 1509 int i; 1510 1511 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1512 do { 1513 atomic_set(&barrier_cbs_invoked, 0); 1514 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1515 /* Ensure barrier_phase ordered after prior assignments. */ 1516 smp_store_release(&barrier_phase, !barrier_phase); 1517 for (i = 0; i < n_barrier_cbs; i++) 1518 wake_up(&barrier_cbs_wq[i]); 1519 wait_event(barrier_wq, 1520 atomic_read(&barrier_cbs_count) == 0 || 1521 torture_must_stop()); 1522 if (torture_must_stop()) 1523 break; 1524 n_barrier_attempts++; 1525 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1526 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1527 n_rcu_torture_barrier_error++; 1528 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 1529 atomic_read(&barrier_cbs_invoked), 1530 n_barrier_cbs); 1531 WARN_ON_ONCE(1); 1532 } 1533 n_barrier_successes++; 1534 schedule_timeout_interruptible(HZ / 10); 1535 } while (!torture_must_stop()); 1536 torture_kthread_stopping("rcu_torture_barrier"); 1537 return 0; 1538 } 1539 1540 /* Initialize RCU barrier testing. */ 1541 static int rcu_torture_barrier_init(void) 1542 { 1543 int i; 1544 int ret; 1545 1546 if (n_barrier_cbs <= 0) 1547 return 0; 1548 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1549 pr_alert("%s" TORTURE_FLAG 1550 " Call or barrier ops missing for %s,\n", 1551 torture_type, cur_ops->name); 1552 pr_alert("%s" TORTURE_FLAG 1553 " RCU barrier testing omitted from run.\n", 1554 torture_type); 1555 return 0; 1556 } 1557 atomic_set(&barrier_cbs_count, 0); 1558 atomic_set(&barrier_cbs_invoked, 0); 1559 barrier_cbs_tasks = 1560 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1561 GFP_KERNEL); 1562 barrier_cbs_wq = 1563 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1564 GFP_KERNEL); 1565 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1566 return -ENOMEM; 1567 for (i = 0; i < n_barrier_cbs; i++) { 1568 init_waitqueue_head(&barrier_cbs_wq[i]); 1569 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1570 (void *)(long)i, 1571 barrier_cbs_tasks[i]); 1572 if (ret) 1573 return ret; 1574 } 1575 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1576 } 1577 1578 /* Clean up after RCU barrier testing. */ 1579 static void rcu_torture_barrier_cleanup(void) 1580 { 1581 int i; 1582 1583 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1584 if (barrier_cbs_tasks != NULL) { 1585 for (i = 0; i < n_barrier_cbs; i++) 1586 torture_stop_kthread(rcu_torture_barrier_cbs, 1587 barrier_cbs_tasks[i]); 1588 kfree(barrier_cbs_tasks); 1589 barrier_cbs_tasks = NULL; 1590 } 1591 if (barrier_cbs_wq != NULL) { 1592 kfree(barrier_cbs_wq); 1593 barrier_cbs_wq = NULL; 1594 } 1595 } 1596 1597 static enum cpuhp_state rcutor_hp; 1598 1599 static void 1600 rcu_torture_cleanup(void) 1601 { 1602 int i; 1603 1604 rcutorture_record_test_transition(); 1605 if (torture_cleanup_begin()) { 1606 if (cur_ops->cb_barrier != NULL) 1607 cur_ops->cb_barrier(); 1608 return; 1609 } 1610 1611 rcu_torture_barrier_cleanup(); 1612 torture_stop_kthread(rcu_torture_stall, stall_task); 1613 torture_stop_kthread(rcu_torture_writer, writer_task); 1614 1615 if (reader_tasks) { 1616 for (i = 0; i < nrealreaders; i++) 1617 torture_stop_kthread(rcu_torture_reader, 1618 reader_tasks[i]); 1619 kfree(reader_tasks); 1620 } 1621 rcu_torture_current = NULL; 1622 1623 if (fakewriter_tasks) { 1624 for (i = 0; i < nfakewriters; i++) { 1625 torture_stop_kthread(rcu_torture_fakewriter, 1626 fakewriter_tasks[i]); 1627 } 1628 kfree(fakewriter_tasks); 1629 fakewriter_tasks = NULL; 1630 } 1631 1632 torture_stop_kthread(rcu_torture_stats, stats_task); 1633 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1634 for (i = 0; i < ncbflooders; i++) 1635 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1636 if ((test_boost == 1 && cur_ops->can_boost) || 1637 test_boost == 2) 1638 cpuhp_remove_state(rcutor_hp); 1639 1640 /* 1641 * Wait for all RCU callbacks to fire, then do flavor-specific 1642 * cleanup operations. 1643 */ 1644 if (cur_ops->cb_barrier != NULL) 1645 cur_ops->cb_barrier(); 1646 if (cur_ops->cleanup != NULL) 1647 cur_ops->cleanup(); 1648 1649 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1650 1651 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1652 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1653 else if (torture_onoff_failures()) 1654 rcu_torture_print_module_parms(cur_ops, 1655 "End of test: RCU_HOTPLUG"); 1656 else 1657 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1658 torture_cleanup_end(); 1659 } 1660 1661 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1662 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1663 { 1664 } 1665 1666 static void rcu_torture_err_cb(struct rcu_head *rhp) 1667 { 1668 /* 1669 * This -might- happen due to race conditions, but is unlikely. 1670 * The scenario that leads to this happening is that the 1671 * first of the pair of duplicate callbacks is queued, 1672 * someone else starts a grace period that includes that 1673 * callback, then the second of the pair must wait for the 1674 * next grace period. Unlikely, but can happen. If it 1675 * does happen, the debug-objects subsystem won't have splatted. 1676 */ 1677 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1678 } 1679 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1680 1681 /* 1682 * Verify that double-free causes debug-objects to complain, but only 1683 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1684 * cannot be carried out. 1685 */ 1686 static void rcu_test_debug_objects(void) 1687 { 1688 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1689 struct rcu_head rh1; 1690 struct rcu_head rh2; 1691 1692 init_rcu_head_on_stack(&rh1); 1693 init_rcu_head_on_stack(&rh2); 1694 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1695 1696 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1697 preempt_disable(); /* Prevent preemption from interrupting test. */ 1698 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1699 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1700 local_irq_disable(); /* Make it harder to start a new grace period. */ 1701 call_rcu(&rh2, rcu_torture_leak_cb); 1702 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1703 local_irq_enable(); 1704 rcu_read_unlock(); 1705 preempt_enable(); 1706 1707 /* Wait for them all to get done so we can safely return. */ 1708 rcu_barrier(); 1709 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1710 destroy_rcu_head_on_stack(&rh1); 1711 destroy_rcu_head_on_stack(&rh2); 1712 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1713 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1714 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1715 } 1716 1717 static int __init 1718 rcu_torture_init(void) 1719 { 1720 int i; 1721 int cpu; 1722 int firsterr = 0; 1723 static struct rcu_torture_ops *torture_ops[] = { 1724 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 1725 &sched_ops, &tasks_ops, 1726 }; 1727 1728 if (!torture_init_begin(torture_type, verbose)) 1729 return -EBUSY; 1730 1731 /* Process args and tell the world that the torturer is on the job. */ 1732 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1733 cur_ops = torture_ops[i]; 1734 if (strcmp(torture_type, cur_ops->name) == 0) 1735 break; 1736 } 1737 if (i == ARRAY_SIZE(torture_ops)) { 1738 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1739 torture_type); 1740 pr_alert("rcu-torture types:"); 1741 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1742 pr_alert(" %s", torture_ops[i]->name); 1743 pr_alert("\n"); 1744 firsterr = -EINVAL; 1745 goto unwind; 1746 } 1747 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1748 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1749 fqs_duration = 0; 1750 } 1751 if (cur_ops->init) 1752 cur_ops->init(); 1753 1754 if (nreaders >= 0) { 1755 nrealreaders = nreaders; 1756 } else { 1757 nrealreaders = num_online_cpus() - 2 - nreaders; 1758 if (nrealreaders <= 0) 1759 nrealreaders = 1; 1760 } 1761 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1762 1763 /* Set up the freelist. */ 1764 1765 INIT_LIST_HEAD(&rcu_torture_freelist); 1766 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1767 rcu_tortures[i].rtort_mbtest = 0; 1768 list_add_tail(&rcu_tortures[i].rtort_free, 1769 &rcu_torture_freelist); 1770 } 1771 1772 /* Initialize the statistics so that each run gets its own numbers. */ 1773 1774 rcu_torture_current = NULL; 1775 rcu_torture_current_version = 0; 1776 atomic_set(&n_rcu_torture_alloc, 0); 1777 atomic_set(&n_rcu_torture_alloc_fail, 0); 1778 atomic_set(&n_rcu_torture_free, 0); 1779 atomic_set(&n_rcu_torture_mberror, 0); 1780 atomic_set(&n_rcu_torture_error, 0); 1781 n_rcu_torture_barrier_error = 0; 1782 n_rcu_torture_boost_ktrerror = 0; 1783 n_rcu_torture_boost_rterror = 0; 1784 n_rcu_torture_boost_failure = 0; 1785 n_rcu_torture_boosts = 0; 1786 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1787 atomic_set(&rcu_torture_wcount[i], 0); 1788 for_each_possible_cpu(cpu) { 1789 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1790 per_cpu(rcu_torture_count, cpu)[i] = 0; 1791 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1792 } 1793 } 1794 1795 /* Start up the kthreads. */ 1796 1797 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1798 writer_task); 1799 if (firsterr) 1800 goto unwind; 1801 if (nfakewriters > 0) { 1802 fakewriter_tasks = kzalloc(nfakewriters * 1803 sizeof(fakewriter_tasks[0]), 1804 GFP_KERNEL); 1805 if (fakewriter_tasks == NULL) { 1806 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1807 firsterr = -ENOMEM; 1808 goto unwind; 1809 } 1810 } 1811 for (i = 0; i < nfakewriters; i++) { 1812 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1813 NULL, fakewriter_tasks[i]); 1814 if (firsterr) 1815 goto unwind; 1816 } 1817 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1818 GFP_KERNEL); 1819 if (reader_tasks == NULL) { 1820 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1821 firsterr = -ENOMEM; 1822 goto unwind; 1823 } 1824 for (i = 0; i < nrealreaders; i++) { 1825 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1826 reader_tasks[i]); 1827 if (firsterr) 1828 goto unwind; 1829 } 1830 if (stat_interval > 0) { 1831 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1832 stats_task); 1833 if (firsterr) 1834 goto unwind; 1835 } 1836 if (test_no_idle_hz && shuffle_interval > 0) { 1837 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1838 if (firsterr) 1839 goto unwind; 1840 } 1841 if (stutter < 0) 1842 stutter = 0; 1843 if (stutter) { 1844 firsterr = torture_stutter_init(stutter * HZ); 1845 if (firsterr) 1846 goto unwind; 1847 } 1848 if (fqs_duration < 0) 1849 fqs_duration = 0; 1850 if (fqs_duration) { 1851 /* Create the fqs thread */ 1852 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1853 fqs_task); 1854 if (firsterr) 1855 goto unwind; 1856 } 1857 if (test_boost_interval < 1) 1858 test_boost_interval = 1; 1859 if (test_boost_duration < 2) 1860 test_boost_duration = 2; 1861 if ((test_boost == 1 && cur_ops->can_boost) || 1862 test_boost == 2) { 1863 1864 boost_starttime = jiffies + test_boost_interval * HZ; 1865 1866 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 1867 rcutorture_booster_init, 1868 rcutorture_booster_cleanup); 1869 if (firsterr < 0) 1870 goto unwind; 1871 rcutor_hp = firsterr; 1872 } 1873 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1874 if (firsterr) 1875 goto unwind; 1876 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1877 if (firsterr) 1878 goto unwind; 1879 firsterr = rcu_torture_stall_init(); 1880 if (firsterr) 1881 goto unwind; 1882 firsterr = rcu_torture_barrier_init(); 1883 if (firsterr) 1884 goto unwind; 1885 if (object_debug) 1886 rcu_test_debug_objects(); 1887 if (cbflood_n_burst > 0) { 1888 /* Create the cbflood threads */ 1889 ncbflooders = (num_online_cpus() + 3) / 4; 1890 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1891 GFP_KERNEL); 1892 if (!cbflood_task) { 1893 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1894 firsterr = -ENOMEM; 1895 goto unwind; 1896 } 1897 for (i = 0; i < ncbflooders; i++) { 1898 firsterr = torture_create_kthread(rcu_torture_cbflood, 1899 NULL, 1900 cbflood_task[i]); 1901 if (firsterr) 1902 goto unwind; 1903 } 1904 } 1905 rcutorture_record_test_transition(); 1906 torture_init_end(); 1907 return 0; 1908 1909 unwind: 1910 torture_init_end(); 1911 rcu_torture_cleanup(); 1912 return firsterr; 1913 } 1914 1915 module_init(rcu_torture_init); 1916 module_exit(rcu_torture_cleanup); 1917