1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched/signal.h> 36 #include <uapi/linux/sched/types.h> 37 #include <linux/atomic.h> 38 #include <linux/bitops.h> 39 #include <linux/completion.h> 40 #include <linux/moduleparam.h> 41 #include <linux/percpu.h> 42 #include <linux/notifier.h> 43 #include <linux/reboot.h> 44 #include <linux/freezer.h> 45 #include <linux/cpu.h> 46 #include <linux/delay.h> 47 #include <linux/stat.h> 48 #include <linux/srcu.h> 49 #include <linux/slab.h> 50 #include <linux/trace_clock.h> 51 #include <asm/byteorder.h> 52 #include <linux/torture.h> 53 #include <linux/vmalloc.h> 54 55 #include "rcu.h" 56 57 MODULE_LICENSE("GPL"); 58 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 59 60 61 torture_param(int, cbflood_inter_holdoff, HZ, 62 "Holdoff between floods (jiffies)"); 63 torture_param(int, cbflood_intra_holdoff, 1, 64 "Holdoff between bursts (jiffies)"); 65 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 66 torture_param(int, cbflood_n_per_burst, 20000, 67 "# callbacks per burst in flood"); 68 torture_param(int, fqs_duration, 0, 69 "Duration of fqs bursts (us), 0 to disable"); 70 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 71 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 72 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 73 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 74 torture_param(bool, gp_normal, false, 75 "Use normal (non-expedited) GP wait primitives"); 76 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 77 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 78 torture_param(int, n_barrier_cbs, 0, 79 "# of callbacks/kthreads for barrier testing"); 80 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 81 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 82 torture_param(int, object_debug, 0, 83 "Enable debug-object double call_rcu() testing"); 84 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 85 torture_param(int, onoff_interval, 0, 86 "Time between CPU hotplugs (s), 0=disable"); 87 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 88 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 89 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 90 torture_param(int, stall_cpu_holdoff, 10, 91 "Time to wait before starting stall (s)."); 92 torture_param(int, stat_interval, 60, 93 "Number of seconds between stats printk()s"); 94 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 95 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 96 torture_param(int, test_boost_duration, 4, 97 "Duration of each boost test, seconds."); 98 torture_param(int, test_boost_interval, 7, 99 "Interval between boost tests, seconds."); 100 torture_param(bool, test_no_idle_hz, true, 101 "Test support for tickless idle CPUs"); 102 torture_param(bool, verbose, true, 103 "Enable verbose debugging printk()s"); 104 105 static char *torture_type = "rcu"; 106 module_param(torture_type, charp, 0444); 107 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 108 109 static int nrealreaders; 110 static int ncbflooders; 111 static struct task_struct *writer_task; 112 static struct task_struct **fakewriter_tasks; 113 static struct task_struct **reader_tasks; 114 static struct task_struct *stats_task; 115 static struct task_struct **cbflood_task; 116 static struct task_struct *fqs_task; 117 static struct task_struct *boost_tasks[NR_CPUS]; 118 static struct task_struct *stall_task; 119 static struct task_struct **barrier_cbs_tasks; 120 static struct task_struct *barrier_task; 121 122 #define RCU_TORTURE_PIPE_LEN 10 123 124 struct rcu_torture { 125 struct rcu_head rtort_rcu; 126 int rtort_pipe_count; 127 struct list_head rtort_free; 128 int rtort_mbtest; 129 }; 130 131 static LIST_HEAD(rcu_torture_freelist); 132 static struct rcu_torture __rcu *rcu_torture_current; 133 static unsigned long rcu_torture_current_version; 134 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 135 static DEFINE_SPINLOCK(rcu_torture_lock); 136 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 137 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 138 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 139 static atomic_t n_rcu_torture_alloc; 140 static atomic_t n_rcu_torture_alloc_fail; 141 static atomic_t n_rcu_torture_free; 142 static atomic_t n_rcu_torture_mberror; 143 static atomic_t n_rcu_torture_error; 144 static long n_rcu_torture_barrier_error; 145 static long n_rcu_torture_boost_ktrerror; 146 static long n_rcu_torture_boost_rterror; 147 static long n_rcu_torture_boost_failure; 148 static long n_rcu_torture_boosts; 149 static long n_rcu_torture_timers; 150 static long n_barrier_attempts; 151 static long n_barrier_successes; 152 static atomic_long_t n_cbfloods; 153 static struct list_head rcu_torture_removed; 154 155 static int rcu_torture_writer_state; 156 #define RTWS_FIXED_DELAY 0 157 #define RTWS_DELAY 1 158 #define RTWS_REPLACE 2 159 #define RTWS_DEF_FREE 3 160 #define RTWS_EXP_SYNC 4 161 #define RTWS_COND_GET 5 162 #define RTWS_COND_SYNC 6 163 #define RTWS_SYNC 7 164 #define RTWS_STUTTER 8 165 #define RTWS_STOPPING 9 166 static const char * const rcu_torture_writer_state_names[] = { 167 "RTWS_FIXED_DELAY", 168 "RTWS_DELAY", 169 "RTWS_REPLACE", 170 "RTWS_DEF_FREE", 171 "RTWS_EXP_SYNC", 172 "RTWS_COND_GET", 173 "RTWS_COND_SYNC", 174 "RTWS_SYNC", 175 "RTWS_STUTTER", 176 "RTWS_STOPPING", 177 }; 178 179 static const char *rcu_torture_writer_state_getname(void) 180 { 181 unsigned int i = READ_ONCE(rcu_torture_writer_state); 182 183 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 184 return "???"; 185 return rcu_torture_writer_state_names[i]; 186 } 187 188 static int torture_runnable = IS_ENABLED(MODULE); 189 module_param(torture_runnable, int, 0444); 190 MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot"); 191 192 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 193 #define rcu_can_boost() 1 194 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 195 #define rcu_can_boost() 0 196 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 197 198 #ifdef CONFIG_RCU_TRACE 199 static u64 notrace rcu_trace_clock_local(void) 200 { 201 u64 ts = trace_clock_local(); 202 203 (void)do_div(ts, NSEC_PER_USEC); 204 return ts; 205 } 206 #else /* #ifdef CONFIG_RCU_TRACE */ 207 static u64 notrace rcu_trace_clock_local(void) 208 { 209 return 0ULL; 210 } 211 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 212 213 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 214 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 215 /* and boost task create/destroy. */ 216 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 217 static bool barrier_phase; /* Test phase. */ 218 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 219 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 220 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 221 222 /* 223 * Allocate an element from the rcu_tortures pool. 224 */ 225 static struct rcu_torture * 226 rcu_torture_alloc(void) 227 { 228 struct list_head *p; 229 230 spin_lock_bh(&rcu_torture_lock); 231 if (list_empty(&rcu_torture_freelist)) { 232 atomic_inc(&n_rcu_torture_alloc_fail); 233 spin_unlock_bh(&rcu_torture_lock); 234 return NULL; 235 } 236 atomic_inc(&n_rcu_torture_alloc); 237 p = rcu_torture_freelist.next; 238 list_del_init(p); 239 spin_unlock_bh(&rcu_torture_lock); 240 return container_of(p, struct rcu_torture, rtort_free); 241 } 242 243 /* 244 * Free an element to the rcu_tortures pool. 245 */ 246 static void 247 rcu_torture_free(struct rcu_torture *p) 248 { 249 atomic_inc(&n_rcu_torture_free); 250 spin_lock_bh(&rcu_torture_lock); 251 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 252 spin_unlock_bh(&rcu_torture_lock); 253 } 254 255 /* 256 * Operations vector for selecting different types of tests. 257 */ 258 259 struct rcu_torture_ops { 260 int ttype; 261 void (*init)(void); 262 void (*cleanup)(void); 263 int (*readlock)(void); 264 void (*read_delay)(struct torture_random_state *rrsp); 265 void (*readunlock)(int idx); 266 unsigned long (*started)(void); 267 unsigned long (*completed)(void); 268 void (*deferred_free)(struct rcu_torture *p); 269 void (*sync)(void); 270 void (*exp_sync)(void); 271 unsigned long (*get_state)(void); 272 void (*cond_sync)(unsigned long oldstate); 273 call_rcu_func_t call; 274 void (*cb_barrier)(void); 275 void (*fqs)(void); 276 void (*stats)(void); 277 int irq_capable; 278 int can_boost; 279 const char *name; 280 }; 281 282 static struct rcu_torture_ops *cur_ops; 283 284 /* 285 * Definitions for rcu torture testing. 286 */ 287 288 static int rcu_torture_read_lock(void) __acquires(RCU) 289 { 290 rcu_read_lock(); 291 return 0; 292 } 293 294 static void rcu_read_delay(struct torture_random_state *rrsp) 295 { 296 unsigned long started; 297 unsigned long completed; 298 const unsigned long shortdelay_us = 200; 299 const unsigned long longdelay_ms = 50; 300 unsigned long long ts; 301 302 /* We want a short delay sometimes to make a reader delay the grace 303 * period, and we want a long delay occasionally to trigger 304 * force_quiescent_state. */ 305 306 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 307 started = cur_ops->completed(); 308 ts = rcu_trace_clock_local(); 309 mdelay(longdelay_ms); 310 completed = cur_ops->completed(); 311 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 312 started, completed); 313 } 314 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 315 udelay(shortdelay_us); 316 #ifdef CONFIG_PREEMPT 317 if (!preempt_count() && 318 !(torture_random(rrsp) % (nrealreaders * 20000))) 319 preempt_schedule(); /* No QS if preempt_disable() in effect */ 320 #endif 321 } 322 323 static void rcu_torture_read_unlock(int idx) __releases(RCU) 324 { 325 rcu_read_unlock(); 326 } 327 328 /* 329 * Update callback in the pipe. This should be invoked after a grace period. 330 */ 331 static bool 332 rcu_torture_pipe_update_one(struct rcu_torture *rp) 333 { 334 int i; 335 336 i = rp->rtort_pipe_count; 337 if (i > RCU_TORTURE_PIPE_LEN) 338 i = RCU_TORTURE_PIPE_LEN; 339 atomic_inc(&rcu_torture_wcount[i]); 340 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 341 rp->rtort_mbtest = 0; 342 return true; 343 } 344 return false; 345 } 346 347 /* 348 * Update all callbacks in the pipe. Suitable for synchronous grace-period 349 * primitives. 350 */ 351 static void 352 rcu_torture_pipe_update(struct rcu_torture *old_rp) 353 { 354 struct rcu_torture *rp; 355 struct rcu_torture *rp1; 356 357 if (old_rp) 358 list_add(&old_rp->rtort_free, &rcu_torture_removed); 359 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 360 if (rcu_torture_pipe_update_one(rp)) { 361 list_del(&rp->rtort_free); 362 rcu_torture_free(rp); 363 } 364 } 365 } 366 367 static void 368 rcu_torture_cb(struct rcu_head *p) 369 { 370 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 371 372 if (torture_must_stop_irq()) { 373 /* Test is ending, just drop callbacks on the floor. */ 374 /* The next initialization will pick up the pieces. */ 375 return; 376 } 377 if (rcu_torture_pipe_update_one(rp)) 378 rcu_torture_free(rp); 379 else 380 cur_ops->deferred_free(rp); 381 } 382 383 static unsigned long rcu_no_completed(void) 384 { 385 return 0; 386 } 387 388 static void rcu_torture_deferred_free(struct rcu_torture *p) 389 { 390 call_rcu(&p->rtort_rcu, rcu_torture_cb); 391 } 392 393 static void rcu_sync_torture_init(void) 394 { 395 INIT_LIST_HEAD(&rcu_torture_removed); 396 } 397 398 static struct rcu_torture_ops rcu_ops = { 399 .ttype = RCU_FLAVOR, 400 .init = rcu_sync_torture_init, 401 .readlock = rcu_torture_read_lock, 402 .read_delay = rcu_read_delay, 403 .readunlock = rcu_torture_read_unlock, 404 .started = rcu_batches_started, 405 .completed = rcu_batches_completed, 406 .deferred_free = rcu_torture_deferred_free, 407 .sync = synchronize_rcu, 408 .exp_sync = synchronize_rcu_expedited, 409 .get_state = get_state_synchronize_rcu, 410 .cond_sync = cond_synchronize_rcu, 411 .call = call_rcu, 412 .cb_barrier = rcu_barrier, 413 .fqs = rcu_force_quiescent_state, 414 .stats = NULL, 415 .irq_capable = 1, 416 .can_boost = rcu_can_boost(), 417 .name = "rcu" 418 }; 419 420 /* 421 * Definitions for rcu_bh torture testing. 422 */ 423 424 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 425 { 426 rcu_read_lock_bh(); 427 return 0; 428 } 429 430 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 431 { 432 rcu_read_unlock_bh(); 433 } 434 435 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 436 { 437 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 438 } 439 440 static struct rcu_torture_ops rcu_bh_ops = { 441 .ttype = RCU_BH_FLAVOR, 442 .init = rcu_sync_torture_init, 443 .readlock = rcu_bh_torture_read_lock, 444 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 445 .readunlock = rcu_bh_torture_read_unlock, 446 .started = rcu_batches_started_bh, 447 .completed = rcu_batches_completed_bh, 448 .deferred_free = rcu_bh_torture_deferred_free, 449 .sync = synchronize_rcu_bh, 450 .exp_sync = synchronize_rcu_bh_expedited, 451 .call = call_rcu_bh, 452 .cb_barrier = rcu_barrier_bh, 453 .fqs = rcu_bh_force_quiescent_state, 454 .stats = NULL, 455 .irq_capable = 1, 456 .name = "rcu_bh" 457 }; 458 459 /* 460 * Don't even think about trying any of these in real life!!! 461 * The names includes "busted", and they really means it! 462 * The only purpose of these functions is to provide a buggy RCU 463 * implementation to make sure that rcutorture correctly emits 464 * buggy-RCU error messages. 465 */ 466 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 467 { 468 /* This is a deliberate bug for testing purposes only! */ 469 rcu_torture_cb(&p->rtort_rcu); 470 } 471 472 static void synchronize_rcu_busted(void) 473 { 474 /* This is a deliberate bug for testing purposes only! */ 475 } 476 477 static void 478 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 479 { 480 /* This is a deliberate bug for testing purposes only! */ 481 func(head); 482 } 483 484 static struct rcu_torture_ops rcu_busted_ops = { 485 .ttype = INVALID_RCU_FLAVOR, 486 .init = rcu_sync_torture_init, 487 .readlock = rcu_torture_read_lock, 488 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 489 .readunlock = rcu_torture_read_unlock, 490 .started = rcu_no_completed, 491 .completed = rcu_no_completed, 492 .deferred_free = rcu_busted_torture_deferred_free, 493 .sync = synchronize_rcu_busted, 494 .exp_sync = synchronize_rcu_busted, 495 .call = call_rcu_busted, 496 .cb_barrier = NULL, 497 .fqs = NULL, 498 .stats = NULL, 499 .irq_capable = 1, 500 .name = "busted" 501 }; 502 503 /* 504 * Definitions for srcu torture testing. 505 */ 506 507 DEFINE_STATIC_SRCU(srcu_ctl); 508 static struct srcu_struct srcu_ctld; 509 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 510 511 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 512 { 513 return srcu_read_lock(srcu_ctlp); 514 } 515 516 static void srcu_read_delay(struct torture_random_state *rrsp) 517 { 518 long delay; 519 const long uspertick = 1000000 / HZ; 520 const long longdelay = 10; 521 522 /* We want there to be long-running readers, but not all the time. */ 523 524 delay = torture_random(rrsp) % 525 (nrealreaders * 2 * longdelay * uspertick); 526 if (!delay && in_task()) 527 schedule_timeout_interruptible(longdelay); 528 else 529 rcu_read_delay(rrsp); 530 } 531 532 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 533 { 534 srcu_read_unlock(srcu_ctlp, idx); 535 } 536 537 static unsigned long srcu_torture_completed(void) 538 { 539 return srcu_batches_completed(srcu_ctlp); 540 } 541 542 static void srcu_torture_deferred_free(struct rcu_torture *rp) 543 { 544 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 545 } 546 547 static void srcu_torture_synchronize(void) 548 { 549 synchronize_srcu(srcu_ctlp); 550 } 551 552 static void srcu_torture_call(struct rcu_head *head, 553 rcu_callback_t func) 554 { 555 call_srcu(srcu_ctlp, head, func); 556 } 557 558 static void srcu_torture_barrier(void) 559 { 560 srcu_barrier(srcu_ctlp); 561 } 562 563 static void srcu_torture_stats(void) 564 { 565 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 566 } 567 568 static void srcu_torture_synchronize_expedited(void) 569 { 570 synchronize_srcu_expedited(srcu_ctlp); 571 } 572 573 static struct rcu_torture_ops srcu_ops = { 574 .ttype = SRCU_FLAVOR, 575 .init = rcu_sync_torture_init, 576 .readlock = srcu_torture_read_lock, 577 .read_delay = srcu_read_delay, 578 .readunlock = srcu_torture_read_unlock, 579 .started = NULL, 580 .completed = srcu_torture_completed, 581 .deferred_free = srcu_torture_deferred_free, 582 .sync = srcu_torture_synchronize, 583 .exp_sync = srcu_torture_synchronize_expedited, 584 .call = srcu_torture_call, 585 .cb_barrier = srcu_torture_barrier, 586 .stats = srcu_torture_stats, 587 .irq_capable = 1, 588 .name = "srcu" 589 }; 590 591 static void srcu_torture_init(void) 592 { 593 rcu_sync_torture_init(); 594 WARN_ON(init_srcu_struct(&srcu_ctld)); 595 srcu_ctlp = &srcu_ctld; 596 } 597 598 static void srcu_torture_cleanup(void) 599 { 600 cleanup_srcu_struct(&srcu_ctld); 601 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 602 } 603 604 /* As above, but dynamically allocated. */ 605 static struct rcu_torture_ops srcud_ops = { 606 .ttype = SRCU_FLAVOR, 607 .init = srcu_torture_init, 608 .cleanup = srcu_torture_cleanup, 609 .readlock = srcu_torture_read_lock, 610 .read_delay = srcu_read_delay, 611 .readunlock = srcu_torture_read_unlock, 612 .started = NULL, 613 .completed = srcu_torture_completed, 614 .deferred_free = srcu_torture_deferred_free, 615 .sync = srcu_torture_synchronize, 616 .exp_sync = srcu_torture_synchronize_expedited, 617 .call = srcu_torture_call, 618 .cb_barrier = srcu_torture_barrier, 619 .stats = srcu_torture_stats, 620 .irq_capable = 1, 621 .name = "srcud" 622 }; 623 624 /* 625 * Definitions for sched torture testing. 626 */ 627 628 static int sched_torture_read_lock(void) 629 { 630 preempt_disable(); 631 return 0; 632 } 633 634 static void sched_torture_read_unlock(int idx) 635 { 636 preempt_enable(); 637 } 638 639 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 640 { 641 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 642 } 643 644 static struct rcu_torture_ops sched_ops = { 645 .ttype = RCU_SCHED_FLAVOR, 646 .init = rcu_sync_torture_init, 647 .readlock = sched_torture_read_lock, 648 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 649 .readunlock = sched_torture_read_unlock, 650 .started = rcu_batches_started_sched, 651 .completed = rcu_batches_completed_sched, 652 .deferred_free = rcu_sched_torture_deferred_free, 653 .sync = synchronize_sched, 654 .exp_sync = synchronize_sched_expedited, 655 .get_state = get_state_synchronize_sched, 656 .cond_sync = cond_synchronize_sched, 657 .call = call_rcu_sched, 658 .cb_barrier = rcu_barrier_sched, 659 .fqs = rcu_sched_force_quiescent_state, 660 .stats = NULL, 661 .irq_capable = 1, 662 .name = "sched" 663 }; 664 665 /* 666 * Definitions for RCU-tasks torture testing. 667 */ 668 669 static int tasks_torture_read_lock(void) 670 { 671 return 0; 672 } 673 674 static void tasks_torture_read_unlock(int idx) 675 { 676 } 677 678 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 679 { 680 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 681 } 682 683 static struct rcu_torture_ops tasks_ops = { 684 .ttype = RCU_TASKS_FLAVOR, 685 .init = rcu_sync_torture_init, 686 .readlock = tasks_torture_read_lock, 687 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 688 .readunlock = tasks_torture_read_unlock, 689 .started = rcu_no_completed, 690 .completed = rcu_no_completed, 691 .deferred_free = rcu_tasks_torture_deferred_free, 692 .sync = synchronize_rcu_tasks, 693 .exp_sync = synchronize_rcu_tasks, 694 .call = call_rcu_tasks, 695 .cb_barrier = rcu_barrier_tasks, 696 .fqs = NULL, 697 .stats = NULL, 698 .irq_capable = 1, 699 .name = "tasks" 700 }; 701 702 static bool __maybe_unused torturing_tasks(void) 703 { 704 return cur_ops == &tasks_ops; 705 } 706 707 /* 708 * RCU torture priority-boost testing. Runs one real-time thread per 709 * CPU for moderate bursts, repeatedly registering RCU callbacks and 710 * spinning waiting for them to be invoked. If a given callback takes 711 * too long to be invoked, we assume that priority inversion has occurred. 712 */ 713 714 struct rcu_boost_inflight { 715 struct rcu_head rcu; 716 int inflight; 717 }; 718 719 static void rcu_torture_boost_cb(struct rcu_head *head) 720 { 721 struct rcu_boost_inflight *rbip = 722 container_of(head, struct rcu_boost_inflight, rcu); 723 724 /* Ensure RCU-core accesses precede clearing ->inflight */ 725 smp_store_release(&rbip->inflight, 0); 726 } 727 728 static int rcu_torture_boost(void *arg) 729 { 730 unsigned long call_rcu_time; 731 unsigned long endtime; 732 unsigned long oldstarttime; 733 struct rcu_boost_inflight rbi = { .inflight = 0 }; 734 struct sched_param sp; 735 736 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 737 738 /* Set real-time priority. */ 739 sp.sched_priority = 1; 740 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 741 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 742 n_rcu_torture_boost_rterror++; 743 } 744 745 init_rcu_head_on_stack(&rbi.rcu); 746 /* Each pass through the following loop does one boost-test cycle. */ 747 do { 748 /* Wait for the next test interval. */ 749 oldstarttime = boost_starttime; 750 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 751 schedule_timeout_interruptible(oldstarttime - jiffies); 752 stutter_wait("rcu_torture_boost"); 753 if (torture_must_stop()) 754 goto checkwait; 755 } 756 757 /* Do one boost-test interval. */ 758 endtime = oldstarttime + test_boost_duration * HZ; 759 call_rcu_time = jiffies; 760 while (ULONG_CMP_LT(jiffies, endtime)) { 761 /* If we don't have a callback in flight, post one. */ 762 if (!smp_load_acquire(&rbi.inflight)) { 763 /* RCU core before ->inflight = 1. */ 764 smp_store_release(&rbi.inflight, 1); 765 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 766 if (jiffies - call_rcu_time > 767 test_boost_duration * HZ - HZ / 2) { 768 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 769 n_rcu_torture_boost_failure++; 770 } 771 call_rcu_time = jiffies; 772 } 773 stutter_wait("rcu_torture_boost"); 774 if (torture_must_stop()) 775 goto checkwait; 776 } 777 778 /* 779 * Set the start time of the next test interval. 780 * Yes, this is vulnerable to long delays, but such 781 * delays simply cause a false negative for the next 782 * interval. Besides, we are running at RT priority, 783 * so delays should be relatively rare. 784 */ 785 while (oldstarttime == boost_starttime && 786 !kthread_should_stop()) { 787 if (mutex_trylock(&boost_mutex)) { 788 boost_starttime = jiffies + 789 test_boost_interval * HZ; 790 n_rcu_torture_boosts++; 791 mutex_unlock(&boost_mutex); 792 break; 793 } 794 schedule_timeout_uninterruptible(1); 795 } 796 797 /* Go do the stutter. */ 798 checkwait: stutter_wait("rcu_torture_boost"); 799 } while (!torture_must_stop()); 800 801 /* Clean up and exit. */ 802 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 803 torture_shutdown_absorb("rcu_torture_boost"); 804 schedule_timeout_uninterruptible(1); 805 } 806 destroy_rcu_head_on_stack(&rbi.rcu); 807 torture_kthread_stopping("rcu_torture_boost"); 808 return 0; 809 } 810 811 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 812 { 813 } 814 815 /* 816 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 817 * to call_rcu() or analogous, increasing the probability of occurrence 818 * of callback-overflow corner cases. 819 */ 820 static int 821 rcu_torture_cbflood(void *arg) 822 { 823 int err = 1; 824 int i; 825 int j; 826 struct rcu_head *rhp; 827 828 if (cbflood_n_per_burst > 0 && 829 cbflood_inter_holdoff > 0 && 830 cbflood_intra_holdoff > 0 && 831 cur_ops->call && 832 cur_ops->cb_barrier) { 833 rhp = vmalloc(sizeof(*rhp) * 834 cbflood_n_burst * cbflood_n_per_burst); 835 err = !rhp; 836 } 837 if (err) { 838 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 839 goto wait_for_stop; 840 } 841 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 842 do { 843 schedule_timeout_interruptible(cbflood_inter_holdoff); 844 atomic_long_inc(&n_cbfloods); 845 WARN_ON(signal_pending(current)); 846 for (i = 0; i < cbflood_n_burst; i++) { 847 for (j = 0; j < cbflood_n_per_burst; j++) { 848 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 849 rcu_torture_cbflood_cb); 850 } 851 schedule_timeout_interruptible(cbflood_intra_holdoff); 852 WARN_ON(signal_pending(current)); 853 } 854 cur_ops->cb_barrier(); 855 stutter_wait("rcu_torture_cbflood"); 856 } while (!torture_must_stop()); 857 vfree(rhp); 858 wait_for_stop: 859 torture_kthread_stopping("rcu_torture_cbflood"); 860 return 0; 861 } 862 863 /* 864 * RCU torture force-quiescent-state kthread. Repeatedly induces 865 * bursts of calls to force_quiescent_state(), increasing the probability 866 * of occurrence of some important types of race conditions. 867 */ 868 static int 869 rcu_torture_fqs(void *arg) 870 { 871 unsigned long fqs_resume_time; 872 int fqs_burst_remaining; 873 874 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 875 do { 876 fqs_resume_time = jiffies + fqs_stutter * HZ; 877 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 878 !kthread_should_stop()) { 879 schedule_timeout_interruptible(1); 880 } 881 fqs_burst_remaining = fqs_duration; 882 while (fqs_burst_remaining > 0 && 883 !kthread_should_stop()) { 884 cur_ops->fqs(); 885 udelay(fqs_holdoff); 886 fqs_burst_remaining -= fqs_holdoff; 887 } 888 stutter_wait("rcu_torture_fqs"); 889 } while (!torture_must_stop()); 890 torture_kthread_stopping("rcu_torture_fqs"); 891 return 0; 892 } 893 894 /* 895 * RCU torture writer kthread. Repeatedly substitutes a new structure 896 * for that pointed to by rcu_torture_current, freeing the old structure 897 * after a series of grace periods (the "pipeline"). 898 */ 899 static int 900 rcu_torture_writer(void *arg) 901 { 902 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 903 int expediting = 0; 904 unsigned long gp_snap; 905 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 906 bool gp_sync1 = gp_sync; 907 int i; 908 struct rcu_torture *rp; 909 struct rcu_torture *old_rp; 910 static DEFINE_TORTURE_RANDOM(rand); 911 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 912 RTWS_COND_GET, RTWS_SYNC }; 913 int nsynctypes = 0; 914 915 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 916 if (!can_expedite) { 917 pr_alert("%s" TORTURE_FLAG 918 " GP expediting controlled from boot/sysfs for %s,\n", 919 torture_type, cur_ops->name); 920 pr_alert("%s" TORTURE_FLAG 921 " Disabled dynamic grace-period expediting.\n", 922 torture_type); 923 } 924 925 /* Initialize synctype[] array. If none set, take default. */ 926 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 927 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 928 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) 929 synctype[nsynctypes++] = RTWS_COND_GET; 930 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) 931 pr_alert("rcu_torture_writer: gp_cond without primitives.\n"); 932 if (gp_exp1 && cur_ops->exp_sync) 933 synctype[nsynctypes++] = RTWS_EXP_SYNC; 934 else if (gp_exp && !cur_ops->exp_sync) 935 pr_alert("rcu_torture_writer: gp_exp without primitives.\n"); 936 if (gp_normal1 && cur_ops->deferred_free) 937 synctype[nsynctypes++] = RTWS_DEF_FREE; 938 else if (gp_normal && !cur_ops->deferred_free) 939 pr_alert("rcu_torture_writer: gp_normal without primitives.\n"); 940 if (gp_sync1 && cur_ops->sync) 941 synctype[nsynctypes++] = RTWS_SYNC; 942 else if (gp_sync && !cur_ops->sync) 943 pr_alert("rcu_torture_writer: gp_sync without primitives.\n"); 944 if (WARN_ONCE(nsynctypes == 0, 945 "rcu_torture_writer: No update-side primitives.\n")) { 946 /* 947 * No updates primitives, so don't try updating. 948 * The resulting test won't be testing much, hence the 949 * above WARN_ONCE(). 950 */ 951 rcu_torture_writer_state = RTWS_STOPPING; 952 torture_kthread_stopping("rcu_torture_writer"); 953 } 954 955 do { 956 rcu_torture_writer_state = RTWS_FIXED_DELAY; 957 schedule_timeout_uninterruptible(1); 958 rp = rcu_torture_alloc(); 959 if (rp == NULL) 960 continue; 961 rp->rtort_pipe_count = 0; 962 rcu_torture_writer_state = RTWS_DELAY; 963 udelay(torture_random(&rand) & 0x3ff); 964 rcu_torture_writer_state = RTWS_REPLACE; 965 old_rp = rcu_dereference_check(rcu_torture_current, 966 current == writer_task); 967 rp->rtort_mbtest = 1; 968 rcu_assign_pointer(rcu_torture_current, rp); 969 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 970 if (old_rp) { 971 i = old_rp->rtort_pipe_count; 972 if (i > RCU_TORTURE_PIPE_LEN) 973 i = RCU_TORTURE_PIPE_LEN; 974 atomic_inc(&rcu_torture_wcount[i]); 975 old_rp->rtort_pipe_count++; 976 switch (synctype[torture_random(&rand) % nsynctypes]) { 977 case RTWS_DEF_FREE: 978 rcu_torture_writer_state = RTWS_DEF_FREE; 979 cur_ops->deferred_free(old_rp); 980 break; 981 case RTWS_EXP_SYNC: 982 rcu_torture_writer_state = RTWS_EXP_SYNC; 983 cur_ops->exp_sync(); 984 rcu_torture_pipe_update(old_rp); 985 break; 986 case RTWS_COND_GET: 987 rcu_torture_writer_state = RTWS_COND_GET; 988 gp_snap = cur_ops->get_state(); 989 i = torture_random(&rand) % 16; 990 if (i != 0) 991 schedule_timeout_interruptible(i); 992 udelay(torture_random(&rand) % 1000); 993 rcu_torture_writer_state = RTWS_COND_SYNC; 994 cur_ops->cond_sync(gp_snap); 995 rcu_torture_pipe_update(old_rp); 996 break; 997 case RTWS_SYNC: 998 rcu_torture_writer_state = RTWS_SYNC; 999 cur_ops->sync(); 1000 rcu_torture_pipe_update(old_rp); 1001 break; 1002 default: 1003 WARN_ON_ONCE(1); 1004 break; 1005 } 1006 } 1007 rcutorture_record_progress(++rcu_torture_current_version); 1008 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1009 if (can_expedite && 1010 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1011 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1012 if (expediting >= 0) 1013 rcu_expedite_gp(); 1014 else 1015 rcu_unexpedite_gp(); 1016 if (++expediting > 3) 1017 expediting = -expediting; 1018 } 1019 rcu_torture_writer_state = RTWS_STUTTER; 1020 stutter_wait("rcu_torture_writer"); 1021 } while (!torture_must_stop()); 1022 /* Reset expediting back to unexpedited. */ 1023 if (expediting > 0) 1024 expediting = -expediting; 1025 while (can_expedite && expediting++ < 0) 1026 rcu_unexpedite_gp(); 1027 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1028 rcu_torture_writer_state = RTWS_STOPPING; 1029 torture_kthread_stopping("rcu_torture_writer"); 1030 return 0; 1031 } 1032 1033 /* 1034 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1035 * delay between calls. 1036 */ 1037 static int 1038 rcu_torture_fakewriter(void *arg) 1039 { 1040 DEFINE_TORTURE_RANDOM(rand); 1041 1042 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1043 set_user_nice(current, MAX_NICE); 1044 1045 do { 1046 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1047 udelay(torture_random(&rand) & 0x3ff); 1048 if (cur_ops->cb_barrier != NULL && 1049 torture_random(&rand) % (nfakewriters * 8) == 0) { 1050 cur_ops->cb_barrier(); 1051 } else if (gp_normal == gp_exp) { 1052 if (torture_random(&rand) & 0x80) 1053 cur_ops->sync(); 1054 else 1055 cur_ops->exp_sync(); 1056 } else if (gp_normal) { 1057 cur_ops->sync(); 1058 } else { 1059 cur_ops->exp_sync(); 1060 } 1061 stutter_wait("rcu_torture_fakewriter"); 1062 } while (!torture_must_stop()); 1063 1064 torture_kthread_stopping("rcu_torture_fakewriter"); 1065 return 0; 1066 } 1067 1068 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1069 { 1070 kfree(rhp); 1071 } 1072 1073 /* 1074 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1075 * incrementing the corresponding element of the pipeline array. The 1076 * counter in the element should never be greater than 1, otherwise, the 1077 * RCU implementation is broken. 1078 */ 1079 static void rcu_torture_timer(unsigned long unused) 1080 { 1081 int idx; 1082 unsigned long started; 1083 unsigned long completed; 1084 static DEFINE_TORTURE_RANDOM(rand); 1085 static DEFINE_SPINLOCK(rand_lock); 1086 struct rcu_torture *p; 1087 int pipe_count; 1088 unsigned long long ts; 1089 1090 idx = cur_ops->readlock(); 1091 if (cur_ops->started) 1092 started = cur_ops->started(); 1093 else 1094 started = cur_ops->completed(); 1095 ts = rcu_trace_clock_local(); 1096 p = rcu_dereference_check(rcu_torture_current, 1097 rcu_read_lock_bh_held() || 1098 rcu_read_lock_sched_held() || 1099 srcu_read_lock_held(srcu_ctlp) || 1100 torturing_tasks()); 1101 if (p == NULL) { 1102 /* Leave because rcu_torture_writer is not yet underway */ 1103 cur_ops->readunlock(idx); 1104 return; 1105 } 1106 if (p->rtort_mbtest == 0) 1107 atomic_inc(&n_rcu_torture_mberror); 1108 spin_lock(&rand_lock); 1109 cur_ops->read_delay(&rand); 1110 n_rcu_torture_timers++; 1111 spin_unlock(&rand_lock); 1112 preempt_disable(); 1113 pipe_count = p->rtort_pipe_count; 1114 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1115 /* Should not happen, but... */ 1116 pipe_count = RCU_TORTURE_PIPE_LEN; 1117 } 1118 completed = cur_ops->completed(); 1119 if (pipe_count > 1) { 1120 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1121 started, completed); 1122 rcu_ftrace_dump(DUMP_ALL); 1123 } 1124 __this_cpu_inc(rcu_torture_count[pipe_count]); 1125 completed = completed - started; 1126 if (cur_ops->started) 1127 completed++; 1128 if (completed > RCU_TORTURE_PIPE_LEN) { 1129 /* Should not happen, but... */ 1130 completed = RCU_TORTURE_PIPE_LEN; 1131 } 1132 __this_cpu_inc(rcu_torture_batch[completed]); 1133 preempt_enable(); 1134 cur_ops->readunlock(idx); 1135 1136 /* Test call_rcu() invocation from interrupt handler. */ 1137 if (cur_ops->call) { 1138 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 1139 1140 if (rhp) 1141 cur_ops->call(rhp, rcu_torture_timer_cb); 1142 } 1143 } 1144 1145 /* 1146 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1147 * incrementing the corresponding element of the pipeline array. The 1148 * counter in the element should never be greater than 1, otherwise, the 1149 * RCU implementation is broken. 1150 */ 1151 static int 1152 rcu_torture_reader(void *arg) 1153 { 1154 unsigned long started; 1155 unsigned long completed; 1156 int idx; 1157 DEFINE_TORTURE_RANDOM(rand); 1158 struct rcu_torture *p; 1159 int pipe_count; 1160 struct timer_list t; 1161 unsigned long long ts; 1162 1163 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1164 set_user_nice(current, MAX_NICE); 1165 if (irqreader && cur_ops->irq_capable) 1166 setup_timer_on_stack(&t, rcu_torture_timer, 0); 1167 1168 do { 1169 if (irqreader && cur_ops->irq_capable) { 1170 if (!timer_pending(&t)) 1171 mod_timer(&t, jiffies + 1); 1172 } 1173 idx = cur_ops->readlock(); 1174 if (cur_ops->started) 1175 started = cur_ops->started(); 1176 else 1177 started = cur_ops->completed(); 1178 ts = rcu_trace_clock_local(); 1179 p = rcu_dereference_check(rcu_torture_current, 1180 rcu_read_lock_bh_held() || 1181 rcu_read_lock_sched_held() || 1182 srcu_read_lock_held(srcu_ctlp) || 1183 torturing_tasks()); 1184 if (p == NULL) { 1185 /* Wait for rcu_torture_writer to get underway */ 1186 cur_ops->readunlock(idx); 1187 schedule_timeout_interruptible(HZ); 1188 continue; 1189 } 1190 if (p->rtort_mbtest == 0) 1191 atomic_inc(&n_rcu_torture_mberror); 1192 cur_ops->read_delay(&rand); 1193 preempt_disable(); 1194 pipe_count = p->rtort_pipe_count; 1195 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1196 /* Should not happen, but... */ 1197 pipe_count = RCU_TORTURE_PIPE_LEN; 1198 } 1199 completed = cur_ops->completed(); 1200 if (pipe_count > 1) { 1201 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1202 ts, started, completed); 1203 rcu_ftrace_dump(DUMP_ALL); 1204 } 1205 __this_cpu_inc(rcu_torture_count[pipe_count]); 1206 completed = completed - started; 1207 if (cur_ops->started) 1208 completed++; 1209 if (completed > RCU_TORTURE_PIPE_LEN) { 1210 /* Should not happen, but... */ 1211 completed = RCU_TORTURE_PIPE_LEN; 1212 } 1213 __this_cpu_inc(rcu_torture_batch[completed]); 1214 preempt_enable(); 1215 cur_ops->readunlock(idx); 1216 stutter_wait("rcu_torture_reader"); 1217 } while (!torture_must_stop()); 1218 if (irqreader && cur_ops->irq_capable) { 1219 del_timer_sync(&t); 1220 destroy_timer_on_stack(&t); 1221 } 1222 torture_kthread_stopping("rcu_torture_reader"); 1223 return 0; 1224 } 1225 1226 /* 1227 * Print torture statistics. Caller must ensure that there is only 1228 * one call to this function at a given time!!! This is normally 1229 * accomplished by relying on the module system to only have one copy 1230 * of the module loaded, and then by giving the rcu_torture_stats 1231 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1232 * thread is not running). 1233 */ 1234 static void 1235 rcu_torture_stats_print(void) 1236 { 1237 int cpu; 1238 int i; 1239 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1240 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1241 static unsigned long rtcv_snap = ULONG_MAX; 1242 struct task_struct *wtp; 1243 1244 for_each_possible_cpu(cpu) { 1245 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1246 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1247 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1248 } 1249 } 1250 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1251 if (pipesummary[i] != 0) 1252 break; 1253 } 1254 1255 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1256 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1257 rcu_torture_current, 1258 rcu_torture_current_version, 1259 list_empty(&rcu_torture_freelist), 1260 atomic_read(&n_rcu_torture_alloc), 1261 atomic_read(&n_rcu_torture_alloc_fail), 1262 atomic_read(&n_rcu_torture_free)); 1263 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ", 1264 atomic_read(&n_rcu_torture_mberror), 1265 n_rcu_torture_barrier_error, 1266 n_rcu_torture_boost_ktrerror, 1267 n_rcu_torture_boost_rterror); 1268 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1269 n_rcu_torture_boost_failure, 1270 n_rcu_torture_boosts, 1271 n_rcu_torture_timers); 1272 torture_onoff_stats(); 1273 pr_cont("barrier: %ld/%ld:%ld ", 1274 n_barrier_successes, 1275 n_barrier_attempts, 1276 n_rcu_torture_barrier_error); 1277 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1278 1279 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1280 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1281 n_rcu_torture_barrier_error != 0 || 1282 n_rcu_torture_boost_ktrerror != 0 || 1283 n_rcu_torture_boost_rterror != 0 || 1284 n_rcu_torture_boost_failure != 0 || 1285 i > 1) { 1286 pr_cont("%s", "!!! "); 1287 atomic_inc(&n_rcu_torture_error); 1288 WARN_ON_ONCE(1); 1289 } 1290 pr_cont("Reader Pipe: "); 1291 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1292 pr_cont(" %ld", pipesummary[i]); 1293 pr_cont("\n"); 1294 1295 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1296 pr_cont("Reader Batch: "); 1297 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1298 pr_cont(" %ld", batchsummary[i]); 1299 pr_cont("\n"); 1300 1301 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1302 pr_cont("Free-Block Circulation: "); 1303 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1304 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1305 } 1306 pr_cont("\n"); 1307 1308 if (cur_ops->stats) 1309 cur_ops->stats(); 1310 if (rtcv_snap == rcu_torture_current_version && 1311 rcu_torture_current != NULL) { 1312 int __maybe_unused flags = 0; 1313 unsigned long __maybe_unused gpnum = 0; 1314 unsigned long __maybe_unused completed = 0; 1315 1316 rcutorture_get_gp_data(cur_ops->ttype, 1317 &flags, &gpnum, &completed); 1318 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, 1319 &flags, &gpnum, &completed); 1320 wtp = READ_ONCE(writer_task); 1321 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx cpu %d\n", 1322 rcu_torture_writer_state_getname(), 1323 rcu_torture_writer_state, 1324 gpnum, completed, flags, 1325 wtp == NULL ? ~0UL : wtp->state, 1326 wtp == NULL ? -1 : (int)task_cpu(wtp)); 1327 show_rcu_gp_kthreads(); 1328 rcu_ftrace_dump(DUMP_ALL); 1329 } 1330 rtcv_snap = rcu_torture_current_version; 1331 } 1332 1333 /* 1334 * Periodically prints torture statistics, if periodic statistics printing 1335 * was specified via the stat_interval module parameter. 1336 */ 1337 static int 1338 rcu_torture_stats(void *arg) 1339 { 1340 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1341 do { 1342 schedule_timeout_interruptible(stat_interval * HZ); 1343 rcu_torture_stats_print(); 1344 torture_shutdown_absorb("rcu_torture_stats"); 1345 } while (!torture_must_stop()); 1346 torture_kthread_stopping("rcu_torture_stats"); 1347 return 0; 1348 } 1349 1350 static inline void 1351 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1352 { 1353 pr_alert("%s" TORTURE_FLAG 1354 "--- %s: nreaders=%d nfakewriters=%d " 1355 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1356 "shuffle_interval=%d stutter=%d irqreader=%d " 1357 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1358 "test_boost=%d/%d test_boost_interval=%d " 1359 "test_boost_duration=%d shutdown_secs=%d " 1360 "stall_cpu=%d stall_cpu_holdoff=%d " 1361 "n_barrier_cbs=%d " 1362 "onoff_interval=%d onoff_holdoff=%d\n", 1363 torture_type, tag, nrealreaders, nfakewriters, 1364 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1365 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1366 test_boost, cur_ops->can_boost, 1367 test_boost_interval, test_boost_duration, shutdown_secs, 1368 stall_cpu, stall_cpu_holdoff, 1369 n_barrier_cbs, 1370 onoff_interval, onoff_holdoff); 1371 } 1372 1373 static int rcutorture_booster_cleanup(unsigned int cpu) 1374 { 1375 struct task_struct *t; 1376 1377 if (boost_tasks[cpu] == NULL) 1378 return 0; 1379 mutex_lock(&boost_mutex); 1380 t = boost_tasks[cpu]; 1381 boost_tasks[cpu] = NULL; 1382 mutex_unlock(&boost_mutex); 1383 1384 /* This must be outside of the mutex, otherwise deadlock! */ 1385 torture_stop_kthread(rcu_torture_boost, t); 1386 return 0; 1387 } 1388 1389 static int rcutorture_booster_init(unsigned int cpu) 1390 { 1391 int retval; 1392 1393 if (boost_tasks[cpu] != NULL) 1394 return 0; /* Already created, nothing more to do. */ 1395 1396 /* Don't allow time recalculation while creating a new task. */ 1397 mutex_lock(&boost_mutex); 1398 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1399 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1400 cpu_to_node(cpu), 1401 "rcu_torture_boost"); 1402 if (IS_ERR(boost_tasks[cpu])) { 1403 retval = PTR_ERR(boost_tasks[cpu]); 1404 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1405 n_rcu_torture_boost_ktrerror++; 1406 boost_tasks[cpu] = NULL; 1407 mutex_unlock(&boost_mutex); 1408 return retval; 1409 } 1410 kthread_bind(boost_tasks[cpu], cpu); 1411 wake_up_process(boost_tasks[cpu]); 1412 mutex_unlock(&boost_mutex); 1413 return 0; 1414 } 1415 1416 /* 1417 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1418 * induces a CPU stall for the time specified by stall_cpu. 1419 */ 1420 static int rcu_torture_stall(void *args) 1421 { 1422 unsigned long stop_at; 1423 1424 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1425 if (stall_cpu_holdoff > 0) { 1426 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1427 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1428 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1429 } 1430 if (!kthread_should_stop()) { 1431 stop_at = get_seconds() + stall_cpu; 1432 /* RCU CPU stall is expected behavior in following code. */ 1433 pr_alert("rcu_torture_stall start.\n"); 1434 rcu_read_lock(); 1435 preempt_disable(); 1436 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1437 continue; /* Induce RCU CPU stall warning. */ 1438 preempt_enable(); 1439 rcu_read_unlock(); 1440 pr_alert("rcu_torture_stall end.\n"); 1441 } 1442 torture_shutdown_absorb("rcu_torture_stall"); 1443 while (!kthread_should_stop()) 1444 schedule_timeout_interruptible(10 * HZ); 1445 return 0; 1446 } 1447 1448 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1449 static int __init rcu_torture_stall_init(void) 1450 { 1451 if (stall_cpu <= 0) 1452 return 0; 1453 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1454 } 1455 1456 /* Callback function for RCU barrier testing. */ 1457 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1458 { 1459 atomic_inc(&barrier_cbs_invoked); 1460 } 1461 1462 /* kthread function to register callbacks used to test RCU barriers. */ 1463 static int rcu_torture_barrier_cbs(void *arg) 1464 { 1465 long myid = (long)arg; 1466 bool lastphase = 0; 1467 bool newphase; 1468 struct rcu_head rcu; 1469 1470 init_rcu_head_on_stack(&rcu); 1471 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1472 set_user_nice(current, MAX_NICE); 1473 do { 1474 wait_event(barrier_cbs_wq[myid], 1475 (newphase = 1476 smp_load_acquire(&barrier_phase)) != lastphase || 1477 torture_must_stop()); 1478 lastphase = newphase; 1479 if (torture_must_stop()) 1480 break; 1481 /* 1482 * The above smp_load_acquire() ensures barrier_phase load 1483 * is ordered before the following ->call(). 1484 */ 1485 local_irq_disable(); /* Just to test no-irq call_rcu(). */ 1486 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1487 local_irq_enable(); 1488 if (atomic_dec_and_test(&barrier_cbs_count)) 1489 wake_up(&barrier_wq); 1490 } while (!torture_must_stop()); 1491 if (cur_ops->cb_barrier != NULL) 1492 cur_ops->cb_barrier(); 1493 destroy_rcu_head_on_stack(&rcu); 1494 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1495 return 0; 1496 } 1497 1498 /* kthread function to drive and coordinate RCU barrier testing. */ 1499 static int rcu_torture_barrier(void *arg) 1500 { 1501 int i; 1502 1503 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1504 do { 1505 atomic_set(&barrier_cbs_invoked, 0); 1506 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1507 /* Ensure barrier_phase ordered after prior assignments. */ 1508 smp_store_release(&barrier_phase, !barrier_phase); 1509 for (i = 0; i < n_barrier_cbs; i++) 1510 wake_up(&barrier_cbs_wq[i]); 1511 wait_event(barrier_wq, 1512 atomic_read(&barrier_cbs_count) == 0 || 1513 torture_must_stop()); 1514 if (torture_must_stop()) 1515 break; 1516 n_barrier_attempts++; 1517 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1518 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1519 n_rcu_torture_barrier_error++; 1520 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 1521 atomic_read(&barrier_cbs_invoked), 1522 n_barrier_cbs); 1523 WARN_ON_ONCE(1); 1524 } 1525 n_barrier_successes++; 1526 schedule_timeout_interruptible(HZ / 10); 1527 } while (!torture_must_stop()); 1528 torture_kthread_stopping("rcu_torture_barrier"); 1529 return 0; 1530 } 1531 1532 /* Initialize RCU barrier testing. */ 1533 static int rcu_torture_barrier_init(void) 1534 { 1535 int i; 1536 int ret; 1537 1538 if (n_barrier_cbs <= 0) 1539 return 0; 1540 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1541 pr_alert("%s" TORTURE_FLAG 1542 " Call or barrier ops missing for %s,\n", 1543 torture_type, cur_ops->name); 1544 pr_alert("%s" TORTURE_FLAG 1545 " RCU barrier testing omitted from run.\n", 1546 torture_type); 1547 return 0; 1548 } 1549 atomic_set(&barrier_cbs_count, 0); 1550 atomic_set(&barrier_cbs_invoked, 0); 1551 barrier_cbs_tasks = 1552 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1553 GFP_KERNEL); 1554 barrier_cbs_wq = 1555 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1556 GFP_KERNEL); 1557 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1558 return -ENOMEM; 1559 for (i = 0; i < n_barrier_cbs; i++) { 1560 init_waitqueue_head(&barrier_cbs_wq[i]); 1561 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1562 (void *)(long)i, 1563 barrier_cbs_tasks[i]); 1564 if (ret) 1565 return ret; 1566 } 1567 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1568 } 1569 1570 /* Clean up after RCU barrier testing. */ 1571 static void rcu_torture_barrier_cleanup(void) 1572 { 1573 int i; 1574 1575 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1576 if (barrier_cbs_tasks != NULL) { 1577 for (i = 0; i < n_barrier_cbs; i++) 1578 torture_stop_kthread(rcu_torture_barrier_cbs, 1579 barrier_cbs_tasks[i]); 1580 kfree(barrier_cbs_tasks); 1581 barrier_cbs_tasks = NULL; 1582 } 1583 if (barrier_cbs_wq != NULL) { 1584 kfree(barrier_cbs_wq); 1585 barrier_cbs_wq = NULL; 1586 } 1587 } 1588 1589 static enum cpuhp_state rcutor_hp; 1590 1591 static void 1592 rcu_torture_cleanup(void) 1593 { 1594 int i; 1595 1596 rcutorture_record_test_transition(); 1597 if (torture_cleanup_begin()) { 1598 if (cur_ops->cb_barrier != NULL) 1599 cur_ops->cb_barrier(); 1600 return; 1601 } 1602 1603 rcu_torture_barrier_cleanup(); 1604 torture_stop_kthread(rcu_torture_stall, stall_task); 1605 torture_stop_kthread(rcu_torture_writer, writer_task); 1606 1607 if (reader_tasks) { 1608 for (i = 0; i < nrealreaders; i++) 1609 torture_stop_kthread(rcu_torture_reader, 1610 reader_tasks[i]); 1611 kfree(reader_tasks); 1612 } 1613 rcu_torture_current = NULL; 1614 1615 if (fakewriter_tasks) { 1616 for (i = 0; i < nfakewriters; i++) { 1617 torture_stop_kthread(rcu_torture_fakewriter, 1618 fakewriter_tasks[i]); 1619 } 1620 kfree(fakewriter_tasks); 1621 fakewriter_tasks = NULL; 1622 } 1623 1624 torture_stop_kthread(rcu_torture_stats, stats_task); 1625 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1626 for (i = 0; i < ncbflooders; i++) 1627 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1628 if ((test_boost == 1 && cur_ops->can_boost) || 1629 test_boost == 2) 1630 cpuhp_remove_state(rcutor_hp); 1631 1632 /* 1633 * Wait for all RCU callbacks to fire, then do flavor-specific 1634 * cleanup operations. 1635 */ 1636 if (cur_ops->cb_barrier != NULL) 1637 cur_ops->cb_barrier(); 1638 if (cur_ops->cleanup != NULL) 1639 cur_ops->cleanup(); 1640 1641 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1642 1643 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1644 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1645 else if (torture_onoff_failures()) 1646 rcu_torture_print_module_parms(cur_ops, 1647 "End of test: RCU_HOTPLUG"); 1648 else 1649 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1650 torture_cleanup_end(); 1651 } 1652 1653 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1654 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1655 { 1656 } 1657 1658 static void rcu_torture_err_cb(struct rcu_head *rhp) 1659 { 1660 /* 1661 * This -might- happen due to race conditions, but is unlikely. 1662 * The scenario that leads to this happening is that the 1663 * first of the pair of duplicate callbacks is queued, 1664 * someone else starts a grace period that includes that 1665 * callback, then the second of the pair must wait for the 1666 * next grace period. Unlikely, but can happen. If it 1667 * does happen, the debug-objects subsystem won't have splatted. 1668 */ 1669 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1670 } 1671 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1672 1673 /* 1674 * Verify that double-free causes debug-objects to complain, but only 1675 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1676 * cannot be carried out. 1677 */ 1678 static void rcu_test_debug_objects(void) 1679 { 1680 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1681 struct rcu_head rh1; 1682 struct rcu_head rh2; 1683 1684 init_rcu_head_on_stack(&rh1); 1685 init_rcu_head_on_stack(&rh2); 1686 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1687 1688 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1689 preempt_disable(); /* Prevent preemption from interrupting test. */ 1690 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1691 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1692 local_irq_disable(); /* Make it harder to start a new grace period. */ 1693 call_rcu(&rh2, rcu_torture_leak_cb); 1694 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1695 local_irq_enable(); 1696 rcu_read_unlock(); 1697 preempt_enable(); 1698 1699 /* Wait for them all to get done so we can safely return. */ 1700 rcu_barrier(); 1701 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1702 destroy_rcu_head_on_stack(&rh1); 1703 destroy_rcu_head_on_stack(&rh2); 1704 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1705 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1706 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1707 } 1708 1709 static int __init 1710 rcu_torture_init(void) 1711 { 1712 int i; 1713 int cpu; 1714 int firsterr = 0; 1715 static struct rcu_torture_ops *torture_ops[] = { 1716 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 1717 &sched_ops, &tasks_ops, 1718 }; 1719 1720 if (!torture_init_begin(torture_type, verbose, &torture_runnable)) 1721 return -EBUSY; 1722 1723 /* Process args and tell the world that the torturer is on the job. */ 1724 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1725 cur_ops = torture_ops[i]; 1726 if (strcmp(torture_type, cur_ops->name) == 0) 1727 break; 1728 } 1729 if (i == ARRAY_SIZE(torture_ops)) { 1730 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1731 torture_type); 1732 pr_alert("rcu-torture types:"); 1733 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1734 pr_alert(" %s", torture_ops[i]->name); 1735 pr_alert("\n"); 1736 firsterr = -EINVAL; 1737 goto unwind; 1738 } 1739 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1740 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1741 fqs_duration = 0; 1742 } 1743 if (cur_ops->init) 1744 cur_ops->init(); 1745 1746 if (nreaders >= 0) { 1747 nrealreaders = nreaders; 1748 } else { 1749 nrealreaders = num_online_cpus() - 2 - nreaders; 1750 if (nrealreaders <= 0) 1751 nrealreaders = 1; 1752 } 1753 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1754 1755 /* Set up the freelist. */ 1756 1757 INIT_LIST_HEAD(&rcu_torture_freelist); 1758 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1759 rcu_tortures[i].rtort_mbtest = 0; 1760 list_add_tail(&rcu_tortures[i].rtort_free, 1761 &rcu_torture_freelist); 1762 } 1763 1764 /* Initialize the statistics so that each run gets its own numbers. */ 1765 1766 rcu_torture_current = NULL; 1767 rcu_torture_current_version = 0; 1768 atomic_set(&n_rcu_torture_alloc, 0); 1769 atomic_set(&n_rcu_torture_alloc_fail, 0); 1770 atomic_set(&n_rcu_torture_free, 0); 1771 atomic_set(&n_rcu_torture_mberror, 0); 1772 atomic_set(&n_rcu_torture_error, 0); 1773 n_rcu_torture_barrier_error = 0; 1774 n_rcu_torture_boost_ktrerror = 0; 1775 n_rcu_torture_boost_rterror = 0; 1776 n_rcu_torture_boost_failure = 0; 1777 n_rcu_torture_boosts = 0; 1778 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1779 atomic_set(&rcu_torture_wcount[i], 0); 1780 for_each_possible_cpu(cpu) { 1781 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1782 per_cpu(rcu_torture_count, cpu)[i] = 0; 1783 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1784 } 1785 } 1786 1787 /* Start up the kthreads. */ 1788 1789 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1790 writer_task); 1791 if (firsterr) 1792 goto unwind; 1793 if (nfakewriters > 0) { 1794 fakewriter_tasks = kzalloc(nfakewriters * 1795 sizeof(fakewriter_tasks[0]), 1796 GFP_KERNEL); 1797 if (fakewriter_tasks == NULL) { 1798 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1799 firsterr = -ENOMEM; 1800 goto unwind; 1801 } 1802 } 1803 for (i = 0; i < nfakewriters; i++) { 1804 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1805 NULL, fakewriter_tasks[i]); 1806 if (firsterr) 1807 goto unwind; 1808 } 1809 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1810 GFP_KERNEL); 1811 if (reader_tasks == NULL) { 1812 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1813 firsterr = -ENOMEM; 1814 goto unwind; 1815 } 1816 for (i = 0; i < nrealreaders; i++) { 1817 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1818 reader_tasks[i]); 1819 if (firsterr) 1820 goto unwind; 1821 } 1822 if (stat_interval > 0) { 1823 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1824 stats_task); 1825 if (firsterr) 1826 goto unwind; 1827 } 1828 if (test_no_idle_hz && shuffle_interval > 0) { 1829 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1830 if (firsterr) 1831 goto unwind; 1832 } 1833 if (stutter < 0) 1834 stutter = 0; 1835 if (stutter) { 1836 firsterr = torture_stutter_init(stutter * HZ); 1837 if (firsterr) 1838 goto unwind; 1839 } 1840 if (fqs_duration < 0) 1841 fqs_duration = 0; 1842 if (fqs_duration) { 1843 /* Create the fqs thread */ 1844 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1845 fqs_task); 1846 if (firsterr) 1847 goto unwind; 1848 } 1849 if (test_boost_interval < 1) 1850 test_boost_interval = 1; 1851 if (test_boost_duration < 2) 1852 test_boost_duration = 2; 1853 if ((test_boost == 1 && cur_ops->can_boost) || 1854 test_boost == 2) { 1855 1856 boost_starttime = jiffies + test_boost_interval * HZ; 1857 1858 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 1859 rcutorture_booster_init, 1860 rcutorture_booster_cleanup); 1861 if (firsterr < 0) 1862 goto unwind; 1863 rcutor_hp = firsterr; 1864 } 1865 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1866 if (firsterr) 1867 goto unwind; 1868 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1869 if (firsterr) 1870 goto unwind; 1871 firsterr = rcu_torture_stall_init(); 1872 if (firsterr) 1873 goto unwind; 1874 firsterr = rcu_torture_barrier_init(); 1875 if (firsterr) 1876 goto unwind; 1877 if (object_debug) 1878 rcu_test_debug_objects(); 1879 if (cbflood_n_burst > 0) { 1880 /* Create the cbflood threads */ 1881 ncbflooders = (num_online_cpus() + 3) / 4; 1882 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1883 GFP_KERNEL); 1884 if (!cbflood_task) { 1885 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1886 firsterr = -ENOMEM; 1887 goto unwind; 1888 } 1889 for (i = 0; i < ncbflooders; i++) { 1890 firsterr = torture_create_kthread(rcu_torture_cbflood, 1891 NULL, 1892 cbflood_task[i]); 1893 if (firsterr) 1894 goto unwind; 1895 } 1896 } 1897 rcutorture_record_test_transition(); 1898 torture_init_end(); 1899 return 0; 1900 1901 unwind: 1902 torture_init_end(); 1903 rcu_torture_cleanup(); 1904 return firsterr; 1905 } 1906 1907 module_init(rcu_torture_init); 1908 module_exit(rcu_torture_cleanup); 1909