1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched.h> 36 #include <linux/atomic.h> 37 #include <linux/bitops.h> 38 #include <linux/completion.h> 39 #include <linux/moduleparam.h> 40 #include <linux/percpu.h> 41 #include <linux/notifier.h> 42 #include <linux/reboot.h> 43 #include <linux/freezer.h> 44 #include <linux/cpu.h> 45 #include <linux/delay.h> 46 #include <linux/stat.h> 47 #include <linux/srcu.h> 48 #include <linux/slab.h> 49 #include <linux/trace_clock.h> 50 #include <asm/byteorder.h> 51 #include <linux/torture.h> 52 #include <linux/vmalloc.h> 53 54 MODULE_LICENSE("GPL"); 55 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 56 57 58 torture_param(int, cbflood_inter_holdoff, HZ, 59 "Holdoff between floods (jiffies)"); 60 torture_param(int, cbflood_intra_holdoff, 1, 61 "Holdoff between bursts (jiffies)"); 62 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 63 torture_param(int, cbflood_n_per_burst, 20000, 64 "# callbacks per burst in flood"); 65 torture_param(int, fqs_duration, 0, 66 "Duration of fqs bursts (us), 0 to disable"); 67 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 68 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 69 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 70 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 71 torture_param(bool, gp_normal, false, 72 "Use normal (non-expedited) GP wait primitives"); 73 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 74 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 75 torture_param(int, n_barrier_cbs, 0, 76 "# of callbacks/kthreads for barrier testing"); 77 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 78 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 79 torture_param(int, object_debug, 0, 80 "Enable debug-object double call_rcu() testing"); 81 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 82 torture_param(int, onoff_interval, 0, 83 "Time between CPU hotplugs (s), 0=disable"); 84 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 85 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 86 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 87 torture_param(int, stall_cpu_holdoff, 10, 88 "Time to wait before starting stall (s)."); 89 torture_param(int, stat_interval, 60, 90 "Number of seconds between stats printk()s"); 91 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 92 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 93 torture_param(int, test_boost_duration, 4, 94 "Duration of each boost test, seconds."); 95 torture_param(int, test_boost_interval, 7, 96 "Interval between boost tests, seconds."); 97 torture_param(bool, test_no_idle_hz, true, 98 "Test support for tickless idle CPUs"); 99 torture_param(bool, verbose, true, 100 "Enable verbose debugging printk()s"); 101 102 static char *torture_type = "rcu"; 103 module_param(torture_type, charp, 0444); 104 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 105 106 static int nrealreaders; 107 static int ncbflooders; 108 static struct task_struct *writer_task; 109 static struct task_struct **fakewriter_tasks; 110 static struct task_struct **reader_tasks; 111 static struct task_struct *stats_task; 112 static struct task_struct **cbflood_task; 113 static struct task_struct *fqs_task; 114 static struct task_struct *boost_tasks[NR_CPUS]; 115 static struct task_struct *stall_task; 116 static struct task_struct **barrier_cbs_tasks; 117 static struct task_struct *barrier_task; 118 119 #define RCU_TORTURE_PIPE_LEN 10 120 121 struct rcu_torture { 122 struct rcu_head rtort_rcu; 123 int rtort_pipe_count; 124 struct list_head rtort_free; 125 int rtort_mbtest; 126 }; 127 128 static LIST_HEAD(rcu_torture_freelist); 129 static struct rcu_torture __rcu *rcu_torture_current; 130 static unsigned long rcu_torture_current_version; 131 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 132 static DEFINE_SPINLOCK(rcu_torture_lock); 133 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 134 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 135 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 136 static atomic_t n_rcu_torture_alloc; 137 static atomic_t n_rcu_torture_alloc_fail; 138 static atomic_t n_rcu_torture_free; 139 static atomic_t n_rcu_torture_mberror; 140 static atomic_t n_rcu_torture_error; 141 static long n_rcu_torture_barrier_error; 142 static long n_rcu_torture_boost_ktrerror; 143 static long n_rcu_torture_boost_rterror; 144 static long n_rcu_torture_boost_failure; 145 static long n_rcu_torture_boosts; 146 static long n_rcu_torture_timers; 147 static long n_barrier_attempts; 148 static long n_barrier_successes; 149 static atomic_long_t n_cbfloods; 150 static struct list_head rcu_torture_removed; 151 152 static int rcu_torture_writer_state; 153 #define RTWS_FIXED_DELAY 0 154 #define RTWS_DELAY 1 155 #define RTWS_REPLACE 2 156 #define RTWS_DEF_FREE 3 157 #define RTWS_EXP_SYNC 4 158 #define RTWS_COND_GET 5 159 #define RTWS_COND_SYNC 6 160 #define RTWS_SYNC 7 161 #define RTWS_STUTTER 8 162 #define RTWS_STOPPING 9 163 static const char * const rcu_torture_writer_state_names[] = { 164 "RTWS_FIXED_DELAY", 165 "RTWS_DELAY", 166 "RTWS_REPLACE", 167 "RTWS_DEF_FREE", 168 "RTWS_EXP_SYNC", 169 "RTWS_COND_GET", 170 "RTWS_COND_SYNC", 171 "RTWS_SYNC", 172 "RTWS_STUTTER", 173 "RTWS_STOPPING", 174 }; 175 176 static const char *rcu_torture_writer_state_getname(void) 177 { 178 unsigned int i = READ_ONCE(rcu_torture_writer_state); 179 180 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 181 return "???"; 182 return rcu_torture_writer_state_names[i]; 183 } 184 185 static int torture_runnable = IS_ENABLED(MODULE); 186 module_param(torture_runnable, int, 0444); 187 MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot"); 188 189 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 190 #define rcu_can_boost() 1 191 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 192 #define rcu_can_boost() 0 193 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 194 195 #ifdef CONFIG_RCU_TRACE 196 static u64 notrace rcu_trace_clock_local(void) 197 { 198 u64 ts = trace_clock_local(); 199 unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC); 200 return ts; 201 } 202 #else /* #ifdef CONFIG_RCU_TRACE */ 203 static u64 notrace rcu_trace_clock_local(void) 204 { 205 return 0ULL; 206 } 207 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 208 209 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 210 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 211 /* and boost task create/destroy. */ 212 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 213 static bool barrier_phase; /* Test phase. */ 214 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 215 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 216 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 217 218 /* 219 * Allocate an element from the rcu_tortures pool. 220 */ 221 static struct rcu_torture * 222 rcu_torture_alloc(void) 223 { 224 struct list_head *p; 225 226 spin_lock_bh(&rcu_torture_lock); 227 if (list_empty(&rcu_torture_freelist)) { 228 atomic_inc(&n_rcu_torture_alloc_fail); 229 spin_unlock_bh(&rcu_torture_lock); 230 return NULL; 231 } 232 atomic_inc(&n_rcu_torture_alloc); 233 p = rcu_torture_freelist.next; 234 list_del_init(p); 235 spin_unlock_bh(&rcu_torture_lock); 236 return container_of(p, struct rcu_torture, rtort_free); 237 } 238 239 /* 240 * Free an element to the rcu_tortures pool. 241 */ 242 static void 243 rcu_torture_free(struct rcu_torture *p) 244 { 245 atomic_inc(&n_rcu_torture_free); 246 spin_lock_bh(&rcu_torture_lock); 247 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 248 spin_unlock_bh(&rcu_torture_lock); 249 } 250 251 /* 252 * Operations vector for selecting different types of tests. 253 */ 254 255 struct rcu_torture_ops { 256 int ttype; 257 void (*init)(void); 258 void (*cleanup)(void); 259 int (*readlock)(void); 260 void (*read_delay)(struct torture_random_state *rrsp); 261 void (*readunlock)(int idx); 262 unsigned long (*started)(void); 263 unsigned long (*completed)(void); 264 void (*deferred_free)(struct rcu_torture *p); 265 void (*sync)(void); 266 void (*exp_sync)(void); 267 unsigned long (*get_state)(void); 268 void (*cond_sync)(unsigned long oldstate); 269 call_rcu_func_t call; 270 void (*cb_barrier)(void); 271 void (*fqs)(void); 272 void (*stats)(void); 273 int irq_capable; 274 int can_boost; 275 const char *name; 276 }; 277 278 static struct rcu_torture_ops *cur_ops; 279 280 /* 281 * Definitions for rcu torture testing. 282 */ 283 284 static int rcu_torture_read_lock(void) __acquires(RCU) 285 { 286 rcu_read_lock(); 287 return 0; 288 } 289 290 static void rcu_read_delay(struct torture_random_state *rrsp) 291 { 292 unsigned long started; 293 unsigned long completed; 294 const unsigned long shortdelay_us = 200; 295 const unsigned long longdelay_ms = 50; 296 unsigned long long ts; 297 298 /* We want a short delay sometimes to make a reader delay the grace 299 * period, and we want a long delay occasionally to trigger 300 * force_quiescent_state. */ 301 302 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 303 started = cur_ops->completed(); 304 ts = rcu_trace_clock_local(); 305 mdelay(longdelay_ms); 306 completed = cur_ops->completed(); 307 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 308 started, completed); 309 } 310 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 311 udelay(shortdelay_us); 312 #ifdef CONFIG_PREEMPT 313 if (!preempt_count() && 314 !(torture_random(rrsp) % (nrealreaders * 20000))) 315 preempt_schedule(); /* No QS if preempt_disable() in effect */ 316 #endif 317 } 318 319 static void rcu_torture_read_unlock(int idx) __releases(RCU) 320 { 321 rcu_read_unlock(); 322 } 323 324 /* 325 * Update callback in the pipe. This should be invoked after a grace period. 326 */ 327 static bool 328 rcu_torture_pipe_update_one(struct rcu_torture *rp) 329 { 330 int i; 331 332 i = rp->rtort_pipe_count; 333 if (i > RCU_TORTURE_PIPE_LEN) 334 i = RCU_TORTURE_PIPE_LEN; 335 atomic_inc(&rcu_torture_wcount[i]); 336 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 337 rp->rtort_mbtest = 0; 338 return true; 339 } 340 return false; 341 } 342 343 /* 344 * Update all callbacks in the pipe. Suitable for synchronous grace-period 345 * primitives. 346 */ 347 static void 348 rcu_torture_pipe_update(struct rcu_torture *old_rp) 349 { 350 struct rcu_torture *rp; 351 struct rcu_torture *rp1; 352 353 if (old_rp) 354 list_add(&old_rp->rtort_free, &rcu_torture_removed); 355 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 356 if (rcu_torture_pipe_update_one(rp)) { 357 list_del(&rp->rtort_free); 358 rcu_torture_free(rp); 359 } 360 } 361 } 362 363 static void 364 rcu_torture_cb(struct rcu_head *p) 365 { 366 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 367 368 if (torture_must_stop_irq()) { 369 /* Test is ending, just drop callbacks on the floor. */ 370 /* The next initialization will pick up the pieces. */ 371 return; 372 } 373 if (rcu_torture_pipe_update_one(rp)) 374 rcu_torture_free(rp); 375 else 376 cur_ops->deferred_free(rp); 377 } 378 379 static unsigned long rcu_no_completed(void) 380 { 381 return 0; 382 } 383 384 static void rcu_torture_deferred_free(struct rcu_torture *p) 385 { 386 call_rcu(&p->rtort_rcu, rcu_torture_cb); 387 } 388 389 static void rcu_sync_torture_init(void) 390 { 391 INIT_LIST_HEAD(&rcu_torture_removed); 392 } 393 394 static struct rcu_torture_ops rcu_ops = { 395 .ttype = RCU_FLAVOR, 396 .init = rcu_sync_torture_init, 397 .readlock = rcu_torture_read_lock, 398 .read_delay = rcu_read_delay, 399 .readunlock = rcu_torture_read_unlock, 400 .started = rcu_batches_started, 401 .completed = rcu_batches_completed, 402 .deferred_free = rcu_torture_deferred_free, 403 .sync = synchronize_rcu, 404 .exp_sync = synchronize_rcu_expedited, 405 .get_state = get_state_synchronize_rcu, 406 .cond_sync = cond_synchronize_rcu, 407 .call = call_rcu, 408 .cb_barrier = rcu_barrier, 409 .fqs = rcu_force_quiescent_state, 410 .stats = NULL, 411 .irq_capable = 1, 412 .can_boost = rcu_can_boost(), 413 .name = "rcu" 414 }; 415 416 /* 417 * Definitions for rcu_bh torture testing. 418 */ 419 420 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 421 { 422 rcu_read_lock_bh(); 423 return 0; 424 } 425 426 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 427 { 428 rcu_read_unlock_bh(); 429 } 430 431 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 432 { 433 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 434 } 435 436 static struct rcu_torture_ops rcu_bh_ops = { 437 .ttype = RCU_BH_FLAVOR, 438 .init = rcu_sync_torture_init, 439 .readlock = rcu_bh_torture_read_lock, 440 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 441 .readunlock = rcu_bh_torture_read_unlock, 442 .started = rcu_batches_started_bh, 443 .completed = rcu_batches_completed_bh, 444 .deferred_free = rcu_bh_torture_deferred_free, 445 .sync = synchronize_rcu_bh, 446 .exp_sync = synchronize_rcu_bh_expedited, 447 .call = call_rcu_bh, 448 .cb_barrier = rcu_barrier_bh, 449 .fqs = rcu_bh_force_quiescent_state, 450 .stats = NULL, 451 .irq_capable = 1, 452 .name = "rcu_bh" 453 }; 454 455 /* 456 * Don't even think about trying any of these in real life!!! 457 * The names includes "busted", and they really means it! 458 * The only purpose of these functions is to provide a buggy RCU 459 * implementation to make sure that rcutorture correctly emits 460 * buggy-RCU error messages. 461 */ 462 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 463 { 464 /* This is a deliberate bug for testing purposes only! */ 465 rcu_torture_cb(&p->rtort_rcu); 466 } 467 468 static void synchronize_rcu_busted(void) 469 { 470 /* This is a deliberate bug for testing purposes only! */ 471 } 472 473 static void 474 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 475 { 476 /* This is a deliberate bug for testing purposes only! */ 477 func(head); 478 } 479 480 static struct rcu_torture_ops rcu_busted_ops = { 481 .ttype = INVALID_RCU_FLAVOR, 482 .init = rcu_sync_torture_init, 483 .readlock = rcu_torture_read_lock, 484 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 485 .readunlock = rcu_torture_read_unlock, 486 .started = rcu_no_completed, 487 .completed = rcu_no_completed, 488 .deferred_free = rcu_busted_torture_deferred_free, 489 .sync = synchronize_rcu_busted, 490 .exp_sync = synchronize_rcu_busted, 491 .call = call_rcu_busted, 492 .cb_barrier = NULL, 493 .fqs = NULL, 494 .stats = NULL, 495 .irq_capable = 1, 496 .name = "rcu_busted" 497 }; 498 499 /* 500 * Definitions for srcu torture testing. 501 */ 502 503 DEFINE_STATIC_SRCU(srcu_ctl); 504 static struct srcu_struct srcu_ctld; 505 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 506 507 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 508 { 509 return srcu_read_lock(srcu_ctlp); 510 } 511 512 static void srcu_read_delay(struct torture_random_state *rrsp) 513 { 514 long delay; 515 const long uspertick = 1000000 / HZ; 516 const long longdelay = 10; 517 518 /* We want there to be long-running readers, but not all the time. */ 519 520 delay = torture_random(rrsp) % 521 (nrealreaders * 2 * longdelay * uspertick); 522 if (!delay) 523 schedule_timeout_interruptible(longdelay); 524 else 525 rcu_read_delay(rrsp); 526 } 527 528 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 529 { 530 srcu_read_unlock(srcu_ctlp, idx); 531 } 532 533 static unsigned long srcu_torture_completed(void) 534 { 535 return srcu_batches_completed(srcu_ctlp); 536 } 537 538 static void srcu_torture_deferred_free(struct rcu_torture *rp) 539 { 540 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 541 } 542 543 static void srcu_torture_synchronize(void) 544 { 545 synchronize_srcu(srcu_ctlp); 546 } 547 548 static void srcu_torture_call(struct rcu_head *head, 549 rcu_callback_t func) 550 { 551 call_srcu(srcu_ctlp, head, func); 552 } 553 554 static void srcu_torture_barrier(void) 555 { 556 srcu_barrier(srcu_ctlp); 557 } 558 559 static void srcu_torture_stats(void) 560 { 561 int cpu; 562 int idx = srcu_ctlp->completed & 0x1; 563 564 pr_alert("%s%s per-CPU(idx=%d):", 565 torture_type, TORTURE_FLAG, idx); 566 for_each_possible_cpu(cpu) { 567 long c0, c1; 568 569 c0 = (long)per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu)->c[!idx]; 570 c1 = (long)per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu)->c[idx]; 571 pr_cont(" %d(%ld,%ld)", cpu, c0, c1); 572 } 573 pr_cont("\n"); 574 } 575 576 static void srcu_torture_synchronize_expedited(void) 577 { 578 synchronize_srcu_expedited(srcu_ctlp); 579 } 580 581 static struct rcu_torture_ops srcu_ops = { 582 .ttype = SRCU_FLAVOR, 583 .init = rcu_sync_torture_init, 584 .readlock = srcu_torture_read_lock, 585 .read_delay = srcu_read_delay, 586 .readunlock = srcu_torture_read_unlock, 587 .started = NULL, 588 .completed = srcu_torture_completed, 589 .deferred_free = srcu_torture_deferred_free, 590 .sync = srcu_torture_synchronize, 591 .exp_sync = srcu_torture_synchronize_expedited, 592 .call = srcu_torture_call, 593 .cb_barrier = srcu_torture_barrier, 594 .stats = srcu_torture_stats, 595 .name = "srcu" 596 }; 597 598 static void srcu_torture_init(void) 599 { 600 rcu_sync_torture_init(); 601 WARN_ON(init_srcu_struct(&srcu_ctld)); 602 srcu_ctlp = &srcu_ctld; 603 } 604 605 static void srcu_torture_cleanup(void) 606 { 607 cleanup_srcu_struct(&srcu_ctld); 608 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 609 } 610 611 /* As above, but dynamically allocated. */ 612 static struct rcu_torture_ops srcud_ops = { 613 .ttype = SRCU_FLAVOR, 614 .init = srcu_torture_init, 615 .cleanup = srcu_torture_cleanup, 616 .readlock = srcu_torture_read_lock, 617 .read_delay = srcu_read_delay, 618 .readunlock = srcu_torture_read_unlock, 619 .started = NULL, 620 .completed = srcu_torture_completed, 621 .deferred_free = srcu_torture_deferred_free, 622 .sync = srcu_torture_synchronize, 623 .exp_sync = srcu_torture_synchronize_expedited, 624 .call = srcu_torture_call, 625 .cb_barrier = srcu_torture_barrier, 626 .stats = srcu_torture_stats, 627 .name = "srcud" 628 }; 629 630 /* 631 * Definitions for sched torture testing. 632 */ 633 634 static int sched_torture_read_lock(void) 635 { 636 preempt_disable(); 637 return 0; 638 } 639 640 static void sched_torture_read_unlock(int idx) 641 { 642 preempt_enable(); 643 } 644 645 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 646 { 647 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 648 } 649 650 static struct rcu_torture_ops sched_ops = { 651 .ttype = RCU_SCHED_FLAVOR, 652 .init = rcu_sync_torture_init, 653 .readlock = sched_torture_read_lock, 654 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 655 .readunlock = sched_torture_read_unlock, 656 .started = rcu_batches_started_sched, 657 .completed = rcu_batches_completed_sched, 658 .deferred_free = rcu_sched_torture_deferred_free, 659 .sync = synchronize_sched, 660 .exp_sync = synchronize_sched_expedited, 661 .get_state = get_state_synchronize_sched, 662 .cond_sync = cond_synchronize_sched, 663 .call = call_rcu_sched, 664 .cb_barrier = rcu_barrier_sched, 665 .fqs = rcu_sched_force_quiescent_state, 666 .stats = NULL, 667 .irq_capable = 1, 668 .name = "sched" 669 }; 670 671 #ifdef CONFIG_TASKS_RCU 672 673 /* 674 * Definitions for RCU-tasks torture testing. 675 */ 676 677 static int tasks_torture_read_lock(void) 678 { 679 return 0; 680 } 681 682 static void tasks_torture_read_unlock(int idx) 683 { 684 } 685 686 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 687 { 688 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 689 } 690 691 static struct rcu_torture_ops tasks_ops = { 692 .ttype = RCU_TASKS_FLAVOR, 693 .init = rcu_sync_torture_init, 694 .readlock = tasks_torture_read_lock, 695 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 696 .readunlock = tasks_torture_read_unlock, 697 .started = rcu_no_completed, 698 .completed = rcu_no_completed, 699 .deferred_free = rcu_tasks_torture_deferred_free, 700 .sync = synchronize_rcu_tasks, 701 .exp_sync = synchronize_rcu_tasks, 702 .call = call_rcu_tasks, 703 .cb_barrier = rcu_barrier_tasks, 704 .fqs = NULL, 705 .stats = NULL, 706 .irq_capable = 1, 707 .name = "tasks" 708 }; 709 710 #define RCUTORTURE_TASKS_OPS &tasks_ops, 711 712 static bool __maybe_unused torturing_tasks(void) 713 { 714 return cur_ops == &tasks_ops; 715 } 716 717 #else /* #ifdef CONFIG_TASKS_RCU */ 718 719 #define RCUTORTURE_TASKS_OPS 720 721 static bool __maybe_unused torturing_tasks(void) 722 { 723 return false; 724 } 725 726 #endif /* #else #ifdef CONFIG_TASKS_RCU */ 727 728 /* 729 * RCU torture priority-boost testing. Runs one real-time thread per 730 * CPU for moderate bursts, repeatedly registering RCU callbacks and 731 * spinning waiting for them to be invoked. If a given callback takes 732 * too long to be invoked, we assume that priority inversion has occurred. 733 */ 734 735 struct rcu_boost_inflight { 736 struct rcu_head rcu; 737 int inflight; 738 }; 739 740 static void rcu_torture_boost_cb(struct rcu_head *head) 741 { 742 struct rcu_boost_inflight *rbip = 743 container_of(head, struct rcu_boost_inflight, rcu); 744 745 /* Ensure RCU-core accesses precede clearing ->inflight */ 746 smp_store_release(&rbip->inflight, 0); 747 } 748 749 static int rcu_torture_boost(void *arg) 750 { 751 unsigned long call_rcu_time; 752 unsigned long endtime; 753 unsigned long oldstarttime; 754 struct rcu_boost_inflight rbi = { .inflight = 0 }; 755 struct sched_param sp; 756 757 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 758 759 /* Set real-time priority. */ 760 sp.sched_priority = 1; 761 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 762 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 763 n_rcu_torture_boost_rterror++; 764 } 765 766 init_rcu_head_on_stack(&rbi.rcu); 767 /* Each pass through the following loop does one boost-test cycle. */ 768 do { 769 /* Wait for the next test interval. */ 770 oldstarttime = boost_starttime; 771 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 772 schedule_timeout_interruptible(oldstarttime - jiffies); 773 stutter_wait("rcu_torture_boost"); 774 if (torture_must_stop()) 775 goto checkwait; 776 } 777 778 /* Do one boost-test interval. */ 779 endtime = oldstarttime + test_boost_duration * HZ; 780 call_rcu_time = jiffies; 781 while (ULONG_CMP_LT(jiffies, endtime)) { 782 /* If we don't have a callback in flight, post one. */ 783 if (!smp_load_acquire(&rbi.inflight)) { 784 /* RCU core before ->inflight = 1. */ 785 smp_store_release(&rbi.inflight, 1); 786 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 787 if (jiffies - call_rcu_time > 788 test_boost_duration * HZ - HZ / 2) { 789 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 790 n_rcu_torture_boost_failure++; 791 } 792 call_rcu_time = jiffies; 793 } 794 stutter_wait("rcu_torture_boost"); 795 if (torture_must_stop()) 796 goto checkwait; 797 } 798 799 /* 800 * Set the start time of the next test interval. 801 * Yes, this is vulnerable to long delays, but such 802 * delays simply cause a false negative for the next 803 * interval. Besides, we are running at RT priority, 804 * so delays should be relatively rare. 805 */ 806 while (oldstarttime == boost_starttime && 807 !kthread_should_stop()) { 808 if (mutex_trylock(&boost_mutex)) { 809 boost_starttime = jiffies + 810 test_boost_interval * HZ; 811 n_rcu_torture_boosts++; 812 mutex_unlock(&boost_mutex); 813 break; 814 } 815 schedule_timeout_uninterruptible(1); 816 } 817 818 /* Go do the stutter. */ 819 checkwait: stutter_wait("rcu_torture_boost"); 820 } while (!torture_must_stop()); 821 822 /* Clean up and exit. */ 823 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 824 torture_shutdown_absorb("rcu_torture_boost"); 825 schedule_timeout_uninterruptible(1); 826 } 827 destroy_rcu_head_on_stack(&rbi.rcu); 828 torture_kthread_stopping("rcu_torture_boost"); 829 return 0; 830 } 831 832 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 833 { 834 } 835 836 /* 837 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 838 * to call_rcu() or analogous, increasing the probability of occurrence 839 * of callback-overflow corner cases. 840 */ 841 static int 842 rcu_torture_cbflood(void *arg) 843 { 844 int err = 1; 845 int i; 846 int j; 847 struct rcu_head *rhp; 848 849 if (cbflood_n_per_burst > 0 && 850 cbflood_inter_holdoff > 0 && 851 cbflood_intra_holdoff > 0 && 852 cur_ops->call && 853 cur_ops->cb_barrier) { 854 rhp = vmalloc(sizeof(*rhp) * 855 cbflood_n_burst * cbflood_n_per_burst); 856 err = !rhp; 857 } 858 if (err) { 859 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 860 goto wait_for_stop; 861 } 862 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 863 do { 864 schedule_timeout_interruptible(cbflood_inter_holdoff); 865 atomic_long_inc(&n_cbfloods); 866 WARN_ON(signal_pending(current)); 867 for (i = 0; i < cbflood_n_burst; i++) { 868 for (j = 0; j < cbflood_n_per_burst; j++) { 869 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 870 rcu_torture_cbflood_cb); 871 } 872 schedule_timeout_interruptible(cbflood_intra_holdoff); 873 WARN_ON(signal_pending(current)); 874 } 875 cur_ops->cb_barrier(); 876 stutter_wait("rcu_torture_cbflood"); 877 } while (!torture_must_stop()); 878 vfree(rhp); 879 wait_for_stop: 880 torture_kthread_stopping("rcu_torture_cbflood"); 881 return 0; 882 } 883 884 /* 885 * RCU torture force-quiescent-state kthread. Repeatedly induces 886 * bursts of calls to force_quiescent_state(), increasing the probability 887 * of occurrence of some important types of race conditions. 888 */ 889 static int 890 rcu_torture_fqs(void *arg) 891 { 892 unsigned long fqs_resume_time; 893 int fqs_burst_remaining; 894 895 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 896 do { 897 fqs_resume_time = jiffies + fqs_stutter * HZ; 898 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 899 !kthread_should_stop()) { 900 schedule_timeout_interruptible(1); 901 } 902 fqs_burst_remaining = fqs_duration; 903 while (fqs_burst_remaining > 0 && 904 !kthread_should_stop()) { 905 cur_ops->fqs(); 906 udelay(fqs_holdoff); 907 fqs_burst_remaining -= fqs_holdoff; 908 } 909 stutter_wait("rcu_torture_fqs"); 910 } while (!torture_must_stop()); 911 torture_kthread_stopping("rcu_torture_fqs"); 912 return 0; 913 } 914 915 /* 916 * RCU torture writer kthread. Repeatedly substitutes a new structure 917 * for that pointed to by rcu_torture_current, freeing the old structure 918 * after a series of grace periods (the "pipeline"). 919 */ 920 static int 921 rcu_torture_writer(void *arg) 922 { 923 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 924 int expediting = 0; 925 unsigned long gp_snap; 926 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 927 bool gp_sync1 = gp_sync; 928 int i; 929 struct rcu_torture *rp; 930 struct rcu_torture *old_rp; 931 static DEFINE_TORTURE_RANDOM(rand); 932 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 933 RTWS_COND_GET, RTWS_SYNC }; 934 int nsynctypes = 0; 935 936 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 937 if (!can_expedite) { 938 pr_alert("%s" TORTURE_FLAG 939 " GP expediting controlled from boot/sysfs for %s,\n", 940 torture_type, cur_ops->name); 941 pr_alert("%s" TORTURE_FLAG 942 " Disabled dynamic grace-period expediting.\n", 943 torture_type); 944 } 945 946 /* Initialize synctype[] array. If none set, take default. */ 947 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 948 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 949 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) 950 synctype[nsynctypes++] = RTWS_COND_GET; 951 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) 952 pr_alert("rcu_torture_writer: gp_cond without primitives.\n"); 953 if (gp_exp1 && cur_ops->exp_sync) 954 synctype[nsynctypes++] = RTWS_EXP_SYNC; 955 else if (gp_exp && !cur_ops->exp_sync) 956 pr_alert("rcu_torture_writer: gp_exp without primitives.\n"); 957 if (gp_normal1 && cur_ops->deferred_free) 958 synctype[nsynctypes++] = RTWS_DEF_FREE; 959 else if (gp_normal && !cur_ops->deferred_free) 960 pr_alert("rcu_torture_writer: gp_normal without primitives.\n"); 961 if (gp_sync1 && cur_ops->sync) 962 synctype[nsynctypes++] = RTWS_SYNC; 963 else if (gp_sync && !cur_ops->sync) 964 pr_alert("rcu_torture_writer: gp_sync without primitives.\n"); 965 if (WARN_ONCE(nsynctypes == 0, 966 "rcu_torture_writer: No update-side primitives.\n")) { 967 /* 968 * No updates primitives, so don't try updating. 969 * The resulting test won't be testing much, hence the 970 * above WARN_ONCE(). 971 */ 972 rcu_torture_writer_state = RTWS_STOPPING; 973 torture_kthread_stopping("rcu_torture_writer"); 974 } 975 976 do { 977 rcu_torture_writer_state = RTWS_FIXED_DELAY; 978 schedule_timeout_uninterruptible(1); 979 rp = rcu_torture_alloc(); 980 if (rp == NULL) 981 continue; 982 rp->rtort_pipe_count = 0; 983 rcu_torture_writer_state = RTWS_DELAY; 984 udelay(torture_random(&rand) & 0x3ff); 985 rcu_torture_writer_state = RTWS_REPLACE; 986 old_rp = rcu_dereference_check(rcu_torture_current, 987 current == writer_task); 988 rp->rtort_mbtest = 1; 989 rcu_assign_pointer(rcu_torture_current, rp); 990 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 991 if (old_rp) { 992 i = old_rp->rtort_pipe_count; 993 if (i > RCU_TORTURE_PIPE_LEN) 994 i = RCU_TORTURE_PIPE_LEN; 995 atomic_inc(&rcu_torture_wcount[i]); 996 old_rp->rtort_pipe_count++; 997 switch (synctype[torture_random(&rand) % nsynctypes]) { 998 case RTWS_DEF_FREE: 999 rcu_torture_writer_state = RTWS_DEF_FREE; 1000 cur_ops->deferred_free(old_rp); 1001 break; 1002 case RTWS_EXP_SYNC: 1003 rcu_torture_writer_state = RTWS_EXP_SYNC; 1004 cur_ops->exp_sync(); 1005 rcu_torture_pipe_update(old_rp); 1006 break; 1007 case RTWS_COND_GET: 1008 rcu_torture_writer_state = RTWS_COND_GET; 1009 gp_snap = cur_ops->get_state(); 1010 i = torture_random(&rand) % 16; 1011 if (i != 0) 1012 schedule_timeout_interruptible(i); 1013 udelay(torture_random(&rand) % 1000); 1014 rcu_torture_writer_state = RTWS_COND_SYNC; 1015 cur_ops->cond_sync(gp_snap); 1016 rcu_torture_pipe_update(old_rp); 1017 break; 1018 case RTWS_SYNC: 1019 rcu_torture_writer_state = RTWS_SYNC; 1020 cur_ops->sync(); 1021 rcu_torture_pipe_update(old_rp); 1022 break; 1023 default: 1024 WARN_ON_ONCE(1); 1025 break; 1026 } 1027 } 1028 rcutorture_record_progress(++rcu_torture_current_version); 1029 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1030 if (can_expedite && 1031 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1032 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1033 if (expediting >= 0) 1034 rcu_expedite_gp(); 1035 else 1036 rcu_unexpedite_gp(); 1037 if (++expediting > 3) 1038 expediting = -expediting; 1039 } 1040 rcu_torture_writer_state = RTWS_STUTTER; 1041 stutter_wait("rcu_torture_writer"); 1042 } while (!torture_must_stop()); 1043 /* Reset expediting back to unexpedited. */ 1044 if (expediting > 0) 1045 expediting = -expediting; 1046 while (can_expedite && expediting++ < 0) 1047 rcu_unexpedite_gp(); 1048 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1049 rcu_torture_writer_state = RTWS_STOPPING; 1050 torture_kthread_stopping("rcu_torture_writer"); 1051 return 0; 1052 } 1053 1054 /* 1055 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1056 * delay between calls. 1057 */ 1058 static int 1059 rcu_torture_fakewriter(void *arg) 1060 { 1061 DEFINE_TORTURE_RANDOM(rand); 1062 1063 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1064 set_user_nice(current, MAX_NICE); 1065 1066 do { 1067 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1068 udelay(torture_random(&rand) & 0x3ff); 1069 if (cur_ops->cb_barrier != NULL && 1070 torture_random(&rand) % (nfakewriters * 8) == 0) { 1071 cur_ops->cb_barrier(); 1072 } else if (gp_normal == gp_exp) { 1073 if (torture_random(&rand) & 0x80) 1074 cur_ops->sync(); 1075 else 1076 cur_ops->exp_sync(); 1077 } else if (gp_normal) { 1078 cur_ops->sync(); 1079 } else { 1080 cur_ops->exp_sync(); 1081 } 1082 stutter_wait("rcu_torture_fakewriter"); 1083 } while (!torture_must_stop()); 1084 1085 torture_kthread_stopping("rcu_torture_fakewriter"); 1086 return 0; 1087 } 1088 1089 /* 1090 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1091 * incrementing the corresponding element of the pipeline array. The 1092 * counter in the element should never be greater than 1, otherwise, the 1093 * RCU implementation is broken. 1094 */ 1095 static void rcu_torture_timer(unsigned long unused) 1096 { 1097 int idx; 1098 unsigned long started; 1099 unsigned long completed; 1100 static DEFINE_TORTURE_RANDOM(rand); 1101 static DEFINE_SPINLOCK(rand_lock); 1102 struct rcu_torture *p; 1103 int pipe_count; 1104 unsigned long long ts; 1105 1106 idx = cur_ops->readlock(); 1107 if (cur_ops->started) 1108 started = cur_ops->started(); 1109 else 1110 started = cur_ops->completed(); 1111 ts = rcu_trace_clock_local(); 1112 p = rcu_dereference_check(rcu_torture_current, 1113 rcu_read_lock_bh_held() || 1114 rcu_read_lock_sched_held() || 1115 srcu_read_lock_held(srcu_ctlp) || 1116 torturing_tasks()); 1117 if (p == NULL) { 1118 /* Leave because rcu_torture_writer is not yet underway */ 1119 cur_ops->readunlock(idx); 1120 return; 1121 } 1122 if (p->rtort_mbtest == 0) 1123 atomic_inc(&n_rcu_torture_mberror); 1124 spin_lock(&rand_lock); 1125 cur_ops->read_delay(&rand); 1126 n_rcu_torture_timers++; 1127 spin_unlock(&rand_lock); 1128 preempt_disable(); 1129 pipe_count = p->rtort_pipe_count; 1130 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1131 /* Should not happen, but... */ 1132 pipe_count = RCU_TORTURE_PIPE_LEN; 1133 } 1134 completed = cur_ops->completed(); 1135 if (pipe_count > 1) { 1136 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1137 started, completed); 1138 rcu_ftrace_dump(DUMP_ALL); 1139 } 1140 __this_cpu_inc(rcu_torture_count[pipe_count]); 1141 completed = completed - started; 1142 if (cur_ops->started) 1143 completed++; 1144 if (completed > RCU_TORTURE_PIPE_LEN) { 1145 /* Should not happen, but... */ 1146 completed = RCU_TORTURE_PIPE_LEN; 1147 } 1148 __this_cpu_inc(rcu_torture_batch[completed]); 1149 preempt_enable(); 1150 cur_ops->readunlock(idx); 1151 } 1152 1153 /* 1154 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1155 * incrementing the corresponding element of the pipeline array. The 1156 * counter in the element should never be greater than 1, otherwise, the 1157 * RCU implementation is broken. 1158 */ 1159 static int 1160 rcu_torture_reader(void *arg) 1161 { 1162 unsigned long started; 1163 unsigned long completed; 1164 int idx; 1165 DEFINE_TORTURE_RANDOM(rand); 1166 struct rcu_torture *p; 1167 int pipe_count; 1168 struct timer_list t; 1169 unsigned long long ts; 1170 1171 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1172 set_user_nice(current, MAX_NICE); 1173 if (irqreader && cur_ops->irq_capable) 1174 setup_timer_on_stack(&t, rcu_torture_timer, 0); 1175 1176 do { 1177 if (irqreader && cur_ops->irq_capable) { 1178 if (!timer_pending(&t)) 1179 mod_timer(&t, jiffies + 1); 1180 } 1181 idx = cur_ops->readlock(); 1182 if (cur_ops->started) 1183 started = cur_ops->started(); 1184 else 1185 started = cur_ops->completed(); 1186 ts = rcu_trace_clock_local(); 1187 p = rcu_dereference_check(rcu_torture_current, 1188 rcu_read_lock_bh_held() || 1189 rcu_read_lock_sched_held() || 1190 srcu_read_lock_held(srcu_ctlp) || 1191 torturing_tasks()); 1192 if (p == NULL) { 1193 /* Wait for rcu_torture_writer to get underway */ 1194 cur_ops->readunlock(idx); 1195 schedule_timeout_interruptible(HZ); 1196 continue; 1197 } 1198 if (p->rtort_mbtest == 0) 1199 atomic_inc(&n_rcu_torture_mberror); 1200 cur_ops->read_delay(&rand); 1201 preempt_disable(); 1202 pipe_count = p->rtort_pipe_count; 1203 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1204 /* Should not happen, but... */ 1205 pipe_count = RCU_TORTURE_PIPE_LEN; 1206 } 1207 completed = cur_ops->completed(); 1208 if (pipe_count > 1) { 1209 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1210 ts, started, completed); 1211 rcu_ftrace_dump(DUMP_ALL); 1212 } 1213 __this_cpu_inc(rcu_torture_count[pipe_count]); 1214 completed = completed - started; 1215 if (cur_ops->started) 1216 completed++; 1217 if (completed > RCU_TORTURE_PIPE_LEN) { 1218 /* Should not happen, but... */ 1219 completed = RCU_TORTURE_PIPE_LEN; 1220 } 1221 __this_cpu_inc(rcu_torture_batch[completed]); 1222 preempt_enable(); 1223 cur_ops->readunlock(idx); 1224 stutter_wait("rcu_torture_reader"); 1225 } while (!torture_must_stop()); 1226 if (irqreader && cur_ops->irq_capable) { 1227 del_timer_sync(&t); 1228 destroy_timer_on_stack(&t); 1229 } 1230 torture_kthread_stopping("rcu_torture_reader"); 1231 return 0; 1232 } 1233 1234 /* 1235 * Print torture statistics. Caller must ensure that there is only 1236 * one call to this function at a given time!!! This is normally 1237 * accomplished by relying on the module system to only have one copy 1238 * of the module loaded, and then by giving the rcu_torture_stats 1239 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1240 * thread is not running). 1241 */ 1242 static void 1243 rcu_torture_stats_print(void) 1244 { 1245 int cpu; 1246 int i; 1247 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1248 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1249 static unsigned long rtcv_snap = ULONG_MAX; 1250 struct task_struct *wtp; 1251 1252 for_each_possible_cpu(cpu) { 1253 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1254 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1255 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1256 } 1257 } 1258 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1259 if (pipesummary[i] != 0) 1260 break; 1261 } 1262 1263 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1264 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1265 rcu_torture_current, 1266 rcu_torture_current_version, 1267 list_empty(&rcu_torture_freelist), 1268 atomic_read(&n_rcu_torture_alloc), 1269 atomic_read(&n_rcu_torture_alloc_fail), 1270 atomic_read(&n_rcu_torture_free)); 1271 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ", 1272 atomic_read(&n_rcu_torture_mberror), 1273 n_rcu_torture_barrier_error, 1274 n_rcu_torture_boost_ktrerror, 1275 n_rcu_torture_boost_rterror); 1276 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1277 n_rcu_torture_boost_failure, 1278 n_rcu_torture_boosts, 1279 n_rcu_torture_timers); 1280 torture_onoff_stats(); 1281 pr_cont("barrier: %ld/%ld:%ld ", 1282 n_barrier_successes, 1283 n_barrier_attempts, 1284 n_rcu_torture_barrier_error); 1285 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1286 1287 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1288 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1289 n_rcu_torture_barrier_error != 0 || 1290 n_rcu_torture_boost_ktrerror != 0 || 1291 n_rcu_torture_boost_rterror != 0 || 1292 n_rcu_torture_boost_failure != 0 || 1293 i > 1) { 1294 pr_cont("%s", "!!! "); 1295 atomic_inc(&n_rcu_torture_error); 1296 WARN_ON_ONCE(1); 1297 } 1298 pr_cont("Reader Pipe: "); 1299 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1300 pr_cont(" %ld", pipesummary[i]); 1301 pr_cont("\n"); 1302 1303 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1304 pr_cont("Reader Batch: "); 1305 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1306 pr_cont(" %ld", batchsummary[i]); 1307 pr_cont("\n"); 1308 1309 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1310 pr_cont("Free-Block Circulation: "); 1311 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1312 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1313 } 1314 pr_cont("\n"); 1315 1316 if (cur_ops->stats) 1317 cur_ops->stats(); 1318 if (rtcv_snap == rcu_torture_current_version && 1319 rcu_torture_current != NULL) { 1320 int __maybe_unused flags; 1321 unsigned long __maybe_unused gpnum; 1322 unsigned long __maybe_unused completed; 1323 1324 rcutorture_get_gp_data(cur_ops->ttype, 1325 &flags, &gpnum, &completed); 1326 wtp = READ_ONCE(writer_task); 1327 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx\n", 1328 rcu_torture_writer_state_getname(), 1329 rcu_torture_writer_state, 1330 gpnum, completed, flags, 1331 wtp == NULL ? ~0UL : wtp->state); 1332 show_rcu_gp_kthreads(); 1333 rcu_ftrace_dump(DUMP_ALL); 1334 } 1335 rtcv_snap = rcu_torture_current_version; 1336 } 1337 1338 /* 1339 * Periodically prints torture statistics, if periodic statistics printing 1340 * was specified via the stat_interval module parameter. 1341 */ 1342 static int 1343 rcu_torture_stats(void *arg) 1344 { 1345 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1346 do { 1347 schedule_timeout_interruptible(stat_interval * HZ); 1348 rcu_torture_stats_print(); 1349 torture_shutdown_absorb("rcu_torture_stats"); 1350 } while (!torture_must_stop()); 1351 torture_kthread_stopping("rcu_torture_stats"); 1352 return 0; 1353 } 1354 1355 static inline void 1356 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1357 { 1358 pr_alert("%s" TORTURE_FLAG 1359 "--- %s: nreaders=%d nfakewriters=%d " 1360 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1361 "shuffle_interval=%d stutter=%d irqreader=%d " 1362 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1363 "test_boost=%d/%d test_boost_interval=%d " 1364 "test_boost_duration=%d shutdown_secs=%d " 1365 "stall_cpu=%d stall_cpu_holdoff=%d " 1366 "n_barrier_cbs=%d " 1367 "onoff_interval=%d onoff_holdoff=%d\n", 1368 torture_type, tag, nrealreaders, nfakewriters, 1369 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1370 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1371 test_boost, cur_ops->can_boost, 1372 test_boost_interval, test_boost_duration, shutdown_secs, 1373 stall_cpu, stall_cpu_holdoff, 1374 n_barrier_cbs, 1375 onoff_interval, onoff_holdoff); 1376 } 1377 1378 static int rcutorture_booster_cleanup(unsigned int cpu) 1379 { 1380 struct task_struct *t; 1381 1382 if (boost_tasks[cpu] == NULL) 1383 return 0; 1384 mutex_lock(&boost_mutex); 1385 t = boost_tasks[cpu]; 1386 boost_tasks[cpu] = NULL; 1387 mutex_unlock(&boost_mutex); 1388 1389 /* This must be outside of the mutex, otherwise deadlock! */ 1390 torture_stop_kthread(rcu_torture_boost, t); 1391 return 0; 1392 } 1393 1394 static int rcutorture_booster_init(unsigned int cpu) 1395 { 1396 int retval; 1397 1398 if (boost_tasks[cpu] != NULL) 1399 return 0; /* Already created, nothing more to do. */ 1400 1401 /* Don't allow time recalculation while creating a new task. */ 1402 mutex_lock(&boost_mutex); 1403 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1404 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1405 cpu_to_node(cpu), 1406 "rcu_torture_boost"); 1407 if (IS_ERR(boost_tasks[cpu])) { 1408 retval = PTR_ERR(boost_tasks[cpu]); 1409 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1410 n_rcu_torture_boost_ktrerror++; 1411 boost_tasks[cpu] = NULL; 1412 mutex_unlock(&boost_mutex); 1413 return retval; 1414 } 1415 kthread_bind(boost_tasks[cpu], cpu); 1416 wake_up_process(boost_tasks[cpu]); 1417 mutex_unlock(&boost_mutex); 1418 return 0; 1419 } 1420 1421 /* 1422 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1423 * induces a CPU stall for the time specified by stall_cpu. 1424 */ 1425 static int rcu_torture_stall(void *args) 1426 { 1427 unsigned long stop_at; 1428 1429 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1430 if (stall_cpu_holdoff > 0) { 1431 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1432 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1433 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1434 } 1435 if (!kthread_should_stop()) { 1436 stop_at = get_seconds() + stall_cpu; 1437 /* RCU CPU stall is expected behavior in following code. */ 1438 pr_alert("rcu_torture_stall start.\n"); 1439 rcu_read_lock(); 1440 preempt_disable(); 1441 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1442 continue; /* Induce RCU CPU stall warning. */ 1443 preempt_enable(); 1444 rcu_read_unlock(); 1445 pr_alert("rcu_torture_stall end.\n"); 1446 } 1447 torture_shutdown_absorb("rcu_torture_stall"); 1448 while (!kthread_should_stop()) 1449 schedule_timeout_interruptible(10 * HZ); 1450 return 0; 1451 } 1452 1453 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1454 static int __init rcu_torture_stall_init(void) 1455 { 1456 if (stall_cpu <= 0) 1457 return 0; 1458 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1459 } 1460 1461 /* Callback function for RCU barrier testing. */ 1462 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1463 { 1464 atomic_inc(&barrier_cbs_invoked); 1465 } 1466 1467 /* kthread function to register callbacks used to test RCU barriers. */ 1468 static int rcu_torture_barrier_cbs(void *arg) 1469 { 1470 long myid = (long)arg; 1471 bool lastphase = 0; 1472 bool newphase; 1473 struct rcu_head rcu; 1474 1475 init_rcu_head_on_stack(&rcu); 1476 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1477 set_user_nice(current, MAX_NICE); 1478 do { 1479 wait_event(barrier_cbs_wq[myid], 1480 (newphase = 1481 smp_load_acquire(&barrier_phase)) != lastphase || 1482 torture_must_stop()); 1483 lastphase = newphase; 1484 if (torture_must_stop()) 1485 break; 1486 /* 1487 * The above smp_load_acquire() ensures barrier_phase load 1488 * is ordered before the following ->call(). 1489 */ 1490 local_irq_disable(); /* Just to test no-irq call_rcu(). */ 1491 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1492 local_irq_enable(); 1493 if (atomic_dec_and_test(&barrier_cbs_count)) 1494 wake_up(&barrier_wq); 1495 } while (!torture_must_stop()); 1496 if (cur_ops->cb_barrier != NULL) 1497 cur_ops->cb_barrier(); 1498 destroy_rcu_head_on_stack(&rcu); 1499 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1500 return 0; 1501 } 1502 1503 /* kthread function to drive and coordinate RCU barrier testing. */ 1504 static int rcu_torture_barrier(void *arg) 1505 { 1506 int i; 1507 1508 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1509 do { 1510 atomic_set(&barrier_cbs_invoked, 0); 1511 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1512 /* Ensure barrier_phase ordered after prior assignments. */ 1513 smp_store_release(&barrier_phase, !barrier_phase); 1514 for (i = 0; i < n_barrier_cbs; i++) 1515 wake_up(&barrier_cbs_wq[i]); 1516 wait_event(barrier_wq, 1517 atomic_read(&barrier_cbs_count) == 0 || 1518 torture_must_stop()); 1519 if (torture_must_stop()) 1520 break; 1521 n_barrier_attempts++; 1522 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1523 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1524 n_rcu_torture_barrier_error++; 1525 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 1526 atomic_read(&barrier_cbs_invoked), 1527 n_barrier_cbs); 1528 WARN_ON_ONCE(1); 1529 } 1530 n_barrier_successes++; 1531 schedule_timeout_interruptible(HZ / 10); 1532 } while (!torture_must_stop()); 1533 torture_kthread_stopping("rcu_torture_barrier"); 1534 return 0; 1535 } 1536 1537 /* Initialize RCU barrier testing. */ 1538 static int rcu_torture_barrier_init(void) 1539 { 1540 int i; 1541 int ret; 1542 1543 if (n_barrier_cbs <= 0) 1544 return 0; 1545 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1546 pr_alert("%s" TORTURE_FLAG 1547 " Call or barrier ops missing for %s,\n", 1548 torture_type, cur_ops->name); 1549 pr_alert("%s" TORTURE_FLAG 1550 " RCU barrier testing omitted from run.\n", 1551 torture_type); 1552 return 0; 1553 } 1554 atomic_set(&barrier_cbs_count, 0); 1555 atomic_set(&barrier_cbs_invoked, 0); 1556 barrier_cbs_tasks = 1557 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1558 GFP_KERNEL); 1559 barrier_cbs_wq = 1560 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1561 GFP_KERNEL); 1562 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1563 return -ENOMEM; 1564 for (i = 0; i < n_barrier_cbs; i++) { 1565 init_waitqueue_head(&barrier_cbs_wq[i]); 1566 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1567 (void *)(long)i, 1568 barrier_cbs_tasks[i]); 1569 if (ret) 1570 return ret; 1571 } 1572 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1573 } 1574 1575 /* Clean up after RCU barrier testing. */ 1576 static void rcu_torture_barrier_cleanup(void) 1577 { 1578 int i; 1579 1580 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1581 if (barrier_cbs_tasks != NULL) { 1582 for (i = 0; i < n_barrier_cbs; i++) 1583 torture_stop_kthread(rcu_torture_barrier_cbs, 1584 barrier_cbs_tasks[i]); 1585 kfree(barrier_cbs_tasks); 1586 barrier_cbs_tasks = NULL; 1587 } 1588 if (barrier_cbs_wq != NULL) { 1589 kfree(barrier_cbs_wq); 1590 barrier_cbs_wq = NULL; 1591 } 1592 } 1593 1594 static enum cpuhp_state rcutor_hp; 1595 1596 static void 1597 rcu_torture_cleanup(void) 1598 { 1599 int i; 1600 1601 rcutorture_record_test_transition(); 1602 if (torture_cleanup_begin()) { 1603 if (cur_ops->cb_barrier != NULL) 1604 cur_ops->cb_barrier(); 1605 return; 1606 } 1607 1608 rcu_torture_barrier_cleanup(); 1609 torture_stop_kthread(rcu_torture_stall, stall_task); 1610 torture_stop_kthread(rcu_torture_writer, writer_task); 1611 1612 if (reader_tasks) { 1613 for (i = 0; i < nrealreaders; i++) 1614 torture_stop_kthread(rcu_torture_reader, 1615 reader_tasks[i]); 1616 kfree(reader_tasks); 1617 } 1618 rcu_torture_current = NULL; 1619 1620 if (fakewriter_tasks) { 1621 for (i = 0; i < nfakewriters; i++) { 1622 torture_stop_kthread(rcu_torture_fakewriter, 1623 fakewriter_tasks[i]); 1624 } 1625 kfree(fakewriter_tasks); 1626 fakewriter_tasks = NULL; 1627 } 1628 1629 torture_stop_kthread(rcu_torture_stats, stats_task); 1630 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1631 for (i = 0; i < ncbflooders; i++) 1632 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1633 if ((test_boost == 1 && cur_ops->can_boost) || 1634 test_boost == 2) 1635 cpuhp_remove_state(rcutor_hp); 1636 1637 /* 1638 * Wait for all RCU callbacks to fire, then do flavor-specific 1639 * cleanup operations. 1640 */ 1641 if (cur_ops->cb_barrier != NULL) 1642 cur_ops->cb_barrier(); 1643 if (cur_ops->cleanup != NULL) 1644 cur_ops->cleanup(); 1645 1646 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1647 1648 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1649 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1650 else if (torture_onoff_failures()) 1651 rcu_torture_print_module_parms(cur_ops, 1652 "End of test: RCU_HOTPLUG"); 1653 else 1654 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1655 torture_cleanup_end(); 1656 } 1657 1658 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1659 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1660 { 1661 } 1662 1663 static void rcu_torture_err_cb(struct rcu_head *rhp) 1664 { 1665 /* 1666 * This -might- happen due to race conditions, but is unlikely. 1667 * The scenario that leads to this happening is that the 1668 * first of the pair of duplicate callbacks is queued, 1669 * someone else starts a grace period that includes that 1670 * callback, then the second of the pair must wait for the 1671 * next grace period. Unlikely, but can happen. If it 1672 * does happen, the debug-objects subsystem won't have splatted. 1673 */ 1674 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1675 } 1676 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1677 1678 /* 1679 * Verify that double-free causes debug-objects to complain, but only 1680 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1681 * cannot be carried out. 1682 */ 1683 static void rcu_test_debug_objects(void) 1684 { 1685 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1686 struct rcu_head rh1; 1687 struct rcu_head rh2; 1688 1689 init_rcu_head_on_stack(&rh1); 1690 init_rcu_head_on_stack(&rh2); 1691 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1692 1693 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1694 preempt_disable(); /* Prevent preemption from interrupting test. */ 1695 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1696 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1697 local_irq_disable(); /* Make it harder to start a new grace period. */ 1698 call_rcu(&rh2, rcu_torture_leak_cb); 1699 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1700 local_irq_enable(); 1701 rcu_read_unlock(); 1702 preempt_enable(); 1703 1704 /* Wait for them all to get done so we can safely return. */ 1705 rcu_barrier(); 1706 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1707 destroy_rcu_head_on_stack(&rh1); 1708 destroy_rcu_head_on_stack(&rh2); 1709 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1710 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1711 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1712 } 1713 1714 static int __init 1715 rcu_torture_init(void) 1716 { 1717 int i; 1718 int cpu; 1719 int firsterr = 0; 1720 static struct rcu_torture_ops *torture_ops[] = { 1721 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 1722 &sched_ops, RCUTORTURE_TASKS_OPS 1723 }; 1724 1725 if (!torture_init_begin(torture_type, verbose, &torture_runnable)) 1726 return -EBUSY; 1727 1728 /* Process args and tell the world that the torturer is on the job. */ 1729 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1730 cur_ops = torture_ops[i]; 1731 if (strcmp(torture_type, cur_ops->name) == 0) 1732 break; 1733 } 1734 if (i == ARRAY_SIZE(torture_ops)) { 1735 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1736 torture_type); 1737 pr_alert("rcu-torture types:"); 1738 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1739 pr_alert(" %s", torture_ops[i]->name); 1740 pr_alert("\n"); 1741 firsterr = -EINVAL; 1742 goto unwind; 1743 } 1744 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1745 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1746 fqs_duration = 0; 1747 } 1748 if (cur_ops->init) 1749 cur_ops->init(); 1750 1751 if (nreaders >= 0) { 1752 nrealreaders = nreaders; 1753 } else { 1754 nrealreaders = num_online_cpus() - 2 - nreaders; 1755 if (nrealreaders <= 0) 1756 nrealreaders = 1; 1757 } 1758 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1759 1760 /* Set up the freelist. */ 1761 1762 INIT_LIST_HEAD(&rcu_torture_freelist); 1763 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1764 rcu_tortures[i].rtort_mbtest = 0; 1765 list_add_tail(&rcu_tortures[i].rtort_free, 1766 &rcu_torture_freelist); 1767 } 1768 1769 /* Initialize the statistics so that each run gets its own numbers. */ 1770 1771 rcu_torture_current = NULL; 1772 rcu_torture_current_version = 0; 1773 atomic_set(&n_rcu_torture_alloc, 0); 1774 atomic_set(&n_rcu_torture_alloc_fail, 0); 1775 atomic_set(&n_rcu_torture_free, 0); 1776 atomic_set(&n_rcu_torture_mberror, 0); 1777 atomic_set(&n_rcu_torture_error, 0); 1778 n_rcu_torture_barrier_error = 0; 1779 n_rcu_torture_boost_ktrerror = 0; 1780 n_rcu_torture_boost_rterror = 0; 1781 n_rcu_torture_boost_failure = 0; 1782 n_rcu_torture_boosts = 0; 1783 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1784 atomic_set(&rcu_torture_wcount[i], 0); 1785 for_each_possible_cpu(cpu) { 1786 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1787 per_cpu(rcu_torture_count, cpu)[i] = 0; 1788 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1789 } 1790 } 1791 1792 /* Start up the kthreads. */ 1793 1794 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1795 writer_task); 1796 if (firsterr) 1797 goto unwind; 1798 if (nfakewriters > 0) { 1799 fakewriter_tasks = kzalloc(nfakewriters * 1800 sizeof(fakewriter_tasks[0]), 1801 GFP_KERNEL); 1802 if (fakewriter_tasks == NULL) { 1803 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1804 firsterr = -ENOMEM; 1805 goto unwind; 1806 } 1807 } 1808 for (i = 0; i < nfakewriters; i++) { 1809 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1810 NULL, fakewriter_tasks[i]); 1811 if (firsterr) 1812 goto unwind; 1813 } 1814 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1815 GFP_KERNEL); 1816 if (reader_tasks == NULL) { 1817 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1818 firsterr = -ENOMEM; 1819 goto unwind; 1820 } 1821 for (i = 0; i < nrealreaders; i++) { 1822 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1823 reader_tasks[i]); 1824 if (firsterr) 1825 goto unwind; 1826 } 1827 if (stat_interval > 0) { 1828 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1829 stats_task); 1830 if (firsterr) 1831 goto unwind; 1832 } 1833 if (test_no_idle_hz && shuffle_interval > 0) { 1834 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1835 if (firsterr) 1836 goto unwind; 1837 } 1838 if (stutter < 0) 1839 stutter = 0; 1840 if (stutter) { 1841 firsterr = torture_stutter_init(stutter * HZ); 1842 if (firsterr) 1843 goto unwind; 1844 } 1845 if (fqs_duration < 0) 1846 fqs_duration = 0; 1847 if (fqs_duration) { 1848 /* Create the fqs thread */ 1849 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1850 fqs_task); 1851 if (firsterr) 1852 goto unwind; 1853 } 1854 if (test_boost_interval < 1) 1855 test_boost_interval = 1; 1856 if (test_boost_duration < 2) 1857 test_boost_duration = 2; 1858 if ((test_boost == 1 && cur_ops->can_boost) || 1859 test_boost == 2) { 1860 1861 boost_starttime = jiffies + test_boost_interval * HZ; 1862 1863 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 1864 rcutorture_booster_init, 1865 rcutorture_booster_cleanup); 1866 if (firsterr < 0) 1867 goto unwind; 1868 rcutor_hp = firsterr; 1869 } 1870 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1871 if (firsterr) 1872 goto unwind; 1873 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1874 if (firsterr) 1875 goto unwind; 1876 firsterr = rcu_torture_stall_init(); 1877 if (firsterr) 1878 goto unwind; 1879 firsterr = rcu_torture_barrier_init(); 1880 if (firsterr) 1881 goto unwind; 1882 if (object_debug) 1883 rcu_test_debug_objects(); 1884 if (cbflood_n_burst > 0) { 1885 /* Create the cbflood threads */ 1886 ncbflooders = (num_online_cpus() + 3) / 4; 1887 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1888 GFP_KERNEL); 1889 if (!cbflood_task) { 1890 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1891 firsterr = -ENOMEM; 1892 goto unwind; 1893 } 1894 for (i = 0; i < ncbflooders; i++) { 1895 firsterr = torture_create_kthread(rcu_torture_cbflood, 1896 NULL, 1897 cbflood_task[i]); 1898 if (firsterr) 1899 goto unwind; 1900 } 1901 } 1902 rcutorture_record_test_transition(); 1903 torture_init_end(); 1904 return 0; 1905 1906 unwind: 1907 torture_init_end(); 1908 rcu_torture_cleanup(); 1909 return firsterr; 1910 } 1911 1912 module_init(rcu_torture_init); 1913 module_exit(rcu_torture_cleanup); 1914