1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched.h> 36 #include <linux/atomic.h> 37 #include <linux/bitops.h> 38 #include <linux/completion.h> 39 #include <linux/moduleparam.h> 40 #include <linux/percpu.h> 41 #include <linux/notifier.h> 42 #include <linux/reboot.h> 43 #include <linux/freezer.h> 44 #include <linux/cpu.h> 45 #include <linux/delay.h> 46 #include <linux/stat.h> 47 #include <linux/srcu.h> 48 #include <linux/slab.h> 49 #include <linux/trace_clock.h> 50 #include <asm/byteorder.h> 51 #include <linux/torture.h> 52 #include <linux/vmalloc.h> 53 54 MODULE_LICENSE("GPL"); 55 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 56 57 58 torture_param(int, cbflood_inter_holdoff, HZ, 59 "Holdoff between floods (jiffies)"); 60 torture_param(int, cbflood_intra_holdoff, 1, 61 "Holdoff between bursts (jiffies)"); 62 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 63 torture_param(int, cbflood_n_per_burst, 20000, 64 "# callbacks per burst in flood"); 65 torture_param(int, fqs_duration, 0, 66 "Duration of fqs bursts (us), 0 to disable"); 67 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 68 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 69 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 70 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 71 torture_param(bool, gp_normal, false, 72 "Use normal (non-expedited) GP wait primitives"); 73 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 74 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 75 torture_param(int, n_barrier_cbs, 0, 76 "# of callbacks/kthreads for barrier testing"); 77 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 78 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 79 torture_param(int, object_debug, 0, 80 "Enable debug-object double call_rcu() testing"); 81 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 82 torture_param(int, onoff_interval, 0, 83 "Time between CPU hotplugs (s), 0=disable"); 84 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 85 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 86 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 87 torture_param(int, stall_cpu_holdoff, 10, 88 "Time to wait before starting stall (s)."); 89 torture_param(int, stat_interval, 60, 90 "Number of seconds between stats printk()s"); 91 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 92 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 93 torture_param(int, test_boost_duration, 4, 94 "Duration of each boost test, seconds."); 95 torture_param(int, test_boost_interval, 7, 96 "Interval between boost tests, seconds."); 97 torture_param(bool, test_no_idle_hz, true, 98 "Test support for tickless idle CPUs"); 99 torture_param(bool, verbose, true, 100 "Enable verbose debugging printk()s"); 101 102 static char *torture_type = "rcu"; 103 module_param(torture_type, charp, 0444); 104 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 105 106 static int nrealreaders; 107 static int ncbflooders; 108 static struct task_struct *writer_task; 109 static struct task_struct **fakewriter_tasks; 110 static struct task_struct **reader_tasks; 111 static struct task_struct *stats_task; 112 static struct task_struct **cbflood_task; 113 static struct task_struct *fqs_task; 114 static struct task_struct *boost_tasks[NR_CPUS]; 115 static struct task_struct *stall_task; 116 static struct task_struct **barrier_cbs_tasks; 117 static struct task_struct *barrier_task; 118 119 #define RCU_TORTURE_PIPE_LEN 10 120 121 struct rcu_torture { 122 struct rcu_head rtort_rcu; 123 int rtort_pipe_count; 124 struct list_head rtort_free; 125 int rtort_mbtest; 126 }; 127 128 static LIST_HEAD(rcu_torture_freelist); 129 static struct rcu_torture __rcu *rcu_torture_current; 130 static unsigned long rcu_torture_current_version; 131 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 132 static DEFINE_SPINLOCK(rcu_torture_lock); 133 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 134 rcu_torture_count) = { 0 }; 135 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 136 rcu_torture_batch) = { 0 }; 137 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 138 static atomic_t n_rcu_torture_alloc; 139 static atomic_t n_rcu_torture_alloc_fail; 140 static atomic_t n_rcu_torture_free; 141 static atomic_t n_rcu_torture_mberror; 142 static atomic_t n_rcu_torture_error; 143 static long n_rcu_torture_barrier_error; 144 static long n_rcu_torture_boost_ktrerror; 145 static long n_rcu_torture_boost_rterror; 146 static long n_rcu_torture_boost_failure; 147 static long n_rcu_torture_boosts; 148 static long n_rcu_torture_timers; 149 static long n_barrier_attempts; 150 static long n_barrier_successes; 151 static atomic_long_t n_cbfloods; 152 static struct list_head rcu_torture_removed; 153 154 static int rcu_torture_writer_state; 155 #define RTWS_FIXED_DELAY 0 156 #define RTWS_DELAY 1 157 #define RTWS_REPLACE 2 158 #define RTWS_DEF_FREE 3 159 #define RTWS_EXP_SYNC 4 160 #define RTWS_COND_GET 5 161 #define RTWS_COND_SYNC 6 162 #define RTWS_SYNC 7 163 #define RTWS_STUTTER 8 164 #define RTWS_STOPPING 9 165 static const char * const rcu_torture_writer_state_names[] = { 166 "RTWS_FIXED_DELAY", 167 "RTWS_DELAY", 168 "RTWS_REPLACE", 169 "RTWS_DEF_FREE", 170 "RTWS_EXP_SYNC", 171 "RTWS_COND_GET", 172 "RTWS_COND_SYNC", 173 "RTWS_SYNC", 174 "RTWS_STUTTER", 175 "RTWS_STOPPING", 176 }; 177 178 static const char *rcu_torture_writer_state_getname(void) 179 { 180 unsigned int i = READ_ONCE(rcu_torture_writer_state); 181 182 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 183 return "???"; 184 return rcu_torture_writer_state_names[i]; 185 } 186 187 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE) 188 #define RCUTORTURE_RUNNABLE_INIT 1 189 #else 190 #define RCUTORTURE_RUNNABLE_INIT 0 191 #endif 192 static int torture_runnable = RCUTORTURE_RUNNABLE_INIT; 193 module_param(torture_runnable, int, 0444); 194 MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot"); 195 196 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 197 #define rcu_can_boost() 1 198 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 199 #define rcu_can_boost() 0 200 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 201 202 #ifdef CONFIG_RCU_TRACE 203 static u64 notrace rcu_trace_clock_local(void) 204 { 205 u64 ts = trace_clock_local(); 206 unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC); 207 return ts; 208 } 209 #else /* #ifdef CONFIG_RCU_TRACE */ 210 static u64 notrace rcu_trace_clock_local(void) 211 { 212 return 0ULL; 213 } 214 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 215 216 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 217 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 218 /* and boost task create/destroy. */ 219 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 220 static bool barrier_phase; /* Test phase. */ 221 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 222 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 223 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 224 225 /* 226 * Allocate an element from the rcu_tortures pool. 227 */ 228 static struct rcu_torture * 229 rcu_torture_alloc(void) 230 { 231 struct list_head *p; 232 233 spin_lock_bh(&rcu_torture_lock); 234 if (list_empty(&rcu_torture_freelist)) { 235 atomic_inc(&n_rcu_torture_alloc_fail); 236 spin_unlock_bh(&rcu_torture_lock); 237 return NULL; 238 } 239 atomic_inc(&n_rcu_torture_alloc); 240 p = rcu_torture_freelist.next; 241 list_del_init(p); 242 spin_unlock_bh(&rcu_torture_lock); 243 return container_of(p, struct rcu_torture, rtort_free); 244 } 245 246 /* 247 * Free an element to the rcu_tortures pool. 248 */ 249 static void 250 rcu_torture_free(struct rcu_torture *p) 251 { 252 atomic_inc(&n_rcu_torture_free); 253 spin_lock_bh(&rcu_torture_lock); 254 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 255 spin_unlock_bh(&rcu_torture_lock); 256 } 257 258 /* 259 * Operations vector for selecting different types of tests. 260 */ 261 262 struct rcu_torture_ops { 263 int ttype; 264 void (*init)(void); 265 void (*cleanup)(void); 266 int (*readlock)(void); 267 void (*read_delay)(struct torture_random_state *rrsp); 268 void (*readunlock)(int idx); 269 unsigned long (*started)(void); 270 unsigned long (*completed)(void); 271 void (*deferred_free)(struct rcu_torture *p); 272 void (*sync)(void); 273 void (*exp_sync)(void); 274 unsigned long (*get_state)(void); 275 void (*cond_sync)(unsigned long oldstate); 276 call_rcu_func_t call; 277 void (*cb_barrier)(void); 278 void (*fqs)(void); 279 void (*stats)(void); 280 int irq_capable; 281 int can_boost; 282 const char *name; 283 }; 284 285 static struct rcu_torture_ops *cur_ops; 286 287 /* 288 * Definitions for rcu torture testing. 289 */ 290 291 static int rcu_torture_read_lock(void) __acquires(RCU) 292 { 293 rcu_read_lock(); 294 return 0; 295 } 296 297 static void rcu_read_delay(struct torture_random_state *rrsp) 298 { 299 const unsigned long shortdelay_us = 200; 300 const unsigned long longdelay_ms = 50; 301 302 /* We want a short delay sometimes to make a reader delay the grace 303 * period, and we want a long delay occasionally to trigger 304 * force_quiescent_state. */ 305 306 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) 307 mdelay(longdelay_ms); 308 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 309 udelay(shortdelay_us); 310 #ifdef CONFIG_PREEMPT 311 if (!preempt_count() && 312 !(torture_random(rrsp) % (nrealreaders * 20000))) 313 preempt_schedule(); /* No QS if preempt_disable() in effect */ 314 #endif 315 } 316 317 static void rcu_torture_read_unlock(int idx) __releases(RCU) 318 { 319 rcu_read_unlock(); 320 } 321 322 /* 323 * Update callback in the pipe. This should be invoked after a grace period. 324 */ 325 static bool 326 rcu_torture_pipe_update_one(struct rcu_torture *rp) 327 { 328 int i; 329 330 i = rp->rtort_pipe_count; 331 if (i > RCU_TORTURE_PIPE_LEN) 332 i = RCU_TORTURE_PIPE_LEN; 333 atomic_inc(&rcu_torture_wcount[i]); 334 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 335 rp->rtort_mbtest = 0; 336 return true; 337 } 338 return false; 339 } 340 341 /* 342 * Update all callbacks in the pipe. Suitable for synchronous grace-period 343 * primitives. 344 */ 345 static void 346 rcu_torture_pipe_update(struct rcu_torture *old_rp) 347 { 348 struct rcu_torture *rp; 349 struct rcu_torture *rp1; 350 351 if (old_rp) 352 list_add(&old_rp->rtort_free, &rcu_torture_removed); 353 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 354 if (rcu_torture_pipe_update_one(rp)) { 355 list_del(&rp->rtort_free); 356 rcu_torture_free(rp); 357 } 358 } 359 } 360 361 static void 362 rcu_torture_cb(struct rcu_head *p) 363 { 364 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 365 366 if (torture_must_stop_irq()) { 367 /* Test is ending, just drop callbacks on the floor. */ 368 /* The next initialization will pick up the pieces. */ 369 return; 370 } 371 if (rcu_torture_pipe_update_one(rp)) 372 rcu_torture_free(rp); 373 else 374 cur_ops->deferred_free(rp); 375 } 376 377 static unsigned long rcu_no_completed(void) 378 { 379 return 0; 380 } 381 382 static void rcu_torture_deferred_free(struct rcu_torture *p) 383 { 384 call_rcu(&p->rtort_rcu, rcu_torture_cb); 385 } 386 387 static void rcu_sync_torture_init(void) 388 { 389 INIT_LIST_HEAD(&rcu_torture_removed); 390 } 391 392 static struct rcu_torture_ops rcu_ops = { 393 .ttype = RCU_FLAVOR, 394 .init = rcu_sync_torture_init, 395 .readlock = rcu_torture_read_lock, 396 .read_delay = rcu_read_delay, 397 .readunlock = rcu_torture_read_unlock, 398 .started = rcu_batches_started, 399 .completed = rcu_batches_completed, 400 .deferred_free = rcu_torture_deferred_free, 401 .sync = synchronize_rcu, 402 .exp_sync = synchronize_rcu_expedited, 403 .get_state = get_state_synchronize_rcu, 404 .cond_sync = cond_synchronize_rcu, 405 .call = call_rcu, 406 .cb_barrier = rcu_barrier, 407 .fqs = rcu_force_quiescent_state, 408 .stats = NULL, 409 .irq_capable = 1, 410 .can_boost = rcu_can_boost(), 411 .name = "rcu" 412 }; 413 414 /* 415 * Definitions for rcu_bh torture testing. 416 */ 417 418 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 419 { 420 rcu_read_lock_bh(); 421 return 0; 422 } 423 424 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 425 { 426 rcu_read_unlock_bh(); 427 } 428 429 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 430 { 431 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 432 } 433 434 static struct rcu_torture_ops rcu_bh_ops = { 435 .ttype = RCU_BH_FLAVOR, 436 .init = rcu_sync_torture_init, 437 .readlock = rcu_bh_torture_read_lock, 438 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 439 .readunlock = rcu_bh_torture_read_unlock, 440 .started = rcu_batches_started_bh, 441 .completed = rcu_batches_completed_bh, 442 .deferred_free = rcu_bh_torture_deferred_free, 443 .sync = synchronize_rcu_bh, 444 .exp_sync = synchronize_rcu_bh_expedited, 445 .call = call_rcu_bh, 446 .cb_barrier = rcu_barrier_bh, 447 .fqs = rcu_bh_force_quiescent_state, 448 .stats = NULL, 449 .irq_capable = 1, 450 .name = "rcu_bh" 451 }; 452 453 /* 454 * Don't even think about trying any of these in real life!!! 455 * The names includes "busted", and they really means it! 456 * The only purpose of these functions is to provide a buggy RCU 457 * implementation to make sure that rcutorture correctly emits 458 * buggy-RCU error messages. 459 */ 460 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 461 { 462 /* This is a deliberate bug for testing purposes only! */ 463 rcu_torture_cb(&p->rtort_rcu); 464 } 465 466 static void synchronize_rcu_busted(void) 467 { 468 /* This is a deliberate bug for testing purposes only! */ 469 } 470 471 static void 472 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 473 { 474 /* This is a deliberate bug for testing purposes only! */ 475 func(head); 476 } 477 478 static struct rcu_torture_ops rcu_busted_ops = { 479 .ttype = INVALID_RCU_FLAVOR, 480 .init = rcu_sync_torture_init, 481 .readlock = rcu_torture_read_lock, 482 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 483 .readunlock = rcu_torture_read_unlock, 484 .started = rcu_no_completed, 485 .completed = rcu_no_completed, 486 .deferred_free = rcu_busted_torture_deferred_free, 487 .sync = synchronize_rcu_busted, 488 .exp_sync = synchronize_rcu_busted, 489 .call = call_rcu_busted, 490 .cb_barrier = NULL, 491 .fqs = NULL, 492 .stats = NULL, 493 .irq_capable = 1, 494 .name = "rcu_busted" 495 }; 496 497 /* 498 * Definitions for srcu torture testing. 499 */ 500 501 DEFINE_STATIC_SRCU(srcu_ctl); 502 static struct srcu_struct srcu_ctld; 503 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 504 505 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 506 { 507 return srcu_read_lock(srcu_ctlp); 508 } 509 510 static void srcu_read_delay(struct torture_random_state *rrsp) 511 { 512 long delay; 513 const long uspertick = 1000000 / HZ; 514 const long longdelay = 10; 515 516 /* We want there to be long-running readers, but not all the time. */ 517 518 delay = torture_random(rrsp) % 519 (nrealreaders * 2 * longdelay * uspertick); 520 if (!delay) 521 schedule_timeout_interruptible(longdelay); 522 else 523 rcu_read_delay(rrsp); 524 } 525 526 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 527 { 528 srcu_read_unlock(srcu_ctlp, idx); 529 } 530 531 static unsigned long srcu_torture_completed(void) 532 { 533 return srcu_batches_completed(srcu_ctlp); 534 } 535 536 static void srcu_torture_deferred_free(struct rcu_torture *rp) 537 { 538 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 539 } 540 541 static void srcu_torture_synchronize(void) 542 { 543 synchronize_srcu(srcu_ctlp); 544 } 545 546 static void srcu_torture_call(struct rcu_head *head, 547 rcu_callback_t func) 548 { 549 call_srcu(srcu_ctlp, head, func); 550 } 551 552 static void srcu_torture_barrier(void) 553 { 554 srcu_barrier(srcu_ctlp); 555 } 556 557 static void srcu_torture_stats(void) 558 { 559 int cpu; 560 int idx = srcu_ctlp->completed & 0x1; 561 562 pr_alert("%s%s per-CPU(idx=%d):", 563 torture_type, TORTURE_FLAG, idx); 564 for_each_possible_cpu(cpu) { 565 long c0, c1; 566 567 c0 = (long)per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu)->c[!idx]; 568 c1 = (long)per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu)->c[idx]; 569 pr_cont(" %d(%ld,%ld)", cpu, c0, c1); 570 } 571 pr_cont("\n"); 572 } 573 574 static void srcu_torture_synchronize_expedited(void) 575 { 576 synchronize_srcu_expedited(srcu_ctlp); 577 } 578 579 static struct rcu_torture_ops srcu_ops = { 580 .ttype = SRCU_FLAVOR, 581 .init = rcu_sync_torture_init, 582 .readlock = srcu_torture_read_lock, 583 .read_delay = srcu_read_delay, 584 .readunlock = srcu_torture_read_unlock, 585 .started = NULL, 586 .completed = srcu_torture_completed, 587 .deferred_free = srcu_torture_deferred_free, 588 .sync = srcu_torture_synchronize, 589 .exp_sync = srcu_torture_synchronize_expedited, 590 .call = srcu_torture_call, 591 .cb_barrier = srcu_torture_barrier, 592 .stats = srcu_torture_stats, 593 .name = "srcu" 594 }; 595 596 static void srcu_torture_init(void) 597 { 598 rcu_sync_torture_init(); 599 WARN_ON(init_srcu_struct(&srcu_ctld)); 600 srcu_ctlp = &srcu_ctld; 601 } 602 603 static void srcu_torture_cleanup(void) 604 { 605 cleanup_srcu_struct(&srcu_ctld); 606 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 607 } 608 609 /* As above, but dynamically allocated. */ 610 static struct rcu_torture_ops srcud_ops = { 611 .ttype = SRCU_FLAVOR, 612 .init = srcu_torture_init, 613 .cleanup = srcu_torture_cleanup, 614 .readlock = srcu_torture_read_lock, 615 .read_delay = srcu_read_delay, 616 .readunlock = srcu_torture_read_unlock, 617 .started = NULL, 618 .completed = srcu_torture_completed, 619 .deferred_free = srcu_torture_deferred_free, 620 .sync = srcu_torture_synchronize, 621 .exp_sync = srcu_torture_synchronize_expedited, 622 .call = srcu_torture_call, 623 .cb_barrier = srcu_torture_barrier, 624 .stats = srcu_torture_stats, 625 .name = "srcud" 626 }; 627 628 /* 629 * Definitions for sched torture testing. 630 */ 631 632 static int sched_torture_read_lock(void) 633 { 634 preempt_disable(); 635 return 0; 636 } 637 638 static void sched_torture_read_unlock(int idx) 639 { 640 preempt_enable(); 641 } 642 643 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 644 { 645 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 646 } 647 648 static struct rcu_torture_ops sched_ops = { 649 .ttype = RCU_SCHED_FLAVOR, 650 .init = rcu_sync_torture_init, 651 .readlock = sched_torture_read_lock, 652 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 653 .readunlock = sched_torture_read_unlock, 654 .started = rcu_batches_started_sched, 655 .completed = rcu_batches_completed_sched, 656 .deferred_free = rcu_sched_torture_deferred_free, 657 .sync = synchronize_sched, 658 .exp_sync = synchronize_sched_expedited, 659 .get_state = get_state_synchronize_sched, 660 .cond_sync = cond_synchronize_sched, 661 .call = call_rcu_sched, 662 .cb_barrier = rcu_barrier_sched, 663 .fqs = rcu_sched_force_quiescent_state, 664 .stats = NULL, 665 .irq_capable = 1, 666 .name = "sched" 667 }; 668 669 #ifdef CONFIG_TASKS_RCU 670 671 /* 672 * Definitions for RCU-tasks torture testing. 673 */ 674 675 static int tasks_torture_read_lock(void) 676 { 677 return 0; 678 } 679 680 static void tasks_torture_read_unlock(int idx) 681 { 682 } 683 684 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 685 { 686 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 687 } 688 689 static struct rcu_torture_ops tasks_ops = { 690 .ttype = RCU_TASKS_FLAVOR, 691 .init = rcu_sync_torture_init, 692 .readlock = tasks_torture_read_lock, 693 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 694 .readunlock = tasks_torture_read_unlock, 695 .started = rcu_no_completed, 696 .completed = rcu_no_completed, 697 .deferred_free = rcu_tasks_torture_deferred_free, 698 .sync = synchronize_rcu_tasks, 699 .exp_sync = synchronize_rcu_tasks, 700 .call = call_rcu_tasks, 701 .cb_barrier = rcu_barrier_tasks, 702 .fqs = NULL, 703 .stats = NULL, 704 .irq_capable = 1, 705 .name = "tasks" 706 }; 707 708 #define RCUTORTURE_TASKS_OPS &tasks_ops, 709 710 static bool __maybe_unused torturing_tasks(void) 711 { 712 return cur_ops == &tasks_ops; 713 } 714 715 #else /* #ifdef CONFIG_TASKS_RCU */ 716 717 #define RCUTORTURE_TASKS_OPS 718 719 static bool __maybe_unused torturing_tasks(void) 720 { 721 return false; 722 } 723 724 #endif /* #else #ifdef CONFIG_TASKS_RCU */ 725 726 /* 727 * RCU torture priority-boost testing. Runs one real-time thread per 728 * CPU for moderate bursts, repeatedly registering RCU callbacks and 729 * spinning waiting for them to be invoked. If a given callback takes 730 * too long to be invoked, we assume that priority inversion has occurred. 731 */ 732 733 struct rcu_boost_inflight { 734 struct rcu_head rcu; 735 int inflight; 736 }; 737 738 static void rcu_torture_boost_cb(struct rcu_head *head) 739 { 740 struct rcu_boost_inflight *rbip = 741 container_of(head, struct rcu_boost_inflight, rcu); 742 743 /* Ensure RCU-core accesses precede clearing ->inflight */ 744 smp_store_release(&rbip->inflight, 0); 745 } 746 747 static int rcu_torture_boost(void *arg) 748 { 749 unsigned long call_rcu_time; 750 unsigned long endtime; 751 unsigned long oldstarttime; 752 struct rcu_boost_inflight rbi = { .inflight = 0 }; 753 struct sched_param sp; 754 755 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 756 757 /* Set real-time priority. */ 758 sp.sched_priority = 1; 759 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 760 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 761 n_rcu_torture_boost_rterror++; 762 } 763 764 init_rcu_head_on_stack(&rbi.rcu); 765 /* Each pass through the following loop does one boost-test cycle. */ 766 do { 767 /* Wait for the next test interval. */ 768 oldstarttime = boost_starttime; 769 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 770 schedule_timeout_interruptible(oldstarttime - jiffies); 771 stutter_wait("rcu_torture_boost"); 772 if (torture_must_stop()) 773 goto checkwait; 774 } 775 776 /* Do one boost-test interval. */ 777 endtime = oldstarttime + test_boost_duration * HZ; 778 call_rcu_time = jiffies; 779 while (ULONG_CMP_LT(jiffies, endtime)) { 780 /* If we don't have a callback in flight, post one. */ 781 if (!smp_load_acquire(&rbi.inflight)) { 782 /* RCU core before ->inflight = 1. */ 783 smp_store_release(&rbi.inflight, 1); 784 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 785 if (jiffies - call_rcu_time > 786 test_boost_duration * HZ - HZ / 2) { 787 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 788 n_rcu_torture_boost_failure++; 789 } 790 call_rcu_time = jiffies; 791 } 792 stutter_wait("rcu_torture_boost"); 793 if (torture_must_stop()) 794 goto checkwait; 795 } 796 797 /* 798 * Set the start time of the next test interval. 799 * Yes, this is vulnerable to long delays, but such 800 * delays simply cause a false negative for the next 801 * interval. Besides, we are running at RT priority, 802 * so delays should be relatively rare. 803 */ 804 while (oldstarttime == boost_starttime && 805 !kthread_should_stop()) { 806 if (mutex_trylock(&boost_mutex)) { 807 boost_starttime = jiffies + 808 test_boost_interval * HZ; 809 n_rcu_torture_boosts++; 810 mutex_unlock(&boost_mutex); 811 break; 812 } 813 schedule_timeout_uninterruptible(1); 814 } 815 816 /* Go do the stutter. */ 817 checkwait: stutter_wait("rcu_torture_boost"); 818 } while (!torture_must_stop()); 819 820 /* Clean up and exit. */ 821 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 822 torture_shutdown_absorb("rcu_torture_boost"); 823 schedule_timeout_uninterruptible(1); 824 } 825 destroy_rcu_head_on_stack(&rbi.rcu); 826 torture_kthread_stopping("rcu_torture_boost"); 827 return 0; 828 } 829 830 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 831 { 832 } 833 834 /* 835 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 836 * to call_rcu() or analogous, increasing the probability of occurrence 837 * of callback-overflow corner cases. 838 */ 839 static int 840 rcu_torture_cbflood(void *arg) 841 { 842 int err = 1; 843 int i; 844 int j; 845 struct rcu_head *rhp; 846 847 if (cbflood_n_per_burst > 0 && 848 cbflood_inter_holdoff > 0 && 849 cbflood_intra_holdoff > 0 && 850 cur_ops->call && 851 cur_ops->cb_barrier) { 852 rhp = vmalloc(sizeof(*rhp) * 853 cbflood_n_burst * cbflood_n_per_burst); 854 err = !rhp; 855 } 856 if (err) { 857 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 858 goto wait_for_stop; 859 } 860 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 861 do { 862 schedule_timeout_interruptible(cbflood_inter_holdoff); 863 atomic_long_inc(&n_cbfloods); 864 WARN_ON(signal_pending(current)); 865 for (i = 0; i < cbflood_n_burst; i++) { 866 for (j = 0; j < cbflood_n_per_burst; j++) { 867 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 868 rcu_torture_cbflood_cb); 869 } 870 schedule_timeout_interruptible(cbflood_intra_holdoff); 871 WARN_ON(signal_pending(current)); 872 } 873 cur_ops->cb_barrier(); 874 stutter_wait("rcu_torture_cbflood"); 875 } while (!torture_must_stop()); 876 vfree(rhp); 877 wait_for_stop: 878 torture_kthread_stopping("rcu_torture_cbflood"); 879 return 0; 880 } 881 882 /* 883 * RCU torture force-quiescent-state kthread. Repeatedly induces 884 * bursts of calls to force_quiescent_state(), increasing the probability 885 * of occurrence of some important types of race conditions. 886 */ 887 static int 888 rcu_torture_fqs(void *arg) 889 { 890 unsigned long fqs_resume_time; 891 int fqs_burst_remaining; 892 893 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 894 do { 895 fqs_resume_time = jiffies + fqs_stutter * HZ; 896 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 897 !kthread_should_stop()) { 898 schedule_timeout_interruptible(1); 899 } 900 fqs_burst_remaining = fqs_duration; 901 while (fqs_burst_remaining > 0 && 902 !kthread_should_stop()) { 903 cur_ops->fqs(); 904 udelay(fqs_holdoff); 905 fqs_burst_remaining -= fqs_holdoff; 906 } 907 stutter_wait("rcu_torture_fqs"); 908 } while (!torture_must_stop()); 909 torture_kthread_stopping("rcu_torture_fqs"); 910 return 0; 911 } 912 913 /* 914 * RCU torture writer kthread. Repeatedly substitutes a new structure 915 * for that pointed to by rcu_torture_current, freeing the old structure 916 * after a series of grace periods (the "pipeline"). 917 */ 918 static int 919 rcu_torture_writer(void *arg) 920 { 921 bool can_expedite = !rcu_gp_is_expedited(); 922 int expediting = 0; 923 unsigned long gp_snap; 924 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 925 bool gp_sync1 = gp_sync; 926 int i; 927 struct rcu_torture *rp; 928 struct rcu_torture *old_rp; 929 static DEFINE_TORTURE_RANDOM(rand); 930 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 931 RTWS_COND_GET, RTWS_SYNC }; 932 int nsynctypes = 0; 933 934 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 935 pr_alert("%s" TORTURE_FLAG 936 " Grace periods expedited from boot/sysfs for %s,\n", 937 torture_type, cur_ops->name); 938 pr_alert("%s" TORTURE_FLAG 939 " Testing of dynamic grace-period expediting diabled.\n", 940 torture_type); 941 942 /* Initialize synctype[] array. If none set, take default. */ 943 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 944 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 945 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) 946 synctype[nsynctypes++] = RTWS_COND_GET; 947 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) 948 pr_alert("rcu_torture_writer: gp_cond without primitives.\n"); 949 if (gp_exp1 && cur_ops->exp_sync) 950 synctype[nsynctypes++] = RTWS_EXP_SYNC; 951 else if (gp_exp && !cur_ops->exp_sync) 952 pr_alert("rcu_torture_writer: gp_exp without primitives.\n"); 953 if (gp_normal1 && cur_ops->deferred_free) 954 synctype[nsynctypes++] = RTWS_DEF_FREE; 955 else if (gp_normal && !cur_ops->deferred_free) 956 pr_alert("rcu_torture_writer: gp_normal without primitives.\n"); 957 if (gp_sync1 && cur_ops->sync) 958 synctype[nsynctypes++] = RTWS_SYNC; 959 else if (gp_sync && !cur_ops->sync) 960 pr_alert("rcu_torture_writer: gp_sync without primitives.\n"); 961 if (WARN_ONCE(nsynctypes == 0, 962 "rcu_torture_writer: No update-side primitives.\n")) { 963 /* 964 * No updates primitives, so don't try updating. 965 * The resulting test won't be testing much, hence the 966 * above WARN_ONCE(). 967 */ 968 rcu_torture_writer_state = RTWS_STOPPING; 969 torture_kthread_stopping("rcu_torture_writer"); 970 } 971 972 do { 973 rcu_torture_writer_state = RTWS_FIXED_DELAY; 974 schedule_timeout_uninterruptible(1); 975 rp = rcu_torture_alloc(); 976 if (rp == NULL) 977 continue; 978 rp->rtort_pipe_count = 0; 979 rcu_torture_writer_state = RTWS_DELAY; 980 udelay(torture_random(&rand) & 0x3ff); 981 rcu_torture_writer_state = RTWS_REPLACE; 982 old_rp = rcu_dereference_check(rcu_torture_current, 983 current == writer_task); 984 rp->rtort_mbtest = 1; 985 rcu_assign_pointer(rcu_torture_current, rp); 986 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 987 if (old_rp) { 988 i = old_rp->rtort_pipe_count; 989 if (i > RCU_TORTURE_PIPE_LEN) 990 i = RCU_TORTURE_PIPE_LEN; 991 atomic_inc(&rcu_torture_wcount[i]); 992 old_rp->rtort_pipe_count++; 993 switch (synctype[torture_random(&rand) % nsynctypes]) { 994 case RTWS_DEF_FREE: 995 rcu_torture_writer_state = RTWS_DEF_FREE; 996 cur_ops->deferred_free(old_rp); 997 break; 998 case RTWS_EXP_SYNC: 999 rcu_torture_writer_state = RTWS_EXP_SYNC; 1000 cur_ops->exp_sync(); 1001 rcu_torture_pipe_update(old_rp); 1002 break; 1003 case RTWS_COND_GET: 1004 rcu_torture_writer_state = RTWS_COND_GET; 1005 gp_snap = cur_ops->get_state(); 1006 i = torture_random(&rand) % 16; 1007 if (i != 0) 1008 schedule_timeout_interruptible(i); 1009 udelay(torture_random(&rand) % 1000); 1010 rcu_torture_writer_state = RTWS_COND_SYNC; 1011 cur_ops->cond_sync(gp_snap); 1012 rcu_torture_pipe_update(old_rp); 1013 break; 1014 case RTWS_SYNC: 1015 rcu_torture_writer_state = RTWS_SYNC; 1016 cur_ops->sync(); 1017 rcu_torture_pipe_update(old_rp); 1018 break; 1019 default: 1020 WARN_ON_ONCE(1); 1021 break; 1022 } 1023 } 1024 rcutorture_record_progress(++rcu_torture_current_version); 1025 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1026 if (can_expedite && 1027 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1028 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1029 if (expediting >= 0) 1030 rcu_expedite_gp(); 1031 else 1032 rcu_unexpedite_gp(); 1033 if (++expediting > 3) 1034 expediting = -expediting; 1035 } 1036 rcu_torture_writer_state = RTWS_STUTTER; 1037 stutter_wait("rcu_torture_writer"); 1038 } while (!torture_must_stop()); 1039 /* Reset expediting back to unexpedited. */ 1040 if (expediting > 0) 1041 expediting = -expediting; 1042 while (can_expedite && expediting++ < 0) 1043 rcu_unexpedite_gp(); 1044 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1045 rcu_torture_writer_state = RTWS_STOPPING; 1046 torture_kthread_stopping("rcu_torture_writer"); 1047 return 0; 1048 } 1049 1050 /* 1051 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1052 * delay between calls. 1053 */ 1054 static int 1055 rcu_torture_fakewriter(void *arg) 1056 { 1057 DEFINE_TORTURE_RANDOM(rand); 1058 1059 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1060 set_user_nice(current, MAX_NICE); 1061 1062 do { 1063 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1064 udelay(torture_random(&rand) & 0x3ff); 1065 if (cur_ops->cb_barrier != NULL && 1066 torture_random(&rand) % (nfakewriters * 8) == 0) { 1067 cur_ops->cb_barrier(); 1068 } else if (gp_normal == gp_exp) { 1069 if (torture_random(&rand) & 0x80) 1070 cur_ops->sync(); 1071 else 1072 cur_ops->exp_sync(); 1073 } else if (gp_normal) { 1074 cur_ops->sync(); 1075 } else { 1076 cur_ops->exp_sync(); 1077 } 1078 stutter_wait("rcu_torture_fakewriter"); 1079 } while (!torture_must_stop()); 1080 1081 torture_kthread_stopping("rcu_torture_fakewriter"); 1082 return 0; 1083 } 1084 1085 static void rcutorture_trace_dump(void) 1086 { 1087 static atomic_t beenhere = ATOMIC_INIT(0); 1088 1089 if (atomic_read(&beenhere)) 1090 return; 1091 if (atomic_xchg(&beenhere, 1) != 0) 1092 return; 1093 ftrace_dump(DUMP_ALL); 1094 } 1095 1096 /* 1097 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1098 * incrementing the corresponding element of the pipeline array. The 1099 * counter in the element should never be greater than 1, otherwise, the 1100 * RCU implementation is broken. 1101 */ 1102 static void rcu_torture_timer(unsigned long unused) 1103 { 1104 int idx; 1105 unsigned long started; 1106 unsigned long completed; 1107 static DEFINE_TORTURE_RANDOM(rand); 1108 static DEFINE_SPINLOCK(rand_lock); 1109 struct rcu_torture *p; 1110 int pipe_count; 1111 unsigned long long ts; 1112 1113 idx = cur_ops->readlock(); 1114 if (cur_ops->started) 1115 started = cur_ops->started(); 1116 else 1117 started = cur_ops->completed(); 1118 ts = rcu_trace_clock_local(); 1119 p = rcu_dereference_check(rcu_torture_current, 1120 rcu_read_lock_bh_held() || 1121 rcu_read_lock_sched_held() || 1122 srcu_read_lock_held(srcu_ctlp) || 1123 torturing_tasks()); 1124 if (p == NULL) { 1125 /* Leave because rcu_torture_writer is not yet underway */ 1126 cur_ops->readunlock(idx); 1127 return; 1128 } 1129 if (p->rtort_mbtest == 0) 1130 atomic_inc(&n_rcu_torture_mberror); 1131 spin_lock(&rand_lock); 1132 cur_ops->read_delay(&rand); 1133 n_rcu_torture_timers++; 1134 spin_unlock(&rand_lock); 1135 preempt_disable(); 1136 pipe_count = p->rtort_pipe_count; 1137 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1138 /* Should not happen, but... */ 1139 pipe_count = RCU_TORTURE_PIPE_LEN; 1140 } 1141 completed = cur_ops->completed(); 1142 if (pipe_count > 1) { 1143 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1144 started, completed); 1145 rcutorture_trace_dump(); 1146 } 1147 __this_cpu_inc(rcu_torture_count[pipe_count]); 1148 completed = completed - started; 1149 if (cur_ops->started) 1150 completed++; 1151 if (completed > RCU_TORTURE_PIPE_LEN) { 1152 /* Should not happen, but... */ 1153 completed = RCU_TORTURE_PIPE_LEN; 1154 } 1155 __this_cpu_inc(rcu_torture_batch[completed]); 1156 preempt_enable(); 1157 cur_ops->readunlock(idx); 1158 } 1159 1160 /* 1161 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1162 * incrementing the corresponding element of the pipeline array. The 1163 * counter in the element should never be greater than 1, otherwise, the 1164 * RCU implementation is broken. 1165 */ 1166 static int 1167 rcu_torture_reader(void *arg) 1168 { 1169 unsigned long started; 1170 unsigned long completed; 1171 int idx; 1172 DEFINE_TORTURE_RANDOM(rand); 1173 struct rcu_torture *p; 1174 int pipe_count; 1175 struct timer_list t; 1176 unsigned long long ts; 1177 1178 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1179 set_user_nice(current, MAX_NICE); 1180 if (irqreader && cur_ops->irq_capable) 1181 setup_timer_on_stack(&t, rcu_torture_timer, 0); 1182 1183 do { 1184 if (irqreader && cur_ops->irq_capable) { 1185 if (!timer_pending(&t)) 1186 mod_timer(&t, jiffies + 1); 1187 } 1188 idx = cur_ops->readlock(); 1189 if (cur_ops->started) 1190 started = cur_ops->started(); 1191 else 1192 started = cur_ops->completed(); 1193 ts = rcu_trace_clock_local(); 1194 p = rcu_dereference_check(rcu_torture_current, 1195 rcu_read_lock_bh_held() || 1196 rcu_read_lock_sched_held() || 1197 srcu_read_lock_held(srcu_ctlp) || 1198 torturing_tasks()); 1199 if (p == NULL) { 1200 /* Wait for rcu_torture_writer to get underway */ 1201 cur_ops->readunlock(idx); 1202 schedule_timeout_interruptible(HZ); 1203 continue; 1204 } 1205 if (p->rtort_mbtest == 0) 1206 atomic_inc(&n_rcu_torture_mberror); 1207 cur_ops->read_delay(&rand); 1208 preempt_disable(); 1209 pipe_count = p->rtort_pipe_count; 1210 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1211 /* Should not happen, but... */ 1212 pipe_count = RCU_TORTURE_PIPE_LEN; 1213 } 1214 completed = cur_ops->completed(); 1215 if (pipe_count > 1) { 1216 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1217 ts, started, completed); 1218 rcutorture_trace_dump(); 1219 } 1220 __this_cpu_inc(rcu_torture_count[pipe_count]); 1221 completed = completed - started; 1222 if (cur_ops->started) 1223 completed++; 1224 if (completed > RCU_TORTURE_PIPE_LEN) { 1225 /* Should not happen, but... */ 1226 completed = RCU_TORTURE_PIPE_LEN; 1227 } 1228 __this_cpu_inc(rcu_torture_batch[completed]); 1229 preempt_enable(); 1230 cur_ops->readunlock(idx); 1231 stutter_wait("rcu_torture_reader"); 1232 } while (!torture_must_stop()); 1233 if (irqreader && cur_ops->irq_capable) { 1234 del_timer_sync(&t); 1235 destroy_timer_on_stack(&t); 1236 } 1237 torture_kthread_stopping("rcu_torture_reader"); 1238 return 0; 1239 } 1240 1241 /* 1242 * Print torture statistics. Caller must ensure that there is only 1243 * one call to this function at a given time!!! This is normally 1244 * accomplished by relying on the module system to only have one copy 1245 * of the module loaded, and then by giving the rcu_torture_stats 1246 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1247 * thread is not running). 1248 */ 1249 static void 1250 rcu_torture_stats_print(void) 1251 { 1252 int cpu; 1253 int i; 1254 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1255 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1256 static unsigned long rtcv_snap = ULONG_MAX; 1257 1258 for_each_possible_cpu(cpu) { 1259 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1260 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1261 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1262 } 1263 } 1264 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1265 if (pipesummary[i] != 0) 1266 break; 1267 } 1268 1269 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1270 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1271 rcu_torture_current, 1272 rcu_torture_current_version, 1273 list_empty(&rcu_torture_freelist), 1274 atomic_read(&n_rcu_torture_alloc), 1275 atomic_read(&n_rcu_torture_alloc_fail), 1276 atomic_read(&n_rcu_torture_free)); 1277 pr_cont("rtmbe: %d rtbke: %ld rtbre: %ld ", 1278 atomic_read(&n_rcu_torture_mberror), 1279 n_rcu_torture_boost_ktrerror, 1280 n_rcu_torture_boost_rterror); 1281 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1282 n_rcu_torture_boost_failure, 1283 n_rcu_torture_boosts, 1284 n_rcu_torture_timers); 1285 torture_onoff_stats(); 1286 pr_cont("barrier: %ld/%ld:%ld ", 1287 n_barrier_successes, 1288 n_barrier_attempts, 1289 n_rcu_torture_barrier_error); 1290 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1291 1292 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1293 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1294 n_rcu_torture_barrier_error != 0 || 1295 n_rcu_torture_boost_ktrerror != 0 || 1296 n_rcu_torture_boost_rterror != 0 || 1297 n_rcu_torture_boost_failure != 0 || 1298 i > 1) { 1299 pr_cont("%s", "!!! "); 1300 atomic_inc(&n_rcu_torture_error); 1301 WARN_ON_ONCE(1); 1302 } 1303 pr_cont("Reader Pipe: "); 1304 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1305 pr_cont(" %ld", pipesummary[i]); 1306 pr_cont("\n"); 1307 1308 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1309 pr_cont("Reader Batch: "); 1310 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1311 pr_cont(" %ld", batchsummary[i]); 1312 pr_cont("\n"); 1313 1314 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1315 pr_cont("Free-Block Circulation: "); 1316 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1317 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1318 } 1319 pr_cont("\n"); 1320 1321 if (cur_ops->stats) 1322 cur_ops->stats(); 1323 if (rtcv_snap == rcu_torture_current_version && 1324 rcu_torture_current != NULL) { 1325 int __maybe_unused flags; 1326 unsigned long __maybe_unused gpnum; 1327 unsigned long __maybe_unused completed; 1328 1329 rcutorture_get_gp_data(cur_ops->ttype, 1330 &flags, &gpnum, &completed); 1331 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x\n", 1332 rcu_torture_writer_state_getname(), 1333 rcu_torture_writer_state, 1334 gpnum, completed, flags); 1335 show_rcu_gp_kthreads(); 1336 rcutorture_trace_dump(); 1337 } 1338 rtcv_snap = rcu_torture_current_version; 1339 } 1340 1341 /* 1342 * Periodically prints torture statistics, if periodic statistics printing 1343 * was specified via the stat_interval module parameter. 1344 */ 1345 static int 1346 rcu_torture_stats(void *arg) 1347 { 1348 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1349 do { 1350 schedule_timeout_interruptible(stat_interval * HZ); 1351 rcu_torture_stats_print(); 1352 torture_shutdown_absorb("rcu_torture_stats"); 1353 } while (!torture_must_stop()); 1354 torture_kthread_stopping("rcu_torture_stats"); 1355 return 0; 1356 } 1357 1358 static inline void 1359 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1360 { 1361 pr_alert("%s" TORTURE_FLAG 1362 "--- %s: nreaders=%d nfakewriters=%d " 1363 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1364 "shuffle_interval=%d stutter=%d irqreader=%d " 1365 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1366 "test_boost=%d/%d test_boost_interval=%d " 1367 "test_boost_duration=%d shutdown_secs=%d " 1368 "stall_cpu=%d stall_cpu_holdoff=%d " 1369 "n_barrier_cbs=%d " 1370 "onoff_interval=%d onoff_holdoff=%d\n", 1371 torture_type, tag, nrealreaders, nfakewriters, 1372 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1373 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1374 test_boost, cur_ops->can_boost, 1375 test_boost_interval, test_boost_duration, shutdown_secs, 1376 stall_cpu, stall_cpu_holdoff, 1377 n_barrier_cbs, 1378 onoff_interval, onoff_holdoff); 1379 } 1380 1381 static void rcutorture_booster_cleanup(int cpu) 1382 { 1383 struct task_struct *t; 1384 1385 if (boost_tasks[cpu] == NULL) 1386 return; 1387 mutex_lock(&boost_mutex); 1388 t = boost_tasks[cpu]; 1389 boost_tasks[cpu] = NULL; 1390 mutex_unlock(&boost_mutex); 1391 1392 /* This must be outside of the mutex, otherwise deadlock! */ 1393 torture_stop_kthread(rcu_torture_boost, t); 1394 } 1395 1396 static int rcutorture_booster_init(int cpu) 1397 { 1398 int retval; 1399 1400 if (boost_tasks[cpu] != NULL) 1401 return 0; /* Already created, nothing more to do. */ 1402 1403 /* Don't allow time recalculation while creating a new task. */ 1404 mutex_lock(&boost_mutex); 1405 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1406 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1407 cpu_to_node(cpu), 1408 "rcu_torture_boost"); 1409 if (IS_ERR(boost_tasks[cpu])) { 1410 retval = PTR_ERR(boost_tasks[cpu]); 1411 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1412 n_rcu_torture_boost_ktrerror++; 1413 boost_tasks[cpu] = NULL; 1414 mutex_unlock(&boost_mutex); 1415 return retval; 1416 } 1417 kthread_bind(boost_tasks[cpu], cpu); 1418 wake_up_process(boost_tasks[cpu]); 1419 mutex_unlock(&boost_mutex); 1420 return 0; 1421 } 1422 1423 /* 1424 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1425 * induces a CPU stall for the time specified by stall_cpu. 1426 */ 1427 static int rcu_torture_stall(void *args) 1428 { 1429 unsigned long stop_at; 1430 1431 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1432 if (stall_cpu_holdoff > 0) { 1433 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1434 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1435 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1436 } 1437 if (!kthread_should_stop()) { 1438 stop_at = get_seconds() + stall_cpu; 1439 /* RCU CPU stall is expected behavior in following code. */ 1440 pr_alert("rcu_torture_stall start.\n"); 1441 rcu_read_lock(); 1442 preempt_disable(); 1443 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1444 continue; /* Induce RCU CPU stall warning. */ 1445 preempt_enable(); 1446 rcu_read_unlock(); 1447 pr_alert("rcu_torture_stall end.\n"); 1448 } 1449 torture_shutdown_absorb("rcu_torture_stall"); 1450 while (!kthread_should_stop()) 1451 schedule_timeout_interruptible(10 * HZ); 1452 return 0; 1453 } 1454 1455 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1456 static int __init rcu_torture_stall_init(void) 1457 { 1458 if (stall_cpu <= 0) 1459 return 0; 1460 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1461 } 1462 1463 /* Callback function for RCU barrier testing. */ 1464 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1465 { 1466 atomic_inc(&barrier_cbs_invoked); 1467 } 1468 1469 /* kthread function to register callbacks used to test RCU barriers. */ 1470 static int rcu_torture_barrier_cbs(void *arg) 1471 { 1472 long myid = (long)arg; 1473 bool lastphase = 0; 1474 bool newphase; 1475 struct rcu_head rcu; 1476 1477 init_rcu_head_on_stack(&rcu); 1478 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1479 set_user_nice(current, MAX_NICE); 1480 do { 1481 wait_event(barrier_cbs_wq[myid], 1482 (newphase = 1483 smp_load_acquire(&barrier_phase)) != lastphase || 1484 torture_must_stop()); 1485 lastphase = newphase; 1486 if (torture_must_stop()) 1487 break; 1488 /* 1489 * The above smp_load_acquire() ensures barrier_phase load 1490 * is ordered before the folloiwng ->call(). 1491 */ 1492 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1493 if (atomic_dec_and_test(&barrier_cbs_count)) 1494 wake_up(&barrier_wq); 1495 } while (!torture_must_stop()); 1496 if (cur_ops->cb_barrier != NULL) 1497 cur_ops->cb_barrier(); 1498 destroy_rcu_head_on_stack(&rcu); 1499 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1500 return 0; 1501 } 1502 1503 /* kthread function to drive and coordinate RCU barrier testing. */ 1504 static int rcu_torture_barrier(void *arg) 1505 { 1506 int i; 1507 1508 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1509 do { 1510 atomic_set(&barrier_cbs_invoked, 0); 1511 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1512 /* Ensure barrier_phase ordered after prior assignments. */ 1513 smp_store_release(&barrier_phase, !barrier_phase); 1514 for (i = 0; i < n_barrier_cbs; i++) 1515 wake_up(&barrier_cbs_wq[i]); 1516 wait_event(barrier_wq, 1517 atomic_read(&barrier_cbs_count) == 0 || 1518 torture_must_stop()); 1519 if (torture_must_stop()) 1520 break; 1521 n_barrier_attempts++; 1522 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1523 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1524 n_rcu_torture_barrier_error++; 1525 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 1526 atomic_read(&barrier_cbs_invoked), 1527 n_barrier_cbs); 1528 WARN_ON_ONCE(1); 1529 } 1530 n_barrier_successes++; 1531 schedule_timeout_interruptible(HZ / 10); 1532 } while (!torture_must_stop()); 1533 torture_kthread_stopping("rcu_torture_barrier"); 1534 return 0; 1535 } 1536 1537 /* Initialize RCU barrier testing. */ 1538 static int rcu_torture_barrier_init(void) 1539 { 1540 int i; 1541 int ret; 1542 1543 if (n_barrier_cbs <= 0) 1544 return 0; 1545 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1546 pr_alert("%s" TORTURE_FLAG 1547 " Call or barrier ops missing for %s,\n", 1548 torture_type, cur_ops->name); 1549 pr_alert("%s" TORTURE_FLAG 1550 " RCU barrier testing omitted from run.\n", 1551 torture_type); 1552 return 0; 1553 } 1554 atomic_set(&barrier_cbs_count, 0); 1555 atomic_set(&barrier_cbs_invoked, 0); 1556 barrier_cbs_tasks = 1557 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1558 GFP_KERNEL); 1559 barrier_cbs_wq = 1560 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1561 GFP_KERNEL); 1562 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1563 return -ENOMEM; 1564 for (i = 0; i < n_barrier_cbs; i++) { 1565 init_waitqueue_head(&barrier_cbs_wq[i]); 1566 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1567 (void *)(long)i, 1568 barrier_cbs_tasks[i]); 1569 if (ret) 1570 return ret; 1571 } 1572 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1573 } 1574 1575 /* Clean up after RCU barrier testing. */ 1576 static void rcu_torture_barrier_cleanup(void) 1577 { 1578 int i; 1579 1580 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1581 if (barrier_cbs_tasks != NULL) { 1582 for (i = 0; i < n_barrier_cbs; i++) 1583 torture_stop_kthread(rcu_torture_barrier_cbs, 1584 barrier_cbs_tasks[i]); 1585 kfree(barrier_cbs_tasks); 1586 barrier_cbs_tasks = NULL; 1587 } 1588 if (barrier_cbs_wq != NULL) { 1589 kfree(barrier_cbs_wq); 1590 barrier_cbs_wq = NULL; 1591 } 1592 } 1593 1594 static int rcutorture_cpu_notify(struct notifier_block *self, 1595 unsigned long action, void *hcpu) 1596 { 1597 long cpu = (long)hcpu; 1598 1599 switch (action) { 1600 case CPU_ONLINE: 1601 case CPU_DOWN_FAILED: 1602 (void)rcutorture_booster_init(cpu); 1603 break; 1604 case CPU_DOWN_PREPARE: 1605 rcutorture_booster_cleanup(cpu); 1606 break; 1607 default: 1608 break; 1609 } 1610 return NOTIFY_OK; 1611 } 1612 1613 static struct notifier_block rcutorture_cpu_nb = { 1614 .notifier_call = rcutorture_cpu_notify, 1615 }; 1616 1617 static void 1618 rcu_torture_cleanup(void) 1619 { 1620 int i; 1621 1622 rcutorture_record_test_transition(); 1623 if (torture_cleanup_begin()) { 1624 if (cur_ops->cb_barrier != NULL) 1625 cur_ops->cb_barrier(); 1626 return; 1627 } 1628 1629 rcu_torture_barrier_cleanup(); 1630 torture_stop_kthread(rcu_torture_stall, stall_task); 1631 torture_stop_kthread(rcu_torture_writer, writer_task); 1632 1633 if (reader_tasks) { 1634 for (i = 0; i < nrealreaders; i++) 1635 torture_stop_kthread(rcu_torture_reader, 1636 reader_tasks[i]); 1637 kfree(reader_tasks); 1638 } 1639 rcu_torture_current = NULL; 1640 1641 if (fakewriter_tasks) { 1642 for (i = 0; i < nfakewriters; i++) { 1643 torture_stop_kthread(rcu_torture_fakewriter, 1644 fakewriter_tasks[i]); 1645 } 1646 kfree(fakewriter_tasks); 1647 fakewriter_tasks = NULL; 1648 } 1649 1650 torture_stop_kthread(rcu_torture_stats, stats_task); 1651 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1652 for (i = 0; i < ncbflooders; i++) 1653 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1654 if ((test_boost == 1 && cur_ops->can_boost) || 1655 test_boost == 2) { 1656 unregister_cpu_notifier(&rcutorture_cpu_nb); 1657 for_each_possible_cpu(i) 1658 rcutorture_booster_cleanup(i); 1659 } 1660 1661 /* 1662 * Wait for all RCU callbacks to fire, then do flavor-specific 1663 * cleanup operations. 1664 */ 1665 if (cur_ops->cb_barrier != NULL) 1666 cur_ops->cb_barrier(); 1667 if (cur_ops->cleanup != NULL) 1668 cur_ops->cleanup(); 1669 1670 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1671 1672 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1673 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1674 else if (torture_onoff_failures()) 1675 rcu_torture_print_module_parms(cur_ops, 1676 "End of test: RCU_HOTPLUG"); 1677 else 1678 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1679 torture_cleanup_end(); 1680 } 1681 1682 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1683 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1684 { 1685 } 1686 1687 static void rcu_torture_err_cb(struct rcu_head *rhp) 1688 { 1689 /* 1690 * This -might- happen due to race conditions, but is unlikely. 1691 * The scenario that leads to this happening is that the 1692 * first of the pair of duplicate callbacks is queued, 1693 * someone else starts a grace period that includes that 1694 * callback, then the second of the pair must wait for the 1695 * next grace period. Unlikely, but can happen. If it 1696 * does happen, the debug-objects subsystem won't have splatted. 1697 */ 1698 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1699 } 1700 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1701 1702 /* 1703 * Verify that double-free causes debug-objects to complain, but only 1704 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1705 * cannot be carried out. 1706 */ 1707 static void rcu_test_debug_objects(void) 1708 { 1709 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1710 struct rcu_head rh1; 1711 struct rcu_head rh2; 1712 1713 init_rcu_head_on_stack(&rh1); 1714 init_rcu_head_on_stack(&rh2); 1715 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1716 1717 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1718 preempt_disable(); /* Prevent preemption from interrupting test. */ 1719 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1720 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1721 local_irq_disable(); /* Make it harder to start a new grace period. */ 1722 call_rcu(&rh2, rcu_torture_leak_cb); 1723 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1724 local_irq_enable(); 1725 rcu_read_unlock(); 1726 preempt_enable(); 1727 1728 /* Wait for them all to get done so we can safely return. */ 1729 rcu_barrier(); 1730 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1731 destroy_rcu_head_on_stack(&rh1); 1732 destroy_rcu_head_on_stack(&rh2); 1733 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1734 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1735 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1736 } 1737 1738 static int __init 1739 rcu_torture_init(void) 1740 { 1741 int i; 1742 int cpu; 1743 int firsterr = 0; 1744 static struct rcu_torture_ops *torture_ops[] = { 1745 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 1746 &sched_ops, RCUTORTURE_TASKS_OPS 1747 }; 1748 1749 if (!torture_init_begin(torture_type, verbose, &torture_runnable)) 1750 return -EBUSY; 1751 1752 /* Process args and tell the world that the torturer is on the job. */ 1753 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1754 cur_ops = torture_ops[i]; 1755 if (strcmp(torture_type, cur_ops->name) == 0) 1756 break; 1757 } 1758 if (i == ARRAY_SIZE(torture_ops)) { 1759 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1760 torture_type); 1761 pr_alert("rcu-torture types:"); 1762 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1763 pr_alert(" %s", torture_ops[i]->name); 1764 pr_alert("\n"); 1765 firsterr = -EINVAL; 1766 goto unwind; 1767 } 1768 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1769 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1770 fqs_duration = 0; 1771 } 1772 if (cur_ops->init) 1773 cur_ops->init(); 1774 1775 if (nreaders >= 0) { 1776 nrealreaders = nreaders; 1777 } else { 1778 nrealreaders = num_online_cpus() - 2 - nreaders; 1779 if (nrealreaders <= 0) 1780 nrealreaders = 1; 1781 } 1782 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1783 1784 /* Set up the freelist. */ 1785 1786 INIT_LIST_HEAD(&rcu_torture_freelist); 1787 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1788 rcu_tortures[i].rtort_mbtest = 0; 1789 list_add_tail(&rcu_tortures[i].rtort_free, 1790 &rcu_torture_freelist); 1791 } 1792 1793 /* Initialize the statistics so that each run gets its own numbers. */ 1794 1795 rcu_torture_current = NULL; 1796 rcu_torture_current_version = 0; 1797 atomic_set(&n_rcu_torture_alloc, 0); 1798 atomic_set(&n_rcu_torture_alloc_fail, 0); 1799 atomic_set(&n_rcu_torture_free, 0); 1800 atomic_set(&n_rcu_torture_mberror, 0); 1801 atomic_set(&n_rcu_torture_error, 0); 1802 n_rcu_torture_barrier_error = 0; 1803 n_rcu_torture_boost_ktrerror = 0; 1804 n_rcu_torture_boost_rterror = 0; 1805 n_rcu_torture_boost_failure = 0; 1806 n_rcu_torture_boosts = 0; 1807 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1808 atomic_set(&rcu_torture_wcount[i], 0); 1809 for_each_possible_cpu(cpu) { 1810 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1811 per_cpu(rcu_torture_count, cpu)[i] = 0; 1812 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1813 } 1814 } 1815 1816 /* Start up the kthreads. */ 1817 1818 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1819 writer_task); 1820 if (firsterr) 1821 goto unwind; 1822 if (nfakewriters > 0) { 1823 fakewriter_tasks = kzalloc(nfakewriters * 1824 sizeof(fakewriter_tasks[0]), 1825 GFP_KERNEL); 1826 if (fakewriter_tasks == NULL) { 1827 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1828 firsterr = -ENOMEM; 1829 goto unwind; 1830 } 1831 } 1832 for (i = 0; i < nfakewriters; i++) { 1833 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1834 NULL, fakewriter_tasks[i]); 1835 if (firsterr) 1836 goto unwind; 1837 } 1838 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1839 GFP_KERNEL); 1840 if (reader_tasks == NULL) { 1841 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1842 firsterr = -ENOMEM; 1843 goto unwind; 1844 } 1845 for (i = 0; i < nrealreaders; i++) { 1846 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1847 reader_tasks[i]); 1848 if (firsterr) 1849 goto unwind; 1850 } 1851 if (stat_interval > 0) { 1852 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1853 stats_task); 1854 if (firsterr) 1855 goto unwind; 1856 } 1857 if (test_no_idle_hz && shuffle_interval > 0) { 1858 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1859 if (firsterr) 1860 goto unwind; 1861 } 1862 if (stutter < 0) 1863 stutter = 0; 1864 if (stutter) { 1865 firsterr = torture_stutter_init(stutter * HZ); 1866 if (firsterr) 1867 goto unwind; 1868 } 1869 if (fqs_duration < 0) 1870 fqs_duration = 0; 1871 if (fqs_duration) { 1872 /* Create the fqs thread */ 1873 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1874 fqs_task); 1875 if (firsterr) 1876 goto unwind; 1877 } 1878 if (test_boost_interval < 1) 1879 test_boost_interval = 1; 1880 if (test_boost_duration < 2) 1881 test_boost_duration = 2; 1882 if ((test_boost == 1 && cur_ops->can_boost) || 1883 test_boost == 2) { 1884 1885 boost_starttime = jiffies + test_boost_interval * HZ; 1886 register_cpu_notifier(&rcutorture_cpu_nb); 1887 for_each_possible_cpu(i) { 1888 if (cpu_is_offline(i)) 1889 continue; /* Heuristic: CPU can go offline. */ 1890 firsterr = rcutorture_booster_init(i); 1891 if (firsterr) 1892 goto unwind; 1893 } 1894 } 1895 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1896 if (firsterr) 1897 goto unwind; 1898 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1899 if (firsterr) 1900 goto unwind; 1901 firsterr = rcu_torture_stall_init(); 1902 if (firsterr) 1903 goto unwind; 1904 firsterr = rcu_torture_barrier_init(); 1905 if (firsterr) 1906 goto unwind; 1907 if (object_debug) 1908 rcu_test_debug_objects(); 1909 if (cbflood_n_burst > 0) { 1910 /* Create the cbflood threads */ 1911 ncbflooders = (num_online_cpus() + 3) / 4; 1912 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1913 GFP_KERNEL); 1914 if (!cbflood_task) { 1915 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1916 firsterr = -ENOMEM; 1917 goto unwind; 1918 } 1919 for (i = 0; i < ncbflooders; i++) { 1920 firsterr = torture_create_kthread(rcu_torture_cbflood, 1921 NULL, 1922 cbflood_task[i]); 1923 if (firsterr) 1924 goto unwind; 1925 } 1926 } 1927 rcutorture_record_test_transition(); 1928 torture_init_end(); 1929 return 0; 1930 1931 unwind: 1932 torture_init_end(); 1933 rcu_torture_cleanup(); 1934 return firsterr; 1935 } 1936 1937 module_init(rcu_torture_init); 1938 module_exit(rcu_torture_cleanup); 1939