1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched/signal.h> 36 #include <uapi/linux/sched/types.h> 37 #include <linux/atomic.h> 38 #include <linux/bitops.h> 39 #include <linux/completion.h> 40 #include <linux/moduleparam.h> 41 #include <linux/percpu.h> 42 #include <linux/notifier.h> 43 #include <linux/reboot.h> 44 #include <linux/freezer.h> 45 #include <linux/cpu.h> 46 #include <linux/delay.h> 47 #include <linux/stat.h> 48 #include <linux/srcu.h> 49 #include <linux/slab.h> 50 #include <linux/trace_clock.h> 51 #include <asm/byteorder.h> 52 #include <linux/torture.h> 53 #include <linux/vmalloc.h> 54 #include <linux/sched/debug.h> 55 56 #include "rcu.h" 57 58 MODULE_LICENSE("GPL"); 59 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 60 61 62 torture_param(int, cbflood_inter_holdoff, HZ, 63 "Holdoff between floods (jiffies)"); 64 torture_param(int, cbflood_intra_holdoff, 1, 65 "Holdoff between bursts (jiffies)"); 66 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 67 torture_param(int, cbflood_n_per_burst, 20000, 68 "# callbacks per burst in flood"); 69 torture_param(int, fqs_duration, 0, 70 "Duration of fqs bursts (us), 0 to disable"); 71 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 72 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 73 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 74 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 75 torture_param(bool, gp_normal, false, 76 "Use normal (non-expedited) GP wait primitives"); 77 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 78 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 79 torture_param(int, n_barrier_cbs, 0, 80 "# of callbacks/kthreads for barrier testing"); 81 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 82 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 83 torture_param(int, object_debug, 0, 84 "Enable debug-object double call_rcu() testing"); 85 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 86 torture_param(int, onoff_interval, 0, 87 "Time between CPU hotplugs (s), 0=disable"); 88 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 89 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 90 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 91 torture_param(int, stall_cpu_holdoff, 10, 92 "Time to wait before starting stall (s)."); 93 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 94 torture_param(int, stat_interval, 60, 95 "Number of seconds between stats printk()s"); 96 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 97 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 98 torture_param(int, test_boost_duration, 4, 99 "Duration of each boost test, seconds."); 100 torture_param(int, test_boost_interval, 7, 101 "Interval between boost tests, seconds."); 102 torture_param(bool, test_no_idle_hz, true, 103 "Test support for tickless idle CPUs"); 104 torture_param(bool, verbose, true, 105 "Enable verbose debugging printk()s"); 106 107 static char *torture_type = "rcu"; 108 module_param(torture_type, charp, 0444); 109 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 110 111 static int nrealreaders; 112 static int ncbflooders; 113 static struct task_struct *writer_task; 114 static struct task_struct **fakewriter_tasks; 115 static struct task_struct **reader_tasks; 116 static struct task_struct *stats_task; 117 static struct task_struct **cbflood_task; 118 static struct task_struct *fqs_task; 119 static struct task_struct *boost_tasks[NR_CPUS]; 120 static struct task_struct *stall_task; 121 static struct task_struct **barrier_cbs_tasks; 122 static struct task_struct *barrier_task; 123 124 #define RCU_TORTURE_PIPE_LEN 10 125 126 struct rcu_torture { 127 struct rcu_head rtort_rcu; 128 int rtort_pipe_count; 129 struct list_head rtort_free; 130 int rtort_mbtest; 131 }; 132 133 static LIST_HEAD(rcu_torture_freelist); 134 static struct rcu_torture __rcu *rcu_torture_current; 135 static unsigned long rcu_torture_current_version; 136 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 137 static DEFINE_SPINLOCK(rcu_torture_lock); 138 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 139 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 140 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 141 static atomic_t n_rcu_torture_alloc; 142 static atomic_t n_rcu_torture_alloc_fail; 143 static atomic_t n_rcu_torture_free; 144 static atomic_t n_rcu_torture_mberror; 145 static atomic_t n_rcu_torture_error; 146 static long n_rcu_torture_barrier_error; 147 static long n_rcu_torture_boost_ktrerror; 148 static long n_rcu_torture_boost_rterror; 149 static long n_rcu_torture_boost_failure; 150 static long n_rcu_torture_boosts; 151 static long n_rcu_torture_timers; 152 static long n_barrier_attempts; 153 static long n_barrier_successes; 154 static atomic_long_t n_cbfloods; 155 static struct list_head rcu_torture_removed; 156 157 static int rcu_torture_writer_state; 158 #define RTWS_FIXED_DELAY 0 159 #define RTWS_DELAY 1 160 #define RTWS_REPLACE 2 161 #define RTWS_DEF_FREE 3 162 #define RTWS_EXP_SYNC 4 163 #define RTWS_COND_GET 5 164 #define RTWS_COND_SYNC 6 165 #define RTWS_SYNC 7 166 #define RTWS_STUTTER 8 167 #define RTWS_STOPPING 9 168 static const char * const rcu_torture_writer_state_names[] = { 169 "RTWS_FIXED_DELAY", 170 "RTWS_DELAY", 171 "RTWS_REPLACE", 172 "RTWS_DEF_FREE", 173 "RTWS_EXP_SYNC", 174 "RTWS_COND_GET", 175 "RTWS_COND_SYNC", 176 "RTWS_SYNC", 177 "RTWS_STUTTER", 178 "RTWS_STOPPING", 179 }; 180 181 static const char *rcu_torture_writer_state_getname(void) 182 { 183 unsigned int i = READ_ONCE(rcu_torture_writer_state); 184 185 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 186 return "???"; 187 return rcu_torture_writer_state_names[i]; 188 } 189 190 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 191 #define rcu_can_boost() 1 192 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 193 #define rcu_can_boost() 0 194 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 195 196 #ifdef CONFIG_RCU_TRACE 197 static u64 notrace rcu_trace_clock_local(void) 198 { 199 u64 ts = trace_clock_local(); 200 201 (void)do_div(ts, NSEC_PER_USEC); 202 return ts; 203 } 204 #else /* #ifdef CONFIG_RCU_TRACE */ 205 static u64 notrace rcu_trace_clock_local(void) 206 { 207 return 0ULL; 208 } 209 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 210 211 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 212 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 213 /* and boost task create/destroy. */ 214 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 215 static bool barrier_phase; /* Test phase. */ 216 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 217 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 218 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 219 220 /* 221 * Allocate an element from the rcu_tortures pool. 222 */ 223 static struct rcu_torture * 224 rcu_torture_alloc(void) 225 { 226 struct list_head *p; 227 228 spin_lock_bh(&rcu_torture_lock); 229 if (list_empty(&rcu_torture_freelist)) { 230 atomic_inc(&n_rcu_torture_alloc_fail); 231 spin_unlock_bh(&rcu_torture_lock); 232 return NULL; 233 } 234 atomic_inc(&n_rcu_torture_alloc); 235 p = rcu_torture_freelist.next; 236 list_del_init(p); 237 spin_unlock_bh(&rcu_torture_lock); 238 return container_of(p, struct rcu_torture, rtort_free); 239 } 240 241 /* 242 * Free an element to the rcu_tortures pool. 243 */ 244 static void 245 rcu_torture_free(struct rcu_torture *p) 246 { 247 atomic_inc(&n_rcu_torture_free); 248 spin_lock_bh(&rcu_torture_lock); 249 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 250 spin_unlock_bh(&rcu_torture_lock); 251 } 252 253 /* 254 * Operations vector for selecting different types of tests. 255 */ 256 257 struct rcu_torture_ops { 258 int ttype; 259 void (*init)(void); 260 void (*cleanup)(void); 261 int (*readlock)(void); 262 void (*read_delay)(struct torture_random_state *rrsp); 263 void (*readunlock)(int idx); 264 unsigned long (*started)(void); 265 unsigned long (*completed)(void); 266 void (*deferred_free)(struct rcu_torture *p); 267 void (*sync)(void); 268 void (*exp_sync)(void); 269 unsigned long (*get_state)(void); 270 void (*cond_sync)(unsigned long oldstate); 271 call_rcu_func_t call; 272 void (*cb_barrier)(void); 273 void (*fqs)(void); 274 void (*stats)(void); 275 int irq_capable; 276 int can_boost; 277 const char *name; 278 }; 279 280 static struct rcu_torture_ops *cur_ops; 281 282 /* 283 * Definitions for rcu torture testing. 284 */ 285 286 static int rcu_torture_read_lock(void) __acquires(RCU) 287 { 288 rcu_read_lock(); 289 return 0; 290 } 291 292 static void rcu_read_delay(struct torture_random_state *rrsp) 293 { 294 unsigned long started; 295 unsigned long completed; 296 const unsigned long shortdelay_us = 200; 297 const unsigned long longdelay_ms = 50; 298 unsigned long long ts; 299 300 /* We want a short delay sometimes to make a reader delay the grace 301 * period, and we want a long delay occasionally to trigger 302 * force_quiescent_state. */ 303 304 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 305 started = cur_ops->completed(); 306 ts = rcu_trace_clock_local(); 307 mdelay(longdelay_ms); 308 completed = cur_ops->completed(); 309 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 310 started, completed); 311 } 312 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 313 udelay(shortdelay_us); 314 if (!preempt_count() && 315 !(torture_random(rrsp) % (nrealreaders * 500))) 316 torture_preempt_schedule(); /* QS only if preemptible. */ 317 } 318 319 static void rcu_torture_read_unlock(int idx) __releases(RCU) 320 { 321 rcu_read_unlock(); 322 } 323 324 /* 325 * Update callback in the pipe. This should be invoked after a grace period. 326 */ 327 static bool 328 rcu_torture_pipe_update_one(struct rcu_torture *rp) 329 { 330 int i; 331 332 i = rp->rtort_pipe_count; 333 if (i > RCU_TORTURE_PIPE_LEN) 334 i = RCU_TORTURE_PIPE_LEN; 335 atomic_inc(&rcu_torture_wcount[i]); 336 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 337 rp->rtort_mbtest = 0; 338 return true; 339 } 340 return false; 341 } 342 343 /* 344 * Update all callbacks in the pipe. Suitable for synchronous grace-period 345 * primitives. 346 */ 347 static void 348 rcu_torture_pipe_update(struct rcu_torture *old_rp) 349 { 350 struct rcu_torture *rp; 351 struct rcu_torture *rp1; 352 353 if (old_rp) 354 list_add(&old_rp->rtort_free, &rcu_torture_removed); 355 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 356 if (rcu_torture_pipe_update_one(rp)) { 357 list_del(&rp->rtort_free); 358 rcu_torture_free(rp); 359 } 360 } 361 } 362 363 static void 364 rcu_torture_cb(struct rcu_head *p) 365 { 366 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 367 368 if (torture_must_stop_irq()) { 369 /* Test is ending, just drop callbacks on the floor. */ 370 /* The next initialization will pick up the pieces. */ 371 return; 372 } 373 if (rcu_torture_pipe_update_one(rp)) 374 rcu_torture_free(rp); 375 else 376 cur_ops->deferred_free(rp); 377 } 378 379 static unsigned long rcu_no_completed(void) 380 { 381 return 0; 382 } 383 384 static void rcu_torture_deferred_free(struct rcu_torture *p) 385 { 386 call_rcu(&p->rtort_rcu, rcu_torture_cb); 387 } 388 389 static void rcu_sync_torture_init(void) 390 { 391 INIT_LIST_HEAD(&rcu_torture_removed); 392 } 393 394 static struct rcu_torture_ops rcu_ops = { 395 .ttype = RCU_FLAVOR, 396 .init = rcu_sync_torture_init, 397 .readlock = rcu_torture_read_lock, 398 .read_delay = rcu_read_delay, 399 .readunlock = rcu_torture_read_unlock, 400 .started = rcu_batches_started, 401 .completed = rcu_batches_completed, 402 .deferred_free = rcu_torture_deferred_free, 403 .sync = synchronize_rcu, 404 .exp_sync = synchronize_rcu_expedited, 405 .get_state = get_state_synchronize_rcu, 406 .cond_sync = cond_synchronize_rcu, 407 .call = call_rcu, 408 .cb_barrier = rcu_barrier, 409 .fqs = rcu_force_quiescent_state, 410 .stats = NULL, 411 .irq_capable = 1, 412 .can_boost = rcu_can_boost(), 413 .name = "rcu" 414 }; 415 416 /* 417 * Definitions for rcu_bh torture testing. 418 */ 419 420 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 421 { 422 rcu_read_lock_bh(); 423 return 0; 424 } 425 426 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 427 { 428 rcu_read_unlock_bh(); 429 } 430 431 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 432 { 433 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 434 } 435 436 static struct rcu_torture_ops rcu_bh_ops = { 437 .ttype = RCU_BH_FLAVOR, 438 .init = rcu_sync_torture_init, 439 .readlock = rcu_bh_torture_read_lock, 440 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 441 .readunlock = rcu_bh_torture_read_unlock, 442 .started = rcu_batches_started_bh, 443 .completed = rcu_batches_completed_bh, 444 .deferred_free = rcu_bh_torture_deferred_free, 445 .sync = synchronize_rcu_bh, 446 .exp_sync = synchronize_rcu_bh_expedited, 447 .call = call_rcu_bh, 448 .cb_barrier = rcu_barrier_bh, 449 .fqs = rcu_bh_force_quiescent_state, 450 .stats = NULL, 451 .irq_capable = 1, 452 .name = "rcu_bh" 453 }; 454 455 /* 456 * Don't even think about trying any of these in real life!!! 457 * The names includes "busted", and they really means it! 458 * The only purpose of these functions is to provide a buggy RCU 459 * implementation to make sure that rcutorture correctly emits 460 * buggy-RCU error messages. 461 */ 462 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 463 { 464 /* This is a deliberate bug for testing purposes only! */ 465 rcu_torture_cb(&p->rtort_rcu); 466 } 467 468 static void synchronize_rcu_busted(void) 469 { 470 /* This is a deliberate bug for testing purposes only! */ 471 } 472 473 static void 474 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 475 { 476 /* This is a deliberate bug for testing purposes only! */ 477 func(head); 478 } 479 480 static struct rcu_torture_ops rcu_busted_ops = { 481 .ttype = INVALID_RCU_FLAVOR, 482 .init = rcu_sync_torture_init, 483 .readlock = rcu_torture_read_lock, 484 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 485 .readunlock = rcu_torture_read_unlock, 486 .started = rcu_no_completed, 487 .completed = rcu_no_completed, 488 .deferred_free = rcu_busted_torture_deferred_free, 489 .sync = synchronize_rcu_busted, 490 .exp_sync = synchronize_rcu_busted, 491 .call = call_rcu_busted, 492 .cb_barrier = NULL, 493 .fqs = NULL, 494 .stats = NULL, 495 .irq_capable = 1, 496 .name = "busted" 497 }; 498 499 /* 500 * Definitions for srcu torture testing. 501 */ 502 503 DEFINE_STATIC_SRCU(srcu_ctl); 504 static struct srcu_struct srcu_ctld; 505 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 506 507 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 508 { 509 return srcu_read_lock(srcu_ctlp); 510 } 511 512 static void srcu_read_delay(struct torture_random_state *rrsp) 513 { 514 long delay; 515 const long uspertick = 1000000 / HZ; 516 const long longdelay = 10; 517 518 /* We want there to be long-running readers, but not all the time. */ 519 520 delay = torture_random(rrsp) % 521 (nrealreaders * 2 * longdelay * uspertick); 522 if (!delay && in_task()) 523 schedule_timeout_interruptible(longdelay); 524 else 525 rcu_read_delay(rrsp); 526 } 527 528 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 529 { 530 srcu_read_unlock(srcu_ctlp, idx); 531 } 532 533 static unsigned long srcu_torture_completed(void) 534 { 535 return srcu_batches_completed(srcu_ctlp); 536 } 537 538 static void srcu_torture_deferred_free(struct rcu_torture *rp) 539 { 540 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 541 } 542 543 static void srcu_torture_synchronize(void) 544 { 545 synchronize_srcu(srcu_ctlp); 546 } 547 548 static void srcu_torture_call(struct rcu_head *head, 549 rcu_callback_t func) 550 { 551 call_srcu(srcu_ctlp, head, func); 552 } 553 554 static void srcu_torture_barrier(void) 555 { 556 srcu_barrier(srcu_ctlp); 557 } 558 559 static void srcu_torture_stats(void) 560 { 561 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 562 } 563 564 static void srcu_torture_synchronize_expedited(void) 565 { 566 synchronize_srcu_expedited(srcu_ctlp); 567 } 568 569 static struct rcu_torture_ops srcu_ops = { 570 .ttype = SRCU_FLAVOR, 571 .init = rcu_sync_torture_init, 572 .readlock = srcu_torture_read_lock, 573 .read_delay = srcu_read_delay, 574 .readunlock = srcu_torture_read_unlock, 575 .started = NULL, 576 .completed = srcu_torture_completed, 577 .deferred_free = srcu_torture_deferred_free, 578 .sync = srcu_torture_synchronize, 579 .exp_sync = srcu_torture_synchronize_expedited, 580 .call = srcu_torture_call, 581 .cb_barrier = srcu_torture_barrier, 582 .stats = srcu_torture_stats, 583 .irq_capable = 1, 584 .name = "srcu" 585 }; 586 587 static void srcu_torture_init(void) 588 { 589 rcu_sync_torture_init(); 590 WARN_ON(init_srcu_struct(&srcu_ctld)); 591 srcu_ctlp = &srcu_ctld; 592 } 593 594 static void srcu_torture_cleanup(void) 595 { 596 static DEFINE_TORTURE_RANDOM(rand); 597 598 if (torture_random(&rand) & 0x800) 599 cleanup_srcu_struct(&srcu_ctld); 600 else 601 cleanup_srcu_struct_quiesced(&srcu_ctld); 602 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 603 } 604 605 /* As above, but dynamically allocated. */ 606 static struct rcu_torture_ops srcud_ops = { 607 .ttype = SRCU_FLAVOR, 608 .init = srcu_torture_init, 609 .cleanup = srcu_torture_cleanup, 610 .readlock = srcu_torture_read_lock, 611 .read_delay = srcu_read_delay, 612 .readunlock = srcu_torture_read_unlock, 613 .started = NULL, 614 .completed = srcu_torture_completed, 615 .deferred_free = srcu_torture_deferred_free, 616 .sync = srcu_torture_synchronize, 617 .exp_sync = srcu_torture_synchronize_expedited, 618 .call = srcu_torture_call, 619 .cb_barrier = srcu_torture_barrier, 620 .stats = srcu_torture_stats, 621 .irq_capable = 1, 622 .name = "srcud" 623 }; 624 625 /* 626 * Definitions for sched torture testing. 627 */ 628 629 static int sched_torture_read_lock(void) 630 { 631 preempt_disable(); 632 return 0; 633 } 634 635 static void sched_torture_read_unlock(int idx) 636 { 637 preempt_enable(); 638 } 639 640 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 641 { 642 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 643 } 644 645 static struct rcu_torture_ops sched_ops = { 646 .ttype = RCU_SCHED_FLAVOR, 647 .init = rcu_sync_torture_init, 648 .readlock = sched_torture_read_lock, 649 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 650 .readunlock = sched_torture_read_unlock, 651 .started = rcu_batches_started_sched, 652 .completed = rcu_batches_completed_sched, 653 .deferred_free = rcu_sched_torture_deferred_free, 654 .sync = synchronize_sched, 655 .exp_sync = synchronize_sched_expedited, 656 .get_state = get_state_synchronize_sched, 657 .cond_sync = cond_synchronize_sched, 658 .call = call_rcu_sched, 659 .cb_barrier = rcu_barrier_sched, 660 .fqs = rcu_sched_force_quiescent_state, 661 .stats = NULL, 662 .irq_capable = 1, 663 .name = "sched" 664 }; 665 666 /* 667 * Definitions for RCU-tasks torture testing. 668 */ 669 670 static int tasks_torture_read_lock(void) 671 { 672 return 0; 673 } 674 675 static void tasks_torture_read_unlock(int idx) 676 { 677 } 678 679 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 680 { 681 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 682 } 683 684 static struct rcu_torture_ops tasks_ops = { 685 .ttype = RCU_TASKS_FLAVOR, 686 .init = rcu_sync_torture_init, 687 .readlock = tasks_torture_read_lock, 688 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 689 .readunlock = tasks_torture_read_unlock, 690 .started = rcu_no_completed, 691 .completed = rcu_no_completed, 692 .deferred_free = rcu_tasks_torture_deferred_free, 693 .sync = synchronize_rcu_tasks, 694 .exp_sync = synchronize_rcu_tasks, 695 .call = call_rcu_tasks, 696 .cb_barrier = rcu_barrier_tasks, 697 .fqs = NULL, 698 .stats = NULL, 699 .irq_capable = 1, 700 .name = "tasks" 701 }; 702 703 static bool __maybe_unused torturing_tasks(void) 704 { 705 return cur_ops == &tasks_ops; 706 } 707 708 /* 709 * RCU torture priority-boost testing. Runs one real-time thread per 710 * CPU for moderate bursts, repeatedly registering RCU callbacks and 711 * spinning waiting for them to be invoked. If a given callback takes 712 * too long to be invoked, we assume that priority inversion has occurred. 713 */ 714 715 struct rcu_boost_inflight { 716 struct rcu_head rcu; 717 int inflight; 718 }; 719 720 static void rcu_torture_boost_cb(struct rcu_head *head) 721 { 722 struct rcu_boost_inflight *rbip = 723 container_of(head, struct rcu_boost_inflight, rcu); 724 725 /* Ensure RCU-core accesses precede clearing ->inflight */ 726 smp_store_release(&rbip->inflight, 0); 727 } 728 729 static int rcu_torture_boost(void *arg) 730 { 731 unsigned long call_rcu_time; 732 unsigned long endtime; 733 unsigned long oldstarttime; 734 struct rcu_boost_inflight rbi = { .inflight = 0 }; 735 struct sched_param sp; 736 737 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 738 739 /* Set real-time priority. */ 740 sp.sched_priority = 1; 741 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 742 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 743 n_rcu_torture_boost_rterror++; 744 } 745 746 init_rcu_head_on_stack(&rbi.rcu); 747 /* Each pass through the following loop does one boost-test cycle. */ 748 do { 749 /* Wait for the next test interval. */ 750 oldstarttime = boost_starttime; 751 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 752 schedule_timeout_interruptible(oldstarttime - jiffies); 753 stutter_wait("rcu_torture_boost"); 754 if (torture_must_stop()) 755 goto checkwait; 756 } 757 758 /* Do one boost-test interval. */ 759 endtime = oldstarttime + test_boost_duration * HZ; 760 call_rcu_time = jiffies; 761 while (ULONG_CMP_LT(jiffies, endtime)) { 762 /* If we don't have a callback in flight, post one. */ 763 if (!smp_load_acquire(&rbi.inflight)) { 764 /* RCU core before ->inflight = 1. */ 765 smp_store_release(&rbi.inflight, 1); 766 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 767 if (jiffies - call_rcu_time > 768 test_boost_duration * HZ - HZ / 2) { 769 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 770 n_rcu_torture_boost_failure++; 771 } 772 call_rcu_time = jiffies; 773 } 774 stutter_wait("rcu_torture_boost"); 775 if (torture_must_stop()) 776 goto checkwait; 777 } 778 779 /* 780 * Set the start time of the next test interval. 781 * Yes, this is vulnerable to long delays, but such 782 * delays simply cause a false negative for the next 783 * interval. Besides, we are running at RT priority, 784 * so delays should be relatively rare. 785 */ 786 while (oldstarttime == boost_starttime && 787 !kthread_should_stop()) { 788 if (mutex_trylock(&boost_mutex)) { 789 boost_starttime = jiffies + 790 test_boost_interval * HZ; 791 n_rcu_torture_boosts++; 792 mutex_unlock(&boost_mutex); 793 break; 794 } 795 schedule_timeout_uninterruptible(1); 796 } 797 798 /* Go do the stutter. */ 799 checkwait: stutter_wait("rcu_torture_boost"); 800 } while (!torture_must_stop()); 801 802 /* Clean up and exit. */ 803 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 804 torture_shutdown_absorb("rcu_torture_boost"); 805 schedule_timeout_uninterruptible(1); 806 } 807 destroy_rcu_head_on_stack(&rbi.rcu); 808 torture_kthread_stopping("rcu_torture_boost"); 809 return 0; 810 } 811 812 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 813 { 814 } 815 816 /* 817 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 818 * to call_rcu() or analogous, increasing the probability of occurrence 819 * of callback-overflow corner cases. 820 */ 821 static int 822 rcu_torture_cbflood(void *arg) 823 { 824 int err = 1; 825 int i; 826 int j; 827 struct rcu_head *rhp; 828 829 if (cbflood_n_per_burst > 0 && 830 cbflood_inter_holdoff > 0 && 831 cbflood_intra_holdoff > 0 && 832 cur_ops->call && 833 cur_ops->cb_barrier) { 834 rhp = vmalloc(sizeof(*rhp) * 835 cbflood_n_burst * cbflood_n_per_burst); 836 err = !rhp; 837 } 838 if (err) { 839 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 840 goto wait_for_stop; 841 } 842 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 843 do { 844 schedule_timeout_interruptible(cbflood_inter_holdoff); 845 atomic_long_inc(&n_cbfloods); 846 WARN_ON(signal_pending(current)); 847 for (i = 0; i < cbflood_n_burst; i++) { 848 for (j = 0; j < cbflood_n_per_burst; j++) { 849 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 850 rcu_torture_cbflood_cb); 851 } 852 schedule_timeout_interruptible(cbflood_intra_holdoff); 853 WARN_ON(signal_pending(current)); 854 } 855 cur_ops->cb_barrier(); 856 stutter_wait("rcu_torture_cbflood"); 857 } while (!torture_must_stop()); 858 vfree(rhp); 859 wait_for_stop: 860 torture_kthread_stopping("rcu_torture_cbflood"); 861 return 0; 862 } 863 864 /* 865 * RCU torture force-quiescent-state kthread. Repeatedly induces 866 * bursts of calls to force_quiescent_state(), increasing the probability 867 * of occurrence of some important types of race conditions. 868 */ 869 static int 870 rcu_torture_fqs(void *arg) 871 { 872 unsigned long fqs_resume_time; 873 int fqs_burst_remaining; 874 875 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 876 do { 877 fqs_resume_time = jiffies + fqs_stutter * HZ; 878 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 879 !kthread_should_stop()) { 880 schedule_timeout_interruptible(1); 881 } 882 fqs_burst_remaining = fqs_duration; 883 while (fqs_burst_remaining > 0 && 884 !kthread_should_stop()) { 885 cur_ops->fqs(); 886 udelay(fqs_holdoff); 887 fqs_burst_remaining -= fqs_holdoff; 888 } 889 stutter_wait("rcu_torture_fqs"); 890 } while (!torture_must_stop()); 891 torture_kthread_stopping("rcu_torture_fqs"); 892 return 0; 893 } 894 895 /* 896 * RCU torture writer kthread. Repeatedly substitutes a new structure 897 * for that pointed to by rcu_torture_current, freeing the old structure 898 * after a series of grace periods (the "pipeline"). 899 */ 900 static int 901 rcu_torture_writer(void *arg) 902 { 903 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 904 int expediting = 0; 905 unsigned long gp_snap; 906 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 907 bool gp_sync1 = gp_sync; 908 int i; 909 struct rcu_torture *rp; 910 struct rcu_torture *old_rp; 911 static DEFINE_TORTURE_RANDOM(rand); 912 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 913 RTWS_COND_GET, RTWS_SYNC }; 914 int nsynctypes = 0; 915 916 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 917 if (!can_expedite) 918 pr_alert("%s" TORTURE_FLAG 919 " GP expediting controlled from boot/sysfs for %s.\n", 920 torture_type, cur_ops->name); 921 922 /* Initialize synctype[] array. If none set, take default. */ 923 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 924 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 925 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) { 926 synctype[nsynctypes++] = RTWS_COND_GET; 927 pr_info("%s: Testing conditional GPs.\n", __func__); 928 } else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) { 929 pr_alert("%s: gp_cond without primitives.\n", __func__); 930 } 931 if (gp_exp1 && cur_ops->exp_sync) { 932 synctype[nsynctypes++] = RTWS_EXP_SYNC; 933 pr_info("%s: Testing expedited GPs.\n", __func__); 934 } else if (gp_exp && !cur_ops->exp_sync) { 935 pr_alert("%s: gp_exp without primitives.\n", __func__); 936 } 937 if (gp_normal1 && cur_ops->deferred_free) { 938 synctype[nsynctypes++] = RTWS_DEF_FREE; 939 pr_info("%s: Testing asynchronous GPs.\n", __func__); 940 } else if (gp_normal && !cur_ops->deferred_free) { 941 pr_alert("%s: gp_normal without primitives.\n", __func__); 942 } 943 if (gp_sync1 && cur_ops->sync) { 944 synctype[nsynctypes++] = RTWS_SYNC; 945 pr_info("%s: Testing normal GPs.\n", __func__); 946 } else if (gp_sync && !cur_ops->sync) { 947 pr_alert("%s: gp_sync without primitives.\n", __func__); 948 } 949 if (WARN_ONCE(nsynctypes == 0, 950 "rcu_torture_writer: No update-side primitives.\n")) { 951 /* 952 * No updates primitives, so don't try updating. 953 * The resulting test won't be testing much, hence the 954 * above WARN_ONCE(). 955 */ 956 rcu_torture_writer_state = RTWS_STOPPING; 957 torture_kthread_stopping("rcu_torture_writer"); 958 } 959 960 do { 961 rcu_torture_writer_state = RTWS_FIXED_DELAY; 962 schedule_timeout_uninterruptible(1); 963 rp = rcu_torture_alloc(); 964 if (rp == NULL) 965 continue; 966 rp->rtort_pipe_count = 0; 967 rcu_torture_writer_state = RTWS_DELAY; 968 udelay(torture_random(&rand) & 0x3ff); 969 rcu_torture_writer_state = RTWS_REPLACE; 970 old_rp = rcu_dereference_check(rcu_torture_current, 971 current == writer_task); 972 rp->rtort_mbtest = 1; 973 rcu_assign_pointer(rcu_torture_current, rp); 974 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 975 if (old_rp) { 976 i = old_rp->rtort_pipe_count; 977 if (i > RCU_TORTURE_PIPE_LEN) 978 i = RCU_TORTURE_PIPE_LEN; 979 atomic_inc(&rcu_torture_wcount[i]); 980 old_rp->rtort_pipe_count++; 981 switch (synctype[torture_random(&rand) % nsynctypes]) { 982 case RTWS_DEF_FREE: 983 rcu_torture_writer_state = RTWS_DEF_FREE; 984 cur_ops->deferred_free(old_rp); 985 break; 986 case RTWS_EXP_SYNC: 987 rcu_torture_writer_state = RTWS_EXP_SYNC; 988 cur_ops->exp_sync(); 989 rcu_torture_pipe_update(old_rp); 990 break; 991 case RTWS_COND_GET: 992 rcu_torture_writer_state = RTWS_COND_GET; 993 gp_snap = cur_ops->get_state(); 994 i = torture_random(&rand) % 16; 995 if (i != 0) 996 schedule_timeout_interruptible(i); 997 udelay(torture_random(&rand) % 1000); 998 rcu_torture_writer_state = RTWS_COND_SYNC; 999 cur_ops->cond_sync(gp_snap); 1000 rcu_torture_pipe_update(old_rp); 1001 break; 1002 case RTWS_SYNC: 1003 rcu_torture_writer_state = RTWS_SYNC; 1004 cur_ops->sync(); 1005 rcu_torture_pipe_update(old_rp); 1006 break; 1007 default: 1008 WARN_ON_ONCE(1); 1009 break; 1010 } 1011 } 1012 rcutorture_record_progress(++rcu_torture_current_version); 1013 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1014 if (can_expedite && 1015 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1016 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1017 if (expediting >= 0) 1018 rcu_expedite_gp(); 1019 else 1020 rcu_unexpedite_gp(); 1021 if (++expediting > 3) 1022 expediting = -expediting; 1023 } else if (!can_expedite) { /* Disabled during boot, recheck. */ 1024 can_expedite = !rcu_gp_is_expedited() && 1025 !rcu_gp_is_normal(); 1026 } 1027 rcu_torture_writer_state = RTWS_STUTTER; 1028 stutter_wait("rcu_torture_writer"); 1029 } while (!torture_must_stop()); 1030 /* Reset expediting back to unexpedited. */ 1031 if (expediting > 0) 1032 expediting = -expediting; 1033 while (can_expedite && expediting++ < 0) 1034 rcu_unexpedite_gp(); 1035 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1036 if (!can_expedite) 1037 pr_alert("%s" TORTURE_FLAG 1038 " Dynamic grace-period expediting was disabled.\n", 1039 torture_type); 1040 rcu_torture_writer_state = RTWS_STOPPING; 1041 torture_kthread_stopping("rcu_torture_writer"); 1042 return 0; 1043 } 1044 1045 /* 1046 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1047 * delay between calls. 1048 */ 1049 static int 1050 rcu_torture_fakewriter(void *arg) 1051 { 1052 DEFINE_TORTURE_RANDOM(rand); 1053 1054 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1055 set_user_nice(current, MAX_NICE); 1056 1057 do { 1058 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1059 udelay(torture_random(&rand) & 0x3ff); 1060 if (cur_ops->cb_barrier != NULL && 1061 torture_random(&rand) % (nfakewriters * 8) == 0) { 1062 cur_ops->cb_barrier(); 1063 } else if (gp_normal == gp_exp) { 1064 if (cur_ops->sync && torture_random(&rand) & 0x80) 1065 cur_ops->sync(); 1066 else if (cur_ops->exp_sync) 1067 cur_ops->exp_sync(); 1068 } else if (gp_normal && cur_ops->sync) { 1069 cur_ops->sync(); 1070 } else if (cur_ops->exp_sync) { 1071 cur_ops->exp_sync(); 1072 } 1073 stutter_wait("rcu_torture_fakewriter"); 1074 } while (!torture_must_stop()); 1075 1076 torture_kthread_stopping("rcu_torture_fakewriter"); 1077 return 0; 1078 } 1079 1080 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1081 { 1082 kfree(rhp); 1083 } 1084 1085 /* 1086 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1087 * incrementing the corresponding element of the pipeline array. The 1088 * counter in the element should never be greater than 1, otherwise, the 1089 * RCU implementation is broken. 1090 */ 1091 static void rcu_torture_timer(struct timer_list *unused) 1092 { 1093 int idx; 1094 unsigned long started; 1095 unsigned long completed; 1096 static DEFINE_TORTURE_RANDOM(rand); 1097 static DEFINE_SPINLOCK(rand_lock); 1098 struct rcu_torture *p; 1099 int pipe_count; 1100 unsigned long long ts; 1101 1102 idx = cur_ops->readlock(); 1103 if (cur_ops->started) 1104 started = cur_ops->started(); 1105 else 1106 started = cur_ops->completed(); 1107 ts = rcu_trace_clock_local(); 1108 p = rcu_dereference_check(rcu_torture_current, 1109 rcu_read_lock_bh_held() || 1110 rcu_read_lock_sched_held() || 1111 srcu_read_lock_held(srcu_ctlp) || 1112 torturing_tasks()); 1113 if (p == NULL) { 1114 /* Leave because rcu_torture_writer is not yet underway */ 1115 cur_ops->readunlock(idx); 1116 return; 1117 } 1118 if (p->rtort_mbtest == 0) 1119 atomic_inc(&n_rcu_torture_mberror); 1120 spin_lock(&rand_lock); 1121 cur_ops->read_delay(&rand); 1122 n_rcu_torture_timers++; 1123 spin_unlock(&rand_lock); 1124 preempt_disable(); 1125 pipe_count = p->rtort_pipe_count; 1126 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1127 /* Should not happen, but... */ 1128 pipe_count = RCU_TORTURE_PIPE_LEN; 1129 } 1130 completed = cur_ops->completed(); 1131 if (pipe_count > 1) { 1132 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1133 started, completed); 1134 rcu_ftrace_dump(DUMP_ALL); 1135 } 1136 __this_cpu_inc(rcu_torture_count[pipe_count]); 1137 completed = completed - started; 1138 if (cur_ops->started) 1139 completed++; 1140 if (completed > RCU_TORTURE_PIPE_LEN) { 1141 /* Should not happen, but... */ 1142 completed = RCU_TORTURE_PIPE_LEN; 1143 } 1144 __this_cpu_inc(rcu_torture_batch[completed]); 1145 preempt_enable(); 1146 cur_ops->readunlock(idx); 1147 1148 /* Test call_rcu() invocation from interrupt handler. */ 1149 if (cur_ops->call) { 1150 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 1151 1152 if (rhp) 1153 cur_ops->call(rhp, rcu_torture_timer_cb); 1154 } 1155 } 1156 1157 /* 1158 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1159 * incrementing the corresponding element of the pipeline array. The 1160 * counter in the element should never be greater than 1, otherwise, the 1161 * RCU implementation is broken. 1162 */ 1163 static int 1164 rcu_torture_reader(void *arg) 1165 { 1166 unsigned long started; 1167 unsigned long completed; 1168 int idx; 1169 DEFINE_TORTURE_RANDOM(rand); 1170 struct rcu_torture *p; 1171 int pipe_count; 1172 struct timer_list t; 1173 unsigned long long ts; 1174 1175 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1176 set_user_nice(current, MAX_NICE); 1177 if (irqreader && cur_ops->irq_capable) 1178 timer_setup_on_stack(&t, rcu_torture_timer, 0); 1179 1180 do { 1181 if (irqreader && cur_ops->irq_capable) { 1182 if (!timer_pending(&t)) 1183 mod_timer(&t, jiffies + 1); 1184 } 1185 idx = cur_ops->readlock(); 1186 if (cur_ops->started) 1187 started = cur_ops->started(); 1188 else 1189 started = cur_ops->completed(); 1190 ts = rcu_trace_clock_local(); 1191 p = rcu_dereference_check(rcu_torture_current, 1192 rcu_read_lock_bh_held() || 1193 rcu_read_lock_sched_held() || 1194 srcu_read_lock_held(srcu_ctlp) || 1195 torturing_tasks()); 1196 if (p == NULL) { 1197 /* Wait for rcu_torture_writer to get underway */ 1198 cur_ops->readunlock(idx); 1199 schedule_timeout_interruptible(HZ); 1200 continue; 1201 } 1202 if (p->rtort_mbtest == 0) 1203 atomic_inc(&n_rcu_torture_mberror); 1204 cur_ops->read_delay(&rand); 1205 preempt_disable(); 1206 pipe_count = p->rtort_pipe_count; 1207 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1208 /* Should not happen, but... */ 1209 pipe_count = RCU_TORTURE_PIPE_LEN; 1210 } 1211 completed = cur_ops->completed(); 1212 if (pipe_count > 1) { 1213 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1214 ts, started, completed); 1215 rcu_ftrace_dump(DUMP_ALL); 1216 } 1217 __this_cpu_inc(rcu_torture_count[pipe_count]); 1218 completed = completed - started; 1219 if (cur_ops->started) 1220 completed++; 1221 if (completed > RCU_TORTURE_PIPE_LEN) { 1222 /* Should not happen, but... */ 1223 completed = RCU_TORTURE_PIPE_LEN; 1224 } 1225 __this_cpu_inc(rcu_torture_batch[completed]); 1226 preempt_enable(); 1227 cur_ops->readunlock(idx); 1228 stutter_wait("rcu_torture_reader"); 1229 } while (!torture_must_stop()); 1230 if (irqreader && cur_ops->irq_capable) { 1231 del_timer_sync(&t); 1232 destroy_timer_on_stack(&t); 1233 } 1234 torture_kthread_stopping("rcu_torture_reader"); 1235 return 0; 1236 } 1237 1238 /* 1239 * Print torture statistics. Caller must ensure that there is only 1240 * one call to this function at a given time!!! This is normally 1241 * accomplished by relying on the module system to only have one copy 1242 * of the module loaded, and then by giving the rcu_torture_stats 1243 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1244 * thread is not running). 1245 */ 1246 static void 1247 rcu_torture_stats_print(void) 1248 { 1249 int cpu; 1250 int i; 1251 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1252 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1253 static unsigned long rtcv_snap = ULONG_MAX; 1254 static bool splatted; 1255 struct task_struct *wtp; 1256 1257 for_each_possible_cpu(cpu) { 1258 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1259 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1260 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1261 } 1262 } 1263 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1264 if (pipesummary[i] != 0) 1265 break; 1266 } 1267 1268 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1269 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1270 rcu_torture_current, 1271 rcu_torture_current_version, 1272 list_empty(&rcu_torture_freelist), 1273 atomic_read(&n_rcu_torture_alloc), 1274 atomic_read(&n_rcu_torture_alloc_fail), 1275 atomic_read(&n_rcu_torture_free)); 1276 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ", 1277 atomic_read(&n_rcu_torture_mberror), 1278 n_rcu_torture_barrier_error, 1279 n_rcu_torture_boost_ktrerror, 1280 n_rcu_torture_boost_rterror); 1281 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1282 n_rcu_torture_boost_failure, 1283 n_rcu_torture_boosts, 1284 n_rcu_torture_timers); 1285 torture_onoff_stats(); 1286 pr_cont("barrier: %ld/%ld:%ld ", 1287 n_barrier_successes, 1288 n_barrier_attempts, 1289 n_rcu_torture_barrier_error); 1290 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1291 1292 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1293 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1294 n_rcu_torture_barrier_error != 0 || 1295 n_rcu_torture_boost_ktrerror != 0 || 1296 n_rcu_torture_boost_rterror != 0 || 1297 n_rcu_torture_boost_failure != 0 || 1298 i > 1) { 1299 pr_cont("%s", "!!! "); 1300 atomic_inc(&n_rcu_torture_error); 1301 WARN_ON_ONCE(1); 1302 } 1303 pr_cont("Reader Pipe: "); 1304 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1305 pr_cont(" %ld", pipesummary[i]); 1306 pr_cont("\n"); 1307 1308 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1309 pr_cont("Reader Batch: "); 1310 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1311 pr_cont(" %ld", batchsummary[i]); 1312 pr_cont("\n"); 1313 1314 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1315 pr_cont("Free-Block Circulation: "); 1316 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1317 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1318 } 1319 pr_cont("\n"); 1320 1321 if (cur_ops->stats) 1322 cur_ops->stats(); 1323 if (rtcv_snap == rcu_torture_current_version && 1324 rcu_torture_current != NULL) { 1325 int __maybe_unused flags = 0; 1326 unsigned long __maybe_unused gpnum = 0; 1327 unsigned long __maybe_unused completed = 0; 1328 1329 rcutorture_get_gp_data(cur_ops->ttype, 1330 &flags, &gpnum, &completed); 1331 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, 1332 &flags, &gpnum, &completed); 1333 wtp = READ_ONCE(writer_task); 1334 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx cpu %d\n", 1335 rcu_torture_writer_state_getname(), 1336 rcu_torture_writer_state, 1337 gpnum, completed, flags, 1338 wtp == NULL ? ~0UL : wtp->state, 1339 wtp == NULL ? -1 : (int)task_cpu(wtp)); 1340 if (!splatted && wtp) { 1341 sched_show_task(wtp); 1342 splatted = true; 1343 } 1344 show_rcu_gp_kthreads(); 1345 rcu_ftrace_dump(DUMP_ALL); 1346 } 1347 rtcv_snap = rcu_torture_current_version; 1348 } 1349 1350 /* 1351 * Periodically prints torture statistics, if periodic statistics printing 1352 * was specified via the stat_interval module parameter. 1353 */ 1354 static int 1355 rcu_torture_stats(void *arg) 1356 { 1357 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1358 do { 1359 schedule_timeout_interruptible(stat_interval * HZ); 1360 rcu_torture_stats_print(); 1361 torture_shutdown_absorb("rcu_torture_stats"); 1362 } while (!torture_must_stop()); 1363 torture_kthread_stopping("rcu_torture_stats"); 1364 return 0; 1365 } 1366 1367 static inline void 1368 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1369 { 1370 pr_alert("%s" TORTURE_FLAG 1371 "--- %s: nreaders=%d nfakewriters=%d " 1372 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1373 "shuffle_interval=%d stutter=%d irqreader=%d " 1374 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1375 "test_boost=%d/%d test_boost_interval=%d " 1376 "test_boost_duration=%d shutdown_secs=%d " 1377 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 1378 "n_barrier_cbs=%d " 1379 "onoff_interval=%d onoff_holdoff=%d\n", 1380 torture_type, tag, nrealreaders, nfakewriters, 1381 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1382 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1383 test_boost, cur_ops->can_boost, 1384 test_boost_interval, test_boost_duration, shutdown_secs, 1385 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 1386 n_barrier_cbs, 1387 onoff_interval, onoff_holdoff); 1388 } 1389 1390 static int rcutorture_booster_cleanup(unsigned int cpu) 1391 { 1392 struct task_struct *t; 1393 1394 if (boost_tasks[cpu] == NULL) 1395 return 0; 1396 mutex_lock(&boost_mutex); 1397 t = boost_tasks[cpu]; 1398 boost_tasks[cpu] = NULL; 1399 mutex_unlock(&boost_mutex); 1400 1401 /* This must be outside of the mutex, otherwise deadlock! */ 1402 torture_stop_kthread(rcu_torture_boost, t); 1403 return 0; 1404 } 1405 1406 static int rcutorture_booster_init(unsigned int cpu) 1407 { 1408 int retval; 1409 1410 if (boost_tasks[cpu] != NULL) 1411 return 0; /* Already created, nothing more to do. */ 1412 1413 /* Don't allow time recalculation while creating a new task. */ 1414 mutex_lock(&boost_mutex); 1415 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1416 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1417 cpu_to_node(cpu), 1418 "rcu_torture_boost"); 1419 if (IS_ERR(boost_tasks[cpu])) { 1420 retval = PTR_ERR(boost_tasks[cpu]); 1421 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1422 n_rcu_torture_boost_ktrerror++; 1423 boost_tasks[cpu] = NULL; 1424 mutex_unlock(&boost_mutex); 1425 return retval; 1426 } 1427 kthread_bind(boost_tasks[cpu], cpu); 1428 wake_up_process(boost_tasks[cpu]); 1429 mutex_unlock(&boost_mutex); 1430 return 0; 1431 } 1432 1433 /* 1434 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1435 * induces a CPU stall for the time specified by stall_cpu. 1436 */ 1437 static int rcu_torture_stall(void *args) 1438 { 1439 unsigned long stop_at; 1440 1441 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1442 if (stall_cpu_holdoff > 0) { 1443 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1444 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1445 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1446 } 1447 if (!kthread_should_stop()) { 1448 stop_at = get_seconds() + stall_cpu; 1449 /* RCU CPU stall is expected behavior in following code. */ 1450 rcu_read_lock(); 1451 if (stall_cpu_irqsoff) 1452 local_irq_disable(); 1453 else 1454 preempt_disable(); 1455 pr_alert("rcu_torture_stall start on CPU %d.\n", 1456 smp_processor_id()); 1457 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1458 continue; /* Induce RCU CPU stall warning. */ 1459 if (stall_cpu_irqsoff) 1460 local_irq_enable(); 1461 else 1462 preempt_enable(); 1463 rcu_read_unlock(); 1464 pr_alert("rcu_torture_stall end.\n"); 1465 } 1466 torture_shutdown_absorb("rcu_torture_stall"); 1467 while (!kthread_should_stop()) 1468 schedule_timeout_interruptible(10 * HZ); 1469 return 0; 1470 } 1471 1472 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1473 static int __init rcu_torture_stall_init(void) 1474 { 1475 if (stall_cpu <= 0) 1476 return 0; 1477 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1478 } 1479 1480 /* Callback function for RCU barrier testing. */ 1481 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1482 { 1483 atomic_inc(&barrier_cbs_invoked); 1484 } 1485 1486 /* kthread function to register callbacks used to test RCU barriers. */ 1487 static int rcu_torture_barrier_cbs(void *arg) 1488 { 1489 long myid = (long)arg; 1490 bool lastphase = 0; 1491 bool newphase; 1492 struct rcu_head rcu; 1493 1494 init_rcu_head_on_stack(&rcu); 1495 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1496 set_user_nice(current, MAX_NICE); 1497 do { 1498 wait_event(barrier_cbs_wq[myid], 1499 (newphase = 1500 smp_load_acquire(&barrier_phase)) != lastphase || 1501 torture_must_stop()); 1502 lastphase = newphase; 1503 if (torture_must_stop()) 1504 break; 1505 /* 1506 * The above smp_load_acquire() ensures barrier_phase load 1507 * is ordered before the following ->call(). 1508 */ 1509 local_irq_disable(); /* Just to test no-irq call_rcu(). */ 1510 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1511 local_irq_enable(); 1512 if (atomic_dec_and_test(&barrier_cbs_count)) 1513 wake_up(&barrier_wq); 1514 } while (!torture_must_stop()); 1515 if (cur_ops->cb_barrier != NULL) 1516 cur_ops->cb_barrier(); 1517 destroy_rcu_head_on_stack(&rcu); 1518 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1519 return 0; 1520 } 1521 1522 /* kthread function to drive and coordinate RCU barrier testing. */ 1523 static int rcu_torture_barrier(void *arg) 1524 { 1525 int i; 1526 1527 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1528 do { 1529 atomic_set(&barrier_cbs_invoked, 0); 1530 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1531 /* Ensure barrier_phase ordered after prior assignments. */ 1532 smp_store_release(&barrier_phase, !barrier_phase); 1533 for (i = 0; i < n_barrier_cbs; i++) 1534 wake_up(&barrier_cbs_wq[i]); 1535 wait_event(barrier_wq, 1536 atomic_read(&barrier_cbs_count) == 0 || 1537 torture_must_stop()); 1538 if (torture_must_stop()) 1539 break; 1540 n_barrier_attempts++; 1541 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1542 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1543 n_rcu_torture_barrier_error++; 1544 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 1545 atomic_read(&barrier_cbs_invoked), 1546 n_barrier_cbs); 1547 WARN_ON_ONCE(1); 1548 } 1549 n_barrier_successes++; 1550 schedule_timeout_interruptible(HZ / 10); 1551 } while (!torture_must_stop()); 1552 torture_kthread_stopping("rcu_torture_barrier"); 1553 return 0; 1554 } 1555 1556 /* Initialize RCU barrier testing. */ 1557 static int rcu_torture_barrier_init(void) 1558 { 1559 int i; 1560 int ret; 1561 1562 if (n_barrier_cbs <= 0) 1563 return 0; 1564 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1565 pr_alert("%s" TORTURE_FLAG 1566 " Call or barrier ops missing for %s,\n", 1567 torture_type, cur_ops->name); 1568 pr_alert("%s" TORTURE_FLAG 1569 " RCU barrier testing omitted from run.\n", 1570 torture_type); 1571 return 0; 1572 } 1573 atomic_set(&barrier_cbs_count, 0); 1574 atomic_set(&barrier_cbs_invoked, 0); 1575 barrier_cbs_tasks = 1576 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]), 1577 GFP_KERNEL); 1578 barrier_cbs_wq = 1579 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL); 1580 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1581 return -ENOMEM; 1582 for (i = 0; i < n_barrier_cbs; i++) { 1583 init_waitqueue_head(&barrier_cbs_wq[i]); 1584 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1585 (void *)(long)i, 1586 barrier_cbs_tasks[i]); 1587 if (ret) 1588 return ret; 1589 } 1590 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1591 } 1592 1593 /* Clean up after RCU barrier testing. */ 1594 static void rcu_torture_barrier_cleanup(void) 1595 { 1596 int i; 1597 1598 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1599 if (barrier_cbs_tasks != NULL) { 1600 for (i = 0; i < n_barrier_cbs; i++) 1601 torture_stop_kthread(rcu_torture_barrier_cbs, 1602 barrier_cbs_tasks[i]); 1603 kfree(barrier_cbs_tasks); 1604 barrier_cbs_tasks = NULL; 1605 } 1606 if (barrier_cbs_wq != NULL) { 1607 kfree(barrier_cbs_wq); 1608 barrier_cbs_wq = NULL; 1609 } 1610 } 1611 1612 static enum cpuhp_state rcutor_hp; 1613 1614 static void 1615 rcu_torture_cleanup(void) 1616 { 1617 int flags = 0; 1618 unsigned long gpnum = 0; 1619 unsigned long completed = 0; 1620 int i; 1621 1622 rcutorture_record_test_transition(); 1623 if (torture_cleanup_begin()) { 1624 if (cur_ops->cb_barrier != NULL) 1625 cur_ops->cb_barrier(); 1626 return; 1627 } 1628 1629 rcu_torture_barrier_cleanup(); 1630 torture_stop_kthread(rcu_torture_stall, stall_task); 1631 torture_stop_kthread(rcu_torture_writer, writer_task); 1632 1633 if (reader_tasks) { 1634 for (i = 0; i < nrealreaders; i++) 1635 torture_stop_kthread(rcu_torture_reader, 1636 reader_tasks[i]); 1637 kfree(reader_tasks); 1638 } 1639 rcu_torture_current = NULL; 1640 1641 if (fakewriter_tasks) { 1642 for (i = 0; i < nfakewriters; i++) { 1643 torture_stop_kthread(rcu_torture_fakewriter, 1644 fakewriter_tasks[i]); 1645 } 1646 kfree(fakewriter_tasks); 1647 fakewriter_tasks = NULL; 1648 } 1649 1650 rcutorture_get_gp_data(cur_ops->ttype, &flags, &gpnum, &completed); 1651 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, 1652 &flags, &gpnum, &completed); 1653 pr_alert("%s: End-test grace-period state: g%lu c%lu f%#x\n", 1654 cur_ops->name, gpnum, completed, flags); 1655 torture_stop_kthread(rcu_torture_stats, stats_task); 1656 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1657 for (i = 0; i < ncbflooders; i++) 1658 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1659 if ((test_boost == 1 && cur_ops->can_boost) || 1660 test_boost == 2) 1661 cpuhp_remove_state(rcutor_hp); 1662 1663 /* 1664 * Wait for all RCU callbacks to fire, then do flavor-specific 1665 * cleanup operations. 1666 */ 1667 if (cur_ops->cb_barrier != NULL) 1668 cur_ops->cb_barrier(); 1669 if (cur_ops->cleanup != NULL) 1670 cur_ops->cleanup(); 1671 1672 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1673 1674 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1675 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1676 else if (torture_onoff_failures()) 1677 rcu_torture_print_module_parms(cur_ops, 1678 "End of test: RCU_HOTPLUG"); 1679 else 1680 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1681 torture_cleanup_end(); 1682 } 1683 1684 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1685 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1686 { 1687 } 1688 1689 static void rcu_torture_err_cb(struct rcu_head *rhp) 1690 { 1691 /* 1692 * This -might- happen due to race conditions, but is unlikely. 1693 * The scenario that leads to this happening is that the 1694 * first of the pair of duplicate callbacks is queued, 1695 * someone else starts a grace period that includes that 1696 * callback, then the second of the pair must wait for the 1697 * next grace period. Unlikely, but can happen. If it 1698 * does happen, the debug-objects subsystem won't have splatted. 1699 */ 1700 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); 1701 } 1702 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1703 1704 /* 1705 * Verify that double-free causes debug-objects to complain, but only 1706 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1707 * cannot be carried out. 1708 */ 1709 static void rcu_test_debug_objects(void) 1710 { 1711 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1712 struct rcu_head rh1; 1713 struct rcu_head rh2; 1714 1715 init_rcu_head_on_stack(&rh1); 1716 init_rcu_head_on_stack(&rh2); 1717 pr_alert("%s: WARN: Duplicate call_rcu() test starting.\n", KBUILD_MODNAME); 1718 1719 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1720 preempt_disable(); /* Prevent preemption from interrupting test. */ 1721 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1722 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1723 local_irq_disable(); /* Make it harder to start a new grace period. */ 1724 call_rcu(&rh2, rcu_torture_leak_cb); 1725 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1726 local_irq_enable(); 1727 rcu_read_unlock(); 1728 preempt_enable(); 1729 1730 /* Wait for them all to get done so we can safely return. */ 1731 rcu_barrier(); 1732 pr_alert("%s: WARN: Duplicate call_rcu() test complete.\n", KBUILD_MODNAME); 1733 destroy_rcu_head_on_stack(&rh1); 1734 destroy_rcu_head_on_stack(&rh2); 1735 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1736 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n", KBUILD_MODNAME); 1737 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1738 } 1739 1740 static int __init 1741 rcu_torture_init(void) 1742 { 1743 int i; 1744 int cpu; 1745 int firsterr = 0; 1746 static struct rcu_torture_ops *torture_ops[] = { 1747 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 1748 &sched_ops, &tasks_ops, 1749 }; 1750 1751 if (!torture_init_begin(torture_type, verbose)) 1752 return -EBUSY; 1753 1754 /* Process args and tell the world that the torturer is on the job. */ 1755 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1756 cur_ops = torture_ops[i]; 1757 if (strcmp(torture_type, cur_ops->name) == 0) 1758 break; 1759 } 1760 if (i == ARRAY_SIZE(torture_ops)) { 1761 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1762 torture_type); 1763 pr_alert("rcu-torture types:"); 1764 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1765 pr_alert(" %s", torture_ops[i]->name); 1766 pr_alert("\n"); 1767 firsterr = -EINVAL; 1768 goto unwind; 1769 } 1770 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1771 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1772 fqs_duration = 0; 1773 } 1774 if (cur_ops->init) 1775 cur_ops->init(); 1776 1777 if (nreaders >= 0) { 1778 nrealreaders = nreaders; 1779 } else { 1780 nrealreaders = num_online_cpus() - 2 - nreaders; 1781 if (nrealreaders <= 0) 1782 nrealreaders = 1; 1783 } 1784 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1785 1786 /* Set up the freelist. */ 1787 1788 INIT_LIST_HEAD(&rcu_torture_freelist); 1789 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1790 rcu_tortures[i].rtort_mbtest = 0; 1791 list_add_tail(&rcu_tortures[i].rtort_free, 1792 &rcu_torture_freelist); 1793 } 1794 1795 /* Initialize the statistics so that each run gets its own numbers. */ 1796 1797 rcu_torture_current = NULL; 1798 rcu_torture_current_version = 0; 1799 atomic_set(&n_rcu_torture_alloc, 0); 1800 atomic_set(&n_rcu_torture_alloc_fail, 0); 1801 atomic_set(&n_rcu_torture_free, 0); 1802 atomic_set(&n_rcu_torture_mberror, 0); 1803 atomic_set(&n_rcu_torture_error, 0); 1804 n_rcu_torture_barrier_error = 0; 1805 n_rcu_torture_boost_ktrerror = 0; 1806 n_rcu_torture_boost_rterror = 0; 1807 n_rcu_torture_boost_failure = 0; 1808 n_rcu_torture_boosts = 0; 1809 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1810 atomic_set(&rcu_torture_wcount[i], 0); 1811 for_each_possible_cpu(cpu) { 1812 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1813 per_cpu(rcu_torture_count, cpu)[i] = 0; 1814 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1815 } 1816 } 1817 1818 /* Start up the kthreads. */ 1819 1820 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1821 writer_task); 1822 if (firsterr) 1823 goto unwind; 1824 if (nfakewriters > 0) { 1825 fakewriter_tasks = kcalloc(nfakewriters, 1826 sizeof(fakewriter_tasks[0]), 1827 GFP_KERNEL); 1828 if (fakewriter_tasks == NULL) { 1829 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1830 firsterr = -ENOMEM; 1831 goto unwind; 1832 } 1833 } 1834 for (i = 0; i < nfakewriters; i++) { 1835 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1836 NULL, fakewriter_tasks[i]); 1837 if (firsterr) 1838 goto unwind; 1839 } 1840 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 1841 GFP_KERNEL); 1842 if (reader_tasks == NULL) { 1843 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1844 firsterr = -ENOMEM; 1845 goto unwind; 1846 } 1847 for (i = 0; i < nrealreaders; i++) { 1848 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1849 reader_tasks[i]); 1850 if (firsterr) 1851 goto unwind; 1852 } 1853 if (stat_interval > 0) { 1854 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1855 stats_task); 1856 if (firsterr) 1857 goto unwind; 1858 } 1859 if (test_no_idle_hz && shuffle_interval > 0) { 1860 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1861 if (firsterr) 1862 goto unwind; 1863 } 1864 if (stutter < 0) 1865 stutter = 0; 1866 if (stutter) { 1867 firsterr = torture_stutter_init(stutter * HZ); 1868 if (firsterr) 1869 goto unwind; 1870 } 1871 if (fqs_duration < 0) 1872 fqs_duration = 0; 1873 if (fqs_duration) { 1874 /* Create the fqs thread */ 1875 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1876 fqs_task); 1877 if (firsterr) 1878 goto unwind; 1879 } 1880 if (test_boost_interval < 1) 1881 test_boost_interval = 1; 1882 if (test_boost_duration < 2) 1883 test_boost_duration = 2; 1884 if ((test_boost == 1 && cur_ops->can_boost) || 1885 test_boost == 2) { 1886 1887 boost_starttime = jiffies + test_boost_interval * HZ; 1888 1889 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 1890 rcutorture_booster_init, 1891 rcutorture_booster_cleanup); 1892 if (firsterr < 0) 1893 goto unwind; 1894 rcutor_hp = firsterr; 1895 } 1896 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1897 if (firsterr) 1898 goto unwind; 1899 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1900 if (firsterr) 1901 goto unwind; 1902 firsterr = rcu_torture_stall_init(); 1903 if (firsterr) 1904 goto unwind; 1905 firsterr = rcu_torture_barrier_init(); 1906 if (firsterr) 1907 goto unwind; 1908 if (object_debug) 1909 rcu_test_debug_objects(); 1910 if (cbflood_n_burst > 0) { 1911 /* Create the cbflood threads */ 1912 ncbflooders = (num_online_cpus() + 3) / 4; 1913 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1914 GFP_KERNEL); 1915 if (!cbflood_task) { 1916 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1917 firsterr = -ENOMEM; 1918 goto unwind; 1919 } 1920 for (i = 0; i < ncbflooders; i++) { 1921 firsterr = torture_create_kthread(rcu_torture_cbflood, 1922 NULL, 1923 cbflood_task[i]); 1924 if (firsterr) 1925 goto unwind; 1926 } 1927 } 1928 rcutorture_record_test_transition(); 1929 torture_init_end(); 1930 return 0; 1931 1932 unwind: 1933 torture_init_end(); 1934 rcu_torture_cleanup(); 1935 return firsterr; 1936 } 1937 1938 module_init(rcu_torture_init); 1939 module_exit(rcu_torture_cleanup); 1940