1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@freedesktop.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched.h> 36 #include <linux/atomic.h> 37 #include <linux/bitops.h> 38 #include <linux/completion.h> 39 #include <linux/moduleparam.h> 40 #include <linux/percpu.h> 41 #include <linux/notifier.h> 42 #include <linux/reboot.h> 43 #include <linux/freezer.h> 44 #include <linux/cpu.h> 45 #include <linux/delay.h> 46 #include <linux/stat.h> 47 #include <linux/srcu.h> 48 #include <linux/slab.h> 49 #include <linux/trace_clock.h> 50 #include <asm/byteorder.h> 51 #include <linux/torture.h> 52 53 MODULE_LICENSE("GPL"); 54 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@freedesktop.org>"); 55 56 MODULE_ALIAS("rcutorture"); 57 #ifdef MODULE_PARAM_PREFIX 58 #undef MODULE_PARAM_PREFIX 59 #endif 60 #define MODULE_PARAM_PREFIX "rcutorture." 61 62 torture_param(int, fqs_duration, 0, 63 "Duration of fqs bursts (us), 0 to disable"); 64 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 65 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 66 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 67 torture_param(bool, gp_normal, false, 68 "Use normal (non-expedited) GP wait primitives"); 69 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 70 torture_param(int, n_barrier_cbs, 0, 71 "# of callbacks/kthreads for barrier testing"); 72 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 73 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 74 torture_param(int, object_debug, 0, 75 "Enable debug-object double call_rcu() testing"); 76 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 77 torture_param(int, onoff_interval, 0, 78 "Time between CPU hotplugs (s), 0=disable"); 79 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 80 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 81 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 82 torture_param(int, stall_cpu_holdoff, 10, 83 "Time to wait before starting stall (s)."); 84 torture_param(int, stat_interval, 60, 85 "Number of seconds between stats printk()s"); 86 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 87 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 88 torture_param(int, test_boost_duration, 4, 89 "Duration of each boost test, seconds."); 90 torture_param(int, test_boost_interval, 7, 91 "Interval between boost tests, seconds."); 92 torture_param(bool, test_no_idle_hz, true, 93 "Test support for tickless idle CPUs"); 94 torture_param(bool, verbose, true, 95 "Enable verbose debugging printk()s"); 96 97 static char *torture_type = "rcu"; 98 module_param(torture_type, charp, 0444); 99 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 100 101 static int nrealreaders; 102 static struct task_struct *writer_task; 103 static struct task_struct **fakewriter_tasks; 104 static struct task_struct **reader_tasks; 105 static struct task_struct *stats_task; 106 static struct task_struct *fqs_task; 107 static struct task_struct *boost_tasks[NR_CPUS]; 108 static struct task_struct *shutdown_task; 109 static struct task_struct *stall_task; 110 static struct task_struct **barrier_cbs_tasks; 111 static struct task_struct *barrier_task; 112 113 #define RCU_TORTURE_PIPE_LEN 10 114 115 struct rcu_torture { 116 struct rcu_head rtort_rcu; 117 int rtort_pipe_count; 118 struct list_head rtort_free; 119 int rtort_mbtest; 120 }; 121 122 static LIST_HEAD(rcu_torture_freelist); 123 static struct rcu_torture __rcu *rcu_torture_current; 124 static unsigned long rcu_torture_current_version; 125 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 126 static DEFINE_SPINLOCK(rcu_torture_lock); 127 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 128 rcu_torture_count) = { 0 }; 129 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 130 rcu_torture_batch) = { 0 }; 131 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 132 static atomic_t n_rcu_torture_alloc; 133 static atomic_t n_rcu_torture_alloc_fail; 134 static atomic_t n_rcu_torture_free; 135 static atomic_t n_rcu_torture_mberror; 136 static atomic_t n_rcu_torture_error; 137 static long n_rcu_torture_barrier_error; 138 static long n_rcu_torture_boost_ktrerror; 139 static long n_rcu_torture_boost_rterror; 140 static long n_rcu_torture_boost_failure; 141 static long n_rcu_torture_boosts; 142 static long n_rcu_torture_timers; 143 static long n_barrier_attempts; 144 static long n_barrier_successes; 145 static struct list_head rcu_torture_removed; 146 147 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE) 148 #define RCUTORTURE_RUNNABLE_INIT 1 149 #else 150 #define RCUTORTURE_RUNNABLE_INIT 0 151 #endif 152 int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT; 153 module_param(rcutorture_runnable, int, 0444); 154 MODULE_PARM_DESC(rcutorture_runnable, "Start rcutorture at boot"); 155 156 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 157 #define rcu_can_boost() 1 158 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 159 #define rcu_can_boost() 0 160 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 161 162 #ifdef CONFIG_RCU_TRACE 163 static u64 notrace rcu_trace_clock_local(void) 164 { 165 u64 ts = trace_clock_local(); 166 unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC); 167 return ts; 168 } 169 #else /* #ifdef CONFIG_RCU_TRACE */ 170 static u64 notrace rcu_trace_clock_local(void) 171 { 172 return 0ULL; 173 } 174 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 175 176 static unsigned long shutdown_time; /* jiffies to system shutdown. */ 177 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 178 DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 179 /* and boost task create/destroy. */ 180 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 181 static bool barrier_phase; /* Test phase. */ 182 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 183 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 184 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 185 186 /* Forward reference. */ 187 static void rcu_torture_cleanup(void); 188 189 /* 190 * Allocate an element from the rcu_tortures pool. 191 */ 192 static struct rcu_torture * 193 rcu_torture_alloc(void) 194 { 195 struct list_head *p; 196 197 spin_lock_bh(&rcu_torture_lock); 198 if (list_empty(&rcu_torture_freelist)) { 199 atomic_inc(&n_rcu_torture_alloc_fail); 200 spin_unlock_bh(&rcu_torture_lock); 201 return NULL; 202 } 203 atomic_inc(&n_rcu_torture_alloc); 204 p = rcu_torture_freelist.next; 205 list_del_init(p); 206 spin_unlock_bh(&rcu_torture_lock); 207 return container_of(p, struct rcu_torture, rtort_free); 208 } 209 210 /* 211 * Free an element to the rcu_tortures pool. 212 */ 213 static void 214 rcu_torture_free(struct rcu_torture *p) 215 { 216 atomic_inc(&n_rcu_torture_free); 217 spin_lock_bh(&rcu_torture_lock); 218 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 219 spin_unlock_bh(&rcu_torture_lock); 220 } 221 222 /* 223 * Operations vector for selecting different types of tests. 224 */ 225 226 struct rcu_torture_ops { 227 void (*init)(void); 228 int (*readlock)(void); 229 void (*read_delay)(struct torture_random_state *rrsp); 230 void (*readunlock)(int idx); 231 int (*completed)(void); 232 void (*deferred_free)(struct rcu_torture *p); 233 void (*sync)(void); 234 void (*exp_sync)(void); 235 void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 236 void (*cb_barrier)(void); 237 void (*fqs)(void); 238 void (*stats)(char *page); 239 int irq_capable; 240 int can_boost; 241 const char *name; 242 }; 243 244 static struct rcu_torture_ops *cur_ops; 245 246 /* 247 * Definitions for rcu torture testing. 248 */ 249 250 static int rcu_torture_read_lock(void) __acquires(RCU) 251 { 252 rcu_read_lock(); 253 return 0; 254 } 255 256 static void rcu_read_delay(struct torture_random_state *rrsp) 257 { 258 const unsigned long shortdelay_us = 200; 259 const unsigned long longdelay_ms = 50; 260 261 /* We want a short delay sometimes to make a reader delay the grace 262 * period, and we want a long delay occasionally to trigger 263 * force_quiescent_state. */ 264 265 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) 266 mdelay(longdelay_ms); 267 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 268 udelay(shortdelay_us); 269 #ifdef CONFIG_PREEMPT 270 if (!preempt_count() && 271 !(torture_random(rrsp) % (nrealreaders * 20000))) 272 preempt_schedule(); /* No QS if preempt_disable() in effect */ 273 #endif 274 } 275 276 static void rcu_torture_read_unlock(int idx) __releases(RCU) 277 { 278 rcu_read_unlock(); 279 } 280 281 static int rcu_torture_completed(void) 282 { 283 return rcu_batches_completed(); 284 } 285 286 static void 287 rcu_torture_cb(struct rcu_head *p) 288 { 289 int i; 290 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 291 292 if (torture_must_stop_irq()) { 293 /* Test is ending, just drop callbacks on the floor. */ 294 /* The next initialization will pick up the pieces. */ 295 return; 296 } 297 i = rp->rtort_pipe_count; 298 if (i > RCU_TORTURE_PIPE_LEN) 299 i = RCU_TORTURE_PIPE_LEN; 300 atomic_inc(&rcu_torture_wcount[i]); 301 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 302 rp->rtort_mbtest = 0; 303 rcu_torture_free(rp); 304 } else { 305 cur_ops->deferred_free(rp); 306 } 307 } 308 309 static int rcu_no_completed(void) 310 { 311 return 0; 312 } 313 314 static void rcu_torture_deferred_free(struct rcu_torture *p) 315 { 316 call_rcu(&p->rtort_rcu, rcu_torture_cb); 317 } 318 319 static void rcu_sync_torture_init(void) 320 { 321 INIT_LIST_HEAD(&rcu_torture_removed); 322 } 323 324 static struct rcu_torture_ops rcu_ops = { 325 .init = rcu_sync_torture_init, 326 .readlock = rcu_torture_read_lock, 327 .read_delay = rcu_read_delay, 328 .readunlock = rcu_torture_read_unlock, 329 .completed = rcu_torture_completed, 330 .deferred_free = rcu_torture_deferred_free, 331 .sync = synchronize_rcu, 332 .exp_sync = synchronize_rcu_expedited, 333 .call = call_rcu, 334 .cb_barrier = rcu_barrier, 335 .fqs = rcu_force_quiescent_state, 336 .stats = NULL, 337 .irq_capable = 1, 338 .can_boost = rcu_can_boost(), 339 .name = "rcu" 340 }; 341 342 /* 343 * Definitions for rcu_bh torture testing. 344 */ 345 346 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 347 { 348 rcu_read_lock_bh(); 349 return 0; 350 } 351 352 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 353 { 354 rcu_read_unlock_bh(); 355 } 356 357 static int rcu_bh_torture_completed(void) 358 { 359 return rcu_batches_completed_bh(); 360 } 361 362 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 363 { 364 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 365 } 366 367 static struct rcu_torture_ops rcu_bh_ops = { 368 .init = rcu_sync_torture_init, 369 .readlock = rcu_bh_torture_read_lock, 370 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 371 .readunlock = rcu_bh_torture_read_unlock, 372 .completed = rcu_bh_torture_completed, 373 .deferred_free = rcu_bh_torture_deferred_free, 374 .sync = synchronize_rcu_bh, 375 .exp_sync = synchronize_rcu_bh_expedited, 376 .call = call_rcu_bh, 377 .cb_barrier = rcu_barrier_bh, 378 .fqs = rcu_bh_force_quiescent_state, 379 .stats = NULL, 380 .irq_capable = 1, 381 .name = "rcu_bh" 382 }; 383 384 /* 385 * Definitions for srcu torture testing. 386 */ 387 388 DEFINE_STATIC_SRCU(srcu_ctl); 389 390 static int srcu_torture_read_lock(void) __acquires(&srcu_ctl) 391 { 392 return srcu_read_lock(&srcu_ctl); 393 } 394 395 static void srcu_read_delay(struct torture_random_state *rrsp) 396 { 397 long delay; 398 const long uspertick = 1000000 / HZ; 399 const long longdelay = 10; 400 401 /* We want there to be long-running readers, but not all the time. */ 402 403 delay = torture_random(rrsp) % 404 (nrealreaders * 2 * longdelay * uspertick); 405 if (!delay) 406 schedule_timeout_interruptible(longdelay); 407 else 408 rcu_read_delay(rrsp); 409 } 410 411 static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl) 412 { 413 srcu_read_unlock(&srcu_ctl, idx); 414 } 415 416 static int srcu_torture_completed(void) 417 { 418 return srcu_batches_completed(&srcu_ctl); 419 } 420 421 static void srcu_torture_deferred_free(struct rcu_torture *rp) 422 { 423 call_srcu(&srcu_ctl, &rp->rtort_rcu, rcu_torture_cb); 424 } 425 426 static void srcu_torture_synchronize(void) 427 { 428 synchronize_srcu(&srcu_ctl); 429 } 430 431 static void srcu_torture_call(struct rcu_head *head, 432 void (*func)(struct rcu_head *head)) 433 { 434 call_srcu(&srcu_ctl, head, func); 435 } 436 437 static void srcu_torture_barrier(void) 438 { 439 srcu_barrier(&srcu_ctl); 440 } 441 442 static void srcu_torture_stats(char *page) 443 { 444 int cpu; 445 int idx = srcu_ctl.completed & 0x1; 446 447 page += sprintf(page, "%s%s per-CPU(idx=%d):", 448 torture_type, TORTURE_FLAG, idx); 449 for_each_possible_cpu(cpu) { 450 page += sprintf(page, " %d(%lu,%lu)", cpu, 451 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx], 452 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]); 453 } 454 sprintf(page, "\n"); 455 } 456 457 static void srcu_torture_synchronize_expedited(void) 458 { 459 synchronize_srcu_expedited(&srcu_ctl); 460 } 461 462 static struct rcu_torture_ops srcu_ops = { 463 .init = rcu_sync_torture_init, 464 .readlock = srcu_torture_read_lock, 465 .read_delay = srcu_read_delay, 466 .readunlock = srcu_torture_read_unlock, 467 .completed = srcu_torture_completed, 468 .deferred_free = srcu_torture_deferred_free, 469 .sync = srcu_torture_synchronize, 470 .exp_sync = srcu_torture_synchronize_expedited, 471 .call = srcu_torture_call, 472 .cb_barrier = srcu_torture_barrier, 473 .stats = srcu_torture_stats, 474 .name = "srcu" 475 }; 476 477 /* 478 * Definitions for sched torture testing. 479 */ 480 481 static int sched_torture_read_lock(void) 482 { 483 preempt_disable(); 484 return 0; 485 } 486 487 static void sched_torture_read_unlock(int idx) 488 { 489 preempt_enable(); 490 } 491 492 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 493 { 494 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 495 } 496 497 static struct rcu_torture_ops sched_ops = { 498 .init = rcu_sync_torture_init, 499 .readlock = sched_torture_read_lock, 500 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 501 .readunlock = sched_torture_read_unlock, 502 .completed = rcu_no_completed, 503 .deferred_free = rcu_sched_torture_deferred_free, 504 .sync = synchronize_sched, 505 .exp_sync = synchronize_sched_expedited, 506 .call = call_rcu_sched, 507 .cb_barrier = rcu_barrier_sched, 508 .fqs = rcu_sched_force_quiescent_state, 509 .stats = NULL, 510 .irq_capable = 1, 511 .name = "sched" 512 }; 513 514 /* 515 * RCU torture priority-boost testing. Runs one real-time thread per 516 * CPU for moderate bursts, repeatedly registering RCU callbacks and 517 * spinning waiting for them to be invoked. If a given callback takes 518 * too long to be invoked, we assume that priority inversion has occurred. 519 */ 520 521 struct rcu_boost_inflight { 522 struct rcu_head rcu; 523 int inflight; 524 }; 525 526 static void rcu_torture_boost_cb(struct rcu_head *head) 527 { 528 struct rcu_boost_inflight *rbip = 529 container_of(head, struct rcu_boost_inflight, rcu); 530 531 smp_mb(); /* Ensure RCU-core accesses precede clearing ->inflight */ 532 rbip->inflight = 0; 533 } 534 535 static int rcu_torture_boost(void *arg) 536 { 537 unsigned long call_rcu_time; 538 unsigned long endtime; 539 unsigned long oldstarttime; 540 struct rcu_boost_inflight rbi = { .inflight = 0 }; 541 struct sched_param sp; 542 543 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 544 545 /* Set real-time priority. */ 546 sp.sched_priority = 1; 547 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 548 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 549 n_rcu_torture_boost_rterror++; 550 } 551 552 init_rcu_head_on_stack(&rbi.rcu); 553 /* Each pass through the following loop does one boost-test cycle. */ 554 do { 555 /* Wait for the next test interval. */ 556 oldstarttime = boost_starttime; 557 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 558 schedule_timeout_interruptible(oldstarttime - jiffies); 559 stutter_wait("rcu_torture_boost"); 560 if (torture_must_stop()) 561 goto checkwait; 562 } 563 564 /* Do one boost-test interval. */ 565 endtime = oldstarttime + test_boost_duration * HZ; 566 call_rcu_time = jiffies; 567 while (ULONG_CMP_LT(jiffies, endtime)) { 568 /* If we don't have a callback in flight, post one. */ 569 if (!rbi.inflight) { 570 smp_mb(); /* RCU core before ->inflight = 1. */ 571 rbi.inflight = 1; 572 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 573 if (jiffies - call_rcu_time > 574 test_boost_duration * HZ - HZ / 2) { 575 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 576 n_rcu_torture_boost_failure++; 577 } 578 call_rcu_time = jiffies; 579 } 580 cond_resched(); 581 stutter_wait("rcu_torture_boost"); 582 if (torture_must_stop()) 583 goto checkwait; 584 } 585 586 /* 587 * Set the start time of the next test interval. 588 * Yes, this is vulnerable to long delays, but such 589 * delays simply cause a false negative for the next 590 * interval. Besides, we are running at RT priority, 591 * so delays should be relatively rare. 592 */ 593 while (oldstarttime == boost_starttime && 594 !kthread_should_stop()) { 595 if (mutex_trylock(&boost_mutex)) { 596 boost_starttime = jiffies + 597 test_boost_interval * HZ; 598 n_rcu_torture_boosts++; 599 mutex_unlock(&boost_mutex); 600 break; 601 } 602 schedule_timeout_uninterruptible(1); 603 } 604 605 /* Go do the stutter. */ 606 checkwait: stutter_wait("rcu_torture_boost"); 607 } while (!torture_must_stop()); 608 609 /* Clean up and exit. */ 610 VERBOSE_TOROUT_STRING("rcu_torture_boost task stopping"); 611 torture_shutdown_absorb("rcu_torture_boost"); 612 while (!kthread_should_stop() || rbi.inflight) 613 schedule_timeout_uninterruptible(1); 614 smp_mb(); /* order accesses to ->inflight before stack-frame death. */ 615 destroy_rcu_head_on_stack(&rbi.rcu); 616 return 0; 617 } 618 619 /* 620 * RCU torture force-quiescent-state kthread. Repeatedly induces 621 * bursts of calls to force_quiescent_state(), increasing the probability 622 * of occurrence of some important types of race conditions. 623 */ 624 static int 625 rcu_torture_fqs(void *arg) 626 { 627 unsigned long fqs_resume_time; 628 int fqs_burst_remaining; 629 630 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 631 do { 632 fqs_resume_time = jiffies + fqs_stutter * HZ; 633 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 634 !kthread_should_stop()) { 635 schedule_timeout_interruptible(1); 636 } 637 fqs_burst_remaining = fqs_duration; 638 while (fqs_burst_remaining > 0 && 639 !kthread_should_stop()) { 640 cur_ops->fqs(); 641 udelay(fqs_holdoff); 642 fqs_burst_remaining -= fqs_holdoff; 643 } 644 stutter_wait("rcu_torture_fqs"); 645 } while (!torture_must_stop()); 646 VERBOSE_TOROUT_STRING("rcu_torture_fqs task stopping"); 647 torture_shutdown_absorb("rcu_torture_fqs"); 648 while (!kthread_should_stop()) 649 schedule_timeout_uninterruptible(1); 650 return 0; 651 } 652 653 /* 654 * RCU torture writer kthread. Repeatedly substitutes a new structure 655 * for that pointed to by rcu_torture_current, freeing the old structure 656 * after a series of grace periods (the "pipeline"). 657 */ 658 static int 659 rcu_torture_writer(void *arg) 660 { 661 bool exp; 662 int i; 663 struct rcu_torture *rp; 664 struct rcu_torture *rp1; 665 struct rcu_torture *old_rp; 666 static DEFINE_TORTURE_RANDOM(rand); 667 668 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 669 set_user_nice(current, 19); 670 671 do { 672 schedule_timeout_uninterruptible(1); 673 rp = rcu_torture_alloc(); 674 if (rp == NULL) 675 continue; 676 rp->rtort_pipe_count = 0; 677 udelay(torture_random(&rand) & 0x3ff); 678 old_rp = rcu_dereference_check(rcu_torture_current, 679 current == writer_task); 680 rp->rtort_mbtest = 1; 681 rcu_assign_pointer(rcu_torture_current, rp); 682 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 683 if (old_rp) { 684 i = old_rp->rtort_pipe_count; 685 if (i > RCU_TORTURE_PIPE_LEN) 686 i = RCU_TORTURE_PIPE_LEN; 687 atomic_inc(&rcu_torture_wcount[i]); 688 old_rp->rtort_pipe_count++; 689 if (gp_normal == gp_exp) 690 exp = !!(torture_random(&rand) & 0x80); 691 else 692 exp = gp_exp; 693 if (!exp) { 694 cur_ops->deferred_free(old_rp); 695 } else { 696 cur_ops->exp_sync(); 697 list_add(&old_rp->rtort_free, 698 &rcu_torture_removed); 699 list_for_each_entry_safe(rp, rp1, 700 &rcu_torture_removed, 701 rtort_free) { 702 i = rp->rtort_pipe_count; 703 if (i > RCU_TORTURE_PIPE_LEN) 704 i = RCU_TORTURE_PIPE_LEN; 705 atomic_inc(&rcu_torture_wcount[i]); 706 if (++rp->rtort_pipe_count >= 707 RCU_TORTURE_PIPE_LEN) { 708 rp->rtort_mbtest = 0; 709 list_del(&rp->rtort_free); 710 rcu_torture_free(rp); 711 } 712 } 713 } 714 } 715 rcutorture_record_progress(++rcu_torture_current_version); 716 stutter_wait("rcu_torture_writer"); 717 } while (!torture_must_stop()); 718 VERBOSE_TOROUT_STRING("rcu_torture_writer task stopping"); 719 torture_shutdown_absorb("rcu_torture_writer"); 720 while (!kthread_should_stop()) 721 schedule_timeout_uninterruptible(1); 722 return 0; 723 } 724 725 /* 726 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 727 * delay between calls. 728 */ 729 static int 730 rcu_torture_fakewriter(void *arg) 731 { 732 DEFINE_TORTURE_RANDOM(rand); 733 734 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 735 set_user_nice(current, 19); 736 737 do { 738 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 739 udelay(torture_random(&rand) & 0x3ff); 740 if (cur_ops->cb_barrier != NULL && 741 torture_random(&rand) % (nfakewriters * 8) == 0) { 742 cur_ops->cb_barrier(); 743 } else if (gp_normal == gp_exp) { 744 if (torture_random(&rand) & 0x80) 745 cur_ops->sync(); 746 else 747 cur_ops->exp_sync(); 748 } else if (gp_normal) { 749 cur_ops->sync(); 750 } else { 751 cur_ops->exp_sync(); 752 } 753 stutter_wait("rcu_torture_fakewriter"); 754 } while (!torture_must_stop()); 755 756 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task stopping"); 757 torture_shutdown_absorb("rcu_torture_fakewriter"); 758 while (!kthread_should_stop()) 759 schedule_timeout_uninterruptible(1); 760 return 0; 761 } 762 763 void rcutorture_trace_dump(void) 764 { 765 static atomic_t beenhere = ATOMIC_INIT(0); 766 767 if (atomic_read(&beenhere)) 768 return; 769 if (atomic_xchg(&beenhere, 1) != 0) 770 return; 771 ftrace_dump(DUMP_ALL); 772 } 773 774 /* 775 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 776 * incrementing the corresponding element of the pipeline array. The 777 * counter in the element should never be greater than 1, otherwise, the 778 * RCU implementation is broken. 779 */ 780 static void rcu_torture_timer(unsigned long unused) 781 { 782 int idx; 783 int completed; 784 int completed_end; 785 static DEFINE_TORTURE_RANDOM(rand); 786 static DEFINE_SPINLOCK(rand_lock); 787 struct rcu_torture *p; 788 int pipe_count; 789 unsigned long long ts; 790 791 idx = cur_ops->readlock(); 792 completed = cur_ops->completed(); 793 ts = rcu_trace_clock_local(); 794 p = rcu_dereference_check(rcu_torture_current, 795 rcu_read_lock_bh_held() || 796 rcu_read_lock_sched_held() || 797 srcu_read_lock_held(&srcu_ctl)); 798 if (p == NULL) { 799 /* Leave because rcu_torture_writer is not yet underway */ 800 cur_ops->readunlock(idx); 801 return; 802 } 803 if (p->rtort_mbtest == 0) 804 atomic_inc(&n_rcu_torture_mberror); 805 spin_lock(&rand_lock); 806 cur_ops->read_delay(&rand); 807 n_rcu_torture_timers++; 808 spin_unlock(&rand_lock); 809 preempt_disable(); 810 pipe_count = p->rtort_pipe_count; 811 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 812 /* Should not happen, but... */ 813 pipe_count = RCU_TORTURE_PIPE_LEN; 814 } 815 completed_end = cur_ops->completed(); 816 if (pipe_count > 1) { 817 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 818 completed, completed_end); 819 rcutorture_trace_dump(); 820 } 821 __this_cpu_inc(rcu_torture_count[pipe_count]); 822 completed = completed_end - completed; 823 if (completed > RCU_TORTURE_PIPE_LEN) { 824 /* Should not happen, but... */ 825 completed = RCU_TORTURE_PIPE_LEN; 826 } 827 __this_cpu_inc(rcu_torture_batch[completed]); 828 preempt_enable(); 829 cur_ops->readunlock(idx); 830 } 831 832 /* 833 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 834 * incrementing the corresponding element of the pipeline array. The 835 * counter in the element should never be greater than 1, otherwise, the 836 * RCU implementation is broken. 837 */ 838 static int 839 rcu_torture_reader(void *arg) 840 { 841 int completed; 842 int completed_end; 843 int idx; 844 DEFINE_TORTURE_RANDOM(rand); 845 struct rcu_torture *p; 846 int pipe_count; 847 struct timer_list t; 848 unsigned long long ts; 849 850 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 851 set_user_nice(current, 19); 852 if (irqreader && cur_ops->irq_capable) 853 setup_timer_on_stack(&t, rcu_torture_timer, 0); 854 855 do { 856 if (irqreader && cur_ops->irq_capable) { 857 if (!timer_pending(&t)) 858 mod_timer(&t, jiffies + 1); 859 } 860 idx = cur_ops->readlock(); 861 completed = cur_ops->completed(); 862 ts = rcu_trace_clock_local(); 863 p = rcu_dereference_check(rcu_torture_current, 864 rcu_read_lock_bh_held() || 865 rcu_read_lock_sched_held() || 866 srcu_read_lock_held(&srcu_ctl)); 867 if (p == NULL) { 868 /* Wait for rcu_torture_writer to get underway */ 869 cur_ops->readunlock(idx); 870 schedule_timeout_interruptible(HZ); 871 continue; 872 } 873 if (p->rtort_mbtest == 0) 874 atomic_inc(&n_rcu_torture_mberror); 875 cur_ops->read_delay(&rand); 876 preempt_disable(); 877 pipe_count = p->rtort_pipe_count; 878 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 879 /* Should not happen, but... */ 880 pipe_count = RCU_TORTURE_PIPE_LEN; 881 } 882 completed_end = cur_ops->completed(); 883 if (pipe_count > 1) { 884 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 885 ts, completed, completed_end); 886 rcutorture_trace_dump(); 887 } 888 __this_cpu_inc(rcu_torture_count[pipe_count]); 889 completed = completed_end - completed; 890 if (completed > RCU_TORTURE_PIPE_LEN) { 891 /* Should not happen, but... */ 892 completed = RCU_TORTURE_PIPE_LEN; 893 } 894 __this_cpu_inc(rcu_torture_batch[completed]); 895 preempt_enable(); 896 cur_ops->readunlock(idx); 897 schedule(); 898 stutter_wait("rcu_torture_reader"); 899 } while (!torture_must_stop()); 900 VERBOSE_TOROUT_STRING("rcu_torture_reader task stopping"); 901 torture_shutdown_absorb("rcu_torture_reader"); 902 if (irqreader && cur_ops->irq_capable) 903 del_timer_sync(&t); 904 while (!kthread_should_stop()) 905 schedule_timeout_uninterruptible(1); 906 return 0; 907 } 908 909 /* 910 * Create an RCU-torture statistics message in the specified buffer. 911 */ 912 static void 913 rcu_torture_printk(char *page) 914 { 915 int cpu; 916 int i; 917 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 918 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 919 920 for_each_possible_cpu(cpu) { 921 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 922 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 923 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 924 } 925 } 926 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 927 if (pipesummary[i] != 0) 928 break; 929 } 930 page += sprintf(page, "%s%s ", torture_type, TORTURE_FLAG); 931 page += sprintf(page, 932 "rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 933 rcu_torture_current, 934 rcu_torture_current_version, 935 list_empty(&rcu_torture_freelist), 936 atomic_read(&n_rcu_torture_alloc), 937 atomic_read(&n_rcu_torture_alloc_fail), 938 atomic_read(&n_rcu_torture_free)); 939 page += sprintf(page, "rtmbe: %d rtbke: %ld rtbre: %ld ", 940 atomic_read(&n_rcu_torture_mberror), 941 n_rcu_torture_boost_ktrerror, 942 n_rcu_torture_boost_rterror); 943 page += sprintf(page, "rtbf: %ld rtb: %ld nt: %ld ", 944 n_rcu_torture_boost_failure, 945 n_rcu_torture_boosts, 946 n_rcu_torture_timers); 947 page = torture_onoff_stats(page); 948 page += sprintf(page, "barrier: %ld/%ld:%ld", 949 n_barrier_successes, 950 n_barrier_attempts, 951 n_rcu_torture_barrier_error); 952 page += sprintf(page, "\n%s%s ", torture_type, TORTURE_FLAG); 953 if (atomic_read(&n_rcu_torture_mberror) != 0 || 954 n_rcu_torture_barrier_error != 0 || 955 n_rcu_torture_boost_ktrerror != 0 || 956 n_rcu_torture_boost_rterror != 0 || 957 n_rcu_torture_boost_failure != 0 || 958 i > 1) { 959 page += sprintf(page, "!!! "); 960 atomic_inc(&n_rcu_torture_error); 961 WARN_ON_ONCE(1); 962 } 963 page += sprintf(page, "Reader Pipe: "); 964 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 965 page += sprintf(page, " %ld", pipesummary[i]); 966 page += sprintf(page, "\n%s%s ", torture_type, TORTURE_FLAG); 967 page += sprintf(page, "Reader Batch: "); 968 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 969 page += sprintf(page, " %ld", batchsummary[i]); 970 page += sprintf(page, "\n%s%s ", torture_type, TORTURE_FLAG); 971 page += sprintf(page, "Free-Block Circulation: "); 972 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 973 page += sprintf(page, " %d", 974 atomic_read(&rcu_torture_wcount[i])); 975 } 976 page += sprintf(page, "\n"); 977 if (cur_ops->stats) 978 cur_ops->stats(page); 979 } 980 981 /* 982 * Print torture statistics. Caller must ensure that there is only 983 * one call to this function at a given time!!! This is normally 984 * accomplished by relying on the module system to only have one copy 985 * of the module loaded, and then by giving the rcu_torture_stats 986 * kthread full control (or the init/cleanup functions when rcu_torture_stats 987 * thread is not running). 988 */ 989 static void 990 rcu_torture_stats_print(void) 991 { 992 int size = nr_cpu_ids * 200 + 8192; 993 char *buf; 994 995 buf = kmalloc(size, GFP_KERNEL); 996 if (!buf) { 997 pr_err("rcu-torture: Out of memory, need: %d", size); 998 return; 999 } 1000 rcu_torture_printk(buf); 1001 pr_alert("%s", buf); 1002 kfree(buf); 1003 } 1004 1005 /* 1006 * Periodically prints torture statistics, if periodic statistics printing 1007 * was specified via the stat_interval module parameter. 1008 */ 1009 static int 1010 rcu_torture_stats(void *arg) 1011 { 1012 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1013 do { 1014 schedule_timeout_interruptible(stat_interval * HZ); 1015 rcu_torture_stats_print(); 1016 torture_shutdown_absorb("rcu_torture_stats"); 1017 } while (!torture_must_stop()); 1018 VERBOSE_TOROUT_STRING("rcu_torture_stats task stopping"); 1019 return 0; 1020 } 1021 1022 static inline void 1023 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1024 { 1025 pr_alert("%s" TORTURE_FLAG 1026 "--- %s: nreaders=%d nfakewriters=%d " 1027 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1028 "shuffle_interval=%d stutter=%d irqreader=%d " 1029 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1030 "test_boost=%d/%d test_boost_interval=%d " 1031 "test_boost_duration=%d shutdown_secs=%d " 1032 "stall_cpu=%d stall_cpu_holdoff=%d " 1033 "n_barrier_cbs=%d " 1034 "onoff_interval=%d onoff_holdoff=%d\n", 1035 torture_type, tag, nrealreaders, nfakewriters, 1036 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1037 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1038 test_boost, cur_ops->can_boost, 1039 test_boost_interval, test_boost_duration, shutdown_secs, 1040 stall_cpu, stall_cpu_holdoff, 1041 n_barrier_cbs, 1042 onoff_interval, onoff_holdoff); 1043 } 1044 1045 static void rcutorture_booster_cleanup(int cpu) 1046 { 1047 struct task_struct *t; 1048 1049 if (boost_tasks[cpu] == NULL) 1050 return; 1051 mutex_lock(&boost_mutex); 1052 VERBOSE_TOROUT_STRING("Stopping rcu_torture_boost task"); 1053 t = boost_tasks[cpu]; 1054 boost_tasks[cpu] = NULL; 1055 mutex_unlock(&boost_mutex); 1056 1057 /* This must be outside of the mutex, otherwise deadlock! */ 1058 kthread_stop(t); 1059 boost_tasks[cpu] = NULL; 1060 } 1061 1062 static int rcutorture_booster_init(int cpu) 1063 { 1064 int retval; 1065 1066 if (boost_tasks[cpu] != NULL) 1067 return 0; /* Already created, nothing more to do. */ 1068 1069 /* Don't allow time recalculation while creating a new task. */ 1070 mutex_lock(&boost_mutex); 1071 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1072 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1073 cpu_to_node(cpu), 1074 "rcu_torture_boost"); 1075 if (IS_ERR(boost_tasks[cpu])) { 1076 retval = PTR_ERR(boost_tasks[cpu]); 1077 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1078 n_rcu_torture_boost_ktrerror++; 1079 boost_tasks[cpu] = NULL; 1080 mutex_unlock(&boost_mutex); 1081 return retval; 1082 } 1083 kthread_bind(boost_tasks[cpu], cpu); 1084 wake_up_process(boost_tasks[cpu]); 1085 mutex_unlock(&boost_mutex); 1086 return 0; 1087 } 1088 1089 /* 1090 * Cause the rcutorture test to shutdown the system after the test has 1091 * run for the time specified by the shutdown_secs module parameter. 1092 */ 1093 static int 1094 rcu_torture_shutdown(void *arg) 1095 { 1096 long delta; 1097 unsigned long jiffies_snap; 1098 1099 VERBOSE_TOROUT_STRING("rcu_torture_shutdown task started"); 1100 jiffies_snap = ACCESS_ONCE(jiffies); 1101 while (ULONG_CMP_LT(jiffies_snap, shutdown_time) && 1102 !kthread_should_stop()) { 1103 delta = shutdown_time - jiffies_snap; 1104 if (verbose) 1105 pr_alert("%s" TORTURE_FLAG 1106 "rcu_torture_shutdown task: %lu jiffies remaining\n", 1107 torture_type, delta); 1108 schedule_timeout_interruptible(delta); 1109 jiffies_snap = ACCESS_ONCE(jiffies); 1110 } 1111 if (kthread_should_stop()) { 1112 VERBOSE_TOROUT_STRING("rcu_torture_shutdown task stopping"); 1113 return 0; 1114 } 1115 1116 /* OK, shut down the system. */ 1117 1118 VERBOSE_TOROUT_STRING("rcu_torture_shutdown task shutting down system"); 1119 shutdown_task = NULL; /* Avoid self-kill deadlock. */ 1120 rcu_torture_cleanup(); /* Get the success/failure message. */ 1121 kernel_power_off(); /* Shut down the system. */ 1122 return 0; 1123 } 1124 1125 /* 1126 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1127 * induces a CPU stall for the time specified by stall_cpu. 1128 */ 1129 static int rcu_torture_stall(void *args) 1130 { 1131 unsigned long stop_at; 1132 1133 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1134 if (stall_cpu_holdoff > 0) { 1135 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1136 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1137 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1138 } 1139 if (!kthread_should_stop()) { 1140 stop_at = get_seconds() + stall_cpu; 1141 /* RCU CPU stall is expected behavior in following code. */ 1142 pr_alert("rcu_torture_stall start.\n"); 1143 rcu_read_lock(); 1144 preempt_disable(); 1145 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1146 continue; /* Induce RCU CPU stall warning. */ 1147 preempt_enable(); 1148 rcu_read_unlock(); 1149 pr_alert("rcu_torture_stall end.\n"); 1150 } 1151 torture_shutdown_absorb("rcu_torture_stall"); 1152 while (!kthread_should_stop()) 1153 schedule_timeout_interruptible(10 * HZ); 1154 return 0; 1155 } 1156 1157 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1158 static int __init rcu_torture_stall_init(void) 1159 { 1160 int ret; 1161 1162 if (stall_cpu <= 0) 1163 return 0; 1164 stall_task = kthread_run(rcu_torture_stall, NULL, "rcu_torture_stall"); 1165 if (IS_ERR(stall_task)) { 1166 ret = PTR_ERR(stall_task); 1167 stall_task = NULL; 1168 return ret; 1169 } 1170 torture_shuffle_task_register(stall_task); 1171 return 0; 1172 } 1173 1174 /* Clean up after the CPU-stall kthread, if one was spawned. */ 1175 static void rcu_torture_stall_cleanup(void) 1176 { 1177 if (stall_task == NULL) 1178 return; 1179 VERBOSE_TOROUT_STRING("Stopping rcu_torture_stall_task."); 1180 kthread_stop(stall_task); 1181 stall_task = NULL; 1182 } 1183 1184 /* Callback function for RCU barrier testing. */ 1185 void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1186 { 1187 atomic_inc(&barrier_cbs_invoked); 1188 } 1189 1190 /* kthread function to register callbacks used to test RCU barriers. */ 1191 static int rcu_torture_barrier_cbs(void *arg) 1192 { 1193 long myid = (long)arg; 1194 bool lastphase = 0; 1195 bool newphase; 1196 struct rcu_head rcu; 1197 1198 init_rcu_head_on_stack(&rcu); 1199 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1200 set_user_nice(current, 19); 1201 do { 1202 wait_event(barrier_cbs_wq[myid], 1203 (newphase = 1204 ACCESS_ONCE(barrier_phase)) != lastphase || 1205 torture_must_stop()); 1206 lastphase = newphase; 1207 smp_mb(); /* ensure barrier_phase load before ->call(). */ 1208 if (torture_must_stop()) 1209 break; 1210 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1211 if (atomic_dec_and_test(&barrier_cbs_count)) 1212 wake_up(&barrier_wq); 1213 } while (!torture_must_stop()); 1214 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task stopping"); 1215 torture_shutdown_absorb("rcu_torture_barrier_cbs"); 1216 while (!kthread_should_stop()) 1217 schedule_timeout_interruptible(1); 1218 cur_ops->cb_barrier(); 1219 destroy_rcu_head_on_stack(&rcu); 1220 return 0; 1221 } 1222 1223 /* kthread function to drive and coordinate RCU barrier testing. */ 1224 static int rcu_torture_barrier(void *arg) 1225 { 1226 int i; 1227 1228 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1229 do { 1230 atomic_set(&barrier_cbs_invoked, 0); 1231 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1232 smp_mb(); /* Ensure barrier_phase after prior assignments. */ 1233 barrier_phase = !barrier_phase; 1234 for (i = 0; i < n_barrier_cbs; i++) 1235 wake_up(&barrier_cbs_wq[i]); 1236 wait_event(barrier_wq, 1237 atomic_read(&barrier_cbs_count) == 0 || 1238 torture_must_stop()); 1239 if (torture_must_stop()) 1240 break; 1241 n_barrier_attempts++; 1242 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1243 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1244 n_rcu_torture_barrier_error++; 1245 WARN_ON_ONCE(1); 1246 } 1247 n_barrier_successes++; 1248 schedule_timeout_interruptible(HZ / 10); 1249 } while (!torture_must_stop()); 1250 VERBOSE_TOROUT_STRING("rcu_torture_barrier task stopping"); 1251 torture_shutdown_absorb("rcu_torture_barrier"); 1252 while (!kthread_should_stop()) 1253 schedule_timeout_interruptible(1); 1254 return 0; 1255 } 1256 1257 /* Initialize RCU barrier testing. */ 1258 static int rcu_torture_barrier_init(void) 1259 { 1260 int i; 1261 int ret; 1262 1263 if (n_barrier_cbs == 0) 1264 return 0; 1265 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1266 pr_alert("%s" TORTURE_FLAG 1267 " Call or barrier ops missing for %s,\n", 1268 torture_type, cur_ops->name); 1269 pr_alert("%s" TORTURE_FLAG 1270 " RCU barrier testing omitted from run.\n", 1271 torture_type); 1272 return 0; 1273 } 1274 atomic_set(&barrier_cbs_count, 0); 1275 atomic_set(&barrier_cbs_invoked, 0); 1276 barrier_cbs_tasks = 1277 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1278 GFP_KERNEL); 1279 barrier_cbs_wq = 1280 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1281 GFP_KERNEL); 1282 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1283 return -ENOMEM; 1284 for (i = 0; i < n_barrier_cbs; i++) { 1285 init_waitqueue_head(&barrier_cbs_wq[i]); 1286 barrier_cbs_tasks[i] = kthread_run(rcu_torture_barrier_cbs, 1287 (void *)(long)i, 1288 "rcu_torture_barrier_cbs"); 1289 if (IS_ERR(barrier_cbs_tasks[i])) { 1290 ret = PTR_ERR(barrier_cbs_tasks[i]); 1291 VERBOSE_TOROUT_ERRSTRING("Failed to create rcu_torture_barrier_cbs"); 1292 barrier_cbs_tasks[i] = NULL; 1293 return ret; 1294 } 1295 torture_shuffle_task_register(barrier_cbs_tasks[i]); 1296 } 1297 barrier_task = kthread_run(rcu_torture_barrier, NULL, 1298 "rcu_torture_barrier"); 1299 if (IS_ERR(barrier_task)) { 1300 ret = PTR_ERR(barrier_task); 1301 VERBOSE_TOROUT_ERRSTRING("Failed to create rcu_torture_barrier"); 1302 barrier_task = NULL; 1303 } 1304 torture_shuffle_task_register(barrier_task); 1305 return 0; 1306 } 1307 1308 /* Clean up after RCU barrier testing. */ 1309 static void rcu_torture_barrier_cleanup(void) 1310 { 1311 int i; 1312 1313 if (barrier_task != NULL) { 1314 VERBOSE_TOROUT_STRING("Stopping rcu_torture_barrier task"); 1315 kthread_stop(barrier_task); 1316 barrier_task = NULL; 1317 } 1318 if (barrier_cbs_tasks != NULL) { 1319 for (i = 0; i < n_barrier_cbs; i++) { 1320 if (barrier_cbs_tasks[i] != NULL) { 1321 VERBOSE_TOROUT_STRING("Stopping rcu_torture_barrier_cbs task"); 1322 kthread_stop(barrier_cbs_tasks[i]); 1323 barrier_cbs_tasks[i] = NULL; 1324 } 1325 } 1326 kfree(barrier_cbs_tasks); 1327 barrier_cbs_tasks = NULL; 1328 } 1329 if (barrier_cbs_wq != NULL) { 1330 kfree(barrier_cbs_wq); 1331 barrier_cbs_wq = NULL; 1332 } 1333 } 1334 1335 static int rcutorture_cpu_notify(struct notifier_block *self, 1336 unsigned long action, void *hcpu) 1337 { 1338 long cpu = (long)hcpu; 1339 1340 switch (action) { 1341 case CPU_ONLINE: 1342 case CPU_DOWN_FAILED: 1343 (void)rcutorture_booster_init(cpu); 1344 break; 1345 case CPU_DOWN_PREPARE: 1346 rcutorture_booster_cleanup(cpu); 1347 break; 1348 default: 1349 break; 1350 } 1351 return NOTIFY_OK; 1352 } 1353 1354 static struct notifier_block rcutorture_cpu_nb = { 1355 .notifier_call = rcutorture_cpu_notify, 1356 }; 1357 1358 static void 1359 rcu_torture_cleanup(void) 1360 { 1361 int i; 1362 1363 rcutorture_record_test_transition(); 1364 if (torture_cleanup()) { 1365 if (cur_ops->cb_barrier != NULL) 1366 cur_ops->cb_barrier(); 1367 return; 1368 } 1369 1370 rcu_torture_barrier_cleanup(); 1371 rcu_torture_stall_cleanup(); 1372 torture_stutter_cleanup(); 1373 1374 if (writer_task) { 1375 VERBOSE_TOROUT_STRING("Stopping rcu_torture_writer task"); 1376 kthread_stop(writer_task); 1377 } 1378 writer_task = NULL; 1379 1380 if (reader_tasks) { 1381 for (i = 0; i < nrealreaders; i++) { 1382 if (reader_tasks[i]) { 1383 VERBOSE_TOROUT_STRING( 1384 "Stopping rcu_torture_reader task"); 1385 kthread_stop(reader_tasks[i]); 1386 } 1387 reader_tasks[i] = NULL; 1388 } 1389 kfree(reader_tasks); 1390 reader_tasks = NULL; 1391 } 1392 rcu_torture_current = NULL; 1393 1394 if (fakewriter_tasks) { 1395 for (i = 0; i < nfakewriters; i++) { 1396 if (fakewriter_tasks[i]) { 1397 VERBOSE_TOROUT_STRING( 1398 "Stopping rcu_torture_fakewriter task"); 1399 kthread_stop(fakewriter_tasks[i]); 1400 } 1401 fakewriter_tasks[i] = NULL; 1402 } 1403 kfree(fakewriter_tasks); 1404 fakewriter_tasks = NULL; 1405 } 1406 1407 if (stats_task) { 1408 VERBOSE_TOROUT_STRING("Stopping rcu_torture_stats task"); 1409 kthread_stop(stats_task); 1410 } 1411 stats_task = NULL; 1412 1413 if (fqs_task) { 1414 VERBOSE_TOROUT_STRING("Stopping rcu_torture_fqs task"); 1415 kthread_stop(fqs_task); 1416 } 1417 fqs_task = NULL; 1418 if ((test_boost == 1 && cur_ops->can_boost) || 1419 test_boost == 2) { 1420 unregister_cpu_notifier(&rcutorture_cpu_nb); 1421 for_each_possible_cpu(i) 1422 rcutorture_booster_cleanup(i); 1423 } 1424 if (shutdown_task != NULL) { 1425 VERBOSE_TOROUT_STRING("Stopping rcu_torture_shutdown task"); 1426 kthread_stop(shutdown_task); 1427 } 1428 shutdown_task = NULL; 1429 1430 /* Wait for all RCU callbacks to fire. */ 1431 1432 if (cur_ops->cb_barrier != NULL) 1433 cur_ops->cb_barrier(); 1434 1435 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1436 1437 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1438 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1439 else if (torture_onoff_failures()) 1440 rcu_torture_print_module_parms(cur_ops, 1441 "End of test: RCU_HOTPLUG"); 1442 else 1443 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1444 } 1445 1446 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1447 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1448 { 1449 } 1450 1451 static void rcu_torture_err_cb(struct rcu_head *rhp) 1452 { 1453 /* 1454 * This -might- happen due to race conditions, but is unlikely. 1455 * The scenario that leads to this happening is that the 1456 * first of the pair of duplicate callbacks is queued, 1457 * someone else starts a grace period that includes that 1458 * callback, then the second of the pair must wait for the 1459 * next grace period. Unlikely, but can happen. If it 1460 * does happen, the debug-objects subsystem won't have splatted. 1461 */ 1462 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1463 } 1464 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1465 1466 /* 1467 * Verify that double-free causes debug-objects to complain, but only 1468 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1469 * cannot be carried out. 1470 */ 1471 static void rcu_test_debug_objects(void) 1472 { 1473 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1474 struct rcu_head rh1; 1475 struct rcu_head rh2; 1476 1477 init_rcu_head_on_stack(&rh1); 1478 init_rcu_head_on_stack(&rh2); 1479 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1480 1481 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1482 preempt_disable(); /* Prevent preemption from interrupting test. */ 1483 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1484 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1485 local_irq_disable(); /* Make it harder to start a new grace period. */ 1486 call_rcu(&rh2, rcu_torture_leak_cb); 1487 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1488 local_irq_enable(); 1489 rcu_read_unlock(); 1490 preempt_enable(); 1491 1492 /* Wait for them all to get done so we can safely return. */ 1493 rcu_barrier(); 1494 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1495 destroy_rcu_head_on_stack(&rh1); 1496 destroy_rcu_head_on_stack(&rh2); 1497 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1498 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1499 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1500 } 1501 1502 static int __init 1503 rcu_torture_init(void) 1504 { 1505 int i; 1506 int cpu; 1507 int firsterr = 0; 1508 int retval; 1509 static struct rcu_torture_ops *torture_ops[] = { 1510 &rcu_ops, &rcu_bh_ops, &srcu_ops, &sched_ops, 1511 }; 1512 1513 torture_init_begin(torture_type, verbose, &rcutorture_runnable); 1514 1515 /* Process args and tell the world that the torturer is on the job. */ 1516 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1517 cur_ops = torture_ops[i]; 1518 if (strcmp(torture_type, cur_ops->name) == 0) 1519 break; 1520 } 1521 if (i == ARRAY_SIZE(torture_ops)) { 1522 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1523 torture_type); 1524 pr_alert("rcu-torture types:"); 1525 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1526 pr_alert(" %s", torture_ops[i]->name); 1527 pr_alert("\n"); 1528 torture_init_end(); 1529 return -EINVAL; 1530 } 1531 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1532 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1533 fqs_duration = 0; 1534 } 1535 if (cur_ops->init) 1536 cur_ops->init(); /* no "goto unwind" prior to this point!!! */ 1537 1538 if (nreaders >= 0) 1539 nrealreaders = nreaders; 1540 else 1541 nrealreaders = 2 * num_online_cpus(); 1542 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1543 1544 /* Set up the freelist. */ 1545 1546 INIT_LIST_HEAD(&rcu_torture_freelist); 1547 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1548 rcu_tortures[i].rtort_mbtest = 0; 1549 list_add_tail(&rcu_tortures[i].rtort_free, 1550 &rcu_torture_freelist); 1551 } 1552 1553 /* Initialize the statistics so that each run gets its own numbers. */ 1554 1555 rcu_torture_current = NULL; 1556 rcu_torture_current_version = 0; 1557 atomic_set(&n_rcu_torture_alloc, 0); 1558 atomic_set(&n_rcu_torture_alloc_fail, 0); 1559 atomic_set(&n_rcu_torture_free, 0); 1560 atomic_set(&n_rcu_torture_mberror, 0); 1561 atomic_set(&n_rcu_torture_error, 0); 1562 n_rcu_torture_barrier_error = 0; 1563 n_rcu_torture_boost_ktrerror = 0; 1564 n_rcu_torture_boost_rterror = 0; 1565 n_rcu_torture_boost_failure = 0; 1566 n_rcu_torture_boosts = 0; 1567 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1568 atomic_set(&rcu_torture_wcount[i], 0); 1569 for_each_possible_cpu(cpu) { 1570 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1571 per_cpu(rcu_torture_count, cpu)[i] = 0; 1572 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1573 } 1574 } 1575 1576 /* Start up the kthreads. */ 1577 1578 VERBOSE_TOROUT_STRING("Creating rcu_torture_writer task"); 1579 writer_task = kthread_create(rcu_torture_writer, NULL, 1580 "rcu_torture_writer"); 1581 if (IS_ERR(writer_task)) { 1582 firsterr = PTR_ERR(writer_task); 1583 VERBOSE_TOROUT_ERRSTRING("Failed to create writer"); 1584 writer_task = NULL; 1585 goto unwind; 1586 } 1587 torture_shuffle_task_register(writer_task); 1588 wake_up_process(writer_task); 1589 fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]), 1590 GFP_KERNEL); 1591 if (fakewriter_tasks == NULL) { 1592 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1593 firsterr = -ENOMEM; 1594 goto unwind; 1595 } 1596 for (i = 0; i < nfakewriters; i++) { 1597 VERBOSE_TOROUT_STRING("Creating rcu_torture_fakewriter task"); 1598 fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL, 1599 "rcu_torture_fakewriter"); 1600 if (IS_ERR(fakewriter_tasks[i])) { 1601 firsterr = PTR_ERR(fakewriter_tasks[i]); 1602 VERBOSE_TOROUT_ERRSTRING("Failed to create fakewriter"); 1603 fakewriter_tasks[i] = NULL; 1604 goto unwind; 1605 } 1606 torture_shuffle_task_register(fakewriter_tasks[i]); 1607 } 1608 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1609 GFP_KERNEL); 1610 if (reader_tasks == NULL) { 1611 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1612 firsterr = -ENOMEM; 1613 goto unwind; 1614 } 1615 for (i = 0; i < nrealreaders; i++) { 1616 VERBOSE_TOROUT_STRING("Creating rcu_torture_reader task"); 1617 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL, 1618 "rcu_torture_reader"); 1619 if (IS_ERR(reader_tasks[i])) { 1620 firsterr = PTR_ERR(reader_tasks[i]); 1621 VERBOSE_TOROUT_ERRSTRING("Failed to create reader"); 1622 reader_tasks[i] = NULL; 1623 goto unwind; 1624 } 1625 torture_shuffle_task_register(reader_tasks[i]); 1626 } 1627 if (stat_interval > 0) { 1628 VERBOSE_TOROUT_STRING("Creating rcu_torture_stats task"); 1629 stats_task = kthread_run(rcu_torture_stats, NULL, 1630 "rcu_torture_stats"); 1631 if (IS_ERR(stats_task)) { 1632 firsterr = PTR_ERR(stats_task); 1633 VERBOSE_TOROUT_ERRSTRING("Failed to create stats"); 1634 stats_task = NULL; 1635 goto unwind; 1636 } 1637 torture_shuffle_task_register(stats_task); 1638 } 1639 if (test_no_idle_hz) { 1640 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1641 if (firsterr) 1642 goto unwind; 1643 } 1644 if (stutter < 0) 1645 stutter = 0; 1646 if (stutter) { 1647 firsterr = torture_stutter_init(stutter * HZ); 1648 if (firsterr) 1649 goto unwind; 1650 } 1651 if (fqs_duration < 0) 1652 fqs_duration = 0; 1653 if (fqs_duration) { 1654 /* Create the fqs thread */ 1655 fqs_task = kthread_run(rcu_torture_fqs, NULL, 1656 "rcu_torture_fqs"); 1657 if (IS_ERR(fqs_task)) { 1658 firsterr = PTR_ERR(fqs_task); 1659 VERBOSE_TOROUT_ERRSTRING("Failed to create fqs"); 1660 fqs_task = NULL; 1661 goto unwind; 1662 } 1663 torture_shuffle_task_register(fqs_task); 1664 } 1665 if (test_boost_interval < 1) 1666 test_boost_interval = 1; 1667 if (test_boost_duration < 2) 1668 test_boost_duration = 2; 1669 if ((test_boost == 1 && cur_ops->can_boost) || 1670 test_boost == 2) { 1671 1672 boost_starttime = jiffies + test_boost_interval * HZ; 1673 register_cpu_notifier(&rcutorture_cpu_nb); 1674 for_each_possible_cpu(i) { 1675 if (cpu_is_offline(i)) 1676 continue; /* Heuristic: CPU can go offline. */ 1677 retval = rcutorture_booster_init(i); 1678 if (retval < 0) { 1679 firsterr = retval; 1680 goto unwind; 1681 } 1682 } 1683 } 1684 if (shutdown_secs > 0) { 1685 shutdown_time = jiffies + shutdown_secs * HZ; 1686 shutdown_task = kthread_create(rcu_torture_shutdown, NULL, 1687 "rcu_torture_shutdown"); 1688 if (IS_ERR(shutdown_task)) { 1689 firsterr = PTR_ERR(shutdown_task); 1690 VERBOSE_TOROUT_ERRSTRING("Failed to create shutdown"); 1691 shutdown_task = NULL; 1692 goto unwind; 1693 } 1694 torture_shuffle_task_register(shutdown_task); 1695 wake_up_process(shutdown_task); 1696 } 1697 i = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1698 if (i != 0) { 1699 firsterr = i; 1700 goto unwind; 1701 } 1702 i = rcu_torture_stall_init(); 1703 if (i != 0) { 1704 firsterr = i; 1705 goto unwind; 1706 } 1707 retval = rcu_torture_barrier_init(); 1708 if (retval != 0) { 1709 firsterr = retval; 1710 goto unwind; 1711 } 1712 if (object_debug) 1713 rcu_test_debug_objects(); 1714 rcutorture_record_test_transition(); 1715 torture_init_end(); 1716 return 0; 1717 1718 unwind: 1719 torture_init_end(); 1720 rcu_torture_cleanup(); 1721 return firsterr; 1722 } 1723 1724 module_init(rcu_torture_init); 1725 module_exit(rcu_torture_cleanup); 1726