1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched.h> 36 #include <linux/atomic.h> 37 #include <linux/bitops.h> 38 #include <linux/completion.h> 39 #include <linux/moduleparam.h> 40 #include <linux/percpu.h> 41 #include <linux/notifier.h> 42 #include <linux/reboot.h> 43 #include <linux/freezer.h> 44 #include <linux/cpu.h> 45 #include <linux/delay.h> 46 #include <linux/stat.h> 47 #include <linux/srcu.h> 48 #include <linux/slab.h> 49 #include <linux/trace_clock.h> 50 #include <asm/byteorder.h> 51 #include <linux/torture.h> 52 #include <linux/vmalloc.h> 53 54 MODULE_LICENSE("GPL"); 55 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 56 57 58 torture_param(int, cbflood_inter_holdoff, HZ, 59 "Holdoff between floods (jiffies)"); 60 torture_param(int, cbflood_intra_holdoff, 1, 61 "Holdoff between bursts (jiffies)"); 62 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 63 torture_param(int, cbflood_n_per_burst, 20000, 64 "# callbacks per burst in flood"); 65 torture_param(int, fqs_duration, 0, 66 "Duration of fqs bursts (us), 0 to disable"); 67 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 68 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 69 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 70 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 71 torture_param(bool, gp_normal, false, 72 "Use normal (non-expedited) GP wait primitives"); 73 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 74 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 75 torture_param(int, n_barrier_cbs, 0, 76 "# of callbacks/kthreads for barrier testing"); 77 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 78 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 79 torture_param(int, object_debug, 0, 80 "Enable debug-object double call_rcu() testing"); 81 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 82 torture_param(int, onoff_interval, 0, 83 "Time between CPU hotplugs (s), 0=disable"); 84 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 85 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 86 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 87 torture_param(int, stall_cpu_holdoff, 10, 88 "Time to wait before starting stall (s)."); 89 torture_param(int, stat_interval, 60, 90 "Number of seconds between stats printk()s"); 91 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 92 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 93 torture_param(int, test_boost_duration, 4, 94 "Duration of each boost test, seconds."); 95 torture_param(int, test_boost_interval, 7, 96 "Interval between boost tests, seconds."); 97 torture_param(bool, test_no_idle_hz, true, 98 "Test support for tickless idle CPUs"); 99 torture_param(bool, verbose, true, 100 "Enable verbose debugging printk()s"); 101 102 static char *torture_type = "rcu"; 103 module_param(torture_type, charp, 0444); 104 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 105 106 static int nrealreaders; 107 static int ncbflooders; 108 static struct task_struct *writer_task; 109 static struct task_struct **fakewriter_tasks; 110 static struct task_struct **reader_tasks; 111 static struct task_struct *stats_task; 112 static struct task_struct **cbflood_task; 113 static struct task_struct *fqs_task; 114 static struct task_struct *boost_tasks[NR_CPUS]; 115 static struct task_struct *stall_task; 116 static struct task_struct **barrier_cbs_tasks; 117 static struct task_struct *barrier_task; 118 119 #define RCU_TORTURE_PIPE_LEN 10 120 121 struct rcu_torture { 122 struct rcu_head rtort_rcu; 123 int rtort_pipe_count; 124 struct list_head rtort_free; 125 int rtort_mbtest; 126 }; 127 128 static LIST_HEAD(rcu_torture_freelist); 129 static struct rcu_torture __rcu *rcu_torture_current; 130 static unsigned long rcu_torture_current_version; 131 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 132 static DEFINE_SPINLOCK(rcu_torture_lock); 133 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 134 rcu_torture_count) = { 0 }; 135 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 136 rcu_torture_batch) = { 0 }; 137 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 138 static atomic_t n_rcu_torture_alloc; 139 static atomic_t n_rcu_torture_alloc_fail; 140 static atomic_t n_rcu_torture_free; 141 static atomic_t n_rcu_torture_mberror; 142 static atomic_t n_rcu_torture_error; 143 static long n_rcu_torture_barrier_error; 144 static long n_rcu_torture_boost_ktrerror; 145 static long n_rcu_torture_boost_rterror; 146 static long n_rcu_torture_boost_failure; 147 static long n_rcu_torture_boosts; 148 static long n_rcu_torture_timers; 149 static long n_barrier_attempts; 150 static long n_barrier_successes; 151 static atomic_long_t n_cbfloods; 152 static struct list_head rcu_torture_removed; 153 154 static int rcu_torture_writer_state; 155 #define RTWS_FIXED_DELAY 0 156 #define RTWS_DELAY 1 157 #define RTWS_REPLACE 2 158 #define RTWS_DEF_FREE 3 159 #define RTWS_EXP_SYNC 4 160 #define RTWS_COND_GET 5 161 #define RTWS_COND_SYNC 6 162 #define RTWS_SYNC 7 163 #define RTWS_STUTTER 8 164 #define RTWS_STOPPING 9 165 166 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE) 167 #define RCUTORTURE_RUNNABLE_INIT 1 168 #else 169 #define RCUTORTURE_RUNNABLE_INIT 0 170 #endif 171 static int torture_runnable = RCUTORTURE_RUNNABLE_INIT; 172 module_param(torture_runnable, int, 0444); 173 MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot"); 174 175 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 176 #define rcu_can_boost() 1 177 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 178 #define rcu_can_boost() 0 179 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 180 181 #ifdef CONFIG_RCU_TRACE 182 static u64 notrace rcu_trace_clock_local(void) 183 { 184 u64 ts = trace_clock_local(); 185 unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC); 186 return ts; 187 } 188 #else /* #ifdef CONFIG_RCU_TRACE */ 189 static u64 notrace rcu_trace_clock_local(void) 190 { 191 return 0ULL; 192 } 193 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 194 195 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 196 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 197 /* and boost task create/destroy. */ 198 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 199 static bool barrier_phase; /* Test phase. */ 200 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 201 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 202 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 203 204 /* 205 * Allocate an element from the rcu_tortures pool. 206 */ 207 static struct rcu_torture * 208 rcu_torture_alloc(void) 209 { 210 struct list_head *p; 211 212 spin_lock_bh(&rcu_torture_lock); 213 if (list_empty(&rcu_torture_freelist)) { 214 atomic_inc(&n_rcu_torture_alloc_fail); 215 spin_unlock_bh(&rcu_torture_lock); 216 return NULL; 217 } 218 atomic_inc(&n_rcu_torture_alloc); 219 p = rcu_torture_freelist.next; 220 list_del_init(p); 221 spin_unlock_bh(&rcu_torture_lock); 222 return container_of(p, struct rcu_torture, rtort_free); 223 } 224 225 /* 226 * Free an element to the rcu_tortures pool. 227 */ 228 static void 229 rcu_torture_free(struct rcu_torture *p) 230 { 231 atomic_inc(&n_rcu_torture_free); 232 spin_lock_bh(&rcu_torture_lock); 233 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 234 spin_unlock_bh(&rcu_torture_lock); 235 } 236 237 /* 238 * Operations vector for selecting different types of tests. 239 */ 240 241 struct rcu_torture_ops { 242 int ttype; 243 void (*init)(void); 244 int (*readlock)(void); 245 void (*read_delay)(struct torture_random_state *rrsp); 246 void (*readunlock)(int idx); 247 int (*completed)(void); 248 void (*deferred_free)(struct rcu_torture *p); 249 void (*sync)(void); 250 void (*exp_sync)(void); 251 unsigned long (*get_state)(void); 252 void (*cond_sync)(unsigned long oldstate); 253 void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 254 void (*cb_barrier)(void); 255 void (*fqs)(void); 256 void (*stats)(void); 257 int irq_capable; 258 int can_boost; 259 const char *name; 260 }; 261 262 static struct rcu_torture_ops *cur_ops; 263 264 /* 265 * Definitions for rcu torture testing. 266 */ 267 268 static int rcu_torture_read_lock(void) __acquires(RCU) 269 { 270 rcu_read_lock(); 271 return 0; 272 } 273 274 static void rcu_read_delay(struct torture_random_state *rrsp) 275 { 276 const unsigned long shortdelay_us = 200; 277 const unsigned long longdelay_ms = 50; 278 279 /* We want a short delay sometimes to make a reader delay the grace 280 * period, and we want a long delay occasionally to trigger 281 * force_quiescent_state. */ 282 283 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) 284 mdelay(longdelay_ms); 285 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 286 udelay(shortdelay_us); 287 #ifdef CONFIG_PREEMPT 288 if (!preempt_count() && 289 !(torture_random(rrsp) % (nrealreaders * 20000))) 290 preempt_schedule(); /* No QS if preempt_disable() in effect */ 291 #endif 292 } 293 294 static void rcu_torture_read_unlock(int idx) __releases(RCU) 295 { 296 rcu_read_unlock(); 297 } 298 299 static int rcu_torture_completed(void) 300 { 301 return rcu_batches_completed(); 302 } 303 304 /* 305 * Update callback in the pipe. This should be invoked after a grace period. 306 */ 307 static bool 308 rcu_torture_pipe_update_one(struct rcu_torture *rp) 309 { 310 int i; 311 312 i = rp->rtort_pipe_count; 313 if (i > RCU_TORTURE_PIPE_LEN) 314 i = RCU_TORTURE_PIPE_LEN; 315 atomic_inc(&rcu_torture_wcount[i]); 316 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 317 rp->rtort_mbtest = 0; 318 return true; 319 } 320 return false; 321 } 322 323 /* 324 * Update all callbacks in the pipe. Suitable for synchronous grace-period 325 * primitives. 326 */ 327 static void 328 rcu_torture_pipe_update(struct rcu_torture *old_rp) 329 { 330 struct rcu_torture *rp; 331 struct rcu_torture *rp1; 332 333 if (old_rp) 334 list_add(&old_rp->rtort_free, &rcu_torture_removed); 335 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 336 if (rcu_torture_pipe_update_one(rp)) { 337 list_del(&rp->rtort_free); 338 rcu_torture_free(rp); 339 } 340 } 341 } 342 343 static void 344 rcu_torture_cb(struct rcu_head *p) 345 { 346 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 347 348 if (torture_must_stop_irq()) { 349 /* Test is ending, just drop callbacks on the floor. */ 350 /* The next initialization will pick up the pieces. */ 351 return; 352 } 353 if (rcu_torture_pipe_update_one(rp)) 354 rcu_torture_free(rp); 355 else 356 cur_ops->deferred_free(rp); 357 } 358 359 static int rcu_no_completed(void) 360 { 361 return 0; 362 } 363 364 static void rcu_torture_deferred_free(struct rcu_torture *p) 365 { 366 call_rcu(&p->rtort_rcu, rcu_torture_cb); 367 } 368 369 static void rcu_sync_torture_init(void) 370 { 371 INIT_LIST_HEAD(&rcu_torture_removed); 372 } 373 374 static struct rcu_torture_ops rcu_ops = { 375 .ttype = RCU_FLAVOR, 376 .init = rcu_sync_torture_init, 377 .readlock = rcu_torture_read_lock, 378 .read_delay = rcu_read_delay, 379 .readunlock = rcu_torture_read_unlock, 380 .completed = rcu_torture_completed, 381 .deferred_free = rcu_torture_deferred_free, 382 .sync = synchronize_rcu, 383 .exp_sync = synchronize_rcu_expedited, 384 .get_state = get_state_synchronize_rcu, 385 .cond_sync = cond_synchronize_rcu, 386 .call = call_rcu, 387 .cb_barrier = rcu_barrier, 388 .fqs = rcu_force_quiescent_state, 389 .stats = NULL, 390 .irq_capable = 1, 391 .can_boost = rcu_can_boost(), 392 .name = "rcu" 393 }; 394 395 /* 396 * Definitions for rcu_bh torture testing. 397 */ 398 399 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 400 { 401 rcu_read_lock_bh(); 402 return 0; 403 } 404 405 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 406 { 407 rcu_read_unlock_bh(); 408 } 409 410 static int rcu_bh_torture_completed(void) 411 { 412 return rcu_batches_completed_bh(); 413 } 414 415 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 416 { 417 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 418 } 419 420 static struct rcu_torture_ops rcu_bh_ops = { 421 .ttype = RCU_BH_FLAVOR, 422 .init = rcu_sync_torture_init, 423 .readlock = rcu_bh_torture_read_lock, 424 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 425 .readunlock = rcu_bh_torture_read_unlock, 426 .completed = rcu_bh_torture_completed, 427 .deferred_free = rcu_bh_torture_deferred_free, 428 .sync = synchronize_rcu_bh, 429 .exp_sync = synchronize_rcu_bh_expedited, 430 .call = call_rcu_bh, 431 .cb_barrier = rcu_barrier_bh, 432 .fqs = rcu_bh_force_quiescent_state, 433 .stats = NULL, 434 .irq_capable = 1, 435 .name = "rcu_bh" 436 }; 437 438 /* 439 * Don't even think about trying any of these in real life!!! 440 * The names includes "busted", and they really means it! 441 * The only purpose of these functions is to provide a buggy RCU 442 * implementation to make sure that rcutorture correctly emits 443 * buggy-RCU error messages. 444 */ 445 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 446 { 447 /* This is a deliberate bug for testing purposes only! */ 448 rcu_torture_cb(&p->rtort_rcu); 449 } 450 451 static void synchronize_rcu_busted(void) 452 { 453 /* This is a deliberate bug for testing purposes only! */ 454 } 455 456 static void 457 call_rcu_busted(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) 458 { 459 /* This is a deliberate bug for testing purposes only! */ 460 func(head); 461 } 462 463 static struct rcu_torture_ops rcu_busted_ops = { 464 .ttype = INVALID_RCU_FLAVOR, 465 .init = rcu_sync_torture_init, 466 .readlock = rcu_torture_read_lock, 467 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 468 .readunlock = rcu_torture_read_unlock, 469 .completed = rcu_no_completed, 470 .deferred_free = rcu_busted_torture_deferred_free, 471 .sync = synchronize_rcu_busted, 472 .exp_sync = synchronize_rcu_busted, 473 .call = call_rcu_busted, 474 .cb_barrier = NULL, 475 .fqs = NULL, 476 .stats = NULL, 477 .irq_capable = 1, 478 .name = "rcu_busted" 479 }; 480 481 /* 482 * Definitions for srcu torture testing. 483 */ 484 485 DEFINE_STATIC_SRCU(srcu_ctl); 486 487 static int srcu_torture_read_lock(void) __acquires(&srcu_ctl) 488 { 489 return srcu_read_lock(&srcu_ctl); 490 } 491 492 static void srcu_read_delay(struct torture_random_state *rrsp) 493 { 494 long delay; 495 const long uspertick = 1000000 / HZ; 496 const long longdelay = 10; 497 498 /* We want there to be long-running readers, but not all the time. */ 499 500 delay = torture_random(rrsp) % 501 (nrealreaders * 2 * longdelay * uspertick); 502 if (!delay) 503 schedule_timeout_interruptible(longdelay); 504 else 505 rcu_read_delay(rrsp); 506 } 507 508 static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl) 509 { 510 srcu_read_unlock(&srcu_ctl, idx); 511 } 512 513 static int srcu_torture_completed(void) 514 { 515 return srcu_batches_completed(&srcu_ctl); 516 } 517 518 static void srcu_torture_deferred_free(struct rcu_torture *rp) 519 { 520 call_srcu(&srcu_ctl, &rp->rtort_rcu, rcu_torture_cb); 521 } 522 523 static void srcu_torture_synchronize(void) 524 { 525 synchronize_srcu(&srcu_ctl); 526 } 527 528 static void srcu_torture_call(struct rcu_head *head, 529 void (*func)(struct rcu_head *head)) 530 { 531 call_srcu(&srcu_ctl, head, func); 532 } 533 534 static void srcu_torture_barrier(void) 535 { 536 srcu_barrier(&srcu_ctl); 537 } 538 539 static void srcu_torture_stats(void) 540 { 541 int cpu; 542 int idx = srcu_ctl.completed & 0x1; 543 544 pr_alert("%s%s per-CPU(idx=%d):", 545 torture_type, TORTURE_FLAG, idx); 546 for_each_possible_cpu(cpu) { 547 long c0, c1; 548 549 c0 = (long)per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx]; 550 c1 = (long)per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]; 551 pr_cont(" %d(%ld,%ld)", cpu, c0, c1); 552 } 553 pr_cont("\n"); 554 } 555 556 static void srcu_torture_synchronize_expedited(void) 557 { 558 synchronize_srcu_expedited(&srcu_ctl); 559 } 560 561 static struct rcu_torture_ops srcu_ops = { 562 .ttype = SRCU_FLAVOR, 563 .init = rcu_sync_torture_init, 564 .readlock = srcu_torture_read_lock, 565 .read_delay = srcu_read_delay, 566 .readunlock = srcu_torture_read_unlock, 567 .completed = srcu_torture_completed, 568 .deferred_free = srcu_torture_deferred_free, 569 .sync = srcu_torture_synchronize, 570 .exp_sync = srcu_torture_synchronize_expedited, 571 .call = srcu_torture_call, 572 .cb_barrier = srcu_torture_barrier, 573 .stats = srcu_torture_stats, 574 .name = "srcu" 575 }; 576 577 /* 578 * Definitions for sched torture testing. 579 */ 580 581 static int sched_torture_read_lock(void) 582 { 583 preempt_disable(); 584 return 0; 585 } 586 587 static void sched_torture_read_unlock(int idx) 588 { 589 preempt_enable(); 590 } 591 592 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 593 { 594 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 595 } 596 597 static struct rcu_torture_ops sched_ops = { 598 .ttype = RCU_SCHED_FLAVOR, 599 .init = rcu_sync_torture_init, 600 .readlock = sched_torture_read_lock, 601 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 602 .readunlock = sched_torture_read_unlock, 603 .completed = rcu_no_completed, 604 .deferred_free = rcu_sched_torture_deferred_free, 605 .sync = synchronize_sched, 606 .exp_sync = synchronize_sched_expedited, 607 .call = call_rcu_sched, 608 .cb_barrier = rcu_barrier_sched, 609 .fqs = rcu_sched_force_quiescent_state, 610 .stats = NULL, 611 .irq_capable = 1, 612 .name = "sched" 613 }; 614 615 #ifdef CONFIG_TASKS_RCU 616 617 /* 618 * Definitions for RCU-tasks torture testing. 619 */ 620 621 static int tasks_torture_read_lock(void) 622 { 623 return 0; 624 } 625 626 static void tasks_torture_read_unlock(int idx) 627 { 628 } 629 630 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 631 { 632 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 633 } 634 635 static struct rcu_torture_ops tasks_ops = { 636 .ttype = RCU_TASKS_FLAVOR, 637 .init = rcu_sync_torture_init, 638 .readlock = tasks_torture_read_lock, 639 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 640 .readunlock = tasks_torture_read_unlock, 641 .completed = rcu_no_completed, 642 .deferred_free = rcu_tasks_torture_deferred_free, 643 .sync = synchronize_rcu_tasks, 644 .exp_sync = synchronize_rcu_tasks, 645 .call = call_rcu_tasks, 646 .cb_barrier = rcu_barrier_tasks, 647 .fqs = NULL, 648 .stats = NULL, 649 .irq_capable = 1, 650 .name = "tasks" 651 }; 652 653 #define RCUTORTURE_TASKS_OPS &tasks_ops, 654 655 #else /* #ifdef CONFIG_TASKS_RCU */ 656 657 #define RCUTORTURE_TASKS_OPS 658 659 #endif /* #else #ifdef CONFIG_TASKS_RCU */ 660 661 /* 662 * RCU torture priority-boost testing. Runs one real-time thread per 663 * CPU for moderate bursts, repeatedly registering RCU callbacks and 664 * spinning waiting for them to be invoked. If a given callback takes 665 * too long to be invoked, we assume that priority inversion has occurred. 666 */ 667 668 struct rcu_boost_inflight { 669 struct rcu_head rcu; 670 int inflight; 671 }; 672 673 static void rcu_torture_boost_cb(struct rcu_head *head) 674 { 675 struct rcu_boost_inflight *rbip = 676 container_of(head, struct rcu_boost_inflight, rcu); 677 678 smp_mb(); /* Ensure RCU-core accesses precede clearing ->inflight */ 679 rbip->inflight = 0; 680 } 681 682 static int rcu_torture_boost(void *arg) 683 { 684 unsigned long call_rcu_time; 685 unsigned long endtime; 686 unsigned long oldstarttime; 687 struct rcu_boost_inflight rbi = { .inflight = 0 }; 688 struct sched_param sp; 689 690 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 691 692 /* Set real-time priority. */ 693 sp.sched_priority = 1; 694 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 695 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 696 n_rcu_torture_boost_rterror++; 697 } 698 699 init_rcu_head_on_stack(&rbi.rcu); 700 /* Each pass through the following loop does one boost-test cycle. */ 701 do { 702 /* Wait for the next test interval. */ 703 oldstarttime = boost_starttime; 704 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 705 schedule_timeout_interruptible(oldstarttime - jiffies); 706 stutter_wait("rcu_torture_boost"); 707 if (torture_must_stop()) 708 goto checkwait; 709 } 710 711 /* Do one boost-test interval. */ 712 endtime = oldstarttime + test_boost_duration * HZ; 713 call_rcu_time = jiffies; 714 while (ULONG_CMP_LT(jiffies, endtime)) { 715 /* If we don't have a callback in flight, post one. */ 716 if (!rbi.inflight) { 717 smp_mb(); /* RCU core before ->inflight = 1. */ 718 rbi.inflight = 1; 719 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 720 if (jiffies - call_rcu_time > 721 test_boost_duration * HZ - HZ / 2) { 722 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 723 n_rcu_torture_boost_failure++; 724 } 725 call_rcu_time = jiffies; 726 } 727 cond_resched_rcu_qs(); 728 stutter_wait("rcu_torture_boost"); 729 if (torture_must_stop()) 730 goto checkwait; 731 } 732 733 /* 734 * Set the start time of the next test interval. 735 * Yes, this is vulnerable to long delays, but such 736 * delays simply cause a false negative for the next 737 * interval. Besides, we are running at RT priority, 738 * so delays should be relatively rare. 739 */ 740 while (oldstarttime == boost_starttime && 741 !kthread_should_stop()) { 742 if (mutex_trylock(&boost_mutex)) { 743 boost_starttime = jiffies + 744 test_boost_interval * HZ; 745 n_rcu_torture_boosts++; 746 mutex_unlock(&boost_mutex); 747 break; 748 } 749 schedule_timeout_uninterruptible(1); 750 } 751 752 /* Go do the stutter. */ 753 checkwait: stutter_wait("rcu_torture_boost"); 754 } while (!torture_must_stop()); 755 756 /* Clean up and exit. */ 757 while (!kthread_should_stop() || rbi.inflight) { 758 torture_shutdown_absorb("rcu_torture_boost"); 759 schedule_timeout_uninterruptible(1); 760 } 761 smp_mb(); /* order accesses to ->inflight before stack-frame death. */ 762 destroy_rcu_head_on_stack(&rbi.rcu); 763 torture_kthread_stopping("rcu_torture_boost"); 764 return 0; 765 } 766 767 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 768 { 769 } 770 771 /* 772 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 773 * to call_rcu() or analogous, increasing the probability of occurrence 774 * of callback-overflow corner cases. 775 */ 776 static int 777 rcu_torture_cbflood(void *arg) 778 { 779 int err = 1; 780 int i; 781 int j; 782 struct rcu_head *rhp; 783 784 if (cbflood_n_per_burst > 0 && 785 cbflood_inter_holdoff > 0 && 786 cbflood_intra_holdoff > 0 && 787 cur_ops->call && 788 cur_ops->cb_barrier) { 789 rhp = vmalloc(sizeof(*rhp) * 790 cbflood_n_burst * cbflood_n_per_burst); 791 err = !rhp; 792 } 793 if (err) { 794 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 795 while (!torture_must_stop()) 796 schedule_timeout_interruptible(HZ); 797 return 0; 798 } 799 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 800 do { 801 schedule_timeout_interruptible(cbflood_inter_holdoff); 802 atomic_long_inc(&n_cbfloods); 803 WARN_ON(signal_pending(current)); 804 for (i = 0; i < cbflood_n_burst; i++) { 805 for (j = 0; j < cbflood_n_per_burst; j++) { 806 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 807 rcu_torture_cbflood_cb); 808 } 809 schedule_timeout_interruptible(cbflood_intra_holdoff); 810 WARN_ON(signal_pending(current)); 811 } 812 cur_ops->cb_barrier(); 813 stutter_wait("rcu_torture_cbflood"); 814 } while (!torture_must_stop()); 815 vfree(rhp); 816 torture_kthread_stopping("rcu_torture_cbflood"); 817 return 0; 818 } 819 820 /* 821 * RCU torture force-quiescent-state kthread. Repeatedly induces 822 * bursts of calls to force_quiescent_state(), increasing the probability 823 * of occurrence of some important types of race conditions. 824 */ 825 static int 826 rcu_torture_fqs(void *arg) 827 { 828 unsigned long fqs_resume_time; 829 int fqs_burst_remaining; 830 831 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 832 do { 833 fqs_resume_time = jiffies + fqs_stutter * HZ; 834 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 835 !kthread_should_stop()) { 836 schedule_timeout_interruptible(1); 837 } 838 fqs_burst_remaining = fqs_duration; 839 while (fqs_burst_remaining > 0 && 840 !kthread_should_stop()) { 841 cur_ops->fqs(); 842 udelay(fqs_holdoff); 843 fqs_burst_remaining -= fqs_holdoff; 844 } 845 stutter_wait("rcu_torture_fqs"); 846 } while (!torture_must_stop()); 847 torture_kthread_stopping("rcu_torture_fqs"); 848 return 0; 849 } 850 851 /* 852 * RCU torture writer kthread. Repeatedly substitutes a new structure 853 * for that pointed to by rcu_torture_current, freeing the old structure 854 * after a series of grace periods (the "pipeline"). 855 */ 856 static int 857 rcu_torture_writer(void *arg) 858 { 859 unsigned long gp_snap; 860 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 861 bool gp_sync1 = gp_sync; 862 int i; 863 struct rcu_torture *rp; 864 struct rcu_torture *old_rp; 865 static DEFINE_TORTURE_RANDOM(rand); 866 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 867 RTWS_COND_GET, RTWS_SYNC }; 868 int nsynctypes = 0; 869 870 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 871 872 /* Initialize synctype[] array. If none set, take default. */ 873 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync) 874 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 875 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) 876 synctype[nsynctypes++] = RTWS_COND_GET; 877 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) 878 pr_alert("rcu_torture_writer: gp_cond without primitives.\n"); 879 if (gp_exp1 && cur_ops->exp_sync) 880 synctype[nsynctypes++] = RTWS_EXP_SYNC; 881 else if (gp_exp && !cur_ops->exp_sync) 882 pr_alert("rcu_torture_writer: gp_exp without primitives.\n"); 883 if (gp_normal1 && cur_ops->deferred_free) 884 synctype[nsynctypes++] = RTWS_DEF_FREE; 885 else if (gp_normal && !cur_ops->deferred_free) 886 pr_alert("rcu_torture_writer: gp_normal without primitives.\n"); 887 if (gp_sync1 && cur_ops->sync) 888 synctype[nsynctypes++] = RTWS_SYNC; 889 else if (gp_sync && !cur_ops->sync) 890 pr_alert("rcu_torture_writer: gp_sync without primitives.\n"); 891 if (WARN_ONCE(nsynctypes == 0, 892 "rcu_torture_writer: No update-side primitives.\n")) { 893 /* 894 * No updates primitives, so don't try updating. 895 * The resulting test won't be testing much, hence the 896 * above WARN_ONCE(). 897 */ 898 rcu_torture_writer_state = RTWS_STOPPING; 899 torture_kthread_stopping("rcu_torture_writer"); 900 } 901 902 do { 903 rcu_torture_writer_state = RTWS_FIXED_DELAY; 904 schedule_timeout_uninterruptible(1); 905 rp = rcu_torture_alloc(); 906 if (rp == NULL) 907 continue; 908 rp->rtort_pipe_count = 0; 909 rcu_torture_writer_state = RTWS_DELAY; 910 udelay(torture_random(&rand) & 0x3ff); 911 rcu_torture_writer_state = RTWS_REPLACE; 912 old_rp = rcu_dereference_check(rcu_torture_current, 913 current == writer_task); 914 rp->rtort_mbtest = 1; 915 rcu_assign_pointer(rcu_torture_current, rp); 916 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 917 if (old_rp) { 918 i = old_rp->rtort_pipe_count; 919 if (i > RCU_TORTURE_PIPE_LEN) 920 i = RCU_TORTURE_PIPE_LEN; 921 atomic_inc(&rcu_torture_wcount[i]); 922 old_rp->rtort_pipe_count++; 923 switch (synctype[torture_random(&rand) % nsynctypes]) { 924 case RTWS_DEF_FREE: 925 rcu_torture_writer_state = RTWS_DEF_FREE; 926 cur_ops->deferred_free(old_rp); 927 break; 928 case RTWS_EXP_SYNC: 929 rcu_torture_writer_state = RTWS_EXP_SYNC; 930 cur_ops->exp_sync(); 931 rcu_torture_pipe_update(old_rp); 932 break; 933 case RTWS_COND_GET: 934 rcu_torture_writer_state = RTWS_COND_GET; 935 gp_snap = cur_ops->get_state(); 936 i = torture_random(&rand) % 16; 937 if (i != 0) 938 schedule_timeout_interruptible(i); 939 udelay(torture_random(&rand) % 1000); 940 rcu_torture_writer_state = RTWS_COND_SYNC; 941 cur_ops->cond_sync(gp_snap); 942 rcu_torture_pipe_update(old_rp); 943 break; 944 case RTWS_SYNC: 945 rcu_torture_writer_state = RTWS_SYNC; 946 cur_ops->sync(); 947 rcu_torture_pipe_update(old_rp); 948 break; 949 default: 950 WARN_ON_ONCE(1); 951 break; 952 } 953 } 954 rcutorture_record_progress(++rcu_torture_current_version); 955 rcu_torture_writer_state = RTWS_STUTTER; 956 stutter_wait("rcu_torture_writer"); 957 } while (!torture_must_stop()); 958 rcu_torture_writer_state = RTWS_STOPPING; 959 torture_kthread_stopping("rcu_torture_writer"); 960 return 0; 961 } 962 963 /* 964 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 965 * delay between calls. 966 */ 967 static int 968 rcu_torture_fakewriter(void *arg) 969 { 970 DEFINE_TORTURE_RANDOM(rand); 971 972 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 973 set_user_nice(current, MAX_NICE); 974 975 do { 976 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 977 udelay(torture_random(&rand) & 0x3ff); 978 if (cur_ops->cb_barrier != NULL && 979 torture_random(&rand) % (nfakewriters * 8) == 0) { 980 cur_ops->cb_barrier(); 981 } else if (gp_normal == gp_exp) { 982 if (torture_random(&rand) & 0x80) 983 cur_ops->sync(); 984 else 985 cur_ops->exp_sync(); 986 } else if (gp_normal) { 987 cur_ops->sync(); 988 } else { 989 cur_ops->exp_sync(); 990 } 991 stutter_wait("rcu_torture_fakewriter"); 992 } while (!torture_must_stop()); 993 994 torture_kthread_stopping("rcu_torture_fakewriter"); 995 return 0; 996 } 997 998 static void rcutorture_trace_dump(void) 999 { 1000 static atomic_t beenhere = ATOMIC_INIT(0); 1001 1002 if (atomic_read(&beenhere)) 1003 return; 1004 if (atomic_xchg(&beenhere, 1) != 0) 1005 return; 1006 ftrace_dump(DUMP_ALL); 1007 } 1008 1009 /* 1010 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1011 * incrementing the corresponding element of the pipeline array. The 1012 * counter in the element should never be greater than 1, otherwise, the 1013 * RCU implementation is broken. 1014 */ 1015 static void rcu_torture_timer(unsigned long unused) 1016 { 1017 int idx; 1018 int completed; 1019 int completed_end; 1020 static DEFINE_TORTURE_RANDOM(rand); 1021 static DEFINE_SPINLOCK(rand_lock); 1022 struct rcu_torture *p; 1023 int pipe_count; 1024 unsigned long long ts; 1025 1026 idx = cur_ops->readlock(); 1027 completed = cur_ops->completed(); 1028 ts = rcu_trace_clock_local(); 1029 p = rcu_dereference_check(rcu_torture_current, 1030 rcu_read_lock_bh_held() || 1031 rcu_read_lock_sched_held() || 1032 srcu_read_lock_held(&srcu_ctl)); 1033 if (p == NULL) { 1034 /* Leave because rcu_torture_writer is not yet underway */ 1035 cur_ops->readunlock(idx); 1036 return; 1037 } 1038 if (p->rtort_mbtest == 0) 1039 atomic_inc(&n_rcu_torture_mberror); 1040 spin_lock(&rand_lock); 1041 cur_ops->read_delay(&rand); 1042 n_rcu_torture_timers++; 1043 spin_unlock(&rand_lock); 1044 preempt_disable(); 1045 pipe_count = p->rtort_pipe_count; 1046 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1047 /* Should not happen, but... */ 1048 pipe_count = RCU_TORTURE_PIPE_LEN; 1049 } 1050 completed_end = cur_ops->completed(); 1051 if (pipe_count > 1) { 1052 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1053 completed, completed_end); 1054 rcutorture_trace_dump(); 1055 } 1056 __this_cpu_inc(rcu_torture_count[pipe_count]); 1057 completed = completed_end - completed; 1058 if (completed > RCU_TORTURE_PIPE_LEN) { 1059 /* Should not happen, but... */ 1060 completed = RCU_TORTURE_PIPE_LEN; 1061 } 1062 __this_cpu_inc(rcu_torture_batch[completed]); 1063 preempt_enable(); 1064 cur_ops->readunlock(idx); 1065 } 1066 1067 /* 1068 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1069 * incrementing the corresponding element of the pipeline array. The 1070 * counter in the element should never be greater than 1, otherwise, the 1071 * RCU implementation is broken. 1072 */ 1073 static int 1074 rcu_torture_reader(void *arg) 1075 { 1076 int completed; 1077 int completed_end; 1078 int idx; 1079 DEFINE_TORTURE_RANDOM(rand); 1080 struct rcu_torture *p; 1081 int pipe_count; 1082 struct timer_list t; 1083 unsigned long long ts; 1084 1085 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1086 set_user_nice(current, MAX_NICE); 1087 if (irqreader && cur_ops->irq_capable) 1088 setup_timer_on_stack(&t, rcu_torture_timer, 0); 1089 1090 do { 1091 if (irqreader && cur_ops->irq_capable) { 1092 if (!timer_pending(&t)) 1093 mod_timer(&t, jiffies + 1); 1094 } 1095 idx = cur_ops->readlock(); 1096 completed = cur_ops->completed(); 1097 ts = rcu_trace_clock_local(); 1098 p = rcu_dereference_check(rcu_torture_current, 1099 rcu_read_lock_bh_held() || 1100 rcu_read_lock_sched_held() || 1101 srcu_read_lock_held(&srcu_ctl)); 1102 if (p == NULL) { 1103 /* Wait for rcu_torture_writer to get underway */ 1104 cur_ops->readunlock(idx); 1105 schedule_timeout_interruptible(HZ); 1106 continue; 1107 } 1108 if (p->rtort_mbtest == 0) 1109 atomic_inc(&n_rcu_torture_mberror); 1110 cur_ops->read_delay(&rand); 1111 preempt_disable(); 1112 pipe_count = p->rtort_pipe_count; 1113 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1114 /* Should not happen, but... */ 1115 pipe_count = RCU_TORTURE_PIPE_LEN; 1116 } 1117 completed_end = cur_ops->completed(); 1118 if (pipe_count > 1) { 1119 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1120 ts, completed, completed_end); 1121 rcutorture_trace_dump(); 1122 } 1123 __this_cpu_inc(rcu_torture_count[pipe_count]); 1124 completed = completed_end - completed; 1125 if (completed > RCU_TORTURE_PIPE_LEN) { 1126 /* Should not happen, but... */ 1127 completed = RCU_TORTURE_PIPE_LEN; 1128 } 1129 __this_cpu_inc(rcu_torture_batch[completed]); 1130 preempt_enable(); 1131 cur_ops->readunlock(idx); 1132 cond_resched_rcu_qs(); 1133 stutter_wait("rcu_torture_reader"); 1134 } while (!torture_must_stop()); 1135 if (irqreader && cur_ops->irq_capable) { 1136 del_timer_sync(&t); 1137 destroy_timer_on_stack(&t); 1138 } 1139 torture_kthread_stopping("rcu_torture_reader"); 1140 return 0; 1141 } 1142 1143 /* 1144 * Print torture statistics. Caller must ensure that there is only 1145 * one call to this function at a given time!!! This is normally 1146 * accomplished by relying on the module system to only have one copy 1147 * of the module loaded, and then by giving the rcu_torture_stats 1148 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1149 * thread is not running). 1150 */ 1151 static void 1152 rcu_torture_stats_print(void) 1153 { 1154 int cpu; 1155 int i; 1156 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1157 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1158 static unsigned long rtcv_snap = ULONG_MAX; 1159 1160 for_each_possible_cpu(cpu) { 1161 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1162 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1163 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1164 } 1165 } 1166 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1167 if (pipesummary[i] != 0) 1168 break; 1169 } 1170 1171 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1172 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1173 rcu_torture_current, 1174 rcu_torture_current_version, 1175 list_empty(&rcu_torture_freelist), 1176 atomic_read(&n_rcu_torture_alloc), 1177 atomic_read(&n_rcu_torture_alloc_fail), 1178 atomic_read(&n_rcu_torture_free)); 1179 pr_cont("rtmbe: %d rtbke: %ld rtbre: %ld ", 1180 atomic_read(&n_rcu_torture_mberror), 1181 n_rcu_torture_boost_ktrerror, 1182 n_rcu_torture_boost_rterror); 1183 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1184 n_rcu_torture_boost_failure, 1185 n_rcu_torture_boosts, 1186 n_rcu_torture_timers); 1187 torture_onoff_stats(); 1188 pr_cont("barrier: %ld/%ld:%ld ", 1189 n_barrier_successes, 1190 n_barrier_attempts, 1191 n_rcu_torture_barrier_error); 1192 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1193 1194 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1195 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1196 n_rcu_torture_barrier_error != 0 || 1197 n_rcu_torture_boost_ktrerror != 0 || 1198 n_rcu_torture_boost_rterror != 0 || 1199 n_rcu_torture_boost_failure != 0 || 1200 i > 1) { 1201 pr_cont("%s", "!!! "); 1202 atomic_inc(&n_rcu_torture_error); 1203 WARN_ON_ONCE(1); 1204 } 1205 pr_cont("Reader Pipe: "); 1206 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1207 pr_cont(" %ld", pipesummary[i]); 1208 pr_cont("\n"); 1209 1210 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1211 pr_cont("Reader Batch: "); 1212 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1213 pr_cont(" %ld", batchsummary[i]); 1214 pr_cont("\n"); 1215 1216 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1217 pr_cont("Free-Block Circulation: "); 1218 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1219 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1220 } 1221 pr_cont("\n"); 1222 1223 if (cur_ops->stats) 1224 cur_ops->stats(); 1225 if (rtcv_snap == rcu_torture_current_version && 1226 rcu_torture_current != NULL) { 1227 int __maybe_unused flags; 1228 unsigned long __maybe_unused gpnum; 1229 unsigned long __maybe_unused completed; 1230 1231 rcutorture_get_gp_data(cur_ops->ttype, 1232 &flags, &gpnum, &completed); 1233 pr_alert("??? Writer stall state %d g%lu c%lu f%#x\n", 1234 rcu_torture_writer_state, 1235 gpnum, completed, flags); 1236 show_rcu_gp_kthreads(); 1237 rcutorture_trace_dump(); 1238 } 1239 rtcv_snap = rcu_torture_current_version; 1240 } 1241 1242 /* 1243 * Periodically prints torture statistics, if periodic statistics printing 1244 * was specified via the stat_interval module parameter. 1245 */ 1246 static int 1247 rcu_torture_stats(void *arg) 1248 { 1249 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1250 do { 1251 schedule_timeout_interruptible(stat_interval * HZ); 1252 rcu_torture_stats_print(); 1253 torture_shutdown_absorb("rcu_torture_stats"); 1254 } while (!torture_must_stop()); 1255 torture_kthread_stopping("rcu_torture_stats"); 1256 return 0; 1257 } 1258 1259 static inline void 1260 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1261 { 1262 pr_alert("%s" TORTURE_FLAG 1263 "--- %s: nreaders=%d nfakewriters=%d " 1264 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1265 "shuffle_interval=%d stutter=%d irqreader=%d " 1266 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1267 "test_boost=%d/%d test_boost_interval=%d " 1268 "test_boost_duration=%d shutdown_secs=%d " 1269 "stall_cpu=%d stall_cpu_holdoff=%d " 1270 "n_barrier_cbs=%d " 1271 "onoff_interval=%d onoff_holdoff=%d\n", 1272 torture_type, tag, nrealreaders, nfakewriters, 1273 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1274 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1275 test_boost, cur_ops->can_boost, 1276 test_boost_interval, test_boost_duration, shutdown_secs, 1277 stall_cpu, stall_cpu_holdoff, 1278 n_barrier_cbs, 1279 onoff_interval, onoff_holdoff); 1280 } 1281 1282 static void rcutorture_booster_cleanup(int cpu) 1283 { 1284 struct task_struct *t; 1285 1286 if (boost_tasks[cpu] == NULL) 1287 return; 1288 mutex_lock(&boost_mutex); 1289 t = boost_tasks[cpu]; 1290 boost_tasks[cpu] = NULL; 1291 mutex_unlock(&boost_mutex); 1292 1293 /* This must be outside of the mutex, otherwise deadlock! */ 1294 torture_stop_kthread(rcu_torture_boost, t); 1295 } 1296 1297 static int rcutorture_booster_init(int cpu) 1298 { 1299 int retval; 1300 1301 if (boost_tasks[cpu] != NULL) 1302 return 0; /* Already created, nothing more to do. */ 1303 1304 /* Don't allow time recalculation while creating a new task. */ 1305 mutex_lock(&boost_mutex); 1306 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1307 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1308 cpu_to_node(cpu), 1309 "rcu_torture_boost"); 1310 if (IS_ERR(boost_tasks[cpu])) { 1311 retval = PTR_ERR(boost_tasks[cpu]); 1312 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1313 n_rcu_torture_boost_ktrerror++; 1314 boost_tasks[cpu] = NULL; 1315 mutex_unlock(&boost_mutex); 1316 return retval; 1317 } 1318 kthread_bind(boost_tasks[cpu], cpu); 1319 wake_up_process(boost_tasks[cpu]); 1320 mutex_unlock(&boost_mutex); 1321 return 0; 1322 } 1323 1324 /* 1325 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1326 * induces a CPU stall for the time specified by stall_cpu. 1327 */ 1328 static int rcu_torture_stall(void *args) 1329 { 1330 unsigned long stop_at; 1331 1332 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1333 if (stall_cpu_holdoff > 0) { 1334 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1335 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1336 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1337 } 1338 if (!kthread_should_stop()) { 1339 stop_at = get_seconds() + stall_cpu; 1340 /* RCU CPU stall is expected behavior in following code. */ 1341 pr_alert("rcu_torture_stall start.\n"); 1342 rcu_read_lock(); 1343 preempt_disable(); 1344 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1345 continue; /* Induce RCU CPU stall warning. */ 1346 preempt_enable(); 1347 rcu_read_unlock(); 1348 pr_alert("rcu_torture_stall end.\n"); 1349 } 1350 torture_shutdown_absorb("rcu_torture_stall"); 1351 while (!kthread_should_stop()) 1352 schedule_timeout_interruptible(10 * HZ); 1353 return 0; 1354 } 1355 1356 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1357 static int __init rcu_torture_stall_init(void) 1358 { 1359 if (stall_cpu <= 0) 1360 return 0; 1361 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1362 } 1363 1364 /* Callback function for RCU barrier testing. */ 1365 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1366 { 1367 atomic_inc(&barrier_cbs_invoked); 1368 } 1369 1370 /* kthread function to register callbacks used to test RCU barriers. */ 1371 static int rcu_torture_barrier_cbs(void *arg) 1372 { 1373 long myid = (long)arg; 1374 bool lastphase = 0; 1375 bool newphase; 1376 struct rcu_head rcu; 1377 1378 init_rcu_head_on_stack(&rcu); 1379 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1380 set_user_nice(current, MAX_NICE); 1381 do { 1382 wait_event(barrier_cbs_wq[myid], 1383 (newphase = 1384 ACCESS_ONCE(barrier_phase)) != lastphase || 1385 torture_must_stop()); 1386 lastphase = newphase; 1387 smp_mb(); /* ensure barrier_phase load before ->call(). */ 1388 if (torture_must_stop()) 1389 break; 1390 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1391 if (atomic_dec_and_test(&barrier_cbs_count)) 1392 wake_up(&barrier_wq); 1393 } while (!torture_must_stop()); 1394 if (cur_ops->cb_barrier != NULL) 1395 cur_ops->cb_barrier(); 1396 destroy_rcu_head_on_stack(&rcu); 1397 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1398 return 0; 1399 } 1400 1401 /* kthread function to drive and coordinate RCU barrier testing. */ 1402 static int rcu_torture_barrier(void *arg) 1403 { 1404 int i; 1405 1406 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1407 do { 1408 atomic_set(&barrier_cbs_invoked, 0); 1409 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1410 smp_mb(); /* Ensure barrier_phase after prior assignments. */ 1411 barrier_phase = !barrier_phase; 1412 for (i = 0; i < n_barrier_cbs; i++) 1413 wake_up(&barrier_cbs_wq[i]); 1414 wait_event(barrier_wq, 1415 atomic_read(&barrier_cbs_count) == 0 || 1416 torture_must_stop()); 1417 if (torture_must_stop()) 1418 break; 1419 n_barrier_attempts++; 1420 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1421 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1422 n_rcu_torture_barrier_error++; 1423 WARN_ON_ONCE(1); 1424 } 1425 n_barrier_successes++; 1426 schedule_timeout_interruptible(HZ / 10); 1427 } while (!torture_must_stop()); 1428 torture_kthread_stopping("rcu_torture_barrier"); 1429 return 0; 1430 } 1431 1432 /* Initialize RCU barrier testing. */ 1433 static int rcu_torture_barrier_init(void) 1434 { 1435 int i; 1436 int ret; 1437 1438 if (n_barrier_cbs == 0) 1439 return 0; 1440 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1441 pr_alert("%s" TORTURE_FLAG 1442 " Call or barrier ops missing for %s,\n", 1443 torture_type, cur_ops->name); 1444 pr_alert("%s" TORTURE_FLAG 1445 " RCU barrier testing omitted from run.\n", 1446 torture_type); 1447 return 0; 1448 } 1449 atomic_set(&barrier_cbs_count, 0); 1450 atomic_set(&barrier_cbs_invoked, 0); 1451 barrier_cbs_tasks = 1452 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1453 GFP_KERNEL); 1454 barrier_cbs_wq = 1455 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1456 GFP_KERNEL); 1457 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1458 return -ENOMEM; 1459 for (i = 0; i < n_barrier_cbs; i++) { 1460 init_waitqueue_head(&barrier_cbs_wq[i]); 1461 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1462 (void *)(long)i, 1463 barrier_cbs_tasks[i]); 1464 if (ret) 1465 return ret; 1466 } 1467 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1468 } 1469 1470 /* Clean up after RCU barrier testing. */ 1471 static void rcu_torture_barrier_cleanup(void) 1472 { 1473 int i; 1474 1475 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1476 if (barrier_cbs_tasks != NULL) { 1477 for (i = 0; i < n_barrier_cbs; i++) 1478 torture_stop_kthread(rcu_torture_barrier_cbs, 1479 barrier_cbs_tasks[i]); 1480 kfree(barrier_cbs_tasks); 1481 barrier_cbs_tasks = NULL; 1482 } 1483 if (barrier_cbs_wq != NULL) { 1484 kfree(barrier_cbs_wq); 1485 barrier_cbs_wq = NULL; 1486 } 1487 } 1488 1489 static int rcutorture_cpu_notify(struct notifier_block *self, 1490 unsigned long action, void *hcpu) 1491 { 1492 long cpu = (long)hcpu; 1493 1494 switch (action) { 1495 case CPU_ONLINE: 1496 case CPU_DOWN_FAILED: 1497 (void)rcutorture_booster_init(cpu); 1498 break; 1499 case CPU_DOWN_PREPARE: 1500 rcutorture_booster_cleanup(cpu); 1501 break; 1502 default: 1503 break; 1504 } 1505 return NOTIFY_OK; 1506 } 1507 1508 static struct notifier_block rcutorture_cpu_nb = { 1509 .notifier_call = rcutorture_cpu_notify, 1510 }; 1511 1512 static void 1513 rcu_torture_cleanup(void) 1514 { 1515 int i; 1516 1517 rcutorture_record_test_transition(); 1518 if (torture_cleanup_begin()) { 1519 if (cur_ops->cb_barrier != NULL) 1520 cur_ops->cb_barrier(); 1521 return; 1522 } 1523 1524 rcu_torture_barrier_cleanup(); 1525 torture_stop_kthread(rcu_torture_stall, stall_task); 1526 torture_stop_kthread(rcu_torture_writer, writer_task); 1527 1528 if (reader_tasks) { 1529 for (i = 0; i < nrealreaders; i++) 1530 torture_stop_kthread(rcu_torture_reader, 1531 reader_tasks[i]); 1532 kfree(reader_tasks); 1533 } 1534 rcu_torture_current = NULL; 1535 1536 if (fakewriter_tasks) { 1537 for (i = 0; i < nfakewriters; i++) { 1538 torture_stop_kthread(rcu_torture_fakewriter, 1539 fakewriter_tasks[i]); 1540 } 1541 kfree(fakewriter_tasks); 1542 fakewriter_tasks = NULL; 1543 } 1544 1545 torture_stop_kthread(rcu_torture_stats, stats_task); 1546 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1547 for (i = 0; i < ncbflooders; i++) 1548 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1549 if ((test_boost == 1 && cur_ops->can_boost) || 1550 test_boost == 2) { 1551 unregister_cpu_notifier(&rcutorture_cpu_nb); 1552 for_each_possible_cpu(i) 1553 rcutorture_booster_cleanup(i); 1554 } 1555 1556 /* Wait for all RCU callbacks to fire. */ 1557 1558 if (cur_ops->cb_barrier != NULL) 1559 cur_ops->cb_barrier(); 1560 1561 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1562 1563 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1564 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1565 else if (torture_onoff_failures()) 1566 rcu_torture_print_module_parms(cur_ops, 1567 "End of test: RCU_HOTPLUG"); 1568 else 1569 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1570 torture_cleanup_end(); 1571 } 1572 1573 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1574 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1575 { 1576 } 1577 1578 static void rcu_torture_err_cb(struct rcu_head *rhp) 1579 { 1580 /* 1581 * This -might- happen due to race conditions, but is unlikely. 1582 * The scenario that leads to this happening is that the 1583 * first of the pair of duplicate callbacks is queued, 1584 * someone else starts a grace period that includes that 1585 * callback, then the second of the pair must wait for the 1586 * next grace period. Unlikely, but can happen. If it 1587 * does happen, the debug-objects subsystem won't have splatted. 1588 */ 1589 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1590 } 1591 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1592 1593 /* 1594 * Verify that double-free causes debug-objects to complain, but only 1595 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1596 * cannot be carried out. 1597 */ 1598 static void rcu_test_debug_objects(void) 1599 { 1600 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1601 struct rcu_head rh1; 1602 struct rcu_head rh2; 1603 1604 init_rcu_head_on_stack(&rh1); 1605 init_rcu_head_on_stack(&rh2); 1606 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1607 1608 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1609 preempt_disable(); /* Prevent preemption from interrupting test. */ 1610 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1611 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1612 local_irq_disable(); /* Make it harder to start a new grace period. */ 1613 call_rcu(&rh2, rcu_torture_leak_cb); 1614 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1615 local_irq_enable(); 1616 rcu_read_unlock(); 1617 preempt_enable(); 1618 1619 /* Wait for them all to get done so we can safely return. */ 1620 rcu_barrier(); 1621 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1622 destroy_rcu_head_on_stack(&rh1); 1623 destroy_rcu_head_on_stack(&rh2); 1624 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1625 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1626 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1627 } 1628 1629 static int __init 1630 rcu_torture_init(void) 1631 { 1632 int i; 1633 int cpu; 1634 int firsterr = 0; 1635 static struct rcu_torture_ops *torture_ops[] = { 1636 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &sched_ops, 1637 RCUTORTURE_TASKS_OPS 1638 }; 1639 1640 if (!torture_init_begin(torture_type, verbose, &torture_runnable)) 1641 return -EBUSY; 1642 1643 /* Process args and tell the world that the torturer is on the job. */ 1644 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1645 cur_ops = torture_ops[i]; 1646 if (strcmp(torture_type, cur_ops->name) == 0) 1647 break; 1648 } 1649 if (i == ARRAY_SIZE(torture_ops)) { 1650 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1651 torture_type); 1652 pr_alert("rcu-torture types:"); 1653 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1654 pr_alert(" %s", torture_ops[i]->name); 1655 pr_alert("\n"); 1656 torture_init_end(); 1657 return -EINVAL; 1658 } 1659 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1660 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1661 fqs_duration = 0; 1662 } 1663 if (cur_ops->init) 1664 cur_ops->init(); /* no "goto unwind" prior to this point!!! */ 1665 1666 if (nreaders >= 0) { 1667 nrealreaders = nreaders; 1668 } else { 1669 nrealreaders = num_online_cpus() - 1; 1670 if (nrealreaders <= 0) 1671 nrealreaders = 1; 1672 } 1673 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1674 1675 /* Set up the freelist. */ 1676 1677 INIT_LIST_HEAD(&rcu_torture_freelist); 1678 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1679 rcu_tortures[i].rtort_mbtest = 0; 1680 list_add_tail(&rcu_tortures[i].rtort_free, 1681 &rcu_torture_freelist); 1682 } 1683 1684 /* Initialize the statistics so that each run gets its own numbers. */ 1685 1686 rcu_torture_current = NULL; 1687 rcu_torture_current_version = 0; 1688 atomic_set(&n_rcu_torture_alloc, 0); 1689 atomic_set(&n_rcu_torture_alloc_fail, 0); 1690 atomic_set(&n_rcu_torture_free, 0); 1691 atomic_set(&n_rcu_torture_mberror, 0); 1692 atomic_set(&n_rcu_torture_error, 0); 1693 n_rcu_torture_barrier_error = 0; 1694 n_rcu_torture_boost_ktrerror = 0; 1695 n_rcu_torture_boost_rterror = 0; 1696 n_rcu_torture_boost_failure = 0; 1697 n_rcu_torture_boosts = 0; 1698 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1699 atomic_set(&rcu_torture_wcount[i], 0); 1700 for_each_possible_cpu(cpu) { 1701 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1702 per_cpu(rcu_torture_count, cpu)[i] = 0; 1703 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1704 } 1705 } 1706 1707 /* Start up the kthreads. */ 1708 1709 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1710 writer_task); 1711 if (firsterr) 1712 goto unwind; 1713 fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]), 1714 GFP_KERNEL); 1715 if (fakewriter_tasks == NULL) { 1716 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1717 firsterr = -ENOMEM; 1718 goto unwind; 1719 } 1720 for (i = 0; i < nfakewriters; i++) { 1721 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1722 NULL, fakewriter_tasks[i]); 1723 if (firsterr) 1724 goto unwind; 1725 } 1726 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1727 GFP_KERNEL); 1728 if (reader_tasks == NULL) { 1729 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1730 firsterr = -ENOMEM; 1731 goto unwind; 1732 } 1733 for (i = 0; i < nrealreaders; i++) { 1734 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1735 reader_tasks[i]); 1736 if (firsterr) 1737 goto unwind; 1738 } 1739 if (stat_interval > 0) { 1740 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1741 stats_task); 1742 if (firsterr) 1743 goto unwind; 1744 } 1745 if (test_no_idle_hz) { 1746 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1747 if (firsterr) 1748 goto unwind; 1749 } 1750 if (stutter < 0) 1751 stutter = 0; 1752 if (stutter) { 1753 firsterr = torture_stutter_init(stutter * HZ); 1754 if (firsterr) 1755 goto unwind; 1756 } 1757 if (fqs_duration < 0) 1758 fqs_duration = 0; 1759 if (fqs_duration) { 1760 /* Create the fqs thread */ 1761 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1762 fqs_task); 1763 if (firsterr) 1764 goto unwind; 1765 } 1766 if (test_boost_interval < 1) 1767 test_boost_interval = 1; 1768 if (test_boost_duration < 2) 1769 test_boost_duration = 2; 1770 if ((test_boost == 1 && cur_ops->can_boost) || 1771 test_boost == 2) { 1772 1773 boost_starttime = jiffies + test_boost_interval * HZ; 1774 register_cpu_notifier(&rcutorture_cpu_nb); 1775 for_each_possible_cpu(i) { 1776 if (cpu_is_offline(i)) 1777 continue; /* Heuristic: CPU can go offline. */ 1778 firsterr = rcutorture_booster_init(i); 1779 if (firsterr) 1780 goto unwind; 1781 } 1782 } 1783 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1784 if (firsterr) 1785 goto unwind; 1786 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1787 if (firsterr) 1788 goto unwind; 1789 firsterr = rcu_torture_stall_init(); 1790 if (firsterr) 1791 goto unwind; 1792 firsterr = rcu_torture_barrier_init(); 1793 if (firsterr) 1794 goto unwind; 1795 if (object_debug) 1796 rcu_test_debug_objects(); 1797 if (cbflood_n_burst > 0) { 1798 /* Create the cbflood threads */ 1799 ncbflooders = (num_online_cpus() + 3) / 4; 1800 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1801 GFP_KERNEL); 1802 if (!cbflood_task) { 1803 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1804 firsterr = -ENOMEM; 1805 goto unwind; 1806 } 1807 for (i = 0; i < ncbflooders; i++) { 1808 firsterr = torture_create_kthread(rcu_torture_cbflood, 1809 NULL, 1810 cbflood_task[i]); 1811 if (firsterr) 1812 goto unwind; 1813 } 1814 } 1815 rcutorture_record_test_transition(); 1816 torture_init_end(); 1817 return 0; 1818 1819 unwind: 1820 torture_init_end(); 1821 rcu_torture_cleanup(); 1822 return firsterr; 1823 } 1824 1825 module_init(rcu_torture_init); 1826 module_exit(rcu_torture_cleanup); 1827