1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based torture test facility 4 * 5 * Copyright (C) IBM Corporation, 2005, 2006 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Josh Triplett <josh@joshtriplett.org> 9 * 10 * See also: Documentation/RCU/torture.txt 11 */ 12 13 #define pr_fmt(fmt) fmt 14 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/module.h> 19 #include <linux/kthread.h> 20 #include <linux/err.h> 21 #include <linux/spinlock.h> 22 #include <linux/smp.h> 23 #include <linux/rcupdate_wait.h> 24 #include <linux/interrupt.h> 25 #include <linux/sched/signal.h> 26 #include <uapi/linux/sched/types.h> 27 #include <linux/atomic.h> 28 #include <linux/bitops.h> 29 #include <linux/completion.h> 30 #include <linux/moduleparam.h> 31 #include <linux/percpu.h> 32 #include <linux/notifier.h> 33 #include <linux/reboot.h> 34 #include <linux/freezer.h> 35 #include <linux/cpu.h> 36 #include <linux/delay.h> 37 #include <linux/stat.h> 38 #include <linux/srcu.h> 39 #include <linux/slab.h> 40 #include <linux/trace_clock.h> 41 #include <asm/byteorder.h> 42 #include <linux/torture.h> 43 #include <linux/vmalloc.h> 44 #include <linux/sched/debug.h> 45 #include <linux/sched/sysctl.h> 46 #include <linux/oom.h> 47 #include <linux/tick.h> 48 49 #include "rcu.h" 50 51 MODULE_LICENSE("GPL"); 52 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 53 54 55 /* Bits for ->extendables field, extendables param, and related definitions. */ 56 #define RCUTORTURE_RDR_SHIFT 8 /* Put SRCU index in upper bits. */ 57 #define RCUTORTURE_RDR_MASK ((1 << RCUTORTURE_RDR_SHIFT) - 1) 58 #define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */ 59 #define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */ 60 #define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */ 61 #define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */ 62 #define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */ 63 #define RCUTORTURE_RDR_RCU 0x20 /* ... entering another RCU reader. */ 64 #define RCUTORTURE_RDR_NBITS 6 /* Number of bits defined above. */ 65 #define RCUTORTURE_MAX_EXTEND \ 66 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \ 67 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED) 68 #define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */ 69 /* Must be power of two minus one. */ 70 #define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3) 71 72 torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, 73 "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); 74 torture_param(int, fqs_duration, 0, 75 "Duration of fqs bursts (us), 0 to disable"); 76 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 77 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 78 torture_param(bool, fwd_progress, 1, "Test grace-period forward progress"); 79 torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait"); 80 torture_param(int, fwd_progress_holdoff, 60, 81 "Time between forward-progress tests (s)"); 82 torture_param(bool, fwd_progress_need_resched, 1, 83 "Hide cond_resched() behind need_resched()"); 84 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 85 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 86 torture_param(bool, gp_normal, false, 87 "Use normal (non-expedited) GP wait primitives"); 88 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 89 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 90 torture_param(int, n_barrier_cbs, 0, 91 "# of callbacks/kthreads for barrier testing"); 92 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 93 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 94 torture_param(int, object_debug, 0, 95 "Enable debug-object double call_rcu() testing"); 96 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 97 torture_param(int, onoff_interval, 0, 98 "Time between CPU hotplugs (jiffies), 0=disable"); 99 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 100 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 101 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 102 torture_param(int, stall_cpu_holdoff, 10, 103 "Time to wait before starting stall (s)."); 104 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 105 torture_param(int, stat_interval, 60, 106 "Number of seconds between stats printk()s"); 107 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 108 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 109 torture_param(int, test_boost_duration, 4, 110 "Duration of each boost test, seconds."); 111 torture_param(int, test_boost_interval, 7, 112 "Interval between boost tests, seconds."); 113 torture_param(bool, test_no_idle_hz, true, 114 "Test support for tickless idle CPUs"); 115 torture_param(int, verbose, 1, 116 "Enable verbose debugging printk()s"); 117 118 static char *torture_type = "rcu"; 119 module_param(torture_type, charp, 0444); 120 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)"); 121 122 static int nrealreaders; 123 static struct task_struct *writer_task; 124 static struct task_struct **fakewriter_tasks; 125 static struct task_struct **reader_tasks; 126 static struct task_struct *stats_task; 127 static struct task_struct *fqs_task; 128 static struct task_struct *boost_tasks[NR_CPUS]; 129 static struct task_struct *stall_task; 130 static struct task_struct *fwd_prog_task; 131 static struct task_struct **barrier_cbs_tasks; 132 static struct task_struct *barrier_task; 133 134 #define RCU_TORTURE_PIPE_LEN 10 135 136 struct rcu_torture { 137 struct rcu_head rtort_rcu; 138 int rtort_pipe_count; 139 struct list_head rtort_free; 140 int rtort_mbtest; 141 }; 142 143 static LIST_HEAD(rcu_torture_freelist); 144 static struct rcu_torture __rcu *rcu_torture_current; 145 static unsigned long rcu_torture_current_version; 146 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 147 static DEFINE_SPINLOCK(rcu_torture_lock); 148 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 149 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 150 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 151 static atomic_t n_rcu_torture_alloc; 152 static atomic_t n_rcu_torture_alloc_fail; 153 static atomic_t n_rcu_torture_free; 154 static atomic_t n_rcu_torture_mberror; 155 static atomic_t n_rcu_torture_error; 156 static long n_rcu_torture_barrier_error; 157 static long n_rcu_torture_boost_ktrerror; 158 static long n_rcu_torture_boost_rterror; 159 static long n_rcu_torture_boost_failure; 160 static long n_rcu_torture_boosts; 161 static atomic_long_t n_rcu_torture_timers; 162 static long n_barrier_attempts; 163 static long n_barrier_successes; /* did rcu_barrier test succeed? */ 164 static struct list_head rcu_torture_removed; 165 static unsigned long shutdown_jiffies; 166 167 static int rcu_torture_writer_state; 168 #define RTWS_FIXED_DELAY 0 169 #define RTWS_DELAY 1 170 #define RTWS_REPLACE 2 171 #define RTWS_DEF_FREE 3 172 #define RTWS_EXP_SYNC 4 173 #define RTWS_COND_GET 5 174 #define RTWS_COND_SYNC 6 175 #define RTWS_SYNC 7 176 #define RTWS_STUTTER 8 177 #define RTWS_STOPPING 9 178 static const char * const rcu_torture_writer_state_names[] = { 179 "RTWS_FIXED_DELAY", 180 "RTWS_DELAY", 181 "RTWS_REPLACE", 182 "RTWS_DEF_FREE", 183 "RTWS_EXP_SYNC", 184 "RTWS_COND_GET", 185 "RTWS_COND_SYNC", 186 "RTWS_SYNC", 187 "RTWS_STUTTER", 188 "RTWS_STOPPING", 189 }; 190 191 /* Record reader segment types and duration for first failing read. */ 192 struct rt_read_seg { 193 int rt_readstate; 194 unsigned long rt_delay_jiffies; 195 unsigned long rt_delay_ms; 196 unsigned long rt_delay_us; 197 bool rt_preempted; 198 }; 199 static int err_segs_recorded; 200 static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS]; 201 static int rt_read_nsegs; 202 203 static const char *rcu_torture_writer_state_getname(void) 204 { 205 unsigned int i = READ_ONCE(rcu_torture_writer_state); 206 207 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 208 return "???"; 209 return rcu_torture_writer_state_names[i]; 210 } 211 212 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 213 #define rcu_can_boost() 1 214 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 215 #define rcu_can_boost() 0 216 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 217 218 #ifdef CONFIG_RCU_TRACE 219 static u64 notrace rcu_trace_clock_local(void) 220 { 221 u64 ts = trace_clock_local(); 222 223 (void)do_div(ts, NSEC_PER_USEC); 224 return ts; 225 } 226 #else /* #ifdef CONFIG_RCU_TRACE */ 227 static u64 notrace rcu_trace_clock_local(void) 228 { 229 return 0ULL; 230 } 231 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 232 233 /* 234 * Stop aggressive CPU-hog tests a bit before the end of the test in order 235 * to avoid interfering with test shutdown. 236 */ 237 static bool shutdown_time_arrived(void) 238 { 239 return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ); 240 } 241 242 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 243 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 244 /* and boost task create/destroy. */ 245 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 246 static bool barrier_phase; /* Test phase. */ 247 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 248 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 249 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 250 251 static bool rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */ 252 253 /* 254 * Allocate an element from the rcu_tortures pool. 255 */ 256 static struct rcu_torture * 257 rcu_torture_alloc(void) 258 { 259 struct list_head *p; 260 261 spin_lock_bh(&rcu_torture_lock); 262 if (list_empty(&rcu_torture_freelist)) { 263 atomic_inc(&n_rcu_torture_alloc_fail); 264 spin_unlock_bh(&rcu_torture_lock); 265 return NULL; 266 } 267 atomic_inc(&n_rcu_torture_alloc); 268 p = rcu_torture_freelist.next; 269 list_del_init(p); 270 spin_unlock_bh(&rcu_torture_lock); 271 return container_of(p, struct rcu_torture, rtort_free); 272 } 273 274 /* 275 * Free an element to the rcu_tortures pool. 276 */ 277 static void 278 rcu_torture_free(struct rcu_torture *p) 279 { 280 atomic_inc(&n_rcu_torture_free); 281 spin_lock_bh(&rcu_torture_lock); 282 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 283 spin_unlock_bh(&rcu_torture_lock); 284 } 285 286 /* 287 * Operations vector for selecting different types of tests. 288 */ 289 290 struct rcu_torture_ops { 291 int ttype; 292 void (*init)(void); 293 void (*cleanup)(void); 294 int (*readlock)(void); 295 void (*read_delay)(struct torture_random_state *rrsp, 296 struct rt_read_seg *rtrsp); 297 void (*readunlock)(int idx); 298 unsigned long (*get_gp_seq)(void); 299 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 300 void (*deferred_free)(struct rcu_torture *p); 301 void (*sync)(void); 302 void (*exp_sync)(void); 303 unsigned long (*get_state)(void); 304 void (*cond_sync)(unsigned long oldstate); 305 call_rcu_func_t call; 306 void (*cb_barrier)(void); 307 void (*fqs)(void); 308 void (*stats)(void); 309 int (*stall_dur)(void); 310 int irq_capable; 311 int can_boost; 312 int extendables; 313 int slow_gps; 314 const char *name; 315 }; 316 317 static struct rcu_torture_ops *cur_ops; 318 319 /* 320 * Definitions for rcu torture testing. 321 */ 322 323 static int rcu_torture_read_lock(void) __acquires(RCU) 324 { 325 rcu_read_lock(); 326 return 0; 327 } 328 329 static void 330 rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 331 { 332 unsigned long started; 333 unsigned long completed; 334 const unsigned long shortdelay_us = 200; 335 unsigned long longdelay_ms = 300; 336 unsigned long long ts; 337 338 /* We want a short delay sometimes to make a reader delay the grace 339 * period, and we want a long delay occasionally to trigger 340 * force_quiescent_state. */ 341 342 if (!READ_ONCE(rcu_fwd_cb_nodelay) && 343 !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 344 started = cur_ops->get_gp_seq(); 345 ts = rcu_trace_clock_local(); 346 if (preempt_count() & (SOFTIRQ_MASK | HARDIRQ_MASK)) 347 longdelay_ms = 5; /* Avoid triggering BH limits. */ 348 mdelay(longdelay_ms); 349 rtrsp->rt_delay_ms = longdelay_ms; 350 completed = cur_ops->get_gp_seq(); 351 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 352 started, completed); 353 } 354 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) { 355 udelay(shortdelay_us); 356 rtrsp->rt_delay_us = shortdelay_us; 357 } 358 if (!preempt_count() && 359 !(torture_random(rrsp) % (nrealreaders * 500))) { 360 torture_preempt_schedule(); /* QS only if preemptible. */ 361 rtrsp->rt_preempted = true; 362 } 363 } 364 365 static void rcu_torture_read_unlock(int idx) __releases(RCU) 366 { 367 rcu_read_unlock(); 368 } 369 370 /* 371 * Update callback in the pipe. This should be invoked after a grace period. 372 */ 373 static bool 374 rcu_torture_pipe_update_one(struct rcu_torture *rp) 375 { 376 int i; 377 378 i = READ_ONCE(rp->rtort_pipe_count); 379 if (i > RCU_TORTURE_PIPE_LEN) 380 i = RCU_TORTURE_PIPE_LEN; 381 atomic_inc(&rcu_torture_wcount[i]); 382 WRITE_ONCE(rp->rtort_pipe_count, i + 1); 383 if (rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 384 rp->rtort_mbtest = 0; 385 return true; 386 } 387 return false; 388 } 389 390 /* 391 * Update all callbacks in the pipe. Suitable for synchronous grace-period 392 * primitives. 393 */ 394 static void 395 rcu_torture_pipe_update(struct rcu_torture *old_rp) 396 { 397 struct rcu_torture *rp; 398 struct rcu_torture *rp1; 399 400 if (old_rp) 401 list_add(&old_rp->rtort_free, &rcu_torture_removed); 402 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 403 if (rcu_torture_pipe_update_one(rp)) { 404 list_del(&rp->rtort_free); 405 rcu_torture_free(rp); 406 } 407 } 408 } 409 410 static void 411 rcu_torture_cb(struct rcu_head *p) 412 { 413 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 414 415 if (torture_must_stop_irq()) { 416 /* Test is ending, just drop callbacks on the floor. */ 417 /* The next initialization will pick up the pieces. */ 418 return; 419 } 420 if (rcu_torture_pipe_update_one(rp)) 421 rcu_torture_free(rp); 422 else 423 cur_ops->deferred_free(rp); 424 } 425 426 static unsigned long rcu_no_completed(void) 427 { 428 return 0; 429 } 430 431 static void rcu_torture_deferred_free(struct rcu_torture *p) 432 { 433 call_rcu(&p->rtort_rcu, rcu_torture_cb); 434 } 435 436 static void rcu_sync_torture_init(void) 437 { 438 INIT_LIST_HEAD(&rcu_torture_removed); 439 } 440 441 static struct rcu_torture_ops rcu_ops = { 442 .ttype = RCU_FLAVOR, 443 .init = rcu_sync_torture_init, 444 .readlock = rcu_torture_read_lock, 445 .read_delay = rcu_read_delay, 446 .readunlock = rcu_torture_read_unlock, 447 .get_gp_seq = rcu_get_gp_seq, 448 .gp_diff = rcu_seq_diff, 449 .deferred_free = rcu_torture_deferred_free, 450 .sync = synchronize_rcu, 451 .exp_sync = synchronize_rcu_expedited, 452 .get_state = get_state_synchronize_rcu, 453 .cond_sync = cond_synchronize_rcu, 454 .call = call_rcu, 455 .cb_barrier = rcu_barrier, 456 .fqs = rcu_force_quiescent_state, 457 .stats = NULL, 458 .stall_dur = rcu_jiffies_till_stall_check, 459 .irq_capable = 1, 460 .can_boost = rcu_can_boost(), 461 .extendables = RCUTORTURE_MAX_EXTEND, 462 .name = "rcu" 463 }; 464 465 /* 466 * Don't even think about trying any of these in real life!!! 467 * The names includes "busted", and they really means it! 468 * The only purpose of these functions is to provide a buggy RCU 469 * implementation to make sure that rcutorture correctly emits 470 * buggy-RCU error messages. 471 */ 472 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 473 { 474 /* This is a deliberate bug for testing purposes only! */ 475 rcu_torture_cb(&p->rtort_rcu); 476 } 477 478 static void synchronize_rcu_busted(void) 479 { 480 /* This is a deliberate bug for testing purposes only! */ 481 } 482 483 static void 484 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 485 { 486 /* This is a deliberate bug for testing purposes only! */ 487 func(head); 488 } 489 490 static struct rcu_torture_ops rcu_busted_ops = { 491 .ttype = INVALID_RCU_FLAVOR, 492 .init = rcu_sync_torture_init, 493 .readlock = rcu_torture_read_lock, 494 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 495 .readunlock = rcu_torture_read_unlock, 496 .get_gp_seq = rcu_no_completed, 497 .deferred_free = rcu_busted_torture_deferred_free, 498 .sync = synchronize_rcu_busted, 499 .exp_sync = synchronize_rcu_busted, 500 .call = call_rcu_busted, 501 .cb_barrier = NULL, 502 .fqs = NULL, 503 .stats = NULL, 504 .irq_capable = 1, 505 .name = "busted" 506 }; 507 508 /* 509 * Definitions for srcu torture testing. 510 */ 511 512 DEFINE_STATIC_SRCU(srcu_ctl); 513 static struct srcu_struct srcu_ctld; 514 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 515 516 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 517 { 518 return srcu_read_lock(srcu_ctlp); 519 } 520 521 static void 522 srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 523 { 524 long delay; 525 const long uspertick = 1000000 / HZ; 526 const long longdelay = 10; 527 528 /* We want there to be long-running readers, but not all the time. */ 529 530 delay = torture_random(rrsp) % 531 (nrealreaders * 2 * longdelay * uspertick); 532 if (!delay && in_task()) { 533 schedule_timeout_interruptible(longdelay); 534 rtrsp->rt_delay_jiffies = longdelay; 535 } else { 536 rcu_read_delay(rrsp, rtrsp); 537 } 538 } 539 540 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 541 { 542 srcu_read_unlock(srcu_ctlp, idx); 543 } 544 545 static unsigned long srcu_torture_completed(void) 546 { 547 return srcu_batches_completed(srcu_ctlp); 548 } 549 550 static void srcu_torture_deferred_free(struct rcu_torture *rp) 551 { 552 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 553 } 554 555 static void srcu_torture_synchronize(void) 556 { 557 synchronize_srcu(srcu_ctlp); 558 } 559 560 static void srcu_torture_call(struct rcu_head *head, 561 rcu_callback_t func) 562 { 563 call_srcu(srcu_ctlp, head, func); 564 } 565 566 static void srcu_torture_barrier(void) 567 { 568 srcu_barrier(srcu_ctlp); 569 } 570 571 static void srcu_torture_stats(void) 572 { 573 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 574 } 575 576 static void srcu_torture_synchronize_expedited(void) 577 { 578 synchronize_srcu_expedited(srcu_ctlp); 579 } 580 581 static struct rcu_torture_ops srcu_ops = { 582 .ttype = SRCU_FLAVOR, 583 .init = rcu_sync_torture_init, 584 .readlock = srcu_torture_read_lock, 585 .read_delay = srcu_read_delay, 586 .readunlock = srcu_torture_read_unlock, 587 .get_gp_seq = srcu_torture_completed, 588 .deferred_free = srcu_torture_deferred_free, 589 .sync = srcu_torture_synchronize, 590 .exp_sync = srcu_torture_synchronize_expedited, 591 .call = srcu_torture_call, 592 .cb_barrier = srcu_torture_barrier, 593 .stats = srcu_torture_stats, 594 .irq_capable = 1, 595 .name = "srcu" 596 }; 597 598 static void srcu_torture_init(void) 599 { 600 rcu_sync_torture_init(); 601 WARN_ON(init_srcu_struct(&srcu_ctld)); 602 srcu_ctlp = &srcu_ctld; 603 } 604 605 static void srcu_torture_cleanup(void) 606 { 607 cleanup_srcu_struct(&srcu_ctld); 608 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 609 } 610 611 /* As above, but dynamically allocated. */ 612 static struct rcu_torture_ops srcud_ops = { 613 .ttype = SRCU_FLAVOR, 614 .init = srcu_torture_init, 615 .cleanup = srcu_torture_cleanup, 616 .readlock = srcu_torture_read_lock, 617 .read_delay = srcu_read_delay, 618 .readunlock = srcu_torture_read_unlock, 619 .get_gp_seq = srcu_torture_completed, 620 .deferred_free = srcu_torture_deferred_free, 621 .sync = srcu_torture_synchronize, 622 .exp_sync = srcu_torture_synchronize_expedited, 623 .call = srcu_torture_call, 624 .cb_barrier = srcu_torture_barrier, 625 .stats = srcu_torture_stats, 626 .irq_capable = 1, 627 .name = "srcud" 628 }; 629 630 /* As above, but broken due to inappropriate reader extension. */ 631 static struct rcu_torture_ops busted_srcud_ops = { 632 .ttype = SRCU_FLAVOR, 633 .init = srcu_torture_init, 634 .cleanup = srcu_torture_cleanup, 635 .readlock = srcu_torture_read_lock, 636 .read_delay = rcu_read_delay, 637 .readunlock = srcu_torture_read_unlock, 638 .get_gp_seq = srcu_torture_completed, 639 .deferred_free = srcu_torture_deferred_free, 640 .sync = srcu_torture_synchronize, 641 .exp_sync = srcu_torture_synchronize_expedited, 642 .call = srcu_torture_call, 643 .cb_barrier = srcu_torture_barrier, 644 .stats = srcu_torture_stats, 645 .irq_capable = 1, 646 .extendables = RCUTORTURE_MAX_EXTEND, 647 .name = "busted_srcud" 648 }; 649 650 /* 651 * Definitions for RCU-tasks torture testing. 652 */ 653 654 static int tasks_torture_read_lock(void) 655 { 656 return 0; 657 } 658 659 static void tasks_torture_read_unlock(int idx) 660 { 661 } 662 663 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 664 { 665 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 666 } 667 668 static void synchronize_rcu_mult_test(void) 669 { 670 synchronize_rcu_mult(call_rcu_tasks, call_rcu); 671 } 672 673 static struct rcu_torture_ops tasks_ops = { 674 .ttype = RCU_TASKS_FLAVOR, 675 .init = rcu_sync_torture_init, 676 .readlock = tasks_torture_read_lock, 677 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 678 .readunlock = tasks_torture_read_unlock, 679 .get_gp_seq = rcu_no_completed, 680 .deferred_free = rcu_tasks_torture_deferred_free, 681 .sync = synchronize_rcu_tasks, 682 .exp_sync = synchronize_rcu_mult_test, 683 .call = call_rcu_tasks, 684 .cb_barrier = rcu_barrier_tasks, 685 .fqs = NULL, 686 .stats = NULL, 687 .irq_capable = 1, 688 .slow_gps = 1, 689 .name = "tasks" 690 }; 691 692 /* 693 * Definitions for trivial CONFIG_PREEMPT=n-only torture testing. 694 * This implementation does not necessarily work well with CPU hotplug. 695 */ 696 697 static void synchronize_rcu_trivial(void) 698 { 699 int cpu; 700 701 for_each_online_cpu(cpu) { 702 rcutorture_sched_setaffinity(current->pid, cpumask_of(cpu)); 703 WARN_ON_ONCE(raw_smp_processor_id() != cpu); 704 } 705 } 706 707 static int rcu_torture_read_lock_trivial(void) __acquires(RCU) 708 { 709 preempt_disable(); 710 return 0; 711 } 712 713 static void rcu_torture_read_unlock_trivial(int idx) __releases(RCU) 714 { 715 preempt_enable(); 716 } 717 718 static struct rcu_torture_ops trivial_ops = { 719 .ttype = RCU_TRIVIAL_FLAVOR, 720 .init = rcu_sync_torture_init, 721 .readlock = rcu_torture_read_lock_trivial, 722 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 723 .readunlock = rcu_torture_read_unlock_trivial, 724 .get_gp_seq = rcu_no_completed, 725 .sync = synchronize_rcu_trivial, 726 .exp_sync = synchronize_rcu_trivial, 727 .fqs = NULL, 728 .stats = NULL, 729 .irq_capable = 1, 730 .name = "trivial" 731 }; 732 733 static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old) 734 { 735 if (!cur_ops->gp_diff) 736 return new - old; 737 return cur_ops->gp_diff(new, old); 738 } 739 740 static bool __maybe_unused torturing_tasks(void) 741 { 742 return cur_ops == &tasks_ops; 743 } 744 745 /* 746 * RCU torture priority-boost testing. Runs one real-time thread per 747 * CPU for moderate bursts, repeatedly registering RCU callbacks and 748 * spinning waiting for them to be invoked. If a given callback takes 749 * too long to be invoked, we assume that priority inversion has occurred. 750 */ 751 752 struct rcu_boost_inflight { 753 struct rcu_head rcu; 754 int inflight; 755 }; 756 757 static void rcu_torture_boost_cb(struct rcu_head *head) 758 { 759 struct rcu_boost_inflight *rbip = 760 container_of(head, struct rcu_boost_inflight, rcu); 761 762 /* Ensure RCU-core accesses precede clearing ->inflight */ 763 smp_store_release(&rbip->inflight, 0); 764 } 765 766 static int old_rt_runtime = -1; 767 768 static void rcu_torture_disable_rt_throttle(void) 769 { 770 /* 771 * Disable RT throttling so that rcutorture's boost threads don't get 772 * throttled. Only possible if rcutorture is built-in otherwise the 773 * user should manually do this by setting the sched_rt_period_us and 774 * sched_rt_runtime sysctls. 775 */ 776 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1) 777 return; 778 779 old_rt_runtime = sysctl_sched_rt_runtime; 780 sysctl_sched_rt_runtime = -1; 781 } 782 783 static void rcu_torture_enable_rt_throttle(void) 784 { 785 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1) 786 return; 787 788 sysctl_sched_rt_runtime = old_rt_runtime; 789 old_rt_runtime = -1; 790 } 791 792 static bool rcu_torture_boost_failed(unsigned long start, unsigned long end) 793 { 794 if (end - start > test_boost_duration * HZ - HZ / 2) { 795 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 796 n_rcu_torture_boost_failure++; 797 798 return true; /* failed */ 799 } 800 801 return false; /* passed */ 802 } 803 804 static int rcu_torture_boost(void *arg) 805 { 806 unsigned long call_rcu_time; 807 unsigned long endtime; 808 unsigned long oldstarttime; 809 struct rcu_boost_inflight rbi = { .inflight = 0 }; 810 struct sched_param sp; 811 812 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 813 814 /* Set real-time priority. */ 815 sp.sched_priority = 1; 816 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 817 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 818 n_rcu_torture_boost_rterror++; 819 } 820 821 init_rcu_head_on_stack(&rbi.rcu); 822 /* Each pass through the following loop does one boost-test cycle. */ 823 do { 824 /* Track if the test failed already in this test interval? */ 825 bool failed = false; 826 827 /* Increment n_rcu_torture_boosts once per boost-test */ 828 while (!kthread_should_stop()) { 829 if (mutex_trylock(&boost_mutex)) { 830 n_rcu_torture_boosts++; 831 mutex_unlock(&boost_mutex); 832 break; 833 } 834 schedule_timeout_uninterruptible(1); 835 } 836 if (kthread_should_stop()) 837 goto checkwait; 838 839 /* Wait for the next test interval. */ 840 oldstarttime = boost_starttime; 841 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 842 schedule_timeout_interruptible(oldstarttime - jiffies); 843 stutter_wait("rcu_torture_boost"); 844 if (torture_must_stop()) 845 goto checkwait; 846 } 847 848 /* Do one boost-test interval. */ 849 endtime = oldstarttime + test_boost_duration * HZ; 850 call_rcu_time = jiffies; 851 while (ULONG_CMP_LT(jiffies, endtime)) { 852 /* If we don't have a callback in flight, post one. */ 853 if (!smp_load_acquire(&rbi.inflight)) { 854 /* RCU core before ->inflight = 1. */ 855 smp_store_release(&rbi.inflight, 1); 856 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 857 /* Check if the boost test failed */ 858 failed = failed || 859 rcu_torture_boost_failed(call_rcu_time, 860 jiffies); 861 call_rcu_time = jiffies; 862 } 863 stutter_wait("rcu_torture_boost"); 864 if (torture_must_stop()) 865 goto checkwait; 866 } 867 868 /* 869 * If boost never happened, then inflight will always be 1, in 870 * this case the boost check would never happen in the above 871 * loop so do another one here. 872 */ 873 if (!failed && smp_load_acquire(&rbi.inflight)) 874 rcu_torture_boost_failed(call_rcu_time, jiffies); 875 876 /* 877 * Set the start time of the next test interval. 878 * Yes, this is vulnerable to long delays, but such 879 * delays simply cause a false negative for the next 880 * interval. Besides, we are running at RT priority, 881 * so delays should be relatively rare. 882 */ 883 while (oldstarttime == boost_starttime && 884 !kthread_should_stop()) { 885 if (mutex_trylock(&boost_mutex)) { 886 boost_starttime = jiffies + 887 test_boost_interval * HZ; 888 mutex_unlock(&boost_mutex); 889 break; 890 } 891 schedule_timeout_uninterruptible(1); 892 } 893 894 /* Go do the stutter. */ 895 checkwait: stutter_wait("rcu_torture_boost"); 896 } while (!torture_must_stop()); 897 898 /* Clean up and exit. */ 899 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 900 torture_shutdown_absorb("rcu_torture_boost"); 901 schedule_timeout_uninterruptible(1); 902 } 903 destroy_rcu_head_on_stack(&rbi.rcu); 904 torture_kthread_stopping("rcu_torture_boost"); 905 return 0; 906 } 907 908 /* 909 * RCU torture force-quiescent-state kthread. Repeatedly induces 910 * bursts of calls to force_quiescent_state(), increasing the probability 911 * of occurrence of some important types of race conditions. 912 */ 913 static int 914 rcu_torture_fqs(void *arg) 915 { 916 unsigned long fqs_resume_time; 917 int fqs_burst_remaining; 918 919 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 920 do { 921 fqs_resume_time = jiffies + fqs_stutter * HZ; 922 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 923 !kthread_should_stop()) { 924 schedule_timeout_interruptible(1); 925 } 926 fqs_burst_remaining = fqs_duration; 927 while (fqs_burst_remaining > 0 && 928 !kthread_should_stop()) { 929 cur_ops->fqs(); 930 udelay(fqs_holdoff); 931 fqs_burst_remaining -= fqs_holdoff; 932 } 933 stutter_wait("rcu_torture_fqs"); 934 } while (!torture_must_stop()); 935 torture_kthread_stopping("rcu_torture_fqs"); 936 return 0; 937 } 938 939 /* 940 * RCU torture writer kthread. Repeatedly substitutes a new structure 941 * for that pointed to by rcu_torture_current, freeing the old structure 942 * after a series of grace periods (the "pipeline"). 943 */ 944 static int 945 rcu_torture_writer(void *arg) 946 { 947 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 948 int expediting = 0; 949 unsigned long gp_snap; 950 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 951 bool gp_sync1 = gp_sync; 952 int i; 953 struct rcu_torture *rp; 954 struct rcu_torture *old_rp; 955 static DEFINE_TORTURE_RANDOM(rand); 956 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 957 RTWS_COND_GET, RTWS_SYNC }; 958 int nsynctypes = 0; 959 960 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 961 if (!can_expedite) 962 pr_alert("%s" TORTURE_FLAG 963 " GP expediting controlled from boot/sysfs for %s.\n", 964 torture_type, cur_ops->name); 965 966 /* Initialize synctype[] array. If none set, take default. */ 967 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 968 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 969 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) { 970 synctype[nsynctypes++] = RTWS_COND_GET; 971 pr_info("%s: Testing conditional GPs.\n", __func__); 972 } else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) { 973 pr_alert("%s: gp_cond without primitives.\n", __func__); 974 } 975 if (gp_exp1 && cur_ops->exp_sync) { 976 synctype[nsynctypes++] = RTWS_EXP_SYNC; 977 pr_info("%s: Testing expedited GPs.\n", __func__); 978 } else if (gp_exp && !cur_ops->exp_sync) { 979 pr_alert("%s: gp_exp without primitives.\n", __func__); 980 } 981 if (gp_normal1 && cur_ops->deferred_free) { 982 synctype[nsynctypes++] = RTWS_DEF_FREE; 983 pr_info("%s: Testing asynchronous GPs.\n", __func__); 984 } else if (gp_normal && !cur_ops->deferred_free) { 985 pr_alert("%s: gp_normal without primitives.\n", __func__); 986 } 987 if (gp_sync1 && cur_ops->sync) { 988 synctype[nsynctypes++] = RTWS_SYNC; 989 pr_info("%s: Testing normal GPs.\n", __func__); 990 } else if (gp_sync && !cur_ops->sync) { 991 pr_alert("%s: gp_sync without primitives.\n", __func__); 992 } 993 if (WARN_ONCE(nsynctypes == 0, 994 "rcu_torture_writer: No update-side primitives.\n")) { 995 /* 996 * No updates primitives, so don't try updating. 997 * The resulting test won't be testing much, hence the 998 * above WARN_ONCE(). 999 */ 1000 rcu_torture_writer_state = RTWS_STOPPING; 1001 torture_kthread_stopping("rcu_torture_writer"); 1002 } 1003 1004 do { 1005 rcu_torture_writer_state = RTWS_FIXED_DELAY; 1006 schedule_timeout_uninterruptible(1); 1007 rp = rcu_torture_alloc(); 1008 if (rp == NULL) 1009 continue; 1010 rp->rtort_pipe_count = 0; 1011 rcu_torture_writer_state = RTWS_DELAY; 1012 udelay(torture_random(&rand) & 0x3ff); 1013 rcu_torture_writer_state = RTWS_REPLACE; 1014 old_rp = rcu_dereference_check(rcu_torture_current, 1015 current == writer_task); 1016 rp->rtort_mbtest = 1; 1017 rcu_assign_pointer(rcu_torture_current, rp); 1018 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 1019 if (old_rp) { 1020 i = old_rp->rtort_pipe_count; 1021 if (i > RCU_TORTURE_PIPE_LEN) 1022 i = RCU_TORTURE_PIPE_LEN; 1023 atomic_inc(&rcu_torture_wcount[i]); 1024 WRITE_ONCE(old_rp->rtort_pipe_count, 1025 old_rp->rtort_pipe_count + 1); 1026 switch (synctype[torture_random(&rand) % nsynctypes]) { 1027 case RTWS_DEF_FREE: 1028 rcu_torture_writer_state = RTWS_DEF_FREE; 1029 cur_ops->deferred_free(old_rp); 1030 break; 1031 case RTWS_EXP_SYNC: 1032 rcu_torture_writer_state = RTWS_EXP_SYNC; 1033 cur_ops->exp_sync(); 1034 rcu_torture_pipe_update(old_rp); 1035 break; 1036 case RTWS_COND_GET: 1037 rcu_torture_writer_state = RTWS_COND_GET; 1038 gp_snap = cur_ops->get_state(); 1039 i = torture_random(&rand) % 16; 1040 if (i != 0) 1041 schedule_timeout_interruptible(i); 1042 udelay(torture_random(&rand) % 1000); 1043 rcu_torture_writer_state = RTWS_COND_SYNC; 1044 cur_ops->cond_sync(gp_snap); 1045 rcu_torture_pipe_update(old_rp); 1046 break; 1047 case RTWS_SYNC: 1048 rcu_torture_writer_state = RTWS_SYNC; 1049 cur_ops->sync(); 1050 rcu_torture_pipe_update(old_rp); 1051 break; 1052 default: 1053 WARN_ON_ONCE(1); 1054 break; 1055 } 1056 } 1057 WRITE_ONCE(rcu_torture_current_version, 1058 rcu_torture_current_version + 1); 1059 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1060 if (can_expedite && 1061 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1062 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1063 if (expediting >= 0) 1064 rcu_expedite_gp(); 1065 else 1066 rcu_unexpedite_gp(); 1067 if (++expediting > 3) 1068 expediting = -expediting; 1069 } else if (!can_expedite) { /* Disabled during boot, recheck. */ 1070 can_expedite = !rcu_gp_is_expedited() && 1071 !rcu_gp_is_normal(); 1072 } 1073 rcu_torture_writer_state = RTWS_STUTTER; 1074 if (stutter_wait("rcu_torture_writer") && 1075 !READ_ONCE(rcu_fwd_cb_nodelay) && 1076 !cur_ops->slow_gps && 1077 !torture_must_stop() && 1078 rcu_inkernel_boot_has_ended()) 1079 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) 1080 if (list_empty(&rcu_tortures[i].rtort_free) && 1081 rcu_access_pointer(rcu_torture_current) != 1082 &rcu_tortures[i]) { 1083 rcu_ftrace_dump(DUMP_ALL); 1084 WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); 1085 } 1086 } while (!torture_must_stop()); 1087 /* Reset expediting back to unexpedited. */ 1088 if (expediting > 0) 1089 expediting = -expediting; 1090 while (can_expedite && expediting++ < 0) 1091 rcu_unexpedite_gp(); 1092 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1093 if (!can_expedite) 1094 pr_alert("%s" TORTURE_FLAG 1095 " Dynamic grace-period expediting was disabled.\n", 1096 torture_type); 1097 rcu_torture_writer_state = RTWS_STOPPING; 1098 torture_kthread_stopping("rcu_torture_writer"); 1099 return 0; 1100 } 1101 1102 /* 1103 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1104 * delay between calls. 1105 */ 1106 static int 1107 rcu_torture_fakewriter(void *arg) 1108 { 1109 DEFINE_TORTURE_RANDOM(rand); 1110 1111 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1112 set_user_nice(current, MAX_NICE); 1113 1114 do { 1115 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1116 udelay(torture_random(&rand) & 0x3ff); 1117 if (cur_ops->cb_barrier != NULL && 1118 torture_random(&rand) % (nfakewriters * 8) == 0) { 1119 cur_ops->cb_barrier(); 1120 } else if (gp_normal == gp_exp) { 1121 if (cur_ops->sync && torture_random(&rand) & 0x80) 1122 cur_ops->sync(); 1123 else if (cur_ops->exp_sync) 1124 cur_ops->exp_sync(); 1125 } else if (gp_normal && cur_ops->sync) { 1126 cur_ops->sync(); 1127 } else if (cur_ops->exp_sync) { 1128 cur_ops->exp_sync(); 1129 } 1130 stutter_wait("rcu_torture_fakewriter"); 1131 } while (!torture_must_stop()); 1132 1133 torture_kthread_stopping("rcu_torture_fakewriter"); 1134 return 0; 1135 } 1136 1137 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1138 { 1139 kfree(rhp); 1140 } 1141 1142 /* 1143 * Do one extension of an RCU read-side critical section using the 1144 * current reader state in readstate (set to zero for initial entry 1145 * to extended critical section), set the new state as specified by 1146 * newstate (set to zero for final exit from extended critical section), 1147 * and random-number-generator state in trsp. If this is neither the 1148 * beginning or end of the critical section and if there was actually a 1149 * change, do a ->read_delay(). 1150 */ 1151 static void rcutorture_one_extend(int *readstate, int newstate, 1152 struct torture_random_state *trsp, 1153 struct rt_read_seg *rtrsp) 1154 { 1155 unsigned long flags; 1156 int idxnew = -1; 1157 int idxold = *readstate; 1158 int statesnew = ~*readstate & newstate; 1159 int statesold = *readstate & ~newstate; 1160 1161 WARN_ON_ONCE(idxold < 0); 1162 WARN_ON_ONCE((idxold >> RCUTORTURE_RDR_SHIFT) > 1); 1163 rtrsp->rt_readstate = newstate; 1164 1165 /* First, put new protection in place to avoid critical-section gap. */ 1166 if (statesnew & RCUTORTURE_RDR_BH) 1167 local_bh_disable(); 1168 if (statesnew & RCUTORTURE_RDR_IRQ) 1169 local_irq_disable(); 1170 if (statesnew & RCUTORTURE_RDR_PREEMPT) 1171 preempt_disable(); 1172 if (statesnew & RCUTORTURE_RDR_RBH) 1173 rcu_read_lock_bh(); 1174 if (statesnew & RCUTORTURE_RDR_SCHED) 1175 rcu_read_lock_sched(); 1176 if (statesnew & RCUTORTURE_RDR_RCU) 1177 idxnew = cur_ops->readlock() << RCUTORTURE_RDR_SHIFT; 1178 1179 /* Next, remove old protection, irq first due to bh conflict. */ 1180 if (statesold & RCUTORTURE_RDR_IRQ) 1181 local_irq_enable(); 1182 if (statesold & RCUTORTURE_RDR_BH) 1183 local_bh_enable(); 1184 if (statesold & RCUTORTURE_RDR_PREEMPT) 1185 preempt_enable(); 1186 if (statesold & RCUTORTURE_RDR_RBH) 1187 rcu_read_unlock_bh(); 1188 if (statesold & RCUTORTURE_RDR_SCHED) 1189 rcu_read_unlock_sched(); 1190 if (statesold & RCUTORTURE_RDR_RCU) { 1191 bool lockit = !statesnew && !(torture_random(trsp) & 0xffff); 1192 1193 if (lockit) 1194 raw_spin_lock_irqsave(¤t->pi_lock, flags); 1195 cur_ops->readunlock(idxold >> RCUTORTURE_RDR_SHIFT); 1196 if (lockit) 1197 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); 1198 } 1199 1200 /* Delay if neither beginning nor end and there was a change. */ 1201 if ((statesnew || statesold) && *readstate && newstate) 1202 cur_ops->read_delay(trsp, rtrsp); 1203 1204 /* Update the reader state. */ 1205 if (idxnew == -1) 1206 idxnew = idxold & ~RCUTORTURE_RDR_MASK; 1207 WARN_ON_ONCE(idxnew < 0); 1208 WARN_ON_ONCE((idxnew >> RCUTORTURE_RDR_SHIFT) > 1); 1209 *readstate = idxnew | newstate; 1210 WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT) < 0); 1211 WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT) > 1); 1212 } 1213 1214 /* Return the biggest extendables mask given current RCU and boot parameters. */ 1215 static int rcutorture_extend_mask_max(void) 1216 { 1217 int mask; 1218 1219 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND); 1220 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables; 1221 mask = mask | RCUTORTURE_RDR_RCU; 1222 return mask; 1223 } 1224 1225 /* Return a random protection state mask, but with at least one bit set. */ 1226 static int 1227 rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) 1228 { 1229 int mask = rcutorture_extend_mask_max(); 1230 unsigned long randmask1 = torture_random(trsp) >> 8; 1231 unsigned long randmask2 = randmask1 >> 3; 1232 1233 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT); 1234 /* Mostly only one bit (need preemption!), sometimes lots of bits. */ 1235 if (!(randmask1 & 0x7)) 1236 mask = mask & randmask2; 1237 else 1238 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); 1239 /* Can't enable bh w/irq disabled. */ 1240 if ((mask & RCUTORTURE_RDR_IRQ) && 1241 ((!(mask & RCUTORTURE_RDR_BH) && (oldmask & RCUTORTURE_RDR_BH)) || 1242 (!(mask & RCUTORTURE_RDR_RBH) && (oldmask & RCUTORTURE_RDR_RBH)))) 1243 mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 1244 return mask ?: RCUTORTURE_RDR_RCU; 1245 } 1246 1247 /* 1248 * Do a randomly selected number of extensions of an existing RCU read-side 1249 * critical section. 1250 */ 1251 static struct rt_read_seg * 1252 rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, 1253 struct rt_read_seg *rtrsp) 1254 { 1255 int i; 1256 int j; 1257 int mask = rcutorture_extend_mask_max(); 1258 1259 WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */ 1260 if (!((mask - 1) & mask)) 1261 return rtrsp; /* Current RCU reader not extendable. */ 1262 /* Bias towards larger numbers of loops. */ 1263 i = (torture_random(trsp) >> 3); 1264 i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; 1265 for (j = 0; j < i; j++) { 1266 mask = rcutorture_extend_mask(*readstate, trsp); 1267 rcutorture_one_extend(readstate, mask, trsp, &rtrsp[j]); 1268 } 1269 return &rtrsp[j]; 1270 } 1271 1272 /* 1273 * Do one read-side critical section, returning false if there was 1274 * no data to read. Can be invoked both from process context and 1275 * from a timer handler. 1276 */ 1277 static bool rcu_torture_one_read(struct torture_random_state *trsp) 1278 { 1279 int i; 1280 unsigned long started; 1281 unsigned long completed; 1282 int newstate; 1283 struct rcu_torture *p; 1284 int pipe_count; 1285 int readstate = 0; 1286 struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS] = { { 0 } }; 1287 struct rt_read_seg *rtrsp = &rtseg[0]; 1288 struct rt_read_seg *rtrsp1; 1289 unsigned long long ts; 1290 1291 newstate = rcutorture_extend_mask(readstate, trsp); 1292 rcutorture_one_extend(&readstate, newstate, trsp, rtrsp++); 1293 started = cur_ops->get_gp_seq(); 1294 ts = rcu_trace_clock_local(); 1295 p = rcu_dereference_check(rcu_torture_current, 1296 rcu_read_lock_bh_held() || 1297 rcu_read_lock_sched_held() || 1298 srcu_read_lock_held(srcu_ctlp) || 1299 torturing_tasks()); 1300 if (p == NULL) { 1301 /* Wait for rcu_torture_writer to get underway */ 1302 rcutorture_one_extend(&readstate, 0, trsp, rtrsp); 1303 return false; 1304 } 1305 if (p->rtort_mbtest == 0) 1306 atomic_inc(&n_rcu_torture_mberror); 1307 rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp); 1308 preempt_disable(); 1309 pipe_count = READ_ONCE(p->rtort_pipe_count); 1310 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1311 /* Should not happen, but... */ 1312 pipe_count = RCU_TORTURE_PIPE_LEN; 1313 } 1314 completed = cur_ops->get_gp_seq(); 1315 if (pipe_count > 1) { 1316 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1317 ts, started, completed); 1318 rcu_ftrace_dump(DUMP_ALL); 1319 } 1320 __this_cpu_inc(rcu_torture_count[pipe_count]); 1321 completed = rcutorture_seq_diff(completed, started); 1322 if (completed > RCU_TORTURE_PIPE_LEN) { 1323 /* Should not happen, but... */ 1324 completed = RCU_TORTURE_PIPE_LEN; 1325 } 1326 __this_cpu_inc(rcu_torture_batch[completed]); 1327 preempt_enable(); 1328 rcutorture_one_extend(&readstate, 0, trsp, rtrsp); 1329 WARN_ON_ONCE(readstate & RCUTORTURE_RDR_MASK); 1330 1331 /* If error or close call, record the sequence of reader protections. */ 1332 if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) { 1333 i = 0; 1334 for (rtrsp1 = &rtseg[0]; rtrsp1 < rtrsp; rtrsp1++) 1335 err_segs[i++] = *rtrsp1; 1336 rt_read_nsegs = i; 1337 } 1338 1339 return true; 1340 } 1341 1342 static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); 1343 1344 /* 1345 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1346 * incrementing the corresponding element of the pipeline array. The 1347 * counter in the element should never be greater than 1, otherwise, the 1348 * RCU implementation is broken. 1349 */ 1350 static void rcu_torture_timer(struct timer_list *unused) 1351 { 1352 atomic_long_inc(&n_rcu_torture_timers); 1353 (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand)); 1354 1355 /* Test call_rcu() invocation from interrupt handler. */ 1356 if (cur_ops->call) { 1357 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 1358 1359 if (rhp) 1360 cur_ops->call(rhp, rcu_torture_timer_cb); 1361 } 1362 } 1363 1364 /* 1365 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1366 * incrementing the corresponding element of the pipeline array. The 1367 * counter in the element should never be greater than 1, otherwise, the 1368 * RCU implementation is broken. 1369 */ 1370 static int 1371 rcu_torture_reader(void *arg) 1372 { 1373 unsigned long lastsleep = jiffies; 1374 long myid = (long)arg; 1375 int mynumonline = myid; 1376 DEFINE_TORTURE_RANDOM(rand); 1377 struct timer_list t; 1378 1379 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1380 set_user_nice(current, MAX_NICE); 1381 if (irqreader && cur_ops->irq_capable) 1382 timer_setup_on_stack(&t, rcu_torture_timer, 0); 1383 tick_dep_set_task(current, TICK_DEP_BIT_RCU); 1384 do { 1385 if (irqreader && cur_ops->irq_capable) { 1386 if (!timer_pending(&t)) 1387 mod_timer(&t, jiffies + 1); 1388 } 1389 if (!rcu_torture_one_read(&rand) && !torture_must_stop()) 1390 schedule_timeout_interruptible(HZ); 1391 if (time_after(jiffies, lastsleep) && !torture_must_stop()) { 1392 schedule_timeout_interruptible(1); 1393 lastsleep = jiffies + 10; 1394 } 1395 while (num_online_cpus() < mynumonline && !torture_must_stop()) 1396 schedule_timeout_interruptible(HZ / 5); 1397 stutter_wait("rcu_torture_reader"); 1398 } while (!torture_must_stop()); 1399 if (irqreader && cur_ops->irq_capable) { 1400 del_timer_sync(&t); 1401 destroy_timer_on_stack(&t); 1402 } 1403 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 1404 torture_kthread_stopping("rcu_torture_reader"); 1405 return 0; 1406 } 1407 1408 /* 1409 * Print torture statistics. Caller must ensure that there is only 1410 * one call to this function at a given time!!! This is normally 1411 * accomplished by relying on the module system to only have one copy 1412 * of the module loaded, and then by giving the rcu_torture_stats 1413 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1414 * thread is not running). 1415 */ 1416 static void 1417 rcu_torture_stats_print(void) 1418 { 1419 int cpu; 1420 int i; 1421 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1422 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1423 struct rcu_torture *rtcp; 1424 static unsigned long rtcv_snap = ULONG_MAX; 1425 static bool splatted; 1426 struct task_struct *wtp; 1427 1428 for_each_possible_cpu(cpu) { 1429 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1430 pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]); 1431 batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]); 1432 } 1433 } 1434 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1435 if (pipesummary[i] != 0) 1436 break; 1437 } 1438 1439 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1440 rtcp = rcu_access_pointer(rcu_torture_current); 1441 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1442 rtcp, 1443 rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER", 1444 rcu_torture_current_version, 1445 list_empty(&rcu_torture_freelist), 1446 atomic_read(&n_rcu_torture_alloc), 1447 atomic_read(&n_rcu_torture_alloc_fail), 1448 atomic_read(&n_rcu_torture_free)); 1449 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ", 1450 atomic_read(&n_rcu_torture_mberror), 1451 n_rcu_torture_barrier_error, 1452 n_rcu_torture_boost_ktrerror, 1453 n_rcu_torture_boost_rterror); 1454 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1455 n_rcu_torture_boost_failure, 1456 n_rcu_torture_boosts, 1457 atomic_long_read(&n_rcu_torture_timers)); 1458 torture_onoff_stats(); 1459 pr_cont("barrier: %ld/%ld:%ld\n", 1460 n_barrier_successes, 1461 n_barrier_attempts, 1462 n_rcu_torture_barrier_error); 1463 1464 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1465 if (atomic_read(&n_rcu_torture_mberror) || 1466 n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror || 1467 n_rcu_torture_boost_rterror || n_rcu_torture_boost_failure || 1468 i > 1) { 1469 pr_cont("%s", "!!! "); 1470 atomic_inc(&n_rcu_torture_error); 1471 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror)); 1472 WARN_ON_ONCE(n_rcu_torture_barrier_error); // rcu_barrier() 1473 WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread 1474 WARN_ON_ONCE(n_rcu_torture_boost_rterror); // can't set RT prio 1475 WARN_ON_ONCE(n_rcu_torture_boost_failure); // RCU boost failed 1476 WARN_ON_ONCE(i > 1); // Too-short grace period 1477 } 1478 pr_cont("Reader Pipe: "); 1479 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1480 pr_cont(" %ld", pipesummary[i]); 1481 pr_cont("\n"); 1482 1483 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1484 pr_cont("Reader Batch: "); 1485 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1486 pr_cont(" %ld", batchsummary[i]); 1487 pr_cont("\n"); 1488 1489 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1490 pr_cont("Free-Block Circulation: "); 1491 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1492 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1493 } 1494 pr_cont("\n"); 1495 1496 if (cur_ops->stats) 1497 cur_ops->stats(); 1498 if (rtcv_snap == rcu_torture_current_version && 1499 rcu_access_pointer(rcu_torture_current) && 1500 !rcu_stall_is_suppressed()) { 1501 int __maybe_unused flags = 0; 1502 unsigned long __maybe_unused gp_seq = 0; 1503 1504 rcutorture_get_gp_data(cur_ops->ttype, 1505 &flags, &gp_seq); 1506 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, 1507 &flags, &gp_seq); 1508 wtp = READ_ONCE(writer_task); 1509 pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#lx cpu %d\n", 1510 rcu_torture_writer_state_getname(), 1511 rcu_torture_writer_state, gp_seq, flags, 1512 wtp == NULL ? ~0UL : wtp->state, 1513 wtp == NULL ? -1 : (int)task_cpu(wtp)); 1514 if (!splatted && wtp) { 1515 sched_show_task(wtp); 1516 splatted = true; 1517 } 1518 show_rcu_gp_kthreads(); 1519 rcu_ftrace_dump(DUMP_ALL); 1520 } 1521 rtcv_snap = rcu_torture_current_version; 1522 } 1523 1524 /* 1525 * Periodically prints torture statistics, if periodic statistics printing 1526 * was specified via the stat_interval module parameter. 1527 */ 1528 static int 1529 rcu_torture_stats(void *arg) 1530 { 1531 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1532 do { 1533 schedule_timeout_interruptible(stat_interval * HZ); 1534 rcu_torture_stats_print(); 1535 torture_shutdown_absorb("rcu_torture_stats"); 1536 } while (!torture_must_stop()); 1537 torture_kthread_stopping("rcu_torture_stats"); 1538 return 0; 1539 } 1540 1541 static void 1542 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1543 { 1544 pr_alert("%s" TORTURE_FLAG 1545 "--- %s: nreaders=%d nfakewriters=%d " 1546 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1547 "shuffle_interval=%d stutter=%d irqreader=%d " 1548 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1549 "test_boost=%d/%d test_boost_interval=%d " 1550 "test_boost_duration=%d shutdown_secs=%d " 1551 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 1552 "n_barrier_cbs=%d " 1553 "onoff_interval=%d onoff_holdoff=%d\n", 1554 torture_type, tag, nrealreaders, nfakewriters, 1555 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1556 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1557 test_boost, cur_ops->can_boost, 1558 test_boost_interval, test_boost_duration, shutdown_secs, 1559 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 1560 n_barrier_cbs, 1561 onoff_interval, onoff_holdoff); 1562 } 1563 1564 static int rcutorture_booster_cleanup(unsigned int cpu) 1565 { 1566 struct task_struct *t; 1567 1568 if (boost_tasks[cpu] == NULL) 1569 return 0; 1570 mutex_lock(&boost_mutex); 1571 t = boost_tasks[cpu]; 1572 boost_tasks[cpu] = NULL; 1573 rcu_torture_enable_rt_throttle(); 1574 mutex_unlock(&boost_mutex); 1575 1576 /* This must be outside of the mutex, otherwise deadlock! */ 1577 torture_stop_kthread(rcu_torture_boost, t); 1578 return 0; 1579 } 1580 1581 static int rcutorture_booster_init(unsigned int cpu) 1582 { 1583 int retval; 1584 1585 if (boost_tasks[cpu] != NULL) 1586 return 0; /* Already created, nothing more to do. */ 1587 1588 /* Don't allow time recalculation while creating a new task. */ 1589 mutex_lock(&boost_mutex); 1590 rcu_torture_disable_rt_throttle(); 1591 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1592 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1593 cpu_to_node(cpu), 1594 "rcu_torture_boost"); 1595 if (IS_ERR(boost_tasks[cpu])) { 1596 retval = PTR_ERR(boost_tasks[cpu]); 1597 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1598 n_rcu_torture_boost_ktrerror++; 1599 boost_tasks[cpu] = NULL; 1600 mutex_unlock(&boost_mutex); 1601 return retval; 1602 } 1603 kthread_bind(boost_tasks[cpu], cpu); 1604 wake_up_process(boost_tasks[cpu]); 1605 mutex_unlock(&boost_mutex); 1606 return 0; 1607 } 1608 1609 /* 1610 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1611 * induces a CPU stall for the time specified by stall_cpu. 1612 */ 1613 static int rcu_torture_stall(void *args) 1614 { 1615 unsigned long stop_at; 1616 1617 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1618 if (stall_cpu_holdoff > 0) { 1619 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1620 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1621 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1622 } 1623 if (!kthread_should_stop()) { 1624 stop_at = ktime_get_seconds() + stall_cpu; 1625 /* RCU CPU stall is expected behavior in following code. */ 1626 rcu_read_lock(); 1627 if (stall_cpu_irqsoff) 1628 local_irq_disable(); 1629 else 1630 preempt_disable(); 1631 pr_alert("rcu_torture_stall start on CPU %d.\n", 1632 smp_processor_id()); 1633 while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), 1634 stop_at)) 1635 continue; /* Induce RCU CPU stall warning. */ 1636 if (stall_cpu_irqsoff) 1637 local_irq_enable(); 1638 else 1639 preempt_enable(); 1640 rcu_read_unlock(); 1641 pr_alert("rcu_torture_stall end.\n"); 1642 } 1643 torture_shutdown_absorb("rcu_torture_stall"); 1644 while (!kthread_should_stop()) 1645 schedule_timeout_interruptible(10 * HZ); 1646 return 0; 1647 } 1648 1649 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1650 static int __init rcu_torture_stall_init(void) 1651 { 1652 if (stall_cpu <= 0) 1653 return 0; 1654 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1655 } 1656 1657 /* State structure for forward-progress self-propagating RCU callback. */ 1658 struct fwd_cb_state { 1659 struct rcu_head rh; 1660 int stop; 1661 }; 1662 1663 /* 1664 * Forward-progress self-propagating RCU callback function. Because 1665 * callbacks run from softirq, this function is an implicit RCU read-side 1666 * critical section. 1667 */ 1668 static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) 1669 { 1670 struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh); 1671 1672 if (READ_ONCE(fcsp->stop)) { 1673 WRITE_ONCE(fcsp->stop, 2); 1674 return; 1675 } 1676 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb); 1677 } 1678 1679 /* State for continuous-flood RCU callbacks. */ 1680 struct rcu_fwd_cb { 1681 struct rcu_head rh; 1682 struct rcu_fwd_cb *rfc_next; 1683 struct rcu_fwd *rfc_rfp; 1684 int rfc_gps; 1685 }; 1686 1687 #define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */ 1688 #define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */ 1689 #define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */ 1690 #define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */ 1691 #define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV)) 1692 1693 struct rcu_launder_hist { 1694 long n_launders; 1695 unsigned long launder_gp_seq; 1696 }; 1697 1698 struct rcu_fwd { 1699 spinlock_t rcu_fwd_lock; 1700 struct rcu_fwd_cb *rcu_fwd_cb_head; 1701 struct rcu_fwd_cb **rcu_fwd_cb_tail; 1702 long n_launders_cb; 1703 unsigned long rcu_fwd_startat; 1704 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST]; 1705 unsigned long rcu_launder_gp_seq_start; 1706 }; 1707 1708 struct rcu_fwd *rcu_fwds; 1709 bool rcu_fwd_emergency_stop; 1710 1711 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) 1712 { 1713 unsigned long gps; 1714 unsigned long gps_old; 1715 int i; 1716 int j; 1717 1718 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--) 1719 if (rfp->n_launders_hist[i].n_launders > 0) 1720 break; 1721 pr_alert("%s: Callback-invocation histogram (duration %lu jiffies):", 1722 __func__, jiffies - rfp->rcu_fwd_startat); 1723 gps_old = rfp->rcu_launder_gp_seq_start; 1724 for (j = 0; j <= i; j++) { 1725 gps = rfp->n_launders_hist[j].launder_gp_seq; 1726 pr_cont(" %ds/%d: %ld:%ld", 1727 j + 1, FWD_CBS_HIST_DIV, 1728 rfp->n_launders_hist[j].n_launders, 1729 rcutorture_seq_diff(gps, gps_old)); 1730 gps_old = gps; 1731 } 1732 pr_cont("\n"); 1733 } 1734 1735 /* Callback function for continuous-flood RCU callbacks. */ 1736 static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) 1737 { 1738 unsigned long flags; 1739 int i; 1740 struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); 1741 struct rcu_fwd_cb **rfcpp; 1742 struct rcu_fwd *rfp = rfcp->rfc_rfp; 1743 1744 rfcp->rfc_next = NULL; 1745 rfcp->rfc_gps++; 1746 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 1747 rfcpp = rfp->rcu_fwd_cb_tail; 1748 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next; 1749 WRITE_ONCE(*rfcpp, rfcp); 1750 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1); 1751 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV)); 1752 if (i >= ARRAY_SIZE(rfp->n_launders_hist)) 1753 i = ARRAY_SIZE(rfp->n_launders_hist) - 1; 1754 rfp->n_launders_hist[i].n_launders++; 1755 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); 1756 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 1757 } 1758 1759 // Give the scheduler a chance, even on nohz_full CPUs. 1760 static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) 1761 { 1762 if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) { 1763 // Real call_rcu() floods hit userspace, so emulate that. 1764 if (need_resched() || (iter & 0xfff)) 1765 schedule(); 1766 return; 1767 } 1768 // No userspace emulation: CB invocation throttles call_rcu() 1769 cond_resched(); 1770 } 1771 1772 /* 1773 * Free all callbacks on the rcu_fwd_cb_head list, either because the 1774 * test is over or because we hit an OOM event. 1775 */ 1776 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) 1777 { 1778 unsigned long flags; 1779 unsigned long freed = 0; 1780 struct rcu_fwd_cb *rfcp; 1781 1782 for (;;) { 1783 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 1784 rfcp = rfp->rcu_fwd_cb_head; 1785 if (!rfcp) { 1786 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 1787 break; 1788 } 1789 rfp->rcu_fwd_cb_head = rfcp->rfc_next; 1790 if (!rfp->rcu_fwd_cb_head) 1791 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; 1792 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 1793 kfree(rfcp); 1794 freed++; 1795 rcu_torture_fwd_prog_cond_resched(freed); 1796 if (tick_nohz_full_enabled()) { 1797 local_irq_save(flags); 1798 rcu_momentary_dyntick_idle(); 1799 local_irq_restore(flags); 1800 } 1801 } 1802 return freed; 1803 } 1804 1805 /* Carry out need_resched()/cond_resched() forward-progress testing. */ 1806 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, 1807 int *tested, int *tested_tries) 1808 { 1809 unsigned long cver; 1810 unsigned long dur; 1811 struct fwd_cb_state fcs; 1812 unsigned long gps; 1813 int idx; 1814 int sd; 1815 int sd4; 1816 bool selfpropcb = false; 1817 unsigned long stopat; 1818 static DEFINE_TORTURE_RANDOM(trs); 1819 1820 if (cur_ops->call && cur_ops->sync && cur_ops->cb_barrier) { 1821 init_rcu_head_on_stack(&fcs.rh); 1822 selfpropcb = true; 1823 } 1824 1825 /* Tight loop containing cond_resched(). */ 1826 WRITE_ONCE(rcu_fwd_cb_nodelay, true); 1827 cur_ops->sync(); /* Later readers see above write. */ 1828 if (selfpropcb) { 1829 WRITE_ONCE(fcs.stop, 0); 1830 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb); 1831 } 1832 cver = READ_ONCE(rcu_torture_current_version); 1833 gps = cur_ops->get_gp_seq(); 1834 sd = cur_ops->stall_dur() + 1; 1835 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div; 1836 dur = sd4 + torture_random(&trs) % (sd - sd4); 1837 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 1838 stopat = rfp->rcu_fwd_startat + dur; 1839 while (time_before(jiffies, stopat) && 1840 !shutdown_time_arrived() && 1841 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 1842 idx = cur_ops->readlock(); 1843 udelay(10); 1844 cur_ops->readunlock(idx); 1845 if (!fwd_progress_need_resched || need_resched()) 1846 cond_resched(); 1847 } 1848 (*tested_tries)++; 1849 if (!time_before(jiffies, stopat) && 1850 !shutdown_time_arrived() && 1851 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 1852 (*tested)++; 1853 cver = READ_ONCE(rcu_torture_current_version) - cver; 1854 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 1855 WARN_ON(!cver && gps < 2); 1856 pr_alert("%s: Duration %ld cver %ld gps %ld\n", __func__, dur, cver, gps); 1857 } 1858 if (selfpropcb) { 1859 WRITE_ONCE(fcs.stop, 1); 1860 cur_ops->sync(); /* Wait for running CB to complete. */ 1861 cur_ops->cb_barrier(); /* Wait for queued callbacks. */ 1862 } 1863 1864 if (selfpropcb) { 1865 WARN_ON(READ_ONCE(fcs.stop) != 2); 1866 destroy_rcu_head_on_stack(&fcs.rh); 1867 } 1868 schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */ 1869 WRITE_ONCE(rcu_fwd_cb_nodelay, false); 1870 } 1871 1872 /* Carry out call_rcu() forward-progress testing. */ 1873 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) 1874 { 1875 unsigned long cver; 1876 unsigned long flags; 1877 unsigned long gps; 1878 int i; 1879 long n_launders; 1880 long n_launders_cb_snap; 1881 long n_launders_sa; 1882 long n_max_cbs; 1883 long n_max_gps; 1884 struct rcu_fwd_cb *rfcp; 1885 struct rcu_fwd_cb *rfcpn; 1886 unsigned long stopat; 1887 unsigned long stoppedat; 1888 1889 if (READ_ONCE(rcu_fwd_emergency_stop)) 1890 return; /* Get out of the way quickly, no GP wait! */ 1891 if (!cur_ops->call) 1892 return; /* Can't do call_rcu() fwd prog without ->call. */ 1893 1894 /* Loop continuously posting RCU callbacks. */ 1895 WRITE_ONCE(rcu_fwd_cb_nodelay, true); 1896 cur_ops->sync(); /* Later readers see above write. */ 1897 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 1898 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES; 1899 n_launders = 0; 1900 rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread 1901 n_launders_sa = 0; 1902 n_max_cbs = 0; 1903 n_max_gps = 0; 1904 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++) 1905 rfp->n_launders_hist[i].n_launders = 0; 1906 cver = READ_ONCE(rcu_torture_current_version); 1907 gps = cur_ops->get_gp_seq(); 1908 rfp->rcu_launder_gp_seq_start = gps; 1909 tick_dep_set_task(current, TICK_DEP_BIT_RCU); 1910 while (time_before(jiffies, stopat) && 1911 !shutdown_time_arrived() && 1912 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 1913 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head); 1914 rfcpn = NULL; 1915 if (rfcp) 1916 rfcpn = READ_ONCE(rfcp->rfc_next); 1917 if (rfcpn) { 1918 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS && 1919 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED) 1920 break; 1921 rfp->rcu_fwd_cb_head = rfcpn; 1922 n_launders++; 1923 n_launders_sa++; 1924 } else { 1925 rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL); 1926 if (WARN_ON_ONCE(!rfcp)) { 1927 schedule_timeout_interruptible(1); 1928 continue; 1929 } 1930 n_max_cbs++; 1931 n_launders_sa = 0; 1932 rfcp->rfc_gps = 0; 1933 rfcp->rfc_rfp = rfp; 1934 } 1935 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); 1936 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); 1937 if (tick_nohz_full_enabled()) { 1938 local_irq_save(flags); 1939 rcu_momentary_dyntick_idle(); 1940 local_irq_restore(flags); 1941 } 1942 } 1943 stoppedat = jiffies; 1944 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb); 1945 cver = READ_ONCE(rcu_torture_current_version) - cver; 1946 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 1947 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ 1948 (void)rcu_torture_fwd_prog_cbfree(rfp); 1949 1950 if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) && 1951 !shutdown_time_arrived()) { 1952 WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED); 1953 pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n", 1954 __func__, 1955 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat, 1956 n_launders + n_max_cbs - n_launders_cb_snap, 1957 n_launders, n_launders_sa, 1958 n_max_gps, n_max_cbs, cver, gps); 1959 rcu_torture_fwd_cb_hist(rfp); 1960 } 1961 schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */ 1962 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 1963 WRITE_ONCE(rcu_fwd_cb_nodelay, false); 1964 } 1965 1966 1967 /* 1968 * OOM notifier, but this only prints diagnostic information for the 1969 * current forward-progress test. 1970 */ 1971 static int rcutorture_oom_notify(struct notifier_block *self, 1972 unsigned long notused, void *nfreed) 1973 { 1974 struct rcu_fwd *rfp = rcu_fwds; 1975 1976 WARN(1, "%s invoked upon OOM during forward-progress testing.\n", 1977 __func__); 1978 rcu_torture_fwd_cb_hist(rfp); 1979 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp->rcu_fwd_startat)) / 2); 1980 WRITE_ONCE(rcu_fwd_emergency_stop, true); 1981 smp_mb(); /* Emergency stop before free and wait to avoid hangs. */ 1982 pr_info("%s: Freed %lu RCU callbacks.\n", 1983 __func__, rcu_torture_fwd_prog_cbfree(rfp)); 1984 rcu_barrier(); 1985 pr_info("%s: Freed %lu RCU callbacks.\n", 1986 __func__, rcu_torture_fwd_prog_cbfree(rfp)); 1987 rcu_barrier(); 1988 pr_info("%s: Freed %lu RCU callbacks.\n", 1989 __func__, rcu_torture_fwd_prog_cbfree(rfp)); 1990 smp_mb(); /* Frees before return to avoid redoing OOM. */ 1991 (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */ 1992 pr_info("%s returning after OOM processing.\n", __func__); 1993 return NOTIFY_OK; 1994 } 1995 1996 static struct notifier_block rcutorture_oom_nb = { 1997 .notifier_call = rcutorture_oom_notify 1998 }; 1999 2000 /* Carry out grace-period forward-progress testing. */ 2001 static int rcu_torture_fwd_prog(void *args) 2002 { 2003 struct rcu_fwd *rfp = args; 2004 int tested = 0; 2005 int tested_tries = 0; 2006 2007 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started"); 2008 rcu_bind_current_to_nocb(); 2009 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST)) 2010 set_user_nice(current, MAX_NICE); 2011 do { 2012 schedule_timeout_interruptible(fwd_progress_holdoff * HZ); 2013 WRITE_ONCE(rcu_fwd_emergency_stop, false); 2014 register_oom_notifier(&rcutorture_oom_nb); 2015 if (!IS_ENABLED(CONFIG_TINY_RCU) || 2016 rcu_inkernel_boot_has_ended()) 2017 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); 2018 if (rcu_inkernel_boot_has_ended()) 2019 rcu_torture_fwd_prog_cr(rfp); 2020 unregister_oom_notifier(&rcutorture_oom_nb); 2021 2022 /* Avoid slow periods, better to test when busy. */ 2023 stutter_wait("rcu_torture_fwd_prog"); 2024 } while (!torture_must_stop()); 2025 /* Short runs might not contain a valid forward-progress attempt. */ 2026 WARN_ON(!tested && tested_tries >= 5); 2027 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries); 2028 torture_kthread_stopping("rcu_torture_fwd_prog"); 2029 return 0; 2030 } 2031 2032 /* If forward-progress checking is requested and feasible, spawn the thread. */ 2033 static int __init rcu_torture_fwd_prog_init(void) 2034 { 2035 struct rcu_fwd *rfp; 2036 2037 if (!fwd_progress) 2038 return 0; /* Not requested, so don't do it. */ 2039 if (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0 || 2040 cur_ops == &rcu_busted_ops) { 2041 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); 2042 return 0; 2043 } 2044 if (stall_cpu > 0) { 2045 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing"); 2046 if (IS_MODULE(CONFIG_RCU_TORTURE_TESTS)) 2047 return -EINVAL; /* In module, can fail back to user. */ 2048 WARN_ON(1); /* Make sure rcutorture notices conflict. */ 2049 return 0; 2050 } 2051 if (fwd_progress_holdoff <= 0) 2052 fwd_progress_holdoff = 1; 2053 if (fwd_progress_div <= 0) 2054 fwd_progress_div = 4; 2055 rfp = kzalloc(sizeof(*rfp), GFP_KERNEL); 2056 if (!rfp) 2057 return -ENOMEM; 2058 spin_lock_init(&rfp->rcu_fwd_lock); 2059 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; 2060 return torture_create_kthread(rcu_torture_fwd_prog, rfp, fwd_prog_task); 2061 } 2062 2063 /* Callback function for RCU barrier testing. */ 2064 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 2065 { 2066 atomic_inc(&barrier_cbs_invoked); 2067 } 2068 2069 /* IPI handler to get callback posted on desired CPU, if online. */ 2070 static void rcu_torture_barrier1cb(void *rcu_void) 2071 { 2072 struct rcu_head *rhp = rcu_void; 2073 2074 cur_ops->call(rhp, rcu_torture_barrier_cbf); 2075 } 2076 2077 /* kthread function to register callbacks used to test RCU barriers. */ 2078 static int rcu_torture_barrier_cbs(void *arg) 2079 { 2080 long myid = (long)arg; 2081 bool lastphase = 0; 2082 bool newphase; 2083 struct rcu_head rcu; 2084 2085 init_rcu_head_on_stack(&rcu); 2086 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 2087 set_user_nice(current, MAX_NICE); 2088 do { 2089 wait_event(barrier_cbs_wq[myid], 2090 (newphase = 2091 smp_load_acquire(&barrier_phase)) != lastphase || 2092 torture_must_stop()); 2093 lastphase = newphase; 2094 if (torture_must_stop()) 2095 break; 2096 /* 2097 * The above smp_load_acquire() ensures barrier_phase load 2098 * is ordered before the following ->call(). 2099 */ 2100 if (smp_call_function_single(myid, rcu_torture_barrier1cb, 2101 &rcu, 1)) { 2102 // IPI failed, so use direct call from current CPU. 2103 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 2104 } 2105 if (atomic_dec_and_test(&barrier_cbs_count)) 2106 wake_up(&barrier_wq); 2107 } while (!torture_must_stop()); 2108 if (cur_ops->cb_barrier != NULL) 2109 cur_ops->cb_barrier(); 2110 destroy_rcu_head_on_stack(&rcu); 2111 torture_kthread_stopping("rcu_torture_barrier_cbs"); 2112 return 0; 2113 } 2114 2115 /* kthread function to drive and coordinate RCU barrier testing. */ 2116 static int rcu_torture_barrier(void *arg) 2117 { 2118 int i; 2119 2120 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 2121 do { 2122 atomic_set(&barrier_cbs_invoked, 0); 2123 atomic_set(&barrier_cbs_count, n_barrier_cbs); 2124 /* Ensure barrier_phase ordered after prior assignments. */ 2125 smp_store_release(&barrier_phase, !barrier_phase); 2126 for (i = 0; i < n_barrier_cbs; i++) 2127 wake_up(&barrier_cbs_wq[i]); 2128 wait_event(barrier_wq, 2129 atomic_read(&barrier_cbs_count) == 0 || 2130 torture_must_stop()); 2131 if (torture_must_stop()) 2132 break; 2133 n_barrier_attempts++; 2134 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 2135 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 2136 n_rcu_torture_barrier_error++; 2137 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 2138 atomic_read(&barrier_cbs_invoked), 2139 n_barrier_cbs); 2140 WARN_ON(1); 2141 // Wait manually for the remaining callbacks 2142 i = 0; 2143 do { 2144 if (WARN_ON(i++ > HZ)) 2145 i = INT_MIN; 2146 schedule_timeout_interruptible(1); 2147 cur_ops->cb_barrier(); 2148 } while (atomic_read(&barrier_cbs_invoked) != 2149 n_barrier_cbs && 2150 !torture_must_stop()); 2151 smp_mb(); // Can't trust ordering if broken. 2152 if (!torture_must_stop()) 2153 pr_err("Recovered: barrier_cbs_invoked = %d\n", 2154 atomic_read(&barrier_cbs_invoked)); 2155 } else { 2156 n_barrier_successes++; 2157 } 2158 schedule_timeout_interruptible(HZ / 10); 2159 } while (!torture_must_stop()); 2160 torture_kthread_stopping("rcu_torture_barrier"); 2161 return 0; 2162 } 2163 2164 /* Initialize RCU barrier testing. */ 2165 static int rcu_torture_barrier_init(void) 2166 { 2167 int i; 2168 int ret; 2169 2170 if (n_barrier_cbs <= 0) 2171 return 0; 2172 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 2173 pr_alert("%s" TORTURE_FLAG 2174 " Call or barrier ops missing for %s,\n", 2175 torture_type, cur_ops->name); 2176 pr_alert("%s" TORTURE_FLAG 2177 " RCU barrier testing omitted from run.\n", 2178 torture_type); 2179 return 0; 2180 } 2181 atomic_set(&barrier_cbs_count, 0); 2182 atomic_set(&barrier_cbs_invoked, 0); 2183 barrier_cbs_tasks = 2184 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]), 2185 GFP_KERNEL); 2186 barrier_cbs_wq = 2187 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL); 2188 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 2189 return -ENOMEM; 2190 for (i = 0; i < n_barrier_cbs; i++) { 2191 init_waitqueue_head(&barrier_cbs_wq[i]); 2192 ret = torture_create_kthread(rcu_torture_barrier_cbs, 2193 (void *)(long)i, 2194 barrier_cbs_tasks[i]); 2195 if (ret) 2196 return ret; 2197 } 2198 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 2199 } 2200 2201 /* Clean up after RCU barrier testing. */ 2202 static void rcu_torture_barrier_cleanup(void) 2203 { 2204 int i; 2205 2206 torture_stop_kthread(rcu_torture_barrier, barrier_task); 2207 if (barrier_cbs_tasks != NULL) { 2208 for (i = 0; i < n_barrier_cbs; i++) 2209 torture_stop_kthread(rcu_torture_barrier_cbs, 2210 barrier_cbs_tasks[i]); 2211 kfree(barrier_cbs_tasks); 2212 barrier_cbs_tasks = NULL; 2213 } 2214 if (barrier_cbs_wq != NULL) { 2215 kfree(barrier_cbs_wq); 2216 barrier_cbs_wq = NULL; 2217 } 2218 } 2219 2220 static bool rcu_torture_can_boost(void) 2221 { 2222 static int boost_warn_once; 2223 int prio; 2224 2225 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2) 2226 return false; 2227 2228 prio = rcu_get_gp_kthreads_prio(); 2229 if (!prio) 2230 return false; 2231 2232 if (prio < 2) { 2233 if (boost_warn_once == 1) 2234 return false; 2235 2236 pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME); 2237 boost_warn_once = 1; 2238 return false; 2239 } 2240 2241 return true; 2242 } 2243 2244 static enum cpuhp_state rcutor_hp; 2245 2246 static void 2247 rcu_torture_cleanup(void) 2248 { 2249 int firsttime; 2250 int flags = 0; 2251 unsigned long gp_seq = 0; 2252 int i; 2253 2254 if (torture_cleanup_begin()) { 2255 if (cur_ops->cb_barrier != NULL) 2256 cur_ops->cb_barrier(); 2257 return; 2258 } 2259 if (!cur_ops) { 2260 torture_cleanup_end(); 2261 return; 2262 } 2263 2264 show_rcu_gp_kthreads(); 2265 rcu_torture_barrier_cleanup(); 2266 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_task); 2267 torture_stop_kthread(rcu_torture_stall, stall_task); 2268 torture_stop_kthread(rcu_torture_writer, writer_task); 2269 2270 if (reader_tasks) { 2271 for (i = 0; i < nrealreaders; i++) 2272 torture_stop_kthread(rcu_torture_reader, 2273 reader_tasks[i]); 2274 kfree(reader_tasks); 2275 } 2276 rcu_torture_current = NULL; 2277 2278 if (fakewriter_tasks) { 2279 for (i = 0; i < nfakewriters; i++) { 2280 torture_stop_kthread(rcu_torture_fakewriter, 2281 fakewriter_tasks[i]); 2282 } 2283 kfree(fakewriter_tasks); 2284 fakewriter_tasks = NULL; 2285 } 2286 2287 rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq); 2288 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq); 2289 pr_alert("%s: End-test grace-period state: g%lu f%#x\n", 2290 cur_ops->name, gp_seq, flags); 2291 torture_stop_kthread(rcu_torture_stats, stats_task); 2292 torture_stop_kthread(rcu_torture_fqs, fqs_task); 2293 if (rcu_torture_can_boost()) 2294 cpuhp_remove_state(rcutor_hp); 2295 2296 /* 2297 * Wait for all RCU callbacks to fire, then do torture-type-specific 2298 * cleanup operations. 2299 */ 2300 if (cur_ops->cb_barrier != NULL) 2301 cur_ops->cb_barrier(); 2302 if (cur_ops->cleanup != NULL) 2303 cur_ops->cleanup(); 2304 2305 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 2306 2307 if (err_segs_recorded) { 2308 pr_alert("Failure/close-call rcutorture reader segments:\n"); 2309 if (rt_read_nsegs == 0) 2310 pr_alert("\t: No segments recorded!!!\n"); 2311 firsttime = 1; 2312 for (i = 0; i < rt_read_nsegs; i++) { 2313 pr_alert("\t%d: %#x ", i, err_segs[i].rt_readstate); 2314 if (err_segs[i].rt_delay_jiffies != 0) { 2315 pr_cont("%s%ldjiffies", firsttime ? "" : "+", 2316 err_segs[i].rt_delay_jiffies); 2317 firsttime = 0; 2318 } 2319 if (err_segs[i].rt_delay_ms != 0) { 2320 pr_cont("%s%ldms", firsttime ? "" : "+", 2321 err_segs[i].rt_delay_ms); 2322 firsttime = 0; 2323 } 2324 if (err_segs[i].rt_delay_us != 0) { 2325 pr_cont("%s%ldus", firsttime ? "" : "+", 2326 err_segs[i].rt_delay_us); 2327 firsttime = 0; 2328 } 2329 pr_cont("%s\n", 2330 err_segs[i].rt_preempted ? "preempted" : ""); 2331 2332 } 2333 } 2334 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 2335 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 2336 else if (torture_onoff_failures()) 2337 rcu_torture_print_module_parms(cur_ops, 2338 "End of test: RCU_HOTPLUG"); 2339 else 2340 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 2341 torture_cleanup_end(); 2342 } 2343 2344 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 2345 static void rcu_torture_leak_cb(struct rcu_head *rhp) 2346 { 2347 } 2348 2349 static void rcu_torture_err_cb(struct rcu_head *rhp) 2350 { 2351 /* 2352 * This -might- happen due to race conditions, but is unlikely. 2353 * The scenario that leads to this happening is that the 2354 * first of the pair of duplicate callbacks is queued, 2355 * someone else starts a grace period that includes that 2356 * callback, then the second of the pair must wait for the 2357 * next grace period. Unlikely, but can happen. If it 2358 * does happen, the debug-objects subsystem won't have splatted. 2359 */ 2360 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); 2361 } 2362 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 2363 2364 /* 2365 * Verify that double-free causes debug-objects to complain, but only 2366 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 2367 * cannot be carried out. 2368 */ 2369 static void rcu_test_debug_objects(void) 2370 { 2371 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 2372 struct rcu_head rh1; 2373 struct rcu_head rh2; 2374 2375 init_rcu_head_on_stack(&rh1); 2376 init_rcu_head_on_stack(&rh2); 2377 pr_alert("%s: WARN: Duplicate call_rcu() test starting.\n", KBUILD_MODNAME); 2378 2379 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 2380 preempt_disable(); /* Prevent preemption from interrupting test. */ 2381 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 2382 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 2383 local_irq_disable(); /* Make it harder to start a new grace period. */ 2384 call_rcu(&rh2, rcu_torture_leak_cb); 2385 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 2386 local_irq_enable(); 2387 rcu_read_unlock(); 2388 preempt_enable(); 2389 2390 /* Wait for them all to get done so we can safely return. */ 2391 rcu_barrier(); 2392 pr_alert("%s: WARN: Duplicate call_rcu() test complete.\n", KBUILD_MODNAME); 2393 destroy_rcu_head_on_stack(&rh1); 2394 destroy_rcu_head_on_stack(&rh2); 2395 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 2396 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n", KBUILD_MODNAME); 2397 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 2398 } 2399 2400 static void rcutorture_sync(void) 2401 { 2402 static unsigned long n; 2403 2404 if (cur_ops->sync && !(++n & 0xfff)) 2405 cur_ops->sync(); 2406 } 2407 2408 static int __init 2409 rcu_torture_init(void) 2410 { 2411 long i; 2412 int cpu; 2413 int firsterr = 0; 2414 static struct rcu_torture_ops *torture_ops[] = { 2415 &rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 2416 &busted_srcud_ops, &tasks_ops, &trivial_ops, 2417 }; 2418 2419 if (!torture_init_begin(torture_type, verbose)) 2420 return -EBUSY; 2421 2422 /* Process args and tell the world that the torturer is on the job. */ 2423 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 2424 cur_ops = torture_ops[i]; 2425 if (strcmp(torture_type, cur_ops->name) == 0) 2426 break; 2427 } 2428 if (i == ARRAY_SIZE(torture_ops)) { 2429 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 2430 torture_type); 2431 pr_alert("rcu-torture types:"); 2432 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 2433 pr_cont(" %s", torture_ops[i]->name); 2434 pr_cont("\n"); 2435 WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST)); 2436 firsterr = -EINVAL; 2437 cur_ops = NULL; 2438 goto unwind; 2439 } 2440 if (cur_ops->fqs == NULL && fqs_duration != 0) { 2441 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 2442 fqs_duration = 0; 2443 } 2444 if (cur_ops->init) 2445 cur_ops->init(); 2446 2447 if (nreaders >= 0) { 2448 nrealreaders = nreaders; 2449 } else { 2450 nrealreaders = num_online_cpus() - 2 - nreaders; 2451 if (nrealreaders <= 0) 2452 nrealreaders = 1; 2453 } 2454 rcu_torture_print_module_parms(cur_ops, "Start of test"); 2455 2456 /* Set up the freelist. */ 2457 2458 INIT_LIST_HEAD(&rcu_torture_freelist); 2459 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 2460 rcu_tortures[i].rtort_mbtest = 0; 2461 list_add_tail(&rcu_tortures[i].rtort_free, 2462 &rcu_torture_freelist); 2463 } 2464 2465 /* Initialize the statistics so that each run gets its own numbers. */ 2466 2467 rcu_torture_current = NULL; 2468 rcu_torture_current_version = 0; 2469 atomic_set(&n_rcu_torture_alloc, 0); 2470 atomic_set(&n_rcu_torture_alloc_fail, 0); 2471 atomic_set(&n_rcu_torture_free, 0); 2472 atomic_set(&n_rcu_torture_mberror, 0); 2473 atomic_set(&n_rcu_torture_error, 0); 2474 n_rcu_torture_barrier_error = 0; 2475 n_rcu_torture_boost_ktrerror = 0; 2476 n_rcu_torture_boost_rterror = 0; 2477 n_rcu_torture_boost_failure = 0; 2478 n_rcu_torture_boosts = 0; 2479 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2480 atomic_set(&rcu_torture_wcount[i], 0); 2481 for_each_possible_cpu(cpu) { 2482 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2483 per_cpu(rcu_torture_count, cpu)[i] = 0; 2484 per_cpu(rcu_torture_batch, cpu)[i] = 0; 2485 } 2486 } 2487 err_segs_recorded = 0; 2488 rt_read_nsegs = 0; 2489 2490 /* Start up the kthreads. */ 2491 2492 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 2493 writer_task); 2494 if (firsterr) 2495 goto unwind; 2496 if (nfakewriters > 0) { 2497 fakewriter_tasks = kcalloc(nfakewriters, 2498 sizeof(fakewriter_tasks[0]), 2499 GFP_KERNEL); 2500 if (fakewriter_tasks == NULL) { 2501 VERBOSE_TOROUT_ERRSTRING("out of memory"); 2502 firsterr = -ENOMEM; 2503 goto unwind; 2504 } 2505 } 2506 for (i = 0; i < nfakewriters; i++) { 2507 firsterr = torture_create_kthread(rcu_torture_fakewriter, 2508 NULL, fakewriter_tasks[i]); 2509 if (firsterr) 2510 goto unwind; 2511 } 2512 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 2513 GFP_KERNEL); 2514 if (reader_tasks == NULL) { 2515 VERBOSE_TOROUT_ERRSTRING("out of memory"); 2516 firsterr = -ENOMEM; 2517 goto unwind; 2518 } 2519 for (i = 0; i < nrealreaders; i++) { 2520 firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, 2521 reader_tasks[i]); 2522 if (firsterr) 2523 goto unwind; 2524 } 2525 if (stat_interval > 0) { 2526 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 2527 stats_task); 2528 if (firsterr) 2529 goto unwind; 2530 } 2531 if (test_no_idle_hz && shuffle_interval > 0) { 2532 firsterr = torture_shuffle_init(shuffle_interval * HZ); 2533 if (firsterr) 2534 goto unwind; 2535 } 2536 if (stutter < 0) 2537 stutter = 0; 2538 if (stutter) { 2539 int t; 2540 2541 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; 2542 firsterr = torture_stutter_init(stutter * HZ, t); 2543 if (firsterr) 2544 goto unwind; 2545 } 2546 if (fqs_duration < 0) 2547 fqs_duration = 0; 2548 if (fqs_duration) { 2549 /* Create the fqs thread */ 2550 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 2551 fqs_task); 2552 if (firsterr) 2553 goto unwind; 2554 } 2555 if (test_boost_interval < 1) 2556 test_boost_interval = 1; 2557 if (test_boost_duration < 2) 2558 test_boost_duration = 2; 2559 if (rcu_torture_can_boost()) { 2560 2561 boost_starttime = jiffies + test_boost_interval * HZ; 2562 2563 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 2564 rcutorture_booster_init, 2565 rcutorture_booster_cleanup); 2566 if (firsterr < 0) 2567 goto unwind; 2568 rcutor_hp = firsterr; 2569 } 2570 shutdown_jiffies = jiffies + shutdown_secs * HZ; 2571 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 2572 if (firsterr) 2573 goto unwind; 2574 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, 2575 rcutorture_sync); 2576 if (firsterr) 2577 goto unwind; 2578 firsterr = rcu_torture_stall_init(); 2579 if (firsterr) 2580 goto unwind; 2581 firsterr = rcu_torture_fwd_prog_init(); 2582 if (firsterr) 2583 goto unwind; 2584 firsterr = rcu_torture_barrier_init(); 2585 if (firsterr) 2586 goto unwind; 2587 if (object_debug) 2588 rcu_test_debug_objects(); 2589 torture_init_end(); 2590 return 0; 2591 2592 unwind: 2593 torture_init_end(); 2594 rcu_torture_cleanup(); 2595 return firsterr; 2596 } 2597 2598 module_init(rcu_torture_init); 2599 module_exit(rcu_torture_cleanup); 2600