1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Module-based torture test facility for locking 4 * 5 * Copyright (C) IBM Corporation, 2014 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Davidlohr Bueso <dave@stgolabs.net> 9 * Based on kernel/rcu/torture.c. 10 */ 11 12 #define pr_fmt(fmt) fmt 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/kthread.h> 17 #include <linux/sched/rt.h> 18 #include <linux/spinlock.h> 19 #include <linux/mutex.h> 20 #include <linux/rwsem.h> 21 #include <linux/smp.h> 22 #include <linux/interrupt.h> 23 #include <linux/sched.h> 24 #include <uapi/linux/sched/types.h> 25 #include <linux/rtmutex.h> 26 #include <linux/atomic.h> 27 #include <linux/moduleparam.h> 28 #include <linux/delay.h> 29 #include <linux/slab.h> 30 #include <linux/torture.h> 31 #include <linux/reboot.h> 32 33 MODULE_LICENSE("GPL"); 34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 35 36 torture_param(int, nwriters_stress, -1, "Number of write-locking stress-test threads"); 37 torture_param(int, nreaders_stress, -1, "Number of read-locking stress-test threads"); 38 torture_param(int, long_hold, 100, "Do occasional long hold of lock (ms), 0=disable"); 39 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 40 torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (s), 0=disable"); 41 torture_param(int, shuffle_interval, 3, "Number of jiffies between shuffles, 0=disable"); 42 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable."); 43 torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); 44 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); 45 torture_param(int, rt_boost, 2, 46 "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types."); 47 torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens."); 48 torture_param(int, writer_fifo, 0, "Run writers at sched_set_fifo() priority"); 49 torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); 50 torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)"); 51 /* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */ 52 #define MAX_NESTED_LOCKS 8 53 54 static char *torture_type = IS_ENABLED(CONFIG_PREEMPT_RT) ? "raw_spin_lock" : "spin_lock"; 55 module_param(torture_type, charp, 0444); 56 MODULE_PARM_DESC(torture_type, 57 "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)"); 58 59 static struct task_struct *stats_task; 60 static struct task_struct **writer_tasks; 61 static struct task_struct **reader_tasks; 62 63 static bool lock_is_write_held; 64 static atomic_t lock_is_read_held; 65 static unsigned long last_lock_release; 66 67 struct lock_stress_stats { 68 long n_lock_fail; 69 long n_lock_acquired; 70 }; 71 72 /* Forward reference. */ 73 static void lock_torture_cleanup(void); 74 75 /* 76 * Operations vector for selecting different types of tests. 77 */ 78 struct lock_torture_ops { 79 void (*init)(void); 80 void (*exit)(void); 81 int (*nested_lock)(int tid, u32 lockset); 82 int (*writelock)(int tid); 83 void (*write_delay)(struct torture_random_state *trsp); 84 void (*task_boost)(struct torture_random_state *trsp); 85 void (*writeunlock)(int tid); 86 void (*nested_unlock)(int tid, u32 lockset); 87 int (*readlock)(int tid); 88 void (*read_delay)(struct torture_random_state *trsp); 89 void (*readunlock)(int tid); 90 91 unsigned long flags; /* for irq spinlocks */ 92 const char *name; 93 }; 94 95 struct lock_torture_cxt { 96 int nrealwriters_stress; 97 int nrealreaders_stress; 98 bool debug_lock; 99 bool init_called; 100 atomic_t n_lock_torture_errors; 101 struct lock_torture_ops *cur_ops; 102 struct lock_stress_stats *lwsa; /* writer statistics */ 103 struct lock_stress_stats *lrsa; /* reader statistics */ 104 }; 105 static struct lock_torture_cxt cxt = { 0, 0, false, false, 106 ATOMIC_INIT(0), 107 NULL, NULL}; 108 /* 109 * Definitions for lock torture testing. 110 */ 111 112 static int torture_lock_busted_write_lock(int tid __maybe_unused) 113 { 114 return 0; /* BUGGY, do not use in real life!!! */ 115 } 116 117 static void torture_lock_busted_write_delay(struct torture_random_state *trsp) 118 { 119 const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX; 120 121 /* We want a long delay occasionally to force massive contention. */ 122 if (!(torture_random(trsp) % 123 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 124 mdelay(longdelay_ms); 125 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 126 torture_preempt_schedule(); /* Allow test to be preempted. */ 127 } 128 129 static void torture_lock_busted_write_unlock(int tid __maybe_unused) 130 { 131 /* BUGGY, do not use in real life!!! */ 132 } 133 134 static void __torture_rt_boost(struct torture_random_state *trsp) 135 { 136 const unsigned int factor = rt_boost_factor; 137 138 if (!rt_task(current)) { 139 /* 140 * Boost priority once every rt_boost_factor operations. When 141 * the task tries to take the lock, the rtmutex it will account 142 * for the new priority, and do any corresponding pi-dance. 143 */ 144 if (trsp && !(torture_random(trsp) % 145 (cxt.nrealwriters_stress * factor))) { 146 sched_set_fifo(current); 147 } else /* common case, do nothing */ 148 return; 149 } else { 150 /* 151 * The task will remain boosted for another 10 * rt_boost_factor 152 * operations, then restored back to its original prio, and so 153 * forth. 154 * 155 * When @trsp is nil, we want to force-reset the task for 156 * stopping the kthread. 157 */ 158 if (!trsp || !(torture_random(trsp) % 159 (cxt.nrealwriters_stress * factor * 2))) { 160 sched_set_normal(current, 0); 161 } else /* common case, do nothing */ 162 return; 163 } 164 } 165 166 static void torture_rt_boost(struct torture_random_state *trsp) 167 { 168 if (rt_boost != 2) 169 return; 170 171 __torture_rt_boost(trsp); 172 } 173 174 static struct lock_torture_ops lock_busted_ops = { 175 .writelock = torture_lock_busted_write_lock, 176 .write_delay = torture_lock_busted_write_delay, 177 .task_boost = torture_rt_boost, 178 .writeunlock = torture_lock_busted_write_unlock, 179 .readlock = NULL, 180 .read_delay = NULL, 181 .readunlock = NULL, 182 .name = "lock_busted" 183 }; 184 185 static DEFINE_SPINLOCK(torture_spinlock); 186 187 static int torture_spin_lock_write_lock(int tid __maybe_unused) 188 __acquires(torture_spinlock) 189 { 190 spin_lock(&torture_spinlock); 191 return 0; 192 } 193 194 static void torture_spin_lock_write_delay(struct torture_random_state *trsp) 195 { 196 const unsigned long shortdelay_us = 2; 197 const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX; 198 unsigned long j; 199 200 /* We want a short delay mostly to emulate likely code, and 201 * we want a long delay occasionally to force massive contention. 202 */ 203 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * longdelay_ms))) { 204 j = jiffies; 205 mdelay(longdelay_ms); 206 pr_alert("%s: delay = %lu jiffies.\n", __func__, jiffies - j); 207 } 208 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 200 * shortdelay_us))) 209 udelay(shortdelay_us); 210 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 211 torture_preempt_schedule(); /* Allow test to be preempted. */ 212 } 213 214 static void torture_spin_lock_write_unlock(int tid __maybe_unused) 215 __releases(torture_spinlock) 216 { 217 spin_unlock(&torture_spinlock); 218 } 219 220 static struct lock_torture_ops spin_lock_ops = { 221 .writelock = torture_spin_lock_write_lock, 222 .write_delay = torture_spin_lock_write_delay, 223 .task_boost = torture_rt_boost, 224 .writeunlock = torture_spin_lock_write_unlock, 225 .readlock = NULL, 226 .read_delay = NULL, 227 .readunlock = NULL, 228 .name = "spin_lock" 229 }; 230 231 static int torture_spin_lock_write_lock_irq(int tid __maybe_unused) 232 __acquires(torture_spinlock) 233 { 234 unsigned long flags; 235 236 spin_lock_irqsave(&torture_spinlock, flags); 237 cxt.cur_ops->flags = flags; 238 return 0; 239 } 240 241 static void torture_lock_spin_write_unlock_irq(int tid __maybe_unused) 242 __releases(torture_spinlock) 243 { 244 spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags); 245 } 246 247 static struct lock_torture_ops spin_lock_irq_ops = { 248 .writelock = torture_spin_lock_write_lock_irq, 249 .write_delay = torture_spin_lock_write_delay, 250 .task_boost = torture_rt_boost, 251 .writeunlock = torture_lock_spin_write_unlock_irq, 252 .readlock = NULL, 253 .read_delay = NULL, 254 .readunlock = NULL, 255 .name = "spin_lock_irq" 256 }; 257 258 static DEFINE_RAW_SPINLOCK(torture_raw_spinlock); 259 260 static int torture_raw_spin_lock_write_lock(int tid __maybe_unused) 261 __acquires(torture_raw_spinlock) 262 { 263 raw_spin_lock(&torture_raw_spinlock); 264 return 0; 265 } 266 267 static void torture_raw_spin_lock_write_unlock(int tid __maybe_unused) 268 __releases(torture_raw_spinlock) 269 { 270 raw_spin_unlock(&torture_raw_spinlock); 271 } 272 273 static struct lock_torture_ops raw_spin_lock_ops = { 274 .writelock = torture_raw_spin_lock_write_lock, 275 .write_delay = torture_spin_lock_write_delay, 276 .task_boost = torture_rt_boost, 277 .writeunlock = torture_raw_spin_lock_write_unlock, 278 .readlock = NULL, 279 .read_delay = NULL, 280 .readunlock = NULL, 281 .name = "raw_spin_lock" 282 }; 283 284 static int torture_raw_spin_lock_write_lock_irq(int tid __maybe_unused) 285 __acquires(torture_raw_spinlock) 286 { 287 unsigned long flags; 288 289 raw_spin_lock_irqsave(&torture_raw_spinlock, flags); 290 cxt.cur_ops->flags = flags; 291 return 0; 292 } 293 294 static void torture_raw_spin_lock_write_unlock_irq(int tid __maybe_unused) 295 __releases(torture_raw_spinlock) 296 { 297 raw_spin_unlock_irqrestore(&torture_raw_spinlock, cxt.cur_ops->flags); 298 } 299 300 static struct lock_torture_ops raw_spin_lock_irq_ops = { 301 .writelock = torture_raw_spin_lock_write_lock_irq, 302 .write_delay = torture_spin_lock_write_delay, 303 .task_boost = torture_rt_boost, 304 .writeunlock = torture_raw_spin_lock_write_unlock_irq, 305 .readlock = NULL, 306 .read_delay = NULL, 307 .readunlock = NULL, 308 .name = "raw_spin_lock_irq" 309 }; 310 311 static DEFINE_RWLOCK(torture_rwlock); 312 313 static int torture_rwlock_write_lock(int tid __maybe_unused) 314 __acquires(torture_rwlock) 315 { 316 write_lock(&torture_rwlock); 317 return 0; 318 } 319 320 static void torture_rwlock_write_delay(struct torture_random_state *trsp) 321 { 322 const unsigned long shortdelay_us = 2; 323 const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX; 324 325 /* We want a short delay mostly to emulate likely code, and 326 * we want a long delay occasionally to force massive contention. 327 */ 328 if (!(torture_random(trsp) % 329 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 330 mdelay(longdelay_ms); 331 else 332 udelay(shortdelay_us); 333 } 334 335 static void torture_rwlock_write_unlock(int tid __maybe_unused) 336 __releases(torture_rwlock) 337 { 338 write_unlock(&torture_rwlock); 339 } 340 341 static int torture_rwlock_read_lock(int tid __maybe_unused) 342 __acquires(torture_rwlock) 343 { 344 read_lock(&torture_rwlock); 345 return 0; 346 } 347 348 static void torture_rwlock_read_delay(struct torture_random_state *trsp) 349 { 350 const unsigned long shortdelay_us = 10; 351 const unsigned long longdelay_ms = 100; 352 353 /* We want a short delay mostly to emulate likely code, and 354 * we want a long delay occasionally to force massive contention. 355 */ 356 if (!(torture_random(trsp) % 357 (cxt.nrealreaders_stress * 2000 * longdelay_ms))) 358 mdelay(longdelay_ms); 359 else 360 udelay(shortdelay_us); 361 } 362 363 static void torture_rwlock_read_unlock(int tid __maybe_unused) 364 __releases(torture_rwlock) 365 { 366 read_unlock(&torture_rwlock); 367 } 368 369 static struct lock_torture_ops rw_lock_ops = { 370 .writelock = torture_rwlock_write_lock, 371 .write_delay = torture_rwlock_write_delay, 372 .task_boost = torture_rt_boost, 373 .writeunlock = torture_rwlock_write_unlock, 374 .readlock = torture_rwlock_read_lock, 375 .read_delay = torture_rwlock_read_delay, 376 .readunlock = torture_rwlock_read_unlock, 377 .name = "rw_lock" 378 }; 379 380 static int torture_rwlock_write_lock_irq(int tid __maybe_unused) 381 __acquires(torture_rwlock) 382 { 383 unsigned long flags; 384 385 write_lock_irqsave(&torture_rwlock, flags); 386 cxt.cur_ops->flags = flags; 387 return 0; 388 } 389 390 static void torture_rwlock_write_unlock_irq(int tid __maybe_unused) 391 __releases(torture_rwlock) 392 { 393 write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); 394 } 395 396 static int torture_rwlock_read_lock_irq(int tid __maybe_unused) 397 __acquires(torture_rwlock) 398 { 399 unsigned long flags; 400 401 read_lock_irqsave(&torture_rwlock, flags); 402 cxt.cur_ops->flags = flags; 403 return 0; 404 } 405 406 static void torture_rwlock_read_unlock_irq(int tid __maybe_unused) 407 __releases(torture_rwlock) 408 { 409 read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); 410 } 411 412 static struct lock_torture_ops rw_lock_irq_ops = { 413 .writelock = torture_rwlock_write_lock_irq, 414 .write_delay = torture_rwlock_write_delay, 415 .task_boost = torture_rt_boost, 416 .writeunlock = torture_rwlock_write_unlock_irq, 417 .readlock = torture_rwlock_read_lock_irq, 418 .read_delay = torture_rwlock_read_delay, 419 .readunlock = torture_rwlock_read_unlock_irq, 420 .name = "rw_lock_irq" 421 }; 422 423 static DEFINE_MUTEX(torture_mutex); 424 static struct mutex torture_nested_mutexes[MAX_NESTED_LOCKS]; 425 static struct lock_class_key nested_mutex_keys[MAX_NESTED_LOCKS]; 426 427 static void torture_mutex_init(void) 428 { 429 int i; 430 431 for (i = 0; i < MAX_NESTED_LOCKS; i++) 432 __mutex_init(&torture_nested_mutexes[i], __func__, 433 &nested_mutex_keys[i]); 434 } 435 436 static int torture_mutex_nested_lock(int tid __maybe_unused, 437 u32 lockset) 438 { 439 int i; 440 441 for (i = 0; i < nested_locks; i++) 442 if (lockset & (1 << i)) 443 mutex_lock(&torture_nested_mutexes[i]); 444 return 0; 445 } 446 447 static int torture_mutex_lock(int tid __maybe_unused) 448 __acquires(torture_mutex) 449 { 450 mutex_lock(&torture_mutex); 451 return 0; 452 } 453 454 static void torture_mutex_delay(struct torture_random_state *trsp) 455 { 456 const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX; 457 458 /* We want a long delay occasionally to force massive contention. */ 459 if (!(torture_random(trsp) % 460 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 461 mdelay(longdelay_ms * 5); 462 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 463 torture_preempt_schedule(); /* Allow test to be preempted. */ 464 } 465 466 static void torture_mutex_unlock(int tid __maybe_unused) 467 __releases(torture_mutex) 468 { 469 mutex_unlock(&torture_mutex); 470 } 471 472 static void torture_mutex_nested_unlock(int tid __maybe_unused, 473 u32 lockset) 474 { 475 int i; 476 477 for (i = nested_locks - 1; i >= 0; i--) 478 if (lockset & (1 << i)) 479 mutex_unlock(&torture_nested_mutexes[i]); 480 } 481 482 static struct lock_torture_ops mutex_lock_ops = { 483 .init = torture_mutex_init, 484 .nested_lock = torture_mutex_nested_lock, 485 .writelock = torture_mutex_lock, 486 .write_delay = torture_mutex_delay, 487 .task_boost = torture_rt_boost, 488 .writeunlock = torture_mutex_unlock, 489 .nested_unlock = torture_mutex_nested_unlock, 490 .readlock = NULL, 491 .read_delay = NULL, 492 .readunlock = NULL, 493 .name = "mutex_lock" 494 }; 495 496 #include <linux/ww_mutex.h> 497 /* 498 * The torture ww_mutexes should belong to the same lock class as 499 * torture_ww_class to avoid lockdep problem. The ww_mutex_init() 500 * function is called for initialization to ensure that. 501 */ 502 static DEFINE_WD_CLASS(torture_ww_class); 503 static struct ww_mutex torture_ww_mutex_0, torture_ww_mutex_1, torture_ww_mutex_2; 504 static struct ww_acquire_ctx *ww_acquire_ctxs; 505 506 static void torture_ww_mutex_init(void) 507 { 508 ww_mutex_init(&torture_ww_mutex_0, &torture_ww_class); 509 ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class); 510 ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class); 511 512 ww_acquire_ctxs = kmalloc_array(cxt.nrealwriters_stress, 513 sizeof(*ww_acquire_ctxs), 514 GFP_KERNEL); 515 if (!ww_acquire_ctxs) 516 VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory"); 517 } 518 519 static void torture_ww_mutex_exit(void) 520 { 521 kfree(ww_acquire_ctxs); 522 } 523 524 static int torture_ww_mutex_lock(int tid) 525 __acquires(torture_ww_mutex_0) 526 __acquires(torture_ww_mutex_1) 527 __acquires(torture_ww_mutex_2) 528 { 529 LIST_HEAD(list); 530 struct reorder_lock { 531 struct list_head link; 532 struct ww_mutex *lock; 533 } locks[3], *ll, *ln; 534 struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; 535 536 locks[0].lock = &torture_ww_mutex_0; 537 list_add(&locks[0].link, &list); 538 539 locks[1].lock = &torture_ww_mutex_1; 540 list_add(&locks[1].link, &list); 541 542 locks[2].lock = &torture_ww_mutex_2; 543 list_add(&locks[2].link, &list); 544 545 ww_acquire_init(ctx, &torture_ww_class); 546 547 list_for_each_entry(ll, &list, link) { 548 int err; 549 550 err = ww_mutex_lock(ll->lock, ctx); 551 if (!err) 552 continue; 553 554 ln = ll; 555 list_for_each_entry_continue_reverse(ln, &list, link) 556 ww_mutex_unlock(ln->lock); 557 558 if (err != -EDEADLK) 559 return err; 560 561 ww_mutex_lock_slow(ll->lock, ctx); 562 list_move(&ll->link, &list); 563 } 564 565 return 0; 566 } 567 568 static void torture_ww_mutex_unlock(int tid) 569 __releases(torture_ww_mutex_0) 570 __releases(torture_ww_mutex_1) 571 __releases(torture_ww_mutex_2) 572 { 573 struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; 574 575 ww_mutex_unlock(&torture_ww_mutex_0); 576 ww_mutex_unlock(&torture_ww_mutex_1); 577 ww_mutex_unlock(&torture_ww_mutex_2); 578 ww_acquire_fini(ctx); 579 } 580 581 static struct lock_torture_ops ww_mutex_lock_ops = { 582 .init = torture_ww_mutex_init, 583 .exit = torture_ww_mutex_exit, 584 .writelock = torture_ww_mutex_lock, 585 .write_delay = torture_mutex_delay, 586 .task_boost = torture_rt_boost, 587 .writeunlock = torture_ww_mutex_unlock, 588 .readlock = NULL, 589 .read_delay = NULL, 590 .readunlock = NULL, 591 .name = "ww_mutex_lock" 592 }; 593 594 #ifdef CONFIG_RT_MUTEXES 595 static DEFINE_RT_MUTEX(torture_rtmutex); 596 static struct rt_mutex torture_nested_rtmutexes[MAX_NESTED_LOCKS]; 597 static struct lock_class_key nested_rtmutex_keys[MAX_NESTED_LOCKS]; 598 599 static void torture_rtmutex_init(void) 600 { 601 int i; 602 603 for (i = 0; i < MAX_NESTED_LOCKS; i++) 604 __rt_mutex_init(&torture_nested_rtmutexes[i], __func__, 605 &nested_rtmutex_keys[i]); 606 } 607 608 static int torture_rtmutex_nested_lock(int tid __maybe_unused, 609 u32 lockset) 610 { 611 int i; 612 613 for (i = 0; i < nested_locks; i++) 614 if (lockset & (1 << i)) 615 rt_mutex_lock(&torture_nested_rtmutexes[i]); 616 return 0; 617 } 618 619 static int torture_rtmutex_lock(int tid __maybe_unused) 620 __acquires(torture_rtmutex) 621 { 622 rt_mutex_lock(&torture_rtmutex); 623 return 0; 624 } 625 626 static void torture_rtmutex_delay(struct torture_random_state *trsp) 627 { 628 const unsigned long shortdelay_us = 2; 629 const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX; 630 631 /* 632 * We want a short delay mostly to emulate likely code, and 633 * we want a long delay occasionally to force massive contention. 634 */ 635 if (!(torture_random(trsp) % 636 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 637 mdelay(longdelay_ms); 638 if (!(torture_random(trsp) % 639 (cxt.nrealwriters_stress * 200 * shortdelay_us))) 640 udelay(shortdelay_us); 641 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 642 torture_preempt_schedule(); /* Allow test to be preempted. */ 643 } 644 645 static void torture_rtmutex_unlock(int tid __maybe_unused) 646 __releases(torture_rtmutex) 647 { 648 rt_mutex_unlock(&torture_rtmutex); 649 } 650 651 static void torture_rt_boost_rtmutex(struct torture_random_state *trsp) 652 { 653 if (!rt_boost) 654 return; 655 656 __torture_rt_boost(trsp); 657 } 658 659 static void torture_rtmutex_nested_unlock(int tid __maybe_unused, 660 u32 lockset) 661 { 662 int i; 663 664 for (i = nested_locks - 1; i >= 0; i--) 665 if (lockset & (1 << i)) 666 rt_mutex_unlock(&torture_nested_rtmutexes[i]); 667 } 668 669 static struct lock_torture_ops rtmutex_lock_ops = { 670 .init = torture_rtmutex_init, 671 .nested_lock = torture_rtmutex_nested_lock, 672 .writelock = torture_rtmutex_lock, 673 .write_delay = torture_rtmutex_delay, 674 .task_boost = torture_rt_boost_rtmutex, 675 .writeunlock = torture_rtmutex_unlock, 676 .nested_unlock = torture_rtmutex_nested_unlock, 677 .readlock = NULL, 678 .read_delay = NULL, 679 .readunlock = NULL, 680 .name = "rtmutex_lock" 681 }; 682 #endif 683 684 static DECLARE_RWSEM(torture_rwsem); 685 static int torture_rwsem_down_write(int tid __maybe_unused) 686 __acquires(torture_rwsem) 687 { 688 down_write(&torture_rwsem); 689 return 0; 690 } 691 692 static void torture_rwsem_write_delay(struct torture_random_state *trsp) 693 { 694 const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX; 695 696 /* We want a long delay occasionally to force massive contention. */ 697 if (!(torture_random(trsp) % 698 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 699 mdelay(longdelay_ms * 10); 700 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 701 torture_preempt_schedule(); /* Allow test to be preempted. */ 702 } 703 704 static void torture_rwsem_up_write(int tid __maybe_unused) 705 __releases(torture_rwsem) 706 { 707 up_write(&torture_rwsem); 708 } 709 710 static int torture_rwsem_down_read(int tid __maybe_unused) 711 __acquires(torture_rwsem) 712 { 713 down_read(&torture_rwsem); 714 return 0; 715 } 716 717 static void torture_rwsem_read_delay(struct torture_random_state *trsp) 718 { 719 const unsigned long longdelay_ms = 100; 720 721 /* We want a long delay occasionally to force massive contention. */ 722 if (!(torture_random(trsp) % 723 (cxt.nrealreaders_stress * 2000 * longdelay_ms))) 724 mdelay(longdelay_ms * 2); 725 else 726 mdelay(longdelay_ms / 2); 727 if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000))) 728 torture_preempt_schedule(); /* Allow test to be preempted. */ 729 } 730 731 static void torture_rwsem_up_read(int tid __maybe_unused) 732 __releases(torture_rwsem) 733 { 734 up_read(&torture_rwsem); 735 } 736 737 static struct lock_torture_ops rwsem_lock_ops = { 738 .writelock = torture_rwsem_down_write, 739 .write_delay = torture_rwsem_write_delay, 740 .task_boost = torture_rt_boost, 741 .writeunlock = torture_rwsem_up_write, 742 .readlock = torture_rwsem_down_read, 743 .read_delay = torture_rwsem_read_delay, 744 .readunlock = torture_rwsem_up_read, 745 .name = "rwsem_lock" 746 }; 747 748 #include <linux/percpu-rwsem.h> 749 static struct percpu_rw_semaphore pcpu_rwsem; 750 751 static void torture_percpu_rwsem_init(void) 752 { 753 BUG_ON(percpu_init_rwsem(&pcpu_rwsem)); 754 } 755 756 static void torture_percpu_rwsem_exit(void) 757 { 758 percpu_free_rwsem(&pcpu_rwsem); 759 } 760 761 static int torture_percpu_rwsem_down_write(int tid __maybe_unused) 762 __acquires(pcpu_rwsem) 763 { 764 percpu_down_write(&pcpu_rwsem); 765 return 0; 766 } 767 768 static void torture_percpu_rwsem_up_write(int tid __maybe_unused) 769 __releases(pcpu_rwsem) 770 { 771 percpu_up_write(&pcpu_rwsem); 772 } 773 774 static int torture_percpu_rwsem_down_read(int tid __maybe_unused) 775 __acquires(pcpu_rwsem) 776 { 777 percpu_down_read(&pcpu_rwsem); 778 return 0; 779 } 780 781 static void torture_percpu_rwsem_up_read(int tid __maybe_unused) 782 __releases(pcpu_rwsem) 783 { 784 percpu_up_read(&pcpu_rwsem); 785 } 786 787 static struct lock_torture_ops percpu_rwsem_lock_ops = { 788 .init = torture_percpu_rwsem_init, 789 .exit = torture_percpu_rwsem_exit, 790 .writelock = torture_percpu_rwsem_down_write, 791 .write_delay = torture_rwsem_write_delay, 792 .task_boost = torture_rt_boost, 793 .writeunlock = torture_percpu_rwsem_up_write, 794 .readlock = torture_percpu_rwsem_down_read, 795 .read_delay = torture_rwsem_read_delay, 796 .readunlock = torture_percpu_rwsem_up_read, 797 .name = "percpu_rwsem_lock" 798 }; 799 800 /* 801 * Lock torture writer kthread. Repeatedly acquires and releases 802 * the lock, checking for duplicate acquisitions. 803 */ 804 static int lock_torture_writer(void *arg) 805 { 806 struct lock_stress_stats *lwsp = arg; 807 int tid = lwsp - cxt.lwsa; 808 DEFINE_TORTURE_RANDOM(rand); 809 u32 lockset_mask; 810 bool skip_main_lock; 811 812 VERBOSE_TOROUT_STRING("lock_torture_writer task started"); 813 if (!rt_task(current)) 814 set_user_nice(current, MAX_NICE); 815 816 do { 817 if ((torture_random(&rand) & 0xfffff) == 0) 818 schedule_timeout_uninterruptible(1); 819 820 lockset_mask = torture_random(&rand); 821 /* 822 * When using nested_locks, we want to occasionally 823 * skip the main lock so we can avoid always serializing 824 * the lock chains on that central lock. By skipping the 825 * main lock occasionally, we can create different 826 * contention patterns (allowing for multiple disjoint 827 * blocked trees) 828 */ 829 skip_main_lock = (nested_locks && 830 !(torture_random(&rand) % 100)); 831 832 cxt.cur_ops->task_boost(&rand); 833 if (cxt.cur_ops->nested_lock) 834 cxt.cur_ops->nested_lock(tid, lockset_mask); 835 836 if (!skip_main_lock) { 837 cxt.cur_ops->writelock(tid); 838 if (WARN_ON_ONCE(lock_is_write_held)) 839 lwsp->n_lock_fail++; 840 lock_is_write_held = true; 841 if (WARN_ON_ONCE(atomic_read(&lock_is_read_held))) 842 lwsp->n_lock_fail++; /* rare, but... */ 843 844 lwsp->n_lock_acquired++; 845 } 846 if (!skip_main_lock) { 847 cxt.cur_ops->write_delay(&rand); 848 lock_is_write_held = false; 849 WRITE_ONCE(last_lock_release, jiffies); 850 cxt.cur_ops->writeunlock(tid); 851 } 852 if (cxt.cur_ops->nested_unlock) 853 cxt.cur_ops->nested_unlock(tid, lockset_mask); 854 855 stutter_wait("lock_torture_writer"); 856 } while (!torture_must_stop()); 857 858 cxt.cur_ops->task_boost(NULL); /* reset prio */ 859 torture_kthread_stopping("lock_torture_writer"); 860 return 0; 861 } 862 863 /* 864 * Lock torture reader kthread. Repeatedly acquires and releases 865 * the reader lock. 866 */ 867 static int lock_torture_reader(void *arg) 868 { 869 struct lock_stress_stats *lrsp = arg; 870 int tid = lrsp - cxt.lrsa; 871 DEFINE_TORTURE_RANDOM(rand); 872 873 VERBOSE_TOROUT_STRING("lock_torture_reader task started"); 874 set_user_nice(current, MAX_NICE); 875 876 do { 877 if ((torture_random(&rand) & 0xfffff) == 0) 878 schedule_timeout_uninterruptible(1); 879 880 cxt.cur_ops->readlock(tid); 881 atomic_inc(&lock_is_read_held); 882 if (WARN_ON_ONCE(lock_is_write_held)) 883 lrsp->n_lock_fail++; /* rare, but... */ 884 885 lrsp->n_lock_acquired++; 886 cxt.cur_ops->read_delay(&rand); 887 atomic_dec(&lock_is_read_held); 888 cxt.cur_ops->readunlock(tid); 889 890 stutter_wait("lock_torture_reader"); 891 } while (!torture_must_stop()); 892 torture_kthread_stopping("lock_torture_reader"); 893 return 0; 894 } 895 896 /* 897 * Create an lock-torture-statistics message in the specified buffer. 898 */ 899 static void __torture_print_stats(char *page, 900 struct lock_stress_stats *statp, bool write) 901 { 902 long cur; 903 bool fail = false; 904 int i, n_stress; 905 long max = 0, min = statp ? data_race(statp[0].n_lock_acquired) : 0; 906 long long sum = 0; 907 908 n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress; 909 for (i = 0; i < n_stress; i++) { 910 if (data_race(statp[i].n_lock_fail)) 911 fail = true; 912 cur = data_race(statp[i].n_lock_acquired); 913 sum += cur; 914 if (max < cur) 915 max = cur; 916 if (min > cur) 917 min = cur; 918 } 919 page += sprintf(page, 920 "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n", 921 write ? "Writes" : "Reads ", 922 sum, max, min, 923 !onoff_interval && max / 2 > min ? "???" : "", 924 fail, fail ? "!!!" : ""); 925 if (fail) 926 atomic_inc(&cxt.n_lock_torture_errors); 927 } 928 929 /* 930 * Print torture statistics. Caller must ensure that there is only one 931 * call to this function at a given time!!! This is normally accomplished 932 * by relying on the module system to only have one copy of the module 933 * loaded, and then by giving the lock_torture_stats kthread full control 934 * (or the init/cleanup functions when lock_torture_stats thread is not 935 * running). 936 */ 937 static void lock_torture_stats_print(void) 938 { 939 int size = cxt.nrealwriters_stress * 200 + 8192; 940 char *buf; 941 942 if (cxt.cur_ops->readlock) 943 size += cxt.nrealreaders_stress * 200 + 8192; 944 945 buf = kmalloc(size, GFP_KERNEL); 946 if (!buf) { 947 pr_err("lock_torture_stats_print: Out of memory, need: %d", 948 size); 949 return; 950 } 951 952 __torture_print_stats(buf, cxt.lwsa, true); 953 pr_alert("%s", buf); 954 kfree(buf); 955 956 if (cxt.cur_ops->readlock) { 957 buf = kmalloc(size, GFP_KERNEL); 958 if (!buf) { 959 pr_err("lock_torture_stats_print: Out of memory, need: %d", 960 size); 961 return; 962 } 963 964 __torture_print_stats(buf, cxt.lrsa, false); 965 pr_alert("%s", buf); 966 kfree(buf); 967 } 968 } 969 970 /* 971 * Periodically prints torture statistics, if periodic statistics printing 972 * was specified via the stat_interval module parameter. 973 * 974 * No need to worry about fullstop here, since this one doesn't reference 975 * volatile state or register callbacks. 976 */ 977 static int lock_torture_stats(void *arg) 978 { 979 VERBOSE_TOROUT_STRING("lock_torture_stats task started"); 980 do { 981 schedule_timeout_interruptible(stat_interval * HZ); 982 lock_torture_stats_print(); 983 torture_shutdown_absorb("lock_torture_stats"); 984 } while (!torture_must_stop()); 985 torture_kthread_stopping("lock_torture_stats"); 986 return 0; 987 } 988 989 static inline void 990 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops, 991 const char *tag) 992 { 993 pr_alert("%s" TORTURE_FLAG 994 "--- %s%s: nwriters_stress=%d nreaders_stress=%d nested_locks=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n", 995 torture_type, tag, cxt.debug_lock ? " [debug]": "", 996 cxt.nrealwriters_stress, cxt.nrealreaders_stress, 997 nested_locks, stat_interval, verbose, shuffle_interval, 998 stutter, shutdown_secs, onoff_interval, onoff_holdoff); 999 } 1000 1001 static void lock_torture_cleanup(void) 1002 { 1003 int i; 1004 1005 if (torture_cleanup_begin()) 1006 return; 1007 1008 /* 1009 * Indicates early cleanup, meaning that the test has not run, 1010 * such as when passing bogus args when loading the module. 1011 * However cxt->cur_ops.init() may have been invoked, so beside 1012 * perform the underlying torture-specific cleanups, cur_ops.exit() 1013 * will be invoked if needed. 1014 */ 1015 if (!cxt.lwsa && !cxt.lrsa) 1016 goto end; 1017 1018 if (writer_tasks) { 1019 for (i = 0; i < cxt.nrealwriters_stress; i++) 1020 torture_stop_kthread(lock_torture_writer, writer_tasks[i]); 1021 kfree(writer_tasks); 1022 writer_tasks = NULL; 1023 } 1024 1025 if (reader_tasks) { 1026 for (i = 0; i < cxt.nrealreaders_stress; i++) 1027 torture_stop_kthread(lock_torture_reader, 1028 reader_tasks[i]); 1029 kfree(reader_tasks); 1030 reader_tasks = NULL; 1031 } 1032 1033 torture_stop_kthread(lock_torture_stats, stats_task); 1034 lock_torture_stats_print(); /* -After- the stats thread is stopped! */ 1035 1036 if (atomic_read(&cxt.n_lock_torture_errors)) 1037 lock_torture_print_module_parms(cxt.cur_ops, 1038 "End of test: FAILURE"); 1039 else if (torture_onoff_failures()) 1040 lock_torture_print_module_parms(cxt.cur_ops, 1041 "End of test: LOCK_HOTPLUG"); 1042 else 1043 lock_torture_print_module_parms(cxt.cur_ops, 1044 "End of test: SUCCESS"); 1045 1046 kfree(cxt.lwsa); 1047 cxt.lwsa = NULL; 1048 kfree(cxt.lrsa); 1049 cxt.lrsa = NULL; 1050 1051 end: 1052 if (cxt.init_called) { 1053 if (cxt.cur_ops->exit) 1054 cxt.cur_ops->exit(); 1055 cxt.init_called = false; 1056 } 1057 torture_cleanup_end(); 1058 } 1059 1060 static int __init lock_torture_init(void) 1061 { 1062 int i, j; 1063 int firsterr = 0; 1064 static struct lock_torture_ops *torture_ops[] = { 1065 &lock_busted_ops, 1066 &spin_lock_ops, &spin_lock_irq_ops, 1067 &raw_spin_lock_ops, &raw_spin_lock_irq_ops, 1068 &rw_lock_ops, &rw_lock_irq_ops, 1069 &mutex_lock_ops, 1070 &ww_mutex_lock_ops, 1071 #ifdef CONFIG_RT_MUTEXES 1072 &rtmutex_lock_ops, 1073 #endif 1074 &rwsem_lock_ops, 1075 &percpu_rwsem_lock_ops, 1076 }; 1077 1078 if (!torture_init_begin(torture_type, verbose)) 1079 return -EBUSY; 1080 1081 /* Process args and tell the world that the torturer is on the job. */ 1082 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1083 cxt.cur_ops = torture_ops[i]; 1084 if (strcmp(torture_type, cxt.cur_ops->name) == 0) 1085 break; 1086 } 1087 if (i == ARRAY_SIZE(torture_ops)) { 1088 pr_alert("lock-torture: invalid torture type: \"%s\"\n", 1089 torture_type); 1090 pr_alert("lock-torture types:"); 1091 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1092 pr_alert(" %s", torture_ops[i]->name); 1093 pr_alert("\n"); 1094 firsterr = -EINVAL; 1095 goto unwind; 1096 } 1097 1098 if (nwriters_stress == 0 && 1099 (!cxt.cur_ops->readlock || nreaders_stress == 0)) { 1100 pr_alert("lock-torture: must run at least one locking thread\n"); 1101 firsterr = -EINVAL; 1102 goto unwind; 1103 } 1104 1105 if (nwriters_stress >= 0) 1106 cxt.nrealwriters_stress = nwriters_stress; 1107 else 1108 cxt.nrealwriters_stress = 2 * num_online_cpus(); 1109 1110 if (cxt.cur_ops->init) { 1111 cxt.cur_ops->init(); 1112 cxt.init_called = true; 1113 } 1114 1115 #ifdef CONFIG_DEBUG_MUTEXES 1116 if (str_has_prefix(torture_type, "mutex")) 1117 cxt.debug_lock = true; 1118 #endif 1119 #ifdef CONFIG_DEBUG_RT_MUTEXES 1120 if (str_has_prefix(torture_type, "rtmutex")) 1121 cxt.debug_lock = true; 1122 #endif 1123 #ifdef CONFIG_DEBUG_SPINLOCK 1124 if ((str_has_prefix(torture_type, "spin")) || 1125 (str_has_prefix(torture_type, "rw_lock"))) 1126 cxt.debug_lock = true; 1127 #endif 1128 1129 /* Initialize the statistics so that each run gets its own numbers. */ 1130 if (nwriters_stress) { 1131 lock_is_write_held = false; 1132 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress, 1133 sizeof(*cxt.lwsa), 1134 GFP_KERNEL); 1135 if (cxt.lwsa == NULL) { 1136 VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory"); 1137 firsterr = -ENOMEM; 1138 goto unwind; 1139 } 1140 1141 for (i = 0; i < cxt.nrealwriters_stress; i++) { 1142 cxt.lwsa[i].n_lock_fail = 0; 1143 cxt.lwsa[i].n_lock_acquired = 0; 1144 } 1145 } 1146 1147 if (cxt.cur_ops->readlock) { 1148 if (nreaders_stress >= 0) 1149 cxt.nrealreaders_stress = nreaders_stress; 1150 else { 1151 /* 1152 * By default distribute evenly the number of 1153 * readers and writers. We still run the same number 1154 * of threads as the writer-only locks default. 1155 */ 1156 if (nwriters_stress < 0) /* user doesn't care */ 1157 cxt.nrealwriters_stress = num_online_cpus(); 1158 cxt.nrealreaders_stress = cxt.nrealwriters_stress; 1159 } 1160 1161 if (nreaders_stress) { 1162 cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress, 1163 sizeof(*cxt.lrsa), 1164 GFP_KERNEL); 1165 if (cxt.lrsa == NULL) { 1166 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory"); 1167 firsterr = -ENOMEM; 1168 kfree(cxt.lwsa); 1169 cxt.lwsa = NULL; 1170 goto unwind; 1171 } 1172 1173 for (i = 0; i < cxt.nrealreaders_stress; i++) { 1174 cxt.lrsa[i].n_lock_fail = 0; 1175 cxt.lrsa[i].n_lock_acquired = 0; 1176 } 1177 } 1178 } 1179 1180 lock_torture_print_module_parms(cxt.cur_ops, "Start of test"); 1181 1182 /* Prepare torture context. */ 1183 if (onoff_interval > 0) { 1184 firsterr = torture_onoff_init(onoff_holdoff * HZ, 1185 onoff_interval * HZ, NULL); 1186 if (torture_init_error(firsterr)) 1187 goto unwind; 1188 } 1189 if (shuffle_interval > 0) { 1190 firsterr = torture_shuffle_init(shuffle_interval); 1191 if (torture_init_error(firsterr)) 1192 goto unwind; 1193 } 1194 if (shutdown_secs > 0) { 1195 firsterr = torture_shutdown_init(shutdown_secs, 1196 lock_torture_cleanup); 1197 if (torture_init_error(firsterr)) 1198 goto unwind; 1199 } 1200 if (stutter > 0) { 1201 firsterr = torture_stutter_init(stutter, stutter); 1202 if (torture_init_error(firsterr)) 1203 goto unwind; 1204 } 1205 1206 if (nwriters_stress) { 1207 writer_tasks = kcalloc(cxt.nrealwriters_stress, 1208 sizeof(writer_tasks[0]), 1209 GFP_KERNEL); 1210 if (writer_tasks == NULL) { 1211 TOROUT_ERRSTRING("writer_tasks: Out of memory"); 1212 firsterr = -ENOMEM; 1213 goto unwind; 1214 } 1215 } 1216 1217 /* cap nested_locks to MAX_NESTED_LOCKS */ 1218 if (nested_locks > MAX_NESTED_LOCKS) 1219 nested_locks = MAX_NESTED_LOCKS; 1220 1221 if (cxt.cur_ops->readlock) { 1222 reader_tasks = kcalloc(cxt.nrealreaders_stress, 1223 sizeof(reader_tasks[0]), 1224 GFP_KERNEL); 1225 if (reader_tasks == NULL) { 1226 TOROUT_ERRSTRING("reader_tasks: Out of memory"); 1227 kfree(writer_tasks); 1228 writer_tasks = NULL; 1229 firsterr = -ENOMEM; 1230 goto unwind; 1231 } 1232 } 1233 1234 /* 1235 * Create the kthreads and start torturing (oh, those poor little locks). 1236 * 1237 * TODO: Note that we interleave writers with readers, giving writers a 1238 * slight advantage, by creating its kthread first. This can be modified 1239 * for very specific needs, or even let the user choose the policy, if 1240 * ever wanted. 1241 */ 1242 for (i = 0, j = 0; i < cxt.nrealwriters_stress || 1243 j < cxt.nrealreaders_stress; i++, j++) { 1244 if (i >= cxt.nrealwriters_stress) 1245 goto create_reader; 1246 1247 /* Create writer. */ 1248 firsterr = torture_create_kthread_cb(lock_torture_writer, &cxt.lwsa[i], 1249 writer_tasks[i], 1250 writer_fifo ? sched_set_fifo : NULL); 1251 if (torture_init_error(firsterr)) 1252 goto unwind; 1253 1254 create_reader: 1255 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress)) 1256 continue; 1257 /* Create reader. */ 1258 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j], 1259 reader_tasks[j]); 1260 if (torture_init_error(firsterr)) 1261 goto unwind; 1262 } 1263 if (stat_interval > 0) { 1264 firsterr = torture_create_kthread(lock_torture_stats, NULL, 1265 stats_task); 1266 if (torture_init_error(firsterr)) 1267 goto unwind; 1268 } 1269 torture_init_end(); 1270 return 0; 1271 1272 unwind: 1273 torture_init_end(); 1274 lock_torture_cleanup(); 1275 if (shutdown_secs) { 1276 WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST)); 1277 kernel_power_off(); 1278 } 1279 return firsterr; 1280 } 1281 1282 module_init(lock_torture_init); 1283 module_exit(lock_torture_cleanup); 1284