1 /* 2 * Common functions for in-kernel torture tests. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2014 19 * 20 * Author: Paul E. McKenney <paulmck@us.ibm.com> 21 * Based on kernel/rcu/torture.c. 22 */ 23 #include <linux/types.h> 24 #include <linux/kernel.h> 25 #include <linux/init.h> 26 #include <linux/module.h> 27 #include <linux/kthread.h> 28 #include <linux/err.h> 29 #include <linux/spinlock.h> 30 #include <linux/smp.h> 31 #include <linux/interrupt.h> 32 #include <linux/sched.h> 33 #include <linux/atomic.h> 34 #include <linux/bitops.h> 35 #include <linux/completion.h> 36 #include <linux/moduleparam.h> 37 #include <linux/percpu.h> 38 #include <linux/notifier.h> 39 #include <linux/reboot.h> 40 #include <linux/freezer.h> 41 #include <linux/cpu.h> 42 #include <linux/delay.h> 43 #include <linux/stat.h> 44 #include <linux/slab.h> 45 #include <linux/trace_clock.h> 46 #include <asm/byteorder.h> 47 #include <linux/torture.h> 48 49 MODULE_LICENSE("GPL"); 50 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>"); 51 52 static char *torture_type; 53 static bool verbose; 54 55 /* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ 56 #define FULLSTOP_DONTSTOP 0 /* Normal operation. */ 57 #define FULLSTOP_SHUTDOWN 1 /* System shutdown with torture running. */ 58 #define FULLSTOP_RMMOD 2 /* Normal rmmod of torture. */ 59 static int fullstop = FULLSTOP_RMMOD; 60 static DEFINE_MUTEX(fullstop_mutex); 61 62 #ifdef CONFIG_HOTPLUG_CPU 63 64 /* 65 * Variables for online-offline handling. Only present if CPU hotplug 66 * is enabled, otherwise does nothing. 67 */ 68 69 static struct task_struct *onoff_task; 70 static long onoff_holdoff; 71 static long onoff_interval; 72 static long n_offline_attempts; 73 static long n_offline_successes; 74 static unsigned long sum_offline; 75 static int min_offline = -1; 76 static int max_offline; 77 static long n_online_attempts; 78 static long n_online_successes; 79 static unsigned long sum_online; 80 static int min_online = -1; 81 static int max_online; 82 83 /* 84 * Execute random CPU-hotplug operations at the interval specified 85 * by the onoff_interval. 86 */ 87 static int 88 torture_onoff(void *arg) 89 { 90 int cpu; 91 unsigned long delta; 92 int maxcpu = -1; 93 DEFINE_TORTURE_RANDOM(rand); 94 int ret; 95 unsigned long starttime; 96 97 VERBOSE_TOROUT_STRING("torture_onoff task started"); 98 for_each_online_cpu(cpu) 99 maxcpu = cpu; 100 WARN_ON(maxcpu < 0); 101 if (onoff_holdoff > 0) { 102 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff"); 103 schedule_timeout_interruptible(onoff_holdoff); 104 VERBOSE_TOROUT_STRING("torture_onoff end holdoff"); 105 } 106 while (!torture_must_stop()) { 107 cpu = (torture_random(&rand) >> 4) % (maxcpu + 1); 108 if (cpu_online(cpu) && cpu_is_hotpluggable(cpu)) { 109 if (verbose) 110 pr_alert("%s" TORTURE_FLAG 111 "torture_onoff task: offlining %d\n", 112 torture_type, cpu); 113 starttime = jiffies; 114 n_offline_attempts++; 115 ret = cpu_down(cpu); 116 if (ret) { 117 if (verbose) 118 pr_alert("%s" TORTURE_FLAG 119 "torture_onoff task: offline %d failed: errno %d\n", 120 torture_type, cpu, ret); 121 } else { 122 if (verbose) 123 pr_alert("%s" TORTURE_FLAG 124 "torture_onoff task: offlined %d\n", 125 torture_type, cpu); 126 n_offline_successes++; 127 delta = jiffies - starttime; 128 sum_offline += delta; 129 if (min_offline < 0) { 130 min_offline = delta; 131 max_offline = delta; 132 } 133 if (min_offline > delta) 134 min_offline = delta; 135 if (max_offline < delta) 136 max_offline = delta; 137 } 138 } else if (cpu_is_hotpluggable(cpu)) { 139 if (verbose) 140 pr_alert("%s" TORTURE_FLAG 141 "torture_onoff task: onlining %d\n", 142 torture_type, cpu); 143 starttime = jiffies; 144 n_online_attempts++; 145 ret = cpu_up(cpu); 146 if (ret) { 147 if (verbose) 148 pr_alert("%s" TORTURE_FLAG 149 "torture_onoff task: online %d failed: errno %d\n", 150 torture_type, cpu, ret); 151 } else { 152 if (verbose) 153 pr_alert("%s" TORTURE_FLAG 154 "torture_onoff task: onlined %d\n", 155 torture_type, cpu); 156 n_online_successes++; 157 delta = jiffies - starttime; 158 sum_online += delta; 159 if (min_online < 0) { 160 min_online = delta; 161 max_online = delta; 162 } 163 if (min_online > delta) 164 min_online = delta; 165 if (max_online < delta) 166 max_online = delta; 167 } 168 } 169 schedule_timeout_interruptible(onoff_interval); 170 } 171 VERBOSE_TOROUT_STRING("torture_onoff task stopping"); 172 return 0; 173 } 174 175 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 176 177 /* 178 * Initiate online-offline handling. 179 */ 180 int torture_onoff_init(long ooholdoff, long oointerval) 181 { 182 #ifdef CONFIG_HOTPLUG_CPU 183 int ret; 184 185 onoff_holdoff = ooholdoff; 186 onoff_interval = oointerval; 187 if (onoff_interval <= 0) 188 return 0; 189 onoff_task = kthread_run(torture_onoff, NULL, "torture_onoff"); 190 if (IS_ERR(onoff_task)) { 191 ret = PTR_ERR(onoff_task); 192 onoff_task = NULL; 193 return ret; 194 } 195 torture_shuffle_task_register(onoff_task); 196 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 197 return 0; 198 } 199 EXPORT_SYMBOL_GPL(torture_onoff_init); 200 201 /* 202 * Clean up after online/offline testing. 203 */ 204 static void torture_onoff_cleanup(void) 205 { 206 #ifdef CONFIG_HOTPLUG_CPU 207 if (onoff_task == NULL) 208 return; 209 VERBOSE_TOROUT_STRING("Stopping torture_onoff task"); 210 kthread_stop(onoff_task); 211 onoff_task = NULL; 212 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 213 } 214 EXPORT_SYMBOL_GPL(torture_onoff_cleanup); 215 216 /* 217 * Print online/offline testing statistics. 218 */ 219 char *torture_onoff_stats(char *page) 220 { 221 #ifdef CONFIG_HOTPLUG_CPU 222 page += sprintf(page, 223 "onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ", 224 n_online_successes, n_online_attempts, 225 n_offline_successes, n_offline_attempts, 226 min_online, max_online, 227 min_offline, max_offline, 228 sum_online, sum_offline, HZ); 229 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 230 return page; 231 } 232 EXPORT_SYMBOL_GPL(torture_onoff_stats); 233 234 /* 235 * Were all the online/offline operations successful? 236 */ 237 bool torture_onoff_failures(void) 238 { 239 #ifdef CONFIG_HOTPLUG_CPU 240 return n_online_successes != n_online_attempts || 241 n_offline_successes != n_offline_attempts; 242 #else /* #ifdef CONFIG_HOTPLUG_CPU */ 243 return false; 244 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ 245 } 246 EXPORT_SYMBOL_GPL(torture_onoff_failures); 247 248 #define TORTURE_RANDOM_MULT 39916801 /* prime */ 249 #define TORTURE_RANDOM_ADD 479001701 /* prime */ 250 #define TORTURE_RANDOM_REFRESH 10000 251 252 /* 253 * Crude but fast random-number generator. Uses a linear congruential 254 * generator, with occasional help from cpu_clock(). 255 */ 256 unsigned long 257 torture_random(struct torture_random_state *trsp) 258 { 259 if (--trsp->trs_count < 0) { 260 trsp->trs_state += (unsigned long)local_clock(); 261 trsp->trs_count = TORTURE_RANDOM_REFRESH; 262 } 263 trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT + 264 TORTURE_RANDOM_ADD; 265 return swahw32(trsp->trs_state); 266 } 267 EXPORT_SYMBOL_GPL(torture_random); 268 269 /* 270 * Variables for shuffling. The idea is to ensure that each CPU stays 271 * idle for an extended period to test interactions with dyntick idle, 272 * as well as interactions with any per-CPU varibles. 273 */ 274 struct shuffle_task { 275 struct list_head st_l; 276 struct task_struct *st_t; 277 }; 278 279 static long shuffle_interval; /* In jiffies. */ 280 static struct task_struct *shuffler_task; 281 static cpumask_var_t shuffle_tmp_mask; 282 static int shuffle_idle_cpu; /* Force all torture tasks off this CPU */ 283 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list); 284 static DEFINE_MUTEX(shuffle_task_mutex); 285 286 /* 287 * Register a task to be shuffled. If there is no memory, just splat 288 * and don't bother registering. 289 */ 290 void torture_shuffle_task_register(struct task_struct *tp) 291 { 292 struct shuffle_task *stp; 293 294 if (WARN_ON_ONCE(tp == NULL)) 295 return; 296 stp = kmalloc(sizeof(*stp), GFP_KERNEL); 297 if (WARN_ON_ONCE(stp == NULL)) 298 return; 299 stp->st_t = tp; 300 mutex_lock(&shuffle_task_mutex); 301 list_add(&stp->st_l, &shuffle_task_list); 302 mutex_unlock(&shuffle_task_mutex); 303 } 304 EXPORT_SYMBOL_GPL(torture_shuffle_task_register); 305 306 /* 307 * Unregister all tasks, for example, at the end of the torture run. 308 */ 309 static void torture_shuffle_task_unregister_all(void) 310 { 311 struct shuffle_task *stp; 312 struct shuffle_task *p; 313 314 mutex_lock(&shuffle_task_mutex); 315 list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) { 316 list_del(&stp->st_l); 317 kfree(stp); 318 } 319 mutex_unlock(&shuffle_task_mutex); 320 } 321 322 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle. 323 * A special case is when shuffle_idle_cpu = -1, in which case we allow 324 * the tasks to run on all CPUs. 325 */ 326 static void torture_shuffle_tasks(void) 327 { 328 struct shuffle_task *stp; 329 330 cpumask_setall(shuffle_tmp_mask); 331 get_online_cpus(); 332 333 /* No point in shuffling if there is only one online CPU (ex: UP) */ 334 if (num_online_cpus() == 1) { 335 put_online_cpus(); 336 return; 337 } 338 339 /* Advance to the next CPU. Upon overflow, don't idle any CPUs. */ 340 shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask); 341 if (shuffle_idle_cpu >= nr_cpu_ids) 342 shuffle_idle_cpu = -1; 343 if (shuffle_idle_cpu != -1) { 344 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask); 345 if (cpumask_empty(shuffle_tmp_mask)) { 346 put_online_cpus(); 347 return; 348 } 349 } 350 351 mutex_lock(&shuffle_task_mutex); 352 list_for_each_entry(stp, &shuffle_task_list, st_l) 353 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask); 354 mutex_unlock(&shuffle_task_mutex); 355 356 put_online_cpus(); 357 } 358 359 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the 360 * system to become idle at a time and cut off its timer ticks. This is meant 361 * to test the support for such tickless idle CPU in RCU. 362 */ 363 static int torture_shuffle(void *arg) 364 { 365 VERBOSE_TOROUT_STRING("torture_shuffle task started"); 366 do { 367 schedule_timeout_interruptible(shuffle_interval); 368 torture_shuffle_tasks(); 369 torture_shutdown_absorb("torture_shuffle"); 370 } while (!torture_must_stop()); 371 VERBOSE_TOROUT_STRING("torture_shuffle task stopping"); 372 return 0; 373 } 374 375 /* 376 * Start the shuffler, with shuffint in jiffies. 377 */ 378 int torture_shuffle_init(long shuffint) 379 { 380 int ret; 381 382 shuffle_interval = shuffint; 383 384 shuffle_idle_cpu = -1; 385 386 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) { 387 VERBOSE_TOROUT_ERRSTRING("Failed to alloc mask"); 388 return -ENOMEM; 389 } 390 391 /* Create the shuffler thread */ 392 shuffler_task = kthread_run(torture_shuffle, NULL, "torture_shuffle"); 393 if (IS_ERR(shuffler_task)) { 394 ret = PTR_ERR(shuffler_task); 395 free_cpumask_var(shuffle_tmp_mask); 396 VERBOSE_TOROUT_ERRSTRING("Failed to create shuffler"); 397 shuffler_task = NULL; 398 return ret; 399 } 400 torture_shuffle_task_register(shuffler_task); 401 return 0; 402 } 403 EXPORT_SYMBOL_GPL(torture_shuffle_init); 404 405 /* 406 * Stop the shuffling. 407 */ 408 static void torture_shuffle_cleanup(void) 409 { 410 torture_shuffle_task_unregister_all(); 411 if (shuffler_task) { 412 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task"); 413 kthread_stop(shuffler_task); 414 free_cpumask_var(shuffle_tmp_mask); 415 } 416 shuffler_task = NULL; 417 } 418 EXPORT_SYMBOL_GPL(torture_shuffle_cleanup); 419 420 /* 421 * Absorb kthreads into a kernel function that won't return, so that 422 * they won't ever access module text or data again. 423 */ 424 void torture_shutdown_absorb(const char *title) 425 { 426 while (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { 427 pr_notice("torture thread %s parking due to system shutdown\n", 428 title); 429 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT); 430 } 431 } 432 EXPORT_SYMBOL_GPL(torture_shutdown_absorb); 433 434 /* 435 * Detect and respond to a system shutdown. 436 */ 437 static int torture_shutdown_notify(struct notifier_block *unused1, 438 unsigned long unused2, void *unused3) 439 { 440 mutex_lock(&fullstop_mutex); 441 if (fullstop == FULLSTOP_DONTSTOP) { 442 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected"); 443 fullstop = FULLSTOP_SHUTDOWN; 444 } else { 445 pr_warn("Concurrent rmmod and shutdown illegal!\n"); 446 } 447 mutex_unlock(&fullstop_mutex); 448 return NOTIFY_DONE; 449 } 450 451 static struct notifier_block torture_shutdown_nb = { 452 .notifier_call = torture_shutdown_notify, 453 }; 454 455 /* 456 * Initialize torture module. Please note that this is -not- invoked via 457 * the usual module_init() mechanism, but rather by an explicit call from 458 * the client torture module. This call must be paired with a later 459 * torture_init_end(). 460 */ 461 void __init torture_init_begin(char *ttype, bool v) 462 { 463 mutex_lock(&fullstop_mutex); 464 torture_type = ttype; 465 verbose = v; 466 fullstop = FULLSTOP_DONTSTOP; 467 468 } 469 EXPORT_SYMBOL_GPL(torture_init_begin); 470 471 /* 472 * Tell the torture module that initialization is complete. 473 */ 474 void __init torture_init_end(void) 475 { 476 mutex_unlock(&fullstop_mutex); 477 register_reboot_notifier(&torture_shutdown_nb); 478 } 479 EXPORT_SYMBOL_GPL(torture_init_end); 480 481 /* 482 * Clean up torture module. Please note that this is -not- invoked via 483 * the usual module_exit() mechanism, but rather by an explicit call from 484 * the client torture module. Returns true if a race with system shutdown 485 * is detected. 486 * 487 * This must be called before the caller starts shutting down its own 488 * kthreads. 489 */ 490 bool torture_cleanup(void) 491 { 492 mutex_lock(&fullstop_mutex); 493 if (fullstop == FULLSTOP_SHUTDOWN) { 494 pr_warn("Concurrent rmmod and shutdown illegal!\n"); 495 mutex_unlock(&fullstop_mutex); 496 schedule_timeout_uninterruptible(10); 497 return true; 498 } 499 fullstop = FULLSTOP_RMMOD; 500 mutex_unlock(&fullstop_mutex); 501 unregister_reboot_notifier(&torture_shutdown_nb); 502 torture_shuffle_cleanup(); 503 torture_onoff_cleanup(); 504 return false; 505 } 506 EXPORT_SYMBOL_GPL(torture_cleanup); 507 508 /* 509 * Is it time for the current torture test to stop? 510 */ 511 bool torture_must_stop(void) 512 { 513 return torture_must_stop_irq() || kthread_should_stop(); 514 } 515 EXPORT_SYMBOL_GPL(torture_must_stop); 516 517 /* 518 * Is it time for the current torture test to stop? This is the irq-safe 519 * version, hence no check for kthread_should_stop(). 520 */ 521 bool torture_must_stop_irq(void) 522 { 523 return fullstop != FULLSTOP_DONTSTOP; 524 } 525 EXPORT_SYMBOL_GPL(torture_must_stop_irq); 526