1 /* 2 * Detect hard and soft lockups on a system 3 * 4 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc. 5 * 6 * Note: Most of this code is borrowed heavily from the original softlockup 7 * detector, so thanks to Ingo for the initial implementation. 8 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks 9 * to those contributors as well. 10 */ 11 12 #define pr_fmt(fmt) "NMI watchdog: " fmt 13 14 #include <linux/mm.h> 15 #include <linux/cpu.h> 16 #include <linux/nmi.h> 17 #include <linux/init.h> 18 #include <linux/module.h> 19 #include <linux/sysctl.h> 20 #include <linux/smpboot.h> 21 #include <linux/sched/rt.h> 22 23 #include <asm/irq_regs.h> 24 #include <linux/kvm_para.h> 25 #include <linux/perf_event.h> 26 27 /* 28 * The run state of the lockup detectors is controlled by the content of the 29 * 'watchdog_enabled' variable. Each lockup detector has its dedicated bit - 30 * bit 0 for the hard lockup detector and bit 1 for the soft lockup detector. 31 * 32 * 'watchdog_user_enabled', 'nmi_watchdog_enabled' and 'soft_watchdog_enabled' 33 * are variables that are only used as an 'interface' between the parameters 34 * in /proc/sys/kernel and the internal state bits in 'watchdog_enabled'. The 35 * 'watchdog_thresh' variable is handled differently because its value is not 36 * boolean, and the lockup detectors are 'suspended' while 'watchdog_thresh' 37 * is equal zero. 38 */ 39 #define NMI_WATCHDOG_ENABLED_BIT 0 40 #define SOFT_WATCHDOG_ENABLED_BIT 1 41 #define NMI_WATCHDOG_ENABLED (1 << NMI_WATCHDOG_ENABLED_BIT) 42 #define SOFT_WATCHDOG_ENABLED (1 << SOFT_WATCHDOG_ENABLED_BIT) 43 44 static DEFINE_MUTEX(watchdog_proc_mutex); 45 46 #ifdef CONFIG_HARDLOCKUP_DETECTOR 47 static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED|NMI_WATCHDOG_ENABLED; 48 #else 49 static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED; 50 #endif 51 int __read_mostly nmi_watchdog_enabled; 52 int __read_mostly soft_watchdog_enabled; 53 int __read_mostly watchdog_user_enabled; 54 int __read_mostly watchdog_thresh = 10; 55 56 #ifdef CONFIG_SMP 57 int __read_mostly sysctl_softlockup_all_cpu_backtrace; 58 #else 59 #define sysctl_softlockup_all_cpu_backtrace 0 60 #endif 61 62 static int __read_mostly watchdog_running; 63 static u64 __read_mostly sample_period; 64 65 static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts); 66 static DEFINE_PER_CPU(struct task_struct *, softlockup_watchdog); 67 static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer); 68 static DEFINE_PER_CPU(bool, softlockup_touch_sync); 69 static DEFINE_PER_CPU(bool, soft_watchdog_warn); 70 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts); 71 static DEFINE_PER_CPU(unsigned long, soft_lockup_hrtimer_cnt); 72 static DEFINE_PER_CPU(struct task_struct *, softlockup_task_ptr_saved); 73 #ifdef CONFIG_HARDLOCKUP_DETECTOR 74 static DEFINE_PER_CPU(bool, hard_watchdog_warn); 75 static DEFINE_PER_CPU(bool, watchdog_nmi_touch); 76 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved); 77 static DEFINE_PER_CPU(struct perf_event *, watchdog_ev); 78 #endif 79 static unsigned long soft_lockup_nmi_warn; 80 81 /* boot commands */ 82 /* 83 * Should we panic when a soft-lockup or hard-lockup occurs: 84 */ 85 #ifdef CONFIG_HARDLOCKUP_DETECTOR 86 static int hardlockup_panic = 87 CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE; 88 /* 89 * We may not want to enable hard lockup detection by default in all cases, 90 * for example when running the kernel as a guest on a hypervisor. In these 91 * cases this function can be called to disable hard lockup detection. This 92 * function should only be executed once by the boot processor before the 93 * kernel command line parameters are parsed, because otherwise it is not 94 * possible to override this in hardlockup_panic_setup(). 95 */ 96 void hardlockup_detector_disable(void) 97 { 98 watchdog_enabled &= ~NMI_WATCHDOG_ENABLED; 99 } 100 101 static int __init hardlockup_panic_setup(char *str) 102 { 103 if (!strncmp(str, "panic", 5)) 104 hardlockup_panic = 1; 105 else if (!strncmp(str, "nopanic", 7)) 106 hardlockup_panic = 0; 107 else if (!strncmp(str, "0", 1)) 108 watchdog_enabled &= ~NMI_WATCHDOG_ENABLED; 109 else if (!strncmp(str, "1", 1)) 110 watchdog_enabled |= NMI_WATCHDOG_ENABLED; 111 return 1; 112 } 113 __setup("nmi_watchdog=", hardlockup_panic_setup); 114 #endif 115 116 unsigned int __read_mostly softlockup_panic = 117 CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE; 118 119 static int __init softlockup_panic_setup(char *str) 120 { 121 softlockup_panic = simple_strtoul(str, NULL, 0); 122 123 return 1; 124 } 125 __setup("softlockup_panic=", softlockup_panic_setup); 126 127 static int __init nowatchdog_setup(char *str) 128 { 129 watchdog_enabled = 0; 130 return 1; 131 } 132 __setup("nowatchdog", nowatchdog_setup); 133 134 static int __init nosoftlockup_setup(char *str) 135 { 136 watchdog_enabled &= ~SOFT_WATCHDOG_ENABLED; 137 return 1; 138 } 139 __setup("nosoftlockup", nosoftlockup_setup); 140 141 #ifdef CONFIG_SMP 142 static int __init softlockup_all_cpu_backtrace_setup(char *str) 143 { 144 sysctl_softlockup_all_cpu_backtrace = 145 !!simple_strtol(str, NULL, 0); 146 return 1; 147 } 148 __setup("softlockup_all_cpu_backtrace=", softlockup_all_cpu_backtrace_setup); 149 #endif 150 151 /* 152 * Hard-lockup warnings should be triggered after just a few seconds. Soft- 153 * lockups can have false positives under extreme conditions. So we generally 154 * want a higher threshold for soft lockups than for hard lockups. So we couple 155 * the thresholds with a factor: we make the soft threshold twice the amount of 156 * time the hard threshold is. 157 */ 158 static int get_softlockup_thresh(void) 159 { 160 return watchdog_thresh * 2; 161 } 162 163 /* 164 * Returns seconds, approximately. We don't need nanosecond 165 * resolution, and we don't need to waste time with a big divide when 166 * 2^30ns == 1.074s. 167 */ 168 static unsigned long get_timestamp(void) 169 { 170 return running_clock() >> 30LL; /* 2^30 ~= 10^9 */ 171 } 172 173 static void set_sample_period(void) 174 { 175 /* 176 * convert watchdog_thresh from seconds to ns 177 * the divide by 5 is to give hrtimer several chances (two 178 * or three with the current relation between the soft 179 * and hard thresholds) to increment before the 180 * hardlockup detector generates a warning 181 */ 182 sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5); 183 } 184 185 /* Commands for resetting the watchdog */ 186 static void __touch_watchdog(void) 187 { 188 __this_cpu_write(watchdog_touch_ts, get_timestamp()); 189 } 190 191 void touch_softlockup_watchdog(void) 192 { 193 /* 194 * Preemption can be enabled. It doesn't matter which CPU's timestamp 195 * gets zeroed here, so use the raw_ operation. 196 */ 197 raw_cpu_write(watchdog_touch_ts, 0); 198 } 199 EXPORT_SYMBOL(touch_softlockup_watchdog); 200 201 void touch_all_softlockup_watchdogs(void) 202 { 203 int cpu; 204 205 /* 206 * this is done lockless 207 * do we care if a 0 races with a timestamp? 208 * all it means is the softlock check starts one cycle later 209 */ 210 for_each_online_cpu(cpu) 211 per_cpu(watchdog_touch_ts, cpu) = 0; 212 } 213 214 #ifdef CONFIG_HARDLOCKUP_DETECTOR 215 void touch_nmi_watchdog(void) 216 { 217 /* 218 * Using __raw here because some code paths have 219 * preemption enabled. If preemption is enabled 220 * then interrupts should be enabled too, in which 221 * case we shouldn't have to worry about the watchdog 222 * going off. 223 */ 224 raw_cpu_write(watchdog_nmi_touch, true); 225 touch_softlockup_watchdog(); 226 } 227 EXPORT_SYMBOL(touch_nmi_watchdog); 228 229 #endif 230 231 void touch_softlockup_watchdog_sync(void) 232 { 233 __this_cpu_write(softlockup_touch_sync, true); 234 __this_cpu_write(watchdog_touch_ts, 0); 235 } 236 237 #ifdef CONFIG_HARDLOCKUP_DETECTOR 238 /* watchdog detector functions */ 239 static int is_hardlockup(void) 240 { 241 unsigned long hrint = __this_cpu_read(hrtimer_interrupts); 242 243 if (__this_cpu_read(hrtimer_interrupts_saved) == hrint) 244 return 1; 245 246 __this_cpu_write(hrtimer_interrupts_saved, hrint); 247 return 0; 248 } 249 #endif 250 251 static int is_softlockup(unsigned long touch_ts) 252 { 253 unsigned long now = get_timestamp(); 254 255 if (watchdog_enabled & SOFT_WATCHDOG_ENABLED) { 256 /* Warn about unreasonable delays. */ 257 if (time_after(now, touch_ts + get_softlockup_thresh())) 258 return now - touch_ts; 259 } 260 return 0; 261 } 262 263 #ifdef CONFIG_HARDLOCKUP_DETECTOR 264 265 static struct perf_event_attr wd_hw_attr = { 266 .type = PERF_TYPE_HARDWARE, 267 .config = PERF_COUNT_HW_CPU_CYCLES, 268 .size = sizeof(struct perf_event_attr), 269 .pinned = 1, 270 .disabled = 1, 271 }; 272 273 /* Callback function for perf event subsystem */ 274 static void watchdog_overflow_callback(struct perf_event *event, 275 struct perf_sample_data *data, 276 struct pt_regs *regs) 277 { 278 /* Ensure the watchdog never gets throttled */ 279 event->hw.interrupts = 0; 280 281 if (__this_cpu_read(watchdog_nmi_touch) == true) { 282 __this_cpu_write(watchdog_nmi_touch, false); 283 return; 284 } 285 286 /* check for a hardlockup 287 * This is done by making sure our timer interrupt 288 * is incrementing. The timer interrupt should have 289 * fired multiple times before we overflow'd. If it hasn't 290 * then this is a good indication the cpu is stuck 291 */ 292 if (is_hardlockup()) { 293 int this_cpu = smp_processor_id(); 294 295 /* only print hardlockups once */ 296 if (__this_cpu_read(hard_watchdog_warn) == true) 297 return; 298 299 if (hardlockup_panic) 300 panic("Watchdog detected hard LOCKUP on cpu %d", 301 this_cpu); 302 else 303 WARN(1, "Watchdog detected hard LOCKUP on cpu %d", 304 this_cpu); 305 306 __this_cpu_write(hard_watchdog_warn, true); 307 return; 308 } 309 310 __this_cpu_write(hard_watchdog_warn, false); 311 return; 312 } 313 #endif /* CONFIG_HARDLOCKUP_DETECTOR */ 314 315 static void watchdog_interrupt_count(void) 316 { 317 __this_cpu_inc(hrtimer_interrupts); 318 } 319 320 static int watchdog_nmi_enable(unsigned int cpu); 321 static void watchdog_nmi_disable(unsigned int cpu); 322 323 /* watchdog kicker functions */ 324 static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) 325 { 326 unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts); 327 struct pt_regs *regs = get_irq_regs(); 328 int duration; 329 int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace; 330 331 /* kick the hardlockup detector */ 332 watchdog_interrupt_count(); 333 334 /* kick the softlockup detector */ 335 wake_up_process(__this_cpu_read(softlockup_watchdog)); 336 337 /* .. and repeat */ 338 hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); 339 340 if (touch_ts == 0) { 341 if (unlikely(__this_cpu_read(softlockup_touch_sync))) { 342 /* 343 * If the time stamp was touched atomically 344 * make sure the scheduler tick is up to date. 345 */ 346 __this_cpu_write(softlockup_touch_sync, false); 347 sched_clock_tick(); 348 } 349 350 /* Clear the guest paused flag on watchdog reset */ 351 kvm_check_and_clear_guest_paused(); 352 __touch_watchdog(); 353 return HRTIMER_RESTART; 354 } 355 356 /* check for a softlockup 357 * This is done by making sure a high priority task is 358 * being scheduled. The task touches the watchdog to 359 * indicate it is getting cpu time. If it hasn't then 360 * this is a good indication some task is hogging the cpu 361 */ 362 duration = is_softlockup(touch_ts); 363 if (unlikely(duration)) { 364 /* 365 * If a virtual machine is stopped by the host it can look to 366 * the watchdog like a soft lockup, check to see if the host 367 * stopped the vm before we issue the warning 368 */ 369 if (kvm_check_and_clear_guest_paused()) 370 return HRTIMER_RESTART; 371 372 /* only warn once */ 373 if (__this_cpu_read(soft_watchdog_warn) == true) { 374 /* 375 * When multiple processes are causing softlockups the 376 * softlockup detector only warns on the first one 377 * because the code relies on a full quiet cycle to 378 * re-arm. The second process prevents the quiet cycle 379 * and never gets reported. Use task pointers to detect 380 * this. 381 */ 382 if (__this_cpu_read(softlockup_task_ptr_saved) != 383 current) { 384 __this_cpu_write(soft_watchdog_warn, false); 385 __touch_watchdog(); 386 } 387 return HRTIMER_RESTART; 388 } 389 390 if (softlockup_all_cpu_backtrace) { 391 /* Prevent multiple soft-lockup reports if one cpu is already 392 * engaged in dumping cpu back traces 393 */ 394 if (test_and_set_bit(0, &soft_lockup_nmi_warn)) { 395 /* Someone else will report us. Let's give up */ 396 __this_cpu_write(soft_watchdog_warn, true); 397 return HRTIMER_RESTART; 398 } 399 } 400 401 pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n", 402 smp_processor_id(), duration, 403 current->comm, task_pid_nr(current)); 404 __this_cpu_write(softlockup_task_ptr_saved, current); 405 print_modules(); 406 print_irqtrace_events(current); 407 if (regs) 408 show_regs(regs); 409 else 410 dump_stack(); 411 412 if (softlockup_all_cpu_backtrace) { 413 /* Avoid generating two back traces for current 414 * given that one is already made above 415 */ 416 trigger_allbutself_cpu_backtrace(); 417 418 clear_bit(0, &soft_lockup_nmi_warn); 419 /* Barrier to sync with other cpus */ 420 smp_mb__after_atomic(); 421 } 422 423 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 424 if (softlockup_panic) 425 panic("softlockup: hung tasks"); 426 __this_cpu_write(soft_watchdog_warn, true); 427 } else 428 __this_cpu_write(soft_watchdog_warn, false); 429 430 return HRTIMER_RESTART; 431 } 432 433 static void watchdog_set_prio(unsigned int policy, unsigned int prio) 434 { 435 struct sched_param param = { .sched_priority = prio }; 436 437 sched_setscheduler(current, policy, ¶m); 438 } 439 440 static void watchdog_enable(unsigned int cpu) 441 { 442 struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); 443 444 /* kick off the timer for the hardlockup detector */ 445 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 446 hrtimer->function = watchdog_timer_fn; 447 448 /* Enable the perf event */ 449 watchdog_nmi_enable(cpu); 450 451 /* done here because hrtimer_start can only pin to smp_processor_id() */ 452 hrtimer_start(hrtimer, ns_to_ktime(sample_period), 453 HRTIMER_MODE_REL_PINNED); 454 455 /* initialize timestamp */ 456 watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1); 457 __touch_watchdog(); 458 } 459 460 static void watchdog_disable(unsigned int cpu) 461 { 462 struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); 463 464 watchdog_set_prio(SCHED_NORMAL, 0); 465 hrtimer_cancel(hrtimer); 466 /* disable the perf event */ 467 watchdog_nmi_disable(cpu); 468 } 469 470 static void watchdog_cleanup(unsigned int cpu, bool online) 471 { 472 watchdog_disable(cpu); 473 } 474 475 static int watchdog_should_run(unsigned int cpu) 476 { 477 return __this_cpu_read(hrtimer_interrupts) != 478 __this_cpu_read(soft_lockup_hrtimer_cnt); 479 } 480 481 /* 482 * The watchdog thread function - touches the timestamp. 483 * 484 * It only runs once every sample_period seconds (4 seconds by 485 * default) to reset the softlockup timestamp. If this gets delayed 486 * for more than 2*watchdog_thresh seconds then the debug-printout 487 * triggers in watchdog_timer_fn(). 488 */ 489 static void watchdog(unsigned int cpu) 490 { 491 __this_cpu_write(soft_lockup_hrtimer_cnt, 492 __this_cpu_read(hrtimer_interrupts)); 493 __touch_watchdog(); 494 495 /* 496 * watchdog_nmi_enable() clears the NMI_WATCHDOG_ENABLED bit in the 497 * failure path. Check for failures that can occur asynchronously - 498 * for example, when CPUs are on-lined - and shut down the hardware 499 * perf event on each CPU accordingly. 500 * 501 * The only non-obvious place this bit can be cleared is through 502 * watchdog_nmi_enable(), so a pr_info() is placed there. Placing a 503 * pr_info here would be too noisy as it would result in a message 504 * every few seconds if the hardlockup was disabled but the softlockup 505 * enabled. 506 */ 507 if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) 508 watchdog_nmi_disable(cpu); 509 } 510 511 #ifdef CONFIG_HARDLOCKUP_DETECTOR 512 /* 513 * People like the simple clean cpu node info on boot. 514 * Reduce the watchdog noise by only printing messages 515 * that are different from what cpu0 displayed. 516 */ 517 static unsigned long cpu0_err; 518 519 static int watchdog_nmi_enable(unsigned int cpu) 520 { 521 struct perf_event_attr *wd_attr; 522 struct perf_event *event = per_cpu(watchdog_ev, cpu); 523 524 /* nothing to do if the hard lockup detector is disabled */ 525 if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) 526 goto out; 527 528 /* is it already setup and enabled? */ 529 if (event && event->state > PERF_EVENT_STATE_OFF) 530 goto out; 531 532 /* it is setup but not enabled */ 533 if (event != NULL) 534 goto out_enable; 535 536 wd_attr = &wd_hw_attr; 537 wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); 538 539 /* Try to register using hardware perf events */ 540 event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL); 541 542 /* save cpu0 error for future comparision */ 543 if (cpu == 0 && IS_ERR(event)) 544 cpu0_err = PTR_ERR(event); 545 546 if (!IS_ERR(event)) { 547 /* only print for cpu0 or different than cpu0 */ 548 if (cpu == 0 || cpu0_err) 549 pr_info("enabled on all CPUs, permanently consumes one hw-PMU counter.\n"); 550 goto out_save; 551 } 552 553 /* 554 * Disable the hard lockup detector if _any_ CPU fails to set up 555 * set up the hardware perf event. The watchdog() function checks 556 * the NMI_WATCHDOG_ENABLED bit periodically. 557 * 558 * The barriers are for syncing up watchdog_enabled across all the 559 * cpus, as clear_bit() does not use barriers. 560 */ 561 smp_mb__before_atomic(); 562 clear_bit(NMI_WATCHDOG_ENABLED_BIT, &watchdog_enabled); 563 smp_mb__after_atomic(); 564 565 /* skip displaying the same error again */ 566 if (cpu > 0 && (PTR_ERR(event) == cpu0_err)) 567 return PTR_ERR(event); 568 569 /* vary the KERN level based on the returned errno */ 570 if (PTR_ERR(event) == -EOPNOTSUPP) 571 pr_info("disabled (cpu%i): not supported (no LAPIC?)\n", cpu); 572 else if (PTR_ERR(event) == -ENOENT) 573 pr_warn("disabled (cpu%i): hardware events not enabled\n", 574 cpu); 575 else 576 pr_err("disabled (cpu%i): unable to create perf event: %ld\n", 577 cpu, PTR_ERR(event)); 578 579 pr_info("Shutting down hard lockup detector on all cpus\n"); 580 581 return PTR_ERR(event); 582 583 /* success path */ 584 out_save: 585 per_cpu(watchdog_ev, cpu) = event; 586 out_enable: 587 perf_event_enable(per_cpu(watchdog_ev, cpu)); 588 out: 589 return 0; 590 } 591 592 static void watchdog_nmi_disable(unsigned int cpu) 593 { 594 struct perf_event *event = per_cpu(watchdog_ev, cpu); 595 596 if (event) { 597 perf_event_disable(event); 598 per_cpu(watchdog_ev, cpu) = NULL; 599 600 /* should be in cleanup, but blocks oprofile */ 601 perf_event_release_kernel(event); 602 } 603 if (cpu == 0) { 604 /* watchdog_nmi_enable() expects this to be zero initially. */ 605 cpu0_err = 0; 606 } 607 } 608 609 void watchdog_nmi_enable_all(void) 610 { 611 int cpu; 612 613 mutex_lock(&watchdog_proc_mutex); 614 615 if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) 616 goto unlock; 617 618 get_online_cpus(); 619 for_each_online_cpu(cpu) 620 watchdog_nmi_enable(cpu); 621 put_online_cpus(); 622 623 unlock: 624 mutex_unlock(&watchdog_proc_mutex); 625 } 626 627 void watchdog_nmi_disable_all(void) 628 { 629 int cpu; 630 631 mutex_lock(&watchdog_proc_mutex); 632 633 if (!watchdog_running) 634 goto unlock; 635 636 get_online_cpus(); 637 for_each_online_cpu(cpu) 638 watchdog_nmi_disable(cpu); 639 put_online_cpus(); 640 641 unlock: 642 mutex_unlock(&watchdog_proc_mutex); 643 } 644 #else 645 static int watchdog_nmi_enable(unsigned int cpu) { return 0; } 646 static void watchdog_nmi_disable(unsigned int cpu) { return; } 647 void watchdog_nmi_enable_all(void) {} 648 void watchdog_nmi_disable_all(void) {} 649 #endif /* CONFIG_HARDLOCKUP_DETECTOR */ 650 651 static struct smp_hotplug_thread watchdog_threads = { 652 .store = &softlockup_watchdog, 653 .thread_should_run = watchdog_should_run, 654 .thread_fn = watchdog, 655 .thread_comm = "watchdog/%u", 656 .setup = watchdog_enable, 657 .cleanup = watchdog_cleanup, 658 .park = watchdog_disable, 659 .unpark = watchdog_enable, 660 }; 661 662 static void restart_watchdog_hrtimer(void *info) 663 { 664 struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); 665 int ret; 666 667 /* 668 * No need to cancel and restart hrtimer if it is currently executing 669 * because it will reprogram itself with the new period now. 670 * We should never see it unqueued here because we are running per-cpu 671 * with interrupts disabled. 672 */ 673 ret = hrtimer_try_to_cancel(hrtimer); 674 if (ret == 1) 675 hrtimer_start(hrtimer, ns_to_ktime(sample_period), 676 HRTIMER_MODE_REL_PINNED); 677 } 678 679 static void update_watchdog(int cpu) 680 { 681 /* 682 * Make sure that perf event counter will adopt to a new 683 * sampling period. Updating the sampling period directly would 684 * be much nicer but we do not have an API for that now so 685 * let's use a big hammer. 686 * Hrtimer will adopt the new period on the next tick but this 687 * might be late already so we have to restart the timer as well. 688 */ 689 watchdog_nmi_disable(cpu); 690 smp_call_function_single(cpu, restart_watchdog_hrtimer, NULL, 1); 691 watchdog_nmi_enable(cpu); 692 } 693 694 static void update_watchdog_all_cpus(void) 695 { 696 int cpu; 697 698 get_online_cpus(); 699 for_each_online_cpu(cpu) 700 update_watchdog(cpu); 701 put_online_cpus(); 702 } 703 704 static int watchdog_enable_all_cpus(void) 705 { 706 int err = 0; 707 708 if (!watchdog_running) { 709 err = smpboot_register_percpu_thread(&watchdog_threads); 710 if (err) 711 pr_err("Failed to create watchdog threads, disabled\n"); 712 else 713 watchdog_running = 1; 714 } else { 715 /* 716 * Enable/disable the lockup detectors or 717 * change the sample period 'on the fly'. 718 */ 719 update_watchdog_all_cpus(); 720 } 721 722 return err; 723 } 724 725 /* prepare/enable/disable routines */ 726 /* sysctl functions */ 727 #ifdef CONFIG_SYSCTL 728 static void watchdog_disable_all_cpus(void) 729 { 730 if (watchdog_running) { 731 watchdog_running = 0; 732 smpboot_unregister_percpu_thread(&watchdog_threads); 733 } 734 } 735 736 /* 737 * Update the run state of the lockup detectors. 738 */ 739 static int proc_watchdog_update(void) 740 { 741 int err = 0; 742 743 /* 744 * Watchdog threads won't be started if they are already active. 745 * The 'watchdog_running' variable in watchdog_*_all_cpus() takes 746 * care of this. If those threads are already active, the sample 747 * period will be updated and the lockup detectors will be enabled 748 * or disabled 'on the fly'. 749 */ 750 if (watchdog_enabled && watchdog_thresh) 751 err = watchdog_enable_all_cpus(); 752 else 753 watchdog_disable_all_cpus(); 754 755 return err; 756 757 } 758 759 /* 760 * common function for watchdog, nmi_watchdog and soft_watchdog parameter 761 * 762 * caller | table->data points to | 'which' contains the flag(s) 763 * -------------------|-----------------------|----------------------------- 764 * proc_watchdog | watchdog_user_enabled | NMI_WATCHDOG_ENABLED or'ed 765 * | | with SOFT_WATCHDOG_ENABLED 766 * -------------------|-----------------------|----------------------------- 767 * proc_nmi_watchdog | nmi_watchdog_enabled | NMI_WATCHDOG_ENABLED 768 * -------------------|-----------------------|----------------------------- 769 * proc_soft_watchdog | soft_watchdog_enabled | SOFT_WATCHDOG_ENABLED 770 */ 771 static int proc_watchdog_common(int which, struct ctl_table *table, int write, 772 void __user *buffer, size_t *lenp, loff_t *ppos) 773 { 774 int err, old, new; 775 int *watchdog_param = (int *)table->data; 776 777 mutex_lock(&watchdog_proc_mutex); 778 779 /* 780 * If the parameter is being read return the state of the corresponding 781 * bit(s) in 'watchdog_enabled', else update 'watchdog_enabled' and the 782 * run state of the lockup detectors. 783 */ 784 if (!write) { 785 *watchdog_param = (watchdog_enabled & which) != 0; 786 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 787 } else { 788 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 789 if (err) 790 goto out; 791 792 /* 793 * There is a race window between fetching the current value 794 * from 'watchdog_enabled' and storing the new value. During 795 * this race window, watchdog_nmi_enable() can sneak in and 796 * clear the NMI_WATCHDOG_ENABLED bit in 'watchdog_enabled'. 797 * The 'cmpxchg' detects this race and the loop retries. 798 */ 799 do { 800 old = watchdog_enabled; 801 /* 802 * If the parameter value is not zero set the 803 * corresponding bit(s), else clear it(them). 804 */ 805 if (*watchdog_param) 806 new = old | which; 807 else 808 new = old & ~which; 809 } while (cmpxchg(&watchdog_enabled, old, new) != old); 810 811 /* 812 * Update the run state of the lockup detectors. 813 * Restore 'watchdog_enabled' on failure. 814 */ 815 err = proc_watchdog_update(); 816 if (err) 817 watchdog_enabled = old; 818 } 819 out: 820 mutex_unlock(&watchdog_proc_mutex); 821 return err; 822 } 823 824 /* 825 * /proc/sys/kernel/watchdog 826 */ 827 int proc_watchdog(struct ctl_table *table, int write, 828 void __user *buffer, size_t *lenp, loff_t *ppos) 829 { 830 return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED, 831 table, write, buffer, lenp, ppos); 832 } 833 834 /* 835 * /proc/sys/kernel/nmi_watchdog 836 */ 837 int proc_nmi_watchdog(struct ctl_table *table, int write, 838 void __user *buffer, size_t *lenp, loff_t *ppos) 839 { 840 return proc_watchdog_common(NMI_WATCHDOG_ENABLED, 841 table, write, buffer, lenp, ppos); 842 } 843 844 /* 845 * /proc/sys/kernel/soft_watchdog 846 */ 847 int proc_soft_watchdog(struct ctl_table *table, int write, 848 void __user *buffer, size_t *lenp, loff_t *ppos) 849 { 850 return proc_watchdog_common(SOFT_WATCHDOG_ENABLED, 851 table, write, buffer, lenp, ppos); 852 } 853 854 /* 855 * /proc/sys/kernel/watchdog_thresh 856 */ 857 int proc_watchdog_thresh(struct ctl_table *table, int write, 858 void __user *buffer, size_t *lenp, loff_t *ppos) 859 { 860 int err, old; 861 862 mutex_lock(&watchdog_proc_mutex); 863 864 old = ACCESS_ONCE(watchdog_thresh); 865 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 866 867 if (err || !write) 868 goto out; 869 870 /* 871 * Update the sample period. 872 * Restore 'watchdog_thresh' on failure. 873 */ 874 set_sample_period(); 875 err = proc_watchdog_update(); 876 if (err) 877 watchdog_thresh = old; 878 out: 879 mutex_unlock(&watchdog_proc_mutex); 880 return err; 881 } 882 #endif /* CONFIG_SYSCTL */ 883 884 void __init lockup_detector_init(void) 885 { 886 set_sample_period(); 887 888 if (watchdog_enabled) 889 watchdog_enable_all_cpus(); 890 } 891