158687acbSDon Zickus /* 258687acbSDon Zickus * Detect hard and soft lockups on a system 358687acbSDon Zickus * 458687acbSDon Zickus * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc. 558687acbSDon Zickus * 686f5e6a7SFernando Luis Vázquez Cao * Note: Most of this code is borrowed heavily from the original softlockup 786f5e6a7SFernando Luis Vázquez Cao * detector, so thanks to Ingo for the initial implementation. 886f5e6a7SFernando Luis Vázquez Cao * Some chunks also taken from the old x86-specific nmi watchdog code, thanks 958687acbSDon Zickus * to those contributors as well. 1058687acbSDon Zickus */ 1158687acbSDon Zickus 124501980aSAndrew Morton #define pr_fmt(fmt) "NMI watchdog: " fmt 134501980aSAndrew Morton 1458687acbSDon Zickus #include <linux/mm.h> 1558687acbSDon Zickus #include <linux/cpu.h> 1658687acbSDon Zickus #include <linux/nmi.h> 1758687acbSDon Zickus #include <linux/init.h> 1858687acbSDon Zickus #include <linux/module.h> 1958687acbSDon Zickus #include <linux/sysctl.h> 20bcd951cfSThomas Gleixner #include <linux/smpboot.h> 218bd75c77SClark Williams #include <linux/sched/rt.h> 2258687acbSDon Zickus 2358687acbSDon Zickus #include <asm/irq_regs.h> 245d1c0f4aSEric B Munson #include <linux/kvm_para.h> 2558687acbSDon Zickus #include <linux/perf_event.h> 2658687acbSDon Zickus 2784d56e66SUlrich Obergfell /* 2884d56e66SUlrich Obergfell * The run state of the lockup detectors is controlled by the content of the 2984d56e66SUlrich Obergfell * 'watchdog_enabled' variable. Each lockup detector has its dedicated bit - 3084d56e66SUlrich Obergfell * bit 0 for the hard lockup detector and bit 1 for the soft lockup detector. 3184d56e66SUlrich Obergfell * 3284d56e66SUlrich Obergfell * 'watchdog_user_enabled', 'nmi_watchdog_enabled' and 'soft_watchdog_enabled' 3384d56e66SUlrich Obergfell * are variables that are only used as an 'interface' between the parameters 3484d56e66SUlrich Obergfell * in /proc/sys/kernel and the internal state bits in 'watchdog_enabled'. The 3584d56e66SUlrich Obergfell * 'watchdog_thresh' variable is handled differently because its value is not 3684d56e66SUlrich Obergfell * boolean, and the lockup detectors are 'suspended' while 'watchdog_thresh' 3784d56e66SUlrich Obergfell * is equal zero. 3884d56e66SUlrich Obergfell */ 3984d56e66SUlrich Obergfell #define NMI_WATCHDOG_ENABLED_BIT 0 4084d56e66SUlrich Obergfell #define SOFT_WATCHDOG_ENABLED_BIT 1 4184d56e66SUlrich Obergfell #define NMI_WATCHDOG_ENABLED (1 << NMI_WATCHDOG_ENABLED_BIT) 4284d56e66SUlrich Obergfell #define SOFT_WATCHDOG_ENABLED (1 << SOFT_WATCHDOG_ENABLED_BIT) 4384d56e66SUlrich Obergfell 44ab992dc3SPeter Zijlstra static DEFINE_MUTEX(watchdog_proc_mutex); 45ab992dc3SPeter Zijlstra 4684d56e66SUlrich Obergfell #ifdef CONFIG_HARDLOCKUP_DETECTOR 4784d56e66SUlrich Obergfell static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED|NMI_WATCHDOG_ENABLED; 4884d56e66SUlrich Obergfell #else 4984d56e66SUlrich Obergfell static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED; 5084d56e66SUlrich Obergfell #endif 5184d56e66SUlrich Obergfell int __read_mostly nmi_watchdog_enabled; 5284d56e66SUlrich Obergfell int __read_mostly soft_watchdog_enabled; 5384d56e66SUlrich Obergfell int __read_mostly watchdog_user_enabled; 544eec42f3SMandeep Singh Baines int __read_mostly watchdog_thresh = 10; 5584d56e66SUlrich Obergfell 56ed235875SAaron Tomlin #ifdef CONFIG_SMP 57ed235875SAaron Tomlin int __read_mostly sysctl_softlockup_all_cpu_backtrace; 58ed235875SAaron Tomlin #else 59ed235875SAaron Tomlin #define sysctl_softlockup_all_cpu_backtrace 0 60ed235875SAaron Tomlin #endif 61ed235875SAaron Tomlin 623c00ea82SFrederic Weisbecker static int __read_mostly watchdog_running; 630f34c400SChuansheng Liu static u64 __read_mostly sample_period; 6458687acbSDon Zickus 6558687acbSDon Zickus static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts); 6658687acbSDon Zickus static DEFINE_PER_CPU(struct task_struct *, softlockup_watchdog); 6758687acbSDon Zickus static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer); 6858687acbSDon Zickus static DEFINE_PER_CPU(bool, softlockup_touch_sync); 6958687acbSDon Zickus static DEFINE_PER_CPU(bool, soft_watchdog_warn); 70bcd951cfSThomas Gleixner static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts); 71bcd951cfSThomas Gleixner static DEFINE_PER_CPU(unsigned long, soft_lockup_hrtimer_cnt); 72b1a8de1fSchai wen static DEFINE_PER_CPU(struct task_struct *, softlockup_task_ptr_saved); 7323637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 74cafcd80dSDon Zickus static DEFINE_PER_CPU(bool, hard_watchdog_warn); 75cafcd80dSDon Zickus static DEFINE_PER_CPU(bool, watchdog_nmi_touch); 7658687acbSDon Zickus static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved); 7758687acbSDon Zickus static DEFINE_PER_CPU(struct perf_event *, watchdog_ev); 7858687acbSDon Zickus #endif 79ed235875SAaron Tomlin static unsigned long soft_lockup_nmi_warn; 8058687acbSDon Zickus 8158687acbSDon Zickus /* boot commands */ 8258687acbSDon Zickus /* 8358687acbSDon Zickus * Should we panic when a soft-lockup or hard-lockup occurs: 8458687acbSDon Zickus */ 8523637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 86fef2c9bcSDon Zickus static int hardlockup_panic = 87fef2c9bcSDon Zickus CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE; 886e7458a6SUlrich Obergfell /* 896e7458a6SUlrich Obergfell * We may not want to enable hard lockup detection by default in all cases, 906e7458a6SUlrich Obergfell * for example when running the kernel as a guest on a hypervisor. In these 916e7458a6SUlrich Obergfell * cases this function can be called to disable hard lockup detection. This 926e7458a6SUlrich Obergfell * function should only be executed once by the boot processor before the 936e7458a6SUlrich Obergfell * kernel command line parameters are parsed, because otherwise it is not 946e7458a6SUlrich Obergfell * possible to override this in hardlockup_panic_setup(). 956e7458a6SUlrich Obergfell */ 96692297d8SUlrich Obergfell void hardlockup_detector_disable(void) 976e7458a6SUlrich Obergfell { 98692297d8SUlrich Obergfell watchdog_enabled &= ~NMI_WATCHDOG_ENABLED; 996e7458a6SUlrich Obergfell } 1006e7458a6SUlrich Obergfell 10158687acbSDon Zickus static int __init hardlockup_panic_setup(char *str) 10258687acbSDon Zickus { 10358687acbSDon Zickus if (!strncmp(str, "panic", 5)) 10458687acbSDon Zickus hardlockup_panic = 1; 105fef2c9bcSDon Zickus else if (!strncmp(str, "nopanic", 7)) 106fef2c9bcSDon Zickus hardlockup_panic = 0; 1075dc30558SDon Zickus else if (!strncmp(str, "0", 1)) 108195daf66SUlrich Obergfell watchdog_enabled &= ~NMI_WATCHDOG_ENABLED; 109195daf66SUlrich Obergfell else if (!strncmp(str, "1", 1)) 110195daf66SUlrich Obergfell watchdog_enabled |= NMI_WATCHDOG_ENABLED; 11158687acbSDon Zickus return 1; 11258687acbSDon Zickus } 11358687acbSDon Zickus __setup("nmi_watchdog=", hardlockup_panic_setup); 11458687acbSDon Zickus #endif 11558687acbSDon Zickus 11658687acbSDon Zickus unsigned int __read_mostly softlockup_panic = 11758687acbSDon Zickus CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE; 11858687acbSDon Zickus 11958687acbSDon Zickus static int __init softlockup_panic_setup(char *str) 12058687acbSDon Zickus { 12158687acbSDon Zickus softlockup_panic = simple_strtoul(str, NULL, 0); 12258687acbSDon Zickus 12358687acbSDon Zickus return 1; 12458687acbSDon Zickus } 12558687acbSDon Zickus __setup("softlockup_panic=", softlockup_panic_setup); 12658687acbSDon Zickus 12758687acbSDon Zickus static int __init nowatchdog_setup(char *str) 12858687acbSDon Zickus { 129195daf66SUlrich Obergfell watchdog_enabled = 0; 13058687acbSDon Zickus return 1; 13158687acbSDon Zickus } 13258687acbSDon Zickus __setup("nowatchdog", nowatchdog_setup); 13358687acbSDon Zickus 13458687acbSDon Zickus static int __init nosoftlockup_setup(char *str) 13558687acbSDon Zickus { 136195daf66SUlrich Obergfell watchdog_enabled &= ~SOFT_WATCHDOG_ENABLED; 13758687acbSDon Zickus return 1; 13858687acbSDon Zickus } 13958687acbSDon Zickus __setup("nosoftlockup", nosoftlockup_setup); 140195daf66SUlrich Obergfell 141ed235875SAaron Tomlin #ifdef CONFIG_SMP 142ed235875SAaron Tomlin static int __init softlockup_all_cpu_backtrace_setup(char *str) 143ed235875SAaron Tomlin { 144ed235875SAaron Tomlin sysctl_softlockup_all_cpu_backtrace = 145ed235875SAaron Tomlin !!simple_strtol(str, NULL, 0); 146ed235875SAaron Tomlin return 1; 147ed235875SAaron Tomlin } 148ed235875SAaron Tomlin __setup("softlockup_all_cpu_backtrace=", softlockup_all_cpu_backtrace_setup); 149ed235875SAaron Tomlin #endif 15058687acbSDon Zickus 1514eec42f3SMandeep Singh Baines /* 1524eec42f3SMandeep Singh Baines * Hard-lockup warnings should be triggered after just a few seconds. Soft- 1534eec42f3SMandeep Singh Baines * lockups can have false positives under extreme conditions. So we generally 1544eec42f3SMandeep Singh Baines * want a higher threshold for soft lockups than for hard lockups. So we couple 1554eec42f3SMandeep Singh Baines * the thresholds with a factor: we make the soft threshold twice the amount of 1564eec42f3SMandeep Singh Baines * time the hard threshold is. 1574eec42f3SMandeep Singh Baines */ 1586e9101aeSIngo Molnar static int get_softlockup_thresh(void) 1594eec42f3SMandeep Singh Baines { 1604eec42f3SMandeep Singh Baines return watchdog_thresh * 2; 1614eec42f3SMandeep Singh Baines } 16258687acbSDon Zickus 16358687acbSDon Zickus /* 16458687acbSDon Zickus * Returns seconds, approximately. We don't need nanosecond 16558687acbSDon Zickus * resolution, and we don't need to waste time with a big divide when 16658687acbSDon Zickus * 2^30ns == 1.074s. 16758687acbSDon Zickus */ 168c06b4f19SNamhyung Kim static unsigned long get_timestamp(void) 16958687acbSDon Zickus { 170545a2bf7SCyril Bur return running_clock() >> 30LL; /* 2^30 ~= 10^9 */ 17158687acbSDon Zickus } 17258687acbSDon Zickus 1730f34c400SChuansheng Liu static void set_sample_period(void) 17458687acbSDon Zickus { 17558687acbSDon Zickus /* 176586692a5SMandeep Singh Baines * convert watchdog_thresh from seconds to ns 17786f5e6a7SFernando Luis Vázquez Cao * the divide by 5 is to give hrtimer several chances (two 17886f5e6a7SFernando Luis Vázquez Cao * or three with the current relation between the soft 17986f5e6a7SFernando Luis Vázquez Cao * and hard thresholds) to increment before the 18086f5e6a7SFernando Luis Vázquez Cao * hardlockup detector generates a warning 18158687acbSDon Zickus */ 1820f34c400SChuansheng Liu sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5); 18358687acbSDon Zickus } 18458687acbSDon Zickus 18558687acbSDon Zickus /* Commands for resetting the watchdog */ 18658687acbSDon Zickus static void __touch_watchdog(void) 18758687acbSDon Zickus { 188c06b4f19SNamhyung Kim __this_cpu_write(watchdog_touch_ts, get_timestamp()); 18958687acbSDon Zickus } 19058687acbSDon Zickus 191332fbdbcSDon Zickus void touch_softlockup_watchdog(void) 19258687acbSDon Zickus { 1937861144bSAndrew Morton /* 1947861144bSAndrew Morton * Preemption can be enabled. It doesn't matter which CPU's timestamp 1957861144bSAndrew Morton * gets zeroed here, so use the raw_ operation. 1967861144bSAndrew Morton */ 1977861144bSAndrew Morton raw_cpu_write(watchdog_touch_ts, 0); 19858687acbSDon Zickus } 1990167c781SIngo Molnar EXPORT_SYMBOL(touch_softlockup_watchdog); 20058687acbSDon Zickus 201332fbdbcSDon Zickus void touch_all_softlockup_watchdogs(void) 20258687acbSDon Zickus { 20358687acbSDon Zickus int cpu; 20458687acbSDon Zickus 20558687acbSDon Zickus /* 20658687acbSDon Zickus * this is done lockless 20758687acbSDon Zickus * do we care if a 0 races with a timestamp? 20858687acbSDon Zickus * all it means is the softlock check starts one cycle later 20958687acbSDon Zickus */ 21058687acbSDon Zickus for_each_online_cpu(cpu) 21158687acbSDon Zickus per_cpu(watchdog_touch_ts, cpu) = 0; 21258687acbSDon Zickus } 21358687acbSDon Zickus 214cafcd80dSDon Zickus #ifdef CONFIG_HARDLOCKUP_DETECTOR 21558687acbSDon Zickus void touch_nmi_watchdog(void) 21658687acbSDon Zickus { 21762572e29SBen Zhang /* 21862572e29SBen Zhang * Using __raw here because some code paths have 21962572e29SBen Zhang * preemption enabled. If preemption is enabled 22062572e29SBen Zhang * then interrupts should be enabled too, in which 22162572e29SBen Zhang * case we shouldn't have to worry about the watchdog 22262572e29SBen Zhang * going off. 22362572e29SBen Zhang */ 224f7f66b05SChristoph Lameter raw_cpu_write(watchdog_nmi_touch, true); 225332fbdbcSDon Zickus touch_softlockup_watchdog(); 22658687acbSDon Zickus } 22758687acbSDon Zickus EXPORT_SYMBOL(touch_nmi_watchdog); 22858687acbSDon Zickus 229cafcd80dSDon Zickus #endif 230cafcd80dSDon Zickus 23158687acbSDon Zickus void touch_softlockup_watchdog_sync(void) 23258687acbSDon Zickus { 233f7f66b05SChristoph Lameter __this_cpu_write(softlockup_touch_sync, true); 234f7f66b05SChristoph Lameter __this_cpu_write(watchdog_touch_ts, 0); 23558687acbSDon Zickus } 23658687acbSDon Zickus 23723637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 23858687acbSDon Zickus /* watchdog detector functions */ 23926e09c6eSDon Zickus static int is_hardlockup(void) 24058687acbSDon Zickus { 241909ea964SChristoph Lameter unsigned long hrint = __this_cpu_read(hrtimer_interrupts); 24258687acbSDon Zickus 243909ea964SChristoph Lameter if (__this_cpu_read(hrtimer_interrupts_saved) == hrint) 24458687acbSDon Zickus return 1; 24558687acbSDon Zickus 246909ea964SChristoph Lameter __this_cpu_write(hrtimer_interrupts_saved, hrint); 24758687acbSDon Zickus return 0; 24858687acbSDon Zickus } 24958687acbSDon Zickus #endif 25058687acbSDon Zickus 25126e09c6eSDon Zickus static int is_softlockup(unsigned long touch_ts) 25258687acbSDon Zickus { 253c06b4f19SNamhyung Kim unsigned long now = get_timestamp(); 25458687acbSDon Zickus 255195daf66SUlrich Obergfell if (watchdog_enabled & SOFT_WATCHDOG_ENABLED) { 256195daf66SUlrich Obergfell /* Warn about unreasonable delays. */ 2574eec42f3SMandeep Singh Baines if (time_after(now, touch_ts + get_softlockup_thresh())) 25858687acbSDon Zickus return now - touch_ts; 259195daf66SUlrich Obergfell } 26058687acbSDon Zickus return 0; 26158687acbSDon Zickus } 26258687acbSDon Zickus 26323637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 2641880c4aeSCyrill Gorcunov 26558687acbSDon Zickus static struct perf_event_attr wd_hw_attr = { 26658687acbSDon Zickus .type = PERF_TYPE_HARDWARE, 26758687acbSDon Zickus .config = PERF_COUNT_HW_CPU_CYCLES, 26858687acbSDon Zickus .size = sizeof(struct perf_event_attr), 26958687acbSDon Zickus .pinned = 1, 27058687acbSDon Zickus .disabled = 1, 27158687acbSDon Zickus }; 27258687acbSDon Zickus 27358687acbSDon Zickus /* Callback function for perf event subsystem */ 274a8b0ca17SPeter Zijlstra static void watchdog_overflow_callback(struct perf_event *event, 27558687acbSDon Zickus struct perf_sample_data *data, 27658687acbSDon Zickus struct pt_regs *regs) 27758687acbSDon Zickus { 278c6db67cdSPeter Zijlstra /* Ensure the watchdog never gets throttled */ 279c6db67cdSPeter Zijlstra event->hw.interrupts = 0; 280c6db67cdSPeter Zijlstra 281909ea964SChristoph Lameter if (__this_cpu_read(watchdog_nmi_touch) == true) { 282909ea964SChristoph Lameter __this_cpu_write(watchdog_nmi_touch, false); 28358687acbSDon Zickus return; 28458687acbSDon Zickus } 28558687acbSDon Zickus 28658687acbSDon Zickus /* check for a hardlockup 28758687acbSDon Zickus * This is done by making sure our timer interrupt 28858687acbSDon Zickus * is incrementing. The timer interrupt should have 28958687acbSDon Zickus * fired multiple times before we overflow'd. If it hasn't 29058687acbSDon Zickus * then this is a good indication the cpu is stuck 29158687acbSDon Zickus */ 29226e09c6eSDon Zickus if (is_hardlockup()) { 29326e09c6eSDon Zickus int this_cpu = smp_processor_id(); 29426e09c6eSDon Zickus 29558687acbSDon Zickus /* only print hardlockups once */ 296909ea964SChristoph Lameter if (__this_cpu_read(hard_watchdog_warn) == true) 29758687acbSDon Zickus return; 29858687acbSDon Zickus 29958687acbSDon Zickus if (hardlockup_panic) 300656c3b79SFabian Frederick panic("Watchdog detected hard LOCKUP on cpu %d", 301656c3b79SFabian Frederick this_cpu); 30258687acbSDon Zickus else 303656c3b79SFabian Frederick WARN(1, "Watchdog detected hard LOCKUP on cpu %d", 304656c3b79SFabian Frederick this_cpu); 30558687acbSDon Zickus 306909ea964SChristoph Lameter __this_cpu_write(hard_watchdog_warn, true); 30758687acbSDon Zickus return; 30858687acbSDon Zickus } 30958687acbSDon Zickus 310909ea964SChristoph Lameter __this_cpu_write(hard_watchdog_warn, false); 31158687acbSDon Zickus return; 31258687acbSDon Zickus } 313bcd951cfSThomas Gleixner #endif /* CONFIG_HARDLOCKUP_DETECTOR */ 314bcd951cfSThomas Gleixner 31558687acbSDon Zickus static void watchdog_interrupt_count(void) 31658687acbSDon Zickus { 317909ea964SChristoph Lameter __this_cpu_inc(hrtimer_interrupts); 31858687acbSDon Zickus } 319bcd951cfSThomas Gleixner 320bcd951cfSThomas Gleixner static int watchdog_nmi_enable(unsigned int cpu); 321bcd951cfSThomas Gleixner static void watchdog_nmi_disable(unsigned int cpu); 32258687acbSDon Zickus 32358687acbSDon Zickus /* watchdog kicker functions */ 32458687acbSDon Zickus static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) 32558687acbSDon Zickus { 326909ea964SChristoph Lameter unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts); 32758687acbSDon Zickus struct pt_regs *regs = get_irq_regs(); 32858687acbSDon Zickus int duration; 329ed235875SAaron Tomlin int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace; 33058687acbSDon Zickus 33158687acbSDon Zickus /* kick the hardlockup detector */ 33258687acbSDon Zickus watchdog_interrupt_count(); 33358687acbSDon Zickus 33458687acbSDon Zickus /* kick the softlockup detector */ 335909ea964SChristoph Lameter wake_up_process(__this_cpu_read(softlockup_watchdog)); 33658687acbSDon Zickus 33758687acbSDon Zickus /* .. and repeat */ 3380f34c400SChuansheng Liu hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); 33958687acbSDon Zickus 34058687acbSDon Zickus if (touch_ts == 0) { 341909ea964SChristoph Lameter if (unlikely(__this_cpu_read(softlockup_touch_sync))) { 34258687acbSDon Zickus /* 34358687acbSDon Zickus * If the time stamp was touched atomically 34458687acbSDon Zickus * make sure the scheduler tick is up to date. 34558687acbSDon Zickus */ 346909ea964SChristoph Lameter __this_cpu_write(softlockup_touch_sync, false); 34758687acbSDon Zickus sched_clock_tick(); 34858687acbSDon Zickus } 3495d1c0f4aSEric B Munson 3505d1c0f4aSEric B Munson /* Clear the guest paused flag on watchdog reset */ 3515d1c0f4aSEric B Munson kvm_check_and_clear_guest_paused(); 35258687acbSDon Zickus __touch_watchdog(); 35358687acbSDon Zickus return HRTIMER_RESTART; 35458687acbSDon Zickus } 35558687acbSDon Zickus 35658687acbSDon Zickus /* check for a softlockup 35758687acbSDon Zickus * This is done by making sure a high priority task is 35858687acbSDon Zickus * being scheduled. The task touches the watchdog to 35958687acbSDon Zickus * indicate it is getting cpu time. If it hasn't then 36058687acbSDon Zickus * this is a good indication some task is hogging the cpu 36158687acbSDon Zickus */ 36226e09c6eSDon Zickus duration = is_softlockup(touch_ts); 36358687acbSDon Zickus if (unlikely(duration)) { 3645d1c0f4aSEric B Munson /* 3655d1c0f4aSEric B Munson * If a virtual machine is stopped by the host it can look to 3665d1c0f4aSEric B Munson * the watchdog like a soft lockup, check to see if the host 3675d1c0f4aSEric B Munson * stopped the vm before we issue the warning 3685d1c0f4aSEric B Munson */ 3695d1c0f4aSEric B Munson if (kvm_check_and_clear_guest_paused()) 3705d1c0f4aSEric B Munson return HRTIMER_RESTART; 3715d1c0f4aSEric B Munson 37258687acbSDon Zickus /* only warn once */ 373b1a8de1fSchai wen if (__this_cpu_read(soft_watchdog_warn) == true) { 374b1a8de1fSchai wen /* 375b1a8de1fSchai wen * When multiple processes are causing softlockups the 376b1a8de1fSchai wen * softlockup detector only warns on the first one 377b1a8de1fSchai wen * because the code relies on a full quiet cycle to 378b1a8de1fSchai wen * re-arm. The second process prevents the quiet cycle 379b1a8de1fSchai wen * and never gets reported. Use task pointers to detect 380b1a8de1fSchai wen * this. 381b1a8de1fSchai wen */ 382b1a8de1fSchai wen if (__this_cpu_read(softlockup_task_ptr_saved) != 383b1a8de1fSchai wen current) { 384b1a8de1fSchai wen __this_cpu_write(soft_watchdog_warn, false); 385b1a8de1fSchai wen __touch_watchdog(); 386b1a8de1fSchai wen } 38758687acbSDon Zickus return HRTIMER_RESTART; 388b1a8de1fSchai wen } 38958687acbSDon Zickus 390ed235875SAaron Tomlin if (softlockup_all_cpu_backtrace) { 391ed235875SAaron Tomlin /* Prevent multiple soft-lockup reports if one cpu is already 392ed235875SAaron Tomlin * engaged in dumping cpu back traces 393ed235875SAaron Tomlin */ 394ed235875SAaron Tomlin if (test_and_set_bit(0, &soft_lockup_nmi_warn)) { 395ed235875SAaron Tomlin /* Someone else will report us. Let's give up */ 396ed235875SAaron Tomlin __this_cpu_write(soft_watchdog_warn, true); 397ed235875SAaron Tomlin return HRTIMER_RESTART; 398ed235875SAaron Tomlin } 399ed235875SAaron Tomlin } 400ed235875SAaron Tomlin 401656c3b79SFabian Frederick pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n", 40226e09c6eSDon Zickus smp_processor_id(), duration, 40358687acbSDon Zickus current->comm, task_pid_nr(current)); 404b1a8de1fSchai wen __this_cpu_write(softlockup_task_ptr_saved, current); 40558687acbSDon Zickus print_modules(); 40658687acbSDon Zickus print_irqtrace_events(current); 40758687acbSDon Zickus if (regs) 40858687acbSDon Zickus show_regs(regs); 40958687acbSDon Zickus else 41058687acbSDon Zickus dump_stack(); 41158687acbSDon Zickus 412ed235875SAaron Tomlin if (softlockup_all_cpu_backtrace) { 413ed235875SAaron Tomlin /* Avoid generating two back traces for current 414ed235875SAaron Tomlin * given that one is already made above 415ed235875SAaron Tomlin */ 416ed235875SAaron Tomlin trigger_allbutself_cpu_backtrace(); 417ed235875SAaron Tomlin 418ed235875SAaron Tomlin clear_bit(0, &soft_lockup_nmi_warn); 419ed235875SAaron Tomlin /* Barrier to sync with other cpus */ 420ed235875SAaron Tomlin smp_mb__after_atomic(); 421ed235875SAaron Tomlin } 422ed235875SAaron Tomlin 42369361eefSJosh Hunt add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 42458687acbSDon Zickus if (softlockup_panic) 42558687acbSDon Zickus panic("softlockup: hung tasks"); 426909ea964SChristoph Lameter __this_cpu_write(soft_watchdog_warn, true); 42758687acbSDon Zickus } else 428909ea964SChristoph Lameter __this_cpu_write(soft_watchdog_warn, false); 42958687acbSDon Zickus 43058687acbSDon Zickus return HRTIMER_RESTART; 43158687acbSDon Zickus } 43258687acbSDon Zickus 433bcd951cfSThomas Gleixner static void watchdog_set_prio(unsigned int policy, unsigned int prio) 43458687acbSDon Zickus { 435bcd951cfSThomas Gleixner struct sched_param param = { .sched_priority = prio }; 436bcd951cfSThomas Gleixner 437bcd951cfSThomas Gleixner sched_setscheduler(current, policy, ¶m); 438bcd951cfSThomas Gleixner } 439bcd951cfSThomas Gleixner 440bcd951cfSThomas Gleixner static void watchdog_enable(unsigned int cpu) 441bcd951cfSThomas Gleixner { 442f7f66b05SChristoph Lameter struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); 44358687acbSDon Zickus 4443935e895SBjørn Mork /* kick off the timer for the hardlockup detector */ 4453935e895SBjørn Mork hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 4463935e895SBjørn Mork hrtimer->function = watchdog_timer_fn; 4473935e895SBjørn Mork 448bcd951cfSThomas Gleixner /* Enable the perf event */ 449bcd951cfSThomas Gleixner watchdog_nmi_enable(cpu); 45058687acbSDon Zickus 45158687acbSDon Zickus /* done here because hrtimer_start can only pin to smp_processor_id() */ 4520f34c400SChuansheng Liu hrtimer_start(hrtimer, ns_to_ktime(sample_period), 45358687acbSDon Zickus HRTIMER_MODE_REL_PINNED); 45458687acbSDon Zickus 455bcd951cfSThomas Gleixner /* initialize timestamp */ 456bcd951cfSThomas Gleixner watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1); 45758687acbSDon Zickus __touch_watchdog(); 45858687acbSDon Zickus } 459bcd951cfSThomas Gleixner 460bcd951cfSThomas Gleixner static void watchdog_disable(unsigned int cpu) 461bcd951cfSThomas Gleixner { 462f7f66b05SChristoph Lameter struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); 463bcd951cfSThomas Gleixner 464bcd951cfSThomas Gleixner watchdog_set_prio(SCHED_NORMAL, 0); 465bcd951cfSThomas Gleixner hrtimer_cancel(hrtimer); 466bcd951cfSThomas Gleixner /* disable the perf event */ 467bcd951cfSThomas Gleixner watchdog_nmi_disable(cpu); 468bcd951cfSThomas Gleixner } 469bcd951cfSThomas Gleixner 470b8900bc0SFrederic Weisbecker static void watchdog_cleanup(unsigned int cpu, bool online) 471b8900bc0SFrederic Weisbecker { 472b8900bc0SFrederic Weisbecker watchdog_disable(cpu); 473b8900bc0SFrederic Weisbecker } 474b8900bc0SFrederic Weisbecker 475bcd951cfSThomas Gleixner static int watchdog_should_run(unsigned int cpu) 476bcd951cfSThomas Gleixner { 477bcd951cfSThomas Gleixner return __this_cpu_read(hrtimer_interrupts) != 478bcd951cfSThomas Gleixner __this_cpu_read(soft_lockup_hrtimer_cnt); 479bcd951cfSThomas Gleixner } 480bcd951cfSThomas Gleixner 481b60f796cSAndrew Morton /* 482bcd951cfSThomas Gleixner * The watchdog thread function - touches the timestamp. 483bcd951cfSThomas Gleixner * 4840f34c400SChuansheng Liu * It only runs once every sample_period seconds (4 seconds by 485bcd951cfSThomas Gleixner * default) to reset the softlockup timestamp. If this gets delayed 486bcd951cfSThomas Gleixner * for more than 2*watchdog_thresh seconds then the debug-printout 487bcd951cfSThomas Gleixner * triggers in watchdog_timer_fn(). 488b60f796cSAndrew Morton */ 489bcd951cfSThomas Gleixner static void watchdog(unsigned int cpu) 490bcd951cfSThomas Gleixner { 491bcd951cfSThomas Gleixner __this_cpu_write(soft_lockup_hrtimer_cnt, 492bcd951cfSThomas Gleixner __this_cpu_read(hrtimer_interrupts)); 493bcd951cfSThomas Gleixner __touch_watchdog(); 494bcfba4f4SUlrich Obergfell 495bcfba4f4SUlrich Obergfell /* 496bcfba4f4SUlrich Obergfell * watchdog_nmi_enable() clears the NMI_WATCHDOG_ENABLED bit in the 497bcfba4f4SUlrich Obergfell * failure path. Check for failures that can occur asynchronously - 498bcfba4f4SUlrich Obergfell * for example, when CPUs are on-lined - and shut down the hardware 499bcfba4f4SUlrich Obergfell * perf event on each CPU accordingly. 500bcfba4f4SUlrich Obergfell * 501bcfba4f4SUlrich Obergfell * The only non-obvious place this bit can be cleared is through 502bcfba4f4SUlrich Obergfell * watchdog_nmi_enable(), so a pr_info() is placed there. Placing a 503bcfba4f4SUlrich Obergfell * pr_info here would be too noisy as it would result in a message 504bcfba4f4SUlrich Obergfell * every few seconds if the hardlockup was disabled but the softlockup 505bcfba4f4SUlrich Obergfell * enabled. 506bcfba4f4SUlrich Obergfell */ 507bcfba4f4SUlrich Obergfell if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) 508bcfba4f4SUlrich Obergfell watchdog_nmi_disable(cpu); 50958687acbSDon Zickus } 51058687acbSDon Zickus 51123637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 512a7027046SDon Zickus /* 513a7027046SDon Zickus * People like the simple clean cpu node info on boot. 514a7027046SDon Zickus * Reduce the watchdog noise by only printing messages 515a7027046SDon Zickus * that are different from what cpu0 displayed. 516a7027046SDon Zickus */ 517a7027046SDon Zickus static unsigned long cpu0_err; 518a7027046SDon Zickus 519bcd951cfSThomas Gleixner static int watchdog_nmi_enable(unsigned int cpu) 52058687acbSDon Zickus { 52158687acbSDon Zickus struct perf_event_attr *wd_attr; 52258687acbSDon Zickus struct perf_event *event = per_cpu(watchdog_ev, cpu); 52358687acbSDon Zickus 524195daf66SUlrich Obergfell /* nothing to do if the hard lockup detector is disabled */ 525195daf66SUlrich Obergfell if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) 526195daf66SUlrich Obergfell goto out; 5276e7458a6SUlrich Obergfell 52858687acbSDon Zickus /* is it already setup and enabled? */ 52958687acbSDon Zickus if (event && event->state > PERF_EVENT_STATE_OFF) 53058687acbSDon Zickus goto out; 53158687acbSDon Zickus 53258687acbSDon Zickus /* it is setup but not enabled */ 53358687acbSDon Zickus if (event != NULL) 53458687acbSDon Zickus goto out_enable; 53558687acbSDon Zickus 53658687acbSDon Zickus wd_attr = &wd_hw_attr; 5374eec42f3SMandeep Singh Baines wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); 5381880c4aeSCyrill Gorcunov 5391880c4aeSCyrill Gorcunov /* Try to register using hardware perf events */ 5404dc0da86SAvi Kivity event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL); 541a7027046SDon Zickus 542a7027046SDon Zickus /* save cpu0 error for future comparision */ 543a7027046SDon Zickus if (cpu == 0 && IS_ERR(event)) 544a7027046SDon Zickus cpu0_err = PTR_ERR(event); 545a7027046SDon Zickus 54658687acbSDon Zickus if (!IS_ERR(event)) { 547a7027046SDon Zickus /* only print for cpu0 or different than cpu0 */ 548a7027046SDon Zickus if (cpu == 0 || cpu0_err) 549a7027046SDon Zickus pr_info("enabled on all CPUs, permanently consumes one hw-PMU counter.\n"); 55058687acbSDon Zickus goto out_save; 55158687acbSDon Zickus } 55258687acbSDon Zickus 553bcfba4f4SUlrich Obergfell /* 554bcfba4f4SUlrich Obergfell * Disable the hard lockup detector if _any_ CPU fails to set up 555bcfba4f4SUlrich Obergfell * set up the hardware perf event. The watchdog() function checks 556bcfba4f4SUlrich Obergfell * the NMI_WATCHDOG_ENABLED bit periodically. 557bcfba4f4SUlrich Obergfell * 558bcfba4f4SUlrich Obergfell * The barriers are for syncing up watchdog_enabled across all the 559bcfba4f4SUlrich Obergfell * cpus, as clear_bit() does not use barriers. 560bcfba4f4SUlrich Obergfell */ 561bcfba4f4SUlrich Obergfell smp_mb__before_atomic(); 562bcfba4f4SUlrich Obergfell clear_bit(NMI_WATCHDOG_ENABLED_BIT, &watchdog_enabled); 563bcfba4f4SUlrich Obergfell smp_mb__after_atomic(); 564bcfba4f4SUlrich Obergfell 565a7027046SDon Zickus /* skip displaying the same error again */ 566a7027046SDon Zickus if (cpu > 0 && (PTR_ERR(event) == cpu0_err)) 567a7027046SDon Zickus return PTR_ERR(event); 5685651f7f4SDon Zickus 5695651f7f4SDon Zickus /* vary the KERN level based on the returned errno */ 5705651f7f4SDon Zickus if (PTR_ERR(event) == -EOPNOTSUPP) 5714501980aSAndrew Morton pr_info("disabled (cpu%i): not supported (no LAPIC?)\n", cpu); 5725651f7f4SDon Zickus else if (PTR_ERR(event) == -ENOENT) 573656c3b79SFabian Frederick pr_warn("disabled (cpu%i): hardware events not enabled\n", 5744501980aSAndrew Morton cpu); 5755651f7f4SDon Zickus else 5764501980aSAndrew Morton pr_err("disabled (cpu%i): unable to create perf event: %ld\n", 5774501980aSAndrew Morton cpu, PTR_ERR(event)); 578bcfba4f4SUlrich Obergfell 579bcfba4f4SUlrich Obergfell pr_info("Shutting down hard lockup detector on all cpus\n"); 580bcfba4f4SUlrich Obergfell 581eac24335SAkinobu Mita return PTR_ERR(event); 58258687acbSDon Zickus 58358687acbSDon Zickus /* success path */ 58458687acbSDon Zickus out_save: 58558687acbSDon Zickus per_cpu(watchdog_ev, cpu) = event; 58658687acbSDon Zickus out_enable: 58758687acbSDon Zickus perf_event_enable(per_cpu(watchdog_ev, cpu)); 58858687acbSDon Zickus out: 58958687acbSDon Zickus return 0; 59058687acbSDon Zickus } 59158687acbSDon Zickus 592bcd951cfSThomas Gleixner static void watchdog_nmi_disable(unsigned int cpu) 59358687acbSDon Zickus { 59458687acbSDon Zickus struct perf_event *event = per_cpu(watchdog_ev, cpu); 59558687acbSDon Zickus 59658687acbSDon Zickus if (event) { 59758687acbSDon Zickus perf_event_disable(event); 59858687acbSDon Zickus per_cpu(watchdog_ev, cpu) = NULL; 59958687acbSDon Zickus 60058687acbSDon Zickus /* should be in cleanup, but blocks oprofile */ 60158687acbSDon Zickus perf_event_release_kernel(event); 60258687acbSDon Zickus } 603df577149SUlrich Obergfell if (cpu == 0) { 604df577149SUlrich Obergfell /* watchdog_nmi_enable() expects this to be zero initially. */ 605df577149SUlrich Obergfell cpu0_err = 0; 606df577149SUlrich Obergfell } 60758687acbSDon Zickus } 608b3738d29SStephane Eranian 609b3738d29SStephane Eranian void watchdog_nmi_enable_all(void) 610b3738d29SStephane Eranian { 611b3738d29SStephane Eranian int cpu; 612b3738d29SStephane Eranian 613ab992dc3SPeter Zijlstra mutex_lock(&watchdog_proc_mutex); 614ab992dc3SPeter Zijlstra 615ab992dc3SPeter Zijlstra if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) 616ab992dc3SPeter Zijlstra goto unlock; 617b3738d29SStephane Eranian 618b3738d29SStephane Eranian get_online_cpus(); 619b3738d29SStephane Eranian for_each_online_cpu(cpu) 620b3738d29SStephane Eranian watchdog_nmi_enable(cpu); 621b3738d29SStephane Eranian put_online_cpus(); 622ab992dc3SPeter Zijlstra 623ab992dc3SPeter Zijlstra unlock: 6241173ff09SMichal Hocko mutex_unlock(&watchdog_proc_mutex); 625b3738d29SStephane Eranian } 626b3738d29SStephane Eranian 627b3738d29SStephane Eranian void watchdog_nmi_disable_all(void) 628b3738d29SStephane Eranian { 629b3738d29SStephane Eranian int cpu; 630b3738d29SStephane Eranian 631ab992dc3SPeter Zijlstra mutex_lock(&watchdog_proc_mutex); 632ab992dc3SPeter Zijlstra 633b3738d29SStephane Eranian if (!watchdog_running) 634ab992dc3SPeter Zijlstra goto unlock; 635b3738d29SStephane Eranian 636b3738d29SStephane Eranian get_online_cpus(); 637b3738d29SStephane Eranian for_each_online_cpu(cpu) 638b3738d29SStephane Eranian watchdog_nmi_disable(cpu); 639b3738d29SStephane Eranian put_online_cpus(); 640ab992dc3SPeter Zijlstra 641ab992dc3SPeter Zijlstra unlock: 642ab992dc3SPeter Zijlstra mutex_unlock(&watchdog_proc_mutex); 643b3738d29SStephane Eranian } 64458687acbSDon Zickus #else 645bcd951cfSThomas Gleixner static int watchdog_nmi_enable(unsigned int cpu) { return 0; } 646bcd951cfSThomas Gleixner static void watchdog_nmi_disable(unsigned int cpu) { return; } 647b3738d29SStephane Eranian void watchdog_nmi_enable_all(void) {} 648b3738d29SStephane Eranian void watchdog_nmi_disable_all(void) {} 64923637d47SFrederic Weisbecker #endif /* CONFIG_HARDLOCKUP_DETECTOR */ 65058687acbSDon Zickus 651b8900bc0SFrederic Weisbecker static struct smp_hotplug_thread watchdog_threads = { 652b8900bc0SFrederic Weisbecker .store = &softlockup_watchdog, 653b8900bc0SFrederic Weisbecker .thread_should_run = watchdog_should_run, 654b8900bc0SFrederic Weisbecker .thread_fn = watchdog, 655b8900bc0SFrederic Weisbecker .thread_comm = "watchdog/%u", 656b8900bc0SFrederic Weisbecker .setup = watchdog_enable, 657b8900bc0SFrederic Weisbecker .cleanup = watchdog_cleanup, 658b8900bc0SFrederic Weisbecker .park = watchdog_disable, 659b8900bc0SFrederic Weisbecker .unpark = watchdog_enable, 660b8900bc0SFrederic Weisbecker }; 661b8900bc0SFrederic Weisbecker 6629809b18fSMichal Hocko static void restart_watchdog_hrtimer(void *info) 6639809b18fSMichal Hocko { 664f7f66b05SChristoph Lameter struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); 6659809b18fSMichal Hocko int ret; 6669809b18fSMichal Hocko 6679809b18fSMichal Hocko /* 6689809b18fSMichal Hocko * No need to cancel and restart hrtimer if it is currently executing 6699809b18fSMichal Hocko * because it will reprogram itself with the new period now. 6709809b18fSMichal Hocko * We should never see it unqueued here because we are running per-cpu 6719809b18fSMichal Hocko * with interrupts disabled. 6729809b18fSMichal Hocko */ 6739809b18fSMichal Hocko ret = hrtimer_try_to_cancel(hrtimer); 6749809b18fSMichal Hocko if (ret == 1) 6759809b18fSMichal Hocko hrtimer_start(hrtimer, ns_to_ktime(sample_period), 6769809b18fSMichal Hocko HRTIMER_MODE_REL_PINNED); 6779809b18fSMichal Hocko } 6789809b18fSMichal Hocko 679b2f57c3aSUlrich Obergfell static void update_watchdog(int cpu) 6809809b18fSMichal Hocko { 6819809b18fSMichal Hocko /* 6829809b18fSMichal Hocko * Make sure that perf event counter will adopt to a new 6839809b18fSMichal Hocko * sampling period. Updating the sampling period directly would 6849809b18fSMichal Hocko * be much nicer but we do not have an API for that now so 6859809b18fSMichal Hocko * let's use a big hammer. 6869809b18fSMichal Hocko * Hrtimer will adopt the new period on the next tick but this 6879809b18fSMichal Hocko * might be late already so we have to restart the timer as well. 6889809b18fSMichal Hocko */ 6899809b18fSMichal Hocko watchdog_nmi_disable(cpu); 690e0a23b06SFrederic Weisbecker smp_call_function_single(cpu, restart_watchdog_hrtimer, NULL, 1); 6919809b18fSMichal Hocko watchdog_nmi_enable(cpu); 6929809b18fSMichal Hocko } 6939809b18fSMichal Hocko 694b2f57c3aSUlrich Obergfell static void update_watchdog_all_cpus(void) 6959809b18fSMichal Hocko { 6969809b18fSMichal Hocko int cpu; 6979809b18fSMichal Hocko 6989809b18fSMichal Hocko get_online_cpus(); 6999809b18fSMichal Hocko for_each_online_cpu(cpu) 700b2f57c3aSUlrich Obergfell update_watchdog(cpu); 7019809b18fSMichal Hocko put_online_cpus(); 7029809b18fSMichal Hocko } 7039809b18fSMichal Hocko 704b2f57c3aSUlrich Obergfell static int watchdog_enable_all_cpus(void) 705b8900bc0SFrederic Weisbecker { 706b8900bc0SFrederic Weisbecker int err = 0; 707b8900bc0SFrederic Weisbecker 7083c00ea82SFrederic Weisbecker if (!watchdog_running) { 709b8900bc0SFrederic Weisbecker err = smpboot_register_percpu_thread(&watchdog_threads); 710b8900bc0SFrederic Weisbecker if (err) 711b8900bc0SFrederic Weisbecker pr_err("Failed to create watchdog threads, disabled\n"); 712b8900bc0SFrederic Weisbecker else 7133c00ea82SFrederic Weisbecker watchdog_running = 1; 714b2f57c3aSUlrich Obergfell } else { 715b2f57c3aSUlrich Obergfell /* 716b2f57c3aSUlrich Obergfell * Enable/disable the lockup detectors or 717b2f57c3aSUlrich Obergfell * change the sample period 'on the fly'. 718b2f57c3aSUlrich Obergfell */ 719b2f57c3aSUlrich Obergfell update_watchdog_all_cpus(); 720b8900bc0SFrederic Weisbecker } 721b8900bc0SFrederic Weisbecker 722b8900bc0SFrederic Weisbecker return err; 723b8900bc0SFrederic Weisbecker } 724b8900bc0SFrederic Weisbecker 72558687acbSDon Zickus /* prepare/enable/disable routines */ 7264ff81951SVasily Averin /* sysctl functions */ 7274ff81951SVasily Averin #ifdef CONFIG_SYSCTL 72858687acbSDon Zickus static void watchdog_disable_all_cpus(void) 72958687acbSDon Zickus { 7303c00ea82SFrederic Weisbecker if (watchdog_running) { 7313c00ea82SFrederic Weisbecker watchdog_running = 0; 732b8900bc0SFrederic Weisbecker smpboot_unregister_percpu_thread(&watchdog_threads); 73358687acbSDon Zickus } 734bcd951cfSThomas Gleixner } 73558687acbSDon Zickus 73658687acbSDon Zickus /* 737a0c9cbb9SUlrich Obergfell * Update the run state of the lockup detectors. 73858687acbSDon Zickus */ 739a0c9cbb9SUlrich Obergfell static int proc_watchdog_update(void) 74058687acbSDon Zickus { 741a0c9cbb9SUlrich Obergfell int err = 0; 742a0c9cbb9SUlrich Obergfell 743a0c9cbb9SUlrich Obergfell /* 744a0c9cbb9SUlrich Obergfell * Watchdog threads won't be started if they are already active. 745a0c9cbb9SUlrich Obergfell * The 'watchdog_running' variable in watchdog_*_all_cpus() takes 746a0c9cbb9SUlrich Obergfell * care of this. If those threads are already active, the sample 747a0c9cbb9SUlrich Obergfell * period will be updated and the lockup detectors will be enabled 748a0c9cbb9SUlrich Obergfell * or disabled 'on the fly'. 749a0c9cbb9SUlrich Obergfell */ 750a0c9cbb9SUlrich Obergfell if (watchdog_enabled && watchdog_thresh) 751b2f57c3aSUlrich Obergfell err = watchdog_enable_all_cpus(); 752a0c9cbb9SUlrich Obergfell else 753a0c9cbb9SUlrich Obergfell watchdog_disable_all_cpus(); 754a0c9cbb9SUlrich Obergfell 755a0c9cbb9SUlrich Obergfell return err; 756a0c9cbb9SUlrich Obergfell 757a0c9cbb9SUlrich Obergfell } 758a0c9cbb9SUlrich Obergfell 759a0c9cbb9SUlrich Obergfell /* 760ef246a21SUlrich Obergfell * common function for watchdog, nmi_watchdog and soft_watchdog parameter 761ef246a21SUlrich Obergfell * 762ef246a21SUlrich Obergfell * caller | table->data points to | 'which' contains the flag(s) 763ef246a21SUlrich Obergfell * -------------------|-----------------------|----------------------------- 764ef246a21SUlrich Obergfell * proc_watchdog | watchdog_user_enabled | NMI_WATCHDOG_ENABLED or'ed 765ef246a21SUlrich Obergfell * | | with SOFT_WATCHDOG_ENABLED 766ef246a21SUlrich Obergfell * -------------------|-----------------------|----------------------------- 767ef246a21SUlrich Obergfell * proc_nmi_watchdog | nmi_watchdog_enabled | NMI_WATCHDOG_ENABLED 768ef246a21SUlrich Obergfell * -------------------|-----------------------|----------------------------- 769ef246a21SUlrich Obergfell * proc_soft_watchdog | soft_watchdog_enabled | SOFT_WATCHDOG_ENABLED 770ef246a21SUlrich Obergfell */ 771ef246a21SUlrich Obergfell static int proc_watchdog_common(int which, struct ctl_table *table, int write, 772ef246a21SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 773ef246a21SUlrich Obergfell { 774ef246a21SUlrich Obergfell int err, old, new; 775ef246a21SUlrich Obergfell int *watchdog_param = (int *)table->data; 776bcd951cfSThomas Gleixner 777ef246a21SUlrich Obergfell mutex_lock(&watchdog_proc_mutex); 778ef246a21SUlrich Obergfell 779ef246a21SUlrich Obergfell /* 780ef246a21SUlrich Obergfell * If the parameter is being read return the state of the corresponding 781ef246a21SUlrich Obergfell * bit(s) in 'watchdog_enabled', else update 'watchdog_enabled' and the 782ef246a21SUlrich Obergfell * run state of the lockup detectors. 783ef246a21SUlrich Obergfell */ 784ef246a21SUlrich Obergfell if (!write) { 785ef246a21SUlrich Obergfell *watchdog_param = (watchdog_enabled & which) != 0; 786b8900bc0SFrederic Weisbecker err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 787ef246a21SUlrich Obergfell } else { 788ef246a21SUlrich Obergfell err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 789ef246a21SUlrich Obergfell if (err) 790ef246a21SUlrich Obergfell goto out; 791ef246a21SUlrich Obergfell 792ef246a21SUlrich Obergfell /* 793ef246a21SUlrich Obergfell * There is a race window between fetching the current value 794ef246a21SUlrich Obergfell * from 'watchdog_enabled' and storing the new value. During 795ef246a21SUlrich Obergfell * this race window, watchdog_nmi_enable() can sneak in and 796ef246a21SUlrich Obergfell * clear the NMI_WATCHDOG_ENABLED bit in 'watchdog_enabled'. 797ef246a21SUlrich Obergfell * The 'cmpxchg' detects this race and the loop retries. 798ef246a21SUlrich Obergfell */ 799ef246a21SUlrich Obergfell do { 800ef246a21SUlrich Obergfell old = watchdog_enabled; 801ef246a21SUlrich Obergfell /* 802ef246a21SUlrich Obergfell * If the parameter value is not zero set the 803ef246a21SUlrich Obergfell * corresponding bit(s), else clear it(them). 804ef246a21SUlrich Obergfell */ 805ef246a21SUlrich Obergfell if (*watchdog_param) 806ef246a21SUlrich Obergfell new = old | which; 807ef246a21SUlrich Obergfell else 808ef246a21SUlrich Obergfell new = old & ~which; 809ef246a21SUlrich Obergfell } while (cmpxchg(&watchdog_enabled, old, new) != old); 810ef246a21SUlrich Obergfell 811ef246a21SUlrich Obergfell /* 812ef246a21SUlrich Obergfell * Update the run state of the lockup detectors. 813ef246a21SUlrich Obergfell * Restore 'watchdog_enabled' on failure. 814ef246a21SUlrich Obergfell */ 815ef246a21SUlrich Obergfell err = proc_watchdog_update(); 816ef246a21SUlrich Obergfell if (err) 817ef246a21SUlrich Obergfell watchdog_enabled = old; 818ef246a21SUlrich Obergfell } 819ef246a21SUlrich Obergfell out: 820ef246a21SUlrich Obergfell mutex_unlock(&watchdog_proc_mutex); 821ef246a21SUlrich Obergfell return err; 822ef246a21SUlrich Obergfell } 823ef246a21SUlrich Obergfell 824ef246a21SUlrich Obergfell /* 82583a80a39SUlrich Obergfell * /proc/sys/kernel/watchdog 82683a80a39SUlrich Obergfell */ 82783a80a39SUlrich Obergfell int proc_watchdog(struct ctl_table *table, int write, 82883a80a39SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 82983a80a39SUlrich Obergfell { 83083a80a39SUlrich Obergfell return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED, 83183a80a39SUlrich Obergfell table, write, buffer, lenp, ppos); 83283a80a39SUlrich Obergfell } 83383a80a39SUlrich Obergfell 83483a80a39SUlrich Obergfell /* 83583a80a39SUlrich Obergfell * /proc/sys/kernel/nmi_watchdog 83683a80a39SUlrich Obergfell */ 83783a80a39SUlrich Obergfell int proc_nmi_watchdog(struct ctl_table *table, int write, 83883a80a39SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 83983a80a39SUlrich Obergfell { 84083a80a39SUlrich Obergfell return proc_watchdog_common(NMI_WATCHDOG_ENABLED, 84183a80a39SUlrich Obergfell table, write, buffer, lenp, ppos); 84283a80a39SUlrich Obergfell } 84383a80a39SUlrich Obergfell 84483a80a39SUlrich Obergfell /* 84583a80a39SUlrich Obergfell * /proc/sys/kernel/soft_watchdog 84683a80a39SUlrich Obergfell */ 84783a80a39SUlrich Obergfell int proc_soft_watchdog(struct ctl_table *table, int write, 84883a80a39SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 84983a80a39SUlrich Obergfell { 85083a80a39SUlrich Obergfell return proc_watchdog_common(SOFT_WATCHDOG_ENABLED, 85183a80a39SUlrich Obergfell table, write, buffer, lenp, ppos); 85283a80a39SUlrich Obergfell } 85383a80a39SUlrich Obergfell 85483a80a39SUlrich Obergfell /* 85583a80a39SUlrich Obergfell * /proc/sys/kernel/watchdog_thresh 85683a80a39SUlrich Obergfell */ 85783a80a39SUlrich Obergfell int proc_watchdog_thresh(struct ctl_table *table, int write, 85883a80a39SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 85983a80a39SUlrich Obergfell { 86083a80a39SUlrich Obergfell int err, old; 86183a80a39SUlrich Obergfell 86283a80a39SUlrich Obergfell mutex_lock(&watchdog_proc_mutex); 86383a80a39SUlrich Obergfell 86483a80a39SUlrich Obergfell old = ACCESS_ONCE(watchdog_thresh); 86583a80a39SUlrich Obergfell err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 86683a80a39SUlrich Obergfell 867b8900bc0SFrederic Weisbecker if (err || !write) 868359e6fabSMichal Hocko goto out; 869e04ab2bcSMandeep Singh Baines 87083a80a39SUlrich Obergfell /* 87183a80a39SUlrich Obergfell * Update the sample period. 87283a80a39SUlrich Obergfell * Restore 'watchdog_thresh' on failure. 87383a80a39SUlrich Obergfell */ 8740f34c400SChuansheng Liu set_sample_period(); 87583a80a39SUlrich Obergfell err = proc_watchdog_update(); 87683a80a39SUlrich Obergfell if (err) 87783a80a39SUlrich Obergfell watchdog_thresh = old; 878359e6fabSMichal Hocko out: 879359e6fabSMichal Hocko mutex_unlock(&watchdog_proc_mutex); 880b8900bc0SFrederic Weisbecker return err; 88158687acbSDon Zickus } 88258687acbSDon Zickus #endif /* CONFIG_SYSCTL */ 88358687acbSDon Zickus 884004417a6SPeter Zijlstra void __init lockup_detector_init(void) 88558687acbSDon Zickus { 8860f34c400SChuansheng Liu set_sample_period(); 887b8900bc0SFrederic Weisbecker 888195daf66SUlrich Obergfell if (watchdog_enabled) 889b2f57c3aSUlrich Obergfell watchdog_enable_all_cpus(); 89058687acbSDon Zickus } 891