158687acbSDon Zickus /* 258687acbSDon Zickus * Detect hard and soft lockups on a system 358687acbSDon Zickus * 458687acbSDon Zickus * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc. 558687acbSDon Zickus * 686f5e6a7SFernando Luis Vázquez Cao * Note: Most of this code is borrowed heavily from the original softlockup 786f5e6a7SFernando Luis Vázquez Cao * detector, so thanks to Ingo for the initial implementation. 886f5e6a7SFernando Luis Vázquez Cao * Some chunks also taken from the old x86-specific nmi watchdog code, thanks 958687acbSDon Zickus * to those contributors as well. 1058687acbSDon Zickus */ 1158687acbSDon Zickus 124501980aSAndrew Morton #define pr_fmt(fmt) "NMI watchdog: " fmt 134501980aSAndrew Morton 1458687acbSDon Zickus #include <linux/mm.h> 1558687acbSDon Zickus #include <linux/cpu.h> 1658687acbSDon Zickus #include <linux/nmi.h> 1758687acbSDon Zickus #include <linux/init.h> 1858687acbSDon Zickus #include <linux/module.h> 1958687acbSDon Zickus #include <linux/sysctl.h> 20bcd951cfSThomas Gleixner #include <linux/smpboot.h> 218bd75c77SClark Williams #include <linux/sched/rt.h> 22fe4ba3c3SChris Metcalf #include <linux/tick.h> 2358687acbSDon Zickus 2458687acbSDon Zickus #include <asm/irq_regs.h> 255d1c0f4aSEric B Munson #include <linux/kvm_para.h> 2658687acbSDon Zickus #include <linux/perf_event.h> 2758687acbSDon Zickus 2884d56e66SUlrich Obergfell /* 2984d56e66SUlrich Obergfell * The run state of the lockup detectors is controlled by the content of the 3084d56e66SUlrich Obergfell * 'watchdog_enabled' variable. Each lockup detector has its dedicated bit - 3184d56e66SUlrich Obergfell * bit 0 for the hard lockup detector and bit 1 for the soft lockup detector. 3284d56e66SUlrich Obergfell * 3384d56e66SUlrich Obergfell * 'watchdog_user_enabled', 'nmi_watchdog_enabled' and 'soft_watchdog_enabled' 3484d56e66SUlrich Obergfell * are variables that are only used as an 'interface' between the parameters 3584d56e66SUlrich Obergfell * in /proc/sys/kernel and the internal state bits in 'watchdog_enabled'. The 3684d56e66SUlrich Obergfell * 'watchdog_thresh' variable is handled differently because its value is not 3784d56e66SUlrich Obergfell * boolean, and the lockup detectors are 'suspended' while 'watchdog_thresh' 3884d56e66SUlrich Obergfell * is equal zero. 3984d56e66SUlrich Obergfell */ 4084d56e66SUlrich Obergfell #define NMI_WATCHDOG_ENABLED_BIT 0 4184d56e66SUlrich Obergfell #define SOFT_WATCHDOG_ENABLED_BIT 1 4284d56e66SUlrich Obergfell #define NMI_WATCHDOG_ENABLED (1 << NMI_WATCHDOG_ENABLED_BIT) 4384d56e66SUlrich Obergfell #define SOFT_WATCHDOG_ENABLED (1 << SOFT_WATCHDOG_ENABLED_BIT) 4484d56e66SUlrich Obergfell 45ab992dc3SPeter Zijlstra static DEFINE_MUTEX(watchdog_proc_mutex); 46ab992dc3SPeter Zijlstra 4784d56e66SUlrich Obergfell #ifdef CONFIG_HARDLOCKUP_DETECTOR 4884d56e66SUlrich Obergfell static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED|NMI_WATCHDOG_ENABLED; 4984d56e66SUlrich Obergfell #else 5084d56e66SUlrich Obergfell static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED; 5184d56e66SUlrich Obergfell #endif 5284d56e66SUlrich Obergfell int __read_mostly nmi_watchdog_enabled; 5384d56e66SUlrich Obergfell int __read_mostly soft_watchdog_enabled; 5484d56e66SUlrich Obergfell int __read_mostly watchdog_user_enabled; 554eec42f3SMandeep Singh Baines int __read_mostly watchdog_thresh = 10; 5684d56e66SUlrich Obergfell 57ed235875SAaron Tomlin #ifdef CONFIG_SMP 58ed235875SAaron Tomlin int __read_mostly sysctl_softlockup_all_cpu_backtrace; 59ed235875SAaron Tomlin #else 60ed235875SAaron Tomlin #define sysctl_softlockup_all_cpu_backtrace 0 61ed235875SAaron Tomlin #endif 62fe4ba3c3SChris Metcalf static struct cpumask watchdog_cpumask __read_mostly; 63fe4ba3c3SChris Metcalf unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask); 64fe4ba3c3SChris Metcalf 65fe4ba3c3SChris Metcalf /* Helper for online, unparked cpus. */ 66fe4ba3c3SChris Metcalf #define for_each_watchdog_cpu(cpu) \ 67fe4ba3c3SChris Metcalf for_each_cpu_and((cpu), cpu_online_mask, &watchdog_cpumask) 68ed235875SAaron Tomlin 693c00ea82SFrederic Weisbecker static int __read_mostly watchdog_running; 700f34c400SChuansheng Liu static u64 __read_mostly sample_period; 7158687acbSDon Zickus 7258687acbSDon Zickus static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts); 7358687acbSDon Zickus static DEFINE_PER_CPU(struct task_struct *, softlockup_watchdog); 7458687acbSDon Zickus static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer); 7558687acbSDon Zickus static DEFINE_PER_CPU(bool, softlockup_touch_sync); 7658687acbSDon Zickus static DEFINE_PER_CPU(bool, soft_watchdog_warn); 77bcd951cfSThomas Gleixner static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts); 78bcd951cfSThomas Gleixner static DEFINE_PER_CPU(unsigned long, soft_lockup_hrtimer_cnt); 79b1a8de1fSchai wen static DEFINE_PER_CPU(struct task_struct *, softlockup_task_ptr_saved); 8023637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 81cafcd80dSDon Zickus static DEFINE_PER_CPU(bool, hard_watchdog_warn); 82cafcd80dSDon Zickus static DEFINE_PER_CPU(bool, watchdog_nmi_touch); 8358687acbSDon Zickus static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved); 8458687acbSDon Zickus static DEFINE_PER_CPU(struct perf_event *, watchdog_ev); 8558687acbSDon Zickus #endif 86ed235875SAaron Tomlin static unsigned long soft_lockup_nmi_warn; 8758687acbSDon Zickus 8858687acbSDon Zickus /* boot commands */ 8958687acbSDon Zickus /* 9058687acbSDon Zickus * Should we panic when a soft-lockup or hard-lockup occurs: 9158687acbSDon Zickus */ 9223637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 93fef2c9bcSDon Zickus static int hardlockup_panic = 94fef2c9bcSDon Zickus CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE; 956e7458a6SUlrich Obergfell /* 966e7458a6SUlrich Obergfell * We may not want to enable hard lockup detection by default in all cases, 976e7458a6SUlrich Obergfell * for example when running the kernel as a guest on a hypervisor. In these 986e7458a6SUlrich Obergfell * cases this function can be called to disable hard lockup detection. This 996e7458a6SUlrich Obergfell * function should only be executed once by the boot processor before the 1006e7458a6SUlrich Obergfell * kernel command line parameters are parsed, because otherwise it is not 1016e7458a6SUlrich Obergfell * possible to override this in hardlockup_panic_setup(). 1026e7458a6SUlrich Obergfell */ 103692297d8SUlrich Obergfell void hardlockup_detector_disable(void) 1046e7458a6SUlrich Obergfell { 105692297d8SUlrich Obergfell watchdog_enabled &= ~NMI_WATCHDOG_ENABLED; 1066e7458a6SUlrich Obergfell } 1076e7458a6SUlrich Obergfell 10858687acbSDon Zickus static int __init hardlockup_panic_setup(char *str) 10958687acbSDon Zickus { 11058687acbSDon Zickus if (!strncmp(str, "panic", 5)) 11158687acbSDon Zickus hardlockup_panic = 1; 112fef2c9bcSDon Zickus else if (!strncmp(str, "nopanic", 7)) 113fef2c9bcSDon Zickus hardlockup_panic = 0; 1145dc30558SDon Zickus else if (!strncmp(str, "0", 1)) 115195daf66SUlrich Obergfell watchdog_enabled &= ~NMI_WATCHDOG_ENABLED; 116195daf66SUlrich Obergfell else if (!strncmp(str, "1", 1)) 117195daf66SUlrich Obergfell watchdog_enabled |= NMI_WATCHDOG_ENABLED; 11858687acbSDon Zickus return 1; 11958687acbSDon Zickus } 12058687acbSDon Zickus __setup("nmi_watchdog=", hardlockup_panic_setup); 12158687acbSDon Zickus #endif 12258687acbSDon Zickus 12358687acbSDon Zickus unsigned int __read_mostly softlockup_panic = 12458687acbSDon Zickus CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE; 12558687acbSDon Zickus 12658687acbSDon Zickus static int __init softlockup_panic_setup(char *str) 12758687acbSDon Zickus { 12858687acbSDon Zickus softlockup_panic = simple_strtoul(str, NULL, 0); 12958687acbSDon Zickus 13058687acbSDon Zickus return 1; 13158687acbSDon Zickus } 13258687acbSDon Zickus __setup("softlockup_panic=", softlockup_panic_setup); 13358687acbSDon Zickus 13458687acbSDon Zickus static int __init nowatchdog_setup(char *str) 13558687acbSDon Zickus { 136195daf66SUlrich Obergfell watchdog_enabled = 0; 13758687acbSDon Zickus return 1; 13858687acbSDon Zickus } 13958687acbSDon Zickus __setup("nowatchdog", nowatchdog_setup); 14058687acbSDon Zickus 14158687acbSDon Zickus static int __init nosoftlockup_setup(char *str) 14258687acbSDon Zickus { 143195daf66SUlrich Obergfell watchdog_enabled &= ~SOFT_WATCHDOG_ENABLED; 14458687acbSDon Zickus return 1; 14558687acbSDon Zickus } 14658687acbSDon Zickus __setup("nosoftlockup", nosoftlockup_setup); 147195daf66SUlrich Obergfell 148ed235875SAaron Tomlin #ifdef CONFIG_SMP 149ed235875SAaron Tomlin static int __init softlockup_all_cpu_backtrace_setup(char *str) 150ed235875SAaron Tomlin { 151ed235875SAaron Tomlin sysctl_softlockup_all_cpu_backtrace = 152ed235875SAaron Tomlin !!simple_strtol(str, NULL, 0); 153ed235875SAaron Tomlin return 1; 154ed235875SAaron Tomlin } 155ed235875SAaron Tomlin __setup("softlockup_all_cpu_backtrace=", softlockup_all_cpu_backtrace_setup); 156ed235875SAaron Tomlin #endif 15758687acbSDon Zickus 1584eec42f3SMandeep Singh Baines /* 1594eec42f3SMandeep Singh Baines * Hard-lockup warnings should be triggered after just a few seconds. Soft- 1604eec42f3SMandeep Singh Baines * lockups can have false positives under extreme conditions. So we generally 1614eec42f3SMandeep Singh Baines * want a higher threshold for soft lockups than for hard lockups. So we couple 1624eec42f3SMandeep Singh Baines * the thresholds with a factor: we make the soft threshold twice the amount of 1634eec42f3SMandeep Singh Baines * time the hard threshold is. 1644eec42f3SMandeep Singh Baines */ 1656e9101aeSIngo Molnar static int get_softlockup_thresh(void) 1664eec42f3SMandeep Singh Baines { 1674eec42f3SMandeep Singh Baines return watchdog_thresh * 2; 1684eec42f3SMandeep Singh Baines } 16958687acbSDon Zickus 17058687acbSDon Zickus /* 17158687acbSDon Zickus * Returns seconds, approximately. We don't need nanosecond 17258687acbSDon Zickus * resolution, and we don't need to waste time with a big divide when 17358687acbSDon Zickus * 2^30ns == 1.074s. 17458687acbSDon Zickus */ 175c06b4f19SNamhyung Kim static unsigned long get_timestamp(void) 17658687acbSDon Zickus { 177545a2bf7SCyril Bur return running_clock() >> 30LL; /* 2^30 ~= 10^9 */ 17858687acbSDon Zickus } 17958687acbSDon Zickus 1800f34c400SChuansheng Liu static void set_sample_period(void) 18158687acbSDon Zickus { 18258687acbSDon Zickus /* 183586692a5SMandeep Singh Baines * convert watchdog_thresh from seconds to ns 18486f5e6a7SFernando Luis Vázquez Cao * the divide by 5 is to give hrtimer several chances (two 18586f5e6a7SFernando Luis Vázquez Cao * or three with the current relation between the soft 18686f5e6a7SFernando Luis Vázquez Cao * and hard thresholds) to increment before the 18786f5e6a7SFernando Luis Vázquez Cao * hardlockup detector generates a warning 18858687acbSDon Zickus */ 1890f34c400SChuansheng Liu sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5); 19058687acbSDon Zickus } 19158687acbSDon Zickus 19258687acbSDon Zickus /* Commands for resetting the watchdog */ 19358687acbSDon Zickus static void __touch_watchdog(void) 19458687acbSDon Zickus { 195c06b4f19SNamhyung Kim __this_cpu_write(watchdog_touch_ts, get_timestamp()); 19658687acbSDon Zickus } 19758687acbSDon Zickus 198332fbdbcSDon Zickus void touch_softlockup_watchdog(void) 19958687acbSDon Zickus { 2007861144bSAndrew Morton /* 2017861144bSAndrew Morton * Preemption can be enabled. It doesn't matter which CPU's timestamp 2027861144bSAndrew Morton * gets zeroed here, so use the raw_ operation. 2037861144bSAndrew Morton */ 2047861144bSAndrew Morton raw_cpu_write(watchdog_touch_ts, 0); 20558687acbSDon Zickus } 2060167c781SIngo Molnar EXPORT_SYMBOL(touch_softlockup_watchdog); 20758687acbSDon Zickus 208332fbdbcSDon Zickus void touch_all_softlockup_watchdogs(void) 20958687acbSDon Zickus { 21058687acbSDon Zickus int cpu; 21158687acbSDon Zickus 21258687acbSDon Zickus /* 21358687acbSDon Zickus * this is done lockless 21458687acbSDon Zickus * do we care if a 0 races with a timestamp? 21558687acbSDon Zickus * all it means is the softlock check starts one cycle later 21658687acbSDon Zickus */ 217fe4ba3c3SChris Metcalf for_each_watchdog_cpu(cpu) 21858687acbSDon Zickus per_cpu(watchdog_touch_ts, cpu) = 0; 21958687acbSDon Zickus } 22058687acbSDon Zickus 221cafcd80dSDon Zickus #ifdef CONFIG_HARDLOCKUP_DETECTOR 22258687acbSDon Zickus void touch_nmi_watchdog(void) 22358687acbSDon Zickus { 22462572e29SBen Zhang /* 22562572e29SBen Zhang * Using __raw here because some code paths have 22662572e29SBen Zhang * preemption enabled. If preemption is enabled 22762572e29SBen Zhang * then interrupts should be enabled too, in which 22862572e29SBen Zhang * case we shouldn't have to worry about the watchdog 22962572e29SBen Zhang * going off. 23062572e29SBen Zhang */ 231f7f66b05SChristoph Lameter raw_cpu_write(watchdog_nmi_touch, true); 232332fbdbcSDon Zickus touch_softlockup_watchdog(); 23358687acbSDon Zickus } 23458687acbSDon Zickus EXPORT_SYMBOL(touch_nmi_watchdog); 23558687acbSDon Zickus 236cafcd80dSDon Zickus #endif 237cafcd80dSDon Zickus 23858687acbSDon Zickus void touch_softlockup_watchdog_sync(void) 23958687acbSDon Zickus { 240f7f66b05SChristoph Lameter __this_cpu_write(softlockup_touch_sync, true); 241f7f66b05SChristoph Lameter __this_cpu_write(watchdog_touch_ts, 0); 24258687acbSDon Zickus } 24358687acbSDon Zickus 24423637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 24558687acbSDon Zickus /* watchdog detector functions */ 24626e09c6eSDon Zickus static int is_hardlockup(void) 24758687acbSDon Zickus { 248909ea964SChristoph Lameter unsigned long hrint = __this_cpu_read(hrtimer_interrupts); 24958687acbSDon Zickus 250909ea964SChristoph Lameter if (__this_cpu_read(hrtimer_interrupts_saved) == hrint) 25158687acbSDon Zickus return 1; 25258687acbSDon Zickus 253909ea964SChristoph Lameter __this_cpu_write(hrtimer_interrupts_saved, hrint); 25458687acbSDon Zickus return 0; 25558687acbSDon Zickus } 25658687acbSDon Zickus #endif 25758687acbSDon Zickus 25826e09c6eSDon Zickus static int is_softlockup(unsigned long touch_ts) 25958687acbSDon Zickus { 260c06b4f19SNamhyung Kim unsigned long now = get_timestamp(); 26158687acbSDon Zickus 262195daf66SUlrich Obergfell if (watchdog_enabled & SOFT_WATCHDOG_ENABLED) { 263195daf66SUlrich Obergfell /* Warn about unreasonable delays. */ 2644eec42f3SMandeep Singh Baines if (time_after(now, touch_ts + get_softlockup_thresh())) 26558687acbSDon Zickus return now - touch_ts; 266195daf66SUlrich Obergfell } 26758687acbSDon Zickus return 0; 26858687acbSDon Zickus } 26958687acbSDon Zickus 27023637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 2711880c4aeSCyrill Gorcunov 27258687acbSDon Zickus static struct perf_event_attr wd_hw_attr = { 27358687acbSDon Zickus .type = PERF_TYPE_HARDWARE, 27458687acbSDon Zickus .config = PERF_COUNT_HW_CPU_CYCLES, 27558687acbSDon Zickus .size = sizeof(struct perf_event_attr), 27658687acbSDon Zickus .pinned = 1, 27758687acbSDon Zickus .disabled = 1, 27858687acbSDon Zickus }; 27958687acbSDon Zickus 28058687acbSDon Zickus /* Callback function for perf event subsystem */ 281a8b0ca17SPeter Zijlstra static void watchdog_overflow_callback(struct perf_event *event, 28258687acbSDon Zickus struct perf_sample_data *data, 28358687acbSDon Zickus struct pt_regs *regs) 28458687acbSDon Zickus { 285c6db67cdSPeter Zijlstra /* Ensure the watchdog never gets throttled */ 286c6db67cdSPeter Zijlstra event->hw.interrupts = 0; 287c6db67cdSPeter Zijlstra 288909ea964SChristoph Lameter if (__this_cpu_read(watchdog_nmi_touch) == true) { 289909ea964SChristoph Lameter __this_cpu_write(watchdog_nmi_touch, false); 29058687acbSDon Zickus return; 29158687acbSDon Zickus } 29258687acbSDon Zickus 29358687acbSDon Zickus /* check for a hardlockup 29458687acbSDon Zickus * This is done by making sure our timer interrupt 29558687acbSDon Zickus * is incrementing. The timer interrupt should have 29658687acbSDon Zickus * fired multiple times before we overflow'd. If it hasn't 29758687acbSDon Zickus * then this is a good indication the cpu is stuck 29858687acbSDon Zickus */ 29926e09c6eSDon Zickus if (is_hardlockup()) { 30026e09c6eSDon Zickus int this_cpu = smp_processor_id(); 30126e09c6eSDon Zickus 30258687acbSDon Zickus /* only print hardlockups once */ 303909ea964SChristoph Lameter if (__this_cpu_read(hard_watchdog_warn) == true) 30458687acbSDon Zickus return; 30558687acbSDon Zickus 30658687acbSDon Zickus if (hardlockup_panic) 307656c3b79SFabian Frederick panic("Watchdog detected hard LOCKUP on cpu %d", 308656c3b79SFabian Frederick this_cpu); 30958687acbSDon Zickus else 310656c3b79SFabian Frederick WARN(1, "Watchdog detected hard LOCKUP on cpu %d", 311656c3b79SFabian Frederick this_cpu); 31258687acbSDon Zickus 313909ea964SChristoph Lameter __this_cpu_write(hard_watchdog_warn, true); 31458687acbSDon Zickus return; 31558687acbSDon Zickus } 31658687acbSDon Zickus 317909ea964SChristoph Lameter __this_cpu_write(hard_watchdog_warn, false); 31858687acbSDon Zickus return; 31958687acbSDon Zickus } 320bcd951cfSThomas Gleixner #endif /* CONFIG_HARDLOCKUP_DETECTOR */ 321bcd951cfSThomas Gleixner 32258687acbSDon Zickus static void watchdog_interrupt_count(void) 32358687acbSDon Zickus { 324909ea964SChristoph Lameter __this_cpu_inc(hrtimer_interrupts); 32558687acbSDon Zickus } 326bcd951cfSThomas Gleixner 327bcd951cfSThomas Gleixner static int watchdog_nmi_enable(unsigned int cpu); 328bcd951cfSThomas Gleixner static void watchdog_nmi_disable(unsigned int cpu); 32958687acbSDon Zickus 33058687acbSDon Zickus /* watchdog kicker functions */ 33158687acbSDon Zickus static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) 33258687acbSDon Zickus { 333909ea964SChristoph Lameter unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts); 33458687acbSDon Zickus struct pt_regs *regs = get_irq_regs(); 33558687acbSDon Zickus int duration; 336ed235875SAaron Tomlin int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace; 33758687acbSDon Zickus 33858687acbSDon Zickus /* kick the hardlockup detector */ 33958687acbSDon Zickus watchdog_interrupt_count(); 34058687acbSDon Zickus 34158687acbSDon Zickus /* kick the softlockup detector */ 342909ea964SChristoph Lameter wake_up_process(__this_cpu_read(softlockup_watchdog)); 34358687acbSDon Zickus 34458687acbSDon Zickus /* .. and repeat */ 3450f34c400SChuansheng Liu hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); 34658687acbSDon Zickus 34758687acbSDon Zickus if (touch_ts == 0) { 348909ea964SChristoph Lameter if (unlikely(__this_cpu_read(softlockup_touch_sync))) { 34958687acbSDon Zickus /* 35058687acbSDon Zickus * If the time stamp was touched atomically 35158687acbSDon Zickus * make sure the scheduler tick is up to date. 35258687acbSDon Zickus */ 353909ea964SChristoph Lameter __this_cpu_write(softlockup_touch_sync, false); 35458687acbSDon Zickus sched_clock_tick(); 35558687acbSDon Zickus } 3565d1c0f4aSEric B Munson 3575d1c0f4aSEric B Munson /* Clear the guest paused flag on watchdog reset */ 3585d1c0f4aSEric B Munson kvm_check_and_clear_guest_paused(); 35958687acbSDon Zickus __touch_watchdog(); 36058687acbSDon Zickus return HRTIMER_RESTART; 36158687acbSDon Zickus } 36258687acbSDon Zickus 36358687acbSDon Zickus /* check for a softlockup 36458687acbSDon Zickus * This is done by making sure a high priority task is 36558687acbSDon Zickus * being scheduled. The task touches the watchdog to 36658687acbSDon Zickus * indicate it is getting cpu time. If it hasn't then 36758687acbSDon Zickus * this is a good indication some task is hogging the cpu 36858687acbSDon Zickus */ 36926e09c6eSDon Zickus duration = is_softlockup(touch_ts); 37058687acbSDon Zickus if (unlikely(duration)) { 3715d1c0f4aSEric B Munson /* 3725d1c0f4aSEric B Munson * If a virtual machine is stopped by the host it can look to 3735d1c0f4aSEric B Munson * the watchdog like a soft lockup, check to see if the host 3745d1c0f4aSEric B Munson * stopped the vm before we issue the warning 3755d1c0f4aSEric B Munson */ 3765d1c0f4aSEric B Munson if (kvm_check_and_clear_guest_paused()) 3775d1c0f4aSEric B Munson return HRTIMER_RESTART; 3785d1c0f4aSEric B Munson 37958687acbSDon Zickus /* only warn once */ 380b1a8de1fSchai wen if (__this_cpu_read(soft_watchdog_warn) == true) { 381b1a8de1fSchai wen /* 382b1a8de1fSchai wen * When multiple processes are causing softlockups the 383b1a8de1fSchai wen * softlockup detector only warns on the first one 384b1a8de1fSchai wen * because the code relies on a full quiet cycle to 385b1a8de1fSchai wen * re-arm. The second process prevents the quiet cycle 386b1a8de1fSchai wen * and never gets reported. Use task pointers to detect 387b1a8de1fSchai wen * this. 388b1a8de1fSchai wen */ 389b1a8de1fSchai wen if (__this_cpu_read(softlockup_task_ptr_saved) != 390b1a8de1fSchai wen current) { 391b1a8de1fSchai wen __this_cpu_write(soft_watchdog_warn, false); 392b1a8de1fSchai wen __touch_watchdog(); 393b1a8de1fSchai wen } 39458687acbSDon Zickus return HRTIMER_RESTART; 395b1a8de1fSchai wen } 39658687acbSDon Zickus 397ed235875SAaron Tomlin if (softlockup_all_cpu_backtrace) { 398ed235875SAaron Tomlin /* Prevent multiple soft-lockup reports if one cpu is already 399ed235875SAaron Tomlin * engaged in dumping cpu back traces 400ed235875SAaron Tomlin */ 401ed235875SAaron Tomlin if (test_and_set_bit(0, &soft_lockup_nmi_warn)) { 402ed235875SAaron Tomlin /* Someone else will report us. Let's give up */ 403ed235875SAaron Tomlin __this_cpu_write(soft_watchdog_warn, true); 404ed235875SAaron Tomlin return HRTIMER_RESTART; 405ed235875SAaron Tomlin } 406ed235875SAaron Tomlin } 407ed235875SAaron Tomlin 408656c3b79SFabian Frederick pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n", 40926e09c6eSDon Zickus smp_processor_id(), duration, 41058687acbSDon Zickus current->comm, task_pid_nr(current)); 411b1a8de1fSchai wen __this_cpu_write(softlockup_task_ptr_saved, current); 41258687acbSDon Zickus print_modules(); 41358687acbSDon Zickus print_irqtrace_events(current); 41458687acbSDon Zickus if (regs) 41558687acbSDon Zickus show_regs(regs); 41658687acbSDon Zickus else 41758687acbSDon Zickus dump_stack(); 41858687acbSDon Zickus 419ed235875SAaron Tomlin if (softlockup_all_cpu_backtrace) { 420ed235875SAaron Tomlin /* Avoid generating two back traces for current 421ed235875SAaron Tomlin * given that one is already made above 422ed235875SAaron Tomlin */ 423ed235875SAaron Tomlin trigger_allbutself_cpu_backtrace(); 424ed235875SAaron Tomlin 425ed235875SAaron Tomlin clear_bit(0, &soft_lockup_nmi_warn); 426ed235875SAaron Tomlin /* Barrier to sync with other cpus */ 427ed235875SAaron Tomlin smp_mb__after_atomic(); 428ed235875SAaron Tomlin } 429ed235875SAaron Tomlin 43069361eefSJosh Hunt add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 43158687acbSDon Zickus if (softlockup_panic) 43258687acbSDon Zickus panic("softlockup: hung tasks"); 433909ea964SChristoph Lameter __this_cpu_write(soft_watchdog_warn, true); 43458687acbSDon Zickus } else 435909ea964SChristoph Lameter __this_cpu_write(soft_watchdog_warn, false); 43658687acbSDon Zickus 43758687acbSDon Zickus return HRTIMER_RESTART; 43858687acbSDon Zickus } 43958687acbSDon Zickus 440bcd951cfSThomas Gleixner static void watchdog_set_prio(unsigned int policy, unsigned int prio) 44158687acbSDon Zickus { 442bcd951cfSThomas Gleixner struct sched_param param = { .sched_priority = prio }; 443bcd951cfSThomas Gleixner 444bcd951cfSThomas Gleixner sched_setscheduler(current, policy, ¶m); 445bcd951cfSThomas Gleixner } 446bcd951cfSThomas Gleixner 447bcd951cfSThomas Gleixner static void watchdog_enable(unsigned int cpu) 448bcd951cfSThomas Gleixner { 449f7f66b05SChristoph Lameter struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); 45058687acbSDon Zickus 4513935e895SBjørn Mork /* kick off the timer for the hardlockup detector */ 4523935e895SBjørn Mork hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 4533935e895SBjørn Mork hrtimer->function = watchdog_timer_fn; 4543935e895SBjørn Mork 455bcd951cfSThomas Gleixner /* Enable the perf event */ 456bcd951cfSThomas Gleixner watchdog_nmi_enable(cpu); 45758687acbSDon Zickus 45858687acbSDon Zickus /* done here because hrtimer_start can only pin to smp_processor_id() */ 4590f34c400SChuansheng Liu hrtimer_start(hrtimer, ns_to_ktime(sample_period), 46058687acbSDon Zickus HRTIMER_MODE_REL_PINNED); 46158687acbSDon Zickus 462bcd951cfSThomas Gleixner /* initialize timestamp */ 463bcd951cfSThomas Gleixner watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1); 46458687acbSDon Zickus __touch_watchdog(); 46558687acbSDon Zickus } 466bcd951cfSThomas Gleixner 467bcd951cfSThomas Gleixner static void watchdog_disable(unsigned int cpu) 468bcd951cfSThomas Gleixner { 469f7f66b05SChristoph Lameter struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); 470bcd951cfSThomas Gleixner 471bcd951cfSThomas Gleixner watchdog_set_prio(SCHED_NORMAL, 0); 472bcd951cfSThomas Gleixner hrtimer_cancel(hrtimer); 473bcd951cfSThomas Gleixner /* disable the perf event */ 474bcd951cfSThomas Gleixner watchdog_nmi_disable(cpu); 475bcd951cfSThomas Gleixner } 476bcd951cfSThomas Gleixner 477b8900bc0SFrederic Weisbecker static void watchdog_cleanup(unsigned int cpu, bool online) 478b8900bc0SFrederic Weisbecker { 479b8900bc0SFrederic Weisbecker watchdog_disable(cpu); 480b8900bc0SFrederic Weisbecker } 481b8900bc0SFrederic Weisbecker 482bcd951cfSThomas Gleixner static int watchdog_should_run(unsigned int cpu) 483bcd951cfSThomas Gleixner { 484bcd951cfSThomas Gleixner return __this_cpu_read(hrtimer_interrupts) != 485bcd951cfSThomas Gleixner __this_cpu_read(soft_lockup_hrtimer_cnt); 486bcd951cfSThomas Gleixner } 487bcd951cfSThomas Gleixner 488b60f796cSAndrew Morton /* 489bcd951cfSThomas Gleixner * The watchdog thread function - touches the timestamp. 490bcd951cfSThomas Gleixner * 4910f34c400SChuansheng Liu * It only runs once every sample_period seconds (4 seconds by 492bcd951cfSThomas Gleixner * default) to reset the softlockup timestamp. If this gets delayed 493bcd951cfSThomas Gleixner * for more than 2*watchdog_thresh seconds then the debug-printout 494bcd951cfSThomas Gleixner * triggers in watchdog_timer_fn(). 495b60f796cSAndrew Morton */ 496bcd951cfSThomas Gleixner static void watchdog(unsigned int cpu) 497bcd951cfSThomas Gleixner { 498bcd951cfSThomas Gleixner __this_cpu_write(soft_lockup_hrtimer_cnt, 499bcd951cfSThomas Gleixner __this_cpu_read(hrtimer_interrupts)); 500bcd951cfSThomas Gleixner __touch_watchdog(); 501bcfba4f4SUlrich Obergfell 502bcfba4f4SUlrich Obergfell /* 503bcfba4f4SUlrich Obergfell * watchdog_nmi_enable() clears the NMI_WATCHDOG_ENABLED bit in the 504bcfba4f4SUlrich Obergfell * failure path. Check for failures that can occur asynchronously - 505bcfba4f4SUlrich Obergfell * for example, when CPUs are on-lined - and shut down the hardware 506bcfba4f4SUlrich Obergfell * perf event on each CPU accordingly. 507bcfba4f4SUlrich Obergfell * 508bcfba4f4SUlrich Obergfell * The only non-obvious place this bit can be cleared is through 509bcfba4f4SUlrich Obergfell * watchdog_nmi_enable(), so a pr_info() is placed there. Placing a 510bcfba4f4SUlrich Obergfell * pr_info here would be too noisy as it would result in a message 511bcfba4f4SUlrich Obergfell * every few seconds if the hardlockup was disabled but the softlockup 512bcfba4f4SUlrich Obergfell * enabled. 513bcfba4f4SUlrich Obergfell */ 514bcfba4f4SUlrich Obergfell if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) 515bcfba4f4SUlrich Obergfell watchdog_nmi_disable(cpu); 51658687acbSDon Zickus } 51758687acbSDon Zickus 51823637d47SFrederic Weisbecker #ifdef CONFIG_HARDLOCKUP_DETECTOR 519a7027046SDon Zickus /* 520a7027046SDon Zickus * People like the simple clean cpu node info on boot. 521a7027046SDon Zickus * Reduce the watchdog noise by only printing messages 522a7027046SDon Zickus * that are different from what cpu0 displayed. 523a7027046SDon Zickus */ 524a7027046SDon Zickus static unsigned long cpu0_err; 525a7027046SDon Zickus 526bcd951cfSThomas Gleixner static int watchdog_nmi_enable(unsigned int cpu) 52758687acbSDon Zickus { 52858687acbSDon Zickus struct perf_event_attr *wd_attr; 52958687acbSDon Zickus struct perf_event *event = per_cpu(watchdog_ev, cpu); 53058687acbSDon Zickus 531195daf66SUlrich Obergfell /* nothing to do if the hard lockup detector is disabled */ 532195daf66SUlrich Obergfell if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) 533195daf66SUlrich Obergfell goto out; 5346e7458a6SUlrich Obergfell 53558687acbSDon Zickus /* is it already setup and enabled? */ 53658687acbSDon Zickus if (event && event->state > PERF_EVENT_STATE_OFF) 53758687acbSDon Zickus goto out; 53858687acbSDon Zickus 53958687acbSDon Zickus /* it is setup but not enabled */ 54058687acbSDon Zickus if (event != NULL) 54158687acbSDon Zickus goto out_enable; 54258687acbSDon Zickus 54358687acbSDon Zickus wd_attr = &wd_hw_attr; 5444eec42f3SMandeep Singh Baines wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); 5451880c4aeSCyrill Gorcunov 5461880c4aeSCyrill Gorcunov /* Try to register using hardware perf events */ 5474dc0da86SAvi Kivity event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL); 548a7027046SDon Zickus 549a7027046SDon Zickus /* save cpu0 error for future comparision */ 550a7027046SDon Zickus if (cpu == 0 && IS_ERR(event)) 551a7027046SDon Zickus cpu0_err = PTR_ERR(event); 552a7027046SDon Zickus 55358687acbSDon Zickus if (!IS_ERR(event)) { 554a7027046SDon Zickus /* only print for cpu0 or different than cpu0 */ 555a7027046SDon Zickus if (cpu == 0 || cpu0_err) 556a7027046SDon Zickus pr_info("enabled on all CPUs, permanently consumes one hw-PMU counter.\n"); 55758687acbSDon Zickus goto out_save; 55858687acbSDon Zickus } 55958687acbSDon Zickus 560bcfba4f4SUlrich Obergfell /* 561bcfba4f4SUlrich Obergfell * Disable the hard lockup detector if _any_ CPU fails to set up 562bcfba4f4SUlrich Obergfell * set up the hardware perf event. The watchdog() function checks 563bcfba4f4SUlrich Obergfell * the NMI_WATCHDOG_ENABLED bit periodically. 564bcfba4f4SUlrich Obergfell * 565bcfba4f4SUlrich Obergfell * The barriers are for syncing up watchdog_enabled across all the 566bcfba4f4SUlrich Obergfell * cpus, as clear_bit() does not use barriers. 567bcfba4f4SUlrich Obergfell */ 568bcfba4f4SUlrich Obergfell smp_mb__before_atomic(); 569bcfba4f4SUlrich Obergfell clear_bit(NMI_WATCHDOG_ENABLED_BIT, &watchdog_enabled); 570bcfba4f4SUlrich Obergfell smp_mb__after_atomic(); 571bcfba4f4SUlrich Obergfell 572a7027046SDon Zickus /* skip displaying the same error again */ 573a7027046SDon Zickus if (cpu > 0 && (PTR_ERR(event) == cpu0_err)) 574a7027046SDon Zickus return PTR_ERR(event); 5755651f7f4SDon Zickus 5765651f7f4SDon Zickus /* vary the KERN level based on the returned errno */ 5775651f7f4SDon Zickus if (PTR_ERR(event) == -EOPNOTSUPP) 5784501980aSAndrew Morton pr_info("disabled (cpu%i): not supported (no LAPIC?)\n", cpu); 5795651f7f4SDon Zickus else if (PTR_ERR(event) == -ENOENT) 580656c3b79SFabian Frederick pr_warn("disabled (cpu%i): hardware events not enabled\n", 5814501980aSAndrew Morton cpu); 5825651f7f4SDon Zickus else 5834501980aSAndrew Morton pr_err("disabled (cpu%i): unable to create perf event: %ld\n", 5844501980aSAndrew Morton cpu, PTR_ERR(event)); 585bcfba4f4SUlrich Obergfell 586bcfba4f4SUlrich Obergfell pr_info("Shutting down hard lockup detector on all cpus\n"); 587bcfba4f4SUlrich Obergfell 588eac24335SAkinobu Mita return PTR_ERR(event); 58958687acbSDon Zickus 59058687acbSDon Zickus /* success path */ 59158687acbSDon Zickus out_save: 59258687acbSDon Zickus per_cpu(watchdog_ev, cpu) = event; 59358687acbSDon Zickus out_enable: 59458687acbSDon Zickus perf_event_enable(per_cpu(watchdog_ev, cpu)); 59558687acbSDon Zickus out: 59658687acbSDon Zickus return 0; 59758687acbSDon Zickus } 59858687acbSDon Zickus 599bcd951cfSThomas Gleixner static void watchdog_nmi_disable(unsigned int cpu) 60058687acbSDon Zickus { 60158687acbSDon Zickus struct perf_event *event = per_cpu(watchdog_ev, cpu); 60258687acbSDon Zickus 60358687acbSDon Zickus if (event) { 60458687acbSDon Zickus perf_event_disable(event); 60558687acbSDon Zickus per_cpu(watchdog_ev, cpu) = NULL; 60658687acbSDon Zickus 60758687acbSDon Zickus /* should be in cleanup, but blocks oprofile */ 60858687acbSDon Zickus perf_event_release_kernel(event); 60958687acbSDon Zickus } 610df577149SUlrich Obergfell if (cpu == 0) { 611df577149SUlrich Obergfell /* watchdog_nmi_enable() expects this to be zero initially. */ 612df577149SUlrich Obergfell cpu0_err = 0; 613df577149SUlrich Obergfell } 61458687acbSDon Zickus } 615b3738d29SStephane Eranian 616b3738d29SStephane Eranian void watchdog_nmi_enable_all(void) 617b3738d29SStephane Eranian { 618b3738d29SStephane Eranian int cpu; 619b3738d29SStephane Eranian 620ab992dc3SPeter Zijlstra mutex_lock(&watchdog_proc_mutex); 621ab992dc3SPeter Zijlstra 622ab992dc3SPeter Zijlstra if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) 623ab992dc3SPeter Zijlstra goto unlock; 624b3738d29SStephane Eranian 625b3738d29SStephane Eranian get_online_cpus(); 626fe4ba3c3SChris Metcalf for_each_watchdog_cpu(cpu) 627b3738d29SStephane Eranian watchdog_nmi_enable(cpu); 628b3738d29SStephane Eranian put_online_cpus(); 629ab992dc3SPeter Zijlstra 630ab992dc3SPeter Zijlstra unlock: 6311173ff09SMichal Hocko mutex_unlock(&watchdog_proc_mutex); 632b3738d29SStephane Eranian } 633b3738d29SStephane Eranian 634b3738d29SStephane Eranian void watchdog_nmi_disable_all(void) 635b3738d29SStephane Eranian { 636b3738d29SStephane Eranian int cpu; 637b3738d29SStephane Eranian 638ab992dc3SPeter Zijlstra mutex_lock(&watchdog_proc_mutex); 639ab992dc3SPeter Zijlstra 640b3738d29SStephane Eranian if (!watchdog_running) 641ab992dc3SPeter Zijlstra goto unlock; 642b3738d29SStephane Eranian 643b3738d29SStephane Eranian get_online_cpus(); 644fe4ba3c3SChris Metcalf for_each_watchdog_cpu(cpu) 645b3738d29SStephane Eranian watchdog_nmi_disable(cpu); 646b3738d29SStephane Eranian put_online_cpus(); 647ab992dc3SPeter Zijlstra 648ab992dc3SPeter Zijlstra unlock: 649ab992dc3SPeter Zijlstra mutex_unlock(&watchdog_proc_mutex); 650b3738d29SStephane Eranian } 65158687acbSDon Zickus #else 652bcd951cfSThomas Gleixner static int watchdog_nmi_enable(unsigned int cpu) { return 0; } 653bcd951cfSThomas Gleixner static void watchdog_nmi_disable(unsigned int cpu) { return; } 654b3738d29SStephane Eranian void watchdog_nmi_enable_all(void) {} 655b3738d29SStephane Eranian void watchdog_nmi_disable_all(void) {} 65623637d47SFrederic Weisbecker #endif /* CONFIG_HARDLOCKUP_DETECTOR */ 65758687acbSDon Zickus 658b8900bc0SFrederic Weisbecker static struct smp_hotplug_thread watchdog_threads = { 659b8900bc0SFrederic Weisbecker .store = &softlockup_watchdog, 660b8900bc0SFrederic Weisbecker .thread_should_run = watchdog_should_run, 661b8900bc0SFrederic Weisbecker .thread_fn = watchdog, 662b8900bc0SFrederic Weisbecker .thread_comm = "watchdog/%u", 663b8900bc0SFrederic Weisbecker .setup = watchdog_enable, 664b8900bc0SFrederic Weisbecker .cleanup = watchdog_cleanup, 665b8900bc0SFrederic Weisbecker .park = watchdog_disable, 666b8900bc0SFrederic Weisbecker .unpark = watchdog_enable, 667b8900bc0SFrederic Weisbecker }; 668b8900bc0SFrederic Weisbecker 6699809b18fSMichal Hocko static void restart_watchdog_hrtimer(void *info) 6709809b18fSMichal Hocko { 671f7f66b05SChristoph Lameter struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); 6729809b18fSMichal Hocko int ret; 6739809b18fSMichal Hocko 6749809b18fSMichal Hocko /* 6759809b18fSMichal Hocko * No need to cancel and restart hrtimer if it is currently executing 6769809b18fSMichal Hocko * because it will reprogram itself with the new period now. 6779809b18fSMichal Hocko * We should never see it unqueued here because we are running per-cpu 6789809b18fSMichal Hocko * with interrupts disabled. 6799809b18fSMichal Hocko */ 6809809b18fSMichal Hocko ret = hrtimer_try_to_cancel(hrtimer); 6819809b18fSMichal Hocko if (ret == 1) 6829809b18fSMichal Hocko hrtimer_start(hrtimer, ns_to_ktime(sample_period), 6839809b18fSMichal Hocko HRTIMER_MODE_REL_PINNED); 6849809b18fSMichal Hocko } 6859809b18fSMichal Hocko 686b2f57c3aSUlrich Obergfell static void update_watchdog(int cpu) 6879809b18fSMichal Hocko { 6889809b18fSMichal Hocko /* 6899809b18fSMichal Hocko * Make sure that perf event counter will adopt to a new 6909809b18fSMichal Hocko * sampling period. Updating the sampling period directly would 6919809b18fSMichal Hocko * be much nicer but we do not have an API for that now so 6929809b18fSMichal Hocko * let's use a big hammer. 6939809b18fSMichal Hocko * Hrtimer will adopt the new period on the next tick but this 6949809b18fSMichal Hocko * might be late already so we have to restart the timer as well. 6959809b18fSMichal Hocko */ 6969809b18fSMichal Hocko watchdog_nmi_disable(cpu); 697e0a23b06SFrederic Weisbecker smp_call_function_single(cpu, restart_watchdog_hrtimer, NULL, 1); 6989809b18fSMichal Hocko watchdog_nmi_enable(cpu); 6999809b18fSMichal Hocko } 7009809b18fSMichal Hocko 701b2f57c3aSUlrich Obergfell static void update_watchdog_all_cpus(void) 7029809b18fSMichal Hocko { 7039809b18fSMichal Hocko int cpu; 7049809b18fSMichal Hocko 7059809b18fSMichal Hocko get_online_cpus(); 706fe4ba3c3SChris Metcalf for_each_watchdog_cpu(cpu) 707b2f57c3aSUlrich Obergfell update_watchdog(cpu); 7089809b18fSMichal Hocko put_online_cpus(); 7099809b18fSMichal Hocko } 7109809b18fSMichal Hocko 711b2f57c3aSUlrich Obergfell static int watchdog_enable_all_cpus(void) 712b8900bc0SFrederic Weisbecker { 713b8900bc0SFrederic Weisbecker int err = 0; 714b8900bc0SFrederic Weisbecker 7153c00ea82SFrederic Weisbecker if (!watchdog_running) { 716b8900bc0SFrederic Weisbecker err = smpboot_register_percpu_thread(&watchdog_threads); 717b8900bc0SFrederic Weisbecker if (err) 718b8900bc0SFrederic Weisbecker pr_err("Failed to create watchdog threads, disabled\n"); 719fe4ba3c3SChris Metcalf else { 720fe4ba3c3SChris Metcalf if (smpboot_update_cpumask_percpu_thread( 721fe4ba3c3SChris Metcalf &watchdog_threads, &watchdog_cpumask)) 722fe4ba3c3SChris Metcalf pr_err("Failed to set cpumask for watchdog threads\n"); 7233c00ea82SFrederic Weisbecker watchdog_running = 1; 724fe4ba3c3SChris Metcalf } 725b2f57c3aSUlrich Obergfell } else { 726b2f57c3aSUlrich Obergfell /* 727b2f57c3aSUlrich Obergfell * Enable/disable the lockup detectors or 728b2f57c3aSUlrich Obergfell * change the sample period 'on the fly'. 729b2f57c3aSUlrich Obergfell */ 730b2f57c3aSUlrich Obergfell update_watchdog_all_cpus(); 731b8900bc0SFrederic Weisbecker } 732b8900bc0SFrederic Weisbecker 733b8900bc0SFrederic Weisbecker return err; 734b8900bc0SFrederic Weisbecker } 735b8900bc0SFrederic Weisbecker 73658687acbSDon Zickus /* prepare/enable/disable routines */ 7374ff81951SVasily Averin /* sysctl functions */ 7384ff81951SVasily Averin #ifdef CONFIG_SYSCTL 73958687acbSDon Zickus static void watchdog_disable_all_cpus(void) 74058687acbSDon Zickus { 7413c00ea82SFrederic Weisbecker if (watchdog_running) { 7423c00ea82SFrederic Weisbecker watchdog_running = 0; 743b8900bc0SFrederic Weisbecker smpboot_unregister_percpu_thread(&watchdog_threads); 74458687acbSDon Zickus } 745bcd951cfSThomas Gleixner } 74658687acbSDon Zickus 74758687acbSDon Zickus /* 748a0c9cbb9SUlrich Obergfell * Update the run state of the lockup detectors. 74958687acbSDon Zickus */ 750a0c9cbb9SUlrich Obergfell static int proc_watchdog_update(void) 75158687acbSDon Zickus { 752a0c9cbb9SUlrich Obergfell int err = 0; 753a0c9cbb9SUlrich Obergfell 754a0c9cbb9SUlrich Obergfell /* 755a0c9cbb9SUlrich Obergfell * Watchdog threads won't be started if they are already active. 756a0c9cbb9SUlrich Obergfell * The 'watchdog_running' variable in watchdog_*_all_cpus() takes 757a0c9cbb9SUlrich Obergfell * care of this. If those threads are already active, the sample 758a0c9cbb9SUlrich Obergfell * period will be updated and the lockup detectors will be enabled 759a0c9cbb9SUlrich Obergfell * or disabled 'on the fly'. 760a0c9cbb9SUlrich Obergfell */ 761a0c9cbb9SUlrich Obergfell if (watchdog_enabled && watchdog_thresh) 762b2f57c3aSUlrich Obergfell err = watchdog_enable_all_cpus(); 763a0c9cbb9SUlrich Obergfell else 764a0c9cbb9SUlrich Obergfell watchdog_disable_all_cpus(); 765a0c9cbb9SUlrich Obergfell 766a0c9cbb9SUlrich Obergfell return err; 767a0c9cbb9SUlrich Obergfell 768a0c9cbb9SUlrich Obergfell } 769a0c9cbb9SUlrich Obergfell 770a0c9cbb9SUlrich Obergfell /* 771ef246a21SUlrich Obergfell * common function for watchdog, nmi_watchdog and soft_watchdog parameter 772ef246a21SUlrich Obergfell * 773ef246a21SUlrich Obergfell * caller | table->data points to | 'which' contains the flag(s) 774ef246a21SUlrich Obergfell * -------------------|-----------------------|----------------------------- 775ef246a21SUlrich Obergfell * proc_watchdog | watchdog_user_enabled | NMI_WATCHDOG_ENABLED or'ed 776ef246a21SUlrich Obergfell * | | with SOFT_WATCHDOG_ENABLED 777ef246a21SUlrich Obergfell * -------------------|-----------------------|----------------------------- 778ef246a21SUlrich Obergfell * proc_nmi_watchdog | nmi_watchdog_enabled | NMI_WATCHDOG_ENABLED 779ef246a21SUlrich Obergfell * -------------------|-----------------------|----------------------------- 780ef246a21SUlrich Obergfell * proc_soft_watchdog | soft_watchdog_enabled | SOFT_WATCHDOG_ENABLED 781ef246a21SUlrich Obergfell */ 782ef246a21SUlrich Obergfell static int proc_watchdog_common(int which, struct ctl_table *table, int write, 783ef246a21SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 784ef246a21SUlrich Obergfell { 785ef246a21SUlrich Obergfell int err, old, new; 786ef246a21SUlrich Obergfell int *watchdog_param = (int *)table->data; 787bcd951cfSThomas Gleixner 788ef246a21SUlrich Obergfell mutex_lock(&watchdog_proc_mutex); 789ef246a21SUlrich Obergfell 790ef246a21SUlrich Obergfell /* 791ef246a21SUlrich Obergfell * If the parameter is being read return the state of the corresponding 792ef246a21SUlrich Obergfell * bit(s) in 'watchdog_enabled', else update 'watchdog_enabled' and the 793ef246a21SUlrich Obergfell * run state of the lockup detectors. 794ef246a21SUlrich Obergfell */ 795ef246a21SUlrich Obergfell if (!write) { 796ef246a21SUlrich Obergfell *watchdog_param = (watchdog_enabled & which) != 0; 797b8900bc0SFrederic Weisbecker err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 798ef246a21SUlrich Obergfell } else { 799ef246a21SUlrich Obergfell err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 800ef246a21SUlrich Obergfell if (err) 801ef246a21SUlrich Obergfell goto out; 802ef246a21SUlrich Obergfell 803ef246a21SUlrich Obergfell /* 804ef246a21SUlrich Obergfell * There is a race window between fetching the current value 805ef246a21SUlrich Obergfell * from 'watchdog_enabled' and storing the new value. During 806ef246a21SUlrich Obergfell * this race window, watchdog_nmi_enable() can sneak in and 807ef246a21SUlrich Obergfell * clear the NMI_WATCHDOG_ENABLED bit in 'watchdog_enabled'. 808ef246a21SUlrich Obergfell * The 'cmpxchg' detects this race and the loop retries. 809ef246a21SUlrich Obergfell */ 810ef246a21SUlrich Obergfell do { 811ef246a21SUlrich Obergfell old = watchdog_enabled; 812ef246a21SUlrich Obergfell /* 813ef246a21SUlrich Obergfell * If the parameter value is not zero set the 814ef246a21SUlrich Obergfell * corresponding bit(s), else clear it(them). 815ef246a21SUlrich Obergfell */ 816ef246a21SUlrich Obergfell if (*watchdog_param) 817ef246a21SUlrich Obergfell new = old | which; 818ef246a21SUlrich Obergfell else 819ef246a21SUlrich Obergfell new = old & ~which; 820ef246a21SUlrich Obergfell } while (cmpxchg(&watchdog_enabled, old, new) != old); 821ef246a21SUlrich Obergfell 822ef246a21SUlrich Obergfell /* 823ef246a21SUlrich Obergfell * Update the run state of the lockup detectors. 824ef246a21SUlrich Obergfell * Restore 'watchdog_enabled' on failure. 825ef246a21SUlrich Obergfell */ 826ef246a21SUlrich Obergfell err = proc_watchdog_update(); 827ef246a21SUlrich Obergfell if (err) 828ef246a21SUlrich Obergfell watchdog_enabled = old; 829ef246a21SUlrich Obergfell } 830ef246a21SUlrich Obergfell out: 831ef246a21SUlrich Obergfell mutex_unlock(&watchdog_proc_mutex); 832ef246a21SUlrich Obergfell return err; 833ef246a21SUlrich Obergfell } 834ef246a21SUlrich Obergfell 835ef246a21SUlrich Obergfell /* 83683a80a39SUlrich Obergfell * /proc/sys/kernel/watchdog 83783a80a39SUlrich Obergfell */ 83883a80a39SUlrich Obergfell int proc_watchdog(struct ctl_table *table, int write, 83983a80a39SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 84083a80a39SUlrich Obergfell { 84183a80a39SUlrich Obergfell return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED, 84283a80a39SUlrich Obergfell table, write, buffer, lenp, ppos); 84383a80a39SUlrich Obergfell } 84483a80a39SUlrich Obergfell 84583a80a39SUlrich Obergfell /* 84683a80a39SUlrich Obergfell * /proc/sys/kernel/nmi_watchdog 84783a80a39SUlrich Obergfell */ 84883a80a39SUlrich Obergfell int proc_nmi_watchdog(struct ctl_table *table, int write, 84983a80a39SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 85083a80a39SUlrich Obergfell { 85183a80a39SUlrich Obergfell return proc_watchdog_common(NMI_WATCHDOG_ENABLED, 85283a80a39SUlrich Obergfell table, write, buffer, lenp, ppos); 85383a80a39SUlrich Obergfell } 85483a80a39SUlrich Obergfell 85583a80a39SUlrich Obergfell /* 85683a80a39SUlrich Obergfell * /proc/sys/kernel/soft_watchdog 85783a80a39SUlrich Obergfell */ 85883a80a39SUlrich Obergfell int proc_soft_watchdog(struct ctl_table *table, int write, 85983a80a39SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 86083a80a39SUlrich Obergfell { 86183a80a39SUlrich Obergfell return proc_watchdog_common(SOFT_WATCHDOG_ENABLED, 86283a80a39SUlrich Obergfell table, write, buffer, lenp, ppos); 86383a80a39SUlrich Obergfell } 86483a80a39SUlrich Obergfell 86583a80a39SUlrich Obergfell /* 86683a80a39SUlrich Obergfell * /proc/sys/kernel/watchdog_thresh 86783a80a39SUlrich Obergfell */ 86883a80a39SUlrich Obergfell int proc_watchdog_thresh(struct ctl_table *table, int write, 86983a80a39SUlrich Obergfell void __user *buffer, size_t *lenp, loff_t *ppos) 87083a80a39SUlrich Obergfell { 87183a80a39SUlrich Obergfell int err, old; 87283a80a39SUlrich Obergfell 87383a80a39SUlrich Obergfell mutex_lock(&watchdog_proc_mutex); 87483a80a39SUlrich Obergfell 87583a80a39SUlrich Obergfell old = ACCESS_ONCE(watchdog_thresh); 87683a80a39SUlrich Obergfell err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 87783a80a39SUlrich Obergfell 878b8900bc0SFrederic Weisbecker if (err || !write) 879359e6fabSMichal Hocko goto out; 880e04ab2bcSMandeep Singh Baines 88183a80a39SUlrich Obergfell /* 88283a80a39SUlrich Obergfell * Update the sample period. 88383a80a39SUlrich Obergfell * Restore 'watchdog_thresh' on failure. 88483a80a39SUlrich Obergfell */ 8850f34c400SChuansheng Liu set_sample_period(); 88683a80a39SUlrich Obergfell err = proc_watchdog_update(); 88783a80a39SUlrich Obergfell if (err) 88883a80a39SUlrich Obergfell watchdog_thresh = old; 889359e6fabSMichal Hocko out: 890359e6fabSMichal Hocko mutex_unlock(&watchdog_proc_mutex); 891b8900bc0SFrederic Weisbecker return err; 89258687acbSDon Zickus } 893fe4ba3c3SChris Metcalf 894fe4ba3c3SChris Metcalf /* 895fe4ba3c3SChris Metcalf * The cpumask is the mask of possible cpus that the watchdog can run 896fe4ba3c3SChris Metcalf * on, not the mask of cpus it is actually running on. This allows the 897fe4ba3c3SChris Metcalf * user to specify a mask that will include cpus that have not yet 898fe4ba3c3SChris Metcalf * been brought online, if desired. 899fe4ba3c3SChris Metcalf */ 900fe4ba3c3SChris Metcalf int proc_watchdog_cpumask(struct ctl_table *table, int write, 901fe4ba3c3SChris Metcalf void __user *buffer, size_t *lenp, loff_t *ppos) 902fe4ba3c3SChris Metcalf { 903fe4ba3c3SChris Metcalf int err; 904fe4ba3c3SChris Metcalf 905fe4ba3c3SChris Metcalf mutex_lock(&watchdog_proc_mutex); 906fe4ba3c3SChris Metcalf err = proc_do_large_bitmap(table, write, buffer, lenp, ppos); 907fe4ba3c3SChris Metcalf if (!err && write) { 908fe4ba3c3SChris Metcalf /* Remove impossible cpus to keep sysctl output cleaner. */ 909fe4ba3c3SChris Metcalf cpumask_and(&watchdog_cpumask, &watchdog_cpumask, 910fe4ba3c3SChris Metcalf cpu_possible_mask); 911fe4ba3c3SChris Metcalf 912fe4ba3c3SChris Metcalf if (watchdog_running) { 913fe4ba3c3SChris Metcalf /* 914fe4ba3c3SChris Metcalf * Failure would be due to being unable to allocate 915fe4ba3c3SChris Metcalf * a temporary cpumask, so we are likely not in a 916fe4ba3c3SChris Metcalf * position to do much else to make things better. 917fe4ba3c3SChris Metcalf */ 918fe4ba3c3SChris Metcalf if (smpboot_update_cpumask_percpu_thread( 919fe4ba3c3SChris Metcalf &watchdog_threads, &watchdog_cpumask) != 0) 920fe4ba3c3SChris Metcalf pr_err("cpumask update failed\n"); 921fe4ba3c3SChris Metcalf } 922fe4ba3c3SChris Metcalf } 923fe4ba3c3SChris Metcalf mutex_unlock(&watchdog_proc_mutex); 924fe4ba3c3SChris Metcalf return err; 925fe4ba3c3SChris Metcalf } 926fe4ba3c3SChris Metcalf 92758687acbSDon Zickus #endif /* CONFIG_SYSCTL */ 92858687acbSDon Zickus 929004417a6SPeter Zijlstra void __init lockup_detector_init(void) 93058687acbSDon Zickus { 9310f34c400SChuansheng Liu set_sample_period(); 932b8900bc0SFrederic Weisbecker 933fe4ba3c3SChris Metcalf #ifdef CONFIG_NO_HZ_FULL 934fe4ba3c3SChris Metcalf if (tick_nohz_full_enabled()) { 935fe4ba3c3SChris Metcalf if (!cpumask_empty(tick_nohz_full_mask)) 936fe4ba3c3SChris Metcalf pr_info("Disabling watchdog on nohz_full cores by default\n"); 937fe4ba3c3SChris Metcalf cpumask_andnot(&watchdog_cpumask, cpu_possible_mask, 938fe4ba3c3SChris Metcalf tick_nohz_full_mask); 939fe4ba3c3SChris Metcalf } else 940fe4ba3c3SChris Metcalf cpumask_copy(&watchdog_cpumask, cpu_possible_mask); 941fe4ba3c3SChris Metcalf #else 942fe4ba3c3SChris Metcalf cpumask_copy(&watchdog_cpumask, cpu_possible_mask); 943fe4ba3c3SChris Metcalf #endif 944fe4ba3c3SChris Metcalf 945195daf66SUlrich Obergfell if (watchdog_enabled) 946b2f57c3aSUlrich Obergfell watchdog_enable_all_cpus(); 94758687acbSDon Zickus } 948