1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
258687acbSDon Zickus /*
358687acbSDon Zickus * Detect hard and soft lockups on a system
458687acbSDon Zickus *
558687acbSDon Zickus * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
658687acbSDon Zickus *
786f5e6a7SFernando Luis Vázquez Cao * Note: Most of this code is borrowed heavily from the original softlockup
886f5e6a7SFernando Luis Vázquez Cao * detector, so thanks to Ingo for the initial implementation.
986f5e6a7SFernando Luis Vázquez Cao * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
1058687acbSDon Zickus * to those contributors as well.
1158687acbSDon Zickus */
1258687acbSDon Zickus
135f92a7b0SKefeng Wang #define pr_fmt(fmt) "watchdog: " fmt
144501980aSAndrew Morton
1558687acbSDon Zickus #include <linux/mm.h>
1658687acbSDon Zickus #include <linux/cpu.h>
1758687acbSDon Zickus #include <linux/nmi.h>
1858687acbSDon Zickus #include <linux/init.h>
1958687acbSDon Zickus #include <linux/module.h>
2058687acbSDon Zickus #include <linux/sysctl.h>
21fe4ba3c3SChris Metcalf #include <linux/tick.h>
22e6017571SIngo Molnar #include <linux/sched/clock.h>
23b17b0153SIngo Molnar #include <linux/sched/debug.h>
2478634061SFrederic Weisbecker #include <linux/sched/isolation.h>
259cf57731SPeter Zijlstra #include <linux/stop_machine.h>
2658687acbSDon Zickus
2758687acbSDon Zickus #include <asm/irq_regs.h>
285d1c0f4aSEric B Munson #include <linux/kvm_para.h>
2958687acbSDon Zickus
30946d1977SThomas Gleixner static DEFINE_MUTEX(watchdog_mutex);
31ab992dc3SPeter Zijlstra
3247f4cb43SPetr Mladek #if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64)
33df95d308SDouglas Anderson # define WATCHDOG_HARDLOCKUP_DEFAULT 1
3484d56e66SUlrich Obergfell #else
35df95d308SDouglas Anderson # define WATCHDOG_HARDLOCKUP_DEFAULT 0
3684d56e66SUlrich Obergfell #endif
3705a4a952SNicholas Piggin
3809154985SThomas Gleixner unsigned long __read_mostly watchdog_enabled;
3909154985SThomas Gleixner int __read_mostly watchdog_user_enabled = 1;
40df95d308SDouglas Anderson static int __read_mostly watchdog_hardlockup_user_enabled = WATCHDOG_HARDLOCKUP_DEFAULT;
41df95d308SDouglas Anderson static int __read_mostly watchdog_softlockup_user_enabled = 1;
427feeb9cdSThomas Gleixner int __read_mostly watchdog_thresh = 10;
43df95d308SDouglas Anderson static int __read_mostly watchdog_hardlockup_available;
447feeb9cdSThomas Gleixner
457feeb9cdSThomas Gleixner struct cpumask watchdog_cpumask __read_mostly;
467feeb9cdSThomas Gleixner unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
477feeb9cdSThomas Gleixner
4805a4a952SNicholas Piggin #ifdef CONFIG_HARDLOCKUP_DETECTOR
49f117955aSGuilherme G. Piccoli
50f117955aSGuilherme G. Piccoli # ifdef CONFIG_SMP
51f117955aSGuilherme G. Piccoli int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
52f117955aSGuilherme G. Piccoli # endif /* CONFIG_SMP */
53f117955aSGuilherme G. Piccoli
5405a4a952SNicholas Piggin /*
5505a4a952SNicholas Piggin * Should we panic when a soft-lockup or hard-lockup occurs:
5605a4a952SNicholas Piggin */
5705a4a952SNicholas Piggin unsigned int __read_mostly hardlockup_panic =
5867fca000SRasmus Villemoes IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);
5905a4a952SNicholas Piggin /*
6005a4a952SNicholas Piggin * We may not want to enable hard lockup detection by default in all cases,
6105a4a952SNicholas Piggin * for example when running the kernel as a guest on a hypervisor. In these
6205a4a952SNicholas Piggin * cases this function can be called to disable hard lockup detection. This
6305a4a952SNicholas Piggin * function should only be executed once by the boot processor before the
6405a4a952SNicholas Piggin * kernel command line parameters are parsed, because otherwise it is not
6505a4a952SNicholas Piggin * possible to override this in hardlockup_panic_setup().
6605a4a952SNicholas Piggin */
hardlockup_detector_disable(void)677a355820SThomas Gleixner void __init hardlockup_detector_disable(void)
6805a4a952SNicholas Piggin {
69df95d308SDouglas Anderson watchdog_hardlockup_user_enabled = 0;
7005a4a952SNicholas Piggin }
7105a4a952SNicholas Piggin
hardlockup_panic_setup(char * str)7205a4a952SNicholas Piggin static int __init hardlockup_panic_setup(char *str)
7305a4a952SNicholas Piggin {
7405a4a952SNicholas Piggin if (!strncmp(str, "panic", 5))
7505a4a952SNicholas Piggin hardlockup_panic = 1;
7605a4a952SNicholas Piggin else if (!strncmp(str, "nopanic", 7))
7705a4a952SNicholas Piggin hardlockup_panic = 0;
7805a4a952SNicholas Piggin else if (!strncmp(str, "0", 1))
79df95d308SDouglas Anderson watchdog_hardlockup_user_enabled = 0;
8005a4a952SNicholas Piggin else if (!strncmp(str, "1", 1))
81df95d308SDouglas Anderson watchdog_hardlockup_user_enabled = 1;
8205a4a952SNicholas Piggin return 1;
8305a4a952SNicholas Piggin }
8405a4a952SNicholas Piggin __setup("nmi_watchdog=", hardlockup_panic_setup);
8505a4a952SNicholas Piggin
86368a7e2cSThomas Gleixner #endif /* CONFIG_HARDLOCKUP_DETECTOR */
8705a4a952SNicholas Piggin
881f423c90SDouglas Anderson #if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER)
8981972551SDouglas Anderson
9077c12fc9SDouglas Anderson static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts);
9177c12fc9SDouglas Anderson static DEFINE_PER_CPU(int, hrtimer_interrupts_saved);
921610611aSDouglas Anderson static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned);
93ed92e1efSDouglas Anderson static DEFINE_PER_CPU(bool, watchdog_hardlockup_touched);
941610611aSDouglas Anderson static unsigned long watchdog_hardlockup_all_cpu_dumped;
9581972551SDouglas Anderson
arch_touch_nmi_watchdog(void)96ed92e1efSDouglas Anderson notrace void arch_touch_nmi_watchdog(void)
97ed92e1efSDouglas Anderson {
98ed92e1efSDouglas Anderson /*
99ed92e1efSDouglas Anderson * Using __raw here because some code paths have
100ed92e1efSDouglas Anderson * preemption enabled. If preemption is enabled
101ed92e1efSDouglas Anderson * then interrupts should be enabled too, in which
102ed92e1efSDouglas Anderson * case we shouldn't have to worry about the watchdog
103ed92e1efSDouglas Anderson * going off.
104ed92e1efSDouglas Anderson */
105ed92e1efSDouglas Anderson raw_cpu_write(watchdog_hardlockup_touched, true);
106ed92e1efSDouglas Anderson }
107ed92e1efSDouglas Anderson EXPORT_SYMBOL(arch_touch_nmi_watchdog);
108ed92e1efSDouglas Anderson
watchdog_hardlockup_touch_cpu(unsigned int cpu)1091f423c90SDouglas Anderson void watchdog_hardlockup_touch_cpu(unsigned int cpu)
1101f423c90SDouglas Anderson {
1111f423c90SDouglas Anderson per_cpu(watchdog_hardlockup_touched, cpu) = true;
1121f423c90SDouglas Anderson }
1131f423c90SDouglas Anderson
is_hardlockup(unsigned int cpu)11477c12fc9SDouglas Anderson static bool is_hardlockup(unsigned int cpu)
11581972551SDouglas Anderson {
11677c12fc9SDouglas Anderson int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu));
11781972551SDouglas Anderson
11877c12fc9SDouglas Anderson if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint)
11981972551SDouglas Anderson return true;
12081972551SDouglas Anderson
12177c12fc9SDouglas Anderson /*
12277c12fc9SDouglas Anderson * NOTE: we don't need any fancy atomic_t or READ_ONCE/WRITE_ONCE
12377c12fc9SDouglas Anderson * for hrtimer_interrupts_saved. hrtimer_interrupts_saved is
12477c12fc9SDouglas Anderson * written/read by a single CPU.
12577c12fc9SDouglas Anderson */
12677c12fc9SDouglas Anderson per_cpu(hrtimer_interrupts_saved, cpu) = hrint;
1271610611aSDouglas Anderson
12881972551SDouglas Anderson return false;
12981972551SDouglas Anderson }
13081972551SDouglas Anderson
watchdog_hardlockup_kick(void)131d3b62aceSDouglas Anderson static void watchdog_hardlockup_kick(void)
13281972551SDouglas Anderson {
133d3b62aceSDouglas Anderson int new_interrupts;
134d3b62aceSDouglas Anderson
135d3b62aceSDouglas Anderson new_interrupts = atomic_inc_return(this_cpu_ptr(&hrtimer_interrupts));
136d3b62aceSDouglas Anderson watchdog_buddy_check_hardlockup(new_interrupts);
13781972551SDouglas Anderson }
13881972551SDouglas Anderson
watchdog_hardlockup_check(unsigned int cpu,struct pt_regs * regs)13977c12fc9SDouglas Anderson void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
14081972551SDouglas Anderson {
141ed92e1efSDouglas Anderson if (per_cpu(watchdog_hardlockup_touched, cpu)) {
142ed92e1efSDouglas Anderson per_cpu(watchdog_hardlockup_touched, cpu) = false;
143ed92e1efSDouglas Anderson return;
144ed92e1efSDouglas Anderson }
145ed92e1efSDouglas Anderson
1461610611aSDouglas Anderson /*
1471610611aSDouglas Anderson * Check for a hardlockup by making sure the CPU's timer
1481610611aSDouglas Anderson * interrupt is incrementing. The timer interrupt should have
14981972551SDouglas Anderson * fired multiple times before we overflow'd. If it hasn't
15081972551SDouglas Anderson * then this is a good indication the cpu is stuck
15181972551SDouglas Anderson */
15277c12fc9SDouglas Anderson if (is_hardlockup(cpu)) {
1531610611aSDouglas Anderson unsigned int this_cpu = smp_processor_id();
15481972551SDouglas Anderson
1551610611aSDouglas Anderson /* Only print hardlockups once. */
15677c12fc9SDouglas Anderson if (per_cpu(watchdog_hardlockup_warned, cpu))
15781972551SDouglas Anderson return;
15881972551SDouglas Anderson
15977c12fc9SDouglas Anderson pr_emerg("Watchdog detected hard LOCKUP on cpu %d\n", cpu);
16081972551SDouglas Anderson print_modules();
16181972551SDouglas Anderson print_irqtrace_events(current);
16277c12fc9SDouglas Anderson if (cpu == this_cpu) {
16381972551SDouglas Anderson if (regs)
16481972551SDouglas Anderson show_regs(regs);
16581972551SDouglas Anderson else
16681972551SDouglas Anderson dump_stack();
16777c12fc9SDouglas Anderson } else {
1681f38c86bSDouglas Anderson trigger_single_cpu_backtrace(cpu);
16977c12fc9SDouglas Anderson }
17081972551SDouglas Anderson
17181972551SDouglas Anderson /*
17277c12fc9SDouglas Anderson * Perform multi-CPU dump only once to avoid multiple
17377c12fc9SDouglas Anderson * hardlockups generating interleaving traces
17481972551SDouglas Anderson */
17581972551SDouglas Anderson if (sysctl_hardlockup_all_cpu_backtrace &&
1761610611aSDouglas Anderson !test_and_set_bit(0, &watchdog_hardlockup_all_cpu_dumped))
1771f38c86bSDouglas Anderson trigger_allbutcpu_cpu_backtrace(cpu);
17881972551SDouglas Anderson
17981972551SDouglas Anderson if (hardlockup_panic)
18081972551SDouglas Anderson nmi_panic(regs, "Hard LOCKUP");
18181972551SDouglas Anderson
18277c12fc9SDouglas Anderson per_cpu(watchdog_hardlockup_warned, cpu) = true;
1831610611aSDouglas Anderson } else {
18477c12fc9SDouglas Anderson per_cpu(watchdog_hardlockup_warned, cpu) = false;
18581972551SDouglas Anderson }
18681972551SDouglas Anderson }
18781972551SDouglas Anderson
1881f423c90SDouglas Anderson #else /* CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */
18981972551SDouglas Anderson
watchdog_hardlockup_kick(void)190d3b62aceSDouglas Anderson static inline void watchdog_hardlockup_kick(void) { }
19181972551SDouglas Anderson
1921f423c90SDouglas Anderson #endif /* !CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */
19381972551SDouglas Anderson
194ec6a9066SUlrich Obergfell /*
195d9b3629aSDouglas Anderson * These functions can be overridden based on the configured hardlockdup detector.
196a10a842fSNicholas Piggin *
197df95d308SDouglas Anderson * watchdog_hardlockup_enable/disable can be implemented to start and stop when
198d9b3629aSDouglas Anderson * softlockup watchdog start and stop. The detector must select the
199a10a842fSNicholas Piggin * SOFTLOCKUP_DETECTOR Kconfig.
20005a4a952SNicholas Piggin */
watchdog_hardlockup_enable(unsigned int cpu)201d9b3629aSDouglas Anderson void __weak watchdog_hardlockup_enable(unsigned int cpu) { }
202941154bdSThomas Gleixner
watchdog_hardlockup_disable(unsigned int cpu)203d9b3629aSDouglas Anderson void __weak watchdog_hardlockup_disable(unsigned int cpu) { }
20405a4a952SNicholas Piggin
205930d8f8dSLecopzer Chen /*
206930d8f8dSLecopzer Chen * Watchdog-detector specific API.
207930d8f8dSLecopzer Chen *
208930d8f8dSLecopzer Chen * Return 0 when hardlockup watchdog is available, negative value otherwise.
209930d8f8dSLecopzer Chen * Note that the negative value means that a delayed probe might
210930d8f8dSLecopzer Chen * succeed later.
211930d8f8dSLecopzer Chen */
watchdog_hardlockup_probe(void)212df95d308SDouglas Anderson int __weak __init watchdog_hardlockup_probe(void)
213a994a314SThomas Gleixner {
214d9b3629aSDouglas Anderson return -ENODEV;
215a994a314SThomas Gleixner }
216a994a314SThomas Gleixner
2176592ad2fSThomas Gleixner /**
218df95d308SDouglas Anderson * watchdog_hardlockup_stop - Stop the watchdog for reconfiguration
2196592ad2fSThomas Gleixner *
2206b9dc480SThomas Gleixner * The reconfiguration steps are:
221df95d308SDouglas Anderson * watchdog_hardlockup_stop();
2226592ad2fSThomas Gleixner * update_variables();
223df95d308SDouglas Anderson * watchdog_hardlockup_start();
2246b9dc480SThomas Gleixner */
watchdog_hardlockup_stop(void)225df95d308SDouglas Anderson void __weak watchdog_hardlockup_stop(void) { }
2266b9dc480SThomas Gleixner
2276b9dc480SThomas Gleixner /**
228df95d308SDouglas Anderson * watchdog_hardlockup_start - Start the watchdog after reconfiguration
2296592ad2fSThomas Gleixner *
230df95d308SDouglas Anderson * Counterpart to watchdog_hardlockup_stop().
2316b9dc480SThomas Gleixner *
2326b9dc480SThomas Gleixner * The following variables have been updated in update_variables() and
2336b9dc480SThomas Gleixner * contain the currently valid configuration:
2347feeb9cdSThomas Gleixner * - watchdog_enabled
235a10a842fSNicholas Piggin * - watchdog_thresh
236a10a842fSNicholas Piggin * - watchdog_cpumask
237a10a842fSNicholas Piggin */
watchdog_hardlockup_start(void)238df95d308SDouglas Anderson void __weak watchdog_hardlockup_start(void) { }
239a10a842fSNicholas Piggin
24009154985SThomas Gleixner /**
24109154985SThomas Gleixner * lockup_detector_update_enable - Update the sysctl enable bit
24209154985SThomas Gleixner *
243df95d308SDouglas Anderson * Caller needs to make sure that the hard watchdogs are off, so this
244df95d308SDouglas Anderson * can't race with watchdog_hardlockup_disable().
24509154985SThomas Gleixner */
lockup_detector_update_enable(void)24609154985SThomas Gleixner static void lockup_detector_update_enable(void)
24709154985SThomas Gleixner {
24809154985SThomas Gleixner watchdog_enabled = 0;
24909154985SThomas Gleixner if (!watchdog_user_enabled)
25009154985SThomas Gleixner return;
251df95d308SDouglas Anderson if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled)
252df95d308SDouglas Anderson watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED;
253df95d308SDouglas Anderson if (watchdog_softlockup_user_enabled)
254df95d308SDouglas Anderson watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED;
25509154985SThomas Gleixner }
25609154985SThomas Gleixner
25705a4a952SNicholas Piggin #ifdef CONFIG_SOFTLOCKUP_DETECTOR
25805a4a952SNicholas Piggin
259fef06efcSPetr Mladek /*
260fef06efcSPetr Mladek * Delay the soflockup report when running a known slow code.
261fef06efcSPetr Mladek * It does _not_ affect the timestamp of the last successdul reschedule.
262fef06efcSPetr Mladek */
263fef06efcSPetr Mladek #define SOFTLOCKUP_DELAY_REPORT ULONG_MAX
26411e31f60SThomas Gleixner
265f117955aSGuilherme G. Piccoli #ifdef CONFIG_SMP
266f117955aSGuilherme G. Piccoli int __read_mostly sysctl_softlockup_all_cpu_backtrace;
267f117955aSGuilherme G. Piccoli #endif
268f117955aSGuilherme G. Piccoli
269e7e04615SSantosh Sivaraj static struct cpumask watchdog_allowed_mask __read_mostly;
270e7e04615SSantosh Sivaraj
2712b9d7f23SThomas Gleixner /* Global variables, exported for sysctl */
2722b9d7f23SThomas Gleixner unsigned int __read_mostly softlockup_panic =
27367fca000SRasmus Villemoes IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC);
2742eb2527fSThomas Gleixner
2759cf57731SPeter Zijlstra static bool softlockup_initialized __read_mostly;
2760f34c400SChuansheng Liu static u64 __read_mostly sample_period;
27758687acbSDon Zickus
278fef06efcSPetr Mladek /* Timestamp taken after the last successful reschedule. */
27958687acbSDon Zickus static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
280fef06efcSPetr Mladek /* Timestamp of the last softlockup report. */
281fef06efcSPetr Mladek static DEFINE_PER_CPU(unsigned long, watchdog_report_ts);
28258687acbSDon Zickus static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
28358687acbSDon Zickus static DEFINE_PER_CPU(bool, softlockup_touch_sync);
284ed235875SAaron Tomlin static unsigned long soft_lockup_nmi_warn;
28558687acbSDon Zickus
softlockup_panic_setup(char * str)286*dbfbac0fSKrister Johansen static int __init softlockup_panic_setup(char *str)
287*dbfbac0fSKrister Johansen {
288*dbfbac0fSKrister Johansen softlockup_panic = simple_strtoul(str, NULL, 0);
289*dbfbac0fSKrister Johansen return 1;
290*dbfbac0fSKrister Johansen }
291*dbfbac0fSKrister Johansen __setup("softlockup_panic=", softlockup_panic_setup);
292*dbfbac0fSKrister Johansen
nowatchdog_setup(char * str)29358687acbSDon Zickus static int __init nowatchdog_setup(char *str)
29458687acbSDon Zickus {
29509154985SThomas Gleixner watchdog_user_enabled = 0;
29658687acbSDon Zickus return 1;
29758687acbSDon Zickus }
29858687acbSDon Zickus __setup("nowatchdog", nowatchdog_setup);
29958687acbSDon Zickus
nosoftlockup_setup(char * str)30058687acbSDon Zickus static int __init nosoftlockup_setup(char *str)
30158687acbSDon Zickus {
302df95d308SDouglas Anderson watchdog_softlockup_user_enabled = 0;
30358687acbSDon Zickus return 1;
30458687acbSDon Zickus }
30558687acbSDon Zickus __setup("nosoftlockup", nosoftlockup_setup);
306195daf66SUlrich Obergfell
watchdog_thresh_setup(char * str)30711295055SLaurence Oberman static int __init watchdog_thresh_setup(char *str)
30811295055SLaurence Oberman {
30911295055SLaurence Oberman get_option(&str, &watchdog_thresh);
31011295055SLaurence Oberman return 1;
31111295055SLaurence Oberman }
31211295055SLaurence Oberman __setup("watchdog_thresh=", watchdog_thresh_setup);
31311295055SLaurence Oberman
314941154bdSThomas Gleixner static void __lockup_detector_cleanup(void);
315941154bdSThomas Gleixner
3164eec42f3SMandeep Singh Baines /*
3174eec42f3SMandeep Singh Baines * Hard-lockup warnings should be triggered after just a few seconds. Soft-
3184eec42f3SMandeep Singh Baines * lockups can have false positives under extreme conditions. So we generally
3194eec42f3SMandeep Singh Baines * want a higher threshold for soft lockups than for hard lockups. So we couple
3204eec42f3SMandeep Singh Baines * the thresholds with a factor: we make the soft threshold twice the amount of
3214eec42f3SMandeep Singh Baines * time the hard threshold is.
3224eec42f3SMandeep Singh Baines */
get_softlockup_thresh(void)3236e9101aeSIngo Molnar static int get_softlockup_thresh(void)
3244eec42f3SMandeep Singh Baines {
3254eec42f3SMandeep Singh Baines return watchdog_thresh * 2;
3264eec42f3SMandeep Singh Baines }
32758687acbSDon Zickus
32858687acbSDon Zickus /*
32958687acbSDon Zickus * Returns seconds, approximately. We don't need nanosecond
33058687acbSDon Zickus * resolution, and we don't need to waste time with a big divide when
33158687acbSDon Zickus * 2^30ns == 1.074s.
33258687acbSDon Zickus */
get_timestamp(void)333c06b4f19SNamhyung Kim static unsigned long get_timestamp(void)
33458687acbSDon Zickus {
335545a2bf7SCyril Bur return running_clock() >> 30LL; /* 2^30 ~= 10^9 */
33658687acbSDon Zickus }
33758687acbSDon Zickus
set_sample_period(void)3380f34c400SChuansheng Liu static void set_sample_period(void)
33958687acbSDon Zickus {
34058687acbSDon Zickus /*
341586692a5SMandeep Singh Baines * convert watchdog_thresh from seconds to ns
34286f5e6a7SFernando Luis Vázquez Cao * the divide by 5 is to give hrtimer several chances (two
34386f5e6a7SFernando Luis Vázquez Cao * or three with the current relation between the soft
34486f5e6a7SFernando Luis Vázquez Cao * and hard thresholds) to increment before the
34586f5e6a7SFernando Luis Vázquez Cao * hardlockup detector generates a warning
34658687acbSDon Zickus */
3470f34c400SChuansheng Liu sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5);
3487edaeb68SThomas Gleixner watchdog_update_hrtimer_threshold(sample_period);
34958687acbSDon Zickus }
35058687acbSDon Zickus
update_report_ts(void)351fef06efcSPetr Mladek static void update_report_ts(void)
352fef06efcSPetr Mladek {
353fef06efcSPetr Mladek __this_cpu_write(watchdog_report_ts, get_timestamp());
354fef06efcSPetr Mladek }
355fef06efcSPetr Mladek
35658687acbSDon Zickus /* Commands for resetting the watchdog */
update_touch_ts(void)3577c0012f5SPetr Mladek static void update_touch_ts(void)
35858687acbSDon Zickus {
359c06b4f19SNamhyung Kim __this_cpu_write(watchdog_touch_ts, get_timestamp());
360fef06efcSPetr Mladek update_report_ts();
36158687acbSDon Zickus }
36258687acbSDon Zickus
36303e0d461STejun Heo /**
36403e0d461STejun Heo * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
36503e0d461STejun Heo *
36603e0d461STejun Heo * Call when the scheduler may have stalled for legitimate reasons
36703e0d461STejun Heo * preventing the watchdog task from executing - e.g. the scheduler
36803e0d461STejun Heo * entering idle state. This should only be used for scheduler events.
36903e0d461STejun Heo * Use touch_softlockup_watchdog() for everything else.
37003e0d461STejun Heo */
touch_softlockup_watchdog_sched(void)371cb9d7fd5SVincent Whitchurch notrace void touch_softlockup_watchdog_sched(void)
37258687acbSDon Zickus {
3737861144bSAndrew Morton /*
374fef06efcSPetr Mladek * Preemption can be enabled. It doesn't matter which CPU's watchdog
375fef06efcSPetr Mladek * report period gets restarted here, so use the raw_ operation.
3767861144bSAndrew Morton */
377fef06efcSPetr Mladek raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
37858687acbSDon Zickus }
37903e0d461STejun Heo
touch_softlockup_watchdog(void)380cb9d7fd5SVincent Whitchurch notrace void touch_softlockup_watchdog(void)
38103e0d461STejun Heo {
38203e0d461STejun Heo touch_softlockup_watchdog_sched();
38382607adcSTejun Heo wq_watchdog_touch(raw_smp_processor_id());
38403e0d461STejun Heo }
3850167c781SIngo Molnar EXPORT_SYMBOL(touch_softlockup_watchdog);
38658687acbSDon Zickus
touch_all_softlockup_watchdogs(void)387332fbdbcSDon Zickus void touch_all_softlockup_watchdogs(void)
38858687acbSDon Zickus {
38958687acbSDon Zickus int cpu;
39058687acbSDon Zickus
39158687acbSDon Zickus /*
392d57108d4SThomas Gleixner * watchdog_mutex cannpt be taken here, as this might be called
393d57108d4SThomas Gleixner * from (soft)interrupt context, so the access to
394d57108d4SThomas Gleixner * watchdog_allowed_cpumask might race with a concurrent update.
395d57108d4SThomas Gleixner *
396d57108d4SThomas Gleixner * The watchdog time stamp can race against a concurrent real
397d57108d4SThomas Gleixner * update as well, the only side effect might be a cycle delay for
398d57108d4SThomas Gleixner * the softlockup check.
39958687acbSDon Zickus */
40089e28ce6SWang Qing for_each_cpu(cpu, &watchdog_allowed_mask) {
401fef06efcSPetr Mladek per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT;
40289e28ce6SWang Qing wq_watchdog_touch(cpu);
40389e28ce6SWang Qing }
40458687acbSDon Zickus }
40558687acbSDon Zickus
touch_softlockup_watchdog_sync(void)40658687acbSDon Zickus void touch_softlockup_watchdog_sync(void)
40758687acbSDon Zickus {
408f7f66b05SChristoph Lameter __this_cpu_write(softlockup_touch_sync, true);
409fef06efcSPetr Mladek __this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
41058687acbSDon Zickus }
41158687acbSDon Zickus
is_softlockup(unsigned long touch_ts,unsigned long period_ts,unsigned long now)4120f90b88dSPetr Mladek static int is_softlockup(unsigned long touch_ts,
4130f90b88dSPetr Mladek unsigned long period_ts,
4140f90b88dSPetr Mladek unsigned long now)
41558687acbSDon Zickus {
416df95d308SDouglas Anderson if ((watchdog_enabled & WATCHDOG_SOFTOCKUP_ENABLED) && watchdog_thresh) {
417195daf66SUlrich Obergfell /* Warn about unreasonable delays. */
418fef06efcSPetr Mladek if (time_after(now, period_ts + get_softlockup_thresh()))
41958687acbSDon Zickus return now - touch_ts;
420195daf66SUlrich Obergfell }
42158687acbSDon Zickus return 0;
42258687acbSDon Zickus }
42358687acbSDon Zickus
42405a4a952SNicholas Piggin /* watchdog detector functions */
425be45bf53SPeter Zijlstra static DEFINE_PER_CPU(struct completion, softlockup_completion);
426be45bf53SPeter Zijlstra static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work);
427be45bf53SPeter Zijlstra
4289cf57731SPeter Zijlstra /*
429b124ac45SWang Qing * The watchdog feed function - touches the timestamp.
4309cf57731SPeter Zijlstra *
4319cf57731SPeter Zijlstra * It only runs once every sample_period seconds (4 seconds by
4329cf57731SPeter Zijlstra * default) to reset the softlockup timestamp. If this gets delayed
4339cf57731SPeter Zijlstra * for more than 2*watchdog_thresh seconds then the debug-printout
4349cf57731SPeter Zijlstra * triggers in watchdog_timer_fn().
4359cf57731SPeter Zijlstra */
softlockup_fn(void * data)4369cf57731SPeter Zijlstra static int softlockup_fn(void *data)
4379cf57731SPeter Zijlstra {
4387c0012f5SPetr Mladek update_touch_ts();
439be45bf53SPeter Zijlstra complete(this_cpu_ptr(&softlockup_completion));
4409cf57731SPeter Zijlstra
4419cf57731SPeter Zijlstra return 0;
4429cf57731SPeter Zijlstra }
4439cf57731SPeter Zijlstra
44458687acbSDon Zickus /* watchdog kicker functions */
watchdog_timer_fn(struct hrtimer * hrtimer)44558687acbSDon Zickus static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
44658687acbSDon Zickus {
4470f90b88dSPetr Mladek unsigned long touch_ts, period_ts, now;
44858687acbSDon Zickus struct pt_regs *regs = get_irq_regs();
44958687acbSDon Zickus int duration;
450ed235875SAaron Tomlin int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
45158687acbSDon Zickus
45201f0a027SThomas Gleixner if (!watchdog_enabled)
453b94f5118SDon Zickus return HRTIMER_NORESTART;
454b94f5118SDon Zickus
455d3b62aceSDouglas Anderson watchdog_hardlockup_kick();
45658687acbSDon Zickus
45758687acbSDon Zickus /* kick the softlockup detector */
458be45bf53SPeter Zijlstra if (completion_done(this_cpu_ptr(&softlockup_completion))) {
459be45bf53SPeter Zijlstra reinit_completion(this_cpu_ptr(&softlockup_completion));
4609cf57731SPeter Zijlstra stop_one_cpu_nowait(smp_processor_id(),
4619cf57731SPeter Zijlstra softlockup_fn, NULL,
4629cf57731SPeter Zijlstra this_cpu_ptr(&softlockup_stop_work));
463be45bf53SPeter Zijlstra }
46458687acbSDon Zickus
46558687acbSDon Zickus /* .. and repeat */
4660f34c400SChuansheng Liu hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
46758687acbSDon Zickus
4689bf3bc94SPetr Mladek /*
4690f90b88dSPetr Mladek * Read the current timestamp first. It might become invalid anytime
4700f90b88dSPetr Mladek * when a virtual machine is stopped by the host or when the watchog
4710f90b88dSPetr Mladek * is touched from NMI.
4720f90b88dSPetr Mladek */
4730f90b88dSPetr Mladek now = get_timestamp();
4740f90b88dSPetr Mladek /*
4759bf3bc94SPetr Mladek * If a virtual machine is stopped by the host it can look to
4760f90b88dSPetr Mladek * the watchdog like a soft lockup. This function touches the watchdog.
4779bf3bc94SPetr Mladek */
4789bf3bc94SPetr Mladek kvm_check_and_clear_guest_paused();
4790f90b88dSPetr Mladek /*
4800f90b88dSPetr Mladek * The stored timestamp is comparable with @now only when not touched.
4810f90b88dSPetr Mladek * It might get touched anytime from NMI. Make sure that is_softlockup()
4820f90b88dSPetr Mladek * uses the same (valid) value.
4830f90b88dSPetr Mladek */
4840f90b88dSPetr Mladek period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts));
4859bf3bc94SPetr Mladek
4869bf3bc94SPetr Mladek /* Reset the interval when touched by known problematic code. */
487fef06efcSPetr Mladek if (period_ts == SOFTLOCKUP_DELAY_REPORT) {
488909ea964SChristoph Lameter if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
48958687acbSDon Zickus /*
49058687acbSDon Zickus * If the time stamp was touched atomically
49158687acbSDon Zickus * make sure the scheduler tick is up to date.
49258687acbSDon Zickus */
493909ea964SChristoph Lameter __this_cpu_write(softlockup_touch_sync, false);
49458687acbSDon Zickus sched_clock_tick();
49558687acbSDon Zickus }
4965d1c0f4aSEric B Munson
497fef06efcSPetr Mladek update_report_ts();
49858687acbSDon Zickus return HRTIMER_RESTART;
49958687acbSDon Zickus }
50058687acbSDon Zickus
5010f90b88dSPetr Mladek /* Check for a softlockup. */
5020f90b88dSPetr Mladek touch_ts = __this_cpu_read(watchdog_touch_ts);
5030f90b88dSPetr Mladek duration = is_softlockup(touch_ts, period_ts, now);
50458687acbSDon Zickus if (unlikely(duration)) {
5055d1c0f4aSEric B Munson /*
5069f113bf7SPetr Mladek * Prevent multiple soft-lockup reports if one cpu is already
5079f113bf7SPetr Mladek * engaged in dumping all cpu back traces.
508ed235875SAaron Tomlin */
5099f113bf7SPetr Mladek if (softlockup_all_cpu_backtrace) {
5109f113bf7SPetr Mladek if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn))
511ed235875SAaron Tomlin return HRTIMER_RESTART;
512ed235875SAaron Tomlin }
513ed235875SAaron Tomlin
514c9ad17c9SPetr Mladek /* Start period for the next softlockup warning. */
515fef06efcSPetr Mladek update_report_ts();
516c9ad17c9SPetr Mladek
517656c3b79SFabian Frederick pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
51826e09c6eSDon Zickus smp_processor_id(), duration,
51958687acbSDon Zickus current->comm, task_pid_nr(current));
52058687acbSDon Zickus print_modules();
52158687acbSDon Zickus print_irqtrace_events(current);
52258687acbSDon Zickus if (regs)
52358687acbSDon Zickus show_regs(regs);
52458687acbSDon Zickus else
52558687acbSDon Zickus dump_stack();
52658687acbSDon Zickus
527ed235875SAaron Tomlin if (softlockup_all_cpu_backtrace) {
5288d539b84SDouglas Anderson trigger_allbutcpu_cpu_backtrace(smp_processor_id());
5299f113bf7SPetr Mladek clear_bit_unlock(0, &soft_lockup_nmi_warn);
530ed235875SAaron Tomlin }
531ed235875SAaron Tomlin
53269361eefSJosh Hunt add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
53358687acbSDon Zickus if (softlockup_panic)
53458687acbSDon Zickus panic("softlockup: hung tasks");
5351bc503cbSPetr Mladek }
53658687acbSDon Zickus
53758687acbSDon Zickus return HRTIMER_RESTART;
53858687acbSDon Zickus }
53958687acbSDon Zickus
watchdog_enable(unsigned int cpu)540bcd951cfSThomas Gleixner static void watchdog_enable(unsigned int cpu)
541bcd951cfSThomas Gleixner {
54201f0a027SThomas Gleixner struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
543be45bf53SPeter Zijlstra struct completion *done = this_cpu_ptr(&softlockup_completion);
54458687acbSDon Zickus
5459cf57731SPeter Zijlstra WARN_ON_ONCE(cpu != smp_processor_id());
5469cf57731SPeter Zijlstra
547be45bf53SPeter Zijlstra init_completion(done);
548be45bf53SPeter Zijlstra complete(done);
549be45bf53SPeter Zijlstra
55001f0a027SThomas Gleixner /*
551df95d308SDouglas Anderson * Start the timer first to prevent the hardlockup watchdog triggering
55201f0a027SThomas Gleixner * before the timer has a chance to fire.
55301f0a027SThomas Gleixner */
554d2ab4cf4SSebastian Andrzej Siewior hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
5553935e895SBjørn Mork hrtimer->function = watchdog_timer_fn;
5560f34c400SChuansheng Liu hrtimer_start(hrtimer, ns_to_ktime(sample_period),
557d2ab4cf4SSebastian Andrzej Siewior HRTIMER_MODE_REL_PINNED_HARD);
55858687acbSDon Zickus
55901f0a027SThomas Gleixner /* Initialize timestamp */
5607c0012f5SPetr Mladek update_touch_ts();
561df95d308SDouglas Anderson /* Enable the hardlockup detector */
562df95d308SDouglas Anderson if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)
563df95d308SDouglas Anderson watchdog_hardlockup_enable(cpu);
56458687acbSDon Zickus }
565bcd951cfSThomas Gleixner
watchdog_disable(unsigned int cpu)566bcd951cfSThomas Gleixner static void watchdog_disable(unsigned int cpu)
567bcd951cfSThomas Gleixner {
56801f0a027SThomas Gleixner struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
569bcd951cfSThomas Gleixner
5709cf57731SPeter Zijlstra WARN_ON_ONCE(cpu != smp_processor_id());
5719cf57731SPeter Zijlstra
57201f0a027SThomas Gleixner /*
573df95d308SDouglas Anderson * Disable the hardlockup detector first. That prevents that a large
574df95d308SDouglas Anderson * delay between disabling the timer and disabling the hardlockup
575df95d308SDouglas Anderson * detector causes a false positive.
57601f0a027SThomas Gleixner */
577df95d308SDouglas Anderson watchdog_hardlockup_disable(cpu);
57801f0a027SThomas Gleixner hrtimer_cancel(hrtimer);
579be45bf53SPeter Zijlstra wait_for_completion(this_cpu_ptr(&softlockup_completion));
580bcd951cfSThomas Gleixner }
581bcd951cfSThomas Gleixner
softlockup_stop_fn(void * data)5829cf57731SPeter Zijlstra static int softlockup_stop_fn(void *data)
583b8900bc0SFrederic Weisbecker {
5849cf57731SPeter Zijlstra watchdog_disable(smp_processor_id());
5859cf57731SPeter Zijlstra return 0;
586b8900bc0SFrederic Weisbecker }
587b8900bc0SFrederic Weisbecker
softlockup_stop_all(void)5889cf57731SPeter Zijlstra static void softlockup_stop_all(void)
589bcd951cfSThomas Gleixner {
5909cf57731SPeter Zijlstra int cpu;
591bcd951cfSThomas Gleixner
5929cf57731SPeter Zijlstra if (!softlockup_initialized)
5932eb2527fSThomas Gleixner return;
5942eb2527fSThomas Gleixner
5959cf57731SPeter Zijlstra for_each_cpu(cpu, &watchdog_allowed_mask)
5969cf57731SPeter Zijlstra smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false);
5972eb2527fSThomas Gleixner
5982eb2527fSThomas Gleixner cpumask_clear(&watchdog_allowed_mask);
5992eb2527fSThomas Gleixner }
6002eb2527fSThomas Gleixner
softlockup_start_fn(void * data)6019cf57731SPeter Zijlstra static int softlockup_start_fn(void *data)
6022eb2527fSThomas Gleixner {
6039cf57731SPeter Zijlstra watchdog_enable(smp_processor_id());
6049cf57731SPeter Zijlstra return 0;
6059cf57731SPeter Zijlstra }
6069cf57731SPeter Zijlstra
softlockup_start_all(void)6079cf57731SPeter Zijlstra static void softlockup_start_all(void)
6089cf57731SPeter Zijlstra {
6099cf57731SPeter Zijlstra int cpu;
6109cf57731SPeter Zijlstra
6112eb2527fSThomas Gleixner cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask);
6129cf57731SPeter Zijlstra for_each_cpu(cpu, &watchdog_allowed_mask)
6139cf57731SPeter Zijlstra smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false);
6149cf57731SPeter Zijlstra }
6159cf57731SPeter Zijlstra
lockup_detector_online_cpu(unsigned int cpu)6169cf57731SPeter Zijlstra int lockup_detector_online_cpu(unsigned int cpu)
6179cf57731SPeter Zijlstra {
6187dd47617SThomas Gleixner if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
6199cf57731SPeter Zijlstra watchdog_enable(cpu);
6209cf57731SPeter Zijlstra return 0;
6219cf57731SPeter Zijlstra }
6229cf57731SPeter Zijlstra
lockup_detector_offline_cpu(unsigned int cpu)6239cf57731SPeter Zijlstra int lockup_detector_offline_cpu(unsigned int cpu)
6249cf57731SPeter Zijlstra {
6257dd47617SThomas Gleixner if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
6269cf57731SPeter Zijlstra watchdog_disable(cpu);
6279cf57731SPeter Zijlstra return 0;
6282eb2527fSThomas Gleixner }
6292eb2527fSThomas Gleixner
__lockup_detector_reconfigure(void)6307c56a873SLaurent Dufour static void __lockup_detector_reconfigure(void)
6312eb2527fSThomas Gleixner {
632e31d6883SThomas Gleixner cpus_read_lock();
633df95d308SDouglas Anderson watchdog_hardlockup_stop();
6349cf57731SPeter Zijlstra
6359cf57731SPeter Zijlstra softlockup_stop_all();
6362eb2527fSThomas Gleixner set_sample_period();
63709154985SThomas Gleixner lockup_detector_update_enable();
63809154985SThomas Gleixner if (watchdog_enabled && watchdog_thresh)
6399cf57731SPeter Zijlstra softlockup_start_all();
6409cf57731SPeter Zijlstra
641df95d308SDouglas Anderson watchdog_hardlockup_start();
642e31d6883SThomas Gleixner cpus_read_unlock();
643e31d6883SThomas Gleixner /*
644e31d6883SThomas Gleixner * Must be called outside the cpus locked section to prevent
645e31d6883SThomas Gleixner * recursive locking in the perf code.
646e31d6883SThomas Gleixner */
647e31d6883SThomas Gleixner __lockup_detector_cleanup();
6482eb2527fSThomas Gleixner }
6492eb2527fSThomas Gleixner
lockup_detector_reconfigure(void)6507c56a873SLaurent Dufour void lockup_detector_reconfigure(void)
6517c56a873SLaurent Dufour {
6527c56a873SLaurent Dufour mutex_lock(&watchdog_mutex);
6537c56a873SLaurent Dufour __lockup_detector_reconfigure();
6547c56a873SLaurent Dufour mutex_unlock(&watchdog_mutex);
6557c56a873SLaurent Dufour }
6567c56a873SLaurent Dufour
6572eb2527fSThomas Gleixner /*
658b124ac45SWang Qing * Create the watchdog infrastructure and configure the detector(s).
6592eb2527fSThomas Gleixner */
lockup_detector_setup(void)6605587185dSThomas Gleixner static __init void lockup_detector_setup(void)
6612eb2527fSThomas Gleixner {
6622eb2527fSThomas Gleixner /*
6632eb2527fSThomas Gleixner * If sysctl is off and watchdog got disabled on the command line,
6642eb2527fSThomas Gleixner * nothing to do here.
6652eb2527fSThomas Gleixner */
66609154985SThomas Gleixner lockup_detector_update_enable();
66709154985SThomas Gleixner
6682eb2527fSThomas Gleixner if (!IS_ENABLED(CONFIG_SYSCTL) &&
6692eb2527fSThomas Gleixner !(watchdog_enabled && watchdog_thresh))
6702eb2527fSThomas Gleixner return;
6712eb2527fSThomas Gleixner
6722eb2527fSThomas Gleixner mutex_lock(&watchdog_mutex);
6737c56a873SLaurent Dufour __lockup_detector_reconfigure();
6749cf57731SPeter Zijlstra softlockup_initialized = true;
6752eb2527fSThomas Gleixner mutex_unlock(&watchdog_mutex);
6762eb2527fSThomas Gleixner }
6772eb2527fSThomas Gleixner
6782b9d7f23SThomas Gleixner #else /* CONFIG_SOFTLOCKUP_DETECTOR */
__lockup_detector_reconfigure(void)6797c56a873SLaurent Dufour static void __lockup_detector_reconfigure(void)
6806592ad2fSThomas Gleixner {
681e31d6883SThomas Gleixner cpus_read_lock();
682df95d308SDouglas Anderson watchdog_hardlockup_stop();
68309154985SThomas Gleixner lockup_detector_update_enable();
684df95d308SDouglas Anderson watchdog_hardlockup_start();
685e31d6883SThomas Gleixner cpus_read_unlock();
6866592ad2fSThomas Gleixner }
lockup_detector_reconfigure(void)6877c56a873SLaurent Dufour void lockup_detector_reconfigure(void)
6887c56a873SLaurent Dufour {
6897c56a873SLaurent Dufour __lockup_detector_reconfigure();
6907c56a873SLaurent Dufour }
lockup_detector_setup(void)6915587185dSThomas Gleixner static inline void lockup_detector_setup(void)
69234ddaa3eSThomas Gleixner {
6937c56a873SLaurent Dufour __lockup_detector_reconfigure();
69434ddaa3eSThomas Gleixner }
6952b9d7f23SThomas Gleixner #endif /* !CONFIG_SOFTLOCKUP_DETECTOR */
69605a4a952SNicholas Piggin
__lockup_detector_cleanup(void)697941154bdSThomas Gleixner static void __lockup_detector_cleanup(void)
698941154bdSThomas Gleixner {
699941154bdSThomas Gleixner lockdep_assert_held(&watchdog_mutex);
700941154bdSThomas Gleixner hardlockup_detector_perf_cleanup();
701941154bdSThomas Gleixner }
702941154bdSThomas Gleixner
703941154bdSThomas Gleixner /**
704941154bdSThomas Gleixner * lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes
705941154bdSThomas Gleixner *
706941154bdSThomas Gleixner * Caller must not hold the cpu hotplug rwsem.
707941154bdSThomas Gleixner */
lockup_detector_cleanup(void)708941154bdSThomas Gleixner void lockup_detector_cleanup(void)
709941154bdSThomas Gleixner {
710941154bdSThomas Gleixner mutex_lock(&watchdog_mutex);
711941154bdSThomas Gleixner __lockup_detector_cleanup();
712941154bdSThomas Gleixner mutex_unlock(&watchdog_mutex);
713941154bdSThomas Gleixner }
714941154bdSThomas Gleixner
7156554fd8cSThomas Gleixner /**
7166554fd8cSThomas Gleixner * lockup_detector_soft_poweroff - Interface to stop lockup detector(s)
7176554fd8cSThomas Gleixner *
7186554fd8cSThomas Gleixner * Special interface for parisc. It prevents lockup detector warnings from
7196554fd8cSThomas Gleixner * the default pm_poweroff() function which busy loops forever.
7206554fd8cSThomas Gleixner */
lockup_detector_soft_poweroff(void)7216554fd8cSThomas Gleixner void lockup_detector_soft_poweroff(void)
7226554fd8cSThomas Gleixner {
7236554fd8cSThomas Gleixner watchdog_enabled = 0;
7246554fd8cSThomas Gleixner }
7256554fd8cSThomas Gleixner
72658cf690aSUlrich Obergfell #ifdef CONFIG_SYSCTL
72758cf690aSUlrich Obergfell
728b124ac45SWang Qing /* Propagate any changes to the watchdog infrastructure */
proc_watchdog_update(void)729d57108d4SThomas Gleixner static void proc_watchdog_update(void)
73058687acbSDon Zickus {
731e8b62b2dSThomas Gleixner /* Remove impossible cpus to keep sysctl output clean. */
732e8b62b2dSThomas Gleixner cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask);
7337c56a873SLaurent Dufour __lockup_detector_reconfigure();
734a0c9cbb9SUlrich Obergfell }
735a0c9cbb9SUlrich Obergfell
736a0c9cbb9SUlrich Obergfell /*
737ef246a21SUlrich Obergfell * common function for watchdog, nmi_watchdog and soft_watchdog parameter
738ef246a21SUlrich Obergfell *
7397feeb9cdSThomas Gleixner * caller | table->data points to | 'which'
740df95d308SDouglas Anderson * -------------------|----------------------------------|-------------------------------
741df95d308SDouglas Anderson * proc_watchdog | watchdog_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED |
742df95d308SDouglas Anderson * | | WATCHDOG_SOFTOCKUP_ENABLED
743df95d308SDouglas Anderson * -------------------|----------------------------------|-------------------------------
744df95d308SDouglas Anderson * proc_nmi_watchdog | watchdog_hardlockup_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED
745df95d308SDouglas Anderson * -------------------|----------------------------------|-------------------------------
746df95d308SDouglas Anderson * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTOCKUP_ENABLED
747ef246a21SUlrich Obergfell */
proc_watchdog_common(int which,struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)748ef246a21SUlrich Obergfell static int proc_watchdog_common(int which, struct ctl_table *table, int write,
74932927393SChristoph Hellwig void *buffer, size_t *lenp, loff_t *ppos)
750ef246a21SUlrich Obergfell {
75109154985SThomas Gleixner int err, old, *param = table->data;
752bcd951cfSThomas Gleixner
753946d1977SThomas Gleixner mutex_lock(&watchdog_mutex);
754ef246a21SUlrich Obergfell
755ef246a21SUlrich Obergfell if (!write) {
75609154985SThomas Gleixner /*
75709154985SThomas Gleixner * On read synchronize the userspace interface. This is a
75809154985SThomas Gleixner * racy snapshot.
75909154985SThomas Gleixner */
76009154985SThomas Gleixner *param = (watchdog_enabled & which) != 0;
761b8900bc0SFrederic Weisbecker err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
762ef246a21SUlrich Obergfell } else {
76309154985SThomas Gleixner old = READ_ONCE(*param);
764ef246a21SUlrich Obergfell err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
76509154985SThomas Gleixner if (!err && old != READ_ONCE(*param))
766d57108d4SThomas Gleixner proc_watchdog_update();
767ef246a21SUlrich Obergfell }
768946d1977SThomas Gleixner mutex_unlock(&watchdog_mutex);
769ef246a21SUlrich Obergfell return err;
770ef246a21SUlrich Obergfell }
771ef246a21SUlrich Obergfell
772ef246a21SUlrich Obergfell /*
77383a80a39SUlrich Obergfell * /proc/sys/kernel/watchdog
77483a80a39SUlrich Obergfell */
proc_watchdog(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)77583a80a39SUlrich Obergfell int proc_watchdog(struct ctl_table *table, int write,
77632927393SChristoph Hellwig void *buffer, size_t *lenp, loff_t *ppos)
77783a80a39SUlrich Obergfell {
778df95d308SDouglas Anderson return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED |
779df95d308SDouglas Anderson WATCHDOG_SOFTOCKUP_ENABLED,
78083a80a39SUlrich Obergfell table, write, buffer, lenp, ppos);
78183a80a39SUlrich Obergfell }
78283a80a39SUlrich Obergfell
78383a80a39SUlrich Obergfell /*
78483a80a39SUlrich Obergfell * /proc/sys/kernel/nmi_watchdog
78583a80a39SUlrich Obergfell */
proc_nmi_watchdog(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)78683a80a39SUlrich Obergfell int proc_nmi_watchdog(struct ctl_table *table, int write,
78732927393SChristoph Hellwig void *buffer, size_t *lenp, loff_t *ppos)
78883a80a39SUlrich Obergfell {
789df95d308SDouglas Anderson if (!watchdog_hardlockup_available && write)
790a994a314SThomas Gleixner return -ENOTSUPP;
791df95d308SDouglas Anderson return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED,
79283a80a39SUlrich Obergfell table, write, buffer, lenp, ppos);
79383a80a39SUlrich Obergfell }
79483a80a39SUlrich Obergfell
79583a80a39SUlrich Obergfell /*
79683a80a39SUlrich Obergfell * /proc/sys/kernel/soft_watchdog
79783a80a39SUlrich Obergfell */
proc_soft_watchdog(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)79883a80a39SUlrich Obergfell int proc_soft_watchdog(struct ctl_table *table, int write,
79932927393SChristoph Hellwig void *buffer, size_t *lenp, loff_t *ppos)
80083a80a39SUlrich Obergfell {
801df95d308SDouglas Anderson return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED,
80283a80a39SUlrich Obergfell table, write, buffer, lenp, ppos);
80383a80a39SUlrich Obergfell }
80483a80a39SUlrich Obergfell
80583a80a39SUlrich Obergfell /*
80683a80a39SUlrich Obergfell * /proc/sys/kernel/watchdog_thresh
80783a80a39SUlrich Obergfell */
proc_watchdog_thresh(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)80883a80a39SUlrich Obergfell int proc_watchdog_thresh(struct ctl_table *table, int write,
80932927393SChristoph Hellwig void *buffer, size_t *lenp, loff_t *ppos)
81083a80a39SUlrich Obergfell {
811d57108d4SThomas Gleixner int err, old;
81283a80a39SUlrich Obergfell
813946d1977SThomas Gleixner mutex_lock(&watchdog_mutex);
81483a80a39SUlrich Obergfell
815d57108d4SThomas Gleixner old = READ_ONCE(watchdog_thresh);
81683a80a39SUlrich Obergfell err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
81783a80a39SUlrich Obergfell
818d57108d4SThomas Gleixner if (!err && write && old != READ_ONCE(watchdog_thresh))
819d57108d4SThomas Gleixner proc_watchdog_update();
820e04ab2bcSMandeep Singh Baines
821946d1977SThomas Gleixner mutex_unlock(&watchdog_mutex);
822b8900bc0SFrederic Weisbecker return err;
82358687acbSDon Zickus }
824fe4ba3c3SChris Metcalf
825fe4ba3c3SChris Metcalf /*
826fe4ba3c3SChris Metcalf * The cpumask is the mask of possible cpus that the watchdog can run
827fe4ba3c3SChris Metcalf * on, not the mask of cpus it is actually running on. This allows the
828fe4ba3c3SChris Metcalf * user to specify a mask that will include cpus that have not yet
829fe4ba3c3SChris Metcalf * been brought online, if desired.
830fe4ba3c3SChris Metcalf */
proc_watchdog_cpumask(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)831fe4ba3c3SChris Metcalf int proc_watchdog_cpumask(struct ctl_table *table, int write,
83232927393SChristoph Hellwig void *buffer, size_t *lenp, loff_t *ppos)
833fe4ba3c3SChris Metcalf {
834fe4ba3c3SChris Metcalf int err;
835fe4ba3c3SChris Metcalf
836946d1977SThomas Gleixner mutex_lock(&watchdog_mutex);
8378c073d27SUlrich Obergfell
838fe4ba3c3SChris Metcalf err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
83905ba3de7SThomas Gleixner if (!err && write)
840e8b62b2dSThomas Gleixner proc_watchdog_update();
8415490125dSThomas Gleixner
842946d1977SThomas Gleixner mutex_unlock(&watchdog_mutex);
843fe4ba3c3SChris Metcalf return err;
844fe4ba3c3SChris Metcalf }
845dd0693fdSXiaoming Ni
846dd0693fdSXiaoming Ni static const int sixty = 60;
847dd0693fdSXiaoming Ni
848dd0693fdSXiaoming Ni static struct ctl_table watchdog_sysctls[] = {
849dd0693fdSXiaoming Ni {
850dd0693fdSXiaoming Ni .procname = "watchdog",
851dd0693fdSXiaoming Ni .data = &watchdog_user_enabled,
852dd0693fdSXiaoming Ni .maxlen = sizeof(int),
853dd0693fdSXiaoming Ni .mode = 0644,
854dd0693fdSXiaoming Ni .proc_handler = proc_watchdog,
855dd0693fdSXiaoming Ni .extra1 = SYSCTL_ZERO,
856dd0693fdSXiaoming Ni .extra2 = SYSCTL_ONE,
857dd0693fdSXiaoming Ni },
858dd0693fdSXiaoming Ni {
859dd0693fdSXiaoming Ni .procname = "watchdog_thresh",
860dd0693fdSXiaoming Ni .data = &watchdog_thresh,
861dd0693fdSXiaoming Ni .maxlen = sizeof(int),
862dd0693fdSXiaoming Ni .mode = 0644,
863dd0693fdSXiaoming Ni .proc_handler = proc_watchdog_thresh,
864dd0693fdSXiaoming Ni .extra1 = SYSCTL_ZERO,
865dd0693fdSXiaoming Ni .extra2 = (void *)&sixty,
866dd0693fdSXiaoming Ni },
867dd0693fdSXiaoming Ni {
868dd0693fdSXiaoming Ni .procname = "watchdog_cpumask",
869dd0693fdSXiaoming Ni .data = &watchdog_cpumask_bits,
870dd0693fdSXiaoming Ni .maxlen = NR_CPUS,
871dd0693fdSXiaoming Ni .mode = 0644,
872dd0693fdSXiaoming Ni .proc_handler = proc_watchdog_cpumask,
873dd0693fdSXiaoming Ni },
874dd0693fdSXiaoming Ni #ifdef CONFIG_SOFTLOCKUP_DETECTOR
875dd0693fdSXiaoming Ni {
876dd0693fdSXiaoming Ni .procname = "soft_watchdog",
877df95d308SDouglas Anderson .data = &watchdog_softlockup_user_enabled,
878dd0693fdSXiaoming Ni .maxlen = sizeof(int),
879dd0693fdSXiaoming Ni .mode = 0644,
880dd0693fdSXiaoming Ni .proc_handler = proc_soft_watchdog,
881dd0693fdSXiaoming Ni .extra1 = SYSCTL_ZERO,
882dd0693fdSXiaoming Ni .extra2 = SYSCTL_ONE,
883dd0693fdSXiaoming Ni },
884dd0693fdSXiaoming Ni {
885dd0693fdSXiaoming Ni .procname = "softlockup_panic",
886dd0693fdSXiaoming Ni .data = &softlockup_panic,
887dd0693fdSXiaoming Ni .maxlen = sizeof(int),
888dd0693fdSXiaoming Ni .mode = 0644,
889dd0693fdSXiaoming Ni .proc_handler = proc_dointvec_minmax,
890dd0693fdSXiaoming Ni .extra1 = SYSCTL_ZERO,
891dd0693fdSXiaoming Ni .extra2 = SYSCTL_ONE,
892dd0693fdSXiaoming Ni },
893dd0693fdSXiaoming Ni #ifdef CONFIG_SMP
894dd0693fdSXiaoming Ni {
895dd0693fdSXiaoming Ni .procname = "softlockup_all_cpu_backtrace",
896dd0693fdSXiaoming Ni .data = &sysctl_softlockup_all_cpu_backtrace,
897dd0693fdSXiaoming Ni .maxlen = sizeof(int),
898dd0693fdSXiaoming Ni .mode = 0644,
899dd0693fdSXiaoming Ni .proc_handler = proc_dointvec_minmax,
900dd0693fdSXiaoming Ni .extra1 = SYSCTL_ZERO,
901dd0693fdSXiaoming Ni .extra2 = SYSCTL_ONE,
902dd0693fdSXiaoming Ni },
903dd0693fdSXiaoming Ni #endif /* CONFIG_SMP */
904dd0693fdSXiaoming Ni #endif
905dd0693fdSXiaoming Ni #ifdef CONFIG_HARDLOCKUP_DETECTOR
906dd0693fdSXiaoming Ni {
907dd0693fdSXiaoming Ni .procname = "hardlockup_panic",
908dd0693fdSXiaoming Ni .data = &hardlockup_panic,
909dd0693fdSXiaoming Ni .maxlen = sizeof(int),
910dd0693fdSXiaoming Ni .mode = 0644,
911dd0693fdSXiaoming Ni .proc_handler = proc_dointvec_minmax,
912dd0693fdSXiaoming Ni .extra1 = SYSCTL_ZERO,
913dd0693fdSXiaoming Ni .extra2 = SYSCTL_ONE,
914dd0693fdSXiaoming Ni },
915dd0693fdSXiaoming Ni #ifdef CONFIG_SMP
916dd0693fdSXiaoming Ni {
917dd0693fdSXiaoming Ni .procname = "hardlockup_all_cpu_backtrace",
918dd0693fdSXiaoming Ni .data = &sysctl_hardlockup_all_cpu_backtrace,
919dd0693fdSXiaoming Ni .maxlen = sizeof(int),
920dd0693fdSXiaoming Ni .mode = 0644,
921dd0693fdSXiaoming Ni .proc_handler = proc_dointvec_minmax,
922dd0693fdSXiaoming Ni .extra1 = SYSCTL_ZERO,
923dd0693fdSXiaoming Ni .extra2 = SYSCTL_ONE,
924dd0693fdSXiaoming Ni },
925dd0693fdSXiaoming Ni #endif /* CONFIG_SMP */
926dd0693fdSXiaoming Ni #endif
927dd0693fdSXiaoming Ni {}
928dd0693fdSXiaoming Ni };
929dd0693fdSXiaoming Ni
9309ec272c5SDouglas Anderson static struct ctl_table watchdog_hardlockup_sysctl[] = {
9319ec272c5SDouglas Anderson {
9329ec272c5SDouglas Anderson .procname = "nmi_watchdog",
9339ec272c5SDouglas Anderson .data = &watchdog_hardlockup_user_enabled,
9349ec272c5SDouglas Anderson .maxlen = sizeof(int),
9359ec272c5SDouglas Anderson .mode = 0444,
9369ec272c5SDouglas Anderson .proc_handler = proc_nmi_watchdog,
9379ec272c5SDouglas Anderson .extra1 = SYSCTL_ZERO,
9389ec272c5SDouglas Anderson .extra2 = SYSCTL_ONE,
9399ec272c5SDouglas Anderson },
9409ec272c5SDouglas Anderson {}
9419ec272c5SDouglas Anderson };
9429ec272c5SDouglas Anderson
watchdog_sysctl_init(void)943dd0693fdSXiaoming Ni static void __init watchdog_sysctl_init(void)
944dd0693fdSXiaoming Ni {
945dd0693fdSXiaoming Ni register_sysctl_init("kernel", watchdog_sysctls);
9469ec272c5SDouglas Anderson
9479ec272c5SDouglas Anderson if (watchdog_hardlockup_available)
9489ec272c5SDouglas Anderson watchdog_hardlockup_sysctl[0].mode = 0644;
9499ec272c5SDouglas Anderson register_sysctl_init("kernel", watchdog_hardlockup_sysctl);
950dd0693fdSXiaoming Ni }
9519ec272c5SDouglas Anderson
952dd0693fdSXiaoming Ni #else
953dd0693fdSXiaoming Ni #define watchdog_sysctl_init() do { } while (0)
95458687acbSDon Zickus #endif /* CONFIG_SYSCTL */
95558687acbSDon Zickus
956930d8f8dSLecopzer Chen static void __init lockup_detector_delay_init(struct work_struct *work);
957930d8f8dSLecopzer Chen static bool allow_lockup_detector_init_retry __initdata;
958930d8f8dSLecopzer Chen
959930d8f8dSLecopzer Chen static struct work_struct detector_work __initdata =
960930d8f8dSLecopzer Chen __WORK_INITIALIZER(detector_work, lockup_detector_delay_init);
961930d8f8dSLecopzer Chen
lockup_detector_delay_init(struct work_struct * work)962930d8f8dSLecopzer Chen static void __init lockup_detector_delay_init(struct work_struct *work)
963930d8f8dSLecopzer Chen {
964930d8f8dSLecopzer Chen int ret;
965930d8f8dSLecopzer Chen
966930d8f8dSLecopzer Chen ret = watchdog_hardlockup_probe();
967930d8f8dSLecopzer Chen if (ret) {
968930d8f8dSLecopzer Chen pr_info("Delayed init of the lockup detector failed: %d\n", ret);
969930d8f8dSLecopzer Chen pr_info("Hard watchdog permanently disabled\n");
970930d8f8dSLecopzer Chen return;
971930d8f8dSLecopzer Chen }
972930d8f8dSLecopzer Chen
973930d8f8dSLecopzer Chen allow_lockup_detector_init_retry = false;
974930d8f8dSLecopzer Chen
975930d8f8dSLecopzer Chen watchdog_hardlockup_available = true;
976930d8f8dSLecopzer Chen lockup_detector_setup();
977930d8f8dSLecopzer Chen }
978930d8f8dSLecopzer Chen
979930d8f8dSLecopzer Chen /*
980930d8f8dSLecopzer Chen * lockup_detector_retry_init - retry init lockup detector if possible.
981930d8f8dSLecopzer Chen *
982930d8f8dSLecopzer Chen * Retry hardlockup detector init. It is useful when it requires some
983930d8f8dSLecopzer Chen * functionality that has to be initialized later on a particular
984930d8f8dSLecopzer Chen * platform.
985930d8f8dSLecopzer Chen */
lockup_detector_retry_init(void)986930d8f8dSLecopzer Chen void __init lockup_detector_retry_init(void)
987930d8f8dSLecopzer Chen {
988930d8f8dSLecopzer Chen /* Must be called before late init calls */
989930d8f8dSLecopzer Chen if (!allow_lockup_detector_init_retry)
990930d8f8dSLecopzer Chen return;
991930d8f8dSLecopzer Chen
992930d8f8dSLecopzer Chen schedule_work(&detector_work);
993930d8f8dSLecopzer Chen }
994930d8f8dSLecopzer Chen
995930d8f8dSLecopzer Chen /*
996930d8f8dSLecopzer Chen * Ensure that optional delayed hardlockup init is proceed before
997930d8f8dSLecopzer Chen * the init code and memory is freed.
998930d8f8dSLecopzer Chen */
lockup_detector_check(void)999930d8f8dSLecopzer Chen static int __init lockup_detector_check(void)
1000930d8f8dSLecopzer Chen {
1001930d8f8dSLecopzer Chen /* Prevent any later retry. */
1002930d8f8dSLecopzer Chen allow_lockup_detector_init_retry = false;
1003930d8f8dSLecopzer Chen
1004930d8f8dSLecopzer Chen /* Make sure no work is pending. */
1005930d8f8dSLecopzer Chen flush_work(&detector_work);
1006930d8f8dSLecopzer Chen
10079ec272c5SDouglas Anderson watchdog_sysctl_init();
10089ec272c5SDouglas Anderson
1009930d8f8dSLecopzer Chen return 0;
1010930d8f8dSLecopzer Chen
1011930d8f8dSLecopzer Chen }
1012930d8f8dSLecopzer Chen late_initcall_sync(lockup_detector_check);
1013930d8f8dSLecopzer Chen
lockup_detector_init(void)1014004417a6SPeter Zijlstra void __init lockup_detector_init(void)
101558687acbSDon Zickus {
101613316b31SFrederic Weisbecker if (tick_nohz_full_enabled())
1017fe4ba3c3SChris Metcalf pr_info("Disabling watchdog on nohz_full cores by default\n");
101813316b31SFrederic Weisbecker
1019de201559SFrederic Weisbecker cpumask_copy(&watchdog_cpumask,
102004d4e665SFrederic Weisbecker housekeeping_cpumask(HK_TYPE_TIMER));
1021fe4ba3c3SChris Metcalf
1022df95d308SDouglas Anderson if (!watchdog_hardlockup_probe())
1023df95d308SDouglas Anderson watchdog_hardlockup_available = true;
1024930d8f8dSLecopzer Chen else
1025930d8f8dSLecopzer Chen allow_lockup_detector_init_retry = true;
1026930d8f8dSLecopzer Chen
10275587185dSThomas Gleixner lockup_detector_setup();
102858687acbSDon Zickus }
1029