xref: /openbmc/linux/kernel/watchdog.c (revision e2f1cf25)
1 /*
2  * Detect hard and soft lockups on a system
3  *
4  * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
5  *
6  * Note: Most of this code is borrowed heavily from the original softlockup
7  * detector, so thanks to Ingo for the initial implementation.
8  * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
9  * to those contributors as well.
10  */
11 
12 #define pr_fmt(fmt) "NMI watchdog: " fmt
13 
14 #include <linux/mm.h>
15 #include <linux/cpu.h>
16 #include <linux/nmi.h>
17 #include <linux/init.h>
18 #include <linux/module.h>
19 #include <linux/sysctl.h>
20 #include <linux/smpboot.h>
21 #include <linux/sched/rt.h>
22 #include <linux/tick.h>
23 
24 #include <asm/irq_regs.h>
25 #include <linux/kvm_para.h>
26 #include <linux/perf_event.h>
27 
28 /*
29  * The run state of the lockup detectors is controlled by the content of the
30  * 'watchdog_enabled' variable. Each lockup detector has its dedicated bit -
31  * bit 0 for the hard lockup detector and bit 1 for the soft lockup detector.
32  *
33  * 'watchdog_user_enabled', 'nmi_watchdog_enabled' and 'soft_watchdog_enabled'
34  * are variables that are only used as an 'interface' between the parameters
35  * in /proc/sys/kernel and the internal state bits in 'watchdog_enabled'. The
36  * 'watchdog_thresh' variable is handled differently because its value is not
37  * boolean, and the lockup detectors are 'suspended' while 'watchdog_thresh'
38  * is equal zero.
39  */
40 #define NMI_WATCHDOG_ENABLED_BIT   0
41 #define SOFT_WATCHDOG_ENABLED_BIT  1
42 #define NMI_WATCHDOG_ENABLED      (1 << NMI_WATCHDOG_ENABLED_BIT)
43 #define SOFT_WATCHDOG_ENABLED     (1 << SOFT_WATCHDOG_ENABLED_BIT)
44 
45 static DEFINE_MUTEX(watchdog_proc_mutex);
46 
47 #ifdef CONFIG_HARDLOCKUP_DETECTOR
48 static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED|NMI_WATCHDOG_ENABLED;
49 #else
50 static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED;
51 #endif
52 int __read_mostly nmi_watchdog_enabled;
53 int __read_mostly soft_watchdog_enabled;
54 int __read_mostly watchdog_user_enabled;
55 int __read_mostly watchdog_thresh = 10;
56 
57 #ifdef CONFIG_SMP
58 int __read_mostly sysctl_softlockup_all_cpu_backtrace;
59 #else
60 #define sysctl_softlockup_all_cpu_backtrace 0
61 #endif
62 static struct cpumask watchdog_cpumask __read_mostly;
63 unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
64 
65 /* Helper for online, unparked cpus. */
66 #define for_each_watchdog_cpu(cpu) \
67 	for_each_cpu_and((cpu), cpu_online_mask, &watchdog_cpumask)
68 
69 static int __read_mostly watchdog_running;
70 static u64 __read_mostly sample_period;
71 
72 static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
73 static DEFINE_PER_CPU(struct task_struct *, softlockup_watchdog);
74 static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
75 static DEFINE_PER_CPU(bool, softlockup_touch_sync);
76 static DEFINE_PER_CPU(bool, soft_watchdog_warn);
77 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
78 static DEFINE_PER_CPU(unsigned long, soft_lockup_hrtimer_cnt);
79 static DEFINE_PER_CPU(struct task_struct *, softlockup_task_ptr_saved);
80 #ifdef CONFIG_HARDLOCKUP_DETECTOR
81 static DEFINE_PER_CPU(bool, hard_watchdog_warn);
82 static DEFINE_PER_CPU(bool, watchdog_nmi_touch);
83 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
84 static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
85 #endif
86 static unsigned long soft_lockup_nmi_warn;
87 
88 /* boot commands */
89 /*
90  * Should we panic when a soft-lockup or hard-lockup occurs:
91  */
92 #ifdef CONFIG_HARDLOCKUP_DETECTOR
93 static int hardlockup_panic =
94 			CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE;
95 /*
96  * We may not want to enable hard lockup detection by default in all cases,
97  * for example when running the kernel as a guest on a hypervisor. In these
98  * cases this function can be called to disable hard lockup detection. This
99  * function should only be executed once by the boot processor before the
100  * kernel command line parameters are parsed, because otherwise it is not
101  * possible to override this in hardlockup_panic_setup().
102  */
103 void hardlockup_detector_disable(void)
104 {
105 	watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
106 }
107 
108 static int __init hardlockup_panic_setup(char *str)
109 {
110 	if (!strncmp(str, "panic", 5))
111 		hardlockup_panic = 1;
112 	else if (!strncmp(str, "nopanic", 7))
113 		hardlockup_panic = 0;
114 	else if (!strncmp(str, "0", 1))
115 		watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
116 	else if (!strncmp(str, "1", 1))
117 		watchdog_enabled |= NMI_WATCHDOG_ENABLED;
118 	return 1;
119 }
120 __setup("nmi_watchdog=", hardlockup_panic_setup);
121 #endif
122 
123 unsigned int __read_mostly softlockup_panic =
124 			CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE;
125 
126 static int __init softlockup_panic_setup(char *str)
127 {
128 	softlockup_panic = simple_strtoul(str, NULL, 0);
129 
130 	return 1;
131 }
132 __setup("softlockup_panic=", softlockup_panic_setup);
133 
134 static int __init nowatchdog_setup(char *str)
135 {
136 	watchdog_enabled = 0;
137 	return 1;
138 }
139 __setup("nowatchdog", nowatchdog_setup);
140 
141 static int __init nosoftlockup_setup(char *str)
142 {
143 	watchdog_enabled &= ~SOFT_WATCHDOG_ENABLED;
144 	return 1;
145 }
146 __setup("nosoftlockup", nosoftlockup_setup);
147 
148 #ifdef CONFIG_SMP
149 static int __init softlockup_all_cpu_backtrace_setup(char *str)
150 {
151 	sysctl_softlockup_all_cpu_backtrace =
152 		!!simple_strtol(str, NULL, 0);
153 	return 1;
154 }
155 __setup("softlockup_all_cpu_backtrace=", softlockup_all_cpu_backtrace_setup);
156 #endif
157 
158 /*
159  * Hard-lockup warnings should be triggered after just a few seconds. Soft-
160  * lockups can have false positives under extreme conditions. So we generally
161  * want a higher threshold for soft lockups than for hard lockups. So we couple
162  * the thresholds with a factor: we make the soft threshold twice the amount of
163  * time the hard threshold is.
164  */
165 static int get_softlockup_thresh(void)
166 {
167 	return watchdog_thresh * 2;
168 }
169 
170 /*
171  * Returns seconds, approximately.  We don't need nanosecond
172  * resolution, and we don't need to waste time with a big divide when
173  * 2^30ns == 1.074s.
174  */
175 static unsigned long get_timestamp(void)
176 {
177 	return running_clock() >> 30LL;  /* 2^30 ~= 10^9 */
178 }
179 
180 static void set_sample_period(void)
181 {
182 	/*
183 	 * convert watchdog_thresh from seconds to ns
184 	 * the divide by 5 is to give hrtimer several chances (two
185 	 * or three with the current relation between the soft
186 	 * and hard thresholds) to increment before the
187 	 * hardlockup detector generates a warning
188 	 */
189 	sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5);
190 }
191 
192 /* Commands for resetting the watchdog */
193 static void __touch_watchdog(void)
194 {
195 	__this_cpu_write(watchdog_touch_ts, get_timestamp());
196 }
197 
198 void touch_softlockup_watchdog(void)
199 {
200 	/*
201 	 * Preemption can be enabled.  It doesn't matter which CPU's timestamp
202 	 * gets zeroed here, so use the raw_ operation.
203 	 */
204 	raw_cpu_write(watchdog_touch_ts, 0);
205 }
206 EXPORT_SYMBOL(touch_softlockup_watchdog);
207 
208 void touch_all_softlockup_watchdogs(void)
209 {
210 	int cpu;
211 
212 	/*
213 	 * this is done lockless
214 	 * do we care if a 0 races with a timestamp?
215 	 * all it means is the softlock check starts one cycle later
216 	 */
217 	for_each_watchdog_cpu(cpu)
218 		per_cpu(watchdog_touch_ts, cpu) = 0;
219 }
220 
221 #ifdef CONFIG_HARDLOCKUP_DETECTOR
222 void touch_nmi_watchdog(void)
223 {
224 	/*
225 	 * Using __raw here because some code paths have
226 	 * preemption enabled.  If preemption is enabled
227 	 * then interrupts should be enabled too, in which
228 	 * case we shouldn't have to worry about the watchdog
229 	 * going off.
230 	 */
231 	raw_cpu_write(watchdog_nmi_touch, true);
232 	touch_softlockup_watchdog();
233 }
234 EXPORT_SYMBOL(touch_nmi_watchdog);
235 
236 #endif
237 
238 void touch_softlockup_watchdog_sync(void)
239 {
240 	__this_cpu_write(softlockup_touch_sync, true);
241 	__this_cpu_write(watchdog_touch_ts, 0);
242 }
243 
244 #ifdef CONFIG_HARDLOCKUP_DETECTOR
245 /* watchdog detector functions */
246 static int is_hardlockup(void)
247 {
248 	unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
249 
250 	if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
251 		return 1;
252 
253 	__this_cpu_write(hrtimer_interrupts_saved, hrint);
254 	return 0;
255 }
256 #endif
257 
258 static int is_softlockup(unsigned long touch_ts)
259 {
260 	unsigned long now = get_timestamp();
261 
262 	if (watchdog_enabled & SOFT_WATCHDOG_ENABLED) {
263 		/* Warn about unreasonable delays. */
264 		if (time_after(now, touch_ts + get_softlockup_thresh()))
265 			return now - touch_ts;
266 	}
267 	return 0;
268 }
269 
270 #ifdef CONFIG_HARDLOCKUP_DETECTOR
271 
272 static struct perf_event_attr wd_hw_attr = {
273 	.type		= PERF_TYPE_HARDWARE,
274 	.config		= PERF_COUNT_HW_CPU_CYCLES,
275 	.size		= sizeof(struct perf_event_attr),
276 	.pinned		= 1,
277 	.disabled	= 1,
278 };
279 
280 /* Callback function for perf event subsystem */
281 static void watchdog_overflow_callback(struct perf_event *event,
282 		 struct perf_sample_data *data,
283 		 struct pt_regs *regs)
284 {
285 	/* Ensure the watchdog never gets throttled */
286 	event->hw.interrupts = 0;
287 
288 	if (__this_cpu_read(watchdog_nmi_touch) == true) {
289 		__this_cpu_write(watchdog_nmi_touch, false);
290 		return;
291 	}
292 
293 	/* check for a hardlockup
294 	 * This is done by making sure our timer interrupt
295 	 * is incrementing.  The timer interrupt should have
296 	 * fired multiple times before we overflow'd.  If it hasn't
297 	 * then this is a good indication the cpu is stuck
298 	 */
299 	if (is_hardlockup()) {
300 		int this_cpu = smp_processor_id();
301 
302 		/* only print hardlockups once */
303 		if (__this_cpu_read(hard_watchdog_warn) == true)
304 			return;
305 
306 		if (hardlockup_panic)
307 			panic("Watchdog detected hard LOCKUP on cpu %d",
308 			      this_cpu);
309 		else
310 			WARN(1, "Watchdog detected hard LOCKUP on cpu %d",
311 			     this_cpu);
312 
313 		__this_cpu_write(hard_watchdog_warn, true);
314 		return;
315 	}
316 
317 	__this_cpu_write(hard_watchdog_warn, false);
318 	return;
319 }
320 #endif /* CONFIG_HARDLOCKUP_DETECTOR */
321 
322 static void watchdog_interrupt_count(void)
323 {
324 	__this_cpu_inc(hrtimer_interrupts);
325 }
326 
327 static int watchdog_nmi_enable(unsigned int cpu);
328 static void watchdog_nmi_disable(unsigned int cpu);
329 
330 /* watchdog kicker functions */
331 static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
332 {
333 	unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
334 	struct pt_regs *regs = get_irq_regs();
335 	int duration;
336 	int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
337 
338 	/* kick the hardlockup detector */
339 	watchdog_interrupt_count();
340 
341 	/* kick the softlockup detector */
342 	wake_up_process(__this_cpu_read(softlockup_watchdog));
343 
344 	/* .. and repeat */
345 	hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
346 
347 	if (touch_ts == 0) {
348 		if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
349 			/*
350 			 * If the time stamp was touched atomically
351 			 * make sure the scheduler tick is up to date.
352 			 */
353 			__this_cpu_write(softlockup_touch_sync, false);
354 			sched_clock_tick();
355 		}
356 
357 		/* Clear the guest paused flag on watchdog reset */
358 		kvm_check_and_clear_guest_paused();
359 		__touch_watchdog();
360 		return HRTIMER_RESTART;
361 	}
362 
363 	/* check for a softlockup
364 	 * This is done by making sure a high priority task is
365 	 * being scheduled.  The task touches the watchdog to
366 	 * indicate it is getting cpu time.  If it hasn't then
367 	 * this is a good indication some task is hogging the cpu
368 	 */
369 	duration = is_softlockup(touch_ts);
370 	if (unlikely(duration)) {
371 		/*
372 		 * If a virtual machine is stopped by the host it can look to
373 		 * the watchdog like a soft lockup, check to see if the host
374 		 * stopped the vm before we issue the warning
375 		 */
376 		if (kvm_check_and_clear_guest_paused())
377 			return HRTIMER_RESTART;
378 
379 		/* only warn once */
380 		if (__this_cpu_read(soft_watchdog_warn) == true) {
381 			/*
382 			 * When multiple processes are causing softlockups the
383 			 * softlockup detector only warns on the first one
384 			 * because the code relies on a full quiet cycle to
385 			 * re-arm.  The second process prevents the quiet cycle
386 			 * and never gets reported.  Use task pointers to detect
387 			 * this.
388 			 */
389 			if (__this_cpu_read(softlockup_task_ptr_saved) !=
390 			    current) {
391 				__this_cpu_write(soft_watchdog_warn, false);
392 				__touch_watchdog();
393 			}
394 			return HRTIMER_RESTART;
395 		}
396 
397 		if (softlockup_all_cpu_backtrace) {
398 			/* Prevent multiple soft-lockup reports if one cpu is already
399 			 * engaged in dumping cpu back traces
400 			 */
401 			if (test_and_set_bit(0, &soft_lockup_nmi_warn)) {
402 				/* Someone else will report us. Let's give up */
403 				__this_cpu_write(soft_watchdog_warn, true);
404 				return HRTIMER_RESTART;
405 			}
406 		}
407 
408 		pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
409 			smp_processor_id(), duration,
410 			current->comm, task_pid_nr(current));
411 		__this_cpu_write(softlockup_task_ptr_saved, current);
412 		print_modules();
413 		print_irqtrace_events(current);
414 		if (regs)
415 			show_regs(regs);
416 		else
417 			dump_stack();
418 
419 		if (softlockup_all_cpu_backtrace) {
420 			/* Avoid generating two back traces for current
421 			 * given that one is already made above
422 			 */
423 			trigger_allbutself_cpu_backtrace();
424 
425 			clear_bit(0, &soft_lockup_nmi_warn);
426 			/* Barrier to sync with other cpus */
427 			smp_mb__after_atomic();
428 		}
429 
430 		add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
431 		if (softlockup_panic)
432 			panic("softlockup: hung tasks");
433 		__this_cpu_write(soft_watchdog_warn, true);
434 	} else
435 		__this_cpu_write(soft_watchdog_warn, false);
436 
437 	return HRTIMER_RESTART;
438 }
439 
440 static void watchdog_set_prio(unsigned int policy, unsigned int prio)
441 {
442 	struct sched_param param = { .sched_priority = prio };
443 
444 	sched_setscheduler(current, policy, &param);
445 }
446 
447 static void watchdog_enable(unsigned int cpu)
448 {
449 	struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer);
450 
451 	/* kick off the timer for the hardlockup detector */
452 	hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
453 	hrtimer->function = watchdog_timer_fn;
454 
455 	/* Enable the perf event */
456 	watchdog_nmi_enable(cpu);
457 
458 	/* done here because hrtimer_start can only pin to smp_processor_id() */
459 	hrtimer_start(hrtimer, ns_to_ktime(sample_period),
460 		      HRTIMER_MODE_REL_PINNED);
461 
462 	/* initialize timestamp */
463 	watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1);
464 	__touch_watchdog();
465 }
466 
467 static void watchdog_disable(unsigned int cpu)
468 {
469 	struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer);
470 
471 	watchdog_set_prio(SCHED_NORMAL, 0);
472 	hrtimer_cancel(hrtimer);
473 	/* disable the perf event */
474 	watchdog_nmi_disable(cpu);
475 }
476 
477 static void watchdog_cleanup(unsigned int cpu, bool online)
478 {
479 	watchdog_disable(cpu);
480 }
481 
482 static int watchdog_should_run(unsigned int cpu)
483 {
484 	return __this_cpu_read(hrtimer_interrupts) !=
485 		__this_cpu_read(soft_lockup_hrtimer_cnt);
486 }
487 
488 /*
489  * The watchdog thread function - touches the timestamp.
490  *
491  * It only runs once every sample_period seconds (4 seconds by
492  * default) to reset the softlockup timestamp. If this gets delayed
493  * for more than 2*watchdog_thresh seconds then the debug-printout
494  * triggers in watchdog_timer_fn().
495  */
496 static void watchdog(unsigned int cpu)
497 {
498 	__this_cpu_write(soft_lockup_hrtimer_cnt,
499 			 __this_cpu_read(hrtimer_interrupts));
500 	__touch_watchdog();
501 
502 	/*
503 	 * watchdog_nmi_enable() clears the NMI_WATCHDOG_ENABLED bit in the
504 	 * failure path. Check for failures that can occur asynchronously -
505 	 * for example, when CPUs are on-lined - and shut down the hardware
506 	 * perf event on each CPU accordingly.
507 	 *
508 	 * The only non-obvious place this bit can be cleared is through
509 	 * watchdog_nmi_enable(), so a pr_info() is placed there.  Placing a
510 	 * pr_info here would be too noisy as it would result in a message
511 	 * every few seconds if the hardlockup was disabled but the softlockup
512 	 * enabled.
513 	 */
514 	if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
515 		watchdog_nmi_disable(cpu);
516 }
517 
518 #ifdef CONFIG_HARDLOCKUP_DETECTOR
519 /*
520  * People like the simple clean cpu node info on boot.
521  * Reduce the watchdog noise by only printing messages
522  * that are different from what cpu0 displayed.
523  */
524 static unsigned long cpu0_err;
525 
526 static int watchdog_nmi_enable(unsigned int cpu)
527 {
528 	struct perf_event_attr *wd_attr;
529 	struct perf_event *event = per_cpu(watchdog_ev, cpu);
530 
531 	/* nothing to do if the hard lockup detector is disabled */
532 	if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
533 		goto out;
534 
535 	/* is it already setup and enabled? */
536 	if (event && event->state > PERF_EVENT_STATE_OFF)
537 		goto out;
538 
539 	/* it is setup but not enabled */
540 	if (event != NULL)
541 		goto out_enable;
542 
543 	wd_attr = &wd_hw_attr;
544 	wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh);
545 
546 	/* Try to register using hardware perf events */
547 	event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL);
548 
549 	/* save cpu0 error for future comparision */
550 	if (cpu == 0 && IS_ERR(event))
551 		cpu0_err = PTR_ERR(event);
552 
553 	if (!IS_ERR(event)) {
554 		/* only print for cpu0 or different than cpu0 */
555 		if (cpu == 0 || cpu0_err)
556 			pr_info("enabled on all CPUs, permanently consumes one hw-PMU counter.\n");
557 		goto out_save;
558 	}
559 
560 	/*
561 	 * Disable the hard lockup detector if _any_ CPU fails to set up
562 	 * set up the hardware perf event. The watchdog() function checks
563 	 * the NMI_WATCHDOG_ENABLED bit periodically.
564 	 *
565 	 * The barriers are for syncing up watchdog_enabled across all the
566 	 * cpus, as clear_bit() does not use barriers.
567 	 */
568 	smp_mb__before_atomic();
569 	clear_bit(NMI_WATCHDOG_ENABLED_BIT, &watchdog_enabled);
570 	smp_mb__after_atomic();
571 
572 	/* skip displaying the same error again */
573 	if (cpu > 0 && (PTR_ERR(event) == cpu0_err))
574 		return PTR_ERR(event);
575 
576 	/* vary the KERN level based on the returned errno */
577 	if (PTR_ERR(event) == -EOPNOTSUPP)
578 		pr_info("disabled (cpu%i): not supported (no LAPIC?)\n", cpu);
579 	else if (PTR_ERR(event) == -ENOENT)
580 		pr_warn("disabled (cpu%i): hardware events not enabled\n",
581 			 cpu);
582 	else
583 		pr_err("disabled (cpu%i): unable to create perf event: %ld\n",
584 			cpu, PTR_ERR(event));
585 
586 	pr_info("Shutting down hard lockup detector on all cpus\n");
587 
588 	return PTR_ERR(event);
589 
590 	/* success path */
591 out_save:
592 	per_cpu(watchdog_ev, cpu) = event;
593 out_enable:
594 	perf_event_enable(per_cpu(watchdog_ev, cpu));
595 out:
596 	return 0;
597 }
598 
599 static void watchdog_nmi_disable(unsigned int cpu)
600 {
601 	struct perf_event *event = per_cpu(watchdog_ev, cpu);
602 
603 	if (event) {
604 		perf_event_disable(event);
605 		per_cpu(watchdog_ev, cpu) = NULL;
606 
607 		/* should be in cleanup, but blocks oprofile */
608 		perf_event_release_kernel(event);
609 	}
610 	if (cpu == 0) {
611 		/* watchdog_nmi_enable() expects this to be zero initially. */
612 		cpu0_err = 0;
613 	}
614 }
615 
616 void watchdog_nmi_enable_all(void)
617 {
618 	int cpu;
619 
620 	mutex_lock(&watchdog_proc_mutex);
621 
622 	if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
623 		goto unlock;
624 
625 	get_online_cpus();
626 	for_each_watchdog_cpu(cpu)
627 		watchdog_nmi_enable(cpu);
628 	put_online_cpus();
629 
630 unlock:
631 	mutex_unlock(&watchdog_proc_mutex);
632 }
633 
634 void watchdog_nmi_disable_all(void)
635 {
636 	int cpu;
637 
638 	mutex_lock(&watchdog_proc_mutex);
639 
640 	if (!watchdog_running)
641 		goto unlock;
642 
643 	get_online_cpus();
644 	for_each_watchdog_cpu(cpu)
645 		watchdog_nmi_disable(cpu);
646 	put_online_cpus();
647 
648 unlock:
649 	mutex_unlock(&watchdog_proc_mutex);
650 }
651 #else
652 static int watchdog_nmi_enable(unsigned int cpu) { return 0; }
653 static void watchdog_nmi_disable(unsigned int cpu) { return; }
654 void watchdog_nmi_enable_all(void) {}
655 void watchdog_nmi_disable_all(void) {}
656 #endif /* CONFIG_HARDLOCKUP_DETECTOR */
657 
658 static struct smp_hotplug_thread watchdog_threads = {
659 	.store			= &softlockup_watchdog,
660 	.thread_should_run	= watchdog_should_run,
661 	.thread_fn		= watchdog,
662 	.thread_comm		= "watchdog/%u",
663 	.setup			= watchdog_enable,
664 	.cleanup		= watchdog_cleanup,
665 	.park			= watchdog_disable,
666 	.unpark			= watchdog_enable,
667 };
668 
669 static void restart_watchdog_hrtimer(void *info)
670 {
671 	struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer);
672 	int ret;
673 
674 	/*
675 	 * No need to cancel and restart hrtimer if it is currently executing
676 	 * because it will reprogram itself with the new period now.
677 	 * We should never see it unqueued here because we are running per-cpu
678 	 * with interrupts disabled.
679 	 */
680 	ret = hrtimer_try_to_cancel(hrtimer);
681 	if (ret == 1)
682 		hrtimer_start(hrtimer, ns_to_ktime(sample_period),
683 				HRTIMER_MODE_REL_PINNED);
684 }
685 
686 static void update_watchdog(int cpu)
687 {
688 	/*
689 	 * Make sure that perf event counter will adopt to a new
690 	 * sampling period. Updating the sampling period directly would
691 	 * be much nicer but we do not have an API for that now so
692 	 * let's use a big hammer.
693 	 * Hrtimer will adopt the new period on the next tick but this
694 	 * might be late already so we have to restart the timer as well.
695 	 */
696 	watchdog_nmi_disable(cpu);
697 	smp_call_function_single(cpu, restart_watchdog_hrtimer, NULL, 1);
698 	watchdog_nmi_enable(cpu);
699 }
700 
701 static void update_watchdog_all_cpus(void)
702 {
703 	int cpu;
704 
705 	get_online_cpus();
706 	for_each_watchdog_cpu(cpu)
707 		update_watchdog(cpu);
708 	put_online_cpus();
709 }
710 
711 static int watchdog_enable_all_cpus(void)
712 {
713 	int err = 0;
714 
715 	if (!watchdog_running) {
716 		err = smpboot_register_percpu_thread(&watchdog_threads);
717 		if (err)
718 			pr_err("Failed to create watchdog threads, disabled\n");
719 		else {
720 			if (smpboot_update_cpumask_percpu_thread(
721 				    &watchdog_threads, &watchdog_cpumask))
722 				pr_err("Failed to set cpumask for watchdog threads\n");
723 			watchdog_running = 1;
724 		}
725 	} else {
726 		/*
727 		 * Enable/disable the lockup detectors or
728 		 * change the sample period 'on the fly'.
729 		 */
730 		update_watchdog_all_cpus();
731 	}
732 
733 	return err;
734 }
735 
736 /* prepare/enable/disable routines */
737 /* sysctl functions */
738 #ifdef CONFIG_SYSCTL
739 static void watchdog_disable_all_cpus(void)
740 {
741 	if (watchdog_running) {
742 		watchdog_running = 0;
743 		smpboot_unregister_percpu_thread(&watchdog_threads);
744 	}
745 }
746 
747 /*
748  * Update the run state of the lockup detectors.
749  */
750 static int proc_watchdog_update(void)
751 {
752 	int err = 0;
753 
754 	/*
755 	 * Watchdog threads won't be started if they are already active.
756 	 * The 'watchdog_running' variable in watchdog_*_all_cpus() takes
757 	 * care of this. If those threads are already active, the sample
758 	 * period will be updated and the lockup detectors will be enabled
759 	 * or disabled 'on the fly'.
760 	 */
761 	if (watchdog_enabled && watchdog_thresh)
762 		err = watchdog_enable_all_cpus();
763 	else
764 		watchdog_disable_all_cpus();
765 
766 	return err;
767 
768 }
769 
770 /*
771  * common function for watchdog, nmi_watchdog and soft_watchdog parameter
772  *
773  * caller             | table->data points to | 'which' contains the flag(s)
774  * -------------------|-----------------------|-----------------------------
775  * proc_watchdog      | watchdog_user_enabled | NMI_WATCHDOG_ENABLED or'ed
776  *                    |                       | with SOFT_WATCHDOG_ENABLED
777  * -------------------|-----------------------|-----------------------------
778  * proc_nmi_watchdog  | nmi_watchdog_enabled  | NMI_WATCHDOG_ENABLED
779  * -------------------|-----------------------|-----------------------------
780  * proc_soft_watchdog | soft_watchdog_enabled | SOFT_WATCHDOG_ENABLED
781  */
782 static int proc_watchdog_common(int which, struct ctl_table *table, int write,
783 				void __user *buffer, size_t *lenp, loff_t *ppos)
784 {
785 	int err, old, new;
786 	int *watchdog_param = (int *)table->data;
787 
788 	mutex_lock(&watchdog_proc_mutex);
789 
790 	/*
791 	 * If the parameter is being read return the state of the corresponding
792 	 * bit(s) in 'watchdog_enabled', else update 'watchdog_enabled' and the
793 	 * run state of the lockup detectors.
794 	 */
795 	if (!write) {
796 		*watchdog_param = (watchdog_enabled & which) != 0;
797 		err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
798 	} else {
799 		err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
800 		if (err)
801 			goto out;
802 
803 		/*
804 		 * There is a race window between fetching the current value
805 		 * from 'watchdog_enabled' and storing the new value. During
806 		 * this race window, watchdog_nmi_enable() can sneak in and
807 		 * clear the NMI_WATCHDOG_ENABLED bit in 'watchdog_enabled'.
808 		 * The 'cmpxchg' detects this race and the loop retries.
809 		 */
810 		do {
811 			old = watchdog_enabled;
812 			/*
813 			 * If the parameter value is not zero set the
814 			 * corresponding bit(s), else clear it(them).
815 			 */
816 			if (*watchdog_param)
817 				new = old | which;
818 			else
819 				new = old & ~which;
820 		} while (cmpxchg(&watchdog_enabled, old, new) != old);
821 
822 		/*
823 		 * Update the run state of the lockup detectors.
824 		 * Restore 'watchdog_enabled' on failure.
825 		 */
826 		err = proc_watchdog_update();
827 		if (err)
828 			watchdog_enabled = old;
829 	}
830 out:
831 	mutex_unlock(&watchdog_proc_mutex);
832 	return err;
833 }
834 
835 /*
836  * /proc/sys/kernel/watchdog
837  */
838 int proc_watchdog(struct ctl_table *table, int write,
839 		  void __user *buffer, size_t *lenp, loff_t *ppos)
840 {
841 	return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED,
842 				    table, write, buffer, lenp, ppos);
843 }
844 
845 /*
846  * /proc/sys/kernel/nmi_watchdog
847  */
848 int proc_nmi_watchdog(struct ctl_table *table, int write,
849 		      void __user *buffer, size_t *lenp, loff_t *ppos)
850 {
851 	return proc_watchdog_common(NMI_WATCHDOG_ENABLED,
852 				    table, write, buffer, lenp, ppos);
853 }
854 
855 /*
856  * /proc/sys/kernel/soft_watchdog
857  */
858 int proc_soft_watchdog(struct ctl_table *table, int write,
859 			void __user *buffer, size_t *lenp, loff_t *ppos)
860 {
861 	return proc_watchdog_common(SOFT_WATCHDOG_ENABLED,
862 				    table, write, buffer, lenp, ppos);
863 }
864 
865 /*
866  * /proc/sys/kernel/watchdog_thresh
867  */
868 int proc_watchdog_thresh(struct ctl_table *table, int write,
869 			 void __user *buffer, size_t *lenp, loff_t *ppos)
870 {
871 	int err, old;
872 
873 	mutex_lock(&watchdog_proc_mutex);
874 
875 	old = ACCESS_ONCE(watchdog_thresh);
876 	err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
877 
878 	if (err || !write)
879 		goto out;
880 
881 	/*
882 	 * Update the sample period.
883 	 * Restore 'watchdog_thresh' on failure.
884 	 */
885 	set_sample_period();
886 	err = proc_watchdog_update();
887 	if (err)
888 		watchdog_thresh = old;
889 out:
890 	mutex_unlock(&watchdog_proc_mutex);
891 	return err;
892 }
893 
894 /*
895  * The cpumask is the mask of possible cpus that the watchdog can run
896  * on, not the mask of cpus it is actually running on.  This allows the
897  * user to specify a mask that will include cpus that have not yet
898  * been brought online, if desired.
899  */
900 int proc_watchdog_cpumask(struct ctl_table *table, int write,
901 			  void __user *buffer, size_t *lenp, loff_t *ppos)
902 {
903 	int err;
904 
905 	mutex_lock(&watchdog_proc_mutex);
906 	err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
907 	if (!err && write) {
908 		/* Remove impossible cpus to keep sysctl output cleaner. */
909 		cpumask_and(&watchdog_cpumask, &watchdog_cpumask,
910 			    cpu_possible_mask);
911 
912 		if (watchdog_running) {
913 			/*
914 			 * Failure would be due to being unable to allocate
915 			 * a temporary cpumask, so we are likely not in a
916 			 * position to do much else to make things better.
917 			 */
918 			if (smpboot_update_cpumask_percpu_thread(
919 				    &watchdog_threads, &watchdog_cpumask) != 0)
920 				pr_err("cpumask update failed\n");
921 		}
922 	}
923 	mutex_unlock(&watchdog_proc_mutex);
924 	return err;
925 }
926 
927 #endif /* CONFIG_SYSCTL */
928 
929 void __init lockup_detector_init(void)
930 {
931 	set_sample_period();
932 
933 #ifdef CONFIG_NO_HZ_FULL
934 	if (tick_nohz_full_enabled()) {
935 		if (!cpumask_empty(tick_nohz_full_mask))
936 			pr_info("Disabling watchdog on nohz_full cores by default\n");
937 		cpumask_andnot(&watchdog_cpumask, cpu_possible_mask,
938 			       tick_nohz_full_mask);
939 	} else
940 		cpumask_copy(&watchdog_cpumask, cpu_possible_mask);
941 #else
942 	cpumask_copy(&watchdog_cpumask, cpu_possible_mask);
943 #endif
944 
945 	if (watchdog_enabled)
946 		watchdog_enable_all_cpus();
947 }
948