1 /* 2 * Detect Hung Task 3 * 4 * kernel/hung_task.c - kernel thread for detecting tasks stuck in D state 5 * 6 */ 7 8 #include <linux/mm.h> 9 #include <linux/cpu.h> 10 #include <linux/nmi.h> 11 #include <linux/init.h> 12 #include <linux/delay.h> 13 #include <linux/freezer.h> 14 #include <linux/kthread.h> 15 #include <linux/lockdep.h> 16 #include <linux/export.h> 17 #include <linux/sysctl.h> 18 #include <linux/utsname.h> 19 #include <linux/sched/signal.h> 20 #include <linux/sched/debug.h> 21 22 #include <trace/events/sched.h> 23 24 /* 25 * The number of tasks checked: 26 */ 27 int __read_mostly sysctl_hung_task_check_count = PID_MAX_LIMIT; 28 29 /* 30 * Limit number of tasks checked in a batch. 31 * 32 * This value controls the preemptibility of khungtaskd since preemption 33 * is disabled during the critical section. It also controls the size of 34 * the RCU grace period. So it needs to be upper-bound. 35 */ 36 #define HUNG_TASK_BATCHING 1024 37 38 /* 39 * Zero means infinite timeout - no checking done: 40 */ 41 unsigned long __read_mostly sysctl_hung_task_timeout_secs = CONFIG_DEFAULT_HUNG_TASK_TIMEOUT; 42 43 /* 44 * Zero (default value) means use sysctl_hung_task_timeout_secs: 45 */ 46 unsigned long __read_mostly sysctl_hung_task_check_interval_secs; 47 48 int __read_mostly sysctl_hung_task_warnings = 10; 49 50 static int __read_mostly did_panic; 51 static bool hung_task_show_lock; 52 static bool hung_task_call_panic; 53 54 static struct task_struct *watchdog_task; 55 56 /* 57 * Should we panic (and reboot, if panic_timeout= is set) when a 58 * hung task is detected: 59 */ 60 unsigned int __read_mostly sysctl_hung_task_panic = 61 CONFIG_BOOTPARAM_HUNG_TASK_PANIC_VALUE; 62 63 static int __init hung_task_panic_setup(char *str) 64 { 65 int rc = kstrtouint(str, 0, &sysctl_hung_task_panic); 66 67 if (rc) 68 return rc; 69 return 1; 70 } 71 __setup("hung_task_panic=", hung_task_panic_setup); 72 73 static int 74 hung_task_panic(struct notifier_block *this, unsigned long event, void *ptr) 75 { 76 did_panic = 1; 77 78 return NOTIFY_DONE; 79 } 80 81 static struct notifier_block panic_block = { 82 .notifier_call = hung_task_panic, 83 }; 84 85 static void check_hung_task(struct task_struct *t, unsigned long timeout) 86 { 87 unsigned long switch_count = t->nvcsw + t->nivcsw; 88 89 /* 90 * Ensure the task is not frozen. 91 * Also, skip vfork and any other user process that freezer should skip. 92 */ 93 if (unlikely(t->flags & (PF_FROZEN | PF_FREEZER_SKIP))) 94 return; 95 96 /* 97 * When a freshly created task is scheduled once, changes its state to 98 * TASK_UNINTERRUPTIBLE without having ever been switched out once, it 99 * musn't be checked. 100 */ 101 if (unlikely(!switch_count)) 102 return; 103 104 if (switch_count != t->last_switch_count) { 105 t->last_switch_count = switch_count; 106 t->last_switch_time = jiffies; 107 return; 108 } 109 if (time_is_after_jiffies(t->last_switch_time + timeout * HZ)) 110 return; 111 112 trace_sched_process_hang(t); 113 114 if (!sysctl_hung_task_warnings && !sysctl_hung_task_panic) 115 return; 116 117 /* 118 * Ok, the task did not get scheduled for more than 2 minutes, 119 * complain: 120 */ 121 if (sysctl_hung_task_warnings) { 122 if (sysctl_hung_task_warnings > 0) 123 sysctl_hung_task_warnings--; 124 pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n", 125 t->comm, t->pid, timeout); 126 pr_err(" %s %s %.*s\n", 127 print_tainted(), init_utsname()->release, 128 (int)strcspn(init_utsname()->version, " "), 129 init_utsname()->version); 130 pr_err("\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\"" 131 " disables this message.\n"); 132 sched_show_task(t); 133 hung_task_show_lock = true; 134 } 135 136 touch_nmi_watchdog(); 137 138 if (sysctl_hung_task_panic) { 139 hung_task_show_lock = true; 140 hung_task_call_panic = true; 141 } 142 } 143 144 /* 145 * To avoid extending the RCU grace period for an unbounded amount of time, 146 * periodically exit the critical section and enter a new one. 147 * 148 * For preemptible RCU it is sufficient to call rcu_read_unlock in order 149 * to exit the grace period. For classic RCU, a reschedule is required. 150 */ 151 static bool rcu_lock_break(struct task_struct *g, struct task_struct *t) 152 { 153 bool can_cont; 154 155 get_task_struct(g); 156 get_task_struct(t); 157 rcu_read_unlock(); 158 cond_resched(); 159 rcu_read_lock(); 160 can_cont = pid_alive(g) && pid_alive(t); 161 put_task_struct(t); 162 put_task_struct(g); 163 164 return can_cont; 165 } 166 167 /* 168 * Check whether a TASK_UNINTERRUPTIBLE does not get woken up for 169 * a really long time (120 seconds). If that happens, print out 170 * a warning. 171 */ 172 static void check_hung_uninterruptible_tasks(unsigned long timeout) 173 { 174 int max_count = sysctl_hung_task_check_count; 175 int batch_count = HUNG_TASK_BATCHING; 176 struct task_struct *g, *t; 177 178 /* 179 * If the system crashed already then all bets are off, 180 * do not report extra hung tasks: 181 */ 182 if (test_taint(TAINT_DIE) || did_panic) 183 return; 184 185 hung_task_show_lock = false; 186 rcu_read_lock(); 187 for_each_process_thread(g, t) { 188 if (!max_count--) 189 goto unlock; 190 if (!--batch_count) { 191 batch_count = HUNG_TASK_BATCHING; 192 if (!rcu_lock_break(g, t)) 193 goto unlock; 194 } 195 /* use "==" to skip the TASK_KILLABLE tasks waiting on NFS */ 196 if (t->state == TASK_UNINTERRUPTIBLE) 197 check_hung_task(t, timeout); 198 } 199 unlock: 200 rcu_read_unlock(); 201 if (hung_task_show_lock) 202 debug_show_all_locks(); 203 if (hung_task_call_panic) { 204 trigger_all_cpu_backtrace(); 205 panic("hung_task: blocked tasks"); 206 } 207 } 208 209 static long hung_timeout_jiffies(unsigned long last_checked, 210 unsigned long timeout) 211 { 212 /* timeout of 0 will disable the watchdog */ 213 return timeout ? last_checked - jiffies + timeout * HZ : 214 MAX_SCHEDULE_TIMEOUT; 215 } 216 217 /* 218 * Process updating of timeout sysctl 219 */ 220 int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, 221 void __user *buffer, 222 size_t *lenp, loff_t *ppos) 223 { 224 int ret; 225 226 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 227 228 if (ret || !write) 229 goto out; 230 231 wake_up_process(watchdog_task); 232 233 out: 234 return ret; 235 } 236 237 static atomic_t reset_hung_task = ATOMIC_INIT(0); 238 239 void reset_hung_task_detector(void) 240 { 241 atomic_set(&reset_hung_task, 1); 242 } 243 EXPORT_SYMBOL_GPL(reset_hung_task_detector); 244 245 /* 246 * kthread which checks for tasks stuck in D state 247 */ 248 static int watchdog(void *dummy) 249 { 250 unsigned long hung_last_checked = jiffies; 251 252 set_user_nice(current, 0); 253 254 for ( ; ; ) { 255 unsigned long timeout = sysctl_hung_task_timeout_secs; 256 unsigned long interval = sysctl_hung_task_check_interval_secs; 257 long t; 258 259 if (interval == 0) 260 interval = timeout; 261 interval = min_t(unsigned long, interval, timeout); 262 t = hung_timeout_jiffies(hung_last_checked, interval); 263 if (t <= 0) { 264 if (!atomic_xchg(&reset_hung_task, 0)) 265 check_hung_uninterruptible_tasks(timeout); 266 hung_last_checked = jiffies; 267 continue; 268 } 269 schedule_timeout_interruptible(t); 270 } 271 272 return 0; 273 } 274 275 static int __init hung_task_init(void) 276 { 277 atomic_notifier_chain_register(&panic_notifier_list, &panic_block); 278 watchdog_task = kthread_run(watchdog, NULL, "khungtaskd"); 279 280 return 0; 281 } 282 subsys_initcall(hung_task_init); 283