1 /* 2 * linux/mm/oom_kill.c 3 * 4 * Copyright (C) 1998,2000 Rik van Riel 5 * Thanks go out to Claus Fischer for some serious inspiration and 6 * for goading me into coding this file... 7 * Copyright (C) 2010 Google, Inc. 8 * Rewritten by David Rientjes 9 * 10 * The routines in this file are used to kill a process when 11 * we're seriously out of memory. This gets called from __alloc_pages() 12 * in mm/page_alloc.c when we really run out of memory. 13 * 14 * Since we won't call these routines often (on a well-configured 15 * machine) this file will double as a 'coding guide' and a signpost 16 * for newbie kernel hackers. It features several pointers to major 17 * kernel subsystems and hints as to where to find out what things do. 18 */ 19 20 #include <linux/oom.h> 21 #include <linux/mm.h> 22 #include <linux/err.h> 23 #include <linux/gfp.h> 24 #include <linux/sched.h> 25 #include <linux/sched/mm.h> 26 #include <linux/sched/coredump.h> 27 #include <linux/sched/task.h> 28 #include <linux/swap.h> 29 #include <linux/timex.h> 30 #include <linux/jiffies.h> 31 #include <linux/cpuset.h> 32 #include <linux/export.h> 33 #include <linux/notifier.h> 34 #include <linux/memcontrol.h> 35 #include <linux/mempolicy.h> 36 #include <linux/security.h> 37 #include <linux/ptrace.h> 38 #include <linux/freezer.h> 39 #include <linux/ftrace.h> 40 #include <linux/ratelimit.h> 41 #include <linux/kthread.h> 42 #include <linux/init.h> 43 #include <linux/mmu_notifier.h> 44 45 #include <asm/tlb.h> 46 #include "internal.h" 47 #include "slab.h" 48 49 #define CREATE_TRACE_POINTS 50 #include <trace/events/oom.h> 51 52 int sysctl_panic_on_oom; 53 int sysctl_oom_kill_allocating_task; 54 int sysctl_oom_dump_tasks = 1; 55 56 DEFINE_MUTEX(oom_lock); 57 58 #ifdef CONFIG_NUMA 59 /** 60 * has_intersects_mems_allowed() - check task eligiblity for kill 61 * @start: task struct of which task to consider 62 * @mask: nodemask passed to page allocator for mempolicy ooms 63 * 64 * Task eligibility is determined by whether or not a candidate task, @tsk, 65 * shares the same mempolicy nodes as current if it is bound by such a policy 66 * and whether or not it has the same set of allowed cpuset nodes. 67 */ 68 static bool has_intersects_mems_allowed(struct task_struct *start, 69 const nodemask_t *mask) 70 { 71 struct task_struct *tsk; 72 bool ret = false; 73 74 rcu_read_lock(); 75 for_each_thread(start, tsk) { 76 if (mask) { 77 /* 78 * If this is a mempolicy constrained oom, tsk's 79 * cpuset is irrelevant. Only return true if its 80 * mempolicy intersects current, otherwise it may be 81 * needlessly killed. 82 */ 83 ret = mempolicy_nodemask_intersects(tsk, mask); 84 } else { 85 /* 86 * This is not a mempolicy constrained oom, so only 87 * check the mems of tsk's cpuset. 88 */ 89 ret = cpuset_mems_allowed_intersects(current, tsk); 90 } 91 if (ret) 92 break; 93 } 94 rcu_read_unlock(); 95 96 return ret; 97 } 98 #else 99 static bool has_intersects_mems_allowed(struct task_struct *tsk, 100 const nodemask_t *mask) 101 { 102 return true; 103 } 104 #endif /* CONFIG_NUMA */ 105 106 /* 107 * The process p may have detached its own ->mm while exiting or through 108 * use_mm(), but one or more of its subthreads may still have a valid 109 * pointer. Return p, or any of its subthreads with a valid ->mm, with 110 * task_lock() held. 111 */ 112 struct task_struct *find_lock_task_mm(struct task_struct *p) 113 { 114 struct task_struct *t; 115 116 rcu_read_lock(); 117 118 for_each_thread(p, t) { 119 task_lock(t); 120 if (likely(t->mm)) 121 goto found; 122 task_unlock(t); 123 } 124 t = NULL; 125 found: 126 rcu_read_unlock(); 127 128 return t; 129 } 130 131 /* 132 * order == -1 means the oom kill is required by sysrq, otherwise only 133 * for display purposes. 134 */ 135 static inline bool is_sysrq_oom(struct oom_control *oc) 136 { 137 return oc->order == -1; 138 } 139 140 static inline bool is_memcg_oom(struct oom_control *oc) 141 { 142 return oc->memcg != NULL; 143 } 144 145 /* return true if the task is not adequate as candidate victim task. */ 146 static bool oom_unkillable_task(struct task_struct *p, 147 struct mem_cgroup *memcg, const nodemask_t *nodemask) 148 { 149 if (is_global_init(p)) 150 return true; 151 if (p->flags & PF_KTHREAD) 152 return true; 153 154 /* When mem_cgroup_out_of_memory() and p is not member of the group */ 155 if (memcg && !task_in_mem_cgroup(p, memcg)) 156 return true; 157 158 /* p may not have freeable memory in nodemask */ 159 if (!has_intersects_mems_allowed(p, nodemask)) 160 return true; 161 162 return false; 163 } 164 165 /* 166 * Print out unreclaimble slabs info when unreclaimable slabs amount is greater 167 * than all user memory (LRU pages) 168 */ 169 static bool is_dump_unreclaim_slabs(void) 170 { 171 unsigned long nr_lru; 172 173 nr_lru = global_node_page_state(NR_ACTIVE_ANON) + 174 global_node_page_state(NR_INACTIVE_ANON) + 175 global_node_page_state(NR_ACTIVE_FILE) + 176 global_node_page_state(NR_INACTIVE_FILE) + 177 global_node_page_state(NR_ISOLATED_ANON) + 178 global_node_page_state(NR_ISOLATED_FILE) + 179 global_node_page_state(NR_UNEVICTABLE); 180 181 return (global_node_page_state(NR_SLAB_UNRECLAIMABLE) > nr_lru); 182 } 183 184 /** 185 * oom_badness - heuristic function to determine which candidate task to kill 186 * @p: task struct of which task we should calculate 187 * @totalpages: total present RAM allowed for page allocation 188 * 189 * The heuristic for determining which task to kill is made to be as simple and 190 * predictable as possible. The goal is to return the highest value for the 191 * task consuming the most memory to avoid subsequent oom failures. 192 */ 193 unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg, 194 const nodemask_t *nodemask, unsigned long totalpages) 195 { 196 long points; 197 long adj; 198 199 if (oom_unkillable_task(p, memcg, nodemask)) 200 return 0; 201 202 p = find_lock_task_mm(p); 203 if (!p) 204 return 0; 205 206 /* 207 * Do not even consider tasks which are explicitly marked oom 208 * unkillable or have been already oom reaped or the are in 209 * the middle of vfork 210 */ 211 adj = (long)p->signal->oom_score_adj; 212 if (adj == OOM_SCORE_ADJ_MIN || 213 test_bit(MMF_OOM_SKIP, &p->mm->flags) || 214 in_vfork(p)) { 215 task_unlock(p); 216 return 0; 217 } 218 219 /* 220 * The baseline for the badness score is the proportion of RAM that each 221 * task's rss, pagetable and swap space use. 222 */ 223 points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) + 224 mm_pgtables_bytes(p->mm) / PAGE_SIZE; 225 task_unlock(p); 226 227 /* 228 * Root processes get 3% bonus, just like the __vm_enough_memory() 229 * implementation used by LSMs. 230 */ 231 if (has_capability_noaudit(p, CAP_SYS_ADMIN)) 232 points -= (points * 3) / 100; 233 234 /* Normalize to oom_score_adj units */ 235 adj *= totalpages / 1000; 236 points += adj; 237 238 /* 239 * Never return 0 for an eligible task regardless of the root bonus and 240 * oom_score_adj (oom_score_adj can't be OOM_SCORE_ADJ_MIN here). 241 */ 242 return points > 0 ? points : 1; 243 } 244 245 enum oom_constraint { 246 CONSTRAINT_NONE, 247 CONSTRAINT_CPUSET, 248 CONSTRAINT_MEMORY_POLICY, 249 CONSTRAINT_MEMCG, 250 }; 251 252 /* 253 * Determine the type of allocation constraint. 254 */ 255 static enum oom_constraint constrained_alloc(struct oom_control *oc) 256 { 257 struct zone *zone; 258 struct zoneref *z; 259 enum zone_type high_zoneidx = gfp_zone(oc->gfp_mask); 260 bool cpuset_limited = false; 261 int nid; 262 263 if (is_memcg_oom(oc)) { 264 oc->totalpages = mem_cgroup_get_limit(oc->memcg) ?: 1; 265 return CONSTRAINT_MEMCG; 266 } 267 268 /* Default to all available memory */ 269 oc->totalpages = totalram_pages + total_swap_pages; 270 271 if (!IS_ENABLED(CONFIG_NUMA)) 272 return CONSTRAINT_NONE; 273 274 if (!oc->zonelist) 275 return CONSTRAINT_NONE; 276 /* 277 * Reach here only when __GFP_NOFAIL is used. So, we should avoid 278 * to kill current.We have to random task kill in this case. 279 * Hopefully, CONSTRAINT_THISNODE...but no way to handle it, now. 280 */ 281 if (oc->gfp_mask & __GFP_THISNODE) 282 return CONSTRAINT_NONE; 283 284 /* 285 * This is not a __GFP_THISNODE allocation, so a truncated nodemask in 286 * the page allocator means a mempolicy is in effect. Cpuset policy 287 * is enforced in get_page_from_freelist(). 288 */ 289 if (oc->nodemask && 290 !nodes_subset(node_states[N_MEMORY], *oc->nodemask)) { 291 oc->totalpages = total_swap_pages; 292 for_each_node_mask(nid, *oc->nodemask) 293 oc->totalpages += node_spanned_pages(nid); 294 return CONSTRAINT_MEMORY_POLICY; 295 } 296 297 /* Check this allocation failure is caused by cpuset's wall function */ 298 for_each_zone_zonelist_nodemask(zone, z, oc->zonelist, 299 high_zoneidx, oc->nodemask) 300 if (!cpuset_zone_allowed(zone, oc->gfp_mask)) 301 cpuset_limited = true; 302 303 if (cpuset_limited) { 304 oc->totalpages = total_swap_pages; 305 for_each_node_mask(nid, cpuset_current_mems_allowed) 306 oc->totalpages += node_spanned_pages(nid); 307 return CONSTRAINT_CPUSET; 308 } 309 return CONSTRAINT_NONE; 310 } 311 312 static int oom_evaluate_task(struct task_struct *task, void *arg) 313 { 314 struct oom_control *oc = arg; 315 unsigned long points; 316 317 if (oom_unkillable_task(task, NULL, oc->nodemask)) 318 goto next; 319 320 /* 321 * This task already has access to memory reserves and is being killed. 322 * Don't allow any other task to have access to the reserves unless 323 * the task has MMF_OOM_SKIP because chances that it would release 324 * any memory is quite low. 325 */ 326 if (!is_sysrq_oom(oc) && tsk_is_oom_victim(task)) { 327 if (test_bit(MMF_OOM_SKIP, &task->signal->oom_mm->flags)) 328 goto next; 329 goto abort; 330 } 331 332 /* 333 * If task is allocating a lot of memory and has been marked to be 334 * killed first if it triggers an oom, then select it. 335 */ 336 if (oom_task_origin(task)) { 337 points = ULONG_MAX; 338 goto select; 339 } 340 341 points = oom_badness(task, NULL, oc->nodemask, oc->totalpages); 342 if (!points || points < oc->chosen_points) 343 goto next; 344 345 /* Prefer thread group leaders for display purposes */ 346 if (points == oc->chosen_points && thread_group_leader(oc->chosen)) 347 goto next; 348 select: 349 if (oc->chosen) 350 put_task_struct(oc->chosen); 351 get_task_struct(task); 352 oc->chosen = task; 353 oc->chosen_points = points; 354 next: 355 return 0; 356 abort: 357 if (oc->chosen) 358 put_task_struct(oc->chosen); 359 oc->chosen = (void *)-1UL; 360 return 1; 361 } 362 363 /* 364 * Simple selection loop. We choose the process with the highest number of 365 * 'points'. In case scan was aborted, oc->chosen is set to -1. 366 */ 367 static void select_bad_process(struct oom_control *oc) 368 { 369 if (is_memcg_oom(oc)) 370 mem_cgroup_scan_tasks(oc->memcg, oom_evaluate_task, oc); 371 else { 372 struct task_struct *p; 373 374 rcu_read_lock(); 375 for_each_process(p) 376 if (oom_evaluate_task(p, oc)) 377 break; 378 rcu_read_unlock(); 379 } 380 381 oc->chosen_points = oc->chosen_points * 1000 / oc->totalpages; 382 } 383 384 /** 385 * dump_tasks - dump current memory state of all system tasks 386 * @memcg: current's memory controller, if constrained 387 * @nodemask: nodemask passed to page allocator for mempolicy ooms 388 * 389 * Dumps the current memory state of all eligible tasks. Tasks not in the same 390 * memcg, not in the same cpuset, or bound to a disjoint set of mempolicy nodes 391 * are not shown. 392 * State information includes task's pid, uid, tgid, vm size, rss, 393 * pgtables_bytes, swapents, oom_score_adj value, and name. 394 */ 395 static void dump_tasks(struct mem_cgroup *memcg, const nodemask_t *nodemask) 396 { 397 struct task_struct *p; 398 struct task_struct *task; 399 400 pr_info("[ pid ] uid tgid total_vm rss pgtables_bytes swapents oom_score_adj name\n"); 401 rcu_read_lock(); 402 for_each_process(p) { 403 if (oom_unkillable_task(p, memcg, nodemask)) 404 continue; 405 406 task = find_lock_task_mm(p); 407 if (!task) { 408 /* 409 * This is a kthread or all of p's threads have already 410 * detached their mm's. There's no need to report 411 * them; they can't be oom killed anyway. 412 */ 413 continue; 414 } 415 416 pr_info("[%5d] %5d %5d %8lu %8lu %8ld %8lu %5hd %s\n", 417 task->pid, from_kuid(&init_user_ns, task_uid(task)), 418 task->tgid, task->mm->total_vm, get_mm_rss(task->mm), 419 mm_pgtables_bytes(task->mm), 420 get_mm_counter(task->mm, MM_SWAPENTS), 421 task->signal->oom_score_adj, task->comm); 422 task_unlock(task); 423 } 424 rcu_read_unlock(); 425 } 426 427 static void dump_header(struct oom_control *oc, struct task_struct *p) 428 { 429 pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), nodemask=%*pbl, order=%d, oom_score_adj=%hd\n", 430 current->comm, oc->gfp_mask, &oc->gfp_mask, 431 nodemask_pr_args(oc->nodemask), oc->order, 432 current->signal->oom_score_adj); 433 if (!IS_ENABLED(CONFIG_COMPACTION) && oc->order) 434 pr_warn("COMPACTION is disabled!!!\n"); 435 436 cpuset_print_current_mems_allowed(); 437 dump_stack(); 438 if (is_memcg_oom(oc)) 439 mem_cgroup_print_oom_info(oc->memcg, p); 440 else { 441 show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask); 442 if (is_dump_unreclaim_slabs()) 443 dump_unreclaimable_slab(); 444 } 445 if (sysctl_oom_dump_tasks) 446 dump_tasks(oc->memcg, oc->nodemask); 447 } 448 449 /* 450 * Number of OOM victims in flight 451 */ 452 static atomic_t oom_victims = ATOMIC_INIT(0); 453 static DECLARE_WAIT_QUEUE_HEAD(oom_victims_wait); 454 455 static bool oom_killer_disabled __read_mostly; 456 457 #define K(x) ((x) << (PAGE_SHIFT-10)) 458 459 /* 460 * task->mm can be NULL if the task is the exited group leader. So to 461 * determine whether the task is using a particular mm, we examine all the 462 * task's threads: if one of those is using this mm then this task was also 463 * using it. 464 */ 465 bool process_shares_mm(struct task_struct *p, struct mm_struct *mm) 466 { 467 struct task_struct *t; 468 469 for_each_thread(p, t) { 470 struct mm_struct *t_mm = READ_ONCE(t->mm); 471 if (t_mm) 472 return t_mm == mm; 473 } 474 return false; 475 } 476 477 478 #ifdef CONFIG_MMU 479 /* 480 * OOM Reaper kernel thread which tries to reap the memory used by the OOM 481 * victim (if that is possible) to help the OOM killer to move on. 482 */ 483 static struct task_struct *oom_reaper_th; 484 static DECLARE_WAIT_QUEUE_HEAD(oom_reaper_wait); 485 static struct task_struct *oom_reaper_list; 486 static DEFINE_SPINLOCK(oom_reaper_lock); 487 488 static bool __oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) 489 { 490 struct mmu_gather tlb; 491 struct vm_area_struct *vma; 492 bool ret = true; 493 494 /* 495 * We have to make sure to not race with the victim exit path 496 * and cause premature new oom victim selection: 497 * __oom_reap_task_mm exit_mm 498 * mmget_not_zero 499 * mmput 500 * atomic_dec_and_test 501 * exit_oom_victim 502 * [...] 503 * out_of_memory 504 * select_bad_process 505 * # no TIF_MEMDIE task selects new victim 506 * unmap_page_range # frees some memory 507 */ 508 mutex_lock(&oom_lock); 509 510 if (!down_read_trylock(&mm->mmap_sem)) { 511 ret = false; 512 trace_skip_task_reaping(tsk->pid); 513 goto unlock_oom; 514 } 515 516 /* 517 * If the mm has notifiers then we would need to invalidate them around 518 * unmap_page_range and that is risky because notifiers can sleep and 519 * what they do is basically undeterministic. So let's have a short 520 * sleep to give the oom victim some more time. 521 * TODO: we really want to get rid of this ugly hack and make sure that 522 * notifiers cannot block for unbounded amount of time and add 523 * mmu_notifier_invalidate_range_{start,end} around unmap_page_range 524 */ 525 if (mm_has_notifiers(mm)) { 526 up_read(&mm->mmap_sem); 527 schedule_timeout_idle(HZ); 528 goto unlock_oom; 529 } 530 531 /* 532 * MMF_OOM_SKIP is set by exit_mmap when the OOM reaper can't 533 * work on the mm anymore. The check for MMF_OOM_SKIP must run 534 * under mmap_sem for reading because it serializes against the 535 * down_write();up_write() cycle in exit_mmap(). 536 */ 537 if (test_bit(MMF_OOM_SKIP, &mm->flags)) { 538 up_read(&mm->mmap_sem); 539 trace_skip_task_reaping(tsk->pid); 540 goto unlock_oom; 541 } 542 543 trace_start_task_reaping(tsk->pid); 544 545 /* 546 * Tell all users of get_user/copy_from_user etc... that the content 547 * is no longer stable. No barriers really needed because unmapping 548 * should imply barriers already and the reader would hit a page fault 549 * if it stumbled over a reaped memory. 550 */ 551 set_bit(MMF_UNSTABLE, &mm->flags); 552 553 tlb_gather_mmu(&tlb, mm, 0, -1); 554 for (vma = mm->mmap ; vma; vma = vma->vm_next) { 555 if (!can_madv_dontneed_vma(vma)) 556 continue; 557 558 /* 559 * Only anonymous pages have a good chance to be dropped 560 * without additional steps which we cannot afford as we 561 * are OOM already. 562 * 563 * We do not even care about fs backed pages because all 564 * which are reclaimable have already been reclaimed and 565 * we do not want to block exit_mmap by keeping mm ref 566 * count elevated without a good reason. 567 */ 568 if (vma_is_anonymous(vma) || !(vma->vm_flags & VM_SHARED)) 569 unmap_page_range(&tlb, vma, vma->vm_start, vma->vm_end, 570 NULL); 571 } 572 tlb_finish_mmu(&tlb, 0, -1); 573 pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", 574 task_pid_nr(tsk), tsk->comm, 575 K(get_mm_counter(mm, MM_ANONPAGES)), 576 K(get_mm_counter(mm, MM_FILEPAGES)), 577 K(get_mm_counter(mm, MM_SHMEMPAGES))); 578 up_read(&mm->mmap_sem); 579 580 trace_finish_task_reaping(tsk->pid); 581 unlock_oom: 582 mutex_unlock(&oom_lock); 583 return ret; 584 } 585 586 #define MAX_OOM_REAP_RETRIES 10 587 static void oom_reap_task(struct task_struct *tsk) 588 { 589 int attempts = 0; 590 struct mm_struct *mm = tsk->signal->oom_mm; 591 592 /* Retry the down_read_trylock(mmap_sem) a few times */ 593 while (attempts++ < MAX_OOM_REAP_RETRIES && !__oom_reap_task_mm(tsk, mm)) 594 schedule_timeout_idle(HZ/10); 595 596 if (attempts <= MAX_OOM_REAP_RETRIES) 597 goto done; 598 599 600 pr_info("oom_reaper: unable to reap pid:%d (%s)\n", 601 task_pid_nr(tsk), tsk->comm); 602 debug_show_all_locks(); 603 604 done: 605 tsk->oom_reaper_list = NULL; 606 607 /* 608 * Hide this mm from OOM killer because it has been either reaped or 609 * somebody can't call up_write(mmap_sem). 610 */ 611 set_bit(MMF_OOM_SKIP, &mm->flags); 612 613 /* Drop a reference taken by wake_oom_reaper */ 614 put_task_struct(tsk); 615 } 616 617 static int oom_reaper(void *unused) 618 { 619 while (true) { 620 struct task_struct *tsk = NULL; 621 622 wait_event_freezable(oom_reaper_wait, oom_reaper_list != NULL); 623 spin_lock(&oom_reaper_lock); 624 if (oom_reaper_list != NULL) { 625 tsk = oom_reaper_list; 626 oom_reaper_list = tsk->oom_reaper_list; 627 } 628 spin_unlock(&oom_reaper_lock); 629 630 if (tsk) 631 oom_reap_task(tsk); 632 } 633 634 return 0; 635 } 636 637 static void wake_oom_reaper(struct task_struct *tsk) 638 { 639 /* tsk is already queued? */ 640 if (tsk == oom_reaper_list || tsk->oom_reaper_list) 641 return; 642 643 get_task_struct(tsk); 644 645 spin_lock(&oom_reaper_lock); 646 tsk->oom_reaper_list = oom_reaper_list; 647 oom_reaper_list = tsk; 648 spin_unlock(&oom_reaper_lock); 649 trace_wake_reaper(tsk->pid); 650 wake_up(&oom_reaper_wait); 651 } 652 653 static int __init oom_init(void) 654 { 655 oom_reaper_th = kthread_run(oom_reaper, NULL, "oom_reaper"); 656 return 0; 657 } 658 subsys_initcall(oom_init) 659 #else 660 static inline void wake_oom_reaper(struct task_struct *tsk) 661 { 662 } 663 #endif /* CONFIG_MMU */ 664 665 /** 666 * mark_oom_victim - mark the given task as OOM victim 667 * @tsk: task to mark 668 * 669 * Has to be called with oom_lock held and never after 670 * oom has been disabled already. 671 * 672 * tsk->mm has to be non NULL and caller has to guarantee it is stable (either 673 * under task_lock or operate on the current). 674 */ 675 static void mark_oom_victim(struct task_struct *tsk) 676 { 677 struct mm_struct *mm = tsk->mm; 678 679 WARN_ON(oom_killer_disabled); 680 /* OOM killer might race with memcg OOM */ 681 if (test_and_set_tsk_thread_flag(tsk, TIF_MEMDIE)) 682 return; 683 684 /* oom_mm is bound to the signal struct life time. */ 685 if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm)) 686 mmgrab(tsk->signal->oom_mm); 687 688 /* 689 * Make sure that the task is woken up from uninterruptible sleep 690 * if it is frozen because OOM killer wouldn't be able to free 691 * any memory and livelock. freezing_slow_path will tell the freezer 692 * that TIF_MEMDIE tasks should be ignored. 693 */ 694 __thaw_task(tsk); 695 atomic_inc(&oom_victims); 696 trace_mark_victim(tsk->pid); 697 } 698 699 /** 700 * exit_oom_victim - note the exit of an OOM victim 701 */ 702 void exit_oom_victim(void) 703 { 704 clear_thread_flag(TIF_MEMDIE); 705 706 if (!atomic_dec_return(&oom_victims)) 707 wake_up_all(&oom_victims_wait); 708 } 709 710 /** 711 * oom_killer_enable - enable OOM killer 712 */ 713 void oom_killer_enable(void) 714 { 715 oom_killer_disabled = false; 716 pr_info("OOM killer enabled.\n"); 717 } 718 719 /** 720 * oom_killer_disable - disable OOM killer 721 * @timeout: maximum timeout to wait for oom victims in jiffies 722 * 723 * Forces all page allocations to fail rather than trigger OOM killer. 724 * Will block and wait until all OOM victims are killed or the given 725 * timeout expires. 726 * 727 * The function cannot be called when there are runnable user tasks because 728 * the userspace would see unexpected allocation failures as a result. Any 729 * new usage of this function should be consulted with MM people. 730 * 731 * Returns true if successful and false if the OOM killer cannot be 732 * disabled. 733 */ 734 bool oom_killer_disable(signed long timeout) 735 { 736 signed long ret; 737 738 /* 739 * Make sure to not race with an ongoing OOM killer. Check that the 740 * current is not killed (possibly due to sharing the victim's memory). 741 */ 742 if (mutex_lock_killable(&oom_lock)) 743 return false; 744 oom_killer_disabled = true; 745 mutex_unlock(&oom_lock); 746 747 ret = wait_event_interruptible_timeout(oom_victims_wait, 748 !atomic_read(&oom_victims), timeout); 749 if (ret <= 0) { 750 oom_killer_enable(); 751 return false; 752 } 753 pr_info("OOM killer disabled.\n"); 754 755 return true; 756 } 757 758 static inline bool __task_will_free_mem(struct task_struct *task) 759 { 760 struct signal_struct *sig = task->signal; 761 762 /* 763 * A coredumping process may sleep for an extended period in exit_mm(), 764 * so the oom killer cannot assume that the process will promptly exit 765 * and release memory. 766 */ 767 if (sig->flags & SIGNAL_GROUP_COREDUMP) 768 return false; 769 770 if (sig->flags & SIGNAL_GROUP_EXIT) 771 return true; 772 773 if (thread_group_empty(task) && (task->flags & PF_EXITING)) 774 return true; 775 776 return false; 777 } 778 779 /* 780 * Checks whether the given task is dying or exiting and likely to 781 * release its address space. This means that all threads and processes 782 * sharing the same mm have to be killed or exiting. 783 * Caller has to make sure that task->mm is stable (hold task_lock or 784 * it operates on the current). 785 */ 786 static bool task_will_free_mem(struct task_struct *task) 787 { 788 struct mm_struct *mm = task->mm; 789 struct task_struct *p; 790 bool ret = true; 791 792 /* 793 * Skip tasks without mm because it might have passed its exit_mm and 794 * exit_oom_victim. oom_reaper could have rescued that but do not rely 795 * on that for now. We can consider find_lock_task_mm in future. 796 */ 797 if (!mm) 798 return false; 799 800 if (!__task_will_free_mem(task)) 801 return false; 802 803 /* 804 * This task has already been drained by the oom reaper so there are 805 * only small chances it will free some more 806 */ 807 if (test_bit(MMF_OOM_SKIP, &mm->flags)) 808 return false; 809 810 if (atomic_read(&mm->mm_users) <= 1) 811 return true; 812 813 /* 814 * Make sure that all tasks which share the mm with the given tasks 815 * are dying as well to make sure that a) nobody pins its mm and 816 * b) the task is also reapable by the oom reaper. 817 */ 818 rcu_read_lock(); 819 for_each_process(p) { 820 if (!process_shares_mm(p, mm)) 821 continue; 822 if (same_thread_group(task, p)) 823 continue; 824 ret = __task_will_free_mem(p); 825 if (!ret) 826 break; 827 } 828 rcu_read_unlock(); 829 830 return ret; 831 } 832 833 static void oom_kill_process(struct oom_control *oc, const char *message) 834 { 835 struct task_struct *p = oc->chosen; 836 unsigned int points = oc->chosen_points; 837 struct task_struct *victim = p; 838 struct task_struct *child; 839 struct task_struct *t; 840 struct mm_struct *mm; 841 unsigned int victim_points = 0; 842 static DEFINE_RATELIMIT_STATE(oom_rs, DEFAULT_RATELIMIT_INTERVAL, 843 DEFAULT_RATELIMIT_BURST); 844 bool can_oom_reap = true; 845 846 /* 847 * If the task is already exiting, don't alarm the sysadmin or kill 848 * its children or threads, just give it access to memory reserves 849 * so it can die quickly 850 */ 851 task_lock(p); 852 if (task_will_free_mem(p)) { 853 mark_oom_victim(p); 854 wake_oom_reaper(p); 855 task_unlock(p); 856 put_task_struct(p); 857 return; 858 } 859 task_unlock(p); 860 861 if (__ratelimit(&oom_rs)) 862 dump_header(oc, p); 863 864 pr_err("%s: Kill process %d (%s) score %u or sacrifice child\n", 865 message, task_pid_nr(p), p->comm, points); 866 867 /* 868 * If any of p's children has a different mm and is eligible for kill, 869 * the one with the highest oom_badness() score is sacrificed for its 870 * parent. This attempts to lose the minimal amount of work done while 871 * still freeing memory. 872 */ 873 read_lock(&tasklist_lock); 874 for_each_thread(p, t) { 875 list_for_each_entry(child, &t->children, sibling) { 876 unsigned int child_points; 877 878 if (process_shares_mm(child, p->mm)) 879 continue; 880 /* 881 * oom_badness() returns 0 if the thread is unkillable 882 */ 883 child_points = oom_badness(child, 884 oc->memcg, oc->nodemask, oc->totalpages); 885 if (child_points > victim_points) { 886 put_task_struct(victim); 887 victim = child; 888 victim_points = child_points; 889 get_task_struct(victim); 890 } 891 } 892 } 893 read_unlock(&tasklist_lock); 894 895 p = find_lock_task_mm(victim); 896 if (!p) { 897 put_task_struct(victim); 898 return; 899 } else if (victim != p) { 900 get_task_struct(p); 901 put_task_struct(victim); 902 victim = p; 903 } 904 905 /* Get a reference to safely compare mm after task_unlock(victim) */ 906 mm = victim->mm; 907 mmgrab(mm); 908 909 /* Raise event before sending signal: task reaper must see this */ 910 count_vm_event(OOM_KILL); 911 count_memcg_event_mm(mm, OOM_KILL); 912 913 /* 914 * We should send SIGKILL before granting access to memory reserves 915 * in order to prevent the OOM victim from depleting the memory 916 * reserves from the user space under its control. 917 */ 918 do_send_sig_info(SIGKILL, SEND_SIG_FORCED, victim, true); 919 mark_oom_victim(victim); 920 pr_err("Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", 921 task_pid_nr(victim), victim->comm, K(victim->mm->total_vm), 922 K(get_mm_counter(victim->mm, MM_ANONPAGES)), 923 K(get_mm_counter(victim->mm, MM_FILEPAGES)), 924 K(get_mm_counter(victim->mm, MM_SHMEMPAGES))); 925 task_unlock(victim); 926 927 /* 928 * Kill all user processes sharing victim->mm in other thread groups, if 929 * any. They don't get access to memory reserves, though, to avoid 930 * depletion of all memory. This prevents mm->mmap_sem livelock when an 931 * oom killed thread cannot exit because it requires the semaphore and 932 * its contended by another thread trying to allocate memory itself. 933 * That thread will now get access to memory reserves since it has a 934 * pending fatal signal. 935 */ 936 rcu_read_lock(); 937 for_each_process(p) { 938 if (!process_shares_mm(p, mm)) 939 continue; 940 if (same_thread_group(p, victim)) 941 continue; 942 if (is_global_init(p)) { 943 can_oom_reap = false; 944 set_bit(MMF_OOM_SKIP, &mm->flags); 945 pr_info("oom killer %d (%s) has mm pinned by %d (%s)\n", 946 task_pid_nr(victim), victim->comm, 947 task_pid_nr(p), p->comm); 948 continue; 949 } 950 /* 951 * No use_mm() user needs to read from the userspace so we are 952 * ok to reap it. 953 */ 954 if (unlikely(p->flags & PF_KTHREAD)) 955 continue; 956 do_send_sig_info(SIGKILL, SEND_SIG_FORCED, p, true); 957 } 958 rcu_read_unlock(); 959 960 if (can_oom_reap) 961 wake_oom_reaper(victim); 962 963 mmdrop(mm); 964 put_task_struct(victim); 965 } 966 #undef K 967 968 /* 969 * Determines whether the kernel must panic because of the panic_on_oom sysctl. 970 */ 971 static void check_panic_on_oom(struct oom_control *oc, 972 enum oom_constraint constraint) 973 { 974 if (likely(!sysctl_panic_on_oom)) 975 return; 976 if (sysctl_panic_on_oom != 2) { 977 /* 978 * panic_on_oom == 1 only affects CONSTRAINT_NONE, the kernel 979 * does not panic for cpuset, mempolicy, or memcg allocation 980 * failures. 981 */ 982 if (constraint != CONSTRAINT_NONE) 983 return; 984 } 985 /* Do not panic for oom kills triggered by sysrq */ 986 if (is_sysrq_oom(oc)) 987 return; 988 dump_header(oc, NULL); 989 panic("Out of memory: %s panic_on_oom is enabled\n", 990 sysctl_panic_on_oom == 2 ? "compulsory" : "system-wide"); 991 } 992 993 static BLOCKING_NOTIFIER_HEAD(oom_notify_list); 994 995 int register_oom_notifier(struct notifier_block *nb) 996 { 997 return blocking_notifier_chain_register(&oom_notify_list, nb); 998 } 999 EXPORT_SYMBOL_GPL(register_oom_notifier); 1000 1001 int unregister_oom_notifier(struct notifier_block *nb) 1002 { 1003 return blocking_notifier_chain_unregister(&oom_notify_list, nb); 1004 } 1005 EXPORT_SYMBOL_GPL(unregister_oom_notifier); 1006 1007 /** 1008 * out_of_memory - kill the "best" process when we run out of memory 1009 * @oc: pointer to struct oom_control 1010 * 1011 * If we run out of memory, we have the choice between either 1012 * killing a random task (bad), letting the system crash (worse) 1013 * OR try to be smart about which process to kill. Note that we 1014 * don't have to be perfect here, we just have to be good. 1015 */ 1016 bool out_of_memory(struct oom_control *oc) 1017 { 1018 unsigned long freed = 0; 1019 enum oom_constraint constraint = CONSTRAINT_NONE; 1020 1021 if (oom_killer_disabled) 1022 return false; 1023 1024 if (!is_memcg_oom(oc)) { 1025 blocking_notifier_call_chain(&oom_notify_list, 0, &freed); 1026 if (freed > 0) 1027 /* Got some memory back in the last second. */ 1028 return true; 1029 } 1030 1031 /* 1032 * If current has a pending SIGKILL or is exiting, then automatically 1033 * select it. The goal is to allow it to allocate so that it may 1034 * quickly exit and free its memory. 1035 */ 1036 if (task_will_free_mem(current)) { 1037 mark_oom_victim(current); 1038 wake_oom_reaper(current); 1039 return true; 1040 } 1041 1042 /* 1043 * The OOM killer does not compensate for IO-less reclaim. 1044 * pagefault_out_of_memory lost its gfp context so we have to 1045 * make sure exclude 0 mask - all other users should have at least 1046 * ___GFP_DIRECT_RECLAIM to get here. 1047 */ 1048 if (oc->gfp_mask && !(oc->gfp_mask & __GFP_FS)) 1049 return true; 1050 1051 /* 1052 * Check if there were limitations on the allocation (only relevant for 1053 * NUMA and memcg) that may require different handling. 1054 */ 1055 constraint = constrained_alloc(oc); 1056 if (constraint != CONSTRAINT_MEMORY_POLICY) 1057 oc->nodemask = NULL; 1058 check_panic_on_oom(oc, constraint); 1059 1060 if (!is_memcg_oom(oc) && sysctl_oom_kill_allocating_task && 1061 current->mm && !oom_unkillable_task(current, NULL, oc->nodemask) && 1062 current->signal->oom_score_adj != OOM_SCORE_ADJ_MIN) { 1063 get_task_struct(current); 1064 oc->chosen = current; 1065 oom_kill_process(oc, "Out of memory (oom_kill_allocating_task)"); 1066 return true; 1067 } 1068 1069 select_bad_process(oc); 1070 /* Found nothing?!?! Either we hang forever, or we panic. */ 1071 if (!oc->chosen && !is_sysrq_oom(oc) && !is_memcg_oom(oc)) { 1072 dump_header(oc, NULL); 1073 panic("Out of memory and no killable processes...\n"); 1074 } 1075 if (oc->chosen && oc->chosen != (void *)-1UL) { 1076 oom_kill_process(oc, !is_memcg_oom(oc) ? "Out of memory" : 1077 "Memory cgroup out of memory"); 1078 /* 1079 * Give the killed process a good chance to exit before trying 1080 * to allocate memory again. 1081 */ 1082 schedule_timeout_killable(1); 1083 } 1084 return !!oc->chosen; 1085 } 1086 1087 /* 1088 * The pagefault handler calls here because it is out of memory, so kill a 1089 * memory-hogging task. If oom_lock is held by somebody else, a parallel oom 1090 * killing is already in progress so do nothing. 1091 */ 1092 void pagefault_out_of_memory(void) 1093 { 1094 struct oom_control oc = { 1095 .zonelist = NULL, 1096 .nodemask = NULL, 1097 .memcg = NULL, 1098 .gfp_mask = 0, 1099 .order = 0, 1100 }; 1101 1102 if (mem_cgroup_oom_synchronize(true)) 1103 return; 1104 1105 if (!mutex_trylock(&oom_lock)) 1106 return; 1107 out_of_memory(&oc); 1108 mutex_unlock(&oom_lock); 1109 } 1110