1 // SPDX-License-Identifier: GPL-2.0-only
15  *  Since we won't call these routines often (on a well-configured
75 	return oc->memcg != NULL;  in is_memcg_oom()
80  * oom_cpuset_eligible() - check task eligibility for kill
88  * This function is assuming oom-killer context and 'current' has triggered
89  * the oom-killer.
96 	const nodemask_t *mask = oc->nodemask;  in oom_cpuset_eligible()
130  * The process p may have detached its own ->mm while exiting or through
132  * pointer.  Return p, or any of its subthreads with a valid ->mm, with
143 		if (likely(t->mm))  in find_lock_task_mm()
155  * order == -1 means the oom kill is required by sysrq, otherwise only
160 	return oc->order == -1;  in is_sysrq_oom()
168 	if (p->flags & PF_KTHREAD)  in oom_unkillable_task()
195  * oom_badness - heuristic function to determine which candidate task to kill
220 	adj = (long)p->signal->oom_score_adj;  in oom_badness()
222 			test_bit(MMF_OOM_SKIP, &p->mm->flags) ||  in oom_badness()
230 	 * task's rss, pagetable and swap space use.  in oom_badness()
232 	points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) +  in oom_badness()
233 		mm_pgtables_bytes(p->mm) / PAGE_SIZE;  in oom_badness()
251  * Determine the type of allocation constraint.
257 	enum zone_type highest_zoneidx = gfp_zone(oc->gfp_mask);  in constrained_alloc()
262 		oc->totalpages = mem_cgroup_get_max(oc->memcg) ?: 1;  in constrained_alloc()
267 	oc->totalpages = totalram_pages() + total_swap_pages;  in constrained_alloc()
272 	if (!oc->zonelist)  in constrained_alloc()
279 	if (oc->gfp_mask & __GFP_THISNODE)  in constrained_alloc()
287 	if (oc->nodemask &&  in constrained_alloc()
288 	    !nodes_subset(node_states[N_MEMORY], *oc->nodemask)) {  in constrained_alloc()
289 		oc->totalpages = total_swap_pages;  in constrained_alloc()
290 		for_each_node_mask(nid, *oc->nodemask)  in constrained_alloc()
291 			oc->totalpages += node_present_pages(nid);  in constrained_alloc()
296 	for_each_zone_zonelist_nodemask(zone, z, oc->zonelist,  in constrained_alloc()
297 			highest_zoneidx, oc->nodemask)  in constrained_alloc()
298 		if (!cpuset_zone_allowed(zone, oc->gfp_mask))  in constrained_alloc()
302 		oc->totalpages = total_swap_pages;  in constrained_alloc()
304 			oc->totalpages += node_present_pages(nid);  in constrained_alloc()
329 		if (test_bit(MMF_OOM_SKIP, &task->signal->oom_mm->flags))  in oom_evaluate_task()
343 	points = oom_badness(task, oc->totalpages);  in oom_evaluate_task()
344 	if (points == LONG_MIN || points < oc->chosen_points)  in oom_evaluate_task()
348 	if (oc->chosen)  in oom_evaluate_task()
349 		put_task_struct(oc->chosen);  in oom_evaluate_task()
351 	oc->chosen = task;  in oom_evaluate_task()
352 	oc->chosen_points = points;  in oom_evaluate_task()
356 	if (oc->chosen)  in oom_evaluate_task()
357 		put_task_struct(oc->chosen);  in oom_evaluate_task()
358 	oc->chosen = (void *)-1UL;  in oom_evaluate_task()
364  * 'points'. In case scan was aborted, oc->chosen is set to -1.
368 	oc->chosen_points = LONG_MIN;  in select_bad_process()
371 		mem_cgroup_scan_tasks(oc->memcg, oom_evaluate_task, oc);  in select_bad_process()
405 		task->pid, from_kuid(&init_user_ns, task_uid(task)),  in dump_task()
406 		task->tgid, task->mm->total_vm, get_mm_rss(task->mm),  in dump_task()
407 		mm_pgtables_bytes(task->mm),  in dump_task()
408 		get_mm_counter(task->mm, MM_SWAPENTS),  in dump_task()
409 		task->signal->oom_score_adj, task->comm);  in dump_task()
416  * dump_tasks - dump current memory state of all system tasks
431 		mem_cgroup_scan_tasks(oc->memcg, dump_task, oc);  in dump_tasks()
450 	pr_info("oom-kill:constraint=%s,nodemask=%*pbl",  in dump_oom_summary()
451 			oom_constraint_text[oc->constraint],  in dump_oom_summary()
452 			nodemask_pr_args(oc->nodemask));  in dump_oom_summary()
454 	mem_cgroup_print_oom_context(oc->memcg, victim);  in dump_oom_summary()
455 	pr_cont(",task=%s,pid=%d,uid=%d\n", victim->comm, victim->pid,  in dump_oom_summary()
461 	pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), order=%d, oom_score_adj=%hd\n",  in dump_header()
462 		current->comm, oc->gfp_mask, &oc->gfp_mask, oc->order,  in dump_header()
463 			current->signal->oom_score_adj);  in dump_header()
464 	if (!IS_ENABLED(CONFIG_COMPACTION) && oc->order)  in dump_header()
469 		mem_cgroup_print_oom_meminfo(oc->memcg);  in dump_header()
471 		__show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask, gfp_zone(oc->gfp_mask));  in dump_header()
490  * task->mm can be NULL if the task is the exited group leader.  So to
500 		struct mm_struct *t_mm = READ_ONCE(t->mm);  in process_shares_mm()
529 	set_bit(MMF_UNSTABLE, &mm->flags);  in __oom_reap_task_mm()
532 		if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP))  in __oom_reap_task_mm()
545 		if (vma_is_anonymous(vma) || !(vma->vm_flags & VM_SHARED)) {  in __oom_reap_task_mm()
550 						mm, vma->vm_start,  in __oom_reap_task_mm()
551 						vma->vm_end);  in __oom_reap_task_mm()
568  * Reaps the address space of the give task.
570  * Returns true on success and false if none or part of the address space
578 		trace_skip_task_reaping(tsk->pid);  in oom_reap_task_mm()
588 	if (test_bit(MMF_OOM_SKIP, &mm->flags)) {  in oom_reap_task_mm()
589 		trace_skip_task_reaping(tsk->pid);  in oom_reap_task_mm()
593 	trace_start_task_reaping(tsk->pid);  in oom_reap_task_mm()
595 	/* failed to reap part of the address space. Try again later */  in oom_reap_task_mm()
600 …pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n…  in oom_reap_task_mm()
601 			task_pid_nr(tsk), tsk->comm,  in oom_reap_task_mm()
606 	trace_finish_task_reaping(tsk->pid);  in oom_reap_task_mm()
617 	struct mm_struct *mm = tsk->signal->oom_mm;  in oom_reap_task()
624 	    test_bit(MMF_OOM_SKIP, &mm->flags))  in oom_reap_task()
628 		task_pid_nr(tsk), tsk->comm);  in oom_reap_task()
633 	tsk->oom_reaper_list = NULL;  in oom_reap_task()
639 	set_bit(MMF_OOM_SKIP, &mm->flags);  in oom_reap_task()
656 			oom_reaper_list = tsk->oom_reaper_list;  in oom_reaper()
671 	struct mm_struct *mm = tsk->signal->oom_mm;  in wake_oom_reaper()
674 	/* The victim managed to terminate on its own - see exit_mmap */  in wake_oom_reaper()
675 	if (test_bit(MMF_OOM_SKIP, &mm->flags)) {  in wake_oom_reaper()
681 	tsk->oom_reaper_list = oom_reaper_list;  in wake_oom_reaper()
684 	trace_wake_reaper(tsk->pid);  in wake_oom_reaper()
700 	if (test_and_set_bit(MMF_OOM_REAP_QUEUED, &tsk->signal->oom_mm->flags))  in queue_oom_reaper()
704 	timer_setup(&tsk->oom_reaper_timer, wake_oom_reaper, 0);  in queue_oom_reaper()
705 	tsk->oom_reaper_timer.expires = jiffies + OOM_REAPER_DELAY;  in queue_oom_reaper()
706 	add_timer(&tsk->oom_reaper_timer);  in queue_oom_reaper()
754  * mark_oom_victim - mark the given task as OOM victim
760  * tsk->mm has to be non NULL and caller has to guarantee it is stable (either
766 	struct mm_struct *mm = tsk->mm;
774 	if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm))
775 		mmgrab(tsk->signal->oom_mm);
786 	trace_mark_victim(tsk, cred->uid.val);
791  * exit_oom_victim - note the exit of an OOM victim
802  * oom_killer_enable - enable OOM killer
811  * oom_killer_disable - disable OOM killer
851 	struct signal_struct *sig = task->signal;  in __task_will_free_mem()
858 	if (sig->core_state)  in __task_will_free_mem()
861 	if (sig->flags & SIGNAL_GROUP_EXIT)  in __task_will_free_mem()
864 	if (thread_group_empty(task) && (task->flags & PF_EXITING))  in __task_will_free_mem()
872  * release its address space. This means that all threads and processes
874  * Caller has to make sure that task->mm is stable (hold task_lock or
879 	struct mm_struct *mm = task->mm;  in task_will_free_mem()
898 	if (test_bit(MMF_OOM_SKIP, &mm->flags))  in task_will_free_mem()
901 	if (atomic_read(&mm->mm_users) <= 1)  in task_will_free_mem()
933 			message, task_pid_nr(victim), victim->comm);  in __oom_kill_process()
943 	mm = victim->mm;  in __oom_kill_process()
953 	 * reserves from the user space under its control.  in __oom_kill_process()
957 …pr_err("%s: Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB…  in __oom_kill_process()
958 		message, task_pid_nr(victim), victim->comm, K(mm->total_vm),  in __oom_kill_process()
963 		mm_pgtables_bytes(mm) >> 10, victim->signal->oom_score_adj);  in __oom_kill_process()
967 	 * Kill all user processes sharing victim->mm in other thread groups, if  in __oom_kill_process()
969 	 * depletion of all memory.  This prevents mm->mmap_lock livelock when an  in __oom_kill_process()
983 			set_bit(MMF_OOM_SKIP, &mm->flags);  in __oom_kill_process()
985 					task_pid_nr(victim), victim->comm,  in __oom_kill_process()
986 					task_pid_nr(p), p->comm);  in __oom_kill_process()
993 		if (unlikely(p->flags & PF_KTHREAD))  in __oom_kill_process()
1012 	if (task->signal->oom_score_adj != OOM_SCORE_ADJ_MIN &&  in oom_kill_memcg_member()
1022 	struct task_struct *victim = oc->chosen;  in oom_kill_process()
1050 	oom_group = mem_cgroup_get_oom_group(victim, oc->memcg);  in oom_kill_process()
1079 		if (oc->constraint != CONSTRAINT_NONE)  in check_panic_on_oom()
1087 		sysctl_panic_on_oom == 2 ? "compulsory" : "system-wide");  in check_panic_on_oom()
1105  * out_of_memory - kill the "best" process when we run out of memory
1139 	 * The OOM killer does not compensate for IO-less reclaim.  in out_of_memory()
1143 	if (!(oc->gfp_mask & __GFP_FS) && !is_memcg_oom(oc))  in out_of_memory()
1150 	oc->constraint = constrained_alloc(oc);  in out_of_memory()
1151 	if (oc->constraint != CONSTRAINT_MEMORY_POLICY)  in out_of_memory()
1152 		oc->nodemask = NULL;  in out_of_memory()
1156 	    current->mm && !oom_unkillable_task(current) &&  in out_of_memory()
1158 	    current->signal->oom_score_adj != OOM_SCORE_ADJ_MIN) {  in out_of_memory()
1160 		oc->chosen = current;  in out_of_memory()
1167 	if (!oc->chosen) {  in out_of_memory()
1178 	if (oc->chosen && oc->chosen != (void *)-1UL)  in out_of_memory()
1181 	return !!oc->chosen;  in out_of_memory()
1216 		return -EINVAL;  in SYSCALL_DEFINE2()
1228 		ret = -ESRCH;  in SYSCALL_DEFINE2()
1232 	mm = p->mm;  in SYSCALL_DEFINE2()
1239 		if (!test_bit(MMF_OOM_SKIP, &mm->flags))  in SYSCALL_DEFINE2()
1240 			ret = -EINVAL;  in SYSCALL_DEFINE2()
1248 		ret = -EINTR;  in SYSCALL_DEFINE2()
1255 	if (!test_bit(MMF_OOM_SKIP, &mm->flags) && !__oom_reap_task_mm(mm))  in SYSCALL_DEFINE2()
1256 		ret = -EAGAIN;  in SYSCALL_DEFINE2()
1265 	return -ENOSYS;  in SYSCALL_DEFINE2()