xref: /openbmc/linux/kernel/fork.c (revision ef2c41cf38a7559bbf91af42d5b6a4429db8fc68)
1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds  *  linux/kernel/fork.c
41da177e4SLinus Torvalds  *
51da177e4SLinus Torvalds  *  Copyright (C) 1991, 1992  Linus Torvalds
61da177e4SLinus Torvalds  */
71da177e4SLinus Torvalds 
81da177e4SLinus Torvalds /*
91da177e4SLinus Torvalds  *  'fork.c' contains the help-routines for the 'fork' system call
101da177e4SLinus Torvalds  * (see also entry.S and others).
111da177e4SLinus Torvalds  * Fork is rather simple, once you get the hang of it, but the memory
121da177e4SLinus Torvalds  * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
131da177e4SLinus Torvalds  */
141da177e4SLinus Torvalds 
15b3e58382SChristian Brauner #include <linux/anon_inodes.h>
161da177e4SLinus Torvalds #include <linux/slab.h>
174eb5aaa3SIngo Molnar #include <linux/sched/autogroup.h>
186e84f315SIngo Molnar #include <linux/sched/mm.h>
19f7ccbae4SIngo Molnar #include <linux/sched/coredump.h>
208703e8a4SIngo Molnar #include <linux/sched/user.h>
216a3827d7SIngo Molnar #include <linux/sched/numa_balancing.h>
2203441a34SIngo Molnar #include <linux/sched/stat.h>
2329930025SIngo Molnar #include <linux/sched/task.h>
2468db0cf1SIngo Molnar #include <linux/sched/task_stack.h>
2532ef5517SIngo Molnar #include <linux/sched/cputime.h>
26b3e58382SChristian Brauner #include <linux/seq_file.h>
27037741a6SIngo Molnar #include <linux/rtmutex.h>
281da177e4SLinus Torvalds #include <linux/init.h>
291da177e4SLinus Torvalds #include <linux/unistd.h>
301da177e4SLinus Torvalds #include <linux/module.h>
311da177e4SLinus Torvalds #include <linux/vmalloc.h>
321da177e4SLinus Torvalds #include <linux/completion.h>
331da177e4SLinus Torvalds #include <linux/personality.h>
341da177e4SLinus Torvalds #include <linux/mempolicy.h>
351da177e4SLinus Torvalds #include <linux/sem.h>
361da177e4SLinus Torvalds #include <linux/file.h>
379f3acc31SAl Viro #include <linux/fdtable.h>
38da9cbc87SJens Axboe #include <linux/iocontext.h>
391da177e4SLinus Torvalds #include <linux/key.h>
401da177e4SLinus Torvalds #include <linux/binfmts.h>
411da177e4SLinus Torvalds #include <linux/mman.h>
42cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h>
431da177e4SLinus Torvalds #include <linux/fs.h>
44615d6e87SDavidlohr Bueso #include <linux/mm.h>
45615d6e87SDavidlohr Bueso #include <linux/vmacache.h>
46ab516013SSerge E. Hallyn #include <linux/nsproxy.h>
47c59ede7bSRandy.Dunlap #include <linux/capability.h>
481da177e4SLinus Torvalds #include <linux/cpu.h>
49b4f48b63SPaul Menage #include <linux/cgroup.h>
501da177e4SLinus Torvalds #include <linux/security.h>
51a1e78772SMel Gorman #include <linux/hugetlb.h>
52e2cfabdfSWill Drewry #include <linux/seccomp.h>
531da177e4SLinus Torvalds #include <linux/swap.h>
541da177e4SLinus Torvalds #include <linux/syscalls.h>
551da177e4SLinus Torvalds #include <linux/jiffies.h>
561da177e4SLinus Torvalds #include <linux/futex.h>
578141c7f3SLinus Torvalds #include <linux/compat.h>
58207205a2SEric Dumazet #include <linux/kthread.h>
597c3ab738SAndrew Morton #include <linux/task_io_accounting_ops.h>
60ab2af1f5SDipankar Sarma #include <linux/rcupdate.h>
611da177e4SLinus Torvalds #include <linux/ptrace.h>
621da177e4SLinus Torvalds #include <linux/mount.h>
631da177e4SLinus Torvalds #include <linux/audit.h>
6478fb7466SPavel Emelianov #include <linux/memcontrol.h>
65f201ae23SFrederic Weisbecker #include <linux/ftrace.h>
665e2bf014SMike Galbraith #include <linux/proc_fs.h>
671da177e4SLinus Torvalds #include <linux/profile.h>
681da177e4SLinus Torvalds #include <linux/rmap.h>
69f8af4da3SHugh Dickins #include <linux/ksm.h>
701da177e4SLinus Torvalds #include <linux/acct.h>
71893e26e6SPavel Emelyanov #include <linux/userfaultfd_k.h>
728f0ab514SJay Lan #include <linux/tsacct_kern.h>
739f46080cSMatt Helsley #include <linux/cn_proc.h>
74ba96a0c8SRafael J. Wysocki #include <linux/freezer.h>
75ca74e92bSShailabh Nagar #include <linux/delayacct.h>
76ad4ecbcbSShailabh Nagar #include <linux/taskstats_kern.h>
770a425405SArjan van de Ven #include <linux/random.h>
78522ed776SMiloslav Trmac #include <linux/tty.h>
79fd0928dfSJens Axboe #include <linux/blkdev.h>
805ad4e53bSAl Viro #include <linux/fs_struct.h>
817c9f8861SEric Sandeen #include <linux/magic.h>
82cdd6c482SIngo Molnar #include <linux/perf_event.h>
8342c4ab41SStanislaw Gruszka #include <linux/posix-timers.h>
848e7cac79SAvi Kivity #include <linux/user-return-notifier.h>
853d5992d2SYing Han #include <linux/oom.h>
86ba76149fSAndrea Arcangeli #include <linux/khugepaged.h>
87d80e731eSOleg Nesterov #include <linux/signalfd.h>
880326f5a9SSrikar Dronamraju #include <linux/uprobes.h>
89a27bb332SKent Overstreet #include <linux/aio.h>
9052f5684cSGideon Israel Dsouza #include <linux/compiler.h>
9116db3d3fSHeinrich Schuchardt #include <linux/sysctl.h>
925c9a8750SDmitry Vyukov #include <linux/kcov.h>
93d83a7cb3SJosh Poimboeuf #include <linux/livepatch.h>
9448ac3c18SMark Rutland #include <linux/thread_info.h>
95afaef01cSAlexander Popov #include <linux/stackleak.h>
96eafb149eSDaniel Axtens #include <linux/kasan.h>
971da177e4SLinus Torvalds 
981da177e4SLinus Torvalds #include <asm/pgtable.h>
991da177e4SLinus Torvalds #include <asm/pgalloc.h>
1007c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
1011da177e4SLinus Torvalds #include <asm/mmu_context.h>
1021da177e4SLinus Torvalds #include <asm/cacheflush.h>
1031da177e4SLinus Torvalds #include <asm/tlbflush.h>
1041da177e4SLinus Torvalds 
105ad8d75ffSSteven Rostedt #include <trace/events/sched.h>
106ad8d75ffSSteven Rostedt 
10743d2b113SKAMEZAWA Hiroyuki #define CREATE_TRACE_POINTS
10843d2b113SKAMEZAWA Hiroyuki #include <trace/events/task.h>
10943d2b113SKAMEZAWA Hiroyuki 
1101da177e4SLinus Torvalds /*
111ac1b398dSHeinrich Schuchardt  * Minimum number of threads to boot the kernel
112ac1b398dSHeinrich Schuchardt  */
113ac1b398dSHeinrich Schuchardt #define MIN_THREADS 20
114ac1b398dSHeinrich Schuchardt 
115ac1b398dSHeinrich Schuchardt /*
116ac1b398dSHeinrich Schuchardt  * Maximum number of threads
117ac1b398dSHeinrich Schuchardt  */
118ac1b398dSHeinrich Schuchardt #define MAX_THREADS FUTEX_TID_MASK
119ac1b398dSHeinrich Schuchardt 
120ac1b398dSHeinrich Schuchardt /*
1211da177e4SLinus Torvalds  * Protected counters by write_lock_irq(&tasklist_lock)
1221da177e4SLinus Torvalds  */
1231da177e4SLinus Torvalds unsigned long total_forks;	/* Handle normal Linux uptimes. */
1241da177e4SLinus Torvalds int nr_threads;			/* The idle threads do not count.. */
1251da177e4SLinus Torvalds 
1268856ae4dSKefeng Wang static int max_threads;		/* tunable limit on nr_threads */
1271da177e4SLinus Torvalds 
1288495f7e6SSai Praneeth Prakhya #define NAMED_ARRAY_INDEX(x)	[x] = __stringify(x)
1298495f7e6SSai Praneeth Prakhya 
1308495f7e6SSai Praneeth Prakhya static const char * const resident_page_types[] = {
1318495f7e6SSai Praneeth Prakhya 	NAMED_ARRAY_INDEX(MM_FILEPAGES),
1328495f7e6SSai Praneeth Prakhya 	NAMED_ARRAY_INDEX(MM_ANONPAGES),
1338495f7e6SSai Praneeth Prakhya 	NAMED_ARRAY_INDEX(MM_SWAPENTS),
1348495f7e6SSai Praneeth Prakhya 	NAMED_ARRAY_INDEX(MM_SHMEMPAGES),
1358495f7e6SSai Praneeth Prakhya };
1368495f7e6SSai Praneeth Prakhya 
1371da177e4SLinus Torvalds DEFINE_PER_CPU(unsigned long, process_counts) = 0;
1381da177e4SLinus Torvalds 
1391da177e4SLinus Torvalds __cacheline_aligned DEFINE_RWLOCK(tasklist_lock);  /* outer */
140db1466b3SPaul E. McKenney 
141db1466b3SPaul E. McKenney #ifdef CONFIG_PROVE_RCU
142db1466b3SPaul E. McKenney int lockdep_tasklist_lock_is_held(void)
143db1466b3SPaul E. McKenney {
144db1466b3SPaul E. McKenney 	return lockdep_is_held(&tasklist_lock);
145db1466b3SPaul E. McKenney }
146db1466b3SPaul E. McKenney EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
147db1466b3SPaul E. McKenney #endif /* #ifdef CONFIG_PROVE_RCU */
1481da177e4SLinus Torvalds 
1491da177e4SLinus Torvalds int nr_processes(void)
1501da177e4SLinus Torvalds {
1511da177e4SLinus Torvalds 	int cpu;
1521da177e4SLinus Torvalds 	int total = 0;
1531da177e4SLinus Torvalds 
1541d510750SIan Campbell 	for_each_possible_cpu(cpu)
1551da177e4SLinus Torvalds 		total += per_cpu(process_counts, cpu);
1561da177e4SLinus Torvalds 
1571da177e4SLinus Torvalds 	return total;
1581da177e4SLinus Torvalds }
1591da177e4SLinus Torvalds 
160f19b9f74SAkinobu Mita void __weak arch_release_task_struct(struct task_struct *tsk)
161f19b9f74SAkinobu Mita {
162f19b9f74SAkinobu Mita }
163f19b9f74SAkinobu Mita 
164f5e10287SThomas Gleixner #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
165e18b890bSChristoph Lameter static struct kmem_cache *task_struct_cachep;
16641101809SThomas Gleixner 
16741101809SThomas Gleixner static inline struct task_struct *alloc_task_struct_node(int node)
16841101809SThomas Gleixner {
16941101809SThomas Gleixner 	return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
17041101809SThomas Gleixner }
17141101809SThomas Gleixner 
17241101809SThomas Gleixner static inline void free_task_struct(struct task_struct *tsk)
17341101809SThomas Gleixner {
17441101809SThomas Gleixner 	kmem_cache_free(task_struct_cachep, tsk);
17541101809SThomas Gleixner }
1761da177e4SLinus Torvalds #endif
1771da177e4SLinus Torvalds 
178b235beeaSLinus Torvalds #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR
17941101809SThomas Gleixner 
1800d15d74aSThomas Gleixner /*
1810d15d74aSThomas Gleixner  * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
1820d15d74aSThomas Gleixner  * kmemcache based allocator.
1830d15d74aSThomas Gleixner  */
184ba14a194SAndy Lutomirski # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
185ac496bf4SAndy Lutomirski 
186ac496bf4SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
187ac496bf4SAndy Lutomirski /*
188ac496bf4SAndy Lutomirski  * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
189ac496bf4SAndy Lutomirski  * flush.  Try to minimize the number of calls by caching stacks.
190ac496bf4SAndy Lutomirski  */
191ac496bf4SAndy Lutomirski #define NR_CACHED_STACKS 2
192ac496bf4SAndy Lutomirski static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
19319659c59SHoeun Ryu 
19419659c59SHoeun Ryu static int free_vm_stack_cache(unsigned int cpu)
19519659c59SHoeun Ryu {
19619659c59SHoeun Ryu 	struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu);
19719659c59SHoeun Ryu 	int i;
19819659c59SHoeun Ryu 
19919659c59SHoeun Ryu 	for (i = 0; i < NR_CACHED_STACKS; i++) {
20019659c59SHoeun Ryu 		struct vm_struct *vm_stack = cached_vm_stacks[i];
20119659c59SHoeun Ryu 
20219659c59SHoeun Ryu 		if (!vm_stack)
20319659c59SHoeun Ryu 			continue;
20419659c59SHoeun Ryu 
20519659c59SHoeun Ryu 		vfree(vm_stack->addr);
20619659c59SHoeun Ryu 		cached_vm_stacks[i] = NULL;
20719659c59SHoeun Ryu 	}
20819659c59SHoeun Ryu 
20919659c59SHoeun Ryu 	return 0;
21019659c59SHoeun Ryu }
211ac496bf4SAndy Lutomirski #endif
212ac496bf4SAndy Lutomirski 
213ba14a194SAndy Lutomirski static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
214b69c49b7SFUJITA Tomonori {
215ba14a194SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
216ac496bf4SAndy Lutomirski 	void *stack;
217ac496bf4SAndy Lutomirski 	int i;
218ac496bf4SAndy Lutomirski 
219ac496bf4SAndy Lutomirski 	for (i = 0; i < NR_CACHED_STACKS; i++) {
220112166f8SChristoph Lameter 		struct vm_struct *s;
221112166f8SChristoph Lameter 
222112166f8SChristoph Lameter 		s = this_cpu_xchg(cached_stacks[i], NULL);
223ac496bf4SAndy Lutomirski 
224ac496bf4SAndy Lutomirski 		if (!s)
225ac496bf4SAndy Lutomirski 			continue;
226ac496bf4SAndy Lutomirski 
227eafb149eSDaniel Axtens 		/* Clear the KASAN shadow of the stack. */
228eafb149eSDaniel Axtens 		kasan_unpoison_shadow(s->addr, THREAD_SIZE);
229eafb149eSDaniel Axtens 
230ca182551SKonstantin Khlebnikov 		/* Clear stale pointers from reused stack. */
231ca182551SKonstantin Khlebnikov 		memset(s->addr, 0, THREAD_SIZE);
232e01e8063SKees Cook 
233ac496bf4SAndy Lutomirski 		tsk->stack_vm_area = s;
234ba4a4574SShakeel Butt 		tsk->stack = s->addr;
235ac496bf4SAndy Lutomirski 		return s->addr;
236ac496bf4SAndy Lutomirski 	}
237ac496bf4SAndy Lutomirski 
2389b6f7e16SRoman Gushchin 	/*
2399b6f7e16SRoman Gushchin 	 * Allocated stacks are cached and later reused by new threads,
2409b6f7e16SRoman Gushchin 	 * so memcg accounting is performed manually on assigning/releasing
2419b6f7e16SRoman Gushchin 	 * stacks to tasks. Drop __GFP_ACCOUNT.
2429b6f7e16SRoman Gushchin 	 */
24348ac3c18SMark Rutland 	stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
244ba14a194SAndy Lutomirski 				     VMALLOC_START, VMALLOC_END,
2459b6f7e16SRoman Gushchin 				     THREADINFO_GFP & ~__GFP_ACCOUNT,
246ba14a194SAndy Lutomirski 				     PAGE_KERNEL,
247ac496bf4SAndy Lutomirski 				     0, node, __builtin_return_address(0));
248ba14a194SAndy Lutomirski 
249ba14a194SAndy Lutomirski 	/*
250ba14a194SAndy Lutomirski 	 * We can't call find_vm_area() in interrupt context, and
251ba14a194SAndy Lutomirski 	 * free_thread_stack() can be called in interrupt context,
252ba14a194SAndy Lutomirski 	 * so cache the vm_struct.
253ba14a194SAndy Lutomirski 	 */
2545eed6f1dSRik van Riel 	if (stack) {
255ba14a194SAndy Lutomirski 		tsk->stack_vm_area = find_vm_area(stack);
2565eed6f1dSRik van Riel 		tsk->stack = stack;
2575eed6f1dSRik van Riel 	}
258ba14a194SAndy Lutomirski 	return stack;
259ba14a194SAndy Lutomirski #else
2604949148aSVladimir Davydov 	struct page *page = alloc_pages_node(node, THREADINFO_GFP,
2612889f608SThomas Gleixner 					     THREAD_SIZE_ORDER);
262b6a84016SEric Dumazet 
2631bf4580eSAndrea Arcangeli 	if (likely(page)) {
2641bf4580eSAndrea Arcangeli 		tsk->stack = page_address(page);
2651bf4580eSAndrea Arcangeli 		return tsk->stack;
2661bf4580eSAndrea Arcangeli 	}
2671bf4580eSAndrea Arcangeli 	return NULL;
268ba14a194SAndy Lutomirski #endif
269b69c49b7SFUJITA Tomonori }
270b69c49b7SFUJITA Tomonori 
271ba14a194SAndy Lutomirski static inline void free_thread_stack(struct task_struct *tsk)
272b69c49b7SFUJITA Tomonori {
273ac496bf4SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
2749b6f7e16SRoman Gushchin 	struct vm_struct *vm = task_stack_vm_area(tsk);
2759b6f7e16SRoman Gushchin 
2769b6f7e16SRoman Gushchin 	if (vm) {
277ac496bf4SAndy Lutomirski 		int i;
278ac496bf4SAndy Lutomirski 
2799b6f7e16SRoman Gushchin 		for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
2809b6f7e16SRoman Gushchin 			mod_memcg_page_state(vm->pages[i],
2819b6f7e16SRoman Gushchin 					     MEMCG_KERNEL_STACK_KB,
2829b6f7e16SRoman Gushchin 					     -(int)(PAGE_SIZE / 1024));
2839b6f7e16SRoman Gushchin 
2849b6f7e16SRoman Gushchin 			memcg_kmem_uncharge(vm->pages[i], 0);
2859b6f7e16SRoman Gushchin 		}
2869b6f7e16SRoman Gushchin 
287ac496bf4SAndy Lutomirski 		for (i = 0; i < NR_CACHED_STACKS; i++) {
288112166f8SChristoph Lameter 			if (this_cpu_cmpxchg(cached_stacks[i],
289112166f8SChristoph Lameter 					NULL, tsk->stack_vm_area) != NULL)
290ac496bf4SAndy Lutomirski 				continue;
291ac496bf4SAndy Lutomirski 
292ac496bf4SAndy Lutomirski 			return;
293ac496bf4SAndy Lutomirski 		}
294ac496bf4SAndy Lutomirski 
2950f110a9bSAndrey Ryabinin 		vfree_atomic(tsk->stack);
296ac496bf4SAndy Lutomirski 		return;
297ac496bf4SAndy Lutomirski 	}
298ac496bf4SAndy Lutomirski #endif
299ac496bf4SAndy Lutomirski 
300ba14a194SAndy Lutomirski 	__free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
301b69c49b7SFUJITA Tomonori }
3020d15d74aSThomas Gleixner # else
303b235beeaSLinus Torvalds static struct kmem_cache *thread_stack_cache;
3040d15d74aSThomas Gleixner 
3059521d399SMichael Ellerman static unsigned long *alloc_thread_stack_node(struct task_struct *tsk,
3060d15d74aSThomas Gleixner 						  int node)
3070d15d74aSThomas Gleixner {
3085eed6f1dSRik van Riel 	unsigned long *stack;
3095eed6f1dSRik van Riel 	stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
3105eed6f1dSRik van Riel 	tsk->stack = stack;
3115eed6f1dSRik van Riel 	return stack;
3120d15d74aSThomas Gleixner }
3130d15d74aSThomas Gleixner 
314ba14a194SAndy Lutomirski static void free_thread_stack(struct task_struct *tsk)
3150d15d74aSThomas Gleixner {
316ba14a194SAndy Lutomirski 	kmem_cache_free(thread_stack_cache, tsk->stack);
3170d15d74aSThomas Gleixner }
3180d15d74aSThomas Gleixner 
319b235beeaSLinus Torvalds void thread_stack_cache_init(void)
3200d15d74aSThomas Gleixner {
321f9d29946SDavid Windsor 	thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
322f9d29946SDavid Windsor 					THREAD_SIZE, THREAD_SIZE, 0, 0,
323f9d29946SDavid Windsor 					THREAD_SIZE, NULL);
324b235beeaSLinus Torvalds 	BUG_ON(thread_stack_cache == NULL);
3250d15d74aSThomas Gleixner }
3260d15d74aSThomas Gleixner # endif
327b69c49b7SFUJITA Tomonori #endif
328b69c49b7SFUJITA Tomonori 
3291da177e4SLinus Torvalds /* SLAB cache for signal_struct structures (tsk->signal) */
330e18b890bSChristoph Lameter static struct kmem_cache *signal_cachep;
3311da177e4SLinus Torvalds 
3321da177e4SLinus Torvalds /* SLAB cache for sighand_struct structures (tsk->sighand) */
333e18b890bSChristoph Lameter struct kmem_cache *sighand_cachep;
3341da177e4SLinus Torvalds 
3351da177e4SLinus Torvalds /* SLAB cache for files_struct structures (tsk->files) */
336e18b890bSChristoph Lameter struct kmem_cache *files_cachep;
3371da177e4SLinus Torvalds 
3381da177e4SLinus Torvalds /* SLAB cache for fs_struct structures (tsk->fs) */
339e18b890bSChristoph Lameter struct kmem_cache *fs_cachep;
3401da177e4SLinus Torvalds 
3411da177e4SLinus Torvalds /* SLAB cache for vm_area_struct structures */
3423928d4f5SLinus Torvalds static struct kmem_cache *vm_area_cachep;
3431da177e4SLinus Torvalds 
3441da177e4SLinus Torvalds /* SLAB cache for mm_struct structures (tsk->mm) */
345e18b890bSChristoph Lameter static struct kmem_cache *mm_cachep;
3461da177e4SLinus Torvalds 
347490fc053SLinus Torvalds struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
3483928d4f5SLinus Torvalds {
349a670468fSAndrew Morton 	struct vm_area_struct *vma;
350490fc053SLinus Torvalds 
351a670468fSAndrew Morton 	vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
352027232daSKirill A. Shutemov 	if (vma)
353027232daSKirill A. Shutemov 		vma_init(vma, mm);
354490fc053SLinus Torvalds 	return vma;
3553928d4f5SLinus Torvalds }
3563928d4f5SLinus Torvalds 
3573928d4f5SLinus Torvalds struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
3583928d4f5SLinus Torvalds {
35995faf699SLinus Torvalds 	struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
36095faf699SLinus Torvalds 
36195faf699SLinus Torvalds 	if (new) {
36295faf699SLinus Torvalds 		*new = *orig;
36395faf699SLinus Torvalds 		INIT_LIST_HEAD(&new->anon_vma_chain);
36495faf699SLinus Torvalds 	}
36595faf699SLinus Torvalds 	return new;
3663928d4f5SLinus Torvalds }
3673928d4f5SLinus Torvalds 
3683928d4f5SLinus Torvalds void vm_area_free(struct vm_area_struct *vma)
3693928d4f5SLinus Torvalds {
3703928d4f5SLinus Torvalds 	kmem_cache_free(vm_area_cachep, vma);
3713928d4f5SLinus Torvalds }
3723928d4f5SLinus Torvalds 
373ba14a194SAndy Lutomirski static void account_kernel_stack(struct task_struct *tsk, int account)
374c6a7f572SKOSAKI Motohiro {
375ba14a194SAndy Lutomirski 	void *stack = task_stack_page(tsk);
376ba14a194SAndy Lutomirski 	struct vm_struct *vm = task_stack_vm_area(tsk);
377ba14a194SAndy Lutomirski 
378ba14a194SAndy Lutomirski 	BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0);
379ba14a194SAndy Lutomirski 
380ba14a194SAndy Lutomirski 	if (vm) {
381ba14a194SAndy Lutomirski 		int i;
382ba14a194SAndy Lutomirski 
383ba14a194SAndy Lutomirski 		BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
384ba14a194SAndy Lutomirski 
385ba14a194SAndy Lutomirski 		for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
386ba14a194SAndy Lutomirski 			mod_zone_page_state(page_zone(vm->pages[i]),
387ba14a194SAndy Lutomirski 					    NR_KERNEL_STACK_KB,
388ba14a194SAndy Lutomirski 					    PAGE_SIZE / 1024 * account);
389ba14a194SAndy Lutomirski 		}
390ba14a194SAndy Lutomirski 	} else {
391ba14a194SAndy Lutomirski 		/*
392ba14a194SAndy Lutomirski 		 * All stack pages are in the same zone and belong to the
393ba14a194SAndy Lutomirski 		 * same memcg.
394ba14a194SAndy Lutomirski 		 */
395efdc9490SAndy Lutomirski 		struct page *first_page = virt_to_page(stack);
396c6a7f572SKOSAKI Motohiro 
397efdc9490SAndy Lutomirski 		mod_zone_page_state(page_zone(first_page), NR_KERNEL_STACK_KB,
398d30dd8beSAndy Lutomirski 				    THREAD_SIZE / 1024 * account);
399efdc9490SAndy Lutomirski 
400ed52be7bSJohannes Weiner 		mod_memcg_page_state(first_page, MEMCG_KERNEL_STACK_KB,
401efdc9490SAndy Lutomirski 				     account * (THREAD_SIZE / 1024));
402c6a7f572SKOSAKI Motohiro 	}
403ba14a194SAndy Lutomirski }
404c6a7f572SKOSAKI Motohiro 
4059b6f7e16SRoman Gushchin static int memcg_charge_kernel_stack(struct task_struct *tsk)
4069b6f7e16SRoman Gushchin {
4079b6f7e16SRoman Gushchin #ifdef CONFIG_VMAP_STACK
4089b6f7e16SRoman Gushchin 	struct vm_struct *vm = task_stack_vm_area(tsk);
4099b6f7e16SRoman Gushchin 	int ret;
4109b6f7e16SRoman Gushchin 
4119b6f7e16SRoman Gushchin 	if (vm) {
4129b6f7e16SRoman Gushchin 		int i;
4139b6f7e16SRoman Gushchin 
4149b6f7e16SRoman Gushchin 		for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
4159b6f7e16SRoman Gushchin 			/*
4169b6f7e16SRoman Gushchin 			 * If memcg_kmem_charge() fails, page->mem_cgroup
4179b6f7e16SRoman Gushchin 			 * pointer is NULL, and both memcg_kmem_uncharge()
4189b6f7e16SRoman Gushchin 			 * and mod_memcg_page_state() in free_thread_stack()
4199b6f7e16SRoman Gushchin 			 * will ignore this page. So it's safe.
4209b6f7e16SRoman Gushchin 			 */
4219b6f7e16SRoman Gushchin 			ret = memcg_kmem_charge(vm->pages[i], GFP_KERNEL, 0);
4229b6f7e16SRoman Gushchin 			if (ret)
4239b6f7e16SRoman Gushchin 				return ret;
4249b6f7e16SRoman Gushchin 
4259b6f7e16SRoman Gushchin 			mod_memcg_page_state(vm->pages[i],
4269b6f7e16SRoman Gushchin 					     MEMCG_KERNEL_STACK_KB,
4279b6f7e16SRoman Gushchin 					     PAGE_SIZE / 1024);
4289b6f7e16SRoman Gushchin 		}
4299b6f7e16SRoman Gushchin 	}
4309b6f7e16SRoman Gushchin #endif
4319b6f7e16SRoman Gushchin 	return 0;
4329b6f7e16SRoman Gushchin }
4339b6f7e16SRoman Gushchin 
43468f24b08SAndy Lutomirski static void release_task_stack(struct task_struct *tsk)
4351da177e4SLinus Torvalds {
436405c0759SAndy Lutomirski 	if (WARN_ON(tsk->state != TASK_DEAD))
437405c0759SAndy Lutomirski 		return;  /* Better to leak the stack than to free prematurely */
438405c0759SAndy Lutomirski 
439ba14a194SAndy Lutomirski 	account_kernel_stack(tsk, -1);
440ba14a194SAndy Lutomirski 	free_thread_stack(tsk);
44168f24b08SAndy Lutomirski 	tsk->stack = NULL;
44268f24b08SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
44368f24b08SAndy Lutomirski 	tsk->stack_vm_area = NULL;
44468f24b08SAndy Lutomirski #endif
44568f24b08SAndy Lutomirski }
44668f24b08SAndy Lutomirski 
44768f24b08SAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
44868f24b08SAndy Lutomirski void put_task_stack(struct task_struct *tsk)
44968f24b08SAndy Lutomirski {
450f0b89d39SElena Reshetova 	if (refcount_dec_and_test(&tsk->stack_refcount))
45168f24b08SAndy Lutomirski 		release_task_stack(tsk);
45268f24b08SAndy Lutomirski }
45368f24b08SAndy Lutomirski #endif
45468f24b08SAndy Lutomirski 
45568f24b08SAndy Lutomirski void free_task(struct task_struct *tsk)
45668f24b08SAndy Lutomirski {
45768f24b08SAndy Lutomirski #ifndef CONFIG_THREAD_INFO_IN_TASK
45868f24b08SAndy Lutomirski 	/*
45968f24b08SAndy Lutomirski 	 * The task is finally done with both the stack and thread_info,
46068f24b08SAndy Lutomirski 	 * so free both.
46168f24b08SAndy Lutomirski 	 */
46268f24b08SAndy Lutomirski 	release_task_stack(tsk);
46368f24b08SAndy Lutomirski #else
46468f24b08SAndy Lutomirski 	/*
46568f24b08SAndy Lutomirski 	 * If the task had a separate stack allocation, it should be gone
46668f24b08SAndy Lutomirski 	 * by now.
46768f24b08SAndy Lutomirski 	 */
468f0b89d39SElena Reshetova 	WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
46968f24b08SAndy Lutomirski #endif
47023f78d4aSIngo Molnar 	rt_mutex_debug_task_free(tsk);
471fb52607aSFrederic Weisbecker 	ftrace_graph_exit_task(tsk);
472e2cfabdfSWill Drewry 	put_seccomp_filter(tsk);
473f19b9f74SAkinobu Mita 	arch_release_task_struct(tsk);
4741da5c46fSOleg Nesterov 	if (tsk->flags & PF_KTHREAD)
4751da5c46fSOleg Nesterov 		free_kthread_struct(tsk);
4761da177e4SLinus Torvalds 	free_task_struct(tsk);
4771da177e4SLinus Torvalds }
4781da177e4SLinus Torvalds EXPORT_SYMBOL(free_task);
4791da177e4SLinus Torvalds 
4801da177e4SLinus Torvalds #ifdef CONFIG_MMU
4810766f788SEmese Revfy static __latent_entropy int dup_mmap(struct mm_struct *mm,
4820766f788SEmese Revfy 					struct mm_struct *oldmm)
4831da177e4SLinus Torvalds {
484297c5eeeSLinus Torvalds 	struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
4851da177e4SLinus Torvalds 	struct rb_node **rb_link, *rb_parent;
4861da177e4SLinus Torvalds 	int retval;
4871da177e4SLinus Torvalds 	unsigned long charge;
488893e26e6SPavel Emelyanov 	LIST_HEAD(uf);
4891da177e4SLinus Torvalds 
49032cdba1eSOleg Nesterov 	uprobe_start_dup_mmap();
4917c051267SMichal Hocko 	if (down_write_killable(&oldmm->mmap_sem)) {
4927c051267SMichal Hocko 		retval = -EINTR;
4937c051267SMichal Hocko 		goto fail_uprobe_end;
4947c051267SMichal Hocko 	}
495ec8c0446SRalf Baechle 	flush_cache_dup_mm(oldmm);
496f8ac4ec9SOleg Nesterov 	uprobe_dup_mmap(oldmm, mm);
497ad339451SIngo Molnar 	/*
498ad339451SIngo Molnar 	 * Not linked in yet - no deadlock potential:
499ad339451SIngo Molnar 	 */
500ad339451SIngo Molnar 	down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
5017ee78232SHugh Dickins 
50290f31d0eSKonstantin Khlebnikov 	/* No ordering required: file already has been exposed. */
50390f31d0eSKonstantin Khlebnikov 	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
50490f31d0eSKonstantin Khlebnikov 
5054f7d4614SVladimir Davydov 	mm->total_vm = oldmm->total_vm;
50684638335SKonstantin Khlebnikov 	mm->data_vm = oldmm->data_vm;
5074f7d4614SVladimir Davydov 	mm->exec_vm = oldmm->exec_vm;
5084f7d4614SVladimir Davydov 	mm->stack_vm = oldmm->stack_vm;
5094f7d4614SVladimir Davydov 
5101da177e4SLinus Torvalds 	rb_link = &mm->mm_rb.rb_node;
5111da177e4SLinus Torvalds 	rb_parent = NULL;
5121da177e4SLinus Torvalds 	pprev = &mm->mmap;
513f8af4da3SHugh Dickins 	retval = ksm_fork(mm, oldmm);
514f8af4da3SHugh Dickins 	if (retval)
515f8af4da3SHugh Dickins 		goto out;
516ba76149fSAndrea Arcangeli 	retval = khugepaged_fork(mm, oldmm);
517ba76149fSAndrea Arcangeli 	if (retval)
518ba76149fSAndrea Arcangeli 		goto out;
5191da177e4SLinus Torvalds 
520297c5eeeSLinus Torvalds 	prev = NULL;
521fd3e42fcSHugh Dickins 	for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
5221da177e4SLinus Torvalds 		struct file *file;
5231da177e4SLinus Torvalds 
5241da177e4SLinus Torvalds 		if (mpnt->vm_flags & VM_DONTCOPY) {
52584638335SKonstantin Khlebnikov 			vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
5261da177e4SLinus Torvalds 			continue;
5271da177e4SLinus Torvalds 		}
5281da177e4SLinus Torvalds 		charge = 0;
529655c79bbSTetsuo Handa 		/*
530655c79bbSTetsuo Handa 		 * Don't duplicate many vmas if we've been oom-killed (for
531655c79bbSTetsuo Handa 		 * example)
532655c79bbSTetsuo Handa 		 */
533655c79bbSTetsuo Handa 		if (fatal_signal_pending(current)) {
534655c79bbSTetsuo Handa 			retval = -EINTR;
535655c79bbSTetsuo Handa 			goto out;
536655c79bbSTetsuo Handa 		}
5371da177e4SLinus Torvalds 		if (mpnt->vm_flags & VM_ACCOUNT) {
538b2412b7fSHuang Shijie 			unsigned long len = vma_pages(mpnt);
539b2412b7fSHuang Shijie 
540191c5424SAl Viro 			if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
5411da177e4SLinus Torvalds 				goto fail_nomem;
5421da177e4SLinus Torvalds 			charge = len;
5431da177e4SLinus Torvalds 		}
5443928d4f5SLinus Torvalds 		tmp = vm_area_dup(mpnt);
5451da177e4SLinus Torvalds 		if (!tmp)
5461da177e4SLinus Torvalds 			goto fail_nomem;
547ef0855d3SOleg Nesterov 		retval = vma_dup_policy(mpnt, tmp);
548ef0855d3SOleg Nesterov 		if (retval)
5491da177e4SLinus Torvalds 			goto fail_nomem_policy;
550a247c3a9SAndrea Arcangeli 		tmp->vm_mm = mm;
551893e26e6SPavel Emelyanov 		retval = dup_userfaultfd(tmp, &uf);
552893e26e6SPavel Emelyanov 		if (retval)
553893e26e6SPavel Emelyanov 			goto fail_nomem_anon_vma_fork;
554d2cd9edeSRik van Riel 		if (tmp->vm_flags & VM_WIPEONFORK) {
555d2cd9edeSRik van Riel 			/* VM_WIPEONFORK gets a clean slate in the child. */
556d2cd9edeSRik van Riel 			tmp->anon_vma = NULL;
557d2cd9edeSRik van Riel 			if (anon_vma_prepare(tmp))
558d2cd9edeSRik van Riel 				goto fail_nomem_anon_vma_fork;
559d2cd9edeSRik van Riel 		} else if (anon_vma_fork(tmp, mpnt))
5605beb4930SRik van Riel 			goto fail_nomem_anon_vma_fork;
561893e26e6SPavel Emelyanov 		tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
562297c5eeeSLinus Torvalds 		tmp->vm_next = tmp->vm_prev = NULL;
5631da177e4SLinus Torvalds 		file = tmp->vm_file;
5641da177e4SLinus Torvalds 		if (file) {
565496ad9aaSAl Viro 			struct inode *inode = file_inode(file);
566b88ed205SHugh Dickins 			struct address_space *mapping = file->f_mapping;
567b88ed205SHugh Dickins 
5681da177e4SLinus Torvalds 			get_file(file);
5691da177e4SLinus Torvalds 			if (tmp->vm_flags & VM_DENYWRITE)
5701da177e4SLinus Torvalds 				atomic_dec(&inode->i_writecount);
57183cde9e8SDavidlohr Bueso 			i_mmap_lock_write(mapping);
572b88ed205SHugh Dickins 			if (tmp->vm_flags & VM_SHARED)
5734bb5f5d9SDavid Herrmann 				atomic_inc(&mapping->i_mmap_writable);
574b88ed205SHugh Dickins 			flush_dcache_mmap_lock(mapping);
575b88ed205SHugh Dickins 			/* insert tmp into the share list, just after mpnt */
5769826a516SMichel Lespinasse 			vma_interval_tree_insert_after(tmp, mpnt,
5779826a516SMichel Lespinasse 					&mapping->i_mmap);
578b88ed205SHugh Dickins 			flush_dcache_mmap_unlock(mapping);
57983cde9e8SDavidlohr Bueso 			i_mmap_unlock_write(mapping);
5801da177e4SLinus Torvalds 		}
5811da177e4SLinus Torvalds 
5821da177e4SLinus Torvalds 		/*
583a1e78772SMel Gorman 		 * Clear hugetlb-related page reserves for children. This only
584a1e78772SMel Gorman 		 * affects MAP_PRIVATE mappings. Faults generated by the child
585a1e78772SMel Gorman 		 * are not guaranteed to succeed, even if read-only
586a1e78772SMel Gorman 		 */
587a1e78772SMel Gorman 		if (is_vm_hugetlb_page(tmp))
588a1e78772SMel Gorman 			reset_vma_resv_huge_pages(tmp);
589a1e78772SMel Gorman 
590a1e78772SMel Gorman 		/*
5917ee78232SHugh Dickins 		 * Link in the new vma and copy the page table entries.
5921da177e4SLinus Torvalds 		 */
5931da177e4SLinus Torvalds 		*pprev = tmp;
5941da177e4SLinus Torvalds 		pprev = &tmp->vm_next;
595297c5eeeSLinus Torvalds 		tmp->vm_prev = prev;
596297c5eeeSLinus Torvalds 		prev = tmp;
5971da177e4SLinus Torvalds 
5981da177e4SLinus Torvalds 		__vma_link_rb(mm, tmp, rb_link, rb_parent);
5991da177e4SLinus Torvalds 		rb_link = &tmp->vm_rb.rb_right;
6001da177e4SLinus Torvalds 		rb_parent = &tmp->vm_rb;
6011da177e4SLinus Torvalds 
6021da177e4SLinus Torvalds 		mm->map_count++;
603d2cd9edeSRik van Riel 		if (!(tmp->vm_flags & VM_WIPEONFORK))
6040b0db14cSHugh Dickins 			retval = copy_page_range(mm, oldmm, mpnt);
6051da177e4SLinus Torvalds 
6061da177e4SLinus Torvalds 		if (tmp->vm_ops && tmp->vm_ops->open)
6071da177e4SLinus Torvalds 			tmp->vm_ops->open(tmp);
6081da177e4SLinus Torvalds 
6091da177e4SLinus Torvalds 		if (retval)
6101da177e4SLinus Torvalds 			goto out;
6111da177e4SLinus Torvalds 	}
612d6dd61c8SJeremy Fitzhardinge 	/* a new mm has just been created */
6131ed0cc5aSNadav Amit 	retval = arch_dup_mmap(oldmm, mm);
6141da177e4SLinus Torvalds out:
6157ee78232SHugh Dickins 	up_write(&mm->mmap_sem);
616fd3e42fcSHugh Dickins 	flush_tlb_mm(oldmm);
6171da177e4SLinus Torvalds 	up_write(&oldmm->mmap_sem);
618893e26e6SPavel Emelyanov 	dup_userfaultfd_complete(&uf);
6197c051267SMichal Hocko fail_uprobe_end:
62032cdba1eSOleg Nesterov 	uprobe_end_dup_mmap();
6211da177e4SLinus Torvalds 	return retval;
6225beb4930SRik van Riel fail_nomem_anon_vma_fork:
623ef0855d3SOleg Nesterov 	mpol_put(vma_policy(tmp));
6241da177e4SLinus Torvalds fail_nomem_policy:
6253928d4f5SLinus Torvalds 	vm_area_free(tmp);
6261da177e4SLinus Torvalds fail_nomem:
6271da177e4SLinus Torvalds 	retval = -ENOMEM;
6281da177e4SLinus Torvalds 	vm_unacct_memory(charge);
6291da177e4SLinus Torvalds 	goto out;
6301da177e4SLinus Torvalds }
6311da177e4SLinus Torvalds 
6321da177e4SLinus Torvalds static inline int mm_alloc_pgd(struct mm_struct *mm)
6331da177e4SLinus Torvalds {
6341da177e4SLinus Torvalds 	mm->pgd = pgd_alloc(mm);
6351da177e4SLinus Torvalds 	if (unlikely(!mm->pgd))
6361da177e4SLinus Torvalds 		return -ENOMEM;
6371da177e4SLinus Torvalds 	return 0;
6381da177e4SLinus Torvalds }
6391da177e4SLinus Torvalds 
6401da177e4SLinus Torvalds static inline void mm_free_pgd(struct mm_struct *mm)
6411da177e4SLinus Torvalds {
6425e541973SBenjamin Herrenschmidt 	pgd_free(mm, mm->pgd);
6431da177e4SLinus Torvalds }
6441da177e4SLinus Torvalds #else
64590f31d0eSKonstantin Khlebnikov static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
64690f31d0eSKonstantin Khlebnikov {
64790f31d0eSKonstantin Khlebnikov 	down_write(&oldmm->mmap_sem);
64890f31d0eSKonstantin Khlebnikov 	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
64990f31d0eSKonstantin Khlebnikov 	up_write(&oldmm->mmap_sem);
65090f31d0eSKonstantin Khlebnikov 	return 0;
65190f31d0eSKonstantin Khlebnikov }
6521da177e4SLinus Torvalds #define mm_alloc_pgd(mm)	(0)
6531da177e4SLinus Torvalds #define mm_free_pgd(mm)
6541da177e4SLinus Torvalds #endif /* CONFIG_MMU */
6551da177e4SLinus Torvalds 
656d70f2a14SAndrew Morton static void check_mm(struct mm_struct *mm)
657d70f2a14SAndrew Morton {
658d70f2a14SAndrew Morton 	int i;
659d70f2a14SAndrew Morton 
6608495f7e6SSai Praneeth Prakhya 	BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
6618495f7e6SSai Praneeth Prakhya 			 "Please make sure 'struct resident_page_types[]' is updated as well");
6628495f7e6SSai Praneeth Prakhya 
663d70f2a14SAndrew Morton 	for (i = 0; i < NR_MM_COUNTERS; i++) {
664d70f2a14SAndrew Morton 		long x = atomic_long_read(&mm->rss_stat.count[i]);
665d70f2a14SAndrew Morton 
666d70f2a14SAndrew Morton 		if (unlikely(x))
6678495f7e6SSai Praneeth Prakhya 			pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n",
6688495f7e6SSai Praneeth Prakhya 				 mm, resident_page_types[i], x);
669d70f2a14SAndrew Morton 	}
670d70f2a14SAndrew Morton 
671d70f2a14SAndrew Morton 	if (mm_pgtables_bytes(mm))
672d70f2a14SAndrew Morton 		pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
673d70f2a14SAndrew Morton 				mm_pgtables_bytes(mm));
674d70f2a14SAndrew Morton 
675d70f2a14SAndrew Morton #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
676d70f2a14SAndrew Morton 	VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
677d70f2a14SAndrew Morton #endif
678d70f2a14SAndrew Morton }
6791da177e4SLinus Torvalds 
680e94b1766SChristoph Lameter #define allocate_mm()	(kmem_cache_alloc(mm_cachep, GFP_KERNEL))
6811da177e4SLinus Torvalds #define free_mm(mm)	(kmem_cache_free(mm_cachep, (mm)))
6821da177e4SLinus Torvalds 
683d70f2a14SAndrew Morton /*
684d70f2a14SAndrew Morton  * Called when the last reference to the mm
685d70f2a14SAndrew Morton  * is dropped: either by a lazy thread or by
686d70f2a14SAndrew Morton  * mmput. Free the page directory and the mm.
687d70f2a14SAndrew Morton  */
688d34bc48fSAndrew Morton void __mmdrop(struct mm_struct *mm)
689d70f2a14SAndrew Morton {
690d70f2a14SAndrew Morton 	BUG_ON(mm == &init_mm);
6913eda69c9SMark Rutland 	WARN_ON_ONCE(mm == current->mm);
6923eda69c9SMark Rutland 	WARN_ON_ONCE(mm == current->active_mm);
693d70f2a14SAndrew Morton 	mm_free_pgd(mm);
694d70f2a14SAndrew Morton 	destroy_context(mm);
695984cfe4eSJason Gunthorpe 	mmu_notifier_subscriptions_destroy(mm);
696d70f2a14SAndrew Morton 	check_mm(mm);
697d70f2a14SAndrew Morton 	put_user_ns(mm->user_ns);
698d70f2a14SAndrew Morton 	free_mm(mm);
699d70f2a14SAndrew Morton }
700d34bc48fSAndrew Morton EXPORT_SYMBOL_GPL(__mmdrop);
701d70f2a14SAndrew Morton 
702d70f2a14SAndrew Morton static void mmdrop_async_fn(struct work_struct *work)
703d70f2a14SAndrew Morton {
704d70f2a14SAndrew Morton 	struct mm_struct *mm;
705d70f2a14SAndrew Morton 
706d70f2a14SAndrew Morton 	mm = container_of(work, struct mm_struct, async_put_work);
707d70f2a14SAndrew Morton 	__mmdrop(mm);
708d70f2a14SAndrew Morton }
709d70f2a14SAndrew Morton 
710d70f2a14SAndrew Morton static void mmdrop_async(struct mm_struct *mm)
711d70f2a14SAndrew Morton {
712d70f2a14SAndrew Morton 	if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
713d70f2a14SAndrew Morton 		INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
714d70f2a14SAndrew Morton 		schedule_work(&mm->async_put_work);
715d70f2a14SAndrew Morton 	}
716d70f2a14SAndrew Morton }
717d70f2a14SAndrew Morton 
7181da177e4SLinus Torvalds static inline void free_signal_struct(struct signal_struct *sig)
7191da177e4SLinus Torvalds {
7201da177e4SLinus Torvalds 	taskstats_tgid_free(sig);
7211da177e4SLinus Torvalds 	sched_autogroup_exit(sig);
7221da177e4SLinus Torvalds 	/*
7231da177e4SLinus Torvalds 	 * __mmdrop is not safe to call from softirq context on x86 due to
7241da177e4SLinus Torvalds 	 * pgd_dtor so postpone it to the async context
7251da177e4SLinus Torvalds 	 */
7261da177e4SLinus Torvalds 	if (sig->oom_mm)
7271da177e4SLinus Torvalds 		mmdrop_async(sig->oom_mm);
7281da177e4SLinus Torvalds 	kmem_cache_free(signal_cachep, sig);
7291da177e4SLinus Torvalds }
7301da177e4SLinus Torvalds 
7311da177e4SLinus Torvalds static inline void put_signal_struct(struct signal_struct *sig)
7321da177e4SLinus Torvalds {
73360d4de3fSElena Reshetova 	if (refcount_dec_and_test(&sig->sigcnt))
7341da177e4SLinus Torvalds 		free_signal_struct(sig);
7351da177e4SLinus Torvalds }
7361da177e4SLinus Torvalds 
7371da177e4SLinus Torvalds void __put_task_struct(struct task_struct *tsk)
7381da177e4SLinus Torvalds {
7391da177e4SLinus Torvalds 	WARN_ON(!tsk->exit_state);
740ec1d2819SElena Reshetova 	WARN_ON(refcount_read(&tsk->usage));
7411da177e4SLinus Torvalds 	WARN_ON(tsk == current);
7421da177e4SLinus Torvalds 
7431da177e4SLinus Torvalds 	cgroup_free(tsk);
74416d51a59SJann Horn 	task_numa_free(tsk, true);
7451da177e4SLinus Torvalds 	security_task_free(tsk);
7461da177e4SLinus Torvalds 	exit_creds(tsk);
7471da177e4SLinus Torvalds 	delayacct_tsk_free(tsk);
7481da177e4SLinus Torvalds 	put_signal_struct(tsk->signal);
7491da177e4SLinus Torvalds 
7501da177e4SLinus Torvalds 	if (!profile_handoff_task(tsk))
7511da177e4SLinus Torvalds 		free_task(tsk);
7521da177e4SLinus Torvalds }
7531da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(__put_task_struct);
7541da177e4SLinus Torvalds 
7551da177e4SLinus Torvalds void __init __weak arch_task_cache_init(void) { }
7561da177e4SLinus Torvalds 
7571da177e4SLinus Torvalds /*
7581da177e4SLinus Torvalds  * set_max_threads
7591da177e4SLinus Torvalds  */
7601da177e4SLinus Torvalds static void set_max_threads(unsigned int max_threads_suggested)
7611da177e4SLinus Torvalds {
7621da177e4SLinus Torvalds 	u64 threads;
763ca79b0c2SArun KS 	unsigned long nr_pages = totalram_pages();
7641da177e4SLinus Torvalds 
7651da177e4SLinus Torvalds 	/*
7661da177e4SLinus Torvalds 	 * The number of threads shall be limited such that the thread
7671da177e4SLinus Torvalds 	 * structures may only consume a small part of the available memory.
7681da177e4SLinus Torvalds 	 */
7693d6357deSArun KS 	if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)
7701da177e4SLinus Torvalds 		threads = MAX_THREADS;
7711da177e4SLinus Torvalds 	else
7723d6357deSArun KS 		threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,
7731da177e4SLinus Torvalds 				    (u64) THREAD_SIZE * 8UL);
7741da177e4SLinus Torvalds 
7751da177e4SLinus Torvalds 	if (threads > max_threads_suggested)
7761da177e4SLinus Torvalds 		threads = max_threads_suggested;
7771da177e4SLinus Torvalds 
7781da177e4SLinus Torvalds 	max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
7791da177e4SLinus Torvalds }
7801da177e4SLinus Torvalds 
7811da177e4SLinus Torvalds #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
7821da177e4SLinus Torvalds /* Initialized by the architecture: */
7831da177e4SLinus Torvalds int arch_task_struct_size __read_mostly;
7841da177e4SLinus Torvalds #endif
7851da177e4SLinus Torvalds 
7864189ff23SChristoph Hellwig #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
7875905429aSKees Cook static void task_struct_whitelist(unsigned long *offset, unsigned long *size)
7885905429aSKees Cook {
7895905429aSKees Cook 	/* Fetch thread_struct whitelist for the architecture. */
7905905429aSKees Cook 	arch_thread_struct_whitelist(offset, size);
7915905429aSKees Cook 
7925905429aSKees Cook 	/*
7935905429aSKees Cook 	 * Handle zero-sized whitelist or empty thread_struct, otherwise
7945905429aSKees Cook 	 * adjust offset to position of thread_struct in task_struct.
7955905429aSKees Cook 	 */
7965905429aSKees Cook 	if (unlikely(*size == 0))
7975905429aSKees Cook 		*offset = 0;
7985905429aSKees Cook 	else
7995905429aSKees Cook 		*offset += offsetof(struct task_struct, thread);
8005905429aSKees Cook }
8014189ff23SChristoph Hellwig #endif /* CONFIG_ARCH_TASK_STRUCT_ALLOCATOR */
8025905429aSKees Cook 
8031da177e4SLinus Torvalds void __init fork_init(void)
8041da177e4SLinus Torvalds {
8051da177e4SLinus Torvalds 	int i;
8061da177e4SLinus Torvalds #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
8071da177e4SLinus Torvalds #ifndef ARCH_MIN_TASKALIGN
8081da177e4SLinus Torvalds #define ARCH_MIN_TASKALIGN	0
8091da177e4SLinus Torvalds #endif
8101da177e4SLinus Torvalds 	int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
8115905429aSKees Cook 	unsigned long useroffset, usersize;
8121da177e4SLinus Torvalds 
8131da177e4SLinus Torvalds 	/* create a slab on which task_structs can be allocated */
8145905429aSKees Cook 	task_struct_whitelist(&useroffset, &usersize);
8155905429aSKees Cook 	task_struct_cachep = kmem_cache_create_usercopy("task_struct",
8161da177e4SLinus Torvalds 			arch_task_struct_size, align,
8175905429aSKees Cook 			SLAB_PANIC|SLAB_ACCOUNT,
8185905429aSKees Cook 			useroffset, usersize, NULL);
8191da177e4SLinus Torvalds #endif
8201da177e4SLinus Torvalds 
8211da177e4SLinus Torvalds 	/* do the arch specific task caches init */
8221da177e4SLinus Torvalds 	arch_task_cache_init();
8231da177e4SLinus Torvalds 
8241da177e4SLinus Torvalds 	set_max_threads(MAX_THREADS);
8251da177e4SLinus Torvalds 
8261da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
8271da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
8281da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_SIGPENDING] =
8291da177e4SLinus Torvalds 		init_task.signal->rlim[RLIMIT_NPROC];
8301da177e4SLinus Torvalds 
8311da177e4SLinus Torvalds 	for (i = 0; i < UCOUNT_COUNTS; i++) {
8321da177e4SLinus Torvalds 		init_user_ns.ucount_max[i] = max_threads/2;
8331da177e4SLinus Torvalds 	}
8341da177e4SLinus Torvalds 
8351da177e4SLinus Torvalds #ifdef CONFIG_VMAP_STACK
8361da177e4SLinus Torvalds 	cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
8371da177e4SLinus Torvalds 			  NULL, free_vm_stack_cache);
8381da177e4SLinus Torvalds #endif
8391da177e4SLinus Torvalds 
8401da177e4SLinus Torvalds 	lockdep_init_task(&init_task);
841aad42dd4SNadav Amit 	uprobes_init();
8421da177e4SLinus Torvalds }
8431da177e4SLinus Torvalds 
8441da177e4SLinus Torvalds int __weak arch_dup_task_struct(struct task_struct *dst,
8451da177e4SLinus Torvalds 					       struct task_struct *src)
8461da177e4SLinus Torvalds {
8471da177e4SLinus Torvalds 	*dst = *src;
8481da177e4SLinus Torvalds 	return 0;
8491da177e4SLinus Torvalds }
8501da177e4SLinus Torvalds 
8511da177e4SLinus Torvalds void set_task_stack_end_magic(struct task_struct *tsk)
8521da177e4SLinus Torvalds {
8531da177e4SLinus Torvalds 	unsigned long *stackend;
8541da177e4SLinus Torvalds 
8551da177e4SLinus Torvalds 	stackend = end_of_stack(tsk);
8561da177e4SLinus Torvalds 	*stackend = STACK_END_MAGIC;	/* for overflow detection */
8571da177e4SLinus Torvalds }
8581da177e4SLinus Torvalds 
8591da177e4SLinus Torvalds static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
8601da177e4SLinus Torvalds {
8611da177e4SLinus Torvalds 	struct task_struct *tsk;
8621da177e4SLinus Torvalds 	unsigned long *stack;
8630f4991e8SYueHaibing 	struct vm_struct *stack_vm_area __maybe_unused;
8641da177e4SLinus Torvalds 	int err;
8651da177e4SLinus Torvalds 
8661da177e4SLinus Torvalds 	if (node == NUMA_NO_NODE)
8671da177e4SLinus Torvalds 		node = tsk_fork_get_node(orig);
8681da177e4SLinus Torvalds 	tsk = alloc_task_struct_node(node);
8691da177e4SLinus Torvalds 	if (!tsk)
8701da177e4SLinus Torvalds 		return NULL;
8711da177e4SLinus Torvalds 
8721da177e4SLinus Torvalds 	stack = alloc_thread_stack_node(tsk, node);
8731da177e4SLinus Torvalds 	if (!stack)
8741da177e4SLinus Torvalds 		goto free_tsk;
8751da177e4SLinus Torvalds 
8769b6f7e16SRoman Gushchin 	if (memcg_charge_kernel_stack(tsk))
8779b6f7e16SRoman Gushchin 		goto free_stack;
8789b6f7e16SRoman Gushchin 
8791da177e4SLinus Torvalds 	stack_vm_area = task_stack_vm_area(tsk);
8801da177e4SLinus Torvalds 
8811da177e4SLinus Torvalds 	err = arch_dup_task_struct(tsk, orig);
8821da177e4SLinus Torvalds 
8831da177e4SLinus Torvalds 	/*
8841da177e4SLinus Torvalds 	 * arch_dup_task_struct() clobbers the stack-related fields.  Make
8851da177e4SLinus Torvalds 	 * sure they're properly initialized before using any stack-related
8861da177e4SLinus Torvalds 	 * functions again.
8871da177e4SLinus Torvalds 	 */
8881da177e4SLinus Torvalds 	tsk->stack = stack;
8891da177e4SLinus Torvalds #ifdef CONFIG_VMAP_STACK
8901da177e4SLinus Torvalds 	tsk->stack_vm_area = stack_vm_area;
8911da177e4SLinus Torvalds #endif
8921da177e4SLinus Torvalds #ifdef CONFIG_THREAD_INFO_IN_TASK
893f0b89d39SElena Reshetova 	refcount_set(&tsk->stack_refcount, 1);
8941da177e4SLinus Torvalds #endif
8951da177e4SLinus Torvalds 
8961da177e4SLinus Torvalds 	if (err)
8971da177e4SLinus Torvalds 		goto free_stack;
8981da177e4SLinus Torvalds 
8991da177e4SLinus Torvalds #ifdef CONFIG_SECCOMP
9001da177e4SLinus Torvalds 	/*
9011da177e4SLinus Torvalds 	 * We must handle setting up seccomp filters once we're under
9021da177e4SLinus Torvalds 	 * the sighand lock in case orig has changed between now and
9031da177e4SLinus Torvalds 	 * then. Until then, filter must be NULL to avoid messing up
9041da177e4SLinus Torvalds 	 * the usage counts on the error path calling free_task.
9051da177e4SLinus Torvalds 	 */
9061da177e4SLinus Torvalds 	tsk->seccomp.filter = NULL;
9071da177e4SLinus Torvalds #endif
9081da177e4SLinus Torvalds 
9091da177e4SLinus Torvalds 	setup_thread_stack(tsk, orig);
9101da177e4SLinus Torvalds 	clear_user_return_notifier(tsk);
9111da177e4SLinus Torvalds 	clear_tsk_need_resched(tsk);
9121da177e4SLinus Torvalds 	set_task_stack_end_magic(tsk);
9131da177e4SLinus Torvalds 
914050e9baaSLinus Torvalds #ifdef CONFIG_STACKPROTECTOR
9151da177e4SLinus Torvalds 	tsk->stack_canary = get_random_canary();
9161da177e4SLinus Torvalds #endif
9173bd37062SSebastian Andrzej Siewior 	if (orig->cpus_ptr == &orig->cpus_mask)
9183bd37062SSebastian Andrzej Siewior 		tsk->cpus_ptr = &tsk->cpus_mask;
9191da177e4SLinus Torvalds 
9201da177e4SLinus Torvalds 	/*
9210ff7b2cfSEric W. Biederman 	 * One for the user space visible state that goes away when reaped.
9220ff7b2cfSEric W. Biederman 	 * One for the scheduler.
9231da177e4SLinus Torvalds 	 */
9240ff7b2cfSEric W. Biederman 	refcount_set(&tsk->rcu_users, 2);
9250ff7b2cfSEric W. Biederman 	/* One for the rcu users */
9260ff7b2cfSEric W. Biederman 	refcount_set(&tsk->usage, 1);
9271da177e4SLinus Torvalds #ifdef CONFIG_BLK_DEV_IO_TRACE
9281da177e4SLinus Torvalds 	tsk->btrace_seq = 0;
9291da177e4SLinus Torvalds #endif
9301da177e4SLinus Torvalds 	tsk->splice_pipe = NULL;
9311da177e4SLinus Torvalds 	tsk->task_frag.page = NULL;
9321da177e4SLinus Torvalds 	tsk->wake_q.next = NULL;
9331da177e4SLinus Torvalds 
9341da177e4SLinus Torvalds 	account_kernel_stack(tsk, 1);
9351da177e4SLinus Torvalds 
9361da177e4SLinus Torvalds 	kcov_task_init(tsk);
9371da177e4SLinus Torvalds 
9381da177e4SLinus Torvalds #ifdef CONFIG_FAULT_INJECTION
9391da177e4SLinus Torvalds 	tsk->fail_nth = 0;
9401da177e4SLinus Torvalds #endif
9411da177e4SLinus Torvalds 
9422c323017SJosef Bacik #ifdef CONFIG_BLK_CGROUP
9432c323017SJosef Bacik 	tsk->throttle_queue = NULL;
9442c323017SJosef Bacik 	tsk->use_memdelay = 0;
9452c323017SJosef Bacik #endif
9462c323017SJosef Bacik 
947d46eb14bSShakeel Butt #ifdef CONFIG_MEMCG
948d46eb14bSShakeel Butt 	tsk->active_memcg = NULL;
949d46eb14bSShakeel Butt #endif
9501da177e4SLinus Torvalds 	return tsk;
9511da177e4SLinus Torvalds 
9521da177e4SLinus Torvalds free_stack:
9531da177e4SLinus Torvalds 	free_thread_stack(tsk);
9541da177e4SLinus Torvalds free_tsk:
9551da177e4SLinus Torvalds 	free_task_struct(tsk);
9561da177e4SLinus Torvalds 	return NULL;
9571da177e4SLinus Torvalds }
9581da177e4SLinus Torvalds 
9591da177e4SLinus Torvalds __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
9601da177e4SLinus Torvalds 
9614cb0e11bSHidehiro Kawai static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
9624cb0e11bSHidehiro Kawai 
9634cb0e11bSHidehiro Kawai static int __init coredump_filter_setup(char *s)
9644cb0e11bSHidehiro Kawai {
9654cb0e11bSHidehiro Kawai 	default_dump_filter =
9664cb0e11bSHidehiro Kawai 		(simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
9674cb0e11bSHidehiro Kawai 		MMF_DUMP_FILTER_MASK;
9684cb0e11bSHidehiro Kawai 	return 1;
9694cb0e11bSHidehiro Kawai }
9704cb0e11bSHidehiro Kawai 
9714cb0e11bSHidehiro Kawai __setup("coredump_filter=", coredump_filter_setup);
9724cb0e11bSHidehiro Kawai 
9731da177e4SLinus Torvalds #include <linux/init_task.h>
9741da177e4SLinus Torvalds 
975858f0993SAlexey Dobriyan static void mm_init_aio(struct mm_struct *mm)
976858f0993SAlexey Dobriyan {
977858f0993SAlexey Dobriyan #ifdef CONFIG_AIO
978858f0993SAlexey Dobriyan 	spin_lock_init(&mm->ioctx_lock);
979db446a08SBenjamin LaHaise 	mm->ioctx_table = NULL;
980858f0993SAlexey Dobriyan #endif
981858f0993SAlexey Dobriyan }
982858f0993SAlexey Dobriyan 
983c3f3ce04SAndrea Arcangeli static __always_inline void mm_clear_owner(struct mm_struct *mm,
984c3f3ce04SAndrea Arcangeli 					   struct task_struct *p)
985c3f3ce04SAndrea Arcangeli {
986c3f3ce04SAndrea Arcangeli #ifdef CONFIG_MEMCG
987c3f3ce04SAndrea Arcangeli 	if (mm->owner == p)
988c3f3ce04SAndrea Arcangeli 		WRITE_ONCE(mm->owner, NULL);
989c3f3ce04SAndrea Arcangeli #endif
990c3f3ce04SAndrea Arcangeli }
991c3f3ce04SAndrea Arcangeli 
99233144e84SVladimir Davydov static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
99333144e84SVladimir Davydov {
99433144e84SVladimir Davydov #ifdef CONFIG_MEMCG
99533144e84SVladimir Davydov 	mm->owner = p;
99633144e84SVladimir Davydov #endif
99733144e84SVladimir Davydov }
99833144e84SVladimir Davydov 
999355627f5SEric Biggers static void mm_init_uprobes_state(struct mm_struct *mm)
1000355627f5SEric Biggers {
1001355627f5SEric Biggers #ifdef CONFIG_UPROBES
1002355627f5SEric Biggers 	mm->uprobes_state.xol_area = NULL;
1003355627f5SEric Biggers #endif
1004355627f5SEric Biggers }
1005355627f5SEric Biggers 
1006bfedb589SEric W. Biederman static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
1007bfedb589SEric W. Biederman 	struct user_namespace *user_ns)
10081da177e4SLinus Torvalds {
100941f727fdSVladimir Davydov 	mm->mmap = NULL;
101041f727fdSVladimir Davydov 	mm->mm_rb = RB_ROOT;
101141f727fdSVladimir Davydov 	mm->vmacache_seqnum = 0;
10121da177e4SLinus Torvalds 	atomic_set(&mm->mm_users, 1);
10131da177e4SLinus Torvalds 	atomic_set(&mm->mm_count, 1);
10141da177e4SLinus Torvalds 	init_rwsem(&mm->mmap_sem);
10151da177e4SLinus Torvalds 	INIT_LIST_HEAD(&mm->mmlist);
1016999d9fc1SOleg Nesterov 	mm->core_state = NULL;
1017af5b0f6aSKirill A. Shutemov 	mm_pgtables_bytes_init(mm);
101841f727fdSVladimir Davydov 	mm->map_count = 0;
101941f727fdSVladimir Davydov 	mm->locked_vm = 0;
102070f8a3caSDavidlohr Bueso 	atomic64_set(&mm->pinned_vm, 0);
1021d559db08SKAMEZAWA Hiroyuki 	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
10221da177e4SLinus Torvalds 	spin_lock_init(&mm->page_table_lock);
102388aa7cc6SYang Shi 	spin_lock_init(&mm->arg_lock);
102441f727fdSVladimir Davydov 	mm_init_cpumask(mm);
1025858f0993SAlexey Dobriyan 	mm_init_aio(mm);
1026cf475ad2SBalbir Singh 	mm_init_owner(mm, p);
10272b7e8665SEric Biggers 	RCU_INIT_POINTER(mm->exe_file, NULL);
1028984cfe4eSJason Gunthorpe 	mmu_notifier_subscriptions_init(mm);
102916af97dcSNadav Amit 	init_tlb_flush_pending(mm);
103041f727fdSVladimir Davydov #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
103141f727fdSVladimir Davydov 	mm->pmd_huge_pte = NULL;
103241f727fdSVladimir Davydov #endif
1033355627f5SEric Biggers 	mm_init_uprobes_state(mm);
10341da177e4SLinus Torvalds 
1035a0715cc2SAlex Thorlton 	if (current->mm) {
1036a0715cc2SAlex Thorlton 		mm->flags = current->mm->flags & MMF_INIT_MASK;
1037a0715cc2SAlex Thorlton 		mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
1038a0715cc2SAlex Thorlton 	} else {
1039a0715cc2SAlex Thorlton 		mm->flags = default_dump_filter;
10401da177e4SLinus Torvalds 		mm->def_flags = 0;
1041a0715cc2SAlex Thorlton 	}
1042a0715cc2SAlex Thorlton 
104341f727fdSVladimir Davydov 	if (mm_alloc_pgd(mm))
104441f727fdSVladimir Davydov 		goto fail_nopgd;
104578fb7466SPavel Emelianov 
104641f727fdSVladimir Davydov 	if (init_new_context(p, mm))
104741f727fdSVladimir Davydov 		goto fail_nocontext;
104841f727fdSVladimir Davydov 
1049bfedb589SEric W. Biederman 	mm->user_ns = get_user_ns(user_ns);
105041f727fdSVladimir Davydov 	return mm;
105141f727fdSVladimir Davydov 
105241f727fdSVladimir Davydov fail_nocontext:
105341f727fdSVladimir Davydov 	mm_free_pgd(mm);
105441f727fdSVladimir Davydov fail_nopgd:
10551da177e4SLinus Torvalds 	free_mm(mm);
10561da177e4SLinus Torvalds 	return NULL;
10571da177e4SLinus Torvalds }
10581da177e4SLinus Torvalds 
10591da177e4SLinus Torvalds /*
10601da177e4SLinus Torvalds  * Allocate and initialize an mm_struct.
10611da177e4SLinus Torvalds  */
10621da177e4SLinus Torvalds struct mm_struct *mm_alloc(void)
10631da177e4SLinus Torvalds {
10641da177e4SLinus Torvalds 	struct mm_struct *mm;
10651da177e4SLinus Torvalds 
10661da177e4SLinus Torvalds 	mm = allocate_mm();
1067de03c72cSKOSAKI Motohiro 	if (!mm)
1068de03c72cSKOSAKI Motohiro 		return NULL;
1069de03c72cSKOSAKI Motohiro 
10701da177e4SLinus Torvalds 	memset(mm, 0, sizeof(*mm));
1071bfedb589SEric W. Biederman 	return mm_init(mm, current, current_user_ns());
10721da177e4SLinus Torvalds }
10731da177e4SLinus Torvalds 
1074ec8d7c14SMichal Hocko static inline void __mmput(struct mm_struct *mm)
10751da177e4SLinus Torvalds {
1076ec8d7c14SMichal Hocko 	VM_BUG_ON(atomic_read(&mm->mm_users));
10770ae26f1bSAndrew Morton 
1078d4b3b638SSrikar Dronamraju 	uprobe_clear_state(mm);
10791da177e4SLinus Torvalds 	exit_aio(mm);
10801c2fb7a4SAndrea Arcangeli 	ksm_exit(mm);
1081ba76149fSAndrea Arcangeli 	khugepaged_exit(mm); /* must run before exit_mmap */
10821da177e4SLinus Torvalds 	exit_mmap(mm);
10836fcb52a5SAaron Lu 	mm_put_huge_zero_page(mm);
1084925d1c40SMatt Helsley 	set_mm_exe_file(mm, NULL);
10851da177e4SLinus Torvalds 	if (!list_empty(&mm->mmlist)) {
10861da177e4SLinus Torvalds 		spin_lock(&mmlist_lock);
10871da177e4SLinus Torvalds 		list_del(&mm->mmlist);
10881da177e4SLinus Torvalds 		spin_unlock(&mmlist_lock);
10891da177e4SLinus Torvalds 	}
1090801460d0SHiroshi Shimamoto 	if (mm->binfmt)
1091801460d0SHiroshi Shimamoto 		module_put(mm->binfmt->module);
10921da177e4SLinus Torvalds 	mmdrop(mm);
10931da177e4SLinus Torvalds }
1094ec8d7c14SMichal Hocko 
1095ec8d7c14SMichal Hocko /*
1096ec8d7c14SMichal Hocko  * Decrement the use count and release all resources for an mm.
1097ec8d7c14SMichal Hocko  */
1098ec8d7c14SMichal Hocko void mmput(struct mm_struct *mm)
1099ec8d7c14SMichal Hocko {
1100ec8d7c14SMichal Hocko 	might_sleep();
1101ec8d7c14SMichal Hocko 
1102ec8d7c14SMichal Hocko 	if (atomic_dec_and_test(&mm->mm_users))
1103ec8d7c14SMichal Hocko 		__mmput(mm);
11041da177e4SLinus Torvalds }
11051da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(mmput);
11061da177e4SLinus Torvalds 
1107a1b2289cSSherry Yang #ifdef CONFIG_MMU
1108a1b2289cSSherry Yang static void mmput_async_fn(struct work_struct *work)
1109a1b2289cSSherry Yang {
1110a1b2289cSSherry Yang 	struct mm_struct *mm = container_of(work, struct mm_struct,
1111a1b2289cSSherry Yang 					    async_put_work);
1112a1b2289cSSherry Yang 
1113a1b2289cSSherry Yang 	__mmput(mm);
1114a1b2289cSSherry Yang }
1115a1b2289cSSherry Yang 
1116a1b2289cSSherry Yang void mmput_async(struct mm_struct *mm)
1117a1b2289cSSherry Yang {
1118a1b2289cSSherry Yang 	if (atomic_dec_and_test(&mm->mm_users)) {
1119a1b2289cSSherry Yang 		INIT_WORK(&mm->async_put_work, mmput_async_fn);
1120a1b2289cSSherry Yang 		schedule_work(&mm->async_put_work);
1121a1b2289cSSherry Yang 	}
1122a1b2289cSSherry Yang }
1123a1b2289cSSherry Yang #endif
1124a1b2289cSSherry Yang 
112590f31d0eSKonstantin Khlebnikov /**
112690f31d0eSKonstantin Khlebnikov  * set_mm_exe_file - change a reference to the mm's executable file
112790f31d0eSKonstantin Khlebnikov  *
112890f31d0eSKonstantin Khlebnikov  * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
112990f31d0eSKonstantin Khlebnikov  *
11306e399cd1SDavidlohr Bueso  * Main users are mmput() and sys_execve(). Callers prevent concurrent
11316e399cd1SDavidlohr Bueso  * invocations: in mmput() nobody alive left, in execve task is single
11326e399cd1SDavidlohr Bueso  * threaded. sys_prctl(PR_SET_MM_MAP/EXE_FILE) also needs to set the
11336e399cd1SDavidlohr Bueso  * mm->exe_file, but does so without using set_mm_exe_file() in order
11346e399cd1SDavidlohr Bueso  * to do avoid the need for any locks.
113590f31d0eSKonstantin Khlebnikov  */
113638646013SJiri Slaby void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
113738646013SJiri Slaby {
11386e399cd1SDavidlohr Bueso 	struct file *old_exe_file;
11396e399cd1SDavidlohr Bueso 
11406e399cd1SDavidlohr Bueso 	/*
11416e399cd1SDavidlohr Bueso 	 * It is safe to dereference the exe_file without RCU as
11426e399cd1SDavidlohr Bueso 	 * this function is only called if nobody else can access
11436e399cd1SDavidlohr Bueso 	 * this mm -- see comment above for justification.
11446e399cd1SDavidlohr Bueso 	 */
11456e399cd1SDavidlohr Bueso 	old_exe_file = rcu_dereference_raw(mm->exe_file);
114690f31d0eSKonstantin Khlebnikov 
114738646013SJiri Slaby 	if (new_exe_file)
114838646013SJiri Slaby 		get_file(new_exe_file);
114990f31d0eSKonstantin Khlebnikov 	rcu_assign_pointer(mm->exe_file, new_exe_file);
115090f31d0eSKonstantin Khlebnikov 	if (old_exe_file)
115190f31d0eSKonstantin Khlebnikov 		fput(old_exe_file);
115238646013SJiri Slaby }
115338646013SJiri Slaby 
115490f31d0eSKonstantin Khlebnikov /**
115590f31d0eSKonstantin Khlebnikov  * get_mm_exe_file - acquire a reference to the mm's executable file
115690f31d0eSKonstantin Khlebnikov  *
115790f31d0eSKonstantin Khlebnikov  * Returns %NULL if mm has no associated executable file.
115890f31d0eSKonstantin Khlebnikov  * User must release file via fput().
115990f31d0eSKonstantin Khlebnikov  */
116038646013SJiri Slaby struct file *get_mm_exe_file(struct mm_struct *mm)
116138646013SJiri Slaby {
116238646013SJiri Slaby 	struct file *exe_file;
116338646013SJiri Slaby 
116490f31d0eSKonstantin Khlebnikov 	rcu_read_lock();
116590f31d0eSKonstantin Khlebnikov 	exe_file = rcu_dereference(mm->exe_file);
116690f31d0eSKonstantin Khlebnikov 	if (exe_file && !get_file_rcu(exe_file))
116790f31d0eSKonstantin Khlebnikov 		exe_file = NULL;
116890f31d0eSKonstantin Khlebnikov 	rcu_read_unlock();
116938646013SJiri Slaby 	return exe_file;
117038646013SJiri Slaby }
117111163348SDavidlohr Bueso EXPORT_SYMBOL(get_mm_exe_file);
117238646013SJiri Slaby 
11731da177e4SLinus Torvalds /**
1174cd81a917SMateusz Guzik  * get_task_exe_file - acquire a reference to the task's executable file
1175cd81a917SMateusz Guzik  *
1176cd81a917SMateusz Guzik  * Returns %NULL if task's mm (if any) has no associated executable file or
1177cd81a917SMateusz Guzik  * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
1178cd81a917SMateusz Guzik  * User must release file via fput().
1179cd81a917SMateusz Guzik  */
1180cd81a917SMateusz Guzik struct file *get_task_exe_file(struct task_struct *task)
1181cd81a917SMateusz Guzik {
1182cd81a917SMateusz Guzik 	struct file *exe_file = NULL;
1183cd81a917SMateusz Guzik 	struct mm_struct *mm;
1184cd81a917SMateusz Guzik 
1185cd81a917SMateusz Guzik 	task_lock(task);
1186cd81a917SMateusz Guzik 	mm = task->mm;
1187cd81a917SMateusz Guzik 	if (mm) {
1188cd81a917SMateusz Guzik 		if (!(task->flags & PF_KTHREAD))
1189cd81a917SMateusz Guzik 			exe_file = get_mm_exe_file(mm);
1190cd81a917SMateusz Guzik 	}
1191cd81a917SMateusz Guzik 	task_unlock(task);
1192cd81a917SMateusz Guzik 	return exe_file;
1193cd81a917SMateusz Guzik }
1194cd81a917SMateusz Guzik EXPORT_SYMBOL(get_task_exe_file);
1195cd81a917SMateusz Guzik 
1196cd81a917SMateusz Guzik /**
11971da177e4SLinus Torvalds  * get_task_mm - acquire a reference to the task's mm
11981da177e4SLinus Torvalds  *
1199246bb0b1SOleg Nesterov  * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning
12001da177e4SLinus Torvalds  * this kernel workthread has transiently adopted a user mm with use_mm,
12011da177e4SLinus Torvalds  * to do its AIO) is not set and if so returns a reference to it, after
12021da177e4SLinus Torvalds  * bumping up the use count.  User must release the mm via mmput()
12031da177e4SLinus Torvalds  * after use.  Typically used by /proc and ptrace.
12041da177e4SLinus Torvalds  */
12051da177e4SLinus Torvalds struct mm_struct *get_task_mm(struct task_struct *task)
12061da177e4SLinus Torvalds {
12071da177e4SLinus Torvalds 	struct mm_struct *mm;
12081da177e4SLinus Torvalds 
12091da177e4SLinus Torvalds 	task_lock(task);
12101da177e4SLinus Torvalds 	mm = task->mm;
12111da177e4SLinus Torvalds 	if (mm) {
1212246bb0b1SOleg Nesterov 		if (task->flags & PF_KTHREAD)
12131da177e4SLinus Torvalds 			mm = NULL;
12141da177e4SLinus Torvalds 		else
12153fce371bSVegard Nossum 			mmget(mm);
12161da177e4SLinus Torvalds 	}
12171da177e4SLinus Torvalds 	task_unlock(task);
12181da177e4SLinus Torvalds 	return mm;
12191da177e4SLinus Torvalds }
12201da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(get_task_mm);
12211da177e4SLinus Torvalds 
12228cdb878dSChristopher Yeoh struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
12238cdb878dSChristopher Yeoh {
12248cdb878dSChristopher Yeoh 	struct mm_struct *mm;
12258cdb878dSChristopher Yeoh 	int err;
12268cdb878dSChristopher Yeoh 
12278cdb878dSChristopher Yeoh 	err =  mutex_lock_killable(&task->signal->cred_guard_mutex);
12288cdb878dSChristopher Yeoh 	if (err)
12298cdb878dSChristopher Yeoh 		return ERR_PTR(err);
12308cdb878dSChristopher Yeoh 
12318cdb878dSChristopher Yeoh 	mm = get_task_mm(task);
12328cdb878dSChristopher Yeoh 	if (mm && mm != current->mm &&
12338cdb878dSChristopher Yeoh 			!ptrace_may_access(task, mode)) {
12348cdb878dSChristopher Yeoh 		mmput(mm);
12358cdb878dSChristopher Yeoh 		mm = ERR_PTR(-EACCES);
12368cdb878dSChristopher Yeoh 	}
12378cdb878dSChristopher Yeoh 	mutex_unlock(&task->signal->cred_guard_mutex);
12388cdb878dSChristopher Yeoh 
12398cdb878dSChristopher Yeoh 	return mm;
12408cdb878dSChristopher Yeoh }
12418cdb878dSChristopher Yeoh 
124257b59c4aSOleg Nesterov static void complete_vfork_done(struct task_struct *tsk)
1243c415c3b4SOleg Nesterov {
1244d68b46feSOleg Nesterov 	struct completion *vfork;
1245c415c3b4SOleg Nesterov 
1246d68b46feSOleg Nesterov 	task_lock(tsk);
1247d68b46feSOleg Nesterov 	vfork = tsk->vfork_done;
1248d68b46feSOleg Nesterov 	if (likely(vfork)) {
1249c415c3b4SOleg Nesterov 		tsk->vfork_done = NULL;
1250d68b46feSOleg Nesterov 		complete(vfork);
1251d68b46feSOleg Nesterov 	}
1252d68b46feSOleg Nesterov 	task_unlock(tsk);
1253d68b46feSOleg Nesterov }
1254d68b46feSOleg Nesterov 
1255d68b46feSOleg Nesterov static int wait_for_vfork_done(struct task_struct *child,
1256d68b46feSOleg Nesterov 				struct completion *vfork)
1257d68b46feSOleg Nesterov {
1258d68b46feSOleg Nesterov 	int killed;
1259d68b46feSOleg Nesterov 
1260d68b46feSOleg Nesterov 	freezer_do_not_count();
126176f969e8SRoman Gushchin 	cgroup_enter_frozen();
1262d68b46feSOleg Nesterov 	killed = wait_for_completion_killable(vfork);
126376f969e8SRoman Gushchin 	cgroup_leave_frozen(false);
1264d68b46feSOleg Nesterov 	freezer_count();
1265d68b46feSOleg Nesterov 
1266d68b46feSOleg Nesterov 	if (killed) {
1267d68b46feSOleg Nesterov 		task_lock(child);
1268d68b46feSOleg Nesterov 		child->vfork_done = NULL;
1269d68b46feSOleg Nesterov 		task_unlock(child);
1270d68b46feSOleg Nesterov 	}
1271d68b46feSOleg Nesterov 
1272d68b46feSOleg Nesterov 	put_task_struct(child);
1273d68b46feSOleg Nesterov 	return killed;
1274c415c3b4SOleg Nesterov }
1275c415c3b4SOleg Nesterov 
12761da177e4SLinus Torvalds /* Please note the differences between mmput and mm_release.
12771da177e4SLinus Torvalds  * mmput is called whenever we stop holding onto a mm_struct,
12781da177e4SLinus Torvalds  * error success whatever.
12791da177e4SLinus Torvalds  *
12801da177e4SLinus Torvalds  * mm_release is called after a mm_struct has been removed
12811da177e4SLinus Torvalds  * from the current process.
12821da177e4SLinus Torvalds  *
12831da177e4SLinus Torvalds  * This difference is important for error handling, when we
12841da177e4SLinus Torvalds  * only half set up a mm_struct for a new process and need to restore
12851da177e4SLinus Torvalds  * the old one.  Because we mmput the new mm_struct before
12861da177e4SLinus Torvalds  * restoring the old one. . .
12871da177e4SLinus Torvalds  * Eric Biederman 10 January 1998
12881da177e4SLinus Torvalds  */
12894610ba7aSThomas Gleixner static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
12901da177e4SLinus Torvalds {
12910326f5a9SSrikar Dronamraju 	uprobe_free_utask(tsk);
12920326f5a9SSrikar Dronamraju 
12931da177e4SLinus Torvalds 	/* Get rid of any cached register state */
12941da177e4SLinus Torvalds 	deactivate_mm(tsk, mm);
12951da177e4SLinus Torvalds 
1296fec1d011SRoland McGrath 	/*
1297735f2770SMichal Hocko 	 * Signal userspace if we're not exiting with a core dump
1298735f2770SMichal Hocko 	 * because we want to leave the value intact for debugging
1299735f2770SMichal Hocko 	 * purposes.
1300fec1d011SRoland McGrath 	 */
13019c8a8228SEric Dumazet 	if (tsk->clear_child_tid) {
1302735f2770SMichal Hocko 		if (!(tsk->signal->flags & SIGNAL_GROUP_COREDUMP) &&
13039c8a8228SEric Dumazet 		    atomic_read(&mm->mm_users) > 1) {
13041da177e4SLinus Torvalds 			/*
13051da177e4SLinus Torvalds 			 * We don't check the error code - if userspace has
13061da177e4SLinus Torvalds 			 * not set up a proper pointer then tough luck.
13071da177e4SLinus Torvalds 			 */
13089c8a8228SEric Dumazet 			put_user(0, tsk->clear_child_tid);
13092de0db99SDominik Brodowski 			do_futex(tsk->clear_child_tid, FUTEX_WAKE,
13102de0db99SDominik Brodowski 					1, NULL, NULL, 0, 0);
13119c8a8228SEric Dumazet 		}
13129c8a8228SEric Dumazet 		tsk->clear_child_tid = NULL;
13131da177e4SLinus Torvalds 	}
1314f7505d64SKonstantin Khlebnikov 
1315f7505d64SKonstantin Khlebnikov 	/*
1316f7505d64SKonstantin Khlebnikov 	 * All done, finally we can wake up parent and return this mm to him.
1317f7505d64SKonstantin Khlebnikov 	 * Also kthread_stop() uses this completion for synchronization.
1318f7505d64SKonstantin Khlebnikov 	 */
1319f7505d64SKonstantin Khlebnikov 	if (tsk->vfork_done)
1320f7505d64SKonstantin Khlebnikov 		complete_vfork_done(tsk);
13211da177e4SLinus Torvalds }
13221da177e4SLinus Torvalds 
13234610ba7aSThomas Gleixner void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)
13244610ba7aSThomas Gleixner {
1325150d7158SThomas Gleixner 	futex_exit_release(tsk);
13264610ba7aSThomas Gleixner 	mm_release(tsk, mm);
13274610ba7aSThomas Gleixner }
13284610ba7aSThomas Gleixner 
13294610ba7aSThomas Gleixner void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)
13304610ba7aSThomas Gleixner {
1331150d7158SThomas Gleixner 	futex_exec_release(tsk);
13324610ba7aSThomas Gleixner 	mm_release(tsk, mm);
13334610ba7aSThomas Gleixner }
13344610ba7aSThomas Gleixner 
133513585fa0SNadav Amit /**
133613585fa0SNadav Amit  * dup_mm() - duplicates an existing mm structure
133713585fa0SNadav Amit  * @tsk: the task_struct with which the new mm will be associated.
133813585fa0SNadav Amit  * @oldmm: the mm to duplicate.
133913585fa0SNadav Amit  *
134013585fa0SNadav Amit  * Allocates a new mm structure and duplicates the provided @oldmm structure
134113585fa0SNadav Amit  * content into it.
134213585fa0SNadav Amit  *
134313585fa0SNadav Amit  * Return: the duplicated mm or NULL on failure.
1344a0a7ec30SJANAK DESAI  */
134513585fa0SNadav Amit static struct mm_struct *dup_mm(struct task_struct *tsk,
134613585fa0SNadav Amit 				struct mm_struct *oldmm)
1347a0a7ec30SJANAK DESAI {
134813585fa0SNadav Amit 	struct mm_struct *mm;
1349a0a7ec30SJANAK DESAI 	int err;
1350a0a7ec30SJANAK DESAI 
1351a0a7ec30SJANAK DESAI 	mm = allocate_mm();
1352a0a7ec30SJANAK DESAI 	if (!mm)
1353a0a7ec30SJANAK DESAI 		goto fail_nomem;
1354a0a7ec30SJANAK DESAI 
1355a0a7ec30SJANAK DESAI 	memcpy(mm, oldmm, sizeof(*mm));
1356a0a7ec30SJANAK DESAI 
1357bfedb589SEric W. Biederman 	if (!mm_init(mm, tsk, mm->user_ns))
1358a0a7ec30SJANAK DESAI 		goto fail_nomem;
1359a0a7ec30SJANAK DESAI 
1360a0a7ec30SJANAK DESAI 	err = dup_mmap(mm, oldmm);
1361a0a7ec30SJANAK DESAI 	if (err)
1362a0a7ec30SJANAK DESAI 		goto free_pt;
1363a0a7ec30SJANAK DESAI 
1364a0a7ec30SJANAK DESAI 	mm->hiwater_rss = get_mm_rss(mm);
1365a0a7ec30SJANAK DESAI 	mm->hiwater_vm = mm->total_vm;
1366a0a7ec30SJANAK DESAI 
1367801460d0SHiroshi Shimamoto 	if (mm->binfmt && !try_module_get(mm->binfmt->module))
1368801460d0SHiroshi Shimamoto 		goto free_pt;
1369801460d0SHiroshi Shimamoto 
1370a0a7ec30SJANAK DESAI 	return mm;
1371a0a7ec30SJANAK DESAI 
1372a0a7ec30SJANAK DESAI free_pt:
1373801460d0SHiroshi Shimamoto 	/* don't put binfmt in mmput, we haven't got module yet */
1374801460d0SHiroshi Shimamoto 	mm->binfmt = NULL;
1375c3f3ce04SAndrea Arcangeli 	mm_init_owner(mm, NULL);
1376a0a7ec30SJANAK DESAI 	mmput(mm);
1377a0a7ec30SJANAK DESAI 
1378a0a7ec30SJANAK DESAI fail_nomem:
1379a0a7ec30SJANAK DESAI 	return NULL;
1380a0a7ec30SJANAK DESAI }
1381a0a7ec30SJANAK DESAI 
13821da177e4SLinus Torvalds static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
13831da177e4SLinus Torvalds {
13841da177e4SLinus Torvalds 	struct mm_struct *mm, *oldmm;
13851da177e4SLinus Torvalds 	int retval;
13861da177e4SLinus Torvalds 
13871da177e4SLinus Torvalds 	tsk->min_flt = tsk->maj_flt = 0;
13881da177e4SLinus Torvalds 	tsk->nvcsw = tsk->nivcsw = 0;
138917406b82SMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK
139017406b82SMandeep Singh Baines 	tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
1391a2e51445SDmitry Vyukov 	tsk->last_switch_time = 0;
139217406b82SMandeep Singh Baines #endif
13931da177e4SLinus Torvalds 
13941da177e4SLinus Torvalds 	tsk->mm = NULL;
13951da177e4SLinus Torvalds 	tsk->active_mm = NULL;
13961da177e4SLinus Torvalds 
13971da177e4SLinus Torvalds 	/*
13981da177e4SLinus Torvalds 	 * Are we cloning a kernel thread?
13991da177e4SLinus Torvalds 	 *
14001da177e4SLinus Torvalds 	 * We need to steal a active VM for that..
14011da177e4SLinus Torvalds 	 */
14021da177e4SLinus Torvalds 	oldmm = current->mm;
14031da177e4SLinus Torvalds 	if (!oldmm)
14041da177e4SLinus Torvalds 		return 0;
14051da177e4SLinus Torvalds 
1406615d6e87SDavidlohr Bueso 	/* initialize the new vmacache entries */
1407615d6e87SDavidlohr Bueso 	vmacache_flush(tsk);
1408615d6e87SDavidlohr Bueso 
14091da177e4SLinus Torvalds 	if (clone_flags & CLONE_VM) {
14103fce371bSVegard Nossum 		mmget(oldmm);
14111da177e4SLinus Torvalds 		mm = oldmm;
14121da177e4SLinus Torvalds 		goto good_mm;
14131da177e4SLinus Torvalds 	}
14141da177e4SLinus Torvalds 
14151da177e4SLinus Torvalds 	retval = -ENOMEM;
141613585fa0SNadav Amit 	mm = dup_mm(tsk, current->mm);
14171da177e4SLinus Torvalds 	if (!mm)
14181da177e4SLinus Torvalds 		goto fail_nomem;
14191da177e4SLinus Torvalds 
14201da177e4SLinus Torvalds good_mm:
14211da177e4SLinus Torvalds 	tsk->mm = mm;
14221da177e4SLinus Torvalds 	tsk->active_mm = mm;
14231da177e4SLinus Torvalds 	return 0;
14241da177e4SLinus Torvalds 
14251da177e4SLinus Torvalds fail_nomem:
14261da177e4SLinus Torvalds 	return retval;
14271da177e4SLinus Torvalds }
14281da177e4SLinus Torvalds 
1429a39bc516SAlexey Dobriyan static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
14301da177e4SLinus Torvalds {
1431498052bbSAl Viro 	struct fs_struct *fs = current->fs;
14321da177e4SLinus Torvalds 	if (clone_flags & CLONE_FS) {
1433498052bbSAl Viro 		/* tsk->fs is already what we want */
14342a4419b5SNick Piggin 		spin_lock(&fs->lock);
1435498052bbSAl Viro 		if (fs->in_exec) {
14362a4419b5SNick Piggin 			spin_unlock(&fs->lock);
1437498052bbSAl Viro 			return -EAGAIN;
1438498052bbSAl Viro 		}
1439498052bbSAl Viro 		fs->users++;
14402a4419b5SNick Piggin 		spin_unlock(&fs->lock);
14411da177e4SLinus Torvalds 		return 0;
14421da177e4SLinus Torvalds 	}
1443498052bbSAl Viro 	tsk->fs = copy_fs_struct(fs);
14441da177e4SLinus Torvalds 	if (!tsk->fs)
14451da177e4SLinus Torvalds 		return -ENOMEM;
14461da177e4SLinus Torvalds 	return 0;
14471da177e4SLinus Torvalds }
14481da177e4SLinus Torvalds 
1449a016f338SJANAK DESAI static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
1450a016f338SJANAK DESAI {
1451a016f338SJANAK DESAI 	struct files_struct *oldf, *newf;
1452a016f338SJANAK DESAI 	int error = 0;
1453a016f338SJANAK DESAI 
1454a016f338SJANAK DESAI 	/*
1455a016f338SJANAK DESAI 	 * A background process may not have any files ...
1456a016f338SJANAK DESAI 	 */
1457a016f338SJANAK DESAI 	oldf = current->files;
1458a016f338SJANAK DESAI 	if (!oldf)
1459a016f338SJANAK DESAI 		goto out;
1460a016f338SJANAK DESAI 
1461a016f338SJANAK DESAI 	if (clone_flags & CLONE_FILES) {
1462a016f338SJANAK DESAI 		atomic_inc(&oldf->count);
1463a016f338SJANAK DESAI 		goto out;
1464a016f338SJANAK DESAI 	}
1465a016f338SJANAK DESAI 
1466a016f338SJANAK DESAI 	newf = dup_fd(oldf, &error);
1467a016f338SJANAK DESAI 	if (!newf)
1468a016f338SJANAK DESAI 		goto out;
1469a016f338SJANAK DESAI 
1470a016f338SJANAK DESAI 	tsk->files = newf;
1471a016f338SJANAK DESAI 	error = 0;
1472a016f338SJANAK DESAI out:
1473a016f338SJANAK DESAI 	return error;
1474a016f338SJANAK DESAI }
1475a016f338SJANAK DESAI 
1476fadad878SJens Axboe static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
1477fd0928dfSJens Axboe {
1478fd0928dfSJens Axboe #ifdef CONFIG_BLOCK
1479fd0928dfSJens Axboe 	struct io_context *ioc = current->io_context;
14806e736be7STejun Heo 	struct io_context *new_ioc;
1481fd0928dfSJens Axboe 
1482fd0928dfSJens Axboe 	if (!ioc)
1483fd0928dfSJens Axboe 		return 0;
1484fadad878SJens Axboe 	/*
1485fadad878SJens Axboe 	 * Share io context with parent, if CLONE_IO is set
1486fadad878SJens Axboe 	 */
1487fadad878SJens Axboe 	if (clone_flags & CLONE_IO) {
14883d48749dSTejun Heo 		ioc_task_link(ioc);
14893d48749dSTejun Heo 		tsk->io_context = ioc;
1490fadad878SJens Axboe 	} else if (ioprio_valid(ioc->ioprio)) {
14916e736be7STejun Heo 		new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
14926e736be7STejun Heo 		if (unlikely(!new_ioc))
1493fd0928dfSJens Axboe 			return -ENOMEM;
1494fd0928dfSJens Axboe 
14956e736be7STejun Heo 		new_ioc->ioprio = ioc->ioprio;
149611a3122fSTejun Heo 		put_io_context(new_ioc);
1497fd0928dfSJens Axboe 	}
1498fd0928dfSJens Axboe #endif
1499fd0928dfSJens Axboe 	return 0;
1500fd0928dfSJens Axboe }
1501fd0928dfSJens Axboe 
1502a39bc516SAlexey Dobriyan static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
15031da177e4SLinus Torvalds {
15041da177e4SLinus Torvalds 	struct sighand_struct *sig;
15051da177e4SLinus Torvalds 
150660348802SZhaolei 	if (clone_flags & CLONE_SIGHAND) {
1507d036bda7SElena Reshetova 		refcount_inc(&current->sighand->count);
15081da177e4SLinus Torvalds 		return 0;
15091da177e4SLinus Torvalds 	}
15101da177e4SLinus Torvalds 	sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
1511e56d0903SIngo Molnar 	rcu_assign_pointer(tsk->sighand, sig);
15121da177e4SLinus Torvalds 	if (!sig)
15131da177e4SLinus Torvalds 		return -ENOMEM;
15149d7fb042SPeter Zijlstra 
1515d036bda7SElena Reshetova 	refcount_set(&sig->count, 1);
151606e62a46SJann Horn 	spin_lock_irq(&current->sighand->siglock);
15171da177e4SLinus Torvalds 	memcpy(sig->action, current->sighand->action, sizeof(sig->action));
151806e62a46SJann Horn 	spin_unlock_irq(&current->sighand->siglock);
1519b612e5dfSChristian Brauner 
1520b612e5dfSChristian Brauner 	/* Reset all signal handler not set to SIG_IGN to SIG_DFL. */
1521b612e5dfSChristian Brauner 	if (clone_flags & CLONE_CLEAR_SIGHAND)
1522b612e5dfSChristian Brauner 		flush_signal_handlers(tsk, 0);
1523b612e5dfSChristian Brauner 
15241da177e4SLinus Torvalds 	return 0;
15251da177e4SLinus Torvalds }
15261da177e4SLinus Torvalds 
1527a7e5328aSOleg Nesterov void __cleanup_sighand(struct sighand_struct *sighand)
1528c81addc9SOleg Nesterov {
1529d036bda7SElena Reshetova 	if (refcount_dec_and_test(&sighand->count)) {
1530d80e731eSOleg Nesterov 		signalfd_cleanup(sighand);
1531392809b2SOleg Nesterov 		/*
15325f0d5a3aSPaul E. McKenney 		 * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it
1533392809b2SOleg Nesterov 		 * without an RCU grace period, see __lock_task_sighand().
1534392809b2SOleg Nesterov 		 */
1535c81addc9SOleg Nesterov 		kmem_cache_free(sighand_cachep, sighand);
1536c81addc9SOleg Nesterov 	}
1537d80e731eSOleg Nesterov }
1538c81addc9SOleg Nesterov 
1539f06febc9SFrank Mayhar /*
1540f06febc9SFrank Mayhar  * Initialize POSIX timer handling for a thread group.
1541f06febc9SFrank Mayhar  */
1542f06febc9SFrank Mayhar static void posix_cpu_timers_init_group(struct signal_struct *sig)
1543f06febc9SFrank Mayhar {
15442b69942fSThomas Gleixner 	struct posix_cputimers *pct = &sig->posix_cputimers;
154578d7d407SJiri Slaby 	unsigned long cpu_limit;
154678d7d407SJiri Slaby 
1547316c1608SJason Low 	cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
15483a245c0fSThomas Gleixner 	posix_cputimers_group_init(pct, cpu_limit);
15496279a751SOleg Nesterov }
15506279a751SOleg Nesterov 
1551a39bc516SAlexey Dobriyan static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
15521da177e4SLinus Torvalds {
15531da177e4SLinus Torvalds 	struct signal_struct *sig;
15541da177e4SLinus Torvalds 
15554ab6c083SOleg Nesterov 	if (clone_flags & CLONE_THREAD)
1556490dea45SPeter Zijlstra 		return 0;
15576279a751SOleg Nesterov 
1558a56704efSVeaceslav Falico 	sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
15591da177e4SLinus Torvalds 	tsk->signal = sig;
15601da177e4SLinus Torvalds 	if (!sig)
15611da177e4SLinus Torvalds 		return -ENOMEM;
15621da177e4SLinus Torvalds 
1563b3ac022cSOleg Nesterov 	sig->nr_threads = 1;
15641da177e4SLinus Torvalds 	atomic_set(&sig->live, 1);
156560d4de3fSElena Reshetova 	refcount_set(&sig->sigcnt, 1);
15660c740d0aSOleg Nesterov 
15670c740d0aSOleg Nesterov 	/* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
15680c740d0aSOleg Nesterov 	sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
15690c740d0aSOleg Nesterov 	tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
15700c740d0aSOleg Nesterov 
15711da177e4SLinus Torvalds 	init_waitqueue_head(&sig->wait_chldexit);
1572db51aeccSOleg Nesterov 	sig->curr_target = tsk;
15731da177e4SLinus Torvalds 	init_sigpending(&sig->shared_pending);
1574c3ad2c3bSEric W. Biederman 	INIT_HLIST_HEAD(&sig->multiprocess);
1575e78c3496SRik van Riel 	seqlock_init(&sig->stats_lock);
15769d7fb042SPeter Zijlstra 	prev_cputime_init(&sig->prev_cputime);
15771da177e4SLinus Torvalds 
1578baa73d9eSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS
1579b18b6a9cSNicolas Pitre 	INIT_LIST_HEAD(&sig->posix_timers);
1580c9cb2e3dSThomas Gleixner 	hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
15811da177e4SLinus Torvalds 	sig->real_timer.function = it_real_fn;
1582baa73d9eSNicolas Pitre #endif
15831da177e4SLinus Torvalds 
15841da177e4SLinus Torvalds 	task_lock(current->group_leader);
15851da177e4SLinus Torvalds 	memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
15861da177e4SLinus Torvalds 	task_unlock(current->group_leader);
15871da177e4SLinus Torvalds 
15886279a751SOleg Nesterov 	posix_cpu_timers_init_group(sig);
15896279a751SOleg Nesterov 
1590522ed776SMiloslav Trmac 	tty_audit_fork(sig);
15915091faa4SMike Galbraith 	sched_autogroup_fork(sig);
1592522ed776SMiloslav Trmac 
1593a63d83f4SDavid Rientjes 	sig->oom_score_adj = current->signal->oom_score_adj;
1594dabb16f6SMandeep Singh Baines 	sig->oom_score_adj_min = current->signal->oom_score_adj_min;
159528b83c51SKOSAKI Motohiro 
15969b1bf12dSKOSAKI Motohiro 	mutex_init(&sig->cred_guard_mutex);
15979b1bf12dSKOSAKI Motohiro 
15981da177e4SLinus Torvalds 	return 0;
15991da177e4SLinus Torvalds }
16001da177e4SLinus Torvalds 
1601dbd95212SKees Cook static void copy_seccomp(struct task_struct *p)
1602dbd95212SKees Cook {
1603dbd95212SKees Cook #ifdef CONFIG_SECCOMP
1604dbd95212SKees Cook 	/*
1605dbd95212SKees Cook 	 * Must be called with sighand->lock held, which is common to
1606dbd95212SKees Cook 	 * all threads in the group. Holding cred_guard_mutex is not
1607dbd95212SKees Cook 	 * needed because this new task is not yet running and cannot
1608dbd95212SKees Cook 	 * be racing exec.
1609dbd95212SKees Cook 	 */
161069f6a34bSGuenter Roeck 	assert_spin_locked(&current->sighand->siglock);
1611dbd95212SKees Cook 
1612dbd95212SKees Cook 	/* Ref-count the new filter user, and assign it. */
1613dbd95212SKees Cook 	get_seccomp_filter(current);
1614dbd95212SKees Cook 	p->seccomp = current->seccomp;
1615dbd95212SKees Cook 
1616dbd95212SKees Cook 	/*
1617dbd95212SKees Cook 	 * Explicitly enable no_new_privs here in case it got set
1618dbd95212SKees Cook 	 * between the task_struct being duplicated and holding the
1619dbd95212SKees Cook 	 * sighand lock. The seccomp state and nnp must be in sync.
1620dbd95212SKees Cook 	 */
1621dbd95212SKees Cook 	if (task_no_new_privs(current))
1622dbd95212SKees Cook 		task_set_no_new_privs(p);
1623dbd95212SKees Cook 
1624dbd95212SKees Cook 	/*
1625dbd95212SKees Cook 	 * If the parent gained a seccomp mode after copying thread
1626dbd95212SKees Cook 	 * flags and between before we held the sighand lock, we have
1627dbd95212SKees Cook 	 * to manually enable the seccomp thread flag here.
1628dbd95212SKees Cook 	 */
1629dbd95212SKees Cook 	if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
1630dbd95212SKees Cook 		set_tsk_thread_flag(p, TIF_SECCOMP);
1631dbd95212SKees Cook #endif
1632dbd95212SKees Cook }
1633dbd95212SKees Cook 
163417da2bd9SHeiko Carstens SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
16351da177e4SLinus Torvalds {
16361da177e4SLinus Torvalds 	current->clear_child_tid = tidptr;
16371da177e4SLinus Torvalds 
1638b488893aSPavel Emelyanov 	return task_pid_vnr(current);
16391da177e4SLinus Torvalds }
16401da177e4SLinus Torvalds 
1641a39bc516SAlexey Dobriyan static void rt_mutex_init_task(struct task_struct *p)
164223f78d4aSIngo Molnar {
16431d615482SThomas Gleixner 	raw_spin_lock_init(&p->pi_lock);
1644e29e175bSZilvinas Valinskas #ifdef CONFIG_RT_MUTEXES
1645a23ba907SDavidlohr Bueso 	p->pi_waiters = RB_ROOT_CACHED;
1646e96a7705SXunlei Pang 	p->pi_top_task = NULL;
164723f78d4aSIngo Molnar 	p->pi_blocked_on = NULL;
164823f78d4aSIngo Molnar #endif
164923f78d4aSIngo Molnar }
165023f78d4aSIngo Molnar 
16512c470475SEric W. Biederman static inline void init_task_pid_links(struct task_struct *task)
16522c470475SEric W. Biederman {
16532c470475SEric W. Biederman 	enum pid_type type;
16542c470475SEric W. Biederman 
16552c470475SEric W. Biederman 	for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
16562c470475SEric W. Biederman 		INIT_HLIST_NODE(&task->pid_links[type]);
16572c470475SEric W. Biederman 	}
16582c470475SEric W. Biederman }
16592c470475SEric W. Biederman 
166081907739SOleg Nesterov static inline void
166181907739SOleg Nesterov init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
166281907739SOleg Nesterov {
16632c470475SEric W. Biederman 	if (type == PIDTYPE_PID)
16642c470475SEric W. Biederman 		task->thread_pid = pid;
16652c470475SEric W. Biederman 	else
16662c470475SEric W. Biederman 		task->signal->pids[type] = pid;
166781907739SOleg Nesterov }
166881907739SOleg Nesterov 
16696bfbaa51SIngo Molnar static inline void rcu_copy_process(struct task_struct *p)
16706bfbaa51SIngo Molnar {
16716bfbaa51SIngo Molnar #ifdef CONFIG_PREEMPT_RCU
16726bfbaa51SIngo Molnar 	p->rcu_read_lock_nesting = 0;
16736bfbaa51SIngo Molnar 	p->rcu_read_unlock_special.s = 0;
16746bfbaa51SIngo Molnar 	p->rcu_blocked_node = NULL;
16756bfbaa51SIngo Molnar 	INIT_LIST_HEAD(&p->rcu_node_entry);
16766bfbaa51SIngo Molnar #endif /* #ifdef CONFIG_PREEMPT_RCU */
16776bfbaa51SIngo Molnar #ifdef CONFIG_TASKS_RCU
16786bfbaa51SIngo Molnar 	p->rcu_tasks_holdout = false;
16796bfbaa51SIngo Molnar 	INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
16806bfbaa51SIngo Molnar 	p->rcu_tasks_idle_cpu = -1;
16816bfbaa51SIngo Molnar #endif /* #ifdef CONFIG_TASKS_RCU */
16826bfbaa51SIngo Molnar }
16836bfbaa51SIngo Molnar 
16843695eae5SChristian Brauner struct pid *pidfd_pid(const struct file *file)
16853695eae5SChristian Brauner {
16863695eae5SChristian Brauner 	if (file->f_op == &pidfd_fops)
16873695eae5SChristian Brauner 		return file->private_data;
16883695eae5SChristian Brauner 
16893695eae5SChristian Brauner 	return ERR_PTR(-EBADF);
16903695eae5SChristian Brauner }
16913695eae5SChristian Brauner 
1692b3e58382SChristian Brauner static int pidfd_release(struct inode *inode, struct file *file)
1693b3e58382SChristian Brauner {
1694b3e58382SChristian Brauner 	struct pid *pid = file->private_data;
1695b3e58382SChristian Brauner 
1696b3e58382SChristian Brauner 	file->private_data = NULL;
1697b3e58382SChristian Brauner 	put_pid(pid);
1698b3e58382SChristian Brauner 	return 0;
1699b3e58382SChristian Brauner }
1700b3e58382SChristian Brauner 
1701b3e58382SChristian Brauner #ifdef CONFIG_PROC_FS
170215d42eb2SChristian Kellner /**
170315d42eb2SChristian Kellner  * pidfd_show_fdinfo - print information about a pidfd
170415d42eb2SChristian Kellner  * @m: proc fdinfo file
170515d42eb2SChristian Kellner  * @f: file referencing a pidfd
170615d42eb2SChristian Kellner  *
170715d42eb2SChristian Kellner  * Pid:
170815d42eb2SChristian Kellner  * This function will print the pid that a given pidfd refers to in the
170915d42eb2SChristian Kellner  * pid namespace of the procfs instance.
171015d42eb2SChristian Kellner  * If the pid namespace of the process is not a descendant of the pid
171115d42eb2SChristian Kellner  * namespace of the procfs instance 0 will be shown as its pid. This is
171215d42eb2SChristian Kellner  * similar to calling getppid() on a process whose parent is outside of
171315d42eb2SChristian Kellner  * its pid namespace.
171415d42eb2SChristian Kellner  *
171515d42eb2SChristian Kellner  * NSpid:
171615d42eb2SChristian Kellner  * If pid namespaces are supported then this function will also print
171715d42eb2SChristian Kellner  * the pid of a given pidfd refers to for all descendant pid namespaces
171815d42eb2SChristian Kellner  * starting from the current pid namespace of the instance, i.e. the
171915d42eb2SChristian Kellner  * Pid field and the first entry in the NSpid field will be identical.
172015d42eb2SChristian Kellner  * If the pid namespace of the process is not a descendant of the pid
172115d42eb2SChristian Kellner  * namespace of the procfs instance 0 will be shown as its first NSpid
172215d42eb2SChristian Kellner  * entry and no others will be shown.
172315d42eb2SChristian Kellner  * Note that this differs from the Pid and NSpid fields in
172415d42eb2SChristian Kellner  * /proc/<pid>/status where Pid and NSpid are always shown relative to
172515d42eb2SChristian Kellner  * the  pid namespace of the procfs instance. The difference becomes
172615d42eb2SChristian Kellner  * obvious when sending around a pidfd between pid namespaces from a
172715d42eb2SChristian Kellner  * different branch of the tree, i.e. where no ancestoral relation is
172815d42eb2SChristian Kellner  * present between the pid namespaces:
172915d42eb2SChristian Kellner  * - create two new pid namespaces ns1 and ns2 in the initial pid
173015d42eb2SChristian Kellner  *   namespace (also take care to create new mount namespaces in the
173115d42eb2SChristian Kellner  *   new pid namespace and mount procfs)
173215d42eb2SChristian Kellner  * - create a process with a pidfd in ns1
173315d42eb2SChristian Kellner  * - send pidfd from ns1 to ns2
173415d42eb2SChristian Kellner  * - read /proc/self/fdinfo/<pidfd> and observe that both Pid and NSpid
173515d42eb2SChristian Kellner  *   have exactly one entry, which is 0
173615d42eb2SChristian Kellner  */
1737b3e58382SChristian Brauner static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
1738b3e58382SChristian Brauner {
1739b3e58382SChristian Brauner 	struct pid *pid = f->private_data;
17403d6d8da4SChristian Brauner 	struct pid_namespace *ns;
17413d6d8da4SChristian Brauner 	pid_t nr = -1;
1742b3e58382SChristian Brauner 
17433d6d8da4SChristian Brauner 	if (likely(pid_has_task(pid, PIDTYPE_PID))) {
17443d6d8da4SChristian Brauner 		ns = proc_pid_ns(file_inode(m->file));
17453d6d8da4SChristian Brauner 		nr = pid_nr_ns(pid, ns);
17463d6d8da4SChristian Brauner 	}
17473d6d8da4SChristian Brauner 
17483d6d8da4SChristian Brauner 	seq_put_decimal_ll(m, "Pid:\t", nr);
174915d42eb2SChristian Kellner 
175015d42eb2SChristian Kellner #ifdef CONFIG_PID_NS
17513d6d8da4SChristian Brauner 	seq_put_decimal_ll(m, "\nNSpid:\t", nr);
17523d6d8da4SChristian Brauner 	if (nr > 0) {
175315d42eb2SChristian Kellner 		int i;
175415d42eb2SChristian Kellner 
175515d42eb2SChristian Kellner 		/* If nr is non-zero it means that 'pid' is valid and that
175615d42eb2SChristian Kellner 		 * ns, i.e. the pid namespace associated with the procfs
175715d42eb2SChristian Kellner 		 * instance, is in the pid namespace hierarchy of pid.
175815d42eb2SChristian Kellner 		 * Start at one below the already printed level.
175915d42eb2SChristian Kellner 		 */
176015d42eb2SChristian Kellner 		for (i = ns->level + 1; i <= pid->level; i++)
17613d6d8da4SChristian Brauner 			seq_put_decimal_ll(m, "\t", pid->numbers[i].nr);
176215d42eb2SChristian Kellner 	}
176315d42eb2SChristian Kellner #endif
1764b3e58382SChristian Brauner 	seq_putc(m, '\n');
1765b3e58382SChristian Brauner }
1766b3e58382SChristian Brauner #endif
1767b3e58382SChristian Brauner 
1768b53b0b9dSJoel Fernandes (Google) /*
1769b53b0b9dSJoel Fernandes (Google)  * Poll support for process exit notification.
1770b53b0b9dSJoel Fernandes (Google)  */
17719e77716aSLuc Van Oostenryck static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts)
1772b53b0b9dSJoel Fernandes (Google) {
1773b53b0b9dSJoel Fernandes (Google) 	struct task_struct *task;
1774b53b0b9dSJoel Fernandes (Google) 	struct pid *pid = file->private_data;
17759e77716aSLuc Van Oostenryck 	__poll_t poll_flags = 0;
1776b53b0b9dSJoel Fernandes (Google) 
1777b53b0b9dSJoel Fernandes (Google) 	poll_wait(file, &pid->wait_pidfd, pts);
1778b53b0b9dSJoel Fernandes (Google) 
1779b53b0b9dSJoel Fernandes (Google) 	rcu_read_lock();
1780b53b0b9dSJoel Fernandes (Google) 	task = pid_task(pid, PIDTYPE_PID);
1781b53b0b9dSJoel Fernandes (Google) 	/*
1782b53b0b9dSJoel Fernandes (Google) 	 * Inform pollers only when the whole thread group exits.
1783b53b0b9dSJoel Fernandes (Google) 	 * If the thread group leader exits before all other threads in the
1784b53b0b9dSJoel Fernandes (Google) 	 * group, then poll(2) should block, similar to the wait(2) family.
1785b53b0b9dSJoel Fernandes (Google) 	 */
1786b53b0b9dSJoel Fernandes (Google) 	if (!task || (task->exit_state && thread_group_empty(task)))
17879e77716aSLuc Van Oostenryck 		poll_flags = EPOLLIN | EPOLLRDNORM;
1788b53b0b9dSJoel Fernandes (Google) 	rcu_read_unlock();
1789b53b0b9dSJoel Fernandes (Google) 
1790b53b0b9dSJoel Fernandes (Google) 	return poll_flags;
1791b53b0b9dSJoel Fernandes (Google) }
1792b53b0b9dSJoel Fernandes (Google) 
1793b3e58382SChristian Brauner const struct file_operations pidfd_fops = {
1794b3e58382SChristian Brauner 	.release = pidfd_release,
1795b53b0b9dSJoel Fernandes (Google) 	.poll = pidfd_poll,
1796b3e58382SChristian Brauner #ifdef CONFIG_PROC_FS
1797b3e58382SChristian Brauner 	.show_fdinfo = pidfd_show_fdinfo,
1798b3e58382SChristian Brauner #endif
1799b3e58382SChristian Brauner };
1800b3e58382SChristian Brauner 
1801c3f3ce04SAndrea Arcangeli static void __delayed_free_task(struct rcu_head *rhp)
1802c3f3ce04SAndrea Arcangeli {
1803c3f3ce04SAndrea Arcangeli 	struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
1804c3f3ce04SAndrea Arcangeli 
1805c3f3ce04SAndrea Arcangeli 	free_task(tsk);
1806c3f3ce04SAndrea Arcangeli }
1807c3f3ce04SAndrea Arcangeli 
1808c3f3ce04SAndrea Arcangeli static __always_inline void delayed_free_task(struct task_struct *tsk)
1809c3f3ce04SAndrea Arcangeli {
1810c3f3ce04SAndrea Arcangeli 	if (IS_ENABLED(CONFIG_MEMCG))
1811c3f3ce04SAndrea Arcangeli 		call_rcu(&tsk->rcu, __delayed_free_task);
1812c3f3ce04SAndrea Arcangeli 	else
1813c3f3ce04SAndrea Arcangeli 		free_task(tsk);
1814c3f3ce04SAndrea Arcangeli }
1815c3f3ce04SAndrea Arcangeli 
1816f06febc9SFrank Mayhar /*
18171da177e4SLinus Torvalds  * This creates a new process as a copy of the old one,
18181da177e4SLinus Torvalds  * but does not actually start it yet.
18191da177e4SLinus Torvalds  *
18201da177e4SLinus Torvalds  * It copies the registers, and all the appropriate
18211da177e4SLinus Torvalds  * parts of the process environment (as per the clone
18221da177e4SLinus Torvalds  * flags). The actual kick-off is left to the caller.
18231da177e4SLinus Torvalds  */
18240766f788SEmese Revfy static __latent_entropy struct task_struct *copy_process(
182509a05394SRoland McGrath 					struct pid *pid,
18263033f14aSJosh Triplett 					int trace,
18277f192e3cSChristian Brauner 					int node,
18287f192e3cSChristian Brauner 					struct kernel_clone_args *args)
18291da177e4SLinus Torvalds {
1830b3e58382SChristian Brauner 	int pidfd = -1, retval;
1831a24efe62SMariusz Kozlowski 	struct task_struct *p;
1832c3ad2c3bSEric W. Biederman 	struct multiprocess_signals delayed;
18336fd2fe49SAl Viro 	struct file *pidfile = NULL;
18347f192e3cSChristian Brauner 	u64 clone_flags = args->flags;
1835769071acSAndrei Vagin 	struct nsproxy *nsp = current->nsproxy;
18361da177e4SLinus Torvalds 
1837667b6094SMarcos Paulo de Souza 	/*
1838667b6094SMarcos Paulo de Souza 	 * Don't allow sharing the root directory with processes in a different
1839667b6094SMarcos Paulo de Souza 	 * namespace
1840667b6094SMarcos Paulo de Souza 	 */
18411da177e4SLinus Torvalds 	if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
18421da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
18431da177e4SLinus Torvalds 
1844e66eded8SEric W. Biederman 	if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
1845e66eded8SEric W. Biederman 		return ERR_PTR(-EINVAL);
1846e66eded8SEric W. Biederman 
18471da177e4SLinus Torvalds 	/*
18481da177e4SLinus Torvalds 	 * Thread groups must share signals as well, and detached threads
18491da177e4SLinus Torvalds 	 * can only be started up within the thread group.
18501da177e4SLinus Torvalds 	 */
18511da177e4SLinus Torvalds 	if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
18521da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
18531da177e4SLinus Torvalds 
18541da177e4SLinus Torvalds 	/*
18551da177e4SLinus Torvalds 	 * Shared signal handlers imply shared VM. By way of the above,
18561da177e4SLinus Torvalds 	 * thread groups also imply shared VM. Blocking this case allows
18571da177e4SLinus Torvalds 	 * for various simplifications in other code.
18581da177e4SLinus Torvalds 	 */
18591da177e4SLinus Torvalds 	if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
18601da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
18611da177e4SLinus Torvalds 
1862123be07bSSukadev Bhattiprolu 	/*
1863123be07bSSukadev Bhattiprolu 	 * Siblings of global init remain as zombies on exit since they are
1864123be07bSSukadev Bhattiprolu 	 * not reaped by their parent (swapper). To solve this and to avoid
1865123be07bSSukadev Bhattiprolu 	 * multi-rooted process trees, prevent global and container-inits
1866123be07bSSukadev Bhattiprolu 	 * from creating siblings.
1867123be07bSSukadev Bhattiprolu 	 */
1868123be07bSSukadev Bhattiprolu 	if ((clone_flags & CLONE_PARENT) &&
1869123be07bSSukadev Bhattiprolu 				current->signal->flags & SIGNAL_UNKILLABLE)
1870123be07bSSukadev Bhattiprolu 		return ERR_PTR(-EINVAL);
1871123be07bSSukadev Bhattiprolu 
18728382fcacSEric W. Biederman 	/*
187340a0d32dSOleg Nesterov 	 * If the new process will be in a different pid or user namespace
1874faf00da5SEric W. Biederman 	 * do not allow it to share a thread group with the forking task.
18758382fcacSEric W. Biederman 	 */
1876faf00da5SEric W. Biederman 	if (clone_flags & CLONE_THREAD) {
187740a0d32dSOleg Nesterov 		if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
1878769071acSAndrei Vagin 		    (task_active_pid_ns(current) != nsp->pid_ns_for_children))
1879769071acSAndrei Vagin 			return ERR_PTR(-EINVAL);
1880769071acSAndrei Vagin 	}
1881769071acSAndrei Vagin 
1882769071acSAndrei Vagin 	/*
1883769071acSAndrei Vagin 	 * If the new process will be in a different time namespace
1884769071acSAndrei Vagin 	 * do not allow it to share VM or a thread group with the forking task.
1885769071acSAndrei Vagin 	 */
1886769071acSAndrei Vagin 	if (clone_flags & (CLONE_THREAD | CLONE_VM)) {
1887769071acSAndrei Vagin 		if (nsp->time_ns != nsp->time_ns_for_children)
18888382fcacSEric W. Biederman 			return ERR_PTR(-EINVAL);
188940a0d32dSOleg Nesterov 	}
18908382fcacSEric W. Biederman 
1891b3e58382SChristian Brauner 	if (clone_flags & CLONE_PIDFD) {
1892b3e58382SChristian Brauner 		/*
1893b3e58382SChristian Brauner 		 * - CLONE_DETACHED is blocked so that we can potentially
1894b3e58382SChristian Brauner 		 *   reuse it later for CLONE_PIDFD.
1895b3e58382SChristian Brauner 		 * - CLONE_THREAD is blocked until someone really needs it.
1896b3e58382SChristian Brauner 		 */
18977f192e3cSChristian Brauner 		if (clone_flags & (CLONE_DETACHED | CLONE_THREAD))
1898b3e58382SChristian Brauner 			return ERR_PTR(-EINVAL);
1899b3e58382SChristian Brauner 	}
1900b3e58382SChristian Brauner 
1901c3ad2c3bSEric W. Biederman 	/*
1902c3ad2c3bSEric W. Biederman 	 * Force any signals received before this point to be delivered
1903c3ad2c3bSEric W. Biederman 	 * before the fork happens.  Collect up signals sent to multiple
1904c3ad2c3bSEric W. Biederman 	 * processes that happen during the fork and delay them so that
1905c3ad2c3bSEric W. Biederman 	 * they appear to happen after the fork.
1906c3ad2c3bSEric W. Biederman 	 */
1907c3ad2c3bSEric W. Biederman 	sigemptyset(&delayed.signal);
1908c3ad2c3bSEric W. Biederman 	INIT_HLIST_NODE(&delayed.node);
1909c3ad2c3bSEric W. Biederman 
1910c3ad2c3bSEric W. Biederman 	spin_lock_irq(&current->sighand->siglock);
1911c3ad2c3bSEric W. Biederman 	if (!(clone_flags & CLONE_THREAD))
1912c3ad2c3bSEric W. Biederman 		hlist_add_head(&delayed.node, &current->signal->multiprocess);
1913c3ad2c3bSEric W. Biederman 	recalc_sigpending();
1914c3ad2c3bSEric W. Biederman 	spin_unlock_irq(&current->sighand->siglock);
1915c3ad2c3bSEric W. Biederman 	retval = -ERESTARTNOINTR;
1916c3ad2c3bSEric W. Biederman 	if (signal_pending(current))
1917c3ad2c3bSEric W. Biederman 		goto fork_out;
1918c3ad2c3bSEric W. Biederman 
19191da177e4SLinus Torvalds 	retval = -ENOMEM;
1920725fc629SAndi Kleen 	p = dup_task_struct(current, node);
19211da177e4SLinus Torvalds 	if (!p)
19221da177e4SLinus Torvalds 		goto fork_out;
19231da177e4SLinus Torvalds 
19244d6501dcSVegard Nossum 	/*
19254d6501dcSVegard Nossum 	 * This _must_ happen before we call free_task(), i.e. before we jump
19264d6501dcSVegard Nossum 	 * to any of the bad_fork_* labels. This is to avoid freeing
19274d6501dcSVegard Nossum 	 * p->set_child_tid which is (ab)used as a kthread's data pointer for
19284d6501dcSVegard Nossum 	 * kernel threads (PF_KTHREAD).
19294d6501dcSVegard Nossum 	 */
19307f192e3cSChristian Brauner 	p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL;
19314d6501dcSVegard Nossum 	/*
19324d6501dcSVegard Nossum 	 * Clear TID on mm_release()?
19334d6501dcSVegard Nossum 	 */
19347f192e3cSChristian Brauner 	p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;
19354d6501dcSVegard Nossum 
1936f7e8b616SSteven Rostedt 	ftrace_graph_init_task(p);
1937f7e8b616SSteven Rostedt 
1938bea493a0SPeter Zijlstra 	rt_mutex_init_task(p);
1939bea493a0SPeter Zijlstra 
1940d12c1a37SIngo Molnar #ifdef CONFIG_PROVE_LOCKING
1941de30a2b3SIngo Molnar 	DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
1942de30a2b3SIngo Molnar 	DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1943de30a2b3SIngo Molnar #endif
19441da177e4SLinus Torvalds 	retval = -EAGAIN;
19453b11a1deSDavid Howells 	if (atomic_read(&p->real_cred->user->processes) >=
194678d7d407SJiri Slaby 			task_rlimit(p, RLIMIT_NPROC)) {
1947b57922b6SEric Paris 		if (p->real_cred->user != INIT_USER &&
1948b57922b6SEric Paris 		    !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
19491da177e4SLinus Torvalds 			goto bad_fork_free;
19501da177e4SLinus Torvalds 	}
195172fa5997SVasiliy Kulikov 	current->flags &= ~PF_NPROC_EXCEEDED;
19521da177e4SLinus Torvalds 
1953f1752eecSDavid Howells 	retval = copy_creds(p, clone_flags);
1954f1752eecSDavid Howells 	if (retval < 0)
1955f1752eecSDavid Howells 		goto bad_fork_free;
19561da177e4SLinus Torvalds 
19571da177e4SLinus Torvalds 	/*
19581da177e4SLinus Torvalds 	 * If multiple threads are within copy_process(), then this check
19591da177e4SLinus Torvalds 	 * triggers too late. This doesn't hurt, the check is only there
19601da177e4SLinus Torvalds 	 * to stop root fork bombs.
19611da177e4SLinus Torvalds 	 */
196204ec93feSLi Zefan 	retval = -EAGAIN;
19631da177e4SLinus Torvalds 	if (nr_threads >= max_threads)
19641da177e4SLinus Torvalds 		goto bad_fork_cleanup_count;
19651da177e4SLinus Torvalds 
1966ca74e92bSShailabh Nagar 	delayacct_tsk_init(p);	/* Must remain after dup_task_struct() */
1967c1de45caSPeter Zijlstra 	p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE);
1968514ddb44SDavid Rientjes 	p->flags |= PF_FORKNOEXEC;
19691da177e4SLinus Torvalds 	INIT_LIST_HEAD(&p->children);
19701da177e4SLinus Torvalds 	INIT_LIST_HEAD(&p->sibling);
1971f41d911fSPaul E. McKenney 	rcu_copy_process(p);
19721da177e4SLinus Torvalds 	p->vfork_done = NULL;
19731da177e4SLinus Torvalds 	spin_lock_init(&p->alloc_lock);
19741da177e4SLinus Torvalds 
19751da177e4SLinus Torvalds 	init_sigpending(&p->pending);
19761da177e4SLinus Torvalds 
197764861634SMartin Schwidefsky 	p->utime = p->stime = p->gtime = 0;
197840565b5aSStanislaw Gruszka #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
197964861634SMartin Schwidefsky 	p->utimescaled = p->stimescaled = 0;
198040565b5aSStanislaw Gruszka #endif
19819d7fb042SPeter Zijlstra 	prev_cputime_init(&p->prev_cputime);
19829d7fb042SPeter Zijlstra 
19836a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
1984bac5b6b6SFrederic Weisbecker 	seqcount_init(&p->vtime.seqcount);
1985bac5b6b6SFrederic Weisbecker 	p->vtime.starttime = 0;
1986bac5b6b6SFrederic Weisbecker 	p->vtime.state = VTIME_INACTIVE;
19876a61671bSFrederic Weisbecker #endif
19886a61671bSFrederic Weisbecker 
1989a3a2e76cSKAMEZAWA Hiroyuki #if defined(SPLIT_RSS_COUNTING)
1990a3a2e76cSKAMEZAWA Hiroyuki 	memset(&p->rss_stat, 0, sizeof(p->rss_stat));
1991a3a2e76cSKAMEZAWA Hiroyuki #endif
1992172ba844SBalbir Singh 
19936976675dSArjan van de Ven 	p->default_timer_slack_ns = current->timer_slack_ns;
19946976675dSArjan van de Ven 
1995eb414681SJohannes Weiner #ifdef CONFIG_PSI
1996eb414681SJohannes Weiner 	p->psi_flags = 0;
1997eb414681SJohannes Weiner #endif
1998eb414681SJohannes Weiner 
19995995477aSAndrea Righi 	task_io_accounting_init(&p->ioac);
20001da177e4SLinus Torvalds 	acct_clear_integrals(p);
20011da177e4SLinus Torvalds 
20023a245c0fSThomas Gleixner 	posix_cputimers_init(&p->posix_cputimers);
20031da177e4SLinus Torvalds 
20041da177e4SLinus Torvalds 	p->io_context = NULL;
2005c0b0ae8aSRichard Guy Briggs 	audit_set_context(p, NULL);
2006b4f48b63SPaul Menage 	cgroup_fork(p);
20071da177e4SLinus Torvalds #ifdef CONFIG_NUMA
2008846a16bfSLee Schermerhorn 	p->mempolicy = mpol_dup(p->mempolicy);
20091da177e4SLinus Torvalds 	if (IS_ERR(p->mempolicy)) {
20101da177e4SLinus Torvalds 		retval = PTR_ERR(p->mempolicy);
20111da177e4SLinus Torvalds 		p->mempolicy = NULL;
2012e8604cb4SLi Zefan 		goto bad_fork_cleanup_threadgroup_lock;
20131da177e4SLinus Torvalds 	}
20141da177e4SLinus Torvalds #endif
2015778d3b0fSMichal Hocko #ifdef CONFIG_CPUSETS
2016778d3b0fSMichal Hocko 	p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
2017778d3b0fSMichal Hocko 	p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
2018cc9a6c87SMel Gorman 	seqcount_init(&p->mems_allowed_seq);
2019778d3b0fSMichal Hocko #endif
2020de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS
2021de30a2b3SIngo Molnar 	p->irq_events = 0;
2022de30a2b3SIngo Molnar 	p->hardirqs_enabled = 0;
2023de30a2b3SIngo Molnar 	p->hardirq_enable_ip = 0;
2024de30a2b3SIngo Molnar 	p->hardirq_enable_event = 0;
2025de30a2b3SIngo Molnar 	p->hardirq_disable_ip = _THIS_IP_;
2026de30a2b3SIngo Molnar 	p->hardirq_disable_event = 0;
2027de30a2b3SIngo Molnar 	p->softirqs_enabled = 1;
2028de30a2b3SIngo Molnar 	p->softirq_enable_ip = _THIS_IP_;
2029de30a2b3SIngo Molnar 	p->softirq_enable_event = 0;
2030de30a2b3SIngo Molnar 	p->softirq_disable_ip = 0;
2031de30a2b3SIngo Molnar 	p->softirq_disable_event = 0;
2032de30a2b3SIngo Molnar 	p->hardirq_context = 0;
2033de30a2b3SIngo Molnar 	p->softirq_context = 0;
2034de30a2b3SIngo Molnar #endif
20358bcbde54SDavid Hildenbrand 
20368bcbde54SDavid Hildenbrand 	p->pagefault_disabled = 0;
20378bcbde54SDavid Hildenbrand 
2038fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP
2039b09be676SByungchul Park 	lockdep_init_task(p);
2040fbb9ce95SIngo Molnar #endif
20411da177e4SLinus Torvalds 
2042408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES
2043408894eeSIngo Molnar 	p->blocked_on = NULL; /* not blocked yet */
2044408894eeSIngo Molnar #endif
2045cafe5635SKent Overstreet #ifdef CONFIG_BCACHE
2046cafe5635SKent Overstreet 	p->sequential_io	= 0;
2047cafe5635SKent Overstreet 	p->sequential_io_avg	= 0;
2048cafe5635SKent Overstreet #endif
20490f481406SMarkus Metzger 
20503c90e6e9SSrivatsa Vaddagiri 	/* Perform scheduler related setup. Assign this task to a CPU. */
2051aab03e05SDario Faggioli 	retval = sched_fork(clone_flags, p);
2052aab03e05SDario Faggioli 	if (retval)
2053aab03e05SDario Faggioli 		goto bad_fork_cleanup_policy;
20546ab423e0SPeter Zijlstra 
2055cdd6c482SIngo Molnar 	retval = perf_event_init_task(p);
20566ab423e0SPeter Zijlstra 	if (retval)
20576ab423e0SPeter Zijlstra 		goto bad_fork_cleanup_policy;
2058fb0a685cSDaniel Rebelo de Oliveira 	retval = audit_alloc(p);
2059fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20606c72e350SPeter Zijlstra 		goto bad_fork_cleanup_perf;
20611da177e4SLinus Torvalds 	/* copy all the process information */
2062ab602f79SJack Miller 	shm_init_task(p);
2063e4e55b47STetsuo Handa 	retval = security_task_alloc(p, clone_flags);
2064fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20651da177e4SLinus Torvalds 		goto bad_fork_cleanup_audit;
2066e4e55b47STetsuo Handa 	retval = copy_semundo(clone_flags, p);
2067e4e55b47STetsuo Handa 	if (retval)
2068e4e55b47STetsuo Handa 		goto bad_fork_cleanup_security;
2069fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_files(clone_flags, p);
2070fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20711da177e4SLinus Torvalds 		goto bad_fork_cleanup_semundo;
2072fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_fs(clone_flags, p);
2073fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20741da177e4SLinus Torvalds 		goto bad_fork_cleanup_files;
2075fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_sighand(clone_flags, p);
2076fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20771da177e4SLinus Torvalds 		goto bad_fork_cleanup_fs;
2078fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_signal(clone_flags, p);
2079fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20801da177e4SLinus Torvalds 		goto bad_fork_cleanup_sighand;
2081fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_mm(clone_flags, p);
2082fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20831da177e4SLinus Torvalds 		goto bad_fork_cleanup_signal;
2084fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_namespaces(clone_flags, p);
2085fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
2086d84f4f99SDavid Howells 		goto bad_fork_cleanup_mm;
2087fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_io(clone_flags, p);
2088fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
2089fd0928dfSJens Axboe 		goto bad_fork_cleanup_namespaces;
20907f192e3cSChristian Brauner 	retval = copy_thread_tls(clone_flags, args->stack, args->stack_size, p,
20917f192e3cSChristian Brauner 				 args->tls);
20921da177e4SLinus Torvalds 	if (retval)
2093fd0928dfSJens Axboe 		goto bad_fork_cleanup_io;
20941da177e4SLinus Torvalds 
2095afaef01cSAlexander Popov 	stackleak_task_init(p);
2096afaef01cSAlexander Popov 
2097425fb2b4SPavel Emelyanov 	if (pid != &init_struct_pid) {
209849cb2fc4SAdrian Reber 		pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid,
209949cb2fc4SAdrian Reber 				args->set_tid_size);
210035f71bc0SMichal Hocko 		if (IS_ERR(pid)) {
210135f71bc0SMichal Hocko 			retval = PTR_ERR(pid);
21020740aa5fSJiri Slaby 			goto bad_fork_cleanup_thread;
2103425fb2b4SPavel Emelyanov 		}
210435f71bc0SMichal Hocko 	}
2105425fb2b4SPavel Emelyanov 
2106b3e58382SChristian Brauner 	/*
2107b3e58382SChristian Brauner 	 * This has to happen after we've potentially unshared the file
2108b3e58382SChristian Brauner 	 * descriptor table (so that the pidfd doesn't leak into the child
2109b3e58382SChristian Brauner 	 * if the fd table isn't shared).
2110b3e58382SChristian Brauner 	 */
2111b3e58382SChristian Brauner 	if (clone_flags & CLONE_PIDFD) {
21126fd2fe49SAl Viro 		retval = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
2113b3e58382SChristian Brauner 		if (retval < 0)
2114b3e58382SChristian Brauner 			goto bad_fork_free_pid;
2115b3e58382SChristian Brauner 
2116b3e58382SChristian Brauner 		pidfd = retval;
21176fd2fe49SAl Viro 
21186fd2fe49SAl Viro 		pidfile = anon_inode_getfile("[pidfd]", &pidfd_fops, pid,
21196fd2fe49SAl Viro 					      O_RDWR | O_CLOEXEC);
21206fd2fe49SAl Viro 		if (IS_ERR(pidfile)) {
21216fd2fe49SAl Viro 			put_unused_fd(pidfd);
212228dd29c0SChristian Brauner 			retval = PTR_ERR(pidfile);
21236fd2fe49SAl Viro 			goto bad_fork_free_pid;
21246fd2fe49SAl Viro 		}
21256fd2fe49SAl Viro 		get_pid(pid);	/* held by pidfile now */
21266fd2fe49SAl Viro 
21277f192e3cSChristian Brauner 		retval = put_user(pidfd, args->pidfd);
2128b3e58382SChristian Brauner 		if (retval)
2129b3e58382SChristian Brauner 			goto bad_fork_put_pidfd;
2130b3e58382SChristian Brauner 	}
2131b3e58382SChristian Brauner 
213273c10101SJens Axboe #ifdef CONFIG_BLOCK
213373c10101SJens Axboe 	p->plug = NULL;
213473c10101SJens Axboe #endif
2135ba31c1a4SThomas Gleixner 	futex_init_task(p);
2136ba31c1a4SThomas Gleixner 
21371da177e4SLinus Torvalds 	/*
2138f9a3879aSGOTO Masanori 	 * sigaltstack should be cleared when sharing the same VM
2139f9a3879aSGOTO Masanori 	 */
2140f9a3879aSGOTO Masanori 	if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
21412a742138SStas Sergeev 		sas_ss_reset(p);
2142f9a3879aSGOTO Masanori 
2143f9a3879aSGOTO Masanori 	/*
21446580807dSOleg Nesterov 	 * Syscall tracing and stepping should be turned off in the
21456580807dSOleg Nesterov 	 * child regardless of CLONE_PTRACE.
21461da177e4SLinus Torvalds 	 */
21476580807dSOleg Nesterov 	user_disable_single_step(p);
21481da177e4SLinus Torvalds 	clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
2149ed75e8d5SLaurent Vivier #ifdef TIF_SYSCALL_EMU
2150ed75e8d5SLaurent Vivier 	clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
2151ed75e8d5SLaurent Vivier #endif
2152e02c9b0dSLin Feng 	clear_tsk_latency_tracing(p);
21531da177e4SLinus Torvalds 
21541da177e4SLinus Torvalds 	/* ok, now we should be set up.. */
215518c830dfSOleg Nesterov 	p->pid = pid_nr(pid);
215618c830dfSOleg Nesterov 	if (clone_flags & CLONE_THREAD) {
21575f8aadd8SOleg Nesterov 		p->exit_signal = -1;
215818c830dfSOleg Nesterov 		p->group_leader = current->group_leader;
215918c830dfSOleg Nesterov 		p->tgid = current->tgid;
216018c830dfSOleg Nesterov 	} else {
216118c830dfSOleg Nesterov 		if (clone_flags & CLONE_PARENT)
21625f8aadd8SOleg Nesterov 			p->exit_signal = current->group_leader->exit_signal;
21635f8aadd8SOleg Nesterov 		else
21647f192e3cSChristian Brauner 			p->exit_signal = args->exit_signal;
216518c830dfSOleg Nesterov 		p->group_leader = p;
216618c830dfSOleg Nesterov 		p->tgid = p->pid;
216718c830dfSOleg Nesterov 	}
21685f8aadd8SOleg Nesterov 
21699d823e8fSWu Fengguang 	p->nr_dirtied = 0;
21709d823e8fSWu Fengguang 	p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
217183712358SWu Fengguang 	p->dirty_paused_when = 0;
21729d823e8fSWu Fengguang 
2173bb8cbbfeSOleg Nesterov 	p->pdeath_signal = 0;
217447e65328SOleg Nesterov 	INIT_LIST_HEAD(&p->thread_group);
2175158e1645SAl Viro 	p->task_works = NULL;
21761da177e4SLinus Torvalds 
217718c830dfSOleg Nesterov 	/*
21787e47682eSAleksa Sarai 	 * Ensure that the cgroup subsystem policies allow the new process to be
21797e47682eSAleksa Sarai 	 * forked. It should be noted the the new process's css_set can be changed
21807e47682eSAleksa Sarai 	 * between here and cgroup_post_fork() if an organisation operation is in
21817e47682eSAleksa Sarai 	 * progress.
21827e47682eSAleksa Sarai 	 */
2183*ef2c41cfSChristian Brauner 	retval = cgroup_can_fork(p, args);
21847e47682eSAleksa Sarai 	if (retval)
21855a5cf5cbSChristian Brauner 		goto bad_fork_put_pidfd;
21867e47682eSAleksa Sarai 
21877e47682eSAleksa Sarai 	/*
21887b558513SDavid Herrmann 	 * From this point on we must avoid any synchronous user-space
21897b558513SDavid Herrmann 	 * communication until we take the tasklist-lock. In particular, we do
21907b558513SDavid Herrmann 	 * not want user-space to be able to predict the process start-time by
21917b558513SDavid Herrmann 	 * stalling fork(2) after we recorded the start_time but before it is
21927b558513SDavid Herrmann 	 * visible to the system.
21937b558513SDavid Herrmann 	 */
21947b558513SDavid Herrmann 
21957b558513SDavid Herrmann 	p->start_time = ktime_get_ns();
2196cf25e24dSPeter Zijlstra 	p->start_boottime = ktime_get_boottime_ns();
21977b558513SDavid Herrmann 
21987b558513SDavid Herrmann 	/*
219918c830dfSOleg Nesterov 	 * Make it visible to the rest of the system, but dont wake it up yet.
220018c830dfSOleg Nesterov 	 * Need tasklist lock for parent etc handling!
220118c830dfSOleg Nesterov 	 */
22021da177e4SLinus Torvalds 	write_lock_irq(&tasklist_lock);
22031da177e4SLinus Torvalds 
22041da177e4SLinus Torvalds 	/* CLONE_PARENT re-uses the old parent */
22052d5516cbSOleg Nesterov 	if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
22061da177e4SLinus Torvalds 		p->real_parent = current->real_parent;
22072d5516cbSOleg Nesterov 		p->parent_exec_id = current->parent_exec_id;
22082d5516cbSOleg Nesterov 	} else {
22091da177e4SLinus Torvalds 		p->real_parent = current;
22102d5516cbSOleg Nesterov 		p->parent_exec_id = current->self_exec_id;
22112d5516cbSOleg Nesterov 	}
22121da177e4SLinus Torvalds 
2213d83a7cb3SJosh Poimboeuf 	klp_copy_process(p);
2214d83a7cb3SJosh Poimboeuf 
22151da177e4SLinus Torvalds 	spin_lock(&current->sighand->siglock);
22164a2c7a78SOleg Nesterov 
22174a2c7a78SOleg Nesterov 	/*
2218dbd95212SKees Cook 	 * Copy seccomp details explicitly here, in case they were changed
2219dbd95212SKees Cook 	 * before holding sighand lock.
2220dbd95212SKees Cook 	 */
2221dbd95212SKees Cook 	copy_seccomp(p);
2222dbd95212SKees Cook 
2223d7822b1eSMathieu Desnoyers 	rseq_fork(p, clone_flags);
2224d7822b1eSMathieu Desnoyers 
22254ca1d3eeSEric W. Biederman 	/* Don't start children in a dying pid namespace */
2226e8cfbc24SGargi Sharma 	if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
22273fd37226SKirill Tkhai 		retval = -ENOMEM;
22283fd37226SKirill Tkhai 		goto bad_fork_cancel_cgroup;
22293fd37226SKirill Tkhai 	}
22304a2c7a78SOleg Nesterov 
22317673bf55SEric W. Biederman 	/* Let kill terminate clone/fork in the middle */
22327673bf55SEric W. Biederman 	if (fatal_signal_pending(current)) {
22337673bf55SEric W. Biederman 		retval = -EINTR;
22347673bf55SEric W. Biederman 		goto bad_fork_cancel_cgroup;
22357673bf55SEric W. Biederman 	}
22367673bf55SEric W. Biederman 
22376fd2fe49SAl Viro 	/* past the last point of failure */
22386fd2fe49SAl Viro 	if (pidfile)
22396fd2fe49SAl Viro 		fd_install(pidfd, pidfile);
22401da177e4SLinus Torvalds 
22412c470475SEric W. Biederman 	init_task_pid_links(p);
224273b9ebfeSOleg Nesterov 	if (likely(p->pid)) {
22434b9d33e6STejun Heo 		ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
22441da177e4SLinus Torvalds 
224581907739SOleg Nesterov 		init_task_pid(p, PIDTYPE_PID, pid);
22461da177e4SLinus Torvalds 		if (thread_group_leader(p)) {
22476883f81aSEric W. Biederman 			init_task_pid(p, PIDTYPE_TGID, pid);
224881907739SOleg Nesterov 			init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
224981907739SOleg Nesterov 			init_task_pid(p, PIDTYPE_SID, task_session(current));
225081907739SOleg Nesterov 
22511c4042c2SEric W. Biederman 			if (is_child_reaper(pid)) {
225217cf22c3SEric W. Biederman 				ns_of_pid(pid)->child_reaper = p;
22531c4042c2SEric W. Biederman 				p->signal->flags |= SIGNAL_UNKILLABLE;
22541c4042c2SEric W. Biederman 			}
2255c3ad2c3bSEric W. Biederman 			p->signal->shared_pending.signal = delayed.signal;
22569c9f4dedSAlan Cox 			p->signal->tty = tty_kref_get(current->signal->tty);
2257749860ceSPavel Tikhomirov 			/*
2258749860ceSPavel Tikhomirov 			 * Inherit has_child_subreaper flag under the same
2259749860ceSPavel Tikhomirov 			 * tasklist_lock with adding child to the process tree
2260749860ceSPavel Tikhomirov 			 * for propagate_has_child_subreaper optimization.
2261749860ceSPavel Tikhomirov 			 */
2262749860ceSPavel Tikhomirov 			p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||
2263749860ceSPavel Tikhomirov 							 p->real_parent->signal->is_child_subreaper;
22649cd80bbbSOleg Nesterov 			list_add_tail(&p->sibling, &p->real_parent->children);
22655e85d4abSEric W. Biederman 			list_add_tail_rcu(&p->tasks, &init_task.tasks);
22666883f81aSEric W. Biederman 			attach_pid(p, PIDTYPE_TGID);
226781907739SOleg Nesterov 			attach_pid(p, PIDTYPE_PGID);
226881907739SOleg Nesterov 			attach_pid(p, PIDTYPE_SID);
2269909ea964SChristoph Lameter 			__this_cpu_inc(process_counts);
227080628ca0SOleg Nesterov 		} else {
227180628ca0SOleg Nesterov 			current->signal->nr_threads++;
227280628ca0SOleg Nesterov 			atomic_inc(&current->signal->live);
227360d4de3fSElena Reshetova 			refcount_inc(&current->signal->sigcnt);
2274924de3b8SEric W. Biederman 			task_join_group_stop(p);
227580628ca0SOleg Nesterov 			list_add_tail_rcu(&p->thread_group,
227680628ca0SOleg Nesterov 					  &p->group_leader->thread_group);
22770c740d0aSOleg Nesterov 			list_add_tail_rcu(&p->thread_node,
22780c740d0aSOleg Nesterov 					  &p->signal->thread_head);
22791da177e4SLinus Torvalds 		}
228081907739SOleg Nesterov 		attach_pid(p, PIDTYPE_PID);
22811da177e4SLinus Torvalds 		nr_threads++;
228273b9ebfeSOleg Nesterov 	}
22831da177e4SLinus Torvalds 	total_forks++;
2284c3ad2c3bSEric W. Biederman 	hlist_del_init(&delayed.node);
22853f17da69SOleg Nesterov 	spin_unlock(&current->sighand->siglock);
22864af4206bSOleg Nesterov 	syscall_tracepoint_update(p);
22871da177e4SLinus Torvalds 	write_unlock_irq(&tasklist_lock);
22884af4206bSOleg Nesterov 
2289c13cf856SAndrew Morton 	proc_fork_connector(p);
2290*ef2c41cfSChristian Brauner 	cgroup_post_fork(p, args);
2291cdd6c482SIngo Molnar 	perf_event_fork(p);
229243d2b113SKAMEZAWA Hiroyuki 
229343d2b113SKAMEZAWA Hiroyuki 	trace_task_newtask(p, clone_flags);
22943ab67966SOleg Nesterov 	uprobe_copy_process(p, clone_flags);
229543d2b113SKAMEZAWA Hiroyuki 
22961da177e4SLinus Torvalds 	return p;
22971da177e4SLinus Torvalds 
22987e47682eSAleksa Sarai bad_fork_cancel_cgroup:
22993fd37226SKirill Tkhai 	spin_unlock(&current->sighand->siglock);
23003fd37226SKirill Tkhai 	write_unlock_irq(&tasklist_lock);
2301*ef2c41cfSChristian Brauner 	cgroup_cancel_fork(p, args);
2302b3e58382SChristian Brauner bad_fork_put_pidfd:
23036fd2fe49SAl Viro 	if (clone_flags & CLONE_PIDFD) {
23046fd2fe49SAl Viro 		fput(pidfile);
23056fd2fe49SAl Viro 		put_unused_fd(pidfd);
23066fd2fe49SAl Viro 	}
2307425fb2b4SPavel Emelyanov bad_fork_free_pid:
2308425fb2b4SPavel Emelyanov 	if (pid != &init_struct_pid)
2309425fb2b4SPavel Emelyanov 		free_pid(pid);
23100740aa5fSJiri Slaby bad_fork_cleanup_thread:
23110740aa5fSJiri Slaby 	exit_thread(p);
2312fd0928dfSJens Axboe bad_fork_cleanup_io:
2313b69f2292SLouis Rilling 	if (p->io_context)
2314b69f2292SLouis Rilling 		exit_io_context(p);
2315ab516013SSerge E. Hallyn bad_fork_cleanup_namespaces:
2316444f378bSLinus Torvalds 	exit_task_namespaces(p);
23171da177e4SLinus Torvalds bad_fork_cleanup_mm:
2318c3f3ce04SAndrea Arcangeli 	if (p->mm) {
2319c3f3ce04SAndrea Arcangeli 		mm_clear_owner(p->mm, p);
23201da177e4SLinus Torvalds 		mmput(p->mm);
2321c3f3ce04SAndrea Arcangeli 	}
23221da177e4SLinus Torvalds bad_fork_cleanup_signal:
23234ab6c083SOleg Nesterov 	if (!(clone_flags & CLONE_THREAD))
23241c5354deSMike Galbraith 		free_signal_struct(p->signal);
23251da177e4SLinus Torvalds bad_fork_cleanup_sighand:
2326a7e5328aSOleg Nesterov 	__cleanup_sighand(p->sighand);
23271da177e4SLinus Torvalds bad_fork_cleanup_fs:
23281da177e4SLinus Torvalds 	exit_fs(p); /* blocking */
23291da177e4SLinus Torvalds bad_fork_cleanup_files:
23301da177e4SLinus Torvalds 	exit_files(p); /* blocking */
23311da177e4SLinus Torvalds bad_fork_cleanup_semundo:
23321da177e4SLinus Torvalds 	exit_sem(p);
2333e4e55b47STetsuo Handa bad_fork_cleanup_security:
2334e4e55b47STetsuo Handa 	security_task_free(p);
23351da177e4SLinus Torvalds bad_fork_cleanup_audit:
23361da177e4SLinus Torvalds 	audit_free(p);
23376c72e350SPeter Zijlstra bad_fork_cleanup_perf:
2338cdd6c482SIngo Molnar 	perf_event_free_task(p);
23396c72e350SPeter Zijlstra bad_fork_cleanup_policy:
2340b09be676SByungchul Park 	lockdep_free_task(p);
23411da177e4SLinus Torvalds #ifdef CONFIG_NUMA
2342f0be3d32SLee Schermerhorn 	mpol_put(p->mempolicy);
2343e8604cb4SLi Zefan bad_fork_cleanup_threadgroup_lock:
23441da177e4SLinus Torvalds #endif
234535df17c5SShailabh Nagar 	delayacct_tsk_free(p);
23461da177e4SLinus Torvalds bad_fork_cleanup_count:
2347d84f4f99SDavid Howells 	atomic_dec(&p->cred->user->processes);
2348e0e81739SDavid Howells 	exit_creds(p);
23491da177e4SLinus Torvalds bad_fork_free:
2350405c0759SAndy Lutomirski 	p->state = TASK_DEAD;
235168f24b08SAndy Lutomirski 	put_task_stack(p);
2352c3f3ce04SAndrea Arcangeli 	delayed_free_task(p);
2353fe7d37d1SOleg Nesterov fork_out:
2354c3ad2c3bSEric W. Biederman 	spin_lock_irq(&current->sighand->siglock);
2355c3ad2c3bSEric W. Biederman 	hlist_del_init(&delayed.node);
2356c3ad2c3bSEric W. Biederman 	spin_unlock_irq(&current->sighand->siglock);
2357fe7d37d1SOleg Nesterov 	return ERR_PTR(retval);
23581da177e4SLinus Torvalds }
23591da177e4SLinus Torvalds 
23602c470475SEric W. Biederman static inline void init_idle_pids(struct task_struct *idle)
2361f106eee1SOleg Nesterov {
2362f106eee1SOleg Nesterov 	enum pid_type type;
2363f106eee1SOleg Nesterov 
2364f106eee1SOleg Nesterov 	for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
23652c470475SEric W. Biederman 		INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */
23662c470475SEric W. Biederman 		init_task_pid(idle, type, &init_struct_pid);
2367f106eee1SOleg Nesterov 	}
2368f106eee1SOleg Nesterov }
2369f106eee1SOleg Nesterov 
23700db0628dSPaul Gortmaker struct task_struct *fork_idle(int cpu)
23711da177e4SLinus Torvalds {
237236c8b586SIngo Molnar 	struct task_struct *task;
23737f192e3cSChristian Brauner 	struct kernel_clone_args args = {
23747f192e3cSChristian Brauner 		.flags = CLONE_VM,
23757f192e3cSChristian Brauner 	};
23767f192e3cSChristian Brauner 
23777f192e3cSChristian Brauner 	task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args);
2378f106eee1SOleg Nesterov 	if (!IS_ERR(task)) {
23792c470475SEric W. Biederman 		init_idle_pids(task);
23801da177e4SLinus Torvalds 		init_idle(task, cpu);
2381f106eee1SOleg Nesterov 	}
238273b9ebfeSOleg Nesterov 
23831da177e4SLinus Torvalds 	return task;
23841da177e4SLinus Torvalds }
23851da177e4SLinus Torvalds 
238613585fa0SNadav Amit struct mm_struct *copy_init_mm(void)
238713585fa0SNadav Amit {
238813585fa0SNadav Amit 	return dup_mm(NULL, &init_mm);
238913585fa0SNadav Amit }
239013585fa0SNadav Amit 
23911da177e4SLinus Torvalds /*
23921da177e4SLinus Torvalds  *  Ok, this is the main fork-routine.
23931da177e4SLinus Torvalds  *
23941da177e4SLinus Torvalds  * It copies the process, and if successful kick-starts
23951da177e4SLinus Torvalds  * it and waits for it to finish using the VM if required.
2396a0eb9abdSEugene Syromiatnikov  *
2397a0eb9abdSEugene Syromiatnikov  * args->exit_signal is expected to be checked for sanity by the caller.
23981da177e4SLinus Torvalds  */
23997f192e3cSChristian Brauner long _do_fork(struct kernel_clone_args *args)
24001da177e4SLinus Torvalds {
24017f192e3cSChristian Brauner 	u64 clone_flags = args->flags;
24029f5325aaSMarcos Paulo de Souza 	struct completion vfork;
24039f5325aaSMarcos Paulo de Souza 	struct pid *pid;
24041da177e4SLinus Torvalds 	struct task_struct *p;
24051da177e4SLinus Torvalds 	int trace = 0;
240692476d7fSEric W. Biederman 	long nr;
24071da177e4SLinus Torvalds 
2408bdff746aSAndrew Morton 	/*
24094b9d33e6STejun Heo 	 * Determine whether and which event to report to ptracer.  When
24104b9d33e6STejun Heo 	 * called from kernel_thread or CLONE_UNTRACED is explicitly
24114b9d33e6STejun Heo 	 * requested, no event is reported; otherwise, report if the event
24124b9d33e6STejun Heo 	 * for the type of forking is enabled.
241309a05394SRoland McGrath 	 */
2414e80d6661SAl Viro 	if (!(clone_flags & CLONE_UNTRACED)) {
24154b9d33e6STejun Heo 		if (clone_flags & CLONE_VFORK)
24164b9d33e6STejun Heo 			trace = PTRACE_EVENT_VFORK;
24177f192e3cSChristian Brauner 		else if (args->exit_signal != SIGCHLD)
24184b9d33e6STejun Heo 			trace = PTRACE_EVENT_CLONE;
24194b9d33e6STejun Heo 		else
24204b9d33e6STejun Heo 			trace = PTRACE_EVENT_FORK;
24214b9d33e6STejun Heo 
24224b9d33e6STejun Heo 		if (likely(!ptrace_event_enabled(current, trace)))
24234b9d33e6STejun Heo 			trace = 0;
24244b9d33e6STejun Heo 	}
24251da177e4SLinus Torvalds 
24267f192e3cSChristian Brauner 	p = copy_process(NULL, trace, NUMA_NO_NODE, args);
242738addce8SEmese Revfy 	add_latent_entropy();
24289f5325aaSMarcos Paulo de Souza 
24299f5325aaSMarcos Paulo de Souza 	if (IS_ERR(p))
24309f5325aaSMarcos Paulo de Souza 		return PTR_ERR(p);
24319f5325aaSMarcos Paulo de Souza 
24321da177e4SLinus Torvalds 	/*
24331da177e4SLinus Torvalds 	 * Do this prior waking up the new thread - the thread pointer
24341da177e4SLinus Torvalds 	 * might get invalid after that point, if the thread exits quickly.
24351da177e4SLinus Torvalds 	 */
24360a16b607SMathieu Desnoyers 	trace_sched_process_fork(current, p);
24370a16b607SMathieu Desnoyers 
24384e52365fSMatthew Dempsky 	pid = get_task_pid(p, PIDTYPE_PID);
24394e52365fSMatthew Dempsky 	nr = pid_vnr(pid);
244030e49c26SPavel Emelyanov 
244130e49c26SPavel Emelyanov 	if (clone_flags & CLONE_PARENT_SETTID)
24427f192e3cSChristian Brauner 		put_user(nr, args->parent_tid);
2443a6f5e063SSukadev Bhattiprolu 
24441da177e4SLinus Torvalds 	if (clone_flags & CLONE_VFORK) {
24451da177e4SLinus Torvalds 		p->vfork_done = &vfork;
24461da177e4SLinus Torvalds 		init_completion(&vfork);
2447d68b46feSOleg Nesterov 		get_task_struct(p);
24481da177e4SLinus Torvalds 	}
24491da177e4SLinus Torvalds 
24503e51e3edSSamir Bellabes 	wake_up_new_task(p);
24511da177e4SLinus Torvalds 
24524b9d33e6STejun Heo 	/* forking complete and child started to run, tell ptracer */
24534b9d33e6STejun Heo 	if (unlikely(trace))
24544e52365fSMatthew Dempsky 		ptrace_event_pid(trace, pid);
245509a05394SRoland McGrath 
24561da177e4SLinus Torvalds 	if (clone_flags & CLONE_VFORK) {
2457d68b46feSOleg Nesterov 		if (!wait_for_vfork_done(p, &vfork))
24584e52365fSMatthew Dempsky 			ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
24599f59ce5dSChuck Ebbert 	}
24604e52365fSMatthew Dempsky 
24614e52365fSMatthew Dempsky 	put_pid(pid);
246292476d7fSEric W. Biederman 	return nr;
24631da177e4SLinus Torvalds }
24641da177e4SLinus Torvalds 
2465028b6e8aSDmitry V. Levin bool legacy_clone_args_valid(const struct kernel_clone_args *kargs)
2466028b6e8aSDmitry V. Levin {
2467028b6e8aSDmitry V. Levin 	/* clone(CLONE_PIDFD) uses parent_tidptr to return a pidfd */
2468028b6e8aSDmitry V. Levin 	if ((kargs->flags & CLONE_PIDFD) &&
2469028b6e8aSDmitry V. Levin 	    (kargs->flags & CLONE_PARENT_SETTID))
2470028b6e8aSDmitry V. Levin 		return false;
2471028b6e8aSDmitry V. Levin 
2472028b6e8aSDmitry V. Levin 	return true;
2473028b6e8aSDmitry V. Levin }
2474028b6e8aSDmitry V. Levin 
24753033f14aSJosh Triplett #ifndef CONFIG_HAVE_COPY_THREAD_TLS
24763033f14aSJosh Triplett /* For compatibility with architectures that call do_fork directly rather than
24773033f14aSJosh Triplett  * using the syscall entry points below. */
24783033f14aSJosh Triplett long do_fork(unsigned long clone_flags,
24793033f14aSJosh Triplett 	      unsigned long stack_start,
24803033f14aSJosh Triplett 	      unsigned long stack_size,
24813033f14aSJosh Triplett 	      int __user *parent_tidptr,
24823033f14aSJosh Triplett 	      int __user *child_tidptr)
24833033f14aSJosh Triplett {
24847f192e3cSChristian Brauner 	struct kernel_clone_args args = {
24857f192e3cSChristian Brauner 		.flags		= (clone_flags & ~CSIGNAL),
2486028b6e8aSDmitry V. Levin 		.pidfd		= parent_tidptr,
24877f192e3cSChristian Brauner 		.child_tid	= child_tidptr,
24887f192e3cSChristian Brauner 		.parent_tid	= parent_tidptr,
24897f192e3cSChristian Brauner 		.exit_signal	= (clone_flags & CSIGNAL),
24907f192e3cSChristian Brauner 		.stack		= stack_start,
24917f192e3cSChristian Brauner 		.stack_size	= stack_size,
24927f192e3cSChristian Brauner 	};
24937f192e3cSChristian Brauner 
2494028b6e8aSDmitry V. Levin 	if (!legacy_clone_args_valid(&args))
2495028b6e8aSDmitry V. Levin 		return -EINVAL;
2496028b6e8aSDmitry V. Levin 
24977f192e3cSChristian Brauner 	return _do_fork(&args);
24983033f14aSJosh Triplett }
24993033f14aSJosh Triplett #endif
25003033f14aSJosh Triplett 
25012aa3a7f8SAl Viro /*
25022aa3a7f8SAl Viro  * Create a kernel thread.
25032aa3a7f8SAl Viro  */
25042aa3a7f8SAl Viro pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
25052aa3a7f8SAl Viro {
25067f192e3cSChristian Brauner 	struct kernel_clone_args args = {
25077f192e3cSChristian Brauner 		.flags		= ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL),
25087f192e3cSChristian Brauner 		.exit_signal	= (flags & CSIGNAL),
25097f192e3cSChristian Brauner 		.stack		= (unsigned long)fn,
25107f192e3cSChristian Brauner 		.stack_size	= (unsigned long)arg,
25117f192e3cSChristian Brauner 	};
25127f192e3cSChristian Brauner 
25137f192e3cSChristian Brauner 	return _do_fork(&args);
25142aa3a7f8SAl Viro }
25152aa3a7f8SAl Viro 
2516d2125043SAl Viro #ifdef __ARCH_WANT_SYS_FORK
2517d2125043SAl Viro SYSCALL_DEFINE0(fork)
2518d2125043SAl Viro {
2519d2125043SAl Viro #ifdef CONFIG_MMU
25207f192e3cSChristian Brauner 	struct kernel_clone_args args = {
25217f192e3cSChristian Brauner 		.exit_signal = SIGCHLD,
25227f192e3cSChristian Brauner 	};
25237f192e3cSChristian Brauner 
25247f192e3cSChristian Brauner 	return _do_fork(&args);
2525d2125043SAl Viro #else
2526d2125043SAl Viro 	/* can not support in nommu mode */
25275d59e182SDaeseok Youn 	return -EINVAL;
2528d2125043SAl Viro #endif
2529d2125043SAl Viro }
2530d2125043SAl Viro #endif
2531d2125043SAl Viro 
2532d2125043SAl Viro #ifdef __ARCH_WANT_SYS_VFORK
2533d2125043SAl Viro SYSCALL_DEFINE0(vfork)
2534d2125043SAl Viro {
25357f192e3cSChristian Brauner 	struct kernel_clone_args args = {
25367f192e3cSChristian Brauner 		.flags		= CLONE_VFORK | CLONE_VM,
25377f192e3cSChristian Brauner 		.exit_signal	= SIGCHLD,
25387f192e3cSChristian Brauner 	};
25397f192e3cSChristian Brauner 
25407f192e3cSChristian Brauner 	return _do_fork(&args);
2541d2125043SAl Viro }
2542d2125043SAl Viro #endif
2543d2125043SAl Viro 
2544d2125043SAl Viro #ifdef __ARCH_WANT_SYS_CLONE
2545d2125043SAl Viro #ifdef CONFIG_CLONE_BACKWARDS
2546d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2547d2125043SAl Viro 		 int __user *, parent_tidptr,
25483033f14aSJosh Triplett 		 unsigned long, tls,
2549d2125043SAl Viro 		 int __user *, child_tidptr)
2550d2125043SAl Viro #elif defined(CONFIG_CLONE_BACKWARDS2)
2551d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
2552d2125043SAl Viro 		 int __user *, parent_tidptr,
2553d2125043SAl Viro 		 int __user *, child_tidptr,
25543033f14aSJosh Triplett 		 unsigned long, tls)
2555dfa9771aSMichal Simek #elif defined(CONFIG_CLONE_BACKWARDS3)
2556dfa9771aSMichal Simek SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
2557dfa9771aSMichal Simek 		int, stack_size,
2558dfa9771aSMichal Simek 		int __user *, parent_tidptr,
2559dfa9771aSMichal Simek 		int __user *, child_tidptr,
25603033f14aSJosh Triplett 		unsigned long, tls)
2561d2125043SAl Viro #else
2562d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2563d2125043SAl Viro 		 int __user *, parent_tidptr,
2564d2125043SAl Viro 		 int __user *, child_tidptr,
25653033f14aSJosh Triplett 		 unsigned long, tls)
2566d2125043SAl Viro #endif
2567d2125043SAl Viro {
25687f192e3cSChristian Brauner 	struct kernel_clone_args args = {
25697f192e3cSChristian Brauner 		.flags		= (clone_flags & ~CSIGNAL),
25707f192e3cSChristian Brauner 		.pidfd		= parent_tidptr,
25717f192e3cSChristian Brauner 		.child_tid	= child_tidptr,
25727f192e3cSChristian Brauner 		.parent_tid	= parent_tidptr,
25737f192e3cSChristian Brauner 		.exit_signal	= (clone_flags & CSIGNAL),
25747f192e3cSChristian Brauner 		.stack		= newsp,
25757f192e3cSChristian Brauner 		.tls		= tls,
25767f192e3cSChristian Brauner 	};
25777f192e3cSChristian Brauner 
2578028b6e8aSDmitry V. Levin 	if (!legacy_clone_args_valid(&args))
25797f192e3cSChristian Brauner 		return -EINVAL;
25807f192e3cSChristian Brauner 
25817f192e3cSChristian Brauner 	return _do_fork(&args);
25827f192e3cSChristian Brauner }
2583d68dbb0cSChristian Brauner #endif
25847f192e3cSChristian Brauner 
2585d68dbb0cSChristian Brauner #ifdef __ARCH_WANT_SYS_CLONE3
2586dd499f7aSAmanieu d'Antras 
2587dd499f7aSAmanieu d'Antras /*
2588dd499f7aSAmanieu d'Antras  * copy_thread implementations handle CLONE_SETTLS by reading the TLS value from
2589dd499f7aSAmanieu d'Antras  * the registers containing the syscall arguments for clone. This doesn't work
2590dd499f7aSAmanieu d'Antras  * with clone3 since the TLS value is passed in clone_args instead.
2591dd499f7aSAmanieu d'Antras  */
2592dd499f7aSAmanieu d'Antras #ifndef CONFIG_HAVE_COPY_THREAD_TLS
2593dd499f7aSAmanieu d'Antras #error clone3 requires copy_thread_tls support in arch
2594dd499f7aSAmanieu d'Antras #endif
2595dd499f7aSAmanieu d'Antras 
25967f192e3cSChristian Brauner noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs,
25977f192e3cSChristian Brauner 					      struct clone_args __user *uargs,
2598f14c234bSAleksa Sarai 					      size_t usize)
25997f192e3cSChristian Brauner {
2600f14c234bSAleksa Sarai 	int err;
26017f192e3cSChristian Brauner 	struct clone_args args;
260249cb2fc4SAdrian Reber 	pid_t *kset_tid = kargs->set_tid;
26037f192e3cSChristian Brauner 
2604f14c234bSAleksa Sarai 	if (unlikely(usize > PAGE_SIZE))
26057f192e3cSChristian Brauner 		return -E2BIG;
2606f14c234bSAleksa Sarai 	if (unlikely(usize < CLONE_ARGS_SIZE_VER0))
26077f192e3cSChristian Brauner 		return -EINVAL;
26087f192e3cSChristian Brauner 
2609f14c234bSAleksa Sarai 	err = copy_struct_from_user(&args, sizeof(args), uargs, usize);
2610f14c234bSAleksa Sarai 	if (err)
2611f14c234bSAleksa Sarai 		return err;
26127f192e3cSChristian Brauner 
261349cb2fc4SAdrian Reber 	if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL))
261449cb2fc4SAdrian Reber 		return -EINVAL;
261549cb2fc4SAdrian Reber 
261649cb2fc4SAdrian Reber 	if (unlikely(!args.set_tid && args.set_tid_size > 0))
261749cb2fc4SAdrian Reber 		return -EINVAL;
261849cb2fc4SAdrian Reber 
261949cb2fc4SAdrian Reber 	if (unlikely(args.set_tid && args.set_tid_size == 0))
262049cb2fc4SAdrian Reber 		return -EINVAL;
262149cb2fc4SAdrian Reber 
2622a0eb9abdSEugene Syromiatnikov 	/*
2623a0eb9abdSEugene Syromiatnikov 	 * Verify that higher 32bits of exit_signal are unset and that
2624a0eb9abdSEugene Syromiatnikov 	 * it is a valid signal
2625a0eb9abdSEugene Syromiatnikov 	 */
2626a0eb9abdSEugene Syromiatnikov 	if (unlikely((args.exit_signal & ~((u64)CSIGNAL)) ||
2627a0eb9abdSEugene Syromiatnikov 		     !valid_signal(args.exit_signal)))
2628a0eb9abdSEugene Syromiatnikov 		return -EINVAL;
2629a0eb9abdSEugene Syromiatnikov 
2630*ef2c41cfSChristian Brauner 	if ((args.flags & CLONE_INTO_CGROUP) && args.cgroup < 0)
2631*ef2c41cfSChristian Brauner 		return -EINVAL;
2632*ef2c41cfSChristian Brauner 
26337f192e3cSChristian Brauner 	*kargs = (struct kernel_clone_args){
26347f192e3cSChristian Brauner 		.flags		= args.flags,
26357f192e3cSChristian Brauner 		.pidfd		= u64_to_user_ptr(args.pidfd),
26367f192e3cSChristian Brauner 		.child_tid	= u64_to_user_ptr(args.child_tid),
26377f192e3cSChristian Brauner 		.parent_tid	= u64_to_user_ptr(args.parent_tid),
26387f192e3cSChristian Brauner 		.exit_signal	= args.exit_signal,
26397f192e3cSChristian Brauner 		.stack		= args.stack,
26407f192e3cSChristian Brauner 		.stack_size	= args.stack_size,
26417f192e3cSChristian Brauner 		.tls		= args.tls,
264249cb2fc4SAdrian Reber 		.set_tid_size	= args.set_tid_size,
2643*ef2c41cfSChristian Brauner 		.cgroup		= args.cgroup,
26447f192e3cSChristian Brauner 	};
26457f192e3cSChristian Brauner 
264649cb2fc4SAdrian Reber 	if (args.set_tid &&
264749cb2fc4SAdrian Reber 		copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),
264849cb2fc4SAdrian Reber 			(kargs->set_tid_size * sizeof(pid_t))))
264949cb2fc4SAdrian Reber 		return -EFAULT;
265049cb2fc4SAdrian Reber 
265149cb2fc4SAdrian Reber 	kargs->set_tid = kset_tid;
265249cb2fc4SAdrian Reber 
26537f192e3cSChristian Brauner 	return 0;
26547f192e3cSChristian Brauner }
26557f192e3cSChristian Brauner 
2656fa729c4dSChristian Brauner /**
2657fa729c4dSChristian Brauner  * clone3_stack_valid - check and prepare stack
2658fa729c4dSChristian Brauner  * @kargs: kernel clone args
2659fa729c4dSChristian Brauner  *
2660fa729c4dSChristian Brauner  * Verify that the stack arguments userspace gave us are sane.
2661fa729c4dSChristian Brauner  * In addition, set the stack direction for userspace since it's easy for us to
2662fa729c4dSChristian Brauner  * determine.
2663fa729c4dSChristian Brauner  */
2664fa729c4dSChristian Brauner static inline bool clone3_stack_valid(struct kernel_clone_args *kargs)
2665fa729c4dSChristian Brauner {
2666fa729c4dSChristian Brauner 	if (kargs->stack == 0) {
2667fa729c4dSChristian Brauner 		if (kargs->stack_size > 0)
2668fa729c4dSChristian Brauner 			return false;
2669fa729c4dSChristian Brauner 	} else {
2670fa729c4dSChristian Brauner 		if (kargs->stack_size == 0)
2671fa729c4dSChristian Brauner 			return false;
2672fa729c4dSChristian Brauner 
2673fa729c4dSChristian Brauner 		if (!access_ok((void __user *)kargs->stack, kargs->stack_size))
2674fa729c4dSChristian Brauner 			return false;
2675fa729c4dSChristian Brauner 
2676fa729c4dSChristian Brauner #if !defined(CONFIG_STACK_GROWSUP) && !defined(CONFIG_IA64)
2677fa729c4dSChristian Brauner 		kargs->stack += kargs->stack_size;
2678fa729c4dSChristian Brauner #endif
2679fa729c4dSChristian Brauner 	}
2680fa729c4dSChristian Brauner 
2681fa729c4dSChristian Brauner 	return true;
2682fa729c4dSChristian Brauner }
2683fa729c4dSChristian Brauner 
2684fa729c4dSChristian Brauner static bool clone3_args_valid(struct kernel_clone_args *kargs)
26857f192e3cSChristian Brauner {
2686b612e5dfSChristian Brauner 	/* Verify that no unknown flags are passed along. */
2687*ef2c41cfSChristian Brauner 	if (kargs->flags &
2688*ef2c41cfSChristian Brauner 	    ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND | CLONE_INTO_CGROUP))
26897f192e3cSChristian Brauner 		return false;
26907f192e3cSChristian Brauner 
26917f192e3cSChristian Brauner 	/*
26927f192e3cSChristian Brauner 	 * - make the CLONE_DETACHED bit reuseable for clone3
26937f192e3cSChristian Brauner 	 * - make the CSIGNAL bits reuseable for clone3
26947f192e3cSChristian Brauner 	 */
26957f192e3cSChristian Brauner 	if (kargs->flags & (CLONE_DETACHED | CSIGNAL))
26967f192e3cSChristian Brauner 		return false;
26977f192e3cSChristian Brauner 
2698b612e5dfSChristian Brauner 	if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) ==
2699b612e5dfSChristian Brauner 	    (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND))
2700b612e5dfSChristian Brauner 		return false;
2701b612e5dfSChristian Brauner 
27027f192e3cSChristian Brauner 	if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) &&
27037f192e3cSChristian Brauner 	    kargs->exit_signal)
27047f192e3cSChristian Brauner 		return false;
27057f192e3cSChristian Brauner 
2706fa729c4dSChristian Brauner 	if (!clone3_stack_valid(kargs))
2707fa729c4dSChristian Brauner 		return false;
2708fa729c4dSChristian Brauner 
27097f192e3cSChristian Brauner 	return true;
27107f192e3cSChristian Brauner }
27117f192e3cSChristian Brauner 
2712501bd016SChristian Brauner /**
2713501bd016SChristian Brauner  * clone3 - create a new process with specific properties
2714501bd016SChristian Brauner  * @uargs: argument structure
2715501bd016SChristian Brauner  * @size:  size of @uargs
2716501bd016SChristian Brauner  *
2717501bd016SChristian Brauner  * clone3() is the extensible successor to clone()/clone2().
2718501bd016SChristian Brauner  * It takes a struct as argument that is versioned by its size.
2719501bd016SChristian Brauner  *
2720501bd016SChristian Brauner  * Return: On success, a positive PID for the child process.
2721501bd016SChristian Brauner  *         On error, a negative errno number.
2722501bd016SChristian Brauner  */
27237f192e3cSChristian Brauner SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size)
27247f192e3cSChristian Brauner {
27257f192e3cSChristian Brauner 	int err;
27267f192e3cSChristian Brauner 
27277f192e3cSChristian Brauner 	struct kernel_clone_args kargs;
272849cb2fc4SAdrian Reber 	pid_t set_tid[MAX_PID_NS_LEVEL];
272949cb2fc4SAdrian Reber 
273049cb2fc4SAdrian Reber 	kargs.set_tid = set_tid;
27317f192e3cSChristian Brauner 
27327f192e3cSChristian Brauner 	err = copy_clone_args_from_user(&kargs, uargs, size);
27337f192e3cSChristian Brauner 	if (err)
27347f192e3cSChristian Brauner 		return err;
27357f192e3cSChristian Brauner 
27367f192e3cSChristian Brauner 	if (!clone3_args_valid(&kargs))
27377f192e3cSChristian Brauner 		return -EINVAL;
27387f192e3cSChristian Brauner 
27397f192e3cSChristian Brauner 	return _do_fork(&kargs);
2740d2125043SAl Viro }
2741d2125043SAl Viro #endif
2742d2125043SAl Viro 
27430f1b92cbSOleg Nesterov void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data)
27440f1b92cbSOleg Nesterov {
27450f1b92cbSOleg Nesterov 	struct task_struct *leader, *parent, *child;
27460f1b92cbSOleg Nesterov 	int res;
27470f1b92cbSOleg Nesterov 
27480f1b92cbSOleg Nesterov 	read_lock(&tasklist_lock);
27490f1b92cbSOleg Nesterov 	leader = top = top->group_leader;
27500f1b92cbSOleg Nesterov down:
27510f1b92cbSOleg Nesterov 	for_each_thread(leader, parent) {
27520f1b92cbSOleg Nesterov 		list_for_each_entry(child, &parent->children, sibling) {
27530f1b92cbSOleg Nesterov 			res = visitor(child, data);
27540f1b92cbSOleg Nesterov 			if (res) {
27550f1b92cbSOleg Nesterov 				if (res < 0)
27560f1b92cbSOleg Nesterov 					goto out;
27570f1b92cbSOleg Nesterov 				leader = child;
27580f1b92cbSOleg Nesterov 				goto down;
27590f1b92cbSOleg Nesterov 			}
27600f1b92cbSOleg Nesterov up:
27610f1b92cbSOleg Nesterov 			;
27620f1b92cbSOleg Nesterov 		}
27630f1b92cbSOleg Nesterov 	}
27640f1b92cbSOleg Nesterov 
27650f1b92cbSOleg Nesterov 	if (leader != top) {
27660f1b92cbSOleg Nesterov 		child = leader;
27670f1b92cbSOleg Nesterov 		parent = child->real_parent;
27680f1b92cbSOleg Nesterov 		leader = parent->group_leader;
27690f1b92cbSOleg Nesterov 		goto up;
27700f1b92cbSOleg Nesterov 	}
27710f1b92cbSOleg Nesterov out:
27720f1b92cbSOleg Nesterov 	read_unlock(&tasklist_lock);
27730f1b92cbSOleg Nesterov }
27740f1b92cbSOleg Nesterov 
27755fd63b30SRavikiran G Thirumalai #ifndef ARCH_MIN_MMSTRUCT_ALIGN
27765fd63b30SRavikiran G Thirumalai #define ARCH_MIN_MMSTRUCT_ALIGN 0
27775fd63b30SRavikiran G Thirumalai #endif
27785fd63b30SRavikiran G Thirumalai 
277951cc5068SAlexey Dobriyan static void sighand_ctor(void *data)
2780aa1757f9SOleg Nesterov {
2781aa1757f9SOleg Nesterov 	struct sighand_struct *sighand = data;
2782aa1757f9SOleg Nesterov 
2783aa1757f9SOleg Nesterov 	spin_lock_init(&sighand->siglock);
2784b8fceee1SDavide Libenzi 	init_waitqueue_head(&sighand->signalfd_wqh);
2785fba2afaaSDavide Libenzi }
2786aa1757f9SOleg Nesterov 
27871da177e4SLinus Torvalds void __init proc_caches_init(void)
27881da177e4SLinus Torvalds {
2789c1a2f7f0SRik van Riel 	unsigned int mm_size;
2790c1a2f7f0SRik van Riel 
27911da177e4SLinus Torvalds 	sighand_cachep = kmem_cache_create("sighand_cache",
27921da177e4SLinus Torvalds 			sizeof(struct sighand_struct), 0,
27935f0d5a3aSPaul E. McKenney 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
279475f296d9SLevin, Alexander (Sasha Levin) 			SLAB_ACCOUNT, sighand_ctor);
27951da177e4SLinus Torvalds 	signal_cachep = kmem_cache_create("signal_cache",
27961da177e4SLinus Torvalds 			sizeof(struct signal_struct), 0,
279775f296d9SLevin, Alexander (Sasha Levin) 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
27985d097056SVladimir Davydov 			NULL);
27991da177e4SLinus Torvalds 	files_cachep = kmem_cache_create("files_cache",
28001da177e4SLinus Torvalds 			sizeof(struct files_struct), 0,
280175f296d9SLevin, Alexander (Sasha Levin) 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
28025d097056SVladimir Davydov 			NULL);
28031da177e4SLinus Torvalds 	fs_cachep = kmem_cache_create("fs_cache",
28041da177e4SLinus Torvalds 			sizeof(struct fs_struct), 0,
280575f296d9SLevin, Alexander (Sasha Levin) 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
28065d097056SVladimir Davydov 			NULL);
2807c1a2f7f0SRik van Riel 
28086345d24dSLinus Torvalds 	/*
2809c1a2f7f0SRik van Riel 	 * The mm_cpumask is located at the end of mm_struct, and is
2810c1a2f7f0SRik van Riel 	 * dynamically sized based on the maximum CPU number this system
2811c1a2f7f0SRik van Riel 	 * can have, taking hotplug into account (nr_cpu_ids).
28126345d24dSLinus Torvalds 	 */
2813c1a2f7f0SRik van Riel 	mm_size = sizeof(struct mm_struct) + cpumask_size();
2814c1a2f7f0SRik van Riel 
281507dcd7feSDavid Windsor 	mm_cachep = kmem_cache_create_usercopy("mm_struct",
2816c1a2f7f0SRik van Riel 			mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
281775f296d9SLevin, Alexander (Sasha Levin) 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
281807dcd7feSDavid Windsor 			offsetof(struct mm_struct, saved_auxv),
281907dcd7feSDavid Windsor 			sizeof_field(struct mm_struct, saved_auxv),
28205d097056SVladimir Davydov 			NULL);
28215d097056SVladimir Davydov 	vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT);
28228feae131SDavid Howells 	mmap_init();
282366577193SAl Viro 	nsproxy_cache_init();
28241da177e4SLinus Torvalds }
2825cf2e340fSJANAK DESAI 
2826cf2e340fSJANAK DESAI /*
28279bfb23fcSOleg Nesterov  * Check constraints on flags passed to the unshare system call.
2828cf2e340fSJANAK DESAI  */
28299bfb23fcSOleg Nesterov static int check_unshare_flags(unsigned long unshare_flags)
2830cf2e340fSJANAK DESAI {
28319bfb23fcSOleg Nesterov 	if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
28329bfb23fcSOleg Nesterov 				CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
283350804fe3SEric W. Biederman 				CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET|
2834769071acSAndrei Vagin 				CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP|
2835769071acSAndrei Vagin 				CLONE_NEWTIME))
2836cf2e340fSJANAK DESAI 		return -EINVAL;
28379bfb23fcSOleg Nesterov 	/*
283812c641abSEric W. Biederman 	 * Not implemented, but pretend it works if there is nothing
283912c641abSEric W. Biederman 	 * to unshare.  Note that unsharing the address space or the
284012c641abSEric W. Biederman 	 * signal handlers also need to unshare the signal queues (aka
284112c641abSEric W. Biederman 	 * CLONE_THREAD).
28429bfb23fcSOleg Nesterov 	 */
28439bfb23fcSOleg Nesterov 	if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
284412c641abSEric W. Biederman 		if (!thread_group_empty(current))
284512c641abSEric W. Biederman 			return -EINVAL;
284612c641abSEric W. Biederman 	}
284712c641abSEric W. Biederman 	if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) {
2848d036bda7SElena Reshetova 		if (refcount_read(&current->sighand->count) > 1)
284912c641abSEric W. Biederman 			return -EINVAL;
285012c641abSEric W. Biederman 	}
285112c641abSEric W. Biederman 	if (unshare_flags & CLONE_VM) {
285212c641abSEric W. Biederman 		if (!current_is_single_threaded())
28539bfb23fcSOleg Nesterov 			return -EINVAL;
28549bfb23fcSOleg Nesterov 	}
2855cf2e340fSJANAK DESAI 
2856cf2e340fSJANAK DESAI 	return 0;
2857cf2e340fSJANAK DESAI }
2858cf2e340fSJANAK DESAI 
2859cf2e340fSJANAK DESAI /*
286099d1419dSJANAK DESAI  * Unshare the filesystem structure if it is being shared
2861cf2e340fSJANAK DESAI  */
2862cf2e340fSJANAK DESAI static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
2863cf2e340fSJANAK DESAI {
2864cf2e340fSJANAK DESAI 	struct fs_struct *fs = current->fs;
2865cf2e340fSJANAK DESAI 
2866498052bbSAl Viro 	if (!(unshare_flags & CLONE_FS) || !fs)
2867498052bbSAl Viro 		return 0;
2868498052bbSAl Viro 
2869498052bbSAl Viro 	/* don't need lock here; in the worst case we'll do useless copy */
2870498052bbSAl Viro 	if (fs->users == 1)
2871498052bbSAl Viro 		return 0;
2872498052bbSAl Viro 
2873498052bbSAl Viro 	*new_fsp = copy_fs_struct(fs);
287499d1419dSJANAK DESAI 	if (!*new_fsp)
287599d1419dSJANAK DESAI 		return -ENOMEM;
2876cf2e340fSJANAK DESAI 
2877cf2e340fSJANAK DESAI 	return 0;
2878cf2e340fSJANAK DESAI }
2879cf2e340fSJANAK DESAI 
2880cf2e340fSJANAK DESAI /*
2881a016f338SJANAK DESAI  * Unshare file descriptor table if it is being shared
2882cf2e340fSJANAK DESAI  */
2883cf2e340fSJANAK DESAI static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
2884cf2e340fSJANAK DESAI {
2885cf2e340fSJANAK DESAI 	struct files_struct *fd = current->files;
2886a016f338SJANAK DESAI 	int error = 0;
2887cf2e340fSJANAK DESAI 
2888cf2e340fSJANAK DESAI 	if ((unshare_flags & CLONE_FILES) &&
2889a016f338SJANAK DESAI 	    (fd && atomic_read(&fd->count) > 1)) {
2890a016f338SJANAK DESAI 		*new_fdp = dup_fd(fd, &error);
2891a016f338SJANAK DESAI 		if (!*new_fdp)
2892a016f338SJANAK DESAI 			return error;
2893a016f338SJANAK DESAI 	}
2894cf2e340fSJANAK DESAI 
2895cf2e340fSJANAK DESAI 	return 0;
2896cf2e340fSJANAK DESAI }
2897cf2e340fSJANAK DESAI 
2898cf2e340fSJANAK DESAI /*
2899cf2e340fSJANAK DESAI  * unshare allows a process to 'unshare' part of the process
2900cf2e340fSJANAK DESAI  * context which was originally shared using clone.  copy_*
2901cf2e340fSJANAK DESAI  * functions used by do_fork() cannot be used here directly
2902cf2e340fSJANAK DESAI  * because they modify an inactive task_struct that is being
2903cf2e340fSJANAK DESAI  * constructed. Here we are modifying the current, active,
2904cf2e340fSJANAK DESAI  * task_struct.
2905cf2e340fSJANAK DESAI  */
29069b32105eSDominik Brodowski int ksys_unshare(unsigned long unshare_flags)
2907cf2e340fSJANAK DESAI {
2908cf2e340fSJANAK DESAI 	struct fs_struct *fs, *new_fs = NULL;
2909cf2e340fSJANAK DESAI 	struct files_struct *fd, *new_fd = NULL;
2910b2e0d987SEric W. Biederman 	struct cred *new_cred = NULL;
2911cf7b708cSPavel Emelyanov 	struct nsproxy *new_nsproxy = NULL;
29129edff4abSManfred Spraul 	int do_sysvsem = 0;
29139bfb23fcSOleg Nesterov 	int err;
2914cf2e340fSJANAK DESAI 
291550804fe3SEric W. Biederman 	/*
2916faf00da5SEric W. Biederman 	 * If unsharing a user namespace must also unshare the thread group
2917faf00da5SEric W. Biederman 	 * and unshare the filesystem root and working directories.
2918b2e0d987SEric W. Biederman 	 */
2919b2e0d987SEric W. Biederman 	if (unshare_flags & CLONE_NEWUSER)
2920e66eded8SEric W. Biederman 		unshare_flags |= CLONE_THREAD | CLONE_FS;
2921b2e0d987SEric W. Biederman 	/*
292250804fe3SEric W. Biederman 	 * If unsharing vm, must also unshare signal handlers.
292350804fe3SEric W. Biederman 	 */
292450804fe3SEric W. Biederman 	if (unshare_flags & CLONE_VM)
292550804fe3SEric W. Biederman 		unshare_flags |= CLONE_SIGHAND;
29266013f67fSManfred Spraul 	/*
292712c641abSEric W. Biederman 	 * If unsharing a signal handlers, must also unshare the signal queues.
292812c641abSEric W. Biederman 	 */
292912c641abSEric W. Biederman 	if (unshare_flags & CLONE_SIGHAND)
293012c641abSEric W. Biederman 		unshare_flags |= CLONE_THREAD;
293112c641abSEric W. Biederman 	/*
29329bfb23fcSOleg Nesterov 	 * If unsharing namespace, must also unshare filesystem information.
29339bfb23fcSOleg Nesterov 	 */
29349bfb23fcSOleg Nesterov 	if (unshare_flags & CLONE_NEWNS)
29359bfb23fcSOleg Nesterov 		unshare_flags |= CLONE_FS;
293650804fe3SEric W. Biederman 
293750804fe3SEric W. Biederman 	err = check_unshare_flags(unshare_flags);
293850804fe3SEric W. Biederman 	if (err)
293950804fe3SEric W. Biederman 		goto bad_unshare_out;
29409bfb23fcSOleg Nesterov 	/*
29416013f67fSManfred Spraul 	 * CLONE_NEWIPC must also detach from the undolist: after switching
29426013f67fSManfred Spraul 	 * to a new ipc namespace, the semaphore arrays from the old
29436013f67fSManfred Spraul 	 * namespace are unreachable.
29446013f67fSManfred Spraul 	 */
29456013f67fSManfred Spraul 	if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
29469edff4abSManfred Spraul 		do_sysvsem = 1;
2947fb0a685cSDaniel Rebelo de Oliveira 	err = unshare_fs(unshare_flags, &new_fs);
2948fb0a685cSDaniel Rebelo de Oliveira 	if (err)
29499bfb23fcSOleg Nesterov 		goto bad_unshare_out;
2950fb0a685cSDaniel Rebelo de Oliveira 	err = unshare_fd(unshare_flags, &new_fd);
2951fb0a685cSDaniel Rebelo de Oliveira 	if (err)
29529bfb23fcSOleg Nesterov 		goto bad_unshare_cleanup_fs;
2953b2e0d987SEric W. Biederman 	err = unshare_userns(unshare_flags, &new_cred);
2954fb0a685cSDaniel Rebelo de Oliveira 	if (err)
29559edff4abSManfred Spraul 		goto bad_unshare_cleanup_fd;
2956b2e0d987SEric W. Biederman 	err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
2957b2e0d987SEric W. Biederman 					 new_cred, new_fs);
2958b2e0d987SEric W. Biederman 	if (err)
2959b2e0d987SEric W. Biederman 		goto bad_unshare_cleanup_cred;
2960cf2e340fSJANAK DESAI 
2961b2e0d987SEric W. Biederman 	if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
29629edff4abSManfred Spraul 		if (do_sysvsem) {
29639edff4abSManfred Spraul 			/*
29649edff4abSManfred Spraul 			 * CLONE_SYSVSEM is equivalent to sys_exit().
29659edff4abSManfred Spraul 			 */
29669edff4abSManfred Spraul 			exit_sem(current);
29679edff4abSManfred Spraul 		}
2968ab602f79SJack Miller 		if (unshare_flags & CLONE_NEWIPC) {
2969ab602f79SJack Miller 			/* Orphan segments in old ns (see sem above). */
2970ab602f79SJack Miller 			exit_shm(current);
2971ab602f79SJack Miller 			shm_init_task(current);
2972ab602f79SJack Miller 		}
2973ab516013SSerge E. Hallyn 
29746f977e6bSAlan Cox 		if (new_nsproxy)
2975cf7b708cSPavel Emelyanov 			switch_task_namespaces(current, new_nsproxy);
2976cf2e340fSJANAK DESAI 
2977cf7b708cSPavel Emelyanov 		task_lock(current);
2978cf7b708cSPavel Emelyanov 
2979cf2e340fSJANAK DESAI 		if (new_fs) {
2980cf2e340fSJANAK DESAI 			fs = current->fs;
29812a4419b5SNick Piggin 			spin_lock(&fs->lock);
2982cf2e340fSJANAK DESAI 			current->fs = new_fs;
2983498052bbSAl Viro 			if (--fs->users)
2984498052bbSAl Viro 				new_fs = NULL;
2985498052bbSAl Viro 			else
2986cf2e340fSJANAK DESAI 				new_fs = fs;
29872a4419b5SNick Piggin 			spin_unlock(&fs->lock);
2988cf2e340fSJANAK DESAI 		}
2989cf2e340fSJANAK DESAI 
2990cf2e340fSJANAK DESAI 		if (new_fd) {
2991cf2e340fSJANAK DESAI 			fd = current->files;
2992cf2e340fSJANAK DESAI 			current->files = new_fd;
2993cf2e340fSJANAK DESAI 			new_fd = fd;
2994cf2e340fSJANAK DESAI 		}
2995cf2e340fSJANAK DESAI 
2996cf2e340fSJANAK DESAI 		task_unlock(current);
2997b2e0d987SEric W. Biederman 
2998b2e0d987SEric W. Biederman 		if (new_cred) {
2999b2e0d987SEric W. Biederman 			/* Install the new user namespace */
3000b2e0d987SEric W. Biederman 			commit_creds(new_cred);
3001b2e0d987SEric W. Biederman 			new_cred = NULL;
3002b2e0d987SEric W. Biederman 		}
3003cf2e340fSJANAK DESAI 	}
3004cf2e340fSJANAK DESAI 
3005e4222673SHari Bathini 	perf_event_namespaces(current);
3006e4222673SHari Bathini 
3007b2e0d987SEric W. Biederman bad_unshare_cleanup_cred:
3008b2e0d987SEric W. Biederman 	if (new_cred)
3009b2e0d987SEric W. Biederman 		put_cred(new_cred);
3010cf2e340fSJANAK DESAI bad_unshare_cleanup_fd:
3011cf2e340fSJANAK DESAI 	if (new_fd)
3012cf2e340fSJANAK DESAI 		put_files_struct(new_fd);
3013cf2e340fSJANAK DESAI 
3014cf2e340fSJANAK DESAI bad_unshare_cleanup_fs:
3015cf2e340fSJANAK DESAI 	if (new_fs)
3016498052bbSAl Viro 		free_fs_struct(new_fs);
3017cf2e340fSJANAK DESAI 
3018cf2e340fSJANAK DESAI bad_unshare_out:
3019cf2e340fSJANAK DESAI 	return err;
3020cf2e340fSJANAK DESAI }
30213b125388SAl Viro 
30229b32105eSDominik Brodowski SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
30239b32105eSDominik Brodowski {
30249b32105eSDominik Brodowski 	return ksys_unshare(unshare_flags);
30259b32105eSDominik Brodowski }
30269b32105eSDominik Brodowski 
30273b125388SAl Viro /*
30283b125388SAl Viro  *	Helper to unshare the files of the current task.
30293b125388SAl Viro  *	We don't want to expose copy_files internals to
30303b125388SAl Viro  *	the exec layer of the kernel.
30313b125388SAl Viro  */
30323b125388SAl Viro 
30333b125388SAl Viro int unshare_files(struct files_struct **displaced)
30343b125388SAl Viro {
30353b125388SAl Viro 	struct task_struct *task = current;
303650704516SAl Viro 	struct files_struct *copy = NULL;
30373b125388SAl Viro 	int error;
30383b125388SAl Viro 
30393b125388SAl Viro 	error = unshare_fd(CLONE_FILES, &copy);
30403b125388SAl Viro 	if (error || !copy) {
30413b125388SAl Viro 		*displaced = NULL;
30423b125388SAl Viro 		return error;
30433b125388SAl Viro 	}
30443b125388SAl Viro 	*displaced = task->files;
30453b125388SAl Viro 	task_lock(task);
30463b125388SAl Viro 	task->files = copy;
30473b125388SAl Viro 	task_unlock(task);
30483b125388SAl Viro 	return 0;
30493b125388SAl Viro }
305016db3d3fSHeinrich Schuchardt 
305116db3d3fSHeinrich Schuchardt int sysctl_max_threads(struct ctl_table *table, int write,
305216db3d3fSHeinrich Schuchardt 		       void __user *buffer, size_t *lenp, loff_t *ppos)
305316db3d3fSHeinrich Schuchardt {
305416db3d3fSHeinrich Schuchardt 	struct ctl_table t;
305516db3d3fSHeinrich Schuchardt 	int ret;
305616db3d3fSHeinrich Schuchardt 	int threads = max_threads;
3057b0f53dbcSMichal Hocko 	int min = 1;
305816db3d3fSHeinrich Schuchardt 	int max = MAX_THREADS;
305916db3d3fSHeinrich Schuchardt 
306016db3d3fSHeinrich Schuchardt 	t = *table;
306116db3d3fSHeinrich Schuchardt 	t.data = &threads;
306216db3d3fSHeinrich Schuchardt 	t.extra1 = &min;
306316db3d3fSHeinrich Schuchardt 	t.extra2 = &max;
306416db3d3fSHeinrich Schuchardt 
306516db3d3fSHeinrich Schuchardt 	ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
306616db3d3fSHeinrich Schuchardt 	if (ret || !write)
306716db3d3fSHeinrich Schuchardt 		return ret;
306816db3d3fSHeinrich Schuchardt 
3069b0f53dbcSMichal Hocko 	max_threads = threads;
307016db3d3fSHeinrich Schuchardt 
307116db3d3fSHeinrich Schuchardt 	return 0;
307216db3d3fSHeinrich Schuchardt }
3073