xref: /openbmc/linux/kernel/fork.c (revision e02c9b0d65a7493180db45320f82482c6ba8ea57)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  *  linux/kernel/fork.c
31da177e4SLinus Torvalds  *
41da177e4SLinus Torvalds  *  Copyright (C) 1991, 1992  Linus Torvalds
51da177e4SLinus Torvalds  */
61da177e4SLinus Torvalds 
71da177e4SLinus Torvalds /*
81da177e4SLinus Torvalds  *  'fork.c' contains the help-routines for the 'fork' system call
91da177e4SLinus Torvalds  * (see also entry.S and others).
101da177e4SLinus Torvalds  * Fork is rather simple, once you get the hang of it, but the memory
111da177e4SLinus Torvalds  * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
121da177e4SLinus Torvalds  */
131da177e4SLinus Torvalds 
14b3e58382SChristian Brauner #include <linux/anon_inodes.h>
151da177e4SLinus Torvalds #include <linux/slab.h>
164eb5aaa3SIngo Molnar #include <linux/sched/autogroup.h>
176e84f315SIngo Molnar #include <linux/sched/mm.h>
18f7ccbae4SIngo Molnar #include <linux/sched/coredump.h>
198703e8a4SIngo Molnar #include <linux/sched/user.h>
206a3827d7SIngo Molnar #include <linux/sched/numa_balancing.h>
2103441a34SIngo Molnar #include <linux/sched/stat.h>
2229930025SIngo Molnar #include <linux/sched/task.h>
2368db0cf1SIngo Molnar #include <linux/sched/task_stack.h>
2432ef5517SIngo Molnar #include <linux/sched/cputime.h>
25b3e58382SChristian Brauner #include <linux/seq_file.h>
26037741a6SIngo Molnar #include <linux/rtmutex.h>
271da177e4SLinus Torvalds #include <linux/init.h>
281da177e4SLinus Torvalds #include <linux/unistd.h>
291da177e4SLinus Torvalds #include <linux/module.h>
301da177e4SLinus Torvalds #include <linux/vmalloc.h>
311da177e4SLinus Torvalds #include <linux/completion.h>
321da177e4SLinus Torvalds #include <linux/personality.h>
331da177e4SLinus Torvalds #include <linux/mempolicy.h>
341da177e4SLinus Torvalds #include <linux/sem.h>
351da177e4SLinus Torvalds #include <linux/file.h>
369f3acc31SAl Viro #include <linux/fdtable.h>
37da9cbc87SJens Axboe #include <linux/iocontext.h>
381da177e4SLinus Torvalds #include <linux/key.h>
391da177e4SLinus Torvalds #include <linux/binfmts.h>
401da177e4SLinus Torvalds #include <linux/mman.h>
41cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h>
42133ff0eaSJérôme Glisse #include <linux/hmm.h>
431da177e4SLinus Torvalds #include <linux/fs.h>
44615d6e87SDavidlohr Bueso #include <linux/mm.h>
45615d6e87SDavidlohr Bueso #include <linux/vmacache.h>
46ab516013SSerge E. Hallyn #include <linux/nsproxy.h>
47c59ede7bSRandy.Dunlap #include <linux/capability.h>
481da177e4SLinus Torvalds #include <linux/cpu.h>
49b4f48b63SPaul Menage #include <linux/cgroup.h>
501da177e4SLinus Torvalds #include <linux/security.h>
51a1e78772SMel Gorman #include <linux/hugetlb.h>
52e2cfabdfSWill Drewry #include <linux/seccomp.h>
531da177e4SLinus Torvalds #include <linux/swap.h>
541da177e4SLinus Torvalds #include <linux/syscalls.h>
551da177e4SLinus Torvalds #include <linux/jiffies.h>
561da177e4SLinus Torvalds #include <linux/futex.h>
578141c7f3SLinus Torvalds #include <linux/compat.h>
58207205a2SEric Dumazet #include <linux/kthread.h>
597c3ab738SAndrew Morton #include <linux/task_io_accounting_ops.h>
60ab2af1f5SDipankar Sarma #include <linux/rcupdate.h>
611da177e4SLinus Torvalds #include <linux/ptrace.h>
621da177e4SLinus Torvalds #include <linux/mount.h>
631da177e4SLinus Torvalds #include <linux/audit.h>
6478fb7466SPavel Emelianov #include <linux/memcontrol.h>
65f201ae23SFrederic Weisbecker #include <linux/ftrace.h>
665e2bf014SMike Galbraith #include <linux/proc_fs.h>
671da177e4SLinus Torvalds #include <linux/profile.h>
681da177e4SLinus Torvalds #include <linux/rmap.h>
69f8af4da3SHugh Dickins #include <linux/ksm.h>
701da177e4SLinus Torvalds #include <linux/acct.h>
71893e26e6SPavel Emelyanov #include <linux/userfaultfd_k.h>
728f0ab514SJay Lan #include <linux/tsacct_kern.h>
739f46080cSMatt Helsley #include <linux/cn_proc.h>
74ba96a0c8SRafael J. Wysocki #include <linux/freezer.h>
75ca74e92bSShailabh Nagar #include <linux/delayacct.h>
76ad4ecbcbSShailabh Nagar #include <linux/taskstats_kern.h>
770a425405SArjan van de Ven #include <linux/random.h>
78522ed776SMiloslav Trmac #include <linux/tty.h>
79fd0928dfSJens Axboe #include <linux/blkdev.h>
805ad4e53bSAl Viro #include <linux/fs_struct.h>
817c9f8861SEric Sandeen #include <linux/magic.h>
82cdd6c482SIngo Molnar #include <linux/perf_event.h>
8342c4ab41SStanislaw Gruszka #include <linux/posix-timers.h>
848e7cac79SAvi Kivity #include <linux/user-return-notifier.h>
853d5992d2SYing Han #include <linux/oom.h>
86ba76149fSAndrea Arcangeli #include <linux/khugepaged.h>
87d80e731eSOleg Nesterov #include <linux/signalfd.h>
880326f5a9SSrikar Dronamraju #include <linux/uprobes.h>
89a27bb332SKent Overstreet #include <linux/aio.h>
9052f5684cSGideon Israel Dsouza #include <linux/compiler.h>
9116db3d3fSHeinrich Schuchardt #include <linux/sysctl.h>
925c9a8750SDmitry Vyukov #include <linux/kcov.h>
93d83a7cb3SJosh Poimboeuf #include <linux/livepatch.h>
9448ac3c18SMark Rutland #include <linux/thread_info.h>
95afaef01cSAlexander Popov #include <linux/stackleak.h>
961da177e4SLinus Torvalds 
971da177e4SLinus Torvalds #include <asm/pgtable.h>
981da177e4SLinus Torvalds #include <asm/pgalloc.h>
997c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
1001da177e4SLinus Torvalds #include <asm/mmu_context.h>
1011da177e4SLinus Torvalds #include <asm/cacheflush.h>
1021da177e4SLinus Torvalds #include <asm/tlbflush.h>
1031da177e4SLinus Torvalds 
104ad8d75ffSSteven Rostedt #include <trace/events/sched.h>
105ad8d75ffSSteven Rostedt 
10643d2b113SKAMEZAWA Hiroyuki #define CREATE_TRACE_POINTS
10743d2b113SKAMEZAWA Hiroyuki #include <trace/events/task.h>
10843d2b113SKAMEZAWA Hiroyuki 
1091da177e4SLinus Torvalds /*
110ac1b398dSHeinrich Schuchardt  * Minimum number of threads to boot the kernel
111ac1b398dSHeinrich Schuchardt  */
112ac1b398dSHeinrich Schuchardt #define MIN_THREADS 20
113ac1b398dSHeinrich Schuchardt 
114ac1b398dSHeinrich Schuchardt /*
115ac1b398dSHeinrich Schuchardt  * Maximum number of threads
116ac1b398dSHeinrich Schuchardt  */
117ac1b398dSHeinrich Schuchardt #define MAX_THREADS FUTEX_TID_MASK
118ac1b398dSHeinrich Schuchardt 
119ac1b398dSHeinrich Schuchardt /*
1201da177e4SLinus Torvalds  * Protected counters by write_lock_irq(&tasklist_lock)
1211da177e4SLinus Torvalds  */
1221da177e4SLinus Torvalds unsigned long total_forks;	/* Handle normal Linux uptimes. */
1231da177e4SLinus Torvalds int nr_threads;			/* The idle threads do not count.. */
1241da177e4SLinus Torvalds 
1251da177e4SLinus Torvalds int max_threads;		/* tunable limit on nr_threads */
1261da177e4SLinus Torvalds 
1271da177e4SLinus Torvalds DEFINE_PER_CPU(unsigned long, process_counts) = 0;
1281da177e4SLinus Torvalds 
1291da177e4SLinus Torvalds __cacheline_aligned DEFINE_RWLOCK(tasklist_lock);  /* outer */
130db1466b3SPaul E. McKenney 
131db1466b3SPaul E. McKenney #ifdef CONFIG_PROVE_RCU
132db1466b3SPaul E. McKenney int lockdep_tasklist_lock_is_held(void)
133db1466b3SPaul E. McKenney {
134db1466b3SPaul E. McKenney 	return lockdep_is_held(&tasklist_lock);
135db1466b3SPaul E. McKenney }
136db1466b3SPaul E. McKenney EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
137db1466b3SPaul E. McKenney #endif /* #ifdef CONFIG_PROVE_RCU */
1381da177e4SLinus Torvalds 
1391da177e4SLinus Torvalds int nr_processes(void)
1401da177e4SLinus Torvalds {
1411da177e4SLinus Torvalds 	int cpu;
1421da177e4SLinus Torvalds 	int total = 0;
1431da177e4SLinus Torvalds 
1441d510750SIan Campbell 	for_each_possible_cpu(cpu)
1451da177e4SLinus Torvalds 		total += per_cpu(process_counts, cpu);
1461da177e4SLinus Torvalds 
1471da177e4SLinus Torvalds 	return total;
1481da177e4SLinus Torvalds }
1491da177e4SLinus Torvalds 
150f19b9f74SAkinobu Mita void __weak arch_release_task_struct(struct task_struct *tsk)
151f19b9f74SAkinobu Mita {
152f19b9f74SAkinobu Mita }
153f19b9f74SAkinobu Mita 
154f5e10287SThomas Gleixner #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
155e18b890bSChristoph Lameter static struct kmem_cache *task_struct_cachep;
15641101809SThomas Gleixner 
15741101809SThomas Gleixner static inline struct task_struct *alloc_task_struct_node(int node)
15841101809SThomas Gleixner {
15941101809SThomas Gleixner 	return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
16041101809SThomas Gleixner }
16141101809SThomas Gleixner 
16241101809SThomas Gleixner static inline void free_task_struct(struct task_struct *tsk)
16341101809SThomas Gleixner {
16441101809SThomas Gleixner 	kmem_cache_free(task_struct_cachep, tsk);
16541101809SThomas Gleixner }
1661da177e4SLinus Torvalds #endif
1671da177e4SLinus Torvalds 
168b235beeaSLinus Torvalds #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR
16941101809SThomas Gleixner 
1700d15d74aSThomas Gleixner /*
1710d15d74aSThomas Gleixner  * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
1720d15d74aSThomas Gleixner  * kmemcache based allocator.
1730d15d74aSThomas Gleixner  */
174ba14a194SAndy Lutomirski # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
175ac496bf4SAndy Lutomirski 
176ac496bf4SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
177ac496bf4SAndy Lutomirski /*
178ac496bf4SAndy Lutomirski  * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
179ac496bf4SAndy Lutomirski  * flush.  Try to minimize the number of calls by caching stacks.
180ac496bf4SAndy Lutomirski  */
181ac496bf4SAndy Lutomirski #define NR_CACHED_STACKS 2
182ac496bf4SAndy Lutomirski static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
18319659c59SHoeun Ryu 
18419659c59SHoeun Ryu static int free_vm_stack_cache(unsigned int cpu)
18519659c59SHoeun Ryu {
18619659c59SHoeun Ryu 	struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu);
18719659c59SHoeun Ryu 	int i;
18819659c59SHoeun Ryu 
18919659c59SHoeun Ryu 	for (i = 0; i < NR_CACHED_STACKS; i++) {
19019659c59SHoeun Ryu 		struct vm_struct *vm_stack = cached_vm_stacks[i];
19119659c59SHoeun Ryu 
19219659c59SHoeun Ryu 		if (!vm_stack)
19319659c59SHoeun Ryu 			continue;
19419659c59SHoeun Ryu 
19519659c59SHoeun Ryu 		vfree(vm_stack->addr);
19619659c59SHoeun Ryu 		cached_vm_stacks[i] = NULL;
19719659c59SHoeun Ryu 	}
19819659c59SHoeun Ryu 
19919659c59SHoeun Ryu 	return 0;
20019659c59SHoeun Ryu }
201ac496bf4SAndy Lutomirski #endif
202ac496bf4SAndy Lutomirski 
203ba14a194SAndy Lutomirski static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
204b69c49b7SFUJITA Tomonori {
205ba14a194SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
206ac496bf4SAndy Lutomirski 	void *stack;
207ac496bf4SAndy Lutomirski 	int i;
208ac496bf4SAndy Lutomirski 
209ac496bf4SAndy Lutomirski 	for (i = 0; i < NR_CACHED_STACKS; i++) {
210112166f8SChristoph Lameter 		struct vm_struct *s;
211112166f8SChristoph Lameter 
212112166f8SChristoph Lameter 		s = this_cpu_xchg(cached_stacks[i], NULL);
213ac496bf4SAndy Lutomirski 
214ac496bf4SAndy Lutomirski 		if (!s)
215ac496bf4SAndy Lutomirski 			continue;
216ac496bf4SAndy Lutomirski 
217ca182551SKonstantin Khlebnikov 		/* Clear stale pointers from reused stack. */
218ca182551SKonstantin Khlebnikov 		memset(s->addr, 0, THREAD_SIZE);
219e01e8063SKees Cook 
220ac496bf4SAndy Lutomirski 		tsk->stack_vm_area = s;
221ba4a4574SShakeel Butt 		tsk->stack = s->addr;
222ac496bf4SAndy Lutomirski 		return s->addr;
223ac496bf4SAndy Lutomirski 	}
224ac496bf4SAndy Lutomirski 
2259b6f7e16SRoman Gushchin 	/*
2269b6f7e16SRoman Gushchin 	 * Allocated stacks are cached and later reused by new threads,
2279b6f7e16SRoman Gushchin 	 * so memcg accounting is performed manually on assigning/releasing
2289b6f7e16SRoman Gushchin 	 * stacks to tasks. Drop __GFP_ACCOUNT.
2299b6f7e16SRoman Gushchin 	 */
23048ac3c18SMark Rutland 	stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
231ba14a194SAndy Lutomirski 				     VMALLOC_START, VMALLOC_END,
2329b6f7e16SRoman Gushchin 				     THREADINFO_GFP & ~__GFP_ACCOUNT,
233ba14a194SAndy Lutomirski 				     PAGE_KERNEL,
234ac496bf4SAndy Lutomirski 				     0, node, __builtin_return_address(0));
235ba14a194SAndy Lutomirski 
236ba14a194SAndy Lutomirski 	/*
237ba14a194SAndy Lutomirski 	 * We can't call find_vm_area() in interrupt context, and
238ba14a194SAndy Lutomirski 	 * free_thread_stack() can be called in interrupt context,
239ba14a194SAndy Lutomirski 	 * so cache the vm_struct.
240ba14a194SAndy Lutomirski 	 */
2415eed6f1dSRik van Riel 	if (stack) {
242ba14a194SAndy Lutomirski 		tsk->stack_vm_area = find_vm_area(stack);
2435eed6f1dSRik van Riel 		tsk->stack = stack;
2445eed6f1dSRik van Riel 	}
245ba14a194SAndy Lutomirski 	return stack;
246ba14a194SAndy Lutomirski #else
2474949148aSVladimir Davydov 	struct page *page = alloc_pages_node(node, THREADINFO_GFP,
2482889f608SThomas Gleixner 					     THREAD_SIZE_ORDER);
249b6a84016SEric Dumazet 
250b6a84016SEric Dumazet 	return page ? page_address(page) : NULL;
251ba14a194SAndy Lutomirski #endif
252b69c49b7SFUJITA Tomonori }
253b69c49b7SFUJITA Tomonori 
254ba14a194SAndy Lutomirski static inline void free_thread_stack(struct task_struct *tsk)
255b69c49b7SFUJITA Tomonori {
256ac496bf4SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
2579b6f7e16SRoman Gushchin 	struct vm_struct *vm = task_stack_vm_area(tsk);
2589b6f7e16SRoman Gushchin 
2599b6f7e16SRoman Gushchin 	if (vm) {
260ac496bf4SAndy Lutomirski 		int i;
261ac496bf4SAndy Lutomirski 
2629b6f7e16SRoman Gushchin 		for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
2639b6f7e16SRoman Gushchin 			mod_memcg_page_state(vm->pages[i],
2649b6f7e16SRoman Gushchin 					     MEMCG_KERNEL_STACK_KB,
2659b6f7e16SRoman Gushchin 					     -(int)(PAGE_SIZE / 1024));
2669b6f7e16SRoman Gushchin 
2679b6f7e16SRoman Gushchin 			memcg_kmem_uncharge(vm->pages[i], 0);
2689b6f7e16SRoman Gushchin 		}
2699b6f7e16SRoman Gushchin 
270ac496bf4SAndy Lutomirski 		for (i = 0; i < NR_CACHED_STACKS; i++) {
271112166f8SChristoph Lameter 			if (this_cpu_cmpxchg(cached_stacks[i],
272112166f8SChristoph Lameter 					NULL, tsk->stack_vm_area) != NULL)
273ac496bf4SAndy Lutomirski 				continue;
274ac496bf4SAndy Lutomirski 
275ac496bf4SAndy Lutomirski 			return;
276ac496bf4SAndy Lutomirski 		}
277ac496bf4SAndy Lutomirski 
2780f110a9bSAndrey Ryabinin 		vfree_atomic(tsk->stack);
279ac496bf4SAndy Lutomirski 		return;
280ac496bf4SAndy Lutomirski 	}
281ac496bf4SAndy Lutomirski #endif
282ac496bf4SAndy Lutomirski 
283ba14a194SAndy Lutomirski 	__free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
284b69c49b7SFUJITA Tomonori }
2850d15d74aSThomas Gleixner # else
286b235beeaSLinus Torvalds static struct kmem_cache *thread_stack_cache;
2870d15d74aSThomas Gleixner 
2889521d399SMichael Ellerman static unsigned long *alloc_thread_stack_node(struct task_struct *tsk,
2890d15d74aSThomas Gleixner 						  int node)
2900d15d74aSThomas Gleixner {
2915eed6f1dSRik van Riel 	unsigned long *stack;
2925eed6f1dSRik van Riel 	stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
2935eed6f1dSRik van Riel 	tsk->stack = stack;
2945eed6f1dSRik van Riel 	return stack;
2950d15d74aSThomas Gleixner }
2960d15d74aSThomas Gleixner 
297ba14a194SAndy Lutomirski static void free_thread_stack(struct task_struct *tsk)
2980d15d74aSThomas Gleixner {
299ba14a194SAndy Lutomirski 	kmem_cache_free(thread_stack_cache, tsk->stack);
3000d15d74aSThomas Gleixner }
3010d15d74aSThomas Gleixner 
302b235beeaSLinus Torvalds void thread_stack_cache_init(void)
3030d15d74aSThomas Gleixner {
304f9d29946SDavid Windsor 	thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
305f9d29946SDavid Windsor 					THREAD_SIZE, THREAD_SIZE, 0, 0,
306f9d29946SDavid Windsor 					THREAD_SIZE, NULL);
307b235beeaSLinus Torvalds 	BUG_ON(thread_stack_cache == NULL);
3080d15d74aSThomas Gleixner }
3090d15d74aSThomas Gleixner # endif
310b69c49b7SFUJITA Tomonori #endif
311b69c49b7SFUJITA Tomonori 
3121da177e4SLinus Torvalds /* SLAB cache for signal_struct structures (tsk->signal) */
313e18b890bSChristoph Lameter static struct kmem_cache *signal_cachep;
3141da177e4SLinus Torvalds 
3151da177e4SLinus Torvalds /* SLAB cache for sighand_struct structures (tsk->sighand) */
316e18b890bSChristoph Lameter struct kmem_cache *sighand_cachep;
3171da177e4SLinus Torvalds 
3181da177e4SLinus Torvalds /* SLAB cache for files_struct structures (tsk->files) */
319e18b890bSChristoph Lameter struct kmem_cache *files_cachep;
3201da177e4SLinus Torvalds 
3211da177e4SLinus Torvalds /* SLAB cache for fs_struct structures (tsk->fs) */
322e18b890bSChristoph Lameter struct kmem_cache *fs_cachep;
3231da177e4SLinus Torvalds 
3241da177e4SLinus Torvalds /* SLAB cache for vm_area_struct structures */
3253928d4f5SLinus Torvalds static struct kmem_cache *vm_area_cachep;
3261da177e4SLinus Torvalds 
3271da177e4SLinus Torvalds /* SLAB cache for mm_struct structures (tsk->mm) */
328e18b890bSChristoph Lameter static struct kmem_cache *mm_cachep;
3291da177e4SLinus Torvalds 
330490fc053SLinus Torvalds struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
3313928d4f5SLinus Torvalds {
332a670468fSAndrew Morton 	struct vm_area_struct *vma;
333490fc053SLinus Torvalds 
334a670468fSAndrew Morton 	vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
335027232daSKirill A. Shutemov 	if (vma)
336027232daSKirill A. Shutemov 		vma_init(vma, mm);
337490fc053SLinus Torvalds 	return vma;
3383928d4f5SLinus Torvalds }
3393928d4f5SLinus Torvalds 
3403928d4f5SLinus Torvalds struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
3413928d4f5SLinus Torvalds {
34295faf699SLinus Torvalds 	struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
34395faf699SLinus Torvalds 
34495faf699SLinus Torvalds 	if (new) {
34595faf699SLinus Torvalds 		*new = *orig;
34695faf699SLinus Torvalds 		INIT_LIST_HEAD(&new->anon_vma_chain);
34795faf699SLinus Torvalds 	}
34895faf699SLinus Torvalds 	return new;
3493928d4f5SLinus Torvalds }
3503928d4f5SLinus Torvalds 
3513928d4f5SLinus Torvalds void vm_area_free(struct vm_area_struct *vma)
3523928d4f5SLinus Torvalds {
3533928d4f5SLinus Torvalds 	kmem_cache_free(vm_area_cachep, vma);
3543928d4f5SLinus Torvalds }
3553928d4f5SLinus Torvalds 
356ba14a194SAndy Lutomirski static void account_kernel_stack(struct task_struct *tsk, int account)
357c6a7f572SKOSAKI Motohiro {
358ba14a194SAndy Lutomirski 	void *stack = task_stack_page(tsk);
359ba14a194SAndy Lutomirski 	struct vm_struct *vm = task_stack_vm_area(tsk);
360ba14a194SAndy Lutomirski 
361ba14a194SAndy Lutomirski 	BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0);
362ba14a194SAndy Lutomirski 
363ba14a194SAndy Lutomirski 	if (vm) {
364ba14a194SAndy Lutomirski 		int i;
365ba14a194SAndy Lutomirski 
366ba14a194SAndy Lutomirski 		BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
367ba14a194SAndy Lutomirski 
368ba14a194SAndy Lutomirski 		for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
369ba14a194SAndy Lutomirski 			mod_zone_page_state(page_zone(vm->pages[i]),
370ba14a194SAndy Lutomirski 					    NR_KERNEL_STACK_KB,
371ba14a194SAndy Lutomirski 					    PAGE_SIZE / 1024 * account);
372ba14a194SAndy Lutomirski 		}
373ba14a194SAndy Lutomirski 	} else {
374ba14a194SAndy Lutomirski 		/*
375ba14a194SAndy Lutomirski 		 * All stack pages are in the same zone and belong to the
376ba14a194SAndy Lutomirski 		 * same memcg.
377ba14a194SAndy Lutomirski 		 */
378efdc9490SAndy Lutomirski 		struct page *first_page = virt_to_page(stack);
379c6a7f572SKOSAKI Motohiro 
380efdc9490SAndy Lutomirski 		mod_zone_page_state(page_zone(first_page), NR_KERNEL_STACK_KB,
381d30dd8beSAndy Lutomirski 				    THREAD_SIZE / 1024 * account);
382efdc9490SAndy Lutomirski 
383ed52be7bSJohannes Weiner 		mod_memcg_page_state(first_page, MEMCG_KERNEL_STACK_KB,
384efdc9490SAndy Lutomirski 				     account * (THREAD_SIZE / 1024));
385c6a7f572SKOSAKI Motohiro 	}
386ba14a194SAndy Lutomirski }
387c6a7f572SKOSAKI Motohiro 
3889b6f7e16SRoman Gushchin static int memcg_charge_kernel_stack(struct task_struct *tsk)
3899b6f7e16SRoman Gushchin {
3909b6f7e16SRoman Gushchin #ifdef CONFIG_VMAP_STACK
3919b6f7e16SRoman Gushchin 	struct vm_struct *vm = task_stack_vm_area(tsk);
3929b6f7e16SRoman Gushchin 	int ret;
3939b6f7e16SRoman Gushchin 
3949b6f7e16SRoman Gushchin 	if (vm) {
3959b6f7e16SRoman Gushchin 		int i;
3969b6f7e16SRoman Gushchin 
3979b6f7e16SRoman Gushchin 		for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
3989b6f7e16SRoman Gushchin 			/*
3999b6f7e16SRoman Gushchin 			 * If memcg_kmem_charge() fails, page->mem_cgroup
4009b6f7e16SRoman Gushchin 			 * pointer is NULL, and both memcg_kmem_uncharge()
4019b6f7e16SRoman Gushchin 			 * and mod_memcg_page_state() in free_thread_stack()
4029b6f7e16SRoman Gushchin 			 * will ignore this page. So it's safe.
4039b6f7e16SRoman Gushchin 			 */
4049b6f7e16SRoman Gushchin 			ret = memcg_kmem_charge(vm->pages[i], GFP_KERNEL, 0);
4059b6f7e16SRoman Gushchin 			if (ret)
4069b6f7e16SRoman Gushchin 				return ret;
4079b6f7e16SRoman Gushchin 
4089b6f7e16SRoman Gushchin 			mod_memcg_page_state(vm->pages[i],
4099b6f7e16SRoman Gushchin 					     MEMCG_KERNEL_STACK_KB,
4109b6f7e16SRoman Gushchin 					     PAGE_SIZE / 1024);
4119b6f7e16SRoman Gushchin 		}
4129b6f7e16SRoman Gushchin 	}
4139b6f7e16SRoman Gushchin #endif
4149b6f7e16SRoman Gushchin 	return 0;
4159b6f7e16SRoman Gushchin }
4169b6f7e16SRoman Gushchin 
41768f24b08SAndy Lutomirski static void release_task_stack(struct task_struct *tsk)
4181da177e4SLinus Torvalds {
419405c0759SAndy Lutomirski 	if (WARN_ON(tsk->state != TASK_DEAD))
420405c0759SAndy Lutomirski 		return;  /* Better to leak the stack than to free prematurely */
421405c0759SAndy Lutomirski 
422ba14a194SAndy Lutomirski 	account_kernel_stack(tsk, -1);
423ba14a194SAndy Lutomirski 	free_thread_stack(tsk);
42468f24b08SAndy Lutomirski 	tsk->stack = NULL;
42568f24b08SAndy Lutomirski #ifdef CONFIG_VMAP_STACK
42668f24b08SAndy Lutomirski 	tsk->stack_vm_area = NULL;
42768f24b08SAndy Lutomirski #endif
42868f24b08SAndy Lutomirski }
42968f24b08SAndy Lutomirski 
43068f24b08SAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK
43168f24b08SAndy Lutomirski void put_task_stack(struct task_struct *tsk)
43268f24b08SAndy Lutomirski {
433f0b89d39SElena Reshetova 	if (refcount_dec_and_test(&tsk->stack_refcount))
43468f24b08SAndy Lutomirski 		release_task_stack(tsk);
43568f24b08SAndy Lutomirski }
43668f24b08SAndy Lutomirski #endif
43768f24b08SAndy Lutomirski 
43868f24b08SAndy Lutomirski void free_task(struct task_struct *tsk)
43968f24b08SAndy Lutomirski {
44068f24b08SAndy Lutomirski #ifndef CONFIG_THREAD_INFO_IN_TASK
44168f24b08SAndy Lutomirski 	/*
44268f24b08SAndy Lutomirski 	 * The task is finally done with both the stack and thread_info,
44368f24b08SAndy Lutomirski 	 * so free both.
44468f24b08SAndy Lutomirski 	 */
44568f24b08SAndy Lutomirski 	release_task_stack(tsk);
44668f24b08SAndy Lutomirski #else
44768f24b08SAndy Lutomirski 	/*
44868f24b08SAndy Lutomirski 	 * If the task had a separate stack allocation, it should be gone
44968f24b08SAndy Lutomirski 	 * by now.
45068f24b08SAndy Lutomirski 	 */
451f0b89d39SElena Reshetova 	WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
45268f24b08SAndy Lutomirski #endif
45323f78d4aSIngo Molnar 	rt_mutex_debug_task_free(tsk);
454fb52607aSFrederic Weisbecker 	ftrace_graph_exit_task(tsk);
455e2cfabdfSWill Drewry 	put_seccomp_filter(tsk);
456f19b9f74SAkinobu Mita 	arch_release_task_struct(tsk);
4571da5c46fSOleg Nesterov 	if (tsk->flags & PF_KTHREAD)
4581da5c46fSOleg Nesterov 		free_kthread_struct(tsk);
4591da177e4SLinus Torvalds 	free_task_struct(tsk);
4601da177e4SLinus Torvalds }
4611da177e4SLinus Torvalds EXPORT_SYMBOL(free_task);
4621da177e4SLinus Torvalds 
4631da177e4SLinus Torvalds #ifdef CONFIG_MMU
4640766f788SEmese Revfy static __latent_entropy int dup_mmap(struct mm_struct *mm,
4650766f788SEmese Revfy 					struct mm_struct *oldmm)
4661da177e4SLinus Torvalds {
467297c5eeeSLinus Torvalds 	struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
4681da177e4SLinus Torvalds 	struct rb_node **rb_link, *rb_parent;
4691da177e4SLinus Torvalds 	int retval;
4701da177e4SLinus Torvalds 	unsigned long charge;
471893e26e6SPavel Emelyanov 	LIST_HEAD(uf);
4721da177e4SLinus Torvalds 
47332cdba1eSOleg Nesterov 	uprobe_start_dup_mmap();
4747c051267SMichal Hocko 	if (down_write_killable(&oldmm->mmap_sem)) {
4757c051267SMichal Hocko 		retval = -EINTR;
4767c051267SMichal Hocko 		goto fail_uprobe_end;
4777c051267SMichal Hocko 	}
478ec8c0446SRalf Baechle 	flush_cache_dup_mm(oldmm);
479f8ac4ec9SOleg Nesterov 	uprobe_dup_mmap(oldmm, mm);
480ad339451SIngo Molnar 	/*
481ad339451SIngo Molnar 	 * Not linked in yet - no deadlock potential:
482ad339451SIngo Molnar 	 */
483ad339451SIngo Molnar 	down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
4847ee78232SHugh Dickins 
48590f31d0eSKonstantin Khlebnikov 	/* No ordering required: file already has been exposed. */
48690f31d0eSKonstantin Khlebnikov 	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
48790f31d0eSKonstantin Khlebnikov 
4884f7d4614SVladimir Davydov 	mm->total_vm = oldmm->total_vm;
48984638335SKonstantin Khlebnikov 	mm->data_vm = oldmm->data_vm;
4904f7d4614SVladimir Davydov 	mm->exec_vm = oldmm->exec_vm;
4914f7d4614SVladimir Davydov 	mm->stack_vm = oldmm->stack_vm;
4924f7d4614SVladimir Davydov 
4931da177e4SLinus Torvalds 	rb_link = &mm->mm_rb.rb_node;
4941da177e4SLinus Torvalds 	rb_parent = NULL;
4951da177e4SLinus Torvalds 	pprev = &mm->mmap;
496f8af4da3SHugh Dickins 	retval = ksm_fork(mm, oldmm);
497f8af4da3SHugh Dickins 	if (retval)
498f8af4da3SHugh Dickins 		goto out;
499ba76149fSAndrea Arcangeli 	retval = khugepaged_fork(mm, oldmm);
500ba76149fSAndrea Arcangeli 	if (retval)
501ba76149fSAndrea Arcangeli 		goto out;
5021da177e4SLinus Torvalds 
503297c5eeeSLinus Torvalds 	prev = NULL;
504fd3e42fcSHugh Dickins 	for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
5051da177e4SLinus Torvalds 		struct file *file;
5061da177e4SLinus Torvalds 
5071da177e4SLinus Torvalds 		if (mpnt->vm_flags & VM_DONTCOPY) {
50884638335SKonstantin Khlebnikov 			vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
5091da177e4SLinus Torvalds 			continue;
5101da177e4SLinus Torvalds 		}
5111da177e4SLinus Torvalds 		charge = 0;
512655c79bbSTetsuo Handa 		/*
513655c79bbSTetsuo Handa 		 * Don't duplicate many vmas if we've been oom-killed (for
514655c79bbSTetsuo Handa 		 * example)
515655c79bbSTetsuo Handa 		 */
516655c79bbSTetsuo Handa 		if (fatal_signal_pending(current)) {
517655c79bbSTetsuo Handa 			retval = -EINTR;
518655c79bbSTetsuo Handa 			goto out;
519655c79bbSTetsuo Handa 		}
5201da177e4SLinus Torvalds 		if (mpnt->vm_flags & VM_ACCOUNT) {
521b2412b7fSHuang Shijie 			unsigned long len = vma_pages(mpnt);
522b2412b7fSHuang Shijie 
523191c5424SAl Viro 			if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
5241da177e4SLinus Torvalds 				goto fail_nomem;
5251da177e4SLinus Torvalds 			charge = len;
5261da177e4SLinus Torvalds 		}
5273928d4f5SLinus Torvalds 		tmp = vm_area_dup(mpnt);
5281da177e4SLinus Torvalds 		if (!tmp)
5291da177e4SLinus Torvalds 			goto fail_nomem;
530ef0855d3SOleg Nesterov 		retval = vma_dup_policy(mpnt, tmp);
531ef0855d3SOleg Nesterov 		if (retval)
5321da177e4SLinus Torvalds 			goto fail_nomem_policy;
533a247c3a9SAndrea Arcangeli 		tmp->vm_mm = mm;
534893e26e6SPavel Emelyanov 		retval = dup_userfaultfd(tmp, &uf);
535893e26e6SPavel Emelyanov 		if (retval)
536893e26e6SPavel Emelyanov 			goto fail_nomem_anon_vma_fork;
537d2cd9edeSRik van Riel 		if (tmp->vm_flags & VM_WIPEONFORK) {
538d2cd9edeSRik van Riel 			/* VM_WIPEONFORK gets a clean slate in the child. */
539d2cd9edeSRik van Riel 			tmp->anon_vma = NULL;
540d2cd9edeSRik van Riel 			if (anon_vma_prepare(tmp))
541d2cd9edeSRik van Riel 				goto fail_nomem_anon_vma_fork;
542d2cd9edeSRik van Riel 		} else if (anon_vma_fork(tmp, mpnt))
5435beb4930SRik van Riel 			goto fail_nomem_anon_vma_fork;
544893e26e6SPavel Emelyanov 		tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
545297c5eeeSLinus Torvalds 		tmp->vm_next = tmp->vm_prev = NULL;
5461da177e4SLinus Torvalds 		file = tmp->vm_file;
5471da177e4SLinus Torvalds 		if (file) {
548496ad9aaSAl Viro 			struct inode *inode = file_inode(file);
549b88ed205SHugh Dickins 			struct address_space *mapping = file->f_mapping;
550b88ed205SHugh Dickins 
5511da177e4SLinus Torvalds 			get_file(file);
5521da177e4SLinus Torvalds 			if (tmp->vm_flags & VM_DENYWRITE)
5531da177e4SLinus Torvalds 				atomic_dec(&inode->i_writecount);
55483cde9e8SDavidlohr Bueso 			i_mmap_lock_write(mapping);
555b88ed205SHugh Dickins 			if (tmp->vm_flags & VM_SHARED)
5564bb5f5d9SDavid Herrmann 				atomic_inc(&mapping->i_mmap_writable);
557b88ed205SHugh Dickins 			flush_dcache_mmap_lock(mapping);
558b88ed205SHugh Dickins 			/* insert tmp into the share list, just after mpnt */
5599826a516SMichel Lespinasse 			vma_interval_tree_insert_after(tmp, mpnt,
5609826a516SMichel Lespinasse 					&mapping->i_mmap);
561b88ed205SHugh Dickins 			flush_dcache_mmap_unlock(mapping);
56283cde9e8SDavidlohr Bueso 			i_mmap_unlock_write(mapping);
5631da177e4SLinus Torvalds 		}
5641da177e4SLinus Torvalds 
5651da177e4SLinus Torvalds 		/*
566a1e78772SMel Gorman 		 * Clear hugetlb-related page reserves for children. This only
567a1e78772SMel Gorman 		 * affects MAP_PRIVATE mappings. Faults generated by the child
568a1e78772SMel Gorman 		 * are not guaranteed to succeed, even if read-only
569a1e78772SMel Gorman 		 */
570a1e78772SMel Gorman 		if (is_vm_hugetlb_page(tmp))
571a1e78772SMel Gorman 			reset_vma_resv_huge_pages(tmp);
572a1e78772SMel Gorman 
573a1e78772SMel Gorman 		/*
5747ee78232SHugh Dickins 		 * Link in the new vma and copy the page table entries.
5751da177e4SLinus Torvalds 		 */
5761da177e4SLinus Torvalds 		*pprev = tmp;
5771da177e4SLinus Torvalds 		pprev = &tmp->vm_next;
578297c5eeeSLinus Torvalds 		tmp->vm_prev = prev;
579297c5eeeSLinus Torvalds 		prev = tmp;
5801da177e4SLinus Torvalds 
5811da177e4SLinus Torvalds 		__vma_link_rb(mm, tmp, rb_link, rb_parent);
5821da177e4SLinus Torvalds 		rb_link = &tmp->vm_rb.rb_right;
5831da177e4SLinus Torvalds 		rb_parent = &tmp->vm_rb;
5841da177e4SLinus Torvalds 
5851da177e4SLinus Torvalds 		mm->map_count++;
586d2cd9edeSRik van Riel 		if (!(tmp->vm_flags & VM_WIPEONFORK))
5870b0db14cSHugh Dickins 			retval = copy_page_range(mm, oldmm, mpnt);
5881da177e4SLinus Torvalds 
5891da177e4SLinus Torvalds 		if (tmp->vm_ops && tmp->vm_ops->open)
5901da177e4SLinus Torvalds 			tmp->vm_ops->open(tmp);
5911da177e4SLinus Torvalds 
5921da177e4SLinus Torvalds 		if (retval)
5931da177e4SLinus Torvalds 			goto out;
5941da177e4SLinus Torvalds 	}
595d6dd61c8SJeremy Fitzhardinge 	/* a new mm has just been created */
5961ed0cc5aSNadav Amit 	retval = arch_dup_mmap(oldmm, mm);
5971da177e4SLinus Torvalds out:
5987ee78232SHugh Dickins 	up_write(&mm->mmap_sem);
599fd3e42fcSHugh Dickins 	flush_tlb_mm(oldmm);
6001da177e4SLinus Torvalds 	up_write(&oldmm->mmap_sem);
601893e26e6SPavel Emelyanov 	dup_userfaultfd_complete(&uf);
6027c051267SMichal Hocko fail_uprobe_end:
60332cdba1eSOleg Nesterov 	uprobe_end_dup_mmap();
6041da177e4SLinus Torvalds 	return retval;
6055beb4930SRik van Riel fail_nomem_anon_vma_fork:
606ef0855d3SOleg Nesterov 	mpol_put(vma_policy(tmp));
6071da177e4SLinus Torvalds fail_nomem_policy:
6083928d4f5SLinus Torvalds 	vm_area_free(tmp);
6091da177e4SLinus Torvalds fail_nomem:
6101da177e4SLinus Torvalds 	retval = -ENOMEM;
6111da177e4SLinus Torvalds 	vm_unacct_memory(charge);
6121da177e4SLinus Torvalds 	goto out;
6131da177e4SLinus Torvalds }
6141da177e4SLinus Torvalds 
6151da177e4SLinus Torvalds static inline int mm_alloc_pgd(struct mm_struct *mm)
6161da177e4SLinus Torvalds {
6171da177e4SLinus Torvalds 	mm->pgd = pgd_alloc(mm);
6181da177e4SLinus Torvalds 	if (unlikely(!mm->pgd))
6191da177e4SLinus Torvalds 		return -ENOMEM;
6201da177e4SLinus Torvalds 	return 0;
6211da177e4SLinus Torvalds }
6221da177e4SLinus Torvalds 
6231da177e4SLinus Torvalds static inline void mm_free_pgd(struct mm_struct *mm)
6241da177e4SLinus Torvalds {
6255e541973SBenjamin Herrenschmidt 	pgd_free(mm, mm->pgd);
6261da177e4SLinus Torvalds }
6271da177e4SLinus Torvalds #else
62890f31d0eSKonstantin Khlebnikov static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
62990f31d0eSKonstantin Khlebnikov {
63090f31d0eSKonstantin Khlebnikov 	down_write(&oldmm->mmap_sem);
63190f31d0eSKonstantin Khlebnikov 	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
63290f31d0eSKonstantin Khlebnikov 	up_write(&oldmm->mmap_sem);
63390f31d0eSKonstantin Khlebnikov 	return 0;
63490f31d0eSKonstantin Khlebnikov }
6351da177e4SLinus Torvalds #define mm_alloc_pgd(mm)	(0)
6361da177e4SLinus Torvalds #define mm_free_pgd(mm)
6371da177e4SLinus Torvalds #endif /* CONFIG_MMU */
6381da177e4SLinus Torvalds 
639d70f2a14SAndrew Morton static void check_mm(struct mm_struct *mm)
640d70f2a14SAndrew Morton {
641d70f2a14SAndrew Morton 	int i;
642d70f2a14SAndrew Morton 
643d70f2a14SAndrew Morton 	for (i = 0; i < NR_MM_COUNTERS; i++) {
644d70f2a14SAndrew Morton 		long x = atomic_long_read(&mm->rss_stat.count[i]);
645d70f2a14SAndrew Morton 
646d70f2a14SAndrew Morton 		if (unlikely(x))
647d70f2a14SAndrew Morton 			printk(KERN_ALERT "BUG: Bad rss-counter state "
648d70f2a14SAndrew Morton 					  "mm:%p idx:%d val:%ld\n", mm, i, x);
649d70f2a14SAndrew Morton 	}
650d70f2a14SAndrew Morton 
651d70f2a14SAndrew Morton 	if (mm_pgtables_bytes(mm))
652d70f2a14SAndrew Morton 		pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
653d70f2a14SAndrew Morton 				mm_pgtables_bytes(mm));
654d70f2a14SAndrew Morton 
655d70f2a14SAndrew Morton #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
656d70f2a14SAndrew Morton 	VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
657d70f2a14SAndrew Morton #endif
658d70f2a14SAndrew Morton }
6591da177e4SLinus Torvalds 
660e94b1766SChristoph Lameter #define allocate_mm()	(kmem_cache_alloc(mm_cachep, GFP_KERNEL))
6611da177e4SLinus Torvalds #define free_mm(mm)	(kmem_cache_free(mm_cachep, (mm)))
6621da177e4SLinus Torvalds 
663d70f2a14SAndrew Morton /*
664d70f2a14SAndrew Morton  * Called when the last reference to the mm
665d70f2a14SAndrew Morton  * is dropped: either by a lazy thread or by
666d70f2a14SAndrew Morton  * mmput. Free the page directory and the mm.
667d70f2a14SAndrew Morton  */
668d34bc48fSAndrew Morton void __mmdrop(struct mm_struct *mm)
669d70f2a14SAndrew Morton {
670d70f2a14SAndrew Morton 	BUG_ON(mm == &init_mm);
6713eda69c9SMark Rutland 	WARN_ON_ONCE(mm == current->mm);
6723eda69c9SMark Rutland 	WARN_ON_ONCE(mm == current->active_mm);
673d70f2a14SAndrew Morton 	mm_free_pgd(mm);
674d70f2a14SAndrew Morton 	destroy_context(mm);
675d70f2a14SAndrew Morton 	hmm_mm_destroy(mm);
676d70f2a14SAndrew Morton 	mmu_notifier_mm_destroy(mm);
677d70f2a14SAndrew Morton 	check_mm(mm);
678d70f2a14SAndrew Morton 	put_user_ns(mm->user_ns);
679d70f2a14SAndrew Morton 	free_mm(mm);
680d70f2a14SAndrew Morton }
681d34bc48fSAndrew Morton EXPORT_SYMBOL_GPL(__mmdrop);
682d70f2a14SAndrew Morton 
683d70f2a14SAndrew Morton static void mmdrop_async_fn(struct work_struct *work)
684d70f2a14SAndrew Morton {
685d70f2a14SAndrew Morton 	struct mm_struct *mm;
686d70f2a14SAndrew Morton 
687d70f2a14SAndrew Morton 	mm = container_of(work, struct mm_struct, async_put_work);
688d70f2a14SAndrew Morton 	__mmdrop(mm);
689d70f2a14SAndrew Morton }
690d70f2a14SAndrew Morton 
691d70f2a14SAndrew Morton static void mmdrop_async(struct mm_struct *mm)
692d70f2a14SAndrew Morton {
693d70f2a14SAndrew Morton 	if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
694d70f2a14SAndrew Morton 		INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
695d70f2a14SAndrew Morton 		schedule_work(&mm->async_put_work);
696d70f2a14SAndrew Morton 	}
697d70f2a14SAndrew Morton }
698d70f2a14SAndrew Morton 
6991da177e4SLinus Torvalds static inline void free_signal_struct(struct signal_struct *sig)
7001da177e4SLinus Torvalds {
7011da177e4SLinus Torvalds 	taskstats_tgid_free(sig);
7021da177e4SLinus Torvalds 	sched_autogroup_exit(sig);
7031da177e4SLinus Torvalds 	/*
7041da177e4SLinus Torvalds 	 * __mmdrop is not safe to call from softirq context on x86 due to
7051da177e4SLinus Torvalds 	 * pgd_dtor so postpone it to the async context
7061da177e4SLinus Torvalds 	 */
7071da177e4SLinus Torvalds 	if (sig->oom_mm)
7081da177e4SLinus Torvalds 		mmdrop_async(sig->oom_mm);
7091da177e4SLinus Torvalds 	kmem_cache_free(signal_cachep, sig);
7101da177e4SLinus Torvalds }
7111da177e4SLinus Torvalds 
7121da177e4SLinus Torvalds static inline void put_signal_struct(struct signal_struct *sig)
7131da177e4SLinus Torvalds {
71460d4de3fSElena Reshetova 	if (refcount_dec_and_test(&sig->sigcnt))
7151da177e4SLinus Torvalds 		free_signal_struct(sig);
7161da177e4SLinus Torvalds }
7171da177e4SLinus Torvalds 
7181da177e4SLinus Torvalds void __put_task_struct(struct task_struct *tsk)
7191da177e4SLinus Torvalds {
7201da177e4SLinus Torvalds 	WARN_ON(!tsk->exit_state);
721ec1d2819SElena Reshetova 	WARN_ON(refcount_read(&tsk->usage));
7221da177e4SLinus Torvalds 	WARN_ON(tsk == current);
7231da177e4SLinus Torvalds 
7241da177e4SLinus Torvalds 	cgroup_free(tsk);
7251da177e4SLinus Torvalds 	task_numa_free(tsk);
7261da177e4SLinus Torvalds 	security_task_free(tsk);
7271da177e4SLinus Torvalds 	exit_creds(tsk);
7281da177e4SLinus Torvalds 	delayacct_tsk_free(tsk);
7291da177e4SLinus Torvalds 	put_signal_struct(tsk->signal);
7301da177e4SLinus Torvalds 
7311da177e4SLinus Torvalds 	if (!profile_handoff_task(tsk))
7321da177e4SLinus Torvalds 		free_task(tsk);
7331da177e4SLinus Torvalds }
7341da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(__put_task_struct);
7351da177e4SLinus Torvalds 
7361da177e4SLinus Torvalds void __init __weak arch_task_cache_init(void) { }
7371da177e4SLinus Torvalds 
7381da177e4SLinus Torvalds /*
7391da177e4SLinus Torvalds  * set_max_threads
7401da177e4SLinus Torvalds  */
7411da177e4SLinus Torvalds static void set_max_threads(unsigned int max_threads_suggested)
7421da177e4SLinus Torvalds {
7431da177e4SLinus Torvalds 	u64 threads;
744ca79b0c2SArun KS 	unsigned long nr_pages = totalram_pages();
7451da177e4SLinus Torvalds 
7461da177e4SLinus Torvalds 	/*
7471da177e4SLinus Torvalds 	 * The number of threads shall be limited such that the thread
7481da177e4SLinus Torvalds 	 * structures may only consume a small part of the available memory.
7491da177e4SLinus Torvalds 	 */
7503d6357deSArun KS 	if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)
7511da177e4SLinus Torvalds 		threads = MAX_THREADS;
7521da177e4SLinus Torvalds 	else
7533d6357deSArun KS 		threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,
7541da177e4SLinus Torvalds 				    (u64) THREAD_SIZE * 8UL);
7551da177e4SLinus Torvalds 
7561da177e4SLinus Torvalds 	if (threads > max_threads_suggested)
7571da177e4SLinus Torvalds 		threads = max_threads_suggested;
7581da177e4SLinus Torvalds 
7591da177e4SLinus Torvalds 	max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
7601da177e4SLinus Torvalds }
7611da177e4SLinus Torvalds 
7621da177e4SLinus Torvalds #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
7631da177e4SLinus Torvalds /* Initialized by the architecture: */
7641da177e4SLinus Torvalds int arch_task_struct_size __read_mostly;
7651da177e4SLinus Torvalds #endif
7661da177e4SLinus Torvalds 
7675905429aSKees Cook static void task_struct_whitelist(unsigned long *offset, unsigned long *size)
7685905429aSKees Cook {
7695905429aSKees Cook 	/* Fetch thread_struct whitelist for the architecture. */
7705905429aSKees Cook 	arch_thread_struct_whitelist(offset, size);
7715905429aSKees Cook 
7725905429aSKees Cook 	/*
7735905429aSKees Cook 	 * Handle zero-sized whitelist or empty thread_struct, otherwise
7745905429aSKees Cook 	 * adjust offset to position of thread_struct in task_struct.
7755905429aSKees Cook 	 */
7765905429aSKees Cook 	if (unlikely(*size == 0))
7775905429aSKees Cook 		*offset = 0;
7785905429aSKees Cook 	else
7795905429aSKees Cook 		*offset += offsetof(struct task_struct, thread);
7805905429aSKees Cook }
7815905429aSKees Cook 
7821da177e4SLinus Torvalds void __init fork_init(void)
7831da177e4SLinus Torvalds {
7841da177e4SLinus Torvalds 	int i;
7851da177e4SLinus Torvalds #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
7861da177e4SLinus Torvalds #ifndef ARCH_MIN_TASKALIGN
7871da177e4SLinus Torvalds #define ARCH_MIN_TASKALIGN	0
7881da177e4SLinus Torvalds #endif
7891da177e4SLinus Torvalds 	int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
7905905429aSKees Cook 	unsigned long useroffset, usersize;
7911da177e4SLinus Torvalds 
7921da177e4SLinus Torvalds 	/* create a slab on which task_structs can be allocated */
7935905429aSKees Cook 	task_struct_whitelist(&useroffset, &usersize);
7945905429aSKees Cook 	task_struct_cachep = kmem_cache_create_usercopy("task_struct",
7951da177e4SLinus Torvalds 			arch_task_struct_size, align,
7965905429aSKees Cook 			SLAB_PANIC|SLAB_ACCOUNT,
7975905429aSKees Cook 			useroffset, usersize, NULL);
7981da177e4SLinus Torvalds #endif
7991da177e4SLinus Torvalds 
8001da177e4SLinus Torvalds 	/* do the arch specific task caches init */
8011da177e4SLinus Torvalds 	arch_task_cache_init();
8021da177e4SLinus Torvalds 
8031da177e4SLinus Torvalds 	set_max_threads(MAX_THREADS);
8041da177e4SLinus Torvalds 
8051da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
8061da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
8071da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_SIGPENDING] =
8081da177e4SLinus Torvalds 		init_task.signal->rlim[RLIMIT_NPROC];
8091da177e4SLinus Torvalds 
8101da177e4SLinus Torvalds 	for (i = 0; i < UCOUNT_COUNTS; i++) {
8111da177e4SLinus Torvalds 		init_user_ns.ucount_max[i] = max_threads/2;
8121da177e4SLinus Torvalds 	}
8131da177e4SLinus Torvalds 
8141da177e4SLinus Torvalds #ifdef CONFIG_VMAP_STACK
8151da177e4SLinus Torvalds 	cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
8161da177e4SLinus Torvalds 			  NULL, free_vm_stack_cache);
8171da177e4SLinus Torvalds #endif
8181da177e4SLinus Torvalds 
8191da177e4SLinus Torvalds 	lockdep_init_task(&init_task);
820aad42dd4SNadav Amit 	uprobes_init();
8211da177e4SLinus Torvalds }
8221da177e4SLinus Torvalds 
8231da177e4SLinus Torvalds int __weak arch_dup_task_struct(struct task_struct *dst,
8241da177e4SLinus Torvalds 					       struct task_struct *src)
8251da177e4SLinus Torvalds {
8261da177e4SLinus Torvalds 	*dst = *src;
8271da177e4SLinus Torvalds 	return 0;
8281da177e4SLinus Torvalds }
8291da177e4SLinus Torvalds 
8301da177e4SLinus Torvalds void set_task_stack_end_magic(struct task_struct *tsk)
8311da177e4SLinus Torvalds {
8321da177e4SLinus Torvalds 	unsigned long *stackend;
8331da177e4SLinus Torvalds 
8341da177e4SLinus Torvalds 	stackend = end_of_stack(tsk);
8351da177e4SLinus Torvalds 	*stackend = STACK_END_MAGIC;	/* for overflow detection */
8361da177e4SLinus Torvalds }
8371da177e4SLinus Torvalds 
8381da177e4SLinus Torvalds static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
8391da177e4SLinus Torvalds {
8401da177e4SLinus Torvalds 	struct task_struct *tsk;
8411da177e4SLinus Torvalds 	unsigned long *stack;
8420f4991e8SYueHaibing 	struct vm_struct *stack_vm_area __maybe_unused;
8431da177e4SLinus Torvalds 	int err;
8441da177e4SLinus Torvalds 
8451da177e4SLinus Torvalds 	if (node == NUMA_NO_NODE)
8461da177e4SLinus Torvalds 		node = tsk_fork_get_node(orig);
8471da177e4SLinus Torvalds 	tsk = alloc_task_struct_node(node);
8481da177e4SLinus Torvalds 	if (!tsk)
8491da177e4SLinus Torvalds 		return NULL;
8501da177e4SLinus Torvalds 
8511da177e4SLinus Torvalds 	stack = alloc_thread_stack_node(tsk, node);
8521da177e4SLinus Torvalds 	if (!stack)
8531da177e4SLinus Torvalds 		goto free_tsk;
8541da177e4SLinus Torvalds 
8559b6f7e16SRoman Gushchin 	if (memcg_charge_kernel_stack(tsk))
8569b6f7e16SRoman Gushchin 		goto free_stack;
8579b6f7e16SRoman Gushchin 
8581da177e4SLinus Torvalds 	stack_vm_area = task_stack_vm_area(tsk);
8591da177e4SLinus Torvalds 
8601da177e4SLinus Torvalds 	err = arch_dup_task_struct(tsk, orig);
8611da177e4SLinus Torvalds 
8621da177e4SLinus Torvalds 	/*
8631da177e4SLinus Torvalds 	 * arch_dup_task_struct() clobbers the stack-related fields.  Make
8641da177e4SLinus Torvalds 	 * sure they're properly initialized before using any stack-related
8651da177e4SLinus Torvalds 	 * functions again.
8661da177e4SLinus Torvalds 	 */
8671da177e4SLinus Torvalds 	tsk->stack = stack;
8681da177e4SLinus Torvalds #ifdef CONFIG_VMAP_STACK
8691da177e4SLinus Torvalds 	tsk->stack_vm_area = stack_vm_area;
8701da177e4SLinus Torvalds #endif
8711da177e4SLinus Torvalds #ifdef CONFIG_THREAD_INFO_IN_TASK
872f0b89d39SElena Reshetova 	refcount_set(&tsk->stack_refcount, 1);
8731da177e4SLinus Torvalds #endif
8741da177e4SLinus Torvalds 
8751da177e4SLinus Torvalds 	if (err)
8761da177e4SLinus Torvalds 		goto free_stack;
8771da177e4SLinus Torvalds 
8781da177e4SLinus Torvalds #ifdef CONFIG_SECCOMP
8791da177e4SLinus Torvalds 	/*
8801da177e4SLinus Torvalds 	 * We must handle setting up seccomp filters once we're under
8811da177e4SLinus Torvalds 	 * the sighand lock in case orig has changed between now and
8821da177e4SLinus Torvalds 	 * then. Until then, filter must be NULL to avoid messing up
8831da177e4SLinus Torvalds 	 * the usage counts on the error path calling free_task.
8841da177e4SLinus Torvalds 	 */
8851da177e4SLinus Torvalds 	tsk->seccomp.filter = NULL;
8861da177e4SLinus Torvalds #endif
8871da177e4SLinus Torvalds 
8881da177e4SLinus Torvalds 	setup_thread_stack(tsk, orig);
8891da177e4SLinus Torvalds 	clear_user_return_notifier(tsk);
8901da177e4SLinus Torvalds 	clear_tsk_need_resched(tsk);
8911da177e4SLinus Torvalds 	set_task_stack_end_magic(tsk);
8921da177e4SLinus Torvalds 
893050e9baaSLinus Torvalds #ifdef CONFIG_STACKPROTECTOR
8941da177e4SLinus Torvalds 	tsk->stack_canary = get_random_canary();
8951da177e4SLinus Torvalds #endif
8961da177e4SLinus Torvalds 
8971da177e4SLinus Torvalds 	/*
8981da177e4SLinus Torvalds 	 * One for us, one for whoever does the "release_task()" (usually
8991da177e4SLinus Torvalds 	 * parent)
9001da177e4SLinus Torvalds 	 */
901ec1d2819SElena Reshetova 	refcount_set(&tsk->usage, 2);
9021da177e4SLinus Torvalds #ifdef CONFIG_BLK_DEV_IO_TRACE
9031da177e4SLinus Torvalds 	tsk->btrace_seq = 0;
9041da177e4SLinus Torvalds #endif
9051da177e4SLinus Torvalds 	tsk->splice_pipe = NULL;
9061da177e4SLinus Torvalds 	tsk->task_frag.page = NULL;
9071da177e4SLinus Torvalds 	tsk->wake_q.next = NULL;
9081da177e4SLinus Torvalds 
9091da177e4SLinus Torvalds 	account_kernel_stack(tsk, 1);
9101da177e4SLinus Torvalds 
9111da177e4SLinus Torvalds 	kcov_task_init(tsk);
9121da177e4SLinus Torvalds 
9131da177e4SLinus Torvalds #ifdef CONFIG_FAULT_INJECTION
9141da177e4SLinus Torvalds 	tsk->fail_nth = 0;
9151da177e4SLinus Torvalds #endif
9161da177e4SLinus Torvalds 
9172c323017SJosef Bacik #ifdef CONFIG_BLK_CGROUP
9182c323017SJosef Bacik 	tsk->throttle_queue = NULL;
9192c323017SJosef Bacik 	tsk->use_memdelay = 0;
9202c323017SJosef Bacik #endif
9212c323017SJosef Bacik 
922d46eb14bSShakeel Butt #ifdef CONFIG_MEMCG
923d46eb14bSShakeel Butt 	tsk->active_memcg = NULL;
924d46eb14bSShakeel Butt #endif
9251da177e4SLinus Torvalds 	return tsk;
9261da177e4SLinus Torvalds 
9271da177e4SLinus Torvalds free_stack:
9281da177e4SLinus Torvalds 	free_thread_stack(tsk);
9291da177e4SLinus Torvalds free_tsk:
9301da177e4SLinus Torvalds 	free_task_struct(tsk);
9311da177e4SLinus Torvalds 	return NULL;
9321da177e4SLinus Torvalds }
9331da177e4SLinus Torvalds 
9341da177e4SLinus Torvalds __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
9351da177e4SLinus Torvalds 
9364cb0e11bSHidehiro Kawai static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
9374cb0e11bSHidehiro Kawai 
9384cb0e11bSHidehiro Kawai static int __init coredump_filter_setup(char *s)
9394cb0e11bSHidehiro Kawai {
9404cb0e11bSHidehiro Kawai 	default_dump_filter =
9414cb0e11bSHidehiro Kawai 		(simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
9424cb0e11bSHidehiro Kawai 		MMF_DUMP_FILTER_MASK;
9434cb0e11bSHidehiro Kawai 	return 1;
9444cb0e11bSHidehiro Kawai }
9454cb0e11bSHidehiro Kawai 
9464cb0e11bSHidehiro Kawai __setup("coredump_filter=", coredump_filter_setup);
9474cb0e11bSHidehiro Kawai 
9481da177e4SLinus Torvalds #include <linux/init_task.h>
9491da177e4SLinus Torvalds 
950858f0993SAlexey Dobriyan static void mm_init_aio(struct mm_struct *mm)
951858f0993SAlexey Dobriyan {
952858f0993SAlexey Dobriyan #ifdef CONFIG_AIO
953858f0993SAlexey Dobriyan 	spin_lock_init(&mm->ioctx_lock);
954db446a08SBenjamin LaHaise 	mm->ioctx_table = NULL;
955858f0993SAlexey Dobriyan #endif
956858f0993SAlexey Dobriyan }
957858f0993SAlexey Dobriyan 
958c3f3ce04SAndrea Arcangeli static __always_inline void mm_clear_owner(struct mm_struct *mm,
959c3f3ce04SAndrea Arcangeli 					   struct task_struct *p)
960c3f3ce04SAndrea Arcangeli {
961c3f3ce04SAndrea Arcangeli #ifdef CONFIG_MEMCG
962c3f3ce04SAndrea Arcangeli 	if (mm->owner == p)
963c3f3ce04SAndrea Arcangeli 		WRITE_ONCE(mm->owner, NULL);
964c3f3ce04SAndrea Arcangeli #endif
965c3f3ce04SAndrea Arcangeli }
966c3f3ce04SAndrea Arcangeli 
96733144e84SVladimir Davydov static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
96833144e84SVladimir Davydov {
96933144e84SVladimir Davydov #ifdef CONFIG_MEMCG
97033144e84SVladimir Davydov 	mm->owner = p;
97133144e84SVladimir Davydov #endif
97233144e84SVladimir Davydov }
97333144e84SVladimir Davydov 
974355627f5SEric Biggers static void mm_init_uprobes_state(struct mm_struct *mm)
975355627f5SEric Biggers {
976355627f5SEric Biggers #ifdef CONFIG_UPROBES
977355627f5SEric Biggers 	mm->uprobes_state.xol_area = NULL;
978355627f5SEric Biggers #endif
979355627f5SEric Biggers }
980355627f5SEric Biggers 
981bfedb589SEric W. Biederman static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
982bfedb589SEric W. Biederman 	struct user_namespace *user_ns)
9831da177e4SLinus Torvalds {
98441f727fdSVladimir Davydov 	mm->mmap = NULL;
98541f727fdSVladimir Davydov 	mm->mm_rb = RB_ROOT;
98641f727fdSVladimir Davydov 	mm->vmacache_seqnum = 0;
9871da177e4SLinus Torvalds 	atomic_set(&mm->mm_users, 1);
9881da177e4SLinus Torvalds 	atomic_set(&mm->mm_count, 1);
9891da177e4SLinus Torvalds 	init_rwsem(&mm->mmap_sem);
9901da177e4SLinus Torvalds 	INIT_LIST_HEAD(&mm->mmlist);
991999d9fc1SOleg Nesterov 	mm->core_state = NULL;
992af5b0f6aSKirill A. Shutemov 	mm_pgtables_bytes_init(mm);
99341f727fdSVladimir Davydov 	mm->map_count = 0;
99441f727fdSVladimir Davydov 	mm->locked_vm = 0;
99570f8a3caSDavidlohr Bueso 	atomic64_set(&mm->pinned_vm, 0);
996d559db08SKAMEZAWA Hiroyuki 	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
9971da177e4SLinus Torvalds 	spin_lock_init(&mm->page_table_lock);
99888aa7cc6SYang Shi 	spin_lock_init(&mm->arg_lock);
99941f727fdSVladimir Davydov 	mm_init_cpumask(mm);
1000858f0993SAlexey Dobriyan 	mm_init_aio(mm);
1001cf475ad2SBalbir Singh 	mm_init_owner(mm, p);
10022b7e8665SEric Biggers 	RCU_INIT_POINTER(mm->exe_file, NULL);
100341f727fdSVladimir Davydov 	mmu_notifier_mm_init(mm);
1004133ff0eaSJérôme Glisse 	hmm_mm_init(mm);
100516af97dcSNadav Amit 	init_tlb_flush_pending(mm);
100641f727fdSVladimir Davydov #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
100741f727fdSVladimir Davydov 	mm->pmd_huge_pte = NULL;
100841f727fdSVladimir Davydov #endif
1009355627f5SEric Biggers 	mm_init_uprobes_state(mm);
10101da177e4SLinus Torvalds 
1011a0715cc2SAlex Thorlton 	if (current->mm) {
1012a0715cc2SAlex Thorlton 		mm->flags = current->mm->flags & MMF_INIT_MASK;
1013a0715cc2SAlex Thorlton 		mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
1014a0715cc2SAlex Thorlton 	} else {
1015a0715cc2SAlex Thorlton 		mm->flags = default_dump_filter;
10161da177e4SLinus Torvalds 		mm->def_flags = 0;
1017a0715cc2SAlex Thorlton 	}
1018a0715cc2SAlex Thorlton 
101941f727fdSVladimir Davydov 	if (mm_alloc_pgd(mm))
102041f727fdSVladimir Davydov 		goto fail_nopgd;
102178fb7466SPavel Emelianov 
102241f727fdSVladimir Davydov 	if (init_new_context(p, mm))
102341f727fdSVladimir Davydov 		goto fail_nocontext;
102441f727fdSVladimir Davydov 
1025bfedb589SEric W. Biederman 	mm->user_ns = get_user_ns(user_ns);
102641f727fdSVladimir Davydov 	return mm;
102741f727fdSVladimir Davydov 
102841f727fdSVladimir Davydov fail_nocontext:
102941f727fdSVladimir Davydov 	mm_free_pgd(mm);
103041f727fdSVladimir Davydov fail_nopgd:
10311da177e4SLinus Torvalds 	free_mm(mm);
10321da177e4SLinus Torvalds 	return NULL;
10331da177e4SLinus Torvalds }
10341da177e4SLinus Torvalds 
10351da177e4SLinus Torvalds /*
10361da177e4SLinus Torvalds  * Allocate and initialize an mm_struct.
10371da177e4SLinus Torvalds  */
10381da177e4SLinus Torvalds struct mm_struct *mm_alloc(void)
10391da177e4SLinus Torvalds {
10401da177e4SLinus Torvalds 	struct mm_struct *mm;
10411da177e4SLinus Torvalds 
10421da177e4SLinus Torvalds 	mm = allocate_mm();
1043de03c72cSKOSAKI Motohiro 	if (!mm)
1044de03c72cSKOSAKI Motohiro 		return NULL;
1045de03c72cSKOSAKI Motohiro 
10461da177e4SLinus Torvalds 	memset(mm, 0, sizeof(*mm));
1047bfedb589SEric W. Biederman 	return mm_init(mm, current, current_user_ns());
10481da177e4SLinus Torvalds }
10491da177e4SLinus Torvalds 
1050ec8d7c14SMichal Hocko static inline void __mmput(struct mm_struct *mm)
10511da177e4SLinus Torvalds {
1052ec8d7c14SMichal Hocko 	VM_BUG_ON(atomic_read(&mm->mm_users));
10530ae26f1bSAndrew Morton 
1054d4b3b638SSrikar Dronamraju 	uprobe_clear_state(mm);
10551da177e4SLinus Torvalds 	exit_aio(mm);
10561c2fb7a4SAndrea Arcangeli 	ksm_exit(mm);
1057ba76149fSAndrea Arcangeli 	khugepaged_exit(mm); /* must run before exit_mmap */
10581da177e4SLinus Torvalds 	exit_mmap(mm);
10596fcb52a5SAaron Lu 	mm_put_huge_zero_page(mm);
1060925d1c40SMatt Helsley 	set_mm_exe_file(mm, NULL);
10611da177e4SLinus Torvalds 	if (!list_empty(&mm->mmlist)) {
10621da177e4SLinus Torvalds 		spin_lock(&mmlist_lock);
10631da177e4SLinus Torvalds 		list_del(&mm->mmlist);
10641da177e4SLinus Torvalds 		spin_unlock(&mmlist_lock);
10651da177e4SLinus Torvalds 	}
1066801460d0SHiroshi Shimamoto 	if (mm->binfmt)
1067801460d0SHiroshi Shimamoto 		module_put(mm->binfmt->module);
10681da177e4SLinus Torvalds 	mmdrop(mm);
10691da177e4SLinus Torvalds }
1070ec8d7c14SMichal Hocko 
1071ec8d7c14SMichal Hocko /*
1072ec8d7c14SMichal Hocko  * Decrement the use count and release all resources for an mm.
1073ec8d7c14SMichal Hocko  */
1074ec8d7c14SMichal Hocko void mmput(struct mm_struct *mm)
1075ec8d7c14SMichal Hocko {
1076ec8d7c14SMichal Hocko 	might_sleep();
1077ec8d7c14SMichal Hocko 
1078ec8d7c14SMichal Hocko 	if (atomic_dec_and_test(&mm->mm_users))
1079ec8d7c14SMichal Hocko 		__mmput(mm);
10801da177e4SLinus Torvalds }
10811da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(mmput);
10821da177e4SLinus Torvalds 
1083a1b2289cSSherry Yang #ifdef CONFIG_MMU
1084a1b2289cSSherry Yang static void mmput_async_fn(struct work_struct *work)
1085a1b2289cSSherry Yang {
1086a1b2289cSSherry Yang 	struct mm_struct *mm = container_of(work, struct mm_struct,
1087a1b2289cSSherry Yang 					    async_put_work);
1088a1b2289cSSherry Yang 
1089a1b2289cSSherry Yang 	__mmput(mm);
1090a1b2289cSSherry Yang }
1091a1b2289cSSherry Yang 
1092a1b2289cSSherry Yang void mmput_async(struct mm_struct *mm)
1093a1b2289cSSherry Yang {
1094a1b2289cSSherry Yang 	if (atomic_dec_and_test(&mm->mm_users)) {
1095a1b2289cSSherry Yang 		INIT_WORK(&mm->async_put_work, mmput_async_fn);
1096a1b2289cSSherry Yang 		schedule_work(&mm->async_put_work);
1097a1b2289cSSherry Yang 	}
1098a1b2289cSSherry Yang }
1099a1b2289cSSherry Yang #endif
1100a1b2289cSSherry Yang 
110190f31d0eSKonstantin Khlebnikov /**
110290f31d0eSKonstantin Khlebnikov  * set_mm_exe_file - change a reference to the mm's executable file
110390f31d0eSKonstantin Khlebnikov  *
110490f31d0eSKonstantin Khlebnikov  * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
110590f31d0eSKonstantin Khlebnikov  *
11066e399cd1SDavidlohr Bueso  * Main users are mmput() and sys_execve(). Callers prevent concurrent
11076e399cd1SDavidlohr Bueso  * invocations: in mmput() nobody alive left, in execve task is single
11086e399cd1SDavidlohr Bueso  * threaded. sys_prctl(PR_SET_MM_MAP/EXE_FILE) also needs to set the
11096e399cd1SDavidlohr Bueso  * mm->exe_file, but does so without using set_mm_exe_file() in order
11106e399cd1SDavidlohr Bueso  * to do avoid the need for any locks.
111190f31d0eSKonstantin Khlebnikov  */
111238646013SJiri Slaby void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
111338646013SJiri Slaby {
11146e399cd1SDavidlohr Bueso 	struct file *old_exe_file;
11156e399cd1SDavidlohr Bueso 
11166e399cd1SDavidlohr Bueso 	/*
11176e399cd1SDavidlohr Bueso 	 * It is safe to dereference the exe_file without RCU as
11186e399cd1SDavidlohr Bueso 	 * this function is only called if nobody else can access
11196e399cd1SDavidlohr Bueso 	 * this mm -- see comment above for justification.
11206e399cd1SDavidlohr Bueso 	 */
11216e399cd1SDavidlohr Bueso 	old_exe_file = rcu_dereference_raw(mm->exe_file);
112290f31d0eSKonstantin Khlebnikov 
112338646013SJiri Slaby 	if (new_exe_file)
112438646013SJiri Slaby 		get_file(new_exe_file);
112590f31d0eSKonstantin Khlebnikov 	rcu_assign_pointer(mm->exe_file, new_exe_file);
112690f31d0eSKonstantin Khlebnikov 	if (old_exe_file)
112790f31d0eSKonstantin Khlebnikov 		fput(old_exe_file);
112838646013SJiri Slaby }
112938646013SJiri Slaby 
113090f31d0eSKonstantin Khlebnikov /**
113190f31d0eSKonstantin Khlebnikov  * get_mm_exe_file - acquire a reference to the mm's executable file
113290f31d0eSKonstantin Khlebnikov  *
113390f31d0eSKonstantin Khlebnikov  * Returns %NULL if mm has no associated executable file.
113490f31d0eSKonstantin Khlebnikov  * User must release file via fput().
113590f31d0eSKonstantin Khlebnikov  */
113638646013SJiri Slaby struct file *get_mm_exe_file(struct mm_struct *mm)
113738646013SJiri Slaby {
113838646013SJiri Slaby 	struct file *exe_file;
113938646013SJiri Slaby 
114090f31d0eSKonstantin Khlebnikov 	rcu_read_lock();
114190f31d0eSKonstantin Khlebnikov 	exe_file = rcu_dereference(mm->exe_file);
114290f31d0eSKonstantin Khlebnikov 	if (exe_file && !get_file_rcu(exe_file))
114390f31d0eSKonstantin Khlebnikov 		exe_file = NULL;
114490f31d0eSKonstantin Khlebnikov 	rcu_read_unlock();
114538646013SJiri Slaby 	return exe_file;
114638646013SJiri Slaby }
114711163348SDavidlohr Bueso EXPORT_SYMBOL(get_mm_exe_file);
114838646013SJiri Slaby 
11491da177e4SLinus Torvalds /**
1150cd81a917SMateusz Guzik  * get_task_exe_file - acquire a reference to the task's executable file
1151cd81a917SMateusz Guzik  *
1152cd81a917SMateusz Guzik  * Returns %NULL if task's mm (if any) has no associated executable file or
1153cd81a917SMateusz Guzik  * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
1154cd81a917SMateusz Guzik  * User must release file via fput().
1155cd81a917SMateusz Guzik  */
1156cd81a917SMateusz Guzik struct file *get_task_exe_file(struct task_struct *task)
1157cd81a917SMateusz Guzik {
1158cd81a917SMateusz Guzik 	struct file *exe_file = NULL;
1159cd81a917SMateusz Guzik 	struct mm_struct *mm;
1160cd81a917SMateusz Guzik 
1161cd81a917SMateusz Guzik 	task_lock(task);
1162cd81a917SMateusz Guzik 	mm = task->mm;
1163cd81a917SMateusz Guzik 	if (mm) {
1164cd81a917SMateusz Guzik 		if (!(task->flags & PF_KTHREAD))
1165cd81a917SMateusz Guzik 			exe_file = get_mm_exe_file(mm);
1166cd81a917SMateusz Guzik 	}
1167cd81a917SMateusz Guzik 	task_unlock(task);
1168cd81a917SMateusz Guzik 	return exe_file;
1169cd81a917SMateusz Guzik }
1170cd81a917SMateusz Guzik EXPORT_SYMBOL(get_task_exe_file);
1171cd81a917SMateusz Guzik 
1172cd81a917SMateusz Guzik /**
11731da177e4SLinus Torvalds  * get_task_mm - acquire a reference to the task's mm
11741da177e4SLinus Torvalds  *
1175246bb0b1SOleg Nesterov  * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning
11761da177e4SLinus Torvalds  * this kernel workthread has transiently adopted a user mm with use_mm,
11771da177e4SLinus Torvalds  * to do its AIO) is not set and if so returns a reference to it, after
11781da177e4SLinus Torvalds  * bumping up the use count.  User must release the mm via mmput()
11791da177e4SLinus Torvalds  * after use.  Typically used by /proc and ptrace.
11801da177e4SLinus Torvalds  */
11811da177e4SLinus Torvalds struct mm_struct *get_task_mm(struct task_struct *task)
11821da177e4SLinus Torvalds {
11831da177e4SLinus Torvalds 	struct mm_struct *mm;
11841da177e4SLinus Torvalds 
11851da177e4SLinus Torvalds 	task_lock(task);
11861da177e4SLinus Torvalds 	mm = task->mm;
11871da177e4SLinus Torvalds 	if (mm) {
1188246bb0b1SOleg Nesterov 		if (task->flags & PF_KTHREAD)
11891da177e4SLinus Torvalds 			mm = NULL;
11901da177e4SLinus Torvalds 		else
11913fce371bSVegard Nossum 			mmget(mm);
11921da177e4SLinus Torvalds 	}
11931da177e4SLinus Torvalds 	task_unlock(task);
11941da177e4SLinus Torvalds 	return mm;
11951da177e4SLinus Torvalds }
11961da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(get_task_mm);
11971da177e4SLinus Torvalds 
11988cdb878dSChristopher Yeoh struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
11998cdb878dSChristopher Yeoh {
12008cdb878dSChristopher Yeoh 	struct mm_struct *mm;
12018cdb878dSChristopher Yeoh 	int err;
12028cdb878dSChristopher Yeoh 
12038cdb878dSChristopher Yeoh 	err =  mutex_lock_killable(&task->signal->cred_guard_mutex);
12048cdb878dSChristopher Yeoh 	if (err)
12058cdb878dSChristopher Yeoh 		return ERR_PTR(err);
12068cdb878dSChristopher Yeoh 
12078cdb878dSChristopher Yeoh 	mm = get_task_mm(task);
12088cdb878dSChristopher Yeoh 	if (mm && mm != current->mm &&
12098cdb878dSChristopher Yeoh 			!ptrace_may_access(task, mode)) {
12108cdb878dSChristopher Yeoh 		mmput(mm);
12118cdb878dSChristopher Yeoh 		mm = ERR_PTR(-EACCES);
12128cdb878dSChristopher Yeoh 	}
12138cdb878dSChristopher Yeoh 	mutex_unlock(&task->signal->cred_guard_mutex);
12148cdb878dSChristopher Yeoh 
12158cdb878dSChristopher Yeoh 	return mm;
12168cdb878dSChristopher Yeoh }
12178cdb878dSChristopher Yeoh 
121857b59c4aSOleg Nesterov static void complete_vfork_done(struct task_struct *tsk)
1219c415c3b4SOleg Nesterov {
1220d68b46feSOleg Nesterov 	struct completion *vfork;
1221c415c3b4SOleg Nesterov 
1222d68b46feSOleg Nesterov 	task_lock(tsk);
1223d68b46feSOleg Nesterov 	vfork = tsk->vfork_done;
1224d68b46feSOleg Nesterov 	if (likely(vfork)) {
1225c415c3b4SOleg Nesterov 		tsk->vfork_done = NULL;
1226d68b46feSOleg Nesterov 		complete(vfork);
1227d68b46feSOleg Nesterov 	}
1228d68b46feSOleg Nesterov 	task_unlock(tsk);
1229d68b46feSOleg Nesterov }
1230d68b46feSOleg Nesterov 
1231d68b46feSOleg Nesterov static int wait_for_vfork_done(struct task_struct *child,
1232d68b46feSOleg Nesterov 				struct completion *vfork)
1233d68b46feSOleg Nesterov {
1234d68b46feSOleg Nesterov 	int killed;
1235d68b46feSOleg Nesterov 
1236d68b46feSOleg Nesterov 	freezer_do_not_count();
123776f969e8SRoman Gushchin 	cgroup_enter_frozen();
1238d68b46feSOleg Nesterov 	killed = wait_for_completion_killable(vfork);
123976f969e8SRoman Gushchin 	cgroup_leave_frozen(false);
1240d68b46feSOleg Nesterov 	freezer_count();
1241d68b46feSOleg Nesterov 
1242d68b46feSOleg Nesterov 	if (killed) {
1243d68b46feSOleg Nesterov 		task_lock(child);
1244d68b46feSOleg Nesterov 		child->vfork_done = NULL;
1245d68b46feSOleg Nesterov 		task_unlock(child);
1246d68b46feSOleg Nesterov 	}
1247d68b46feSOleg Nesterov 
1248d68b46feSOleg Nesterov 	put_task_struct(child);
1249d68b46feSOleg Nesterov 	return killed;
1250c415c3b4SOleg Nesterov }
1251c415c3b4SOleg Nesterov 
12521da177e4SLinus Torvalds /* Please note the differences between mmput and mm_release.
12531da177e4SLinus Torvalds  * mmput is called whenever we stop holding onto a mm_struct,
12541da177e4SLinus Torvalds  * error success whatever.
12551da177e4SLinus Torvalds  *
12561da177e4SLinus Torvalds  * mm_release is called after a mm_struct has been removed
12571da177e4SLinus Torvalds  * from the current process.
12581da177e4SLinus Torvalds  *
12591da177e4SLinus Torvalds  * This difference is important for error handling, when we
12601da177e4SLinus Torvalds  * only half set up a mm_struct for a new process and need to restore
12611da177e4SLinus Torvalds  * the old one.  Because we mmput the new mm_struct before
12621da177e4SLinus Torvalds  * restoring the old one. . .
12631da177e4SLinus Torvalds  * Eric Biederman 10 January 1998
12641da177e4SLinus Torvalds  */
12651da177e4SLinus Torvalds void mm_release(struct task_struct *tsk, struct mm_struct *mm)
12661da177e4SLinus Torvalds {
12678141c7f3SLinus Torvalds 	/* Get rid of any futexes when releasing the mm */
12688141c7f3SLinus Torvalds #ifdef CONFIG_FUTEX
1269fc6b177dSPeter Zijlstra 	if (unlikely(tsk->robust_list)) {
12708141c7f3SLinus Torvalds 		exit_robust_list(tsk);
1271fc6b177dSPeter Zijlstra 		tsk->robust_list = NULL;
1272fc6b177dSPeter Zijlstra 	}
12738141c7f3SLinus Torvalds #ifdef CONFIG_COMPAT
1274fc6b177dSPeter Zijlstra 	if (unlikely(tsk->compat_robust_list)) {
12758141c7f3SLinus Torvalds 		compat_exit_robust_list(tsk);
1276fc6b177dSPeter Zijlstra 		tsk->compat_robust_list = NULL;
1277fc6b177dSPeter Zijlstra 	}
12788141c7f3SLinus Torvalds #endif
1279322a2c10SThomas Gleixner 	if (unlikely(!list_empty(&tsk->pi_state_list)))
1280322a2c10SThomas Gleixner 		exit_pi_state_list(tsk);
12818141c7f3SLinus Torvalds #endif
12828141c7f3SLinus Torvalds 
12830326f5a9SSrikar Dronamraju 	uprobe_free_utask(tsk);
12840326f5a9SSrikar Dronamraju 
12851da177e4SLinus Torvalds 	/* Get rid of any cached register state */
12861da177e4SLinus Torvalds 	deactivate_mm(tsk, mm);
12871da177e4SLinus Torvalds 
1288fec1d011SRoland McGrath 	/*
1289735f2770SMichal Hocko 	 * Signal userspace if we're not exiting with a core dump
1290735f2770SMichal Hocko 	 * because we want to leave the value intact for debugging
1291735f2770SMichal Hocko 	 * purposes.
1292fec1d011SRoland McGrath 	 */
12939c8a8228SEric Dumazet 	if (tsk->clear_child_tid) {
1294735f2770SMichal Hocko 		if (!(tsk->signal->flags & SIGNAL_GROUP_COREDUMP) &&
12959c8a8228SEric Dumazet 		    atomic_read(&mm->mm_users) > 1) {
12961da177e4SLinus Torvalds 			/*
12971da177e4SLinus Torvalds 			 * We don't check the error code - if userspace has
12981da177e4SLinus Torvalds 			 * not set up a proper pointer then tough luck.
12991da177e4SLinus Torvalds 			 */
13009c8a8228SEric Dumazet 			put_user(0, tsk->clear_child_tid);
13012de0db99SDominik Brodowski 			do_futex(tsk->clear_child_tid, FUTEX_WAKE,
13022de0db99SDominik Brodowski 					1, NULL, NULL, 0, 0);
13039c8a8228SEric Dumazet 		}
13049c8a8228SEric Dumazet 		tsk->clear_child_tid = NULL;
13051da177e4SLinus Torvalds 	}
1306f7505d64SKonstantin Khlebnikov 
1307f7505d64SKonstantin Khlebnikov 	/*
1308f7505d64SKonstantin Khlebnikov 	 * All done, finally we can wake up parent and return this mm to him.
1309f7505d64SKonstantin Khlebnikov 	 * Also kthread_stop() uses this completion for synchronization.
1310f7505d64SKonstantin Khlebnikov 	 */
1311f7505d64SKonstantin Khlebnikov 	if (tsk->vfork_done)
1312f7505d64SKonstantin Khlebnikov 		complete_vfork_done(tsk);
13131da177e4SLinus Torvalds }
13141da177e4SLinus Torvalds 
131513585fa0SNadav Amit /**
131613585fa0SNadav Amit  * dup_mm() - duplicates an existing mm structure
131713585fa0SNadav Amit  * @tsk: the task_struct with which the new mm will be associated.
131813585fa0SNadav Amit  * @oldmm: the mm to duplicate.
131913585fa0SNadav Amit  *
132013585fa0SNadav Amit  * Allocates a new mm structure and duplicates the provided @oldmm structure
132113585fa0SNadav Amit  * content into it.
132213585fa0SNadav Amit  *
132313585fa0SNadav Amit  * Return: the duplicated mm or NULL on failure.
1324a0a7ec30SJANAK DESAI  */
132513585fa0SNadav Amit static struct mm_struct *dup_mm(struct task_struct *tsk,
132613585fa0SNadav Amit 				struct mm_struct *oldmm)
1327a0a7ec30SJANAK DESAI {
132813585fa0SNadav Amit 	struct mm_struct *mm;
1329a0a7ec30SJANAK DESAI 	int err;
1330a0a7ec30SJANAK DESAI 
1331a0a7ec30SJANAK DESAI 	mm = allocate_mm();
1332a0a7ec30SJANAK DESAI 	if (!mm)
1333a0a7ec30SJANAK DESAI 		goto fail_nomem;
1334a0a7ec30SJANAK DESAI 
1335a0a7ec30SJANAK DESAI 	memcpy(mm, oldmm, sizeof(*mm));
1336a0a7ec30SJANAK DESAI 
1337bfedb589SEric W. Biederman 	if (!mm_init(mm, tsk, mm->user_ns))
1338a0a7ec30SJANAK DESAI 		goto fail_nomem;
1339a0a7ec30SJANAK DESAI 
1340a0a7ec30SJANAK DESAI 	err = dup_mmap(mm, oldmm);
1341a0a7ec30SJANAK DESAI 	if (err)
1342a0a7ec30SJANAK DESAI 		goto free_pt;
1343a0a7ec30SJANAK DESAI 
1344a0a7ec30SJANAK DESAI 	mm->hiwater_rss = get_mm_rss(mm);
1345a0a7ec30SJANAK DESAI 	mm->hiwater_vm = mm->total_vm;
1346a0a7ec30SJANAK DESAI 
1347801460d0SHiroshi Shimamoto 	if (mm->binfmt && !try_module_get(mm->binfmt->module))
1348801460d0SHiroshi Shimamoto 		goto free_pt;
1349801460d0SHiroshi Shimamoto 
1350a0a7ec30SJANAK DESAI 	return mm;
1351a0a7ec30SJANAK DESAI 
1352a0a7ec30SJANAK DESAI free_pt:
1353801460d0SHiroshi Shimamoto 	/* don't put binfmt in mmput, we haven't got module yet */
1354801460d0SHiroshi Shimamoto 	mm->binfmt = NULL;
1355c3f3ce04SAndrea Arcangeli 	mm_init_owner(mm, NULL);
1356a0a7ec30SJANAK DESAI 	mmput(mm);
1357a0a7ec30SJANAK DESAI 
1358a0a7ec30SJANAK DESAI fail_nomem:
1359a0a7ec30SJANAK DESAI 	return NULL;
1360a0a7ec30SJANAK DESAI }
1361a0a7ec30SJANAK DESAI 
13621da177e4SLinus Torvalds static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
13631da177e4SLinus Torvalds {
13641da177e4SLinus Torvalds 	struct mm_struct *mm, *oldmm;
13651da177e4SLinus Torvalds 	int retval;
13661da177e4SLinus Torvalds 
13671da177e4SLinus Torvalds 	tsk->min_flt = tsk->maj_flt = 0;
13681da177e4SLinus Torvalds 	tsk->nvcsw = tsk->nivcsw = 0;
136917406b82SMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK
137017406b82SMandeep Singh Baines 	tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
1371a2e51445SDmitry Vyukov 	tsk->last_switch_time = 0;
137217406b82SMandeep Singh Baines #endif
13731da177e4SLinus Torvalds 
13741da177e4SLinus Torvalds 	tsk->mm = NULL;
13751da177e4SLinus Torvalds 	tsk->active_mm = NULL;
13761da177e4SLinus Torvalds 
13771da177e4SLinus Torvalds 	/*
13781da177e4SLinus Torvalds 	 * Are we cloning a kernel thread?
13791da177e4SLinus Torvalds 	 *
13801da177e4SLinus Torvalds 	 * We need to steal a active VM for that..
13811da177e4SLinus Torvalds 	 */
13821da177e4SLinus Torvalds 	oldmm = current->mm;
13831da177e4SLinus Torvalds 	if (!oldmm)
13841da177e4SLinus Torvalds 		return 0;
13851da177e4SLinus Torvalds 
1386615d6e87SDavidlohr Bueso 	/* initialize the new vmacache entries */
1387615d6e87SDavidlohr Bueso 	vmacache_flush(tsk);
1388615d6e87SDavidlohr Bueso 
13891da177e4SLinus Torvalds 	if (clone_flags & CLONE_VM) {
13903fce371bSVegard Nossum 		mmget(oldmm);
13911da177e4SLinus Torvalds 		mm = oldmm;
13921da177e4SLinus Torvalds 		goto good_mm;
13931da177e4SLinus Torvalds 	}
13941da177e4SLinus Torvalds 
13951da177e4SLinus Torvalds 	retval = -ENOMEM;
139613585fa0SNadav Amit 	mm = dup_mm(tsk, current->mm);
13971da177e4SLinus Torvalds 	if (!mm)
13981da177e4SLinus Torvalds 		goto fail_nomem;
13991da177e4SLinus Torvalds 
14001da177e4SLinus Torvalds good_mm:
14011da177e4SLinus Torvalds 	tsk->mm = mm;
14021da177e4SLinus Torvalds 	tsk->active_mm = mm;
14031da177e4SLinus Torvalds 	return 0;
14041da177e4SLinus Torvalds 
14051da177e4SLinus Torvalds fail_nomem:
14061da177e4SLinus Torvalds 	return retval;
14071da177e4SLinus Torvalds }
14081da177e4SLinus Torvalds 
1409a39bc516SAlexey Dobriyan static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
14101da177e4SLinus Torvalds {
1411498052bbSAl Viro 	struct fs_struct *fs = current->fs;
14121da177e4SLinus Torvalds 	if (clone_flags & CLONE_FS) {
1413498052bbSAl Viro 		/* tsk->fs is already what we want */
14142a4419b5SNick Piggin 		spin_lock(&fs->lock);
1415498052bbSAl Viro 		if (fs->in_exec) {
14162a4419b5SNick Piggin 			spin_unlock(&fs->lock);
1417498052bbSAl Viro 			return -EAGAIN;
1418498052bbSAl Viro 		}
1419498052bbSAl Viro 		fs->users++;
14202a4419b5SNick Piggin 		spin_unlock(&fs->lock);
14211da177e4SLinus Torvalds 		return 0;
14221da177e4SLinus Torvalds 	}
1423498052bbSAl Viro 	tsk->fs = copy_fs_struct(fs);
14241da177e4SLinus Torvalds 	if (!tsk->fs)
14251da177e4SLinus Torvalds 		return -ENOMEM;
14261da177e4SLinus Torvalds 	return 0;
14271da177e4SLinus Torvalds }
14281da177e4SLinus Torvalds 
1429a016f338SJANAK DESAI static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
1430a016f338SJANAK DESAI {
1431a016f338SJANAK DESAI 	struct files_struct *oldf, *newf;
1432a016f338SJANAK DESAI 	int error = 0;
1433a016f338SJANAK DESAI 
1434a016f338SJANAK DESAI 	/*
1435a016f338SJANAK DESAI 	 * A background process may not have any files ...
1436a016f338SJANAK DESAI 	 */
1437a016f338SJANAK DESAI 	oldf = current->files;
1438a016f338SJANAK DESAI 	if (!oldf)
1439a016f338SJANAK DESAI 		goto out;
1440a016f338SJANAK DESAI 
1441a016f338SJANAK DESAI 	if (clone_flags & CLONE_FILES) {
1442a016f338SJANAK DESAI 		atomic_inc(&oldf->count);
1443a016f338SJANAK DESAI 		goto out;
1444a016f338SJANAK DESAI 	}
1445a016f338SJANAK DESAI 
1446a016f338SJANAK DESAI 	newf = dup_fd(oldf, &error);
1447a016f338SJANAK DESAI 	if (!newf)
1448a016f338SJANAK DESAI 		goto out;
1449a016f338SJANAK DESAI 
1450a016f338SJANAK DESAI 	tsk->files = newf;
1451a016f338SJANAK DESAI 	error = 0;
1452a016f338SJANAK DESAI out:
1453a016f338SJANAK DESAI 	return error;
1454a016f338SJANAK DESAI }
1455a016f338SJANAK DESAI 
1456fadad878SJens Axboe static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
1457fd0928dfSJens Axboe {
1458fd0928dfSJens Axboe #ifdef CONFIG_BLOCK
1459fd0928dfSJens Axboe 	struct io_context *ioc = current->io_context;
14606e736be7STejun Heo 	struct io_context *new_ioc;
1461fd0928dfSJens Axboe 
1462fd0928dfSJens Axboe 	if (!ioc)
1463fd0928dfSJens Axboe 		return 0;
1464fadad878SJens Axboe 	/*
1465fadad878SJens Axboe 	 * Share io context with parent, if CLONE_IO is set
1466fadad878SJens Axboe 	 */
1467fadad878SJens Axboe 	if (clone_flags & CLONE_IO) {
14683d48749dSTejun Heo 		ioc_task_link(ioc);
14693d48749dSTejun Heo 		tsk->io_context = ioc;
1470fadad878SJens Axboe 	} else if (ioprio_valid(ioc->ioprio)) {
14716e736be7STejun Heo 		new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
14726e736be7STejun Heo 		if (unlikely(!new_ioc))
1473fd0928dfSJens Axboe 			return -ENOMEM;
1474fd0928dfSJens Axboe 
14756e736be7STejun Heo 		new_ioc->ioprio = ioc->ioprio;
147611a3122fSTejun Heo 		put_io_context(new_ioc);
1477fd0928dfSJens Axboe 	}
1478fd0928dfSJens Axboe #endif
1479fd0928dfSJens Axboe 	return 0;
1480fd0928dfSJens Axboe }
1481fd0928dfSJens Axboe 
1482a39bc516SAlexey Dobriyan static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
14831da177e4SLinus Torvalds {
14841da177e4SLinus Torvalds 	struct sighand_struct *sig;
14851da177e4SLinus Torvalds 
148660348802SZhaolei 	if (clone_flags & CLONE_SIGHAND) {
1487d036bda7SElena Reshetova 		refcount_inc(&current->sighand->count);
14881da177e4SLinus Torvalds 		return 0;
14891da177e4SLinus Torvalds 	}
14901da177e4SLinus Torvalds 	sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
1491e56d0903SIngo Molnar 	rcu_assign_pointer(tsk->sighand, sig);
14921da177e4SLinus Torvalds 	if (!sig)
14931da177e4SLinus Torvalds 		return -ENOMEM;
14949d7fb042SPeter Zijlstra 
1495d036bda7SElena Reshetova 	refcount_set(&sig->count, 1);
149606e62a46SJann Horn 	spin_lock_irq(&current->sighand->siglock);
14971da177e4SLinus Torvalds 	memcpy(sig->action, current->sighand->action, sizeof(sig->action));
149806e62a46SJann Horn 	spin_unlock_irq(&current->sighand->siglock);
14991da177e4SLinus Torvalds 	return 0;
15001da177e4SLinus Torvalds }
15011da177e4SLinus Torvalds 
1502a7e5328aSOleg Nesterov void __cleanup_sighand(struct sighand_struct *sighand)
1503c81addc9SOleg Nesterov {
1504d036bda7SElena Reshetova 	if (refcount_dec_and_test(&sighand->count)) {
1505d80e731eSOleg Nesterov 		signalfd_cleanup(sighand);
1506392809b2SOleg Nesterov 		/*
15075f0d5a3aSPaul E. McKenney 		 * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it
1508392809b2SOleg Nesterov 		 * without an RCU grace period, see __lock_task_sighand().
1509392809b2SOleg Nesterov 		 */
1510c81addc9SOleg Nesterov 		kmem_cache_free(sighand_cachep, sighand);
1511c81addc9SOleg Nesterov 	}
1512d80e731eSOleg Nesterov }
1513c81addc9SOleg Nesterov 
1514b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS
1515f06febc9SFrank Mayhar /*
1516f06febc9SFrank Mayhar  * Initialize POSIX timer handling for a thread group.
1517f06febc9SFrank Mayhar  */
1518f06febc9SFrank Mayhar static void posix_cpu_timers_init_group(struct signal_struct *sig)
1519f06febc9SFrank Mayhar {
152078d7d407SJiri Slaby 	unsigned long cpu_limit;
152178d7d407SJiri Slaby 
1522316c1608SJason Low 	cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
152378d7d407SJiri Slaby 	if (cpu_limit != RLIM_INFINITY) {
1524ebd7e7fcSFrederic Weisbecker 		sig->cputime_expires.prof_exp = cpu_limit * NSEC_PER_SEC;
1525d5c373ebSJason Low 		sig->cputimer.running = true;
15266279a751SOleg Nesterov 	}
15276279a751SOleg Nesterov 
1528f06febc9SFrank Mayhar 	/* The timer lists. */
1529f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&sig->cpu_timers[0]);
1530f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&sig->cpu_timers[1]);
1531f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&sig->cpu_timers[2]);
1532f06febc9SFrank Mayhar }
1533b18b6a9cSNicolas Pitre #else
1534b18b6a9cSNicolas Pitre static inline void posix_cpu_timers_init_group(struct signal_struct *sig) { }
1535b18b6a9cSNicolas Pitre #endif
1536f06febc9SFrank Mayhar 
1537a39bc516SAlexey Dobriyan static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
15381da177e4SLinus Torvalds {
15391da177e4SLinus Torvalds 	struct signal_struct *sig;
15401da177e4SLinus Torvalds 
15414ab6c083SOleg Nesterov 	if (clone_flags & CLONE_THREAD)
1542490dea45SPeter Zijlstra 		return 0;
15436279a751SOleg Nesterov 
1544a56704efSVeaceslav Falico 	sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
15451da177e4SLinus Torvalds 	tsk->signal = sig;
15461da177e4SLinus Torvalds 	if (!sig)
15471da177e4SLinus Torvalds 		return -ENOMEM;
15481da177e4SLinus Torvalds 
1549b3ac022cSOleg Nesterov 	sig->nr_threads = 1;
15501da177e4SLinus Torvalds 	atomic_set(&sig->live, 1);
155160d4de3fSElena Reshetova 	refcount_set(&sig->sigcnt, 1);
15520c740d0aSOleg Nesterov 
15530c740d0aSOleg Nesterov 	/* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
15540c740d0aSOleg Nesterov 	sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
15550c740d0aSOleg Nesterov 	tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
15560c740d0aSOleg Nesterov 
15571da177e4SLinus Torvalds 	init_waitqueue_head(&sig->wait_chldexit);
1558db51aeccSOleg Nesterov 	sig->curr_target = tsk;
15591da177e4SLinus Torvalds 	init_sigpending(&sig->shared_pending);
1560c3ad2c3bSEric W. Biederman 	INIT_HLIST_HEAD(&sig->multiprocess);
1561e78c3496SRik van Riel 	seqlock_init(&sig->stats_lock);
15629d7fb042SPeter Zijlstra 	prev_cputime_init(&sig->prev_cputime);
15631da177e4SLinus Torvalds 
1564baa73d9eSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS
1565b18b6a9cSNicolas Pitre 	INIT_LIST_HEAD(&sig->posix_timers);
1566c9cb2e3dSThomas Gleixner 	hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
15671da177e4SLinus Torvalds 	sig->real_timer.function = it_real_fn;
1568baa73d9eSNicolas Pitre #endif
15691da177e4SLinus Torvalds 
15701da177e4SLinus Torvalds 	task_lock(current->group_leader);
15711da177e4SLinus Torvalds 	memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
15721da177e4SLinus Torvalds 	task_unlock(current->group_leader);
15731da177e4SLinus Torvalds 
15746279a751SOleg Nesterov 	posix_cpu_timers_init_group(sig);
15756279a751SOleg Nesterov 
1576522ed776SMiloslav Trmac 	tty_audit_fork(sig);
15775091faa4SMike Galbraith 	sched_autogroup_fork(sig);
1578522ed776SMiloslav Trmac 
1579a63d83f4SDavid Rientjes 	sig->oom_score_adj = current->signal->oom_score_adj;
1580dabb16f6SMandeep Singh Baines 	sig->oom_score_adj_min = current->signal->oom_score_adj_min;
158128b83c51SKOSAKI Motohiro 
15829b1bf12dSKOSAKI Motohiro 	mutex_init(&sig->cred_guard_mutex);
15839b1bf12dSKOSAKI Motohiro 
15841da177e4SLinus Torvalds 	return 0;
15851da177e4SLinus Torvalds }
15861da177e4SLinus Torvalds 
1587dbd95212SKees Cook static void copy_seccomp(struct task_struct *p)
1588dbd95212SKees Cook {
1589dbd95212SKees Cook #ifdef CONFIG_SECCOMP
1590dbd95212SKees Cook 	/*
1591dbd95212SKees Cook 	 * Must be called with sighand->lock held, which is common to
1592dbd95212SKees Cook 	 * all threads in the group. Holding cred_guard_mutex is not
1593dbd95212SKees Cook 	 * needed because this new task is not yet running and cannot
1594dbd95212SKees Cook 	 * be racing exec.
1595dbd95212SKees Cook 	 */
159669f6a34bSGuenter Roeck 	assert_spin_locked(&current->sighand->siglock);
1597dbd95212SKees Cook 
1598dbd95212SKees Cook 	/* Ref-count the new filter user, and assign it. */
1599dbd95212SKees Cook 	get_seccomp_filter(current);
1600dbd95212SKees Cook 	p->seccomp = current->seccomp;
1601dbd95212SKees Cook 
1602dbd95212SKees Cook 	/*
1603dbd95212SKees Cook 	 * Explicitly enable no_new_privs here in case it got set
1604dbd95212SKees Cook 	 * between the task_struct being duplicated and holding the
1605dbd95212SKees Cook 	 * sighand lock. The seccomp state and nnp must be in sync.
1606dbd95212SKees Cook 	 */
1607dbd95212SKees Cook 	if (task_no_new_privs(current))
1608dbd95212SKees Cook 		task_set_no_new_privs(p);
1609dbd95212SKees Cook 
1610dbd95212SKees Cook 	/*
1611dbd95212SKees Cook 	 * If the parent gained a seccomp mode after copying thread
1612dbd95212SKees Cook 	 * flags and between before we held the sighand lock, we have
1613dbd95212SKees Cook 	 * to manually enable the seccomp thread flag here.
1614dbd95212SKees Cook 	 */
1615dbd95212SKees Cook 	if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
1616dbd95212SKees Cook 		set_tsk_thread_flag(p, TIF_SECCOMP);
1617dbd95212SKees Cook #endif
1618dbd95212SKees Cook }
1619dbd95212SKees Cook 
162017da2bd9SHeiko Carstens SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
16211da177e4SLinus Torvalds {
16221da177e4SLinus Torvalds 	current->clear_child_tid = tidptr;
16231da177e4SLinus Torvalds 
1624b488893aSPavel Emelyanov 	return task_pid_vnr(current);
16251da177e4SLinus Torvalds }
16261da177e4SLinus Torvalds 
1627a39bc516SAlexey Dobriyan static void rt_mutex_init_task(struct task_struct *p)
162823f78d4aSIngo Molnar {
16291d615482SThomas Gleixner 	raw_spin_lock_init(&p->pi_lock);
1630e29e175bSZilvinas Valinskas #ifdef CONFIG_RT_MUTEXES
1631a23ba907SDavidlohr Bueso 	p->pi_waiters = RB_ROOT_CACHED;
1632e96a7705SXunlei Pang 	p->pi_top_task = NULL;
163323f78d4aSIngo Molnar 	p->pi_blocked_on = NULL;
163423f78d4aSIngo Molnar #endif
163523f78d4aSIngo Molnar }
163623f78d4aSIngo Molnar 
1637b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS
16381da177e4SLinus Torvalds /*
1639f06febc9SFrank Mayhar  * Initialize POSIX timer handling for a single task.
1640f06febc9SFrank Mayhar  */
1641f06febc9SFrank Mayhar static void posix_cpu_timers_init(struct task_struct *tsk)
1642f06febc9SFrank Mayhar {
164364861634SMartin Schwidefsky 	tsk->cputime_expires.prof_exp = 0;
164464861634SMartin Schwidefsky 	tsk->cputime_expires.virt_exp = 0;
1645f06febc9SFrank Mayhar 	tsk->cputime_expires.sched_exp = 0;
1646f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&tsk->cpu_timers[0]);
1647f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&tsk->cpu_timers[1]);
1648f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&tsk->cpu_timers[2]);
1649f06febc9SFrank Mayhar }
1650b18b6a9cSNicolas Pitre #else
1651b18b6a9cSNicolas Pitre static inline void posix_cpu_timers_init(struct task_struct *tsk) { }
1652b18b6a9cSNicolas Pitre #endif
1653f06febc9SFrank Mayhar 
16542c470475SEric W. Biederman static inline void init_task_pid_links(struct task_struct *task)
16552c470475SEric W. Biederman {
16562c470475SEric W. Biederman 	enum pid_type type;
16572c470475SEric W. Biederman 
16582c470475SEric W. Biederman 	for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
16592c470475SEric W. Biederman 		INIT_HLIST_NODE(&task->pid_links[type]);
16602c470475SEric W. Biederman 	}
16612c470475SEric W. Biederman }
16622c470475SEric W. Biederman 
166381907739SOleg Nesterov static inline void
166481907739SOleg Nesterov init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
166581907739SOleg Nesterov {
16662c470475SEric W. Biederman 	if (type == PIDTYPE_PID)
16672c470475SEric W. Biederman 		task->thread_pid = pid;
16682c470475SEric W. Biederman 	else
16692c470475SEric W. Biederman 		task->signal->pids[type] = pid;
167081907739SOleg Nesterov }
167181907739SOleg Nesterov 
16726bfbaa51SIngo Molnar static inline void rcu_copy_process(struct task_struct *p)
16736bfbaa51SIngo Molnar {
16746bfbaa51SIngo Molnar #ifdef CONFIG_PREEMPT_RCU
16756bfbaa51SIngo Molnar 	p->rcu_read_lock_nesting = 0;
16766bfbaa51SIngo Molnar 	p->rcu_read_unlock_special.s = 0;
16776bfbaa51SIngo Molnar 	p->rcu_blocked_node = NULL;
16786bfbaa51SIngo Molnar 	INIT_LIST_HEAD(&p->rcu_node_entry);
16796bfbaa51SIngo Molnar #endif /* #ifdef CONFIG_PREEMPT_RCU */
16806bfbaa51SIngo Molnar #ifdef CONFIG_TASKS_RCU
16816bfbaa51SIngo Molnar 	p->rcu_tasks_holdout = false;
16826bfbaa51SIngo Molnar 	INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
16836bfbaa51SIngo Molnar 	p->rcu_tasks_idle_cpu = -1;
16846bfbaa51SIngo Molnar #endif /* #ifdef CONFIG_TASKS_RCU */
16856bfbaa51SIngo Molnar }
16866bfbaa51SIngo Molnar 
1687b3e58382SChristian Brauner static int pidfd_release(struct inode *inode, struct file *file)
1688b3e58382SChristian Brauner {
1689b3e58382SChristian Brauner 	struct pid *pid = file->private_data;
1690b3e58382SChristian Brauner 
1691b3e58382SChristian Brauner 	file->private_data = NULL;
1692b3e58382SChristian Brauner 	put_pid(pid);
1693b3e58382SChristian Brauner 	return 0;
1694b3e58382SChristian Brauner }
1695b3e58382SChristian Brauner 
1696b3e58382SChristian Brauner #ifdef CONFIG_PROC_FS
1697b3e58382SChristian Brauner static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
1698b3e58382SChristian Brauner {
1699b3e58382SChristian Brauner 	struct pid_namespace *ns = proc_pid_ns(file_inode(m->file));
1700b3e58382SChristian Brauner 	struct pid *pid = f->private_data;
1701b3e58382SChristian Brauner 
1702b3e58382SChristian Brauner 	seq_put_decimal_ull(m, "Pid:\t", pid_nr_ns(pid, ns));
1703b3e58382SChristian Brauner 	seq_putc(m, '\n');
1704b3e58382SChristian Brauner }
1705b3e58382SChristian Brauner #endif
1706b3e58382SChristian Brauner 
1707b3e58382SChristian Brauner const struct file_operations pidfd_fops = {
1708b3e58382SChristian Brauner 	.release = pidfd_release,
1709b3e58382SChristian Brauner #ifdef CONFIG_PROC_FS
1710b3e58382SChristian Brauner 	.show_fdinfo = pidfd_show_fdinfo,
1711b3e58382SChristian Brauner #endif
1712b3e58382SChristian Brauner };
1713b3e58382SChristian Brauner 
1714b3e58382SChristian Brauner /**
1715b3e58382SChristian Brauner  * pidfd_create() - Create a new pid file descriptor.
1716b3e58382SChristian Brauner  *
1717b3e58382SChristian Brauner  * @pid:  struct pid that the pidfd will reference
1718b3e58382SChristian Brauner  *
1719b3e58382SChristian Brauner  * This creates a new pid file descriptor with the O_CLOEXEC flag set.
1720b3e58382SChristian Brauner  *
1721b3e58382SChristian Brauner  * Note, that this function can only be called after the fd table has
1722b3e58382SChristian Brauner  * been unshared to avoid leaking the pidfd to the new process.
1723b3e58382SChristian Brauner  *
1724b3e58382SChristian Brauner  * Return: On success, a cloexec pidfd is returned.
1725b3e58382SChristian Brauner  *         On error, a negative errno number will be returned.
1726b3e58382SChristian Brauner  */
1727b3e58382SChristian Brauner static int pidfd_create(struct pid *pid)
1728b3e58382SChristian Brauner {
1729b3e58382SChristian Brauner 	int fd;
1730b3e58382SChristian Brauner 
1731b3e58382SChristian Brauner 	fd = anon_inode_getfd("[pidfd]", &pidfd_fops, get_pid(pid),
1732b3e58382SChristian Brauner 			      O_RDWR | O_CLOEXEC);
1733b3e58382SChristian Brauner 	if (fd < 0)
1734b3e58382SChristian Brauner 		put_pid(pid);
1735b3e58382SChristian Brauner 
1736b3e58382SChristian Brauner 	return fd;
1737b3e58382SChristian Brauner }
1738b3e58382SChristian Brauner 
1739c3f3ce04SAndrea Arcangeli static void __delayed_free_task(struct rcu_head *rhp)
1740c3f3ce04SAndrea Arcangeli {
1741c3f3ce04SAndrea Arcangeli 	struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
1742c3f3ce04SAndrea Arcangeli 
1743c3f3ce04SAndrea Arcangeli 	free_task(tsk);
1744c3f3ce04SAndrea Arcangeli }
1745c3f3ce04SAndrea Arcangeli 
1746c3f3ce04SAndrea Arcangeli static __always_inline void delayed_free_task(struct task_struct *tsk)
1747c3f3ce04SAndrea Arcangeli {
1748c3f3ce04SAndrea Arcangeli 	if (IS_ENABLED(CONFIG_MEMCG))
1749c3f3ce04SAndrea Arcangeli 		call_rcu(&tsk->rcu, __delayed_free_task);
1750c3f3ce04SAndrea Arcangeli 	else
1751c3f3ce04SAndrea Arcangeli 		free_task(tsk);
1752c3f3ce04SAndrea Arcangeli }
1753c3f3ce04SAndrea Arcangeli 
1754f06febc9SFrank Mayhar /*
17551da177e4SLinus Torvalds  * This creates a new process as a copy of the old one,
17561da177e4SLinus Torvalds  * but does not actually start it yet.
17571da177e4SLinus Torvalds  *
17581da177e4SLinus Torvalds  * It copies the registers, and all the appropriate
17591da177e4SLinus Torvalds  * parts of the process environment (as per the clone
17601da177e4SLinus Torvalds  * flags). The actual kick-off is left to the caller.
17611da177e4SLinus Torvalds  */
17620766f788SEmese Revfy static __latent_entropy struct task_struct *copy_process(
17630766f788SEmese Revfy 					unsigned long clone_flags,
17641da177e4SLinus Torvalds 					unsigned long stack_start,
17651da177e4SLinus Torvalds 					unsigned long stack_size,
1766b3e58382SChristian Brauner 					int __user *parent_tidptr,
17671da177e4SLinus Torvalds 					int __user *child_tidptr,
176809a05394SRoland McGrath 					struct pid *pid,
17693033f14aSJosh Triplett 					int trace,
1770725fc629SAndi Kleen 					unsigned long tls,
1771725fc629SAndi Kleen 					int node)
17721da177e4SLinus Torvalds {
1773b3e58382SChristian Brauner 	int pidfd = -1, retval;
1774a24efe62SMariusz Kozlowski 	struct task_struct *p;
1775c3ad2c3bSEric W. Biederman 	struct multiprocess_signals delayed;
17761da177e4SLinus Torvalds 
1777667b6094SMarcos Paulo de Souza 	/*
1778667b6094SMarcos Paulo de Souza 	 * Don't allow sharing the root directory with processes in a different
1779667b6094SMarcos Paulo de Souza 	 * namespace
1780667b6094SMarcos Paulo de Souza 	 */
17811da177e4SLinus Torvalds 	if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
17821da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
17831da177e4SLinus Torvalds 
1784e66eded8SEric W. Biederman 	if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
1785e66eded8SEric W. Biederman 		return ERR_PTR(-EINVAL);
1786e66eded8SEric W. Biederman 
17871da177e4SLinus Torvalds 	/*
17881da177e4SLinus Torvalds 	 * Thread groups must share signals as well, and detached threads
17891da177e4SLinus Torvalds 	 * can only be started up within the thread group.
17901da177e4SLinus Torvalds 	 */
17911da177e4SLinus Torvalds 	if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
17921da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
17931da177e4SLinus Torvalds 
17941da177e4SLinus Torvalds 	/*
17951da177e4SLinus Torvalds 	 * Shared signal handlers imply shared VM. By way of the above,
17961da177e4SLinus Torvalds 	 * thread groups also imply shared VM. Blocking this case allows
17971da177e4SLinus Torvalds 	 * for various simplifications in other code.
17981da177e4SLinus Torvalds 	 */
17991da177e4SLinus Torvalds 	if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
18001da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
18011da177e4SLinus Torvalds 
1802123be07bSSukadev Bhattiprolu 	/*
1803123be07bSSukadev Bhattiprolu 	 * Siblings of global init remain as zombies on exit since they are
1804123be07bSSukadev Bhattiprolu 	 * not reaped by their parent (swapper). To solve this and to avoid
1805123be07bSSukadev Bhattiprolu 	 * multi-rooted process trees, prevent global and container-inits
1806123be07bSSukadev Bhattiprolu 	 * from creating siblings.
1807123be07bSSukadev Bhattiprolu 	 */
1808123be07bSSukadev Bhattiprolu 	if ((clone_flags & CLONE_PARENT) &&
1809123be07bSSukadev Bhattiprolu 				current->signal->flags & SIGNAL_UNKILLABLE)
1810123be07bSSukadev Bhattiprolu 		return ERR_PTR(-EINVAL);
1811123be07bSSukadev Bhattiprolu 
18128382fcacSEric W. Biederman 	/*
181340a0d32dSOleg Nesterov 	 * If the new process will be in a different pid or user namespace
1814faf00da5SEric W. Biederman 	 * do not allow it to share a thread group with the forking task.
18158382fcacSEric W. Biederman 	 */
1816faf00da5SEric W. Biederman 	if (clone_flags & CLONE_THREAD) {
181740a0d32dSOleg Nesterov 		if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
181840a0d32dSOleg Nesterov 		    (task_active_pid_ns(current) !=
1819c2b1df2eSAndy Lutomirski 				current->nsproxy->pid_ns_for_children))
18208382fcacSEric W. Biederman 			return ERR_PTR(-EINVAL);
182140a0d32dSOleg Nesterov 	}
18228382fcacSEric W. Biederman 
1823b3e58382SChristian Brauner 	if (clone_flags & CLONE_PIDFD) {
1824b3e58382SChristian Brauner 		int reserved;
1825b3e58382SChristian Brauner 
1826b3e58382SChristian Brauner 		/*
1827b3e58382SChristian Brauner 		 * - CLONE_PARENT_SETTID is useless for pidfds and also
1828b3e58382SChristian Brauner 		 *   parent_tidptr is used to return pidfds.
1829b3e58382SChristian Brauner 		 * - CLONE_DETACHED is blocked so that we can potentially
1830b3e58382SChristian Brauner 		 *   reuse it later for CLONE_PIDFD.
1831b3e58382SChristian Brauner 		 * - CLONE_THREAD is blocked until someone really needs it.
1832b3e58382SChristian Brauner 		 */
1833b3e58382SChristian Brauner 		if (clone_flags &
1834b3e58382SChristian Brauner 		    (CLONE_DETACHED | CLONE_PARENT_SETTID | CLONE_THREAD))
1835b3e58382SChristian Brauner 			return ERR_PTR(-EINVAL);
1836b3e58382SChristian Brauner 
1837b3e58382SChristian Brauner 		/*
1838b3e58382SChristian Brauner 		 * Verify that parent_tidptr is sane so we can potentially
1839b3e58382SChristian Brauner 		 * reuse it later.
1840b3e58382SChristian Brauner 		 */
1841b3e58382SChristian Brauner 		if (get_user(reserved, parent_tidptr))
1842b3e58382SChristian Brauner 			return ERR_PTR(-EFAULT);
1843b3e58382SChristian Brauner 
1844b3e58382SChristian Brauner 		if (reserved != 0)
1845b3e58382SChristian Brauner 			return ERR_PTR(-EINVAL);
1846b3e58382SChristian Brauner 	}
1847b3e58382SChristian Brauner 
1848c3ad2c3bSEric W. Biederman 	/*
1849c3ad2c3bSEric W. Biederman 	 * Force any signals received before this point to be delivered
1850c3ad2c3bSEric W. Biederman 	 * before the fork happens.  Collect up signals sent to multiple
1851c3ad2c3bSEric W. Biederman 	 * processes that happen during the fork and delay them so that
1852c3ad2c3bSEric W. Biederman 	 * they appear to happen after the fork.
1853c3ad2c3bSEric W. Biederman 	 */
1854c3ad2c3bSEric W. Biederman 	sigemptyset(&delayed.signal);
1855c3ad2c3bSEric W. Biederman 	INIT_HLIST_NODE(&delayed.node);
1856c3ad2c3bSEric W. Biederman 
1857c3ad2c3bSEric W. Biederman 	spin_lock_irq(&current->sighand->siglock);
1858c3ad2c3bSEric W. Biederman 	if (!(clone_flags & CLONE_THREAD))
1859c3ad2c3bSEric W. Biederman 		hlist_add_head(&delayed.node, &current->signal->multiprocess);
1860c3ad2c3bSEric W. Biederman 	recalc_sigpending();
1861c3ad2c3bSEric W. Biederman 	spin_unlock_irq(&current->sighand->siglock);
1862c3ad2c3bSEric W. Biederman 	retval = -ERESTARTNOINTR;
1863c3ad2c3bSEric W. Biederman 	if (signal_pending(current))
1864c3ad2c3bSEric W. Biederman 		goto fork_out;
1865c3ad2c3bSEric W. Biederman 
18661da177e4SLinus Torvalds 	retval = -ENOMEM;
1867725fc629SAndi Kleen 	p = dup_task_struct(current, node);
18681da177e4SLinus Torvalds 	if (!p)
18691da177e4SLinus Torvalds 		goto fork_out;
18701da177e4SLinus Torvalds 
18714d6501dcSVegard Nossum 	/*
18724d6501dcSVegard Nossum 	 * This _must_ happen before we call free_task(), i.e. before we jump
18734d6501dcSVegard Nossum 	 * to any of the bad_fork_* labels. This is to avoid freeing
18744d6501dcSVegard Nossum 	 * p->set_child_tid which is (ab)used as a kthread's data pointer for
18754d6501dcSVegard Nossum 	 * kernel threads (PF_KTHREAD).
18764d6501dcSVegard Nossum 	 */
18774d6501dcSVegard Nossum 	p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
18784d6501dcSVegard Nossum 	/*
18794d6501dcSVegard Nossum 	 * Clear TID on mm_release()?
18804d6501dcSVegard Nossum 	 */
18814d6501dcSVegard Nossum 	p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL;
18824d6501dcSVegard Nossum 
1883f7e8b616SSteven Rostedt 	ftrace_graph_init_task(p);
1884f7e8b616SSteven Rostedt 
1885bea493a0SPeter Zijlstra 	rt_mutex_init_task(p);
1886bea493a0SPeter Zijlstra 
1887d12c1a37SIngo Molnar #ifdef CONFIG_PROVE_LOCKING
1888de30a2b3SIngo Molnar 	DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
1889de30a2b3SIngo Molnar 	DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1890de30a2b3SIngo Molnar #endif
18911da177e4SLinus Torvalds 	retval = -EAGAIN;
18923b11a1deSDavid Howells 	if (atomic_read(&p->real_cred->user->processes) >=
189378d7d407SJiri Slaby 			task_rlimit(p, RLIMIT_NPROC)) {
1894b57922b6SEric Paris 		if (p->real_cred->user != INIT_USER &&
1895b57922b6SEric Paris 		    !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
18961da177e4SLinus Torvalds 			goto bad_fork_free;
18971da177e4SLinus Torvalds 	}
189872fa5997SVasiliy Kulikov 	current->flags &= ~PF_NPROC_EXCEEDED;
18991da177e4SLinus Torvalds 
1900f1752eecSDavid Howells 	retval = copy_creds(p, clone_flags);
1901f1752eecSDavid Howells 	if (retval < 0)
1902f1752eecSDavid Howells 		goto bad_fork_free;
19031da177e4SLinus Torvalds 
19041da177e4SLinus Torvalds 	/*
19051da177e4SLinus Torvalds 	 * If multiple threads are within copy_process(), then this check
19061da177e4SLinus Torvalds 	 * triggers too late. This doesn't hurt, the check is only there
19071da177e4SLinus Torvalds 	 * to stop root fork bombs.
19081da177e4SLinus Torvalds 	 */
190904ec93feSLi Zefan 	retval = -EAGAIN;
19101da177e4SLinus Torvalds 	if (nr_threads >= max_threads)
19111da177e4SLinus Torvalds 		goto bad_fork_cleanup_count;
19121da177e4SLinus Torvalds 
1913ca74e92bSShailabh Nagar 	delayacct_tsk_init(p);	/* Must remain after dup_task_struct() */
1914c1de45caSPeter Zijlstra 	p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE);
1915514ddb44SDavid Rientjes 	p->flags |= PF_FORKNOEXEC;
19161da177e4SLinus Torvalds 	INIT_LIST_HEAD(&p->children);
19171da177e4SLinus Torvalds 	INIT_LIST_HEAD(&p->sibling);
1918f41d911fSPaul E. McKenney 	rcu_copy_process(p);
19191da177e4SLinus Torvalds 	p->vfork_done = NULL;
19201da177e4SLinus Torvalds 	spin_lock_init(&p->alloc_lock);
19211da177e4SLinus Torvalds 
19221da177e4SLinus Torvalds 	init_sigpending(&p->pending);
19231da177e4SLinus Torvalds 
192464861634SMartin Schwidefsky 	p->utime = p->stime = p->gtime = 0;
192540565b5aSStanislaw Gruszka #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
192664861634SMartin Schwidefsky 	p->utimescaled = p->stimescaled = 0;
192740565b5aSStanislaw Gruszka #endif
19289d7fb042SPeter Zijlstra 	prev_cputime_init(&p->prev_cputime);
19299d7fb042SPeter Zijlstra 
19306a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
1931bac5b6b6SFrederic Weisbecker 	seqcount_init(&p->vtime.seqcount);
1932bac5b6b6SFrederic Weisbecker 	p->vtime.starttime = 0;
1933bac5b6b6SFrederic Weisbecker 	p->vtime.state = VTIME_INACTIVE;
19346a61671bSFrederic Weisbecker #endif
19356a61671bSFrederic Weisbecker 
1936a3a2e76cSKAMEZAWA Hiroyuki #if defined(SPLIT_RSS_COUNTING)
1937a3a2e76cSKAMEZAWA Hiroyuki 	memset(&p->rss_stat, 0, sizeof(p->rss_stat));
1938a3a2e76cSKAMEZAWA Hiroyuki #endif
1939172ba844SBalbir Singh 
19406976675dSArjan van de Ven 	p->default_timer_slack_ns = current->timer_slack_ns;
19416976675dSArjan van de Ven 
1942eb414681SJohannes Weiner #ifdef CONFIG_PSI
1943eb414681SJohannes Weiner 	p->psi_flags = 0;
1944eb414681SJohannes Weiner #endif
1945eb414681SJohannes Weiner 
19465995477aSAndrea Righi 	task_io_accounting_init(&p->ioac);
19471da177e4SLinus Torvalds 	acct_clear_integrals(p);
19481da177e4SLinus Torvalds 
1949f06febc9SFrank Mayhar 	posix_cpu_timers_init(p);
19501da177e4SLinus Torvalds 
19511da177e4SLinus Torvalds 	p->io_context = NULL;
1952c0b0ae8aSRichard Guy Briggs 	audit_set_context(p, NULL);
1953b4f48b63SPaul Menage 	cgroup_fork(p);
19541da177e4SLinus Torvalds #ifdef CONFIG_NUMA
1955846a16bfSLee Schermerhorn 	p->mempolicy = mpol_dup(p->mempolicy);
19561da177e4SLinus Torvalds 	if (IS_ERR(p->mempolicy)) {
19571da177e4SLinus Torvalds 		retval = PTR_ERR(p->mempolicy);
19581da177e4SLinus Torvalds 		p->mempolicy = NULL;
1959e8604cb4SLi Zefan 		goto bad_fork_cleanup_threadgroup_lock;
19601da177e4SLinus Torvalds 	}
19611da177e4SLinus Torvalds #endif
1962778d3b0fSMichal Hocko #ifdef CONFIG_CPUSETS
1963778d3b0fSMichal Hocko 	p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
1964778d3b0fSMichal Hocko 	p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
1965cc9a6c87SMel Gorman 	seqcount_init(&p->mems_allowed_seq);
1966778d3b0fSMichal Hocko #endif
1967de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS
1968de30a2b3SIngo Molnar 	p->irq_events = 0;
1969de30a2b3SIngo Molnar 	p->hardirqs_enabled = 0;
1970de30a2b3SIngo Molnar 	p->hardirq_enable_ip = 0;
1971de30a2b3SIngo Molnar 	p->hardirq_enable_event = 0;
1972de30a2b3SIngo Molnar 	p->hardirq_disable_ip = _THIS_IP_;
1973de30a2b3SIngo Molnar 	p->hardirq_disable_event = 0;
1974de30a2b3SIngo Molnar 	p->softirqs_enabled = 1;
1975de30a2b3SIngo Molnar 	p->softirq_enable_ip = _THIS_IP_;
1976de30a2b3SIngo Molnar 	p->softirq_enable_event = 0;
1977de30a2b3SIngo Molnar 	p->softirq_disable_ip = 0;
1978de30a2b3SIngo Molnar 	p->softirq_disable_event = 0;
1979de30a2b3SIngo Molnar 	p->hardirq_context = 0;
1980de30a2b3SIngo Molnar 	p->softirq_context = 0;
1981de30a2b3SIngo Molnar #endif
19828bcbde54SDavid Hildenbrand 
19838bcbde54SDavid Hildenbrand 	p->pagefault_disabled = 0;
19848bcbde54SDavid Hildenbrand 
1985fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP
1986fbb9ce95SIngo Molnar 	p->lockdep_depth = 0; /* no locks held yet */
1987fbb9ce95SIngo Molnar 	p->curr_chain_key = 0;
1988fbb9ce95SIngo Molnar 	p->lockdep_recursion = 0;
1989b09be676SByungchul Park 	lockdep_init_task(p);
1990fbb9ce95SIngo Molnar #endif
19911da177e4SLinus Torvalds 
1992408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES
1993408894eeSIngo Molnar 	p->blocked_on = NULL; /* not blocked yet */
1994408894eeSIngo Molnar #endif
1995cafe5635SKent Overstreet #ifdef CONFIG_BCACHE
1996cafe5635SKent Overstreet 	p->sequential_io	= 0;
1997cafe5635SKent Overstreet 	p->sequential_io_avg	= 0;
1998cafe5635SKent Overstreet #endif
19990f481406SMarkus Metzger 
20003c90e6e9SSrivatsa Vaddagiri 	/* Perform scheduler related setup. Assign this task to a CPU. */
2001aab03e05SDario Faggioli 	retval = sched_fork(clone_flags, p);
2002aab03e05SDario Faggioli 	if (retval)
2003aab03e05SDario Faggioli 		goto bad_fork_cleanup_policy;
20046ab423e0SPeter Zijlstra 
2005cdd6c482SIngo Molnar 	retval = perf_event_init_task(p);
20066ab423e0SPeter Zijlstra 	if (retval)
20076ab423e0SPeter Zijlstra 		goto bad_fork_cleanup_policy;
2008fb0a685cSDaniel Rebelo de Oliveira 	retval = audit_alloc(p);
2009fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20106c72e350SPeter Zijlstra 		goto bad_fork_cleanup_perf;
20111da177e4SLinus Torvalds 	/* copy all the process information */
2012ab602f79SJack Miller 	shm_init_task(p);
2013e4e55b47STetsuo Handa 	retval = security_task_alloc(p, clone_flags);
2014fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20151da177e4SLinus Torvalds 		goto bad_fork_cleanup_audit;
2016e4e55b47STetsuo Handa 	retval = copy_semundo(clone_flags, p);
2017e4e55b47STetsuo Handa 	if (retval)
2018e4e55b47STetsuo Handa 		goto bad_fork_cleanup_security;
2019fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_files(clone_flags, p);
2020fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20211da177e4SLinus Torvalds 		goto bad_fork_cleanup_semundo;
2022fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_fs(clone_flags, p);
2023fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20241da177e4SLinus Torvalds 		goto bad_fork_cleanup_files;
2025fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_sighand(clone_flags, p);
2026fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20271da177e4SLinus Torvalds 		goto bad_fork_cleanup_fs;
2028fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_signal(clone_flags, p);
2029fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20301da177e4SLinus Torvalds 		goto bad_fork_cleanup_sighand;
2031fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_mm(clone_flags, p);
2032fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
20331da177e4SLinus Torvalds 		goto bad_fork_cleanup_signal;
2034fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_namespaces(clone_flags, p);
2035fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
2036d84f4f99SDavid Howells 		goto bad_fork_cleanup_mm;
2037fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_io(clone_flags, p);
2038fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
2039fd0928dfSJens Axboe 		goto bad_fork_cleanup_namespaces;
20403033f14aSJosh Triplett 	retval = copy_thread_tls(clone_flags, stack_start, stack_size, p, tls);
20411da177e4SLinus Torvalds 	if (retval)
2042fd0928dfSJens Axboe 		goto bad_fork_cleanup_io;
20431da177e4SLinus Torvalds 
2044afaef01cSAlexander Popov 	stackleak_task_init(p);
2045afaef01cSAlexander Popov 
2046425fb2b4SPavel Emelyanov 	if (pid != &init_struct_pid) {
2047c2b1df2eSAndy Lutomirski 		pid = alloc_pid(p->nsproxy->pid_ns_for_children);
204835f71bc0SMichal Hocko 		if (IS_ERR(pid)) {
204935f71bc0SMichal Hocko 			retval = PTR_ERR(pid);
20500740aa5fSJiri Slaby 			goto bad_fork_cleanup_thread;
2051425fb2b4SPavel Emelyanov 		}
205235f71bc0SMichal Hocko 	}
2053425fb2b4SPavel Emelyanov 
2054b3e58382SChristian Brauner 	/*
2055b3e58382SChristian Brauner 	 * This has to happen after we've potentially unshared the file
2056b3e58382SChristian Brauner 	 * descriptor table (so that the pidfd doesn't leak into the child
2057b3e58382SChristian Brauner 	 * if the fd table isn't shared).
2058b3e58382SChristian Brauner 	 */
2059b3e58382SChristian Brauner 	if (clone_flags & CLONE_PIDFD) {
2060b3e58382SChristian Brauner 		retval = pidfd_create(pid);
2061b3e58382SChristian Brauner 		if (retval < 0)
2062b3e58382SChristian Brauner 			goto bad_fork_free_pid;
2063b3e58382SChristian Brauner 
2064b3e58382SChristian Brauner 		pidfd = retval;
2065b3e58382SChristian Brauner 		retval = put_user(pidfd, parent_tidptr);
2066b3e58382SChristian Brauner 		if (retval)
2067b3e58382SChristian Brauner 			goto bad_fork_put_pidfd;
2068b3e58382SChristian Brauner 	}
2069b3e58382SChristian Brauner 
207073c10101SJens Axboe #ifdef CONFIG_BLOCK
207173c10101SJens Axboe 	p->plug = NULL;
207273c10101SJens Axboe #endif
207342b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX
20748f17d3a5SIngo Molnar 	p->robust_list = NULL;
20758f17d3a5SIngo Molnar #ifdef CONFIG_COMPAT
20768f17d3a5SIngo Molnar 	p->compat_robust_list = NULL;
20778f17d3a5SIngo Molnar #endif
2078c87e2837SIngo Molnar 	INIT_LIST_HEAD(&p->pi_state_list);
2079c87e2837SIngo Molnar 	p->pi_state_cache = NULL;
208042b2dd0aSAlexey Dobriyan #endif
20811da177e4SLinus Torvalds 	/*
2082f9a3879aSGOTO Masanori 	 * sigaltstack should be cleared when sharing the same VM
2083f9a3879aSGOTO Masanori 	 */
2084f9a3879aSGOTO Masanori 	if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
20852a742138SStas Sergeev 		sas_ss_reset(p);
2086f9a3879aSGOTO Masanori 
2087f9a3879aSGOTO Masanori 	/*
20886580807dSOleg Nesterov 	 * Syscall tracing and stepping should be turned off in the
20896580807dSOleg Nesterov 	 * child regardless of CLONE_PTRACE.
20901da177e4SLinus Torvalds 	 */
20916580807dSOleg Nesterov 	user_disable_single_step(p);
20921da177e4SLinus Torvalds 	clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
2093ed75e8d5SLaurent Vivier #ifdef TIF_SYSCALL_EMU
2094ed75e8d5SLaurent Vivier 	clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
2095ed75e8d5SLaurent Vivier #endif
2096*e02c9b0dSLin Feng 	clear_tsk_latency_tracing(p);
20971da177e4SLinus Torvalds 
20981da177e4SLinus Torvalds 	/* ok, now we should be set up.. */
209918c830dfSOleg Nesterov 	p->pid = pid_nr(pid);
210018c830dfSOleg Nesterov 	if (clone_flags & CLONE_THREAD) {
21015f8aadd8SOleg Nesterov 		p->exit_signal = -1;
210218c830dfSOleg Nesterov 		p->group_leader = current->group_leader;
210318c830dfSOleg Nesterov 		p->tgid = current->tgid;
210418c830dfSOleg Nesterov 	} else {
210518c830dfSOleg Nesterov 		if (clone_flags & CLONE_PARENT)
21065f8aadd8SOleg Nesterov 			p->exit_signal = current->group_leader->exit_signal;
21075f8aadd8SOleg Nesterov 		else
21085f8aadd8SOleg Nesterov 			p->exit_signal = (clone_flags & CSIGNAL);
210918c830dfSOleg Nesterov 		p->group_leader = p;
211018c830dfSOleg Nesterov 		p->tgid = p->pid;
211118c830dfSOleg Nesterov 	}
21125f8aadd8SOleg Nesterov 
21139d823e8fSWu Fengguang 	p->nr_dirtied = 0;
21149d823e8fSWu Fengguang 	p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
211583712358SWu Fengguang 	p->dirty_paused_when = 0;
21169d823e8fSWu Fengguang 
2117bb8cbbfeSOleg Nesterov 	p->pdeath_signal = 0;
211847e65328SOleg Nesterov 	INIT_LIST_HEAD(&p->thread_group);
2119158e1645SAl Viro 	p->task_works = NULL;
21201da177e4SLinus Torvalds 
2121780de9ddSIngo Molnar 	cgroup_threadgroup_change_begin(current);
212218c830dfSOleg Nesterov 	/*
21237e47682eSAleksa Sarai 	 * Ensure that the cgroup subsystem policies allow the new process to be
21247e47682eSAleksa Sarai 	 * forked. It should be noted the the new process's css_set can be changed
21257e47682eSAleksa Sarai 	 * between here and cgroup_post_fork() if an organisation operation is in
21267e47682eSAleksa Sarai 	 * progress.
21277e47682eSAleksa Sarai 	 */
2128b53202e6SOleg Nesterov 	retval = cgroup_can_fork(p);
21297e47682eSAleksa Sarai 	if (retval)
2130c3b7112dSChristian Brauner 		goto bad_fork_cgroup_threadgroup_change_end;
21317e47682eSAleksa Sarai 
21327e47682eSAleksa Sarai 	/*
21337b558513SDavid Herrmann 	 * From this point on we must avoid any synchronous user-space
21347b558513SDavid Herrmann 	 * communication until we take the tasklist-lock. In particular, we do
21357b558513SDavid Herrmann 	 * not want user-space to be able to predict the process start-time by
21367b558513SDavid Herrmann 	 * stalling fork(2) after we recorded the start_time but before it is
21377b558513SDavid Herrmann 	 * visible to the system.
21387b558513SDavid Herrmann 	 */
21397b558513SDavid Herrmann 
21407b558513SDavid Herrmann 	p->start_time = ktime_get_ns();
21417b558513SDavid Herrmann 	p->real_start_time = ktime_get_boot_ns();
21427b558513SDavid Herrmann 
21437b558513SDavid Herrmann 	/*
214418c830dfSOleg Nesterov 	 * Make it visible to the rest of the system, but dont wake it up yet.
214518c830dfSOleg Nesterov 	 * Need tasklist lock for parent etc handling!
214618c830dfSOleg Nesterov 	 */
21471da177e4SLinus Torvalds 	write_lock_irq(&tasklist_lock);
21481da177e4SLinus Torvalds 
21491da177e4SLinus Torvalds 	/* CLONE_PARENT re-uses the old parent */
21502d5516cbSOleg Nesterov 	if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
21511da177e4SLinus Torvalds 		p->real_parent = current->real_parent;
21522d5516cbSOleg Nesterov 		p->parent_exec_id = current->parent_exec_id;
21532d5516cbSOleg Nesterov 	} else {
21541da177e4SLinus Torvalds 		p->real_parent = current;
21552d5516cbSOleg Nesterov 		p->parent_exec_id = current->self_exec_id;
21562d5516cbSOleg Nesterov 	}
21571da177e4SLinus Torvalds 
2158d83a7cb3SJosh Poimboeuf 	klp_copy_process(p);
2159d83a7cb3SJosh Poimboeuf 
21601da177e4SLinus Torvalds 	spin_lock(&current->sighand->siglock);
21614a2c7a78SOleg Nesterov 
21624a2c7a78SOleg Nesterov 	/*
2163dbd95212SKees Cook 	 * Copy seccomp details explicitly here, in case they were changed
2164dbd95212SKees Cook 	 * before holding sighand lock.
2165dbd95212SKees Cook 	 */
2166dbd95212SKees Cook 	copy_seccomp(p);
2167dbd95212SKees Cook 
2168d7822b1eSMathieu Desnoyers 	rseq_fork(p, clone_flags);
2169d7822b1eSMathieu Desnoyers 
21704ca1d3eeSEric W. Biederman 	/* Don't start children in a dying pid namespace */
2171e8cfbc24SGargi Sharma 	if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
21723fd37226SKirill Tkhai 		retval = -ENOMEM;
21733fd37226SKirill Tkhai 		goto bad_fork_cancel_cgroup;
21743fd37226SKirill Tkhai 	}
21754a2c7a78SOleg Nesterov 
21767673bf55SEric W. Biederman 	/* Let kill terminate clone/fork in the middle */
21777673bf55SEric W. Biederman 	if (fatal_signal_pending(current)) {
21787673bf55SEric W. Biederman 		retval = -EINTR;
21797673bf55SEric W. Biederman 		goto bad_fork_cancel_cgroup;
21807673bf55SEric W. Biederman 	}
21817673bf55SEric W. Biederman 
21821da177e4SLinus Torvalds 
21832c470475SEric W. Biederman 	init_task_pid_links(p);
218473b9ebfeSOleg Nesterov 	if (likely(p->pid)) {
21854b9d33e6STejun Heo 		ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
21861da177e4SLinus Torvalds 
218781907739SOleg Nesterov 		init_task_pid(p, PIDTYPE_PID, pid);
21881da177e4SLinus Torvalds 		if (thread_group_leader(p)) {
21896883f81aSEric W. Biederman 			init_task_pid(p, PIDTYPE_TGID, pid);
219081907739SOleg Nesterov 			init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
219181907739SOleg Nesterov 			init_task_pid(p, PIDTYPE_SID, task_session(current));
219281907739SOleg Nesterov 
21931c4042c2SEric W. Biederman 			if (is_child_reaper(pid)) {
219417cf22c3SEric W. Biederman 				ns_of_pid(pid)->child_reaper = p;
21951c4042c2SEric W. Biederman 				p->signal->flags |= SIGNAL_UNKILLABLE;
21961c4042c2SEric W. Biederman 			}
2197c3ad2c3bSEric W. Biederman 			p->signal->shared_pending.signal = delayed.signal;
21989c9f4dedSAlan Cox 			p->signal->tty = tty_kref_get(current->signal->tty);
2199749860ceSPavel Tikhomirov 			/*
2200749860ceSPavel Tikhomirov 			 * Inherit has_child_subreaper flag under the same
2201749860ceSPavel Tikhomirov 			 * tasklist_lock with adding child to the process tree
2202749860ceSPavel Tikhomirov 			 * for propagate_has_child_subreaper optimization.
2203749860ceSPavel Tikhomirov 			 */
2204749860ceSPavel Tikhomirov 			p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||
2205749860ceSPavel Tikhomirov 							 p->real_parent->signal->is_child_subreaper;
22069cd80bbbSOleg Nesterov 			list_add_tail(&p->sibling, &p->real_parent->children);
22075e85d4abSEric W. Biederman 			list_add_tail_rcu(&p->tasks, &init_task.tasks);
22086883f81aSEric W. Biederman 			attach_pid(p, PIDTYPE_TGID);
220981907739SOleg Nesterov 			attach_pid(p, PIDTYPE_PGID);
221081907739SOleg Nesterov 			attach_pid(p, PIDTYPE_SID);
2211909ea964SChristoph Lameter 			__this_cpu_inc(process_counts);
221280628ca0SOleg Nesterov 		} else {
221380628ca0SOleg Nesterov 			current->signal->nr_threads++;
221480628ca0SOleg Nesterov 			atomic_inc(&current->signal->live);
221560d4de3fSElena Reshetova 			refcount_inc(&current->signal->sigcnt);
2216924de3b8SEric W. Biederman 			task_join_group_stop(p);
221780628ca0SOleg Nesterov 			list_add_tail_rcu(&p->thread_group,
221880628ca0SOleg Nesterov 					  &p->group_leader->thread_group);
22190c740d0aSOleg Nesterov 			list_add_tail_rcu(&p->thread_node,
22200c740d0aSOleg Nesterov 					  &p->signal->thread_head);
22211da177e4SLinus Torvalds 		}
222281907739SOleg Nesterov 		attach_pid(p, PIDTYPE_PID);
22231da177e4SLinus Torvalds 		nr_threads++;
222473b9ebfeSOleg Nesterov 	}
22251da177e4SLinus Torvalds 	total_forks++;
2226c3ad2c3bSEric W. Biederman 	hlist_del_init(&delayed.node);
22273f17da69SOleg Nesterov 	spin_unlock(&current->sighand->siglock);
22284af4206bSOleg Nesterov 	syscall_tracepoint_update(p);
22291da177e4SLinus Torvalds 	write_unlock_irq(&tasklist_lock);
22304af4206bSOleg Nesterov 
2231c13cf856SAndrew Morton 	proc_fork_connector(p);
2232b53202e6SOleg Nesterov 	cgroup_post_fork(p);
2233780de9ddSIngo Molnar 	cgroup_threadgroup_change_end(current);
2234cdd6c482SIngo Molnar 	perf_event_fork(p);
223543d2b113SKAMEZAWA Hiroyuki 
223643d2b113SKAMEZAWA Hiroyuki 	trace_task_newtask(p, clone_flags);
22373ab67966SOleg Nesterov 	uprobe_copy_process(p, clone_flags);
223843d2b113SKAMEZAWA Hiroyuki 
22391da177e4SLinus Torvalds 	return p;
22401da177e4SLinus Torvalds 
22417e47682eSAleksa Sarai bad_fork_cancel_cgroup:
22423fd37226SKirill Tkhai 	spin_unlock(&current->sighand->siglock);
22433fd37226SKirill Tkhai 	write_unlock_irq(&tasklist_lock);
2244b53202e6SOleg Nesterov 	cgroup_cancel_fork(p);
2245c3b7112dSChristian Brauner bad_fork_cgroup_threadgroup_change_end:
2246c3b7112dSChristian Brauner 	cgroup_threadgroup_change_end(current);
2247b3e58382SChristian Brauner bad_fork_put_pidfd:
2248b3e58382SChristian Brauner 	if (clone_flags & CLONE_PIDFD)
2249b3e58382SChristian Brauner 		ksys_close(pidfd);
2250425fb2b4SPavel Emelyanov bad_fork_free_pid:
2251425fb2b4SPavel Emelyanov 	if (pid != &init_struct_pid)
2252425fb2b4SPavel Emelyanov 		free_pid(pid);
22530740aa5fSJiri Slaby bad_fork_cleanup_thread:
22540740aa5fSJiri Slaby 	exit_thread(p);
2255fd0928dfSJens Axboe bad_fork_cleanup_io:
2256b69f2292SLouis Rilling 	if (p->io_context)
2257b69f2292SLouis Rilling 		exit_io_context(p);
2258ab516013SSerge E. Hallyn bad_fork_cleanup_namespaces:
2259444f378bSLinus Torvalds 	exit_task_namespaces(p);
22601da177e4SLinus Torvalds bad_fork_cleanup_mm:
2261c3f3ce04SAndrea Arcangeli 	if (p->mm) {
2262c3f3ce04SAndrea Arcangeli 		mm_clear_owner(p->mm, p);
22631da177e4SLinus Torvalds 		mmput(p->mm);
2264c3f3ce04SAndrea Arcangeli 	}
22651da177e4SLinus Torvalds bad_fork_cleanup_signal:
22664ab6c083SOleg Nesterov 	if (!(clone_flags & CLONE_THREAD))
22671c5354deSMike Galbraith 		free_signal_struct(p->signal);
22681da177e4SLinus Torvalds bad_fork_cleanup_sighand:
2269a7e5328aSOleg Nesterov 	__cleanup_sighand(p->sighand);
22701da177e4SLinus Torvalds bad_fork_cleanup_fs:
22711da177e4SLinus Torvalds 	exit_fs(p); /* blocking */
22721da177e4SLinus Torvalds bad_fork_cleanup_files:
22731da177e4SLinus Torvalds 	exit_files(p); /* blocking */
22741da177e4SLinus Torvalds bad_fork_cleanup_semundo:
22751da177e4SLinus Torvalds 	exit_sem(p);
2276e4e55b47STetsuo Handa bad_fork_cleanup_security:
2277e4e55b47STetsuo Handa 	security_task_free(p);
22781da177e4SLinus Torvalds bad_fork_cleanup_audit:
22791da177e4SLinus Torvalds 	audit_free(p);
22806c72e350SPeter Zijlstra bad_fork_cleanup_perf:
2281cdd6c482SIngo Molnar 	perf_event_free_task(p);
22826c72e350SPeter Zijlstra bad_fork_cleanup_policy:
2283b09be676SByungchul Park 	lockdep_free_task(p);
22841da177e4SLinus Torvalds #ifdef CONFIG_NUMA
2285f0be3d32SLee Schermerhorn 	mpol_put(p->mempolicy);
2286e8604cb4SLi Zefan bad_fork_cleanup_threadgroup_lock:
22871da177e4SLinus Torvalds #endif
228835df17c5SShailabh Nagar 	delayacct_tsk_free(p);
22891da177e4SLinus Torvalds bad_fork_cleanup_count:
2290d84f4f99SDavid Howells 	atomic_dec(&p->cred->user->processes);
2291e0e81739SDavid Howells 	exit_creds(p);
22921da177e4SLinus Torvalds bad_fork_free:
2293405c0759SAndy Lutomirski 	p->state = TASK_DEAD;
229468f24b08SAndy Lutomirski 	put_task_stack(p);
2295c3f3ce04SAndrea Arcangeli 	delayed_free_task(p);
2296fe7d37d1SOleg Nesterov fork_out:
2297c3ad2c3bSEric W. Biederman 	spin_lock_irq(&current->sighand->siglock);
2298c3ad2c3bSEric W. Biederman 	hlist_del_init(&delayed.node);
2299c3ad2c3bSEric W. Biederman 	spin_unlock_irq(&current->sighand->siglock);
2300fe7d37d1SOleg Nesterov 	return ERR_PTR(retval);
23011da177e4SLinus Torvalds }
23021da177e4SLinus Torvalds 
23032c470475SEric W. Biederman static inline void init_idle_pids(struct task_struct *idle)
2304f106eee1SOleg Nesterov {
2305f106eee1SOleg Nesterov 	enum pid_type type;
2306f106eee1SOleg Nesterov 
2307f106eee1SOleg Nesterov 	for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
23082c470475SEric W. Biederman 		INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */
23092c470475SEric W. Biederman 		init_task_pid(idle, type, &init_struct_pid);
2310f106eee1SOleg Nesterov 	}
2311f106eee1SOleg Nesterov }
2312f106eee1SOleg Nesterov 
23130db0628dSPaul Gortmaker struct task_struct *fork_idle(int cpu)
23141da177e4SLinus Torvalds {
231536c8b586SIngo Molnar 	struct task_struct *task;
2316b3e58382SChristian Brauner 	task = copy_process(CLONE_VM, 0, 0, NULL, NULL, &init_struct_pid, 0, 0,
2317725fc629SAndi Kleen 			    cpu_to_node(cpu));
2318f106eee1SOleg Nesterov 	if (!IS_ERR(task)) {
23192c470475SEric W. Biederman 		init_idle_pids(task);
23201da177e4SLinus Torvalds 		init_idle(task, cpu);
2321f106eee1SOleg Nesterov 	}
232273b9ebfeSOleg Nesterov 
23231da177e4SLinus Torvalds 	return task;
23241da177e4SLinus Torvalds }
23251da177e4SLinus Torvalds 
232613585fa0SNadav Amit struct mm_struct *copy_init_mm(void)
232713585fa0SNadav Amit {
232813585fa0SNadav Amit 	return dup_mm(NULL, &init_mm);
232913585fa0SNadav Amit }
233013585fa0SNadav Amit 
23311da177e4SLinus Torvalds /*
23321da177e4SLinus Torvalds  *  Ok, this is the main fork-routine.
23331da177e4SLinus Torvalds  *
23341da177e4SLinus Torvalds  * It copies the process, and if successful kick-starts
23351da177e4SLinus Torvalds  * it and waits for it to finish using the VM if required.
23361da177e4SLinus Torvalds  */
23373033f14aSJosh Triplett long _do_fork(unsigned long clone_flags,
23381da177e4SLinus Torvalds 	      unsigned long stack_start,
23391da177e4SLinus Torvalds 	      unsigned long stack_size,
23401da177e4SLinus Torvalds 	      int __user *parent_tidptr,
23413033f14aSJosh Triplett 	      int __user *child_tidptr,
23423033f14aSJosh Triplett 	      unsigned long tls)
23431da177e4SLinus Torvalds {
23449f5325aaSMarcos Paulo de Souza 	struct completion vfork;
23459f5325aaSMarcos Paulo de Souza 	struct pid *pid;
23461da177e4SLinus Torvalds 	struct task_struct *p;
23471da177e4SLinus Torvalds 	int trace = 0;
234892476d7fSEric W. Biederman 	long nr;
23491da177e4SLinus Torvalds 
2350bdff746aSAndrew Morton 	/*
23514b9d33e6STejun Heo 	 * Determine whether and which event to report to ptracer.  When
23524b9d33e6STejun Heo 	 * called from kernel_thread or CLONE_UNTRACED is explicitly
23534b9d33e6STejun Heo 	 * requested, no event is reported; otherwise, report if the event
23544b9d33e6STejun Heo 	 * for the type of forking is enabled.
235509a05394SRoland McGrath 	 */
2356e80d6661SAl Viro 	if (!(clone_flags & CLONE_UNTRACED)) {
23574b9d33e6STejun Heo 		if (clone_flags & CLONE_VFORK)
23584b9d33e6STejun Heo 			trace = PTRACE_EVENT_VFORK;
23594b9d33e6STejun Heo 		else if ((clone_flags & CSIGNAL) != SIGCHLD)
23604b9d33e6STejun Heo 			trace = PTRACE_EVENT_CLONE;
23614b9d33e6STejun Heo 		else
23624b9d33e6STejun Heo 			trace = PTRACE_EVENT_FORK;
23634b9d33e6STejun Heo 
23644b9d33e6STejun Heo 		if (likely(!ptrace_event_enabled(current, trace)))
23654b9d33e6STejun Heo 			trace = 0;
23664b9d33e6STejun Heo 	}
23671da177e4SLinus Torvalds 
2368b3e58382SChristian Brauner 	p = copy_process(clone_flags, stack_start, stack_size, parent_tidptr,
2369725fc629SAndi Kleen 			 child_tidptr, NULL, trace, tls, NUMA_NO_NODE);
237038addce8SEmese Revfy 	add_latent_entropy();
23719f5325aaSMarcos Paulo de Souza 
23729f5325aaSMarcos Paulo de Souza 	if (IS_ERR(p))
23739f5325aaSMarcos Paulo de Souza 		return PTR_ERR(p);
23749f5325aaSMarcos Paulo de Souza 
23751da177e4SLinus Torvalds 	/*
23761da177e4SLinus Torvalds 	 * Do this prior waking up the new thread - the thread pointer
23771da177e4SLinus Torvalds 	 * might get invalid after that point, if the thread exits quickly.
23781da177e4SLinus Torvalds 	 */
23790a16b607SMathieu Desnoyers 	trace_sched_process_fork(current, p);
23800a16b607SMathieu Desnoyers 
23814e52365fSMatthew Dempsky 	pid = get_task_pid(p, PIDTYPE_PID);
23824e52365fSMatthew Dempsky 	nr = pid_vnr(pid);
238330e49c26SPavel Emelyanov 
238430e49c26SPavel Emelyanov 	if (clone_flags & CLONE_PARENT_SETTID)
238530e49c26SPavel Emelyanov 		put_user(nr, parent_tidptr);
2386a6f5e063SSukadev Bhattiprolu 
23871da177e4SLinus Torvalds 	if (clone_flags & CLONE_VFORK) {
23881da177e4SLinus Torvalds 		p->vfork_done = &vfork;
23891da177e4SLinus Torvalds 		init_completion(&vfork);
2390d68b46feSOleg Nesterov 		get_task_struct(p);
23911da177e4SLinus Torvalds 	}
23921da177e4SLinus Torvalds 
23933e51e3edSSamir Bellabes 	wake_up_new_task(p);
23941da177e4SLinus Torvalds 
23954b9d33e6STejun Heo 	/* forking complete and child started to run, tell ptracer */
23964b9d33e6STejun Heo 	if (unlikely(trace))
23974e52365fSMatthew Dempsky 		ptrace_event_pid(trace, pid);
239809a05394SRoland McGrath 
23991da177e4SLinus Torvalds 	if (clone_flags & CLONE_VFORK) {
2400d68b46feSOleg Nesterov 		if (!wait_for_vfork_done(p, &vfork))
24014e52365fSMatthew Dempsky 			ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
24029f59ce5dSChuck Ebbert 	}
24034e52365fSMatthew Dempsky 
24044e52365fSMatthew Dempsky 	put_pid(pid);
240592476d7fSEric W. Biederman 	return nr;
24061da177e4SLinus Torvalds }
24071da177e4SLinus Torvalds 
24083033f14aSJosh Triplett #ifndef CONFIG_HAVE_COPY_THREAD_TLS
24093033f14aSJosh Triplett /* For compatibility with architectures that call do_fork directly rather than
24103033f14aSJosh Triplett  * using the syscall entry points below. */
24113033f14aSJosh Triplett long do_fork(unsigned long clone_flags,
24123033f14aSJosh Triplett 	      unsigned long stack_start,
24133033f14aSJosh Triplett 	      unsigned long stack_size,
24143033f14aSJosh Triplett 	      int __user *parent_tidptr,
24153033f14aSJosh Triplett 	      int __user *child_tidptr)
24163033f14aSJosh Triplett {
24173033f14aSJosh Triplett 	return _do_fork(clone_flags, stack_start, stack_size,
24183033f14aSJosh Triplett 			parent_tidptr, child_tidptr, 0);
24193033f14aSJosh Triplett }
24203033f14aSJosh Triplett #endif
24213033f14aSJosh Triplett 
24222aa3a7f8SAl Viro /*
24232aa3a7f8SAl Viro  * Create a kernel thread.
24242aa3a7f8SAl Viro  */
24252aa3a7f8SAl Viro pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
24262aa3a7f8SAl Viro {
24273033f14aSJosh Triplett 	return _do_fork(flags|CLONE_VM|CLONE_UNTRACED, (unsigned long)fn,
24283033f14aSJosh Triplett 		(unsigned long)arg, NULL, NULL, 0);
24292aa3a7f8SAl Viro }
24302aa3a7f8SAl Viro 
2431d2125043SAl Viro #ifdef __ARCH_WANT_SYS_FORK
2432d2125043SAl Viro SYSCALL_DEFINE0(fork)
2433d2125043SAl Viro {
2434d2125043SAl Viro #ifdef CONFIG_MMU
24353033f14aSJosh Triplett 	return _do_fork(SIGCHLD, 0, 0, NULL, NULL, 0);
2436d2125043SAl Viro #else
2437d2125043SAl Viro 	/* can not support in nommu mode */
24385d59e182SDaeseok Youn 	return -EINVAL;
2439d2125043SAl Viro #endif
2440d2125043SAl Viro }
2441d2125043SAl Viro #endif
2442d2125043SAl Viro 
2443d2125043SAl Viro #ifdef __ARCH_WANT_SYS_VFORK
2444d2125043SAl Viro SYSCALL_DEFINE0(vfork)
2445d2125043SAl Viro {
24463033f14aSJosh Triplett 	return _do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, 0,
24473033f14aSJosh Triplett 			0, NULL, NULL, 0);
2448d2125043SAl Viro }
2449d2125043SAl Viro #endif
2450d2125043SAl Viro 
2451d2125043SAl Viro #ifdef __ARCH_WANT_SYS_CLONE
2452d2125043SAl Viro #ifdef CONFIG_CLONE_BACKWARDS
2453d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2454d2125043SAl Viro 		 int __user *, parent_tidptr,
24553033f14aSJosh Triplett 		 unsigned long, tls,
2456d2125043SAl Viro 		 int __user *, child_tidptr)
2457d2125043SAl Viro #elif defined(CONFIG_CLONE_BACKWARDS2)
2458d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
2459d2125043SAl Viro 		 int __user *, parent_tidptr,
2460d2125043SAl Viro 		 int __user *, child_tidptr,
24613033f14aSJosh Triplett 		 unsigned long, tls)
2462dfa9771aSMichal Simek #elif defined(CONFIG_CLONE_BACKWARDS3)
2463dfa9771aSMichal Simek SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
2464dfa9771aSMichal Simek 		int, stack_size,
2465dfa9771aSMichal Simek 		int __user *, parent_tidptr,
2466dfa9771aSMichal Simek 		int __user *, child_tidptr,
24673033f14aSJosh Triplett 		unsigned long, tls)
2468d2125043SAl Viro #else
2469d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2470d2125043SAl Viro 		 int __user *, parent_tidptr,
2471d2125043SAl Viro 		 int __user *, child_tidptr,
24723033f14aSJosh Triplett 		 unsigned long, tls)
2473d2125043SAl Viro #endif
2474d2125043SAl Viro {
24753033f14aSJosh Triplett 	return _do_fork(clone_flags, newsp, 0, parent_tidptr, child_tidptr, tls);
2476d2125043SAl Viro }
2477d2125043SAl Viro #endif
2478d2125043SAl Viro 
24790f1b92cbSOleg Nesterov void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data)
24800f1b92cbSOleg Nesterov {
24810f1b92cbSOleg Nesterov 	struct task_struct *leader, *parent, *child;
24820f1b92cbSOleg Nesterov 	int res;
24830f1b92cbSOleg Nesterov 
24840f1b92cbSOleg Nesterov 	read_lock(&tasklist_lock);
24850f1b92cbSOleg Nesterov 	leader = top = top->group_leader;
24860f1b92cbSOleg Nesterov down:
24870f1b92cbSOleg Nesterov 	for_each_thread(leader, parent) {
24880f1b92cbSOleg Nesterov 		list_for_each_entry(child, &parent->children, sibling) {
24890f1b92cbSOleg Nesterov 			res = visitor(child, data);
24900f1b92cbSOleg Nesterov 			if (res) {
24910f1b92cbSOleg Nesterov 				if (res < 0)
24920f1b92cbSOleg Nesterov 					goto out;
24930f1b92cbSOleg Nesterov 				leader = child;
24940f1b92cbSOleg Nesterov 				goto down;
24950f1b92cbSOleg Nesterov 			}
24960f1b92cbSOleg Nesterov up:
24970f1b92cbSOleg Nesterov 			;
24980f1b92cbSOleg Nesterov 		}
24990f1b92cbSOleg Nesterov 	}
25000f1b92cbSOleg Nesterov 
25010f1b92cbSOleg Nesterov 	if (leader != top) {
25020f1b92cbSOleg Nesterov 		child = leader;
25030f1b92cbSOleg Nesterov 		parent = child->real_parent;
25040f1b92cbSOleg Nesterov 		leader = parent->group_leader;
25050f1b92cbSOleg Nesterov 		goto up;
25060f1b92cbSOleg Nesterov 	}
25070f1b92cbSOleg Nesterov out:
25080f1b92cbSOleg Nesterov 	read_unlock(&tasklist_lock);
25090f1b92cbSOleg Nesterov }
25100f1b92cbSOleg Nesterov 
25115fd63b30SRavikiran G Thirumalai #ifndef ARCH_MIN_MMSTRUCT_ALIGN
25125fd63b30SRavikiran G Thirumalai #define ARCH_MIN_MMSTRUCT_ALIGN 0
25135fd63b30SRavikiran G Thirumalai #endif
25145fd63b30SRavikiran G Thirumalai 
251551cc5068SAlexey Dobriyan static void sighand_ctor(void *data)
2516aa1757f9SOleg Nesterov {
2517aa1757f9SOleg Nesterov 	struct sighand_struct *sighand = data;
2518aa1757f9SOleg Nesterov 
2519aa1757f9SOleg Nesterov 	spin_lock_init(&sighand->siglock);
2520b8fceee1SDavide Libenzi 	init_waitqueue_head(&sighand->signalfd_wqh);
2521fba2afaaSDavide Libenzi }
2522aa1757f9SOleg Nesterov 
25231da177e4SLinus Torvalds void __init proc_caches_init(void)
25241da177e4SLinus Torvalds {
2525c1a2f7f0SRik van Riel 	unsigned int mm_size;
2526c1a2f7f0SRik van Riel 
25271da177e4SLinus Torvalds 	sighand_cachep = kmem_cache_create("sighand_cache",
25281da177e4SLinus Torvalds 			sizeof(struct sighand_struct), 0,
25295f0d5a3aSPaul E. McKenney 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
253075f296d9SLevin, Alexander (Sasha Levin) 			SLAB_ACCOUNT, sighand_ctor);
25311da177e4SLinus Torvalds 	signal_cachep = kmem_cache_create("signal_cache",
25321da177e4SLinus Torvalds 			sizeof(struct signal_struct), 0,
253375f296d9SLevin, Alexander (Sasha Levin) 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
25345d097056SVladimir Davydov 			NULL);
25351da177e4SLinus Torvalds 	files_cachep = kmem_cache_create("files_cache",
25361da177e4SLinus Torvalds 			sizeof(struct files_struct), 0,
253775f296d9SLevin, Alexander (Sasha Levin) 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
25385d097056SVladimir Davydov 			NULL);
25391da177e4SLinus Torvalds 	fs_cachep = kmem_cache_create("fs_cache",
25401da177e4SLinus Torvalds 			sizeof(struct fs_struct), 0,
254175f296d9SLevin, Alexander (Sasha Levin) 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
25425d097056SVladimir Davydov 			NULL);
2543c1a2f7f0SRik van Riel 
25446345d24dSLinus Torvalds 	/*
2545c1a2f7f0SRik van Riel 	 * The mm_cpumask is located at the end of mm_struct, and is
2546c1a2f7f0SRik van Riel 	 * dynamically sized based on the maximum CPU number this system
2547c1a2f7f0SRik van Riel 	 * can have, taking hotplug into account (nr_cpu_ids).
25486345d24dSLinus Torvalds 	 */
2549c1a2f7f0SRik van Riel 	mm_size = sizeof(struct mm_struct) + cpumask_size();
2550c1a2f7f0SRik van Riel 
255107dcd7feSDavid Windsor 	mm_cachep = kmem_cache_create_usercopy("mm_struct",
2552c1a2f7f0SRik van Riel 			mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
255375f296d9SLevin, Alexander (Sasha Levin) 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
255407dcd7feSDavid Windsor 			offsetof(struct mm_struct, saved_auxv),
255507dcd7feSDavid Windsor 			sizeof_field(struct mm_struct, saved_auxv),
25565d097056SVladimir Davydov 			NULL);
25575d097056SVladimir Davydov 	vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT);
25588feae131SDavid Howells 	mmap_init();
255966577193SAl Viro 	nsproxy_cache_init();
25601da177e4SLinus Torvalds }
2561cf2e340fSJANAK DESAI 
2562cf2e340fSJANAK DESAI /*
25639bfb23fcSOleg Nesterov  * Check constraints on flags passed to the unshare system call.
2564cf2e340fSJANAK DESAI  */
25659bfb23fcSOleg Nesterov static int check_unshare_flags(unsigned long unshare_flags)
2566cf2e340fSJANAK DESAI {
25679bfb23fcSOleg Nesterov 	if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
25689bfb23fcSOleg Nesterov 				CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
256950804fe3SEric W. Biederman 				CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET|
2570a79a908fSAditya Kali 				CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP))
2571cf2e340fSJANAK DESAI 		return -EINVAL;
25729bfb23fcSOleg Nesterov 	/*
257312c641abSEric W. Biederman 	 * Not implemented, but pretend it works if there is nothing
257412c641abSEric W. Biederman 	 * to unshare.  Note that unsharing the address space or the
257512c641abSEric W. Biederman 	 * signal handlers also need to unshare the signal queues (aka
257612c641abSEric W. Biederman 	 * CLONE_THREAD).
25779bfb23fcSOleg Nesterov 	 */
25789bfb23fcSOleg Nesterov 	if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
257912c641abSEric W. Biederman 		if (!thread_group_empty(current))
258012c641abSEric W. Biederman 			return -EINVAL;
258112c641abSEric W. Biederman 	}
258212c641abSEric W. Biederman 	if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) {
2583d036bda7SElena Reshetova 		if (refcount_read(&current->sighand->count) > 1)
258412c641abSEric W. Biederman 			return -EINVAL;
258512c641abSEric W. Biederman 	}
258612c641abSEric W. Biederman 	if (unshare_flags & CLONE_VM) {
258712c641abSEric W. Biederman 		if (!current_is_single_threaded())
25889bfb23fcSOleg Nesterov 			return -EINVAL;
25899bfb23fcSOleg Nesterov 	}
2590cf2e340fSJANAK DESAI 
2591cf2e340fSJANAK DESAI 	return 0;
2592cf2e340fSJANAK DESAI }
2593cf2e340fSJANAK DESAI 
2594cf2e340fSJANAK DESAI /*
259599d1419dSJANAK DESAI  * Unshare the filesystem structure if it is being shared
2596cf2e340fSJANAK DESAI  */
2597cf2e340fSJANAK DESAI static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
2598cf2e340fSJANAK DESAI {
2599cf2e340fSJANAK DESAI 	struct fs_struct *fs = current->fs;
2600cf2e340fSJANAK DESAI 
2601498052bbSAl Viro 	if (!(unshare_flags & CLONE_FS) || !fs)
2602498052bbSAl Viro 		return 0;
2603498052bbSAl Viro 
2604498052bbSAl Viro 	/* don't need lock here; in the worst case we'll do useless copy */
2605498052bbSAl Viro 	if (fs->users == 1)
2606498052bbSAl Viro 		return 0;
2607498052bbSAl Viro 
2608498052bbSAl Viro 	*new_fsp = copy_fs_struct(fs);
260999d1419dSJANAK DESAI 	if (!*new_fsp)
261099d1419dSJANAK DESAI 		return -ENOMEM;
2611cf2e340fSJANAK DESAI 
2612cf2e340fSJANAK DESAI 	return 0;
2613cf2e340fSJANAK DESAI }
2614cf2e340fSJANAK DESAI 
2615cf2e340fSJANAK DESAI /*
2616a016f338SJANAK DESAI  * Unshare file descriptor table if it is being shared
2617cf2e340fSJANAK DESAI  */
2618cf2e340fSJANAK DESAI static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
2619cf2e340fSJANAK DESAI {
2620cf2e340fSJANAK DESAI 	struct files_struct *fd = current->files;
2621a016f338SJANAK DESAI 	int error = 0;
2622cf2e340fSJANAK DESAI 
2623cf2e340fSJANAK DESAI 	if ((unshare_flags & CLONE_FILES) &&
2624a016f338SJANAK DESAI 	    (fd && atomic_read(&fd->count) > 1)) {
2625a016f338SJANAK DESAI 		*new_fdp = dup_fd(fd, &error);
2626a016f338SJANAK DESAI 		if (!*new_fdp)
2627a016f338SJANAK DESAI 			return error;
2628a016f338SJANAK DESAI 	}
2629cf2e340fSJANAK DESAI 
2630cf2e340fSJANAK DESAI 	return 0;
2631cf2e340fSJANAK DESAI }
2632cf2e340fSJANAK DESAI 
2633cf2e340fSJANAK DESAI /*
2634cf2e340fSJANAK DESAI  * unshare allows a process to 'unshare' part of the process
2635cf2e340fSJANAK DESAI  * context which was originally shared using clone.  copy_*
2636cf2e340fSJANAK DESAI  * functions used by do_fork() cannot be used here directly
2637cf2e340fSJANAK DESAI  * because they modify an inactive task_struct that is being
2638cf2e340fSJANAK DESAI  * constructed. Here we are modifying the current, active,
2639cf2e340fSJANAK DESAI  * task_struct.
2640cf2e340fSJANAK DESAI  */
26419b32105eSDominik Brodowski int ksys_unshare(unsigned long unshare_flags)
2642cf2e340fSJANAK DESAI {
2643cf2e340fSJANAK DESAI 	struct fs_struct *fs, *new_fs = NULL;
2644cf2e340fSJANAK DESAI 	struct files_struct *fd, *new_fd = NULL;
2645b2e0d987SEric W. Biederman 	struct cred *new_cred = NULL;
2646cf7b708cSPavel Emelyanov 	struct nsproxy *new_nsproxy = NULL;
26479edff4abSManfred Spraul 	int do_sysvsem = 0;
26489bfb23fcSOleg Nesterov 	int err;
2649cf2e340fSJANAK DESAI 
265050804fe3SEric W. Biederman 	/*
2651faf00da5SEric W. Biederman 	 * If unsharing a user namespace must also unshare the thread group
2652faf00da5SEric W. Biederman 	 * and unshare the filesystem root and working directories.
2653b2e0d987SEric W. Biederman 	 */
2654b2e0d987SEric W. Biederman 	if (unshare_flags & CLONE_NEWUSER)
2655e66eded8SEric W. Biederman 		unshare_flags |= CLONE_THREAD | CLONE_FS;
2656b2e0d987SEric W. Biederman 	/*
265750804fe3SEric W. Biederman 	 * If unsharing vm, must also unshare signal handlers.
265850804fe3SEric W. Biederman 	 */
265950804fe3SEric W. Biederman 	if (unshare_flags & CLONE_VM)
266050804fe3SEric W. Biederman 		unshare_flags |= CLONE_SIGHAND;
26616013f67fSManfred Spraul 	/*
266212c641abSEric W. Biederman 	 * If unsharing a signal handlers, must also unshare the signal queues.
266312c641abSEric W. Biederman 	 */
266412c641abSEric W. Biederman 	if (unshare_flags & CLONE_SIGHAND)
266512c641abSEric W. Biederman 		unshare_flags |= CLONE_THREAD;
266612c641abSEric W. Biederman 	/*
26679bfb23fcSOleg Nesterov 	 * If unsharing namespace, must also unshare filesystem information.
26689bfb23fcSOleg Nesterov 	 */
26699bfb23fcSOleg Nesterov 	if (unshare_flags & CLONE_NEWNS)
26709bfb23fcSOleg Nesterov 		unshare_flags |= CLONE_FS;
267150804fe3SEric W. Biederman 
267250804fe3SEric W. Biederman 	err = check_unshare_flags(unshare_flags);
267350804fe3SEric W. Biederman 	if (err)
267450804fe3SEric W. Biederman 		goto bad_unshare_out;
26759bfb23fcSOleg Nesterov 	/*
26766013f67fSManfred Spraul 	 * CLONE_NEWIPC must also detach from the undolist: after switching
26776013f67fSManfred Spraul 	 * to a new ipc namespace, the semaphore arrays from the old
26786013f67fSManfred Spraul 	 * namespace are unreachable.
26796013f67fSManfred Spraul 	 */
26806013f67fSManfred Spraul 	if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
26819edff4abSManfred Spraul 		do_sysvsem = 1;
2682fb0a685cSDaniel Rebelo de Oliveira 	err = unshare_fs(unshare_flags, &new_fs);
2683fb0a685cSDaniel Rebelo de Oliveira 	if (err)
26849bfb23fcSOleg Nesterov 		goto bad_unshare_out;
2685fb0a685cSDaniel Rebelo de Oliveira 	err = unshare_fd(unshare_flags, &new_fd);
2686fb0a685cSDaniel Rebelo de Oliveira 	if (err)
26879bfb23fcSOleg Nesterov 		goto bad_unshare_cleanup_fs;
2688b2e0d987SEric W. Biederman 	err = unshare_userns(unshare_flags, &new_cred);
2689fb0a685cSDaniel Rebelo de Oliveira 	if (err)
26909edff4abSManfred Spraul 		goto bad_unshare_cleanup_fd;
2691b2e0d987SEric W. Biederman 	err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
2692b2e0d987SEric W. Biederman 					 new_cred, new_fs);
2693b2e0d987SEric W. Biederman 	if (err)
2694b2e0d987SEric W. Biederman 		goto bad_unshare_cleanup_cred;
2695cf2e340fSJANAK DESAI 
2696b2e0d987SEric W. Biederman 	if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
26979edff4abSManfred Spraul 		if (do_sysvsem) {
26989edff4abSManfred Spraul 			/*
26999edff4abSManfred Spraul 			 * CLONE_SYSVSEM is equivalent to sys_exit().
27009edff4abSManfred Spraul 			 */
27019edff4abSManfred Spraul 			exit_sem(current);
27029edff4abSManfred Spraul 		}
2703ab602f79SJack Miller 		if (unshare_flags & CLONE_NEWIPC) {
2704ab602f79SJack Miller 			/* Orphan segments in old ns (see sem above). */
2705ab602f79SJack Miller 			exit_shm(current);
2706ab602f79SJack Miller 			shm_init_task(current);
2707ab602f79SJack Miller 		}
2708ab516013SSerge E. Hallyn 
27096f977e6bSAlan Cox 		if (new_nsproxy)
2710cf7b708cSPavel Emelyanov 			switch_task_namespaces(current, new_nsproxy);
2711cf2e340fSJANAK DESAI 
2712cf7b708cSPavel Emelyanov 		task_lock(current);
2713cf7b708cSPavel Emelyanov 
2714cf2e340fSJANAK DESAI 		if (new_fs) {
2715cf2e340fSJANAK DESAI 			fs = current->fs;
27162a4419b5SNick Piggin 			spin_lock(&fs->lock);
2717cf2e340fSJANAK DESAI 			current->fs = new_fs;
2718498052bbSAl Viro 			if (--fs->users)
2719498052bbSAl Viro 				new_fs = NULL;
2720498052bbSAl Viro 			else
2721cf2e340fSJANAK DESAI 				new_fs = fs;
27222a4419b5SNick Piggin 			spin_unlock(&fs->lock);
2723cf2e340fSJANAK DESAI 		}
2724cf2e340fSJANAK DESAI 
2725cf2e340fSJANAK DESAI 		if (new_fd) {
2726cf2e340fSJANAK DESAI 			fd = current->files;
2727cf2e340fSJANAK DESAI 			current->files = new_fd;
2728cf2e340fSJANAK DESAI 			new_fd = fd;
2729cf2e340fSJANAK DESAI 		}
2730cf2e340fSJANAK DESAI 
2731cf2e340fSJANAK DESAI 		task_unlock(current);
2732b2e0d987SEric W. Biederman 
2733b2e0d987SEric W. Biederman 		if (new_cred) {
2734b2e0d987SEric W. Biederman 			/* Install the new user namespace */
2735b2e0d987SEric W. Biederman 			commit_creds(new_cred);
2736b2e0d987SEric W. Biederman 			new_cred = NULL;
2737b2e0d987SEric W. Biederman 		}
2738cf2e340fSJANAK DESAI 	}
2739cf2e340fSJANAK DESAI 
2740e4222673SHari Bathini 	perf_event_namespaces(current);
2741e4222673SHari Bathini 
2742b2e0d987SEric W. Biederman bad_unshare_cleanup_cred:
2743b2e0d987SEric W. Biederman 	if (new_cred)
2744b2e0d987SEric W. Biederman 		put_cred(new_cred);
2745cf2e340fSJANAK DESAI bad_unshare_cleanup_fd:
2746cf2e340fSJANAK DESAI 	if (new_fd)
2747cf2e340fSJANAK DESAI 		put_files_struct(new_fd);
2748cf2e340fSJANAK DESAI 
2749cf2e340fSJANAK DESAI bad_unshare_cleanup_fs:
2750cf2e340fSJANAK DESAI 	if (new_fs)
2751498052bbSAl Viro 		free_fs_struct(new_fs);
2752cf2e340fSJANAK DESAI 
2753cf2e340fSJANAK DESAI bad_unshare_out:
2754cf2e340fSJANAK DESAI 	return err;
2755cf2e340fSJANAK DESAI }
27563b125388SAl Viro 
27579b32105eSDominik Brodowski SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
27589b32105eSDominik Brodowski {
27599b32105eSDominik Brodowski 	return ksys_unshare(unshare_flags);
27609b32105eSDominik Brodowski }
27619b32105eSDominik Brodowski 
27623b125388SAl Viro /*
27633b125388SAl Viro  *	Helper to unshare the files of the current task.
27643b125388SAl Viro  *	We don't want to expose copy_files internals to
27653b125388SAl Viro  *	the exec layer of the kernel.
27663b125388SAl Viro  */
27673b125388SAl Viro 
27683b125388SAl Viro int unshare_files(struct files_struct **displaced)
27693b125388SAl Viro {
27703b125388SAl Viro 	struct task_struct *task = current;
277150704516SAl Viro 	struct files_struct *copy = NULL;
27723b125388SAl Viro 	int error;
27733b125388SAl Viro 
27743b125388SAl Viro 	error = unshare_fd(CLONE_FILES, &copy);
27753b125388SAl Viro 	if (error || !copy) {
27763b125388SAl Viro 		*displaced = NULL;
27773b125388SAl Viro 		return error;
27783b125388SAl Viro 	}
27793b125388SAl Viro 	*displaced = task->files;
27803b125388SAl Viro 	task_lock(task);
27813b125388SAl Viro 	task->files = copy;
27823b125388SAl Viro 	task_unlock(task);
27833b125388SAl Viro 	return 0;
27843b125388SAl Viro }
278516db3d3fSHeinrich Schuchardt 
278616db3d3fSHeinrich Schuchardt int sysctl_max_threads(struct ctl_table *table, int write,
278716db3d3fSHeinrich Schuchardt 		       void __user *buffer, size_t *lenp, loff_t *ppos)
278816db3d3fSHeinrich Schuchardt {
278916db3d3fSHeinrich Schuchardt 	struct ctl_table t;
279016db3d3fSHeinrich Schuchardt 	int ret;
279116db3d3fSHeinrich Schuchardt 	int threads = max_threads;
279216db3d3fSHeinrich Schuchardt 	int min = MIN_THREADS;
279316db3d3fSHeinrich Schuchardt 	int max = MAX_THREADS;
279416db3d3fSHeinrich Schuchardt 
279516db3d3fSHeinrich Schuchardt 	t = *table;
279616db3d3fSHeinrich Schuchardt 	t.data = &threads;
279716db3d3fSHeinrich Schuchardt 	t.extra1 = &min;
279816db3d3fSHeinrich Schuchardt 	t.extra2 = &max;
279916db3d3fSHeinrich Schuchardt 
280016db3d3fSHeinrich Schuchardt 	ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
280116db3d3fSHeinrich Schuchardt 	if (ret || !write)
280216db3d3fSHeinrich Schuchardt 		return ret;
280316db3d3fSHeinrich Schuchardt 
280416db3d3fSHeinrich Schuchardt 	set_max_threads(threads);
280516db3d3fSHeinrich Schuchardt 
280616db3d3fSHeinrich Schuchardt 	return 0;
280716db3d3fSHeinrich Schuchardt }
2808