xref: /openbmc/linux/kernel/fork.c (revision 7e47682ea555e7c1edef1d8fd96e2aa4c12abe59)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  *  linux/kernel/fork.c
31da177e4SLinus Torvalds  *
41da177e4SLinus Torvalds  *  Copyright (C) 1991, 1992  Linus Torvalds
51da177e4SLinus Torvalds  */
61da177e4SLinus Torvalds 
71da177e4SLinus Torvalds /*
81da177e4SLinus Torvalds  *  'fork.c' contains the help-routines for the 'fork' system call
91da177e4SLinus Torvalds  * (see also entry.S and others).
101da177e4SLinus Torvalds  * Fork is rather simple, once you get the hang of it, but the memory
111da177e4SLinus Torvalds  * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
121da177e4SLinus Torvalds  */
131da177e4SLinus Torvalds 
141da177e4SLinus Torvalds #include <linux/slab.h>
151da177e4SLinus Torvalds #include <linux/init.h>
161da177e4SLinus Torvalds #include <linux/unistd.h>
171da177e4SLinus Torvalds #include <linux/module.h>
181da177e4SLinus Torvalds #include <linux/vmalloc.h>
191da177e4SLinus Torvalds #include <linux/completion.h>
201da177e4SLinus Torvalds #include <linux/personality.h>
211da177e4SLinus Torvalds #include <linux/mempolicy.h>
221da177e4SLinus Torvalds #include <linux/sem.h>
231da177e4SLinus Torvalds #include <linux/file.h>
249f3acc31SAl Viro #include <linux/fdtable.h>
25da9cbc87SJens Axboe #include <linux/iocontext.h>
261da177e4SLinus Torvalds #include <linux/key.h>
271da177e4SLinus Torvalds #include <linux/binfmts.h>
281da177e4SLinus Torvalds #include <linux/mman.h>
29cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h>
301da177e4SLinus Torvalds #include <linux/fs.h>
31615d6e87SDavidlohr Bueso #include <linux/mm.h>
32615d6e87SDavidlohr Bueso #include <linux/vmacache.h>
33ab516013SSerge E. Hallyn #include <linux/nsproxy.h>
34c59ede7bSRandy.Dunlap #include <linux/capability.h>
351da177e4SLinus Torvalds #include <linux/cpu.h>
36b4f48b63SPaul Menage #include <linux/cgroup.h>
371da177e4SLinus Torvalds #include <linux/security.h>
38a1e78772SMel Gorman #include <linux/hugetlb.h>
39e2cfabdfSWill Drewry #include <linux/seccomp.h>
401da177e4SLinus Torvalds #include <linux/swap.h>
411da177e4SLinus Torvalds #include <linux/syscalls.h>
421da177e4SLinus Torvalds #include <linux/jiffies.h>
431da177e4SLinus Torvalds #include <linux/futex.h>
448141c7f3SLinus Torvalds #include <linux/compat.h>
45207205a2SEric Dumazet #include <linux/kthread.h>
467c3ab738SAndrew Morton #include <linux/task_io_accounting_ops.h>
47ab2af1f5SDipankar Sarma #include <linux/rcupdate.h>
481da177e4SLinus Torvalds #include <linux/ptrace.h>
491da177e4SLinus Torvalds #include <linux/mount.h>
501da177e4SLinus Torvalds #include <linux/audit.h>
5178fb7466SPavel Emelianov #include <linux/memcontrol.h>
52f201ae23SFrederic Weisbecker #include <linux/ftrace.h>
535e2bf014SMike Galbraith #include <linux/proc_fs.h>
541da177e4SLinus Torvalds #include <linux/profile.h>
551da177e4SLinus Torvalds #include <linux/rmap.h>
56f8af4da3SHugh Dickins #include <linux/ksm.h>
571da177e4SLinus Torvalds #include <linux/acct.h>
588f0ab514SJay Lan #include <linux/tsacct_kern.h>
599f46080cSMatt Helsley #include <linux/cn_proc.h>
60ba96a0c8SRafael J. Wysocki #include <linux/freezer.h>
61ca74e92bSShailabh Nagar #include <linux/delayacct.h>
62ad4ecbcbSShailabh Nagar #include <linux/taskstats_kern.h>
630a425405SArjan van de Ven #include <linux/random.h>
64522ed776SMiloslav Trmac #include <linux/tty.h>
65fd0928dfSJens Axboe #include <linux/blkdev.h>
665ad4e53bSAl Viro #include <linux/fs_struct.h>
677c9f8861SEric Sandeen #include <linux/magic.h>
68cdd6c482SIngo Molnar #include <linux/perf_event.h>
6942c4ab41SStanislaw Gruszka #include <linux/posix-timers.h>
708e7cac79SAvi Kivity #include <linux/user-return-notifier.h>
713d5992d2SYing Han #include <linux/oom.h>
72ba76149fSAndrea Arcangeli #include <linux/khugepaged.h>
73d80e731eSOleg Nesterov #include <linux/signalfd.h>
740326f5a9SSrikar Dronamraju #include <linux/uprobes.h>
75a27bb332SKent Overstreet #include <linux/aio.h>
7652f5684cSGideon Israel Dsouza #include <linux/compiler.h>
7716db3d3fSHeinrich Schuchardt #include <linux/sysctl.h>
781da177e4SLinus Torvalds 
791da177e4SLinus Torvalds #include <asm/pgtable.h>
801da177e4SLinus Torvalds #include <asm/pgalloc.h>
811da177e4SLinus Torvalds #include <asm/uaccess.h>
821da177e4SLinus Torvalds #include <asm/mmu_context.h>
831da177e4SLinus Torvalds #include <asm/cacheflush.h>
841da177e4SLinus Torvalds #include <asm/tlbflush.h>
851da177e4SLinus Torvalds 
86ad8d75ffSSteven Rostedt #include <trace/events/sched.h>
87ad8d75ffSSteven Rostedt 
8843d2b113SKAMEZAWA Hiroyuki #define CREATE_TRACE_POINTS
8943d2b113SKAMEZAWA Hiroyuki #include <trace/events/task.h>
9043d2b113SKAMEZAWA Hiroyuki 
911da177e4SLinus Torvalds /*
92ac1b398dSHeinrich Schuchardt  * Minimum number of threads to boot the kernel
93ac1b398dSHeinrich Schuchardt  */
94ac1b398dSHeinrich Schuchardt #define MIN_THREADS 20
95ac1b398dSHeinrich Schuchardt 
96ac1b398dSHeinrich Schuchardt /*
97ac1b398dSHeinrich Schuchardt  * Maximum number of threads
98ac1b398dSHeinrich Schuchardt  */
99ac1b398dSHeinrich Schuchardt #define MAX_THREADS FUTEX_TID_MASK
100ac1b398dSHeinrich Schuchardt 
101ac1b398dSHeinrich Schuchardt /*
1021da177e4SLinus Torvalds  * Protected counters by write_lock_irq(&tasklist_lock)
1031da177e4SLinus Torvalds  */
1041da177e4SLinus Torvalds unsigned long total_forks;	/* Handle normal Linux uptimes. */
1051da177e4SLinus Torvalds int nr_threads;			/* The idle threads do not count.. */
1061da177e4SLinus Torvalds 
1071da177e4SLinus Torvalds int max_threads;		/* tunable limit on nr_threads */
1081da177e4SLinus Torvalds 
1091da177e4SLinus Torvalds DEFINE_PER_CPU(unsigned long, process_counts) = 0;
1101da177e4SLinus Torvalds 
1111da177e4SLinus Torvalds __cacheline_aligned DEFINE_RWLOCK(tasklist_lock);  /* outer */
112db1466b3SPaul E. McKenney 
113db1466b3SPaul E. McKenney #ifdef CONFIG_PROVE_RCU
114db1466b3SPaul E. McKenney int lockdep_tasklist_lock_is_held(void)
115db1466b3SPaul E. McKenney {
116db1466b3SPaul E. McKenney 	return lockdep_is_held(&tasklist_lock);
117db1466b3SPaul E. McKenney }
118db1466b3SPaul E. McKenney EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
119db1466b3SPaul E. McKenney #endif /* #ifdef CONFIG_PROVE_RCU */
1201da177e4SLinus Torvalds 
1211da177e4SLinus Torvalds int nr_processes(void)
1221da177e4SLinus Torvalds {
1231da177e4SLinus Torvalds 	int cpu;
1241da177e4SLinus Torvalds 	int total = 0;
1251da177e4SLinus Torvalds 
1261d510750SIan Campbell 	for_each_possible_cpu(cpu)
1271da177e4SLinus Torvalds 		total += per_cpu(process_counts, cpu);
1281da177e4SLinus Torvalds 
1291da177e4SLinus Torvalds 	return total;
1301da177e4SLinus Torvalds }
1311da177e4SLinus Torvalds 
132f19b9f74SAkinobu Mita void __weak arch_release_task_struct(struct task_struct *tsk)
133f19b9f74SAkinobu Mita {
134f19b9f74SAkinobu Mita }
135f19b9f74SAkinobu Mita 
136f5e10287SThomas Gleixner #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
137e18b890bSChristoph Lameter static struct kmem_cache *task_struct_cachep;
13841101809SThomas Gleixner 
13941101809SThomas Gleixner static inline struct task_struct *alloc_task_struct_node(int node)
14041101809SThomas Gleixner {
14141101809SThomas Gleixner 	return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
14241101809SThomas Gleixner }
14341101809SThomas Gleixner 
14441101809SThomas Gleixner static inline void free_task_struct(struct task_struct *tsk)
14541101809SThomas Gleixner {
14641101809SThomas Gleixner 	kmem_cache_free(task_struct_cachep, tsk);
14741101809SThomas Gleixner }
1481da177e4SLinus Torvalds #endif
1491da177e4SLinus Torvalds 
150f19b9f74SAkinobu Mita void __weak arch_release_thread_info(struct thread_info *ti)
151f19b9f74SAkinobu Mita {
152f19b9f74SAkinobu Mita }
153f19b9f74SAkinobu Mita 
154f5e10287SThomas Gleixner #ifndef CONFIG_ARCH_THREAD_INFO_ALLOCATOR
15541101809SThomas Gleixner 
1560d15d74aSThomas Gleixner /*
1570d15d74aSThomas Gleixner  * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
1580d15d74aSThomas Gleixner  * kmemcache based allocator.
1590d15d74aSThomas Gleixner  */
1600d15d74aSThomas Gleixner # if THREAD_SIZE >= PAGE_SIZE
161b6a84016SEric Dumazet static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
162b6a84016SEric Dumazet 						  int node)
163b69c49b7SFUJITA Tomonori {
16452383431SVladimir Davydov 	struct page *page = alloc_kmem_pages_node(node, THREADINFO_GFP,
1652889f608SThomas Gleixner 						  THREAD_SIZE_ORDER);
166b6a84016SEric Dumazet 
167b6a84016SEric Dumazet 	return page ? page_address(page) : NULL;
168b69c49b7SFUJITA Tomonori }
169b69c49b7SFUJITA Tomonori 
170b69c49b7SFUJITA Tomonori static inline void free_thread_info(struct thread_info *ti)
171b69c49b7SFUJITA Tomonori {
17252383431SVladimir Davydov 	free_kmem_pages((unsigned long)ti, THREAD_SIZE_ORDER);
173b69c49b7SFUJITA Tomonori }
1740d15d74aSThomas Gleixner # else
1750d15d74aSThomas Gleixner static struct kmem_cache *thread_info_cache;
1760d15d74aSThomas Gleixner 
1770d15d74aSThomas Gleixner static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
1780d15d74aSThomas Gleixner 						  int node)
1790d15d74aSThomas Gleixner {
1800d15d74aSThomas Gleixner 	return kmem_cache_alloc_node(thread_info_cache, THREADINFO_GFP, node);
1810d15d74aSThomas Gleixner }
1820d15d74aSThomas Gleixner 
1830d15d74aSThomas Gleixner static void free_thread_info(struct thread_info *ti)
1840d15d74aSThomas Gleixner {
1850d15d74aSThomas Gleixner 	kmem_cache_free(thread_info_cache, ti);
1860d15d74aSThomas Gleixner }
1870d15d74aSThomas Gleixner 
1880d15d74aSThomas Gleixner void thread_info_cache_init(void)
1890d15d74aSThomas Gleixner {
1900d15d74aSThomas Gleixner 	thread_info_cache = kmem_cache_create("thread_info", THREAD_SIZE,
1910d15d74aSThomas Gleixner 					      THREAD_SIZE, 0, NULL);
1920d15d74aSThomas Gleixner 	BUG_ON(thread_info_cache == NULL);
1930d15d74aSThomas Gleixner }
1940d15d74aSThomas Gleixner # endif
195b69c49b7SFUJITA Tomonori #endif
196b69c49b7SFUJITA Tomonori 
1971da177e4SLinus Torvalds /* SLAB cache for signal_struct structures (tsk->signal) */
198e18b890bSChristoph Lameter static struct kmem_cache *signal_cachep;
1991da177e4SLinus Torvalds 
2001da177e4SLinus Torvalds /* SLAB cache for sighand_struct structures (tsk->sighand) */
201e18b890bSChristoph Lameter struct kmem_cache *sighand_cachep;
2021da177e4SLinus Torvalds 
2031da177e4SLinus Torvalds /* SLAB cache for files_struct structures (tsk->files) */
204e18b890bSChristoph Lameter struct kmem_cache *files_cachep;
2051da177e4SLinus Torvalds 
2061da177e4SLinus Torvalds /* SLAB cache for fs_struct structures (tsk->fs) */
207e18b890bSChristoph Lameter struct kmem_cache *fs_cachep;
2081da177e4SLinus Torvalds 
2091da177e4SLinus Torvalds /* SLAB cache for vm_area_struct structures */
210e18b890bSChristoph Lameter struct kmem_cache *vm_area_cachep;
2111da177e4SLinus Torvalds 
2121da177e4SLinus Torvalds /* SLAB cache for mm_struct structures (tsk->mm) */
213e18b890bSChristoph Lameter static struct kmem_cache *mm_cachep;
2141da177e4SLinus Torvalds 
215c6a7f572SKOSAKI Motohiro static void account_kernel_stack(struct thread_info *ti, int account)
216c6a7f572SKOSAKI Motohiro {
217c6a7f572SKOSAKI Motohiro 	struct zone *zone = page_zone(virt_to_page(ti));
218c6a7f572SKOSAKI Motohiro 
219c6a7f572SKOSAKI Motohiro 	mod_zone_page_state(zone, NR_KERNEL_STACK, account);
220c6a7f572SKOSAKI Motohiro }
221c6a7f572SKOSAKI Motohiro 
2221da177e4SLinus Torvalds void free_task(struct task_struct *tsk)
2231da177e4SLinus Torvalds {
224c6a7f572SKOSAKI Motohiro 	account_kernel_stack(tsk->stack, -1);
225f19b9f74SAkinobu Mita 	arch_release_thread_info(tsk->stack);
226f7e4217bSRoman Zippel 	free_thread_info(tsk->stack);
22723f78d4aSIngo Molnar 	rt_mutex_debug_task_free(tsk);
228fb52607aSFrederic Weisbecker 	ftrace_graph_exit_task(tsk);
229e2cfabdfSWill Drewry 	put_seccomp_filter(tsk);
230f19b9f74SAkinobu Mita 	arch_release_task_struct(tsk);
2311da177e4SLinus Torvalds 	free_task_struct(tsk);
2321da177e4SLinus Torvalds }
2331da177e4SLinus Torvalds EXPORT_SYMBOL(free_task);
2341da177e4SLinus Torvalds 
235ea6d290cSOleg Nesterov static inline void free_signal_struct(struct signal_struct *sig)
236ea6d290cSOleg Nesterov {
23797101eb4SOleg Nesterov 	taskstats_tgid_free(sig);
2381c5354deSMike Galbraith 	sched_autogroup_exit(sig);
239ea6d290cSOleg Nesterov 	kmem_cache_free(signal_cachep, sig);
240ea6d290cSOleg Nesterov }
241ea6d290cSOleg Nesterov 
242ea6d290cSOleg Nesterov static inline void put_signal_struct(struct signal_struct *sig)
243ea6d290cSOleg Nesterov {
2441c5354deSMike Galbraith 	if (atomic_dec_and_test(&sig->sigcnt))
245ea6d290cSOleg Nesterov 		free_signal_struct(sig);
246ea6d290cSOleg Nesterov }
247ea6d290cSOleg Nesterov 
248158d9ebdSAndrew Morton void __put_task_struct(struct task_struct *tsk)
2491da177e4SLinus Torvalds {
250270f722dSEugene Teo 	WARN_ON(!tsk->exit_state);
2511da177e4SLinus Torvalds 	WARN_ON(atomic_read(&tsk->usage));
2521da177e4SLinus Torvalds 	WARN_ON(tsk == current);
2531da177e4SLinus Torvalds 
254156654f4SMike Galbraith 	task_numa_free(tsk);
2551a2a4d06SKees Cook 	security_task_free(tsk);
256e0e81739SDavid Howells 	exit_creds(tsk);
25735df17c5SShailabh Nagar 	delayacct_tsk_free(tsk);
258ea6d290cSOleg Nesterov 	put_signal_struct(tsk->signal);
2591da177e4SLinus Torvalds 
2601da177e4SLinus Torvalds 	if (!profile_handoff_task(tsk))
2611da177e4SLinus Torvalds 		free_task(tsk);
2621da177e4SLinus Torvalds }
26377c100c8SRik van Riel EXPORT_SYMBOL_GPL(__put_task_struct);
2641da177e4SLinus Torvalds 
2656c0a9fa6SThomas Gleixner void __init __weak arch_task_cache_init(void) { }
26661c4628bSSuresh Siddha 
267ff691f6eSHeinrich Schuchardt /*
268ff691f6eSHeinrich Schuchardt  * set_max_threads
269ff691f6eSHeinrich Schuchardt  */
27016db3d3fSHeinrich Schuchardt static void set_max_threads(unsigned int max_threads_suggested)
271ff691f6eSHeinrich Schuchardt {
272ac1b398dSHeinrich Schuchardt 	u64 threads;
273ff691f6eSHeinrich Schuchardt 
274ff691f6eSHeinrich Schuchardt 	/*
275ac1b398dSHeinrich Schuchardt 	 * The number of threads shall be limited such that the thread
276ac1b398dSHeinrich Schuchardt 	 * structures may only consume a small part of the available memory.
277ff691f6eSHeinrich Schuchardt 	 */
278ac1b398dSHeinrich Schuchardt 	if (fls64(totalram_pages) + fls64(PAGE_SIZE) > 64)
279ac1b398dSHeinrich Schuchardt 		threads = MAX_THREADS;
280ac1b398dSHeinrich Schuchardt 	else
281ac1b398dSHeinrich Schuchardt 		threads = div64_u64((u64) totalram_pages * (u64) PAGE_SIZE,
282ac1b398dSHeinrich Schuchardt 				    (u64) THREAD_SIZE * 8UL);
283ac1b398dSHeinrich Schuchardt 
28416db3d3fSHeinrich Schuchardt 	if (threads > max_threads_suggested)
28516db3d3fSHeinrich Schuchardt 		threads = max_threads_suggested;
28616db3d3fSHeinrich Schuchardt 
287ac1b398dSHeinrich Schuchardt 	max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
288ff691f6eSHeinrich Schuchardt }
289ff691f6eSHeinrich Schuchardt 
290ff691f6eSHeinrich Schuchardt void __init fork_init(void)
2911da177e4SLinus Torvalds {
292f5e10287SThomas Gleixner #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
2931da177e4SLinus Torvalds #ifndef ARCH_MIN_TASKALIGN
2941da177e4SLinus Torvalds #define ARCH_MIN_TASKALIGN	L1_CACHE_BYTES
2951da177e4SLinus Torvalds #endif
2961da177e4SLinus Torvalds 	/* create a slab on which task_structs can be allocated */
2971da177e4SLinus Torvalds 	task_struct_cachep =
2981da177e4SLinus Torvalds 		kmem_cache_create("task_struct", sizeof(struct task_struct),
2992dff4405SVegard Nossum 			ARCH_MIN_TASKALIGN, SLAB_PANIC | SLAB_NOTRACK, NULL);
3001da177e4SLinus Torvalds #endif
3011da177e4SLinus Torvalds 
30261c4628bSSuresh Siddha 	/* do the arch specific task caches init */
30361c4628bSSuresh Siddha 	arch_task_cache_init();
30461c4628bSSuresh Siddha 
30516db3d3fSHeinrich Schuchardt 	set_max_threads(MAX_THREADS);
3061da177e4SLinus Torvalds 
3071da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
3081da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
3091da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_SIGPENDING] =
3101da177e4SLinus Torvalds 		init_task.signal->rlim[RLIMIT_NPROC];
3111da177e4SLinus Torvalds }
3121da177e4SLinus Torvalds 
31352f5684cSGideon Israel Dsouza int __weak arch_dup_task_struct(struct task_struct *dst,
31461c4628bSSuresh Siddha 					       struct task_struct *src)
31561c4628bSSuresh Siddha {
31661c4628bSSuresh Siddha 	*dst = *src;
31761c4628bSSuresh Siddha 	return 0;
31861c4628bSSuresh Siddha }
31961c4628bSSuresh Siddha 
320d4311ff1SAaron Tomlin void set_task_stack_end_magic(struct task_struct *tsk)
321d4311ff1SAaron Tomlin {
322d4311ff1SAaron Tomlin 	unsigned long *stackend;
323d4311ff1SAaron Tomlin 
324d4311ff1SAaron Tomlin 	stackend = end_of_stack(tsk);
325d4311ff1SAaron Tomlin 	*stackend = STACK_END_MAGIC;	/* for overflow detection */
326d4311ff1SAaron Tomlin }
327d4311ff1SAaron Tomlin 
3281da177e4SLinus Torvalds static struct task_struct *dup_task_struct(struct task_struct *orig)
3291da177e4SLinus Torvalds {
3301da177e4SLinus Torvalds 	struct task_struct *tsk;
3311da177e4SLinus Torvalds 	struct thread_info *ti;
332207205a2SEric Dumazet 	int node = tsk_fork_get_node(orig);
3333e26c149SPeter Zijlstra 	int err;
3341da177e4SLinus Torvalds 
335504f52b5SEric Dumazet 	tsk = alloc_task_struct_node(node);
3361da177e4SLinus Torvalds 	if (!tsk)
3371da177e4SLinus Torvalds 		return NULL;
3381da177e4SLinus Torvalds 
339b6a84016SEric Dumazet 	ti = alloc_thread_info_node(tsk, node);
340f19b9f74SAkinobu Mita 	if (!ti)
341f19b9f74SAkinobu Mita 		goto free_tsk;
3421da177e4SLinus Torvalds 
34361c4628bSSuresh Siddha 	err = arch_dup_task_struct(tsk, orig);
34461c4628bSSuresh Siddha 	if (err)
345f19b9f74SAkinobu Mita 		goto free_ti;
34661c4628bSSuresh Siddha 
34787bec58aSAndrew Morton 	tsk->stack = ti;
348dbd95212SKees Cook #ifdef CONFIG_SECCOMP
349dbd95212SKees Cook 	/*
350dbd95212SKees Cook 	 * We must handle setting up seccomp filters once we're under
351dbd95212SKees Cook 	 * the sighand lock in case orig has changed between now and
352dbd95212SKees Cook 	 * then. Until then, filter must be NULL to avoid messing up
353dbd95212SKees Cook 	 * the usage counts on the error path calling free_task.
354dbd95212SKees Cook 	 */
355dbd95212SKees Cook 	tsk->seccomp.filter = NULL;
356dbd95212SKees Cook #endif
35787bec58aSAndrew Morton 
35887bec58aSAndrew Morton 	setup_thread_stack(tsk, orig);
3598e7cac79SAvi Kivity 	clear_user_return_notifier(tsk);
360f26f9affSMike Galbraith 	clear_tsk_need_resched(tsk);
361d4311ff1SAaron Tomlin 	set_task_stack_end_magic(tsk);
3621da177e4SLinus Torvalds 
3630a425405SArjan van de Ven #ifdef CONFIG_CC_STACKPROTECTOR
3640a425405SArjan van de Ven 	tsk->stack_canary = get_random_int();
3650a425405SArjan van de Ven #endif
3660a425405SArjan van de Ven 
367fb0a685cSDaniel Rebelo de Oliveira 	/*
368fb0a685cSDaniel Rebelo de Oliveira 	 * One for us, one for whoever does the "release_task()" (usually
369fb0a685cSDaniel Rebelo de Oliveira 	 * parent)
370fb0a685cSDaniel Rebelo de Oliveira 	 */
3711da177e4SLinus Torvalds 	atomic_set(&tsk->usage, 2);
3726c5c9341SAlexey Dobriyan #ifdef CONFIG_BLK_DEV_IO_TRACE
3732056a782SJens Axboe 	tsk->btrace_seq = 0;
3746c5c9341SAlexey Dobriyan #endif
375a0aa7f68SJens Axboe 	tsk->splice_pipe = NULL;
3765640f768SEric Dumazet 	tsk->task_frag.page = NULL;
377c6a7f572SKOSAKI Motohiro 
378c6a7f572SKOSAKI Motohiro 	account_kernel_stack(ti, 1);
379c6a7f572SKOSAKI Motohiro 
3801da177e4SLinus Torvalds 	return tsk;
38161c4628bSSuresh Siddha 
382f19b9f74SAkinobu Mita free_ti:
38361c4628bSSuresh Siddha 	free_thread_info(ti);
384f19b9f74SAkinobu Mita free_tsk:
38561c4628bSSuresh Siddha 	free_task_struct(tsk);
38661c4628bSSuresh Siddha 	return NULL;
3871da177e4SLinus Torvalds }
3881da177e4SLinus Torvalds 
3891da177e4SLinus Torvalds #ifdef CONFIG_MMU
390a39bc516SAlexey Dobriyan static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
3911da177e4SLinus Torvalds {
392297c5eeeSLinus Torvalds 	struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
3931da177e4SLinus Torvalds 	struct rb_node **rb_link, *rb_parent;
3941da177e4SLinus Torvalds 	int retval;
3951da177e4SLinus Torvalds 	unsigned long charge;
3961da177e4SLinus Torvalds 
39732cdba1eSOleg Nesterov 	uprobe_start_dup_mmap();
3981da177e4SLinus Torvalds 	down_write(&oldmm->mmap_sem);
399ec8c0446SRalf Baechle 	flush_cache_dup_mm(oldmm);
400f8ac4ec9SOleg Nesterov 	uprobe_dup_mmap(oldmm, mm);
401ad339451SIngo Molnar 	/*
402ad339451SIngo Molnar 	 * Not linked in yet - no deadlock potential:
403ad339451SIngo Molnar 	 */
404ad339451SIngo Molnar 	down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
4057ee78232SHugh Dickins 
40690f31d0eSKonstantin Khlebnikov 	/* No ordering required: file already has been exposed. */
40790f31d0eSKonstantin Khlebnikov 	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
40890f31d0eSKonstantin Khlebnikov 
4094f7d4614SVladimir Davydov 	mm->total_vm = oldmm->total_vm;
4104f7d4614SVladimir Davydov 	mm->shared_vm = oldmm->shared_vm;
4114f7d4614SVladimir Davydov 	mm->exec_vm = oldmm->exec_vm;
4124f7d4614SVladimir Davydov 	mm->stack_vm = oldmm->stack_vm;
4134f7d4614SVladimir Davydov 
4141da177e4SLinus Torvalds 	rb_link = &mm->mm_rb.rb_node;
4151da177e4SLinus Torvalds 	rb_parent = NULL;
4161da177e4SLinus Torvalds 	pprev = &mm->mmap;
417f8af4da3SHugh Dickins 	retval = ksm_fork(mm, oldmm);
418f8af4da3SHugh Dickins 	if (retval)
419f8af4da3SHugh Dickins 		goto out;
420ba76149fSAndrea Arcangeli 	retval = khugepaged_fork(mm, oldmm);
421ba76149fSAndrea Arcangeli 	if (retval)
422ba76149fSAndrea Arcangeli 		goto out;
4231da177e4SLinus Torvalds 
424297c5eeeSLinus Torvalds 	prev = NULL;
425fd3e42fcSHugh Dickins 	for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
4261da177e4SLinus Torvalds 		struct file *file;
4271da177e4SLinus Torvalds 
4281da177e4SLinus Torvalds 		if (mpnt->vm_flags & VM_DONTCOPY) {
429ab50b8edSHugh Dickins 			vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
43044de9d0cSHuang Shijie 							-vma_pages(mpnt));
4311da177e4SLinus Torvalds 			continue;
4321da177e4SLinus Torvalds 		}
4331da177e4SLinus Torvalds 		charge = 0;
4341da177e4SLinus Torvalds 		if (mpnt->vm_flags & VM_ACCOUNT) {
435b2412b7fSHuang Shijie 			unsigned long len = vma_pages(mpnt);
436b2412b7fSHuang Shijie 
437191c5424SAl Viro 			if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
4381da177e4SLinus Torvalds 				goto fail_nomem;
4391da177e4SLinus Torvalds 			charge = len;
4401da177e4SLinus Torvalds 		}
441e94b1766SChristoph Lameter 		tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
4421da177e4SLinus Torvalds 		if (!tmp)
4431da177e4SLinus Torvalds 			goto fail_nomem;
4441da177e4SLinus Torvalds 		*tmp = *mpnt;
4455beb4930SRik van Riel 		INIT_LIST_HEAD(&tmp->anon_vma_chain);
446ef0855d3SOleg Nesterov 		retval = vma_dup_policy(mpnt, tmp);
447ef0855d3SOleg Nesterov 		if (retval)
4481da177e4SLinus Torvalds 			goto fail_nomem_policy;
449a247c3a9SAndrea Arcangeli 		tmp->vm_mm = mm;
4505beb4930SRik van Riel 		if (anon_vma_fork(tmp, mpnt))
4515beb4930SRik van Riel 			goto fail_nomem_anon_vma_fork;
4521da177e4SLinus Torvalds 		tmp->vm_flags &= ~VM_LOCKED;
453297c5eeeSLinus Torvalds 		tmp->vm_next = tmp->vm_prev = NULL;
4541da177e4SLinus Torvalds 		file = tmp->vm_file;
4551da177e4SLinus Torvalds 		if (file) {
456496ad9aaSAl Viro 			struct inode *inode = file_inode(file);
457b88ed205SHugh Dickins 			struct address_space *mapping = file->f_mapping;
458b88ed205SHugh Dickins 
4591da177e4SLinus Torvalds 			get_file(file);
4601da177e4SLinus Torvalds 			if (tmp->vm_flags & VM_DENYWRITE)
4611da177e4SLinus Torvalds 				atomic_dec(&inode->i_writecount);
46283cde9e8SDavidlohr Bueso 			i_mmap_lock_write(mapping);
463b88ed205SHugh Dickins 			if (tmp->vm_flags & VM_SHARED)
4644bb5f5d9SDavid Herrmann 				atomic_inc(&mapping->i_mmap_writable);
465b88ed205SHugh Dickins 			flush_dcache_mmap_lock(mapping);
466b88ed205SHugh Dickins 			/* insert tmp into the share list, just after mpnt */
4679826a516SMichel Lespinasse 			vma_interval_tree_insert_after(tmp, mpnt,
4689826a516SMichel Lespinasse 					&mapping->i_mmap);
469b88ed205SHugh Dickins 			flush_dcache_mmap_unlock(mapping);
47083cde9e8SDavidlohr Bueso 			i_mmap_unlock_write(mapping);
4711da177e4SLinus Torvalds 		}
4721da177e4SLinus Torvalds 
4731da177e4SLinus Torvalds 		/*
474a1e78772SMel Gorman 		 * Clear hugetlb-related page reserves for children. This only
475a1e78772SMel Gorman 		 * affects MAP_PRIVATE mappings. Faults generated by the child
476a1e78772SMel Gorman 		 * are not guaranteed to succeed, even if read-only
477a1e78772SMel Gorman 		 */
478a1e78772SMel Gorman 		if (is_vm_hugetlb_page(tmp))
479a1e78772SMel Gorman 			reset_vma_resv_huge_pages(tmp);
480a1e78772SMel Gorman 
481a1e78772SMel Gorman 		/*
4827ee78232SHugh Dickins 		 * Link in the new vma and copy the page table entries.
4831da177e4SLinus Torvalds 		 */
4841da177e4SLinus Torvalds 		*pprev = tmp;
4851da177e4SLinus Torvalds 		pprev = &tmp->vm_next;
486297c5eeeSLinus Torvalds 		tmp->vm_prev = prev;
487297c5eeeSLinus Torvalds 		prev = tmp;
4881da177e4SLinus Torvalds 
4891da177e4SLinus Torvalds 		__vma_link_rb(mm, tmp, rb_link, rb_parent);
4901da177e4SLinus Torvalds 		rb_link = &tmp->vm_rb.rb_right;
4911da177e4SLinus Torvalds 		rb_parent = &tmp->vm_rb;
4921da177e4SLinus Torvalds 
4931da177e4SLinus Torvalds 		mm->map_count++;
4940b0db14cSHugh Dickins 		retval = copy_page_range(mm, oldmm, mpnt);
4951da177e4SLinus Torvalds 
4961da177e4SLinus Torvalds 		if (tmp->vm_ops && tmp->vm_ops->open)
4971da177e4SLinus Torvalds 			tmp->vm_ops->open(tmp);
4981da177e4SLinus Torvalds 
4991da177e4SLinus Torvalds 		if (retval)
5001da177e4SLinus Torvalds 			goto out;
5011da177e4SLinus Torvalds 	}
502d6dd61c8SJeremy Fitzhardinge 	/* a new mm has just been created */
503d6dd61c8SJeremy Fitzhardinge 	arch_dup_mmap(oldmm, mm);
5041da177e4SLinus Torvalds 	retval = 0;
5051da177e4SLinus Torvalds out:
5067ee78232SHugh Dickins 	up_write(&mm->mmap_sem);
507fd3e42fcSHugh Dickins 	flush_tlb_mm(oldmm);
5081da177e4SLinus Torvalds 	up_write(&oldmm->mmap_sem);
50932cdba1eSOleg Nesterov 	uprobe_end_dup_mmap();
5101da177e4SLinus Torvalds 	return retval;
5115beb4930SRik van Riel fail_nomem_anon_vma_fork:
512ef0855d3SOleg Nesterov 	mpol_put(vma_policy(tmp));
5131da177e4SLinus Torvalds fail_nomem_policy:
5141da177e4SLinus Torvalds 	kmem_cache_free(vm_area_cachep, tmp);
5151da177e4SLinus Torvalds fail_nomem:
5161da177e4SLinus Torvalds 	retval = -ENOMEM;
5171da177e4SLinus Torvalds 	vm_unacct_memory(charge);
5181da177e4SLinus Torvalds 	goto out;
5191da177e4SLinus Torvalds }
5201da177e4SLinus Torvalds 
5211da177e4SLinus Torvalds static inline int mm_alloc_pgd(struct mm_struct *mm)
5221da177e4SLinus Torvalds {
5231da177e4SLinus Torvalds 	mm->pgd = pgd_alloc(mm);
5241da177e4SLinus Torvalds 	if (unlikely(!mm->pgd))
5251da177e4SLinus Torvalds 		return -ENOMEM;
5261da177e4SLinus Torvalds 	return 0;
5271da177e4SLinus Torvalds }
5281da177e4SLinus Torvalds 
5291da177e4SLinus Torvalds static inline void mm_free_pgd(struct mm_struct *mm)
5301da177e4SLinus Torvalds {
5315e541973SBenjamin Herrenschmidt 	pgd_free(mm, mm->pgd);
5321da177e4SLinus Torvalds }
5331da177e4SLinus Torvalds #else
53490f31d0eSKonstantin Khlebnikov static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
53590f31d0eSKonstantin Khlebnikov {
53690f31d0eSKonstantin Khlebnikov 	down_write(&oldmm->mmap_sem);
53790f31d0eSKonstantin Khlebnikov 	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
53890f31d0eSKonstantin Khlebnikov 	up_write(&oldmm->mmap_sem);
53990f31d0eSKonstantin Khlebnikov 	return 0;
54090f31d0eSKonstantin Khlebnikov }
5411da177e4SLinus Torvalds #define mm_alloc_pgd(mm)	(0)
5421da177e4SLinus Torvalds #define mm_free_pgd(mm)
5431da177e4SLinus Torvalds #endif /* CONFIG_MMU */
5441da177e4SLinus Torvalds 
5451da177e4SLinus Torvalds __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
5461da177e4SLinus Torvalds 
547e94b1766SChristoph Lameter #define allocate_mm()	(kmem_cache_alloc(mm_cachep, GFP_KERNEL))
5481da177e4SLinus Torvalds #define free_mm(mm)	(kmem_cache_free(mm_cachep, (mm)))
5491da177e4SLinus Torvalds 
5504cb0e11bSHidehiro Kawai static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
5514cb0e11bSHidehiro Kawai 
5524cb0e11bSHidehiro Kawai static int __init coredump_filter_setup(char *s)
5534cb0e11bSHidehiro Kawai {
5544cb0e11bSHidehiro Kawai 	default_dump_filter =
5554cb0e11bSHidehiro Kawai 		(simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
5564cb0e11bSHidehiro Kawai 		MMF_DUMP_FILTER_MASK;
5574cb0e11bSHidehiro Kawai 	return 1;
5584cb0e11bSHidehiro Kawai }
5594cb0e11bSHidehiro Kawai 
5604cb0e11bSHidehiro Kawai __setup("coredump_filter=", coredump_filter_setup);
5614cb0e11bSHidehiro Kawai 
5621da177e4SLinus Torvalds #include <linux/init_task.h>
5631da177e4SLinus Torvalds 
564858f0993SAlexey Dobriyan static void mm_init_aio(struct mm_struct *mm)
565858f0993SAlexey Dobriyan {
566858f0993SAlexey Dobriyan #ifdef CONFIG_AIO
567858f0993SAlexey Dobriyan 	spin_lock_init(&mm->ioctx_lock);
568db446a08SBenjamin LaHaise 	mm->ioctx_table = NULL;
569858f0993SAlexey Dobriyan #endif
570858f0993SAlexey Dobriyan }
571858f0993SAlexey Dobriyan 
57233144e84SVladimir Davydov static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
57333144e84SVladimir Davydov {
57433144e84SVladimir Davydov #ifdef CONFIG_MEMCG
57533144e84SVladimir Davydov 	mm->owner = p;
57633144e84SVladimir Davydov #endif
57733144e84SVladimir Davydov }
57833144e84SVladimir Davydov 
57978fb7466SPavel Emelianov static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
5801da177e4SLinus Torvalds {
58141f727fdSVladimir Davydov 	mm->mmap = NULL;
58241f727fdSVladimir Davydov 	mm->mm_rb = RB_ROOT;
58341f727fdSVladimir Davydov 	mm->vmacache_seqnum = 0;
5841da177e4SLinus Torvalds 	atomic_set(&mm->mm_users, 1);
5851da177e4SLinus Torvalds 	atomic_set(&mm->mm_count, 1);
5861da177e4SLinus Torvalds 	init_rwsem(&mm->mmap_sem);
5871da177e4SLinus Torvalds 	INIT_LIST_HEAD(&mm->mmlist);
588999d9fc1SOleg Nesterov 	mm->core_state = NULL;
589e1f56c89SKirill A. Shutemov 	atomic_long_set(&mm->nr_ptes, 0);
5902d2f5119SKirill A. Shutemov 	mm_nr_pmds_init(mm);
59141f727fdSVladimir Davydov 	mm->map_count = 0;
59241f727fdSVladimir Davydov 	mm->locked_vm = 0;
593ce65cefaSVladimir Davydov 	mm->pinned_vm = 0;
594d559db08SKAMEZAWA Hiroyuki 	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
5951da177e4SLinus Torvalds 	spin_lock_init(&mm->page_table_lock);
59641f727fdSVladimir Davydov 	mm_init_cpumask(mm);
597858f0993SAlexey Dobriyan 	mm_init_aio(mm);
598cf475ad2SBalbir Singh 	mm_init_owner(mm, p);
59941f727fdSVladimir Davydov 	mmu_notifier_mm_init(mm);
60020841405SRik van Riel 	clear_tlb_flush_pending(mm);
60141f727fdSVladimir Davydov #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
60241f727fdSVladimir Davydov 	mm->pmd_huge_pte = NULL;
60341f727fdSVladimir Davydov #endif
6041da177e4SLinus Torvalds 
605a0715cc2SAlex Thorlton 	if (current->mm) {
606a0715cc2SAlex Thorlton 		mm->flags = current->mm->flags & MMF_INIT_MASK;
607a0715cc2SAlex Thorlton 		mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
608a0715cc2SAlex Thorlton 	} else {
609a0715cc2SAlex Thorlton 		mm->flags = default_dump_filter;
6101da177e4SLinus Torvalds 		mm->def_flags = 0;
611a0715cc2SAlex Thorlton 	}
612a0715cc2SAlex Thorlton 
61341f727fdSVladimir Davydov 	if (mm_alloc_pgd(mm))
61441f727fdSVladimir Davydov 		goto fail_nopgd;
61578fb7466SPavel Emelianov 
61641f727fdSVladimir Davydov 	if (init_new_context(p, mm))
61741f727fdSVladimir Davydov 		goto fail_nocontext;
61841f727fdSVladimir Davydov 
61941f727fdSVladimir Davydov 	return mm;
62041f727fdSVladimir Davydov 
62141f727fdSVladimir Davydov fail_nocontext:
62241f727fdSVladimir Davydov 	mm_free_pgd(mm);
62341f727fdSVladimir Davydov fail_nopgd:
6241da177e4SLinus Torvalds 	free_mm(mm);
6251da177e4SLinus Torvalds 	return NULL;
6261da177e4SLinus Torvalds }
6271da177e4SLinus Torvalds 
628c3f0327fSKonstantin Khlebnikov static void check_mm(struct mm_struct *mm)
629c3f0327fSKonstantin Khlebnikov {
630c3f0327fSKonstantin Khlebnikov 	int i;
631c3f0327fSKonstantin Khlebnikov 
632c3f0327fSKonstantin Khlebnikov 	for (i = 0; i < NR_MM_COUNTERS; i++) {
633c3f0327fSKonstantin Khlebnikov 		long x = atomic_long_read(&mm->rss_stat.count[i]);
634c3f0327fSKonstantin Khlebnikov 
635c3f0327fSKonstantin Khlebnikov 		if (unlikely(x))
636c3f0327fSKonstantin Khlebnikov 			printk(KERN_ALERT "BUG: Bad rss-counter state "
637c3f0327fSKonstantin Khlebnikov 					  "mm:%p idx:%d val:%ld\n", mm, i, x);
638c3f0327fSKonstantin Khlebnikov 	}
639b30fe6c7SKirill A. Shutemov 
640b30fe6c7SKirill A. Shutemov 	if (atomic_long_read(&mm->nr_ptes))
641b30fe6c7SKirill A. Shutemov 		pr_alert("BUG: non-zero nr_ptes on freeing mm: %ld\n",
642b30fe6c7SKirill A. Shutemov 				atomic_long_read(&mm->nr_ptes));
643b30fe6c7SKirill A. Shutemov 	if (mm_nr_pmds(mm))
644b30fe6c7SKirill A. Shutemov 		pr_alert("BUG: non-zero nr_pmds on freeing mm: %ld\n",
645b30fe6c7SKirill A. Shutemov 				mm_nr_pmds(mm));
646b30fe6c7SKirill A. Shutemov 
647e009bb30SKirill A. Shutemov #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
64896dad67fSSasha Levin 	VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
649c3f0327fSKonstantin Khlebnikov #endif
650c3f0327fSKonstantin Khlebnikov }
651c3f0327fSKonstantin Khlebnikov 
6521da177e4SLinus Torvalds /*
6531da177e4SLinus Torvalds  * Allocate and initialize an mm_struct.
6541da177e4SLinus Torvalds  */
6551da177e4SLinus Torvalds struct mm_struct *mm_alloc(void)
6561da177e4SLinus Torvalds {
6571da177e4SLinus Torvalds 	struct mm_struct *mm;
6581da177e4SLinus Torvalds 
6591da177e4SLinus Torvalds 	mm = allocate_mm();
660de03c72cSKOSAKI Motohiro 	if (!mm)
661de03c72cSKOSAKI Motohiro 		return NULL;
662de03c72cSKOSAKI Motohiro 
6631da177e4SLinus Torvalds 	memset(mm, 0, sizeof(*mm));
6646345d24dSLinus Torvalds 	return mm_init(mm, current);
6651da177e4SLinus Torvalds }
6661da177e4SLinus Torvalds 
6671da177e4SLinus Torvalds /*
6681da177e4SLinus Torvalds  * Called when the last reference to the mm
6691da177e4SLinus Torvalds  * is dropped: either by a lazy thread or by
6701da177e4SLinus Torvalds  * mmput. Free the page directory and the mm.
6711da177e4SLinus Torvalds  */
6727ad5b3a5SHarvey Harrison void __mmdrop(struct mm_struct *mm)
6731da177e4SLinus Torvalds {
6741da177e4SLinus Torvalds 	BUG_ON(mm == &init_mm);
6751da177e4SLinus Torvalds 	mm_free_pgd(mm);
6761da177e4SLinus Torvalds 	destroy_context(mm);
677cddb8a5cSAndrea Arcangeli 	mmu_notifier_mm_destroy(mm);
678c3f0327fSKonstantin Khlebnikov 	check_mm(mm);
6791da177e4SLinus Torvalds 	free_mm(mm);
6801da177e4SLinus Torvalds }
6816d4e4c4fSAvi Kivity EXPORT_SYMBOL_GPL(__mmdrop);
6821da177e4SLinus Torvalds 
6831da177e4SLinus Torvalds /*
6841da177e4SLinus Torvalds  * Decrement the use count and release all resources for an mm.
6851da177e4SLinus Torvalds  */
6861da177e4SLinus Torvalds void mmput(struct mm_struct *mm)
6871da177e4SLinus Torvalds {
6880ae26f1bSAndrew Morton 	might_sleep();
6890ae26f1bSAndrew Morton 
6901da177e4SLinus Torvalds 	if (atomic_dec_and_test(&mm->mm_users)) {
691d4b3b638SSrikar Dronamraju 		uprobe_clear_state(mm);
6921da177e4SLinus Torvalds 		exit_aio(mm);
6931c2fb7a4SAndrea Arcangeli 		ksm_exit(mm);
694ba76149fSAndrea Arcangeli 		khugepaged_exit(mm); /* must run before exit_mmap */
6951da177e4SLinus Torvalds 		exit_mmap(mm);
696925d1c40SMatt Helsley 		set_mm_exe_file(mm, NULL);
6971da177e4SLinus Torvalds 		if (!list_empty(&mm->mmlist)) {
6981da177e4SLinus Torvalds 			spin_lock(&mmlist_lock);
6991da177e4SLinus Torvalds 			list_del(&mm->mmlist);
7001da177e4SLinus Torvalds 			spin_unlock(&mmlist_lock);
7011da177e4SLinus Torvalds 		}
702801460d0SHiroshi Shimamoto 		if (mm->binfmt)
703801460d0SHiroshi Shimamoto 			module_put(mm->binfmt->module);
7041da177e4SLinus Torvalds 		mmdrop(mm);
7051da177e4SLinus Torvalds 	}
7061da177e4SLinus Torvalds }
7071da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(mmput);
7081da177e4SLinus Torvalds 
70990f31d0eSKonstantin Khlebnikov /**
71090f31d0eSKonstantin Khlebnikov  * set_mm_exe_file - change a reference to the mm's executable file
71190f31d0eSKonstantin Khlebnikov  *
71290f31d0eSKonstantin Khlebnikov  * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
71390f31d0eSKonstantin Khlebnikov  *
7146e399cd1SDavidlohr Bueso  * Main users are mmput() and sys_execve(). Callers prevent concurrent
7156e399cd1SDavidlohr Bueso  * invocations: in mmput() nobody alive left, in execve task is single
7166e399cd1SDavidlohr Bueso  * threaded. sys_prctl(PR_SET_MM_MAP/EXE_FILE) also needs to set the
7176e399cd1SDavidlohr Bueso  * mm->exe_file, but does so without using set_mm_exe_file() in order
7186e399cd1SDavidlohr Bueso  * to do avoid the need for any locks.
71990f31d0eSKonstantin Khlebnikov  */
72038646013SJiri Slaby void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
72138646013SJiri Slaby {
7226e399cd1SDavidlohr Bueso 	struct file *old_exe_file;
7236e399cd1SDavidlohr Bueso 
7246e399cd1SDavidlohr Bueso 	/*
7256e399cd1SDavidlohr Bueso 	 * It is safe to dereference the exe_file without RCU as
7266e399cd1SDavidlohr Bueso 	 * this function is only called if nobody else can access
7276e399cd1SDavidlohr Bueso 	 * this mm -- see comment above for justification.
7286e399cd1SDavidlohr Bueso 	 */
7296e399cd1SDavidlohr Bueso 	old_exe_file = rcu_dereference_raw(mm->exe_file);
73090f31d0eSKonstantin Khlebnikov 
73138646013SJiri Slaby 	if (new_exe_file)
73238646013SJiri Slaby 		get_file(new_exe_file);
73390f31d0eSKonstantin Khlebnikov 	rcu_assign_pointer(mm->exe_file, new_exe_file);
73490f31d0eSKonstantin Khlebnikov 	if (old_exe_file)
73590f31d0eSKonstantin Khlebnikov 		fput(old_exe_file);
73638646013SJiri Slaby }
73738646013SJiri Slaby 
73890f31d0eSKonstantin Khlebnikov /**
73990f31d0eSKonstantin Khlebnikov  * get_mm_exe_file - acquire a reference to the mm's executable file
74090f31d0eSKonstantin Khlebnikov  *
74190f31d0eSKonstantin Khlebnikov  * Returns %NULL if mm has no associated executable file.
74290f31d0eSKonstantin Khlebnikov  * User must release file via fput().
74390f31d0eSKonstantin Khlebnikov  */
74438646013SJiri Slaby struct file *get_mm_exe_file(struct mm_struct *mm)
74538646013SJiri Slaby {
74638646013SJiri Slaby 	struct file *exe_file;
74738646013SJiri Slaby 
74890f31d0eSKonstantin Khlebnikov 	rcu_read_lock();
74990f31d0eSKonstantin Khlebnikov 	exe_file = rcu_dereference(mm->exe_file);
75090f31d0eSKonstantin Khlebnikov 	if (exe_file && !get_file_rcu(exe_file))
75190f31d0eSKonstantin Khlebnikov 		exe_file = NULL;
75290f31d0eSKonstantin Khlebnikov 	rcu_read_unlock();
75338646013SJiri Slaby 	return exe_file;
75438646013SJiri Slaby }
75511163348SDavidlohr Bueso EXPORT_SYMBOL(get_mm_exe_file);
75638646013SJiri Slaby 
7571da177e4SLinus Torvalds /**
7581da177e4SLinus Torvalds  * get_task_mm - acquire a reference to the task's mm
7591da177e4SLinus Torvalds  *
760246bb0b1SOleg Nesterov  * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning
7611da177e4SLinus Torvalds  * this kernel workthread has transiently adopted a user mm with use_mm,
7621da177e4SLinus Torvalds  * to do its AIO) is not set and if so returns a reference to it, after
7631da177e4SLinus Torvalds  * bumping up the use count.  User must release the mm via mmput()
7641da177e4SLinus Torvalds  * after use.  Typically used by /proc and ptrace.
7651da177e4SLinus Torvalds  */
7661da177e4SLinus Torvalds struct mm_struct *get_task_mm(struct task_struct *task)
7671da177e4SLinus Torvalds {
7681da177e4SLinus Torvalds 	struct mm_struct *mm;
7691da177e4SLinus Torvalds 
7701da177e4SLinus Torvalds 	task_lock(task);
7711da177e4SLinus Torvalds 	mm = task->mm;
7721da177e4SLinus Torvalds 	if (mm) {
773246bb0b1SOleg Nesterov 		if (task->flags & PF_KTHREAD)
7741da177e4SLinus Torvalds 			mm = NULL;
7751da177e4SLinus Torvalds 		else
7761da177e4SLinus Torvalds 			atomic_inc(&mm->mm_users);
7771da177e4SLinus Torvalds 	}
7781da177e4SLinus Torvalds 	task_unlock(task);
7791da177e4SLinus Torvalds 	return mm;
7801da177e4SLinus Torvalds }
7811da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(get_task_mm);
7821da177e4SLinus Torvalds 
7838cdb878dSChristopher Yeoh struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
7848cdb878dSChristopher Yeoh {
7858cdb878dSChristopher Yeoh 	struct mm_struct *mm;
7868cdb878dSChristopher Yeoh 	int err;
7878cdb878dSChristopher Yeoh 
7888cdb878dSChristopher Yeoh 	err =  mutex_lock_killable(&task->signal->cred_guard_mutex);
7898cdb878dSChristopher Yeoh 	if (err)
7908cdb878dSChristopher Yeoh 		return ERR_PTR(err);
7918cdb878dSChristopher Yeoh 
7928cdb878dSChristopher Yeoh 	mm = get_task_mm(task);
7938cdb878dSChristopher Yeoh 	if (mm && mm != current->mm &&
7948cdb878dSChristopher Yeoh 			!ptrace_may_access(task, mode)) {
7958cdb878dSChristopher Yeoh 		mmput(mm);
7968cdb878dSChristopher Yeoh 		mm = ERR_PTR(-EACCES);
7978cdb878dSChristopher Yeoh 	}
7988cdb878dSChristopher Yeoh 	mutex_unlock(&task->signal->cred_guard_mutex);
7998cdb878dSChristopher Yeoh 
8008cdb878dSChristopher Yeoh 	return mm;
8018cdb878dSChristopher Yeoh }
8028cdb878dSChristopher Yeoh 
80357b59c4aSOleg Nesterov static void complete_vfork_done(struct task_struct *tsk)
804c415c3b4SOleg Nesterov {
805d68b46feSOleg Nesterov 	struct completion *vfork;
806c415c3b4SOleg Nesterov 
807d68b46feSOleg Nesterov 	task_lock(tsk);
808d68b46feSOleg Nesterov 	vfork = tsk->vfork_done;
809d68b46feSOleg Nesterov 	if (likely(vfork)) {
810c415c3b4SOleg Nesterov 		tsk->vfork_done = NULL;
811d68b46feSOleg Nesterov 		complete(vfork);
812d68b46feSOleg Nesterov 	}
813d68b46feSOleg Nesterov 	task_unlock(tsk);
814d68b46feSOleg Nesterov }
815d68b46feSOleg Nesterov 
816d68b46feSOleg Nesterov static int wait_for_vfork_done(struct task_struct *child,
817d68b46feSOleg Nesterov 				struct completion *vfork)
818d68b46feSOleg Nesterov {
819d68b46feSOleg Nesterov 	int killed;
820d68b46feSOleg Nesterov 
821d68b46feSOleg Nesterov 	freezer_do_not_count();
822d68b46feSOleg Nesterov 	killed = wait_for_completion_killable(vfork);
823d68b46feSOleg Nesterov 	freezer_count();
824d68b46feSOleg Nesterov 
825d68b46feSOleg Nesterov 	if (killed) {
826d68b46feSOleg Nesterov 		task_lock(child);
827d68b46feSOleg Nesterov 		child->vfork_done = NULL;
828d68b46feSOleg Nesterov 		task_unlock(child);
829d68b46feSOleg Nesterov 	}
830d68b46feSOleg Nesterov 
831d68b46feSOleg Nesterov 	put_task_struct(child);
832d68b46feSOleg Nesterov 	return killed;
833c415c3b4SOleg Nesterov }
834c415c3b4SOleg Nesterov 
8351da177e4SLinus Torvalds /* Please note the differences between mmput and mm_release.
8361da177e4SLinus Torvalds  * mmput is called whenever we stop holding onto a mm_struct,
8371da177e4SLinus Torvalds  * error success whatever.
8381da177e4SLinus Torvalds  *
8391da177e4SLinus Torvalds  * mm_release is called after a mm_struct has been removed
8401da177e4SLinus Torvalds  * from the current process.
8411da177e4SLinus Torvalds  *
8421da177e4SLinus Torvalds  * This difference is important for error handling, when we
8431da177e4SLinus Torvalds  * only half set up a mm_struct for a new process and need to restore
8441da177e4SLinus Torvalds  * the old one.  Because we mmput the new mm_struct before
8451da177e4SLinus Torvalds  * restoring the old one. . .
8461da177e4SLinus Torvalds  * Eric Biederman 10 January 1998
8471da177e4SLinus Torvalds  */
8481da177e4SLinus Torvalds void mm_release(struct task_struct *tsk, struct mm_struct *mm)
8491da177e4SLinus Torvalds {
8508141c7f3SLinus Torvalds 	/* Get rid of any futexes when releasing the mm */
8518141c7f3SLinus Torvalds #ifdef CONFIG_FUTEX
852fc6b177dSPeter Zijlstra 	if (unlikely(tsk->robust_list)) {
8538141c7f3SLinus Torvalds 		exit_robust_list(tsk);
854fc6b177dSPeter Zijlstra 		tsk->robust_list = NULL;
855fc6b177dSPeter Zijlstra 	}
8568141c7f3SLinus Torvalds #ifdef CONFIG_COMPAT
857fc6b177dSPeter Zijlstra 	if (unlikely(tsk->compat_robust_list)) {
8588141c7f3SLinus Torvalds 		compat_exit_robust_list(tsk);
859fc6b177dSPeter Zijlstra 		tsk->compat_robust_list = NULL;
860fc6b177dSPeter Zijlstra 	}
8618141c7f3SLinus Torvalds #endif
862322a2c10SThomas Gleixner 	if (unlikely(!list_empty(&tsk->pi_state_list)))
863322a2c10SThomas Gleixner 		exit_pi_state_list(tsk);
8648141c7f3SLinus Torvalds #endif
8658141c7f3SLinus Torvalds 
8660326f5a9SSrikar Dronamraju 	uprobe_free_utask(tsk);
8670326f5a9SSrikar Dronamraju 
8681da177e4SLinus Torvalds 	/* Get rid of any cached register state */
8691da177e4SLinus Torvalds 	deactivate_mm(tsk, mm);
8701da177e4SLinus Torvalds 
871fec1d011SRoland McGrath 	/*
872fec1d011SRoland McGrath 	 * If we're exiting normally, clear a user-space tid field if
873fec1d011SRoland McGrath 	 * requested.  We leave this alone when dying by signal, to leave
874fec1d011SRoland McGrath 	 * the value intact in a core dump, and to save the unnecessary
875d68b46feSOleg Nesterov 	 * trouble, say, a killed vfork parent shouldn't touch this mm.
876d68b46feSOleg Nesterov 	 * Userland only wants this done for a sys_exit.
877fec1d011SRoland McGrath 	 */
8789c8a8228SEric Dumazet 	if (tsk->clear_child_tid) {
8799c8a8228SEric Dumazet 		if (!(tsk->flags & PF_SIGNALED) &&
8809c8a8228SEric Dumazet 		    atomic_read(&mm->mm_users) > 1) {
8811da177e4SLinus Torvalds 			/*
8821da177e4SLinus Torvalds 			 * We don't check the error code - if userspace has
8831da177e4SLinus Torvalds 			 * not set up a proper pointer then tough luck.
8841da177e4SLinus Torvalds 			 */
8859c8a8228SEric Dumazet 			put_user(0, tsk->clear_child_tid);
8869c8a8228SEric Dumazet 			sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
8879c8a8228SEric Dumazet 					1, NULL, NULL, 0);
8889c8a8228SEric Dumazet 		}
8899c8a8228SEric Dumazet 		tsk->clear_child_tid = NULL;
8901da177e4SLinus Torvalds 	}
891f7505d64SKonstantin Khlebnikov 
892f7505d64SKonstantin Khlebnikov 	/*
893f7505d64SKonstantin Khlebnikov 	 * All done, finally we can wake up parent and return this mm to him.
894f7505d64SKonstantin Khlebnikov 	 * Also kthread_stop() uses this completion for synchronization.
895f7505d64SKonstantin Khlebnikov 	 */
896f7505d64SKonstantin Khlebnikov 	if (tsk->vfork_done)
897f7505d64SKonstantin Khlebnikov 		complete_vfork_done(tsk);
8981da177e4SLinus Torvalds }
8991da177e4SLinus Torvalds 
900a0a7ec30SJANAK DESAI /*
901a0a7ec30SJANAK DESAI  * Allocate a new mm structure and copy contents from the
902a0a7ec30SJANAK DESAI  * mm structure of the passed in task structure.
903a0a7ec30SJANAK DESAI  */
904ff252c1fSDaeSeok Youn static struct mm_struct *dup_mm(struct task_struct *tsk)
905a0a7ec30SJANAK DESAI {
906a0a7ec30SJANAK DESAI 	struct mm_struct *mm, *oldmm = current->mm;
907a0a7ec30SJANAK DESAI 	int err;
908a0a7ec30SJANAK DESAI 
909a0a7ec30SJANAK DESAI 	mm = allocate_mm();
910a0a7ec30SJANAK DESAI 	if (!mm)
911a0a7ec30SJANAK DESAI 		goto fail_nomem;
912a0a7ec30SJANAK DESAI 
913a0a7ec30SJANAK DESAI 	memcpy(mm, oldmm, sizeof(*mm));
914a0a7ec30SJANAK DESAI 
91578fb7466SPavel Emelianov 	if (!mm_init(mm, tsk))
916a0a7ec30SJANAK DESAI 		goto fail_nomem;
917a0a7ec30SJANAK DESAI 
918a0a7ec30SJANAK DESAI 	err = dup_mmap(mm, oldmm);
919a0a7ec30SJANAK DESAI 	if (err)
920a0a7ec30SJANAK DESAI 		goto free_pt;
921a0a7ec30SJANAK DESAI 
922a0a7ec30SJANAK DESAI 	mm->hiwater_rss = get_mm_rss(mm);
923a0a7ec30SJANAK DESAI 	mm->hiwater_vm = mm->total_vm;
924a0a7ec30SJANAK DESAI 
925801460d0SHiroshi Shimamoto 	if (mm->binfmt && !try_module_get(mm->binfmt->module))
926801460d0SHiroshi Shimamoto 		goto free_pt;
927801460d0SHiroshi Shimamoto 
928a0a7ec30SJANAK DESAI 	return mm;
929a0a7ec30SJANAK DESAI 
930a0a7ec30SJANAK DESAI free_pt:
931801460d0SHiroshi Shimamoto 	/* don't put binfmt in mmput, we haven't got module yet */
932801460d0SHiroshi Shimamoto 	mm->binfmt = NULL;
933a0a7ec30SJANAK DESAI 	mmput(mm);
934a0a7ec30SJANAK DESAI 
935a0a7ec30SJANAK DESAI fail_nomem:
936a0a7ec30SJANAK DESAI 	return NULL;
937a0a7ec30SJANAK DESAI }
938a0a7ec30SJANAK DESAI 
9391da177e4SLinus Torvalds static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
9401da177e4SLinus Torvalds {
9411da177e4SLinus Torvalds 	struct mm_struct *mm, *oldmm;
9421da177e4SLinus Torvalds 	int retval;
9431da177e4SLinus Torvalds 
9441da177e4SLinus Torvalds 	tsk->min_flt = tsk->maj_flt = 0;
9451da177e4SLinus Torvalds 	tsk->nvcsw = tsk->nivcsw = 0;
94617406b82SMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK
94717406b82SMandeep Singh Baines 	tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
94817406b82SMandeep Singh Baines #endif
9491da177e4SLinus Torvalds 
9501da177e4SLinus Torvalds 	tsk->mm = NULL;
9511da177e4SLinus Torvalds 	tsk->active_mm = NULL;
9521da177e4SLinus Torvalds 
9531da177e4SLinus Torvalds 	/*
9541da177e4SLinus Torvalds 	 * Are we cloning a kernel thread?
9551da177e4SLinus Torvalds 	 *
9561da177e4SLinus Torvalds 	 * We need to steal a active VM for that..
9571da177e4SLinus Torvalds 	 */
9581da177e4SLinus Torvalds 	oldmm = current->mm;
9591da177e4SLinus Torvalds 	if (!oldmm)
9601da177e4SLinus Torvalds 		return 0;
9611da177e4SLinus Torvalds 
962615d6e87SDavidlohr Bueso 	/* initialize the new vmacache entries */
963615d6e87SDavidlohr Bueso 	vmacache_flush(tsk);
964615d6e87SDavidlohr Bueso 
9651da177e4SLinus Torvalds 	if (clone_flags & CLONE_VM) {
9661da177e4SLinus Torvalds 		atomic_inc(&oldmm->mm_users);
9671da177e4SLinus Torvalds 		mm = oldmm;
9681da177e4SLinus Torvalds 		goto good_mm;
9691da177e4SLinus Torvalds 	}
9701da177e4SLinus Torvalds 
9711da177e4SLinus Torvalds 	retval = -ENOMEM;
972a0a7ec30SJANAK DESAI 	mm = dup_mm(tsk);
9731da177e4SLinus Torvalds 	if (!mm)
9741da177e4SLinus Torvalds 		goto fail_nomem;
9751da177e4SLinus Torvalds 
9761da177e4SLinus Torvalds good_mm:
9771da177e4SLinus Torvalds 	tsk->mm = mm;
9781da177e4SLinus Torvalds 	tsk->active_mm = mm;
9791da177e4SLinus Torvalds 	return 0;
9801da177e4SLinus Torvalds 
9811da177e4SLinus Torvalds fail_nomem:
9821da177e4SLinus Torvalds 	return retval;
9831da177e4SLinus Torvalds }
9841da177e4SLinus Torvalds 
985a39bc516SAlexey Dobriyan static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
9861da177e4SLinus Torvalds {
987498052bbSAl Viro 	struct fs_struct *fs = current->fs;
9881da177e4SLinus Torvalds 	if (clone_flags & CLONE_FS) {
989498052bbSAl Viro 		/* tsk->fs is already what we want */
9902a4419b5SNick Piggin 		spin_lock(&fs->lock);
991498052bbSAl Viro 		if (fs->in_exec) {
9922a4419b5SNick Piggin 			spin_unlock(&fs->lock);
993498052bbSAl Viro 			return -EAGAIN;
994498052bbSAl Viro 		}
995498052bbSAl Viro 		fs->users++;
9962a4419b5SNick Piggin 		spin_unlock(&fs->lock);
9971da177e4SLinus Torvalds 		return 0;
9981da177e4SLinus Torvalds 	}
999498052bbSAl Viro 	tsk->fs = copy_fs_struct(fs);
10001da177e4SLinus Torvalds 	if (!tsk->fs)
10011da177e4SLinus Torvalds 		return -ENOMEM;
10021da177e4SLinus Torvalds 	return 0;
10031da177e4SLinus Torvalds }
10041da177e4SLinus Torvalds 
1005a016f338SJANAK DESAI static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
1006a016f338SJANAK DESAI {
1007a016f338SJANAK DESAI 	struct files_struct *oldf, *newf;
1008a016f338SJANAK DESAI 	int error = 0;
1009a016f338SJANAK DESAI 
1010a016f338SJANAK DESAI 	/*
1011a016f338SJANAK DESAI 	 * A background process may not have any files ...
1012a016f338SJANAK DESAI 	 */
1013a016f338SJANAK DESAI 	oldf = current->files;
1014a016f338SJANAK DESAI 	if (!oldf)
1015a016f338SJANAK DESAI 		goto out;
1016a016f338SJANAK DESAI 
1017a016f338SJANAK DESAI 	if (clone_flags & CLONE_FILES) {
1018a016f338SJANAK DESAI 		atomic_inc(&oldf->count);
1019a016f338SJANAK DESAI 		goto out;
1020a016f338SJANAK DESAI 	}
1021a016f338SJANAK DESAI 
1022a016f338SJANAK DESAI 	newf = dup_fd(oldf, &error);
1023a016f338SJANAK DESAI 	if (!newf)
1024a016f338SJANAK DESAI 		goto out;
1025a016f338SJANAK DESAI 
1026a016f338SJANAK DESAI 	tsk->files = newf;
1027a016f338SJANAK DESAI 	error = 0;
1028a016f338SJANAK DESAI out:
1029a016f338SJANAK DESAI 	return error;
1030a016f338SJANAK DESAI }
1031a016f338SJANAK DESAI 
1032fadad878SJens Axboe static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
1033fd0928dfSJens Axboe {
1034fd0928dfSJens Axboe #ifdef CONFIG_BLOCK
1035fd0928dfSJens Axboe 	struct io_context *ioc = current->io_context;
10366e736be7STejun Heo 	struct io_context *new_ioc;
1037fd0928dfSJens Axboe 
1038fd0928dfSJens Axboe 	if (!ioc)
1039fd0928dfSJens Axboe 		return 0;
1040fadad878SJens Axboe 	/*
1041fadad878SJens Axboe 	 * Share io context with parent, if CLONE_IO is set
1042fadad878SJens Axboe 	 */
1043fadad878SJens Axboe 	if (clone_flags & CLONE_IO) {
10443d48749dSTejun Heo 		ioc_task_link(ioc);
10453d48749dSTejun Heo 		tsk->io_context = ioc;
1046fadad878SJens Axboe 	} else if (ioprio_valid(ioc->ioprio)) {
10476e736be7STejun Heo 		new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
10486e736be7STejun Heo 		if (unlikely(!new_ioc))
1049fd0928dfSJens Axboe 			return -ENOMEM;
1050fd0928dfSJens Axboe 
10516e736be7STejun Heo 		new_ioc->ioprio = ioc->ioprio;
105211a3122fSTejun Heo 		put_io_context(new_ioc);
1053fd0928dfSJens Axboe 	}
1054fd0928dfSJens Axboe #endif
1055fd0928dfSJens Axboe 	return 0;
1056fd0928dfSJens Axboe }
1057fd0928dfSJens Axboe 
1058a39bc516SAlexey Dobriyan static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
10591da177e4SLinus Torvalds {
10601da177e4SLinus Torvalds 	struct sighand_struct *sig;
10611da177e4SLinus Torvalds 
106260348802SZhaolei 	if (clone_flags & CLONE_SIGHAND) {
10631da177e4SLinus Torvalds 		atomic_inc(&current->sighand->count);
10641da177e4SLinus Torvalds 		return 0;
10651da177e4SLinus Torvalds 	}
10661da177e4SLinus Torvalds 	sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
1067e56d0903SIngo Molnar 	rcu_assign_pointer(tsk->sighand, sig);
10681da177e4SLinus Torvalds 	if (!sig)
10691da177e4SLinus Torvalds 		return -ENOMEM;
10701da177e4SLinus Torvalds 	atomic_set(&sig->count, 1);
10711da177e4SLinus Torvalds 	memcpy(sig->action, current->sighand->action, sizeof(sig->action));
10721da177e4SLinus Torvalds 	return 0;
10731da177e4SLinus Torvalds }
10741da177e4SLinus Torvalds 
1075a7e5328aSOleg Nesterov void __cleanup_sighand(struct sighand_struct *sighand)
1076c81addc9SOleg Nesterov {
1077d80e731eSOleg Nesterov 	if (atomic_dec_and_test(&sighand->count)) {
1078d80e731eSOleg Nesterov 		signalfd_cleanup(sighand);
1079392809b2SOleg Nesterov 		/*
1080392809b2SOleg Nesterov 		 * sighand_cachep is SLAB_DESTROY_BY_RCU so we can free it
1081392809b2SOleg Nesterov 		 * without an RCU grace period, see __lock_task_sighand().
1082392809b2SOleg Nesterov 		 */
1083c81addc9SOleg Nesterov 		kmem_cache_free(sighand_cachep, sighand);
1084c81addc9SOleg Nesterov 	}
1085d80e731eSOleg Nesterov }
1086c81addc9SOleg Nesterov 
1087f06febc9SFrank Mayhar /*
1088f06febc9SFrank Mayhar  * Initialize POSIX timer handling for a thread group.
1089f06febc9SFrank Mayhar  */
1090f06febc9SFrank Mayhar static void posix_cpu_timers_init_group(struct signal_struct *sig)
1091f06febc9SFrank Mayhar {
109278d7d407SJiri Slaby 	unsigned long cpu_limit;
109378d7d407SJiri Slaby 
1094316c1608SJason Low 	cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
109578d7d407SJiri Slaby 	if (cpu_limit != RLIM_INFINITY) {
109678d7d407SJiri Slaby 		sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit);
10976279a751SOleg Nesterov 		sig->cputimer.running = 1;
10986279a751SOleg Nesterov 	}
10996279a751SOleg Nesterov 
1100f06febc9SFrank Mayhar 	/* The timer lists. */
1101f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&sig->cpu_timers[0]);
1102f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&sig->cpu_timers[1]);
1103f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&sig->cpu_timers[2]);
1104f06febc9SFrank Mayhar }
1105f06febc9SFrank Mayhar 
1106a39bc516SAlexey Dobriyan static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
11071da177e4SLinus Torvalds {
11081da177e4SLinus Torvalds 	struct signal_struct *sig;
11091da177e4SLinus Torvalds 
11104ab6c083SOleg Nesterov 	if (clone_flags & CLONE_THREAD)
1111490dea45SPeter Zijlstra 		return 0;
11126279a751SOleg Nesterov 
1113a56704efSVeaceslav Falico 	sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
11141da177e4SLinus Torvalds 	tsk->signal = sig;
11151da177e4SLinus Torvalds 	if (!sig)
11161da177e4SLinus Torvalds 		return -ENOMEM;
11171da177e4SLinus Torvalds 
1118b3ac022cSOleg Nesterov 	sig->nr_threads = 1;
11191da177e4SLinus Torvalds 	atomic_set(&sig->live, 1);
1120b3ac022cSOleg Nesterov 	atomic_set(&sig->sigcnt, 1);
11210c740d0aSOleg Nesterov 
11220c740d0aSOleg Nesterov 	/* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
11230c740d0aSOleg Nesterov 	sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
11240c740d0aSOleg Nesterov 	tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
11250c740d0aSOleg Nesterov 
11261da177e4SLinus Torvalds 	init_waitqueue_head(&sig->wait_chldexit);
1127db51aeccSOleg Nesterov 	sig->curr_target = tsk;
11281da177e4SLinus Torvalds 	init_sigpending(&sig->shared_pending);
11291da177e4SLinus Torvalds 	INIT_LIST_HEAD(&sig->posix_timers);
1130e78c3496SRik van Riel 	seqlock_init(&sig->stats_lock);
11311da177e4SLinus Torvalds 
1132c9cb2e3dSThomas Gleixner 	hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
11331da177e4SLinus Torvalds 	sig->real_timer.function = it_real_fn;
11341da177e4SLinus Torvalds 
11351da177e4SLinus Torvalds 	task_lock(current->group_leader);
11361da177e4SLinus Torvalds 	memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
11371da177e4SLinus Torvalds 	task_unlock(current->group_leader);
11381da177e4SLinus Torvalds 
11396279a751SOleg Nesterov 	posix_cpu_timers_init_group(sig);
11406279a751SOleg Nesterov 
1141522ed776SMiloslav Trmac 	tty_audit_fork(sig);
11425091faa4SMike Galbraith 	sched_autogroup_fork(sig);
1143522ed776SMiloslav Trmac 
1144a63d83f4SDavid Rientjes 	sig->oom_score_adj = current->signal->oom_score_adj;
1145dabb16f6SMandeep Singh Baines 	sig->oom_score_adj_min = current->signal->oom_score_adj_min;
114628b83c51SKOSAKI Motohiro 
1147ebec18a6SLennart Poettering 	sig->has_child_subreaper = current->signal->has_child_subreaper ||
1148ebec18a6SLennart Poettering 				   current->signal->is_child_subreaper;
1149ebec18a6SLennart Poettering 
11509b1bf12dSKOSAKI Motohiro 	mutex_init(&sig->cred_guard_mutex);
11519b1bf12dSKOSAKI Motohiro 
11521da177e4SLinus Torvalds 	return 0;
11531da177e4SLinus Torvalds }
11541da177e4SLinus Torvalds 
1155dbd95212SKees Cook static void copy_seccomp(struct task_struct *p)
1156dbd95212SKees Cook {
1157dbd95212SKees Cook #ifdef CONFIG_SECCOMP
1158dbd95212SKees Cook 	/*
1159dbd95212SKees Cook 	 * Must be called with sighand->lock held, which is common to
1160dbd95212SKees Cook 	 * all threads in the group. Holding cred_guard_mutex is not
1161dbd95212SKees Cook 	 * needed because this new task is not yet running and cannot
1162dbd95212SKees Cook 	 * be racing exec.
1163dbd95212SKees Cook 	 */
116469f6a34bSGuenter Roeck 	assert_spin_locked(&current->sighand->siglock);
1165dbd95212SKees Cook 
1166dbd95212SKees Cook 	/* Ref-count the new filter user, and assign it. */
1167dbd95212SKees Cook 	get_seccomp_filter(current);
1168dbd95212SKees Cook 	p->seccomp = current->seccomp;
1169dbd95212SKees Cook 
1170dbd95212SKees Cook 	/*
1171dbd95212SKees Cook 	 * Explicitly enable no_new_privs here in case it got set
1172dbd95212SKees Cook 	 * between the task_struct being duplicated and holding the
1173dbd95212SKees Cook 	 * sighand lock. The seccomp state and nnp must be in sync.
1174dbd95212SKees Cook 	 */
1175dbd95212SKees Cook 	if (task_no_new_privs(current))
1176dbd95212SKees Cook 		task_set_no_new_privs(p);
1177dbd95212SKees Cook 
1178dbd95212SKees Cook 	/*
1179dbd95212SKees Cook 	 * If the parent gained a seccomp mode after copying thread
1180dbd95212SKees Cook 	 * flags and between before we held the sighand lock, we have
1181dbd95212SKees Cook 	 * to manually enable the seccomp thread flag here.
1182dbd95212SKees Cook 	 */
1183dbd95212SKees Cook 	if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
1184dbd95212SKees Cook 		set_tsk_thread_flag(p, TIF_SECCOMP);
1185dbd95212SKees Cook #endif
1186dbd95212SKees Cook }
1187dbd95212SKees Cook 
118817da2bd9SHeiko Carstens SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
11891da177e4SLinus Torvalds {
11901da177e4SLinus Torvalds 	current->clear_child_tid = tidptr;
11911da177e4SLinus Torvalds 
1192b488893aSPavel Emelyanov 	return task_pid_vnr(current);
11931da177e4SLinus Torvalds }
11941da177e4SLinus Torvalds 
1195a39bc516SAlexey Dobriyan static void rt_mutex_init_task(struct task_struct *p)
119623f78d4aSIngo Molnar {
11971d615482SThomas Gleixner 	raw_spin_lock_init(&p->pi_lock);
1198e29e175bSZilvinas Valinskas #ifdef CONFIG_RT_MUTEXES
1199fb00aca4SPeter Zijlstra 	p->pi_waiters = RB_ROOT;
1200fb00aca4SPeter Zijlstra 	p->pi_waiters_leftmost = NULL;
120123f78d4aSIngo Molnar 	p->pi_blocked_on = NULL;
120223f78d4aSIngo Molnar #endif
120323f78d4aSIngo Molnar }
120423f78d4aSIngo Molnar 
12051da177e4SLinus Torvalds /*
1206f06febc9SFrank Mayhar  * Initialize POSIX timer handling for a single task.
1207f06febc9SFrank Mayhar  */
1208f06febc9SFrank Mayhar static void posix_cpu_timers_init(struct task_struct *tsk)
1209f06febc9SFrank Mayhar {
121064861634SMartin Schwidefsky 	tsk->cputime_expires.prof_exp = 0;
121164861634SMartin Schwidefsky 	tsk->cputime_expires.virt_exp = 0;
1212f06febc9SFrank Mayhar 	tsk->cputime_expires.sched_exp = 0;
1213f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&tsk->cpu_timers[0]);
1214f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&tsk->cpu_timers[1]);
1215f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&tsk->cpu_timers[2]);
1216f06febc9SFrank Mayhar }
1217f06febc9SFrank Mayhar 
121881907739SOleg Nesterov static inline void
121981907739SOleg Nesterov init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
122081907739SOleg Nesterov {
122181907739SOleg Nesterov 	 task->pids[type].pid = pid;
122281907739SOleg Nesterov }
122381907739SOleg Nesterov 
1224f06febc9SFrank Mayhar /*
12251da177e4SLinus Torvalds  * This creates a new process as a copy of the old one,
12261da177e4SLinus Torvalds  * but does not actually start it yet.
12271da177e4SLinus Torvalds  *
12281da177e4SLinus Torvalds  * It copies the registers, and all the appropriate
12291da177e4SLinus Torvalds  * parts of the process environment (as per the clone
12301da177e4SLinus Torvalds  * flags). The actual kick-off is left to the caller.
12311da177e4SLinus Torvalds  */
123236c8b586SIngo Molnar static struct task_struct *copy_process(unsigned long clone_flags,
12331da177e4SLinus Torvalds 					unsigned long stack_start,
12341da177e4SLinus Torvalds 					unsigned long stack_size,
12351da177e4SLinus Torvalds 					int __user *child_tidptr,
123609a05394SRoland McGrath 					struct pid *pid,
12373033f14aSJosh Triplett 					int trace,
12383033f14aSJosh Triplett 					unsigned long tls)
12391da177e4SLinus Torvalds {
12401da177e4SLinus Torvalds 	int retval;
1241a24efe62SMariusz Kozlowski 	struct task_struct *p;
1242*7e47682eSAleksa Sarai 	void *cgrp_ss_priv[CGROUP_CANFORK_COUNT] = {};
12431da177e4SLinus Torvalds 
12441da177e4SLinus Torvalds 	if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
12451da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
12461da177e4SLinus Torvalds 
1247e66eded8SEric W. Biederman 	if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
1248e66eded8SEric W. Biederman 		return ERR_PTR(-EINVAL);
1249e66eded8SEric W. Biederman 
12501da177e4SLinus Torvalds 	/*
12511da177e4SLinus Torvalds 	 * Thread groups must share signals as well, and detached threads
12521da177e4SLinus Torvalds 	 * can only be started up within the thread group.
12531da177e4SLinus Torvalds 	 */
12541da177e4SLinus Torvalds 	if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
12551da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
12561da177e4SLinus Torvalds 
12571da177e4SLinus Torvalds 	/*
12581da177e4SLinus Torvalds 	 * Shared signal handlers imply shared VM. By way of the above,
12591da177e4SLinus Torvalds 	 * thread groups also imply shared VM. Blocking this case allows
12601da177e4SLinus Torvalds 	 * for various simplifications in other code.
12611da177e4SLinus Torvalds 	 */
12621da177e4SLinus Torvalds 	if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
12631da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
12641da177e4SLinus Torvalds 
1265123be07bSSukadev Bhattiprolu 	/*
1266123be07bSSukadev Bhattiprolu 	 * Siblings of global init remain as zombies on exit since they are
1267123be07bSSukadev Bhattiprolu 	 * not reaped by their parent (swapper). To solve this and to avoid
1268123be07bSSukadev Bhattiprolu 	 * multi-rooted process trees, prevent global and container-inits
1269123be07bSSukadev Bhattiprolu 	 * from creating siblings.
1270123be07bSSukadev Bhattiprolu 	 */
1271123be07bSSukadev Bhattiprolu 	if ((clone_flags & CLONE_PARENT) &&
1272123be07bSSukadev Bhattiprolu 				current->signal->flags & SIGNAL_UNKILLABLE)
1273123be07bSSukadev Bhattiprolu 		return ERR_PTR(-EINVAL);
1274123be07bSSukadev Bhattiprolu 
12758382fcacSEric W. Biederman 	/*
127640a0d32dSOleg Nesterov 	 * If the new process will be in a different pid or user namespace
127740a0d32dSOleg Nesterov 	 * do not allow it to share a thread group or signal handlers or
127840a0d32dSOleg Nesterov 	 * parent with the forking task.
12798382fcacSEric W. Biederman 	 */
12801f7f4ddeSEric W. Biederman 	if (clone_flags & CLONE_SIGHAND) {
128140a0d32dSOleg Nesterov 		if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
128240a0d32dSOleg Nesterov 		    (task_active_pid_ns(current) !=
1283c2b1df2eSAndy Lutomirski 				current->nsproxy->pid_ns_for_children))
12848382fcacSEric W. Biederman 			return ERR_PTR(-EINVAL);
128540a0d32dSOleg Nesterov 	}
12868382fcacSEric W. Biederman 
12871da177e4SLinus Torvalds 	retval = security_task_create(clone_flags);
12881da177e4SLinus Torvalds 	if (retval)
12891da177e4SLinus Torvalds 		goto fork_out;
12901da177e4SLinus Torvalds 
12911da177e4SLinus Torvalds 	retval = -ENOMEM;
12921da177e4SLinus Torvalds 	p = dup_task_struct(current);
12931da177e4SLinus Torvalds 	if (!p)
12941da177e4SLinus Torvalds 		goto fork_out;
12951da177e4SLinus Torvalds 
1296f7e8b616SSteven Rostedt 	ftrace_graph_init_task(p);
1297f7e8b616SSteven Rostedt 
1298bea493a0SPeter Zijlstra 	rt_mutex_init_task(p);
1299bea493a0SPeter Zijlstra 
1300d12c1a37SIngo Molnar #ifdef CONFIG_PROVE_LOCKING
1301de30a2b3SIngo Molnar 	DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
1302de30a2b3SIngo Molnar 	DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1303de30a2b3SIngo Molnar #endif
13041da177e4SLinus Torvalds 	retval = -EAGAIN;
13053b11a1deSDavid Howells 	if (atomic_read(&p->real_cred->user->processes) >=
130678d7d407SJiri Slaby 			task_rlimit(p, RLIMIT_NPROC)) {
1307b57922b6SEric Paris 		if (p->real_cred->user != INIT_USER &&
1308b57922b6SEric Paris 		    !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
13091da177e4SLinus Torvalds 			goto bad_fork_free;
13101da177e4SLinus Torvalds 	}
131172fa5997SVasiliy Kulikov 	current->flags &= ~PF_NPROC_EXCEEDED;
13121da177e4SLinus Torvalds 
1313f1752eecSDavid Howells 	retval = copy_creds(p, clone_flags);
1314f1752eecSDavid Howells 	if (retval < 0)
1315f1752eecSDavid Howells 		goto bad_fork_free;
13161da177e4SLinus Torvalds 
13171da177e4SLinus Torvalds 	/*
13181da177e4SLinus Torvalds 	 * If multiple threads are within copy_process(), then this check
13191da177e4SLinus Torvalds 	 * triggers too late. This doesn't hurt, the check is only there
13201da177e4SLinus Torvalds 	 * to stop root fork bombs.
13211da177e4SLinus Torvalds 	 */
132204ec93feSLi Zefan 	retval = -EAGAIN;
13231da177e4SLinus Torvalds 	if (nr_threads >= max_threads)
13241da177e4SLinus Torvalds 		goto bad_fork_cleanup_count;
13251da177e4SLinus Torvalds 
1326ca74e92bSShailabh Nagar 	delayacct_tsk_init(p);	/* Must remain after dup_task_struct() */
1327514ddb44SDavid Rientjes 	p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
1328514ddb44SDavid Rientjes 	p->flags |= PF_FORKNOEXEC;
13291da177e4SLinus Torvalds 	INIT_LIST_HEAD(&p->children);
13301da177e4SLinus Torvalds 	INIT_LIST_HEAD(&p->sibling);
1331f41d911fSPaul E. McKenney 	rcu_copy_process(p);
13321da177e4SLinus Torvalds 	p->vfork_done = NULL;
13331da177e4SLinus Torvalds 	spin_lock_init(&p->alloc_lock);
13341da177e4SLinus Torvalds 
13351da177e4SLinus Torvalds 	init_sigpending(&p->pending);
13361da177e4SLinus Torvalds 
133764861634SMartin Schwidefsky 	p->utime = p->stime = p->gtime = 0;
133864861634SMartin Schwidefsky 	p->utimescaled = p->stimescaled = 0;
13399fbc42eaSFrederic Weisbecker #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
1340d37f761dSFrederic Weisbecker 	p->prev_cputime.utime = p->prev_cputime.stime = 0;
1341d99ca3b9SHidetoshi Seto #endif
13426a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
13436a61671bSFrederic Weisbecker 	seqlock_init(&p->vtime_seqlock);
13446a61671bSFrederic Weisbecker 	p->vtime_snap = 0;
13456a61671bSFrederic Weisbecker 	p->vtime_snap_whence = VTIME_SLEEPING;
13466a61671bSFrederic Weisbecker #endif
13476a61671bSFrederic Weisbecker 
1348a3a2e76cSKAMEZAWA Hiroyuki #if defined(SPLIT_RSS_COUNTING)
1349a3a2e76cSKAMEZAWA Hiroyuki 	memset(&p->rss_stat, 0, sizeof(p->rss_stat));
1350a3a2e76cSKAMEZAWA Hiroyuki #endif
1351172ba844SBalbir Singh 
13526976675dSArjan van de Ven 	p->default_timer_slack_ns = current->timer_slack_ns;
13536976675dSArjan van de Ven 
13545995477aSAndrea Righi 	task_io_accounting_init(&p->ioac);
13551da177e4SLinus Torvalds 	acct_clear_integrals(p);
13561da177e4SLinus Torvalds 
1357f06febc9SFrank Mayhar 	posix_cpu_timers_init(p);
13581da177e4SLinus Torvalds 
1359ccbf62d8SThomas Gleixner 	p->start_time = ktime_get_ns();
136057e0be04SThomas Gleixner 	p->real_start_time = ktime_get_boot_ns();
13611da177e4SLinus Torvalds 	p->io_context = NULL;
13621da177e4SLinus Torvalds 	p->audit_context = NULL;
13634714d1d3SBen Blum 	if (clone_flags & CLONE_THREAD)
1364257058aeSTejun Heo 		threadgroup_change_begin(current);
1365b4f48b63SPaul Menage 	cgroup_fork(p);
13661da177e4SLinus Torvalds #ifdef CONFIG_NUMA
1367846a16bfSLee Schermerhorn 	p->mempolicy = mpol_dup(p->mempolicy);
13681da177e4SLinus Torvalds 	if (IS_ERR(p->mempolicy)) {
13691da177e4SLinus Torvalds 		retval = PTR_ERR(p->mempolicy);
13701da177e4SLinus Torvalds 		p->mempolicy = NULL;
1371e8604cb4SLi Zefan 		goto bad_fork_cleanup_threadgroup_lock;
13721da177e4SLinus Torvalds 	}
13731da177e4SLinus Torvalds #endif
1374778d3b0fSMichal Hocko #ifdef CONFIG_CPUSETS
1375778d3b0fSMichal Hocko 	p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
1376778d3b0fSMichal Hocko 	p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
1377cc9a6c87SMel Gorman 	seqcount_init(&p->mems_allowed_seq);
1378778d3b0fSMichal Hocko #endif
1379de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS
1380de30a2b3SIngo Molnar 	p->irq_events = 0;
1381de30a2b3SIngo Molnar 	p->hardirqs_enabled = 0;
1382de30a2b3SIngo Molnar 	p->hardirq_enable_ip = 0;
1383de30a2b3SIngo Molnar 	p->hardirq_enable_event = 0;
1384de30a2b3SIngo Molnar 	p->hardirq_disable_ip = _THIS_IP_;
1385de30a2b3SIngo Molnar 	p->hardirq_disable_event = 0;
1386de30a2b3SIngo Molnar 	p->softirqs_enabled = 1;
1387de30a2b3SIngo Molnar 	p->softirq_enable_ip = _THIS_IP_;
1388de30a2b3SIngo Molnar 	p->softirq_enable_event = 0;
1389de30a2b3SIngo Molnar 	p->softirq_disable_ip = 0;
1390de30a2b3SIngo Molnar 	p->softirq_disable_event = 0;
1391de30a2b3SIngo Molnar 	p->hardirq_context = 0;
1392de30a2b3SIngo Molnar 	p->softirq_context = 0;
1393de30a2b3SIngo Molnar #endif
13948bcbde54SDavid Hildenbrand 
13958bcbde54SDavid Hildenbrand 	p->pagefault_disabled = 0;
13968bcbde54SDavid Hildenbrand 
1397fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP
1398fbb9ce95SIngo Molnar 	p->lockdep_depth = 0; /* no locks held yet */
1399fbb9ce95SIngo Molnar 	p->curr_chain_key = 0;
1400fbb9ce95SIngo Molnar 	p->lockdep_recursion = 0;
1401fbb9ce95SIngo Molnar #endif
14021da177e4SLinus Torvalds 
1403408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES
1404408894eeSIngo Molnar 	p->blocked_on = NULL; /* not blocked yet */
1405408894eeSIngo Molnar #endif
1406cafe5635SKent Overstreet #ifdef CONFIG_BCACHE
1407cafe5635SKent Overstreet 	p->sequential_io	= 0;
1408cafe5635SKent Overstreet 	p->sequential_io_avg	= 0;
1409cafe5635SKent Overstreet #endif
14100f481406SMarkus Metzger 
14113c90e6e9SSrivatsa Vaddagiri 	/* Perform scheduler related setup. Assign this task to a CPU. */
1412aab03e05SDario Faggioli 	retval = sched_fork(clone_flags, p);
1413aab03e05SDario Faggioli 	if (retval)
1414aab03e05SDario Faggioli 		goto bad_fork_cleanup_policy;
14156ab423e0SPeter Zijlstra 
1416cdd6c482SIngo Molnar 	retval = perf_event_init_task(p);
14176ab423e0SPeter Zijlstra 	if (retval)
14186ab423e0SPeter Zijlstra 		goto bad_fork_cleanup_policy;
1419fb0a685cSDaniel Rebelo de Oliveira 	retval = audit_alloc(p);
1420fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
14216c72e350SPeter Zijlstra 		goto bad_fork_cleanup_perf;
14221da177e4SLinus Torvalds 	/* copy all the process information */
1423ab602f79SJack Miller 	shm_init_task(p);
1424fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_semundo(clone_flags, p);
1425fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
14261da177e4SLinus Torvalds 		goto bad_fork_cleanup_audit;
1427fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_files(clone_flags, p);
1428fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
14291da177e4SLinus Torvalds 		goto bad_fork_cleanup_semundo;
1430fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_fs(clone_flags, p);
1431fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
14321da177e4SLinus Torvalds 		goto bad_fork_cleanup_files;
1433fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_sighand(clone_flags, p);
1434fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
14351da177e4SLinus Torvalds 		goto bad_fork_cleanup_fs;
1436fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_signal(clone_flags, p);
1437fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
14381da177e4SLinus Torvalds 		goto bad_fork_cleanup_sighand;
1439fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_mm(clone_flags, p);
1440fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
14411da177e4SLinus Torvalds 		goto bad_fork_cleanup_signal;
1442fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_namespaces(clone_flags, p);
1443fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
1444d84f4f99SDavid Howells 		goto bad_fork_cleanup_mm;
1445fb0a685cSDaniel Rebelo de Oliveira 	retval = copy_io(clone_flags, p);
1446fb0a685cSDaniel Rebelo de Oliveira 	if (retval)
1447fd0928dfSJens Axboe 		goto bad_fork_cleanup_namespaces;
14483033f14aSJosh Triplett 	retval = copy_thread_tls(clone_flags, stack_start, stack_size, p, tls);
14491da177e4SLinus Torvalds 	if (retval)
1450fd0928dfSJens Axboe 		goto bad_fork_cleanup_io;
14511da177e4SLinus Torvalds 
1452425fb2b4SPavel Emelyanov 	if (pid != &init_struct_pid) {
1453c2b1df2eSAndy Lutomirski 		pid = alloc_pid(p->nsproxy->pid_ns_for_children);
145435f71bc0SMichal Hocko 		if (IS_ERR(pid)) {
145535f71bc0SMichal Hocko 			retval = PTR_ERR(pid);
1456fd0928dfSJens Axboe 			goto bad_fork_cleanup_io;
1457425fb2b4SPavel Emelyanov 		}
145835f71bc0SMichal Hocko 	}
1459425fb2b4SPavel Emelyanov 
14601da177e4SLinus Torvalds 	p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
14611da177e4SLinus Torvalds 	/*
14621da177e4SLinus Torvalds 	 * Clear TID on mm_release()?
14631da177e4SLinus Torvalds 	 */
14641da177e4SLinus Torvalds 	p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL;
146573c10101SJens Axboe #ifdef CONFIG_BLOCK
146673c10101SJens Axboe 	p->plug = NULL;
146773c10101SJens Axboe #endif
146842b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX
14698f17d3a5SIngo Molnar 	p->robust_list = NULL;
14708f17d3a5SIngo Molnar #ifdef CONFIG_COMPAT
14718f17d3a5SIngo Molnar 	p->compat_robust_list = NULL;
14728f17d3a5SIngo Molnar #endif
1473c87e2837SIngo Molnar 	INIT_LIST_HEAD(&p->pi_state_list);
1474c87e2837SIngo Molnar 	p->pi_state_cache = NULL;
147542b2dd0aSAlexey Dobriyan #endif
14761da177e4SLinus Torvalds 	/*
1477f9a3879aSGOTO Masanori 	 * sigaltstack should be cleared when sharing the same VM
1478f9a3879aSGOTO Masanori 	 */
1479f9a3879aSGOTO Masanori 	if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
1480f9a3879aSGOTO Masanori 		p->sas_ss_sp = p->sas_ss_size = 0;
1481f9a3879aSGOTO Masanori 
1482f9a3879aSGOTO Masanori 	/*
14836580807dSOleg Nesterov 	 * Syscall tracing and stepping should be turned off in the
14846580807dSOleg Nesterov 	 * child regardless of CLONE_PTRACE.
14851da177e4SLinus Torvalds 	 */
14866580807dSOleg Nesterov 	user_disable_single_step(p);
14871da177e4SLinus Torvalds 	clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
1488ed75e8d5SLaurent Vivier #ifdef TIF_SYSCALL_EMU
1489ed75e8d5SLaurent Vivier 	clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
1490ed75e8d5SLaurent Vivier #endif
14919745512cSArjan van de Ven 	clear_all_latency_tracing(p);
14921da177e4SLinus Torvalds 
14931da177e4SLinus Torvalds 	/* ok, now we should be set up.. */
149418c830dfSOleg Nesterov 	p->pid = pid_nr(pid);
149518c830dfSOleg Nesterov 	if (clone_flags & CLONE_THREAD) {
14965f8aadd8SOleg Nesterov 		p->exit_signal = -1;
149718c830dfSOleg Nesterov 		p->group_leader = current->group_leader;
149818c830dfSOleg Nesterov 		p->tgid = current->tgid;
149918c830dfSOleg Nesterov 	} else {
150018c830dfSOleg Nesterov 		if (clone_flags & CLONE_PARENT)
15015f8aadd8SOleg Nesterov 			p->exit_signal = current->group_leader->exit_signal;
15025f8aadd8SOleg Nesterov 		else
15035f8aadd8SOleg Nesterov 			p->exit_signal = (clone_flags & CSIGNAL);
150418c830dfSOleg Nesterov 		p->group_leader = p;
150518c830dfSOleg Nesterov 		p->tgid = p->pid;
150618c830dfSOleg Nesterov 	}
15075f8aadd8SOleg Nesterov 
15089d823e8fSWu Fengguang 	p->nr_dirtied = 0;
15099d823e8fSWu Fengguang 	p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
151083712358SWu Fengguang 	p->dirty_paused_when = 0;
15119d823e8fSWu Fengguang 
1512bb8cbbfeSOleg Nesterov 	p->pdeath_signal = 0;
151347e65328SOleg Nesterov 	INIT_LIST_HEAD(&p->thread_group);
1514158e1645SAl Viro 	p->task_works = NULL;
15151da177e4SLinus Torvalds 
151618c830dfSOleg Nesterov 	/*
1517*7e47682eSAleksa Sarai 	 * Ensure that the cgroup subsystem policies allow the new process to be
1518*7e47682eSAleksa Sarai 	 * forked. It should be noted the the new process's css_set can be changed
1519*7e47682eSAleksa Sarai 	 * between here and cgroup_post_fork() if an organisation operation is in
1520*7e47682eSAleksa Sarai 	 * progress.
1521*7e47682eSAleksa Sarai 	 */
1522*7e47682eSAleksa Sarai 	retval = cgroup_can_fork(p, cgrp_ss_priv);
1523*7e47682eSAleksa Sarai 	if (retval)
1524*7e47682eSAleksa Sarai 		goto bad_fork_free_pid;
1525*7e47682eSAleksa Sarai 
1526*7e47682eSAleksa Sarai 	/*
152718c830dfSOleg Nesterov 	 * Make it visible to the rest of the system, but dont wake it up yet.
152818c830dfSOleg Nesterov 	 * Need tasklist lock for parent etc handling!
152918c830dfSOleg Nesterov 	 */
15301da177e4SLinus Torvalds 	write_lock_irq(&tasklist_lock);
15311da177e4SLinus Torvalds 
15321da177e4SLinus Torvalds 	/* CLONE_PARENT re-uses the old parent */
15332d5516cbSOleg Nesterov 	if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
15341da177e4SLinus Torvalds 		p->real_parent = current->real_parent;
15352d5516cbSOleg Nesterov 		p->parent_exec_id = current->parent_exec_id;
15362d5516cbSOleg Nesterov 	} else {
15371da177e4SLinus Torvalds 		p->real_parent = current;
15382d5516cbSOleg Nesterov 		p->parent_exec_id = current->self_exec_id;
15392d5516cbSOleg Nesterov 	}
15401da177e4SLinus Torvalds 
15411da177e4SLinus Torvalds 	spin_lock(&current->sighand->siglock);
15424a2c7a78SOleg Nesterov 
15434a2c7a78SOleg Nesterov 	/*
1544dbd95212SKees Cook 	 * Copy seccomp details explicitly here, in case they were changed
1545dbd95212SKees Cook 	 * before holding sighand lock.
1546dbd95212SKees Cook 	 */
1547dbd95212SKees Cook 	copy_seccomp(p);
1548dbd95212SKees Cook 
1549dbd95212SKees Cook 	/*
15504a2c7a78SOleg Nesterov 	 * Process group and session signals need to be delivered to just the
15514a2c7a78SOleg Nesterov 	 * parent before the fork or both the parent and the child after the
15524a2c7a78SOleg Nesterov 	 * fork. Restart if a signal comes in before we add the new process to
15534a2c7a78SOleg Nesterov 	 * it's process group.
15544a2c7a78SOleg Nesterov 	 * A fatal signal pending means that current will exit, so the new
15554a2c7a78SOleg Nesterov 	 * thread can't slip out of an OOM kill (or normal SIGKILL).
15564a2c7a78SOleg Nesterov 	*/
15574a2c7a78SOleg Nesterov 	recalc_sigpending();
15584a2c7a78SOleg Nesterov 	if (signal_pending(current)) {
15594a2c7a78SOleg Nesterov 		spin_unlock(&current->sighand->siglock);
15604a2c7a78SOleg Nesterov 		write_unlock_irq(&tasklist_lock);
15614a2c7a78SOleg Nesterov 		retval = -ERESTARTNOINTR;
1562*7e47682eSAleksa Sarai 		goto bad_fork_cancel_cgroup;
15634a2c7a78SOleg Nesterov 	}
15644a2c7a78SOleg Nesterov 
156573b9ebfeSOleg Nesterov 	if (likely(p->pid)) {
15664b9d33e6STejun Heo 		ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
15671da177e4SLinus Torvalds 
156881907739SOleg Nesterov 		init_task_pid(p, PIDTYPE_PID, pid);
15691da177e4SLinus Torvalds 		if (thread_group_leader(p)) {
157081907739SOleg Nesterov 			init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
157181907739SOleg Nesterov 			init_task_pid(p, PIDTYPE_SID, task_session(current));
157281907739SOleg Nesterov 
15731c4042c2SEric W. Biederman 			if (is_child_reaper(pid)) {
157417cf22c3SEric W. Biederman 				ns_of_pid(pid)->child_reaper = p;
15751c4042c2SEric W. Biederman 				p->signal->flags |= SIGNAL_UNKILLABLE;
15761c4042c2SEric W. Biederman 			}
15775cd17569SEric W. Biederman 
1578fea9d175SOleg Nesterov 			p->signal->leader_pid = pid;
15799c9f4dedSAlan Cox 			p->signal->tty = tty_kref_get(current->signal->tty);
15809cd80bbbSOleg Nesterov 			list_add_tail(&p->sibling, &p->real_parent->children);
15815e85d4abSEric W. Biederman 			list_add_tail_rcu(&p->tasks, &init_task.tasks);
158281907739SOleg Nesterov 			attach_pid(p, PIDTYPE_PGID);
158381907739SOleg Nesterov 			attach_pid(p, PIDTYPE_SID);
1584909ea964SChristoph Lameter 			__this_cpu_inc(process_counts);
158580628ca0SOleg Nesterov 		} else {
158680628ca0SOleg Nesterov 			current->signal->nr_threads++;
158780628ca0SOleg Nesterov 			atomic_inc(&current->signal->live);
158880628ca0SOleg Nesterov 			atomic_inc(&current->signal->sigcnt);
158980628ca0SOleg Nesterov 			list_add_tail_rcu(&p->thread_group,
159080628ca0SOleg Nesterov 					  &p->group_leader->thread_group);
15910c740d0aSOleg Nesterov 			list_add_tail_rcu(&p->thread_node,
15920c740d0aSOleg Nesterov 					  &p->signal->thread_head);
15931da177e4SLinus Torvalds 		}
159481907739SOleg Nesterov 		attach_pid(p, PIDTYPE_PID);
15951da177e4SLinus Torvalds 		nr_threads++;
159673b9ebfeSOleg Nesterov 	}
159773b9ebfeSOleg Nesterov 
15981da177e4SLinus Torvalds 	total_forks++;
15993f17da69SOleg Nesterov 	spin_unlock(&current->sighand->siglock);
16004af4206bSOleg Nesterov 	syscall_tracepoint_update(p);
16011da177e4SLinus Torvalds 	write_unlock_irq(&tasklist_lock);
16024af4206bSOleg Nesterov 
1603c13cf856SAndrew Morton 	proc_fork_connector(p);
1604*7e47682eSAleksa Sarai 	cgroup_post_fork(p, cgrp_ss_priv);
16054714d1d3SBen Blum 	if (clone_flags & CLONE_THREAD)
1606257058aeSTejun Heo 		threadgroup_change_end(current);
1607cdd6c482SIngo Molnar 	perf_event_fork(p);
160843d2b113SKAMEZAWA Hiroyuki 
160943d2b113SKAMEZAWA Hiroyuki 	trace_task_newtask(p, clone_flags);
16103ab67966SOleg Nesterov 	uprobe_copy_process(p, clone_flags);
161143d2b113SKAMEZAWA Hiroyuki 
16121da177e4SLinus Torvalds 	return p;
16131da177e4SLinus Torvalds 
1614*7e47682eSAleksa Sarai bad_fork_cancel_cgroup:
1615*7e47682eSAleksa Sarai 	cgroup_cancel_fork(p, cgrp_ss_priv);
1616425fb2b4SPavel Emelyanov bad_fork_free_pid:
1617425fb2b4SPavel Emelyanov 	if (pid != &init_struct_pid)
1618425fb2b4SPavel Emelyanov 		free_pid(pid);
1619fd0928dfSJens Axboe bad_fork_cleanup_io:
1620b69f2292SLouis Rilling 	if (p->io_context)
1621b69f2292SLouis Rilling 		exit_io_context(p);
1622ab516013SSerge E. Hallyn bad_fork_cleanup_namespaces:
1623444f378bSLinus Torvalds 	exit_task_namespaces(p);
16241da177e4SLinus Torvalds bad_fork_cleanup_mm:
1625c9f01245SDavid Rientjes 	if (p->mm)
16261da177e4SLinus Torvalds 		mmput(p->mm);
16271da177e4SLinus Torvalds bad_fork_cleanup_signal:
16284ab6c083SOleg Nesterov 	if (!(clone_flags & CLONE_THREAD))
16291c5354deSMike Galbraith 		free_signal_struct(p->signal);
16301da177e4SLinus Torvalds bad_fork_cleanup_sighand:
1631a7e5328aSOleg Nesterov 	__cleanup_sighand(p->sighand);
16321da177e4SLinus Torvalds bad_fork_cleanup_fs:
16331da177e4SLinus Torvalds 	exit_fs(p); /* blocking */
16341da177e4SLinus Torvalds bad_fork_cleanup_files:
16351da177e4SLinus Torvalds 	exit_files(p); /* blocking */
16361da177e4SLinus Torvalds bad_fork_cleanup_semundo:
16371da177e4SLinus Torvalds 	exit_sem(p);
16381da177e4SLinus Torvalds bad_fork_cleanup_audit:
16391da177e4SLinus Torvalds 	audit_free(p);
16406c72e350SPeter Zijlstra bad_fork_cleanup_perf:
1641cdd6c482SIngo Molnar 	perf_event_free_task(p);
16426c72e350SPeter Zijlstra bad_fork_cleanup_policy:
16431da177e4SLinus Torvalds #ifdef CONFIG_NUMA
1644f0be3d32SLee Schermerhorn 	mpol_put(p->mempolicy);
1645e8604cb4SLi Zefan bad_fork_cleanup_threadgroup_lock:
16461da177e4SLinus Torvalds #endif
16474714d1d3SBen Blum 	if (clone_flags & CLONE_THREAD)
1648257058aeSTejun Heo 		threadgroup_change_end(current);
164935df17c5SShailabh Nagar 	delayacct_tsk_free(p);
16501da177e4SLinus Torvalds bad_fork_cleanup_count:
1651d84f4f99SDavid Howells 	atomic_dec(&p->cred->user->processes);
1652e0e81739SDavid Howells 	exit_creds(p);
16531da177e4SLinus Torvalds bad_fork_free:
16541da177e4SLinus Torvalds 	free_task(p);
1655fe7d37d1SOleg Nesterov fork_out:
1656fe7d37d1SOleg Nesterov 	return ERR_PTR(retval);
16571da177e4SLinus Torvalds }
16581da177e4SLinus Torvalds 
1659f106eee1SOleg Nesterov static inline void init_idle_pids(struct pid_link *links)
1660f106eee1SOleg Nesterov {
1661f106eee1SOleg Nesterov 	enum pid_type type;
1662f106eee1SOleg Nesterov 
1663f106eee1SOleg Nesterov 	for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
1664f106eee1SOleg Nesterov 		INIT_HLIST_NODE(&links[type].node); /* not really needed */
1665f106eee1SOleg Nesterov 		links[type].pid = &init_struct_pid;
1666f106eee1SOleg Nesterov 	}
1667f106eee1SOleg Nesterov }
1668f106eee1SOleg Nesterov 
16690db0628dSPaul Gortmaker struct task_struct *fork_idle(int cpu)
16701da177e4SLinus Torvalds {
167136c8b586SIngo Molnar 	struct task_struct *task;
16723033f14aSJosh Triplett 	task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0, 0);
1673f106eee1SOleg Nesterov 	if (!IS_ERR(task)) {
1674f106eee1SOleg Nesterov 		init_idle_pids(task->pids);
16751da177e4SLinus Torvalds 		init_idle(task, cpu);
1676f106eee1SOleg Nesterov 	}
167773b9ebfeSOleg Nesterov 
16781da177e4SLinus Torvalds 	return task;
16791da177e4SLinus Torvalds }
16801da177e4SLinus Torvalds 
16811da177e4SLinus Torvalds /*
16821da177e4SLinus Torvalds  *  Ok, this is the main fork-routine.
16831da177e4SLinus Torvalds  *
16841da177e4SLinus Torvalds  * It copies the process, and if successful kick-starts
16851da177e4SLinus Torvalds  * it and waits for it to finish using the VM if required.
16861da177e4SLinus Torvalds  */
16873033f14aSJosh Triplett long _do_fork(unsigned long clone_flags,
16881da177e4SLinus Torvalds 	      unsigned long stack_start,
16891da177e4SLinus Torvalds 	      unsigned long stack_size,
16901da177e4SLinus Torvalds 	      int __user *parent_tidptr,
16913033f14aSJosh Triplett 	      int __user *child_tidptr,
16923033f14aSJosh Triplett 	      unsigned long tls)
16931da177e4SLinus Torvalds {
16941da177e4SLinus Torvalds 	struct task_struct *p;
16951da177e4SLinus Torvalds 	int trace = 0;
169692476d7fSEric W. Biederman 	long nr;
16971da177e4SLinus Torvalds 
1698bdff746aSAndrew Morton 	/*
16994b9d33e6STejun Heo 	 * Determine whether and which event to report to ptracer.  When
17004b9d33e6STejun Heo 	 * called from kernel_thread or CLONE_UNTRACED is explicitly
17014b9d33e6STejun Heo 	 * requested, no event is reported; otherwise, report if the event
17024b9d33e6STejun Heo 	 * for the type of forking is enabled.
170309a05394SRoland McGrath 	 */
1704e80d6661SAl Viro 	if (!(clone_flags & CLONE_UNTRACED)) {
17054b9d33e6STejun Heo 		if (clone_flags & CLONE_VFORK)
17064b9d33e6STejun Heo 			trace = PTRACE_EVENT_VFORK;
17074b9d33e6STejun Heo 		else if ((clone_flags & CSIGNAL) != SIGCHLD)
17084b9d33e6STejun Heo 			trace = PTRACE_EVENT_CLONE;
17094b9d33e6STejun Heo 		else
17104b9d33e6STejun Heo 			trace = PTRACE_EVENT_FORK;
17114b9d33e6STejun Heo 
17124b9d33e6STejun Heo 		if (likely(!ptrace_event_enabled(current, trace)))
17134b9d33e6STejun Heo 			trace = 0;
17144b9d33e6STejun Heo 	}
17151da177e4SLinus Torvalds 
171662e791c1SAl Viro 	p = copy_process(clone_flags, stack_start, stack_size,
17173033f14aSJosh Triplett 			 child_tidptr, NULL, trace, tls);
17181da177e4SLinus Torvalds 	/*
17191da177e4SLinus Torvalds 	 * Do this prior waking up the new thread - the thread pointer
17201da177e4SLinus Torvalds 	 * might get invalid after that point, if the thread exits quickly.
17211da177e4SLinus Torvalds 	 */
17221da177e4SLinus Torvalds 	if (!IS_ERR(p)) {
17231da177e4SLinus Torvalds 		struct completion vfork;
17244e52365fSMatthew Dempsky 		struct pid *pid;
17251da177e4SLinus Torvalds 
17260a16b607SMathieu Desnoyers 		trace_sched_process_fork(current, p);
17270a16b607SMathieu Desnoyers 
17284e52365fSMatthew Dempsky 		pid = get_task_pid(p, PIDTYPE_PID);
17294e52365fSMatthew Dempsky 		nr = pid_vnr(pid);
173030e49c26SPavel Emelyanov 
173130e49c26SPavel Emelyanov 		if (clone_flags & CLONE_PARENT_SETTID)
173230e49c26SPavel Emelyanov 			put_user(nr, parent_tidptr);
1733a6f5e063SSukadev Bhattiprolu 
17341da177e4SLinus Torvalds 		if (clone_flags & CLONE_VFORK) {
17351da177e4SLinus Torvalds 			p->vfork_done = &vfork;
17361da177e4SLinus Torvalds 			init_completion(&vfork);
1737d68b46feSOleg Nesterov 			get_task_struct(p);
17381da177e4SLinus Torvalds 		}
17391da177e4SLinus Torvalds 
17403e51e3edSSamir Bellabes 		wake_up_new_task(p);
17411da177e4SLinus Torvalds 
17424b9d33e6STejun Heo 		/* forking complete and child started to run, tell ptracer */
17434b9d33e6STejun Heo 		if (unlikely(trace))
17444e52365fSMatthew Dempsky 			ptrace_event_pid(trace, pid);
174509a05394SRoland McGrath 
17461da177e4SLinus Torvalds 		if (clone_flags & CLONE_VFORK) {
1747d68b46feSOleg Nesterov 			if (!wait_for_vfork_done(p, &vfork))
17484e52365fSMatthew Dempsky 				ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
17499f59ce5dSChuck Ebbert 		}
17504e52365fSMatthew Dempsky 
17514e52365fSMatthew Dempsky 		put_pid(pid);
17521da177e4SLinus Torvalds 	} else {
175392476d7fSEric W. Biederman 		nr = PTR_ERR(p);
17541da177e4SLinus Torvalds 	}
175592476d7fSEric W. Biederman 	return nr;
17561da177e4SLinus Torvalds }
17571da177e4SLinus Torvalds 
17583033f14aSJosh Triplett #ifndef CONFIG_HAVE_COPY_THREAD_TLS
17593033f14aSJosh Triplett /* For compatibility with architectures that call do_fork directly rather than
17603033f14aSJosh Triplett  * using the syscall entry points below. */
17613033f14aSJosh Triplett long do_fork(unsigned long clone_flags,
17623033f14aSJosh Triplett 	      unsigned long stack_start,
17633033f14aSJosh Triplett 	      unsigned long stack_size,
17643033f14aSJosh Triplett 	      int __user *parent_tidptr,
17653033f14aSJosh Triplett 	      int __user *child_tidptr)
17663033f14aSJosh Triplett {
17673033f14aSJosh Triplett 	return _do_fork(clone_flags, stack_start, stack_size,
17683033f14aSJosh Triplett 			parent_tidptr, child_tidptr, 0);
17693033f14aSJosh Triplett }
17703033f14aSJosh Triplett #endif
17713033f14aSJosh Triplett 
17722aa3a7f8SAl Viro /*
17732aa3a7f8SAl Viro  * Create a kernel thread.
17742aa3a7f8SAl Viro  */
17752aa3a7f8SAl Viro pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
17762aa3a7f8SAl Viro {
17773033f14aSJosh Triplett 	return _do_fork(flags|CLONE_VM|CLONE_UNTRACED, (unsigned long)fn,
17783033f14aSJosh Triplett 		(unsigned long)arg, NULL, NULL, 0);
17792aa3a7f8SAl Viro }
17802aa3a7f8SAl Viro 
1781d2125043SAl Viro #ifdef __ARCH_WANT_SYS_FORK
1782d2125043SAl Viro SYSCALL_DEFINE0(fork)
1783d2125043SAl Viro {
1784d2125043SAl Viro #ifdef CONFIG_MMU
17853033f14aSJosh Triplett 	return _do_fork(SIGCHLD, 0, 0, NULL, NULL, 0);
1786d2125043SAl Viro #else
1787d2125043SAl Viro 	/* can not support in nommu mode */
17885d59e182SDaeseok Youn 	return -EINVAL;
1789d2125043SAl Viro #endif
1790d2125043SAl Viro }
1791d2125043SAl Viro #endif
1792d2125043SAl Viro 
1793d2125043SAl Viro #ifdef __ARCH_WANT_SYS_VFORK
1794d2125043SAl Viro SYSCALL_DEFINE0(vfork)
1795d2125043SAl Viro {
17963033f14aSJosh Triplett 	return _do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, 0,
17973033f14aSJosh Triplett 			0, NULL, NULL, 0);
1798d2125043SAl Viro }
1799d2125043SAl Viro #endif
1800d2125043SAl Viro 
1801d2125043SAl Viro #ifdef __ARCH_WANT_SYS_CLONE
1802d2125043SAl Viro #ifdef CONFIG_CLONE_BACKWARDS
1803d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
1804d2125043SAl Viro 		 int __user *, parent_tidptr,
18053033f14aSJosh Triplett 		 unsigned long, tls,
1806d2125043SAl Viro 		 int __user *, child_tidptr)
1807d2125043SAl Viro #elif defined(CONFIG_CLONE_BACKWARDS2)
1808d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
1809d2125043SAl Viro 		 int __user *, parent_tidptr,
1810d2125043SAl Viro 		 int __user *, child_tidptr,
18113033f14aSJosh Triplett 		 unsigned long, tls)
1812dfa9771aSMichal Simek #elif defined(CONFIG_CLONE_BACKWARDS3)
1813dfa9771aSMichal Simek SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
1814dfa9771aSMichal Simek 		int, stack_size,
1815dfa9771aSMichal Simek 		int __user *, parent_tidptr,
1816dfa9771aSMichal Simek 		int __user *, child_tidptr,
18173033f14aSJosh Triplett 		unsigned long, tls)
1818d2125043SAl Viro #else
1819d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
1820d2125043SAl Viro 		 int __user *, parent_tidptr,
1821d2125043SAl Viro 		 int __user *, child_tidptr,
18223033f14aSJosh Triplett 		 unsigned long, tls)
1823d2125043SAl Viro #endif
1824d2125043SAl Viro {
18253033f14aSJosh Triplett 	return _do_fork(clone_flags, newsp, 0, parent_tidptr, child_tidptr, tls);
1826d2125043SAl Viro }
1827d2125043SAl Viro #endif
1828d2125043SAl Viro 
18295fd63b30SRavikiran G Thirumalai #ifndef ARCH_MIN_MMSTRUCT_ALIGN
18305fd63b30SRavikiran G Thirumalai #define ARCH_MIN_MMSTRUCT_ALIGN 0
18315fd63b30SRavikiran G Thirumalai #endif
18325fd63b30SRavikiran G Thirumalai 
183351cc5068SAlexey Dobriyan static void sighand_ctor(void *data)
1834aa1757f9SOleg Nesterov {
1835aa1757f9SOleg Nesterov 	struct sighand_struct *sighand = data;
1836aa1757f9SOleg Nesterov 
1837aa1757f9SOleg Nesterov 	spin_lock_init(&sighand->siglock);
1838b8fceee1SDavide Libenzi 	init_waitqueue_head(&sighand->signalfd_wqh);
1839fba2afaaSDavide Libenzi }
1840aa1757f9SOleg Nesterov 
18411da177e4SLinus Torvalds void __init proc_caches_init(void)
18421da177e4SLinus Torvalds {
18431da177e4SLinus Torvalds 	sighand_cachep = kmem_cache_create("sighand_cache",
18441da177e4SLinus Torvalds 			sizeof(struct sighand_struct), 0,
18452dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU|
18462dff4405SVegard Nossum 			SLAB_NOTRACK, sighand_ctor);
18471da177e4SLinus Torvalds 	signal_cachep = kmem_cache_create("signal_cache",
18481da177e4SLinus Torvalds 			sizeof(struct signal_struct), 0,
18492dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
18501da177e4SLinus Torvalds 	files_cachep = kmem_cache_create("files_cache",
18511da177e4SLinus Torvalds 			sizeof(struct files_struct), 0,
18522dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
18531da177e4SLinus Torvalds 	fs_cachep = kmem_cache_create("fs_cache",
18541da177e4SLinus Torvalds 			sizeof(struct fs_struct), 0,
18552dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
18566345d24dSLinus Torvalds 	/*
18576345d24dSLinus Torvalds 	 * FIXME! The "sizeof(struct mm_struct)" currently includes the
18586345d24dSLinus Torvalds 	 * whole struct cpumask for the OFFSTACK case. We could change
18596345d24dSLinus Torvalds 	 * this to *only* allocate as much of it as required by the
18606345d24dSLinus Torvalds 	 * maximum number of CPU's we can ever have.  The cpumask_allocation
18616345d24dSLinus Torvalds 	 * is at the end of the structure, exactly for that reason.
18626345d24dSLinus Torvalds 	 */
18631da177e4SLinus Torvalds 	mm_cachep = kmem_cache_create("mm_struct",
18645fd63b30SRavikiran G Thirumalai 			sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
18652dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
186633e5d769SDavid Howells 	vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC);
18678feae131SDavid Howells 	mmap_init();
186866577193SAl Viro 	nsproxy_cache_init();
18691da177e4SLinus Torvalds }
1870cf2e340fSJANAK DESAI 
1871cf2e340fSJANAK DESAI /*
18729bfb23fcSOleg Nesterov  * Check constraints on flags passed to the unshare system call.
1873cf2e340fSJANAK DESAI  */
18749bfb23fcSOleg Nesterov static int check_unshare_flags(unsigned long unshare_flags)
1875cf2e340fSJANAK DESAI {
18769bfb23fcSOleg Nesterov 	if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
18779bfb23fcSOleg Nesterov 				CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
187850804fe3SEric W. Biederman 				CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET|
1879b2e0d987SEric W. Biederman 				CLONE_NEWUSER|CLONE_NEWPID))
1880cf2e340fSJANAK DESAI 		return -EINVAL;
18819bfb23fcSOleg Nesterov 	/*
18829bfb23fcSOleg Nesterov 	 * Not implemented, but pretend it works if there is nothing to
18839bfb23fcSOleg Nesterov 	 * unshare. Note that unsharing CLONE_THREAD or CLONE_SIGHAND
18849bfb23fcSOleg Nesterov 	 * needs to unshare vm.
18859bfb23fcSOleg Nesterov 	 */
18869bfb23fcSOleg Nesterov 	if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
18879bfb23fcSOleg Nesterov 		/* FIXME: get_task_mm() increments ->mm_users */
18889bfb23fcSOleg Nesterov 		if (atomic_read(&current->mm->mm_users) > 1)
18899bfb23fcSOleg Nesterov 			return -EINVAL;
18909bfb23fcSOleg Nesterov 	}
1891cf2e340fSJANAK DESAI 
1892cf2e340fSJANAK DESAI 	return 0;
1893cf2e340fSJANAK DESAI }
1894cf2e340fSJANAK DESAI 
1895cf2e340fSJANAK DESAI /*
189699d1419dSJANAK DESAI  * Unshare the filesystem structure if it is being shared
1897cf2e340fSJANAK DESAI  */
1898cf2e340fSJANAK DESAI static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
1899cf2e340fSJANAK DESAI {
1900cf2e340fSJANAK DESAI 	struct fs_struct *fs = current->fs;
1901cf2e340fSJANAK DESAI 
1902498052bbSAl Viro 	if (!(unshare_flags & CLONE_FS) || !fs)
1903498052bbSAl Viro 		return 0;
1904498052bbSAl Viro 
1905498052bbSAl Viro 	/* don't need lock here; in the worst case we'll do useless copy */
1906498052bbSAl Viro 	if (fs->users == 1)
1907498052bbSAl Viro 		return 0;
1908498052bbSAl Viro 
1909498052bbSAl Viro 	*new_fsp = copy_fs_struct(fs);
191099d1419dSJANAK DESAI 	if (!*new_fsp)
191199d1419dSJANAK DESAI 		return -ENOMEM;
1912cf2e340fSJANAK DESAI 
1913cf2e340fSJANAK DESAI 	return 0;
1914cf2e340fSJANAK DESAI }
1915cf2e340fSJANAK DESAI 
1916cf2e340fSJANAK DESAI /*
1917a016f338SJANAK DESAI  * Unshare file descriptor table if it is being shared
1918cf2e340fSJANAK DESAI  */
1919cf2e340fSJANAK DESAI static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
1920cf2e340fSJANAK DESAI {
1921cf2e340fSJANAK DESAI 	struct files_struct *fd = current->files;
1922a016f338SJANAK DESAI 	int error = 0;
1923cf2e340fSJANAK DESAI 
1924cf2e340fSJANAK DESAI 	if ((unshare_flags & CLONE_FILES) &&
1925a016f338SJANAK DESAI 	    (fd && atomic_read(&fd->count) > 1)) {
1926a016f338SJANAK DESAI 		*new_fdp = dup_fd(fd, &error);
1927a016f338SJANAK DESAI 		if (!*new_fdp)
1928a016f338SJANAK DESAI 			return error;
1929a016f338SJANAK DESAI 	}
1930cf2e340fSJANAK DESAI 
1931cf2e340fSJANAK DESAI 	return 0;
1932cf2e340fSJANAK DESAI }
1933cf2e340fSJANAK DESAI 
1934cf2e340fSJANAK DESAI /*
1935cf2e340fSJANAK DESAI  * unshare allows a process to 'unshare' part of the process
1936cf2e340fSJANAK DESAI  * context which was originally shared using clone.  copy_*
1937cf2e340fSJANAK DESAI  * functions used by do_fork() cannot be used here directly
1938cf2e340fSJANAK DESAI  * because they modify an inactive task_struct that is being
1939cf2e340fSJANAK DESAI  * constructed. Here we are modifying the current, active,
1940cf2e340fSJANAK DESAI  * task_struct.
1941cf2e340fSJANAK DESAI  */
19426559eed8SHeiko Carstens SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
1943cf2e340fSJANAK DESAI {
1944cf2e340fSJANAK DESAI 	struct fs_struct *fs, *new_fs = NULL;
1945cf2e340fSJANAK DESAI 	struct files_struct *fd, *new_fd = NULL;
1946b2e0d987SEric W. Biederman 	struct cred *new_cred = NULL;
1947cf7b708cSPavel Emelyanov 	struct nsproxy *new_nsproxy = NULL;
19489edff4abSManfred Spraul 	int do_sysvsem = 0;
19499bfb23fcSOleg Nesterov 	int err;
1950cf2e340fSJANAK DESAI 
195150804fe3SEric W. Biederman 	/*
1952b2e0d987SEric W. Biederman 	 * If unsharing a user namespace must also unshare the thread.
1953b2e0d987SEric W. Biederman 	 */
1954b2e0d987SEric W. Biederman 	if (unshare_flags & CLONE_NEWUSER)
1955e66eded8SEric W. Biederman 		unshare_flags |= CLONE_THREAD | CLONE_FS;
1956b2e0d987SEric W. Biederman 	/*
195750804fe3SEric W. Biederman 	 * If unsharing a thread from a thread group, must also unshare vm.
195850804fe3SEric W. Biederman 	 */
195950804fe3SEric W. Biederman 	if (unshare_flags & CLONE_THREAD)
196050804fe3SEric W. Biederman 		unshare_flags |= CLONE_VM;
196150804fe3SEric W. Biederman 	/*
196250804fe3SEric W. Biederman 	 * If unsharing vm, must also unshare signal handlers.
196350804fe3SEric W. Biederman 	 */
196450804fe3SEric W. Biederman 	if (unshare_flags & CLONE_VM)
196550804fe3SEric W. Biederman 		unshare_flags |= CLONE_SIGHAND;
19666013f67fSManfred Spraul 	/*
19679bfb23fcSOleg Nesterov 	 * If unsharing namespace, must also unshare filesystem information.
19689bfb23fcSOleg Nesterov 	 */
19699bfb23fcSOleg Nesterov 	if (unshare_flags & CLONE_NEWNS)
19709bfb23fcSOleg Nesterov 		unshare_flags |= CLONE_FS;
197150804fe3SEric W. Biederman 
197250804fe3SEric W. Biederman 	err = check_unshare_flags(unshare_flags);
197350804fe3SEric W. Biederman 	if (err)
197450804fe3SEric W. Biederman 		goto bad_unshare_out;
19759bfb23fcSOleg Nesterov 	/*
19766013f67fSManfred Spraul 	 * CLONE_NEWIPC must also detach from the undolist: after switching
19776013f67fSManfred Spraul 	 * to a new ipc namespace, the semaphore arrays from the old
19786013f67fSManfred Spraul 	 * namespace are unreachable.
19796013f67fSManfred Spraul 	 */
19806013f67fSManfred Spraul 	if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
19819edff4abSManfred Spraul 		do_sysvsem = 1;
1982fb0a685cSDaniel Rebelo de Oliveira 	err = unshare_fs(unshare_flags, &new_fs);
1983fb0a685cSDaniel Rebelo de Oliveira 	if (err)
19849bfb23fcSOleg Nesterov 		goto bad_unshare_out;
1985fb0a685cSDaniel Rebelo de Oliveira 	err = unshare_fd(unshare_flags, &new_fd);
1986fb0a685cSDaniel Rebelo de Oliveira 	if (err)
19879bfb23fcSOleg Nesterov 		goto bad_unshare_cleanup_fs;
1988b2e0d987SEric W. Biederman 	err = unshare_userns(unshare_flags, &new_cred);
1989fb0a685cSDaniel Rebelo de Oliveira 	if (err)
19909edff4abSManfred Spraul 		goto bad_unshare_cleanup_fd;
1991b2e0d987SEric W. Biederman 	err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
1992b2e0d987SEric W. Biederman 					 new_cred, new_fs);
1993b2e0d987SEric W. Biederman 	if (err)
1994b2e0d987SEric W. Biederman 		goto bad_unshare_cleanup_cred;
1995cf2e340fSJANAK DESAI 
1996b2e0d987SEric W. Biederman 	if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
19979edff4abSManfred Spraul 		if (do_sysvsem) {
19989edff4abSManfred Spraul 			/*
19999edff4abSManfred Spraul 			 * CLONE_SYSVSEM is equivalent to sys_exit().
20009edff4abSManfred Spraul 			 */
20019edff4abSManfred Spraul 			exit_sem(current);
20029edff4abSManfred Spraul 		}
2003ab602f79SJack Miller 		if (unshare_flags & CLONE_NEWIPC) {
2004ab602f79SJack Miller 			/* Orphan segments in old ns (see sem above). */
2005ab602f79SJack Miller 			exit_shm(current);
2006ab602f79SJack Miller 			shm_init_task(current);
2007ab602f79SJack Miller 		}
2008ab516013SSerge E. Hallyn 
20096f977e6bSAlan Cox 		if (new_nsproxy)
2010cf7b708cSPavel Emelyanov 			switch_task_namespaces(current, new_nsproxy);
2011cf2e340fSJANAK DESAI 
2012cf7b708cSPavel Emelyanov 		task_lock(current);
2013cf7b708cSPavel Emelyanov 
2014cf2e340fSJANAK DESAI 		if (new_fs) {
2015cf2e340fSJANAK DESAI 			fs = current->fs;
20162a4419b5SNick Piggin 			spin_lock(&fs->lock);
2017cf2e340fSJANAK DESAI 			current->fs = new_fs;
2018498052bbSAl Viro 			if (--fs->users)
2019498052bbSAl Viro 				new_fs = NULL;
2020498052bbSAl Viro 			else
2021cf2e340fSJANAK DESAI 				new_fs = fs;
20222a4419b5SNick Piggin 			spin_unlock(&fs->lock);
2023cf2e340fSJANAK DESAI 		}
2024cf2e340fSJANAK DESAI 
2025cf2e340fSJANAK DESAI 		if (new_fd) {
2026cf2e340fSJANAK DESAI 			fd = current->files;
2027cf2e340fSJANAK DESAI 			current->files = new_fd;
2028cf2e340fSJANAK DESAI 			new_fd = fd;
2029cf2e340fSJANAK DESAI 		}
2030cf2e340fSJANAK DESAI 
2031cf2e340fSJANAK DESAI 		task_unlock(current);
2032b2e0d987SEric W. Biederman 
2033b2e0d987SEric W. Biederman 		if (new_cred) {
2034b2e0d987SEric W. Biederman 			/* Install the new user namespace */
2035b2e0d987SEric W. Biederman 			commit_creds(new_cred);
2036b2e0d987SEric W. Biederman 			new_cred = NULL;
2037b2e0d987SEric W. Biederman 		}
2038cf2e340fSJANAK DESAI 	}
2039cf2e340fSJANAK DESAI 
2040b2e0d987SEric W. Biederman bad_unshare_cleanup_cred:
2041b2e0d987SEric W. Biederman 	if (new_cred)
2042b2e0d987SEric W. Biederman 		put_cred(new_cred);
2043cf2e340fSJANAK DESAI bad_unshare_cleanup_fd:
2044cf2e340fSJANAK DESAI 	if (new_fd)
2045cf2e340fSJANAK DESAI 		put_files_struct(new_fd);
2046cf2e340fSJANAK DESAI 
2047cf2e340fSJANAK DESAI bad_unshare_cleanup_fs:
2048cf2e340fSJANAK DESAI 	if (new_fs)
2049498052bbSAl Viro 		free_fs_struct(new_fs);
2050cf2e340fSJANAK DESAI 
2051cf2e340fSJANAK DESAI bad_unshare_out:
2052cf2e340fSJANAK DESAI 	return err;
2053cf2e340fSJANAK DESAI }
20543b125388SAl Viro 
20553b125388SAl Viro /*
20563b125388SAl Viro  *	Helper to unshare the files of the current task.
20573b125388SAl Viro  *	We don't want to expose copy_files internals to
20583b125388SAl Viro  *	the exec layer of the kernel.
20593b125388SAl Viro  */
20603b125388SAl Viro 
20613b125388SAl Viro int unshare_files(struct files_struct **displaced)
20623b125388SAl Viro {
20633b125388SAl Viro 	struct task_struct *task = current;
206450704516SAl Viro 	struct files_struct *copy = NULL;
20653b125388SAl Viro 	int error;
20663b125388SAl Viro 
20673b125388SAl Viro 	error = unshare_fd(CLONE_FILES, &copy);
20683b125388SAl Viro 	if (error || !copy) {
20693b125388SAl Viro 		*displaced = NULL;
20703b125388SAl Viro 		return error;
20713b125388SAl Viro 	}
20723b125388SAl Viro 	*displaced = task->files;
20733b125388SAl Viro 	task_lock(task);
20743b125388SAl Viro 	task->files = copy;
20753b125388SAl Viro 	task_unlock(task);
20763b125388SAl Viro 	return 0;
20773b125388SAl Viro }
207816db3d3fSHeinrich Schuchardt 
207916db3d3fSHeinrich Schuchardt int sysctl_max_threads(struct ctl_table *table, int write,
208016db3d3fSHeinrich Schuchardt 		       void __user *buffer, size_t *lenp, loff_t *ppos)
208116db3d3fSHeinrich Schuchardt {
208216db3d3fSHeinrich Schuchardt 	struct ctl_table t;
208316db3d3fSHeinrich Schuchardt 	int ret;
208416db3d3fSHeinrich Schuchardt 	int threads = max_threads;
208516db3d3fSHeinrich Schuchardt 	int min = MIN_THREADS;
208616db3d3fSHeinrich Schuchardt 	int max = MAX_THREADS;
208716db3d3fSHeinrich Schuchardt 
208816db3d3fSHeinrich Schuchardt 	t = *table;
208916db3d3fSHeinrich Schuchardt 	t.data = &threads;
209016db3d3fSHeinrich Schuchardt 	t.extra1 = &min;
209116db3d3fSHeinrich Schuchardt 	t.extra2 = &max;
209216db3d3fSHeinrich Schuchardt 
209316db3d3fSHeinrich Schuchardt 	ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
209416db3d3fSHeinrich Schuchardt 	if (ret || !write)
209516db3d3fSHeinrich Schuchardt 		return ret;
209616db3d3fSHeinrich Schuchardt 
209716db3d3fSHeinrich Schuchardt 	set_max_threads(threads);
209816db3d3fSHeinrich Schuchardt 
209916db3d3fSHeinrich Schuchardt 	return 0;
210016db3d3fSHeinrich Schuchardt }
2101