xref: /openbmc/linux/kernel/fork.c (revision 3e51e3edfd81bfd9853ad7de91167e4ce33d0fe7)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  *  linux/kernel/fork.c
31da177e4SLinus Torvalds  *
41da177e4SLinus Torvalds  *  Copyright (C) 1991, 1992  Linus Torvalds
51da177e4SLinus Torvalds  */
61da177e4SLinus Torvalds 
71da177e4SLinus Torvalds /*
81da177e4SLinus Torvalds  *  'fork.c' contains the help-routines for the 'fork' system call
91da177e4SLinus Torvalds  * (see also entry.S and others).
101da177e4SLinus Torvalds  * Fork is rather simple, once you get the hang of it, but the memory
111da177e4SLinus Torvalds  * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
121da177e4SLinus Torvalds  */
131da177e4SLinus Torvalds 
141da177e4SLinus Torvalds #include <linux/slab.h>
151da177e4SLinus Torvalds #include <linux/init.h>
161da177e4SLinus Torvalds #include <linux/unistd.h>
171da177e4SLinus Torvalds #include <linux/module.h>
181da177e4SLinus Torvalds #include <linux/vmalloc.h>
191da177e4SLinus Torvalds #include <linux/completion.h>
201da177e4SLinus Torvalds #include <linux/personality.h>
211da177e4SLinus Torvalds #include <linux/mempolicy.h>
221da177e4SLinus Torvalds #include <linux/sem.h>
231da177e4SLinus Torvalds #include <linux/file.h>
249f3acc31SAl Viro #include <linux/fdtable.h>
25da9cbc87SJens Axboe #include <linux/iocontext.h>
261da177e4SLinus Torvalds #include <linux/key.h>
271da177e4SLinus Torvalds #include <linux/binfmts.h>
281da177e4SLinus Torvalds #include <linux/mman.h>
29cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h>
301da177e4SLinus Torvalds #include <linux/fs.h>
31ab516013SSerge E. Hallyn #include <linux/nsproxy.h>
32c59ede7bSRandy.Dunlap #include <linux/capability.h>
331da177e4SLinus Torvalds #include <linux/cpu.h>
34b4f48b63SPaul Menage #include <linux/cgroup.h>
351da177e4SLinus Torvalds #include <linux/security.h>
36a1e78772SMel Gorman #include <linux/hugetlb.h>
371da177e4SLinus Torvalds #include <linux/swap.h>
381da177e4SLinus Torvalds #include <linux/syscalls.h>
391da177e4SLinus Torvalds #include <linux/jiffies.h>
4009a05394SRoland McGrath #include <linux/tracehook.h>
411da177e4SLinus Torvalds #include <linux/futex.h>
428141c7f3SLinus Torvalds #include <linux/compat.h>
43207205a2SEric Dumazet #include <linux/kthread.h>
447c3ab738SAndrew Morton #include <linux/task_io_accounting_ops.h>
45ab2af1f5SDipankar Sarma #include <linux/rcupdate.h>
461da177e4SLinus Torvalds #include <linux/ptrace.h>
471da177e4SLinus Torvalds #include <linux/mount.h>
481da177e4SLinus Torvalds #include <linux/audit.h>
4978fb7466SPavel Emelianov #include <linux/memcontrol.h>
50f201ae23SFrederic Weisbecker #include <linux/ftrace.h>
511da177e4SLinus Torvalds #include <linux/profile.h>
521da177e4SLinus Torvalds #include <linux/rmap.h>
53f8af4da3SHugh Dickins #include <linux/ksm.h>
541da177e4SLinus Torvalds #include <linux/acct.h>
558f0ab514SJay Lan #include <linux/tsacct_kern.h>
569f46080cSMatt Helsley #include <linux/cn_proc.h>
57ba96a0c8SRafael J. Wysocki #include <linux/freezer.h>
58ca74e92bSShailabh Nagar #include <linux/delayacct.h>
59ad4ecbcbSShailabh Nagar #include <linux/taskstats_kern.h>
600a425405SArjan van de Ven #include <linux/random.h>
61522ed776SMiloslav Trmac #include <linux/tty.h>
626f4e6433SPavel Emelyanov #include <linux/proc_fs.h>
63fd0928dfSJens Axboe #include <linux/blkdev.h>
645ad4e53bSAl Viro #include <linux/fs_struct.h>
657c9f8861SEric Sandeen #include <linux/magic.h>
66cdd6c482SIngo Molnar #include <linux/perf_event.h>
6742c4ab41SStanislaw Gruszka #include <linux/posix-timers.h>
688e7cac79SAvi Kivity #include <linux/user-return-notifier.h>
693d5992d2SYing Han #include <linux/oom.h>
70ba76149fSAndrea Arcangeli #include <linux/khugepaged.h>
711da177e4SLinus Torvalds 
721da177e4SLinus Torvalds #include <asm/pgtable.h>
731da177e4SLinus Torvalds #include <asm/pgalloc.h>
741da177e4SLinus Torvalds #include <asm/uaccess.h>
751da177e4SLinus Torvalds #include <asm/mmu_context.h>
761da177e4SLinus Torvalds #include <asm/cacheflush.h>
771da177e4SLinus Torvalds #include <asm/tlbflush.h>
781da177e4SLinus Torvalds 
79ad8d75ffSSteven Rostedt #include <trace/events/sched.h>
80ad8d75ffSSteven Rostedt 
811da177e4SLinus Torvalds /*
821da177e4SLinus Torvalds  * Protected counters by write_lock_irq(&tasklist_lock)
831da177e4SLinus Torvalds  */
841da177e4SLinus Torvalds unsigned long total_forks;	/* Handle normal Linux uptimes. */
851da177e4SLinus Torvalds int nr_threads; 		/* The idle threads do not count.. */
861da177e4SLinus Torvalds 
871da177e4SLinus Torvalds int max_threads;		/* tunable limit on nr_threads */
881da177e4SLinus Torvalds 
891da177e4SLinus Torvalds DEFINE_PER_CPU(unsigned long, process_counts) = 0;
901da177e4SLinus Torvalds 
911da177e4SLinus Torvalds __cacheline_aligned DEFINE_RWLOCK(tasklist_lock);  /* outer */
92db1466b3SPaul E. McKenney 
93db1466b3SPaul E. McKenney #ifdef CONFIG_PROVE_RCU
94db1466b3SPaul E. McKenney int lockdep_tasklist_lock_is_held(void)
95db1466b3SPaul E. McKenney {
96db1466b3SPaul E. McKenney 	return lockdep_is_held(&tasklist_lock);
97db1466b3SPaul E. McKenney }
98db1466b3SPaul E. McKenney EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
99db1466b3SPaul E. McKenney #endif /* #ifdef CONFIG_PROVE_RCU */
1001da177e4SLinus Torvalds 
1011da177e4SLinus Torvalds int nr_processes(void)
1021da177e4SLinus Torvalds {
1031da177e4SLinus Torvalds 	int cpu;
1041da177e4SLinus Torvalds 	int total = 0;
1051da177e4SLinus Torvalds 
1061d510750SIan Campbell 	for_each_possible_cpu(cpu)
1071da177e4SLinus Torvalds 		total += per_cpu(process_counts, cpu);
1081da177e4SLinus Torvalds 
1091da177e4SLinus Torvalds 	return total;
1101da177e4SLinus Torvalds }
1111da177e4SLinus Torvalds 
1121da177e4SLinus Torvalds #ifndef __HAVE_ARCH_TASK_STRUCT_ALLOCATOR
113504f52b5SEric Dumazet # define alloc_task_struct_node(node)		\
114504f52b5SEric Dumazet 		kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node)
115504f52b5SEric Dumazet # define free_task_struct(tsk)			\
116504f52b5SEric Dumazet 		kmem_cache_free(task_struct_cachep, (tsk))
117e18b890bSChristoph Lameter static struct kmem_cache *task_struct_cachep;
1181da177e4SLinus Torvalds #endif
1191da177e4SLinus Torvalds 
120b69c49b7SFUJITA Tomonori #ifndef __HAVE_ARCH_THREAD_INFO_ALLOCATOR
121b6a84016SEric Dumazet static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
122b6a84016SEric Dumazet 						  int node)
123b69c49b7SFUJITA Tomonori {
124b69c49b7SFUJITA Tomonori #ifdef CONFIG_DEBUG_STACK_USAGE
125b69c49b7SFUJITA Tomonori 	gfp_t mask = GFP_KERNEL | __GFP_ZERO;
126b69c49b7SFUJITA Tomonori #else
127b69c49b7SFUJITA Tomonori 	gfp_t mask = GFP_KERNEL;
128b69c49b7SFUJITA Tomonori #endif
129b6a84016SEric Dumazet 	struct page *page = alloc_pages_node(node, mask, THREAD_SIZE_ORDER);
130b6a84016SEric Dumazet 
131b6a84016SEric Dumazet 	return page ? page_address(page) : NULL;
132b69c49b7SFUJITA Tomonori }
133b69c49b7SFUJITA Tomonori 
134b69c49b7SFUJITA Tomonori static inline void free_thread_info(struct thread_info *ti)
135b69c49b7SFUJITA Tomonori {
136b69c49b7SFUJITA Tomonori 	free_pages((unsigned long)ti, THREAD_SIZE_ORDER);
137b69c49b7SFUJITA Tomonori }
138b69c49b7SFUJITA Tomonori #endif
139b69c49b7SFUJITA Tomonori 
1401da177e4SLinus Torvalds /* SLAB cache for signal_struct structures (tsk->signal) */
141e18b890bSChristoph Lameter static struct kmem_cache *signal_cachep;
1421da177e4SLinus Torvalds 
1431da177e4SLinus Torvalds /* SLAB cache for sighand_struct structures (tsk->sighand) */
144e18b890bSChristoph Lameter struct kmem_cache *sighand_cachep;
1451da177e4SLinus Torvalds 
1461da177e4SLinus Torvalds /* SLAB cache for files_struct structures (tsk->files) */
147e18b890bSChristoph Lameter struct kmem_cache *files_cachep;
1481da177e4SLinus Torvalds 
1491da177e4SLinus Torvalds /* SLAB cache for fs_struct structures (tsk->fs) */
150e18b890bSChristoph Lameter struct kmem_cache *fs_cachep;
1511da177e4SLinus Torvalds 
1521da177e4SLinus Torvalds /* SLAB cache for vm_area_struct structures */
153e18b890bSChristoph Lameter struct kmem_cache *vm_area_cachep;
1541da177e4SLinus Torvalds 
1551da177e4SLinus Torvalds /* SLAB cache for mm_struct structures (tsk->mm) */
156e18b890bSChristoph Lameter static struct kmem_cache *mm_cachep;
1571da177e4SLinus Torvalds 
158c6a7f572SKOSAKI Motohiro static void account_kernel_stack(struct thread_info *ti, int account)
159c6a7f572SKOSAKI Motohiro {
160c6a7f572SKOSAKI Motohiro 	struct zone *zone = page_zone(virt_to_page(ti));
161c6a7f572SKOSAKI Motohiro 
162c6a7f572SKOSAKI Motohiro 	mod_zone_page_state(zone, NR_KERNEL_STACK, account);
163c6a7f572SKOSAKI Motohiro }
164c6a7f572SKOSAKI Motohiro 
1651da177e4SLinus Torvalds void free_task(struct task_struct *tsk)
1661da177e4SLinus Torvalds {
1673e26c149SPeter Zijlstra 	prop_local_destroy_single(&tsk->dirties);
168c6a7f572SKOSAKI Motohiro 	account_kernel_stack(tsk->stack, -1);
169f7e4217bSRoman Zippel 	free_thread_info(tsk->stack);
17023f78d4aSIngo Molnar 	rt_mutex_debug_task_free(tsk);
171fb52607aSFrederic Weisbecker 	ftrace_graph_exit_task(tsk);
1721da177e4SLinus Torvalds 	free_task_struct(tsk);
1731da177e4SLinus Torvalds }
1741da177e4SLinus Torvalds EXPORT_SYMBOL(free_task);
1751da177e4SLinus Torvalds 
176ea6d290cSOleg Nesterov static inline void free_signal_struct(struct signal_struct *sig)
177ea6d290cSOleg Nesterov {
17897101eb4SOleg Nesterov 	taskstats_tgid_free(sig);
1791c5354deSMike Galbraith 	sched_autogroup_exit(sig);
180ea6d290cSOleg Nesterov 	kmem_cache_free(signal_cachep, sig);
181ea6d290cSOleg Nesterov }
182ea6d290cSOleg Nesterov 
183ea6d290cSOleg Nesterov static inline void put_signal_struct(struct signal_struct *sig)
184ea6d290cSOleg Nesterov {
1851c5354deSMike Galbraith 	if (atomic_dec_and_test(&sig->sigcnt))
186ea6d290cSOleg Nesterov 		free_signal_struct(sig);
187ea6d290cSOleg Nesterov }
188ea6d290cSOleg Nesterov 
189158d9ebdSAndrew Morton void __put_task_struct(struct task_struct *tsk)
1901da177e4SLinus Torvalds {
191270f722dSEugene Teo 	WARN_ON(!tsk->exit_state);
1921da177e4SLinus Torvalds 	WARN_ON(atomic_read(&tsk->usage));
1931da177e4SLinus Torvalds 	WARN_ON(tsk == current);
1941da177e4SLinus Torvalds 
195e0e81739SDavid Howells 	exit_creds(tsk);
19635df17c5SShailabh Nagar 	delayacct_tsk_free(tsk);
197ea6d290cSOleg Nesterov 	put_signal_struct(tsk->signal);
1981da177e4SLinus Torvalds 
1991da177e4SLinus Torvalds 	if (!profile_handoff_task(tsk))
2001da177e4SLinus Torvalds 		free_task(tsk);
2011da177e4SLinus Torvalds }
20277c100c8SRik van Riel EXPORT_SYMBOL_GPL(__put_task_struct);
2031da177e4SLinus Torvalds 
2042adee9b3SSuresh Siddha /*
2052adee9b3SSuresh Siddha  * macro override instead of weak attribute alias, to workaround
2062adee9b3SSuresh Siddha  * gcc 4.1.0 and 4.1.1 bugs with weak attribute and empty functions.
2072adee9b3SSuresh Siddha  */
2082adee9b3SSuresh Siddha #ifndef arch_task_cache_init
2092adee9b3SSuresh Siddha #define arch_task_cache_init()
2102adee9b3SSuresh Siddha #endif
21161c4628bSSuresh Siddha 
2121da177e4SLinus Torvalds void __init fork_init(unsigned long mempages)
2131da177e4SLinus Torvalds {
2141da177e4SLinus Torvalds #ifndef __HAVE_ARCH_TASK_STRUCT_ALLOCATOR
2151da177e4SLinus Torvalds #ifndef ARCH_MIN_TASKALIGN
2161da177e4SLinus Torvalds #define ARCH_MIN_TASKALIGN	L1_CACHE_BYTES
2171da177e4SLinus Torvalds #endif
2181da177e4SLinus Torvalds 	/* create a slab on which task_structs can be allocated */
2191da177e4SLinus Torvalds 	task_struct_cachep =
2201da177e4SLinus Torvalds 		kmem_cache_create("task_struct", sizeof(struct task_struct),
2212dff4405SVegard Nossum 			ARCH_MIN_TASKALIGN, SLAB_PANIC | SLAB_NOTRACK, NULL);
2221da177e4SLinus Torvalds #endif
2231da177e4SLinus Torvalds 
22461c4628bSSuresh Siddha 	/* do the arch specific task caches init */
22561c4628bSSuresh Siddha 	arch_task_cache_init();
22661c4628bSSuresh Siddha 
2271da177e4SLinus Torvalds 	/*
2281da177e4SLinus Torvalds 	 * The default maximum number of threads is set to a safe
2291da177e4SLinus Torvalds 	 * value: the thread structures can take up at most half
2301da177e4SLinus Torvalds 	 * of memory.
2311da177e4SLinus Torvalds 	 */
2321da177e4SLinus Torvalds 	max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE);
2331da177e4SLinus Torvalds 
2341da177e4SLinus Torvalds 	/*
2351da177e4SLinus Torvalds 	 * we need to allow at least 20 threads to boot a system
2361da177e4SLinus Torvalds 	 */
2371da177e4SLinus Torvalds 	if(max_threads < 20)
2381da177e4SLinus Torvalds 		max_threads = 20;
2391da177e4SLinus Torvalds 
2401da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
2411da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
2421da177e4SLinus Torvalds 	init_task.signal->rlim[RLIMIT_SIGPENDING] =
2431da177e4SLinus Torvalds 		init_task.signal->rlim[RLIMIT_NPROC];
2441da177e4SLinus Torvalds }
2451da177e4SLinus Torvalds 
24661c4628bSSuresh Siddha int __attribute__((weak)) arch_dup_task_struct(struct task_struct *dst,
24761c4628bSSuresh Siddha 					       struct task_struct *src)
24861c4628bSSuresh Siddha {
24961c4628bSSuresh Siddha 	*dst = *src;
25061c4628bSSuresh Siddha 	return 0;
25161c4628bSSuresh Siddha }
25261c4628bSSuresh Siddha 
2531da177e4SLinus Torvalds static struct task_struct *dup_task_struct(struct task_struct *orig)
2541da177e4SLinus Torvalds {
2551da177e4SLinus Torvalds 	struct task_struct *tsk;
2561da177e4SLinus Torvalds 	struct thread_info *ti;
2577c9f8861SEric Sandeen 	unsigned long *stackend;
258207205a2SEric Dumazet 	int node = tsk_fork_get_node(orig);
2593e26c149SPeter Zijlstra 	int err;
2601da177e4SLinus Torvalds 
2611da177e4SLinus Torvalds 	prepare_to_copy(orig);
2621da177e4SLinus Torvalds 
263504f52b5SEric Dumazet 	tsk = alloc_task_struct_node(node);
2641da177e4SLinus Torvalds 	if (!tsk)
2651da177e4SLinus Torvalds 		return NULL;
2661da177e4SLinus Torvalds 
267b6a84016SEric Dumazet 	ti = alloc_thread_info_node(tsk, node);
2681da177e4SLinus Torvalds 	if (!ti) {
2691da177e4SLinus Torvalds 		free_task_struct(tsk);
2701da177e4SLinus Torvalds 		return NULL;
2711da177e4SLinus Torvalds 	}
2721da177e4SLinus Torvalds 
27361c4628bSSuresh Siddha  	err = arch_dup_task_struct(tsk, orig);
27461c4628bSSuresh Siddha 	if (err)
27561c4628bSSuresh Siddha 		goto out;
27661c4628bSSuresh Siddha 
277f7e4217bSRoman Zippel 	tsk->stack = ti;
2783e26c149SPeter Zijlstra 
2793e26c149SPeter Zijlstra 	err = prop_local_init_single(&tsk->dirties);
28061c4628bSSuresh Siddha 	if (err)
28161c4628bSSuresh Siddha 		goto out;
2823e26c149SPeter Zijlstra 
28310ebffdeSAl Viro 	setup_thread_stack(tsk, orig);
2848e7cac79SAvi Kivity 	clear_user_return_notifier(tsk);
285f26f9affSMike Galbraith 	clear_tsk_need_resched(tsk);
2867c9f8861SEric Sandeen 	stackend = end_of_stack(tsk);
2877c9f8861SEric Sandeen 	*stackend = STACK_END_MAGIC;	/* for overflow detection */
2881da177e4SLinus Torvalds 
2890a425405SArjan van de Ven #ifdef CONFIG_CC_STACKPROTECTOR
2900a425405SArjan van de Ven 	tsk->stack_canary = get_random_int();
2910a425405SArjan van de Ven #endif
2920a425405SArjan van de Ven 
2931da177e4SLinus Torvalds 	/* One for us, one for whoever does the "release_task()" (usually parent) */
2941da177e4SLinus Torvalds 	atomic_set(&tsk->usage,2);
2954b5d37acSGiancarlo Formicuccia 	atomic_set(&tsk->fs_excl, 0);
2966c5c9341SAlexey Dobriyan #ifdef CONFIG_BLK_DEV_IO_TRACE
2972056a782SJens Axboe 	tsk->btrace_seq = 0;
2986c5c9341SAlexey Dobriyan #endif
299a0aa7f68SJens Axboe 	tsk->splice_pipe = NULL;
300c6a7f572SKOSAKI Motohiro 
301c6a7f572SKOSAKI Motohiro 	account_kernel_stack(ti, 1);
302c6a7f572SKOSAKI Motohiro 
3031da177e4SLinus Torvalds 	return tsk;
30461c4628bSSuresh Siddha 
30561c4628bSSuresh Siddha out:
30661c4628bSSuresh Siddha 	free_thread_info(ti);
30761c4628bSSuresh Siddha 	free_task_struct(tsk);
30861c4628bSSuresh Siddha 	return NULL;
3091da177e4SLinus Torvalds }
3101da177e4SLinus Torvalds 
3111da177e4SLinus Torvalds #ifdef CONFIG_MMU
312a39bc516SAlexey Dobriyan static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
3131da177e4SLinus Torvalds {
314297c5eeeSLinus Torvalds 	struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
3151da177e4SLinus Torvalds 	struct rb_node **rb_link, *rb_parent;
3161da177e4SLinus Torvalds 	int retval;
3171da177e4SLinus Torvalds 	unsigned long charge;
3181da177e4SLinus Torvalds 	struct mempolicy *pol;
3191da177e4SLinus Torvalds 
3201da177e4SLinus Torvalds 	down_write(&oldmm->mmap_sem);
321ec8c0446SRalf Baechle 	flush_cache_dup_mm(oldmm);
322ad339451SIngo Molnar 	/*
323ad339451SIngo Molnar 	 * Not linked in yet - no deadlock potential:
324ad339451SIngo Molnar 	 */
325ad339451SIngo Molnar 	down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
3267ee78232SHugh Dickins 
3271da177e4SLinus Torvalds 	mm->locked_vm = 0;
3281da177e4SLinus Torvalds 	mm->mmap = NULL;
3291da177e4SLinus Torvalds 	mm->mmap_cache = NULL;
3301da177e4SLinus Torvalds 	mm->free_area_cache = oldmm->mmap_base;
3311363c3cdSWolfgang Wander 	mm->cached_hole_size = ~0UL;
3321da177e4SLinus Torvalds 	mm->map_count = 0;
33394894244SRusty Russell 	cpumask_clear(mm_cpumask(mm));
3341da177e4SLinus Torvalds 	mm->mm_rb = RB_ROOT;
3351da177e4SLinus Torvalds 	rb_link = &mm->mm_rb.rb_node;
3361da177e4SLinus Torvalds 	rb_parent = NULL;
3371da177e4SLinus Torvalds 	pprev = &mm->mmap;
338f8af4da3SHugh Dickins 	retval = ksm_fork(mm, oldmm);
339f8af4da3SHugh Dickins 	if (retval)
340f8af4da3SHugh Dickins 		goto out;
341ba76149fSAndrea Arcangeli 	retval = khugepaged_fork(mm, oldmm);
342ba76149fSAndrea Arcangeli 	if (retval)
343ba76149fSAndrea Arcangeli 		goto out;
3441da177e4SLinus Torvalds 
345297c5eeeSLinus Torvalds 	prev = NULL;
346fd3e42fcSHugh Dickins 	for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
3471da177e4SLinus Torvalds 		struct file *file;
3481da177e4SLinus Torvalds 
3491da177e4SLinus Torvalds 		if (mpnt->vm_flags & VM_DONTCOPY) {
3503b6bfcdbSHugh Dickins 			long pages = vma_pages(mpnt);
3513b6bfcdbSHugh Dickins 			mm->total_vm -= pages;
352ab50b8edSHugh Dickins 			vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
3533b6bfcdbSHugh Dickins 								-pages);
3541da177e4SLinus Torvalds 			continue;
3551da177e4SLinus Torvalds 		}
3561da177e4SLinus Torvalds 		charge = 0;
3571da177e4SLinus Torvalds 		if (mpnt->vm_flags & VM_ACCOUNT) {
3581da177e4SLinus Torvalds 			unsigned int len = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT;
3591da177e4SLinus Torvalds 			if (security_vm_enough_memory(len))
3601da177e4SLinus Torvalds 				goto fail_nomem;
3611da177e4SLinus Torvalds 			charge = len;
3621da177e4SLinus Torvalds 		}
363e94b1766SChristoph Lameter 		tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
3641da177e4SLinus Torvalds 		if (!tmp)
3651da177e4SLinus Torvalds 			goto fail_nomem;
3661da177e4SLinus Torvalds 		*tmp = *mpnt;
3675beb4930SRik van Riel 		INIT_LIST_HEAD(&tmp->anon_vma_chain);
368846a16bfSLee Schermerhorn 		pol = mpol_dup(vma_policy(mpnt));
3691da177e4SLinus Torvalds 		retval = PTR_ERR(pol);
3701da177e4SLinus Torvalds 		if (IS_ERR(pol))
3711da177e4SLinus Torvalds 			goto fail_nomem_policy;
3721da177e4SLinus Torvalds 		vma_set_policy(tmp, pol);
373a247c3a9SAndrea Arcangeli 		tmp->vm_mm = mm;
3745beb4930SRik van Riel 		if (anon_vma_fork(tmp, mpnt))
3755beb4930SRik van Riel 			goto fail_nomem_anon_vma_fork;
3761da177e4SLinus Torvalds 		tmp->vm_flags &= ~VM_LOCKED;
377297c5eeeSLinus Torvalds 		tmp->vm_next = tmp->vm_prev = NULL;
3781da177e4SLinus Torvalds 		file = tmp->vm_file;
3791da177e4SLinus Torvalds 		if (file) {
380f3a43f3fSJosef "Jeff" Sipek 			struct inode *inode = file->f_path.dentry->d_inode;
381b88ed205SHugh Dickins 			struct address_space *mapping = file->f_mapping;
382b88ed205SHugh Dickins 
3831da177e4SLinus Torvalds 			get_file(file);
3841da177e4SLinus Torvalds 			if (tmp->vm_flags & VM_DENYWRITE)
3851da177e4SLinus Torvalds 				atomic_dec(&inode->i_writecount);
386b88ed205SHugh Dickins 			spin_lock(&mapping->i_mmap_lock);
387b88ed205SHugh Dickins 			if (tmp->vm_flags & VM_SHARED)
388b88ed205SHugh Dickins 				mapping->i_mmap_writable++;
3891da177e4SLinus Torvalds 			tmp->vm_truncate_count = mpnt->vm_truncate_count;
390b88ed205SHugh Dickins 			flush_dcache_mmap_lock(mapping);
391b88ed205SHugh Dickins 			/* insert tmp into the share list, just after mpnt */
3921da177e4SLinus Torvalds 			vma_prio_tree_add(tmp, mpnt);
393b88ed205SHugh Dickins 			flush_dcache_mmap_unlock(mapping);
394b88ed205SHugh Dickins 			spin_unlock(&mapping->i_mmap_lock);
3951da177e4SLinus Torvalds 		}
3961da177e4SLinus Torvalds 
3971da177e4SLinus Torvalds 		/*
398a1e78772SMel Gorman 		 * Clear hugetlb-related page reserves for children. This only
399a1e78772SMel Gorman 		 * affects MAP_PRIVATE mappings. Faults generated by the child
400a1e78772SMel Gorman 		 * are not guaranteed to succeed, even if read-only
401a1e78772SMel Gorman 		 */
402a1e78772SMel Gorman 		if (is_vm_hugetlb_page(tmp))
403a1e78772SMel Gorman 			reset_vma_resv_huge_pages(tmp);
404a1e78772SMel Gorman 
405a1e78772SMel Gorman 		/*
4067ee78232SHugh Dickins 		 * Link in the new vma and copy the page table entries.
4071da177e4SLinus Torvalds 		 */
4081da177e4SLinus Torvalds 		*pprev = tmp;
4091da177e4SLinus Torvalds 		pprev = &tmp->vm_next;
410297c5eeeSLinus Torvalds 		tmp->vm_prev = prev;
411297c5eeeSLinus Torvalds 		prev = tmp;
4121da177e4SLinus Torvalds 
4131da177e4SLinus Torvalds 		__vma_link_rb(mm, tmp, rb_link, rb_parent);
4141da177e4SLinus Torvalds 		rb_link = &tmp->vm_rb.rb_right;
4151da177e4SLinus Torvalds 		rb_parent = &tmp->vm_rb;
4161da177e4SLinus Torvalds 
4171da177e4SLinus Torvalds 		mm->map_count++;
4180b0db14cSHugh Dickins 		retval = copy_page_range(mm, oldmm, mpnt);
4191da177e4SLinus Torvalds 
4201da177e4SLinus Torvalds 		if (tmp->vm_ops && tmp->vm_ops->open)
4211da177e4SLinus Torvalds 			tmp->vm_ops->open(tmp);
4221da177e4SLinus Torvalds 
4231da177e4SLinus Torvalds 		if (retval)
4241da177e4SLinus Torvalds 			goto out;
4251da177e4SLinus Torvalds 	}
426d6dd61c8SJeremy Fitzhardinge 	/* a new mm has just been created */
427d6dd61c8SJeremy Fitzhardinge 	arch_dup_mmap(oldmm, mm);
4281da177e4SLinus Torvalds 	retval = 0;
4291da177e4SLinus Torvalds out:
4307ee78232SHugh Dickins 	up_write(&mm->mmap_sem);
431fd3e42fcSHugh Dickins 	flush_tlb_mm(oldmm);
4321da177e4SLinus Torvalds 	up_write(&oldmm->mmap_sem);
4331da177e4SLinus Torvalds 	return retval;
4345beb4930SRik van Riel fail_nomem_anon_vma_fork:
4355beb4930SRik van Riel 	mpol_put(pol);
4361da177e4SLinus Torvalds fail_nomem_policy:
4371da177e4SLinus Torvalds 	kmem_cache_free(vm_area_cachep, tmp);
4381da177e4SLinus Torvalds fail_nomem:
4391da177e4SLinus Torvalds 	retval = -ENOMEM;
4401da177e4SLinus Torvalds 	vm_unacct_memory(charge);
4411da177e4SLinus Torvalds 	goto out;
4421da177e4SLinus Torvalds }
4431da177e4SLinus Torvalds 
4441da177e4SLinus Torvalds static inline int mm_alloc_pgd(struct mm_struct * mm)
4451da177e4SLinus Torvalds {
4461da177e4SLinus Torvalds 	mm->pgd = pgd_alloc(mm);
4471da177e4SLinus Torvalds 	if (unlikely(!mm->pgd))
4481da177e4SLinus Torvalds 		return -ENOMEM;
4491da177e4SLinus Torvalds 	return 0;
4501da177e4SLinus Torvalds }
4511da177e4SLinus Torvalds 
4521da177e4SLinus Torvalds static inline void mm_free_pgd(struct mm_struct * mm)
4531da177e4SLinus Torvalds {
4545e541973SBenjamin Herrenschmidt 	pgd_free(mm, mm->pgd);
4551da177e4SLinus Torvalds }
4561da177e4SLinus Torvalds #else
4571da177e4SLinus Torvalds #define dup_mmap(mm, oldmm)	(0)
4581da177e4SLinus Torvalds #define mm_alloc_pgd(mm)	(0)
4591da177e4SLinus Torvalds #define mm_free_pgd(mm)
4601da177e4SLinus Torvalds #endif /* CONFIG_MMU */
4611da177e4SLinus Torvalds 
4621da177e4SLinus Torvalds __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
4631da177e4SLinus Torvalds 
464e94b1766SChristoph Lameter #define allocate_mm()	(kmem_cache_alloc(mm_cachep, GFP_KERNEL))
4651da177e4SLinus Torvalds #define free_mm(mm)	(kmem_cache_free(mm_cachep, (mm)))
4661da177e4SLinus Torvalds 
4674cb0e11bSHidehiro Kawai static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
4684cb0e11bSHidehiro Kawai 
4694cb0e11bSHidehiro Kawai static int __init coredump_filter_setup(char *s)
4704cb0e11bSHidehiro Kawai {
4714cb0e11bSHidehiro Kawai 	default_dump_filter =
4724cb0e11bSHidehiro Kawai 		(simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
4734cb0e11bSHidehiro Kawai 		MMF_DUMP_FILTER_MASK;
4744cb0e11bSHidehiro Kawai 	return 1;
4754cb0e11bSHidehiro Kawai }
4764cb0e11bSHidehiro Kawai 
4774cb0e11bSHidehiro Kawai __setup("coredump_filter=", coredump_filter_setup);
4784cb0e11bSHidehiro Kawai 
4791da177e4SLinus Torvalds #include <linux/init_task.h>
4801da177e4SLinus Torvalds 
481858f0993SAlexey Dobriyan static void mm_init_aio(struct mm_struct *mm)
482858f0993SAlexey Dobriyan {
483858f0993SAlexey Dobriyan #ifdef CONFIG_AIO
484858f0993SAlexey Dobriyan 	spin_lock_init(&mm->ioctx_lock);
485858f0993SAlexey Dobriyan 	INIT_HLIST_HEAD(&mm->ioctx_list);
486858f0993SAlexey Dobriyan #endif
487858f0993SAlexey Dobriyan }
488858f0993SAlexey Dobriyan 
48978fb7466SPavel Emelianov static struct mm_struct * mm_init(struct mm_struct * mm, struct task_struct *p)
4901da177e4SLinus Torvalds {
4911da177e4SLinus Torvalds 	atomic_set(&mm->mm_users, 1);
4921da177e4SLinus Torvalds 	atomic_set(&mm->mm_count, 1);
4931da177e4SLinus Torvalds 	init_rwsem(&mm->mmap_sem);
4941da177e4SLinus Torvalds 	INIT_LIST_HEAD(&mm->mmlist);
495f8af4da3SHugh Dickins 	mm->flags = (current->mm) ?
496f8af4da3SHugh Dickins 		(current->mm->flags & MMF_INIT_MASK) : default_dump_filter;
497999d9fc1SOleg Nesterov 	mm->core_state = NULL;
4981da177e4SLinus Torvalds 	mm->nr_ptes = 0;
499d559db08SKAMEZAWA Hiroyuki 	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
5001da177e4SLinus Torvalds 	spin_lock_init(&mm->page_table_lock);
5011da177e4SLinus Torvalds 	mm->free_area_cache = TASK_UNMAPPED_BASE;
5021363c3cdSWolfgang Wander 	mm->cached_hole_size = ~0UL;
503858f0993SAlexey Dobriyan 	mm_init_aio(mm);
504cf475ad2SBalbir Singh 	mm_init_owner(mm, p);
5053d5992d2SYing Han 	atomic_set(&mm->oom_disable_count, 0);
5061da177e4SLinus Torvalds 
5071da177e4SLinus Torvalds 	if (likely(!mm_alloc_pgd(mm))) {
5081da177e4SLinus Torvalds 		mm->def_flags = 0;
509cddb8a5cSAndrea Arcangeli 		mmu_notifier_mm_init(mm);
5101da177e4SLinus Torvalds 		return mm;
5111da177e4SLinus Torvalds 	}
51278fb7466SPavel Emelianov 
5131da177e4SLinus Torvalds 	free_mm(mm);
5141da177e4SLinus Torvalds 	return NULL;
5151da177e4SLinus Torvalds }
5161da177e4SLinus Torvalds 
5171da177e4SLinus Torvalds /*
5181da177e4SLinus Torvalds  * Allocate and initialize an mm_struct.
5191da177e4SLinus Torvalds  */
5201da177e4SLinus Torvalds struct mm_struct * mm_alloc(void)
5211da177e4SLinus Torvalds {
5221da177e4SLinus Torvalds 	struct mm_struct * mm;
5231da177e4SLinus Torvalds 
5241da177e4SLinus Torvalds 	mm = allocate_mm();
5251da177e4SLinus Torvalds 	if (mm) {
5261da177e4SLinus Torvalds 		memset(mm, 0, sizeof(*mm));
52778fb7466SPavel Emelianov 		mm = mm_init(mm, current);
5281da177e4SLinus Torvalds 	}
5291da177e4SLinus Torvalds 	return mm;
5301da177e4SLinus Torvalds }
5311da177e4SLinus Torvalds 
5321da177e4SLinus Torvalds /*
5331da177e4SLinus Torvalds  * Called when the last reference to the mm
5341da177e4SLinus Torvalds  * is dropped: either by a lazy thread or by
5351da177e4SLinus Torvalds  * mmput. Free the page directory and the mm.
5361da177e4SLinus Torvalds  */
5377ad5b3a5SHarvey Harrison void __mmdrop(struct mm_struct *mm)
5381da177e4SLinus Torvalds {
5391da177e4SLinus Torvalds 	BUG_ON(mm == &init_mm);
5401da177e4SLinus Torvalds 	mm_free_pgd(mm);
5411da177e4SLinus Torvalds 	destroy_context(mm);
542cddb8a5cSAndrea Arcangeli 	mmu_notifier_mm_destroy(mm);
543e7a00c45SAndrea Arcangeli #ifdef CONFIG_TRANSPARENT_HUGEPAGE
544e7a00c45SAndrea Arcangeli 	VM_BUG_ON(mm->pmd_huge_pte);
545e7a00c45SAndrea Arcangeli #endif
5461da177e4SLinus Torvalds 	free_mm(mm);
5471da177e4SLinus Torvalds }
5486d4e4c4fSAvi Kivity EXPORT_SYMBOL_GPL(__mmdrop);
5491da177e4SLinus Torvalds 
5501da177e4SLinus Torvalds /*
5511da177e4SLinus Torvalds  * Decrement the use count and release all resources for an mm.
5521da177e4SLinus Torvalds  */
5531da177e4SLinus Torvalds void mmput(struct mm_struct *mm)
5541da177e4SLinus Torvalds {
5550ae26f1bSAndrew Morton 	might_sleep();
5560ae26f1bSAndrew Morton 
5571da177e4SLinus Torvalds 	if (atomic_dec_and_test(&mm->mm_users)) {
5581da177e4SLinus Torvalds 		exit_aio(mm);
5591c2fb7a4SAndrea Arcangeli 		ksm_exit(mm);
560ba76149fSAndrea Arcangeli 		khugepaged_exit(mm); /* must run before exit_mmap */
5611da177e4SLinus Torvalds 		exit_mmap(mm);
562925d1c40SMatt Helsley 		set_mm_exe_file(mm, NULL);
5631da177e4SLinus Torvalds 		if (!list_empty(&mm->mmlist)) {
5641da177e4SLinus Torvalds 			spin_lock(&mmlist_lock);
5651da177e4SLinus Torvalds 			list_del(&mm->mmlist);
5661da177e4SLinus Torvalds 			spin_unlock(&mmlist_lock);
5671da177e4SLinus Torvalds 		}
5681da177e4SLinus Torvalds 		put_swap_token(mm);
569801460d0SHiroshi Shimamoto 		if (mm->binfmt)
570801460d0SHiroshi Shimamoto 			module_put(mm->binfmt->module);
5711da177e4SLinus Torvalds 		mmdrop(mm);
5721da177e4SLinus Torvalds 	}
5731da177e4SLinus Torvalds }
5741da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(mmput);
5751da177e4SLinus Torvalds 
5761da177e4SLinus Torvalds /**
5771da177e4SLinus Torvalds  * get_task_mm - acquire a reference to the task's mm
5781da177e4SLinus Torvalds  *
579246bb0b1SOleg Nesterov  * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning
5801da177e4SLinus Torvalds  * this kernel workthread has transiently adopted a user mm with use_mm,
5811da177e4SLinus Torvalds  * to do its AIO) is not set and if so returns a reference to it, after
5821da177e4SLinus Torvalds  * bumping up the use count.  User must release the mm via mmput()
5831da177e4SLinus Torvalds  * after use.  Typically used by /proc and ptrace.
5841da177e4SLinus Torvalds  */
5851da177e4SLinus Torvalds struct mm_struct *get_task_mm(struct task_struct *task)
5861da177e4SLinus Torvalds {
5871da177e4SLinus Torvalds 	struct mm_struct *mm;
5881da177e4SLinus Torvalds 
5891da177e4SLinus Torvalds 	task_lock(task);
5901da177e4SLinus Torvalds 	mm = task->mm;
5911da177e4SLinus Torvalds 	if (mm) {
592246bb0b1SOleg Nesterov 		if (task->flags & PF_KTHREAD)
5931da177e4SLinus Torvalds 			mm = NULL;
5941da177e4SLinus Torvalds 		else
5951da177e4SLinus Torvalds 			atomic_inc(&mm->mm_users);
5961da177e4SLinus Torvalds 	}
5971da177e4SLinus Torvalds 	task_unlock(task);
5981da177e4SLinus Torvalds 	return mm;
5991da177e4SLinus Torvalds }
6001da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(get_task_mm);
6011da177e4SLinus Torvalds 
6021da177e4SLinus Torvalds /* Please note the differences between mmput and mm_release.
6031da177e4SLinus Torvalds  * mmput is called whenever we stop holding onto a mm_struct,
6041da177e4SLinus Torvalds  * error success whatever.
6051da177e4SLinus Torvalds  *
6061da177e4SLinus Torvalds  * mm_release is called after a mm_struct has been removed
6071da177e4SLinus Torvalds  * from the current process.
6081da177e4SLinus Torvalds  *
6091da177e4SLinus Torvalds  * This difference is important for error handling, when we
6101da177e4SLinus Torvalds  * only half set up a mm_struct for a new process and need to restore
6111da177e4SLinus Torvalds  * the old one.  Because we mmput the new mm_struct before
6121da177e4SLinus Torvalds  * restoring the old one. . .
6131da177e4SLinus Torvalds  * Eric Biederman 10 January 1998
6141da177e4SLinus Torvalds  */
6151da177e4SLinus Torvalds void mm_release(struct task_struct *tsk, struct mm_struct *mm)
6161da177e4SLinus Torvalds {
6171da177e4SLinus Torvalds 	struct completion *vfork_done = tsk->vfork_done;
6181da177e4SLinus Torvalds 
6198141c7f3SLinus Torvalds 	/* Get rid of any futexes when releasing the mm */
6208141c7f3SLinus Torvalds #ifdef CONFIG_FUTEX
621fc6b177dSPeter Zijlstra 	if (unlikely(tsk->robust_list)) {
6228141c7f3SLinus Torvalds 		exit_robust_list(tsk);
623fc6b177dSPeter Zijlstra 		tsk->robust_list = NULL;
624fc6b177dSPeter Zijlstra 	}
6258141c7f3SLinus Torvalds #ifdef CONFIG_COMPAT
626fc6b177dSPeter Zijlstra 	if (unlikely(tsk->compat_robust_list)) {
6278141c7f3SLinus Torvalds 		compat_exit_robust_list(tsk);
628fc6b177dSPeter Zijlstra 		tsk->compat_robust_list = NULL;
629fc6b177dSPeter Zijlstra 	}
6308141c7f3SLinus Torvalds #endif
631322a2c10SThomas Gleixner 	if (unlikely(!list_empty(&tsk->pi_state_list)))
632322a2c10SThomas Gleixner 		exit_pi_state_list(tsk);
6338141c7f3SLinus Torvalds #endif
6348141c7f3SLinus Torvalds 
6351da177e4SLinus Torvalds 	/* Get rid of any cached register state */
6361da177e4SLinus Torvalds 	deactivate_mm(tsk, mm);
6371da177e4SLinus Torvalds 
6381da177e4SLinus Torvalds 	/* notify parent sleeping on vfork() */
6391da177e4SLinus Torvalds 	if (vfork_done) {
6401da177e4SLinus Torvalds 		tsk->vfork_done = NULL;
6411da177e4SLinus Torvalds 		complete(vfork_done);
6421da177e4SLinus Torvalds 	}
643fec1d011SRoland McGrath 
644fec1d011SRoland McGrath 	/*
645fec1d011SRoland McGrath 	 * If we're exiting normally, clear a user-space tid field if
646fec1d011SRoland McGrath 	 * requested.  We leave this alone when dying by signal, to leave
647fec1d011SRoland McGrath 	 * the value intact in a core dump, and to save the unnecessary
648fec1d011SRoland McGrath 	 * trouble otherwise.  Userland only wants this done for a sys_exit.
649fec1d011SRoland McGrath 	 */
6509c8a8228SEric Dumazet 	if (tsk->clear_child_tid) {
6519c8a8228SEric Dumazet 		if (!(tsk->flags & PF_SIGNALED) &&
6529c8a8228SEric Dumazet 		    atomic_read(&mm->mm_users) > 1) {
6531da177e4SLinus Torvalds 			/*
6541da177e4SLinus Torvalds 			 * We don't check the error code - if userspace has
6551da177e4SLinus Torvalds 			 * not set up a proper pointer then tough luck.
6561da177e4SLinus Torvalds 			 */
6579c8a8228SEric Dumazet 			put_user(0, tsk->clear_child_tid);
6589c8a8228SEric Dumazet 			sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
6599c8a8228SEric Dumazet 					1, NULL, NULL, 0);
6609c8a8228SEric Dumazet 		}
6619c8a8228SEric Dumazet 		tsk->clear_child_tid = NULL;
6621da177e4SLinus Torvalds 	}
6631da177e4SLinus Torvalds }
6641da177e4SLinus Torvalds 
665a0a7ec30SJANAK DESAI /*
666a0a7ec30SJANAK DESAI  * Allocate a new mm structure and copy contents from the
667a0a7ec30SJANAK DESAI  * mm structure of the passed in task structure.
668a0a7ec30SJANAK DESAI  */
669402b0862SCarsten Otte struct mm_struct *dup_mm(struct task_struct *tsk)
670a0a7ec30SJANAK DESAI {
671a0a7ec30SJANAK DESAI 	struct mm_struct *mm, *oldmm = current->mm;
672a0a7ec30SJANAK DESAI 	int err;
673a0a7ec30SJANAK DESAI 
674a0a7ec30SJANAK DESAI 	if (!oldmm)
675a0a7ec30SJANAK DESAI 		return NULL;
676a0a7ec30SJANAK DESAI 
677a0a7ec30SJANAK DESAI 	mm = allocate_mm();
678a0a7ec30SJANAK DESAI 	if (!mm)
679a0a7ec30SJANAK DESAI 		goto fail_nomem;
680a0a7ec30SJANAK DESAI 
681a0a7ec30SJANAK DESAI 	memcpy(mm, oldmm, sizeof(*mm));
682a0a7ec30SJANAK DESAI 
6837602bdf2SAshwin Chaugule 	/* Initializing for Swap token stuff */
6847602bdf2SAshwin Chaugule 	mm->token_priority = 0;
6857602bdf2SAshwin Chaugule 	mm->last_interval = 0;
6867602bdf2SAshwin Chaugule 
687e7a00c45SAndrea Arcangeli #ifdef CONFIG_TRANSPARENT_HUGEPAGE
688e7a00c45SAndrea Arcangeli 	mm->pmd_huge_pte = NULL;
689e7a00c45SAndrea Arcangeli #endif
690e7a00c45SAndrea Arcangeli 
69178fb7466SPavel Emelianov 	if (!mm_init(mm, tsk))
692a0a7ec30SJANAK DESAI 		goto fail_nomem;
693a0a7ec30SJANAK DESAI 
694a0a7ec30SJANAK DESAI 	if (init_new_context(tsk, mm))
695a0a7ec30SJANAK DESAI 		goto fail_nocontext;
696a0a7ec30SJANAK DESAI 
697925d1c40SMatt Helsley 	dup_mm_exe_file(oldmm, mm);
698925d1c40SMatt Helsley 
699a0a7ec30SJANAK DESAI 	err = dup_mmap(mm, oldmm);
700a0a7ec30SJANAK DESAI 	if (err)
701a0a7ec30SJANAK DESAI 		goto free_pt;
702a0a7ec30SJANAK DESAI 
703a0a7ec30SJANAK DESAI 	mm->hiwater_rss = get_mm_rss(mm);
704a0a7ec30SJANAK DESAI 	mm->hiwater_vm = mm->total_vm;
705a0a7ec30SJANAK DESAI 
706801460d0SHiroshi Shimamoto 	if (mm->binfmt && !try_module_get(mm->binfmt->module))
707801460d0SHiroshi Shimamoto 		goto free_pt;
708801460d0SHiroshi Shimamoto 
709a0a7ec30SJANAK DESAI 	return mm;
710a0a7ec30SJANAK DESAI 
711a0a7ec30SJANAK DESAI free_pt:
712801460d0SHiroshi Shimamoto 	/* don't put binfmt in mmput, we haven't got module yet */
713801460d0SHiroshi Shimamoto 	mm->binfmt = NULL;
714a0a7ec30SJANAK DESAI 	mmput(mm);
715a0a7ec30SJANAK DESAI 
716a0a7ec30SJANAK DESAI fail_nomem:
717a0a7ec30SJANAK DESAI 	return NULL;
718a0a7ec30SJANAK DESAI 
719a0a7ec30SJANAK DESAI fail_nocontext:
720a0a7ec30SJANAK DESAI 	/*
721a0a7ec30SJANAK DESAI 	 * If init_new_context() failed, we cannot use mmput() to free the mm
722a0a7ec30SJANAK DESAI 	 * because it calls destroy_context()
723a0a7ec30SJANAK DESAI 	 */
724a0a7ec30SJANAK DESAI 	mm_free_pgd(mm);
725a0a7ec30SJANAK DESAI 	free_mm(mm);
726a0a7ec30SJANAK DESAI 	return NULL;
727a0a7ec30SJANAK DESAI }
728a0a7ec30SJANAK DESAI 
7291da177e4SLinus Torvalds static int copy_mm(unsigned long clone_flags, struct task_struct * tsk)
7301da177e4SLinus Torvalds {
7311da177e4SLinus Torvalds 	struct mm_struct * mm, *oldmm;
7321da177e4SLinus Torvalds 	int retval;
7331da177e4SLinus Torvalds 
7341da177e4SLinus Torvalds 	tsk->min_flt = tsk->maj_flt = 0;
7351da177e4SLinus Torvalds 	tsk->nvcsw = tsk->nivcsw = 0;
73617406b82SMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK
73717406b82SMandeep Singh Baines 	tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
73817406b82SMandeep Singh Baines #endif
7391da177e4SLinus Torvalds 
7401da177e4SLinus Torvalds 	tsk->mm = NULL;
7411da177e4SLinus Torvalds 	tsk->active_mm = NULL;
7421da177e4SLinus Torvalds 
7431da177e4SLinus Torvalds 	/*
7441da177e4SLinus Torvalds 	 * Are we cloning a kernel thread?
7451da177e4SLinus Torvalds 	 *
7461da177e4SLinus Torvalds 	 * We need to steal a active VM for that..
7471da177e4SLinus Torvalds 	 */
7481da177e4SLinus Torvalds 	oldmm = current->mm;
7491da177e4SLinus Torvalds 	if (!oldmm)
7501da177e4SLinus Torvalds 		return 0;
7511da177e4SLinus Torvalds 
7521da177e4SLinus Torvalds 	if (clone_flags & CLONE_VM) {
7531da177e4SLinus Torvalds 		atomic_inc(&oldmm->mm_users);
7541da177e4SLinus Torvalds 		mm = oldmm;
7551da177e4SLinus Torvalds 		goto good_mm;
7561da177e4SLinus Torvalds 	}
7571da177e4SLinus Torvalds 
7581da177e4SLinus Torvalds 	retval = -ENOMEM;
759a0a7ec30SJANAK DESAI 	mm = dup_mm(tsk);
7601da177e4SLinus Torvalds 	if (!mm)
7611da177e4SLinus Torvalds 		goto fail_nomem;
7621da177e4SLinus Torvalds 
7631da177e4SLinus Torvalds good_mm:
7647602bdf2SAshwin Chaugule 	/* Initializing for Swap token stuff */
7657602bdf2SAshwin Chaugule 	mm->token_priority = 0;
7667602bdf2SAshwin Chaugule 	mm->last_interval = 0;
7673d5992d2SYing Han 	if (tsk->signal->oom_score_adj == OOM_SCORE_ADJ_MIN)
7683d5992d2SYing Han 		atomic_inc(&mm->oom_disable_count);
7697602bdf2SAshwin Chaugule 
7701da177e4SLinus Torvalds 	tsk->mm = mm;
7711da177e4SLinus Torvalds 	tsk->active_mm = mm;
7721da177e4SLinus Torvalds 	return 0;
7731da177e4SLinus Torvalds 
7741da177e4SLinus Torvalds fail_nomem:
7751da177e4SLinus Torvalds 	return retval;
7761da177e4SLinus Torvalds }
7771da177e4SLinus Torvalds 
778a39bc516SAlexey Dobriyan static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
7791da177e4SLinus Torvalds {
780498052bbSAl Viro 	struct fs_struct *fs = current->fs;
7811da177e4SLinus Torvalds 	if (clone_flags & CLONE_FS) {
782498052bbSAl Viro 		/* tsk->fs is already what we want */
7832a4419b5SNick Piggin 		spin_lock(&fs->lock);
784498052bbSAl Viro 		if (fs->in_exec) {
7852a4419b5SNick Piggin 			spin_unlock(&fs->lock);
786498052bbSAl Viro 			return -EAGAIN;
787498052bbSAl Viro 		}
788498052bbSAl Viro 		fs->users++;
7892a4419b5SNick Piggin 		spin_unlock(&fs->lock);
7901da177e4SLinus Torvalds 		return 0;
7911da177e4SLinus Torvalds 	}
792498052bbSAl Viro 	tsk->fs = copy_fs_struct(fs);
7931da177e4SLinus Torvalds 	if (!tsk->fs)
7941da177e4SLinus Torvalds 		return -ENOMEM;
7951da177e4SLinus Torvalds 	return 0;
7961da177e4SLinus Torvalds }
7971da177e4SLinus Torvalds 
798a016f338SJANAK DESAI static int copy_files(unsigned long clone_flags, struct task_struct * tsk)
799a016f338SJANAK DESAI {
800a016f338SJANAK DESAI 	struct files_struct *oldf, *newf;
801a016f338SJANAK DESAI 	int error = 0;
802a016f338SJANAK DESAI 
803a016f338SJANAK DESAI 	/*
804a016f338SJANAK DESAI 	 * A background process may not have any files ...
805a016f338SJANAK DESAI 	 */
806a016f338SJANAK DESAI 	oldf = current->files;
807a016f338SJANAK DESAI 	if (!oldf)
808a016f338SJANAK DESAI 		goto out;
809a016f338SJANAK DESAI 
810a016f338SJANAK DESAI 	if (clone_flags & CLONE_FILES) {
811a016f338SJANAK DESAI 		atomic_inc(&oldf->count);
812a016f338SJANAK DESAI 		goto out;
813a016f338SJANAK DESAI 	}
814a016f338SJANAK DESAI 
815a016f338SJANAK DESAI 	newf = dup_fd(oldf, &error);
816a016f338SJANAK DESAI 	if (!newf)
817a016f338SJANAK DESAI 		goto out;
818a016f338SJANAK DESAI 
819a016f338SJANAK DESAI 	tsk->files = newf;
820a016f338SJANAK DESAI 	error = 0;
821a016f338SJANAK DESAI out:
822a016f338SJANAK DESAI 	return error;
823a016f338SJANAK DESAI }
824a016f338SJANAK DESAI 
825fadad878SJens Axboe static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
826fd0928dfSJens Axboe {
827fd0928dfSJens Axboe #ifdef CONFIG_BLOCK
828fd0928dfSJens Axboe 	struct io_context *ioc = current->io_context;
829fd0928dfSJens Axboe 
830fd0928dfSJens Axboe 	if (!ioc)
831fd0928dfSJens Axboe 		return 0;
832fadad878SJens Axboe 	/*
833fadad878SJens Axboe 	 * Share io context with parent, if CLONE_IO is set
834fadad878SJens Axboe 	 */
835fadad878SJens Axboe 	if (clone_flags & CLONE_IO) {
836fadad878SJens Axboe 		tsk->io_context = ioc_task_link(ioc);
837fadad878SJens Axboe 		if (unlikely(!tsk->io_context))
838fadad878SJens Axboe 			return -ENOMEM;
839fadad878SJens Axboe 	} else if (ioprio_valid(ioc->ioprio)) {
840fd0928dfSJens Axboe 		tsk->io_context = alloc_io_context(GFP_KERNEL, -1);
841fd0928dfSJens Axboe 		if (unlikely(!tsk->io_context))
842fd0928dfSJens Axboe 			return -ENOMEM;
843fd0928dfSJens Axboe 
844fd0928dfSJens Axboe 		tsk->io_context->ioprio = ioc->ioprio;
845fd0928dfSJens Axboe 	}
846fd0928dfSJens Axboe #endif
847fd0928dfSJens Axboe 	return 0;
848fd0928dfSJens Axboe }
849fd0928dfSJens Axboe 
850a39bc516SAlexey Dobriyan static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
8511da177e4SLinus Torvalds {
8521da177e4SLinus Torvalds 	struct sighand_struct *sig;
8531da177e4SLinus Torvalds 
85460348802SZhaolei 	if (clone_flags & CLONE_SIGHAND) {
8551da177e4SLinus Torvalds 		atomic_inc(&current->sighand->count);
8561da177e4SLinus Torvalds 		return 0;
8571da177e4SLinus Torvalds 	}
8581da177e4SLinus Torvalds 	sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
859e56d0903SIngo Molnar 	rcu_assign_pointer(tsk->sighand, sig);
8601da177e4SLinus Torvalds 	if (!sig)
8611da177e4SLinus Torvalds 		return -ENOMEM;
8621da177e4SLinus Torvalds 	atomic_set(&sig->count, 1);
8631da177e4SLinus Torvalds 	memcpy(sig->action, current->sighand->action, sizeof(sig->action));
8641da177e4SLinus Torvalds 	return 0;
8651da177e4SLinus Torvalds }
8661da177e4SLinus Torvalds 
867a7e5328aSOleg Nesterov void __cleanup_sighand(struct sighand_struct *sighand)
868c81addc9SOleg Nesterov {
869c81addc9SOleg Nesterov 	if (atomic_dec_and_test(&sighand->count))
870c81addc9SOleg Nesterov 		kmem_cache_free(sighand_cachep, sighand);
871c81addc9SOleg Nesterov }
872c81addc9SOleg Nesterov 
873f06febc9SFrank Mayhar 
874f06febc9SFrank Mayhar /*
875f06febc9SFrank Mayhar  * Initialize POSIX timer handling for a thread group.
876f06febc9SFrank Mayhar  */
877f06febc9SFrank Mayhar static void posix_cpu_timers_init_group(struct signal_struct *sig)
878f06febc9SFrank Mayhar {
87978d7d407SJiri Slaby 	unsigned long cpu_limit;
88078d7d407SJiri Slaby 
881f06febc9SFrank Mayhar 	/* Thread group counters. */
882f06febc9SFrank Mayhar 	thread_group_cputime_init(sig);
883f06febc9SFrank Mayhar 
88478d7d407SJiri Slaby 	cpu_limit = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
88578d7d407SJiri Slaby 	if (cpu_limit != RLIM_INFINITY) {
88678d7d407SJiri Slaby 		sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit);
8876279a751SOleg Nesterov 		sig->cputimer.running = 1;
8886279a751SOleg Nesterov 	}
8896279a751SOleg Nesterov 
890f06febc9SFrank Mayhar 	/* The timer lists. */
891f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&sig->cpu_timers[0]);
892f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&sig->cpu_timers[1]);
893f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&sig->cpu_timers[2]);
894f06febc9SFrank Mayhar }
895f06febc9SFrank Mayhar 
896a39bc516SAlexey Dobriyan static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
8971da177e4SLinus Torvalds {
8981da177e4SLinus Torvalds 	struct signal_struct *sig;
8991da177e4SLinus Torvalds 
9004ab6c083SOleg Nesterov 	if (clone_flags & CLONE_THREAD)
901490dea45SPeter Zijlstra 		return 0;
9026279a751SOleg Nesterov 
903a56704efSVeaceslav Falico 	sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
9041da177e4SLinus Torvalds 	tsk->signal = sig;
9051da177e4SLinus Torvalds 	if (!sig)
9061da177e4SLinus Torvalds 		return -ENOMEM;
9071da177e4SLinus Torvalds 
908b3ac022cSOleg Nesterov 	sig->nr_threads = 1;
9091da177e4SLinus Torvalds 	atomic_set(&sig->live, 1);
910b3ac022cSOleg Nesterov 	atomic_set(&sig->sigcnt, 1);
9111da177e4SLinus Torvalds 	init_waitqueue_head(&sig->wait_chldexit);
912b3bfa0cbSSukadev Bhattiprolu 	if (clone_flags & CLONE_NEWPID)
913b3bfa0cbSSukadev Bhattiprolu 		sig->flags |= SIGNAL_UNKILLABLE;
914db51aeccSOleg Nesterov 	sig->curr_target = tsk;
9151da177e4SLinus Torvalds 	init_sigpending(&sig->shared_pending);
9161da177e4SLinus Torvalds 	INIT_LIST_HEAD(&sig->posix_timers);
9171da177e4SLinus Torvalds 
918c9cb2e3dSThomas Gleixner 	hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
9191da177e4SLinus Torvalds 	sig->real_timer.function = it_real_fn;
9201da177e4SLinus Torvalds 
9211da177e4SLinus Torvalds 	task_lock(current->group_leader);
9221da177e4SLinus Torvalds 	memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
9231da177e4SLinus Torvalds 	task_unlock(current->group_leader);
9241da177e4SLinus Torvalds 
9256279a751SOleg Nesterov 	posix_cpu_timers_init_group(sig);
9266279a751SOleg Nesterov 
927522ed776SMiloslav Trmac 	tty_audit_fork(sig);
9285091faa4SMike Galbraith 	sched_autogroup_fork(sig);
929522ed776SMiloslav Trmac 
93028b83c51SKOSAKI Motohiro 	sig->oom_adj = current->signal->oom_adj;
931a63d83f4SDavid Rientjes 	sig->oom_score_adj = current->signal->oom_score_adj;
932dabb16f6SMandeep Singh Baines 	sig->oom_score_adj_min = current->signal->oom_score_adj_min;
93328b83c51SKOSAKI Motohiro 
9349b1bf12dSKOSAKI Motohiro 	mutex_init(&sig->cred_guard_mutex);
9359b1bf12dSKOSAKI Motohiro 
9361da177e4SLinus Torvalds 	return 0;
9371da177e4SLinus Torvalds }
9381da177e4SLinus Torvalds 
939a39bc516SAlexey Dobriyan static void copy_flags(unsigned long clone_flags, struct task_struct *p)
9401da177e4SLinus Torvalds {
9411da177e4SLinus Torvalds 	unsigned long new_flags = p->flags;
9421da177e4SLinus Torvalds 
94321aa9af0STejun Heo 	new_flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
9441da177e4SLinus Torvalds 	new_flags |= PF_FORKNOEXEC;
94509a05394SRoland McGrath 	new_flags |= PF_STARTING;
9461da177e4SLinus Torvalds 	p->flags = new_flags;
9472e131895SRafael J. Wysocki 	clear_freeze_flag(p);
9481da177e4SLinus Torvalds }
9491da177e4SLinus Torvalds 
95017da2bd9SHeiko Carstens SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
9511da177e4SLinus Torvalds {
9521da177e4SLinus Torvalds 	current->clear_child_tid = tidptr;
9531da177e4SLinus Torvalds 
954b488893aSPavel Emelyanov 	return task_pid_vnr(current);
9551da177e4SLinus Torvalds }
9561da177e4SLinus Torvalds 
957a39bc516SAlexey Dobriyan static void rt_mutex_init_task(struct task_struct *p)
95823f78d4aSIngo Molnar {
9591d615482SThomas Gleixner 	raw_spin_lock_init(&p->pi_lock);
960e29e175bSZilvinas Valinskas #ifdef CONFIG_RT_MUTEXES
9611d615482SThomas Gleixner 	plist_head_init_raw(&p->pi_waiters, &p->pi_lock);
96223f78d4aSIngo Molnar 	p->pi_blocked_on = NULL;
96323f78d4aSIngo Molnar #endif
96423f78d4aSIngo Molnar }
96523f78d4aSIngo Molnar 
966cf475ad2SBalbir Singh #ifdef CONFIG_MM_OWNER
967cf475ad2SBalbir Singh void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
968cf475ad2SBalbir Singh {
969cf475ad2SBalbir Singh 	mm->owner = p;
970cf475ad2SBalbir Singh }
971cf475ad2SBalbir Singh #endif /* CONFIG_MM_OWNER */
972cf475ad2SBalbir Singh 
9731da177e4SLinus Torvalds /*
974f06febc9SFrank Mayhar  * Initialize POSIX timer handling for a single task.
975f06febc9SFrank Mayhar  */
976f06febc9SFrank Mayhar static void posix_cpu_timers_init(struct task_struct *tsk)
977f06febc9SFrank Mayhar {
978f06febc9SFrank Mayhar 	tsk->cputime_expires.prof_exp = cputime_zero;
979f06febc9SFrank Mayhar 	tsk->cputime_expires.virt_exp = cputime_zero;
980f06febc9SFrank Mayhar 	tsk->cputime_expires.sched_exp = 0;
981f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&tsk->cpu_timers[0]);
982f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&tsk->cpu_timers[1]);
983f06febc9SFrank Mayhar 	INIT_LIST_HEAD(&tsk->cpu_timers[2]);
984f06febc9SFrank Mayhar }
985f06febc9SFrank Mayhar 
986f06febc9SFrank Mayhar /*
9871da177e4SLinus Torvalds  * This creates a new process as a copy of the old one,
9881da177e4SLinus Torvalds  * but does not actually start it yet.
9891da177e4SLinus Torvalds  *
9901da177e4SLinus Torvalds  * It copies the registers, and all the appropriate
9911da177e4SLinus Torvalds  * parts of the process environment (as per the clone
9921da177e4SLinus Torvalds  * flags). The actual kick-off is left to the caller.
9931da177e4SLinus Torvalds  */
99436c8b586SIngo Molnar static struct task_struct *copy_process(unsigned long clone_flags,
9951da177e4SLinus Torvalds 					unsigned long stack_start,
9961da177e4SLinus Torvalds 					struct pt_regs *regs,
9971da177e4SLinus Torvalds 					unsigned long stack_size,
9981da177e4SLinus Torvalds 					int __user *child_tidptr,
99909a05394SRoland McGrath 					struct pid *pid,
100009a05394SRoland McGrath 					int trace)
10011da177e4SLinus Torvalds {
10021da177e4SLinus Torvalds 	int retval;
1003a24efe62SMariusz Kozlowski 	struct task_struct *p;
1004b4f48b63SPaul Menage 	int cgroup_callbacks_done = 0;
10051da177e4SLinus Torvalds 
10061da177e4SLinus Torvalds 	if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
10071da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
10081da177e4SLinus Torvalds 
10091da177e4SLinus Torvalds 	/*
10101da177e4SLinus Torvalds 	 * Thread groups must share signals as well, and detached threads
10111da177e4SLinus Torvalds 	 * can only be started up within the thread group.
10121da177e4SLinus Torvalds 	 */
10131da177e4SLinus Torvalds 	if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
10141da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
10151da177e4SLinus Torvalds 
10161da177e4SLinus Torvalds 	/*
10171da177e4SLinus Torvalds 	 * Shared signal handlers imply shared VM. By way of the above,
10181da177e4SLinus Torvalds 	 * thread groups also imply shared VM. Blocking this case allows
10191da177e4SLinus Torvalds 	 * for various simplifications in other code.
10201da177e4SLinus Torvalds 	 */
10211da177e4SLinus Torvalds 	if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
10221da177e4SLinus Torvalds 		return ERR_PTR(-EINVAL);
10231da177e4SLinus Torvalds 
1024123be07bSSukadev Bhattiprolu 	/*
1025123be07bSSukadev Bhattiprolu 	 * Siblings of global init remain as zombies on exit since they are
1026123be07bSSukadev Bhattiprolu 	 * not reaped by their parent (swapper). To solve this and to avoid
1027123be07bSSukadev Bhattiprolu 	 * multi-rooted process trees, prevent global and container-inits
1028123be07bSSukadev Bhattiprolu 	 * from creating siblings.
1029123be07bSSukadev Bhattiprolu 	 */
1030123be07bSSukadev Bhattiprolu 	if ((clone_flags & CLONE_PARENT) &&
1031123be07bSSukadev Bhattiprolu 				current->signal->flags & SIGNAL_UNKILLABLE)
1032123be07bSSukadev Bhattiprolu 		return ERR_PTR(-EINVAL);
1033123be07bSSukadev Bhattiprolu 
10341da177e4SLinus Torvalds 	retval = security_task_create(clone_flags);
10351da177e4SLinus Torvalds 	if (retval)
10361da177e4SLinus Torvalds 		goto fork_out;
10371da177e4SLinus Torvalds 
10381da177e4SLinus Torvalds 	retval = -ENOMEM;
10391da177e4SLinus Torvalds 	p = dup_task_struct(current);
10401da177e4SLinus Torvalds 	if (!p)
10411da177e4SLinus Torvalds 		goto fork_out;
10421da177e4SLinus Torvalds 
1043f7e8b616SSteven Rostedt 	ftrace_graph_init_task(p);
1044f7e8b616SSteven Rostedt 
1045bea493a0SPeter Zijlstra 	rt_mutex_init_task(p);
1046bea493a0SPeter Zijlstra 
1047d12c1a37SIngo Molnar #ifdef CONFIG_PROVE_LOCKING
1048de30a2b3SIngo Molnar 	DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
1049de30a2b3SIngo Molnar 	DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1050de30a2b3SIngo Molnar #endif
10511da177e4SLinus Torvalds 	retval = -EAGAIN;
10523b11a1deSDavid Howells 	if (atomic_read(&p->real_cred->user->processes) >=
105378d7d407SJiri Slaby 			task_rlimit(p, RLIMIT_NPROC)) {
10541da177e4SLinus Torvalds 		if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) &&
105518b6e041SSerge Hallyn 		    p->real_cred->user != INIT_USER)
10561da177e4SLinus Torvalds 			goto bad_fork_free;
10571da177e4SLinus Torvalds 	}
10581da177e4SLinus Torvalds 
1059f1752eecSDavid Howells 	retval = copy_creds(p, clone_flags);
1060f1752eecSDavid Howells 	if (retval < 0)
1061f1752eecSDavid Howells 		goto bad_fork_free;
10621da177e4SLinus Torvalds 
10631da177e4SLinus Torvalds 	/*
10641da177e4SLinus Torvalds 	 * If multiple threads are within copy_process(), then this check
10651da177e4SLinus Torvalds 	 * triggers too late. This doesn't hurt, the check is only there
10661da177e4SLinus Torvalds 	 * to stop root fork bombs.
10671da177e4SLinus Torvalds 	 */
106804ec93feSLi Zefan 	retval = -EAGAIN;
10691da177e4SLinus Torvalds 	if (nr_threads >= max_threads)
10701da177e4SLinus Torvalds 		goto bad_fork_cleanup_count;
10711da177e4SLinus Torvalds 
1072a1261f54SAl Viro 	if (!try_module_get(task_thread_info(p)->exec_domain->module))
10731da177e4SLinus Torvalds 		goto bad_fork_cleanup_count;
10741da177e4SLinus Torvalds 
10751da177e4SLinus Torvalds 	p->did_exec = 0;
1076ca74e92bSShailabh Nagar 	delayacct_tsk_init(p);	/* Must remain after dup_task_struct() */
10771da177e4SLinus Torvalds 	copy_flags(clone_flags, p);
10781da177e4SLinus Torvalds 	INIT_LIST_HEAD(&p->children);
10791da177e4SLinus Torvalds 	INIT_LIST_HEAD(&p->sibling);
1080f41d911fSPaul E. McKenney 	rcu_copy_process(p);
10811da177e4SLinus Torvalds 	p->vfork_done = NULL;
10821da177e4SLinus Torvalds 	spin_lock_init(&p->alloc_lock);
10831da177e4SLinus Torvalds 
10841da177e4SLinus Torvalds 	init_sigpending(&p->pending);
10851da177e4SLinus Torvalds 
10861da177e4SLinus Torvalds 	p->utime = cputime_zero;
10871da177e4SLinus Torvalds 	p->stime = cputime_zero;
10889ac52315SLaurent Vivier 	p->gtime = cputime_zero;
1089c66f08beSMichael Neuling 	p->utimescaled = cputime_zero;
1090c66f08beSMichael Neuling 	p->stimescaled = cputime_zero;
1091d99ca3b9SHidetoshi Seto #ifndef CONFIG_VIRT_CPU_ACCOUNTING
109273a2bcb0SPeter Zijlstra 	p->prev_utime = cputime_zero;
10939301899bSBalbir Singh 	p->prev_stime = cputime_zero;
1094d99ca3b9SHidetoshi Seto #endif
1095a3a2e76cSKAMEZAWA Hiroyuki #if defined(SPLIT_RSS_COUNTING)
1096a3a2e76cSKAMEZAWA Hiroyuki 	memset(&p->rss_stat, 0, sizeof(p->rss_stat));
1097a3a2e76cSKAMEZAWA Hiroyuki #endif
1098172ba844SBalbir Singh 
10996976675dSArjan van de Ven 	p->default_timer_slack_ns = current->timer_slack_ns;
11006976675dSArjan van de Ven 
11015995477aSAndrea Righi 	task_io_accounting_init(&p->ioac);
11021da177e4SLinus Torvalds 	acct_clear_integrals(p);
11031da177e4SLinus Torvalds 
1104f06febc9SFrank Mayhar 	posix_cpu_timers_init(p);
11051da177e4SLinus Torvalds 
11061da177e4SLinus Torvalds 	do_posix_clock_monotonic_gettime(&p->start_time);
1107924b42d5STomas Janousek 	p->real_start_time = p->start_time;
1108924b42d5STomas Janousek 	monotonic_to_bootbased(&p->real_start_time);
11091da177e4SLinus Torvalds 	p->io_context = NULL;
11101da177e4SLinus Torvalds 	p->audit_context = NULL;
1111b4f48b63SPaul Menage 	cgroup_fork(p);
11121da177e4SLinus Torvalds #ifdef CONFIG_NUMA
1113846a16bfSLee Schermerhorn 	p->mempolicy = mpol_dup(p->mempolicy);
11141da177e4SLinus Torvalds  	if (IS_ERR(p->mempolicy)) {
11151da177e4SLinus Torvalds  		retval = PTR_ERR(p->mempolicy);
11161da177e4SLinus Torvalds  		p->mempolicy = NULL;
1117b4f48b63SPaul Menage  		goto bad_fork_cleanup_cgroup;
11181da177e4SLinus Torvalds  	}
1119c61afb18SPaul Jackson 	mpol_fix_fork_child_flag(p);
11201da177e4SLinus Torvalds #endif
1121de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS
1122de30a2b3SIngo Molnar 	p->irq_events = 0;
1123b36e4758SRussell King #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
1124b36e4758SRussell King 	p->hardirqs_enabled = 1;
1125b36e4758SRussell King #else
1126de30a2b3SIngo Molnar 	p->hardirqs_enabled = 0;
1127b36e4758SRussell King #endif
1128de30a2b3SIngo Molnar 	p->hardirq_enable_ip = 0;
1129de30a2b3SIngo Molnar 	p->hardirq_enable_event = 0;
1130de30a2b3SIngo Molnar 	p->hardirq_disable_ip = _THIS_IP_;
1131de30a2b3SIngo Molnar 	p->hardirq_disable_event = 0;
1132de30a2b3SIngo Molnar 	p->softirqs_enabled = 1;
1133de30a2b3SIngo Molnar 	p->softirq_enable_ip = _THIS_IP_;
1134de30a2b3SIngo Molnar 	p->softirq_enable_event = 0;
1135de30a2b3SIngo Molnar 	p->softirq_disable_ip = 0;
1136de30a2b3SIngo Molnar 	p->softirq_disable_event = 0;
1137de30a2b3SIngo Molnar 	p->hardirq_context = 0;
1138de30a2b3SIngo Molnar 	p->softirq_context = 0;
1139de30a2b3SIngo Molnar #endif
1140fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP
1141fbb9ce95SIngo Molnar 	p->lockdep_depth = 0; /* no locks held yet */
1142fbb9ce95SIngo Molnar 	p->curr_chain_key = 0;
1143fbb9ce95SIngo Molnar 	p->lockdep_recursion = 0;
1144fbb9ce95SIngo Molnar #endif
11451da177e4SLinus Torvalds 
1146408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES
1147408894eeSIngo Molnar 	p->blocked_on = NULL; /* not blocked yet */
1148408894eeSIngo Molnar #endif
1149569b846dSKAMEZAWA Hiroyuki #ifdef CONFIG_CGROUP_MEM_RES_CTLR
1150569b846dSKAMEZAWA Hiroyuki 	p->memcg_batch.do_batch = 0;
1151569b846dSKAMEZAWA Hiroyuki 	p->memcg_batch.memcg = NULL;
1152569b846dSKAMEZAWA Hiroyuki #endif
11530f481406SMarkus Metzger 
11543c90e6e9SSrivatsa Vaddagiri 	/* Perform scheduler related setup. Assign this task to a CPU. */
1155*3e51e3edSSamir Bellabes 	sched_fork(p);
11566ab423e0SPeter Zijlstra 
1157cdd6c482SIngo Molnar 	retval = perf_event_init_task(p);
11586ab423e0SPeter Zijlstra 	if (retval)
11596ab423e0SPeter Zijlstra 		goto bad_fork_cleanup_policy;
11603c90e6e9SSrivatsa Vaddagiri 
11611da177e4SLinus Torvalds 	if ((retval = audit_alloc(p)))
1162f1752eecSDavid Howells 		goto bad_fork_cleanup_policy;
11631da177e4SLinus Torvalds 	/* copy all the process information */
11641da177e4SLinus Torvalds 	if ((retval = copy_semundo(clone_flags, p)))
11651da177e4SLinus Torvalds 		goto bad_fork_cleanup_audit;
11661da177e4SLinus Torvalds 	if ((retval = copy_files(clone_flags, p)))
11671da177e4SLinus Torvalds 		goto bad_fork_cleanup_semundo;
11681da177e4SLinus Torvalds 	if ((retval = copy_fs(clone_flags, p)))
11691da177e4SLinus Torvalds 		goto bad_fork_cleanup_files;
11701da177e4SLinus Torvalds 	if ((retval = copy_sighand(clone_flags, p)))
11711da177e4SLinus Torvalds 		goto bad_fork_cleanup_fs;
11721da177e4SLinus Torvalds 	if ((retval = copy_signal(clone_flags, p)))
11731da177e4SLinus Torvalds 		goto bad_fork_cleanup_sighand;
11741da177e4SLinus Torvalds 	if ((retval = copy_mm(clone_flags, p)))
11751da177e4SLinus Torvalds 		goto bad_fork_cleanup_signal;
1176ab516013SSerge E. Hallyn 	if ((retval = copy_namespaces(clone_flags, p)))
1177d84f4f99SDavid Howells 		goto bad_fork_cleanup_mm;
1178fadad878SJens Axboe 	if ((retval = copy_io(clone_flags, p)))
1179fd0928dfSJens Axboe 		goto bad_fork_cleanup_namespaces;
11806f2c55b8SAlexey Dobriyan 	retval = copy_thread(clone_flags, stack_start, stack_size, p, regs);
11811da177e4SLinus Torvalds 	if (retval)
1182fd0928dfSJens Axboe 		goto bad_fork_cleanup_io;
11831da177e4SLinus Torvalds 
1184425fb2b4SPavel Emelyanov 	if (pid != &init_struct_pid) {
1185425fb2b4SPavel Emelyanov 		retval = -ENOMEM;
118661bce0f1SEric W. Biederman 		pid = alloc_pid(p->nsproxy->pid_ns);
1187425fb2b4SPavel Emelyanov 		if (!pid)
1188fd0928dfSJens Axboe 			goto bad_fork_cleanup_io;
1189425fb2b4SPavel Emelyanov 	}
1190425fb2b4SPavel Emelyanov 
1191425fb2b4SPavel Emelyanov 	p->pid = pid_nr(pid);
1192425fb2b4SPavel Emelyanov 	p->tgid = p->pid;
1193425fb2b4SPavel Emelyanov 	if (clone_flags & CLONE_THREAD)
1194425fb2b4SPavel Emelyanov 		p->tgid = current->tgid;
1195425fb2b4SPavel Emelyanov 
1196e885dcdeSSerge E. Hallyn 	if (current->nsproxy != p->nsproxy) {
1197e885dcdeSSerge E. Hallyn 		retval = ns_cgroup_clone(p, pid);
1198e885dcdeSSerge E. Hallyn 		if (retval)
1199f7e8b616SSteven Rostedt 			goto bad_fork_free_pid;
1200e885dcdeSSerge E. Hallyn 	}
1201e885dcdeSSerge E. Hallyn 
12021da177e4SLinus Torvalds 	p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
12031da177e4SLinus Torvalds 	/*
12041da177e4SLinus Torvalds 	 * Clear TID on mm_release()?
12051da177e4SLinus Torvalds 	 */
12061da177e4SLinus Torvalds 	p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr: NULL;
120773c10101SJens Axboe #ifdef CONFIG_BLOCK
120873c10101SJens Axboe 	p->plug = NULL;
120973c10101SJens Axboe #endif
121042b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX
12118f17d3a5SIngo Molnar 	p->robust_list = NULL;
12128f17d3a5SIngo Molnar #ifdef CONFIG_COMPAT
12138f17d3a5SIngo Molnar 	p->compat_robust_list = NULL;
12148f17d3a5SIngo Molnar #endif
1215c87e2837SIngo Molnar 	INIT_LIST_HEAD(&p->pi_state_list);
1216c87e2837SIngo Molnar 	p->pi_state_cache = NULL;
121742b2dd0aSAlexey Dobriyan #endif
12181da177e4SLinus Torvalds 	/*
1219f9a3879aSGOTO Masanori 	 * sigaltstack should be cleared when sharing the same VM
1220f9a3879aSGOTO Masanori 	 */
1221f9a3879aSGOTO Masanori 	if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
1222f9a3879aSGOTO Masanori 		p->sas_ss_sp = p->sas_ss_size = 0;
1223f9a3879aSGOTO Masanori 
1224f9a3879aSGOTO Masanori 	/*
12256580807dSOleg Nesterov 	 * Syscall tracing and stepping should be turned off in the
12266580807dSOleg Nesterov 	 * child regardless of CLONE_PTRACE.
12271da177e4SLinus Torvalds 	 */
12286580807dSOleg Nesterov 	user_disable_single_step(p);
12291da177e4SLinus Torvalds 	clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
1230ed75e8d5SLaurent Vivier #ifdef TIF_SYSCALL_EMU
1231ed75e8d5SLaurent Vivier 	clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
1232ed75e8d5SLaurent Vivier #endif
12339745512cSArjan van de Ven 	clear_all_latency_tracing(p);
12341da177e4SLinus Torvalds 
12351da177e4SLinus Torvalds 	/* ok, now we should be set up.. */
12361da177e4SLinus Torvalds 	p->exit_signal = (clone_flags & CLONE_THREAD) ? -1 : (clone_flags & CSIGNAL);
12371da177e4SLinus Torvalds 	p->pdeath_signal = 0;
12381da177e4SLinus Torvalds 	p->exit_state = 0;
12391da177e4SLinus Torvalds 
12401da177e4SLinus Torvalds 	/*
12411da177e4SLinus Torvalds 	 * Ok, make it visible to the rest of the system.
12421da177e4SLinus Torvalds 	 * We dont wake it up yet.
12431da177e4SLinus Torvalds 	 */
12441da177e4SLinus Torvalds 	p->group_leader = p;
124547e65328SOleg Nesterov 	INIT_LIST_HEAD(&p->thread_group);
12461da177e4SLinus Torvalds 
1247b4f48b63SPaul Menage 	/* Now that the task is set up, run cgroup callbacks if
1248b4f48b63SPaul Menage 	 * necessary. We need to run them before the task is visible
1249b4f48b63SPaul Menage 	 * on the tasklist. */
1250b4f48b63SPaul Menage 	cgroup_fork_callbacks(p);
1251b4f48b63SPaul Menage 	cgroup_callbacks_done = 1;
1252b4f48b63SPaul Menage 
12531da177e4SLinus Torvalds 	/* Need tasklist lock for parent etc handling! */
12541da177e4SLinus Torvalds 	write_lock_irq(&tasklist_lock);
12551da177e4SLinus Torvalds 
12561da177e4SLinus Torvalds 	/* CLONE_PARENT re-uses the old parent */
12572d5516cbSOleg Nesterov 	if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
12581da177e4SLinus Torvalds 		p->real_parent = current->real_parent;
12592d5516cbSOleg Nesterov 		p->parent_exec_id = current->parent_exec_id;
12602d5516cbSOleg Nesterov 	} else {
12611da177e4SLinus Torvalds 		p->real_parent = current;
12622d5516cbSOleg Nesterov 		p->parent_exec_id = current->self_exec_id;
12632d5516cbSOleg Nesterov 	}
12641da177e4SLinus Torvalds 
12651da177e4SLinus Torvalds 	spin_lock(&current->sighand->siglock);
12664a2c7a78SOleg Nesterov 
12674a2c7a78SOleg Nesterov 	/*
12684a2c7a78SOleg Nesterov 	 * Process group and session signals need to be delivered to just the
12694a2c7a78SOleg Nesterov 	 * parent before the fork or both the parent and the child after the
12704a2c7a78SOleg Nesterov 	 * fork. Restart if a signal comes in before we add the new process to
12714a2c7a78SOleg Nesterov 	 * it's process group.
12724a2c7a78SOleg Nesterov 	 * A fatal signal pending means that current will exit, so the new
12734a2c7a78SOleg Nesterov 	 * thread can't slip out of an OOM kill (or normal SIGKILL).
12744a2c7a78SOleg Nesterov  	 */
12754a2c7a78SOleg Nesterov 	recalc_sigpending();
12764a2c7a78SOleg Nesterov 	if (signal_pending(current)) {
12774a2c7a78SOleg Nesterov 		spin_unlock(&current->sighand->siglock);
12784a2c7a78SOleg Nesterov 		write_unlock_irq(&tasklist_lock);
12794a2c7a78SOleg Nesterov 		retval = -ERESTARTNOINTR;
1280f7e8b616SSteven Rostedt 		goto bad_fork_free_pid;
12814a2c7a78SOleg Nesterov 	}
12824a2c7a78SOleg Nesterov 
12833f17da69SOleg Nesterov 	if (clone_flags & CLONE_THREAD) {
1284b3ac022cSOleg Nesterov 		current->signal->nr_threads++;
12854ab6c083SOleg Nesterov 		atomic_inc(&current->signal->live);
1286b3ac022cSOleg Nesterov 		atomic_inc(&current->signal->sigcnt);
12871da177e4SLinus Torvalds 		p->group_leader = current->group_leader;
128847e65328SOleg Nesterov 		list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group);
12891da177e4SLinus Torvalds 	}
12901da177e4SLinus Torvalds 
129173b9ebfeSOleg Nesterov 	if (likely(p->pid)) {
129209a05394SRoland McGrath 		tracehook_finish_clone(p, clone_flags, trace);
12931da177e4SLinus Torvalds 
12941da177e4SLinus Torvalds 		if (thread_group_leader(p)) {
129545a68628SEric W. Biederman 			if (is_child_reaper(pid))
129630e49c26SPavel Emelyanov 				p->nsproxy->pid_ns->child_reaper = p;
12975cd17569SEric W. Biederman 
1298fea9d175SOleg Nesterov 			p->signal->leader_pid = pid;
12999c9f4dedSAlan Cox 			p->signal->tty = tty_kref_get(current->signal->tty);
13005cd17569SEric W. Biederman 			attach_pid(p, PIDTYPE_PGID, task_pgrp(current));
13015cd17569SEric W. Biederman 			attach_pid(p, PIDTYPE_SID, task_session(current));
13029cd80bbbSOleg Nesterov 			list_add_tail(&p->sibling, &p->real_parent->children);
13035e85d4abSEric W. Biederman 			list_add_tail_rcu(&p->tasks, &init_task.tasks);
1304909ea964SChristoph Lameter 			__this_cpu_inc(process_counts);
13051da177e4SLinus Torvalds 		}
130685868995SSukadev Bhattiprolu 		attach_pid(p, PIDTYPE_PID, pid);
13071da177e4SLinus Torvalds 		nr_threads++;
130873b9ebfeSOleg Nesterov 	}
130973b9ebfeSOleg Nesterov 
13101da177e4SLinus Torvalds 	total_forks++;
13113f17da69SOleg Nesterov 	spin_unlock(&current->sighand->siglock);
13121da177e4SLinus Torvalds 	write_unlock_irq(&tasklist_lock);
1313c13cf856SAndrew Morton 	proc_fork_connector(p);
1314817929ecSPaul Menage 	cgroup_post_fork(p);
1315cdd6c482SIngo Molnar 	perf_event_fork(p);
13161da177e4SLinus Torvalds 	return p;
13171da177e4SLinus Torvalds 
1318425fb2b4SPavel Emelyanov bad_fork_free_pid:
1319425fb2b4SPavel Emelyanov 	if (pid != &init_struct_pid)
1320425fb2b4SPavel Emelyanov 		free_pid(pid);
1321fd0928dfSJens Axboe bad_fork_cleanup_io:
1322b69f2292SLouis Rilling 	if (p->io_context)
1323b69f2292SLouis Rilling 		exit_io_context(p);
1324ab516013SSerge E. Hallyn bad_fork_cleanup_namespaces:
1325444f378bSLinus Torvalds 	exit_task_namespaces(p);
13261da177e4SLinus Torvalds bad_fork_cleanup_mm:
13273d5992d2SYing Han 	if (p->mm) {
13283d5992d2SYing Han 		task_lock(p);
13293d5992d2SYing Han 		if (p->signal->oom_score_adj == OOM_SCORE_ADJ_MIN)
13303d5992d2SYing Han 			atomic_dec(&p->mm->oom_disable_count);
13313d5992d2SYing Han 		task_unlock(p);
13321da177e4SLinus Torvalds 		mmput(p->mm);
13333d5992d2SYing Han 	}
13341da177e4SLinus Torvalds bad_fork_cleanup_signal:
13354ab6c083SOleg Nesterov 	if (!(clone_flags & CLONE_THREAD))
13361c5354deSMike Galbraith 		free_signal_struct(p->signal);
13371da177e4SLinus Torvalds bad_fork_cleanup_sighand:
1338a7e5328aSOleg Nesterov 	__cleanup_sighand(p->sighand);
13391da177e4SLinus Torvalds bad_fork_cleanup_fs:
13401da177e4SLinus Torvalds 	exit_fs(p); /* blocking */
13411da177e4SLinus Torvalds bad_fork_cleanup_files:
13421da177e4SLinus Torvalds 	exit_files(p); /* blocking */
13431da177e4SLinus Torvalds bad_fork_cleanup_semundo:
13441da177e4SLinus Torvalds 	exit_sem(p);
13451da177e4SLinus Torvalds bad_fork_cleanup_audit:
13461da177e4SLinus Torvalds 	audit_free(p);
13471da177e4SLinus Torvalds bad_fork_cleanup_policy:
1348cdd6c482SIngo Molnar 	perf_event_free_task(p);
13491da177e4SLinus Torvalds #ifdef CONFIG_NUMA
1350f0be3d32SLee Schermerhorn 	mpol_put(p->mempolicy);
1351b4f48b63SPaul Menage bad_fork_cleanup_cgroup:
13521da177e4SLinus Torvalds #endif
1353b4f48b63SPaul Menage 	cgroup_exit(p, cgroup_callbacks_done);
135435df17c5SShailabh Nagar 	delayacct_tsk_free(p);
1355a1261f54SAl Viro 	module_put(task_thread_info(p)->exec_domain->module);
13561da177e4SLinus Torvalds bad_fork_cleanup_count:
1357d84f4f99SDavid Howells 	atomic_dec(&p->cred->user->processes);
1358e0e81739SDavid Howells 	exit_creds(p);
13591da177e4SLinus Torvalds bad_fork_free:
13601da177e4SLinus Torvalds 	free_task(p);
1361fe7d37d1SOleg Nesterov fork_out:
1362fe7d37d1SOleg Nesterov 	return ERR_PTR(retval);
13631da177e4SLinus Torvalds }
13641da177e4SLinus Torvalds 
13656b2fb3c6SAdrian Bunk noinline struct pt_regs * __cpuinit __attribute__((weak)) idle_regs(struct pt_regs *regs)
13661da177e4SLinus Torvalds {
13671da177e4SLinus Torvalds 	memset(regs, 0, sizeof(struct pt_regs));
13681da177e4SLinus Torvalds 	return regs;
13691da177e4SLinus Torvalds }
13701da177e4SLinus Torvalds 
1371f106eee1SOleg Nesterov static inline void init_idle_pids(struct pid_link *links)
1372f106eee1SOleg Nesterov {
1373f106eee1SOleg Nesterov 	enum pid_type type;
1374f106eee1SOleg Nesterov 
1375f106eee1SOleg Nesterov 	for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
1376f106eee1SOleg Nesterov 		INIT_HLIST_NODE(&links[type].node); /* not really needed */
1377f106eee1SOleg Nesterov 		links[type].pid = &init_struct_pid;
1378f106eee1SOleg Nesterov 	}
1379f106eee1SOleg Nesterov }
1380f106eee1SOleg Nesterov 
13819abcf40bSAl Viro struct task_struct * __cpuinit fork_idle(int cpu)
13821da177e4SLinus Torvalds {
138336c8b586SIngo Molnar 	struct task_struct *task;
13841da177e4SLinus Torvalds 	struct pt_regs regs;
13851da177e4SLinus Torvalds 
138630e49c26SPavel Emelyanov 	task = copy_process(CLONE_VM, 0, idle_regs(&regs), 0, NULL,
138709a05394SRoland McGrath 			    &init_struct_pid, 0);
1388f106eee1SOleg Nesterov 	if (!IS_ERR(task)) {
1389f106eee1SOleg Nesterov 		init_idle_pids(task->pids);
13901da177e4SLinus Torvalds 		init_idle(task, cpu);
1391f106eee1SOleg Nesterov 	}
139273b9ebfeSOleg Nesterov 
13931da177e4SLinus Torvalds 	return task;
13941da177e4SLinus Torvalds }
13951da177e4SLinus Torvalds 
13961da177e4SLinus Torvalds /*
13971da177e4SLinus Torvalds  *  Ok, this is the main fork-routine.
13981da177e4SLinus Torvalds  *
13991da177e4SLinus Torvalds  * It copies the process, and if successful kick-starts
14001da177e4SLinus Torvalds  * it and waits for it to finish using the VM if required.
14011da177e4SLinus Torvalds  */
14021da177e4SLinus Torvalds long do_fork(unsigned long clone_flags,
14031da177e4SLinus Torvalds 	      unsigned long stack_start,
14041da177e4SLinus Torvalds 	      struct pt_regs *regs,
14051da177e4SLinus Torvalds 	      unsigned long stack_size,
14061da177e4SLinus Torvalds 	      int __user *parent_tidptr,
14071da177e4SLinus Torvalds 	      int __user *child_tidptr)
14081da177e4SLinus Torvalds {
14091da177e4SLinus Torvalds 	struct task_struct *p;
14101da177e4SLinus Torvalds 	int trace = 0;
141192476d7fSEric W. Biederman 	long nr;
14121da177e4SLinus Torvalds 
1413bdff746aSAndrew Morton 	/*
141418b6e041SSerge Hallyn 	 * Do some preliminary argument and permissions checking before we
141518b6e041SSerge Hallyn 	 * actually start allocating stuff
141618b6e041SSerge Hallyn 	 */
141718b6e041SSerge Hallyn 	if (clone_flags & CLONE_NEWUSER) {
141818b6e041SSerge Hallyn 		if (clone_flags & CLONE_THREAD)
141918b6e041SSerge Hallyn 			return -EINVAL;
142018b6e041SSerge Hallyn 		/* hopefully this check will go away when userns support is
142118b6e041SSerge Hallyn 		 * complete
142218b6e041SSerge Hallyn 		 */
14237657d904SSerge E. Hallyn 		if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SETUID) ||
14247657d904SSerge E. Hallyn 				!capable(CAP_SETGID))
142518b6e041SSerge Hallyn 			return -EPERM;
142618b6e041SSerge Hallyn 	}
142718b6e041SSerge Hallyn 
142818b6e041SSerge Hallyn 	/*
142909a05394SRoland McGrath 	 * When called from kernel_thread, don't do user tracing stuff.
143009a05394SRoland McGrath 	 */
143109a05394SRoland McGrath 	if (likely(user_mode(regs)))
143209a05394SRoland McGrath 		trace = tracehook_prepare_clone(clone_flags);
14331da177e4SLinus Torvalds 
1434a6f5e063SSukadev Bhattiprolu 	p = copy_process(clone_flags, stack_start, regs, stack_size,
143509a05394SRoland McGrath 			 child_tidptr, NULL, trace);
14361da177e4SLinus Torvalds 	/*
14371da177e4SLinus Torvalds 	 * Do this prior waking up the new thread - the thread pointer
14381da177e4SLinus Torvalds 	 * might get invalid after that point, if the thread exits quickly.
14391da177e4SLinus Torvalds 	 */
14401da177e4SLinus Torvalds 	if (!IS_ERR(p)) {
14411da177e4SLinus Torvalds 		struct completion vfork;
14421da177e4SLinus Torvalds 
14430a16b607SMathieu Desnoyers 		trace_sched_process_fork(current, p);
14440a16b607SMathieu Desnoyers 
14456c5f3e7bSPavel Emelyanov 		nr = task_pid_vnr(p);
144630e49c26SPavel Emelyanov 
144730e49c26SPavel Emelyanov 		if (clone_flags & CLONE_PARENT_SETTID)
144830e49c26SPavel Emelyanov 			put_user(nr, parent_tidptr);
1449a6f5e063SSukadev Bhattiprolu 
14501da177e4SLinus Torvalds 		if (clone_flags & CLONE_VFORK) {
14511da177e4SLinus Torvalds 			p->vfork_done = &vfork;
14521da177e4SLinus Torvalds 			init_completion(&vfork);
14531da177e4SLinus Torvalds 		}
14541da177e4SLinus Torvalds 
1455a64e6494SAl Viro 		audit_finish_fork(p);
1456087eb437SOleg Nesterov 		tracehook_report_clone(regs, clone_flags, nr, p);
145709a05394SRoland McGrath 
145809a05394SRoland McGrath 		/*
145909a05394SRoland McGrath 		 * We set PF_STARTING at creation in case tracing wants to
146009a05394SRoland McGrath 		 * use this to distinguish a fully live task from one that
146109a05394SRoland McGrath 		 * hasn't gotten to tracehook_report_clone() yet.  Now we
146209a05394SRoland McGrath 		 * clear it and set the child going.
146309a05394SRoland McGrath 		 */
146409a05394SRoland McGrath 		p->flags &= ~PF_STARTING;
146509a05394SRoland McGrath 
1466*3e51e3edSSamir Bellabes 		wake_up_new_task(p);
14671da177e4SLinus Torvalds 
146809a05394SRoland McGrath 		tracehook_report_clone_complete(trace, regs,
146909a05394SRoland McGrath 						clone_flags, nr, p);
147009a05394SRoland McGrath 
14711da177e4SLinus Torvalds 		if (clone_flags & CLONE_VFORK) {
1472ba96a0c8SRafael J. Wysocki 			freezer_do_not_count();
14731da177e4SLinus Torvalds 			wait_for_completion(&vfork);
1474ba96a0c8SRafael J. Wysocki 			freezer_count();
1475daded34bSRoland McGrath 			tracehook_report_vfork_done(p, nr);
14769f59ce5dSChuck Ebbert 		}
14771da177e4SLinus Torvalds 	} else {
147892476d7fSEric W. Biederman 		nr = PTR_ERR(p);
14791da177e4SLinus Torvalds 	}
148092476d7fSEric W. Biederman 	return nr;
14811da177e4SLinus Torvalds }
14821da177e4SLinus Torvalds 
14835fd63b30SRavikiran G Thirumalai #ifndef ARCH_MIN_MMSTRUCT_ALIGN
14845fd63b30SRavikiran G Thirumalai #define ARCH_MIN_MMSTRUCT_ALIGN 0
14855fd63b30SRavikiran G Thirumalai #endif
14865fd63b30SRavikiran G Thirumalai 
148751cc5068SAlexey Dobriyan static void sighand_ctor(void *data)
1488aa1757f9SOleg Nesterov {
1489aa1757f9SOleg Nesterov 	struct sighand_struct *sighand = data;
1490aa1757f9SOleg Nesterov 
1491aa1757f9SOleg Nesterov 	spin_lock_init(&sighand->siglock);
1492b8fceee1SDavide Libenzi 	init_waitqueue_head(&sighand->signalfd_wqh);
1493fba2afaaSDavide Libenzi }
1494aa1757f9SOleg Nesterov 
14951da177e4SLinus Torvalds void __init proc_caches_init(void)
14961da177e4SLinus Torvalds {
14971da177e4SLinus Torvalds 	sighand_cachep = kmem_cache_create("sighand_cache",
14981da177e4SLinus Torvalds 			sizeof(struct sighand_struct), 0,
14992dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU|
15002dff4405SVegard Nossum 			SLAB_NOTRACK, sighand_ctor);
15011da177e4SLinus Torvalds 	signal_cachep = kmem_cache_create("signal_cache",
15021da177e4SLinus Torvalds 			sizeof(struct signal_struct), 0,
15032dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
15041da177e4SLinus Torvalds 	files_cachep = kmem_cache_create("files_cache",
15051da177e4SLinus Torvalds 			sizeof(struct files_struct), 0,
15062dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
15071da177e4SLinus Torvalds 	fs_cachep = kmem_cache_create("fs_cache",
15081da177e4SLinus Torvalds 			sizeof(struct fs_struct), 0,
15092dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
15101da177e4SLinus Torvalds 	mm_cachep = kmem_cache_create("mm_struct",
15115fd63b30SRavikiran G Thirumalai 			sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
15122dff4405SVegard Nossum 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
151333e5d769SDavid Howells 	vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC);
15148feae131SDavid Howells 	mmap_init();
15151da177e4SLinus Torvalds }
1516cf2e340fSJANAK DESAI 
1517cf2e340fSJANAK DESAI /*
15189bfb23fcSOleg Nesterov  * Check constraints on flags passed to the unshare system call.
1519cf2e340fSJANAK DESAI  */
15209bfb23fcSOleg Nesterov static int check_unshare_flags(unsigned long unshare_flags)
1521cf2e340fSJANAK DESAI {
15229bfb23fcSOleg Nesterov 	if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
15239bfb23fcSOleg Nesterov 				CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
15249bfb23fcSOleg Nesterov 				CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET))
1525cf2e340fSJANAK DESAI 		return -EINVAL;
15269bfb23fcSOleg Nesterov 	/*
15279bfb23fcSOleg Nesterov 	 * Not implemented, but pretend it works if there is nothing to
15289bfb23fcSOleg Nesterov 	 * unshare. Note that unsharing CLONE_THREAD or CLONE_SIGHAND
15299bfb23fcSOleg Nesterov 	 * needs to unshare vm.
15309bfb23fcSOleg Nesterov 	 */
15319bfb23fcSOleg Nesterov 	if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
15329bfb23fcSOleg Nesterov 		/* FIXME: get_task_mm() increments ->mm_users */
15339bfb23fcSOleg Nesterov 		if (atomic_read(&current->mm->mm_users) > 1)
15349bfb23fcSOleg Nesterov 			return -EINVAL;
15359bfb23fcSOleg Nesterov 	}
1536cf2e340fSJANAK DESAI 
1537cf2e340fSJANAK DESAI 	return 0;
1538cf2e340fSJANAK DESAI }
1539cf2e340fSJANAK DESAI 
1540cf2e340fSJANAK DESAI /*
154199d1419dSJANAK DESAI  * Unshare the filesystem structure if it is being shared
1542cf2e340fSJANAK DESAI  */
1543cf2e340fSJANAK DESAI static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
1544cf2e340fSJANAK DESAI {
1545cf2e340fSJANAK DESAI 	struct fs_struct *fs = current->fs;
1546cf2e340fSJANAK DESAI 
1547498052bbSAl Viro 	if (!(unshare_flags & CLONE_FS) || !fs)
1548498052bbSAl Viro 		return 0;
1549498052bbSAl Viro 
1550498052bbSAl Viro 	/* don't need lock here; in the worst case we'll do useless copy */
1551498052bbSAl Viro 	if (fs->users == 1)
1552498052bbSAl Viro 		return 0;
1553498052bbSAl Viro 
1554498052bbSAl Viro 	*new_fsp = copy_fs_struct(fs);
155599d1419dSJANAK DESAI 	if (!*new_fsp)
155699d1419dSJANAK DESAI 		return -ENOMEM;
1557cf2e340fSJANAK DESAI 
1558cf2e340fSJANAK DESAI 	return 0;
1559cf2e340fSJANAK DESAI }
1560cf2e340fSJANAK DESAI 
1561cf2e340fSJANAK DESAI /*
1562a016f338SJANAK DESAI  * Unshare file descriptor table if it is being shared
1563cf2e340fSJANAK DESAI  */
1564cf2e340fSJANAK DESAI static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
1565cf2e340fSJANAK DESAI {
1566cf2e340fSJANAK DESAI 	struct files_struct *fd = current->files;
1567a016f338SJANAK DESAI 	int error = 0;
1568cf2e340fSJANAK DESAI 
1569cf2e340fSJANAK DESAI 	if ((unshare_flags & CLONE_FILES) &&
1570a016f338SJANAK DESAI 	    (fd && atomic_read(&fd->count) > 1)) {
1571a016f338SJANAK DESAI 		*new_fdp = dup_fd(fd, &error);
1572a016f338SJANAK DESAI 		if (!*new_fdp)
1573a016f338SJANAK DESAI 			return error;
1574a016f338SJANAK DESAI 	}
1575cf2e340fSJANAK DESAI 
1576cf2e340fSJANAK DESAI 	return 0;
1577cf2e340fSJANAK DESAI }
1578cf2e340fSJANAK DESAI 
1579cf2e340fSJANAK DESAI /*
1580cf2e340fSJANAK DESAI  * unshare allows a process to 'unshare' part of the process
1581cf2e340fSJANAK DESAI  * context which was originally shared using clone.  copy_*
1582cf2e340fSJANAK DESAI  * functions used by do_fork() cannot be used here directly
1583cf2e340fSJANAK DESAI  * because they modify an inactive task_struct that is being
1584cf2e340fSJANAK DESAI  * constructed. Here we are modifying the current, active,
1585cf2e340fSJANAK DESAI  * task_struct.
1586cf2e340fSJANAK DESAI  */
15876559eed8SHeiko Carstens SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
1588cf2e340fSJANAK DESAI {
1589cf2e340fSJANAK DESAI 	struct fs_struct *fs, *new_fs = NULL;
1590cf2e340fSJANAK DESAI 	struct files_struct *fd, *new_fd = NULL;
1591cf7b708cSPavel Emelyanov 	struct nsproxy *new_nsproxy = NULL;
15929edff4abSManfred Spraul 	int do_sysvsem = 0;
15939bfb23fcSOleg Nesterov 	int err;
1594cf2e340fSJANAK DESAI 
15959bfb23fcSOleg Nesterov 	err = check_unshare_flags(unshare_flags);
15969bfb23fcSOleg Nesterov 	if (err)
159706f9d4f9SEric W. Biederman 		goto bad_unshare_out;
159806f9d4f9SEric W. Biederman 
15996013f67fSManfred Spraul 	/*
16009bfb23fcSOleg Nesterov 	 * If unsharing namespace, must also unshare filesystem information.
16019bfb23fcSOleg Nesterov 	 */
16029bfb23fcSOleg Nesterov 	if (unshare_flags & CLONE_NEWNS)
16039bfb23fcSOleg Nesterov 		unshare_flags |= CLONE_FS;
16049bfb23fcSOleg Nesterov 	/*
16056013f67fSManfred Spraul 	 * CLONE_NEWIPC must also detach from the undolist: after switching
16066013f67fSManfred Spraul 	 * to a new ipc namespace, the semaphore arrays from the old
16076013f67fSManfred Spraul 	 * namespace are unreachable.
16086013f67fSManfred Spraul 	 */
16096013f67fSManfred Spraul 	if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
16109edff4abSManfred Spraul 		do_sysvsem = 1;
1611cf2e340fSJANAK DESAI 	if ((err = unshare_fs(unshare_flags, &new_fs)))
16129bfb23fcSOleg Nesterov 		goto bad_unshare_out;
1613cf2e340fSJANAK DESAI 	if ((err = unshare_fd(unshare_flags, &new_fd)))
16149bfb23fcSOleg Nesterov 		goto bad_unshare_cleanup_fs;
1615e3222c4eSBadari Pulavarty 	if ((err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
1616e3222c4eSBadari Pulavarty 			new_fs)))
16179edff4abSManfred Spraul 		goto bad_unshare_cleanup_fd;
1618cf2e340fSJANAK DESAI 
16199bfb23fcSOleg Nesterov 	if (new_fs || new_fd || do_sysvsem || new_nsproxy) {
16209edff4abSManfred Spraul 		if (do_sysvsem) {
16219edff4abSManfred Spraul 			/*
16229edff4abSManfred Spraul 			 * CLONE_SYSVSEM is equivalent to sys_exit().
16239edff4abSManfred Spraul 			 */
16249edff4abSManfred Spraul 			exit_sem(current);
16259edff4abSManfred Spraul 		}
1626ab516013SSerge E. Hallyn 
1627c0b2fc31SSerge Hallyn 		if (new_nsproxy) {
1628cf7b708cSPavel Emelyanov 			switch_task_namespaces(current, new_nsproxy);
1629cf7b708cSPavel Emelyanov 			new_nsproxy = NULL;
1630c0b2fc31SSerge Hallyn 		}
1631cf2e340fSJANAK DESAI 
1632cf7b708cSPavel Emelyanov 		task_lock(current);
1633cf7b708cSPavel Emelyanov 
1634cf2e340fSJANAK DESAI 		if (new_fs) {
1635cf2e340fSJANAK DESAI 			fs = current->fs;
16362a4419b5SNick Piggin 			spin_lock(&fs->lock);
1637cf2e340fSJANAK DESAI 			current->fs = new_fs;
1638498052bbSAl Viro 			if (--fs->users)
1639498052bbSAl Viro 				new_fs = NULL;
1640498052bbSAl Viro 			else
1641cf2e340fSJANAK DESAI 				new_fs = fs;
16422a4419b5SNick Piggin 			spin_unlock(&fs->lock);
1643cf2e340fSJANAK DESAI 		}
1644cf2e340fSJANAK DESAI 
1645cf2e340fSJANAK DESAI 		if (new_fd) {
1646cf2e340fSJANAK DESAI 			fd = current->files;
1647cf2e340fSJANAK DESAI 			current->files = new_fd;
1648cf2e340fSJANAK DESAI 			new_fd = fd;
1649cf2e340fSJANAK DESAI 		}
1650cf2e340fSJANAK DESAI 
1651cf2e340fSJANAK DESAI 		task_unlock(current);
1652cf2e340fSJANAK DESAI 	}
1653cf2e340fSJANAK DESAI 
1654c0b2fc31SSerge Hallyn 	if (new_nsproxy)
1655444f378bSLinus Torvalds 		put_nsproxy(new_nsproxy);
1656c0b2fc31SSerge Hallyn 
1657cf2e340fSJANAK DESAI bad_unshare_cleanup_fd:
1658cf2e340fSJANAK DESAI 	if (new_fd)
1659cf2e340fSJANAK DESAI 		put_files_struct(new_fd);
1660cf2e340fSJANAK DESAI 
1661cf2e340fSJANAK DESAI bad_unshare_cleanup_fs:
1662cf2e340fSJANAK DESAI 	if (new_fs)
1663498052bbSAl Viro 		free_fs_struct(new_fs);
1664cf2e340fSJANAK DESAI 
1665cf2e340fSJANAK DESAI bad_unshare_out:
1666cf2e340fSJANAK DESAI 	return err;
1667cf2e340fSJANAK DESAI }
16683b125388SAl Viro 
16693b125388SAl Viro /*
16703b125388SAl Viro  *	Helper to unshare the files of the current task.
16713b125388SAl Viro  *	We don't want to expose copy_files internals to
16723b125388SAl Viro  *	the exec layer of the kernel.
16733b125388SAl Viro  */
16743b125388SAl Viro 
16753b125388SAl Viro int unshare_files(struct files_struct **displaced)
16763b125388SAl Viro {
16773b125388SAl Viro 	struct task_struct *task = current;
167850704516SAl Viro 	struct files_struct *copy = NULL;
16793b125388SAl Viro 	int error;
16803b125388SAl Viro 
16813b125388SAl Viro 	error = unshare_fd(CLONE_FILES, &copy);
16823b125388SAl Viro 	if (error || !copy) {
16833b125388SAl Viro 		*displaced = NULL;
16843b125388SAl Viro 		return error;
16853b125388SAl Viro 	}
16863b125388SAl Viro 	*displaced = task->files;
16873b125388SAl Viro 	task_lock(task);
16883b125388SAl Viro 	task->files = copy;
16893b125388SAl Viro 	task_unlock(task);
16903b125388SAl Viro 	return 0;
16913b125388SAl Viro }
1692