11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/kernel/fork.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1991, 1992 Linus Torvalds 51da177e4SLinus Torvalds */ 61da177e4SLinus Torvalds 71da177e4SLinus Torvalds /* 81da177e4SLinus Torvalds * 'fork.c' contains the help-routines for the 'fork' system call 91da177e4SLinus Torvalds * (see also entry.S and others). 101da177e4SLinus Torvalds * Fork is rather simple, once you get the hang of it, but the memory 111da177e4SLinus Torvalds * management can be a bitch. See 'mm/memory.c': 'copy_page_range()' 121da177e4SLinus Torvalds */ 131da177e4SLinus Torvalds 14b3e58382SChristian Brauner #include <linux/anon_inodes.h> 151da177e4SLinus Torvalds #include <linux/slab.h> 164eb5aaa3SIngo Molnar #include <linux/sched/autogroup.h> 176e84f315SIngo Molnar #include <linux/sched/mm.h> 18f7ccbae4SIngo Molnar #include <linux/sched/coredump.h> 198703e8a4SIngo Molnar #include <linux/sched/user.h> 206a3827d7SIngo Molnar #include <linux/sched/numa_balancing.h> 2103441a34SIngo Molnar #include <linux/sched/stat.h> 2229930025SIngo Molnar #include <linux/sched/task.h> 2368db0cf1SIngo Molnar #include <linux/sched/task_stack.h> 2432ef5517SIngo Molnar #include <linux/sched/cputime.h> 25b3e58382SChristian Brauner #include <linux/seq_file.h> 26037741a6SIngo Molnar #include <linux/rtmutex.h> 271da177e4SLinus Torvalds #include <linux/init.h> 281da177e4SLinus Torvalds #include <linux/unistd.h> 291da177e4SLinus Torvalds #include <linux/module.h> 301da177e4SLinus Torvalds #include <linux/vmalloc.h> 311da177e4SLinus Torvalds #include <linux/completion.h> 321da177e4SLinus Torvalds #include <linux/personality.h> 331da177e4SLinus Torvalds #include <linux/mempolicy.h> 341da177e4SLinus Torvalds #include <linux/sem.h> 351da177e4SLinus Torvalds #include <linux/file.h> 369f3acc31SAl Viro #include <linux/fdtable.h> 37da9cbc87SJens Axboe #include <linux/iocontext.h> 381da177e4SLinus Torvalds #include <linux/key.h> 391da177e4SLinus Torvalds #include <linux/binfmts.h> 401da177e4SLinus Torvalds #include <linux/mman.h> 41cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h> 42133ff0eaSJérôme Glisse #include <linux/hmm.h> 431da177e4SLinus Torvalds #include <linux/fs.h> 44615d6e87SDavidlohr Bueso #include <linux/mm.h> 45615d6e87SDavidlohr Bueso #include <linux/vmacache.h> 46ab516013SSerge E. Hallyn #include <linux/nsproxy.h> 47c59ede7bSRandy.Dunlap #include <linux/capability.h> 481da177e4SLinus Torvalds #include <linux/cpu.h> 49b4f48b63SPaul Menage #include <linux/cgroup.h> 501da177e4SLinus Torvalds #include <linux/security.h> 51a1e78772SMel Gorman #include <linux/hugetlb.h> 52e2cfabdfSWill Drewry #include <linux/seccomp.h> 531da177e4SLinus Torvalds #include <linux/swap.h> 541da177e4SLinus Torvalds #include <linux/syscalls.h> 551da177e4SLinus Torvalds #include <linux/jiffies.h> 561da177e4SLinus Torvalds #include <linux/futex.h> 578141c7f3SLinus Torvalds #include <linux/compat.h> 58207205a2SEric Dumazet #include <linux/kthread.h> 597c3ab738SAndrew Morton #include <linux/task_io_accounting_ops.h> 60ab2af1f5SDipankar Sarma #include <linux/rcupdate.h> 611da177e4SLinus Torvalds #include <linux/ptrace.h> 621da177e4SLinus Torvalds #include <linux/mount.h> 631da177e4SLinus Torvalds #include <linux/audit.h> 6478fb7466SPavel Emelianov #include <linux/memcontrol.h> 65f201ae23SFrederic Weisbecker #include <linux/ftrace.h> 665e2bf014SMike Galbraith #include <linux/proc_fs.h> 671da177e4SLinus Torvalds #include <linux/profile.h> 681da177e4SLinus Torvalds #include <linux/rmap.h> 69f8af4da3SHugh Dickins #include <linux/ksm.h> 701da177e4SLinus Torvalds #include <linux/acct.h> 71893e26e6SPavel Emelyanov #include <linux/userfaultfd_k.h> 728f0ab514SJay Lan #include <linux/tsacct_kern.h> 739f46080cSMatt Helsley #include <linux/cn_proc.h> 74ba96a0c8SRafael J. Wysocki #include <linux/freezer.h> 75ca74e92bSShailabh Nagar #include <linux/delayacct.h> 76ad4ecbcbSShailabh Nagar #include <linux/taskstats_kern.h> 770a425405SArjan van de Ven #include <linux/random.h> 78522ed776SMiloslav Trmac #include <linux/tty.h> 79fd0928dfSJens Axboe #include <linux/blkdev.h> 805ad4e53bSAl Viro #include <linux/fs_struct.h> 817c9f8861SEric Sandeen #include <linux/magic.h> 82cdd6c482SIngo Molnar #include <linux/perf_event.h> 8342c4ab41SStanislaw Gruszka #include <linux/posix-timers.h> 848e7cac79SAvi Kivity #include <linux/user-return-notifier.h> 853d5992d2SYing Han #include <linux/oom.h> 86ba76149fSAndrea Arcangeli #include <linux/khugepaged.h> 87d80e731eSOleg Nesterov #include <linux/signalfd.h> 880326f5a9SSrikar Dronamraju #include <linux/uprobes.h> 89a27bb332SKent Overstreet #include <linux/aio.h> 9052f5684cSGideon Israel Dsouza #include <linux/compiler.h> 9116db3d3fSHeinrich Schuchardt #include <linux/sysctl.h> 925c9a8750SDmitry Vyukov #include <linux/kcov.h> 93d83a7cb3SJosh Poimboeuf #include <linux/livepatch.h> 9448ac3c18SMark Rutland #include <linux/thread_info.h> 95afaef01cSAlexander Popov #include <linux/stackleak.h> 961da177e4SLinus Torvalds 971da177e4SLinus Torvalds #include <asm/pgtable.h> 981da177e4SLinus Torvalds #include <asm/pgalloc.h> 997c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 1001da177e4SLinus Torvalds #include <asm/mmu_context.h> 1011da177e4SLinus Torvalds #include <asm/cacheflush.h> 1021da177e4SLinus Torvalds #include <asm/tlbflush.h> 1031da177e4SLinus Torvalds 104ad8d75ffSSteven Rostedt #include <trace/events/sched.h> 105ad8d75ffSSteven Rostedt 10643d2b113SKAMEZAWA Hiroyuki #define CREATE_TRACE_POINTS 10743d2b113SKAMEZAWA Hiroyuki #include <trace/events/task.h> 10843d2b113SKAMEZAWA Hiroyuki 1091da177e4SLinus Torvalds /* 110ac1b398dSHeinrich Schuchardt * Minimum number of threads to boot the kernel 111ac1b398dSHeinrich Schuchardt */ 112ac1b398dSHeinrich Schuchardt #define MIN_THREADS 20 113ac1b398dSHeinrich Schuchardt 114ac1b398dSHeinrich Schuchardt /* 115ac1b398dSHeinrich Schuchardt * Maximum number of threads 116ac1b398dSHeinrich Schuchardt */ 117ac1b398dSHeinrich Schuchardt #define MAX_THREADS FUTEX_TID_MASK 118ac1b398dSHeinrich Schuchardt 119ac1b398dSHeinrich Schuchardt /* 1201da177e4SLinus Torvalds * Protected counters by write_lock_irq(&tasklist_lock) 1211da177e4SLinus Torvalds */ 1221da177e4SLinus Torvalds unsigned long total_forks; /* Handle normal Linux uptimes. */ 1231da177e4SLinus Torvalds int nr_threads; /* The idle threads do not count.. */ 1241da177e4SLinus Torvalds 1251da177e4SLinus Torvalds int max_threads; /* tunable limit on nr_threads */ 1261da177e4SLinus Torvalds 1271da177e4SLinus Torvalds DEFINE_PER_CPU(unsigned long, process_counts) = 0; 1281da177e4SLinus Torvalds 1291da177e4SLinus Torvalds __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ 130db1466b3SPaul E. McKenney 131db1466b3SPaul E. McKenney #ifdef CONFIG_PROVE_RCU 132db1466b3SPaul E. McKenney int lockdep_tasklist_lock_is_held(void) 133db1466b3SPaul E. McKenney { 134db1466b3SPaul E. McKenney return lockdep_is_held(&tasklist_lock); 135db1466b3SPaul E. McKenney } 136db1466b3SPaul E. McKenney EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held); 137db1466b3SPaul E. McKenney #endif /* #ifdef CONFIG_PROVE_RCU */ 1381da177e4SLinus Torvalds 1391da177e4SLinus Torvalds int nr_processes(void) 1401da177e4SLinus Torvalds { 1411da177e4SLinus Torvalds int cpu; 1421da177e4SLinus Torvalds int total = 0; 1431da177e4SLinus Torvalds 1441d510750SIan Campbell for_each_possible_cpu(cpu) 1451da177e4SLinus Torvalds total += per_cpu(process_counts, cpu); 1461da177e4SLinus Torvalds 1471da177e4SLinus Torvalds return total; 1481da177e4SLinus Torvalds } 1491da177e4SLinus Torvalds 150f19b9f74SAkinobu Mita void __weak arch_release_task_struct(struct task_struct *tsk) 151f19b9f74SAkinobu Mita { 152f19b9f74SAkinobu Mita } 153f19b9f74SAkinobu Mita 154f5e10287SThomas Gleixner #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR 155e18b890bSChristoph Lameter static struct kmem_cache *task_struct_cachep; 15641101809SThomas Gleixner 15741101809SThomas Gleixner static inline struct task_struct *alloc_task_struct_node(int node) 15841101809SThomas Gleixner { 15941101809SThomas Gleixner return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node); 16041101809SThomas Gleixner } 16141101809SThomas Gleixner 16241101809SThomas Gleixner static inline void free_task_struct(struct task_struct *tsk) 16341101809SThomas Gleixner { 16441101809SThomas Gleixner kmem_cache_free(task_struct_cachep, tsk); 16541101809SThomas Gleixner } 1661da177e4SLinus Torvalds #endif 1671da177e4SLinus Torvalds 168b235beeaSLinus Torvalds #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR 16941101809SThomas Gleixner 1700d15d74aSThomas Gleixner /* 1710d15d74aSThomas Gleixner * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a 1720d15d74aSThomas Gleixner * kmemcache based allocator. 1730d15d74aSThomas Gleixner */ 174ba14a194SAndy Lutomirski # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) 175ac496bf4SAndy Lutomirski 176ac496bf4SAndy Lutomirski #ifdef CONFIG_VMAP_STACK 177ac496bf4SAndy Lutomirski /* 178ac496bf4SAndy Lutomirski * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB 179ac496bf4SAndy Lutomirski * flush. Try to minimize the number of calls by caching stacks. 180ac496bf4SAndy Lutomirski */ 181ac496bf4SAndy Lutomirski #define NR_CACHED_STACKS 2 182ac496bf4SAndy Lutomirski static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); 18319659c59SHoeun Ryu 18419659c59SHoeun Ryu static int free_vm_stack_cache(unsigned int cpu) 18519659c59SHoeun Ryu { 18619659c59SHoeun Ryu struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu); 18719659c59SHoeun Ryu int i; 18819659c59SHoeun Ryu 18919659c59SHoeun Ryu for (i = 0; i < NR_CACHED_STACKS; i++) { 19019659c59SHoeun Ryu struct vm_struct *vm_stack = cached_vm_stacks[i]; 19119659c59SHoeun Ryu 19219659c59SHoeun Ryu if (!vm_stack) 19319659c59SHoeun Ryu continue; 19419659c59SHoeun Ryu 19519659c59SHoeun Ryu vfree(vm_stack->addr); 19619659c59SHoeun Ryu cached_vm_stacks[i] = NULL; 19719659c59SHoeun Ryu } 19819659c59SHoeun Ryu 19919659c59SHoeun Ryu return 0; 20019659c59SHoeun Ryu } 201ac496bf4SAndy Lutomirski #endif 202ac496bf4SAndy Lutomirski 203ba14a194SAndy Lutomirski static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) 204b69c49b7SFUJITA Tomonori { 205ba14a194SAndy Lutomirski #ifdef CONFIG_VMAP_STACK 206ac496bf4SAndy Lutomirski void *stack; 207ac496bf4SAndy Lutomirski int i; 208ac496bf4SAndy Lutomirski 209ac496bf4SAndy Lutomirski for (i = 0; i < NR_CACHED_STACKS; i++) { 210112166f8SChristoph Lameter struct vm_struct *s; 211112166f8SChristoph Lameter 212112166f8SChristoph Lameter s = this_cpu_xchg(cached_stacks[i], NULL); 213ac496bf4SAndy Lutomirski 214ac496bf4SAndy Lutomirski if (!s) 215ac496bf4SAndy Lutomirski continue; 216ac496bf4SAndy Lutomirski 217ca182551SKonstantin Khlebnikov /* Clear stale pointers from reused stack. */ 218ca182551SKonstantin Khlebnikov memset(s->addr, 0, THREAD_SIZE); 219e01e8063SKees Cook 220ac496bf4SAndy Lutomirski tsk->stack_vm_area = s; 221ba4a4574SShakeel Butt tsk->stack = s->addr; 222ac496bf4SAndy Lutomirski return s->addr; 223ac496bf4SAndy Lutomirski } 224ac496bf4SAndy Lutomirski 2259b6f7e16SRoman Gushchin /* 2269b6f7e16SRoman Gushchin * Allocated stacks are cached and later reused by new threads, 2279b6f7e16SRoman Gushchin * so memcg accounting is performed manually on assigning/releasing 2289b6f7e16SRoman Gushchin * stacks to tasks. Drop __GFP_ACCOUNT. 2299b6f7e16SRoman Gushchin */ 23048ac3c18SMark Rutland stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, 231ba14a194SAndy Lutomirski VMALLOC_START, VMALLOC_END, 2329b6f7e16SRoman Gushchin THREADINFO_GFP & ~__GFP_ACCOUNT, 233ba14a194SAndy Lutomirski PAGE_KERNEL, 234ac496bf4SAndy Lutomirski 0, node, __builtin_return_address(0)); 235ba14a194SAndy Lutomirski 236ba14a194SAndy Lutomirski /* 237ba14a194SAndy Lutomirski * We can't call find_vm_area() in interrupt context, and 238ba14a194SAndy Lutomirski * free_thread_stack() can be called in interrupt context, 239ba14a194SAndy Lutomirski * so cache the vm_struct. 240ba14a194SAndy Lutomirski */ 2415eed6f1dSRik van Riel if (stack) { 242ba14a194SAndy Lutomirski tsk->stack_vm_area = find_vm_area(stack); 2435eed6f1dSRik van Riel tsk->stack = stack; 2445eed6f1dSRik van Riel } 245ba14a194SAndy Lutomirski return stack; 246ba14a194SAndy Lutomirski #else 2474949148aSVladimir Davydov struct page *page = alloc_pages_node(node, THREADINFO_GFP, 2482889f608SThomas Gleixner THREAD_SIZE_ORDER); 249b6a84016SEric Dumazet 250b6a84016SEric Dumazet return page ? page_address(page) : NULL; 251ba14a194SAndy Lutomirski #endif 252b69c49b7SFUJITA Tomonori } 253b69c49b7SFUJITA Tomonori 254ba14a194SAndy Lutomirski static inline void free_thread_stack(struct task_struct *tsk) 255b69c49b7SFUJITA Tomonori { 256ac496bf4SAndy Lutomirski #ifdef CONFIG_VMAP_STACK 2579b6f7e16SRoman Gushchin struct vm_struct *vm = task_stack_vm_area(tsk); 2589b6f7e16SRoman Gushchin 2599b6f7e16SRoman Gushchin if (vm) { 260ac496bf4SAndy Lutomirski int i; 261ac496bf4SAndy Lutomirski 2629b6f7e16SRoman Gushchin for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { 2639b6f7e16SRoman Gushchin mod_memcg_page_state(vm->pages[i], 2649b6f7e16SRoman Gushchin MEMCG_KERNEL_STACK_KB, 2659b6f7e16SRoman Gushchin -(int)(PAGE_SIZE / 1024)); 2669b6f7e16SRoman Gushchin 2679b6f7e16SRoman Gushchin memcg_kmem_uncharge(vm->pages[i], 0); 2689b6f7e16SRoman Gushchin } 2699b6f7e16SRoman Gushchin 270ac496bf4SAndy Lutomirski for (i = 0; i < NR_CACHED_STACKS; i++) { 271112166f8SChristoph Lameter if (this_cpu_cmpxchg(cached_stacks[i], 272112166f8SChristoph Lameter NULL, tsk->stack_vm_area) != NULL) 273ac496bf4SAndy Lutomirski continue; 274ac496bf4SAndy Lutomirski 275ac496bf4SAndy Lutomirski return; 276ac496bf4SAndy Lutomirski } 277ac496bf4SAndy Lutomirski 2780f110a9bSAndrey Ryabinin vfree_atomic(tsk->stack); 279ac496bf4SAndy Lutomirski return; 280ac496bf4SAndy Lutomirski } 281ac496bf4SAndy Lutomirski #endif 282ac496bf4SAndy Lutomirski 283ba14a194SAndy Lutomirski __free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER); 284b69c49b7SFUJITA Tomonori } 2850d15d74aSThomas Gleixner # else 286b235beeaSLinus Torvalds static struct kmem_cache *thread_stack_cache; 2870d15d74aSThomas Gleixner 2889521d399SMichael Ellerman static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, 2890d15d74aSThomas Gleixner int node) 2900d15d74aSThomas Gleixner { 2915eed6f1dSRik van Riel unsigned long *stack; 2925eed6f1dSRik van Riel stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node); 2935eed6f1dSRik van Riel tsk->stack = stack; 2945eed6f1dSRik van Riel return stack; 2950d15d74aSThomas Gleixner } 2960d15d74aSThomas Gleixner 297ba14a194SAndy Lutomirski static void free_thread_stack(struct task_struct *tsk) 2980d15d74aSThomas Gleixner { 299ba14a194SAndy Lutomirski kmem_cache_free(thread_stack_cache, tsk->stack); 3000d15d74aSThomas Gleixner } 3010d15d74aSThomas Gleixner 302b235beeaSLinus Torvalds void thread_stack_cache_init(void) 3030d15d74aSThomas Gleixner { 304f9d29946SDavid Windsor thread_stack_cache = kmem_cache_create_usercopy("thread_stack", 305f9d29946SDavid Windsor THREAD_SIZE, THREAD_SIZE, 0, 0, 306f9d29946SDavid Windsor THREAD_SIZE, NULL); 307b235beeaSLinus Torvalds BUG_ON(thread_stack_cache == NULL); 3080d15d74aSThomas Gleixner } 3090d15d74aSThomas Gleixner # endif 310b69c49b7SFUJITA Tomonori #endif 311b69c49b7SFUJITA Tomonori 3121da177e4SLinus Torvalds /* SLAB cache for signal_struct structures (tsk->signal) */ 313e18b890bSChristoph Lameter static struct kmem_cache *signal_cachep; 3141da177e4SLinus Torvalds 3151da177e4SLinus Torvalds /* SLAB cache for sighand_struct structures (tsk->sighand) */ 316e18b890bSChristoph Lameter struct kmem_cache *sighand_cachep; 3171da177e4SLinus Torvalds 3181da177e4SLinus Torvalds /* SLAB cache for files_struct structures (tsk->files) */ 319e18b890bSChristoph Lameter struct kmem_cache *files_cachep; 3201da177e4SLinus Torvalds 3211da177e4SLinus Torvalds /* SLAB cache for fs_struct structures (tsk->fs) */ 322e18b890bSChristoph Lameter struct kmem_cache *fs_cachep; 3231da177e4SLinus Torvalds 3241da177e4SLinus Torvalds /* SLAB cache for vm_area_struct structures */ 3253928d4f5SLinus Torvalds static struct kmem_cache *vm_area_cachep; 3261da177e4SLinus Torvalds 3271da177e4SLinus Torvalds /* SLAB cache for mm_struct structures (tsk->mm) */ 328e18b890bSChristoph Lameter static struct kmem_cache *mm_cachep; 3291da177e4SLinus Torvalds 330490fc053SLinus Torvalds struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) 3313928d4f5SLinus Torvalds { 332a670468fSAndrew Morton struct vm_area_struct *vma; 333490fc053SLinus Torvalds 334a670468fSAndrew Morton vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 335027232daSKirill A. Shutemov if (vma) 336027232daSKirill A. Shutemov vma_init(vma, mm); 337490fc053SLinus Torvalds return vma; 3383928d4f5SLinus Torvalds } 3393928d4f5SLinus Torvalds 3403928d4f5SLinus Torvalds struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) 3413928d4f5SLinus Torvalds { 34295faf699SLinus Torvalds struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 34395faf699SLinus Torvalds 34495faf699SLinus Torvalds if (new) { 34595faf699SLinus Torvalds *new = *orig; 34695faf699SLinus Torvalds INIT_LIST_HEAD(&new->anon_vma_chain); 34795faf699SLinus Torvalds } 34895faf699SLinus Torvalds return new; 3493928d4f5SLinus Torvalds } 3503928d4f5SLinus Torvalds 3513928d4f5SLinus Torvalds void vm_area_free(struct vm_area_struct *vma) 3523928d4f5SLinus Torvalds { 3533928d4f5SLinus Torvalds kmem_cache_free(vm_area_cachep, vma); 3543928d4f5SLinus Torvalds } 3553928d4f5SLinus Torvalds 356ba14a194SAndy Lutomirski static void account_kernel_stack(struct task_struct *tsk, int account) 357c6a7f572SKOSAKI Motohiro { 358ba14a194SAndy Lutomirski void *stack = task_stack_page(tsk); 359ba14a194SAndy Lutomirski struct vm_struct *vm = task_stack_vm_area(tsk); 360ba14a194SAndy Lutomirski 361ba14a194SAndy Lutomirski BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0); 362ba14a194SAndy Lutomirski 363ba14a194SAndy Lutomirski if (vm) { 364ba14a194SAndy Lutomirski int i; 365ba14a194SAndy Lutomirski 366ba14a194SAndy Lutomirski BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE); 367ba14a194SAndy Lutomirski 368ba14a194SAndy Lutomirski for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { 369ba14a194SAndy Lutomirski mod_zone_page_state(page_zone(vm->pages[i]), 370ba14a194SAndy Lutomirski NR_KERNEL_STACK_KB, 371ba14a194SAndy Lutomirski PAGE_SIZE / 1024 * account); 372ba14a194SAndy Lutomirski } 373ba14a194SAndy Lutomirski } else { 374ba14a194SAndy Lutomirski /* 375ba14a194SAndy Lutomirski * All stack pages are in the same zone and belong to the 376ba14a194SAndy Lutomirski * same memcg. 377ba14a194SAndy Lutomirski */ 378efdc9490SAndy Lutomirski struct page *first_page = virt_to_page(stack); 379c6a7f572SKOSAKI Motohiro 380efdc9490SAndy Lutomirski mod_zone_page_state(page_zone(first_page), NR_KERNEL_STACK_KB, 381d30dd8beSAndy Lutomirski THREAD_SIZE / 1024 * account); 382efdc9490SAndy Lutomirski 383ed52be7bSJohannes Weiner mod_memcg_page_state(first_page, MEMCG_KERNEL_STACK_KB, 384efdc9490SAndy Lutomirski account * (THREAD_SIZE / 1024)); 385c6a7f572SKOSAKI Motohiro } 386ba14a194SAndy Lutomirski } 387c6a7f572SKOSAKI Motohiro 3889b6f7e16SRoman Gushchin static int memcg_charge_kernel_stack(struct task_struct *tsk) 3899b6f7e16SRoman Gushchin { 3909b6f7e16SRoman Gushchin #ifdef CONFIG_VMAP_STACK 3919b6f7e16SRoman Gushchin struct vm_struct *vm = task_stack_vm_area(tsk); 3929b6f7e16SRoman Gushchin int ret; 3939b6f7e16SRoman Gushchin 3949b6f7e16SRoman Gushchin if (vm) { 3959b6f7e16SRoman Gushchin int i; 3969b6f7e16SRoman Gushchin 3979b6f7e16SRoman Gushchin for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { 3989b6f7e16SRoman Gushchin /* 3999b6f7e16SRoman Gushchin * If memcg_kmem_charge() fails, page->mem_cgroup 4009b6f7e16SRoman Gushchin * pointer is NULL, and both memcg_kmem_uncharge() 4019b6f7e16SRoman Gushchin * and mod_memcg_page_state() in free_thread_stack() 4029b6f7e16SRoman Gushchin * will ignore this page. So it's safe. 4039b6f7e16SRoman Gushchin */ 4049b6f7e16SRoman Gushchin ret = memcg_kmem_charge(vm->pages[i], GFP_KERNEL, 0); 4059b6f7e16SRoman Gushchin if (ret) 4069b6f7e16SRoman Gushchin return ret; 4079b6f7e16SRoman Gushchin 4089b6f7e16SRoman Gushchin mod_memcg_page_state(vm->pages[i], 4099b6f7e16SRoman Gushchin MEMCG_KERNEL_STACK_KB, 4109b6f7e16SRoman Gushchin PAGE_SIZE / 1024); 4119b6f7e16SRoman Gushchin } 4129b6f7e16SRoman Gushchin } 4139b6f7e16SRoman Gushchin #endif 4149b6f7e16SRoman Gushchin return 0; 4159b6f7e16SRoman Gushchin } 4169b6f7e16SRoman Gushchin 41768f24b08SAndy Lutomirski static void release_task_stack(struct task_struct *tsk) 4181da177e4SLinus Torvalds { 419405c0759SAndy Lutomirski if (WARN_ON(tsk->state != TASK_DEAD)) 420405c0759SAndy Lutomirski return; /* Better to leak the stack than to free prematurely */ 421405c0759SAndy Lutomirski 422ba14a194SAndy Lutomirski account_kernel_stack(tsk, -1); 423ba14a194SAndy Lutomirski free_thread_stack(tsk); 42468f24b08SAndy Lutomirski tsk->stack = NULL; 42568f24b08SAndy Lutomirski #ifdef CONFIG_VMAP_STACK 42668f24b08SAndy Lutomirski tsk->stack_vm_area = NULL; 42768f24b08SAndy Lutomirski #endif 42868f24b08SAndy Lutomirski } 42968f24b08SAndy Lutomirski 43068f24b08SAndy Lutomirski #ifdef CONFIG_THREAD_INFO_IN_TASK 43168f24b08SAndy Lutomirski void put_task_stack(struct task_struct *tsk) 43268f24b08SAndy Lutomirski { 433f0b89d39SElena Reshetova if (refcount_dec_and_test(&tsk->stack_refcount)) 43468f24b08SAndy Lutomirski release_task_stack(tsk); 43568f24b08SAndy Lutomirski } 43668f24b08SAndy Lutomirski #endif 43768f24b08SAndy Lutomirski 43868f24b08SAndy Lutomirski void free_task(struct task_struct *tsk) 43968f24b08SAndy Lutomirski { 44068f24b08SAndy Lutomirski #ifndef CONFIG_THREAD_INFO_IN_TASK 44168f24b08SAndy Lutomirski /* 44268f24b08SAndy Lutomirski * The task is finally done with both the stack and thread_info, 44368f24b08SAndy Lutomirski * so free both. 44468f24b08SAndy Lutomirski */ 44568f24b08SAndy Lutomirski release_task_stack(tsk); 44668f24b08SAndy Lutomirski #else 44768f24b08SAndy Lutomirski /* 44868f24b08SAndy Lutomirski * If the task had a separate stack allocation, it should be gone 44968f24b08SAndy Lutomirski * by now. 45068f24b08SAndy Lutomirski */ 451f0b89d39SElena Reshetova WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0); 45268f24b08SAndy Lutomirski #endif 45323f78d4aSIngo Molnar rt_mutex_debug_task_free(tsk); 454fb52607aSFrederic Weisbecker ftrace_graph_exit_task(tsk); 455e2cfabdfSWill Drewry put_seccomp_filter(tsk); 456f19b9f74SAkinobu Mita arch_release_task_struct(tsk); 4571da5c46fSOleg Nesterov if (tsk->flags & PF_KTHREAD) 4581da5c46fSOleg Nesterov free_kthread_struct(tsk); 4591da177e4SLinus Torvalds free_task_struct(tsk); 4601da177e4SLinus Torvalds } 4611da177e4SLinus Torvalds EXPORT_SYMBOL(free_task); 4621da177e4SLinus Torvalds 4631da177e4SLinus Torvalds #ifdef CONFIG_MMU 4640766f788SEmese Revfy static __latent_entropy int dup_mmap(struct mm_struct *mm, 4650766f788SEmese Revfy struct mm_struct *oldmm) 4661da177e4SLinus Torvalds { 467297c5eeeSLinus Torvalds struct vm_area_struct *mpnt, *tmp, *prev, **pprev; 4681da177e4SLinus Torvalds struct rb_node **rb_link, *rb_parent; 4691da177e4SLinus Torvalds int retval; 4701da177e4SLinus Torvalds unsigned long charge; 471893e26e6SPavel Emelyanov LIST_HEAD(uf); 4721da177e4SLinus Torvalds 47332cdba1eSOleg Nesterov uprobe_start_dup_mmap(); 4747c051267SMichal Hocko if (down_write_killable(&oldmm->mmap_sem)) { 4757c051267SMichal Hocko retval = -EINTR; 4767c051267SMichal Hocko goto fail_uprobe_end; 4777c051267SMichal Hocko } 478ec8c0446SRalf Baechle flush_cache_dup_mm(oldmm); 479f8ac4ec9SOleg Nesterov uprobe_dup_mmap(oldmm, mm); 480ad339451SIngo Molnar /* 481ad339451SIngo Molnar * Not linked in yet - no deadlock potential: 482ad339451SIngo Molnar */ 483ad339451SIngo Molnar down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); 4847ee78232SHugh Dickins 48590f31d0eSKonstantin Khlebnikov /* No ordering required: file already has been exposed. */ 48690f31d0eSKonstantin Khlebnikov RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); 48790f31d0eSKonstantin Khlebnikov 4884f7d4614SVladimir Davydov mm->total_vm = oldmm->total_vm; 48984638335SKonstantin Khlebnikov mm->data_vm = oldmm->data_vm; 4904f7d4614SVladimir Davydov mm->exec_vm = oldmm->exec_vm; 4914f7d4614SVladimir Davydov mm->stack_vm = oldmm->stack_vm; 4924f7d4614SVladimir Davydov 4931da177e4SLinus Torvalds rb_link = &mm->mm_rb.rb_node; 4941da177e4SLinus Torvalds rb_parent = NULL; 4951da177e4SLinus Torvalds pprev = &mm->mmap; 496f8af4da3SHugh Dickins retval = ksm_fork(mm, oldmm); 497f8af4da3SHugh Dickins if (retval) 498f8af4da3SHugh Dickins goto out; 499ba76149fSAndrea Arcangeli retval = khugepaged_fork(mm, oldmm); 500ba76149fSAndrea Arcangeli if (retval) 501ba76149fSAndrea Arcangeli goto out; 5021da177e4SLinus Torvalds 503297c5eeeSLinus Torvalds prev = NULL; 504fd3e42fcSHugh Dickins for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { 5051da177e4SLinus Torvalds struct file *file; 5061da177e4SLinus Torvalds 5071da177e4SLinus Torvalds if (mpnt->vm_flags & VM_DONTCOPY) { 50884638335SKonstantin Khlebnikov vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); 5091da177e4SLinus Torvalds continue; 5101da177e4SLinus Torvalds } 5111da177e4SLinus Torvalds charge = 0; 512655c79bbSTetsuo Handa /* 513655c79bbSTetsuo Handa * Don't duplicate many vmas if we've been oom-killed (for 514655c79bbSTetsuo Handa * example) 515655c79bbSTetsuo Handa */ 516655c79bbSTetsuo Handa if (fatal_signal_pending(current)) { 517655c79bbSTetsuo Handa retval = -EINTR; 518655c79bbSTetsuo Handa goto out; 519655c79bbSTetsuo Handa } 5201da177e4SLinus Torvalds if (mpnt->vm_flags & VM_ACCOUNT) { 521b2412b7fSHuang Shijie unsigned long len = vma_pages(mpnt); 522b2412b7fSHuang Shijie 523191c5424SAl Viro if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ 5241da177e4SLinus Torvalds goto fail_nomem; 5251da177e4SLinus Torvalds charge = len; 5261da177e4SLinus Torvalds } 5273928d4f5SLinus Torvalds tmp = vm_area_dup(mpnt); 5281da177e4SLinus Torvalds if (!tmp) 5291da177e4SLinus Torvalds goto fail_nomem; 530ef0855d3SOleg Nesterov retval = vma_dup_policy(mpnt, tmp); 531ef0855d3SOleg Nesterov if (retval) 5321da177e4SLinus Torvalds goto fail_nomem_policy; 533a247c3a9SAndrea Arcangeli tmp->vm_mm = mm; 534893e26e6SPavel Emelyanov retval = dup_userfaultfd(tmp, &uf); 535893e26e6SPavel Emelyanov if (retval) 536893e26e6SPavel Emelyanov goto fail_nomem_anon_vma_fork; 537d2cd9edeSRik van Riel if (tmp->vm_flags & VM_WIPEONFORK) { 538d2cd9edeSRik van Riel /* VM_WIPEONFORK gets a clean slate in the child. */ 539d2cd9edeSRik van Riel tmp->anon_vma = NULL; 540d2cd9edeSRik van Riel if (anon_vma_prepare(tmp)) 541d2cd9edeSRik van Riel goto fail_nomem_anon_vma_fork; 542d2cd9edeSRik van Riel } else if (anon_vma_fork(tmp, mpnt)) 5435beb4930SRik van Riel goto fail_nomem_anon_vma_fork; 544893e26e6SPavel Emelyanov tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT); 545297c5eeeSLinus Torvalds tmp->vm_next = tmp->vm_prev = NULL; 5461da177e4SLinus Torvalds file = tmp->vm_file; 5471da177e4SLinus Torvalds if (file) { 548496ad9aaSAl Viro struct inode *inode = file_inode(file); 549b88ed205SHugh Dickins struct address_space *mapping = file->f_mapping; 550b88ed205SHugh Dickins 5511da177e4SLinus Torvalds get_file(file); 5521da177e4SLinus Torvalds if (tmp->vm_flags & VM_DENYWRITE) 5531da177e4SLinus Torvalds atomic_dec(&inode->i_writecount); 55483cde9e8SDavidlohr Bueso i_mmap_lock_write(mapping); 555b88ed205SHugh Dickins if (tmp->vm_flags & VM_SHARED) 5564bb5f5d9SDavid Herrmann atomic_inc(&mapping->i_mmap_writable); 557b88ed205SHugh Dickins flush_dcache_mmap_lock(mapping); 558b88ed205SHugh Dickins /* insert tmp into the share list, just after mpnt */ 5599826a516SMichel Lespinasse vma_interval_tree_insert_after(tmp, mpnt, 5609826a516SMichel Lespinasse &mapping->i_mmap); 561b88ed205SHugh Dickins flush_dcache_mmap_unlock(mapping); 56283cde9e8SDavidlohr Bueso i_mmap_unlock_write(mapping); 5631da177e4SLinus Torvalds } 5641da177e4SLinus Torvalds 5651da177e4SLinus Torvalds /* 566a1e78772SMel Gorman * Clear hugetlb-related page reserves for children. This only 567a1e78772SMel Gorman * affects MAP_PRIVATE mappings. Faults generated by the child 568a1e78772SMel Gorman * are not guaranteed to succeed, even if read-only 569a1e78772SMel Gorman */ 570a1e78772SMel Gorman if (is_vm_hugetlb_page(tmp)) 571a1e78772SMel Gorman reset_vma_resv_huge_pages(tmp); 572a1e78772SMel Gorman 573a1e78772SMel Gorman /* 5747ee78232SHugh Dickins * Link in the new vma and copy the page table entries. 5751da177e4SLinus Torvalds */ 5761da177e4SLinus Torvalds *pprev = tmp; 5771da177e4SLinus Torvalds pprev = &tmp->vm_next; 578297c5eeeSLinus Torvalds tmp->vm_prev = prev; 579297c5eeeSLinus Torvalds prev = tmp; 5801da177e4SLinus Torvalds 5811da177e4SLinus Torvalds __vma_link_rb(mm, tmp, rb_link, rb_parent); 5821da177e4SLinus Torvalds rb_link = &tmp->vm_rb.rb_right; 5831da177e4SLinus Torvalds rb_parent = &tmp->vm_rb; 5841da177e4SLinus Torvalds 5851da177e4SLinus Torvalds mm->map_count++; 586d2cd9edeSRik van Riel if (!(tmp->vm_flags & VM_WIPEONFORK)) 5870b0db14cSHugh Dickins retval = copy_page_range(mm, oldmm, mpnt); 5881da177e4SLinus Torvalds 5891da177e4SLinus Torvalds if (tmp->vm_ops && tmp->vm_ops->open) 5901da177e4SLinus Torvalds tmp->vm_ops->open(tmp); 5911da177e4SLinus Torvalds 5921da177e4SLinus Torvalds if (retval) 5931da177e4SLinus Torvalds goto out; 5941da177e4SLinus Torvalds } 595d6dd61c8SJeremy Fitzhardinge /* a new mm has just been created */ 5961ed0cc5aSNadav Amit retval = arch_dup_mmap(oldmm, mm); 5971da177e4SLinus Torvalds out: 5987ee78232SHugh Dickins up_write(&mm->mmap_sem); 599fd3e42fcSHugh Dickins flush_tlb_mm(oldmm); 6001da177e4SLinus Torvalds up_write(&oldmm->mmap_sem); 601893e26e6SPavel Emelyanov dup_userfaultfd_complete(&uf); 6027c051267SMichal Hocko fail_uprobe_end: 60332cdba1eSOleg Nesterov uprobe_end_dup_mmap(); 6041da177e4SLinus Torvalds return retval; 6055beb4930SRik van Riel fail_nomem_anon_vma_fork: 606ef0855d3SOleg Nesterov mpol_put(vma_policy(tmp)); 6071da177e4SLinus Torvalds fail_nomem_policy: 6083928d4f5SLinus Torvalds vm_area_free(tmp); 6091da177e4SLinus Torvalds fail_nomem: 6101da177e4SLinus Torvalds retval = -ENOMEM; 6111da177e4SLinus Torvalds vm_unacct_memory(charge); 6121da177e4SLinus Torvalds goto out; 6131da177e4SLinus Torvalds } 6141da177e4SLinus Torvalds 6151da177e4SLinus Torvalds static inline int mm_alloc_pgd(struct mm_struct *mm) 6161da177e4SLinus Torvalds { 6171da177e4SLinus Torvalds mm->pgd = pgd_alloc(mm); 6181da177e4SLinus Torvalds if (unlikely(!mm->pgd)) 6191da177e4SLinus Torvalds return -ENOMEM; 6201da177e4SLinus Torvalds return 0; 6211da177e4SLinus Torvalds } 6221da177e4SLinus Torvalds 6231da177e4SLinus Torvalds static inline void mm_free_pgd(struct mm_struct *mm) 6241da177e4SLinus Torvalds { 6255e541973SBenjamin Herrenschmidt pgd_free(mm, mm->pgd); 6261da177e4SLinus Torvalds } 6271da177e4SLinus Torvalds #else 62890f31d0eSKonstantin Khlebnikov static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) 62990f31d0eSKonstantin Khlebnikov { 63090f31d0eSKonstantin Khlebnikov down_write(&oldmm->mmap_sem); 63190f31d0eSKonstantin Khlebnikov RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); 63290f31d0eSKonstantin Khlebnikov up_write(&oldmm->mmap_sem); 63390f31d0eSKonstantin Khlebnikov return 0; 63490f31d0eSKonstantin Khlebnikov } 6351da177e4SLinus Torvalds #define mm_alloc_pgd(mm) (0) 6361da177e4SLinus Torvalds #define mm_free_pgd(mm) 6371da177e4SLinus Torvalds #endif /* CONFIG_MMU */ 6381da177e4SLinus Torvalds 639d70f2a14SAndrew Morton static void check_mm(struct mm_struct *mm) 640d70f2a14SAndrew Morton { 641d70f2a14SAndrew Morton int i; 642d70f2a14SAndrew Morton 643d70f2a14SAndrew Morton for (i = 0; i < NR_MM_COUNTERS; i++) { 644d70f2a14SAndrew Morton long x = atomic_long_read(&mm->rss_stat.count[i]); 645d70f2a14SAndrew Morton 646d70f2a14SAndrew Morton if (unlikely(x)) 647d70f2a14SAndrew Morton printk(KERN_ALERT "BUG: Bad rss-counter state " 648d70f2a14SAndrew Morton "mm:%p idx:%d val:%ld\n", mm, i, x); 649d70f2a14SAndrew Morton } 650d70f2a14SAndrew Morton 651d70f2a14SAndrew Morton if (mm_pgtables_bytes(mm)) 652d70f2a14SAndrew Morton pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", 653d70f2a14SAndrew Morton mm_pgtables_bytes(mm)); 654d70f2a14SAndrew Morton 655d70f2a14SAndrew Morton #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS 656d70f2a14SAndrew Morton VM_BUG_ON_MM(mm->pmd_huge_pte, mm); 657d70f2a14SAndrew Morton #endif 658d70f2a14SAndrew Morton } 6591da177e4SLinus Torvalds 660e94b1766SChristoph Lameter #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL)) 6611da177e4SLinus Torvalds #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) 6621da177e4SLinus Torvalds 663d70f2a14SAndrew Morton /* 664d70f2a14SAndrew Morton * Called when the last reference to the mm 665d70f2a14SAndrew Morton * is dropped: either by a lazy thread or by 666d70f2a14SAndrew Morton * mmput. Free the page directory and the mm. 667d70f2a14SAndrew Morton */ 668d34bc48fSAndrew Morton void __mmdrop(struct mm_struct *mm) 669d70f2a14SAndrew Morton { 670d70f2a14SAndrew Morton BUG_ON(mm == &init_mm); 6713eda69c9SMark Rutland WARN_ON_ONCE(mm == current->mm); 6723eda69c9SMark Rutland WARN_ON_ONCE(mm == current->active_mm); 673d70f2a14SAndrew Morton mm_free_pgd(mm); 674d70f2a14SAndrew Morton destroy_context(mm); 675d70f2a14SAndrew Morton hmm_mm_destroy(mm); 676d70f2a14SAndrew Morton mmu_notifier_mm_destroy(mm); 677d70f2a14SAndrew Morton check_mm(mm); 678d70f2a14SAndrew Morton put_user_ns(mm->user_ns); 679d70f2a14SAndrew Morton free_mm(mm); 680d70f2a14SAndrew Morton } 681d34bc48fSAndrew Morton EXPORT_SYMBOL_GPL(__mmdrop); 682d70f2a14SAndrew Morton 683d70f2a14SAndrew Morton static void mmdrop_async_fn(struct work_struct *work) 684d70f2a14SAndrew Morton { 685d70f2a14SAndrew Morton struct mm_struct *mm; 686d70f2a14SAndrew Morton 687d70f2a14SAndrew Morton mm = container_of(work, struct mm_struct, async_put_work); 688d70f2a14SAndrew Morton __mmdrop(mm); 689d70f2a14SAndrew Morton } 690d70f2a14SAndrew Morton 691d70f2a14SAndrew Morton static void mmdrop_async(struct mm_struct *mm) 692d70f2a14SAndrew Morton { 693d70f2a14SAndrew Morton if (unlikely(atomic_dec_and_test(&mm->mm_count))) { 694d70f2a14SAndrew Morton INIT_WORK(&mm->async_put_work, mmdrop_async_fn); 695d70f2a14SAndrew Morton schedule_work(&mm->async_put_work); 696d70f2a14SAndrew Morton } 697d70f2a14SAndrew Morton } 698d70f2a14SAndrew Morton 6991da177e4SLinus Torvalds static inline void free_signal_struct(struct signal_struct *sig) 7001da177e4SLinus Torvalds { 7011da177e4SLinus Torvalds taskstats_tgid_free(sig); 7021da177e4SLinus Torvalds sched_autogroup_exit(sig); 7031da177e4SLinus Torvalds /* 7041da177e4SLinus Torvalds * __mmdrop is not safe to call from softirq context on x86 due to 7051da177e4SLinus Torvalds * pgd_dtor so postpone it to the async context 7061da177e4SLinus Torvalds */ 7071da177e4SLinus Torvalds if (sig->oom_mm) 7081da177e4SLinus Torvalds mmdrop_async(sig->oom_mm); 7091da177e4SLinus Torvalds kmem_cache_free(signal_cachep, sig); 7101da177e4SLinus Torvalds } 7111da177e4SLinus Torvalds 7121da177e4SLinus Torvalds static inline void put_signal_struct(struct signal_struct *sig) 7131da177e4SLinus Torvalds { 71460d4de3fSElena Reshetova if (refcount_dec_and_test(&sig->sigcnt)) 7151da177e4SLinus Torvalds free_signal_struct(sig); 7161da177e4SLinus Torvalds } 7171da177e4SLinus Torvalds 7181da177e4SLinus Torvalds void __put_task_struct(struct task_struct *tsk) 7191da177e4SLinus Torvalds { 7201da177e4SLinus Torvalds WARN_ON(!tsk->exit_state); 721ec1d2819SElena Reshetova WARN_ON(refcount_read(&tsk->usage)); 7221da177e4SLinus Torvalds WARN_ON(tsk == current); 7231da177e4SLinus Torvalds 7241da177e4SLinus Torvalds cgroup_free(tsk); 7251da177e4SLinus Torvalds task_numa_free(tsk); 7261da177e4SLinus Torvalds security_task_free(tsk); 7271da177e4SLinus Torvalds exit_creds(tsk); 7281da177e4SLinus Torvalds delayacct_tsk_free(tsk); 7291da177e4SLinus Torvalds put_signal_struct(tsk->signal); 7301da177e4SLinus Torvalds 7311da177e4SLinus Torvalds if (!profile_handoff_task(tsk)) 7321da177e4SLinus Torvalds free_task(tsk); 7331da177e4SLinus Torvalds } 7341da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(__put_task_struct); 7351da177e4SLinus Torvalds 7361da177e4SLinus Torvalds void __init __weak arch_task_cache_init(void) { } 7371da177e4SLinus Torvalds 7381da177e4SLinus Torvalds /* 7391da177e4SLinus Torvalds * set_max_threads 7401da177e4SLinus Torvalds */ 7411da177e4SLinus Torvalds static void set_max_threads(unsigned int max_threads_suggested) 7421da177e4SLinus Torvalds { 7431da177e4SLinus Torvalds u64 threads; 744ca79b0c2SArun KS unsigned long nr_pages = totalram_pages(); 7451da177e4SLinus Torvalds 7461da177e4SLinus Torvalds /* 7471da177e4SLinus Torvalds * The number of threads shall be limited such that the thread 7481da177e4SLinus Torvalds * structures may only consume a small part of the available memory. 7491da177e4SLinus Torvalds */ 7503d6357deSArun KS if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64) 7511da177e4SLinus Torvalds threads = MAX_THREADS; 7521da177e4SLinus Torvalds else 7533d6357deSArun KS threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE, 7541da177e4SLinus Torvalds (u64) THREAD_SIZE * 8UL); 7551da177e4SLinus Torvalds 7561da177e4SLinus Torvalds if (threads > max_threads_suggested) 7571da177e4SLinus Torvalds threads = max_threads_suggested; 7581da177e4SLinus Torvalds 7591da177e4SLinus Torvalds max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS); 7601da177e4SLinus Torvalds } 7611da177e4SLinus Torvalds 7621da177e4SLinus Torvalds #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT 7631da177e4SLinus Torvalds /* Initialized by the architecture: */ 7641da177e4SLinus Torvalds int arch_task_struct_size __read_mostly; 7651da177e4SLinus Torvalds #endif 7661da177e4SLinus Torvalds 7675905429aSKees Cook static void task_struct_whitelist(unsigned long *offset, unsigned long *size) 7685905429aSKees Cook { 7695905429aSKees Cook /* Fetch thread_struct whitelist for the architecture. */ 7705905429aSKees Cook arch_thread_struct_whitelist(offset, size); 7715905429aSKees Cook 7725905429aSKees Cook /* 7735905429aSKees Cook * Handle zero-sized whitelist or empty thread_struct, otherwise 7745905429aSKees Cook * adjust offset to position of thread_struct in task_struct. 7755905429aSKees Cook */ 7765905429aSKees Cook if (unlikely(*size == 0)) 7775905429aSKees Cook *offset = 0; 7785905429aSKees Cook else 7795905429aSKees Cook *offset += offsetof(struct task_struct, thread); 7805905429aSKees Cook } 7815905429aSKees Cook 7821da177e4SLinus Torvalds void __init fork_init(void) 7831da177e4SLinus Torvalds { 7841da177e4SLinus Torvalds int i; 7851da177e4SLinus Torvalds #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR 7861da177e4SLinus Torvalds #ifndef ARCH_MIN_TASKALIGN 7871da177e4SLinus Torvalds #define ARCH_MIN_TASKALIGN 0 7881da177e4SLinus Torvalds #endif 7891da177e4SLinus Torvalds int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN); 7905905429aSKees Cook unsigned long useroffset, usersize; 7911da177e4SLinus Torvalds 7921da177e4SLinus Torvalds /* create a slab on which task_structs can be allocated */ 7935905429aSKees Cook task_struct_whitelist(&useroffset, &usersize); 7945905429aSKees Cook task_struct_cachep = kmem_cache_create_usercopy("task_struct", 7951da177e4SLinus Torvalds arch_task_struct_size, align, 7965905429aSKees Cook SLAB_PANIC|SLAB_ACCOUNT, 7975905429aSKees Cook useroffset, usersize, NULL); 7981da177e4SLinus Torvalds #endif 7991da177e4SLinus Torvalds 8001da177e4SLinus Torvalds /* do the arch specific task caches init */ 8011da177e4SLinus Torvalds arch_task_cache_init(); 8021da177e4SLinus Torvalds 8031da177e4SLinus Torvalds set_max_threads(MAX_THREADS); 8041da177e4SLinus Torvalds 8051da177e4SLinus Torvalds init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; 8061da177e4SLinus Torvalds init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; 8071da177e4SLinus Torvalds init_task.signal->rlim[RLIMIT_SIGPENDING] = 8081da177e4SLinus Torvalds init_task.signal->rlim[RLIMIT_NPROC]; 8091da177e4SLinus Torvalds 8101da177e4SLinus Torvalds for (i = 0; i < UCOUNT_COUNTS; i++) { 8111da177e4SLinus Torvalds init_user_ns.ucount_max[i] = max_threads/2; 8121da177e4SLinus Torvalds } 8131da177e4SLinus Torvalds 8141da177e4SLinus Torvalds #ifdef CONFIG_VMAP_STACK 8151da177e4SLinus Torvalds cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", 8161da177e4SLinus Torvalds NULL, free_vm_stack_cache); 8171da177e4SLinus Torvalds #endif 8181da177e4SLinus Torvalds 8191da177e4SLinus Torvalds lockdep_init_task(&init_task); 820aad42dd4SNadav Amit uprobes_init(); 8211da177e4SLinus Torvalds } 8221da177e4SLinus Torvalds 8231da177e4SLinus Torvalds int __weak arch_dup_task_struct(struct task_struct *dst, 8241da177e4SLinus Torvalds struct task_struct *src) 8251da177e4SLinus Torvalds { 8261da177e4SLinus Torvalds *dst = *src; 8271da177e4SLinus Torvalds return 0; 8281da177e4SLinus Torvalds } 8291da177e4SLinus Torvalds 8301da177e4SLinus Torvalds void set_task_stack_end_magic(struct task_struct *tsk) 8311da177e4SLinus Torvalds { 8321da177e4SLinus Torvalds unsigned long *stackend; 8331da177e4SLinus Torvalds 8341da177e4SLinus Torvalds stackend = end_of_stack(tsk); 8351da177e4SLinus Torvalds *stackend = STACK_END_MAGIC; /* for overflow detection */ 8361da177e4SLinus Torvalds } 8371da177e4SLinus Torvalds 8381da177e4SLinus Torvalds static struct task_struct *dup_task_struct(struct task_struct *orig, int node) 8391da177e4SLinus Torvalds { 8401da177e4SLinus Torvalds struct task_struct *tsk; 8411da177e4SLinus Torvalds unsigned long *stack; 8420f4991e8SYueHaibing struct vm_struct *stack_vm_area __maybe_unused; 8431da177e4SLinus Torvalds int err; 8441da177e4SLinus Torvalds 8451da177e4SLinus Torvalds if (node == NUMA_NO_NODE) 8461da177e4SLinus Torvalds node = tsk_fork_get_node(orig); 8471da177e4SLinus Torvalds tsk = alloc_task_struct_node(node); 8481da177e4SLinus Torvalds if (!tsk) 8491da177e4SLinus Torvalds return NULL; 8501da177e4SLinus Torvalds 8511da177e4SLinus Torvalds stack = alloc_thread_stack_node(tsk, node); 8521da177e4SLinus Torvalds if (!stack) 8531da177e4SLinus Torvalds goto free_tsk; 8541da177e4SLinus Torvalds 8559b6f7e16SRoman Gushchin if (memcg_charge_kernel_stack(tsk)) 8569b6f7e16SRoman Gushchin goto free_stack; 8579b6f7e16SRoman Gushchin 8581da177e4SLinus Torvalds stack_vm_area = task_stack_vm_area(tsk); 8591da177e4SLinus Torvalds 8601da177e4SLinus Torvalds err = arch_dup_task_struct(tsk, orig); 8611da177e4SLinus Torvalds 8621da177e4SLinus Torvalds /* 8631da177e4SLinus Torvalds * arch_dup_task_struct() clobbers the stack-related fields. Make 8641da177e4SLinus Torvalds * sure they're properly initialized before using any stack-related 8651da177e4SLinus Torvalds * functions again. 8661da177e4SLinus Torvalds */ 8671da177e4SLinus Torvalds tsk->stack = stack; 8681da177e4SLinus Torvalds #ifdef CONFIG_VMAP_STACK 8691da177e4SLinus Torvalds tsk->stack_vm_area = stack_vm_area; 8701da177e4SLinus Torvalds #endif 8711da177e4SLinus Torvalds #ifdef CONFIG_THREAD_INFO_IN_TASK 872f0b89d39SElena Reshetova refcount_set(&tsk->stack_refcount, 1); 8731da177e4SLinus Torvalds #endif 8741da177e4SLinus Torvalds 8751da177e4SLinus Torvalds if (err) 8761da177e4SLinus Torvalds goto free_stack; 8771da177e4SLinus Torvalds 8781da177e4SLinus Torvalds #ifdef CONFIG_SECCOMP 8791da177e4SLinus Torvalds /* 8801da177e4SLinus Torvalds * We must handle setting up seccomp filters once we're under 8811da177e4SLinus Torvalds * the sighand lock in case orig has changed between now and 8821da177e4SLinus Torvalds * then. Until then, filter must be NULL to avoid messing up 8831da177e4SLinus Torvalds * the usage counts on the error path calling free_task. 8841da177e4SLinus Torvalds */ 8851da177e4SLinus Torvalds tsk->seccomp.filter = NULL; 8861da177e4SLinus Torvalds #endif 8871da177e4SLinus Torvalds 8881da177e4SLinus Torvalds setup_thread_stack(tsk, orig); 8891da177e4SLinus Torvalds clear_user_return_notifier(tsk); 8901da177e4SLinus Torvalds clear_tsk_need_resched(tsk); 8911da177e4SLinus Torvalds set_task_stack_end_magic(tsk); 8921da177e4SLinus Torvalds 893050e9baaSLinus Torvalds #ifdef CONFIG_STACKPROTECTOR 8941da177e4SLinus Torvalds tsk->stack_canary = get_random_canary(); 8951da177e4SLinus Torvalds #endif 8961da177e4SLinus Torvalds 8971da177e4SLinus Torvalds /* 8981da177e4SLinus Torvalds * One for us, one for whoever does the "release_task()" (usually 8991da177e4SLinus Torvalds * parent) 9001da177e4SLinus Torvalds */ 901ec1d2819SElena Reshetova refcount_set(&tsk->usage, 2); 9021da177e4SLinus Torvalds #ifdef CONFIG_BLK_DEV_IO_TRACE 9031da177e4SLinus Torvalds tsk->btrace_seq = 0; 9041da177e4SLinus Torvalds #endif 9051da177e4SLinus Torvalds tsk->splice_pipe = NULL; 9061da177e4SLinus Torvalds tsk->task_frag.page = NULL; 9071da177e4SLinus Torvalds tsk->wake_q.next = NULL; 9081da177e4SLinus Torvalds 9091da177e4SLinus Torvalds account_kernel_stack(tsk, 1); 9101da177e4SLinus Torvalds 9111da177e4SLinus Torvalds kcov_task_init(tsk); 9121da177e4SLinus Torvalds 9131da177e4SLinus Torvalds #ifdef CONFIG_FAULT_INJECTION 9141da177e4SLinus Torvalds tsk->fail_nth = 0; 9151da177e4SLinus Torvalds #endif 9161da177e4SLinus Torvalds 9172c323017SJosef Bacik #ifdef CONFIG_BLK_CGROUP 9182c323017SJosef Bacik tsk->throttle_queue = NULL; 9192c323017SJosef Bacik tsk->use_memdelay = 0; 9202c323017SJosef Bacik #endif 9212c323017SJosef Bacik 922d46eb14bSShakeel Butt #ifdef CONFIG_MEMCG 923d46eb14bSShakeel Butt tsk->active_memcg = NULL; 924d46eb14bSShakeel Butt #endif 9251da177e4SLinus Torvalds return tsk; 9261da177e4SLinus Torvalds 9271da177e4SLinus Torvalds free_stack: 9281da177e4SLinus Torvalds free_thread_stack(tsk); 9291da177e4SLinus Torvalds free_tsk: 9301da177e4SLinus Torvalds free_task_struct(tsk); 9311da177e4SLinus Torvalds return NULL; 9321da177e4SLinus Torvalds } 9331da177e4SLinus Torvalds 9341da177e4SLinus Torvalds __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); 9351da177e4SLinus Torvalds 9364cb0e11bSHidehiro Kawai static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT; 9374cb0e11bSHidehiro Kawai 9384cb0e11bSHidehiro Kawai static int __init coredump_filter_setup(char *s) 9394cb0e11bSHidehiro Kawai { 9404cb0e11bSHidehiro Kawai default_dump_filter = 9414cb0e11bSHidehiro Kawai (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) & 9424cb0e11bSHidehiro Kawai MMF_DUMP_FILTER_MASK; 9434cb0e11bSHidehiro Kawai return 1; 9444cb0e11bSHidehiro Kawai } 9454cb0e11bSHidehiro Kawai 9464cb0e11bSHidehiro Kawai __setup("coredump_filter=", coredump_filter_setup); 9474cb0e11bSHidehiro Kawai 9481da177e4SLinus Torvalds #include <linux/init_task.h> 9491da177e4SLinus Torvalds 950858f0993SAlexey Dobriyan static void mm_init_aio(struct mm_struct *mm) 951858f0993SAlexey Dobriyan { 952858f0993SAlexey Dobriyan #ifdef CONFIG_AIO 953858f0993SAlexey Dobriyan spin_lock_init(&mm->ioctx_lock); 954db446a08SBenjamin LaHaise mm->ioctx_table = NULL; 955858f0993SAlexey Dobriyan #endif 956858f0993SAlexey Dobriyan } 957858f0993SAlexey Dobriyan 958c3f3ce04SAndrea Arcangeli static __always_inline void mm_clear_owner(struct mm_struct *mm, 959c3f3ce04SAndrea Arcangeli struct task_struct *p) 960c3f3ce04SAndrea Arcangeli { 961c3f3ce04SAndrea Arcangeli #ifdef CONFIG_MEMCG 962c3f3ce04SAndrea Arcangeli if (mm->owner == p) 963c3f3ce04SAndrea Arcangeli WRITE_ONCE(mm->owner, NULL); 964c3f3ce04SAndrea Arcangeli #endif 965c3f3ce04SAndrea Arcangeli } 966c3f3ce04SAndrea Arcangeli 96733144e84SVladimir Davydov static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) 96833144e84SVladimir Davydov { 96933144e84SVladimir Davydov #ifdef CONFIG_MEMCG 97033144e84SVladimir Davydov mm->owner = p; 97133144e84SVladimir Davydov #endif 97233144e84SVladimir Davydov } 97333144e84SVladimir Davydov 974355627f5SEric Biggers static void mm_init_uprobes_state(struct mm_struct *mm) 975355627f5SEric Biggers { 976355627f5SEric Biggers #ifdef CONFIG_UPROBES 977355627f5SEric Biggers mm->uprobes_state.xol_area = NULL; 978355627f5SEric Biggers #endif 979355627f5SEric Biggers } 980355627f5SEric Biggers 981bfedb589SEric W. Biederman static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, 982bfedb589SEric W. Biederman struct user_namespace *user_ns) 9831da177e4SLinus Torvalds { 98441f727fdSVladimir Davydov mm->mmap = NULL; 98541f727fdSVladimir Davydov mm->mm_rb = RB_ROOT; 98641f727fdSVladimir Davydov mm->vmacache_seqnum = 0; 9871da177e4SLinus Torvalds atomic_set(&mm->mm_users, 1); 9881da177e4SLinus Torvalds atomic_set(&mm->mm_count, 1); 9891da177e4SLinus Torvalds init_rwsem(&mm->mmap_sem); 9901da177e4SLinus Torvalds INIT_LIST_HEAD(&mm->mmlist); 991999d9fc1SOleg Nesterov mm->core_state = NULL; 992af5b0f6aSKirill A. Shutemov mm_pgtables_bytes_init(mm); 99341f727fdSVladimir Davydov mm->map_count = 0; 99441f727fdSVladimir Davydov mm->locked_vm = 0; 99570f8a3caSDavidlohr Bueso atomic64_set(&mm->pinned_vm, 0); 996d559db08SKAMEZAWA Hiroyuki memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); 9971da177e4SLinus Torvalds spin_lock_init(&mm->page_table_lock); 99888aa7cc6SYang Shi spin_lock_init(&mm->arg_lock); 99941f727fdSVladimir Davydov mm_init_cpumask(mm); 1000858f0993SAlexey Dobriyan mm_init_aio(mm); 1001cf475ad2SBalbir Singh mm_init_owner(mm, p); 10022b7e8665SEric Biggers RCU_INIT_POINTER(mm->exe_file, NULL); 100341f727fdSVladimir Davydov mmu_notifier_mm_init(mm); 1004133ff0eaSJérôme Glisse hmm_mm_init(mm); 100516af97dcSNadav Amit init_tlb_flush_pending(mm); 100641f727fdSVladimir Davydov #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS 100741f727fdSVladimir Davydov mm->pmd_huge_pte = NULL; 100841f727fdSVladimir Davydov #endif 1009355627f5SEric Biggers mm_init_uprobes_state(mm); 10101da177e4SLinus Torvalds 1011a0715cc2SAlex Thorlton if (current->mm) { 1012a0715cc2SAlex Thorlton mm->flags = current->mm->flags & MMF_INIT_MASK; 1013a0715cc2SAlex Thorlton mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; 1014a0715cc2SAlex Thorlton } else { 1015a0715cc2SAlex Thorlton mm->flags = default_dump_filter; 10161da177e4SLinus Torvalds mm->def_flags = 0; 1017a0715cc2SAlex Thorlton } 1018a0715cc2SAlex Thorlton 101941f727fdSVladimir Davydov if (mm_alloc_pgd(mm)) 102041f727fdSVladimir Davydov goto fail_nopgd; 102178fb7466SPavel Emelianov 102241f727fdSVladimir Davydov if (init_new_context(p, mm)) 102341f727fdSVladimir Davydov goto fail_nocontext; 102441f727fdSVladimir Davydov 1025bfedb589SEric W. Biederman mm->user_ns = get_user_ns(user_ns); 102641f727fdSVladimir Davydov return mm; 102741f727fdSVladimir Davydov 102841f727fdSVladimir Davydov fail_nocontext: 102941f727fdSVladimir Davydov mm_free_pgd(mm); 103041f727fdSVladimir Davydov fail_nopgd: 10311da177e4SLinus Torvalds free_mm(mm); 10321da177e4SLinus Torvalds return NULL; 10331da177e4SLinus Torvalds } 10341da177e4SLinus Torvalds 10351da177e4SLinus Torvalds /* 10361da177e4SLinus Torvalds * Allocate and initialize an mm_struct. 10371da177e4SLinus Torvalds */ 10381da177e4SLinus Torvalds struct mm_struct *mm_alloc(void) 10391da177e4SLinus Torvalds { 10401da177e4SLinus Torvalds struct mm_struct *mm; 10411da177e4SLinus Torvalds 10421da177e4SLinus Torvalds mm = allocate_mm(); 1043de03c72cSKOSAKI Motohiro if (!mm) 1044de03c72cSKOSAKI Motohiro return NULL; 1045de03c72cSKOSAKI Motohiro 10461da177e4SLinus Torvalds memset(mm, 0, sizeof(*mm)); 1047bfedb589SEric W. Biederman return mm_init(mm, current, current_user_ns()); 10481da177e4SLinus Torvalds } 10491da177e4SLinus Torvalds 1050ec8d7c14SMichal Hocko static inline void __mmput(struct mm_struct *mm) 10511da177e4SLinus Torvalds { 1052ec8d7c14SMichal Hocko VM_BUG_ON(atomic_read(&mm->mm_users)); 10530ae26f1bSAndrew Morton 1054d4b3b638SSrikar Dronamraju uprobe_clear_state(mm); 10551da177e4SLinus Torvalds exit_aio(mm); 10561c2fb7a4SAndrea Arcangeli ksm_exit(mm); 1057ba76149fSAndrea Arcangeli khugepaged_exit(mm); /* must run before exit_mmap */ 10581da177e4SLinus Torvalds exit_mmap(mm); 10596fcb52a5SAaron Lu mm_put_huge_zero_page(mm); 1060925d1c40SMatt Helsley set_mm_exe_file(mm, NULL); 10611da177e4SLinus Torvalds if (!list_empty(&mm->mmlist)) { 10621da177e4SLinus Torvalds spin_lock(&mmlist_lock); 10631da177e4SLinus Torvalds list_del(&mm->mmlist); 10641da177e4SLinus Torvalds spin_unlock(&mmlist_lock); 10651da177e4SLinus Torvalds } 1066801460d0SHiroshi Shimamoto if (mm->binfmt) 1067801460d0SHiroshi Shimamoto module_put(mm->binfmt->module); 10681da177e4SLinus Torvalds mmdrop(mm); 10691da177e4SLinus Torvalds } 1070ec8d7c14SMichal Hocko 1071ec8d7c14SMichal Hocko /* 1072ec8d7c14SMichal Hocko * Decrement the use count and release all resources for an mm. 1073ec8d7c14SMichal Hocko */ 1074ec8d7c14SMichal Hocko void mmput(struct mm_struct *mm) 1075ec8d7c14SMichal Hocko { 1076ec8d7c14SMichal Hocko might_sleep(); 1077ec8d7c14SMichal Hocko 1078ec8d7c14SMichal Hocko if (atomic_dec_and_test(&mm->mm_users)) 1079ec8d7c14SMichal Hocko __mmput(mm); 10801da177e4SLinus Torvalds } 10811da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(mmput); 10821da177e4SLinus Torvalds 1083a1b2289cSSherry Yang #ifdef CONFIG_MMU 1084a1b2289cSSherry Yang static void mmput_async_fn(struct work_struct *work) 1085a1b2289cSSherry Yang { 1086a1b2289cSSherry Yang struct mm_struct *mm = container_of(work, struct mm_struct, 1087a1b2289cSSherry Yang async_put_work); 1088a1b2289cSSherry Yang 1089a1b2289cSSherry Yang __mmput(mm); 1090a1b2289cSSherry Yang } 1091a1b2289cSSherry Yang 1092a1b2289cSSherry Yang void mmput_async(struct mm_struct *mm) 1093a1b2289cSSherry Yang { 1094a1b2289cSSherry Yang if (atomic_dec_and_test(&mm->mm_users)) { 1095a1b2289cSSherry Yang INIT_WORK(&mm->async_put_work, mmput_async_fn); 1096a1b2289cSSherry Yang schedule_work(&mm->async_put_work); 1097a1b2289cSSherry Yang } 1098a1b2289cSSherry Yang } 1099a1b2289cSSherry Yang #endif 1100a1b2289cSSherry Yang 110190f31d0eSKonstantin Khlebnikov /** 110290f31d0eSKonstantin Khlebnikov * set_mm_exe_file - change a reference to the mm's executable file 110390f31d0eSKonstantin Khlebnikov * 110490f31d0eSKonstantin Khlebnikov * This changes mm's executable file (shown as symlink /proc/[pid]/exe). 110590f31d0eSKonstantin Khlebnikov * 11066e399cd1SDavidlohr Bueso * Main users are mmput() and sys_execve(). Callers prevent concurrent 11076e399cd1SDavidlohr Bueso * invocations: in mmput() nobody alive left, in execve task is single 11086e399cd1SDavidlohr Bueso * threaded. sys_prctl(PR_SET_MM_MAP/EXE_FILE) also needs to set the 11096e399cd1SDavidlohr Bueso * mm->exe_file, but does so without using set_mm_exe_file() in order 11106e399cd1SDavidlohr Bueso * to do avoid the need for any locks. 111190f31d0eSKonstantin Khlebnikov */ 111238646013SJiri Slaby void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) 111338646013SJiri Slaby { 11146e399cd1SDavidlohr Bueso struct file *old_exe_file; 11156e399cd1SDavidlohr Bueso 11166e399cd1SDavidlohr Bueso /* 11176e399cd1SDavidlohr Bueso * It is safe to dereference the exe_file without RCU as 11186e399cd1SDavidlohr Bueso * this function is only called if nobody else can access 11196e399cd1SDavidlohr Bueso * this mm -- see comment above for justification. 11206e399cd1SDavidlohr Bueso */ 11216e399cd1SDavidlohr Bueso old_exe_file = rcu_dereference_raw(mm->exe_file); 112290f31d0eSKonstantin Khlebnikov 112338646013SJiri Slaby if (new_exe_file) 112438646013SJiri Slaby get_file(new_exe_file); 112590f31d0eSKonstantin Khlebnikov rcu_assign_pointer(mm->exe_file, new_exe_file); 112690f31d0eSKonstantin Khlebnikov if (old_exe_file) 112790f31d0eSKonstantin Khlebnikov fput(old_exe_file); 112838646013SJiri Slaby } 112938646013SJiri Slaby 113090f31d0eSKonstantin Khlebnikov /** 113190f31d0eSKonstantin Khlebnikov * get_mm_exe_file - acquire a reference to the mm's executable file 113290f31d0eSKonstantin Khlebnikov * 113390f31d0eSKonstantin Khlebnikov * Returns %NULL if mm has no associated executable file. 113490f31d0eSKonstantin Khlebnikov * User must release file via fput(). 113590f31d0eSKonstantin Khlebnikov */ 113638646013SJiri Slaby struct file *get_mm_exe_file(struct mm_struct *mm) 113738646013SJiri Slaby { 113838646013SJiri Slaby struct file *exe_file; 113938646013SJiri Slaby 114090f31d0eSKonstantin Khlebnikov rcu_read_lock(); 114190f31d0eSKonstantin Khlebnikov exe_file = rcu_dereference(mm->exe_file); 114290f31d0eSKonstantin Khlebnikov if (exe_file && !get_file_rcu(exe_file)) 114390f31d0eSKonstantin Khlebnikov exe_file = NULL; 114490f31d0eSKonstantin Khlebnikov rcu_read_unlock(); 114538646013SJiri Slaby return exe_file; 114638646013SJiri Slaby } 114711163348SDavidlohr Bueso EXPORT_SYMBOL(get_mm_exe_file); 114838646013SJiri Slaby 11491da177e4SLinus Torvalds /** 1150cd81a917SMateusz Guzik * get_task_exe_file - acquire a reference to the task's executable file 1151cd81a917SMateusz Guzik * 1152cd81a917SMateusz Guzik * Returns %NULL if task's mm (if any) has no associated executable file or 1153cd81a917SMateusz Guzik * this is a kernel thread with borrowed mm (see the comment above get_task_mm). 1154cd81a917SMateusz Guzik * User must release file via fput(). 1155cd81a917SMateusz Guzik */ 1156cd81a917SMateusz Guzik struct file *get_task_exe_file(struct task_struct *task) 1157cd81a917SMateusz Guzik { 1158cd81a917SMateusz Guzik struct file *exe_file = NULL; 1159cd81a917SMateusz Guzik struct mm_struct *mm; 1160cd81a917SMateusz Guzik 1161cd81a917SMateusz Guzik task_lock(task); 1162cd81a917SMateusz Guzik mm = task->mm; 1163cd81a917SMateusz Guzik if (mm) { 1164cd81a917SMateusz Guzik if (!(task->flags & PF_KTHREAD)) 1165cd81a917SMateusz Guzik exe_file = get_mm_exe_file(mm); 1166cd81a917SMateusz Guzik } 1167cd81a917SMateusz Guzik task_unlock(task); 1168cd81a917SMateusz Guzik return exe_file; 1169cd81a917SMateusz Guzik } 1170cd81a917SMateusz Guzik EXPORT_SYMBOL(get_task_exe_file); 1171cd81a917SMateusz Guzik 1172cd81a917SMateusz Guzik /** 11731da177e4SLinus Torvalds * get_task_mm - acquire a reference to the task's mm 11741da177e4SLinus Torvalds * 1175246bb0b1SOleg Nesterov * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning 11761da177e4SLinus Torvalds * this kernel workthread has transiently adopted a user mm with use_mm, 11771da177e4SLinus Torvalds * to do its AIO) is not set and if so returns a reference to it, after 11781da177e4SLinus Torvalds * bumping up the use count. User must release the mm via mmput() 11791da177e4SLinus Torvalds * after use. Typically used by /proc and ptrace. 11801da177e4SLinus Torvalds */ 11811da177e4SLinus Torvalds struct mm_struct *get_task_mm(struct task_struct *task) 11821da177e4SLinus Torvalds { 11831da177e4SLinus Torvalds struct mm_struct *mm; 11841da177e4SLinus Torvalds 11851da177e4SLinus Torvalds task_lock(task); 11861da177e4SLinus Torvalds mm = task->mm; 11871da177e4SLinus Torvalds if (mm) { 1188246bb0b1SOleg Nesterov if (task->flags & PF_KTHREAD) 11891da177e4SLinus Torvalds mm = NULL; 11901da177e4SLinus Torvalds else 11913fce371bSVegard Nossum mmget(mm); 11921da177e4SLinus Torvalds } 11931da177e4SLinus Torvalds task_unlock(task); 11941da177e4SLinus Torvalds return mm; 11951da177e4SLinus Torvalds } 11961da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(get_task_mm); 11971da177e4SLinus Torvalds 11988cdb878dSChristopher Yeoh struct mm_struct *mm_access(struct task_struct *task, unsigned int mode) 11998cdb878dSChristopher Yeoh { 12008cdb878dSChristopher Yeoh struct mm_struct *mm; 12018cdb878dSChristopher Yeoh int err; 12028cdb878dSChristopher Yeoh 12038cdb878dSChristopher Yeoh err = mutex_lock_killable(&task->signal->cred_guard_mutex); 12048cdb878dSChristopher Yeoh if (err) 12058cdb878dSChristopher Yeoh return ERR_PTR(err); 12068cdb878dSChristopher Yeoh 12078cdb878dSChristopher Yeoh mm = get_task_mm(task); 12088cdb878dSChristopher Yeoh if (mm && mm != current->mm && 12098cdb878dSChristopher Yeoh !ptrace_may_access(task, mode)) { 12108cdb878dSChristopher Yeoh mmput(mm); 12118cdb878dSChristopher Yeoh mm = ERR_PTR(-EACCES); 12128cdb878dSChristopher Yeoh } 12138cdb878dSChristopher Yeoh mutex_unlock(&task->signal->cred_guard_mutex); 12148cdb878dSChristopher Yeoh 12158cdb878dSChristopher Yeoh return mm; 12168cdb878dSChristopher Yeoh } 12178cdb878dSChristopher Yeoh 121857b59c4aSOleg Nesterov static void complete_vfork_done(struct task_struct *tsk) 1219c415c3b4SOleg Nesterov { 1220d68b46feSOleg Nesterov struct completion *vfork; 1221c415c3b4SOleg Nesterov 1222d68b46feSOleg Nesterov task_lock(tsk); 1223d68b46feSOleg Nesterov vfork = tsk->vfork_done; 1224d68b46feSOleg Nesterov if (likely(vfork)) { 1225c415c3b4SOleg Nesterov tsk->vfork_done = NULL; 1226d68b46feSOleg Nesterov complete(vfork); 1227d68b46feSOleg Nesterov } 1228d68b46feSOleg Nesterov task_unlock(tsk); 1229d68b46feSOleg Nesterov } 1230d68b46feSOleg Nesterov 1231d68b46feSOleg Nesterov static int wait_for_vfork_done(struct task_struct *child, 1232d68b46feSOleg Nesterov struct completion *vfork) 1233d68b46feSOleg Nesterov { 1234d68b46feSOleg Nesterov int killed; 1235d68b46feSOleg Nesterov 1236d68b46feSOleg Nesterov freezer_do_not_count(); 123776f969e8SRoman Gushchin cgroup_enter_frozen(); 1238d68b46feSOleg Nesterov killed = wait_for_completion_killable(vfork); 123976f969e8SRoman Gushchin cgroup_leave_frozen(false); 1240d68b46feSOleg Nesterov freezer_count(); 1241d68b46feSOleg Nesterov 1242d68b46feSOleg Nesterov if (killed) { 1243d68b46feSOleg Nesterov task_lock(child); 1244d68b46feSOleg Nesterov child->vfork_done = NULL; 1245d68b46feSOleg Nesterov task_unlock(child); 1246d68b46feSOleg Nesterov } 1247d68b46feSOleg Nesterov 1248d68b46feSOleg Nesterov put_task_struct(child); 1249d68b46feSOleg Nesterov return killed; 1250c415c3b4SOleg Nesterov } 1251c415c3b4SOleg Nesterov 12521da177e4SLinus Torvalds /* Please note the differences between mmput and mm_release. 12531da177e4SLinus Torvalds * mmput is called whenever we stop holding onto a mm_struct, 12541da177e4SLinus Torvalds * error success whatever. 12551da177e4SLinus Torvalds * 12561da177e4SLinus Torvalds * mm_release is called after a mm_struct has been removed 12571da177e4SLinus Torvalds * from the current process. 12581da177e4SLinus Torvalds * 12591da177e4SLinus Torvalds * This difference is important for error handling, when we 12601da177e4SLinus Torvalds * only half set up a mm_struct for a new process and need to restore 12611da177e4SLinus Torvalds * the old one. Because we mmput the new mm_struct before 12621da177e4SLinus Torvalds * restoring the old one. . . 12631da177e4SLinus Torvalds * Eric Biederman 10 January 1998 12641da177e4SLinus Torvalds */ 12651da177e4SLinus Torvalds void mm_release(struct task_struct *tsk, struct mm_struct *mm) 12661da177e4SLinus Torvalds { 12678141c7f3SLinus Torvalds /* Get rid of any futexes when releasing the mm */ 12688141c7f3SLinus Torvalds #ifdef CONFIG_FUTEX 1269fc6b177dSPeter Zijlstra if (unlikely(tsk->robust_list)) { 12708141c7f3SLinus Torvalds exit_robust_list(tsk); 1271fc6b177dSPeter Zijlstra tsk->robust_list = NULL; 1272fc6b177dSPeter Zijlstra } 12738141c7f3SLinus Torvalds #ifdef CONFIG_COMPAT 1274fc6b177dSPeter Zijlstra if (unlikely(tsk->compat_robust_list)) { 12758141c7f3SLinus Torvalds compat_exit_robust_list(tsk); 1276fc6b177dSPeter Zijlstra tsk->compat_robust_list = NULL; 1277fc6b177dSPeter Zijlstra } 12788141c7f3SLinus Torvalds #endif 1279322a2c10SThomas Gleixner if (unlikely(!list_empty(&tsk->pi_state_list))) 1280322a2c10SThomas Gleixner exit_pi_state_list(tsk); 12818141c7f3SLinus Torvalds #endif 12828141c7f3SLinus Torvalds 12830326f5a9SSrikar Dronamraju uprobe_free_utask(tsk); 12840326f5a9SSrikar Dronamraju 12851da177e4SLinus Torvalds /* Get rid of any cached register state */ 12861da177e4SLinus Torvalds deactivate_mm(tsk, mm); 12871da177e4SLinus Torvalds 1288fec1d011SRoland McGrath /* 1289735f2770SMichal Hocko * Signal userspace if we're not exiting with a core dump 1290735f2770SMichal Hocko * because we want to leave the value intact for debugging 1291735f2770SMichal Hocko * purposes. 1292fec1d011SRoland McGrath */ 12939c8a8228SEric Dumazet if (tsk->clear_child_tid) { 1294735f2770SMichal Hocko if (!(tsk->signal->flags & SIGNAL_GROUP_COREDUMP) && 12959c8a8228SEric Dumazet atomic_read(&mm->mm_users) > 1) { 12961da177e4SLinus Torvalds /* 12971da177e4SLinus Torvalds * We don't check the error code - if userspace has 12981da177e4SLinus Torvalds * not set up a proper pointer then tough luck. 12991da177e4SLinus Torvalds */ 13009c8a8228SEric Dumazet put_user(0, tsk->clear_child_tid); 13012de0db99SDominik Brodowski do_futex(tsk->clear_child_tid, FUTEX_WAKE, 13022de0db99SDominik Brodowski 1, NULL, NULL, 0, 0); 13039c8a8228SEric Dumazet } 13049c8a8228SEric Dumazet tsk->clear_child_tid = NULL; 13051da177e4SLinus Torvalds } 1306f7505d64SKonstantin Khlebnikov 1307f7505d64SKonstantin Khlebnikov /* 1308f7505d64SKonstantin Khlebnikov * All done, finally we can wake up parent and return this mm to him. 1309f7505d64SKonstantin Khlebnikov * Also kthread_stop() uses this completion for synchronization. 1310f7505d64SKonstantin Khlebnikov */ 1311f7505d64SKonstantin Khlebnikov if (tsk->vfork_done) 1312f7505d64SKonstantin Khlebnikov complete_vfork_done(tsk); 13131da177e4SLinus Torvalds } 13141da177e4SLinus Torvalds 131513585fa0SNadav Amit /** 131613585fa0SNadav Amit * dup_mm() - duplicates an existing mm structure 131713585fa0SNadav Amit * @tsk: the task_struct with which the new mm will be associated. 131813585fa0SNadav Amit * @oldmm: the mm to duplicate. 131913585fa0SNadav Amit * 132013585fa0SNadav Amit * Allocates a new mm structure and duplicates the provided @oldmm structure 132113585fa0SNadav Amit * content into it. 132213585fa0SNadav Amit * 132313585fa0SNadav Amit * Return: the duplicated mm or NULL on failure. 1324a0a7ec30SJANAK DESAI */ 132513585fa0SNadav Amit static struct mm_struct *dup_mm(struct task_struct *tsk, 132613585fa0SNadav Amit struct mm_struct *oldmm) 1327a0a7ec30SJANAK DESAI { 132813585fa0SNadav Amit struct mm_struct *mm; 1329a0a7ec30SJANAK DESAI int err; 1330a0a7ec30SJANAK DESAI 1331a0a7ec30SJANAK DESAI mm = allocate_mm(); 1332a0a7ec30SJANAK DESAI if (!mm) 1333a0a7ec30SJANAK DESAI goto fail_nomem; 1334a0a7ec30SJANAK DESAI 1335a0a7ec30SJANAK DESAI memcpy(mm, oldmm, sizeof(*mm)); 1336a0a7ec30SJANAK DESAI 1337bfedb589SEric W. Biederman if (!mm_init(mm, tsk, mm->user_ns)) 1338a0a7ec30SJANAK DESAI goto fail_nomem; 1339a0a7ec30SJANAK DESAI 1340a0a7ec30SJANAK DESAI err = dup_mmap(mm, oldmm); 1341a0a7ec30SJANAK DESAI if (err) 1342a0a7ec30SJANAK DESAI goto free_pt; 1343a0a7ec30SJANAK DESAI 1344a0a7ec30SJANAK DESAI mm->hiwater_rss = get_mm_rss(mm); 1345a0a7ec30SJANAK DESAI mm->hiwater_vm = mm->total_vm; 1346a0a7ec30SJANAK DESAI 1347801460d0SHiroshi Shimamoto if (mm->binfmt && !try_module_get(mm->binfmt->module)) 1348801460d0SHiroshi Shimamoto goto free_pt; 1349801460d0SHiroshi Shimamoto 1350a0a7ec30SJANAK DESAI return mm; 1351a0a7ec30SJANAK DESAI 1352a0a7ec30SJANAK DESAI free_pt: 1353801460d0SHiroshi Shimamoto /* don't put binfmt in mmput, we haven't got module yet */ 1354801460d0SHiroshi Shimamoto mm->binfmt = NULL; 1355c3f3ce04SAndrea Arcangeli mm_init_owner(mm, NULL); 1356a0a7ec30SJANAK DESAI mmput(mm); 1357a0a7ec30SJANAK DESAI 1358a0a7ec30SJANAK DESAI fail_nomem: 1359a0a7ec30SJANAK DESAI return NULL; 1360a0a7ec30SJANAK DESAI } 1361a0a7ec30SJANAK DESAI 13621da177e4SLinus Torvalds static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) 13631da177e4SLinus Torvalds { 13641da177e4SLinus Torvalds struct mm_struct *mm, *oldmm; 13651da177e4SLinus Torvalds int retval; 13661da177e4SLinus Torvalds 13671da177e4SLinus Torvalds tsk->min_flt = tsk->maj_flt = 0; 13681da177e4SLinus Torvalds tsk->nvcsw = tsk->nivcsw = 0; 136917406b82SMandeep Singh Baines #ifdef CONFIG_DETECT_HUNG_TASK 137017406b82SMandeep Singh Baines tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; 1371a2e51445SDmitry Vyukov tsk->last_switch_time = 0; 137217406b82SMandeep Singh Baines #endif 13731da177e4SLinus Torvalds 13741da177e4SLinus Torvalds tsk->mm = NULL; 13751da177e4SLinus Torvalds tsk->active_mm = NULL; 13761da177e4SLinus Torvalds 13771da177e4SLinus Torvalds /* 13781da177e4SLinus Torvalds * Are we cloning a kernel thread? 13791da177e4SLinus Torvalds * 13801da177e4SLinus Torvalds * We need to steal a active VM for that.. 13811da177e4SLinus Torvalds */ 13821da177e4SLinus Torvalds oldmm = current->mm; 13831da177e4SLinus Torvalds if (!oldmm) 13841da177e4SLinus Torvalds return 0; 13851da177e4SLinus Torvalds 1386615d6e87SDavidlohr Bueso /* initialize the new vmacache entries */ 1387615d6e87SDavidlohr Bueso vmacache_flush(tsk); 1388615d6e87SDavidlohr Bueso 13891da177e4SLinus Torvalds if (clone_flags & CLONE_VM) { 13903fce371bSVegard Nossum mmget(oldmm); 13911da177e4SLinus Torvalds mm = oldmm; 13921da177e4SLinus Torvalds goto good_mm; 13931da177e4SLinus Torvalds } 13941da177e4SLinus Torvalds 13951da177e4SLinus Torvalds retval = -ENOMEM; 139613585fa0SNadav Amit mm = dup_mm(tsk, current->mm); 13971da177e4SLinus Torvalds if (!mm) 13981da177e4SLinus Torvalds goto fail_nomem; 13991da177e4SLinus Torvalds 14001da177e4SLinus Torvalds good_mm: 14011da177e4SLinus Torvalds tsk->mm = mm; 14021da177e4SLinus Torvalds tsk->active_mm = mm; 14031da177e4SLinus Torvalds return 0; 14041da177e4SLinus Torvalds 14051da177e4SLinus Torvalds fail_nomem: 14061da177e4SLinus Torvalds return retval; 14071da177e4SLinus Torvalds } 14081da177e4SLinus Torvalds 1409a39bc516SAlexey Dobriyan static int copy_fs(unsigned long clone_flags, struct task_struct *tsk) 14101da177e4SLinus Torvalds { 1411498052bbSAl Viro struct fs_struct *fs = current->fs; 14121da177e4SLinus Torvalds if (clone_flags & CLONE_FS) { 1413498052bbSAl Viro /* tsk->fs is already what we want */ 14142a4419b5SNick Piggin spin_lock(&fs->lock); 1415498052bbSAl Viro if (fs->in_exec) { 14162a4419b5SNick Piggin spin_unlock(&fs->lock); 1417498052bbSAl Viro return -EAGAIN; 1418498052bbSAl Viro } 1419498052bbSAl Viro fs->users++; 14202a4419b5SNick Piggin spin_unlock(&fs->lock); 14211da177e4SLinus Torvalds return 0; 14221da177e4SLinus Torvalds } 1423498052bbSAl Viro tsk->fs = copy_fs_struct(fs); 14241da177e4SLinus Torvalds if (!tsk->fs) 14251da177e4SLinus Torvalds return -ENOMEM; 14261da177e4SLinus Torvalds return 0; 14271da177e4SLinus Torvalds } 14281da177e4SLinus Torvalds 1429a016f338SJANAK DESAI static int copy_files(unsigned long clone_flags, struct task_struct *tsk) 1430a016f338SJANAK DESAI { 1431a016f338SJANAK DESAI struct files_struct *oldf, *newf; 1432a016f338SJANAK DESAI int error = 0; 1433a016f338SJANAK DESAI 1434a016f338SJANAK DESAI /* 1435a016f338SJANAK DESAI * A background process may not have any files ... 1436a016f338SJANAK DESAI */ 1437a016f338SJANAK DESAI oldf = current->files; 1438a016f338SJANAK DESAI if (!oldf) 1439a016f338SJANAK DESAI goto out; 1440a016f338SJANAK DESAI 1441a016f338SJANAK DESAI if (clone_flags & CLONE_FILES) { 1442a016f338SJANAK DESAI atomic_inc(&oldf->count); 1443a016f338SJANAK DESAI goto out; 1444a016f338SJANAK DESAI } 1445a016f338SJANAK DESAI 1446a016f338SJANAK DESAI newf = dup_fd(oldf, &error); 1447a016f338SJANAK DESAI if (!newf) 1448a016f338SJANAK DESAI goto out; 1449a016f338SJANAK DESAI 1450a016f338SJANAK DESAI tsk->files = newf; 1451a016f338SJANAK DESAI error = 0; 1452a016f338SJANAK DESAI out: 1453a016f338SJANAK DESAI return error; 1454a016f338SJANAK DESAI } 1455a016f338SJANAK DESAI 1456fadad878SJens Axboe static int copy_io(unsigned long clone_flags, struct task_struct *tsk) 1457fd0928dfSJens Axboe { 1458fd0928dfSJens Axboe #ifdef CONFIG_BLOCK 1459fd0928dfSJens Axboe struct io_context *ioc = current->io_context; 14606e736be7STejun Heo struct io_context *new_ioc; 1461fd0928dfSJens Axboe 1462fd0928dfSJens Axboe if (!ioc) 1463fd0928dfSJens Axboe return 0; 1464fadad878SJens Axboe /* 1465fadad878SJens Axboe * Share io context with parent, if CLONE_IO is set 1466fadad878SJens Axboe */ 1467fadad878SJens Axboe if (clone_flags & CLONE_IO) { 14683d48749dSTejun Heo ioc_task_link(ioc); 14693d48749dSTejun Heo tsk->io_context = ioc; 1470fadad878SJens Axboe } else if (ioprio_valid(ioc->ioprio)) { 14716e736be7STejun Heo new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE); 14726e736be7STejun Heo if (unlikely(!new_ioc)) 1473fd0928dfSJens Axboe return -ENOMEM; 1474fd0928dfSJens Axboe 14756e736be7STejun Heo new_ioc->ioprio = ioc->ioprio; 147611a3122fSTejun Heo put_io_context(new_ioc); 1477fd0928dfSJens Axboe } 1478fd0928dfSJens Axboe #endif 1479fd0928dfSJens Axboe return 0; 1480fd0928dfSJens Axboe } 1481fd0928dfSJens Axboe 1482a39bc516SAlexey Dobriyan static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk) 14831da177e4SLinus Torvalds { 14841da177e4SLinus Torvalds struct sighand_struct *sig; 14851da177e4SLinus Torvalds 148660348802SZhaolei if (clone_flags & CLONE_SIGHAND) { 1487d036bda7SElena Reshetova refcount_inc(¤t->sighand->count); 14881da177e4SLinus Torvalds return 0; 14891da177e4SLinus Torvalds } 14901da177e4SLinus Torvalds sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL); 1491e56d0903SIngo Molnar rcu_assign_pointer(tsk->sighand, sig); 14921da177e4SLinus Torvalds if (!sig) 14931da177e4SLinus Torvalds return -ENOMEM; 14949d7fb042SPeter Zijlstra 1495d036bda7SElena Reshetova refcount_set(&sig->count, 1); 149606e62a46SJann Horn spin_lock_irq(¤t->sighand->siglock); 14971da177e4SLinus Torvalds memcpy(sig->action, current->sighand->action, sizeof(sig->action)); 149806e62a46SJann Horn spin_unlock_irq(¤t->sighand->siglock); 14991da177e4SLinus Torvalds return 0; 15001da177e4SLinus Torvalds } 15011da177e4SLinus Torvalds 1502a7e5328aSOleg Nesterov void __cleanup_sighand(struct sighand_struct *sighand) 1503c81addc9SOleg Nesterov { 1504d036bda7SElena Reshetova if (refcount_dec_and_test(&sighand->count)) { 1505d80e731eSOleg Nesterov signalfd_cleanup(sighand); 1506392809b2SOleg Nesterov /* 15075f0d5a3aSPaul E. McKenney * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it 1508392809b2SOleg Nesterov * without an RCU grace period, see __lock_task_sighand(). 1509392809b2SOleg Nesterov */ 1510c81addc9SOleg Nesterov kmem_cache_free(sighand_cachep, sighand); 1511c81addc9SOleg Nesterov } 1512d80e731eSOleg Nesterov } 1513c81addc9SOleg Nesterov 1514b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS 1515f06febc9SFrank Mayhar /* 1516f06febc9SFrank Mayhar * Initialize POSIX timer handling for a thread group. 1517f06febc9SFrank Mayhar */ 1518f06febc9SFrank Mayhar static void posix_cpu_timers_init_group(struct signal_struct *sig) 1519f06febc9SFrank Mayhar { 152078d7d407SJiri Slaby unsigned long cpu_limit; 152178d7d407SJiri Slaby 1522316c1608SJason Low cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); 152378d7d407SJiri Slaby if (cpu_limit != RLIM_INFINITY) { 1524ebd7e7fcSFrederic Weisbecker sig->cputime_expires.prof_exp = cpu_limit * NSEC_PER_SEC; 1525d5c373ebSJason Low sig->cputimer.running = true; 15266279a751SOleg Nesterov } 15276279a751SOleg Nesterov 1528f06febc9SFrank Mayhar /* The timer lists. */ 1529f06febc9SFrank Mayhar INIT_LIST_HEAD(&sig->cpu_timers[0]); 1530f06febc9SFrank Mayhar INIT_LIST_HEAD(&sig->cpu_timers[1]); 1531f06febc9SFrank Mayhar INIT_LIST_HEAD(&sig->cpu_timers[2]); 1532f06febc9SFrank Mayhar } 1533b18b6a9cSNicolas Pitre #else 1534b18b6a9cSNicolas Pitre static inline void posix_cpu_timers_init_group(struct signal_struct *sig) { } 1535b18b6a9cSNicolas Pitre #endif 1536f06febc9SFrank Mayhar 1537a39bc516SAlexey Dobriyan static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) 15381da177e4SLinus Torvalds { 15391da177e4SLinus Torvalds struct signal_struct *sig; 15401da177e4SLinus Torvalds 15414ab6c083SOleg Nesterov if (clone_flags & CLONE_THREAD) 1542490dea45SPeter Zijlstra return 0; 15436279a751SOleg Nesterov 1544a56704efSVeaceslav Falico sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL); 15451da177e4SLinus Torvalds tsk->signal = sig; 15461da177e4SLinus Torvalds if (!sig) 15471da177e4SLinus Torvalds return -ENOMEM; 15481da177e4SLinus Torvalds 1549b3ac022cSOleg Nesterov sig->nr_threads = 1; 15501da177e4SLinus Torvalds atomic_set(&sig->live, 1); 155160d4de3fSElena Reshetova refcount_set(&sig->sigcnt, 1); 15520c740d0aSOleg Nesterov 15530c740d0aSOleg Nesterov /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */ 15540c740d0aSOleg Nesterov sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); 15550c740d0aSOleg Nesterov tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); 15560c740d0aSOleg Nesterov 15571da177e4SLinus Torvalds init_waitqueue_head(&sig->wait_chldexit); 1558db51aeccSOleg Nesterov sig->curr_target = tsk; 15591da177e4SLinus Torvalds init_sigpending(&sig->shared_pending); 1560c3ad2c3bSEric W. Biederman INIT_HLIST_HEAD(&sig->multiprocess); 1561e78c3496SRik van Riel seqlock_init(&sig->stats_lock); 15629d7fb042SPeter Zijlstra prev_cputime_init(&sig->prev_cputime); 15631da177e4SLinus Torvalds 1564baa73d9eSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS 1565b18b6a9cSNicolas Pitre INIT_LIST_HEAD(&sig->posix_timers); 1566c9cb2e3dSThomas Gleixner hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 15671da177e4SLinus Torvalds sig->real_timer.function = it_real_fn; 1568baa73d9eSNicolas Pitre #endif 15691da177e4SLinus Torvalds 15701da177e4SLinus Torvalds task_lock(current->group_leader); 15711da177e4SLinus Torvalds memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); 15721da177e4SLinus Torvalds task_unlock(current->group_leader); 15731da177e4SLinus Torvalds 15746279a751SOleg Nesterov posix_cpu_timers_init_group(sig); 15756279a751SOleg Nesterov 1576522ed776SMiloslav Trmac tty_audit_fork(sig); 15775091faa4SMike Galbraith sched_autogroup_fork(sig); 1578522ed776SMiloslav Trmac 1579a63d83f4SDavid Rientjes sig->oom_score_adj = current->signal->oom_score_adj; 1580dabb16f6SMandeep Singh Baines sig->oom_score_adj_min = current->signal->oom_score_adj_min; 158128b83c51SKOSAKI Motohiro 15829b1bf12dSKOSAKI Motohiro mutex_init(&sig->cred_guard_mutex); 15839b1bf12dSKOSAKI Motohiro 15841da177e4SLinus Torvalds return 0; 15851da177e4SLinus Torvalds } 15861da177e4SLinus Torvalds 1587dbd95212SKees Cook static void copy_seccomp(struct task_struct *p) 1588dbd95212SKees Cook { 1589dbd95212SKees Cook #ifdef CONFIG_SECCOMP 1590dbd95212SKees Cook /* 1591dbd95212SKees Cook * Must be called with sighand->lock held, which is common to 1592dbd95212SKees Cook * all threads in the group. Holding cred_guard_mutex is not 1593dbd95212SKees Cook * needed because this new task is not yet running and cannot 1594dbd95212SKees Cook * be racing exec. 1595dbd95212SKees Cook */ 159669f6a34bSGuenter Roeck assert_spin_locked(¤t->sighand->siglock); 1597dbd95212SKees Cook 1598dbd95212SKees Cook /* Ref-count the new filter user, and assign it. */ 1599dbd95212SKees Cook get_seccomp_filter(current); 1600dbd95212SKees Cook p->seccomp = current->seccomp; 1601dbd95212SKees Cook 1602dbd95212SKees Cook /* 1603dbd95212SKees Cook * Explicitly enable no_new_privs here in case it got set 1604dbd95212SKees Cook * between the task_struct being duplicated and holding the 1605dbd95212SKees Cook * sighand lock. The seccomp state and nnp must be in sync. 1606dbd95212SKees Cook */ 1607dbd95212SKees Cook if (task_no_new_privs(current)) 1608dbd95212SKees Cook task_set_no_new_privs(p); 1609dbd95212SKees Cook 1610dbd95212SKees Cook /* 1611dbd95212SKees Cook * If the parent gained a seccomp mode after copying thread 1612dbd95212SKees Cook * flags and between before we held the sighand lock, we have 1613dbd95212SKees Cook * to manually enable the seccomp thread flag here. 1614dbd95212SKees Cook */ 1615dbd95212SKees Cook if (p->seccomp.mode != SECCOMP_MODE_DISABLED) 1616dbd95212SKees Cook set_tsk_thread_flag(p, TIF_SECCOMP); 1617dbd95212SKees Cook #endif 1618dbd95212SKees Cook } 1619dbd95212SKees Cook 162017da2bd9SHeiko Carstens SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr) 16211da177e4SLinus Torvalds { 16221da177e4SLinus Torvalds current->clear_child_tid = tidptr; 16231da177e4SLinus Torvalds 1624b488893aSPavel Emelyanov return task_pid_vnr(current); 16251da177e4SLinus Torvalds } 16261da177e4SLinus Torvalds 1627a39bc516SAlexey Dobriyan static void rt_mutex_init_task(struct task_struct *p) 162823f78d4aSIngo Molnar { 16291d615482SThomas Gleixner raw_spin_lock_init(&p->pi_lock); 1630e29e175bSZilvinas Valinskas #ifdef CONFIG_RT_MUTEXES 1631a23ba907SDavidlohr Bueso p->pi_waiters = RB_ROOT_CACHED; 1632e96a7705SXunlei Pang p->pi_top_task = NULL; 163323f78d4aSIngo Molnar p->pi_blocked_on = NULL; 163423f78d4aSIngo Molnar #endif 163523f78d4aSIngo Molnar } 163623f78d4aSIngo Molnar 1637b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS 16381da177e4SLinus Torvalds /* 1639f06febc9SFrank Mayhar * Initialize POSIX timer handling for a single task. 1640f06febc9SFrank Mayhar */ 1641f06febc9SFrank Mayhar static void posix_cpu_timers_init(struct task_struct *tsk) 1642f06febc9SFrank Mayhar { 164364861634SMartin Schwidefsky tsk->cputime_expires.prof_exp = 0; 164464861634SMartin Schwidefsky tsk->cputime_expires.virt_exp = 0; 1645f06febc9SFrank Mayhar tsk->cputime_expires.sched_exp = 0; 1646f06febc9SFrank Mayhar INIT_LIST_HEAD(&tsk->cpu_timers[0]); 1647f06febc9SFrank Mayhar INIT_LIST_HEAD(&tsk->cpu_timers[1]); 1648f06febc9SFrank Mayhar INIT_LIST_HEAD(&tsk->cpu_timers[2]); 1649f06febc9SFrank Mayhar } 1650b18b6a9cSNicolas Pitre #else 1651b18b6a9cSNicolas Pitre static inline void posix_cpu_timers_init(struct task_struct *tsk) { } 1652b18b6a9cSNicolas Pitre #endif 1653f06febc9SFrank Mayhar 16542c470475SEric W. Biederman static inline void init_task_pid_links(struct task_struct *task) 16552c470475SEric W. Biederman { 16562c470475SEric W. Biederman enum pid_type type; 16572c470475SEric W. Biederman 16582c470475SEric W. Biederman for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) { 16592c470475SEric W. Biederman INIT_HLIST_NODE(&task->pid_links[type]); 16602c470475SEric W. Biederman } 16612c470475SEric W. Biederman } 16622c470475SEric W. Biederman 166381907739SOleg Nesterov static inline void 166481907739SOleg Nesterov init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid) 166581907739SOleg Nesterov { 16662c470475SEric W. Biederman if (type == PIDTYPE_PID) 16672c470475SEric W. Biederman task->thread_pid = pid; 16682c470475SEric W. Biederman else 16692c470475SEric W. Biederman task->signal->pids[type] = pid; 167081907739SOleg Nesterov } 167181907739SOleg Nesterov 16726bfbaa51SIngo Molnar static inline void rcu_copy_process(struct task_struct *p) 16736bfbaa51SIngo Molnar { 16746bfbaa51SIngo Molnar #ifdef CONFIG_PREEMPT_RCU 16756bfbaa51SIngo Molnar p->rcu_read_lock_nesting = 0; 16766bfbaa51SIngo Molnar p->rcu_read_unlock_special.s = 0; 16776bfbaa51SIngo Molnar p->rcu_blocked_node = NULL; 16786bfbaa51SIngo Molnar INIT_LIST_HEAD(&p->rcu_node_entry); 16796bfbaa51SIngo Molnar #endif /* #ifdef CONFIG_PREEMPT_RCU */ 16806bfbaa51SIngo Molnar #ifdef CONFIG_TASKS_RCU 16816bfbaa51SIngo Molnar p->rcu_tasks_holdout = false; 16826bfbaa51SIngo Molnar INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); 16836bfbaa51SIngo Molnar p->rcu_tasks_idle_cpu = -1; 16846bfbaa51SIngo Molnar #endif /* #ifdef CONFIG_TASKS_RCU */ 16856bfbaa51SIngo Molnar } 16866bfbaa51SIngo Molnar 1687b3e58382SChristian Brauner static int pidfd_release(struct inode *inode, struct file *file) 1688b3e58382SChristian Brauner { 1689b3e58382SChristian Brauner struct pid *pid = file->private_data; 1690b3e58382SChristian Brauner 1691b3e58382SChristian Brauner file->private_data = NULL; 1692b3e58382SChristian Brauner put_pid(pid); 1693b3e58382SChristian Brauner return 0; 1694b3e58382SChristian Brauner } 1695b3e58382SChristian Brauner 1696b3e58382SChristian Brauner #ifdef CONFIG_PROC_FS 1697b3e58382SChristian Brauner static void pidfd_show_fdinfo(struct seq_file *m, struct file *f) 1698b3e58382SChristian Brauner { 1699b3e58382SChristian Brauner struct pid_namespace *ns = proc_pid_ns(file_inode(m->file)); 1700b3e58382SChristian Brauner struct pid *pid = f->private_data; 1701b3e58382SChristian Brauner 1702b3e58382SChristian Brauner seq_put_decimal_ull(m, "Pid:\t", pid_nr_ns(pid, ns)); 1703b3e58382SChristian Brauner seq_putc(m, '\n'); 1704b3e58382SChristian Brauner } 1705b3e58382SChristian Brauner #endif 1706b3e58382SChristian Brauner 1707b3e58382SChristian Brauner const struct file_operations pidfd_fops = { 1708b3e58382SChristian Brauner .release = pidfd_release, 1709b3e58382SChristian Brauner #ifdef CONFIG_PROC_FS 1710b3e58382SChristian Brauner .show_fdinfo = pidfd_show_fdinfo, 1711b3e58382SChristian Brauner #endif 1712b3e58382SChristian Brauner }; 1713b3e58382SChristian Brauner 1714b3e58382SChristian Brauner /** 1715b3e58382SChristian Brauner * pidfd_create() - Create a new pid file descriptor. 1716b3e58382SChristian Brauner * 1717b3e58382SChristian Brauner * @pid: struct pid that the pidfd will reference 1718b3e58382SChristian Brauner * 1719b3e58382SChristian Brauner * This creates a new pid file descriptor with the O_CLOEXEC flag set. 1720b3e58382SChristian Brauner * 1721b3e58382SChristian Brauner * Note, that this function can only be called after the fd table has 1722b3e58382SChristian Brauner * been unshared to avoid leaking the pidfd to the new process. 1723b3e58382SChristian Brauner * 1724b3e58382SChristian Brauner * Return: On success, a cloexec pidfd is returned. 1725b3e58382SChristian Brauner * On error, a negative errno number will be returned. 1726b3e58382SChristian Brauner */ 1727b3e58382SChristian Brauner static int pidfd_create(struct pid *pid) 1728b3e58382SChristian Brauner { 1729b3e58382SChristian Brauner int fd; 1730b3e58382SChristian Brauner 1731b3e58382SChristian Brauner fd = anon_inode_getfd("[pidfd]", &pidfd_fops, get_pid(pid), 1732b3e58382SChristian Brauner O_RDWR | O_CLOEXEC); 1733b3e58382SChristian Brauner if (fd < 0) 1734b3e58382SChristian Brauner put_pid(pid); 1735b3e58382SChristian Brauner 1736b3e58382SChristian Brauner return fd; 1737b3e58382SChristian Brauner } 1738b3e58382SChristian Brauner 1739c3f3ce04SAndrea Arcangeli static void __delayed_free_task(struct rcu_head *rhp) 1740c3f3ce04SAndrea Arcangeli { 1741c3f3ce04SAndrea Arcangeli struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); 1742c3f3ce04SAndrea Arcangeli 1743c3f3ce04SAndrea Arcangeli free_task(tsk); 1744c3f3ce04SAndrea Arcangeli } 1745c3f3ce04SAndrea Arcangeli 1746c3f3ce04SAndrea Arcangeli static __always_inline void delayed_free_task(struct task_struct *tsk) 1747c3f3ce04SAndrea Arcangeli { 1748c3f3ce04SAndrea Arcangeli if (IS_ENABLED(CONFIG_MEMCG)) 1749c3f3ce04SAndrea Arcangeli call_rcu(&tsk->rcu, __delayed_free_task); 1750c3f3ce04SAndrea Arcangeli else 1751c3f3ce04SAndrea Arcangeli free_task(tsk); 1752c3f3ce04SAndrea Arcangeli } 1753c3f3ce04SAndrea Arcangeli 1754f06febc9SFrank Mayhar /* 17551da177e4SLinus Torvalds * This creates a new process as a copy of the old one, 17561da177e4SLinus Torvalds * but does not actually start it yet. 17571da177e4SLinus Torvalds * 17581da177e4SLinus Torvalds * It copies the registers, and all the appropriate 17591da177e4SLinus Torvalds * parts of the process environment (as per the clone 17601da177e4SLinus Torvalds * flags). The actual kick-off is left to the caller. 17611da177e4SLinus Torvalds */ 17620766f788SEmese Revfy static __latent_entropy struct task_struct *copy_process( 17630766f788SEmese Revfy unsigned long clone_flags, 17641da177e4SLinus Torvalds unsigned long stack_start, 17651da177e4SLinus Torvalds unsigned long stack_size, 1766b3e58382SChristian Brauner int __user *parent_tidptr, 17671da177e4SLinus Torvalds int __user *child_tidptr, 176809a05394SRoland McGrath struct pid *pid, 17693033f14aSJosh Triplett int trace, 1770725fc629SAndi Kleen unsigned long tls, 1771725fc629SAndi Kleen int node) 17721da177e4SLinus Torvalds { 1773b3e58382SChristian Brauner int pidfd = -1, retval; 1774a24efe62SMariusz Kozlowski struct task_struct *p; 1775c3ad2c3bSEric W. Biederman struct multiprocess_signals delayed; 17761da177e4SLinus Torvalds 1777667b6094SMarcos Paulo de Souza /* 1778667b6094SMarcos Paulo de Souza * Don't allow sharing the root directory with processes in a different 1779667b6094SMarcos Paulo de Souza * namespace 1780667b6094SMarcos Paulo de Souza */ 17811da177e4SLinus Torvalds if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) 17821da177e4SLinus Torvalds return ERR_PTR(-EINVAL); 17831da177e4SLinus Torvalds 1784e66eded8SEric W. Biederman if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) 1785e66eded8SEric W. Biederman return ERR_PTR(-EINVAL); 1786e66eded8SEric W. Biederman 17871da177e4SLinus Torvalds /* 17881da177e4SLinus Torvalds * Thread groups must share signals as well, and detached threads 17891da177e4SLinus Torvalds * can only be started up within the thread group. 17901da177e4SLinus Torvalds */ 17911da177e4SLinus Torvalds if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND)) 17921da177e4SLinus Torvalds return ERR_PTR(-EINVAL); 17931da177e4SLinus Torvalds 17941da177e4SLinus Torvalds /* 17951da177e4SLinus Torvalds * Shared signal handlers imply shared VM. By way of the above, 17961da177e4SLinus Torvalds * thread groups also imply shared VM. Blocking this case allows 17971da177e4SLinus Torvalds * for various simplifications in other code. 17981da177e4SLinus Torvalds */ 17991da177e4SLinus Torvalds if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM)) 18001da177e4SLinus Torvalds return ERR_PTR(-EINVAL); 18011da177e4SLinus Torvalds 1802123be07bSSukadev Bhattiprolu /* 1803123be07bSSukadev Bhattiprolu * Siblings of global init remain as zombies on exit since they are 1804123be07bSSukadev Bhattiprolu * not reaped by their parent (swapper). To solve this and to avoid 1805123be07bSSukadev Bhattiprolu * multi-rooted process trees, prevent global and container-inits 1806123be07bSSukadev Bhattiprolu * from creating siblings. 1807123be07bSSukadev Bhattiprolu */ 1808123be07bSSukadev Bhattiprolu if ((clone_flags & CLONE_PARENT) && 1809123be07bSSukadev Bhattiprolu current->signal->flags & SIGNAL_UNKILLABLE) 1810123be07bSSukadev Bhattiprolu return ERR_PTR(-EINVAL); 1811123be07bSSukadev Bhattiprolu 18128382fcacSEric W. Biederman /* 181340a0d32dSOleg Nesterov * If the new process will be in a different pid or user namespace 1814faf00da5SEric W. Biederman * do not allow it to share a thread group with the forking task. 18158382fcacSEric W. Biederman */ 1816faf00da5SEric W. Biederman if (clone_flags & CLONE_THREAD) { 181740a0d32dSOleg Nesterov if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) || 181840a0d32dSOleg Nesterov (task_active_pid_ns(current) != 1819c2b1df2eSAndy Lutomirski current->nsproxy->pid_ns_for_children)) 18208382fcacSEric W. Biederman return ERR_PTR(-EINVAL); 182140a0d32dSOleg Nesterov } 18228382fcacSEric W. Biederman 1823b3e58382SChristian Brauner if (clone_flags & CLONE_PIDFD) { 1824b3e58382SChristian Brauner int reserved; 1825b3e58382SChristian Brauner 1826b3e58382SChristian Brauner /* 1827b3e58382SChristian Brauner * - CLONE_PARENT_SETTID is useless for pidfds and also 1828b3e58382SChristian Brauner * parent_tidptr is used to return pidfds. 1829b3e58382SChristian Brauner * - CLONE_DETACHED is blocked so that we can potentially 1830b3e58382SChristian Brauner * reuse it later for CLONE_PIDFD. 1831b3e58382SChristian Brauner * - CLONE_THREAD is blocked until someone really needs it. 1832b3e58382SChristian Brauner */ 1833b3e58382SChristian Brauner if (clone_flags & 1834b3e58382SChristian Brauner (CLONE_DETACHED | CLONE_PARENT_SETTID | CLONE_THREAD)) 1835b3e58382SChristian Brauner return ERR_PTR(-EINVAL); 1836b3e58382SChristian Brauner 1837b3e58382SChristian Brauner /* 1838b3e58382SChristian Brauner * Verify that parent_tidptr is sane so we can potentially 1839b3e58382SChristian Brauner * reuse it later. 1840b3e58382SChristian Brauner */ 1841b3e58382SChristian Brauner if (get_user(reserved, parent_tidptr)) 1842b3e58382SChristian Brauner return ERR_PTR(-EFAULT); 1843b3e58382SChristian Brauner 1844b3e58382SChristian Brauner if (reserved != 0) 1845b3e58382SChristian Brauner return ERR_PTR(-EINVAL); 1846b3e58382SChristian Brauner } 1847b3e58382SChristian Brauner 1848c3ad2c3bSEric W. Biederman /* 1849c3ad2c3bSEric W. Biederman * Force any signals received before this point to be delivered 1850c3ad2c3bSEric W. Biederman * before the fork happens. Collect up signals sent to multiple 1851c3ad2c3bSEric W. Biederman * processes that happen during the fork and delay them so that 1852c3ad2c3bSEric W. Biederman * they appear to happen after the fork. 1853c3ad2c3bSEric W. Biederman */ 1854c3ad2c3bSEric W. Biederman sigemptyset(&delayed.signal); 1855c3ad2c3bSEric W. Biederman INIT_HLIST_NODE(&delayed.node); 1856c3ad2c3bSEric W. Biederman 1857c3ad2c3bSEric W. Biederman spin_lock_irq(¤t->sighand->siglock); 1858c3ad2c3bSEric W. Biederman if (!(clone_flags & CLONE_THREAD)) 1859c3ad2c3bSEric W. Biederman hlist_add_head(&delayed.node, ¤t->signal->multiprocess); 1860c3ad2c3bSEric W. Biederman recalc_sigpending(); 1861c3ad2c3bSEric W. Biederman spin_unlock_irq(¤t->sighand->siglock); 1862c3ad2c3bSEric W. Biederman retval = -ERESTARTNOINTR; 1863c3ad2c3bSEric W. Biederman if (signal_pending(current)) 1864c3ad2c3bSEric W. Biederman goto fork_out; 1865c3ad2c3bSEric W. Biederman 18661da177e4SLinus Torvalds retval = -ENOMEM; 1867725fc629SAndi Kleen p = dup_task_struct(current, node); 18681da177e4SLinus Torvalds if (!p) 18691da177e4SLinus Torvalds goto fork_out; 18701da177e4SLinus Torvalds 18714d6501dcSVegard Nossum /* 18724d6501dcSVegard Nossum * This _must_ happen before we call free_task(), i.e. before we jump 18734d6501dcSVegard Nossum * to any of the bad_fork_* labels. This is to avoid freeing 18744d6501dcSVegard Nossum * p->set_child_tid which is (ab)used as a kthread's data pointer for 18754d6501dcSVegard Nossum * kernel threads (PF_KTHREAD). 18764d6501dcSVegard Nossum */ 18774d6501dcSVegard Nossum p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL; 18784d6501dcSVegard Nossum /* 18794d6501dcSVegard Nossum * Clear TID on mm_release()? 18804d6501dcSVegard Nossum */ 18814d6501dcSVegard Nossum p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL; 18824d6501dcSVegard Nossum 1883f7e8b616SSteven Rostedt ftrace_graph_init_task(p); 1884f7e8b616SSteven Rostedt 1885bea493a0SPeter Zijlstra rt_mutex_init_task(p); 1886bea493a0SPeter Zijlstra 1887d12c1a37SIngo Molnar #ifdef CONFIG_PROVE_LOCKING 1888de30a2b3SIngo Molnar DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled); 1889de30a2b3SIngo Molnar DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); 1890de30a2b3SIngo Molnar #endif 18911da177e4SLinus Torvalds retval = -EAGAIN; 18923b11a1deSDavid Howells if (atomic_read(&p->real_cred->user->processes) >= 189378d7d407SJiri Slaby task_rlimit(p, RLIMIT_NPROC)) { 1894b57922b6SEric Paris if (p->real_cred->user != INIT_USER && 1895b57922b6SEric Paris !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) 18961da177e4SLinus Torvalds goto bad_fork_free; 18971da177e4SLinus Torvalds } 189872fa5997SVasiliy Kulikov current->flags &= ~PF_NPROC_EXCEEDED; 18991da177e4SLinus Torvalds 1900f1752eecSDavid Howells retval = copy_creds(p, clone_flags); 1901f1752eecSDavid Howells if (retval < 0) 1902f1752eecSDavid Howells goto bad_fork_free; 19031da177e4SLinus Torvalds 19041da177e4SLinus Torvalds /* 19051da177e4SLinus Torvalds * If multiple threads are within copy_process(), then this check 19061da177e4SLinus Torvalds * triggers too late. This doesn't hurt, the check is only there 19071da177e4SLinus Torvalds * to stop root fork bombs. 19081da177e4SLinus Torvalds */ 190904ec93feSLi Zefan retval = -EAGAIN; 19101da177e4SLinus Torvalds if (nr_threads >= max_threads) 19111da177e4SLinus Torvalds goto bad_fork_cleanup_count; 19121da177e4SLinus Torvalds 1913ca74e92bSShailabh Nagar delayacct_tsk_init(p); /* Must remain after dup_task_struct() */ 1914c1de45caSPeter Zijlstra p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE); 1915514ddb44SDavid Rientjes p->flags |= PF_FORKNOEXEC; 19161da177e4SLinus Torvalds INIT_LIST_HEAD(&p->children); 19171da177e4SLinus Torvalds INIT_LIST_HEAD(&p->sibling); 1918f41d911fSPaul E. McKenney rcu_copy_process(p); 19191da177e4SLinus Torvalds p->vfork_done = NULL; 19201da177e4SLinus Torvalds spin_lock_init(&p->alloc_lock); 19211da177e4SLinus Torvalds 19221da177e4SLinus Torvalds init_sigpending(&p->pending); 19231da177e4SLinus Torvalds 192464861634SMartin Schwidefsky p->utime = p->stime = p->gtime = 0; 192540565b5aSStanislaw Gruszka #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME 192664861634SMartin Schwidefsky p->utimescaled = p->stimescaled = 0; 192740565b5aSStanislaw Gruszka #endif 19289d7fb042SPeter Zijlstra prev_cputime_init(&p->prev_cputime); 19299d7fb042SPeter Zijlstra 19306a61671bSFrederic Weisbecker #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 1931bac5b6b6SFrederic Weisbecker seqcount_init(&p->vtime.seqcount); 1932bac5b6b6SFrederic Weisbecker p->vtime.starttime = 0; 1933bac5b6b6SFrederic Weisbecker p->vtime.state = VTIME_INACTIVE; 19346a61671bSFrederic Weisbecker #endif 19356a61671bSFrederic Weisbecker 1936a3a2e76cSKAMEZAWA Hiroyuki #if defined(SPLIT_RSS_COUNTING) 1937a3a2e76cSKAMEZAWA Hiroyuki memset(&p->rss_stat, 0, sizeof(p->rss_stat)); 1938a3a2e76cSKAMEZAWA Hiroyuki #endif 1939172ba844SBalbir Singh 19406976675dSArjan van de Ven p->default_timer_slack_ns = current->timer_slack_ns; 19416976675dSArjan van de Ven 1942eb414681SJohannes Weiner #ifdef CONFIG_PSI 1943eb414681SJohannes Weiner p->psi_flags = 0; 1944eb414681SJohannes Weiner #endif 1945eb414681SJohannes Weiner 19465995477aSAndrea Righi task_io_accounting_init(&p->ioac); 19471da177e4SLinus Torvalds acct_clear_integrals(p); 19481da177e4SLinus Torvalds 1949f06febc9SFrank Mayhar posix_cpu_timers_init(p); 19501da177e4SLinus Torvalds 19511da177e4SLinus Torvalds p->io_context = NULL; 1952c0b0ae8aSRichard Guy Briggs audit_set_context(p, NULL); 1953b4f48b63SPaul Menage cgroup_fork(p); 19541da177e4SLinus Torvalds #ifdef CONFIG_NUMA 1955846a16bfSLee Schermerhorn p->mempolicy = mpol_dup(p->mempolicy); 19561da177e4SLinus Torvalds if (IS_ERR(p->mempolicy)) { 19571da177e4SLinus Torvalds retval = PTR_ERR(p->mempolicy); 19581da177e4SLinus Torvalds p->mempolicy = NULL; 1959e8604cb4SLi Zefan goto bad_fork_cleanup_threadgroup_lock; 19601da177e4SLinus Torvalds } 19611da177e4SLinus Torvalds #endif 1962778d3b0fSMichal Hocko #ifdef CONFIG_CPUSETS 1963778d3b0fSMichal Hocko p->cpuset_mem_spread_rotor = NUMA_NO_NODE; 1964778d3b0fSMichal Hocko p->cpuset_slab_spread_rotor = NUMA_NO_NODE; 1965cc9a6c87SMel Gorman seqcount_init(&p->mems_allowed_seq); 1966778d3b0fSMichal Hocko #endif 1967de30a2b3SIngo Molnar #ifdef CONFIG_TRACE_IRQFLAGS 1968de30a2b3SIngo Molnar p->irq_events = 0; 1969de30a2b3SIngo Molnar p->hardirqs_enabled = 0; 1970de30a2b3SIngo Molnar p->hardirq_enable_ip = 0; 1971de30a2b3SIngo Molnar p->hardirq_enable_event = 0; 1972de30a2b3SIngo Molnar p->hardirq_disable_ip = _THIS_IP_; 1973de30a2b3SIngo Molnar p->hardirq_disable_event = 0; 1974de30a2b3SIngo Molnar p->softirqs_enabled = 1; 1975de30a2b3SIngo Molnar p->softirq_enable_ip = _THIS_IP_; 1976de30a2b3SIngo Molnar p->softirq_enable_event = 0; 1977de30a2b3SIngo Molnar p->softirq_disable_ip = 0; 1978de30a2b3SIngo Molnar p->softirq_disable_event = 0; 1979de30a2b3SIngo Molnar p->hardirq_context = 0; 1980de30a2b3SIngo Molnar p->softirq_context = 0; 1981de30a2b3SIngo Molnar #endif 19828bcbde54SDavid Hildenbrand 19838bcbde54SDavid Hildenbrand p->pagefault_disabled = 0; 19848bcbde54SDavid Hildenbrand 1985fbb9ce95SIngo Molnar #ifdef CONFIG_LOCKDEP 1986fbb9ce95SIngo Molnar p->lockdep_depth = 0; /* no locks held yet */ 1987fbb9ce95SIngo Molnar p->curr_chain_key = 0; 1988fbb9ce95SIngo Molnar p->lockdep_recursion = 0; 1989b09be676SByungchul Park lockdep_init_task(p); 1990fbb9ce95SIngo Molnar #endif 19911da177e4SLinus Torvalds 1992408894eeSIngo Molnar #ifdef CONFIG_DEBUG_MUTEXES 1993408894eeSIngo Molnar p->blocked_on = NULL; /* not blocked yet */ 1994408894eeSIngo Molnar #endif 1995cafe5635SKent Overstreet #ifdef CONFIG_BCACHE 1996cafe5635SKent Overstreet p->sequential_io = 0; 1997cafe5635SKent Overstreet p->sequential_io_avg = 0; 1998cafe5635SKent Overstreet #endif 19990f481406SMarkus Metzger 20003c90e6e9SSrivatsa Vaddagiri /* Perform scheduler related setup. Assign this task to a CPU. */ 2001aab03e05SDario Faggioli retval = sched_fork(clone_flags, p); 2002aab03e05SDario Faggioli if (retval) 2003aab03e05SDario Faggioli goto bad_fork_cleanup_policy; 20046ab423e0SPeter Zijlstra 2005cdd6c482SIngo Molnar retval = perf_event_init_task(p); 20066ab423e0SPeter Zijlstra if (retval) 20076ab423e0SPeter Zijlstra goto bad_fork_cleanup_policy; 2008fb0a685cSDaniel Rebelo de Oliveira retval = audit_alloc(p); 2009fb0a685cSDaniel Rebelo de Oliveira if (retval) 20106c72e350SPeter Zijlstra goto bad_fork_cleanup_perf; 20111da177e4SLinus Torvalds /* copy all the process information */ 2012ab602f79SJack Miller shm_init_task(p); 2013e4e55b47STetsuo Handa retval = security_task_alloc(p, clone_flags); 2014fb0a685cSDaniel Rebelo de Oliveira if (retval) 20151da177e4SLinus Torvalds goto bad_fork_cleanup_audit; 2016e4e55b47STetsuo Handa retval = copy_semundo(clone_flags, p); 2017e4e55b47STetsuo Handa if (retval) 2018e4e55b47STetsuo Handa goto bad_fork_cleanup_security; 2019fb0a685cSDaniel Rebelo de Oliveira retval = copy_files(clone_flags, p); 2020fb0a685cSDaniel Rebelo de Oliveira if (retval) 20211da177e4SLinus Torvalds goto bad_fork_cleanup_semundo; 2022fb0a685cSDaniel Rebelo de Oliveira retval = copy_fs(clone_flags, p); 2023fb0a685cSDaniel Rebelo de Oliveira if (retval) 20241da177e4SLinus Torvalds goto bad_fork_cleanup_files; 2025fb0a685cSDaniel Rebelo de Oliveira retval = copy_sighand(clone_flags, p); 2026fb0a685cSDaniel Rebelo de Oliveira if (retval) 20271da177e4SLinus Torvalds goto bad_fork_cleanup_fs; 2028fb0a685cSDaniel Rebelo de Oliveira retval = copy_signal(clone_flags, p); 2029fb0a685cSDaniel Rebelo de Oliveira if (retval) 20301da177e4SLinus Torvalds goto bad_fork_cleanup_sighand; 2031fb0a685cSDaniel Rebelo de Oliveira retval = copy_mm(clone_flags, p); 2032fb0a685cSDaniel Rebelo de Oliveira if (retval) 20331da177e4SLinus Torvalds goto bad_fork_cleanup_signal; 2034fb0a685cSDaniel Rebelo de Oliveira retval = copy_namespaces(clone_flags, p); 2035fb0a685cSDaniel Rebelo de Oliveira if (retval) 2036d84f4f99SDavid Howells goto bad_fork_cleanup_mm; 2037fb0a685cSDaniel Rebelo de Oliveira retval = copy_io(clone_flags, p); 2038fb0a685cSDaniel Rebelo de Oliveira if (retval) 2039fd0928dfSJens Axboe goto bad_fork_cleanup_namespaces; 20403033f14aSJosh Triplett retval = copy_thread_tls(clone_flags, stack_start, stack_size, p, tls); 20411da177e4SLinus Torvalds if (retval) 2042fd0928dfSJens Axboe goto bad_fork_cleanup_io; 20431da177e4SLinus Torvalds 2044afaef01cSAlexander Popov stackleak_task_init(p); 2045afaef01cSAlexander Popov 2046425fb2b4SPavel Emelyanov if (pid != &init_struct_pid) { 2047c2b1df2eSAndy Lutomirski pid = alloc_pid(p->nsproxy->pid_ns_for_children); 204835f71bc0SMichal Hocko if (IS_ERR(pid)) { 204935f71bc0SMichal Hocko retval = PTR_ERR(pid); 20500740aa5fSJiri Slaby goto bad_fork_cleanup_thread; 2051425fb2b4SPavel Emelyanov } 205235f71bc0SMichal Hocko } 2053425fb2b4SPavel Emelyanov 2054b3e58382SChristian Brauner /* 2055b3e58382SChristian Brauner * This has to happen after we've potentially unshared the file 2056b3e58382SChristian Brauner * descriptor table (so that the pidfd doesn't leak into the child 2057b3e58382SChristian Brauner * if the fd table isn't shared). 2058b3e58382SChristian Brauner */ 2059b3e58382SChristian Brauner if (clone_flags & CLONE_PIDFD) { 2060b3e58382SChristian Brauner retval = pidfd_create(pid); 2061b3e58382SChristian Brauner if (retval < 0) 2062b3e58382SChristian Brauner goto bad_fork_free_pid; 2063b3e58382SChristian Brauner 2064b3e58382SChristian Brauner pidfd = retval; 2065b3e58382SChristian Brauner retval = put_user(pidfd, parent_tidptr); 2066b3e58382SChristian Brauner if (retval) 2067b3e58382SChristian Brauner goto bad_fork_put_pidfd; 2068b3e58382SChristian Brauner } 2069b3e58382SChristian Brauner 207073c10101SJens Axboe #ifdef CONFIG_BLOCK 207173c10101SJens Axboe p->plug = NULL; 207273c10101SJens Axboe #endif 207342b2dd0aSAlexey Dobriyan #ifdef CONFIG_FUTEX 20748f17d3a5SIngo Molnar p->robust_list = NULL; 20758f17d3a5SIngo Molnar #ifdef CONFIG_COMPAT 20768f17d3a5SIngo Molnar p->compat_robust_list = NULL; 20778f17d3a5SIngo Molnar #endif 2078c87e2837SIngo Molnar INIT_LIST_HEAD(&p->pi_state_list); 2079c87e2837SIngo Molnar p->pi_state_cache = NULL; 208042b2dd0aSAlexey Dobriyan #endif 20811da177e4SLinus Torvalds /* 2082f9a3879aSGOTO Masanori * sigaltstack should be cleared when sharing the same VM 2083f9a3879aSGOTO Masanori */ 2084f9a3879aSGOTO Masanori if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) 20852a742138SStas Sergeev sas_ss_reset(p); 2086f9a3879aSGOTO Masanori 2087f9a3879aSGOTO Masanori /* 20886580807dSOleg Nesterov * Syscall tracing and stepping should be turned off in the 20896580807dSOleg Nesterov * child regardless of CLONE_PTRACE. 20901da177e4SLinus Torvalds */ 20916580807dSOleg Nesterov user_disable_single_step(p); 20921da177e4SLinus Torvalds clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE); 2093ed75e8d5SLaurent Vivier #ifdef TIF_SYSCALL_EMU 2094ed75e8d5SLaurent Vivier clear_tsk_thread_flag(p, TIF_SYSCALL_EMU); 2095ed75e8d5SLaurent Vivier #endif 2096*e02c9b0dSLin Feng clear_tsk_latency_tracing(p); 20971da177e4SLinus Torvalds 20981da177e4SLinus Torvalds /* ok, now we should be set up.. */ 209918c830dfSOleg Nesterov p->pid = pid_nr(pid); 210018c830dfSOleg Nesterov if (clone_flags & CLONE_THREAD) { 21015f8aadd8SOleg Nesterov p->exit_signal = -1; 210218c830dfSOleg Nesterov p->group_leader = current->group_leader; 210318c830dfSOleg Nesterov p->tgid = current->tgid; 210418c830dfSOleg Nesterov } else { 210518c830dfSOleg Nesterov if (clone_flags & CLONE_PARENT) 21065f8aadd8SOleg Nesterov p->exit_signal = current->group_leader->exit_signal; 21075f8aadd8SOleg Nesterov else 21085f8aadd8SOleg Nesterov p->exit_signal = (clone_flags & CSIGNAL); 210918c830dfSOleg Nesterov p->group_leader = p; 211018c830dfSOleg Nesterov p->tgid = p->pid; 211118c830dfSOleg Nesterov } 21125f8aadd8SOleg Nesterov 21139d823e8fSWu Fengguang p->nr_dirtied = 0; 21149d823e8fSWu Fengguang p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); 211583712358SWu Fengguang p->dirty_paused_when = 0; 21169d823e8fSWu Fengguang 2117bb8cbbfeSOleg Nesterov p->pdeath_signal = 0; 211847e65328SOleg Nesterov INIT_LIST_HEAD(&p->thread_group); 2119158e1645SAl Viro p->task_works = NULL; 21201da177e4SLinus Torvalds 2121780de9ddSIngo Molnar cgroup_threadgroup_change_begin(current); 212218c830dfSOleg Nesterov /* 21237e47682eSAleksa Sarai * Ensure that the cgroup subsystem policies allow the new process to be 21247e47682eSAleksa Sarai * forked. It should be noted the the new process's css_set can be changed 21257e47682eSAleksa Sarai * between here and cgroup_post_fork() if an organisation operation is in 21267e47682eSAleksa Sarai * progress. 21277e47682eSAleksa Sarai */ 2128b53202e6SOleg Nesterov retval = cgroup_can_fork(p); 21297e47682eSAleksa Sarai if (retval) 2130c3b7112dSChristian Brauner goto bad_fork_cgroup_threadgroup_change_end; 21317e47682eSAleksa Sarai 21327e47682eSAleksa Sarai /* 21337b558513SDavid Herrmann * From this point on we must avoid any synchronous user-space 21347b558513SDavid Herrmann * communication until we take the tasklist-lock. In particular, we do 21357b558513SDavid Herrmann * not want user-space to be able to predict the process start-time by 21367b558513SDavid Herrmann * stalling fork(2) after we recorded the start_time but before it is 21377b558513SDavid Herrmann * visible to the system. 21387b558513SDavid Herrmann */ 21397b558513SDavid Herrmann 21407b558513SDavid Herrmann p->start_time = ktime_get_ns(); 21417b558513SDavid Herrmann p->real_start_time = ktime_get_boot_ns(); 21427b558513SDavid Herrmann 21437b558513SDavid Herrmann /* 214418c830dfSOleg Nesterov * Make it visible to the rest of the system, but dont wake it up yet. 214518c830dfSOleg Nesterov * Need tasklist lock for parent etc handling! 214618c830dfSOleg Nesterov */ 21471da177e4SLinus Torvalds write_lock_irq(&tasklist_lock); 21481da177e4SLinus Torvalds 21491da177e4SLinus Torvalds /* CLONE_PARENT re-uses the old parent */ 21502d5516cbSOleg Nesterov if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) { 21511da177e4SLinus Torvalds p->real_parent = current->real_parent; 21522d5516cbSOleg Nesterov p->parent_exec_id = current->parent_exec_id; 21532d5516cbSOleg Nesterov } else { 21541da177e4SLinus Torvalds p->real_parent = current; 21552d5516cbSOleg Nesterov p->parent_exec_id = current->self_exec_id; 21562d5516cbSOleg Nesterov } 21571da177e4SLinus Torvalds 2158d83a7cb3SJosh Poimboeuf klp_copy_process(p); 2159d83a7cb3SJosh Poimboeuf 21601da177e4SLinus Torvalds spin_lock(¤t->sighand->siglock); 21614a2c7a78SOleg Nesterov 21624a2c7a78SOleg Nesterov /* 2163dbd95212SKees Cook * Copy seccomp details explicitly here, in case they were changed 2164dbd95212SKees Cook * before holding sighand lock. 2165dbd95212SKees Cook */ 2166dbd95212SKees Cook copy_seccomp(p); 2167dbd95212SKees Cook 2168d7822b1eSMathieu Desnoyers rseq_fork(p, clone_flags); 2169d7822b1eSMathieu Desnoyers 21704ca1d3eeSEric W. Biederman /* Don't start children in a dying pid namespace */ 2171e8cfbc24SGargi Sharma if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { 21723fd37226SKirill Tkhai retval = -ENOMEM; 21733fd37226SKirill Tkhai goto bad_fork_cancel_cgroup; 21743fd37226SKirill Tkhai } 21754a2c7a78SOleg Nesterov 21767673bf55SEric W. Biederman /* Let kill terminate clone/fork in the middle */ 21777673bf55SEric W. Biederman if (fatal_signal_pending(current)) { 21787673bf55SEric W. Biederman retval = -EINTR; 21797673bf55SEric W. Biederman goto bad_fork_cancel_cgroup; 21807673bf55SEric W. Biederman } 21817673bf55SEric W. Biederman 21821da177e4SLinus Torvalds 21832c470475SEric W. Biederman init_task_pid_links(p); 218473b9ebfeSOleg Nesterov if (likely(p->pid)) { 21854b9d33e6STejun Heo ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace); 21861da177e4SLinus Torvalds 218781907739SOleg Nesterov init_task_pid(p, PIDTYPE_PID, pid); 21881da177e4SLinus Torvalds if (thread_group_leader(p)) { 21896883f81aSEric W. Biederman init_task_pid(p, PIDTYPE_TGID, pid); 219081907739SOleg Nesterov init_task_pid(p, PIDTYPE_PGID, task_pgrp(current)); 219181907739SOleg Nesterov init_task_pid(p, PIDTYPE_SID, task_session(current)); 219281907739SOleg Nesterov 21931c4042c2SEric W. Biederman if (is_child_reaper(pid)) { 219417cf22c3SEric W. Biederman ns_of_pid(pid)->child_reaper = p; 21951c4042c2SEric W. Biederman p->signal->flags |= SIGNAL_UNKILLABLE; 21961c4042c2SEric W. Biederman } 2197c3ad2c3bSEric W. Biederman p->signal->shared_pending.signal = delayed.signal; 21989c9f4dedSAlan Cox p->signal->tty = tty_kref_get(current->signal->tty); 2199749860ceSPavel Tikhomirov /* 2200749860ceSPavel Tikhomirov * Inherit has_child_subreaper flag under the same 2201749860ceSPavel Tikhomirov * tasklist_lock with adding child to the process tree 2202749860ceSPavel Tikhomirov * for propagate_has_child_subreaper optimization. 2203749860ceSPavel Tikhomirov */ 2204749860ceSPavel Tikhomirov p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper || 2205749860ceSPavel Tikhomirov p->real_parent->signal->is_child_subreaper; 22069cd80bbbSOleg Nesterov list_add_tail(&p->sibling, &p->real_parent->children); 22075e85d4abSEric W. Biederman list_add_tail_rcu(&p->tasks, &init_task.tasks); 22086883f81aSEric W. Biederman attach_pid(p, PIDTYPE_TGID); 220981907739SOleg Nesterov attach_pid(p, PIDTYPE_PGID); 221081907739SOleg Nesterov attach_pid(p, PIDTYPE_SID); 2211909ea964SChristoph Lameter __this_cpu_inc(process_counts); 221280628ca0SOleg Nesterov } else { 221380628ca0SOleg Nesterov current->signal->nr_threads++; 221480628ca0SOleg Nesterov atomic_inc(¤t->signal->live); 221560d4de3fSElena Reshetova refcount_inc(¤t->signal->sigcnt); 2216924de3b8SEric W. Biederman task_join_group_stop(p); 221780628ca0SOleg Nesterov list_add_tail_rcu(&p->thread_group, 221880628ca0SOleg Nesterov &p->group_leader->thread_group); 22190c740d0aSOleg Nesterov list_add_tail_rcu(&p->thread_node, 22200c740d0aSOleg Nesterov &p->signal->thread_head); 22211da177e4SLinus Torvalds } 222281907739SOleg Nesterov attach_pid(p, PIDTYPE_PID); 22231da177e4SLinus Torvalds nr_threads++; 222473b9ebfeSOleg Nesterov } 22251da177e4SLinus Torvalds total_forks++; 2226c3ad2c3bSEric W. Biederman hlist_del_init(&delayed.node); 22273f17da69SOleg Nesterov spin_unlock(¤t->sighand->siglock); 22284af4206bSOleg Nesterov syscall_tracepoint_update(p); 22291da177e4SLinus Torvalds write_unlock_irq(&tasklist_lock); 22304af4206bSOleg Nesterov 2231c13cf856SAndrew Morton proc_fork_connector(p); 2232b53202e6SOleg Nesterov cgroup_post_fork(p); 2233780de9ddSIngo Molnar cgroup_threadgroup_change_end(current); 2234cdd6c482SIngo Molnar perf_event_fork(p); 223543d2b113SKAMEZAWA Hiroyuki 223643d2b113SKAMEZAWA Hiroyuki trace_task_newtask(p, clone_flags); 22373ab67966SOleg Nesterov uprobe_copy_process(p, clone_flags); 223843d2b113SKAMEZAWA Hiroyuki 22391da177e4SLinus Torvalds return p; 22401da177e4SLinus Torvalds 22417e47682eSAleksa Sarai bad_fork_cancel_cgroup: 22423fd37226SKirill Tkhai spin_unlock(¤t->sighand->siglock); 22433fd37226SKirill Tkhai write_unlock_irq(&tasklist_lock); 2244b53202e6SOleg Nesterov cgroup_cancel_fork(p); 2245c3b7112dSChristian Brauner bad_fork_cgroup_threadgroup_change_end: 2246c3b7112dSChristian Brauner cgroup_threadgroup_change_end(current); 2247b3e58382SChristian Brauner bad_fork_put_pidfd: 2248b3e58382SChristian Brauner if (clone_flags & CLONE_PIDFD) 2249b3e58382SChristian Brauner ksys_close(pidfd); 2250425fb2b4SPavel Emelyanov bad_fork_free_pid: 2251425fb2b4SPavel Emelyanov if (pid != &init_struct_pid) 2252425fb2b4SPavel Emelyanov free_pid(pid); 22530740aa5fSJiri Slaby bad_fork_cleanup_thread: 22540740aa5fSJiri Slaby exit_thread(p); 2255fd0928dfSJens Axboe bad_fork_cleanup_io: 2256b69f2292SLouis Rilling if (p->io_context) 2257b69f2292SLouis Rilling exit_io_context(p); 2258ab516013SSerge E. Hallyn bad_fork_cleanup_namespaces: 2259444f378bSLinus Torvalds exit_task_namespaces(p); 22601da177e4SLinus Torvalds bad_fork_cleanup_mm: 2261c3f3ce04SAndrea Arcangeli if (p->mm) { 2262c3f3ce04SAndrea Arcangeli mm_clear_owner(p->mm, p); 22631da177e4SLinus Torvalds mmput(p->mm); 2264c3f3ce04SAndrea Arcangeli } 22651da177e4SLinus Torvalds bad_fork_cleanup_signal: 22664ab6c083SOleg Nesterov if (!(clone_flags & CLONE_THREAD)) 22671c5354deSMike Galbraith free_signal_struct(p->signal); 22681da177e4SLinus Torvalds bad_fork_cleanup_sighand: 2269a7e5328aSOleg Nesterov __cleanup_sighand(p->sighand); 22701da177e4SLinus Torvalds bad_fork_cleanup_fs: 22711da177e4SLinus Torvalds exit_fs(p); /* blocking */ 22721da177e4SLinus Torvalds bad_fork_cleanup_files: 22731da177e4SLinus Torvalds exit_files(p); /* blocking */ 22741da177e4SLinus Torvalds bad_fork_cleanup_semundo: 22751da177e4SLinus Torvalds exit_sem(p); 2276e4e55b47STetsuo Handa bad_fork_cleanup_security: 2277e4e55b47STetsuo Handa security_task_free(p); 22781da177e4SLinus Torvalds bad_fork_cleanup_audit: 22791da177e4SLinus Torvalds audit_free(p); 22806c72e350SPeter Zijlstra bad_fork_cleanup_perf: 2281cdd6c482SIngo Molnar perf_event_free_task(p); 22826c72e350SPeter Zijlstra bad_fork_cleanup_policy: 2283b09be676SByungchul Park lockdep_free_task(p); 22841da177e4SLinus Torvalds #ifdef CONFIG_NUMA 2285f0be3d32SLee Schermerhorn mpol_put(p->mempolicy); 2286e8604cb4SLi Zefan bad_fork_cleanup_threadgroup_lock: 22871da177e4SLinus Torvalds #endif 228835df17c5SShailabh Nagar delayacct_tsk_free(p); 22891da177e4SLinus Torvalds bad_fork_cleanup_count: 2290d84f4f99SDavid Howells atomic_dec(&p->cred->user->processes); 2291e0e81739SDavid Howells exit_creds(p); 22921da177e4SLinus Torvalds bad_fork_free: 2293405c0759SAndy Lutomirski p->state = TASK_DEAD; 229468f24b08SAndy Lutomirski put_task_stack(p); 2295c3f3ce04SAndrea Arcangeli delayed_free_task(p); 2296fe7d37d1SOleg Nesterov fork_out: 2297c3ad2c3bSEric W. Biederman spin_lock_irq(¤t->sighand->siglock); 2298c3ad2c3bSEric W. Biederman hlist_del_init(&delayed.node); 2299c3ad2c3bSEric W. Biederman spin_unlock_irq(¤t->sighand->siglock); 2300fe7d37d1SOleg Nesterov return ERR_PTR(retval); 23011da177e4SLinus Torvalds } 23021da177e4SLinus Torvalds 23032c470475SEric W. Biederman static inline void init_idle_pids(struct task_struct *idle) 2304f106eee1SOleg Nesterov { 2305f106eee1SOleg Nesterov enum pid_type type; 2306f106eee1SOleg Nesterov 2307f106eee1SOleg Nesterov for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) { 23082c470475SEric W. Biederman INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */ 23092c470475SEric W. Biederman init_task_pid(idle, type, &init_struct_pid); 2310f106eee1SOleg Nesterov } 2311f106eee1SOleg Nesterov } 2312f106eee1SOleg Nesterov 23130db0628dSPaul Gortmaker struct task_struct *fork_idle(int cpu) 23141da177e4SLinus Torvalds { 231536c8b586SIngo Molnar struct task_struct *task; 2316b3e58382SChristian Brauner task = copy_process(CLONE_VM, 0, 0, NULL, NULL, &init_struct_pid, 0, 0, 2317725fc629SAndi Kleen cpu_to_node(cpu)); 2318f106eee1SOleg Nesterov if (!IS_ERR(task)) { 23192c470475SEric W. Biederman init_idle_pids(task); 23201da177e4SLinus Torvalds init_idle(task, cpu); 2321f106eee1SOleg Nesterov } 232273b9ebfeSOleg Nesterov 23231da177e4SLinus Torvalds return task; 23241da177e4SLinus Torvalds } 23251da177e4SLinus Torvalds 232613585fa0SNadav Amit struct mm_struct *copy_init_mm(void) 232713585fa0SNadav Amit { 232813585fa0SNadav Amit return dup_mm(NULL, &init_mm); 232913585fa0SNadav Amit } 233013585fa0SNadav Amit 23311da177e4SLinus Torvalds /* 23321da177e4SLinus Torvalds * Ok, this is the main fork-routine. 23331da177e4SLinus Torvalds * 23341da177e4SLinus Torvalds * It copies the process, and if successful kick-starts 23351da177e4SLinus Torvalds * it and waits for it to finish using the VM if required. 23361da177e4SLinus Torvalds */ 23373033f14aSJosh Triplett long _do_fork(unsigned long clone_flags, 23381da177e4SLinus Torvalds unsigned long stack_start, 23391da177e4SLinus Torvalds unsigned long stack_size, 23401da177e4SLinus Torvalds int __user *parent_tidptr, 23413033f14aSJosh Triplett int __user *child_tidptr, 23423033f14aSJosh Triplett unsigned long tls) 23431da177e4SLinus Torvalds { 23449f5325aaSMarcos Paulo de Souza struct completion vfork; 23459f5325aaSMarcos Paulo de Souza struct pid *pid; 23461da177e4SLinus Torvalds struct task_struct *p; 23471da177e4SLinus Torvalds int trace = 0; 234892476d7fSEric W. Biederman long nr; 23491da177e4SLinus Torvalds 2350bdff746aSAndrew Morton /* 23514b9d33e6STejun Heo * Determine whether and which event to report to ptracer. When 23524b9d33e6STejun Heo * called from kernel_thread or CLONE_UNTRACED is explicitly 23534b9d33e6STejun Heo * requested, no event is reported; otherwise, report if the event 23544b9d33e6STejun Heo * for the type of forking is enabled. 235509a05394SRoland McGrath */ 2356e80d6661SAl Viro if (!(clone_flags & CLONE_UNTRACED)) { 23574b9d33e6STejun Heo if (clone_flags & CLONE_VFORK) 23584b9d33e6STejun Heo trace = PTRACE_EVENT_VFORK; 23594b9d33e6STejun Heo else if ((clone_flags & CSIGNAL) != SIGCHLD) 23604b9d33e6STejun Heo trace = PTRACE_EVENT_CLONE; 23614b9d33e6STejun Heo else 23624b9d33e6STejun Heo trace = PTRACE_EVENT_FORK; 23634b9d33e6STejun Heo 23644b9d33e6STejun Heo if (likely(!ptrace_event_enabled(current, trace))) 23654b9d33e6STejun Heo trace = 0; 23664b9d33e6STejun Heo } 23671da177e4SLinus Torvalds 2368b3e58382SChristian Brauner p = copy_process(clone_flags, stack_start, stack_size, parent_tidptr, 2369725fc629SAndi Kleen child_tidptr, NULL, trace, tls, NUMA_NO_NODE); 237038addce8SEmese Revfy add_latent_entropy(); 23719f5325aaSMarcos Paulo de Souza 23729f5325aaSMarcos Paulo de Souza if (IS_ERR(p)) 23739f5325aaSMarcos Paulo de Souza return PTR_ERR(p); 23749f5325aaSMarcos Paulo de Souza 23751da177e4SLinus Torvalds /* 23761da177e4SLinus Torvalds * Do this prior waking up the new thread - the thread pointer 23771da177e4SLinus Torvalds * might get invalid after that point, if the thread exits quickly. 23781da177e4SLinus Torvalds */ 23790a16b607SMathieu Desnoyers trace_sched_process_fork(current, p); 23800a16b607SMathieu Desnoyers 23814e52365fSMatthew Dempsky pid = get_task_pid(p, PIDTYPE_PID); 23824e52365fSMatthew Dempsky nr = pid_vnr(pid); 238330e49c26SPavel Emelyanov 238430e49c26SPavel Emelyanov if (clone_flags & CLONE_PARENT_SETTID) 238530e49c26SPavel Emelyanov put_user(nr, parent_tidptr); 2386a6f5e063SSukadev Bhattiprolu 23871da177e4SLinus Torvalds if (clone_flags & CLONE_VFORK) { 23881da177e4SLinus Torvalds p->vfork_done = &vfork; 23891da177e4SLinus Torvalds init_completion(&vfork); 2390d68b46feSOleg Nesterov get_task_struct(p); 23911da177e4SLinus Torvalds } 23921da177e4SLinus Torvalds 23933e51e3edSSamir Bellabes wake_up_new_task(p); 23941da177e4SLinus Torvalds 23954b9d33e6STejun Heo /* forking complete and child started to run, tell ptracer */ 23964b9d33e6STejun Heo if (unlikely(trace)) 23974e52365fSMatthew Dempsky ptrace_event_pid(trace, pid); 239809a05394SRoland McGrath 23991da177e4SLinus Torvalds if (clone_flags & CLONE_VFORK) { 2400d68b46feSOleg Nesterov if (!wait_for_vfork_done(p, &vfork)) 24014e52365fSMatthew Dempsky ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid); 24029f59ce5dSChuck Ebbert } 24034e52365fSMatthew Dempsky 24044e52365fSMatthew Dempsky put_pid(pid); 240592476d7fSEric W. Biederman return nr; 24061da177e4SLinus Torvalds } 24071da177e4SLinus Torvalds 24083033f14aSJosh Triplett #ifndef CONFIG_HAVE_COPY_THREAD_TLS 24093033f14aSJosh Triplett /* For compatibility with architectures that call do_fork directly rather than 24103033f14aSJosh Triplett * using the syscall entry points below. */ 24113033f14aSJosh Triplett long do_fork(unsigned long clone_flags, 24123033f14aSJosh Triplett unsigned long stack_start, 24133033f14aSJosh Triplett unsigned long stack_size, 24143033f14aSJosh Triplett int __user *parent_tidptr, 24153033f14aSJosh Triplett int __user *child_tidptr) 24163033f14aSJosh Triplett { 24173033f14aSJosh Triplett return _do_fork(clone_flags, stack_start, stack_size, 24183033f14aSJosh Triplett parent_tidptr, child_tidptr, 0); 24193033f14aSJosh Triplett } 24203033f14aSJosh Triplett #endif 24213033f14aSJosh Triplett 24222aa3a7f8SAl Viro /* 24232aa3a7f8SAl Viro * Create a kernel thread. 24242aa3a7f8SAl Viro */ 24252aa3a7f8SAl Viro pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) 24262aa3a7f8SAl Viro { 24273033f14aSJosh Triplett return _do_fork(flags|CLONE_VM|CLONE_UNTRACED, (unsigned long)fn, 24283033f14aSJosh Triplett (unsigned long)arg, NULL, NULL, 0); 24292aa3a7f8SAl Viro } 24302aa3a7f8SAl Viro 2431d2125043SAl Viro #ifdef __ARCH_WANT_SYS_FORK 2432d2125043SAl Viro SYSCALL_DEFINE0(fork) 2433d2125043SAl Viro { 2434d2125043SAl Viro #ifdef CONFIG_MMU 24353033f14aSJosh Triplett return _do_fork(SIGCHLD, 0, 0, NULL, NULL, 0); 2436d2125043SAl Viro #else 2437d2125043SAl Viro /* can not support in nommu mode */ 24385d59e182SDaeseok Youn return -EINVAL; 2439d2125043SAl Viro #endif 2440d2125043SAl Viro } 2441d2125043SAl Viro #endif 2442d2125043SAl Viro 2443d2125043SAl Viro #ifdef __ARCH_WANT_SYS_VFORK 2444d2125043SAl Viro SYSCALL_DEFINE0(vfork) 2445d2125043SAl Viro { 24463033f14aSJosh Triplett return _do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, 0, 24473033f14aSJosh Triplett 0, NULL, NULL, 0); 2448d2125043SAl Viro } 2449d2125043SAl Viro #endif 2450d2125043SAl Viro 2451d2125043SAl Viro #ifdef __ARCH_WANT_SYS_CLONE 2452d2125043SAl Viro #ifdef CONFIG_CLONE_BACKWARDS 2453d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 2454d2125043SAl Viro int __user *, parent_tidptr, 24553033f14aSJosh Triplett unsigned long, tls, 2456d2125043SAl Viro int __user *, child_tidptr) 2457d2125043SAl Viro #elif defined(CONFIG_CLONE_BACKWARDS2) 2458d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags, 2459d2125043SAl Viro int __user *, parent_tidptr, 2460d2125043SAl Viro int __user *, child_tidptr, 24613033f14aSJosh Triplett unsigned long, tls) 2462dfa9771aSMichal Simek #elif defined(CONFIG_CLONE_BACKWARDS3) 2463dfa9771aSMichal Simek SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, 2464dfa9771aSMichal Simek int, stack_size, 2465dfa9771aSMichal Simek int __user *, parent_tidptr, 2466dfa9771aSMichal Simek int __user *, child_tidptr, 24673033f14aSJosh Triplett unsigned long, tls) 2468d2125043SAl Viro #else 2469d2125043SAl Viro SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 2470d2125043SAl Viro int __user *, parent_tidptr, 2471d2125043SAl Viro int __user *, child_tidptr, 24723033f14aSJosh Triplett unsigned long, tls) 2473d2125043SAl Viro #endif 2474d2125043SAl Viro { 24753033f14aSJosh Triplett return _do_fork(clone_flags, newsp, 0, parent_tidptr, child_tidptr, tls); 2476d2125043SAl Viro } 2477d2125043SAl Viro #endif 2478d2125043SAl Viro 24790f1b92cbSOleg Nesterov void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data) 24800f1b92cbSOleg Nesterov { 24810f1b92cbSOleg Nesterov struct task_struct *leader, *parent, *child; 24820f1b92cbSOleg Nesterov int res; 24830f1b92cbSOleg Nesterov 24840f1b92cbSOleg Nesterov read_lock(&tasklist_lock); 24850f1b92cbSOleg Nesterov leader = top = top->group_leader; 24860f1b92cbSOleg Nesterov down: 24870f1b92cbSOleg Nesterov for_each_thread(leader, parent) { 24880f1b92cbSOleg Nesterov list_for_each_entry(child, &parent->children, sibling) { 24890f1b92cbSOleg Nesterov res = visitor(child, data); 24900f1b92cbSOleg Nesterov if (res) { 24910f1b92cbSOleg Nesterov if (res < 0) 24920f1b92cbSOleg Nesterov goto out; 24930f1b92cbSOleg Nesterov leader = child; 24940f1b92cbSOleg Nesterov goto down; 24950f1b92cbSOleg Nesterov } 24960f1b92cbSOleg Nesterov up: 24970f1b92cbSOleg Nesterov ; 24980f1b92cbSOleg Nesterov } 24990f1b92cbSOleg Nesterov } 25000f1b92cbSOleg Nesterov 25010f1b92cbSOleg Nesterov if (leader != top) { 25020f1b92cbSOleg Nesterov child = leader; 25030f1b92cbSOleg Nesterov parent = child->real_parent; 25040f1b92cbSOleg Nesterov leader = parent->group_leader; 25050f1b92cbSOleg Nesterov goto up; 25060f1b92cbSOleg Nesterov } 25070f1b92cbSOleg Nesterov out: 25080f1b92cbSOleg Nesterov read_unlock(&tasklist_lock); 25090f1b92cbSOleg Nesterov } 25100f1b92cbSOleg Nesterov 25115fd63b30SRavikiran G Thirumalai #ifndef ARCH_MIN_MMSTRUCT_ALIGN 25125fd63b30SRavikiran G Thirumalai #define ARCH_MIN_MMSTRUCT_ALIGN 0 25135fd63b30SRavikiran G Thirumalai #endif 25145fd63b30SRavikiran G Thirumalai 251551cc5068SAlexey Dobriyan static void sighand_ctor(void *data) 2516aa1757f9SOleg Nesterov { 2517aa1757f9SOleg Nesterov struct sighand_struct *sighand = data; 2518aa1757f9SOleg Nesterov 2519aa1757f9SOleg Nesterov spin_lock_init(&sighand->siglock); 2520b8fceee1SDavide Libenzi init_waitqueue_head(&sighand->signalfd_wqh); 2521fba2afaaSDavide Libenzi } 2522aa1757f9SOleg Nesterov 25231da177e4SLinus Torvalds void __init proc_caches_init(void) 25241da177e4SLinus Torvalds { 2525c1a2f7f0SRik van Riel unsigned int mm_size; 2526c1a2f7f0SRik van Riel 25271da177e4SLinus Torvalds sighand_cachep = kmem_cache_create("sighand_cache", 25281da177e4SLinus Torvalds sizeof(struct sighand_struct), 0, 25295f0d5a3aSPaul E. McKenney SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU| 253075f296d9SLevin, Alexander (Sasha Levin) SLAB_ACCOUNT, sighand_ctor); 25311da177e4SLinus Torvalds signal_cachep = kmem_cache_create("signal_cache", 25321da177e4SLinus Torvalds sizeof(struct signal_struct), 0, 253375f296d9SLevin, Alexander (Sasha Levin) SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 25345d097056SVladimir Davydov NULL); 25351da177e4SLinus Torvalds files_cachep = kmem_cache_create("files_cache", 25361da177e4SLinus Torvalds sizeof(struct files_struct), 0, 253775f296d9SLevin, Alexander (Sasha Levin) SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 25385d097056SVladimir Davydov NULL); 25391da177e4SLinus Torvalds fs_cachep = kmem_cache_create("fs_cache", 25401da177e4SLinus Torvalds sizeof(struct fs_struct), 0, 254175f296d9SLevin, Alexander (Sasha Levin) SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 25425d097056SVladimir Davydov NULL); 2543c1a2f7f0SRik van Riel 25446345d24dSLinus Torvalds /* 2545c1a2f7f0SRik van Riel * The mm_cpumask is located at the end of mm_struct, and is 2546c1a2f7f0SRik van Riel * dynamically sized based on the maximum CPU number this system 2547c1a2f7f0SRik van Riel * can have, taking hotplug into account (nr_cpu_ids). 25486345d24dSLinus Torvalds */ 2549c1a2f7f0SRik van Riel mm_size = sizeof(struct mm_struct) + cpumask_size(); 2550c1a2f7f0SRik van Riel 255107dcd7feSDavid Windsor mm_cachep = kmem_cache_create_usercopy("mm_struct", 2552c1a2f7f0SRik van Riel mm_size, ARCH_MIN_MMSTRUCT_ALIGN, 255375f296d9SLevin, Alexander (Sasha Levin) SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 255407dcd7feSDavid Windsor offsetof(struct mm_struct, saved_auxv), 255507dcd7feSDavid Windsor sizeof_field(struct mm_struct, saved_auxv), 25565d097056SVladimir Davydov NULL); 25575d097056SVladimir Davydov vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT); 25588feae131SDavid Howells mmap_init(); 255966577193SAl Viro nsproxy_cache_init(); 25601da177e4SLinus Torvalds } 2561cf2e340fSJANAK DESAI 2562cf2e340fSJANAK DESAI /* 25639bfb23fcSOleg Nesterov * Check constraints on flags passed to the unshare system call. 2564cf2e340fSJANAK DESAI */ 25659bfb23fcSOleg Nesterov static int check_unshare_flags(unsigned long unshare_flags) 2566cf2e340fSJANAK DESAI { 25679bfb23fcSOleg Nesterov if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND| 25689bfb23fcSOleg Nesterov CLONE_VM|CLONE_FILES|CLONE_SYSVSEM| 256950804fe3SEric W. Biederman CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET| 2570a79a908fSAditya Kali CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP)) 2571cf2e340fSJANAK DESAI return -EINVAL; 25729bfb23fcSOleg Nesterov /* 257312c641abSEric W. Biederman * Not implemented, but pretend it works if there is nothing 257412c641abSEric W. Biederman * to unshare. Note that unsharing the address space or the 257512c641abSEric W. Biederman * signal handlers also need to unshare the signal queues (aka 257612c641abSEric W. Biederman * CLONE_THREAD). 25779bfb23fcSOleg Nesterov */ 25789bfb23fcSOleg Nesterov if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) { 257912c641abSEric W. Biederman if (!thread_group_empty(current)) 258012c641abSEric W. Biederman return -EINVAL; 258112c641abSEric W. Biederman } 258212c641abSEric W. Biederman if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) { 2583d036bda7SElena Reshetova if (refcount_read(¤t->sighand->count) > 1) 258412c641abSEric W. Biederman return -EINVAL; 258512c641abSEric W. Biederman } 258612c641abSEric W. Biederman if (unshare_flags & CLONE_VM) { 258712c641abSEric W. Biederman if (!current_is_single_threaded()) 25889bfb23fcSOleg Nesterov return -EINVAL; 25899bfb23fcSOleg Nesterov } 2590cf2e340fSJANAK DESAI 2591cf2e340fSJANAK DESAI return 0; 2592cf2e340fSJANAK DESAI } 2593cf2e340fSJANAK DESAI 2594cf2e340fSJANAK DESAI /* 259599d1419dSJANAK DESAI * Unshare the filesystem structure if it is being shared 2596cf2e340fSJANAK DESAI */ 2597cf2e340fSJANAK DESAI static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) 2598cf2e340fSJANAK DESAI { 2599cf2e340fSJANAK DESAI struct fs_struct *fs = current->fs; 2600cf2e340fSJANAK DESAI 2601498052bbSAl Viro if (!(unshare_flags & CLONE_FS) || !fs) 2602498052bbSAl Viro return 0; 2603498052bbSAl Viro 2604498052bbSAl Viro /* don't need lock here; in the worst case we'll do useless copy */ 2605498052bbSAl Viro if (fs->users == 1) 2606498052bbSAl Viro return 0; 2607498052bbSAl Viro 2608498052bbSAl Viro *new_fsp = copy_fs_struct(fs); 260999d1419dSJANAK DESAI if (!*new_fsp) 261099d1419dSJANAK DESAI return -ENOMEM; 2611cf2e340fSJANAK DESAI 2612cf2e340fSJANAK DESAI return 0; 2613cf2e340fSJANAK DESAI } 2614cf2e340fSJANAK DESAI 2615cf2e340fSJANAK DESAI /* 2616a016f338SJANAK DESAI * Unshare file descriptor table if it is being shared 2617cf2e340fSJANAK DESAI */ 2618cf2e340fSJANAK DESAI static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp) 2619cf2e340fSJANAK DESAI { 2620cf2e340fSJANAK DESAI struct files_struct *fd = current->files; 2621a016f338SJANAK DESAI int error = 0; 2622cf2e340fSJANAK DESAI 2623cf2e340fSJANAK DESAI if ((unshare_flags & CLONE_FILES) && 2624a016f338SJANAK DESAI (fd && atomic_read(&fd->count) > 1)) { 2625a016f338SJANAK DESAI *new_fdp = dup_fd(fd, &error); 2626a016f338SJANAK DESAI if (!*new_fdp) 2627a016f338SJANAK DESAI return error; 2628a016f338SJANAK DESAI } 2629cf2e340fSJANAK DESAI 2630cf2e340fSJANAK DESAI return 0; 2631cf2e340fSJANAK DESAI } 2632cf2e340fSJANAK DESAI 2633cf2e340fSJANAK DESAI /* 2634cf2e340fSJANAK DESAI * unshare allows a process to 'unshare' part of the process 2635cf2e340fSJANAK DESAI * context which was originally shared using clone. copy_* 2636cf2e340fSJANAK DESAI * functions used by do_fork() cannot be used here directly 2637cf2e340fSJANAK DESAI * because they modify an inactive task_struct that is being 2638cf2e340fSJANAK DESAI * constructed. Here we are modifying the current, active, 2639cf2e340fSJANAK DESAI * task_struct. 2640cf2e340fSJANAK DESAI */ 26419b32105eSDominik Brodowski int ksys_unshare(unsigned long unshare_flags) 2642cf2e340fSJANAK DESAI { 2643cf2e340fSJANAK DESAI struct fs_struct *fs, *new_fs = NULL; 2644cf2e340fSJANAK DESAI struct files_struct *fd, *new_fd = NULL; 2645b2e0d987SEric W. Biederman struct cred *new_cred = NULL; 2646cf7b708cSPavel Emelyanov struct nsproxy *new_nsproxy = NULL; 26479edff4abSManfred Spraul int do_sysvsem = 0; 26489bfb23fcSOleg Nesterov int err; 2649cf2e340fSJANAK DESAI 265050804fe3SEric W. Biederman /* 2651faf00da5SEric W. Biederman * If unsharing a user namespace must also unshare the thread group 2652faf00da5SEric W. Biederman * and unshare the filesystem root and working directories. 2653b2e0d987SEric W. Biederman */ 2654b2e0d987SEric W. Biederman if (unshare_flags & CLONE_NEWUSER) 2655e66eded8SEric W. Biederman unshare_flags |= CLONE_THREAD | CLONE_FS; 2656b2e0d987SEric W. Biederman /* 265750804fe3SEric W. Biederman * If unsharing vm, must also unshare signal handlers. 265850804fe3SEric W. Biederman */ 265950804fe3SEric W. Biederman if (unshare_flags & CLONE_VM) 266050804fe3SEric W. Biederman unshare_flags |= CLONE_SIGHAND; 26616013f67fSManfred Spraul /* 266212c641abSEric W. Biederman * If unsharing a signal handlers, must also unshare the signal queues. 266312c641abSEric W. Biederman */ 266412c641abSEric W. Biederman if (unshare_flags & CLONE_SIGHAND) 266512c641abSEric W. Biederman unshare_flags |= CLONE_THREAD; 266612c641abSEric W. Biederman /* 26679bfb23fcSOleg Nesterov * If unsharing namespace, must also unshare filesystem information. 26689bfb23fcSOleg Nesterov */ 26699bfb23fcSOleg Nesterov if (unshare_flags & CLONE_NEWNS) 26709bfb23fcSOleg Nesterov unshare_flags |= CLONE_FS; 267150804fe3SEric W. Biederman 267250804fe3SEric W. Biederman err = check_unshare_flags(unshare_flags); 267350804fe3SEric W. Biederman if (err) 267450804fe3SEric W. Biederman goto bad_unshare_out; 26759bfb23fcSOleg Nesterov /* 26766013f67fSManfred Spraul * CLONE_NEWIPC must also detach from the undolist: after switching 26776013f67fSManfred Spraul * to a new ipc namespace, the semaphore arrays from the old 26786013f67fSManfred Spraul * namespace are unreachable. 26796013f67fSManfred Spraul */ 26806013f67fSManfred Spraul if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM)) 26819edff4abSManfred Spraul do_sysvsem = 1; 2682fb0a685cSDaniel Rebelo de Oliveira err = unshare_fs(unshare_flags, &new_fs); 2683fb0a685cSDaniel Rebelo de Oliveira if (err) 26849bfb23fcSOleg Nesterov goto bad_unshare_out; 2685fb0a685cSDaniel Rebelo de Oliveira err = unshare_fd(unshare_flags, &new_fd); 2686fb0a685cSDaniel Rebelo de Oliveira if (err) 26879bfb23fcSOleg Nesterov goto bad_unshare_cleanup_fs; 2688b2e0d987SEric W. Biederman err = unshare_userns(unshare_flags, &new_cred); 2689fb0a685cSDaniel Rebelo de Oliveira if (err) 26909edff4abSManfred Spraul goto bad_unshare_cleanup_fd; 2691b2e0d987SEric W. Biederman err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, 2692b2e0d987SEric W. Biederman new_cred, new_fs); 2693b2e0d987SEric W. Biederman if (err) 2694b2e0d987SEric W. Biederman goto bad_unshare_cleanup_cred; 2695cf2e340fSJANAK DESAI 2696b2e0d987SEric W. Biederman if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) { 26979edff4abSManfred Spraul if (do_sysvsem) { 26989edff4abSManfred Spraul /* 26999edff4abSManfred Spraul * CLONE_SYSVSEM is equivalent to sys_exit(). 27009edff4abSManfred Spraul */ 27019edff4abSManfred Spraul exit_sem(current); 27029edff4abSManfred Spraul } 2703ab602f79SJack Miller if (unshare_flags & CLONE_NEWIPC) { 2704ab602f79SJack Miller /* Orphan segments in old ns (see sem above). */ 2705ab602f79SJack Miller exit_shm(current); 2706ab602f79SJack Miller shm_init_task(current); 2707ab602f79SJack Miller } 2708ab516013SSerge E. Hallyn 27096f977e6bSAlan Cox if (new_nsproxy) 2710cf7b708cSPavel Emelyanov switch_task_namespaces(current, new_nsproxy); 2711cf2e340fSJANAK DESAI 2712cf7b708cSPavel Emelyanov task_lock(current); 2713cf7b708cSPavel Emelyanov 2714cf2e340fSJANAK DESAI if (new_fs) { 2715cf2e340fSJANAK DESAI fs = current->fs; 27162a4419b5SNick Piggin spin_lock(&fs->lock); 2717cf2e340fSJANAK DESAI current->fs = new_fs; 2718498052bbSAl Viro if (--fs->users) 2719498052bbSAl Viro new_fs = NULL; 2720498052bbSAl Viro else 2721cf2e340fSJANAK DESAI new_fs = fs; 27222a4419b5SNick Piggin spin_unlock(&fs->lock); 2723cf2e340fSJANAK DESAI } 2724cf2e340fSJANAK DESAI 2725cf2e340fSJANAK DESAI if (new_fd) { 2726cf2e340fSJANAK DESAI fd = current->files; 2727cf2e340fSJANAK DESAI current->files = new_fd; 2728cf2e340fSJANAK DESAI new_fd = fd; 2729cf2e340fSJANAK DESAI } 2730cf2e340fSJANAK DESAI 2731cf2e340fSJANAK DESAI task_unlock(current); 2732b2e0d987SEric W. Biederman 2733b2e0d987SEric W. Biederman if (new_cred) { 2734b2e0d987SEric W. Biederman /* Install the new user namespace */ 2735b2e0d987SEric W. Biederman commit_creds(new_cred); 2736b2e0d987SEric W. Biederman new_cred = NULL; 2737b2e0d987SEric W. Biederman } 2738cf2e340fSJANAK DESAI } 2739cf2e340fSJANAK DESAI 2740e4222673SHari Bathini perf_event_namespaces(current); 2741e4222673SHari Bathini 2742b2e0d987SEric W. Biederman bad_unshare_cleanup_cred: 2743b2e0d987SEric W. Biederman if (new_cred) 2744b2e0d987SEric W. Biederman put_cred(new_cred); 2745cf2e340fSJANAK DESAI bad_unshare_cleanup_fd: 2746cf2e340fSJANAK DESAI if (new_fd) 2747cf2e340fSJANAK DESAI put_files_struct(new_fd); 2748cf2e340fSJANAK DESAI 2749cf2e340fSJANAK DESAI bad_unshare_cleanup_fs: 2750cf2e340fSJANAK DESAI if (new_fs) 2751498052bbSAl Viro free_fs_struct(new_fs); 2752cf2e340fSJANAK DESAI 2753cf2e340fSJANAK DESAI bad_unshare_out: 2754cf2e340fSJANAK DESAI return err; 2755cf2e340fSJANAK DESAI } 27563b125388SAl Viro 27579b32105eSDominik Brodowski SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags) 27589b32105eSDominik Brodowski { 27599b32105eSDominik Brodowski return ksys_unshare(unshare_flags); 27609b32105eSDominik Brodowski } 27619b32105eSDominik Brodowski 27623b125388SAl Viro /* 27633b125388SAl Viro * Helper to unshare the files of the current task. 27643b125388SAl Viro * We don't want to expose copy_files internals to 27653b125388SAl Viro * the exec layer of the kernel. 27663b125388SAl Viro */ 27673b125388SAl Viro 27683b125388SAl Viro int unshare_files(struct files_struct **displaced) 27693b125388SAl Viro { 27703b125388SAl Viro struct task_struct *task = current; 277150704516SAl Viro struct files_struct *copy = NULL; 27723b125388SAl Viro int error; 27733b125388SAl Viro 27743b125388SAl Viro error = unshare_fd(CLONE_FILES, ©); 27753b125388SAl Viro if (error || !copy) { 27763b125388SAl Viro *displaced = NULL; 27773b125388SAl Viro return error; 27783b125388SAl Viro } 27793b125388SAl Viro *displaced = task->files; 27803b125388SAl Viro task_lock(task); 27813b125388SAl Viro task->files = copy; 27823b125388SAl Viro task_unlock(task); 27833b125388SAl Viro return 0; 27843b125388SAl Viro } 278516db3d3fSHeinrich Schuchardt 278616db3d3fSHeinrich Schuchardt int sysctl_max_threads(struct ctl_table *table, int write, 278716db3d3fSHeinrich Schuchardt void __user *buffer, size_t *lenp, loff_t *ppos) 278816db3d3fSHeinrich Schuchardt { 278916db3d3fSHeinrich Schuchardt struct ctl_table t; 279016db3d3fSHeinrich Schuchardt int ret; 279116db3d3fSHeinrich Schuchardt int threads = max_threads; 279216db3d3fSHeinrich Schuchardt int min = MIN_THREADS; 279316db3d3fSHeinrich Schuchardt int max = MAX_THREADS; 279416db3d3fSHeinrich Schuchardt 279516db3d3fSHeinrich Schuchardt t = *table; 279616db3d3fSHeinrich Schuchardt t.data = &threads; 279716db3d3fSHeinrich Schuchardt t.extra1 = &min; 279816db3d3fSHeinrich Schuchardt t.extra2 = &max; 279916db3d3fSHeinrich Schuchardt 280016db3d3fSHeinrich Schuchardt ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 280116db3d3fSHeinrich Schuchardt if (ret || !write) 280216db3d3fSHeinrich Schuchardt return ret; 280316db3d3fSHeinrich Schuchardt 280416db3d3fSHeinrich Schuchardt set_max_threads(threads); 280516db3d3fSHeinrich Schuchardt 280616db3d3fSHeinrich Schuchardt return 0; 280716db3d3fSHeinrich Schuchardt } 2808