1 /* 2 * linux/kernel/fork.c 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 */ 6 7 /* 8 * 'fork.c' contains the help-routines for the 'fork' system call 9 * (see also entry.S and others). 10 * Fork is rather simple, once you get the hang of it, but the memory 11 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()' 12 */ 13 14 #include <linux/slab.h> 15 #include <linux/init.h> 16 #include <linux/unistd.h> 17 #include <linux/module.h> 18 #include <linux/vmalloc.h> 19 #include <linux/completion.h> 20 #include <linux/personality.h> 21 #include <linux/mempolicy.h> 22 #include <linux/sem.h> 23 #include <linux/file.h> 24 #include <linux/fdtable.h> 25 #include <linux/iocontext.h> 26 #include <linux/key.h> 27 #include <linux/binfmts.h> 28 #include <linux/mman.h> 29 #include <linux/mmu_notifier.h> 30 #include <linux/fs.h> 31 #include <linux/mm.h> 32 #include <linux/vmacache.h> 33 #include <linux/nsproxy.h> 34 #include <linux/capability.h> 35 #include <linux/cpu.h> 36 #include <linux/cgroup.h> 37 #include <linux/security.h> 38 #include <linux/hugetlb.h> 39 #include <linux/seccomp.h> 40 #include <linux/swap.h> 41 #include <linux/syscalls.h> 42 #include <linux/jiffies.h> 43 #include <linux/futex.h> 44 #include <linux/compat.h> 45 #include <linux/kthread.h> 46 #include <linux/task_io_accounting_ops.h> 47 #include <linux/rcupdate.h> 48 #include <linux/ptrace.h> 49 #include <linux/mount.h> 50 #include <linux/audit.h> 51 #include <linux/memcontrol.h> 52 #include <linux/ftrace.h> 53 #include <linux/proc_fs.h> 54 #include <linux/profile.h> 55 #include <linux/rmap.h> 56 #include <linux/ksm.h> 57 #include <linux/acct.h> 58 #include <linux/tsacct_kern.h> 59 #include <linux/cn_proc.h> 60 #include <linux/freezer.h> 61 #include <linux/delayacct.h> 62 #include <linux/taskstats_kern.h> 63 #include <linux/random.h> 64 #include <linux/tty.h> 65 #include <linux/blkdev.h> 66 #include <linux/fs_struct.h> 67 #include <linux/magic.h> 68 #include <linux/perf_event.h> 69 #include <linux/posix-timers.h> 70 #include <linux/user-return-notifier.h> 71 #include <linux/oom.h> 72 #include <linux/khugepaged.h> 73 #include <linux/signalfd.h> 74 #include <linux/uprobes.h> 75 #include <linux/aio.h> 76 #include <linux/compiler.h> 77 #include <linux/sysctl.h> 78 #include <linux/kcov.h> 79 80 #include <asm/pgtable.h> 81 #include <asm/pgalloc.h> 82 #include <asm/uaccess.h> 83 #include <asm/mmu_context.h> 84 #include <asm/cacheflush.h> 85 #include <asm/tlbflush.h> 86 87 #include <trace/events/sched.h> 88 89 #define CREATE_TRACE_POINTS 90 #include <trace/events/task.h> 91 92 /* 93 * Minimum number of threads to boot the kernel 94 */ 95 #define MIN_THREADS 20 96 97 /* 98 * Maximum number of threads 99 */ 100 #define MAX_THREADS FUTEX_TID_MASK 101 102 /* 103 * Protected counters by write_lock_irq(&tasklist_lock) 104 */ 105 unsigned long total_forks; /* Handle normal Linux uptimes. */ 106 int nr_threads; /* The idle threads do not count.. */ 107 108 int max_threads; /* tunable limit on nr_threads */ 109 110 DEFINE_PER_CPU(unsigned long, process_counts) = 0; 111 112 __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ 113 114 #ifdef CONFIG_PROVE_RCU 115 int lockdep_tasklist_lock_is_held(void) 116 { 117 return lockdep_is_held(&tasklist_lock); 118 } 119 EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held); 120 #endif /* #ifdef CONFIG_PROVE_RCU */ 121 122 int nr_processes(void) 123 { 124 int cpu; 125 int total = 0; 126 127 for_each_possible_cpu(cpu) 128 total += per_cpu(process_counts, cpu); 129 130 return total; 131 } 132 133 void __weak arch_release_task_struct(struct task_struct *tsk) 134 { 135 } 136 137 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR 138 static struct kmem_cache *task_struct_cachep; 139 140 static inline struct task_struct *alloc_task_struct_node(int node) 141 { 142 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node); 143 } 144 145 static inline void free_task_struct(struct task_struct *tsk) 146 { 147 kmem_cache_free(task_struct_cachep, tsk); 148 } 149 #endif 150 151 void __weak arch_release_thread_stack(unsigned long *stack) 152 { 153 } 154 155 #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR 156 157 /* 158 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a 159 * kmemcache based allocator. 160 */ 161 # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) 162 163 #ifdef CONFIG_VMAP_STACK 164 /* 165 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB 166 * flush. Try to minimize the number of calls by caching stacks. 167 */ 168 #define NR_CACHED_STACKS 2 169 static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); 170 #endif 171 172 static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) 173 { 174 #ifdef CONFIG_VMAP_STACK 175 void *stack; 176 int i; 177 178 local_irq_disable(); 179 for (i = 0; i < NR_CACHED_STACKS; i++) { 180 struct vm_struct *s = this_cpu_read(cached_stacks[i]); 181 182 if (!s) 183 continue; 184 this_cpu_write(cached_stacks[i], NULL); 185 186 tsk->stack_vm_area = s; 187 local_irq_enable(); 188 return s->addr; 189 } 190 local_irq_enable(); 191 192 stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE, 193 VMALLOC_START, VMALLOC_END, 194 THREADINFO_GFP | __GFP_HIGHMEM, 195 PAGE_KERNEL, 196 0, node, __builtin_return_address(0)); 197 198 /* 199 * We can't call find_vm_area() in interrupt context, and 200 * free_thread_stack() can be called in interrupt context, 201 * so cache the vm_struct. 202 */ 203 if (stack) 204 tsk->stack_vm_area = find_vm_area(stack); 205 return stack; 206 #else 207 struct page *page = alloc_pages_node(node, THREADINFO_GFP, 208 THREAD_SIZE_ORDER); 209 210 return page ? page_address(page) : NULL; 211 #endif 212 } 213 214 static inline void free_thread_stack(struct task_struct *tsk) 215 { 216 #ifdef CONFIG_VMAP_STACK 217 if (task_stack_vm_area(tsk)) { 218 unsigned long flags; 219 int i; 220 221 local_irq_save(flags); 222 for (i = 0; i < NR_CACHED_STACKS; i++) { 223 if (this_cpu_read(cached_stacks[i])) 224 continue; 225 226 this_cpu_write(cached_stacks[i], tsk->stack_vm_area); 227 local_irq_restore(flags); 228 return; 229 } 230 local_irq_restore(flags); 231 232 vfree(tsk->stack); 233 return; 234 } 235 #endif 236 237 __free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER); 238 } 239 # else 240 static struct kmem_cache *thread_stack_cache; 241 242 static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, 243 int node) 244 { 245 return kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node); 246 } 247 248 static void free_thread_stack(struct task_struct *tsk) 249 { 250 kmem_cache_free(thread_stack_cache, tsk->stack); 251 } 252 253 void thread_stack_cache_init(void) 254 { 255 thread_stack_cache = kmem_cache_create("thread_stack", THREAD_SIZE, 256 THREAD_SIZE, 0, NULL); 257 BUG_ON(thread_stack_cache == NULL); 258 } 259 # endif 260 #endif 261 262 /* SLAB cache for signal_struct structures (tsk->signal) */ 263 static struct kmem_cache *signal_cachep; 264 265 /* SLAB cache for sighand_struct structures (tsk->sighand) */ 266 struct kmem_cache *sighand_cachep; 267 268 /* SLAB cache for files_struct structures (tsk->files) */ 269 struct kmem_cache *files_cachep; 270 271 /* SLAB cache for fs_struct structures (tsk->fs) */ 272 struct kmem_cache *fs_cachep; 273 274 /* SLAB cache for vm_area_struct structures */ 275 struct kmem_cache *vm_area_cachep; 276 277 /* SLAB cache for mm_struct structures (tsk->mm) */ 278 static struct kmem_cache *mm_cachep; 279 280 static void account_kernel_stack(struct task_struct *tsk, int account) 281 { 282 void *stack = task_stack_page(tsk); 283 struct vm_struct *vm = task_stack_vm_area(tsk); 284 285 BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0); 286 287 if (vm) { 288 int i; 289 290 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE); 291 292 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { 293 mod_zone_page_state(page_zone(vm->pages[i]), 294 NR_KERNEL_STACK_KB, 295 PAGE_SIZE / 1024 * account); 296 } 297 298 /* All stack pages belong to the same memcg. */ 299 memcg_kmem_update_page_stat(vm->pages[0], MEMCG_KERNEL_STACK_KB, 300 account * (THREAD_SIZE / 1024)); 301 } else { 302 /* 303 * All stack pages are in the same zone and belong to the 304 * same memcg. 305 */ 306 struct page *first_page = virt_to_page(stack); 307 308 mod_zone_page_state(page_zone(first_page), NR_KERNEL_STACK_KB, 309 THREAD_SIZE / 1024 * account); 310 311 memcg_kmem_update_page_stat(first_page, MEMCG_KERNEL_STACK_KB, 312 account * (THREAD_SIZE / 1024)); 313 } 314 } 315 316 static void release_task_stack(struct task_struct *tsk) 317 { 318 account_kernel_stack(tsk, -1); 319 arch_release_thread_stack(tsk->stack); 320 free_thread_stack(tsk); 321 tsk->stack = NULL; 322 #ifdef CONFIG_VMAP_STACK 323 tsk->stack_vm_area = NULL; 324 #endif 325 } 326 327 #ifdef CONFIG_THREAD_INFO_IN_TASK 328 void put_task_stack(struct task_struct *tsk) 329 { 330 if (atomic_dec_and_test(&tsk->stack_refcount)) 331 release_task_stack(tsk); 332 } 333 #endif 334 335 void free_task(struct task_struct *tsk) 336 { 337 #ifndef CONFIG_THREAD_INFO_IN_TASK 338 /* 339 * The task is finally done with both the stack and thread_info, 340 * so free both. 341 */ 342 release_task_stack(tsk); 343 #else 344 /* 345 * If the task had a separate stack allocation, it should be gone 346 * by now. 347 */ 348 WARN_ON_ONCE(atomic_read(&tsk->stack_refcount) != 0); 349 #endif 350 rt_mutex_debug_task_free(tsk); 351 ftrace_graph_exit_task(tsk); 352 put_seccomp_filter(tsk); 353 arch_release_task_struct(tsk); 354 free_task_struct(tsk); 355 } 356 EXPORT_SYMBOL(free_task); 357 358 static inline void free_signal_struct(struct signal_struct *sig) 359 { 360 taskstats_tgid_free(sig); 361 sched_autogroup_exit(sig); 362 /* 363 * __mmdrop is not safe to call from softirq context on x86 due to 364 * pgd_dtor so postpone it to the async context 365 */ 366 if (sig->oom_mm) 367 mmdrop_async(sig->oom_mm); 368 kmem_cache_free(signal_cachep, sig); 369 } 370 371 static inline void put_signal_struct(struct signal_struct *sig) 372 { 373 if (atomic_dec_and_test(&sig->sigcnt)) 374 free_signal_struct(sig); 375 } 376 377 void __put_task_struct(struct task_struct *tsk) 378 { 379 WARN_ON(!tsk->exit_state); 380 WARN_ON(atomic_read(&tsk->usage)); 381 WARN_ON(tsk == current); 382 383 cgroup_free(tsk); 384 task_numa_free(tsk); 385 security_task_free(tsk); 386 exit_creds(tsk); 387 delayacct_tsk_free(tsk); 388 put_signal_struct(tsk->signal); 389 390 if (!profile_handoff_task(tsk)) 391 free_task(tsk); 392 } 393 EXPORT_SYMBOL_GPL(__put_task_struct); 394 395 void __init __weak arch_task_cache_init(void) { } 396 397 /* 398 * set_max_threads 399 */ 400 static void set_max_threads(unsigned int max_threads_suggested) 401 { 402 u64 threads; 403 404 /* 405 * The number of threads shall be limited such that the thread 406 * structures may only consume a small part of the available memory. 407 */ 408 if (fls64(totalram_pages) + fls64(PAGE_SIZE) > 64) 409 threads = MAX_THREADS; 410 else 411 threads = div64_u64((u64) totalram_pages * (u64) PAGE_SIZE, 412 (u64) THREAD_SIZE * 8UL); 413 414 if (threads > max_threads_suggested) 415 threads = max_threads_suggested; 416 417 max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS); 418 } 419 420 #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT 421 /* Initialized by the architecture: */ 422 int arch_task_struct_size __read_mostly; 423 #endif 424 425 void __init fork_init(void) 426 { 427 int i; 428 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR 429 #ifndef ARCH_MIN_TASKALIGN 430 #define ARCH_MIN_TASKALIGN L1_CACHE_BYTES 431 #endif 432 /* create a slab on which task_structs can be allocated */ 433 task_struct_cachep = kmem_cache_create("task_struct", 434 arch_task_struct_size, ARCH_MIN_TASKALIGN, 435 SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT, NULL); 436 #endif 437 438 /* do the arch specific task caches init */ 439 arch_task_cache_init(); 440 441 set_max_threads(MAX_THREADS); 442 443 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; 444 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; 445 init_task.signal->rlim[RLIMIT_SIGPENDING] = 446 init_task.signal->rlim[RLIMIT_NPROC]; 447 448 for (i = 0; i < UCOUNT_COUNTS; i++) { 449 init_user_ns.ucount_max[i] = max_threads/2; 450 } 451 } 452 453 int __weak arch_dup_task_struct(struct task_struct *dst, 454 struct task_struct *src) 455 { 456 *dst = *src; 457 return 0; 458 } 459 460 void set_task_stack_end_magic(struct task_struct *tsk) 461 { 462 unsigned long *stackend; 463 464 stackend = end_of_stack(tsk); 465 *stackend = STACK_END_MAGIC; /* for overflow detection */ 466 } 467 468 static struct task_struct *dup_task_struct(struct task_struct *orig, int node) 469 { 470 struct task_struct *tsk; 471 unsigned long *stack; 472 struct vm_struct *stack_vm_area; 473 int err; 474 475 if (node == NUMA_NO_NODE) 476 node = tsk_fork_get_node(orig); 477 tsk = alloc_task_struct_node(node); 478 if (!tsk) 479 return NULL; 480 481 stack = alloc_thread_stack_node(tsk, node); 482 if (!stack) 483 goto free_tsk; 484 485 stack_vm_area = task_stack_vm_area(tsk); 486 487 err = arch_dup_task_struct(tsk, orig); 488 489 /* 490 * arch_dup_task_struct() clobbers the stack-related fields. Make 491 * sure they're properly initialized before using any stack-related 492 * functions again. 493 */ 494 tsk->stack = stack; 495 #ifdef CONFIG_VMAP_STACK 496 tsk->stack_vm_area = stack_vm_area; 497 #endif 498 #ifdef CONFIG_THREAD_INFO_IN_TASK 499 atomic_set(&tsk->stack_refcount, 1); 500 #endif 501 502 if (err) 503 goto free_stack; 504 505 #ifdef CONFIG_SECCOMP 506 /* 507 * We must handle setting up seccomp filters once we're under 508 * the sighand lock in case orig has changed between now and 509 * then. Until then, filter must be NULL to avoid messing up 510 * the usage counts on the error path calling free_task. 511 */ 512 tsk->seccomp.filter = NULL; 513 #endif 514 515 setup_thread_stack(tsk, orig); 516 clear_user_return_notifier(tsk); 517 clear_tsk_need_resched(tsk); 518 set_task_stack_end_magic(tsk); 519 520 #ifdef CONFIG_CC_STACKPROTECTOR 521 tsk->stack_canary = get_random_int(); 522 #endif 523 524 /* 525 * One for us, one for whoever does the "release_task()" (usually 526 * parent) 527 */ 528 atomic_set(&tsk->usage, 2); 529 #ifdef CONFIG_BLK_DEV_IO_TRACE 530 tsk->btrace_seq = 0; 531 #endif 532 tsk->splice_pipe = NULL; 533 tsk->task_frag.page = NULL; 534 tsk->wake_q.next = NULL; 535 536 account_kernel_stack(tsk, 1); 537 538 kcov_task_init(tsk); 539 540 return tsk; 541 542 free_stack: 543 free_thread_stack(tsk); 544 free_tsk: 545 free_task_struct(tsk); 546 return NULL; 547 } 548 549 #ifdef CONFIG_MMU 550 static __latent_entropy int dup_mmap(struct mm_struct *mm, 551 struct mm_struct *oldmm) 552 { 553 struct vm_area_struct *mpnt, *tmp, *prev, **pprev; 554 struct rb_node **rb_link, *rb_parent; 555 int retval; 556 unsigned long charge; 557 558 uprobe_start_dup_mmap(); 559 if (down_write_killable(&oldmm->mmap_sem)) { 560 retval = -EINTR; 561 goto fail_uprobe_end; 562 } 563 flush_cache_dup_mm(oldmm); 564 uprobe_dup_mmap(oldmm, mm); 565 /* 566 * Not linked in yet - no deadlock potential: 567 */ 568 down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); 569 570 /* No ordering required: file already has been exposed. */ 571 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); 572 573 mm->total_vm = oldmm->total_vm; 574 mm->data_vm = oldmm->data_vm; 575 mm->exec_vm = oldmm->exec_vm; 576 mm->stack_vm = oldmm->stack_vm; 577 578 rb_link = &mm->mm_rb.rb_node; 579 rb_parent = NULL; 580 pprev = &mm->mmap; 581 retval = ksm_fork(mm, oldmm); 582 if (retval) 583 goto out; 584 retval = khugepaged_fork(mm, oldmm); 585 if (retval) 586 goto out; 587 588 prev = NULL; 589 for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { 590 struct file *file; 591 592 if (mpnt->vm_flags & VM_DONTCOPY) { 593 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); 594 continue; 595 } 596 charge = 0; 597 if (mpnt->vm_flags & VM_ACCOUNT) { 598 unsigned long len = vma_pages(mpnt); 599 600 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ 601 goto fail_nomem; 602 charge = len; 603 } 604 tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 605 if (!tmp) 606 goto fail_nomem; 607 *tmp = *mpnt; 608 INIT_LIST_HEAD(&tmp->anon_vma_chain); 609 retval = vma_dup_policy(mpnt, tmp); 610 if (retval) 611 goto fail_nomem_policy; 612 tmp->vm_mm = mm; 613 if (anon_vma_fork(tmp, mpnt)) 614 goto fail_nomem_anon_vma_fork; 615 tmp->vm_flags &= 616 ~(VM_LOCKED|VM_LOCKONFAULT|VM_UFFD_MISSING|VM_UFFD_WP); 617 tmp->vm_next = tmp->vm_prev = NULL; 618 tmp->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; 619 file = tmp->vm_file; 620 if (file) { 621 struct inode *inode = file_inode(file); 622 struct address_space *mapping = file->f_mapping; 623 624 get_file(file); 625 if (tmp->vm_flags & VM_DENYWRITE) 626 atomic_dec(&inode->i_writecount); 627 i_mmap_lock_write(mapping); 628 if (tmp->vm_flags & VM_SHARED) 629 atomic_inc(&mapping->i_mmap_writable); 630 flush_dcache_mmap_lock(mapping); 631 /* insert tmp into the share list, just after mpnt */ 632 vma_interval_tree_insert_after(tmp, mpnt, 633 &mapping->i_mmap); 634 flush_dcache_mmap_unlock(mapping); 635 i_mmap_unlock_write(mapping); 636 } 637 638 /* 639 * Clear hugetlb-related page reserves for children. This only 640 * affects MAP_PRIVATE mappings. Faults generated by the child 641 * are not guaranteed to succeed, even if read-only 642 */ 643 if (is_vm_hugetlb_page(tmp)) 644 reset_vma_resv_huge_pages(tmp); 645 646 /* 647 * Link in the new vma and copy the page table entries. 648 */ 649 *pprev = tmp; 650 pprev = &tmp->vm_next; 651 tmp->vm_prev = prev; 652 prev = tmp; 653 654 __vma_link_rb(mm, tmp, rb_link, rb_parent); 655 rb_link = &tmp->vm_rb.rb_right; 656 rb_parent = &tmp->vm_rb; 657 658 mm->map_count++; 659 retval = copy_page_range(mm, oldmm, mpnt); 660 661 if (tmp->vm_ops && tmp->vm_ops->open) 662 tmp->vm_ops->open(tmp); 663 664 if (retval) 665 goto out; 666 } 667 /* a new mm has just been created */ 668 arch_dup_mmap(oldmm, mm); 669 retval = 0; 670 out: 671 up_write(&mm->mmap_sem); 672 flush_tlb_mm(oldmm); 673 up_write(&oldmm->mmap_sem); 674 fail_uprobe_end: 675 uprobe_end_dup_mmap(); 676 return retval; 677 fail_nomem_anon_vma_fork: 678 mpol_put(vma_policy(tmp)); 679 fail_nomem_policy: 680 kmem_cache_free(vm_area_cachep, tmp); 681 fail_nomem: 682 retval = -ENOMEM; 683 vm_unacct_memory(charge); 684 goto out; 685 } 686 687 static inline int mm_alloc_pgd(struct mm_struct *mm) 688 { 689 mm->pgd = pgd_alloc(mm); 690 if (unlikely(!mm->pgd)) 691 return -ENOMEM; 692 return 0; 693 } 694 695 static inline void mm_free_pgd(struct mm_struct *mm) 696 { 697 pgd_free(mm, mm->pgd); 698 } 699 #else 700 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) 701 { 702 down_write(&oldmm->mmap_sem); 703 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); 704 up_write(&oldmm->mmap_sem); 705 return 0; 706 } 707 #define mm_alloc_pgd(mm) (0) 708 #define mm_free_pgd(mm) 709 #endif /* CONFIG_MMU */ 710 711 __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); 712 713 #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL)) 714 #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) 715 716 static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT; 717 718 static int __init coredump_filter_setup(char *s) 719 { 720 default_dump_filter = 721 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) & 722 MMF_DUMP_FILTER_MASK; 723 return 1; 724 } 725 726 __setup("coredump_filter=", coredump_filter_setup); 727 728 #include <linux/init_task.h> 729 730 static void mm_init_aio(struct mm_struct *mm) 731 { 732 #ifdef CONFIG_AIO 733 spin_lock_init(&mm->ioctx_lock); 734 mm->ioctx_table = NULL; 735 #endif 736 } 737 738 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) 739 { 740 #ifdef CONFIG_MEMCG 741 mm->owner = p; 742 #endif 743 } 744 745 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) 746 { 747 mm->mmap = NULL; 748 mm->mm_rb = RB_ROOT; 749 mm->vmacache_seqnum = 0; 750 atomic_set(&mm->mm_users, 1); 751 atomic_set(&mm->mm_count, 1); 752 init_rwsem(&mm->mmap_sem); 753 INIT_LIST_HEAD(&mm->mmlist); 754 mm->core_state = NULL; 755 atomic_long_set(&mm->nr_ptes, 0); 756 mm_nr_pmds_init(mm); 757 mm->map_count = 0; 758 mm->locked_vm = 0; 759 mm->pinned_vm = 0; 760 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); 761 spin_lock_init(&mm->page_table_lock); 762 mm_init_cpumask(mm); 763 mm_init_aio(mm); 764 mm_init_owner(mm, p); 765 mmu_notifier_mm_init(mm); 766 clear_tlb_flush_pending(mm); 767 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS 768 mm->pmd_huge_pte = NULL; 769 #endif 770 771 if (current->mm) { 772 mm->flags = current->mm->flags & MMF_INIT_MASK; 773 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; 774 } else { 775 mm->flags = default_dump_filter; 776 mm->def_flags = 0; 777 } 778 779 if (mm_alloc_pgd(mm)) 780 goto fail_nopgd; 781 782 if (init_new_context(p, mm)) 783 goto fail_nocontext; 784 785 return mm; 786 787 fail_nocontext: 788 mm_free_pgd(mm); 789 fail_nopgd: 790 free_mm(mm); 791 return NULL; 792 } 793 794 static void check_mm(struct mm_struct *mm) 795 { 796 int i; 797 798 for (i = 0; i < NR_MM_COUNTERS; i++) { 799 long x = atomic_long_read(&mm->rss_stat.count[i]); 800 801 if (unlikely(x)) 802 printk(KERN_ALERT "BUG: Bad rss-counter state " 803 "mm:%p idx:%d val:%ld\n", mm, i, x); 804 } 805 806 if (atomic_long_read(&mm->nr_ptes)) 807 pr_alert("BUG: non-zero nr_ptes on freeing mm: %ld\n", 808 atomic_long_read(&mm->nr_ptes)); 809 if (mm_nr_pmds(mm)) 810 pr_alert("BUG: non-zero nr_pmds on freeing mm: %ld\n", 811 mm_nr_pmds(mm)); 812 813 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS 814 VM_BUG_ON_MM(mm->pmd_huge_pte, mm); 815 #endif 816 } 817 818 /* 819 * Allocate and initialize an mm_struct. 820 */ 821 struct mm_struct *mm_alloc(void) 822 { 823 struct mm_struct *mm; 824 825 mm = allocate_mm(); 826 if (!mm) 827 return NULL; 828 829 memset(mm, 0, sizeof(*mm)); 830 return mm_init(mm, current); 831 } 832 833 /* 834 * Called when the last reference to the mm 835 * is dropped: either by a lazy thread or by 836 * mmput. Free the page directory and the mm. 837 */ 838 void __mmdrop(struct mm_struct *mm) 839 { 840 BUG_ON(mm == &init_mm); 841 mm_free_pgd(mm); 842 destroy_context(mm); 843 mmu_notifier_mm_destroy(mm); 844 check_mm(mm); 845 free_mm(mm); 846 } 847 EXPORT_SYMBOL_GPL(__mmdrop); 848 849 static inline void __mmput(struct mm_struct *mm) 850 { 851 VM_BUG_ON(atomic_read(&mm->mm_users)); 852 853 uprobe_clear_state(mm); 854 exit_aio(mm); 855 ksm_exit(mm); 856 khugepaged_exit(mm); /* must run before exit_mmap */ 857 exit_mmap(mm); 858 mm_put_huge_zero_page(mm); 859 set_mm_exe_file(mm, NULL); 860 if (!list_empty(&mm->mmlist)) { 861 spin_lock(&mmlist_lock); 862 list_del(&mm->mmlist); 863 spin_unlock(&mmlist_lock); 864 } 865 if (mm->binfmt) 866 module_put(mm->binfmt->module); 867 set_bit(MMF_OOM_SKIP, &mm->flags); 868 mmdrop(mm); 869 } 870 871 /* 872 * Decrement the use count and release all resources for an mm. 873 */ 874 void mmput(struct mm_struct *mm) 875 { 876 might_sleep(); 877 878 if (atomic_dec_and_test(&mm->mm_users)) 879 __mmput(mm); 880 } 881 EXPORT_SYMBOL_GPL(mmput); 882 883 #ifdef CONFIG_MMU 884 static void mmput_async_fn(struct work_struct *work) 885 { 886 struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work); 887 __mmput(mm); 888 } 889 890 void mmput_async(struct mm_struct *mm) 891 { 892 if (atomic_dec_and_test(&mm->mm_users)) { 893 INIT_WORK(&mm->async_put_work, mmput_async_fn); 894 schedule_work(&mm->async_put_work); 895 } 896 } 897 #endif 898 899 /** 900 * set_mm_exe_file - change a reference to the mm's executable file 901 * 902 * This changes mm's executable file (shown as symlink /proc/[pid]/exe). 903 * 904 * Main users are mmput() and sys_execve(). Callers prevent concurrent 905 * invocations: in mmput() nobody alive left, in execve task is single 906 * threaded. sys_prctl(PR_SET_MM_MAP/EXE_FILE) also needs to set the 907 * mm->exe_file, but does so without using set_mm_exe_file() in order 908 * to do avoid the need for any locks. 909 */ 910 void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) 911 { 912 struct file *old_exe_file; 913 914 /* 915 * It is safe to dereference the exe_file without RCU as 916 * this function is only called if nobody else can access 917 * this mm -- see comment above for justification. 918 */ 919 old_exe_file = rcu_dereference_raw(mm->exe_file); 920 921 if (new_exe_file) 922 get_file(new_exe_file); 923 rcu_assign_pointer(mm->exe_file, new_exe_file); 924 if (old_exe_file) 925 fput(old_exe_file); 926 } 927 928 /** 929 * get_mm_exe_file - acquire a reference to the mm's executable file 930 * 931 * Returns %NULL if mm has no associated executable file. 932 * User must release file via fput(). 933 */ 934 struct file *get_mm_exe_file(struct mm_struct *mm) 935 { 936 struct file *exe_file; 937 938 rcu_read_lock(); 939 exe_file = rcu_dereference(mm->exe_file); 940 if (exe_file && !get_file_rcu(exe_file)) 941 exe_file = NULL; 942 rcu_read_unlock(); 943 return exe_file; 944 } 945 EXPORT_SYMBOL(get_mm_exe_file); 946 947 /** 948 * get_task_exe_file - acquire a reference to the task's executable file 949 * 950 * Returns %NULL if task's mm (if any) has no associated executable file or 951 * this is a kernel thread with borrowed mm (see the comment above get_task_mm). 952 * User must release file via fput(). 953 */ 954 struct file *get_task_exe_file(struct task_struct *task) 955 { 956 struct file *exe_file = NULL; 957 struct mm_struct *mm; 958 959 task_lock(task); 960 mm = task->mm; 961 if (mm) { 962 if (!(task->flags & PF_KTHREAD)) 963 exe_file = get_mm_exe_file(mm); 964 } 965 task_unlock(task); 966 return exe_file; 967 } 968 EXPORT_SYMBOL(get_task_exe_file); 969 970 /** 971 * get_task_mm - acquire a reference to the task's mm 972 * 973 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning 974 * this kernel workthread has transiently adopted a user mm with use_mm, 975 * to do its AIO) is not set and if so returns a reference to it, after 976 * bumping up the use count. User must release the mm via mmput() 977 * after use. Typically used by /proc and ptrace. 978 */ 979 struct mm_struct *get_task_mm(struct task_struct *task) 980 { 981 struct mm_struct *mm; 982 983 task_lock(task); 984 mm = task->mm; 985 if (mm) { 986 if (task->flags & PF_KTHREAD) 987 mm = NULL; 988 else 989 atomic_inc(&mm->mm_users); 990 } 991 task_unlock(task); 992 return mm; 993 } 994 EXPORT_SYMBOL_GPL(get_task_mm); 995 996 struct mm_struct *mm_access(struct task_struct *task, unsigned int mode) 997 { 998 struct mm_struct *mm; 999 int err; 1000 1001 err = mutex_lock_killable(&task->signal->cred_guard_mutex); 1002 if (err) 1003 return ERR_PTR(err); 1004 1005 mm = get_task_mm(task); 1006 if (mm && mm != current->mm && 1007 !ptrace_may_access(task, mode)) { 1008 mmput(mm); 1009 mm = ERR_PTR(-EACCES); 1010 } 1011 mutex_unlock(&task->signal->cred_guard_mutex); 1012 1013 return mm; 1014 } 1015 1016 static void complete_vfork_done(struct task_struct *tsk) 1017 { 1018 struct completion *vfork; 1019 1020 task_lock(tsk); 1021 vfork = tsk->vfork_done; 1022 if (likely(vfork)) { 1023 tsk->vfork_done = NULL; 1024 complete(vfork); 1025 } 1026 task_unlock(tsk); 1027 } 1028 1029 static int wait_for_vfork_done(struct task_struct *child, 1030 struct completion *vfork) 1031 { 1032 int killed; 1033 1034 freezer_do_not_count(); 1035 killed = wait_for_completion_killable(vfork); 1036 freezer_count(); 1037 1038 if (killed) { 1039 task_lock(child); 1040 child->vfork_done = NULL; 1041 task_unlock(child); 1042 } 1043 1044 put_task_struct(child); 1045 return killed; 1046 } 1047 1048 /* Please note the differences between mmput and mm_release. 1049 * mmput is called whenever we stop holding onto a mm_struct, 1050 * error success whatever. 1051 * 1052 * mm_release is called after a mm_struct has been removed 1053 * from the current process. 1054 * 1055 * This difference is important for error handling, when we 1056 * only half set up a mm_struct for a new process and need to restore 1057 * the old one. Because we mmput the new mm_struct before 1058 * restoring the old one. . . 1059 * Eric Biederman 10 January 1998 1060 */ 1061 void mm_release(struct task_struct *tsk, struct mm_struct *mm) 1062 { 1063 /* Get rid of any futexes when releasing the mm */ 1064 #ifdef CONFIG_FUTEX 1065 if (unlikely(tsk->robust_list)) { 1066 exit_robust_list(tsk); 1067 tsk->robust_list = NULL; 1068 } 1069 #ifdef CONFIG_COMPAT 1070 if (unlikely(tsk->compat_robust_list)) { 1071 compat_exit_robust_list(tsk); 1072 tsk->compat_robust_list = NULL; 1073 } 1074 #endif 1075 if (unlikely(!list_empty(&tsk->pi_state_list))) 1076 exit_pi_state_list(tsk); 1077 #endif 1078 1079 uprobe_free_utask(tsk); 1080 1081 /* Get rid of any cached register state */ 1082 deactivate_mm(tsk, mm); 1083 1084 /* 1085 * Signal userspace if we're not exiting with a core dump 1086 * because we want to leave the value intact for debugging 1087 * purposes. 1088 */ 1089 if (tsk->clear_child_tid) { 1090 if (!(tsk->signal->flags & SIGNAL_GROUP_COREDUMP) && 1091 atomic_read(&mm->mm_users) > 1) { 1092 /* 1093 * We don't check the error code - if userspace has 1094 * not set up a proper pointer then tough luck. 1095 */ 1096 put_user(0, tsk->clear_child_tid); 1097 sys_futex(tsk->clear_child_tid, FUTEX_WAKE, 1098 1, NULL, NULL, 0); 1099 } 1100 tsk->clear_child_tid = NULL; 1101 } 1102 1103 /* 1104 * All done, finally we can wake up parent and return this mm to him. 1105 * Also kthread_stop() uses this completion for synchronization. 1106 */ 1107 if (tsk->vfork_done) 1108 complete_vfork_done(tsk); 1109 } 1110 1111 /* 1112 * Allocate a new mm structure and copy contents from the 1113 * mm structure of the passed in task structure. 1114 */ 1115 static struct mm_struct *dup_mm(struct task_struct *tsk) 1116 { 1117 struct mm_struct *mm, *oldmm = current->mm; 1118 int err; 1119 1120 mm = allocate_mm(); 1121 if (!mm) 1122 goto fail_nomem; 1123 1124 memcpy(mm, oldmm, sizeof(*mm)); 1125 1126 if (!mm_init(mm, tsk)) 1127 goto fail_nomem; 1128 1129 err = dup_mmap(mm, oldmm); 1130 if (err) 1131 goto free_pt; 1132 1133 mm->hiwater_rss = get_mm_rss(mm); 1134 mm->hiwater_vm = mm->total_vm; 1135 1136 if (mm->binfmt && !try_module_get(mm->binfmt->module)) 1137 goto free_pt; 1138 1139 return mm; 1140 1141 free_pt: 1142 /* don't put binfmt in mmput, we haven't got module yet */ 1143 mm->binfmt = NULL; 1144 mmput(mm); 1145 1146 fail_nomem: 1147 return NULL; 1148 } 1149 1150 static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) 1151 { 1152 struct mm_struct *mm, *oldmm; 1153 int retval; 1154 1155 tsk->min_flt = tsk->maj_flt = 0; 1156 tsk->nvcsw = tsk->nivcsw = 0; 1157 #ifdef CONFIG_DETECT_HUNG_TASK 1158 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; 1159 #endif 1160 1161 tsk->mm = NULL; 1162 tsk->active_mm = NULL; 1163 1164 /* 1165 * Are we cloning a kernel thread? 1166 * 1167 * We need to steal a active VM for that.. 1168 */ 1169 oldmm = current->mm; 1170 if (!oldmm) 1171 return 0; 1172 1173 /* initialize the new vmacache entries */ 1174 vmacache_flush(tsk); 1175 1176 if (clone_flags & CLONE_VM) { 1177 atomic_inc(&oldmm->mm_users); 1178 mm = oldmm; 1179 goto good_mm; 1180 } 1181 1182 retval = -ENOMEM; 1183 mm = dup_mm(tsk); 1184 if (!mm) 1185 goto fail_nomem; 1186 1187 good_mm: 1188 tsk->mm = mm; 1189 tsk->active_mm = mm; 1190 return 0; 1191 1192 fail_nomem: 1193 return retval; 1194 } 1195 1196 static int copy_fs(unsigned long clone_flags, struct task_struct *tsk) 1197 { 1198 struct fs_struct *fs = current->fs; 1199 if (clone_flags & CLONE_FS) { 1200 /* tsk->fs is already what we want */ 1201 spin_lock(&fs->lock); 1202 if (fs->in_exec) { 1203 spin_unlock(&fs->lock); 1204 return -EAGAIN; 1205 } 1206 fs->users++; 1207 spin_unlock(&fs->lock); 1208 return 0; 1209 } 1210 tsk->fs = copy_fs_struct(fs); 1211 if (!tsk->fs) 1212 return -ENOMEM; 1213 return 0; 1214 } 1215 1216 static int copy_files(unsigned long clone_flags, struct task_struct *tsk) 1217 { 1218 struct files_struct *oldf, *newf; 1219 int error = 0; 1220 1221 /* 1222 * A background process may not have any files ... 1223 */ 1224 oldf = current->files; 1225 if (!oldf) 1226 goto out; 1227 1228 if (clone_flags & CLONE_FILES) { 1229 atomic_inc(&oldf->count); 1230 goto out; 1231 } 1232 1233 newf = dup_fd(oldf, &error); 1234 if (!newf) 1235 goto out; 1236 1237 tsk->files = newf; 1238 error = 0; 1239 out: 1240 return error; 1241 } 1242 1243 static int copy_io(unsigned long clone_flags, struct task_struct *tsk) 1244 { 1245 #ifdef CONFIG_BLOCK 1246 struct io_context *ioc = current->io_context; 1247 struct io_context *new_ioc; 1248 1249 if (!ioc) 1250 return 0; 1251 /* 1252 * Share io context with parent, if CLONE_IO is set 1253 */ 1254 if (clone_flags & CLONE_IO) { 1255 ioc_task_link(ioc); 1256 tsk->io_context = ioc; 1257 } else if (ioprio_valid(ioc->ioprio)) { 1258 new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE); 1259 if (unlikely(!new_ioc)) 1260 return -ENOMEM; 1261 1262 new_ioc->ioprio = ioc->ioprio; 1263 put_io_context(new_ioc); 1264 } 1265 #endif 1266 return 0; 1267 } 1268 1269 static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk) 1270 { 1271 struct sighand_struct *sig; 1272 1273 if (clone_flags & CLONE_SIGHAND) { 1274 atomic_inc(¤t->sighand->count); 1275 return 0; 1276 } 1277 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL); 1278 rcu_assign_pointer(tsk->sighand, sig); 1279 if (!sig) 1280 return -ENOMEM; 1281 1282 atomic_set(&sig->count, 1); 1283 memcpy(sig->action, current->sighand->action, sizeof(sig->action)); 1284 return 0; 1285 } 1286 1287 void __cleanup_sighand(struct sighand_struct *sighand) 1288 { 1289 if (atomic_dec_and_test(&sighand->count)) { 1290 signalfd_cleanup(sighand); 1291 /* 1292 * sighand_cachep is SLAB_DESTROY_BY_RCU so we can free it 1293 * without an RCU grace period, see __lock_task_sighand(). 1294 */ 1295 kmem_cache_free(sighand_cachep, sighand); 1296 } 1297 } 1298 1299 /* 1300 * Initialize POSIX timer handling for a thread group. 1301 */ 1302 static void posix_cpu_timers_init_group(struct signal_struct *sig) 1303 { 1304 unsigned long cpu_limit; 1305 1306 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); 1307 if (cpu_limit != RLIM_INFINITY) { 1308 sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit); 1309 sig->cputimer.running = true; 1310 } 1311 1312 /* The timer lists. */ 1313 INIT_LIST_HEAD(&sig->cpu_timers[0]); 1314 INIT_LIST_HEAD(&sig->cpu_timers[1]); 1315 INIT_LIST_HEAD(&sig->cpu_timers[2]); 1316 } 1317 1318 static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) 1319 { 1320 struct signal_struct *sig; 1321 1322 if (clone_flags & CLONE_THREAD) 1323 return 0; 1324 1325 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL); 1326 tsk->signal = sig; 1327 if (!sig) 1328 return -ENOMEM; 1329 1330 sig->nr_threads = 1; 1331 atomic_set(&sig->live, 1); 1332 atomic_set(&sig->sigcnt, 1); 1333 1334 /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */ 1335 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); 1336 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); 1337 1338 init_waitqueue_head(&sig->wait_chldexit); 1339 sig->curr_target = tsk; 1340 init_sigpending(&sig->shared_pending); 1341 INIT_LIST_HEAD(&sig->posix_timers); 1342 seqlock_init(&sig->stats_lock); 1343 prev_cputime_init(&sig->prev_cputime); 1344 1345 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1346 sig->real_timer.function = it_real_fn; 1347 1348 task_lock(current->group_leader); 1349 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); 1350 task_unlock(current->group_leader); 1351 1352 posix_cpu_timers_init_group(sig); 1353 1354 tty_audit_fork(sig); 1355 sched_autogroup_fork(sig); 1356 1357 sig->oom_score_adj = current->signal->oom_score_adj; 1358 sig->oom_score_adj_min = current->signal->oom_score_adj_min; 1359 1360 sig->has_child_subreaper = current->signal->has_child_subreaper || 1361 current->signal->is_child_subreaper; 1362 1363 mutex_init(&sig->cred_guard_mutex); 1364 1365 return 0; 1366 } 1367 1368 static void copy_seccomp(struct task_struct *p) 1369 { 1370 #ifdef CONFIG_SECCOMP 1371 /* 1372 * Must be called with sighand->lock held, which is common to 1373 * all threads in the group. Holding cred_guard_mutex is not 1374 * needed because this new task is not yet running and cannot 1375 * be racing exec. 1376 */ 1377 assert_spin_locked(¤t->sighand->siglock); 1378 1379 /* Ref-count the new filter user, and assign it. */ 1380 get_seccomp_filter(current); 1381 p->seccomp = current->seccomp; 1382 1383 /* 1384 * Explicitly enable no_new_privs here in case it got set 1385 * between the task_struct being duplicated and holding the 1386 * sighand lock. The seccomp state and nnp must be in sync. 1387 */ 1388 if (task_no_new_privs(current)) 1389 task_set_no_new_privs(p); 1390 1391 /* 1392 * If the parent gained a seccomp mode after copying thread 1393 * flags and between before we held the sighand lock, we have 1394 * to manually enable the seccomp thread flag here. 1395 */ 1396 if (p->seccomp.mode != SECCOMP_MODE_DISABLED) 1397 set_tsk_thread_flag(p, TIF_SECCOMP); 1398 #endif 1399 } 1400 1401 SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr) 1402 { 1403 current->clear_child_tid = tidptr; 1404 1405 return task_pid_vnr(current); 1406 } 1407 1408 static void rt_mutex_init_task(struct task_struct *p) 1409 { 1410 raw_spin_lock_init(&p->pi_lock); 1411 #ifdef CONFIG_RT_MUTEXES 1412 p->pi_waiters = RB_ROOT; 1413 p->pi_waiters_leftmost = NULL; 1414 p->pi_blocked_on = NULL; 1415 #endif 1416 } 1417 1418 /* 1419 * Initialize POSIX timer handling for a single task. 1420 */ 1421 static void posix_cpu_timers_init(struct task_struct *tsk) 1422 { 1423 tsk->cputime_expires.prof_exp = 0; 1424 tsk->cputime_expires.virt_exp = 0; 1425 tsk->cputime_expires.sched_exp = 0; 1426 INIT_LIST_HEAD(&tsk->cpu_timers[0]); 1427 INIT_LIST_HEAD(&tsk->cpu_timers[1]); 1428 INIT_LIST_HEAD(&tsk->cpu_timers[2]); 1429 } 1430 1431 static inline void 1432 init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid) 1433 { 1434 task->pids[type].pid = pid; 1435 } 1436 1437 /* 1438 * This creates a new process as a copy of the old one, 1439 * but does not actually start it yet. 1440 * 1441 * It copies the registers, and all the appropriate 1442 * parts of the process environment (as per the clone 1443 * flags). The actual kick-off is left to the caller. 1444 */ 1445 static __latent_entropy struct task_struct *copy_process( 1446 unsigned long clone_flags, 1447 unsigned long stack_start, 1448 unsigned long stack_size, 1449 int __user *child_tidptr, 1450 struct pid *pid, 1451 int trace, 1452 unsigned long tls, 1453 int node) 1454 { 1455 int retval; 1456 struct task_struct *p; 1457 1458 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) 1459 return ERR_PTR(-EINVAL); 1460 1461 if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) 1462 return ERR_PTR(-EINVAL); 1463 1464 /* 1465 * Thread groups must share signals as well, and detached threads 1466 * can only be started up within the thread group. 1467 */ 1468 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND)) 1469 return ERR_PTR(-EINVAL); 1470 1471 /* 1472 * Shared signal handlers imply shared VM. By way of the above, 1473 * thread groups also imply shared VM. Blocking this case allows 1474 * for various simplifications in other code. 1475 */ 1476 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM)) 1477 return ERR_PTR(-EINVAL); 1478 1479 /* 1480 * Siblings of global init remain as zombies on exit since they are 1481 * not reaped by their parent (swapper). To solve this and to avoid 1482 * multi-rooted process trees, prevent global and container-inits 1483 * from creating siblings. 1484 */ 1485 if ((clone_flags & CLONE_PARENT) && 1486 current->signal->flags & SIGNAL_UNKILLABLE) 1487 return ERR_PTR(-EINVAL); 1488 1489 /* 1490 * If the new process will be in a different pid or user namespace 1491 * do not allow it to share a thread group with the forking task. 1492 */ 1493 if (clone_flags & CLONE_THREAD) { 1494 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) || 1495 (task_active_pid_ns(current) != 1496 current->nsproxy->pid_ns_for_children)) 1497 return ERR_PTR(-EINVAL); 1498 } 1499 1500 retval = security_task_create(clone_flags); 1501 if (retval) 1502 goto fork_out; 1503 1504 retval = -ENOMEM; 1505 p = dup_task_struct(current, node); 1506 if (!p) 1507 goto fork_out; 1508 1509 ftrace_graph_init_task(p); 1510 1511 rt_mutex_init_task(p); 1512 1513 #ifdef CONFIG_PROVE_LOCKING 1514 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled); 1515 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); 1516 #endif 1517 retval = -EAGAIN; 1518 if (atomic_read(&p->real_cred->user->processes) >= 1519 task_rlimit(p, RLIMIT_NPROC)) { 1520 if (p->real_cred->user != INIT_USER && 1521 !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) 1522 goto bad_fork_free; 1523 } 1524 current->flags &= ~PF_NPROC_EXCEEDED; 1525 1526 retval = copy_creds(p, clone_flags); 1527 if (retval < 0) 1528 goto bad_fork_free; 1529 1530 /* 1531 * If multiple threads are within copy_process(), then this check 1532 * triggers too late. This doesn't hurt, the check is only there 1533 * to stop root fork bombs. 1534 */ 1535 retval = -EAGAIN; 1536 if (nr_threads >= max_threads) 1537 goto bad_fork_cleanup_count; 1538 1539 delayacct_tsk_init(p); /* Must remain after dup_task_struct() */ 1540 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER); 1541 p->flags |= PF_FORKNOEXEC; 1542 INIT_LIST_HEAD(&p->children); 1543 INIT_LIST_HEAD(&p->sibling); 1544 rcu_copy_process(p); 1545 p->vfork_done = NULL; 1546 spin_lock_init(&p->alloc_lock); 1547 1548 init_sigpending(&p->pending); 1549 1550 p->utime = p->stime = p->gtime = 0; 1551 p->utimescaled = p->stimescaled = 0; 1552 prev_cputime_init(&p->prev_cputime); 1553 1554 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 1555 seqcount_init(&p->vtime_seqcount); 1556 p->vtime_snap = 0; 1557 p->vtime_snap_whence = VTIME_INACTIVE; 1558 #endif 1559 1560 #if defined(SPLIT_RSS_COUNTING) 1561 memset(&p->rss_stat, 0, sizeof(p->rss_stat)); 1562 #endif 1563 1564 p->default_timer_slack_ns = current->timer_slack_ns; 1565 1566 task_io_accounting_init(&p->ioac); 1567 acct_clear_integrals(p); 1568 1569 posix_cpu_timers_init(p); 1570 1571 p->start_time = ktime_get_ns(); 1572 p->real_start_time = ktime_get_boot_ns(); 1573 p->io_context = NULL; 1574 p->audit_context = NULL; 1575 cgroup_fork(p); 1576 #ifdef CONFIG_NUMA 1577 p->mempolicy = mpol_dup(p->mempolicy); 1578 if (IS_ERR(p->mempolicy)) { 1579 retval = PTR_ERR(p->mempolicy); 1580 p->mempolicy = NULL; 1581 goto bad_fork_cleanup_threadgroup_lock; 1582 } 1583 #endif 1584 #ifdef CONFIG_CPUSETS 1585 p->cpuset_mem_spread_rotor = NUMA_NO_NODE; 1586 p->cpuset_slab_spread_rotor = NUMA_NO_NODE; 1587 seqcount_init(&p->mems_allowed_seq); 1588 #endif 1589 #ifdef CONFIG_TRACE_IRQFLAGS 1590 p->irq_events = 0; 1591 p->hardirqs_enabled = 0; 1592 p->hardirq_enable_ip = 0; 1593 p->hardirq_enable_event = 0; 1594 p->hardirq_disable_ip = _THIS_IP_; 1595 p->hardirq_disable_event = 0; 1596 p->softirqs_enabled = 1; 1597 p->softirq_enable_ip = _THIS_IP_; 1598 p->softirq_enable_event = 0; 1599 p->softirq_disable_ip = 0; 1600 p->softirq_disable_event = 0; 1601 p->hardirq_context = 0; 1602 p->softirq_context = 0; 1603 #endif 1604 1605 p->pagefault_disabled = 0; 1606 1607 #ifdef CONFIG_LOCKDEP 1608 p->lockdep_depth = 0; /* no locks held yet */ 1609 p->curr_chain_key = 0; 1610 p->lockdep_recursion = 0; 1611 #endif 1612 1613 #ifdef CONFIG_DEBUG_MUTEXES 1614 p->blocked_on = NULL; /* not blocked yet */ 1615 #endif 1616 #ifdef CONFIG_BCACHE 1617 p->sequential_io = 0; 1618 p->sequential_io_avg = 0; 1619 #endif 1620 1621 /* Perform scheduler related setup. Assign this task to a CPU. */ 1622 retval = sched_fork(clone_flags, p); 1623 if (retval) 1624 goto bad_fork_cleanup_policy; 1625 1626 retval = perf_event_init_task(p); 1627 if (retval) 1628 goto bad_fork_cleanup_policy; 1629 retval = audit_alloc(p); 1630 if (retval) 1631 goto bad_fork_cleanup_perf; 1632 /* copy all the process information */ 1633 shm_init_task(p); 1634 retval = copy_semundo(clone_flags, p); 1635 if (retval) 1636 goto bad_fork_cleanup_audit; 1637 retval = copy_files(clone_flags, p); 1638 if (retval) 1639 goto bad_fork_cleanup_semundo; 1640 retval = copy_fs(clone_flags, p); 1641 if (retval) 1642 goto bad_fork_cleanup_files; 1643 retval = copy_sighand(clone_flags, p); 1644 if (retval) 1645 goto bad_fork_cleanup_fs; 1646 retval = copy_signal(clone_flags, p); 1647 if (retval) 1648 goto bad_fork_cleanup_sighand; 1649 retval = copy_mm(clone_flags, p); 1650 if (retval) 1651 goto bad_fork_cleanup_signal; 1652 retval = copy_namespaces(clone_flags, p); 1653 if (retval) 1654 goto bad_fork_cleanup_mm; 1655 retval = copy_io(clone_flags, p); 1656 if (retval) 1657 goto bad_fork_cleanup_namespaces; 1658 retval = copy_thread_tls(clone_flags, stack_start, stack_size, p, tls); 1659 if (retval) 1660 goto bad_fork_cleanup_io; 1661 1662 if (pid != &init_struct_pid) { 1663 pid = alloc_pid(p->nsproxy->pid_ns_for_children); 1664 if (IS_ERR(pid)) { 1665 retval = PTR_ERR(pid); 1666 goto bad_fork_cleanup_thread; 1667 } 1668 } 1669 1670 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL; 1671 /* 1672 * Clear TID on mm_release()? 1673 */ 1674 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL; 1675 #ifdef CONFIG_BLOCK 1676 p->plug = NULL; 1677 #endif 1678 #ifdef CONFIG_FUTEX 1679 p->robust_list = NULL; 1680 #ifdef CONFIG_COMPAT 1681 p->compat_robust_list = NULL; 1682 #endif 1683 INIT_LIST_HEAD(&p->pi_state_list); 1684 p->pi_state_cache = NULL; 1685 #endif 1686 /* 1687 * sigaltstack should be cleared when sharing the same VM 1688 */ 1689 if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) 1690 sas_ss_reset(p); 1691 1692 /* 1693 * Syscall tracing and stepping should be turned off in the 1694 * child regardless of CLONE_PTRACE. 1695 */ 1696 user_disable_single_step(p); 1697 clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE); 1698 #ifdef TIF_SYSCALL_EMU 1699 clear_tsk_thread_flag(p, TIF_SYSCALL_EMU); 1700 #endif 1701 clear_all_latency_tracing(p); 1702 1703 /* ok, now we should be set up.. */ 1704 p->pid = pid_nr(pid); 1705 if (clone_flags & CLONE_THREAD) { 1706 p->exit_signal = -1; 1707 p->group_leader = current->group_leader; 1708 p->tgid = current->tgid; 1709 } else { 1710 if (clone_flags & CLONE_PARENT) 1711 p->exit_signal = current->group_leader->exit_signal; 1712 else 1713 p->exit_signal = (clone_flags & CSIGNAL); 1714 p->group_leader = p; 1715 p->tgid = p->pid; 1716 } 1717 1718 p->nr_dirtied = 0; 1719 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); 1720 p->dirty_paused_when = 0; 1721 1722 p->pdeath_signal = 0; 1723 INIT_LIST_HEAD(&p->thread_group); 1724 p->task_works = NULL; 1725 1726 threadgroup_change_begin(current); 1727 /* 1728 * Ensure that the cgroup subsystem policies allow the new process to be 1729 * forked. It should be noted the the new process's css_set can be changed 1730 * between here and cgroup_post_fork() if an organisation operation is in 1731 * progress. 1732 */ 1733 retval = cgroup_can_fork(p); 1734 if (retval) 1735 goto bad_fork_free_pid; 1736 1737 /* 1738 * Make it visible to the rest of the system, but dont wake it up yet. 1739 * Need tasklist lock for parent etc handling! 1740 */ 1741 write_lock_irq(&tasklist_lock); 1742 1743 /* CLONE_PARENT re-uses the old parent */ 1744 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) { 1745 p->real_parent = current->real_parent; 1746 p->parent_exec_id = current->parent_exec_id; 1747 } else { 1748 p->real_parent = current; 1749 p->parent_exec_id = current->self_exec_id; 1750 } 1751 1752 spin_lock(¤t->sighand->siglock); 1753 1754 /* 1755 * Copy seccomp details explicitly here, in case they were changed 1756 * before holding sighand lock. 1757 */ 1758 copy_seccomp(p); 1759 1760 /* 1761 * Process group and session signals need to be delivered to just the 1762 * parent before the fork or both the parent and the child after the 1763 * fork. Restart if a signal comes in before we add the new process to 1764 * it's process group. 1765 * A fatal signal pending means that current will exit, so the new 1766 * thread can't slip out of an OOM kill (or normal SIGKILL). 1767 */ 1768 recalc_sigpending(); 1769 if (signal_pending(current)) { 1770 spin_unlock(¤t->sighand->siglock); 1771 write_unlock_irq(&tasklist_lock); 1772 retval = -ERESTARTNOINTR; 1773 goto bad_fork_cancel_cgroup; 1774 } 1775 1776 if (likely(p->pid)) { 1777 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace); 1778 1779 init_task_pid(p, PIDTYPE_PID, pid); 1780 if (thread_group_leader(p)) { 1781 init_task_pid(p, PIDTYPE_PGID, task_pgrp(current)); 1782 init_task_pid(p, PIDTYPE_SID, task_session(current)); 1783 1784 if (is_child_reaper(pid)) { 1785 ns_of_pid(pid)->child_reaper = p; 1786 p->signal->flags |= SIGNAL_UNKILLABLE; 1787 } 1788 1789 p->signal->leader_pid = pid; 1790 p->signal->tty = tty_kref_get(current->signal->tty); 1791 list_add_tail(&p->sibling, &p->real_parent->children); 1792 list_add_tail_rcu(&p->tasks, &init_task.tasks); 1793 attach_pid(p, PIDTYPE_PGID); 1794 attach_pid(p, PIDTYPE_SID); 1795 __this_cpu_inc(process_counts); 1796 } else { 1797 current->signal->nr_threads++; 1798 atomic_inc(¤t->signal->live); 1799 atomic_inc(¤t->signal->sigcnt); 1800 list_add_tail_rcu(&p->thread_group, 1801 &p->group_leader->thread_group); 1802 list_add_tail_rcu(&p->thread_node, 1803 &p->signal->thread_head); 1804 } 1805 attach_pid(p, PIDTYPE_PID); 1806 nr_threads++; 1807 } 1808 1809 total_forks++; 1810 spin_unlock(¤t->sighand->siglock); 1811 syscall_tracepoint_update(p); 1812 write_unlock_irq(&tasklist_lock); 1813 1814 proc_fork_connector(p); 1815 cgroup_post_fork(p); 1816 threadgroup_change_end(current); 1817 perf_event_fork(p); 1818 1819 trace_task_newtask(p, clone_flags); 1820 uprobe_copy_process(p, clone_flags); 1821 1822 return p; 1823 1824 bad_fork_cancel_cgroup: 1825 cgroup_cancel_fork(p); 1826 bad_fork_free_pid: 1827 threadgroup_change_end(current); 1828 if (pid != &init_struct_pid) 1829 free_pid(pid); 1830 bad_fork_cleanup_thread: 1831 exit_thread(p); 1832 bad_fork_cleanup_io: 1833 if (p->io_context) 1834 exit_io_context(p); 1835 bad_fork_cleanup_namespaces: 1836 exit_task_namespaces(p); 1837 bad_fork_cleanup_mm: 1838 if (p->mm) 1839 mmput(p->mm); 1840 bad_fork_cleanup_signal: 1841 if (!(clone_flags & CLONE_THREAD)) 1842 free_signal_struct(p->signal); 1843 bad_fork_cleanup_sighand: 1844 __cleanup_sighand(p->sighand); 1845 bad_fork_cleanup_fs: 1846 exit_fs(p); /* blocking */ 1847 bad_fork_cleanup_files: 1848 exit_files(p); /* blocking */ 1849 bad_fork_cleanup_semundo: 1850 exit_sem(p); 1851 bad_fork_cleanup_audit: 1852 audit_free(p); 1853 bad_fork_cleanup_perf: 1854 perf_event_free_task(p); 1855 bad_fork_cleanup_policy: 1856 #ifdef CONFIG_NUMA 1857 mpol_put(p->mempolicy); 1858 bad_fork_cleanup_threadgroup_lock: 1859 #endif 1860 delayacct_tsk_free(p); 1861 bad_fork_cleanup_count: 1862 atomic_dec(&p->cred->user->processes); 1863 exit_creds(p); 1864 bad_fork_free: 1865 put_task_stack(p); 1866 free_task(p); 1867 fork_out: 1868 return ERR_PTR(retval); 1869 } 1870 1871 static inline void init_idle_pids(struct pid_link *links) 1872 { 1873 enum pid_type type; 1874 1875 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) { 1876 INIT_HLIST_NODE(&links[type].node); /* not really needed */ 1877 links[type].pid = &init_struct_pid; 1878 } 1879 } 1880 1881 struct task_struct *fork_idle(int cpu) 1882 { 1883 struct task_struct *task; 1884 task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0, 0, 1885 cpu_to_node(cpu)); 1886 if (!IS_ERR(task)) { 1887 init_idle_pids(task->pids); 1888 init_idle(task, cpu); 1889 } 1890 1891 return task; 1892 } 1893 1894 /* 1895 * Ok, this is the main fork-routine. 1896 * 1897 * It copies the process, and if successful kick-starts 1898 * it and waits for it to finish using the VM if required. 1899 */ 1900 long _do_fork(unsigned long clone_flags, 1901 unsigned long stack_start, 1902 unsigned long stack_size, 1903 int __user *parent_tidptr, 1904 int __user *child_tidptr, 1905 unsigned long tls) 1906 { 1907 struct task_struct *p; 1908 int trace = 0; 1909 long nr; 1910 1911 /* 1912 * Determine whether and which event to report to ptracer. When 1913 * called from kernel_thread or CLONE_UNTRACED is explicitly 1914 * requested, no event is reported; otherwise, report if the event 1915 * for the type of forking is enabled. 1916 */ 1917 if (!(clone_flags & CLONE_UNTRACED)) { 1918 if (clone_flags & CLONE_VFORK) 1919 trace = PTRACE_EVENT_VFORK; 1920 else if ((clone_flags & CSIGNAL) != SIGCHLD) 1921 trace = PTRACE_EVENT_CLONE; 1922 else 1923 trace = PTRACE_EVENT_FORK; 1924 1925 if (likely(!ptrace_event_enabled(current, trace))) 1926 trace = 0; 1927 } 1928 1929 p = copy_process(clone_flags, stack_start, stack_size, 1930 child_tidptr, NULL, trace, tls, NUMA_NO_NODE); 1931 add_latent_entropy(); 1932 /* 1933 * Do this prior waking up the new thread - the thread pointer 1934 * might get invalid after that point, if the thread exits quickly. 1935 */ 1936 if (!IS_ERR(p)) { 1937 struct completion vfork; 1938 struct pid *pid; 1939 1940 trace_sched_process_fork(current, p); 1941 1942 pid = get_task_pid(p, PIDTYPE_PID); 1943 nr = pid_vnr(pid); 1944 1945 if (clone_flags & CLONE_PARENT_SETTID) 1946 put_user(nr, parent_tidptr); 1947 1948 if (clone_flags & CLONE_VFORK) { 1949 p->vfork_done = &vfork; 1950 init_completion(&vfork); 1951 get_task_struct(p); 1952 } 1953 1954 wake_up_new_task(p); 1955 1956 /* forking complete and child started to run, tell ptracer */ 1957 if (unlikely(trace)) 1958 ptrace_event_pid(trace, pid); 1959 1960 if (clone_flags & CLONE_VFORK) { 1961 if (!wait_for_vfork_done(p, &vfork)) 1962 ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid); 1963 } 1964 1965 put_pid(pid); 1966 } else { 1967 nr = PTR_ERR(p); 1968 } 1969 return nr; 1970 } 1971 1972 #ifndef CONFIG_HAVE_COPY_THREAD_TLS 1973 /* For compatibility with architectures that call do_fork directly rather than 1974 * using the syscall entry points below. */ 1975 long do_fork(unsigned long clone_flags, 1976 unsigned long stack_start, 1977 unsigned long stack_size, 1978 int __user *parent_tidptr, 1979 int __user *child_tidptr) 1980 { 1981 return _do_fork(clone_flags, stack_start, stack_size, 1982 parent_tidptr, child_tidptr, 0); 1983 } 1984 #endif 1985 1986 /* 1987 * Create a kernel thread. 1988 */ 1989 pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) 1990 { 1991 return _do_fork(flags|CLONE_VM|CLONE_UNTRACED, (unsigned long)fn, 1992 (unsigned long)arg, NULL, NULL, 0); 1993 } 1994 1995 #ifdef __ARCH_WANT_SYS_FORK 1996 SYSCALL_DEFINE0(fork) 1997 { 1998 #ifdef CONFIG_MMU 1999 return _do_fork(SIGCHLD, 0, 0, NULL, NULL, 0); 2000 #else 2001 /* can not support in nommu mode */ 2002 return -EINVAL; 2003 #endif 2004 } 2005 #endif 2006 2007 #ifdef __ARCH_WANT_SYS_VFORK 2008 SYSCALL_DEFINE0(vfork) 2009 { 2010 return _do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, 0, 2011 0, NULL, NULL, 0); 2012 } 2013 #endif 2014 2015 #ifdef __ARCH_WANT_SYS_CLONE 2016 #ifdef CONFIG_CLONE_BACKWARDS 2017 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 2018 int __user *, parent_tidptr, 2019 unsigned long, tls, 2020 int __user *, child_tidptr) 2021 #elif defined(CONFIG_CLONE_BACKWARDS2) 2022 SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags, 2023 int __user *, parent_tidptr, 2024 int __user *, child_tidptr, 2025 unsigned long, tls) 2026 #elif defined(CONFIG_CLONE_BACKWARDS3) 2027 SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, 2028 int, stack_size, 2029 int __user *, parent_tidptr, 2030 int __user *, child_tidptr, 2031 unsigned long, tls) 2032 #else 2033 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 2034 int __user *, parent_tidptr, 2035 int __user *, child_tidptr, 2036 unsigned long, tls) 2037 #endif 2038 { 2039 return _do_fork(clone_flags, newsp, 0, parent_tidptr, child_tidptr, tls); 2040 } 2041 #endif 2042 2043 #ifndef ARCH_MIN_MMSTRUCT_ALIGN 2044 #define ARCH_MIN_MMSTRUCT_ALIGN 0 2045 #endif 2046 2047 static void sighand_ctor(void *data) 2048 { 2049 struct sighand_struct *sighand = data; 2050 2051 spin_lock_init(&sighand->siglock); 2052 init_waitqueue_head(&sighand->signalfd_wqh); 2053 } 2054 2055 void __init proc_caches_init(void) 2056 { 2057 sighand_cachep = kmem_cache_create("sighand_cache", 2058 sizeof(struct sighand_struct), 0, 2059 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU| 2060 SLAB_NOTRACK|SLAB_ACCOUNT, sighand_ctor); 2061 signal_cachep = kmem_cache_create("signal_cache", 2062 sizeof(struct signal_struct), 0, 2063 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT, 2064 NULL); 2065 files_cachep = kmem_cache_create("files_cache", 2066 sizeof(struct files_struct), 0, 2067 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT, 2068 NULL); 2069 fs_cachep = kmem_cache_create("fs_cache", 2070 sizeof(struct fs_struct), 0, 2071 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT, 2072 NULL); 2073 /* 2074 * FIXME! The "sizeof(struct mm_struct)" currently includes the 2075 * whole struct cpumask for the OFFSTACK case. We could change 2076 * this to *only* allocate as much of it as required by the 2077 * maximum number of CPU's we can ever have. The cpumask_allocation 2078 * is at the end of the structure, exactly for that reason. 2079 */ 2080 mm_cachep = kmem_cache_create("mm_struct", 2081 sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN, 2082 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT, 2083 NULL); 2084 vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT); 2085 mmap_init(); 2086 nsproxy_cache_init(); 2087 } 2088 2089 /* 2090 * Check constraints on flags passed to the unshare system call. 2091 */ 2092 static int check_unshare_flags(unsigned long unshare_flags) 2093 { 2094 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND| 2095 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM| 2096 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET| 2097 CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP)) 2098 return -EINVAL; 2099 /* 2100 * Not implemented, but pretend it works if there is nothing 2101 * to unshare. Note that unsharing the address space or the 2102 * signal handlers also need to unshare the signal queues (aka 2103 * CLONE_THREAD). 2104 */ 2105 if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) { 2106 if (!thread_group_empty(current)) 2107 return -EINVAL; 2108 } 2109 if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) { 2110 if (atomic_read(¤t->sighand->count) > 1) 2111 return -EINVAL; 2112 } 2113 if (unshare_flags & CLONE_VM) { 2114 if (!current_is_single_threaded()) 2115 return -EINVAL; 2116 } 2117 2118 return 0; 2119 } 2120 2121 /* 2122 * Unshare the filesystem structure if it is being shared 2123 */ 2124 static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) 2125 { 2126 struct fs_struct *fs = current->fs; 2127 2128 if (!(unshare_flags & CLONE_FS) || !fs) 2129 return 0; 2130 2131 /* don't need lock here; in the worst case we'll do useless copy */ 2132 if (fs->users == 1) 2133 return 0; 2134 2135 *new_fsp = copy_fs_struct(fs); 2136 if (!*new_fsp) 2137 return -ENOMEM; 2138 2139 return 0; 2140 } 2141 2142 /* 2143 * Unshare file descriptor table if it is being shared 2144 */ 2145 static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp) 2146 { 2147 struct files_struct *fd = current->files; 2148 int error = 0; 2149 2150 if ((unshare_flags & CLONE_FILES) && 2151 (fd && atomic_read(&fd->count) > 1)) { 2152 *new_fdp = dup_fd(fd, &error); 2153 if (!*new_fdp) 2154 return error; 2155 } 2156 2157 return 0; 2158 } 2159 2160 /* 2161 * unshare allows a process to 'unshare' part of the process 2162 * context which was originally shared using clone. copy_* 2163 * functions used by do_fork() cannot be used here directly 2164 * because they modify an inactive task_struct that is being 2165 * constructed. Here we are modifying the current, active, 2166 * task_struct. 2167 */ 2168 SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags) 2169 { 2170 struct fs_struct *fs, *new_fs = NULL; 2171 struct files_struct *fd, *new_fd = NULL; 2172 struct cred *new_cred = NULL; 2173 struct nsproxy *new_nsproxy = NULL; 2174 int do_sysvsem = 0; 2175 int err; 2176 2177 /* 2178 * If unsharing a user namespace must also unshare the thread group 2179 * and unshare the filesystem root and working directories. 2180 */ 2181 if (unshare_flags & CLONE_NEWUSER) 2182 unshare_flags |= CLONE_THREAD | CLONE_FS; 2183 /* 2184 * If unsharing vm, must also unshare signal handlers. 2185 */ 2186 if (unshare_flags & CLONE_VM) 2187 unshare_flags |= CLONE_SIGHAND; 2188 /* 2189 * If unsharing a signal handlers, must also unshare the signal queues. 2190 */ 2191 if (unshare_flags & CLONE_SIGHAND) 2192 unshare_flags |= CLONE_THREAD; 2193 /* 2194 * If unsharing namespace, must also unshare filesystem information. 2195 */ 2196 if (unshare_flags & CLONE_NEWNS) 2197 unshare_flags |= CLONE_FS; 2198 2199 err = check_unshare_flags(unshare_flags); 2200 if (err) 2201 goto bad_unshare_out; 2202 /* 2203 * CLONE_NEWIPC must also detach from the undolist: after switching 2204 * to a new ipc namespace, the semaphore arrays from the old 2205 * namespace are unreachable. 2206 */ 2207 if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM)) 2208 do_sysvsem = 1; 2209 err = unshare_fs(unshare_flags, &new_fs); 2210 if (err) 2211 goto bad_unshare_out; 2212 err = unshare_fd(unshare_flags, &new_fd); 2213 if (err) 2214 goto bad_unshare_cleanup_fs; 2215 err = unshare_userns(unshare_flags, &new_cred); 2216 if (err) 2217 goto bad_unshare_cleanup_fd; 2218 err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, 2219 new_cred, new_fs); 2220 if (err) 2221 goto bad_unshare_cleanup_cred; 2222 2223 if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) { 2224 if (do_sysvsem) { 2225 /* 2226 * CLONE_SYSVSEM is equivalent to sys_exit(). 2227 */ 2228 exit_sem(current); 2229 } 2230 if (unshare_flags & CLONE_NEWIPC) { 2231 /* Orphan segments in old ns (see sem above). */ 2232 exit_shm(current); 2233 shm_init_task(current); 2234 } 2235 2236 if (new_nsproxy) 2237 switch_task_namespaces(current, new_nsproxy); 2238 2239 task_lock(current); 2240 2241 if (new_fs) { 2242 fs = current->fs; 2243 spin_lock(&fs->lock); 2244 current->fs = new_fs; 2245 if (--fs->users) 2246 new_fs = NULL; 2247 else 2248 new_fs = fs; 2249 spin_unlock(&fs->lock); 2250 } 2251 2252 if (new_fd) { 2253 fd = current->files; 2254 current->files = new_fd; 2255 new_fd = fd; 2256 } 2257 2258 task_unlock(current); 2259 2260 if (new_cred) { 2261 /* Install the new user namespace */ 2262 commit_creds(new_cred); 2263 new_cred = NULL; 2264 } 2265 } 2266 2267 bad_unshare_cleanup_cred: 2268 if (new_cred) 2269 put_cred(new_cred); 2270 bad_unshare_cleanup_fd: 2271 if (new_fd) 2272 put_files_struct(new_fd); 2273 2274 bad_unshare_cleanup_fs: 2275 if (new_fs) 2276 free_fs_struct(new_fs); 2277 2278 bad_unshare_out: 2279 return err; 2280 } 2281 2282 /* 2283 * Helper to unshare the files of the current task. 2284 * We don't want to expose copy_files internals to 2285 * the exec layer of the kernel. 2286 */ 2287 2288 int unshare_files(struct files_struct **displaced) 2289 { 2290 struct task_struct *task = current; 2291 struct files_struct *copy = NULL; 2292 int error; 2293 2294 error = unshare_fd(CLONE_FILES, ©); 2295 if (error || !copy) { 2296 *displaced = NULL; 2297 return error; 2298 } 2299 *displaced = task->files; 2300 task_lock(task); 2301 task->files = copy; 2302 task_unlock(task); 2303 return 0; 2304 } 2305 2306 int sysctl_max_threads(struct ctl_table *table, int write, 2307 void __user *buffer, size_t *lenp, loff_t *ppos) 2308 { 2309 struct ctl_table t; 2310 int ret; 2311 int threads = max_threads; 2312 int min = MIN_THREADS; 2313 int max = MAX_THREADS; 2314 2315 t = *table; 2316 t.data = &threads; 2317 t.extra1 = &min; 2318 t.extra2 = &max; 2319 2320 ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 2321 if (ret || !write) 2322 return ret; 2323 2324 set_max_threads(threads); 2325 2326 return 0; 2327 } 2328