1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/kernel/fork.c 4 * 5 * Copyright (C) 1991, 1992 Linus Torvalds 6 */ 7 8 /* 9 * 'fork.c' contains the help-routines for the 'fork' system call 10 * (see also entry.S and others). 11 * Fork is rather simple, once you get the hang of it, but the memory 12 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()' 13 */ 14 15 #include <linux/anon_inodes.h> 16 #include <linux/slab.h> 17 #include <linux/sched/autogroup.h> 18 #include <linux/sched/mm.h> 19 #include <linux/sched/coredump.h> 20 #include <linux/sched/user.h> 21 #include <linux/sched/numa_balancing.h> 22 #include <linux/sched/stat.h> 23 #include <linux/sched/task.h> 24 #include <linux/sched/task_stack.h> 25 #include <linux/sched/cputime.h> 26 #include <linux/seq_file.h> 27 #include <linux/rtmutex.h> 28 #include <linux/init.h> 29 #include <linux/unistd.h> 30 #include <linux/module.h> 31 #include <linux/vmalloc.h> 32 #include <linux/completion.h> 33 #include <linux/personality.h> 34 #include <linux/mempolicy.h> 35 #include <linux/sem.h> 36 #include <linux/file.h> 37 #include <linux/fdtable.h> 38 #include <linux/iocontext.h> 39 #include <linux/key.h> 40 #include <linux/kmsan.h> 41 #include <linux/binfmts.h> 42 #include <linux/mman.h> 43 #include <linux/mmu_notifier.h> 44 #include <linux/fs.h> 45 #include <linux/mm.h> 46 #include <linux/mm_inline.h> 47 #include <linux/nsproxy.h> 48 #include <linux/capability.h> 49 #include <linux/cpu.h> 50 #include <linux/cgroup.h> 51 #include <linux/security.h> 52 #include <linux/hugetlb.h> 53 #include <linux/seccomp.h> 54 #include <linux/swap.h> 55 #include <linux/syscalls.h> 56 #include <linux/jiffies.h> 57 #include <linux/futex.h> 58 #include <linux/compat.h> 59 #include <linux/kthread.h> 60 #include <linux/task_io_accounting_ops.h> 61 #include <linux/rcupdate.h> 62 #include <linux/ptrace.h> 63 #include <linux/mount.h> 64 #include <linux/audit.h> 65 #include <linux/memcontrol.h> 66 #include <linux/ftrace.h> 67 #include <linux/proc_fs.h> 68 #include <linux/profile.h> 69 #include <linux/rmap.h> 70 #include <linux/ksm.h> 71 #include <linux/acct.h> 72 #include <linux/userfaultfd_k.h> 73 #include <linux/tsacct_kern.h> 74 #include <linux/cn_proc.h> 75 #include <linux/freezer.h> 76 #include <linux/delayacct.h> 77 #include <linux/taskstats_kern.h> 78 #include <linux/tty.h> 79 #include <linux/fs_struct.h> 80 #include <linux/magic.h> 81 #include <linux/perf_event.h> 82 #include <linux/posix-timers.h> 83 #include <linux/user-return-notifier.h> 84 #include <linux/oom.h> 85 #include <linux/khugepaged.h> 86 #include <linux/signalfd.h> 87 #include <linux/uprobes.h> 88 #include <linux/aio.h> 89 #include <linux/compiler.h> 90 #include <linux/sysctl.h> 91 #include <linux/kcov.h> 92 #include <linux/livepatch.h> 93 #include <linux/thread_info.h> 94 #include <linux/stackleak.h> 95 #include <linux/kasan.h> 96 #include <linux/scs.h> 97 #include <linux/io_uring.h> 98 #include <linux/bpf.h> 99 #include <linux/stackprotector.h> 100 101 #include <asm/pgalloc.h> 102 #include <linux/uaccess.h> 103 #include <asm/mmu_context.h> 104 #include <asm/cacheflush.h> 105 #include <asm/tlbflush.h> 106 107 #include <trace/events/sched.h> 108 109 #define CREATE_TRACE_POINTS 110 #include <trace/events/task.h> 111 112 /* 113 * Minimum number of threads to boot the kernel 114 */ 115 #define MIN_THREADS 20 116 117 /* 118 * Maximum number of threads 119 */ 120 #define MAX_THREADS FUTEX_TID_MASK 121 122 /* 123 * Protected counters by write_lock_irq(&tasklist_lock) 124 */ 125 unsigned long total_forks; /* Handle normal Linux uptimes. */ 126 int nr_threads; /* The idle threads do not count.. */ 127 128 static int max_threads; /* tunable limit on nr_threads */ 129 130 #define NAMED_ARRAY_INDEX(x) [x] = __stringify(x) 131 132 static const char * const resident_page_types[] = { 133 NAMED_ARRAY_INDEX(MM_FILEPAGES), 134 NAMED_ARRAY_INDEX(MM_ANONPAGES), 135 NAMED_ARRAY_INDEX(MM_SWAPENTS), 136 NAMED_ARRAY_INDEX(MM_SHMEMPAGES), 137 }; 138 139 DEFINE_PER_CPU(unsigned long, process_counts) = 0; 140 141 __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ 142 143 #ifdef CONFIG_PROVE_RCU 144 int lockdep_tasklist_lock_is_held(void) 145 { 146 return lockdep_is_held(&tasklist_lock); 147 } 148 EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held); 149 #endif /* #ifdef CONFIG_PROVE_RCU */ 150 151 int nr_processes(void) 152 { 153 int cpu; 154 int total = 0; 155 156 for_each_possible_cpu(cpu) 157 total += per_cpu(process_counts, cpu); 158 159 return total; 160 } 161 162 void __weak arch_release_task_struct(struct task_struct *tsk) 163 { 164 } 165 166 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR 167 static struct kmem_cache *task_struct_cachep; 168 169 static inline struct task_struct *alloc_task_struct_node(int node) 170 { 171 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node); 172 } 173 174 static inline void free_task_struct(struct task_struct *tsk) 175 { 176 kmem_cache_free(task_struct_cachep, tsk); 177 } 178 #endif 179 180 #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR 181 182 /* 183 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a 184 * kmemcache based allocator. 185 */ 186 # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) 187 188 # ifdef CONFIG_VMAP_STACK 189 /* 190 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB 191 * flush. Try to minimize the number of calls by caching stacks. 192 */ 193 #define NR_CACHED_STACKS 2 194 static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); 195 196 struct vm_stack { 197 struct rcu_head rcu; 198 struct vm_struct *stack_vm_area; 199 }; 200 201 static bool try_release_thread_stack_to_cache(struct vm_struct *vm) 202 { 203 unsigned int i; 204 205 for (i = 0; i < NR_CACHED_STACKS; i++) { 206 if (this_cpu_cmpxchg(cached_stacks[i], NULL, vm) != NULL) 207 continue; 208 return true; 209 } 210 return false; 211 } 212 213 static void thread_stack_free_rcu(struct rcu_head *rh) 214 { 215 struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu); 216 217 if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area)) 218 return; 219 220 vfree(vm_stack); 221 } 222 223 static void thread_stack_delayed_free(struct task_struct *tsk) 224 { 225 struct vm_stack *vm_stack = tsk->stack; 226 227 vm_stack->stack_vm_area = tsk->stack_vm_area; 228 call_rcu(&vm_stack->rcu, thread_stack_free_rcu); 229 } 230 231 static int free_vm_stack_cache(unsigned int cpu) 232 { 233 struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu); 234 int i; 235 236 for (i = 0; i < NR_CACHED_STACKS; i++) { 237 struct vm_struct *vm_stack = cached_vm_stacks[i]; 238 239 if (!vm_stack) 240 continue; 241 242 vfree(vm_stack->addr); 243 cached_vm_stacks[i] = NULL; 244 } 245 246 return 0; 247 } 248 249 static int memcg_charge_kernel_stack(struct vm_struct *vm) 250 { 251 int i; 252 int ret; 253 254 BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0); 255 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE); 256 257 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { 258 ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL, 0); 259 if (ret) 260 goto err; 261 } 262 return 0; 263 err: 264 /* 265 * If memcg_kmem_charge_page() fails, page's memory cgroup pointer is 266 * NULL, and memcg_kmem_uncharge_page() in free_thread_stack() will 267 * ignore this page. 268 */ 269 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) 270 memcg_kmem_uncharge_page(vm->pages[i], 0); 271 return ret; 272 } 273 274 static int alloc_thread_stack_node(struct task_struct *tsk, int node) 275 { 276 struct vm_struct *vm; 277 void *stack; 278 int i; 279 280 for (i = 0; i < NR_CACHED_STACKS; i++) { 281 struct vm_struct *s; 282 283 s = this_cpu_xchg(cached_stacks[i], NULL); 284 285 if (!s) 286 continue; 287 288 /* Reset stack metadata. */ 289 kasan_unpoison_range(s->addr, THREAD_SIZE); 290 291 stack = kasan_reset_tag(s->addr); 292 293 /* Clear stale pointers from reused stack. */ 294 memset(stack, 0, THREAD_SIZE); 295 296 if (memcg_charge_kernel_stack(s)) { 297 vfree(s->addr); 298 return -ENOMEM; 299 } 300 301 tsk->stack_vm_area = s; 302 tsk->stack = stack; 303 return 0; 304 } 305 306 /* 307 * Allocated stacks are cached and later reused by new threads, 308 * so memcg accounting is performed manually on assigning/releasing 309 * stacks to tasks. Drop __GFP_ACCOUNT. 310 */ 311 stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, 312 VMALLOC_START, VMALLOC_END, 313 THREADINFO_GFP & ~__GFP_ACCOUNT, 314 PAGE_KERNEL, 315 0, node, __builtin_return_address(0)); 316 if (!stack) 317 return -ENOMEM; 318 319 vm = find_vm_area(stack); 320 if (memcg_charge_kernel_stack(vm)) { 321 vfree(stack); 322 return -ENOMEM; 323 } 324 /* 325 * We can't call find_vm_area() in interrupt context, and 326 * free_thread_stack() can be called in interrupt context, 327 * so cache the vm_struct. 328 */ 329 tsk->stack_vm_area = vm; 330 stack = kasan_reset_tag(stack); 331 tsk->stack = stack; 332 return 0; 333 } 334 335 static void free_thread_stack(struct task_struct *tsk) 336 { 337 if (!try_release_thread_stack_to_cache(tsk->stack_vm_area)) 338 thread_stack_delayed_free(tsk); 339 340 tsk->stack = NULL; 341 tsk->stack_vm_area = NULL; 342 } 343 344 # else /* !CONFIG_VMAP_STACK */ 345 346 static void thread_stack_free_rcu(struct rcu_head *rh) 347 { 348 __free_pages(virt_to_page(rh), THREAD_SIZE_ORDER); 349 } 350 351 static void thread_stack_delayed_free(struct task_struct *tsk) 352 { 353 struct rcu_head *rh = tsk->stack; 354 355 call_rcu(rh, thread_stack_free_rcu); 356 } 357 358 static int alloc_thread_stack_node(struct task_struct *tsk, int node) 359 { 360 struct page *page = alloc_pages_node(node, THREADINFO_GFP, 361 THREAD_SIZE_ORDER); 362 363 if (likely(page)) { 364 tsk->stack = kasan_reset_tag(page_address(page)); 365 return 0; 366 } 367 return -ENOMEM; 368 } 369 370 static void free_thread_stack(struct task_struct *tsk) 371 { 372 thread_stack_delayed_free(tsk); 373 tsk->stack = NULL; 374 } 375 376 # endif /* CONFIG_VMAP_STACK */ 377 # else /* !(THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)) */ 378 379 static struct kmem_cache *thread_stack_cache; 380 381 static void thread_stack_free_rcu(struct rcu_head *rh) 382 { 383 kmem_cache_free(thread_stack_cache, rh); 384 } 385 386 static void thread_stack_delayed_free(struct task_struct *tsk) 387 { 388 struct rcu_head *rh = tsk->stack; 389 390 call_rcu(rh, thread_stack_free_rcu); 391 } 392 393 static int alloc_thread_stack_node(struct task_struct *tsk, int node) 394 { 395 unsigned long *stack; 396 stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node); 397 stack = kasan_reset_tag(stack); 398 tsk->stack = stack; 399 return stack ? 0 : -ENOMEM; 400 } 401 402 static void free_thread_stack(struct task_struct *tsk) 403 { 404 thread_stack_delayed_free(tsk); 405 tsk->stack = NULL; 406 } 407 408 void thread_stack_cache_init(void) 409 { 410 thread_stack_cache = kmem_cache_create_usercopy("thread_stack", 411 THREAD_SIZE, THREAD_SIZE, 0, 0, 412 THREAD_SIZE, NULL); 413 BUG_ON(thread_stack_cache == NULL); 414 } 415 416 # endif /* THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) */ 417 #else /* CONFIG_ARCH_THREAD_STACK_ALLOCATOR */ 418 419 static int alloc_thread_stack_node(struct task_struct *tsk, int node) 420 { 421 unsigned long *stack; 422 423 stack = arch_alloc_thread_stack_node(tsk, node); 424 tsk->stack = stack; 425 return stack ? 0 : -ENOMEM; 426 } 427 428 static void free_thread_stack(struct task_struct *tsk) 429 { 430 arch_free_thread_stack(tsk); 431 tsk->stack = NULL; 432 } 433 434 #endif /* !CONFIG_ARCH_THREAD_STACK_ALLOCATOR */ 435 436 /* SLAB cache for signal_struct structures (tsk->signal) */ 437 static struct kmem_cache *signal_cachep; 438 439 /* SLAB cache for sighand_struct structures (tsk->sighand) */ 440 struct kmem_cache *sighand_cachep; 441 442 /* SLAB cache for files_struct structures (tsk->files) */ 443 struct kmem_cache *files_cachep; 444 445 /* SLAB cache for fs_struct structures (tsk->fs) */ 446 struct kmem_cache *fs_cachep; 447 448 /* SLAB cache for vm_area_struct structures */ 449 static struct kmem_cache *vm_area_cachep; 450 451 /* SLAB cache for mm_struct structures (tsk->mm) */ 452 static struct kmem_cache *mm_cachep; 453 454 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) 455 { 456 struct vm_area_struct *vma; 457 458 vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 459 if (vma) 460 vma_init(vma, mm); 461 return vma; 462 } 463 464 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) 465 { 466 struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 467 468 if (new) { 469 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); 470 ASSERT_EXCLUSIVE_WRITER(orig->vm_file); 471 /* 472 * orig->shared.rb may be modified concurrently, but the clone 473 * will be reinitialized. 474 */ 475 data_race(memcpy(new, orig, sizeof(*new))); 476 INIT_LIST_HEAD(&new->anon_vma_chain); 477 dup_anon_vma_name(orig, new); 478 } 479 return new; 480 } 481 482 void vm_area_free(struct vm_area_struct *vma) 483 { 484 free_anon_vma_name(vma); 485 kmem_cache_free(vm_area_cachep, vma); 486 } 487 488 static void account_kernel_stack(struct task_struct *tsk, int account) 489 { 490 if (IS_ENABLED(CONFIG_VMAP_STACK)) { 491 struct vm_struct *vm = task_stack_vm_area(tsk); 492 int i; 493 494 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) 495 mod_lruvec_page_state(vm->pages[i], NR_KERNEL_STACK_KB, 496 account * (PAGE_SIZE / 1024)); 497 } else { 498 void *stack = task_stack_page(tsk); 499 500 /* All stack pages are in the same node. */ 501 mod_lruvec_kmem_state(stack, NR_KERNEL_STACK_KB, 502 account * (THREAD_SIZE / 1024)); 503 } 504 } 505 506 void exit_task_stack_account(struct task_struct *tsk) 507 { 508 account_kernel_stack(tsk, -1); 509 510 if (IS_ENABLED(CONFIG_VMAP_STACK)) { 511 struct vm_struct *vm; 512 int i; 513 514 vm = task_stack_vm_area(tsk); 515 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) 516 memcg_kmem_uncharge_page(vm->pages[i], 0); 517 } 518 } 519 520 static void release_task_stack(struct task_struct *tsk) 521 { 522 if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD)) 523 return; /* Better to leak the stack than to free prematurely */ 524 525 free_thread_stack(tsk); 526 } 527 528 #ifdef CONFIG_THREAD_INFO_IN_TASK 529 void put_task_stack(struct task_struct *tsk) 530 { 531 if (refcount_dec_and_test(&tsk->stack_refcount)) 532 release_task_stack(tsk); 533 } 534 #endif 535 536 void free_task(struct task_struct *tsk) 537 { 538 #ifdef CONFIG_SECCOMP 539 WARN_ON_ONCE(tsk->seccomp.filter); 540 #endif 541 release_user_cpus_ptr(tsk); 542 scs_release(tsk); 543 544 #ifndef CONFIG_THREAD_INFO_IN_TASK 545 /* 546 * The task is finally done with both the stack and thread_info, 547 * so free both. 548 */ 549 release_task_stack(tsk); 550 #else 551 /* 552 * If the task had a separate stack allocation, it should be gone 553 * by now. 554 */ 555 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0); 556 #endif 557 rt_mutex_debug_task_free(tsk); 558 ftrace_graph_exit_task(tsk); 559 arch_release_task_struct(tsk); 560 if (tsk->flags & PF_KTHREAD) 561 free_kthread_struct(tsk); 562 free_task_struct(tsk); 563 } 564 EXPORT_SYMBOL(free_task); 565 566 static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) 567 { 568 struct file *exe_file; 569 570 exe_file = get_mm_exe_file(oldmm); 571 RCU_INIT_POINTER(mm->exe_file, exe_file); 572 /* 573 * We depend on the oldmm having properly denied write access to the 574 * exe_file already. 575 */ 576 if (exe_file && deny_write_access(exe_file)) 577 pr_warn_once("deny_write_access() failed in %s\n", __func__); 578 } 579 580 #ifdef CONFIG_MMU 581 static __latent_entropy int dup_mmap(struct mm_struct *mm, 582 struct mm_struct *oldmm) 583 { 584 struct vm_area_struct *mpnt, *tmp; 585 int retval; 586 unsigned long charge = 0; 587 LIST_HEAD(uf); 588 VMA_ITERATOR(old_vmi, oldmm, 0); 589 VMA_ITERATOR(vmi, mm, 0); 590 591 uprobe_start_dup_mmap(); 592 if (mmap_write_lock_killable(oldmm)) { 593 retval = -EINTR; 594 goto fail_uprobe_end; 595 } 596 flush_cache_dup_mm(oldmm); 597 uprobe_dup_mmap(oldmm, mm); 598 /* 599 * Not linked in yet - no deadlock potential: 600 */ 601 mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING); 602 603 /* No ordering required: file already has been exposed. */ 604 dup_mm_exe_file(mm, oldmm); 605 606 mm->total_vm = oldmm->total_vm; 607 mm->data_vm = oldmm->data_vm; 608 mm->exec_vm = oldmm->exec_vm; 609 mm->stack_vm = oldmm->stack_vm; 610 611 retval = ksm_fork(mm, oldmm); 612 if (retval) 613 goto out; 614 khugepaged_fork(mm, oldmm); 615 616 retval = vma_iter_bulk_alloc(&vmi, oldmm->map_count); 617 if (retval) 618 goto out; 619 620 mt_clear_in_rcu(vmi.mas.tree); 621 for_each_vma(old_vmi, mpnt) { 622 struct file *file; 623 624 if (mpnt->vm_flags & VM_DONTCOPY) { 625 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); 626 continue; 627 } 628 charge = 0; 629 /* 630 * Don't duplicate many vmas if we've been oom-killed (for 631 * example) 632 */ 633 if (fatal_signal_pending(current)) { 634 retval = -EINTR; 635 goto loop_out; 636 } 637 if (mpnt->vm_flags & VM_ACCOUNT) { 638 unsigned long len = vma_pages(mpnt); 639 640 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ 641 goto fail_nomem; 642 charge = len; 643 } 644 tmp = vm_area_dup(mpnt); 645 if (!tmp) 646 goto fail_nomem; 647 retval = vma_dup_policy(mpnt, tmp); 648 if (retval) 649 goto fail_nomem_policy; 650 tmp->vm_mm = mm; 651 retval = dup_userfaultfd(tmp, &uf); 652 if (retval) 653 goto fail_nomem_anon_vma_fork; 654 if (tmp->vm_flags & VM_WIPEONFORK) { 655 /* 656 * VM_WIPEONFORK gets a clean slate in the child. 657 * Don't prepare anon_vma until fault since we don't 658 * copy page for current vma. 659 */ 660 tmp->anon_vma = NULL; 661 } else if (anon_vma_fork(tmp, mpnt)) 662 goto fail_nomem_anon_vma_fork; 663 vm_flags_clear(tmp, VM_LOCKED_MASK); 664 file = tmp->vm_file; 665 if (file) { 666 struct address_space *mapping = file->f_mapping; 667 668 get_file(file); 669 i_mmap_lock_write(mapping); 670 if (tmp->vm_flags & VM_SHARED) 671 mapping_allow_writable(mapping); 672 flush_dcache_mmap_lock(mapping); 673 /* insert tmp into the share list, just after mpnt */ 674 vma_interval_tree_insert_after(tmp, mpnt, 675 &mapping->i_mmap); 676 flush_dcache_mmap_unlock(mapping); 677 i_mmap_unlock_write(mapping); 678 } 679 680 /* 681 * Copy/update hugetlb private vma information. 682 */ 683 if (is_vm_hugetlb_page(tmp)) 684 hugetlb_dup_vma_private(tmp); 685 686 /* Link the vma into the MT */ 687 if (vma_iter_bulk_store(&vmi, tmp)) 688 goto fail_nomem_vmi_store; 689 690 mm->map_count++; 691 if (!(tmp->vm_flags & VM_WIPEONFORK)) 692 retval = copy_page_range(tmp, mpnt); 693 694 if (tmp->vm_ops && tmp->vm_ops->open) 695 tmp->vm_ops->open(tmp); 696 697 if (retval) 698 goto loop_out; 699 } 700 /* a new mm has just been created */ 701 retval = arch_dup_mmap(oldmm, mm); 702 loop_out: 703 vma_iter_free(&vmi); 704 if (!retval) 705 mt_set_in_rcu(vmi.mas.tree); 706 out: 707 mmap_write_unlock(mm); 708 flush_tlb_mm(oldmm); 709 mmap_write_unlock(oldmm); 710 dup_userfaultfd_complete(&uf); 711 fail_uprobe_end: 712 uprobe_end_dup_mmap(); 713 return retval; 714 715 fail_nomem_vmi_store: 716 unlink_anon_vmas(tmp); 717 fail_nomem_anon_vma_fork: 718 mpol_put(vma_policy(tmp)); 719 fail_nomem_policy: 720 vm_area_free(tmp); 721 fail_nomem: 722 retval = -ENOMEM; 723 vm_unacct_memory(charge); 724 goto loop_out; 725 } 726 727 static inline int mm_alloc_pgd(struct mm_struct *mm) 728 { 729 mm->pgd = pgd_alloc(mm); 730 if (unlikely(!mm->pgd)) 731 return -ENOMEM; 732 return 0; 733 } 734 735 static inline void mm_free_pgd(struct mm_struct *mm) 736 { 737 pgd_free(mm, mm->pgd); 738 } 739 #else 740 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) 741 { 742 mmap_write_lock(oldmm); 743 dup_mm_exe_file(mm, oldmm); 744 mmap_write_unlock(oldmm); 745 return 0; 746 } 747 #define mm_alloc_pgd(mm) (0) 748 #define mm_free_pgd(mm) 749 #endif /* CONFIG_MMU */ 750 751 static void check_mm(struct mm_struct *mm) 752 { 753 int i; 754 755 BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS, 756 "Please make sure 'struct resident_page_types[]' is updated as well"); 757 758 for (i = 0; i < NR_MM_COUNTERS; i++) { 759 long x = percpu_counter_sum(&mm->rss_stat[i]); 760 761 if (unlikely(x)) 762 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", 763 mm, resident_page_types[i], x); 764 } 765 766 if (mm_pgtables_bytes(mm)) 767 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", 768 mm_pgtables_bytes(mm)); 769 770 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS 771 VM_BUG_ON_MM(mm->pmd_huge_pte, mm); 772 #endif 773 } 774 775 #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL)) 776 #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) 777 778 /* 779 * Called when the last reference to the mm 780 * is dropped: either by a lazy thread or by 781 * mmput. Free the page directory and the mm. 782 */ 783 void __mmdrop(struct mm_struct *mm) 784 { 785 int i; 786 787 BUG_ON(mm == &init_mm); 788 WARN_ON_ONCE(mm == current->mm); 789 WARN_ON_ONCE(mm == current->active_mm); 790 mm_free_pgd(mm); 791 destroy_context(mm); 792 mmu_notifier_subscriptions_destroy(mm); 793 check_mm(mm); 794 put_user_ns(mm->user_ns); 795 mm_pasid_drop(mm); 796 797 for (i = 0; i < NR_MM_COUNTERS; i++) 798 percpu_counter_destroy(&mm->rss_stat[i]); 799 free_mm(mm); 800 } 801 EXPORT_SYMBOL_GPL(__mmdrop); 802 803 static void mmdrop_async_fn(struct work_struct *work) 804 { 805 struct mm_struct *mm; 806 807 mm = container_of(work, struct mm_struct, async_put_work); 808 __mmdrop(mm); 809 } 810 811 static void mmdrop_async(struct mm_struct *mm) 812 { 813 if (unlikely(atomic_dec_and_test(&mm->mm_count))) { 814 INIT_WORK(&mm->async_put_work, mmdrop_async_fn); 815 schedule_work(&mm->async_put_work); 816 } 817 } 818 819 static inline void free_signal_struct(struct signal_struct *sig) 820 { 821 taskstats_tgid_free(sig); 822 sched_autogroup_exit(sig); 823 /* 824 * __mmdrop is not safe to call from softirq context on x86 due to 825 * pgd_dtor so postpone it to the async context 826 */ 827 if (sig->oom_mm) 828 mmdrop_async(sig->oom_mm); 829 kmem_cache_free(signal_cachep, sig); 830 } 831 832 static inline void put_signal_struct(struct signal_struct *sig) 833 { 834 if (refcount_dec_and_test(&sig->sigcnt)) 835 free_signal_struct(sig); 836 } 837 838 void __put_task_struct(struct task_struct *tsk) 839 { 840 WARN_ON(!tsk->exit_state); 841 WARN_ON(refcount_read(&tsk->usage)); 842 WARN_ON(tsk == current); 843 844 io_uring_free(tsk); 845 cgroup_free(tsk); 846 task_numa_free(tsk, true); 847 security_task_free(tsk); 848 bpf_task_storage_free(tsk); 849 exit_creds(tsk); 850 delayacct_tsk_free(tsk); 851 put_signal_struct(tsk->signal); 852 sched_core_free(tsk); 853 free_task(tsk); 854 } 855 EXPORT_SYMBOL_GPL(__put_task_struct); 856 857 void __init __weak arch_task_cache_init(void) { } 858 859 /* 860 * set_max_threads 861 */ 862 static void set_max_threads(unsigned int max_threads_suggested) 863 { 864 u64 threads; 865 unsigned long nr_pages = totalram_pages(); 866 867 /* 868 * The number of threads shall be limited such that the thread 869 * structures may only consume a small part of the available memory. 870 */ 871 if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64) 872 threads = MAX_THREADS; 873 else 874 threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE, 875 (u64) THREAD_SIZE * 8UL); 876 877 if (threads > max_threads_suggested) 878 threads = max_threads_suggested; 879 880 max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS); 881 } 882 883 #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT 884 /* Initialized by the architecture: */ 885 int arch_task_struct_size __read_mostly; 886 #endif 887 888 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR 889 static void task_struct_whitelist(unsigned long *offset, unsigned long *size) 890 { 891 /* Fetch thread_struct whitelist for the architecture. */ 892 arch_thread_struct_whitelist(offset, size); 893 894 /* 895 * Handle zero-sized whitelist or empty thread_struct, otherwise 896 * adjust offset to position of thread_struct in task_struct. 897 */ 898 if (unlikely(*size == 0)) 899 *offset = 0; 900 else 901 *offset += offsetof(struct task_struct, thread); 902 } 903 #endif /* CONFIG_ARCH_TASK_STRUCT_ALLOCATOR */ 904 905 void __init fork_init(void) 906 { 907 int i; 908 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR 909 #ifndef ARCH_MIN_TASKALIGN 910 #define ARCH_MIN_TASKALIGN 0 911 #endif 912 int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN); 913 unsigned long useroffset, usersize; 914 915 /* create a slab on which task_structs can be allocated */ 916 task_struct_whitelist(&useroffset, &usersize); 917 task_struct_cachep = kmem_cache_create_usercopy("task_struct", 918 arch_task_struct_size, align, 919 SLAB_PANIC|SLAB_ACCOUNT, 920 useroffset, usersize, NULL); 921 #endif 922 923 /* do the arch specific task caches init */ 924 arch_task_cache_init(); 925 926 set_max_threads(MAX_THREADS); 927 928 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; 929 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; 930 init_task.signal->rlim[RLIMIT_SIGPENDING] = 931 init_task.signal->rlim[RLIMIT_NPROC]; 932 933 for (i = 0; i < UCOUNT_COUNTS; i++) 934 init_user_ns.ucount_max[i] = max_threads/2; 935 936 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_NPROC, RLIM_INFINITY); 937 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MSGQUEUE, RLIM_INFINITY); 938 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY); 939 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY); 940 941 #ifdef CONFIG_VMAP_STACK 942 cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", 943 NULL, free_vm_stack_cache); 944 #endif 945 946 scs_init(); 947 948 lockdep_init_task(&init_task); 949 uprobes_init(); 950 } 951 952 int __weak arch_dup_task_struct(struct task_struct *dst, 953 struct task_struct *src) 954 { 955 *dst = *src; 956 return 0; 957 } 958 959 void set_task_stack_end_magic(struct task_struct *tsk) 960 { 961 unsigned long *stackend; 962 963 stackend = end_of_stack(tsk); 964 *stackend = STACK_END_MAGIC; /* for overflow detection */ 965 } 966 967 static struct task_struct *dup_task_struct(struct task_struct *orig, int node) 968 { 969 struct task_struct *tsk; 970 int err; 971 972 if (node == NUMA_NO_NODE) 973 node = tsk_fork_get_node(orig); 974 tsk = alloc_task_struct_node(node); 975 if (!tsk) 976 return NULL; 977 978 err = arch_dup_task_struct(tsk, orig); 979 if (err) 980 goto free_tsk; 981 982 err = alloc_thread_stack_node(tsk, node); 983 if (err) 984 goto free_tsk; 985 986 #ifdef CONFIG_THREAD_INFO_IN_TASK 987 refcount_set(&tsk->stack_refcount, 1); 988 #endif 989 account_kernel_stack(tsk, 1); 990 991 err = scs_prepare(tsk, node); 992 if (err) 993 goto free_stack; 994 995 #ifdef CONFIG_SECCOMP 996 /* 997 * We must handle setting up seccomp filters once we're under 998 * the sighand lock in case orig has changed between now and 999 * then. Until then, filter must be NULL to avoid messing up 1000 * the usage counts on the error path calling free_task. 1001 */ 1002 tsk->seccomp.filter = NULL; 1003 #endif 1004 1005 setup_thread_stack(tsk, orig); 1006 clear_user_return_notifier(tsk); 1007 clear_tsk_need_resched(tsk); 1008 set_task_stack_end_magic(tsk); 1009 clear_syscall_work_syscall_user_dispatch(tsk); 1010 1011 #ifdef CONFIG_STACKPROTECTOR 1012 tsk->stack_canary = get_random_canary(); 1013 #endif 1014 if (orig->cpus_ptr == &orig->cpus_mask) 1015 tsk->cpus_ptr = &tsk->cpus_mask; 1016 dup_user_cpus_ptr(tsk, orig, node); 1017 1018 /* 1019 * One for the user space visible state that goes away when reaped. 1020 * One for the scheduler. 1021 */ 1022 refcount_set(&tsk->rcu_users, 2); 1023 /* One for the rcu users */ 1024 refcount_set(&tsk->usage, 1); 1025 #ifdef CONFIG_BLK_DEV_IO_TRACE 1026 tsk->btrace_seq = 0; 1027 #endif 1028 tsk->splice_pipe = NULL; 1029 tsk->task_frag.page = NULL; 1030 tsk->wake_q.next = NULL; 1031 tsk->worker_private = NULL; 1032 1033 kcov_task_init(tsk); 1034 kmsan_task_create(tsk); 1035 kmap_local_fork(tsk); 1036 1037 #ifdef CONFIG_FAULT_INJECTION 1038 tsk->fail_nth = 0; 1039 #endif 1040 1041 #ifdef CONFIG_BLK_CGROUP 1042 tsk->throttle_disk = NULL; 1043 tsk->use_memdelay = 0; 1044 #endif 1045 1046 #ifdef CONFIG_IOMMU_SVA 1047 tsk->pasid_activated = 0; 1048 #endif 1049 1050 #ifdef CONFIG_MEMCG 1051 tsk->active_memcg = NULL; 1052 #endif 1053 1054 #ifdef CONFIG_CPU_SUP_INTEL 1055 tsk->reported_split_lock = 0; 1056 #endif 1057 1058 #ifdef CONFIG_SCHED_MM_CID 1059 tsk->mm_cid = -1; 1060 tsk->mm_cid_active = 0; 1061 #endif 1062 return tsk; 1063 1064 free_stack: 1065 exit_task_stack_account(tsk); 1066 free_thread_stack(tsk); 1067 free_tsk: 1068 free_task_struct(tsk); 1069 return NULL; 1070 } 1071 1072 __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); 1073 1074 static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT; 1075 1076 static int __init coredump_filter_setup(char *s) 1077 { 1078 default_dump_filter = 1079 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) & 1080 MMF_DUMP_FILTER_MASK; 1081 return 1; 1082 } 1083 1084 __setup("coredump_filter=", coredump_filter_setup); 1085 1086 #include <linux/init_task.h> 1087 1088 static void mm_init_aio(struct mm_struct *mm) 1089 { 1090 #ifdef CONFIG_AIO 1091 spin_lock_init(&mm->ioctx_lock); 1092 mm->ioctx_table = NULL; 1093 #endif 1094 } 1095 1096 static __always_inline void mm_clear_owner(struct mm_struct *mm, 1097 struct task_struct *p) 1098 { 1099 #ifdef CONFIG_MEMCG 1100 if (mm->owner == p) 1101 WRITE_ONCE(mm->owner, NULL); 1102 #endif 1103 } 1104 1105 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) 1106 { 1107 #ifdef CONFIG_MEMCG 1108 mm->owner = p; 1109 #endif 1110 } 1111 1112 static void mm_init_uprobes_state(struct mm_struct *mm) 1113 { 1114 #ifdef CONFIG_UPROBES 1115 mm->uprobes_state.xol_area = NULL; 1116 #endif 1117 } 1118 1119 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, 1120 struct user_namespace *user_ns) 1121 { 1122 int i; 1123 1124 mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); 1125 mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); 1126 atomic_set(&mm->mm_users, 1); 1127 atomic_set(&mm->mm_count, 1); 1128 seqcount_init(&mm->write_protect_seq); 1129 mmap_init_lock(mm); 1130 INIT_LIST_HEAD(&mm->mmlist); 1131 mm_pgtables_bytes_init(mm); 1132 mm->map_count = 0; 1133 mm->locked_vm = 0; 1134 atomic64_set(&mm->pinned_vm, 0); 1135 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); 1136 spin_lock_init(&mm->page_table_lock); 1137 spin_lock_init(&mm->arg_lock); 1138 mm_init_cpumask(mm); 1139 mm_init_aio(mm); 1140 mm_init_owner(mm, p); 1141 mm_pasid_init(mm); 1142 RCU_INIT_POINTER(mm->exe_file, NULL); 1143 mmu_notifier_subscriptions_init(mm); 1144 init_tlb_flush_pending(mm); 1145 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS 1146 mm->pmd_huge_pte = NULL; 1147 #endif 1148 mm_init_uprobes_state(mm); 1149 hugetlb_count_init(mm); 1150 1151 if (current->mm) { 1152 mm->flags = current->mm->flags & MMF_INIT_MASK; 1153 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; 1154 } else { 1155 mm->flags = default_dump_filter; 1156 mm->def_flags = 0; 1157 } 1158 1159 if (mm_alloc_pgd(mm)) 1160 goto fail_nopgd; 1161 1162 if (init_new_context(p, mm)) 1163 goto fail_nocontext; 1164 1165 for (i = 0; i < NR_MM_COUNTERS; i++) 1166 if (percpu_counter_init(&mm->rss_stat[i], 0, GFP_KERNEL_ACCOUNT)) 1167 goto fail_pcpu; 1168 1169 mm->user_ns = get_user_ns(user_ns); 1170 lru_gen_init_mm(mm); 1171 mm_init_cid(mm); 1172 return mm; 1173 1174 fail_pcpu: 1175 while (i > 0) 1176 percpu_counter_destroy(&mm->rss_stat[--i]); 1177 destroy_context(mm); 1178 fail_nocontext: 1179 mm_free_pgd(mm); 1180 fail_nopgd: 1181 free_mm(mm); 1182 return NULL; 1183 } 1184 1185 /* 1186 * Allocate and initialize an mm_struct. 1187 */ 1188 struct mm_struct *mm_alloc(void) 1189 { 1190 struct mm_struct *mm; 1191 1192 mm = allocate_mm(); 1193 if (!mm) 1194 return NULL; 1195 1196 memset(mm, 0, sizeof(*mm)); 1197 return mm_init(mm, current, current_user_ns()); 1198 } 1199 1200 static inline void __mmput(struct mm_struct *mm) 1201 { 1202 VM_BUG_ON(atomic_read(&mm->mm_users)); 1203 1204 uprobe_clear_state(mm); 1205 exit_aio(mm); 1206 ksm_exit(mm); 1207 khugepaged_exit(mm); /* must run before exit_mmap */ 1208 exit_mmap(mm); 1209 mm_put_huge_zero_page(mm); 1210 set_mm_exe_file(mm, NULL); 1211 if (!list_empty(&mm->mmlist)) { 1212 spin_lock(&mmlist_lock); 1213 list_del(&mm->mmlist); 1214 spin_unlock(&mmlist_lock); 1215 } 1216 if (mm->binfmt) 1217 module_put(mm->binfmt->module); 1218 lru_gen_del_mm(mm); 1219 mmdrop(mm); 1220 } 1221 1222 /* 1223 * Decrement the use count and release all resources for an mm. 1224 */ 1225 void mmput(struct mm_struct *mm) 1226 { 1227 might_sleep(); 1228 1229 if (atomic_dec_and_test(&mm->mm_users)) 1230 __mmput(mm); 1231 } 1232 EXPORT_SYMBOL_GPL(mmput); 1233 1234 #ifdef CONFIG_MMU 1235 static void mmput_async_fn(struct work_struct *work) 1236 { 1237 struct mm_struct *mm = container_of(work, struct mm_struct, 1238 async_put_work); 1239 1240 __mmput(mm); 1241 } 1242 1243 void mmput_async(struct mm_struct *mm) 1244 { 1245 if (atomic_dec_and_test(&mm->mm_users)) { 1246 INIT_WORK(&mm->async_put_work, mmput_async_fn); 1247 schedule_work(&mm->async_put_work); 1248 } 1249 } 1250 EXPORT_SYMBOL_GPL(mmput_async); 1251 #endif 1252 1253 /** 1254 * set_mm_exe_file - change a reference to the mm's executable file 1255 * 1256 * This changes mm's executable file (shown as symlink /proc/[pid]/exe). 1257 * 1258 * Main users are mmput() and sys_execve(). Callers prevent concurrent 1259 * invocations: in mmput() nobody alive left, in execve task is single 1260 * threaded. 1261 * 1262 * Can only fail if new_exe_file != NULL. 1263 */ 1264 int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) 1265 { 1266 struct file *old_exe_file; 1267 1268 /* 1269 * It is safe to dereference the exe_file without RCU as 1270 * this function is only called if nobody else can access 1271 * this mm -- see comment above for justification. 1272 */ 1273 old_exe_file = rcu_dereference_raw(mm->exe_file); 1274 1275 if (new_exe_file) { 1276 /* 1277 * We expect the caller (i.e., sys_execve) to already denied 1278 * write access, so this is unlikely to fail. 1279 */ 1280 if (unlikely(deny_write_access(new_exe_file))) 1281 return -EACCES; 1282 get_file(new_exe_file); 1283 } 1284 rcu_assign_pointer(mm->exe_file, new_exe_file); 1285 if (old_exe_file) { 1286 allow_write_access(old_exe_file); 1287 fput(old_exe_file); 1288 } 1289 return 0; 1290 } 1291 1292 /** 1293 * replace_mm_exe_file - replace a reference to the mm's executable file 1294 * 1295 * This changes mm's executable file (shown as symlink /proc/[pid]/exe), 1296 * dealing with concurrent invocation and without grabbing the mmap lock in 1297 * write mode. 1298 * 1299 * Main user is sys_prctl(PR_SET_MM_MAP/EXE_FILE). 1300 */ 1301 int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) 1302 { 1303 struct vm_area_struct *vma; 1304 struct file *old_exe_file; 1305 int ret = 0; 1306 1307 /* Forbid mm->exe_file change if old file still mapped. */ 1308 old_exe_file = get_mm_exe_file(mm); 1309 if (old_exe_file) { 1310 VMA_ITERATOR(vmi, mm, 0); 1311 mmap_read_lock(mm); 1312 for_each_vma(vmi, vma) { 1313 if (!vma->vm_file) 1314 continue; 1315 if (path_equal(&vma->vm_file->f_path, 1316 &old_exe_file->f_path)) { 1317 ret = -EBUSY; 1318 break; 1319 } 1320 } 1321 mmap_read_unlock(mm); 1322 fput(old_exe_file); 1323 if (ret) 1324 return ret; 1325 } 1326 1327 /* set the new file, lockless */ 1328 ret = deny_write_access(new_exe_file); 1329 if (ret) 1330 return -EACCES; 1331 get_file(new_exe_file); 1332 1333 old_exe_file = xchg(&mm->exe_file, new_exe_file); 1334 if (old_exe_file) { 1335 /* 1336 * Don't race with dup_mmap() getting the file and disallowing 1337 * write access while someone might open the file writable. 1338 */ 1339 mmap_read_lock(mm); 1340 allow_write_access(old_exe_file); 1341 fput(old_exe_file); 1342 mmap_read_unlock(mm); 1343 } 1344 return 0; 1345 } 1346 1347 /** 1348 * get_mm_exe_file - acquire a reference to the mm's executable file 1349 * 1350 * Returns %NULL if mm has no associated executable file. 1351 * User must release file via fput(). 1352 */ 1353 struct file *get_mm_exe_file(struct mm_struct *mm) 1354 { 1355 struct file *exe_file; 1356 1357 rcu_read_lock(); 1358 exe_file = rcu_dereference(mm->exe_file); 1359 if (exe_file && !get_file_rcu(exe_file)) 1360 exe_file = NULL; 1361 rcu_read_unlock(); 1362 return exe_file; 1363 } 1364 1365 /** 1366 * get_task_exe_file - acquire a reference to the task's executable file 1367 * 1368 * Returns %NULL if task's mm (if any) has no associated executable file or 1369 * this is a kernel thread with borrowed mm (see the comment above get_task_mm). 1370 * User must release file via fput(). 1371 */ 1372 struct file *get_task_exe_file(struct task_struct *task) 1373 { 1374 struct file *exe_file = NULL; 1375 struct mm_struct *mm; 1376 1377 task_lock(task); 1378 mm = task->mm; 1379 if (mm) { 1380 if (!(task->flags & PF_KTHREAD)) 1381 exe_file = get_mm_exe_file(mm); 1382 } 1383 task_unlock(task); 1384 return exe_file; 1385 } 1386 1387 /** 1388 * get_task_mm - acquire a reference to the task's mm 1389 * 1390 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning 1391 * this kernel workthread has transiently adopted a user mm with use_mm, 1392 * to do its AIO) is not set and if so returns a reference to it, after 1393 * bumping up the use count. User must release the mm via mmput() 1394 * after use. Typically used by /proc and ptrace. 1395 */ 1396 struct mm_struct *get_task_mm(struct task_struct *task) 1397 { 1398 struct mm_struct *mm; 1399 1400 task_lock(task); 1401 mm = task->mm; 1402 if (mm) { 1403 if (task->flags & PF_KTHREAD) 1404 mm = NULL; 1405 else 1406 mmget(mm); 1407 } 1408 task_unlock(task); 1409 return mm; 1410 } 1411 EXPORT_SYMBOL_GPL(get_task_mm); 1412 1413 struct mm_struct *mm_access(struct task_struct *task, unsigned int mode) 1414 { 1415 struct mm_struct *mm; 1416 int err; 1417 1418 err = down_read_killable(&task->signal->exec_update_lock); 1419 if (err) 1420 return ERR_PTR(err); 1421 1422 mm = get_task_mm(task); 1423 if (mm && mm != current->mm && 1424 !ptrace_may_access(task, mode)) { 1425 mmput(mm); 1426 mm = ERR_PTR(-EACCES); 1427 } 1428 up_read(&task->signal->exec_update_lock); 1429 1430 return mm; 1431 } 1432 1433 static void complete_vfork_done(struct task_struct *tsk) 1434 { 1435 struct completion *vfork; 1436 1437 task_lock(tsk); 1438 vfork = tsk->vfork_done; 1439 if (likely(vfork)) { 1440 tsk->vfork_done = NULL; 1441 complete(vfork); 1442 } 1443 task_unlock(tsk); 1444 } 1445 1446 static int wait_for_vfork_done(struct task_struct *child, 1447 struct completion *vfork) 1448 { 1449 unsigned int state = TASK_UNINTERRUPTIBLE|TASK_KILLABLE|TASK_FREEZABLE; 1450 int killed; 1451 1452 cgroup_enter_frozen(); 1453 killed = wait_for_completion_state(vfork, state); 1454 cgroup_leave_frozen(false); 1455 1456 if (killed) { 1457 task_lock(child); 1458 child->vfork_done = NULL; 1459 task_unlock(child); 1460 } 1461 1462 put_task_struct(child); 1463 return killed; 1464 } 1465 1466 /* Please note the differences between mmput and mm_release. 1467 * mmput is called whenever we stop holding onto a mm_struct, 1468 * error success whatever. 1469 * 1470 * mm_release is called after a mm_struct has been removed 1471 * from the current process. 1472 * 1473 * This difference is important for error handling, when we 1474 * only half set up a mm_struct for a new process and need to restore 1475 * the old one. Because we mmput the new mm_struct before 1476 * restoring the old one. . . 1477 * Eric Biederman 10 January 1998 1478 */ 1479 static void mm_release(struct task_struct *tsk, struct mm_struct *mm) 1480 { 1481 uprobe_free_utask(tsk); 1482 1483 /* Get rid of any cached register state */ 1484 deactivate_mm(tsk, mm); 1485 1486 /* 1487 * Signal userspace if we're not exiting with a core dump 1488 * because we want to leave the value intact for debugging 1489 * purposes. 1490 */ 1491 if (tsk->clear_child_tid) { 1492 if (atomic_read(&mm->mm_users) > 1) { 1493 /* 1494 * We don't check the error code - if userspace has 1495 * not set up a proper pointer then tough luck. 1496 */ 1497 put_user(0, tsk->clear_child_tid); 1498 do_futex(tsk->clear_child_tid, FUTEX_WAKE, 1499 1, NULL, NULL, 0, 0); 1500 } 1501 tsk->clear_child_tid = NULL; 1502 } 1503 1504 /* 1505 * All done, finally we can wake up parent and return this mm to him. 1506 * Also kthread_stop() uses this completion for synchronization. 1507 */ 1508 if (tsk->vfork_done) 1509 complete_vfork_done(tsk); 1510 } 1511 1512 void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm) 1513 { 1514 futex_exit_release(tsk); 1515 mm_release(tsk, mm); 1516 } 1517 1518 void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm) 1519 { 1520 futex_exec_release(tsk); 1521 mm_release(tsk, mm); 1522 } 1523 1524 /** 1525 * dup_mm() - duplicates an existing mm structure 1526 * @tsk: the task_struct with which the new mm will be associated. 1527 * @oldmm: the mm to duplicate. 1528 * 1529 * Allocates a new mm structure and duplicates the provided @oldmm structure 1530 * content into it. 1531 * 1532 * Return: the duplicated mm or NULL on failure. 1533 */ 1534 static struct mm_struct *dup_mm(struct task_struct *tsk, 1535 struct mm_struct *oldmm) 1536 { 1537 struct mm_struct *mm; 1538 int err; 1539 1540 mm = allocate_mm(); 1541 if (!mm) 1542 goto fail_nomem; 1543 1544 memcpy(mm, oldmm, sizeof(*mm)); 1545 1546 if (!mm_init(mm, tsk, mm->user_ns)) 1547 goto fail_nomem; 1548 1549 err = dup_mmap(mm, oldmm); 1550 if (err) 1551 goto free_pt; 1552 1553 mm->hiwater_rss = get_mm_rss(mm); 1554 mm->hiwater_vm = mm->total_vm; 1555 1556 if (mm->binfmt && !try_module_get(mm->binfmt->module)) 1557 goto free_pt; 1558 1559 return mm; 1560 1561 free_pt: 1562 /* don't put binfmt in mmput, we haven't got module yet */ 1563 mm->binfmt = NULL; 1564 mm_init_owner(mm, NULL); 1565 mmput(mm); 1566 1567 fail_nomem: 1568 return NULL; 1569 } 1570 1571 static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) 1572 { 1573 struct mm_struct *mm, *oldmm; 1574 1575 tsk->min_flt = tsk->maj_flt = 0; 1576 tsk->nvcsw = tsk->nivcsw = 0; 1577 #ifdef CONFIG_DETECT_HUNG_TASK 1578 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; 1579 tsk->last_switch_time = 0; 1580 #endif 1581 1582 tsk->mm = NULL; 1583 tsk->active_mm = NULL; 1584 1585 /* 1586 * Are we cloning a kernel thread? 1587 * 1588 * We need to steal a active VM for that.. 1589 */ 1590 oldmm = current->mm; 1591 if (!oldmm) 1592 return 0; 1593 1594 if (clone_flags & CLONE_VM) { 1595 mmget(oldmm); 1596 mm = oldmm; 1597 } else { 1598 mm = dup_mm(tsk, current->mm); 1599 if (!mm) 1600 return -ENOMEM; 1601 } 1602 1603 tsk->mm = mm; 1604 tsk->active_mm = mm; 1605 sched_mm_cid_fork(tsk); 1606 return 0; 1607 } 1608 1609 static int copy_fs(unsigned long clone_flags, struct task_struct *tsk) 1610 { 1611 struct fs_struct *fs = current->fs; 1612 if (clone_flags & CLONE_FS) { 1613 /* tsk->fs is already what we want */ 1614 spin_lock(&fs->lock); 1615 if (fs->in_exec) { 1616 spin_unlock(&fs->lock); 1617 return -EAGAIN; 1618 } 1619 fs->users++; 1620 spin_unlock(&fs->lock); 1621 return 0; 1622 } 1623 tsk->fs = copy_fs_struct(fs); 1624 if (!tsk->fs) 1625 return -ENOMEM; 1626 return 0; 1627 } 1628 1629 static int copy_files(unsigned long clone_flags, struct task_struct *tsk) 1630 { 1631 struct files_struct *oldf, *newf; 1632 int error = 0; 1633 1634 /* 1635 * A background process may not have any files ... 1636 */ 1637 oldf = current->files; 1638 if (!oldf) 1639 goto out; 1640 1641 if (clone_flags & CLONE_FILES) { 1642 atomic_inc(&oldf->count); 1643 goto out; 1644 } 1645 1646 newf = dup_fd(oldf, NR_OPEN_MAX, &error); 1647 if (!newf) 1648 goto out; 1649 1650 tsk->files = newf; 1651 error = 0; 1652 out: 1653 return error; 1654 } 1655 1656 static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk) 1657 { 1658 struct sighand_struct *sig; 1659 1660 if (clone_flags & CLONE_SIGHAND) { 1661 refcount_inc(¤t->sighand->count); 1662 return 0; 1663 } 1664 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL); 1665 RCU_INIT_POINTER(tsk->sighand, sig); 1666 if (!sig) 1667 return -ENOMEM; 1668 1669 refcount_set(&sig->count, 1); 1670 spin_lock_irq(¤t->sighand->siglock); 1671 memcpy(sig->action, current->sighand->action, sizeof(sig->action)); 1672 spin_unlock_irq(¤t->sighand->siglock); 1673 1674 /* Reset all signal handler not set to SIG_IGN to SIG_DFL. */ 1675 if (clone_flags & CLONE_CLEAR_SIGHAND) 1676 flush_signal_handlers(tsk, 0); 1677 1678 return 0; 1679 } 1680 1681 void __cleanup_sighand(struct sighand_struct *sighand) 1682 { 1683 if (refcount_dec_and_test(&sighand->count)) { 1684 signalfd_cleanup(sighand); 1685 /* 1686 * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it 1687 * without an RCU grace period, see __lock_task_sighand(). 1688 */ 1689 kmem_cache_free(sighand_cachep, sighand); 1690 } 1691 } 1692 1693 /* 1694 * Initialize POSIX timer handling for a thread group. 1695 */ 1696 static void posix_cpu_timers_init_group(struct signal_struct *sig) 1697 { 1698 struct posix_cputimers *pct = &sig->posix_cputimers; 1699 unsigned long cpu_limit; 1700 1701 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); 1702 posix_cputimers_group_init(pct, cpu_limit); 1703 } 1704 1705 static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) 1706 { 1707 struct signal_struct *sig; 1708 1709 if (clone_flags & CLONE_THREAD) 1710 return 0; 1711 1712 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL); 1713 tsk->signal = sig; 1714 if (!sig) 1715 return -ENOMEM; 1716 1717 sig->nr_threads = 1; 1718 sig->quick_threads = 1; 1719 atomic_set(&sig->live, 1); 1720 refcount_set(&sig->sigcnt, 1); 1721 1722 /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */ 1723 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); 1724 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); 1725 1726 init_waitqueue_head(&sig->wait_chldexit); 1727 sig->curr_target = tsk; 1728 init_sigpending(&sig->shared_pending); 1729 INIT_HLIST_HEAD(&sig->multiprocess); 1730 seqlock_init(&sig->stats_lock); 1731 prev_cputime_init(&sig->prev_cputime); 1732 1733 #ifdef CONFIG_POSIX_TIMERS 1734 INIT_LIST_HEAD(&sig->posix_timers); 1735 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1736 sig->real_timer.function = it_real_fn; 1737 #endif 1738 1739 task_lock(current->group_leader); 1740 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); 1741 task_unlock(current->group_leader); 1742 1743 posix_cpu_timers_init_group(sig); 1744 1745 tty_audit_fork(sig); 1746 sched_autogroup_fork(sig); 1747 1748 sig->oom_score_adj = current->signal->oom_score_adj; 1749 sig->oom_score_adj_min = current->signal->oom_score_adj_min; 1750 1751 mutex_init(&sig->cred_guard_mutex); 1752 init_rwsem(&sig->exec_update_lock); 1753 1754 return 0; 1755 } 1756 1757 static void copy_seccomp(struct task_struct *p) 1758 { 1759 #ifdef CONFIG_SECCOMP 1760 /* 1761 * Must be called with sighand->lock held, which is common to 1762 * all threads in the group. Holding cred_guard_mutex is not 1763 * needed because this new task is not yet running and cannot 1764 * be racing exec. 1765 */ 1766 assert_spin_locked(¤t->sighand->siglock); 1767 1768 /* Ref-count the new filter user, and assign it. */ 1769 get_seccomp_filter(current); 1770 p->seccomp = current->seccomp; 1771 1772 /* 1773 * Explicitly enable no_new_privs here in case it got set 1774 * between the task_struct being duplicated and holding the 1775 * sighand lock. The seccomp state and nnp must be in sync. 1776 */ 1777 if (task_no_new_privs(current)) 1778 task_set_no_new_privs(p); 1779 1780 /* 1781 * If the parent gained a seccomp mode after copying thread 1782 * flags and between before we held the sighand lock, we have 1783 * to manually enable the seccomp thread flag here. 1784 */ 1785 if (p->seccomp.mode != SECCOMP_MODE_DISABLED) 1786 set_task_syscall_work(p, SECCOMP); 1787 #endif 1788 } 1789 1790 SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr) 1791 { 1792 current->clear_child_tid = tidptr; 1793 1794 return task_pid_vnr(current); 1795 } 1796 1797 static void rt_mutex_init_task(struct task_struct *p) 1798 { 1799 raw_spin_lock_init(&p->pi_lock); 1800 #ifdef CONFIG_RT_MUTEXES 1801 p->pi_waiters = RB_ROOT_CACHED; 1802 p->pi_top_task = NULL; 1803 p->pi_blocked_on = NULL; 1804 #endif 1805 } 1806 1807 static inline void init_task_pid_links(struct task_struct *task) 1808 { 1809 enum pid_type type; 1810 1811 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) 1812 INIT_HLIST_NODE(&task->pid_links[type]); 1813 } 1814 1815 static inline void 1816 init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid) 1817 { 1818 if (type == PIDTYPE_PID) 1819 task->thread_pid = pid; 1820 else 1821 task->signal->pids[type] = pid; 1822 } 1823 1824 static inline void rcu_copy_process(struct task_struct *p) 1825 { 1826 #ifdef CONFIG_PREEMPT_RCU 1827 p->rcu_read_lock_nesting = 0; 1828 p->rcu_read_unlock_special.s = 0; 1829 p->rcu_blocked_node = NULL; 1830 INIT_LIST_HEAD(&p->rcu_node_entry); 1831 #endif /* #ifdef CONFIG_PREEMPT_RCU */ 1832 #ifdef CONFIG_TASKS_RCU 1833 p->rcu_tasks_holdout = false; 1834 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); 1835 p->rcu_tasks_idle_cpu = -1; 1836 #endif /* #ifdef CONFIG_TASKS_RCU */ 1837 #ifdef CONFIG_TASKS_TRACE_RCU 1838 p->trc_reader_nesting = 0; 1839 p->trc_reader_special.s = 0; 1840 INIT_LIST_HEAD(&p->trc_holdout_list); 1841 INIT_LIST_HEAD(&p->trc_blkd_node); 1842 #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ 1843 } 1844 1845 struct pid *pidfd_pid(const struct file *file) 1846 { 1847 if (file->f_op == &pidfd_fops) 1848 return file->private_data; 1849 1850 return ERR_PTR(-EBADF); 1851 } 1852 1853 static int pidfd_release(struct inode *inode, struct file *file) 1854 { 1855 struct pid *pid = file->private_data; 1856 1857 file->private_data = NULL; 1858 put_pid(pid); 1859 return 0; 1860 } 1861 1862 #ifdef CONFIG_PROC_FS 1863 /** 1864 * pidfd_show_fdinfo - print information about a pidfd 1865 * @m: proc fdinfo file 1866 * @f: file referencing a pidfd 1867 * 1868 * Pid: 1869 * This function will print the pid that a given pidfd refers to in the 1870 * pid namespace of the procfs instance. 1871 * If the pid namespace of the process is not a descendant of the pid 1872 * namespace of the procfs instance 0 will be shown as its pid. This is 1873 * similar to calling getppid() on a process whose parent is outside of 1874 * its pid namespace. 1875 * 1876 * NSpid: 1877 * If pid namespaces are supported then this function will also print 1878 * the pid of a given pidfd refers to for all descendant pid namespaces 1879 * starting from the current pid namespace of the instance, i.e. the 1880 * Pid field and the first entry in the NSpid field will be identical. 1881 * If the pid namespace of the process is not a descendant of the pid 1882 * namespace of the procfs instance 0 will be shown as its first NSpid 1883 * entry and no others will be shown. 1884 * Note that this differs from the Pid and NSpid fields in 1885 * /proc/<pid>/status where Pid and NSpid are always shown relative to 1886 * the pid namespace of the procfs instance. The difference becomes 1887 * obvious when sending around a pidfd between pid namespaces from a 1888 * different branch of the tree, i.e. where no ancestral relation is 1889 * present between the pid namespaces: 1890 * - create two new pid namespaces ns1 and ns2 in the initial pid 1891 * namespace (also take care to create new mount namespaces in the 1892 * new pid namespace and mount procfs) 1893 * - create a process with a pidfd in ns1 1894 * - send pidfd from ns1 to ns2 1895 * - read /proc/self/fdinfo/<pidfd> and observe that both Pid and NSpid 1896 * have exactly one entry, which is 0 1897 */ 1898 static void pidfd_show_fdinfo(struct seq_file *m, struct file *f) 1899 { 1900 struct pid *pid = f->private_data; 1901 struct pid_namespace *ns; 1902 pid_t nr = -1; 1903 1904 if (likely(pid_has_task(pid, PIDTYPE_PID))) { 1905 ns = proc_pid_ns(file_inode(m->file)->i_sb); 1906 nr = pid_nr_ns(pid, ns); 1907 } 1908 1909 seq_put_decimal_ll(m, "Pid:\t", nr); 1910 1911 #ifdef CONFIG_PID_NS 1912 seq_put_decimal_ll(m, "\nNSpid:\t", nr); 1913 if (nr > 0) { 1914 int i; 1915 1916 /* If nr is non-zero it means that 'pid' is valid and that 1917 * ns, i.e. the pid namespace associated with the procfs 1918 * instance, is in the pid namespace hierarchy of pid. 1919 * Start at one below the already printed level. 1920 */ 1921 for (i = ns->level + 1; i <= pid->level; i++) 1922 seq_put_decimal_ll(m, "\t", pid->numbers[i].nr); 1923 } 1924 #endif 1925 seq_putc(m, '\n'); 1926 } 1927 #endif 1928 1929 /* 1930 * Poll support for process exit notification. 1931 */ 1932 static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts) 1933 { 1934 struct pid *pid = file->private_data; 1935 __poll_t poll_flags = 0; 1936 1937 poll_wait(file, &pid->wait_pidfd, pts); 1938 1939 /* 1940 * Inform pollers only when the whole thread group exits. 1941 * If the thread group leader exits before all other threads in the 1942 * group, then poll(2) should block, similar to the wait(2) family. 1943 */ 1944 if (thread_group_exited(pid)) 1945 poll_flags = EPOLLIN | EPOLLRDNORM; 1946 1947 return poll_flags; 1948 } 1949 1950 const struct file_operations pidfd_fops = { 1951 .release = pidfd_release, 1952 .poll = pidfd_poll, 1953 #ifdef CONFIG_PROC_FS 1954 .show_fdinfo = pidfd_show_fdinfo, 1955 #endif 1956 }; 1957 1958 static void __delayed_free_task(struct rcu_head *rhp) 1959 { 1960 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); 1961 1962 free_task(tsk); 1963 } 1964 1965 static __always_inline void delayed_free_task(struct task_struct *tsk) 1966 { 1967 if (IS_ENABLED(CONFIG_MEMCG)) 1968 call_rcu(&tsk->rcu, __delayed_free_task); 1969 else 1970 free_task(tsk); 1971 } 1972 1973 static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk) 1974 { 1975 /* Skip if kernel thread */ 1976 if (!tsk->mm) 1977 return; 1978 1979 /* Skip if spawning a thread or using vfork */ 1980 if ((clone_flags & (CLONE_VM | CLONE_THREAD | CLONE_VFORK)) != CLONE_VM) 1981 return; 1982 1983 /* We need to synchronize with __set_oom_adj */ 1984 mutex_lock(&oom_adj_mutex); 1985 set_bit(MMF_MULTIPROCESS, &tsk->mm->flags); 1986 /* Update the values in case they were changed after copy_signal */ 1987 tsk->signal->oom_score_adj = current->signal->oom_score_adj; 1988 tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min; 1989 mutex_unlock(&oom_adj_mutex); 1990 } 1991 1992 #ifdef CONFIG_RV 1993 static void rv_task_fork(struct task_struct *p) 1994 { 1995 int i; 1996 1997 for (i = 0; i < RV_PER_TASK_MONITORS; i++) 1998 p->rv[i].da_mon.monitoring = false; 1999 } 2000 #else 2001 #define rv_task_fork(p) do {} while (0) 2002 #endif 2003 2004 /* 2005 * This creates a new process as a copy of the old one, 2006 * but does not actually start it yet. 2007 * 2008 * It copies the registers, and all the appropriate 2009 * parts of the process environment (as per the clone 2010 * flags). The actual kick-off is left to the caller. 2011 */ 2012 static __latent_entropy struct task_struct *copy_process( 2013 struct pid *pid, 2014 int trace, 2015 int node, 2016 struct kernel_clone_args *args) 2017 { 2018 int pidfd = -1, retval; 2019 struct task_struct *p; 2020 struct multiprocess_signals delayed; 2021 struct file *pidfile = NULL; 2022 const u64 clone_flags = args->flags; 2023 struct nsproxy *nsp = current->nsproxy; 2024 2025 /* 2026 * Don't allow sharing the root directory with processes in a different 2027 * namespace 2028 */ 2029 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) 2030 return ERR_PTR(-EINVAL); 2031 2032 if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) 2033 return ERR_PTR(-EINVAL); 2034 2035 /* 2036 * Thread groups must share signals as well, and detached threads 2037 * can only be started up within the thread group. 2038 */ 2039 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND)) 2040 return ERR_PTR(-EINVAL); 2041 2042 /* 2043 * Shared signal handlers imply shared VM. By way of the above, 2044 * thread groups also imply shared VM. Blocking this case allows 2045 * for various simplifications in other code. 2046 */ 2047 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM)) 2048 return ERR_PTR(-EINVAL); 2049 2050 /* 2051 * Siblings of global init remain as zombies on exit since they are 2052 * not reaped by their parent (swapper). To solve this and to avoid 2053 * multi-rooted process trees, prevent global and container-inits 2054 * from creating siblings. 2055 */ 2056 if ((clone_flags & CLONE_PARENT) && 2057 current->signal->flags & SIGNAL_UNKILLABLE) 2058 return ERR_PTR(-EINVAL); 2059 2060 /* 2061 * If the new process will be in a different pid or user namespace 2062 * do not allow it to share a thread group with the forking task. 2063 */ 2064 if (clone_flags & CLONE_THREAD) { 2065 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) || 2066 (task_active_pid_ns(current) != nsp->pid_ns_for_children)) 2067 return ERR_PTR(-EINVAL); 2068 } 2069 2070 if (clone_flags & CLONE_PIDFD) { 2071 /* 2072 * - CLONE_DETACHED is blocked so that we can potentially 2073 * reuse it later for CLONE_PIDFD. 2074 * - CLONE_THREAD is blocked until someone really needs it. 2075 */ 2076 if (clone_flags & (CLONE_DETACHED | CLONE_THREAD)) 2077 return ERR_PTR(-EINVAL); 2078 } 2079 2080 /* 2081 * Force any signals received before this point to be delivered 2082 * before the fork happens. Collect up signals sent to multiple 2083 * processes that happen during the fork and delay them so that 2084 * they appear to happen after the fork. 2085 */ 2086 sigemptyset(&delayed.signal); 2087 INIT_HLIST_NODE(&delayed.node); 2088 2089 spin_lock_irq(¤t->sighand->siglock); 2090 if (!(clone_flags & CLONE_THREAD)) 2091 hlist_add_head(&delayed.node, ¤t->signal->multiprocess); 2092 recalc_sigpending(); 2093 spin_unlock_irq(¤t->sighand->siglock); 2094 retval = -ERESTARTNOINTR; 2095 if (task_sigpending(current)) 2096 goto fork_out; 2097 2098 retval = -ENOMEM; 2099 p = dup_task_struct(current, node); 2100 if (!p) 2101 goto fork_out; 2102 p->flags &= ~PF_KTHREAD; 2103 if (args->kthread) 2104 p->flags |= PF_KTHREAD; 2105 if (args->io_thread) { 2106 /* 2107 * Mark us an IO worker, and block any signal that isn't 2108 * fatal or STOP 2109 */ 2110 p->flags |= PF_IO_WORKER; 2111 siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); 2112 } 2113 2114 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL; 2115 /* 2116 * Clear TID on mm_release()? 2117 */ 2118 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; 2119 2120 ftrace_graph_init_task(p); 2121 2122 rt_mutex_init_task(p); 2123 2124 lockdep_assert_irqs_enabled(); 2125 #ifdef CONFIG_PROVE_LOCKING 2126 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); 2127 #endif 2128 retval = copy_creds(p, clone_flags); 2129 if (retval < 0) 2130 goto bad_fork_free; 2131 2132 retval = -EAGAIN; 2133 if (is_rlimit_overlimit(task_ucounts(p), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) { 2134 if (p->real_cred->user != INIT_USER && 2135 !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) 2136 goto bad_fork_cleanup_count; 2137 } 2138 current->flags &= ~PF_NPROC_EXCEEDED; 2139 2140 /* 2141 * If multiple threads are within copy_process(), then this check 2142 * triggers too late. This doesn't hurt, the check is only there 2143 * to stop root fork bombs. 2144 */ 2145 retval = -EAGAIN; 2146 if (data_race(nr_threads >= max_threads)) 2147 goto bad_fork_cleanup_count; 2148 2149 delayacct_tsk_init(p); /* Must remain after dup_task_struct() */ 2150 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE | PF_NO_SETAFFINITY); 2151 p->flags |= PF_FORKNOEXEC; 2152 INIT_LIST_HEAD(&p->children); 2153 INIT_LIST_HEAD(&p->sibling); 2154 rcu_copy_process(p); 2155 p->vfork_done = NULL; 2156 spin_lock_init(&p->alloc_lock); 2157 2158 init_sigpending(&p->pending); 2159 2160 p->utime = p->stime = p->gtime = 0; 2161 #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME 2162 p->utimescaled = p->stimescaled = 0; 2163 #endif 2164 prev_cputime_init(&p->prev_cputime); 2165 2166 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 2167 seqcount_init(&p->vtime.seqcount); 2168 p->vtime.starttime = 0; 2169 p->vtime.state = VTIME_INACTIVE; 2170 #endif 2171 2172 #ifdef CONFIG_IO_URING 2173 p->io_uring = NULL; 2174 #endif 2175 2176 #if defined(SPLIT_RSS_COUNTING) 2177 memset(&p->rss_stat, 0, sizeof(p->rss_stat)); 2178 #endif 2179 2180 p->default_timer_slack_ns = current->timer_slack_ns; 2181 2182 #ifdef CONFIG_PSI 2183 p->psi_flags = 0; 2184 #endif 2185 2186 task_io_accounting_init(&p->ioac); 2187 acct_clear_integrals(p); 2188 2189 posix_cputimers_init(&p->posix_cputimers); 2190 2191 p->io_context = NULL; 2192 audit_set_context(p, NULL); 2193 cgroup_fork(p); 2194 if (args->kthread) { 2195 if (!set_kthread_struct(p)) 2196 goto bad_fork_cleanup_delayacct; 2197 } 2198 #ifdef CONFIG_NUMA 2199 p->mempolicy = mpol_dup(p->mempolicy); 2200 if (IS_ERR(p->mempolicy)) { 2201 retval = PTR_ERR(p->mempolicy); 2202 p->mempolicy = NULL; 2203 goto bad_fork_cleanup_delayacct; 2204 } 2205 #endif 2206 #ifdef CONFIG_CPUSETS 2207 p->cpuset_mem_spread_rotor = NUMA_NO_NODE; 2208 p->cpuset_slab_spread_rotor = NUMA_NO_NODE; 2209 seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock); 2210 #endif 2211 #ifdef CONFIG_TRACE_IRQFLAGS 2212 memset(&p->irqtrace, 0, sizeof(p->irqtrace)); 2213 p->irqtrace.hardirq_disable_ip = _THIS_IP_; 2214 p->irqtrace.softirq_enable_ip = _THIS_IP_; 2215 p->softirqs_enabled = 1; 2216 p->softirq_context = 0; 2217 #endif 2218 2219 p->pagefault_disabled = 0; 2220 2221 #ifdef CONFIG_LOCKDEP 2222 lockdep_init_task(p); 2223 #endif 2224 2225 #ifdef CONFIG_DEBUG_MUTEXES 2226 p->blocked_on = NULL; /* not blocked yet */ 2227 #endif 2228 #ifdef CONFIG_BCACHE 2229 p->sequential_io = 0; 2230 p->sequential_io_avg = 0; 2231 #endif 2232 #ifdef CONFIG_BPF_SYSCALL 2233 RCU_INIT_POINTER(p->bpf_storage, NULL); 2234 p->bpf_ctx = NULL; 2235 #endif 2236 2237 /* Perform scheduler related setup. Assign this task to a CPU. */ 2238 retval = sched_fork(clone_flags, p); 2239 if (retval) 2240 goto bad_fork_cleanup_policy; 2241 2242 retval = perf_event_init_task(p, clone_flags); 2243 if (retval) 2244 goto bad_fork_cleanup_policy; 2245 retval = audit_alloc(p); 2246 if (retval) 2247 goto bad_fork_cleanup_perf; 2248 /* copy all the process information */ 2249 shm_init_task(p); 2250 retval = security_task_alloc(p, clone_flags); 2251 if (retval) 2252 goto bad_fork_cleanup_audit; 2253 retval = copy_semundo(clone_flags, p); 2254 if (retval) 2255 goto bad_fork_cleanup_security; 2256 retval = copy_files(clone_flags, p); 2257 if (retval) 2258 goto bad_fork_cleanup_semundo; 2259 retval = copy_fs(clone_flags, p); 2260 if (retval) 2261 goto bad_fork_cleanup_files; 2262 retval = copy_sighand(clone_flags, p); 2263 if (retval) 2264 goto bad_fork_cleanup_fs; 2265 retval = copy_signal(clone_flags, p); 2266 if (retval) 2267 goto bad_fork_cleanup_sighand; 2268 retval = copy_mm(clone_flags, p); 2269 if (retval) 2270 goto bad_fork_cleanup_signal; 2271 retval = copy_namespaces(clone_flags, p); 2272 if (retval) 2273 goto bad_fork_cleanup_mm; 2274 retval = copy_io(clone_flags, p); 2275 if (retval) 2276 goto bad_fork_cleanup_namespaces; 2277 retval = copy_thread(p, args); 2278 if (retval) 2279 goto bad_fork_cleanup_io; 2280 2281 stackleak_task_init(p); 2282 2283 if (pid != &init_struct_pid) { 2284 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid, 2285 args->set_tid_size); 2286 if (IS_ERR(pid)) { 2287 retval = PTR_ERR(pid); 2288 goto bad_fork_cleanup_thread; 2289 } 2290 } 2291 2292 /* 2293 * This has to happen after we've potentially unshared the file 2294 * descriptor table (so that the pidfd doesn't leak into the child 2295 * if the fd table isn't shared). 2296 */ 2297 if (clone_flags & CLONE_PIDFD) { 2298 retval = get_unused_fd_flags(O_RDWR | O_CLOEXEC); 2299 if (retval < 0) 2300 goto bad_fork_free_pid; 2301 2302 pidfd = retval; 2303 2304 pidfile = anon_inode_getfile("[pidfd]", &pidfd_fops, pid, 2305 O_RDWR | O_CLOEXEC); 2306 if (IS_ERR(pidfile)) { 2307 put_unused_fd(pidfd); 2308 retval = PTR_ERR(pidfile); 2309 goto bad_fork_free_pid; 2310 } 2311 get_pid(pid); /* held by pidfile now */ 2312 2313 retval = put_user(pidfd, args->pidfd); 2314 if (retval) 2315 goto bad_fork_put_pidfd; 2316 } 2317 2318 #ifdef CONFIG_BLOCK 2319 p->plug = NULL; 2320 #endif 2321 futex_init_task(p); 2322 2323 /* 2324 * sigaltstack should be cleared when sharing the same VM 2325 */ 2326 if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) 2327 sas_ss_reset(p); 2328 2329 /* 2330 * Syscall tracing and stepping should be turned off in the 2331 * child regardless of CLONE_PTRACE. 2332 */ 2333 user_disable_single_step(p); 2334 clear_task_syscall_work(p, SYSCALL_TRACE); 2335 #if defined(CONFIG_GENERIC_ENTRY) || defined(TIF_SYSCALL_EMU) 2336 clear_task_syscall_work(p, SYSCALL_EMU); 2337 #endif 2338 clear_tsk_latency_tracing(p); 2339 2340 /* ok, now we should be set up.. */ 2341 p->pid = pid_nr(pid); 2342 if (clone_flags & CLONE_THREAD) { 2343 p->group_leader = current->group_leader; 2344 p->tgid = current->tgid; 2345 } else { 2346 p->group_leader = p; 2347 p->tgid = p->pid; 2348 } 2349 2350 p->nr_dirtied = 0; 2351 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); 2352 p->dirty_paused_when = 0; 2353 2354 p->pdeath_signal = 0; 2355 INIT_LIST_HEAD(&p->thread_group); 2356 p->task_works = NULL; 2357 clear_posix_cputimers_work(p); 2358 2359 #ifdef CONFIG_KRETPROBES 2360 p->kretprobe_instances.first = NULL; 2361 #endif 2362 #ifdef CONFIG_RETHOOK 2363 p->rethooks.first = NULL; 2364 #endif 2365 2366 /* 2367 * Ensure that the cgroup subsystem policies allow the new process to be 2368 * forked. It should be noted that the new process's css_set can be changed 2369 * between here and cgroup_post_fork() if an organisation operation is in 2370 * progress. 2371 */ 2372 retval = cgroup_can_fork(p, args); 2373 if (retval) 2374 goto bad_fork_put_pidfd; 2375 2376 /* 2377 * Now that the cgroups are pinned, re-clone the parent cgroup and put 2378 * the new task on the correct runqueue. All this *before* the task 2379 * becomes visible. 2380 * 2381 * This isn't part of ->can_fork() because while the re-cloning is 2382 * cgroup specific, it unconditionally needs to place the task on a 2383 * runqueue. 2384 */ 2385 sched_cgroup_fork(p, args); 2386 2387 /* 2388 * From this point on we must avoid any synchronous user-space 2389 * communication until we take the tasklist-lock. In particular, we do 2390 * not want user-space to be able to predict the process start-time by 2391 * stalling fork(2) after we recorded the start_time but before it is 2392 * visible to the system. 2393 */ 2394 2395 p->start_time = ktime_get_ns(); 2396 p->start_boottime = ktime_get_boottime_ns(); 2397 2398 /* 2399 * Make it visible to the rest of the system, but dont wake it up yet. 2400 * Need tasklist lock for parent etc handling! 2401 */ 2402 write_lock_irq(&tasklist_lock); 2403 2404 /* CLONE_PARENT re-uses the old parent */ 2405 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) { 2406 p->real_parent = current->real_parent; 2407 p->parent_exec_id = current->parent_exec_id; 2408 if (clone_flags & CLONE_THREAD) 2409 p->exit_signal = -1; 2410 else 2411 p->exit_signal = current->group_leader->exit_signal; 2412 } else { 2413 p->real_parent = current; 2414 p->parent_exec_id = current->self_exec_id; 2415 p->exit_signal = args->exit_signal; 2416 } 2417 2418 klp_copy_process(p); 2419 2420 sched_core_fork(p); 2421 2422 spin_lock(¤t->sighand->siglock); 2423 2424 rv_task_fork(p); 2425 2426 rseq_fork(p, clone_flags); 2427 2428 /* Don't start children in a dying pid namespace */ 2429 if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { 2430 retval = -ENOMEM; 2431 goto bad_fork_cancel_cgroup; 2432 } 2433 2434 /* Let kill terminate clone/fork in the middle */ 2435 if (fatal_signal_pending(current)) { 2436 retval = -EINTR; 2437 goto bad_fork_cancel_cgroup; 2438 } 2439 2440 /* No more failure paths after this point. */ 2441 2442 /* 2443 * Copy seccomp details explicitly here, in case they were changed 2444 * before holding sighand lock. 2445 */ 2446 copy_seccomp(p); 2447 2448 init_task_pid_links(p); 2449 if (likely(p->pid)) { 2450 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace); 2451 2452 init_task_pid(p, PIDTYPE_PID, pid); 2453 if (thread_group_leader(p)) { 2454 init_task_pid(p, PIDTYPE_TGID, pid); 2455 init_task_pid(p, PIDTYPE_PGID, task_pgrp(current)); 2456 init_task_pid(p, PIDTYPE_SID, task_session(current)); 2457 2458 if (is_child_reaper(pid)) { 2459 ns_of_pid(pid)->child_reaper = p; 2460 p->signal->flags |= SIGNAL_UNKILLABLE; 2461 } 2462 p->signal->shared_pending.signal = delayed.signal; 2463 p->signal->tty = tty_kref_get(current->signal->tty); 2464 /* 2465 * Inherit has_child_subreaper flag under the same 2466 * tasklist_lock with adding child to the process tree 2467 * for propagate_has_child_subreaper optimization. 2468 */ 2469 p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper || 2470 p->real_parent->signal->is_child_subreaper; 2471 list_add_tail(&p->sibling, &p->real_parent->children); 2472 list_add_tail_rcu(&p->tasks, &init_task.tasks); 2473 attach_pid(p, PIDTYPE_TGID); 2474 attach_pid(p, PIDTYPE_PGID); 2475 attach_pid(p, PIDTYPE_SID); 2476 __this_cpu_inc(process_counts); 2477 } else { 2478 current->signal->nr_threads++; 2479 current->signal->quick_threads++; 2480 atomic_inc(¤t->signal->live); 2481 refcount_inc(¤t->signal->sigcnt); 2482 task_join_group_stop(p); 2483 list_add_tail_rcu(&p->thread_group, 2484 &p->group_leader->thread_group); 2485 list_add_tail_rcu(&p->thread_node, 2486 &p->signal->thread_head); 2487 } 2488 attach_pid(p, PIDTYPE_PID); 2489 nr_threads++; 2490 } 2491 total_forks++; 2492 hlist_del_init(&delayed.node); 2493 spin_unlock(¤t->sighand->siglock); 2494 syscall_tracepoint_update(p); 2495 write_unlock_irq(&tasklist_lock); 2496 2497 if (pidfile) 2498 fd_install(pidfd, pidfile); 2499 2500 proc_fork_connector(p); 2501 sched_post_fork(p); 2502 cgroup_post_fork(p, args); 2503 perf_event_fork(p); 2504 2505 trace_task_newtask(p, clone_flags); 2506 uprobe_copy_process(p, clone_flags); 2507 2508 copy_oom_score_adj(clone_flags, p); 2509 2510 return p; 2511 2512 bad_fork_cancel_cgroup: 2513 sched_core_free(p); 2514 spin_unlock(¤t->sighand->siglock); 2515 write_unlock_irq(&tasklist_lock); 2516 cgroup_cancel_fork(p, args); 2517 bad_fork_put_pidfd: 2518 if (clone_flags & CLONE_PIDFD) { 2519 fput(pidfile); 2520 put_unused_fd(pidfd); 2521 } 2522 bad_fork_free_pid: 2523 if (pid != &init_struct_pid) 2524 free_pid(pid); 2525 bad_fork_cleanup_thread: 2526 exit_thread(p); 2527 bad_fork_cleanup_io: 2528 if (p->io_context) 2529 exit_io_context(p); 2530 bad_fork_cleanup_namespaces: 2531 exit_task_namespaces(p); 2532 bad_fork_cleanup_mm: 2533 if (p->mm) { 2534 mm_clear_owner(p->mm, p); 2535 mmput(p->mm); 2536 } 2537 bad_fork_cleanup_signal: 2538 if (!(clone_flags & CLONE_THREAD)) 2539 free_signal_struct(p->signal); 2540 bad_fork_cleanup_sighand: 2541 __cleanup_sighand(p->sighand); 2542 bad_fork_cleanup_fs: 2543 exit_fs(p); /* blocking */ 2544 bad_fork_cleanup_files: 2545 exit_files(p); /* blocking */ 2546 bad_fork_cleanup_semundo: 2547 exit_sem(p); 2548 bad_fork_cleanup_security: 2549 security_task_free(p); 2550 bad_fork_cleanup_audit: 2551 audit_free(p); 2552 bad_fork_cleanup_perf: 2553 perf_event_free_task(p); 2554 bad_fork_cleanup_policy: 2555 lockdep_free_task(p); 2556 #ifdef CONFIG_NUMA 2557 mpol_put(p->mempolicy); 2558 #endif 2559 bad_fork_cleanup_delayacct: 2560 delayacct_tsk_free(p); 2561 bad_fork_cleanup_count: 2562 dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1); 2563 exit_creds(p); 2564 bad_fork_free: 2565 WRITE_ONCE(p->__state, TASK_DEAD); 2566 exit_task_stack_account(p); 2567 put_task_stack(p); 2568 delayed_free_task(p); 2569 fork_out: 2570 spin_lock_irq(¤t->sighand->siglock); 2571 hlist_del_init(&delayed.node); 2572 spin_unlock_irq(¤t->sighand->siglock); 2573 return ERR_PTR(retval); 2574 } 2575 2576 static inline void init_idle_pids(struct task_struct *idle) 2577 { 2578 enum pid_type type; 2579 2580 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) { 2581 INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */ 2582 init_task_pid(idle, type, &init_struct_pid); 2583 } 2584 } 2585 2586 static int idle_dummy(void *dummy) 2587 { 2588 /* This function is never called */ 2589 return 0; 2590 } 2591 2592 struct task_struct * __init fork_idle(int cpu) 2593 { 2594 struct task_struct *task; 2595 struct kernel_clone_args args = { 2596 .flags = CLONE_VM, 2597 .fn = &idle_dummy, 2598 .fn_arg = NULL, 2599 .kthread = 1, 2600 .idle = 1, 2601 }; 2602 2603 task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args); 2604 if (!IS_ERR(task)) { 2605 init_idle_pids(task); 2606 init_idle(task, cpu); 2607 } 2608 2609 return task; 2610 } 2611 2612 /* 2613 * This is like kernel_clone(), but shaved down and tailored to just 2614 * creating io_uring workers. It returns a created task, or an error pointer. 2615 * The returned task is inactive, and the caller must fire it up through 2616 * wake_up_new_task(p). All signals are blocked in the created task. 2617 */ 2618 struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) 2619 { 2620 unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD| 2621 CLONE_IO; 2622 struct kernel_clone_args args = { 2623 .flags = ((lower_32_bits(flags) | CLONE_VM | 2624 CLONE_UNTRACED) & ~CSIGNAL), 2625 .exit_signal = (lower_32_bits(flags) & CSIGNAL), 2626 .fn = fn, 2627 .fn_arg = arg, 2628 .io_thread = 1, 2629 }; 2630 2631 return copy_process(NULL, 0, node, &args); 2632 } 2633 2634 /* 2635 * Ok, this is the main fork-routine. 2636 * 2637 * It copies the process, and if successful kick-starts 2638 * it and waits for it to finish using the VM if required. 2639 * 2640 * args->exit_signal is expected to be checked for sanity by the caller. 2641 */ 2642 pid_t kernel_clone(struct kernel_clone_args *args) 2643 { 2644 u64 clone_flags = args->flags; 2645 struct completion vfork; 2646 struct pid *pid; 2647 struct task_struct *p; 2648 int trace = 0; 2649 pid_t nr; 2650 2651 /* 2652 * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument 2653 * to return the pidfd. Hence, CLONE_PIDFD and CLONE_PARENT_SETTID are 2654 * mutually exclusive. With clone3() CLONE_PIDFD has grown a separate 2655 * field in struct clone_args and it still doesn't make sense to have 2656 * them both point at the same memory location. Performing this check 2657 * here has the advantage that we don't need to have a separate helper 2658 * to check for legacy clone(). 2659 */ 2660 if ((args->flags & CLONE_PIDFD) && 2661 (args->flags & CLONE_PARENT_SETTID) && 2662 (args->pidfd == args->parent_tid)) 2663 return -EINVAL; 2664 2665 /* 2666 * Determine whether and which event to report to ptracer. When 2667 * called from kernel_thread or CLONE_UNTRACED is explicitly 2668 * requested, no event is reported; otherwise, report if the event 2669 * for the type of forking is enabled. 2670 */ 2671 if (!(clone_flags & CLONE_UNTRACED)) { 2672 if (clone_flags & CLONE_VFORK) 2673 trace = PTRACE_EVENT_VFORK; 2674 else if (args->exit_signal != SIGCHLD) 2675 trace = PTRACE_EVENT_CLONE; 2676 else 2677 trace = PTRACE_EVENT_FORK; 2678 2679 if (likely(!ptrace_event_enabled(current, trace))) 2680 trace = 0; 2681 } 2682 2683 p = copy_process(NULL, trace, NUMA_NO_NODE, args); 2684 add_latent_entropy(); 2685 2686 if (IS_ERR(p)) 2687 return PTR_ERR(p); 2688 2689 /* 2690 * Do this prior waking up the new thread - the thread pointer 2691 * might get invalid after that point, if the thread exits quickly. 2692 */ 2693 trace_sched_process_fork(current, p); 2694 2695 pid = get_task_pid(p, PIDTYPE_PID); 2696 nr = pid_vnr(pid); 2697 2698 if (clone_flags & CLONE_PARENT_SETTID) 2699 put_user(nr, args->parent_tid); 2700 2701 if (clone_flags & CLONE_VFORK) { 2702 p->vfork_done = &vfork; 2703 init_completion(&vfork); 2704 get_task_struct(p); 2705 } 2706 2707 if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) { 2708 /* lock the task to synchronize with memcg migration */ 2709 task_lock(p); 2710 lru_gen_add_mm(p->mm); 2711 task_unlock(p); 2712 } 2713 2714 wake_up_new_task(p); 2715 2716 /* forking complete and child started to run, tell ptracer */ 2717 if (unlikely(trace)) 2718 ptrace_event_pid(trace, pid); 2719 2720 if (clone_flags & CLONE_VFORK) { 2721 if (!wait_for_vfork_done(p, &vfork)) 2722 ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid); 2723 } 2724 2725 put_pid(pid); 2726 return nr; 2727 } 2728 2729 /* 2730 * Create a kernel thread. 2731 */ 2732 pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) 2733 { 2734 struct kernel_clone_args args = { 2735 .flags = ((lower_32_bits(flags) | CLONE_VM | 2736 CLONE_UNTRACED) & ~CSIGNAL), 2737 .exit_signal = (lower_32_bits(flags) & CSIGNAL), 2738 .fn = fn, 2739 .fn_arg = arg, 2740 .kthread = 1, 2741 }; 2742 2743 return kernel_clone(&args); 2744 } 2745 2746 /* 2747 * Create a user mode thread. 2748 */ 2749 pid_t user_mode_thread(int (*fn)(void *), void *arg, unsigned long flags) 2750 { 2751 struct kernel_clone_args args = { 2752 .flags = ((lower_32_bits(flags) | CLONE_VM | 2753 CLONE_UNTRACED) & ~CSIGNAL), 2754 .exit_signal = (lower_32_bits(flags) & CSIGNAL), 2755 .fn = fn, 2756 .fn_arg = arg, 2757 }; 2758 2759 return kernel_clone(&args); 2760 } 2761 2762 #ifdef __ARCH_WANT_SYS_FORK 2763 SYSCALL_DEFINE0(fork) 2764 { 2765 #ifdef CONFIG_MMU 2766 struct kernel_clone_args args = { 2767 .exit_signal = SIGCHLD, 2768 }; 2769 2770 return kernel_clone(&args); 2771 #else 2772 /* can not support in nommu mode */ 2773 return -EINVAL; 2774 #endif 2775 } 2776 #endif 2777 2778 #ifdef __ARCH_WANT_SYS_VFORK 2779 SYSCALL_DEFINE0(vfork) 2780 { 2781 struct kernel_clone_args args = { 2782 .flags = CLONE_VFORK | CLONE_VM, 2783 .exit_signal = SIGCHLD, 2784 }; 2785 2786 return kernel_clone(&args); 2787 } 2788 #endif 2789 2790 #ifdef __ARCH_WANT_SYS_CLONE 2791 #ifdef CONFIG_CLONE_BACKWARDS 2792 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 2793 int __user *, parent_tidptr, 2794 unsigned long, tls, 2795 int __user *, child_tidptr) 2796 #elif defined(CONFIG_CLONE_BACKWARDS2) 2797 SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags, 2798 int __user *, parent_tidptr, 2799 int __user *, child_tidptr, 2800 unsigned long, tls) 2801 #elif defined(CONFIG_CLONE_BACKWARDS3) 2802 SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, 2803 int, stack_size, 2804 int __user *, parent_tidptr, 2805 int __user *, child_tidptr, 2806 unsigned long, tls) 2807 #else 2808 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, 2809 int __user *, parent_tidptr, 2810 int __user *, child_tidptr, 2811 unsigned long, tls) 2812 #endif 2813 { 2814 struct kernel_clone_args args = { 2815 .flags = (lower_32_bits(clone_flags) & ~CSIGNAL), 2816 .pidfd = parent_tidptr, 2817 .child_tid = child_tidptr, 2818 .parent_tid = parent_tidptr, 2819 .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL), 2820 .stack = newsp, 2821 .tls = tls, 2822 }; 2823 2824 return kernel_clone(&args); 2825 } 2826 #endif 2827 2828 #ifdef __ARCH_WANT_SYS_CLONE3 2829 2830 noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs, 2831 struct clone_args __user *uargs, 2832 size_t usize) 2833 { 2834 int err; 2835 struct clone_args args; 2836 pid_t *kset_tid = kargs->set_tid; 2837 2838 BUILD_BUG_ON(offsetofend(struct clone_args, tls) != 2839 CLONE_ARGS_SIZE_VER0); 2840 BUILD_BUG_ON(offsetofend(struct clone_args, set_tid_size) != 2841 CLONE_ARGS_SIZE_VER1); 2842 BUILD_BUG_ON(offsetofend(struct clone_args, cgroup) != 2843 CLONE_ARGS_SIZE_VER2); 2844 BUILD_BUG_ON(sizeof(struct clone_args) != CLONE_ARGS_SIZE_VER2); 2845 2846 if (unlikely(usize > PAGE_SIZE)) 2847 return -E2BIG; 2848 if (unlikely(usize < CLONE_ARGS_SIZE_VER0)) 2849 return -EINVAL; 2850 2851 err = copy_struct_from_user(&args, sizeof(args), uargs, usize); 2852 if (err) 2853 return err; 2854 2855 if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL)) 2856 return -EINVAL; 2857 2858 if (unlikely(!args.set_tid && args.set_tid_size > 0)) 2859 return -EINVAL; 2860 2861 if (unlikely(args.set_tid && args.set_tid_size == 0)) 2862 return -EINVAL; 2863 2864 /* 2865 * Verify that higher 32bits of exit_signal are unset and that 2866 * it is a valid signal 2867 */ 2868 if (unlikely((args.exit_signal & ~((u64)CSIGNAL)) || 2869 !valid_signal(args.exit_signal))) 2870 return -EINVAL; 2871 2872 if ((args.flags & CLONE_INTO_CGROUP) && 2873 (args.cgroup > INT_MAX || usize < CLONE_ARGS_SIZE_VER2)) 2874 return -EINVAL; 2875 2876 *kargs = (struct kernel_clone_args){ 2877 .flags = args.flags, 2878 .pidfd = u64_to_user_ptr(args.pidfd), 2879 .child_tid = u64_to_user_ptr(args.child_tid), 2880 .parent_tid = u64_to_user_ptr(args.parent_tid), 2881 .exit_signal = args.exit_signal, 2882 .stack = args.stack, 2883 .stack_size = args.stack_size, 2884 .tls = args.tls, 2885 .set_tid_size = args.set_tid_size, 2886 .cgroup = args.cgroup, 2887 }; 2888 2889 if (args.set_tid && 2890 copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid), 2891 (kargs->set_tid_size * sizeof(pid_t)))) 2892 return -EFAULT; 2893 2894 kargs->set_tid = kset_tid; 2895 2896 return 0; 2897 } 2898 2899 /** 2900 * clone3_stack_valid - check and prepare stack 2901 * @kargs: kernel clone args 2902 * 2903 * Verify that the stack arguments userspace gave us are sane. 2904 * In addition, set the stack direction for userspace since it's easy for us to 2905 * determine. 2906 */ 2907 static inline bool clone3_stack_valid(struct kernel_clone_args *kargs) 2908 { 2909 if (kargs->stack == 0) { 2910 if (kargs->stack_size > 0) 2911 return false; 2912 } else { 2913 if (kargs->stack_size == 0) 2914 return false; 2915 2916 if (!access_ok((void __user *)kargs->stack, kargs->stack_size)) 2917 return false; 2918 2919 #if !defined(CONFIG_STACK_GROWSUP) && !defined(CONFIG_IA64) 2920 kargs->stack += kargs->stack_size; 2921 #endif 2922 } 2923 2924 return true; 2925 } 2926 2927 static bool clone3_args_valid(struct kernel_clone_args *kargs) 2928 { 2929 /* Verify that no unknown flags are passed along. */ 2930 if (kargs->flags & 2931 ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND | CLONE_INTO_CGROUP)) 2932 return false; 2933 2934 /* 2935 * - make the CLONE_DETACHED bit reusable for clone3 2936 * - make the CSIGNAL bits reusable for clone3 2937 */ 2938 if (kargs->flags & (CLONE_DETACHED | (CSIGNAL & (~CLONE_NEWTIME)))) 2939 return false; 2940 2941 if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) == 2942 (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) 2943 return false; 2944 2945 if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) && 2946 kargs->exit_signal) 2947 return false; 2948 2949 if (!clone3_stack_valid(kargs)) 2950 return false; 2951 2952 return true; 2953 } 2954 2955 /** 2956 * clone3 - create a new process with specific properties 2957 * @uargs: argument structure 2958 * @size: size of @uargs 2959 * 2960 * clone3() is the extensible successor to clone()/clone2(). 2961 * It takes a struct as argument that is versioned by its size. 2962 * 2963 * Return: On success, a positive PID for the child process. 2964 * On error, a negative errno number. 2965 */ 2966 SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size) 2967 { 2968 int err; 2969 2970 struct kernel_clone_args kargs; 2971 pid_t set_tid[MAX_PID_NS_LEVEL]; 2972 2973 kargs.set_tid = set_tid; 2974 2975 err = copy_clone_args_from_user(&kargs, uargs, size); 2976 if (err) 2977 return err; 2978 2979 if (!clone3_args_valid(&kargs)) 2980 return -EINVAL; 2981 2982 return kernel_clone(&kargs); 2983 } 2984 #endif 2985 2986 void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data) 2987 { 2988 struct task_struct *leader, *parent, *child; 2989 int res; 2990 2991 read_lock(&tasklist_lock); 2992 leader = top = top->group_leader; 2993 down: 2994 for_each_thread(leader, parent) { 2995 list_for_each_entry(child, &parent->children, sibling) { 2996 res = visitor(child, data); 2997 if (res) { 2998 if (res < 0) 2999 goto out; 3000 leader = child; 3001 goto down; 3002 } 3003 up: 3004 ; 3005 } 3006 } 3007 3008 if (leader != top) { 3009 child = leader; 3010 parent = child->real_parent; 3011 leader = parent->group_leader; 3012 goto up; 3013 } 3014 out: 3015 read_unlock(&tasklist_lock); 3016 } 3017 3018 #ifndef ARCH_MIN_MMSTRUCT_ALIGN 3019 #define ARCH_MIN_MMSTRUCT_ALIGN 0 3020 #endif 3021 3022 static void sighand_ctor(void *data) 3023 { 3024 struct sighand_struct *sighand = data; 3025 3026 spin_lock_init(&sighand->siglock); 3027 init_waitqueue_head(&sighand->signalfd_wqh); 3028 } 3029 3030 void __init mm_cache_init(void) 3031 { 3032 unsigned int mm_size; 3033 3034 /* 3035 * The mm_cpumask is located at the end of mm_struct, and is 3036 * dynamically sized based on the maximum CPU number this system 3037 * can have, taking hotplug into account (nr_cpu_ids). 3038 */ 3039 mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size(); 3040 3041 mm_cachep = kmem_cache_create_usercopy("mm_struct", 3042 mm_size, ARCH_MIN_MMSTRUCT_ALIGN, 3043 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 3044 offsetof(struct mm_struct, saved_auxv), 3045 sizeof_field(struct mm_struct, saved_auxv), 3046 NULL); 3047 } 3048 3049 void __init proc_caches_init(void) 3050 { 3051 sighand_cachep = kmem_cache_create("sighand_cache", 3052 sizeof(struct sighand_struct), 0, 3053 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU| 3054 SLAB_ACCOUNT, sighand_ctor); 3055 signal_cachep = kmem_cache_create("signal_cache", 3056 sizeof(struct signal_struct), 0, 3057 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 3058 NULL); 3059 files_cachep = kmem_cache_create("files_cache", 3060 sizeof(struct files_struct), 0, 3061 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 3062 NULL); 3063 fs_cachep = kmem_cache_create("fs_cache", 3064 sizeof(struct fs_struct), 0, 3065 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, 3066 NULL); 3067 3068 vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT); 3069 mmap_init(); 3070 nsproxy_cache_init(); 3071 } 3072 3073 /* 3074 * Check constraints on flags passed to the unshare system call. 3075 */ 3076 static int check_unshare_flags(unsigned long unshare_flags) 3077 { 3078 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND| 3079 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM| 3080 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET| 3081 CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP| 3082 CLONE_NEWTIME)) 3083 return -EINVAL; 3084 /* 3085 * Not implemented, but pretend it works if there is nothing 3086 * to unshare. Note that unsharing the address space or the 3087 * signal handlers also need to unshare the signal queues (aka 3088 * CLONE_THREAD). 3089 */ 3090 if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) { 3091 if (!thread_group_empty(current)) 3092 return -EINVAL; 3093 } 3094 if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) { 3095 if (refcount_read(¤t->sighand->count) > 1) 3096 return -EINVAL; 3097 } 3098 if (unshare_flags & CLONE_VM) { 3099 if (!current_is_single_threaded()) 3100 return -EINVAL; 3101 } 3102 3103 return 0; 3104 } 3105 3106 /* 3107 * Unshare the filesystem structure if it is being shared 3108 */ 3109 static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) 3110 { 3111 struct fs_struct *fs = current->fs; 3112 3113 if (!(unshare_flags & CLONE_FS) || !fs) 3114 return 0; 3115 3116 /* don't need lock here; in the worst case we'll do useless copy */ 3117 if (fs->users == 1) 3118 return 0; 3119 3120 *new_fsp = copy_fs_struct(fs); 3121 if (!*new_fsp) 3122 return -ENOMEM; 3123 3124 return 0; 3125 } 3126 3127 /* 3128 * Unshare file descriptor table if it is being shared 3129 */ 3130 int unshare_fd(unsigned long unshare_flags, unsigned int max_fds, 3131 struct files_struct **new_fdp) 3132 { 3133 struct files_struct *fd = current->files; 3134 int error = 0; 3135 3136 if ((unshare_flags & CLONE_FILES) && 3137 (fd && atomic_read(&fd->count) > 1)) { 3138 *new_fdp = dup_fd(fd, max_fds, &error); 3139 if (!*new_fdp) 3140 return error; 3141 } 3142 3143 return 0; 3144 } 3145 3146 /* 3147 * unshare allows a process to 'unshare' part of the process 3148 * context which was originally shared using clone. copy_* 3149 * functions used by kernel_clone() cannot be used here directly 3150 * because they modify an inactive task_struct that is being 3151 * constructed. Here we are modifying the current, active, 3152 * task_struct. 3153 */ 3154 int ksys_unshare(unsigned long unshare_flags) 3155 { 3156 struct fs_struct *fs, *new_fs = NULL; 3157 struct files_struct *new_fd = NULL; 3158 struct cred *new_cred = NULL; 3159 struct nsproxy *new_nsproxy = NULL; 3160 int do_sysvsem = 0; 3161 int err; 3162 3163 /* 3164 * If unsharing a user namespace must also unshare the thread group 3165 * and unshare the filesystem root and working directories. 3166 */ 3167 if (unshare_flags & CLONE_NEWUSER) 3168 unshare_flags |= CLONE_THREAD | CLONE_FS; 3169 /* 3170 * If unsharing vm, must also unshare signal handlers. 3171 */ 3172 if (unshare_flags & CLONE_VM) 3173 unshare_flags |= CLONE_SIGHAND; 3174 /* 3175 * If unsharing a signal handlers, must also unshare the signal queues. 3176 */ 3177 if (unshare_flags & CLONE_SIGHAND) 3178 unshare_flags |= CLONE_THREAD; 3179 /* 3180 * If unsharing namespace, must also unshare filesystem information. 3181 */ 3182 if (unshare_flags & CLONE_NEWNS) 3183 unshare_flags |= CLONE_FS; 3184 3185 err = check_unshare_flags(unshare_flags); 3186 if (err) 3187 goto bad_unshare_out; 3188 /* 3189 * CLONE_NEWIPC must also detach from the undolist: after switching 3190 * to a new ipc namespace, the semaphore arrays from the old 3191 * namespace are unreachable. 3192 */ 3193 if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM)) 3194 do_sysvsem = 1; 3195 err = unshare_fs(unshare_flags, &new_fs); 3196 if (err) 3197 goto bad_unshare_out; 3198 err = unshare_fd(unshare_flags, NR_OPEN_MAX, &new_fd); 3199 if (err) 3200 goto bad_unshare_cleanup_fs; 3201 err = unshare_userns(unshare_flags, &new_cred); 3202 if (err) 3203 goto bad_unshare_cleanup_fd; 3204 err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, 3205 new_cred, new_fs); 3206 if (err) 3207 goto bad_unshare_cleanup_cred; 3208 3209 if (new_cred) { 3210 err = set_cred_ucounts(new_cred); 3211 if (err) 3212 goto bad_unshare_cleanup_cred; 3213 } 3214 3215 if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) { 3216 if (do_sysvsem) { 3217 /* 3218 * CLONE_SYSVSEM is equivalent to sys_exit(). 3219 */ 3220 exit_sem(current); 3221 } 3222 if (unshare_flags & CLONE_NEWIPC) { 3223 /* Orphan segments in old ns (see sem above). */ 3224 exit_shm(current); 3225 shm_init_task(current); 3226 } 3227 3228 if (new_nsproxy) 3229 switch_task_namespaces(current, new_nsproxy); 3230 3231 task_lock(current); 3232 3233 if (new_fs) { 3234 fs = current->fs; 3235 spin_lock(&fs->lock); 3236 current->fs = new_fs; 3237 if (--fs->users) 3238 new_fs = NULL; 3239 else 3240 new_fs = fs; 3241 spin_unlock(&fs->lock); 3242 } 3243 3244 if (new_fd) 3245 swap(current->files, new_fd); 3246 3247 task_unlock(current); 3248 3249 if (new_cred) { 3250 /* Install the new user namespace */ 3251 commit_creds(new_cred); 3252 new_cred = NULL; 3253 } 3254 } 3255 3256 perf_event_namespaces(current); 3257 3258 bad_unshare_cleanup_cred: 3259 if (new_cred) 3260 put_cred(new_cred); 3261 bad_unshare_cleanup_fd: 3262 if (new_fd) 3263 put_files_struct(new_fd); 3264 3265 bad_unshare_cleanup_fs: 3266 if (new_fs) 3267 free_fs_struct(new_fs); 3268 3269 bad_unshare_out: 3270 return err; 3271 } 3272 3273 SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags) 3274 { 3275 return ksys_unshare(unshare_flags); 3276 } 3277 3278 /* 3279 * Helper to unshare the files of the current task. 3280 * We don't want to expose copy_files internals to 3281 * the exec layer of the kernel. 3282 */ 3283 3284 int unshare_files(void) 3285 { 3286 struct task_struct *task = current; 3287 struct files_struct *old, *copy = NULL; 3288 int error; 3289 3290 error = unshare_fd(CLONE_FILES, NR_OPEN_MAX, ©); 3291 if (error || !copy) 3292 return error; 3293 3294 old = task->files; 3295 task_lock(task); 3296 task->files = copy; 3297 task_unlock(task); 3298 put_files_struct(old); 3299 return 0; 3300 } 3301 3302 int sysctl_max_threads(struct ctl_table *table, int write, 3303 void *buffer, size_t *lenp, loff_t *ppos) 3304 { 3305 struct ctl_table t; 3306 int ret; 3307 int threads = max_threads; 3308 int min = 1; 3309 int max = MAX_THREADS; 3310 3311 t = *table; 3312 t.data = &threads; 3313 t.extra1 = &min; 3314 t.extra2 = &max; 3315 3316 ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 3317 if (ret || !write) 3318 return ret; 3319 3320 max_threads = threads; 3321 3322 return 0; 3323 } 3324