Lines Matching +full:mm +full:- +full:0
1 // SPDX-License-Identifier: GPL-2.0-only
9 * 'fork.c' contains the help-routines for the 'fork' system call
12 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
18 #include <linux/sched/mm.h>
45 #include <linux/mm.h>
82 #include <linux/posix-timers.h>
83 #include <linux/user-return-notifier.h>
142 DEFINE_PER_CPU(unsigned long, process_counts) = 0;
157 int total = 0; in nr_processes()
208 for (i = 0; i < NR_CACHED_STACKS; i++) { in try_release_thread_stack_to_cache()
220 if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area)) in thread_stack_free_rcu()
228 struct vm_stack *vm_stack = tsk->stack; in thread_stack_delayed_free()
230 vm_stack->stack_vm_area = tsk->stack_vm_area; in thread_stack_delayed_free()
231 call_rcu(&vm_stack->rcu, thread_stack_free_rcu); in thread_stack_delayed_free()
239 for (i = 0; i < NR_CACHED_STACKS; i++) { in free_vm_stack_cache()
245 vfree(vm_stack->addr); in free_vm_stack_cache()
249 return 0; in free_vm_stack_cache()
256 int nr_charged = 0; in memcg_charge_kernel_stack()
258 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE); in memcg_charge_kernel_stack()
260 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { in memcg_charge_kernel_stack()
261 ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL, 0); in memcg_charge_kernel_stack()
266 return 0; in memcg_charge_kernel_stack()
268 for (i = 0; i < nr_charged; i++) in memcg_charge_kernel_stack()
269 memcg_kmem_uncharge_page(vm->pages[i], 0); in memcg_charge_kernel_stack()
279 for (i = 0; i < NR_CACHED_STACKS; i++) { in alloc_thread_stack_node()
288 kasan_unpoison_range(s->addr, THREAD_SIZE); in alloc_thread_stack_node()
290 stack = kasan_reset_tag(s->addr); in alloc_thread_stack_node()
293 memset(stack, 0, THREAD_SIZE); in alloc_thread_stack_node()
296 vfree(s->addr); in alloc_thread_stack_node()
297 return -ENOMEM; in alloc_thread_stack_node()
300 tsk->stack_vm_area = s; in alloc_thread_stack_node()
301 tsk->stack = stack; in alloc_thread_stack_node()
302 return 0; in alloc_thread_stack_node()
314 0, node, __builtin_return_address(0)); in alloc_thread_stack_node()
316 return -ENOMEM; in alloc_thread_stack_node()
321 return -ENOMEM; in alloc_thread_stack_node()
328 tsk->stack_vm_area = vm; in alloc_thread_stack_node()
330 tsk->stack = stack; in alloc_thread_stack_node()
331 return 0; in alloc_thread_stack_node()
336 if (!try_release_thread_stack_to_cache(tsk->stack_vm_area)) in free_thread_stack()
339 tsk->stack = NULL; in free_thread_stack()
340 tsk->stack_vm_area = NULL; in free_thread_stack()
352 struct rcu_head *rh = tsk->stack; in thread_stack_delayed_free()
363 tsk->stack = kasan_reset_tag(page_address(page)); in alloc_thread_stack_node()
364 return 0; in alloc_thread_stack_node()
366 return -ENOMEM; in alloc_thread_stack_node()
372 tsk->stack = NULL; in free_thread_stack()
387 struct rcu_head *rh = tsk->stack; in thread_stack_delayed_free()
397 tsk->stack = stack; in alloc_thread_stack_node()
398 return stack ? 0 : -ENOMEM; in alloc_thread_stack_node()
404 tsk->stack = NULL; in free_thread_stack()
410 THREAD_SIZE, THREAD_SIZE, 0, 0, in thread_stack_cache_init()
423 tsk->stack = stack; in alloc_thread_stack_node()
424 return stack ? 0 : -ENOMEM; in alloc_thread_stack_node()
430 tsk->stack = NULL; in free_thread_stack()
435 /* SLAB cache for signal_struct structures (tsk->signal) */
438 /* SLAB cache for sighand_struct structures (tsk->sighand) */
441 /* SLAB cache for files_struct structures (tsk->files) */
444 /* SLAB cache for fs_struct structures (tsk->fs) */
450 /* SLAB cache for mm_struct structures (tsk->mm) */
460 vma->vm_lock = kmem_cache_alloc(vma_lock_cachep, GFP_KERNEL); in vma_lock_alloc()
461 if (!vma->vm_lock) in vma_lock_alloc()
464 init_rwsem(&vma->vm_lock->lock); in vma_lock_alloc()
465 vma->vm_lock_seq = -1; in vma_lock_alloc()
472 kmem_cache_free(vma_lock_cachep, vma->vm_lock); in vma_lock_free()
482 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) in vm_area_alloc() argument
490 vma_init(vma, mm); in vm_area_alloc()
506 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); in vm_area_dup()
507 ASSERT_EXCLUSIVE_WRITER(orig->vm_file); in vm_area_dup()
509 * orig->shared.rb may be modified concurrently, but the clone in vm_area_dup()
517 INIT_LIST_HEAD(&new->anon_vma_chain); in vm_area_dup()
539 VM_BUG_ON_VMA(rwsem_is_locked(&vma->vm_lock->lock), vma); in vm_area_free_rcu_cb()
547 call_rcu(&vma->vm_rcu, vm_area_free_rcu_cb); in vm_area_free()
559 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) in account_kernel_stack()
560 mod_lruvec_page_state(vm->pages[i], NR_KERNEL_STACK_KB, in account_kernel_stack()
573 account_kernel_stack(tsk, -1); in exit_task_stack_account()
580 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) in exit_task_stack_account()
581 memcg_kmem_uncharge_page(vm->pages[i], 0); in exit_task_stack_account()
587 if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD)) in release_task_stack()
596 if (refcount_dec_and_test(&tsk->stack_refcount)) in put_task_stack()
604 WARN_ON_ONCE(tsk->seccomp.filter); in free_task()
620 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0); in free_task()
625 if (tsk->flags & PF_KTHREAD) in free_task()
632 static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mm_exe_file() argument
637 RCU_INIT_POINTER(mm->exe_file, exe_file); in dup_mm_exe_file()
647 static __latent_entropy int dup_mmap(struct mm_struct *mm, in dup_mmap() argument
652 unsigned long charge = 0; in dup_mmap()
654 VMA_ITERATOR(old_vmi, oldmm, 0); in dup_mmap()
655 VMA_ITERATOR(vmi, mm, 0); in dup_mmap()
659 retval = -EINTR; in dup_mmap()
663 uprobe_dup_mmap(oldmm, mm); in dup_mmap()
665 * Not linked in yet - no deadlock potential: in dup_mmap()
667 mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING); in dup_mmap()
670 dup_mm_exe_file(mm, oldmm); in dup_mmap()
672 mm->total_vm = oldmm->total_vm; in dup_mmap()
673 mm->data_vm = oldmm->data_vm; in dup_mmap()
674 mm->exec_vm = oldmm->exec_vm; in dup_mmap()
675 mm->stack_vm = oldmm->stack_vm; in dup_mmap()
677 retval = ksm_fork(mm, oldmm); in dup_mmap()
680 khugepaged_fork(mm, oldmm); in dup_mmap()
682 retval = vma_iter_bulk_alloc(&vmi, oldmm->map_count); in dup_mmap()
691 if (mpnt->vm_flags & VM_DONTCOPY) { in dup_mmap()
692 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); in dup_mmap()
695 charge = 0; in dup_mmap()
697 * Don't duplicate many vmas if we've been oom-killed (for in dup_mmap()
701 retval = -EINTR; in dup_mmap()
704 if (mpnt->vm_flags & VM_ACCOUNT) { in dup_mmap()
717 tmp->vm_mm = mm; in dup_mmap()
721 if (tmp->vm_flags & VM_WIPEONFORK) { in dup_mmap()
727 tmp->anon_vma = NULL; in dup_mmap()
731 file = tmp->vm_file; in dup_mmap()
733 struct address_space *mapping = file->f_mapping; in dup_mmap()
737 if (tmp->vm_flags & VM_SHARED) in dup_mmap()
742 &mapping->i_mmap); in dup_mmap()
757 mm->map_count++; in dup_mmap()
758 if (!(tmp->vm_flags & VM_WIPEONFORK)) in dup_mmap()
761 if (tmp->vm_ops && tmp->vm_ops->open) in dup_mmap()
762 tmp->vm_ops->open(tmp); in dup_mmap()
767 /* a new mm has just been created */ in dup_mmap()
768 retval = arch_dup_mmap(oldmm, mm); in dup_mmap()
774 mmap_write_unlock(mm); in dup_mmap()
789 retval = -ENOMEM; in dup_mmap()
794 static inline int mm_alloc_pgd(struct mm_struct *mm) in mm_alloc_pgd() argument
796 mm->pgd = pgd_alloc(mm); in mm_alloc_pgd()
797 if (unlikely(!mm->pgd)) in mm_alloc_pgd()
798 return -ENOMEM; in mm_alloc_pgd()
799 return 0; in mm_alloc_pgd()
802 static inline void mm_free_pgd(struct mm_struct *mm) in mm_free_pgd() argument
804 pgd_free(mm, mm->pgd); in mm_free_pgd()
807 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mmap() argument
810 dup_mm_exe_file(mm, oldmm); in dup_mmap()
812 return 0; in dup_mmap()
814 #define mm_alloc_pgd(mm) (0) argument
815 #define mm_free_pgd(mm) argument
818 static void check_mm(struct mm_struct *mm) in check_mm() argument
825 for (i = 0; i < NR_MM_COUNTERS; i++) { in check_mm()
826 long x = percpu_counter_sum(&mm->rss_stat[i]); in check_mm()
829 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", in check_mm()
830 mm, resident_page_types[i], x); in check_mm()
833 if (mm_pgtables_bytes(mm)) in check_mm()
834 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", in check_mm()
835 mm_pgtables_bytes(mm)); in check_mm()
838 VM_BUG_ON_MM(mm->pmd_huge_pte, mm); in check_mm()
843 #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) argument
847 struct mm_struct *mm = arg; in do_check_lazy_tlb() local
849 WARN_ON_ONCE(current->active_mm == mm); in do_check_lazy_tlb()
854 struct mm_struct *mm = arg; in do_shoot_lazy_tlb() local
856 if (current->active_mm == mm) { in do_shoot_lazy_tlb()
857 WARN_ON_ONCE(current->mm); in do_shoot_lazy_tlb()
858 current->active_mm = &init_mm; in do_shoot_lazy_tlb()
859 switch_mm(mm, &init_mm, current); in do_shoot_lazy_tlb()
863 static void cleanup_lazy_tlbs(struct mm_struct *mm) in cleanup_lazy_tlbs() argument
874 * Lazy mm shootdown does not refcount "lazy tlb mm" usage, rather it in cleanup_lazy_tlbs()
875 * requires lazy mm users to switch to another mm when the refcount in cleanup_lazy_tlbs()
876 * drops to zero, before the mm is freed. This requires IPIs here to in cleanup_lazy_tlbs()
879 * archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm in cleanup_lazy_tlbs()
881 * mm lazy on this CPU but no others, reducing the need for additional in cleanup_lazy_tlbs()
884 * one exiting, or kernel threads using the mm when userspace exits. in cleanup_lazy_tlbs()
889 * - The last lazy reference created by exit_mm() could instead switch in cleanup_lazy_tlbs()
892 * - A batch of mms requiring IPIs could be gathered and freed at once. in cleanup_lazy_tlbs()
893 * - CPUs store active_mm where it can be remotely checked without a in cleanup_lazy_tlbs()
894 * lock, to filter out false-positives in the cpumask. in cleanup_lazy_tlbs()
895 * - After mm_users or mm_count reaches zero, switching away from the in cleanup_lazy_tlbs()
896 * mm could clear mm_cpumask to reduce some IPIs, perhaps together in cleanup_lazy_tlbs()
898 * - A delayed freeing and RCU-like quiescing sequence based on mm in cleanup_lazy_tlbs()
901 on_each_cpu_mask(mm_cpumask(mm), do_shoot_lazy_tlb, (void *)mm, 1); in cleanup_lazy_tlbs()
903 on_each_cpu(do_check_lazy_tlb, (void *)mm, 1); in cleanup_lazy_tlbs()
907 * Called when the last reference to the mm
909 * mmput. Free the page directory and the mm.
911 void __mmdrop(struct mm_struct *mm) in __mmdrop() argument
913 BUG_ON(mm == &init_mm); in __mmdrop()
914 WARN_ON_ONCE(mm == current->mm); in __mmdrop()
916 /* Ensure no CPUs are using this as their lazy tlb mm */ in __mmdrop()
917 cleanup_lazy_tlbs(mm); in __mmdrop()
919 WARN_ON_ONCE(mm == current->active_mm); in __mmdrop()
920 mm_free_pgd(mm); in __mmdrop()
921 destroy_context(mm); in __mmdrop()
922 mmu_notifier_subscriptions_destroy(mm); in __mmdrop()
923 check_mm(mm); in __mmdrop()
924 put_user_ns(mm->user_ns); in __mmdrop()
925 mm_pasid_drop(mm); in __mmdrop()
926 mm_destroy_cid(mm); in __mmdrop()
927 percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS); in __mmdrop()
929 free_mm(mm); in __mmdrop()
935 struct mm_struct *mm; in mmdrop_async_fn() local
937 mm = container_of(work, struct mm_struct, async_put_work); in mmdrop_async_fn()
938 __mmdrop(mm); in mmdrop_async_fn()
941 static void mmdrop_async(struct mm_struct *mm) in mmdrop_async() argument
943 if (unlikely(atomic_dec_and_test(&mm->mm_count))) { in mmdrop_async()
944 INIT_WORK(&mm->async_put_work, mmdrop_async_fn); in mmdrop_async()
945 schedule_work(&mm->async_put_work); in mmdrop_async()
957 if (sig->oom_mm) in free_signal_struct()
958 mmdrop_async(sig->oom_mm); in free_signal_struct()
964 if (refcount_dec_and_test(&sig->sigcnt)) in put_signal_struct()
970 WARN_ON(!tsk->exit_state); in __put_task_struct()
971 WARN_ON(refcount_read(&tsk->usage)); in __put_task_struct()
980 put_signal_struct(tsk->signal); in __put_task_struct()
1032 * Handle zero-sized whitelist or empty thread_struct, otherwise in task_struct_whitelist()
1035 if (unlikely(*size == 0)) in task_struct_whitelist()
1036 *offset = 0; in task_struct_whitelist()
1047 #define ARCH_MIN_TASKALIGN 0 in fork_init()
1065 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; in fork_init()
1066 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; in fork_init()
1067 init_task.signal->rlim[RLIMIT_SIGPENDING] = in fork_init()
1068 init_task.signal->rlim[RLIMIT_NPROC]; in fork_init()
1070 for (i = 0; i < UCOUNT_COUNTS; i++) in fork_init()
1093 return 0; in arch_dup_task_struct()
1124 refcount_set(&tsk->stack_refcount, 1); in dup_task_struct()
1139 tsk->seccomp.filter = NULL; in dup_task_struct()
1149 tsk->stack_canary = get_random_canary(); in dup_task_struct()
1151 if (orig->cpus_ptr == &orig->cpus_mask) in dup_task_struct()
1152 tsk->cpus_ptr = &tsk->cpus_mask; in dup_task_struct()
1159 refcount_set(&tsk->rcu_users, 2); in dup_task_struct()
1161 refcount_set(&tsk->usage, 1); in dup_task_struct()
1163 tsk->btrace_seq = 0; in dup_task_struct()
1165 tsk->splice_pipe = NULL; in dup_task_struct()
1166 tsk->task_frag.page = NULL; in dup_task_struct()
1167 tsk->wake_q.next = NULL; in dup_task_struct()
1168 tsk->worker_private = NULL; in dup_task_struct()
1175 tsk->fail_nth = 0; in dup_task_struct()
1179 tsk->throttle_disk = NULL; in dup_task_struct()
1180 tsk->use_memdelay = 0; in dup_task_struct()
1184 tsk->pasid_activated = 0; in dup_task_struct()
1188 tsk->active_memcg = NULL; in dup_task_struct()
1192 tsk->reported_split_lock = 0; in dup_task_struct()
1196 tsk->mm_cid = -1; in dup_task_struct()
1197 tsk->last_mm_cid = -1; in dup_task_struct()
1198 tsk->mm_cid_active = 0; in dup_task_struct()
1199 tsk->migrate_from_cpu = -1; in dup_task_struct()
1218 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) & in coredump_filter_setup()
1227 static void mm_init_aio(struct mm_struct *mm) in mm_init_aio() argument
1230 spin_lock_init(&mm->ioctx_lock); in mm_init_aio()
1231 mm->ioctx_table = NULL; in mm_init_aio()
1235 static __always_inline void mm_clear_owner(struct mm_struct *mm, in mm_clear_owner() argument
1239 if (mm->owner == p) in mm_clear_owner()
1240 WRITE_ONCE(mm->owner, NULL); in mm_clear_owner()
1244 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) in mm_init_owner() argument
1247 mm->owner = p; in mm_init_owner()
1251 static void mm_init_uprobes_state(struct mm_struct *mm) in mm_init_uprobes_state() argument
1254 mm->uprobes_state.xol_area = NULL; in mm_init_uprobes_state()
1258 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, in mm_init() argument
1261 mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); in mm_init()
1262 mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); in mm_init()
1263 atomic_set(&mm->mm_users, 1); in mm_init()
1264 atomic_set(&mm->mm_count, 1); in mm_init()
1265 seqcount_init(&mm->write_protect_seq); in mm_init()
1266 mmap_init_lock(mm); in mm_init()
1267 INIT_LIST_HEAD(&mm->mmlist); in mm_init()
1269 mm->mm_lock_seq = 0; in mm_init()
1271 mm_pgtables_bytes_init(mm); in mm_init()
1272 mm->map_count = 0; in mm_init()
1273 mm->locked_vm = 0; in mm_init()
1274 atomic64_set(&mm->pinned_vm, 0); in mm_init()
1275 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); in mm_init()
1276 spin_lock_init(&mm->page_table_lock); in mm_init()
1277 spin_lock_init(&mm->arg_lock); in mm_init()
1278 mm_init_cpumask(mm); in mm_init()
1279 mm_init_aio(mm); in mm_init()
1280 mm_init_owner(mm, p); in mm_init()
1281 mm_pasid_init(mm); in mm_init()
1282 RCU_INIT_POINTER(mm->exe_file, NULL); in mm_init()
1283 mmu_notifier_subscriptions_init(mm); in mm_init()
1284 init_tlb_flush_pending(mm); in mm_init()
1286 mm->pmd_huge_pte = NULL; in mm_init()
1288 mm_init_uprobes_state(mm); in mm_init()
1289 hugetlb_count_init(mm); in mm_init()
1291 if (current->mm) { in mm_init()
1292 mm->flags = mmf_init_flags(current->mm->flags); in mm_init()
1293 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; in mm_init()
1295 mm->flags = default_dump_filter; in mm_init()
1296 mm->def_flags = 0; in mm_init()
1299 if (mm_alloc_pgd(mm)) in mm_init()
1302 if (init_new_context(p, mm)) in mm_init()
1305 if (mm_alloc_cid(mm)) in mm_init()
1308 if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT, in mm_init()
1312 mm->user_ns = get_user_ns(user_ns); in mm_init()
1313 lru_gen_init_mm(mm); in mm_init()
1314 return mm; in mm_init()
1317 mm_destroy_cid(mm); in mm_init()
1319 destroy_context(mm); in mm_init()
1321 mm_free_pgd(mm); in mm_init()
1323 free_mm(mm); in mm_init()
1332 struct mm_struct *mm; in mm_alloc() local
1334 mm = allocate_mm(); in mm_alloc()
1335 if (!mm) in mm_alloc()
1338 memset(mm, 0, sizeof(*mm)); in mm_alloc()
1339 return mm_init(mm, current, current_user_ns()); in mm_alloc()
1342 static inline void __mmput(struct mm_struct *mm) in __mmput() argument
1344 VM_BUG_ON(atomic_read(&mm->mm_users)); in __mmput()
1346 uprobe_clear_state(mm); in __mmput()
1347 exit_aio(mm); in __mmput()
1348 ksm_exit(mm); in __mmput()
1349 khugepaged_exit(mm); /* must run before exit_mmap */ in __mmput()
1350 exit_mmap(mm); in __mmput()
1351 mm_put_huge_zero_page(mm); in __mmput()
1352 set_mm_exe_file(mm, NULL); in __mmput()
1353 if (!list_empty(&mm->mmlist)) { in __mmput()
1355 list_del(&mm->mmlist); in __mmput()
1358 if (mm->binfmt) in __mmput()
1359 module_put(mm->binfmt->module); in __mmput()
1360 lru_gen_del_mm(mm); in __mmput()
1361 mmdrop(mm); in __mmput()
1365 * Decrement the use count and release all resources for an mm.
1367 void mmput(struct mm_struct *mm) in mmput() argument
1371 if (atomic_dec_and_test(&mm->mm_users)) in mmput()
1372 __mmput(mm); in mmput()
1379 struct mm_struct *mm = container_of(work, struct mm_struct, in mmput_async_fn() local
1382 __mmput(mm); in mmput_async_fn()
1385 void mmput_async(struct mm_struct *mm) in mmput_async() argument
1387 if (atomic_dec_and_test(&mm->mm_users)) { in mmput_async()
1388 INIT_WORK(&mm->async_put_work, mmput_async_fn); in mmput_async()
1389 schedule_work(&mm->async_put_work); in mmput_async()
1396 * set_mm_exe_file - change a reference to the mm's executable file
1398 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1402 * the new mm is made visible to anyone.
1406 int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) in set_mm_exe_file() argument
1413 * this mm -- see comment above for justification. in set_mm_exe_file()
1415 old_exe_file = rcu_dereference_raw(mm->exe_file); in set_mm_exe_file()
1423 return -EACCES; in set_mm_exe_file()
1426 rcu_assign_pointer(mm->exe_file, new_exe_file); in set_mm_exe_file()
1431 return 0; in set_mm_exe_file()
1435 * replace_mm_exe_file - replace a reference to the mm's executable file
1437 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1441 int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) in replace_mm_exe_file() argument
1445 int ret = 0; in replace_mm_exe_file()
1447 /* Forbid mm->exe_file change if old file still mapped. */ in replace_mm_exe_file()
1448 old_exe_file = get_mm_exe_file(mm); in replace_mm_exe_file()
1450 VMA_ITERATOR(vmi, mm, 0); in replace_mm_exe_file()
1451 mmap_read_lock(mm); in replace_mm_exe_file()
1453 if (!vma->vm_file) in replace_mm_exe_file()
1455 if (path_equal(&vma->vm_file->f_path, in replace_mm_exe_file()
1456 &old_exe_file->f_path)) { in replace_mm_exe_file()
1457 ret = -EBUSY; in replace_mm_exe_file()
1461 mmap_read_unlock(mm); in replace_mm_exe_file()
1469 return -EACCES; in replace_mm_exe_file()
1473 mmap_write_lock(mm); in replace_mm_exe_file()
1474 old_exe_file = rcu_dereference_raw(mm->exe_file); in replace_mm_exe_file()
1475 rcu_assign_pointer(mm->exe_file, new_exe_file); in replace_mm_exe_file()
1476 mmap_write_unlock(mm); in replace_mm_exe_file()
1482 return 0; in replace_mm_exe_file()
1486 * get_mm_exe_file - acquire a reference to the mm's executable file
1488 * Returns %NULL if mm has no associated executable file.
1491 struct file *get_mm_exe_file(struct mm_struct *mm) in get_mm_exe_file() argument
1496 exe_file = rcu_dereference(mm->exe_file); in get_mm_exe_file()
1504 * get_task_exe_file - acquire a reference to the task's executable file
1506 * Returns %NULL if task's mm (if any) has no associated executable file or
1507 * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
1513 struct mm_struct *mm; in get_task_exe_file() local
1516 mm = task->mm; in get_task_exe_file()
1517 if (mm) { in get_task_exe_file()
1518 if (!(task->flags & PF_KTHREAD)) in get_task_exe_file()
1519 exe_file = get_mm_exe_file(mm); in get_task_exe_file()
1526 * get_task_mm - acquire a reference to the task's mm
1528 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
1529 * this kernel workthread has transiently adopted a user mm with use_mm,
1531 * bumping up the use count. User must release the mm via mmput()
1536 struct mm_struct *mm; in get_task_mm() local
1539 mm = task->mm; in get_task_mm()
1540 if (mm) { in get_task_mm()
1541 if (task->flags & PF_KTHREAD) in get_task_mm()
1542 mm = NULL; in get_task_mm()
1544 mmget(mm); in get_task_mm()
1547 return mm; in get_task_mm()
1553 struct mm_struct *mm; in mm_access() local
1556 err = down_read_killable(&task->signal->exec_update_lock); in mm_access()
1560 mm = get_task_mm(task); in mm_access()
1561 if (mm && mm != current->mm && in mm_access()
1563 mmput(mm); in mm_access()
1564 mm = ERR_PTR(-EACCES); in mm_access()
1566 up_read(&task->signal->exec_update_lock); in mm_access()
1568 return mm; in mm_access()
1576 vfork = tsk->vfork_done; in complete_vfork_done()
1578 tsk->vfork_done = NULL; in complete_vfork_done()
1596 child->vfork_done = NULL; in wait_for_vfork_done()
1617 static void mm_release(struct task_struct *tsk, struct mm_struct *mm) in mm_release() argument
1622 deactivate_mm(tsk, mm); in mm_release()
1629 if (tsk->clear_child_tid) { in mm_release()
1630 if (atomic_read(&mm->mm_users) > 1) { in mm_release()
1632 * We don't check the error code - if userspace has in mm_release()
1635 put_user(0, tsk->clear_child_tid); in mm_release()
1636 do_futex(tsk->clear_child_tid, FUTEX_WAKE, in mm_release()
1637 1, NULL, NULL, 0, 0); in mm_release()
1639 tsk->clear_child_tid = NULL; in mm_release()
1643 * All done, finally we can wake up parent and return this mm to him. in mm_release()
1646 if (tsk->vfork_done) in mm_release()
1650 void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm) in exit_mm_release() argument
1653 mm_release(tsk, mm); in exit_mm_release()
1656 void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm) in exec_mm_release() argument
1659 mm_release(tsk, mm); in exec_mm_release()
1663 * dup_mm() - duplicates an existing mm structure
1664 * @tsk: the task_struct with which the new mm will be associated.
1665 * @oldmm: the mm to duplicate.
1667 * Allocates a new mm structure and duplicates the provided @oldmm structure
1670 * Return: the duplicated mm or NULL on failure.
1675 struct mm_struct *mm; in dup_mm() local
1678 mm = allocate_mm(); in dup_mm()
1679 if (!mm) in dup_mm()
1682 memcpy(mm, oldmm, sizeof(*mm)); in dup_mm()
1684 if (!mm_init(mm, tsk, mm->user_ns)) in dup_mm()
1687 err = dup_mmap(mm, oldmm); in dup_mm()
1691 mm->hiwater_rss = get_mm_rss(mm); in dup_mm()
1692 mm->hiwater_vm = mm->total_vm; in dup_mm()
1694 if (mm->binfmt && !try_module_get(mm->binfmt->module)) in dup_mm()
1697 return mm; in dup_mm()
1701 mm->binfmt = NULL; in dup_mm()
1702 mm_init_owner(mm, NULL); in dup_mm()
1703 mmput(mm); in dup_mm()
1711 struct mm_struct *mm, *oldmm; in copy_mm() local
1713 tsk->min_flt = tsk->maj_flt = 0; in copy_mm()
1714 tsk->nvcsw = tsk->nivcsw = 0; in copy_mm()
1716 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; in copy_mm()
1717 tsk->last_switch_time = 0; in copy_mm()
1720 tsk->mm = NULL; in copy_mm()
1721 tsk->active_mm = NULL; in copy_mm()
1728 oldmm = current->mm; in copy_mm()
1730 return 0; in copy_mm()
1734 mm = oldmm; in copy_mm()
1736 mm = dup_mm(tsk, current->mm); in copy_mm()
1737 if (!mm) in copy_mm()
1738 return -ENOMEM; in copy_mm()
1741 tsk->mm = mm; in copy_mm()
1742 tsk->active_mm = mm; in copy_mm()
1744 return 0; in copy_mm()
1749 struct fs_struct *fs = current->fs; in copy_fs()
1751 /* tsk->fs is already what we want */ in copy_fs()
1752 spin_lock(&fs->lock); in copy_fs()
1753 if (fs->in_exec) { in copy_fs()
1754 spin_unlock(&fs->lock); in copy_fs()
1755 return -EAGAIN; in copy_fs()
1757 fs->users++; in copy_fs()
1758 spin_unlock(&fs->lock); in copy_fs()
1759 return 0; in copy_fs()
1761 tsk->fs = copy_fs_struct(fs); in copy_fs()
1762 if (!tsk->fs) in copy_fs()
1763 return -ENOMEM; in copy_fs()
1764 return 0; in copy_fs()
1775 oldf = current->files; in copy_files()
1777 return 0; in copy_files()
1780 tsk->files = NULL; in copy_files()
1781 return 0; in copy_files()
1785 atomic_inc(&oldf->count); in copy_files()
1786 return 0; in copy_files()
1793 tsk->files = newf; in copy_files()
1794 return 0; in copy_files()
1802 refcount_inc(¤t->sighand->count); in copy_sighand()
1803 return 0; in copy_sighand()
1806 RCU_INIT_POINTER(tsk->sighand, sig); in copy_sighand()
1808 return -ENOMEM; in copy_sighand()
1810 refcount_set(&sig->count, 1); in copy_sighand()
1811 spin_lock_irq(¤t->sighand->siglock); in copy_sighand()
1812 memcpy(sig->action, current->sighand->action, sizeof(sig->action)); in copy_sighand()
1813 spin_unlock_irq(¤t->sighand->siglock); in copy_sighand()
1817 flush_signal_handlers(tsk, 0); in copy_sighand()
1819 return 0; in copy_sighand()
1824 if (refcount_dec_and_test(&sighand->count)) { in __cleanup_sighand()
1839 struct posix_cputimers *pct = &sig->posix_cputimers; in posix_cpu_timers_init_group()
1842 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); in posix_cpu_timers_init_group()
1851 return 0; in copy_signal()
1854 tsk->signal = sig; in copy_signal()
1856 return -ENOMEM; in copy_signal()
1858 sig->nr_threads = 1; in copy_signal()
1859 sig->quick_threads = 1; in copy_signal()
1860 atomic_set(&sig->live, 1); in copy_signal()
1861 refcount_set(&sig->sigcnt, 1); in copy_signal()
1864 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); in copy_signal()
1865 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); in copy_signal()
1867 init_waitqueue_head(&sig->wait_chldexit); in copy_signal()
1868 sig->curr_target = tsk; in copy_signal()
1869 init_sigpending(&sig->shared_pending); in copy_signal()
1870 INIT_HLIST_HEAD(&sig->multiprocess); in copy_signal()
1871 seqlock_init(&sig->stats_lock); in copy_signal()
1872 prev_cputime_init(&sig->prev_cputime); in copy_signal()
1875 INIT_LIST_HEAD(&sig->posix_timers); in copy_signal()
1876 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in copy_signal()
1877 sig->real_timer.function = it_real_fn; in copy_signal()
1880 task_lock(current->group_leader); in copy_signal()
1881 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); in copy_signal()
1882 task_unlock(current->group_leader); in copy_signal()
1889 sig->oom_score_adj = current->signal->oom_score_adj; in copy_signal()
1890 sig->oom_score_adj_min = current->signal->oom_score_adj_min; in copy_signal()
1892 mutex_init(&sig->cred_guard_mutex); in copy_signal()
1893 init_rwsem(&sig->exec_update_lock); in copy_signal()
1895 return 0; in copy_signal()
1902 * Must be called with sighand->lock held, which is common to in copy_seccomp()
1907 assert_spin_locked(¤t->sighand->siglock); in copy_seccomp()
1909 /* Ref-count the new filter user, and assign it. */ in copy_seccomp()
1911 p->seccomp = current->seccomp; in copy_seccomp()
1926 if (p->seccomp.mode != SECCOMP_MODE_DISABLED) in copy_seccomp()
1933 current->clear_child_tid = tidptr; in SYSCALL_DEFINE1()
1940 raw_spin_lock_init(&p->pi_lock); in rt_mutex_init_task()
1942 p->pi_waiters = RB_ROOT_CACHED; in rt_mutex_init_task()
1943 p->pi_top_task = NULL; in rt_mutex_init_task()
1944 p->pi_blocked_on = NULL; in rt_mutex_init_task()
1953 INIT_HLIST_NODE(&task->pid_links[type]); in init_task_pid_links()
1960 task->thread_pid = pid; in init_task_pid()
1962 task->signal->pids[type] = pid; in init_task_pid()
1968 p->rcu_read_lock_nesting = 0; in rcu_copy_process()
1969 p->rcu_read_unlock_special.s = 0; in rcu_copy_process()
1970 p->rcu_blocked_node = NULL; in rcu_copy_process()
1971 INIT_LIST_HEAD(&p->rcu_node_entry); in rcu_copy_process()
1974 p->rcu_tasks_holdout = false; in rcu_copy_process()
1975 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); in rcu_copy_process()
1976 p->rcu_tasks_idle_cpu = -1; in rcu_copy_process()
1977 INIT_LIST_HEAD(&p->rcu_tasks_exit_list); in rcu_copy_process()
1980 p->trc_reader_nesting = 0; in rcu_copy_process()
1981 p->trc_reader_special.s = 0; in rcu_copy_process()
1982 INIT_LIST_HEAD(&p->trc_holdout_list); in rcu_copy_process()
1983 INIT_LIST_HEAD(&p->trc_blkd_node); in rcu_copy_process()
1989 if (file->f_op == &pidfd_fops) in pidfd_pid()
1990 return file->private_data; in pidfd_pid()
1992 return ERR_PTR(-EBADF); in pidfd_pid()
1997 struct pid *pid = file->private_data; in pidfd_release()
1999 file->private_data = NULL; in pidfd_release()
2001 return 0; in pidfd_release()
2006 * pidfd_show_fdinfo - print information about a pidfd
2014 * namespace of the procfs instance 0 will be shown as its pid. This is
2024 * namespace of the procfs instance 0 will be shown as its first NSpid
2032 * - create two new pid namespaces ns1 and ns2 in the initial pid
2035 * - create a process with a pidfd in ns1
2036 * - send pidfd from ns1 to ns2
2037 * - read /proc/self/fdinfo/<pidfd> and observe that both Pid and NSpid
2038 * have exactly one entry, which is 0
2042 struct pid *pid = f->private_data; in pidfd_show_fdinfo()
2044 pid_t nr = -1; in pidfd_show_fdinfo()
2047 ns = proc_pid_ns(file_inode(m->file)->i_sb); in pidfd_show_fdinfo()
2055 if (nr > 0) { in pidfd_show_fdinfo()
2058 /* If nr is non-zero it means that 'pid' is valid and that in pidfd_show_fdinfo()
2063 for (i = ns->level + 1; i <= pid->level; i++) in pidfd_show_fdinfo()
2064 seq_put_decimal_ll(m, "\t", pid->numbers[i].nr); in pidfd_show_fdinfo()
2076 struct pid *pid = file->private_data; in pidfd_poll()
2077 __poll_t poll_flags = 0; in pidfd_poll()
2079 poll_wait(file, &pid->wait_pidfd, pts); in pidfd_poll()
2101 * __pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
2134 return -EINVAL; in __pidfd_prepare()
2137 if (pidfd < 0) in __pidfd_prepare()
2152 * pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
2180 return -EINVAL; in pidfd_prepare()
2195 call_rcu(&tsk->rcu, __delayed_free_task); in delayed_free_task()
2203 if (!tsk->mm) in copy_oom_score_adj()
2212 set_bit(MMF_MULTIPROCESS, &tsk->mm->flags); in copy_oom_score_adj()
2214 tsk->signal->oom_score_adj = current->signal->oom_score_adj; in copy_oom_score_adj()
2215 tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min; in copy_oom_score_adj()
2224 for (i = 0; i < RV_PER_TASK_MONITORS; i++) in rv_task_fork()
2225 p->rv[i].da_mon.monitoring = false; in rv_task_fork()
2228 #define rv_task_fork(p) do {} while (0)
2237 * flags). The actual kick-off is left to the caller.
2245 int pidfd = -1, retval; in copy_process()
2249 const u64 clone_flags = args->flags; in copy_process()
2250 struct nsproxy *nsp = current->nsproxy; in copy_process()
2257 return ERR_PTR(-EINVAL); in copy_process()
2260 return ERR_PTR(-EINVAL); in copy_process()
2267 return ERR_PTR(-EINVAL); in copy_process()
2275 return ERR_PTR(-EINVAL); in copy_process()
2280 * multi-rooted process trees, prevent global and container-inits in copy_process()
2284 current->signal->flags & SIGNAL_UNKILLABLE) in copy_process()
2285 return ERR_PTR(-EINVAL); in copy_process()
2293 (task_active_pid_ns(current) != nsp->pid_ns_for_children)) in copy_process()
2294 return ERR_PTR(-EINVAL); in copy_process()
2299 * - CLONE_DETACHED is blocked so that we can potentially in copy_process()
2301 * - CLONE_THREAD is blocked until someone really needs it. in copy_process()
2304 return ERR_PTR(-EINVAL); in copy_process()
2316 spin_lock_irq(¤t->sighand->siglock); in copy_process()
2318 hlist_add_head(&delayed.node, ¤t->signal->multiprocess); in copy_process()
2320 spin_unlock_irq(¤t->sighand->siglock); in copy_process()
2321 retval = -ERESTARTNOINTR; in copy_process()
2325 retval = -ENOMEM; in copy_process()
2329 p->flags &= ~PF_KTHREAD; in copy_process()
2330 if (args->kthread) in copy_process()
2331 p->flags |= PF_KTHREAD; in copy_process()
2332 if (args->user_worker) { in copy_process()
2337 p->flags |= PF_USER_WORKER; in copy_process()
2338 siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); in copy_process()
2340 if (args->io_thread) in copy_process()
2341 p->flags |= PF_IO_WORKER; in copy_process()
2343 if (args->name) in copy_process()
2344 strscpy_pad(p->comm, args->name, sizeof(p->comm)); in copy_process()
2346 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL; in copy_process()
2350 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; in copy_process()
2358 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); in copy_process()
2361 if (retval < 0) in copy_process()
2364 retval = -EAGAIN; in copy_process()
2366 if (p->real_cred->user != INIT_USER && in copy_process()
2370 current->flags &= ~PF_NPROC_EXCEEDED; in copy_process()
2377 retval = -EAGAIN; in copy_process()
2382 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE | PF_NO_SETAFFINITY); in copy_process()
2383 p->flags |= PF_FORKNOEXEC; in copy_process()
2384 INIT_LIST_HEAD(&p->children); in copy_process()
2385 INIT_LIST_HEAD(&p->sibling); in copy_process()
2387 p->vfork_done = NULL; in copy_process()
2388 spin_lock_init(&p->alloc_lock); in copy_process()
2390 init_sigpending(&p->pending); in copy_process()
2392 p->utime = p->stime = p->gtime = 0; in copy_process()
2394 p->utimescaled = p->stimescaled = 0; in copy_process()
2396 prev_cputime_init(&p->prev_cputime); in copy_process()
2399 seqcount_init(&p->vtime.seqcount); in copy_process()
2400 p->vtime.starttime = 0; in copy_process()
2401 p->vtime.state = VTIME_INACTIVE; in copy_process()
2405 p->io_uring = NULL; in copy_process()
2409 memset(&p->rss_stat, 0, sizeof(p->rss_stat)); in copy_process()
2412 p->default_timer_slack_ns = current->timer_slack_ns; in copy_process()
2415 p->psi_flags = 0; in copy_process()
2418 task_io_accounting_init(&p->ioac); in copy_process()
2421 posix_cputimers_init(&p->posix_cputimers); in copy_process()
2424 p->io_context = NULL; in copy_process()
2427 if (args->kthread) { in copy_process()
2432 p->mempolicy = mpol_dup(p->mempolicy); in copy_process()
2433 if (IS_ERR(p->mempolicy)) { in copy_process()
2434 retval = PTR_ERR(p->mempolicy); in copy_process()
2435 p->mempolicy = NULL; in copy_process()
2440 p->cpuset_mem_spread_rotor = NUMA_NO_NODE; in copy_process()
2441 p->cpuset_slab_spread_rotor = NUMA_NO_NODE; in copy_process()
2442 seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock); in copy_process()
2445 memset(&p->irqtrace, 0, sizeof(p->irqtrace)); in copy_process()
2446 p->irqtrace.hardirq_disable_ip = _THIS_IP_; in copy_process()
2447 p->irqtrace.softirq_enable_ip = _THIS_IP_; in copy_process()
2448 p->softirqs_enabled = 1; in copy_process()
2449 p->softirq_context = 0; in copy_process()
2452 p->pagefault_disabled = 0; in copy_process()
2459 p->blocked_on = NULL; /* not blocked yet */ in copy_process()
2462 p->sequential_io = 0; in copy_process()
2463 p->sequential_io_avg = 0; in copy_process()
2466 RCU_INIT_POINTER(p->bpf_storage, NULL); in copy_process()
2467 p->bpf_ctx = NULL; in copy_process()
2489 retval = copy_files(clone_flags, p, args->no_files); in copy_process()
2517 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid, in copy_process()
2518 args->set_tid_size); in copy_process()
2533 if (retval < 0) in copy_process()
2537 retval = put_user(pidfd, args->pidfd); in copy_process()
2543 p->plug = NULL; in copy_process()
2565 p->pid = pid_nr(pid); in copy_process()
2567 p->group_leader = current->group_leader; in copy_process()
2568 p->tgid = current->tgid; in copy_process()
2570 p->group_leader = p; in copy_process()
2571 p->tgid = p->pid; in copy_process()
2574 p->nr_dirtied = 0; in copy_process()
2575 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); in copy_process()
2576 p->dirty_paused_when = 0; in copy_process()
2578 p->pdeath_signal = 0; in copy_process()
2579 INIT_LIST_HEAD(&p->thread_group); in copy_process()
2580 p->task_works = NULL; in copy_process()
2584 p->kretprobe_instances.first = NULL; in copy_process()
2587 p->rethooks.first = NULL; in copy_process()
2601 * Now that the cgroups are pinned, re-clone the parent cgroup and put in copy_process()
2605 * This isn't part of ->can_fork() because while the re-cloning is in copy_process()
2612 * From this point on we must avoid any synchronous user-space in copy_process()
2613 * communication until we take the tasklist-lock. In particular, we do in copy_process()
2614 * not want user-space to be able to predict the process start-time by in copy_process()
2619 p->start_time = ktime_get_ns(); in copy_process()
2620 p->start_boottime = ktime_get_boottime_ns(); in copy_process()
2628 /* CLONE_PARENT re-uses the old parent */ in copy_process()
2630 p->real_parent = current->real_parent; in copy_process()
2631 p->parent_exec_id = current->parent_exec_id; in copy_process()
2633 p->exit_signal = -1; in copy_process()
2635 p->exit_signal = current->group_leader->exit_signal; in copy_process()
2637 p->real_parent = current; in copy_process()
2638 p->parent_exec_id = current->self_exec_id; in copy_process()
2639 p->exit_signal = args->exit_signal; in copy_process()
2646 spin_lock(¤t->sighand->siglock); in copy_process()
2653 if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { in copy_process()
2654 retval = -ENOMEM; in copy_process()
2660 retval = -EINTR; in copy_process()
2673 if (likely(p->pid)) { in copy_process()
2683 ns_of_pid(pid)->child_reaper = p; in copy_process()
2684 p->signal->flags |= SIGNAL_UNKILLABLE; in copy_process()
2686 p->signal->shared_pending.signal = delayed.signal; in copy_process()
2687 p->signal->tty = tty_kref_get(current->signal->tty); in copy_process()
2693 p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper || in copy_process()
2694 p->real_parent->signal->is_child_subreaper; in copy_process()
2695 list_add_tail(&p->sibling, &p->real_parent->children); in copy_process()
2696 list_add_tail_rcu(&p->tasks, &init_task.tasks); in copy_process()
2702 current->signal->nr_threads++; in copy_process()
2703 current->signal->quick_threads++; in copy_process()
2704 atomic_inc(¤t->signal->live); in copy_process()
2705 refcount_inc(¤t->signal->sigcnt); in copy_process()
2707 list_add_tail_rcu(&p->thread_group, in copy_process()
2708 &p->group_leader->thread_group); in copy_process()
2709 list_add_tail_rcu(&p->thread_node, in copy_process()
2710 &p->signal->thread_head); in copy_process()
2717 spin_unlock(¤t->sighand->siglock); in copy_process()
2739 spin_unlock(¤t->sighand->siglock); in copy_process()
2753 if (p->io_context) in copy_process()
2758 if (p->mm) { in copy_process()
2759 mm_clear_owner(p->mm, p); in copy_process()
2760 mmput(p->mm); in copy_process()
2764 free_signal_struct(p->signal); in copy_process()
2766 __cleanup_sighand(p->sighand); in copy_process()
2782 mpol_put(p->mempolicy); in copy_process()
2790 WRITE_ONCE(p->__state, TASK_DEAD); in copy_process()
2795 spin_lock_irq(¤t->sighand->siglock); in copy_process()
2797 spin_unlock_irq(¤t->sighand->siglock); in copy_process()
2806 INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */ in init_idle_pids()
2814 return 0; in idle_dummy()
2828 task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args); in fork_idle()
2857 return copy_process(NULL, 0, node, &args); in create_io_thread()
2861 * Ok, this is the main fork-routine.
2863 * It copies the process, and if successful kick-starts
2866 * args->exit_signal is expected to be checked for sanity by the caller.
2870 u64 clone_flags = args->flags; in kernel_clone()
2874 int trace = 0; in kernel_clone()
2886 if ((args->flags & CLONE_PIDFD) && in kernel_clone()
2887 (args->flags & CLONE_PARENT_SETTID) && in kernel_clone()
2888 (args->pidfd == args->parent_tid)) in kernel_clone()
2889 return -EINVAL; in kernel_clone()
2900 else if (args->exit_signal != SIGCHLD) in kernel_clone()
2906 trace = 0; in kernel_clone()
2916 * Do this prior waking up the new thread - the thread pointer in kernel_clone()
2925 put_user(nr, args->parent_tid); in kernel_clone()
2928 p->vfork_done = &vfork; in kernel_clone()
2936 lru_gen_add_mm(p->mm); in kernel_clone()
3001 return -EINVAL; in SYSCALL_DEFINE0()
3064 pid_t *kset_tid = kargs->set_tid; in copy_clone_args_from_user()
3075 return -E2BIG; in copy_clone_args_from_user()
3077 return -EINVAL; in copy_clone_args_from_user()
3084 return -EINVAL; in copy_clone_args_from_user()
3086 if (unlikely(!args.set_tid && args.set_tid_size > 0)) in copy_clone_args_from_user()
3087 return -EINVAL; in copy_clone_args_from_user()
3089 if (unlikely(args.set_tid && args.set_tid_size == 0)) in copy_clone_args_from_user()
3090 return -EINVAL; in copy_clone_args_from_user()
3098 return -EINVAL; in copy_clone_args_from_user()
3102 return -EINVAL; in copy_clone_args_from_user()
3119 (kargs->set_tid_size * sizeof(pid_t)))) in copy_clone_args_from_user()
3120 return -EFAULT; in copy_clone_args_from_user()
3122 kargs->set_tid = kset_tid; in copy_clone_args_from_user()
3124 return 0; in copy_clone_args_from_user()
3128 * clone3_stack_valid - check and prepare stack
3137 if (kargs->stack == 0) { in clone3_stack_valid()
3138 if (kargs->stack_size > 0) in clone3_stack_valid()
3141 if (kargs->stack_size == 0) in clone3_stack_valid()
3144 if (!access_ok((void __user *)kargs->stack, kargs->stack_size)) in clone3_stack_valid()
3148 kargs->stack += kargs->stack_size; in clone3_stack_valid()
3158 if (kargs->flags & in clone3_args_valid()
3163 * - make the CLONE_DETACHED bit reusable for clone3 in clone3_args_valid()
3164 * - make the CSIGNAL bits reusable for clone3 in clone3_args_valid()
3166 if (kargs->flags & (CLONE_DETACHED | (CSIGNAL & (~CLONE_NEWTIME)))) in clone3_args_valid()
3169 if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) == in clone3_args_valid()
3173 if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) && in clone3_args_valid()
3174 kargs->exit_signal) in clone3_args_valid()
3184 * clone3 - create a new process with specific properties
3208 return -EINVAL; in SYSCALL_DEFINE2()
3220 leader = top = top->group_leader; in walk_process_tree()
3223 list_for_each_entry(child, &parent->children, sibling) { in walk_process_tree()
3226 if (res < 0) in walk_process_tree()
3238 parent = child->real_parent; in walk_process_tree()
3239 leader = parent->group_leader; in walk_process_tree()
3247 #define ARCH_MIN_MMSTRUCT_ALIGN 0
3254 spin_lock_init(&sighand->siglock); in sighand_ctor()
3255 init_waitqueue_head(&sighand->signalfd_wqh); in sighand_ctor()
3280 sizeof(struct sighand_struct), 0, in proc_caches_init()
3284 sizeof(struct signal_struct), 0, in proc_caches_init()
3288 sizeof(struct files_struct), 0, in proc_caches_init()
3292 sizeof(struct fs_struct), 0, in proc_caches_init()
3314 return -EINVAL; in check_unshare_flags()
3323 return -EINVAL; in check_unshare_flags()
3326 if (refcount_read(¤t->sighand->count) > 1) in check_unshare_flags()
3327 return -EINVAL; in check_unshare_flags()
3331 return -EINVAL; in check_unshare_flags()
3334 return 0; in check_unshare_flags()
3342 struct fs_struct *fs = current->fs; in unshare_fs()
3345 return 0; in unshare_fs()
3348 if (fs->users == 1) in unshare_fs()
3349 return 0; in unshare_fs()
3353 return -ENOMEM; in unshare_fs()
3355 return 0; in unshare_fs()
3363 struct files_struct *fd = current->files; in unshare_fd()
3366 (fd && atomic_read(&fd->count) > 1)) { in unshare_fd()
3373 return 0; in unshare_fd()
3390 int do_sysvsem = 0; in ksys_unshare()
3464 fs = current->fs; in ksys_unshare()
3465 spin_lock(&fs->lock); in ksys_unshare()
3466 current->fs = new_fs; in ksys_unshare()
3467 if (--fs->users) in ksys_unshare()
3471 spin_unlock(&fs->lock); in ksys_unshare()
3475 swap(current->files, new_fd); in ksys_unshare()
3524 old = task->files; in unshare_files()
3526 task->files = copy; in unshare_files()
3529 return 0; in unshare_files()
3552 return 0; in sysctl_max_threads()