fork.c (6acf116c9558314d3cac36d5eb17f30368c73fd2) fork.c (4d6501dce079c1eb6bf0b1d8f528a5e81770109e)
1/*
2 * linux/kernel/fork.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7/*
8 * 'fork.c' contains the help-routines for the 'fork' system call

--- 73 unchanged lines hidden (view full) ---

82#include <linux/oom.h>
83#include <linux/khugepaged.h>
84#include <linux/signalfd.h>
85#include <linux/uprobes.h>
86#include <linux/aio.h>
87#include <linux/compiler.h>
88#include <linux/sysctl.h>
89#include <linux/kcov.h>
1/*
2 * linux/kernel/fork.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7/*
8 * 'fork.c' contains the help-routines for the 'fork' system call

--- 73 unchanged lines hidden (view full) ---

82#include <linux/oom.h>
83#include <linux/khugepaged.h>
84#include <linux/signalfd.h>
85#include <linux/uprobes.h>
86#include <linux/aio.h>
87#include <linux/compiler.h>
88#include <linux/sysctl.h>
89#include <linux/kcov.h>
90#include <linux/livepatch.h>
91
92#include <asm/pgtable.h>
93#include <asm/pgalloc.h>
94#include <linux/uaccess.h>
95#include <asm/mmu_context.h>
96#include <asm/cacheflush.h>
97#include <asm/tlbflush.h>
98

--- 75 unchanged lines hidden (view full) ---

174
175#ifdef CONFIG_VMAP_STACK
176/*
177 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
178 * flush. Try to minimize the number of calls by caching stacks.
179 */
180#define NR_CACHED_STACKS 2
181static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
90
91#include <asm/pgtable.h>
92#include <asm/pgalloc.h>
93#include <linux/uaccess.h>
94#include <asm/mmu_context.h>
95#include <asm/cacheflush.h>
96#include <asm/tlbflush.h>
97

--- 75 unchanged lines hidden (view full) ---

173
174#ifdef CONFIG_VMAP_STACK
175/*
176 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
177 * flush. Try to minimize the number of calls by caching stacks.
178 */
179#define NR_CACHED_STACKS 2
180static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
182
183static int free_vm_stack_cache(unsigned int cpu)
184{
185 struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu);
186 int i;
187
188 for (i = 0; i < NR_CACHED_STACKS; i++) {
189 struct vm_struct *vm_stack = cached_vm_stacks[i];
190
191 if (!vm_stack)
192 continue;
193
194 vfree(vm_stack->addr);
195 cached_vm_stacks[i] = NULL;
196 }
197
198 return 0;
199}
200#endif
201
202static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
203{
204#ifdef CONFIG_VMAP_STACK
205 void *stack;
206 int i;
207

--- 8 unchanged lines hidden (view full) ---

216 tsk->stack_vm_area = s;
217 local_irq_enable();
218 return s->addr;
219 }
220 local_irq_enable();
221
222 stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
223 VMALLOC_START, VMALLOC_END,
181#endif
182
183static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
184{
185#ifdef CONFIG_VMAP_STACK
186 void *stack;
187 int i;
188

--- 8 unchanged lines hidden (view full) ---

197 tsk->stack_vm_area = s;
198 local_irq_enable();
199 return s->addr;
200 }
201 local_irq_enable();
202
203 stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
204 VMALLOC_START, VMALLOC_END,
224 THREADINFO_GFP,
205 THREADINFO_GFP | __GFP_HIGHMEM,
225 PAGE_KERNEL,
226 0, node, __builtin_return_address(0));
227
228 /*
229 * We can't call find_vm_area() in interrupt context, and
230 * free_thread_stack() can be called in interrupt context,
231 * so cache the vm_struct.
232 */

--- 247 unchanged lines hidden (view full) ---

480 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
481 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
482 init_task.signal->rlim[RLIMIT_SIGPENDING] =
483 init_task.signal->rlim[RLIMIT_NPROC];
484
485 for (i = 0; i < UCOUNT_COUNTS; i++) {
486 init_user_ns.ucount_max[i] = max_threads/2;
487 }
206 PAGE_KERNEL,
207 0, node, __builtin_return_address(0));
208
209 /*
210 * We can't call find_vm_area() in interrupt context, and
211 * free_thread_stack() can be called in interrupt context,
212 * so cache the vm_struct.
213 */

--- 247 unchanged lines hidden (view full) ---

461 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
462 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
463 init_task.signal->rlim[RLIMIT_SIGPENDING] =
464 init_task.signal->rlim[RLIMIT_NPROC];
465
466 for (i = 0; i < UCOUNT_COUNTS; i++) {
467 init_user_ns.ucount_max[i] = max_threads/2;
468 }
488
489#ifdef CONFIG_VMAP_STACK
490 cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
491 NULL, free_vm_stack_cache);
492#endif
493}
494
495int __weak arch_dup_task_struct(struct task_struct *dst,
496 struct task_struct *src)
497{
498 *dst = *src;
499 return 0;
500}

--- 831 unchanged lines hidden (view full) ---

1332 return 0;
1333}
1334
1335void __cleanup_sighand(struct sighand_struct *sighand)
1336{
1337 if (atomic_dec_and_test(&sighand->count)) {
1338 signalfd_cleanup(sighand);
1339 /*
469}
470
471int __weak arch_dup_task_struct(struct task_struct *dst,
472 struct task_struct *src)
473{
474 *dst = *src;
475 return 0;
476}

--- 831 unchanged lines hidden (view full) ---

1308 return 0;
1309}
1310
1311void __cleanup_sighand(struct sighand_struct *sighand)
1312{
1313 if (atomic_dec_and_test(&sighand->count)) {
1314 signalfd_cleanup(sighand);
1315 /*
1340 * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it
1316 * sighand_cachep is SLAB_DESTROY_BY_RCU so we can free it
1341 * without an RCU grace period, see __lock_task_sighand().
1342 */
1343 kmem_cache_free(sighand_cachep, sighand);
1344 }
1345}
1346
1347#ifdef CONFIG_POSIX_TIMERS
1348/*

--- 223 unchanged lines hidden (view full) ---

1572 if (retval)
1573 goto fork_out;
1574
1575 retval = -ENOMEM;
1576 p = dup_task_struct(current, node);
1577 if (!p)
1578 goto fork_out;
1579
1317 * without an RCU grace period, see __lock_task_sighand().
1318 */
1319 kmem_cache_free(sighand_cachep, sighand);
1320 }
1321}
1322
1323#ifdef CONFIG_POSIX_TIMERS
1324/*

--- 223 unchanged lines hidden (view full) ---

1548 if (retval)
1549 goto fork_out;
1550
1551 retval = -ENOMEM;
1552 p = dup_task_struct(current, node);
1553 if (!p)
1554 goto fork_out;
1555
1556 /*
1557 * This _must_ happen before we call free_task(), i.e. before we jump
1558 * to any of the bad_fork_* labels. This is to avoid freeing
1559 * p->set_child_tid which is (ab)used as a kthread's data pointer for
1560 * kernel threads (PF_KTHREAD).
1561 */
1562 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
1563 /*
1564 * Clear TID on mm_release()?
1565 */
1566 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL;
1567
1580 ftrace_graph_init_task(p);
1581
1582 rt_mutex_init_task(p);
1583
1584#ifdef CONFIG_PROVE_LOCKING
1585 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
1586 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1587#endif

--- 111 unchanged lines hidden (view full) ---

1699 retval = perf_event_init_task(p);
1700 if (retval)
1701 goto bad_fork_cleanup_policy;
1702 retval = audit_alloc(p);
1703 if (retval)
1704 goto bad_fork_cleanup_perf;
1705 /* copy all the process information */
1706 shm_init_task(p);
1568 ftrace_graph_init_task(p);
1569
1570 rt_mutex_init_task(p);
1571
1572#ifdef CONFIG_PROVE_LOCKING
1573 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
1574 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1575#endif

--- 111 unchanged lines hidden (view full) ---

1687 retval = perf_event_init_task(p);
1688 if (retval)
1689 goto bad_fork_cleanup_policy;
1690 retval = audit_alloc(p);
1691 if (retval)
1692 goto bad_fork_cleanup_perf;
1693 /* copy all the process information */
1694 shm_init_task(p);
1707 retval = security_task_alloc(p, clone_flags);
1708 if (retval)
1709 goto bad_fork_cleanup_audit;
1710 retval = copy_semundo(clone_flags, p);
1711 if (retval)
1695 retval = copy_semundo(clone_flags, p);
1696 if (retval)
1712 goto bad_fork_cleanup_security;
1697 goto bad_fork_cleanup_audit;
1713 retval = copy_files(clone_flags, p);
1714 if (retval)
1715 goto bad_fork_cleanup_semundo;
1716 retval = copy_fs(clone_flags, p);
1717 if (retval)
1718 goto bad_fork_cleanup_files;
1719 retval = copy_sighand(clone_flags, p);
1720 if (retval)

--- 17 unchanged lines hidden (view full) ---

1738 if (pid != &init_struct_pid) {
1739 pid = alloc_pid(p->nsproxy->pid_ns_for_children);
1740 if (IS_ERR(pid)) {
1741 retval = PTR_ERR(pid);
1742 goto bad_fork_cleanup_thread;
1743 }
1744 }
1745
1698 retval = copy_files(clone_flags, p);
1699 if (retval)
1700 goto bad_fork_cleanup_semundo;
1701 retval = copy_fs(clone_flags, p);
1702 if (retval)
1703 goto bad_fork_cleanup_files;
1704 retval = copy_sighand(clone_flags, p);
1705 if (retval)

--- 17 unchanged lines hidden (view full) ---

1723 if (pid != &init_struct_pid) {
1724 pid = alloc_pid(p->nsproxy->pid_ns_for_children);
1725 if (IS_ERR(pid)) {
1726 retval = PTR_ERR(pid);
1727 goto bad_fork_cleanup_thread;
1728 }
1729 }
1730
1746 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
1747 /*
1748 * Clear TID on mm_release()?
1749 */
1750 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL;
1751#ifdef CONFIG_BLOCK
1752 p->plug = NULL;
1753#endif
1754#ifdef CONFIG_FUTEX
1755 p->robust_list = NULL;
1756#ifdef CONFIG_COMPAT
1757 p->compat_robust_list = NULL;
1758#endif

--- 61 unchanged lines hidden (view full) ---

1820 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
1821 p->real_parent = current->real_parent;
1822 p->parent_exec_id = current->parent_exec_id;
1823 } else {
1824 p->real_parent = current;
1825 p->parent_exec_id = current->self_exec_id;
1826 }
1827
1731#ifdef CONFIG_BLOCK
1732 p->plug = NULL;
1733#endif
1734#ifdef CONFIG_FUTEX
1735 p->robust_list = NULL;
1736#ifdef CONFIG_COMPAT
1737 p->compat_robust_list = NULL;
1738#endif

--- 61 unchanged lines hidden (view full) ---

1800 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
1801 p->real_parent = current->real_parent;
1802 p->parent_exec_id = current->parent_exec_id;
1803 } else {
1804 p->real_parent = current;
1805 p->parent_exec_id = current->self_exec_id;
1806 }
1807
1828 klp_copy_process(p);
1829
1830 spin_lock(&current->sighand->siglock);
1831
1832 /*
1833 * Copy seccomp details explicitly here, in case they were changed
1834 * before holding sighand lock.
1835 */
1836 copy_seccomp(p);
1837
1838 /*
1839 * Process group and session signals need to be delivered to just the
1840 * parent before the fork or both the parent and the child after the
1841 * fork. Restart if a signal comes in before we add the new process to
1842 * it's process group.
1843 * A fatal signal pending means that current will exit, so the new
1844 * thread can't slip out of an OOM kill (or normal SIGKILL).
1845 */
1846 recalc_sigpending();
1847 if (signal_pending(current)) {
1808 spin_lock(&current->sighand->siglock);
1809
1810 /*
1811 * Copy seccomp details explicitly here, in case they were changed
1812 * before holding sighand lock.
1813 */
1814 copy_seccomp(p);
1815
1816 /*
1817 * Process group and session signals need to be delivered to just the
1818 * parent before the fork or both the parent and the child after the
1819 * fork. Restart if a signal comes in before we add the new process to
1820 * it's process group.
1821 * A fatal signal pending means that current will exit, so the new
1822 * thread can't slip out of an OOM kill (or normal SIGKILL).
1823 */
1824 recalc_sigpending();
1825 if (signal_pending(current)) {
1826 spin_unlock(&current->sighand->siglock);
1827 write_unlock_irq(&tasklist_lock);
1848 retval = -ERESTARTNOINTR;
1849 goto bad_fork_cancel_cgroup;
1850 }
1828 retval = -ERESTARTNOINTR;
1829 goto bad_fork_cancel_cgroup;
1830 }
1851 if (unlikely(!(ns_of_pid(pid)->nr_hashed & PIDNS_HASH_ADDING))) {
1852 retval = -ENOMEM;
1853 goto bad_fork_cancel_cgroup;
1854 }
1855
1856 if (likely(p->pid)) {
1857 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
1858
1859 init_task_pid(p, PIDTYPE_PID, pid);
1860 if (thread_group_leader(p)) {
1861 init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
1862 init_task_pid(p, PIDTYPE_SID, task_session(current));

--- 41 unchanged lines hidden (view full) ---

1904 perf_event_fork(p);
1905
1906 trace_task_newtask(p, clone_flags);
1907 uprobe_copy_process(p, clone_flags);
1908
1909 return p;
1910
1911bad_fork_cancel_cgroup:
1831
1832 if (likely(p->pid)) {
1833 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
1834
1835 init_task_pid(p, PIDTYPE_PID, pid);
1836 if (thread_group_leader(p)) {
1837 init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
1838 init_task_pid(p, PIDTYPE_SID, task_session(current));

--- 41 unchanged lines hidden (view full) ---

1880 perf_event_fork(p);
1881
1882 trace_task_newtask(p, clone_flags);
1883 uprobe_copy_process(p, clone_flags);
1884
1885 return p;
1886
1887bad_fork_cancel_cgroup:
1912 spin_unlock(&current->sighand->siglock);
1913 write_unlock_irq(&tasklist_lock);
1914 cgroup_cancel_fork(p);
1915bad_fork_free_pid:
1916 cgroup_threadgroup_change_end(current);
1917 if (pid != &init_struct_pid)
1918 free_pid(pid);
1919bad_fork_cleanup_thread:
1920 exit_thread(p);
1921bad_fork_cleanup_io:

--- 10 unchanged lines hidden (view full) ---

1932bad_fork_cleanup_sighand:
1933 __cleanup_sighand(p->sighand);
1934bad_fork_cleanup_fs:
1935 exit_fs(p); /* blocking */
1936bad_fork_cleanup_files:
1937 exit_files(p); /* blocking */
1938bad_fork_cleanup_semundo:
1939 exit_sem(p);
1888 cgroup_cancel_fork(p);
1889bad_fork_free_pid:
1890 cgroup_threadgroup_change_end(current);
1891 if (pid != &init_struct_pid)
1892 free_pid(pid);
1893bad_fork_cleanup_thread:
1894 exit_thread(p);
1895bad_fork_cleanup_io:

--- 10 unchanged lines hidden (view full) ---

1906bad_fork_cleanup_sighand:
1907 __cleanup_sighand(p->sighand);
1908bad_fork_cleanup_fs:
1909 exit_fs(p); /* blocking */
1910bad_fork_cleanup_files:
1911 exit_files(p); /* blocking */
1912bad_fork_cleanup_semundo:
1913 exit_sem(p);
1940bad_fork_cleanup_security:
1941 security_task_free(p);
1942bad_fork_cleanup_audit:
1943 audit_free(p);
1944bad_fork_cleanup_perf:
1945 perf_event_free_task(p);
1946bad_fork_cleanup_policy:
1947#ifdef CONFIG_NUMA
1948 mpol_put(p->mempolicy);
1949bad_fork_cleanup_threadgroup_lock:

--- 225 unchanged lines hidden (view full) ---

2175 spin_lock_init(&sighand->siglock);
2176 init_waitqueue_head(&sighand->signalfd_wqh);
2177}
2178
2179void __init proc_caches_init(void)
2180{
2181 sighand_cachep = kmem_cache_create("sighand_cache",
2182 sizeof(struct sighand_struct), 0,
1914bad_fork_cleanup_audit:
1915 audit_free(p);
1916bad_fork_cleanup_perf:
1917 perf_event_free_task(p);
1918bad_fork_cleanup_policy:
1919#ifdef CONFIG_NUMA
1920 mpol_put(p->mempolicy);
1921bad_fork_cleanup_threadgroup_lock:

--- 225 unchanged lines hidden (view full) ---

2147 spin_lock_init(&sighand->siglock);
2148 init_waitqueue_head(&sighand->signalfd_wqh);
2149}
2150
2151void __init proc_caches_init(void)
2152{
2153 sighand_cachep = kmem_cache_create("sighand_cache",
2154 sizeof(struct sighand_struct), 0,
2183 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
2155 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU|
2184 SLAB_NOTRACK|SLAB_ACCOUNT, sighand_ctor);
2185 signal_cachep = kmem_cache_create("signal_cache",
2186 sizeof(struct signal_struct), 0,
2187 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT,
2188 NULL);
2189 files_cachep = kmem_cache_create("files_cache",
2190 sizeof(struct files_struct), 0,
2191 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT,

--- 262 unchanged lines hidden --- 		2156 SLAB_NOTRACK|SLAB_ACCOUNT, sighand_ctor);
2157 signal_cachep = kmem_cache_create("signal_cache",
2158 sizeof(struct signal_struct), 0,
2159 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT,
2160 NULL);
2161 files_cachep = kmem_cache_create("files_cache",
2162 sizeof(struct files_struct), 0,
2163 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT,

--- 262 unchanged lines hidden ---