1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Generic pidhash and scalable, time-bounded PID allocator 4 * 5 * (C) 2002-2003 Nadia Yvette Chambers, IBM 6 * (C) 2004 Nadia Yvette Chambers, Oracle 7 * (C) 2002-2004 Ingo Molnar, Red Hat 8 * 9 * pid-structures are backing objects for tasks sharing a given ID to chain 10 * against. There is very little to them aside from hashing them and 11 * parking tasks using given ID's on a list. 12 * 13 * The hash is always changed with the tasklist_lock write-acquired, 14 * and the hash is only accessed with the tasklist_lock at least 15 * read-acquired, so there's no additional SMP locking needed here. 16 * 17 * We have a list of bitmap pages, which bitmaps represent the PID space. 18 * Allocating and freeing PIDs is completely lockless. The worst-case 19 * allocation scenario when all but one out of 1 million PIDs possible are 20 * allocated already: the scanning of 32 list entries and at most PAGE_SIZE 21 * bytes. The typical fastpath is a single successful setbit. Freeing is O(1). 22 * 23 * Pid namespaces: 24 * (C) 2007 Pavel Emelyanov <xemul@openvz.org>, OpenVZ, SWsoft Inc. 25 * (C) 2007 Sukadev Bhattiprolu <sukadev@us.ibm.com>, IBM 26 * Many thanks to Oleg Nesterov for comments and help 27 * 28 */ 29 30 #include <linux/mm.h> 31 #include <linux/export.h> 32 #include <linux/slab.h> 33 #include <linux/init.h> 34 #include <linux/rculist.h> 35 #include <linux/memblock.h> 36 #include <linux/pid_namespace.h> 37 #include <linux/init_task.h> 38 #include <linux/syscalls.h> 39 #include <linux/proc_ns.h> 40 #include <linux/refcount.h> 41 #include <linux/anon_inodes.h> 42 #include <linux/sched/signal.h> 43 #include <linux/sched/task.h> 44 #include <linux/idr.h> 45 46 struct pid init_struct_pid = { 47 .count = REFCOUNT_INIT(1), 48 .tasks = { 49 { .first = NULL }, 50 { .first = NULL }, 51 { .first = NULL }, 52 }, 53 .level = 0, 54 .numbers = { { 55 .nr = 0, 56 .ns = &init_pid_ns, 57 }, } 58 }; 59 60 int pid_max = PID_MAX_DEFAULT; 61 62 #define RESERVED_PIDS 300 63 64 int pid_max_min = RESERVED_PIDS + 1; 65 int pid_max_max = PID_MAX_LIMIT; 66 67 /* 68 * PID-map pages start out as NULL, they get allocated upon 69 * first use and are never deallocated. This way a low pid_max 70 * value does not cause lots of bitmaps to be allocated, but 71 * the scheme scales to up to 4 million PIDs, runtime. 72 */ 73 struct pid_namespace init_pid_ns = { 74 .kref = KREF_INIT(2), 75 .idr = IDR_INIT(init_pid_ns.idr), 76 .pid_allocated = PIDNS_ADDING, 77 .level = 0, 78 .child_reaper = &init_task, 79 .user_ns = &init_user_ns, 80 .ns.inum = PROC_PID_INIT_INO, 81 #ifdef CONFIG_PID_NS 82 .ns.ops = &pidns_operations, 83 #endif 84 }; 85 EXPORT_SYMBOL_GPL(init_pid_ns); 86 87 /* 88 * Note: disable interrupts while the pidmap_lock is held as an 89 * interrupt might come in and do read_lock(&tasklist_lock). 90 * 91 * If we don't disable interrupts there is a nasty deadlock between 92 * detach_pid()->free_pid() and another cpu that does 93 * spin_lock(&pidmap_lock) followed by an interrupt routine that does 94 * read_lock(&tasklist_lock); 95 * 96 * After we clean up the tasklist_lock and know there are no 97 * irq handlers that take it we can leave the interrupts enabled. 98 * For now it is easier to be safe than to prove it can't happen. 99 */ 100 101 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(pidmap_lock); 102 103 void put_pid(struct pid *pid) 104 { 105 struct pid_namespace *ns; 106 107 if (!pid) 108 return; 109 110 ns = pid->numbers[pid->level].ns; 111 if (refcount_dec_and_test(&pid->count)) { 112 kmem_cache_free(ns->pid_cachep, pid); 113 put_pid_ns(ns); 114 } 115 } 116 EXPORT_SYMBOL_GPL(put_pid); 117 118 static void delayed_put_pid(struct rcu_head *rhp) 119 { 120 struct pid *pid = container_of(rhp, struct pid, rcu); 121 put_pid(pid); 122 } 123 124 void free_pid(struct pid *pid) 125 { 126 /* We can be called with write_lock_irq(&tasklist_lock) held */ 127 int i; 128 unsigned long flags; 129 130 spin_lock_irqsave(&pidmap_lock, flags); 131 for (i = 0; i <= pid->level; i++) { 132 struct upid *upid = pid->numbers + i; 133 struct pid_namespace *ns = upid->ns; 134 switch (--ns->pid_allocated) { 135 case 2: 136 case 1: 137 /* When all that is left in the pid namespace 138 * is the reaper wake up the reaper. The reaper 139 * may be sleeping in zap_pid_ns_processes(). 140 */ 141 wake_up_process(ns->child_reaper); 142 break; 143 case PIDNS_ADDING: 144 /* Handle a fork failure of the first process */ 145 WARN_ON(ns->child_reaper); 146 ns->pid_allocated = 0; 147 /* fall through */ 148 case 0: 149 schedule_work(&ns->proc_work); 150 break; 151 } 152 153 idr_remove(&ns->idr, upid->nr); 154 } 155 spin_unlock_irqrestore(&pidmap_lock, flags); 156 157 call_rcu(&pid->rcu, delayed_put_pid); 158 } 159 160 struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid, 161 size_t set_tid_size) 162 { 163 struct pid *pid; 164 enum pid_type type; 165 int i, nr; 166 struct pid_namespace *tmp; 167 struct upid *upid; 168 int retval = -ENOMEM; 169 170 /* 171 * set_tid_size contains the size of the set_tid array. Starting at 172 * the most nested currently active PID namespace it tells alloc_pid() 173 * which PID to set for a process in that most nested PID namespace 174 * up to set_tid_size PID namespaces. It does not have to set the PID 175 * for a process in all nested PID namespaces but set_tid_size must 176 * never be greater than the current ns->level + 1. 177 */ 178 if (set_tid_size > ns->level + 1) 179 return ERR_PTR(-EINVAL); 180 181 pid = kmem_cache_alloc(ns->pid_cachep, GFP_KERNEL); 182 if (!pid) 183 return ERR_PTR(retval); 184 185 tmp = ns; 186 pid->level = ns->level; 187 188 for (i = ns->level; i >= 0; i--) { 189 int tid = 0; 190 191 if (set_tid_size) { 192 tid = set_tid[ns->level - i]; 193 194 retval = -EINVAL; 195 if (tid < 1 || tid >= pid_max) 196 goto out_free; 197 /* 198 * Also fail if a PID != 1 is requested and 199 * no PID 1 exists. 200 */ 201 if (tid != 1 && !tmp->child_reaper) 202 goto out_free; 203 retval = -EPERM; 204 if (!ns_capable(tmp->user_ns, CAP_SYS_ADMIN)) 205 goto out_free; 206 set_tid_size--; 207 } 208 209 idr_preload(GFP_KERNEL); 210 spin_lock_irq(&pidmap_lock); 211 212 if (tid) { 213 nr = idr_alloc(&tmp->idr, NULL, tid, 214 tid + 1, GFP_ATOMIC); 215 /* 216 * If ENOSPC is returned it means that the PID is 217 * alreay in use. Return EEXIST in that case. 218 */ 219 if (nr == -ENOSPC) 220 nr = -EEXIST; 221 } else { 222 int pid_min = 1; 223 /* 224 * init really needs pid 1, but after reaching the 225 * maximum wrap back to RESERVED_PIDS 226 */ 227 if (idr_get_cursor(&tmp->idr) > RESERVED_PIDS) 228 pid_min = RESERVED_PIDS; 229 230 /* 231 * Store a null pointer so find_pid_ns does not find 232 * a partially initialized PID (see below). 233 */ 234 nr = idr_alloc_cyclic(&tmp->idr, NULL, pid_min, 235 pid_max, GFP_ATOMIC); 236 } 237 spin_unlock_irq(&pidmap_lock); 238 idr_preload_end(); 239 240 if (nr < 0) { 241 retval = (nr == -ENOSPC) ? -EAGAIN : nr; 242 goto out_free; 243 } 244 245 pid->numbers[i].nr = nr; 246 pid->numbers[i].ns = tmp; 247 tmp = tmp->parent; 248 } 249 250 /* 251 * ENOMEM is not the most obvious choice especially for the case 252 * where the child subreaper has already exited and the pid 253 * namespace denies the creation of any new processes. But ENOMEM 254 * is what we have exposed to userspace for a long time and it is 255 * documented behavior for pid namespaces. So we can't easily 256 * change it even if there were an error code better suited. 257 */ 258 retval = -ENOMEM; 259 260 if (unlikely(is_child_reaper(pid))) { 261 if (pid_ns_prepare_proc(ns)) 262 goto out_free; 263 } 264 265 get_pid_ns(ns); 266 refcount_set(&pid->count, 1); 267 for (type = 0; type < PIDTYPE_MAX; ++type) 268 INIT_HLIST_HEAD(&pid->tasks[type]); 269 270 init_waitqueue_head(&pid->wait_pidfd); 271 272 upid = pid->numbers + ns->level; 273 spin_lock_irq(&pidmap_lock); 274 if (!(ns->pid_allocated & PIDNS_ADDING)) 275 goto out_unlock; 276 for ( ; upid >= pid->numbers; --upid) { 277 /* Make the PID visible to find_pid_ns. */ 278 idr_replace(&upid->ns->idr, pid, upid->nr); 279 upid->ns->pid_allocated++; 280 } 281 spin_unlock_irq(&pidmap_lock); 282 283 return pid; 284 285 out_unlock: 286 spin_unlock_irq(&pidmap_lock); 287 put_pid_ns(ns); 288 289 out_free: 290 spin_lock_irq(&pidmap_lock); 291 while (++i <= ns->level) { 292 upid = pid->numbers + i; 293 idr_remove(&upid->ns->idr, upid->nr); 294 } 295 296 /* On failure to allocate the first pid, reset the state */ 297 if (ns->pid_allocated == PIDNS_ADDING) 298 idr_set_cursor(&ns->idr, 0); 299 300 spin_unlock_irq(&pidmap_lock); 301 302 kmem_cache_free(ns->pid_cachep, pid); 303 return ERR_PTR(retval); 304 } 305 306 void disable_pid_allocation(struct pid_namespace *ns) 307 { 308 spin_lock_irq(&pidmap_lock); 309 ns->pid_allocated &= ~PIDNS_ADDING; 310 spin_unlock_irq(&pidmap_lock); 311 } 312 313 struct pid *find_pid_ns(int nr, struct pid_namespace *ns) 314 { 315 return idr_find(&ns->idr, nr); 316 } 317 EXPORT_SYMBOL_GPL(find_pid_ns); 318 319 struct pid *find_vpid(int nr) 320 { 321 return find_pid_ns(nr, task_active_pid_ns(current)); 322 } 323 EXPORT_SYMBOL_GPL(find_vpid); 324 325 static struct pid **task_pid_ptr(struct task_struct *task, enum pid_type type) 326 { 327 return (type == PIDTYPE_PID) ? 328 &task->thread_pid : 329 &task->signal->pids[type]; 330 } 331 332 /* 333 * attach_pid() must be called with the tasklist_lock write-held. 334 */ 335 void attach_pid(struct task_struct *task, enum pid_type type) 336 { 337 struct pid *pid = *task_pid_ptr(task, type); 338 hlist_add_head_rcu(&task->pid_links[type], &pid->tasks[type]); 339 } 340 341 static void __change_pid(struct task_struct *task, enum pid_type type, 342 struct pid *new) 343 { 344 struct pid **pid_ptr = task_pid_ptr(task, type); 345 struct pid *pid; 346 int tmp; 347 348 pid = *pid_ptr; 349 350 hlist_del_rcu(&task->pid_links[type]); 351 *pid_ptr = new; 352 353 for (tmp = PIDTYPE_MAX; --tmp >= 0; ) 354 if (pid_has_task(pid, tmp)) 355 return; 356 357 free_pid(pid); 358 } 359 360 void detach_pid(struct task_struct *task, enum pid_type type) 361 { 362 __change_pid(task, type, NULL); 363 } 364 365 void change_pid(struct task_struct *task, enum pid_type type, 366 struct pid *pid) 367 { 368 __change_pid(task, type, pid); 369 attach_pid(task, type); 370 } 371 372 /* transfer_pid is an optimization of attach_pid(new), detach_pid(old) */ 373 void transfer_pid(struct task_struct *old, struct task_struct *new, 374 enum pid_type type) 375 { 376 if (type == PIDTYPE_PID) 377 new->thread_pid = old->thread_pid; 378 hlist_replace_rcu(&old->pid_links[type], &new->pid_links[type]); 379 } 380 381 struct task_struct *pid_task(struct pid *pid, enum pid_type type) 382 { 383 struct task_struct *result = NULL; 384 if (pid) { 385 struct hlist_node *first; 386 first = rcu_dereference_check(hlist_first_rcu(&pid->tasks[type]), 387 lockdep_tasklist_lock_is_held()); 388 if (first) 389 result = hlist_entry(first, struct task_struct, pid_links[(type)]); 390 } 391 return result; 392 } 393 EXPORT_SYMBOL(pid_task); 394 395 /* 396 * Must be called under rcu_read_lock(). 397 */ 398 struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns) 399 { 400 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), 401 "find_task_by_pid_ns() needs rcu_read_lock() protection"); 402 return pid_task(find_pid_ns(nr, ns), PIDTYPE_PID); 403 } 404 405 struct task_struct *find_task_by_vpid(pid_t vnr) 406 { 407 return find_task_by_pid_ns(vnr, task_active_pid_ns(current)); 408 } 409 410 struct task_struct *find_get_task_by_vpid(pid_t nr) 411 { 412 struct task_struct *task; 413 414 rcu_read_lock(); 415 task = find_task_by_vpid(nr); 416 if (task) 417 get_task_struct(task); 418 rcu_read_unlock(); 419 420 return task; 421 } 422 423 struct pid *get_task_pid(struct task_struct *task, enum pid_type type) 424 { 425 struct pid *pid; 426 rcu_read_lock(); 427 pid = get_pid(rcu_dereference(*task_pid_ptr(task, type))); 428 rcu_read_unlock(); 429 return pid; 430 } 431 EXPORT_SYMBOL_GPL(get_task_pid); 432 433 struct task_struct *get_pid_task(struct pid *pid, enum pid_type type) 434 { 435 struct task_struct *result; 436 rcu_read_lock(); 437 result = pid_task(pid, type); 438 if (result) 439 get_task_struct(result); 440 rcu_read_unlock(); 441 return result; 442 } 443 EXPORT_SYMBOL_GPL(get_pid_task); 444 445 struct pid *find_get_pid(pid_t nr) 446 { 447 struct pid *pid; 448 449 rcu_read_lock(); 450 pid = get_pid(find_vpid(nr)); 451 rcu_read_unlock(); 452 453 return pid; 454 } 455 EXPORT_SYMBOL_GPL(find_get_pid); 456 457 pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns) 458 { 459 struct upid *upid; 460 pid_t nr = 0; 461 462 if (pid && ns->level <= pid->level) { 463 upid = &pid->numbers[ns->level]; 464 if (upid->ns == ns) 465 nr = upid->nr; 466 } 467 return nr; 468 } 469 EXPORT_SYMBOL_GPL(pid_nr_ns); 470 471 pid_t pid_vnr(struct pid *pid) 472 { 473 return pid_nr_ns(pid, task_active_pid_ns(current)); 474 } 475 EXPORT_SYMBOL_GPL(pid_vnr); 476 477 pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, 478 struct pid_namespace *ns) 479 { 480 pid_t nr = 0; 481 482 rcu_read_lock(); 483 if (!ns) 484 ns = task_active_pid_ns(current); 485 if (likely(pid_alive(task))) 486 nr = pid_nr_ns(rcu_dereference(*task_pid_ptr(task, type)), ns); 487 rcu_read_unlock(); 488 489 return nr; 490 } 491 EXPORT_SYMBOL(__task_pid_nr_ns); 492 493 struct pid_namespace *task_active_pid_ns(struct task_struct *tsk) 494 { 495 return ns_of_pid(task_pid(tsk)); 496 } 497 EXPORT_SYMBOL_GPL(task_active_pid_ns); 498 499 /* 500 * Used by proc to find the first pid that is greater than or equal to nr. 501 * 502 * If there is a pid at nr this function is exactly the same as find_pid_ns. 503 */ 504 struct pid *find_ge_pid(int nr, struct pid_namespace *ns) 505 { 506 return idr_get_next(&ns->idr, &nr); 507 } 508 509 /** 510 * pidfd_create() - Create a new pid file descriptor. 511 * 512 * @pid: struct pid that the pidfd will reference 513 * 514 * This creates a new pid file descriptor with the O_CLOEXEC flag set. 515 * 516 * Note, that this function can only be called after the fd table has 517 * been unshared to avoid leaking the pidfd to the new process. 518 * 519 * Return: On success, a cloexec pidfd is returned. 520 * On error, a negative errno number will be returned. 521 */ 522 static int pidfd_create(struct pid *pid) 523 { 524 int fd; 525 526 fd = anon_inode_getfd("[pidfd]", &pidfd_fops, get_pid(pid), 527 O_RDWR | O_CLOEXEC); 528 if (fd < 0) 529 put_pid(pid); 530 531 return fd; 532 } 533 534 /** 535 * pidfd_open() - Open new pid file descriptor. 536 * 537 * @pid: pid for which to retrieve a pidfd 538 * @flags: flags to pass 539 * 540 * This creates a new pid file descriptor with the O_CLOEXEC flag set for 541 * the process identified by @pid. Currently, the process identified by 542 * @pid must be a thread-group leader. This restriction currently exists 543 * for all aspects of pidfds including pidfd creation (CLONE_PIDFD cannot 544 * be used with CLONE_THREAD) and pidfd polling (only supports thread group 545 * leaders). 546 * 547 * Return: On success, a cloexec pidfd is returned. 548 * On error, a negative errno number will be returned. 549 */ 550 SYSCALL_DEFINE2(pidfd_open, pid_t, pid, unsigned int, flags) 551 { 552 int fd; 553 struct pid *p; 554 555 if (flags) 556 return -EINVAL; 557 558 if (pid <= 0) 559 return -EINVAL; 560 561 p = find_get_pid(pid); 562 if (!p) 563 return -ESRCH; 564 565 if (pid_has_task(p, PIDTYPE_TGID)) 566 fd = pidfd_create(p); 567 else 568 fd = -EINVAL; 569 570 put_pid(p); 571 return fd; 572 } 573 574 void __init pid_idr_init(void) 575 { 576 /* Verify no one has done anything silly: */ 577 BUILD_BUG_ON(PID_MAX_LIMIT >= PIDNS_ADDING); 578 579 /* bump default and minimum pid_max based on number of cpus */ 580 pid_max = min(pid_max_max, max_t(int, pid_max, 581 PIDS_PER_CPU_DEFAULT * num_possible_cpus())); 582 pid_max_min = max_t(int, pid_max_min, 583 PIDS_PER_CPU_MIN * num_possible_cpus()); 584 pr_info("pid_max: default: %u minimum: %u\n", pid_max, pid_max_min); 585 586 idr_init(&init_pid_ns.idr); 587 588 init_pid_ns.pid_cachep = KMEM_CACHE(pid, 589 SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT); 590 } 591 592 static struct file *__pidfd_fget(struct task_struct *task, int fd) 593 { 594 struct file *file; 595 int ret; 596 597 ret = mutex_lock_killable(&task->signal->cred_guard_mutex); 598 if (ret) 599 return ERR_PTR(ret); 600 601 if (ptrace_may_access(task, PTRACE_MODE_ATTACH_REALCREDS)) 602 file = fget_task(task, fd); 603 else 604 file = ERR_PTR(-EPERM); 605 606 mutex_unlock(&task->signal->cred_guard_mutex); 607 608 return file ?: ERR_PTR(-EBADF); 609 } 610 611 static int pidfd_getfd(struct pid *pid, int fd) 612 { 613 struct task_struct *task; 614 struct file *file; 615 int ret; 616 617 task = get_pid_task(pid, PIDTYPE_PID); 618 if (!task) 619 return -ESRCH; 620 621 file = __pidfd_fget(task, fd); 622 put_task_struct(task); 623 if (IS_ERR(file)) 624 return PTR_ERR(file); 625 626 ret = security_file_receive(file); 627 if (ret) { 628 fput(file); 629 return ret; 630 } 631 632 ret = get_unused_fd_flags(O_CLOEXEC); 633 if (ret < 0) 634 fput(file); 635 else 636 fd_install(ret, file); 637 638 return ret; 639 } 640 641 /** 642 * sys_pidfd_getfd() - Get a file descriptor from another process 643 * 644 * @pidfd: the pidfd file descriptor of the process 645 * @fd: the file descriptor number to get 646 * @flags: flags on how to get the fd (reserved) 647 * 648 * This syscall gets a copy of a file descriptor from another process 649 * based on the pidfd, and file descriptor number. It requires that 650 * the calling process has the ability to ptrace the process represented 651 * by the pidfd. The process which is having its file descriptor copied 652 * is otherwise unaffected. 653 * 654 * Return: On success, a cloexec file descriptor is returned. 655 * On error, a negative errno number will be returned. 656 */ 657 SYSCALL_DEFINE3(pidfd_getfd, int, pidfd, int, fd, 658 unsigned int, flags) 659 { 660 struct pid *pid; 661 struct fd f; 662 int ret; 663 664 /* flags is currently unused - make sure it's unset */ 665 if (flags) 666 return -EINVAL; 667 668 f = fdget(pidfd); 669 if (!f.file) 670 return -EBADF; 671 672 pid = pidfd_pid(f.file); 673 if (IS_ERR(pid)) 674 ret = PTR_ERR(pid); 675 else 676 ret = pidfd_getfd(pid, fd); 677 678 fdput(f); 679 return ret; 680 } 681