1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/compat.h> 4 #include <linux/syscalls.h> 5 #include <linux/time_namespace.h> 6 7 #include "futex.h" 8 9 /* 10 * Support for robust futexes: the kernel cleans up held futexes at 11 * thread exit time. 12 * 13 * Implementation: user-space maintains a per-thread list of locks it 14 * is holding. Upon do_exit(), the kernel carefully walks this list, 15 * and marks all locks that are owned by this thread with the 16 * FUTEX_OWNER_DIED bit, and wakes up a waiter (if any). The list is 17 * always manipulated with the lock held, so the list is private and 18 * per-thread. Userspace also maintains a per-thread 'list_op_pending' 19 * field, to allow the kernel to clean up if the thread dies after 20 * acquiring the lock, but just before it could have added itself to 21 * the list. There can only be one such pending lock. 22 */ 23 24 /** 25 * sys_set_robust_list() - Set the robust-futex list head of a task 26 * @head: pointer to the list-head 27 * @len: length of the list-head, as userspace expects 28 */ 29 SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, 30 size_t, len) 31 { 32 if (!futex_cmpxchg_enabled) 33 return -ENOSYS; 34 /* 35 * The kernel knows only one size for now: 36 */ 37 if (unlikely(len != sizeof(*head))) 38 return -EINVAL; 39 40 current->robust_list = head; 41 42 return 0; 43 } 44 45 /** 46 * sys_get_robust_list() - Get the robust-futex list head of a task 47 * @pid: pid of the process [zero for current task] 48 * @head_ptr: pointer to a list-head pointer, the kernel fills it in 49 * @len_ptr: pointer to a length field, the kernel fills in the header size 50 */ 51 SYSCALL_DEFINE3(get_robust_list, int, pid, 52 struct robust_list_head __user * __user *, head_ptr, 53 size_t __user *, len_ptr) 54 { 55 struct robust_list_head __user *head; 56 unsigned long ret; 57 struct task_struct *p; 58 59 if (!futex_cmpxchg_enabled) 60 return -ENOSYS; 61 62 rcu_read_lock(); 63 64 ret = -ESRCH; 65 if (!pid) 66 p = current; 67 else { 68 p = find_task_by_vpid(pid); 69 if (!p) 70 goto err_unlock; 71 } 72 73 ret = -EPERM; 74 if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) 75 goto err_unlock; 76 77 head = p->robust_list; 78 rcu_read_unlock(); 79 80 if (put_user(sizeof(*head), len_ptr)) 81 return -EFAULT; 82 return put_user(head, head_ptr); 83 84 err_unlock: 85 rcu_read_unlock(); 86 87 return ret; 88 } 89 90 long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, 91 u32 __user *uaddr2, u32 val2, u32 val3) 92 { 93 int cmd = op & FUTEX_CMD_MASK; 94 unsigned int flags = 0; 95 96 if (!(op & FUTEX_PRIVATE_FLAG)) 97 flags |= FLAGS_SHARED; 98 99 if (op & FUTEX_CLOCK_REALTIME) { 100 flags |= FLAGS_CLOCKRT; 101 if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI && 102 cmd != FUTEX_LOCK_PI2) 103 return -ENOSYS; 104 } 105 106 switch (cmd) { 107 case FUTEX_LOCK_PI: 108 case FUTEX_LOCK_PI2: 109 case FUTEX_UNLOCK_PI: 110 case FUTEX_TRYLOCK_PI: 111 case FUTEX_WAIT_REQUEUE_PI: 112 case FUTEX_CMP_REQUEUE_PI: 113 if (!futex_cmpxchg_enabled) 114 return -ENOSYS; 115 } 116 117 switch (cmd) { 118 case FUTEX_WAIT: 119 val3 = FUTEX_BITSET_MATCH_ANY; 120 fallthrough; 121 case FUTEX_WAIT_BITSET: 122 return futex_wait(uaddr, flags, val, timeout, val3); 123 case FUTEX_WAKE: 124 val3 = FUTEX_BITSET_MATCH_ANY; 125 fallthrough; 126 case FUTEX_WAKE_BITSET: 127 return futex_wake(uaddr, flags, val, val3); 128 case FUTEX_REQUEUE: 129 return futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0); 130 case FUTEX_CMP_REQUEUE: 131 return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0); 132 case FUTEX_WAKE_OP: 133 return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3); 134 case FUTEX_LOCK_PI: 135 flags |= FLAGS_CLOCKRT; 136 fallthrough; 137 case FUTEX_LOCK_PI2: 138 return futex_lock_pi(uaddr, flags, timeout, 0); 139 case FUTEX_UNLOCK_PI: 140 return futex_unlock_pi(uaddr, flags); 141 case FUTEX_TRYLOCK_PI: 142 return futex_lock_pi(uaddr, flags, NULL, 1); 143 case FUTEX_WAIT_REQUEUE_PI: 144 val3 = FUTEX_BITSET_MATCH_ANY; 145 return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3, 146 uaddr2); 147 case FUTEX_CMP_REQUEUE_PI: 148 return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1); 149 } 150 return -ENOSYS; 151 } 152 153 static __always_inline bool futex_cmd_has_timeout(u32 cmd) 154 { 155 switch (cmd) { 156 case FUTEX_WAIT: 157 case FUTEX_LOCK_PI: 158 case FUTEX_LOCK_PI2: 159 case FUTEX_WAIT_BITSET: 160 case FUTEX_WAIT_REQUEUE_PI: 161 return true; 162 } 163 return false; 164 } 165 166 static __always_inline int 167 futex_init_timeout(u32 cmd, u32 op, struct timespec64 *ts, ktime_t *t) 168 { 169 if (!timespec64_valid(ts)) 170 return -EINVAL; 171 172 *t = timespec64_to_ktime(*ts); 173 if (cmd == FUTEX_WAIT) 174 *t = ktime_add_safe(ktime_get(), *t); 175 else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME)) 176 *t = timens_ktime_to_host(CLOCK_MONOTONIC, *t); 177 return 0; 178 } 179 180 SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, 181 const struct __kernel_timespec __user *, utime, 182 u32 __user *, uaddr2, u32, val3) 183 { 184 int ret, cmd = op & FUTEX_CMD_MASK; 185 ktime_t t, *tp = NULL; 186 struct timespec64 ts; 187 188 if (utime && futex_cmd_has_timeout(cmd)) { 189 if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG)))) 190 return -EFAULT; 191 if (get_timespec64(&ts, utime)) 192 return -EFAULT; 193 ret = futex_init_timeout(cmd, op, &ts, &t); 194 if (ret) 195 return ret; 196 tp = &t; 197 } 198 199 return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); 200 } 201 202 /* Mask of available flags for each futex in futex_waitv list */ 203 #define FUTEXV_WAITER_MASK (FUTEX_32 | FUTEX_PRIVATE_FLAG) 204 205 /** 206 * futex_parse_waitv - Parse a waitv array from userspace 207 * @futexv: Kernel side list of waiters to be filled 208 * @uwaitv: Userspace list to be parsed 209 * @nr_futexes: Length of futexv 210 * 211 * Return: Error code on failure, 0 on success 212 */ 213 static int futex_parse_waitv(struct futex_vector *futexv, 214 struct futex_waitv __user *uwaitv, 215 unsigned int nr_futexes) 216 { 217 struct futex_waitv aux; 218 unsigned int i; 219 220 for (i = 0; i < nr_futexes; i++) { 221 if (copy_from_user(&aux, &uwaitv[i], sizeof(aux))) 222 return -EFAULT; 223 224 if ((aux.flags & ~FUTEXV_WAITER_MASK) || aux.__reserved) 225 return -EINVAL; 226 227 if (!(aux.flags & FUTEX_32)) 228 return -EINVAL; 229 230 futexv[i].w.flags = aux.flags; 231 futexv[i].w.val = aux.val; 232 futexv[i].w.uaddr = aux.uaddr; 233 futexv[i].q = futex_q_init; 234 } 235 236 return 0; 237 } 238 239 /** 240 * sys_futex_waitv - Wait on a list of futexes 241 * @waiters: List of futexes to wait on 242 * @nr_futexes: Length of futexv 243 * @flags: Flag for timeout (monotonic/realtime) 244 * @timeout: Optional absolute timeout. 245 * @clockid: Clock to be used for the timeout, realtime or monotonic. 246 * 247 * Given an array of `struct futex_waitv`, wait on each uaddr. The thread wakes 248 * if a futex_wake() is performed at any uaddr. The syscall returns immediately 249 * if any waiter has *uaddr != val. *timeout is an optional timeout value for 250 * the operation. Each waiter has individual flags. The `flags` argument for 251 * the syscall should be used solely for specifying the timeout as realtime, if 252 * needed. Flags for private futexes, sizes, etc. should be used on the 253 * individual flags of each waiter. 254 * 255 * Returns the array index of one of the woken futexes. No further information 256 * is provided: any number of other futexes may also have been woken by the 257 * same event, and if more than one futex was woken, the retrned index may 258 * refer to any one of them. (It is not necessaryily the futex with the 259 * smallest index, nor the one most recently woken, nor...) 260 */ 261 262 SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, 263 unsigned int, nr_futexes, unsigned int, flags, 264 struct __kernel_timespec __user *, timeout, clockid_t, clockid) 265 { 266 struct hrtimer_sleeper to; 267 struct futex_vector *futexv; 268 struct timespec64 ts; 269 ktime_t time; 270 int ret; 271 272 /* This syscall supports no flags for now */ 273 if (flags) 274 return -EINVAL; 275 276 if (!nr_futexes || nr_futexes > FUTEX_WAITV_MAX || !waiters) 277 return -EINVAL; 278 279 if (timeout) { 280 int flag_clkid = 0, flag_init = 0; 281 282 if (clockid == CLOCK_REALTIME) { 283 flag_clkid = FLAGS_CLOCKRT; 284 flag_init = FUTEX_CLOCK_REALTIME; 285 } 286 287 if (clockid != CLOCK_REALTIME && clockid != CLOCK_MONOTONIC) 288 return -EINVAL; 289 290 if (get_timespec64(&ts, timeout)) 291 return -EFAULT; 292 293 /* 294 * Since there's no opcode for futex_waitv, use 295 * FUTEX_WAIT_BITSET that uses absolute timeout as well 296 */ 297 ret = futex_init_timeout(FUTEX_WAIT_BITSET, flag_init, &ts, &time); 298 if (ret) 299 return ret; 300 301 futex_setup_timer(&time, &to, flag_clkid, 0); 302 } 303 304 futexv = kcalloc(nr_futexes, sizeof(*futexv), GFP_KERNEL); 305 if (!futexv) 306 return -ENOMEM; 307 308 ret = futex_parse_waitv(futexv, waiters, nr_futexes); 309 if (!ret) 310 ret = futex_wait_multiple(futexv, nr_futexes, timeout ? &to : NULL); 311 312 if (timeout) { 313 hrtimer_cancel(&to.timer); 314 destroy_hrtimer_on_stack(&to.timer); 315 } 316 317 kfree(futexv); 318 return ret; 319 } 320 321 #ifdef CONFIG_COMPAT 322 COMPAT_SYSCALL_DEFINE2(set_robust_list, 323 struct compat_robust_list_head __user *, head, 324 compat_size_t, len) 325 { 326 if (!futex_cmpxchg_enabled) 327 return -ENOSYS; 328 329 if (unlikely(len != sizeof(*head))) 330 return -EINVAL; 331 332 current->compat_robust_list = head; 333 334 return 0; 335 } 336 337 COMPAT_SYSCALL_DEFINE3(get_robust_list, int, pid, 338 compat_uptr_t __user *, head_ptr, 339 compat_size_t __user *, len_ptr) 340 { 341 struct compat_robust_list_head __user *head; 342 unsigned long ret; 343 struct task_struct *p; 344 345 if (!futex_cmpxchg_enabled) 346 return -ENOSYS; 347 348 rcu_read_lock(); 349 350 ret = -ESRCH; 351 if (!pid) 352 p = current; 353 else { 354 p = find_task_by_vpid(pid); 355 if (!p) 356 goto err_unlock; 357 } 358 359 ret = -EPERM; 360 if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) 361 goto err_unlock; 362 363 head = p->compat_robust_list; 364 rcu_read_unlock(); 365 366 if (put_user(sizeof(*head), len_ptr)) 367 return -EFAULT; 368 return put_user(ptr_to_compat(head), head_ptr); 369 370 err_unlock: 371 rcu_read_unlock(); 372 373 return ret; 374 } 375 #endif /* CONFIG_COMPAT */ 376 377 #ifdef CONFIG_COMPAT_32BIT_TIME 378 SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, 379 const struct old_timespec32 __user *, utime, u32 __user *, uaddr2, 380 u32, val3) 381 { 382 int ret, cmd = op & FUTEX_CMD_MASK; 383 ktime_t t, *tp = NULL; 384 struct timespec64 ts; 385 386 if (utime && futex_cmd_has_timeout(cmd)) { 387 if (get_old_timespec32(&ts, utime)) 388 return -EFAULT; 389 ret = futex_init_timeout(cmd, op, &ts, &t); 390 if (ret) 391 return ret; 392 tp = &t; 393 } 394 395 return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); 396 } 397 #endif /* CONFIG_COMPAT_32BIT_TIME */ 398 399