Lines Matching +full:ps +full:- +full:hold
1 // SPDX-License-Identifier: GPL-2.0-or-later
16 if (likely(current->pi_state_cache)) in refill_pi_state_cache()
22 return -ENOMEM; in refill_pi_state_cache()
24 INIT_LIST_HEAD(&pi_state->list); in refill_pi_state_cache()
26 pi_state->owner = NULL; in refill_pi_state_cache()
27 refcount_set(&pi_state->refcount, 1); in refill_pi_state_cache()
28 pi_state->key = FUTEX_KEY_INIT; in refill_pi_state_cache()
30 current->pi_state_cache = pi_state; in refill_pi_state_cache()
37 struct futex_pi_state *pi_state = current->pi_state_cache; in alloc_pi_state()
40 current->pi_state_cache = NULL; in alloc_pi_state()
48 struct task_struct *old_owner = pi_state->owner; in pi_state_update_owner()
50 lockdep_assert_held(&pi_state->pi_mutex.wait_lock); in pi_state_update_owner()
53 raw_spin_lock(&old_owner->pi_lock); in pi_state_update_owner()
54 WARN_ON(list_empty(&pi_state->list)); in pi_state_update_owner()
55 list_del_init(&pi_state->list); in pi_state_update_owner()
56 raw_spin_unlock(&old_owner->pi_lock); in pi_state_update_owner()
60 raw_spin_lock(&new_owner->pi_lock); in pi_state_update_owner()
61 WARN_ON(!list_empty(&pi_state->list)); in pi_state_update_owner()
62 list_add(&pi_state->list, &new_owner->pi_state_list); in pi_state_update_owner()
63 pi_state->owner = new_owner; in pi_state_update_owner()
64 raw_spin_unlock(&new_owner->pi_lock); in pi_state_update_owner()
70 WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount)); in get_pi_state()
82 if (!refcount_dec_and_test(&pi_state->refcount)) in put_pi_state()
86 * If pi_state->owner is NULL, the owner is most probably dying in put_pi_state()
89 if (pi_state->owner) { in put_pi_state()
92 raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags); in put_pi_state()
94 rt_mutex_proxy_unlock(&pi_state->pi_mutex); in put_pi_state()
95 raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags); in put_pi_state()
98 if (current->pi_state_cache) { in put_pi_state()
102 * pi_state->list is already empty. in put_pi_state()
103 * clear pi_state->owner. in put_pi_state()
104 * refcount is at 0 - put it back to 1. in put_pi_state()
106 pi_state->owner = NULL; in put_pi_state()
107 refcount_set(&pi_state->refcount, 1); in put_pi_state()
108 current->pi_state_cache = pi_state; in put_pi_state()
115 * Waiter | pi_state | pi->owner | uTID | uODIED | ?
117 * [1] NULL | --- | --- | 0 | 0/1 | Valid
118 * [2] NULL | --- | --- | >0 | 0/1 | Valid
120 * [3] Found | NULL | -- | Any | 0/1 | Invalid
150 * [7] pi_state->owner can only be NULL when the OWNER_DIED bit is set.
165 * hb->lock:
167 * hb -> futex_q, relation
168 * futex_q -> pi_state, relation
173 * pi_mutex->wait_lock:
179 * p->pi_lock:
181 * p->pi_state_list -> pi_state->list, relation
182 * pi_mutex->owner -> pi_state->owner, relation
184 * pi_state->refcount:
191 * hb->lock
192 * pi_mutex->wait_lock
193 * p->pi_lock
204 struct futex_pi_state **ps) in attach_to_pi_state() argument
211 * Userspace might have messed up non-PI and PI futexes [3] in attach_to_pi_state()
214 return -EINVAL; in attach_to_pi_state()
217 * We get here with hb->lock held, and having found a in attach_to_pi_state()
219 * has dropped the hb->lock in between futex_queue() and futex_unqueue_pi(), in attach_to_pi_state()
228 WARN_ON(!refcount_read(&pi_state->refcount)); in attach_to_pi_state()
234 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
255 * pi_state->rt_mutex will fixup owner. in attach_to_pi_state()
257 if (!pi_state->owner) { in attach_to_pi_state()
285 if (!pi_state->owner) in attach_to_pi_state()
294 if (pid != task_pid_vnr(pi_state->owner)) in attach_to_pi_state()
299 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
300 *ps = pi_state; in attach_to_pi_state()
304 ret = -EINVAL; in attach_to_pi_state()
308 ret = -EAGAIN; in attach_to_pi_state()
312 ret = -EFAULT; in attach_to_pi_state()
316 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
329 if (tsk && tsk->futex_state != FUTEX_STATE_DEAD) in handle_exit_race()
330 return -EBUSY; in handle_exit_race()
341 * tsk->flags |= PF_EXITING; *uaddr == 0x00000PID in handle_exit_race()
346 * } if (!tsk->flags & PF_EXITING) { in handle_exit_race()
348 * tsk->futex_state = } else { in handle_exit_race()
349 * FUTEX_STATE_DEAD; if (tsk->futex_state != in handle_exit_race()
351 * return -EAGAIN; in handle_exit_race()
352 * return -ESRCH; <--- FAIL in handle_exit_race()
362 return -EFAULT; in handle_exit_race()
366 return -EAGAIN; in handle_exit_race()
373 return -ESRCH; in handle_exit_race()
377 struct futex_pi_state **ps) in __attach_to_pi_owner() argument
382 * This creates pi_state, we have hb->lock held, this means nothing can in __attach_to_pi_owner()
391 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p); in __attach_to_pi_owner()
394 pi_state->key = *key; in __attach_to_pi_owner()
396 WARN_ON(!list_empty(&pi_state->list)); in __attach_to_pi_owner()
397 list_add(&pi_state->list, &p->pi_state_list); in __attach_to_pi_owner()
399 * Assignment without holding pi_state->pi_mutex.wait_lock is safe in __attach_to_pi_owner()
402 pi_state->owner = p; in __attach_to_pi_owner()
404 *ps = pi_state; in __attach_to_pi_owner()
411 struct futex_pi_state **ps, in attach_to_pi_owner() argument
418 * We are the first waiter - try to look up the real owner and attach in attach_to_pi_owner()
425 return -EAGAIN; in attach_to_pi_owner()
430 if (unlikely(p->flags & PF_KTHREAD)) { in attach_to_pi_owner()
432 return -EPERM; in attach_to_pi_owner()
438 * in futex_exit_release(), we do this protected by p->pi_lock: in attach_to_pi_owner()
440 raw_spin_lock_irq(&p->pi_lock); in attach_to_pi_owner()
441 if (unlikely(p->futex_state != FUTEX_STATE_OK)) { in attach_to_pi_owner()
449 raw_spin_unlock_irq(&p->pi_lock); in attach_to_pi_owner()
459 if (ret == -EBUSY) in attach_to_pi_owner()
466 __attach_to_pi_owner(p, key, ps); in attach_to_pi_owner()
467 raw_spin_unlock_irq(&p->pi_lock); in attach_to_pi_owner()
480 return -EFAULT; in lock_pi_update_atomic()
487 return curval != uval ? -EAGAIN : 0; in lock_pi_update_atomic()
491 * futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex
495 * @ps: the pi_state pointer where we store the result of the
504 * - 0 - ready to wait;
505 * - 1 - acquired the lock;
506 * - <0 - error
508 * The hb->lock must be held by the caller.
510 * @exiting is only set when the return value is -EBUSY. If so, this holds
516 struct futex_pi_state **ps, in futex_lock_pi_atomic() argument
530 return -EFAULT; in futex_lock_pi_atomic()
533 return -EFAULT; in futex_lock_pi_atomic()
539 return -EDEADLK; in futex_lock_pi_atomic()
542 return -EDEADLK; in futex_lock_pi_atomic()
550 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps); in futex_lock_pi_atomic()
586 raw_spin_lock_irq(&task->pi_lock); in futex_lock_pi_atomic()
587 __attach_to_pi_owner(task, key, ps); in futex_lock_pi_atomic()
588 raw_spin_unlock_irq(&task->pi_lock); in futex_lock_pi_atomic()
596 * the kernel and blocked on hb->lock. in futex_lock_pi_atomic()
607 return attach_to_pi_owner(uaddr, newval, key, ps, exiting); in futex_lock_pi_atomic()
611 * Caller must hold a reference on @pi_state.
622 top_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex); in wake_futex_pi()
632 ret = -EAGAIN; in wake_futex_pi()
636 new_owner = top_waiter->task; in wake_futex_pi()
646 ret = -EFAULT; in wake_futex_pi()
654 * try the TID->0 transition) raced with a waiter setting the in wake_futex_pi()
659 ret = -EAGAIN; in wake_futex_pi()
661 ret = -EINVAL; in wake_futex_pi()
671 postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wqh); in wake_futex_pi()
675 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in wake_futex_pi()
686 struct futex_pi_state *pi_state = q->pi_state; in __fixup_pi_state_owner()
691 oldowner = pi_state->owner; in __fixup_pi_state_owner()
696 * - we stole the lock and pi_state->owner needs updating to reflect in __fixup_pi_state_owner()
701 * - someone stole our lock and we need to fix things to point to the in __fixup_pi_state_owner()
726 if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) { in __fixup_pi_state_owner()
735 newowner = rt_mutex_owner(&pi_state->pi_mutex); in __fixup_pi_state_owner()
745 err = -EAGAIN; in __fixup_pi_state_owner()
762 if (!pi_state->owner) in __fixup_pi_state_owner()
798 * Note: we hold both hb->lock and pi_mutex->wait_lock. We can safely in __fixup_pi_state_owner()
799 * drop hb->lock since the caller owns the hb -> futex_q relation. in __fixup_pi_state_owner()
800 * Dropping the pi_mutex->wait_lock requires the state revalidate. in __fixup_pi_state_owner()
803 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in __fixup_pi_state_owner()
804 spin_unlock(q->lock_ptr); in __fixup_pi_state_owner()
807 case -EFAULT: in __fixup_pi_state_owner()
811 case -EAGAIN: in __fixup_pi_state_owner()
821 spin_lock(q->lock_ptr); in __fixup_pi_state_owner()
822 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in __fixup_pi_state_owner()
827 if (pi_state->owner != oldowner) in __fixup_pi_state_owner()
830 /* Retry if err was -EAGAIN or the fault in succeeded */ in __fixup_pi_state_owner()
847 * The rtmutex has an owner - either current or some other in __fixup_pi_state_owner()
850 pi_state_update_owner(pi_state, rt_mutex_owner(&pi_state->pi_mutex)); in __fixup_pi_state_owner()
858 struct futex_pi_state *pi_state = q->pi_state; in fixup_pi_state_owner()
861 lockdep_assert_held(q->lock_ptr); in fixup_pi_state_owner()
863 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in fixup_pi_state_owner()
865 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in fixup_pi_state_owner()
870 * fixup_pi_owner() - Post lock pi_state and corner case management
880 * - 1 - success, lock taken;
881 * - 0 - success, lock not taken;
882 * - <0 - on error (-EFAULT)
889 * did a lock-steal - fix up the PI-state in that case: in fixup_pi_owner()
891 * Speculative pi_state->owner read (we don't hold wait_lock); in fixup_pi_owner()
892 * since we own the lock pi_state->owner == current is the in fixup_pi_owner()
895 if (q->pi_state->owner != current) in fixup_pi_owner()
905 * Another speculative read; pi_state->owner == current is unstable in fixup_pi_owner()
908 if (q->pi_state->owner == current) in fixup_pi_owner()
915 if (WARN_ON_ONCE(rt_mutex_owner(&q->pi_state->pi_mutex) == current)) in fixup_pi_owner()
922 * Userspace tried a 0 -> TID atomic transition of the futex value
925 * on rt-mutexes, it does PI, etc. (Due to races the kernel might see
940 return -ENOSYS; in futex_lock_pi()
943 return -ENOMEM; in futex_lock_pi()
967 case -EFAULT: in futex_lock_pi()
969 case -EBUSY: in futex_lock_pi()
970 case -EAGAIN: in futex_lock_pi()
973 * - EBUSY: Task is exiting and we just wait for the in futex_lock_pi()
975 * - EAGAIN: The user space value changed. in futex_lock_pi()
999 ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex); in futex_lock_pi()
1001 ret = ret ? 0 : -EWOULDBLOCK; in futex_lock_pi()
1008 * On PREEMPT_RT, when hb->lock becomes an rt_mutex, we must not in futex_lock_pi()
1009 * hold it while doing rt_mutex_start_proxy(), because then it will in futex_lock_pi()
1010 * include hb->lock in the blocking chain, even through we'll not in in futex_lock_pi()
1011 * fact hold it while blocking. This will lead it to report -EDEADLK in futex_lock_pi()
1014 * Therefore acquire wait_lock while holding hb->lock, but drop the in futex_lock_pi()
1016 * interleaves with futex_unlock_pi() -- which does a similar lock in futex_lock_pi()
1017 * handoff -- such that the latter can observe the futex_q::pi_state in futex_lock_pi()
1020 raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock); in futex_lock_pi()
1027 ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current); in futex_lock_pi()
1028 raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock); in futex_lock_pi()
1039 ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter); in futex_lock_pi()
1045 * first acquire the hb->lock before removing the lock from the in futex_lock_pi()
1052 if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter)) in futex_lock_pi()
1063 * the lock, clear our -ETIMEDOUT or -EINTR. in futex_lock_pi()
1077 hrtimer_cancel(&to->timer); in futex_lock_pi()
1078 destroy_hrtimer_on_stack(&to->timer); in futex_lock_pi()
1080 return ret != -EINTR ? ret : -ERESTARTNOINTR; in futex_lock_pi()
1096 * Userspace attempted a TID -> 0 atomic transition, and failed.
1097 * This is the in-kernel slowpath: we look up the PI state (if any),
1098 * and do the rt-mutex unlock.
1109 return -ENOSYS; in futex_unlock_pi()
1113 return -EFAULT; in futex_unlock_pi()
1118 return -EPERM; in futex_unlock_pi()
1125 spin_lock(&hb->lock); in futex_unlock_pi()
1134 struct futex_pi_state *pi_state = top_waiter->pi_state; in futex_unlock_pi()
1136 ret = -EINVAL; in futex_unlock_pi()
1144 if (pi_state->owner != current) in futex_unlock_pi()
1149 * By taking wait_lock while still holding hb->lock, we ensure in futex_unlock_pi()
1150 * there is no point where we hold neither; and therefore in futex_unlock_pi()
1158 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in futex_unlock_pi()
1159 spin_unlock(&hb->lock); in futex_unlock_pi()
1161 /* drops pi_state->pi_mutex.wait_lock */ in futex_unlock_pi()
1173 * pagefault, so retry the user-access and the wakeup: in futex_unlock_pi()
1175 if (ret == -EFAULT) in futex_unlock_pi()
1181 if (ret == -EAGAIN) in futex_unlock_pi()
1193 * on hb->lock. So we can safely ignore them. We do neither in futex_unlock_pi()
1198 spin_unlock(&hb->lock); in futex_unlock_pi()
1200 case -EFAULT: in futex_unlock_pi()
1203 case -EAGAIN: in futex_unlock_pi()
1215 ret = (curval == uval) ? 0 : -EAGAIN; in futex_unlock_pi()
1218 spin_unlock(&hb->lock); in futex_unlock_pi()