74 void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,  in requeue_futex()  argument
83 		plist_del(&q->list, &hb1->chain);  in requeue_futex()
86 		plist_add(&q->list, &hb2->chain);  in requeue_futex()
87 		q->lock_ptr = &hb2->lock;  in requeue_futex()
89 	q->key = *key2;  in requeue_futex()
92 static inline bool futex_requeue_pi_prepare(struct futex_q *q,  in futex_requeue_pi_prepare()  argument
102 	old = atomic_read_acquire(&q->requeue_state);  in futex_requeue_pi_prepare()
119 	} while (!atomic_try_cmpxchg(&q->requeue_state, &old, new));  in futex_requeue_pi_prepare()
121 	q->pi_state = pi_state;  in futex_requeue_pi_prepare()
125 static inline void futex_requeue_pi_complete(struct futex_q *q, int locked)  in futex_requeue_pi_complete()  argument
129 	old = atomic_read_acquire(&q->requeue_state);  in futex_requeue_pi_complete()
147 	} while (!atomic_try_cmpxchg(&q->requeue_state, &old, new));  in futex_requeue_pi_complete()
152 		rcuwait_wake_up(&q->requeue_wait);  in futex_requeue_pi_complete()
156 static inline int futex_requeue_pi_wakeup_sync(struct futex_q *q)  in futex_requeue_pi_wakeup_sync()  argument
160 	old = atomic_read_acquire(&q->requeue_state);  in futex_requeue_pi_wakeup_sync()
173 	} while (!atomic_try_cmpxchg(&q->requeue_state, &old, new));  in futex_requeue_pi_wakeup_sync()
178 		rcuwait_wait_event(&q->requeue_wait,  in futex_requeue_pi_wakeup_sync()
179 				   atomic_read(&q->requeue_state) != Q_REQUEUE_PI_WAIT,  in futex_requeue_pi_wakeup_sync()
182 		(void)atomic_cond_read_relaxed(&q->requeue_state, VAL != Q_REQUEUE_PI_WAIT);  in futex_requeue_pi_wakeup_sync()
191 	return atomic_read(&q->requeue_state);  in futex_requeue_pi_wakeup_sync()
223 void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,  in requeue_pi_wake_futex()  argument
226 	q->key = *key;  in requeue_pi_wake_futex()
228 	__futex_unqueue(q);  in requeue_pi_wake_futex()
230 	WARN_ON(!q->rt_waiter);  in requeue_pi_wake_futex()
231 	q->rt_waiter = NULL;  in requeue_pi_wake_futex()
233 	q->lock_ptr = &hb->lock;  in requeue_pi_wake_futex()
236 	futex_requeue_pi_complete(q, 1);  in requeue_pi_wake_futex()
237 	wake_up_state(q->task, TASK_NORMAL);  in requeue_pi_wake_futex()
692 				   struct futex_q *q,  in handle_early_requeue_pi_wakeup()  argument
704 	WARN_ON_ONCE(&hb->lock != q->lock_ptr);  in handle_early_requeue_pi_wakeup()
710 	plist_del(&q->list, &hb->chain);  in handle_early_requeue_pi_wakeup()
770 	struct futex_q q = futex_q_init;  in futex_wait_requeue_pi()  local
796 	q.bitset = bitset;  in futex_wait_requeue_pi()
797 	q.rt_waiter = &rt_waiter;  in futex_wait_requeue_pi()
798 	q.requeue_pi_key = &key2;  in futex_wait_requeue_pi()
804 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait_requeue_pi()
812 	if (futex_match(&q.key, &key2)) {  in futex_wait_requeue_pi()
819 	futex_wait_queue(hb, &q, to);  in futex_wait_requeue_pi()
821 	switch (futex_requeue_pi_wakeup_sync(&q)) {  in futex_wait_requeue_pi()
825 		ret = handle_early_requeue_pi_wakeup(hb, &q, to);  in futex_wait_requeue_pi()
831 		if (q.pi_state && (q.pi_state->owner != current)) {  in futex_wait_requeue_pi()
832 			spin_lock(q.lock_ptr);  in futex_wait_requeue_pi()
833 			ret = fixup_pi_owner(uaddr2, &q, true);  in futex_wait_requeue_pi()
838 			put_pi_state(q.pi_state);  in futex_wait_requeue_pi()
839 			spin_unlock(q.lock_ptr);  in futex_wait_requeue_pi()
850 		pi_mutex = &q.pi_state->pi_mutex;  in futex_wait_requeue_pi()
854 		spin_lock(q.lock_ptr);  in futex_wait_requeue_pi()
863 		res = fixup_pi_owner(uaddr2, &q, !ret);  in futex_wait_requeue_pi()
871 		futex_unqueue_pi(&q);  in futex_wait_requeue_pi()
872 		spin_unlock(q.lock_ptr);  in futex_wait_requeue_pi()