1af8cc960SPeter Zijlstra /* SPDX-License-Identifier: GPL-2.0 */ 2af8cc960SPeter Zijlstra #ifndef _FUTEX_H 3af8cc960SPeter Zijlstra #define _FUTEX_H 4af8cc960SPeter Zijlstra 585dc28faSPeter Zijlstra #include <linux/futex.h> 6*e5c68284SPeter Zijlstra #include <linux/sched/wake_q.h> 7*e5c68284SPeter Zijlstra 8af8cc960SPeter Zijlstra #include <asm/futex.h> 9af8cc960SPeter Zijlstra 10af8cc960SPeter Zijlstra /* 11af8cc960SPeter Zijlstra * Futex flags used to encode options to functions and preserve them across 12af8cc960SPeter Zijlstra * restarts. 13af8cc960SPeter Zijlstra */ 14af8cc960SPeter Zijlstra #ifdef CONFIG_MMU 15af8cc960SPeter Zijlstra # define FLAGS_SHARED 0x01 16af8cc960SPeter Zijlstra #else 17af8cc960SPeter Zijlstra /* 18af8cc960SPeter Zijlstra * NOMMU does not have per process address space. Let the compiler optimize 19af8cc960SPeter Zijlstra * code away. 20af8cc960SPeter Zijlstra */ 21af8cc960SPeter Zijlstra # define FLAGS_SHARED 0x00 22af8cc960SPeter Zijlstra #endif 23af8cc960SPeter Zijlstra #define FLAGS_CLOCKRT 0x02 24af8cc960SPeter Zijlstra #define FLAGS_HAS_TIMEOUT 0x04 25af8cc960SPeter Zijlstra 26af8cc960SPeter Zijlstra #ifdef CONFIG_HAVE_FUTEX_CMPXCHG 27af8cc960SPeter Zijlstra #define futex_cmpxchg_enabled 1 28af8cc960SPeter Zijlstra #else 29af8cc960SPeter Zijlstra extern int __read_mostly futex_cmpxchg_enabled; 30af8cc960SPeter Zijlstra #endif 31af8cc960SPeter Zijlstra 32af8cc960SPeter Zijlstra #ifdef CONFIG_FAIL_FUTEX 33af8cc960SPeter Zijlstra extern bool should_fail_futex(bool fshared); 34af8cc960SPeter Zijlstra #else 35af8cc960SPeter Zijlstra static inline bool should_fail_futex(bool fshared) 36af8cc960SPeter Zijlstra { 37af8cc960SPeter Zijlstra return false; 38af8cc960SPeter Zijlstra } 39af8cc960SPeter Zijlstra #endif 40af8cc960SPeter Zijlstra 4185dc28faSPeter Zijlstra /* 4285dc28faSPeter Zijlstra * Hash buckets are shared by all the futex_keys that hash to the same 4385dc28faSPeter Zijlstra * location. Each key may have multiple futex_q structures, one for each task 4485dc28faSPeter Zijlstra * waiting on a futex. 4585dc28faSPeter Zijlstra */ 4685dc28faSPeter Zijlstra struct futex_hash_bucket { 4785dc28faSPeter Zijlstra atomic_t waiters; 4885dc28faSPeter Zijlstra spinlock_t lock; 4985dc28faSPeter Zijlstra struct plist_head chain; 5085dc28faSPeter Zijlstra } ____cacheline_aligned_in_smp; 5185dc28faSPeter Zijlstra 5285dc28faSPeter Zijlstra /* 5385dc28faSPeter Zijlstra * Priority Inheritance state: 5485dc28faSPeter Zijlstra */ 5585dc28faSPeter Zijlstra struct futex_pi_state { 5685dc28faSPeter Zijlstra /* 5785dc28faSPeter Zijlstra * list of 'owned' pi_state instances - these have to be 5885dc28faSPeter Zijlstra * cleaned up in do_exit() if the task exits prematurely: 5985dc28faSPeter Zijlstra */ 6085dc28faSPeter Zijlstra struct list_head list; 6185dc28faSPeter Zijlstra 6285dc28faSPeter Zijlstra /* 6385dc28faSPeter Zijlstra * The PI object: 6485dc28faSPeter Zijlstra */ 6585dc28faSPeter Zijlstra struct rt_mutex_base pi_mutex; 6685dc28faSPeter Zijlstra 6785dc28faSPeter Zijlstra struct task_struct *owner; 6885dc28faSPeter Zijlstra refcount_t refcount; 6985dc28faSPeter Zijlstra 7085dc28faSPeter Zijlstra union futex_key key; 7185dc28faSPeter Zijlstra } __randomize_layout; 7285dc28faSPeter Zijlstra 7385dc28faSPeter Zijlstra /** 7485dc28faSPeter Zijlstra * struct futex_q - The hashed futex queue entry, one per waiting task 7585dc28faSPeter Zijlstra * @list: priority-sorted list of tasks waiting on this futex 7685dc28faSPeter Zijlstra * @task: the task waiting on the futex 7785dc28faSPeter Zijlstra * @lock_ptr: the hash bucket lock 7885dc28faSPeter Zijlstra * @key: the key the futex is hashed on 7985dc28faSPeter Zijlstra * @pi_state: optional priority inheritance state 8085dc28faSPeter Zijlstra * @rt_waiter: rt_waiter storage for use with requeue_pi 8185dc28faSPeter Zijlstra * @requeue_pi_key: the requeue_pi target futex key 8285dc28faSPeter Zijlstra * @bitset: bitset for the optional bitmasked wakeup 8385dc28faSPeter Zijlstra * @requeue_state: State field for futex_requeue_pi() 8485dc28faSPeter Zijlstra * @requeue_wait: RCU wait for futex_requeue_pi() (RT only) 8585dc28faSPeter Zijlstra * 8685dc28faSPeter Zijlstra * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so 8785dc28faSPeter Zijlstra * we can wake only the relevant ones (hashed queues may be shared). 8885dc28faSPeter Zijlstra * 8985dc28faSPeter Zijlstra * A futex_q has a woken state, just like tasks have TASK_RUNNING. 9085dc28faSPeter Zijlstra * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. 9185dc28faSPeter Zijlstra * The order of wakeup is always to make the first condition true, then 9285dc28faSPeter Zijlstra * the second. 9385dc28faSPeter Zijlstra * 9485dc28faSPeter Zijlstra * PI futexes are typically woken before they are removed from the hash list via 9585dc28faSPeter Zijlstra * the rt_mutex code. See futex_unqueue_pi(). 9685dc28faSPeter Zijlstra */ 9785dc28faSPeter Zijlstra struct futex_q { 9885dc28faSPeter Zijlstra struct plist_node list; 9985dc28faSPeter Zijlstra 10085dc28faSPeter Zijlstra struct task_struct *task; 10185dc28faSPeter Zijlstra spinlock_t *lock_ptr; 10285dc28faSPeter Zijlstra union futex_key key; 10385dc28faSPeter Zijlstra struct futex_pi_state *pi_state; 10485dc28faSPeter Zijlstra struct rt_mutex_waiter *rt_waiter; 10585dc28faSPeter Zijlstra union futex_key *requeue_pi_key; 10685dc28faSPeter Zijlstra u32 bitset; 10785dc28faSPeter Zijlstra atomic_t requeue_state; 10885dc28faSPeter Zijlstra #ifdef CONFIG_PREEMPT_RT 10985dc28faSPeter Zijlstra struct rcuwait requeue_wait; 11085dc28faSPeter Zijlstra #endif 11185dc28faSPeter Zijlstra } __randomize_layout; 11285dc28faSPeter Zijlstra 11385dc28faSPeter Zijlstra extern const struct futex_q futex_q_init; 11485dc28faSPeter Zijlstra 11585dc28faSPeter Zijlstra enum futex_access { 11685dc28faSPeter Zijlstra FUTEX_READ, 11785dc28faSPeter Zijlstra FUTEX_WRITE 11885dc28faSPeter Zijlstra }; 11985dc28faSPeter Zijlstra 12085dc28faSPeter Zijlstra extern int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key, 12185dc28faSPeter Zijlstra enum futex_access rw); 12285dc28faSPeter Zijlstra 12385dc28faSPeter Zijlstra extern struct hrtimer_sleeper * 12485dc28faSPeter Zijlstra futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, 12585dc28faSPeter Zijlstra int flags, u64 range_ns); 12685dc28faSPeter Zijlstra 127*e5c68284SPeter Zijlstra extern struct futex_hash_bucket *futex_hash(union futex_key *key); 128*e5c68284SPeter Zijlstra 129*e5c68284SPeter Zijlstra /** 130*e5c68284SPeter Zijlstra * futex_match - Check whether two futex keys are equal 131*e5c68284SPeter Zijlstra * @key1: Pointer to key1 132*e5c68284SPeter Zijlstra * @key2: Pointer to key2 133*e5c68284SPeter Zijlstra * 134*e5c68284SPeter Zijlstra * Return 1 if two futex_keys are equal, 0 otherwise. 135*e5c68284SPeter Zijlstra */ 136*e5c68284SPeter Zijlstra static inline int futex_match(union futex_key *key1, union futex_key *key2) 137*e5c68284SPeter Zijlstra { 138*e5c68284SPeter Zijlstra return (key1 && key2 139*e5c68284SPeter Zijlstra && key1->both.word == key2->both.word 140*e5c68284SPeter Zijlstra && key1->both.ptr == key2->both.ptr 141*e5c68284SPeter Zijlstra && key1->both.offset == key2->both.offset); 142*e5c68284SPeter Zijlstra } 143*e5c68284SPeter Zijlstra 144*e5c68284SPeter Zijlstra extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, 145*e5c68284SPeter Zijlstra struct futex_q *q, struct futex_hash_bucket **hb); 146*e5c68284SPeter Zijlstra extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, 147*e5c68284SPeter Zijlstra struct hrtimer_sleeper *timeout); 148*e5c68284SPeter Zijlstra extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q); 149*e5c68284SPeter Zijlstra 15085dc28faSPeter Zijlstra extern int fault_in_user_writeable(u32 __user *uaddr); 15185dc28faSPeter Zijlstra extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval); 15285dc28faSPeter Zijlstra extern int futex_get_value_locked(u32 *dest, u32 __user *from); 15385dc28faSPeter Zijlstra extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key); 15485dc28faSPeter Zijlstra 155*e5c68284SPeter Zijlstra extern void __futex_unqueue(struct futex_q *q); 15685dc28faSPeter Zijlstra extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb); 15785dc28faSPeter Zijlstra extern void futex_unqueue_pi(struct futex_q *q); 15885dc28faSPeter Zijlstra 15985dc28faSPeter Zijlstra extern void wait_for_owner_exiting(int ret, struct task_struct *exiting); 16085dc28faSPeter Zijlstra 161*e5c68284SPeter Zijlstra /* 162*e5c68284SPeter Zijlstra * Reflects a new waiter being added to the waitqueue. 163*e5c68284SPeter Zijlstra */ 164*e5c68284SPeter Zijlstra static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb) 165*e5c68284SPeter Zijlstra { 166*e5c68284SPeter Zijlstra #ifdef CONFIG_SMP 167*e5c68284SPeter Zijlstra atomic_inc(&hb->waiters); 168*e5c68284SPeter Zijlstra /* 169*e5c68284SPeter Zijlstra * Full barrier (A), see the ordering comment above. 170*e5c68284SPeter Zijlstra */ 171*e5c68284SPeter Zijlstra smp_mb__after_atomic(); 172*e5c68284SPeter Zijlstra #endif 173*e5c68284SPeter Zijlstra } 174*e5c68284SPeter Zijlstra 175*e5c68284SPeter Zijlstra /* 176*e5c68284SPeter Zijlstra * Reflects a waiter being removed from the waitqueue by wakeup 177*e5c68284SPeter Zijlstra * paths. 178*e5c68284SPeter Zijlstra */ 179*e5c68284SPeter Zijlstra static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb) 180*e5c68284SPeter Zijlstra { 181*e5c68284SPeter Zijlstra #ifdef CONFIG_SMP 182*e5c68284SPeter Zijlstra atomic_dec(&hb->waiters); 183*e5c68284SPeter Zijlstra #endif 184*e5c68284SPeter Zijlstra } 185*e5c68284SPeter Zijlstra 18685dc28faSPeter Zijlstra extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q); 18785dc28faSPeter Zijlstra extern void futex_q_unlock(struct futex_hash_bucket *hb); 18885dc28faSPeter Zijlstra 18985dc28faSPeter Zijlstra 19085dc28faSPeter Zijlstra extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb, 19185dc28faSPeter Zijlstra union futex_key *key, 19285dc28faSPeter Zijlstra struct futex_pi_state **ps, 19385dc28faSPeter Zijlstra struct task_struct *task, 19485dc28faSPeter Zijlstra struct task_struct **exiting, 19585dc28faSPeter Zijlstra int set_waiters); 19685dc28faSPeter Zijlstra 19785dc28faSPeter Zijlstra extern int refill_pi_state_cache(void); 19885dc28faSPeter Zijlstra extern void get_pi_state(struct futex_pi_state *pi_state); 19985dc28faSPeter Zijlstra extern void put_pi_state(struct futex_pi_state *pi_state); 20085dc28faSPeter Zijlstra extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked); 20185dc28faSPeter Zijlstra 202*e5c68284SPeter Zijlstra /* 203*e5c68284SPeter Zijlstra * Express the locking dependencies for lockdep: 204*e5c68284SPeter Zijlstra */ 205*e5c68284SPeter Zijlstra static inline void 206*e5c68284SPeter Zijlstra double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 207*e5c68284SPeter Zijlstra { 208*e5c68284SPeter Zijlstra if (hb1 <= hb2) { 209*e5c68284SPeter Zijlstra spin_lock(&hb1->lock); 210*e5c68284SPeter Zijlstra if (hb1 < hb2) 211*e5c68284SPeter Zijlstra spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING); 212*e5c68284SPeter Zijlstra } else { /* hb1 > hb2 */ 213*e5c68284SPeter Zijlstra spin_lock(&hb2->lock); 214*e5c68284SPeter Zijlstra spin_lock_nested(&hb1->lock, SINGLE_DEPTH_NESTING); 215*e5c68284SPeter Zijlstra } 216*e5c68284SPeter Zijlstra } 217*e5c68284SPeter Zijlstra 218*e5c68284SPeter Zijlstra static inline void 219*e5c68284SPeter Zijlstra double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 220*e5c68284SPeter Zijlstra { 221*e5c68284SPeter Zijlstra spin_unlock(&hb1->lock); 222*e5c68284SPeter Zijlstra if (hb1 != hb2) 223*e5c68284SPeter Zijlstra spin_unlock(&hb2->lock); 224*e5c68284SPeter Zijlstra } 225*e5c68284SPeter Zijlstra 22685dc28faSPeter Zijlstra /* syscalls */ 22785dc28faSPeter Zijlstra 228af8cc960SPeter Zijlstra extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32 229af8cc960SPeter Zijlstra val, ktime_t *abs_time, u32 bitset, u32 __user 230af8cc960SPeter Zijlstra *uaddr2); 231af8cc960SPeter Zijlstra 232af8cc960SPeter Zijlstra extern int futex_requeue(u32 __user *uaddr1, unsigned int flags, 233af8cc960SPeter Zijlstra u32 __user *uaddr2, int nr_wake, int nr_requeue, 234af8cc960SPeter Zijlstra u32 *cmpval, int requeue_pi); 235af8cc960SPeter Zijlstra 236af8cc960SPeter Zijlstra extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, 237af8cc960SPeter Zijlstra ktime_t *abs_time, u32 bitset); 238af8cc960SPeter Zijlstra 239af8cc960SPeter Zijlstra extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset); 240af8cc960SPeter Zijlstra 241af8cc960SPeter Zijlstra extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags, 242af8cc960SPeter Zijlstra u32 __user *uaddr2, int nr_wake, int nr_wake2, int op); 243af8cc960SPeter Zijlstra 244af8cc960SPeter Zijlstra extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags); 245af8cc960SPeter Zijlstra 246af8cc960SPeter Zijlstra extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock); 247af8cc960SPeter Zijlstra 248af8cc960SPeter Zijlstra #endif /* _FUTEX_H */ 249