1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _FUTEX_H 3 #define _FUTEX_H 4 5 #include <linux/futex.h> 6 #include <linux/sched/wake_q.h> 7 8 #ifdef CONFIG_PREEMPT_RT 9 #include <linux/rcuwait.h> 10 #endif 11 12 #include <asm/futex.h> 13 14 /* 15 * Futex flags used to encode options to functions and preserve them across 16 * restarts. 17 */ 18 #ifdef CONFIG_MMU 19 # define FLAGS_SHARED 0x01 20 #else 21 /* 22 * NOMMU does not have per process address space. Let the compiler optimize 23 * code away. 24 */ 25 # define FLAGS_SHARED 0x00 26 #endif 27 #define FLAGS_CLOCKRT 0x02 28 #define FLAGS_HAS_TIMEOUT 0x04 29 30 #ifdef CONFIG_HAVE_FUTEX_CMPXCHG 31 #define futex_cmpxchg_enabled 1 32 #else 33 extern int __read_mostly futex_cmpxchg_enabled; 34 #endif 35 36 #ifdef CONFIG_FAIL_FUTEX 37 extern bool should_fail_futex(bool fshared); 38 #else 39 static inline bool should_fail_futex(bool fshared) 40 { 41 return false; 42 } 43 #endif 44 45 /* 46 * Hash buckets are shared by all the futex_keys that hash to the same 47 * location. Each key may have multiple futex_q structures, one for each task 48 * waiting on a futex. 49 */ 50 struct futex_hash_bucket { 51 atomic_t waiters; 52 spinlock_t lock; 53 struct plist_head chain; 54 } ____cacheline_aligned_in_smp; 55 56 /* 57 * Priority Inheritance state: 58 */ 59 struct futex_pi_state { 60 /* 61 * list of 'owned' pi_state instances - these have to be 62 * cleaned up in do_exit() if the task exits prematurely: 63 */ 64 struct list_head list; 65 66 /* 67 * The PI object: 68 */ 69 struct rt_mutex_base pi_mutex; 70 71 struct task_struct *owner; 72 refcount_t refcount; 73 74 union futex_key key; 75 } __randomize_layout; 76 77 /** 78 * struct futex_q - The hashed futex queue entry, one per waiting task 79 * @list: priority-sorted list of tasks waiting on this futex 80 * @task: the task waiting on the futex 81 * @lock_ptr: the hash bucket lock 82 * @key: the key the futex is hashed on 83 * @pi_state: optional priority inheritance state 84 * @rt_waiter: rt_waiter storage for use with requeue_pi 85 * @requeue_pi_key: the requeue_pi target futex key 86 * @bitset: bitset for the optional bitmasked wakeup 87 * @requeue_state: State field for futex_requeue_pi() 88 * @requeue_wait: RCU wait for futex_requeue_pi() (RT only) 89 * 90 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so 91 * we can wake only the relevant ones (hashed queues may be shared). 92 * 93 * A futex_q has a woken state, just like tasks have TASK_RUNNING. 94 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. 95 * The order of wakeup is always to make the first condition true, then 96 * the second. 97 * 98 * PI futexes are typically woken before they are removed from the hash list via 99 * the rt_mutex code. See futex_unqueue_pi(). 100 */ 101 struct futex_q { 102 struct plist_node list; 103 104 struct task_struct *task; 105 spinlock_t *lock_ptr; 106 union futex_key key; 107 struct futex_pi_state *pi_state; 108 struct rt_mutex_waiter *rt_waiter; 109 union futex_key *requeue_pi_key; 110 u32 bitset; 111 atomic_t requeue_state; 112 #ifdef CONFIG_PREEMPT_RT 113 struct rcuwait requeue_wait; 114 #endif 115 } __randomize_layout; 116 117 extern const struct futex_q futex_q_init; 118 119 enum futex_access { 120 FUTEX_READ, 121 FUTEX_WRITE 122 }; 123 124 extern int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key, 125 enum futex_access rw); 126 127 extern struct hrtimer_sleeper * 128 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, 129 int flags, u64 range_ns); 130 131 extern struct futex_hash_bucket *futex_hash(union futex_key *key); 132 133 /** 134 * futex_match - Check whether two futex keys are equal 135 * @key1: Pointer to key1 136 * @key2: Pointer to key2 137 * 138 * Return 1 if two futex_keys are equal, 0 otherwise. 139 */ 140 static inline int futex_match(union futex_key *key1, union futex_key *key2) 141 { 142 return (key1 && key2 143 && key1->both.word == key2->both.word 144 && key1->both.ptr == key2->both.ptr 145 && key1->both.offset == key2->both.offset); 146 } 147 148 extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, 149 struct futex_q *q, struct futex_hash_bucket **hb); 150 extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, 151 struct hrtimer_sleeper *timeout); 152 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q); 153 154 extern int fault_in_user_writeable(u32 __user *uaddr); 155 extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval); 156 extern int futex_get_value_locked(u32 *dest, u32 __user *from); 157 extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key); 158 159 extern void __futex_unqueue(struct futex_q *q); 160 extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb); 161 extern int futex_unqueue(struct futex_q *q); 162 163 /** 164 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket 165 * @q: The futex_q to enqueue 166 * @hb: The destination hash bucket 167 * 168 * The hb->lock must be held by the caller, and is released here. A call to 169 * futex_queue() is typically paired with exactly one call to futex_unqueue(). The 170 * exceptions involve the PI related operations, which may use futex_unqueue_pi() 171 * or nothing if the unqueue is done as part of the wake process and the unqueue 172 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for 173 * an example). 174 */ 175 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb) 176 __releases(&hb->lock) 177 { 178 __futex_queue(q, hb); 179 spin_unlock(&hb->lock); 180 } 181 182 extern void futex_unqueue_pi(struct futex_q *q); 183 184 extern void wait_for_owner_exiting(int ret, struct task_struct *exiting); 185 186 /* 187 * Reflects a new waiter being added to the waitqueue. 188 */ 189 static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb) 190 { 191 #ifdef CONFIG_SMP 192 atomic_inc(&hb->waiters); 193 /* 194 * Full barrier (A), see the ordering comment above. 195 */ 196 smp_mb__after_atomic(); 197 #endif 198 } 199 200 /* 201 * Reflects a waiter being removed from the waitqueue by wakeup 202 * paths. 203 */ 204 static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb) 205 { 206 #ifdef CONFIG_SMP 207 atomic_dec(&hb->waiters); 208 #endif 209 } 210 211 static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb) 212 { 213 #ifdef CONFIG_SMP 214 /* 215 * Full barrier (B), see the ordering comment above. 216 */ 217 smp_mb(); 218 return atomic_read(&hb->waiters); 219 #else 220 return 1; 221 #endif 222 } 223 224 extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q); 225 extern void futex_q_unlock(struct futex_hash_bucket *hb); 226 227 228 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb, 229 union futex_key *key, 230 struct futex_pi_state **ps, 231 struct task_struct *task, 232 struct task_struct **exiting, 233 int set_waiters); 234 235 extern int refill_pi_state_cache(void); 236 extern void get_pi_state(struct futex_pi_state *pi_state); 237 extern void put_pi_state(struct futex_pi_state *pi_state); 238 extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked); 239 240 /* 241 * Express the locking dependencies for lockdep: 242 */ 243 static inline void 244 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 245 { 246 if (hb1 > hb2) 247 swap(hb1, hb2); 248 249 spin_lock(&hb1->lock); 250 if (hb1 != hb2) 251 spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING); 252 } 253 254 static inline void 255 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 256 { 257 spin_unlock(&hb1->lock); 258 if (hb1 != hb2) 259 spin_unlock(&hb2->lock); 260 } 261 262 /* syscalls */ 263 264 extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32 265 val, ktime_t *abs_time, u32 bitset, u32 __user 266 *uaddr2); 267 268 extern int futex_requeue(u32 __user *uaddr1, unsigned int flags, 269 u32 __user *uaddr2, int nr_wake, int nr_requeue, 270 u32 *cmpval, int requeue_pi); 271 272 extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, 273 ktime_t *abs_time, u32 bitset); 274 275 /** 276 * struct futex_vector - Auxiliary struct for futex_waitv() 277 * @w: Userspace provided data 278 * @q: Kernel side data 279 * 280 * Struct used to build an array with all data need for futex_waitv() 281 */ 282 struct futex_vector { 283 struct futex_waitv w; 284 struct futex_q q; 285 }; 286 287 extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count, 288 struct hrtimer_sleeper *to); 289 290 extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset); 291 292 extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags, 293 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op); 294 295 extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags); 296 297 extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock); 298 299 #endif /* _FUTEX_H */ 300