1 /* 2 * The implementation of the wait_bit*() and related waiting APIs: 3 */ 4 #include "sched.h" 5 6 #define WAIT_TABLE_BITS 8 7 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS) 8 9 static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned; 10 11 wait_queue_head_t *bit_waitqueue(void *word, int bit) 12 { 13 const int shift = BITS_PER_LONG == 32 ? 5 : 6; 14 unsigned long val = (unsigned long)word << shift | bit; 15 16 return bit_wait_table + hash_long(val, WAIT_TABLE_BITS); 17 } 18 EXPORT_SYMBOL(bit_waitqueue); 19 20 int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg) 21 { 22 struct wait_bit_key *key = arg; 23 struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); 24 25 if (wait_bit->key.flags != key->flags || 26 wait_bit->key.bit_nr != key->bit_nr || 27 test_bit(key->bit_nr, key->flags)) 28 return 0; 29 30 return autoremove_wake_function(wq_entry, mode, sync, key); 31 } 32 EXPORT_SYMBOL(wake_bit_function); 33 34 /* 35 * To allow interruptible waiting and asynchronous (i.e. nonblocking) 36 * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are 37 * permitted return codes. Nonzero return codes halt waiting and return. 38 */ 39 int __sched 40 __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, 41 wait_bit_action_f *action, unsigned mode) 42 { 43 int ret = 0; 44 45 do { 46 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode); 47 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) 48 ret = (*action)(&wbq_entry->key, mode); 49 } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); 50 51 finish_wait(wq_head, &wbq_entry->wq_entry); 52 53 return ret; 54 } 55 EXPORT_SYMBOL(__wait_on_bit); 56 57 int __sched out_of_line_wait_on_bit(void *word, int bit, 58 wait_bit_action_f *action, unsigned mode) 59 { 60 struct wait_queue_head *wq_head = bit_waitqueue(word, bit); 61 DEFINE_WAIT_BIT(wq_entry, word, bit); 62 63 return __wait_on_bit(wq_head, &wq_entry, action, mode); 64 } 65 EXPORT_SYMBOL(out_of_line_wait_on_bit); 66 67 int __sched out_of_line_wait_on_bit_timeout( 68 void *word, int bit, wait_bit_action_f *action, 69 unsigned mode, unsigned long timeout) 70 { 71 struct wait_queue_head *wq_head = bit_waitqueue(word, bit); 72 DEFINE_WAIT_BIT(wq_entry, word, bit); 73 74 wq_entry.key.timeout = jiffies + timeout; 75 76 return __wait_on_bit(wq_head, &wq_entry, action, mode); 77 } 78 EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout); 79 80 int __sched 81 __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, 82 wait_bit_action_f *action, unsigned mode) 83 { 84 int ret = 0; 85 86 for (;;) { 87 prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode); 88 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) { 89 ret = action(&wbq_entry->key, mode); 90 /* 91 * See the comment in prepare_to_wait_event(). 92 * finish_wait() does not necessarily takes wwq_head->lock, 93 * but test_and_set_bit() implies mb() which pairs with 94 * smp_mb__after_atomic() before wake_up_page(). 95 */ 96 if (ret) 97 finish_wait(wq_head, &wbq_entry->wq_entry); 98 } 99 if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) { 100 if (!ret) 101 finish_wait(wq_head, &wbq_entry->wq_entry); 102 return 0; 103 } else if (ret) { 104 return ret; 105 } 106 } 107 } 108 EXPORT_SYMBOL(__wait_on_bit_lock); 109 110 int __sched out_of_line_wait_on_bit_lock(void *word, int bit, 111 wait_bit_action_f *action, unsigned mode) 112 { 113 struct wait_queue_head *wq_head = bit_waitqueue(word, bit); 114 DEFINE_WAIT_BIT(wq_entry, word, bit); 115 116 return __wait_on_bit_lock(wq_head, &wq_entry, action, mode); 117 } 118 EXPORT_SYMBOL(out_of_line_wait_on_bit_lock); 119 120 void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit) 121 { 122 struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit); 123 124 if (waitqueue_active(wq_head)) 125 __wake_up(wq_head, TASK_NORMAL, 1, &key); 126 } 127 EXPORT_SYMBOL(__wake_up_bit); 128 129 /** 130 * wake_up_bit - wake up a waiter on a bit 131 * @word: the word being waited on, a kernel virtual address 132 * @bit: the bit of the word being waited on 133 * 134 * There is a standard hashed waitqueue table for generic use. This 135 * is the part of the hashtable's accessor API that wakes up waiters 136 * on a bit. For instance, if one were to have waiters on a bitflag, 137 * one would call wake_up_bit() after clearing the bit. 138 * 139 * In order for this to function properly, as it uses waitqueue_active() 140 * internally, some kind of memory barrier must be done prior to calling 141 * this. Typically, this will be smp_mb__after_atomic(), but in some 142 * cases where bitflags are manipulated non-atomically under a lock, one 143 * may need to use a less regular barrier, such fs/inode.c's smp_mb(), 144 * because spin_unlock() does not guarantee a memory barrier. 145 */ 146 void wake_up_bit(void *word, int bit) 147 { 148 __wake_up_bit(bit_waitqueue(word, bit), word, bit); 149 } 150 EXPORT_SYMBOL(wake_up_bit); 151 152 wait_queue_head_t *__var_waitqueue(void *p) 153 { 154 return bit_wait_table + hash_ptr(p, WAIT_TABLE_BITS); 155 } 156 EXPORT_SYMBOL(__var_waitqueue); 157 158 static int 159 var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, 160 int sync, void *arg) 161 { 162 struct wait_bit_key *key = arg; 163 struct wait_bit_queue_entry *wbq_entry = 164 container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); 165 166 if (wbq_entry->key.flags != key->flags || 167 wbq_entry->key.bit_nr != key->bit_nr) 168 return 0; 169 170 return autoremove_wake_function(wq_entry, mode, sync, key); 171 } 172 173 void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, int flags) 174 { 175 *wbq_entry = (struct wait_bit_queue_entry){ 176 .key = { 177 .flags = (var), 178 .bit_nr = -1, 179 }, 180 .wq_entry = { 181 .private = current, 182 .func = var_wake_function, 183 .entry = LIST_HEAD_INIT(wbq_entry->wq_entry.entry), 184 }, 185 }; 186 } 187 EXPORT_SYMBOL(init_wait_var_entry); 188 189 void wake_up_var(void *var) 190 { 191 __wake_up_bit(__var_waitqueue(var), var, -1); 192 } 193 EXPORT_SYMBOL(wake_up_var); 194 195 __sched int bit_wait(struct wait_bit_key *word, int mode) 196 { 197 schedule(); 198 if (signal_pending_state(mode, current)) 199 return -EINTR; 200 201 return 0; 202 } 203 EXPORT_SYMBOL(bit_wait); 204 205 __sched int bit_wait_io(struct wait_bit_key *word, int mode) 206 { 207 io_schedule(); 208 if (signal_pending_state(mode, current)) 209 return -EINTR; 210 211 return 0; 212 } 213 EXPORT_SYMBOL(bit_wait_io); 214 215 __sched int bit_wait_timeout(struct wait_bit_key *word, int mode) 216 { 217 unsigned long now = READ_ONCE(jiffies); 218 219 if (time_after_eq(now, word->timeout)) 220 return -EAGAIN; 221 schedule_timeout(word->timeout - now); 222 if (signal_pending_state(mode, current)) 223 return -EINTR; 224 225 return 0; 226 } 227 EXPORT_SYMBOL_GPL(bit_wait_timeout); 228 229 __sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode) 230 { 231 unsigned long now = READ_ONCE(jiffies); 232 233 if (time_after_eq(now, word->timeout)) 234 return -EAGAIN; 235 io_schedule_timeout(word->timeout - now); 236 if (signal_pending_state(mode, current)) 237 return -EINTR; 238 239 return 0; 240 } 241 EXPORT_SYMBOL_GPL(bit_wait_io_timeout); 242 243 void __init wait_bit_init(void) 244 { 245 int i; 246 247 for (i = 0; i < WAIT_TABLE_SIZE; i++) 248 init_waitqueue_head(bit_wait_table + i); 249 } 250