42 /* *val is the current value of lockcnt->count.
44  * If the lock is free, try a cmpxchg from *val to new_if_free; return
45  * true and set *val to the old value found by the cmpxchg in
49  * *without trying again to take the lock*.  Again, set *val to the
58 static bool qemu_lockcnt_cmpxchg_or_wait(QemuLockCnt *lockcnt, int *val,  in qemu_lockcnt_cmpxchg_or_wait()  argument
62     if ((*val & QEMU_LOCKCNT_STATE_MASK) == QEMU_LOCKCNT_STATE_FREE) {  in qemu_lockcnt_cmpxchg_or_wait()
63         int expected = *val;  in qemu_lockcnt_cmpxchg_or_wait()
66         *val = qatomic_cmpxchg(&lockcnt->count, expected, new_if_free);  in qemu_lockcnt_cmpxchg_or_wait()
67         if (*val == expected) {  in qemu_lockcnt_cmpxchg_or_wait()
69             *val = new_if_free;  in qemu_lockcnt_cmpxchg_or_wait()
80     while ((*val & QEMU_LOCKCNT_STATE_MASK) != QEMU_LOCKCNT_STATE_FREE) {  in qemu_lockcnt_cmpxchg_or_wait()
81         if ((*val & QEMU_LOCKCNT_STATE_MASK) == QEMU_LOCKCNT_STATE_LOCKED) {  in qemu_lockcnt_cmpxchg_or_wait()
82             int expected = *val;  in qemu_lockcnt_cmpxchg_or_wait()
86             *val = qatomic_cmpxchg(&lockcnt->count, expected, new);  in qemu_lockcnt_cmpxchg_or_wait()
87             if (*val == expected) {  in qemu_lockcnt_cmpxchg_or_wait()
88                 *val = new;  in qemu_lockcnt_cmpxchg_or_wait()
93         if ((*val & QEMU_LOCKCNT_STATE_MASK) == QEMU_LOCKCNT_STATE_WAITING) {  in qemu_lockcnt_cmpxchg_or_wait()
95             trace_lockcnt_futex_wait(lockcnt, *val);  in qemu_lockcnt_cmpxchg_or_wait()
96             qemu_futex_wait(&lockcnt->count, *val);  in qemu_lockcnt_cmpxchg_or_wait()
97             *val = qatomic_read(&lockcnt->count);  in qemu_lockcnt_cmpxchg_or_wait()
98             trace_lockcnt_futex_wait_resume(lockcnt, *val);  in qemu_lockcnt_cmpxchg_or_wait()
115     int val = qatomic_read(&lockcnt->count);  in qemu_lockcnt_inc()  local
119         if (val >= QEMU_LOCKCNT_COUNT_STEP) {  in qemu_lockcnt_inc()
120             int expected = val;  in qemu_lockcnt_inc()
121             val = qatomic_cmpxchg(&lockcnt->count, val,  in qemu_lockcnt_inc()
122                                   val + QEMU_LOCKCNT_COUNT_STEP);  in qemu_lockcnt_inc()
123             if (val == expected) {  in qemu_lockcnt_inc()
128             if (qemu_lockcnt_cmpxchg_or_wait(lockcnt, &val, QEMU_LOCKCNT_COUNT_STEP,  in qemu_lockcnt_inc()
157     int val = qatomic_read(&lockcnt->count);  in qemu_lockcnt_dec_and_lock()  local
162         if (val >= 2 * QEMU_LOCKCNT_COUNT_STEP) {  in qemu_lockcnt_dec_and_lock()
163             int expected = val;  in qemu_lockcnt_dec_and_lock()
164             val = qatomic_cmpxchg(&lockcnt->count, val,  in qemu_lockcnt_dec_and_lock()
165                                   val - QEMU_LOCKCNT_COUNT_STEP);  in qemu_lockcnt_dec_and_lock()
166             if (val == expected) {  in qemu_lockcnt_dec_and_lock()
173             if (qemu_lockcnt_cmpxchg_or_wait(lockcnt, &val, locked_state, &waited)) {  in qemu_lockcnt_dec_and_lock()
206     int val = qatomic_read(&lockcnt->count);  in qemu_lockcnt_dec_if_lock()  local
210     while (val < 2 * QEMU_LOCKCNT_COUNT_STEP) {  in qemu_lockcnt_dec_if_lock()
214         if (qemu_lockcnt_cmpxchg_or_wait(lockcnt, &val, locked_state, &waited)) {  in qemu_lockcnt_dec_if_lock()
240     int val = qatomic_read(&lockcnt->count);  in qemu_lockcnt_lock()  local
244     /* The third argument is only used if the low bits of val are 0  in qemu_lockcnt_lock()
248     while (!qemu_lockcnt_cmpxchg_or_wait(lockcnt, &val, val + step, &waited)) {  in qemu_lockcnt_lock()
260     int expected, new, val;  in qemu_lockcnt_inc_and_unlock()  local
262     val = qatomic_read(&lockcnt->count);  in qemu_lockcnt_inc_and_unlock()
264         expected = val;  in qemu_lockcnt_inc_and_unlock()
265         new = (val + QEMU_LOCKCNT_COUNT_STEP) & ~QEMU_LOCKCNT_STATE_MASK;  in qemu_lockcnt_inc_and_unlock()
266         trace_lockcnt_unlock_attempt(lockcnt, val, new);  in qemu_lockcnt_inc_and_unlock()
267         val = qatomic_cmpxchg(&lockcnt->count, val, new);  in qemu_lockcnt_inc_and_unlock()
268     } while (val != expected);  in qemu_lockcnt_inc_and_unlock()
270     trace_lockcnt_unlock_success(lockcnt, val, new);  in qemu_lockcnt_inc_and_unlock()
271     if (val & QEMU_LOCKCNT_STATE_WAITING) {  in qemu_lockcnt_inc_and_unlock()
278     int expected, new, val;  in qemu_lockcnt_unlock()  local
280     val = qatomic_read(&lockcnt->count);  in qemu_lockcnt_unlock()
282         expected = val;  in qemu_lockcnt_unlock()
283         new = val & ~QEMU_LOCKCNT_STATE_MASK;  in qemu_lockcnt_unlock()
284         trace_lockcnt_unlock_attempt(lockcnt, val, new);  in qemu_lockcnt_unlock()
285         val = qatomic_cmpxchg(&lockcnt->count, val, new);  in qemu_lockcnt_unlock()
286     } while (val != expected);  in qemu_lockcnt_unlock()
288     trace_lockcnt_unlock_success(lockcnt, val, new);  in qemu_lockcnt_unlock()
289     if (val & QEMU_LOCKCNT_STATE_WAITING) {  in qemu_lockcnt_unlock()
338     int val = qatomic_read(&lockcnt->count);  in qemu_lockcnt_dec_and_lock()  local
339     while (val > 1) {  in qemu_lockcnt_dec_and_lock()
340         int old = qatomic_cmpxchg(&lockcnt->count, val, val - 1);  in qemu_lockcnt_dec_and_lock()
341         if (old != val) {  in qemu_lockcnt_dec_and_lock()
342             val = old;  in qemu_lockcnt_dec_and_lock()
367     int val = qatomic_read(&lockcnt->count);  in qemu_lockcnt_dec_if_lock()  local
368     if (val > 1) {  in qemu_lockcnt_dec_if_lock()