1 // SPDX-License-Identifier: GPL-2.0-or-later
5  * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
15 #include <trace/events/lock.h>
18  * queued_read_lock_slowpath - acquire read lock of a queued rwlock
19  * @lock: Pointer to queued rwlock structure
21 void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)  in queued_read_lock_slowpath()  argument
24 	 * Readers come here when they cannot get the lock without waiting  in queued_read_lock_slowpath()
28 		 * Readers in interrupt context will get the lock immediately  in queued_read_lock_slowpath()
29 		 * if the writer is just waiting (not holding the lock yet),  in queued_read_lock_slowpath()
30 		 * so spin with ACQUIRE semantics until the lock is available  in queued_read_lock_slowpath()
33 		atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));  in queued_read_lock_slowpath()
36 	atomic_sub(_QR_BIAS, &lock->cnts);  in queued_read_lock_slowpath()
38 	trace_contention_begin(lock, LCB_F_SPIN | LCB_F_READ);  in queued_read_lock_slowpath()
43 	arch_spin_lock(&lock->wait_lock);  in queued_read_lock_slowpath()
44 	atomic_add(_QR_BIAS, &lock->cnts);  in queued_read_lock_slowpath()
49 	 * section in the case that the lock is currently held for write.  in queued_read_lock_slowpath()
51 	atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));  in queued_read_lock_slowpath()
56 	arch_spin_unlock(&lock->wait_lock);  in queued_read_lock_slowpath()
58 	trace_contention_end(lock, 0);  in queued_read_lock_slowpath()
63  * queued_write_lock_slowpath - acquire write lock of a queued rwlock
64  * @lock : Pointer to queued rwlock structure
66 void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)  in queued_write_lock_slowpath()  argument
70 	trace_contention_begin(lock, LCB_F_SPIN | LCB_F_WRITE);  in queued_write_lock_slowpath()
73 	arch_spin_lock(&lock->wait_lock);  in queued_write_lock_slowpath()
75 	/* Try to acquire the lock directly if no reader is present */  in queued_write_lock_slowpath()
76 	if (!(cnts = atomic_read(&lock->cnts)) &&  in queued_write_lock_slowpath()
77 	    atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))  in queued_write_lock_slowpath()
81 	atomic_or(_QW_WAITING, &lock->cnts);  in queued_write_lock_slowpath()
85 		cnts = atomic_cond_read_relaxed(&lock->cnts, VAL == _QW_WAITING);  in queued_write_lock_slowpath()
86 	} while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));  in queued_write_lock_slowpath()
88 	arch_spin_unlock(&lock->wait_lock);  in queued_write_lock_slowpath()
90 	trace_contention_end(lock, 0);  in queued_write_lock_slowpath()