1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
22f4f12e5SLinus Torvalds #include <linux/export.h>
32f4f12e5SLinus Torvalds #include <linux/lockref.h>
42f4f12e5SLinus Torvalds
557f4257eSPeter Zijlstra #if USE_CMPXCHG_LOCKREF
6bc08b449SLinus Torvalds
7bc08b449SLinus Torvalds /*
8bc08b449SLinus Torvalds * Note that the "cmpxchg()" reloads the "old" value for the
9bc08b449SLinus Torvalds * failure case.
10bc08b449SLinus Torvalds */
11bc08b449SLinus Torvalds #define CMPXCHG_LOOP(CODE, SUCCESS) do { \
12893a7d32SJan Glauber int retry = 100; \
13bc08b449SLinus Torvalds struct lockref old; \
14bc08b449SLinus Torvalds BUILD_BUG_ON(sizeof(old) != 8); \
154d3199e4SDavidlohr Bueso old.lock_count = READ_ONCE(lockref->lock_count); \
16bc08b449SLinus Torvalds while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
17*3378323bSUros Bizjak struct lockref new = old; \
18bc08b449SLinus Torvalds CODE \
19*3378323bSUros Bizjak if (likely(try_cmpxchg64_relaxed(&lockref->lock_count, \
20*3378323bSUros Bizjak &old.lock_count, \
21*3378323bSUros Bizjak new.lock_count))) { \
22bc08b449SLinus Torvalds SUCCESS; \
23bc08b449SLinus Torvalds } \
24893a7d32SJan Glauber if (!--retry) \
25893a7d32SJan Glauber break; \
26bc08b449SLinus Torvalds } \
27bc08b449SLinus Torvalds } while (0)
28bc08b449SLinus Torvalds
29bc08b449SLinus Torvalds #else
30bc08b449SLinus Torvalds
31bc08b449SLinus Torvalds #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
32bc08b449SLinus Torvalds
33bc08b449SLinus Torvalds #endif
34bc08b449SLinus Torvalds
352f4f12e5SLinus Torvalds /**
362f4f12e5SLinus Torvalds * lockref_get - Increments reference count unconditionally
3744a0cf92SLinus Torvalds * @lockref: pointer to lockref structure
382f4f12e5SLinus Torvalds *
392f4f12e5SLinus Torvalds * This operation is only valid if you already hold a reference
402f4f12e5SLinus Torvalds * to the object, so you know the count cannot be zero.
412f4f12e5SLinus Torvalds */
lockref_get(struct lockref * lockref)422f4f12e5SLinus Torvalds void lockref_get(struct lockref *lockref)
432f4f12e5SLinus Torvalds {
44bc08b449SLinus Torvalds CMPXCHG_LOOP(
45bc08b449SLinus Torvalds new.count++;
46bc08b449SLinus Torvalds ,
47bc08b449SLinus Torvalds return;
48bc08b449SLinus Torvalds );
49bc08b449SLinus Torvalds
502f4f12e5SLinus Torvalds spin_lock(&lockref->lock);
512f4f12e5SLinus Torvalds lockref->count++;
522f4f12e5SLinus Torvalds spin_unlock(&lockref->lock);
532f4f12e5SLinus Torvalds }
542f4f12e5SLinus Torvalds EXPORT_SYMBOL(lockref_get);
552f4f12e5SLinus Torvalds
562f4f12e5SLinus Torvalds /**
57360f5479SLinus Torvalds * lockref_get_not_zero - Increments count unless the count is 0 or dead
5844a0cf92SLinus Torvalds * @lockref: pointer to lockref structure
592f4f12e5SLinus Torvalds * Return: 1 if count updated successfully or 0 if count was zero
602f4f12e5SLinus Torvalds */
lockref_get_not_zero(struct lockref * lockref)612f4f12e5SLinus Torvalds int lockref_get_not_zero(struct lockref *lockref)
622f4f12e5SLinus Torvalds {
63bc08b449SLinus Torvalds int retval;
64bc08b449SLinus Torvalds
65bc08b449SLinus Torvalds CMPXCHG_LOOP(
66bc08b449SLinus Torvalds new.count++;
67360f5479SLinus Torvalds if (old.count <= 0)
68bc08b449SLinus Torvalds return 0;
69bc08b449SLinus Torvalds ,
70bc08b449SLinus Torvalds return 1;
71bc08b449SLinus Torvalds );
722f4f12e5SLinus Torvalds
732f4f12e5SLinus Torvalds spin_lock(&lockref->lock);
74bc08b449SLinus Torvalds retval = 0;
75360f5479SLinus Torvalds if (lockref->count > 0) {
762f4f12e5SLinus Torvalds lockref->count++;
772f4f12e5SLinus Torvalds retval = 1;
782f4f12e5SLinus Torvalds }
792f4f12e5SLinus Torvalds spin_unlock(&lockref->lock);
802f4f12e5SLinus Torvalds return retval;
812f4f12e5SLinus Torvalds }
822f4f12e5SLinus Torvalds EXPORT_SYMBOL(lockref_get_not_zero);
832f4f12e5SLinus Torvalds
842f4f12e5SLinus Torvalds /**
85450b1f6fSAndreas Gruenbacher * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
86450b1f6fSAndreas Gruenbacher * @lockref: pointer to lockref structure
87450b1f6fSAndreas Gruenbacher * Return: 1 if count updated successfully or 0 if count would become zero
88450b1f6fSAndreas Gruenbacher */
lockref_put_not_zero(struct lockref * lockref)89450b1f6fSAndreas Gruenbacher int lockref_put_not_zero(struct lockref *lockref)
90450b1f6fSAndreas Gruenbacher {
91450b1f6fSAndreas Gruenbacher int retval;
92450b1f6fSAndreas Gruenbacher
93450b1f6fSAndreas Gruenbacher CMPXCHG_LOOP(
94450b1f6fSAndreas Gruenbacher new.count--;
95450b1f6fSAndreas Gruenbacher if (old.count <= 1)
96450b1f6fSAndreas Gruenbacher return 0;
97450b1f6fSAndreas Gruenbacher ,
98450b1f6fSAndreas Gruenbacher return 1;
99450b1f6fSAndreas Gruenbacher );
100450b1f6fSAndreas Gruenbacher
101450b1f6fSAndreas Gruenbacher spin_lock(&lockref->lock);
102450b1f6fSAndreas Gruenbacher retval = 0;
103450b1f6fSAndreas Gruenbacher if (lockref->count > 1) {
104450b1f6fSAndreas Gruenbacher lockref->count--;
105450b1f6fSAndreas Gruenbacher retval = 1;
106450b1f6fSAndreas Gruenbacher }
107450b1f6fSAndreas Gruenbacher spin_unlock(&lockref->lock);
108450b1f6fSAndreas Gruenbacher return retval;
109450b1f6fSAndreas Gruenbacher }
110450b1f6fSAndreas Gruenbacher EXPORT_SYMBOL(lockref_put_not_zero);
111450b1f6fSAndreas Gruenbacher
112450b1f6fSAndreas Gruenbacher /**
113360f5479SLinus Torvalds * lockref_put_return - Decrement reference count if possible
114360f5479SLinus Torvalds * @lockref: pointer to lockref structure
115360f5479SLinus Torvalds *
116360f5479SLinus Torvalds * Decrement the reference count and return the new value.
117360f5479SLinus Torvalds * If the lockref was dead or locked, return an error.
118360f5479SLinus Torvalds */
lockref_put_return(struct lockref * lockref)119360f5479SLinus Torvalds int lockref_put_return(struct lockref *lockref)
120360f5479SLinus Torvalds {
121360f5479SLinus Torvalds CMPXCHG_LOOP(
122360f5479SLinus Torvalds new.count--;
123360f5479SLinus Torvalds if (old.count <= 0)
124360f5479SLinus Torvalds return -1;
125360f5479SLinus Torvalds ,
126360f5479SLinus Torvalds return new.count;
127360f5479SLinus Torvalds );
128360f5479SLinus Torvalds return -1;
129360f5479SLinus Torvalds }
130360f5479SLinus Torvalds EXPORT_SYMBOL(lockref_put_return);
131360f5479SLinus Torvalds
132360f5479SLinus Torvalds /**
1332f4f12e5SLinus Torvalds * lockref_put_or_lock - decrements count unless count <= 1 before decrement
13444a0cf92SLinus Torvalds * @lockref: pointer to lockref structure
1352f4f12e5SLinus Torvalds * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
1362f4f12e5SLinus Torvalds */
lockref_put_or_lock(struct lockref * lockref)1372f4f12e5SLinus Torvalds int lockref_put_or_lock(struct lockref *lockref)
1382f4f12e5SLinus Torvalds {
139bc08b449SLinus Torvalds CMPXCHG_LOOP(
140bc08b449SLinus Torvalds new.count--;
141bc08b449SLinus Torvalds if (old.count <= 1)
142bc08b449SLinus Torvalds break;
143bc08b449SLinus Torvalds ,
144bc08b449SLinus Torvalds return 1;
145bc08b449SLinus Torvalds );
146bc08b449SLinus Torvalds
1472f4f12e5SLinus Torvalds spin_lock(&lockref->lock);
1482f4f12e5SLinus Torvalds if (lockref->count <= 1)
1492f4f12e5SLinus Torvalds return 0;
1502f4f12e5SLinus Torvalds lockref->count--;
1512f4f12e5SLinus Torvalds spin_unlock(&lockref->lock);
1522f4f12e5SLinus Torvalds return 1;
1532f4f12e5SLinus Torvalds }
1542f4f12e5SLinus Torvalds EXPORT_SYMBOL(lockref_put_or_lock);
155e7d33bb5SLinus Torvalds
156e7d33bb5SLinus Torvalds /**
157e7d33bb5SLinus Torvalds * lockref_mark_dead - mark lockref dead
158e7d33bb5SLinus Torvalds * @lockref: pointer to lockref structure
159e7d33bb5SLinus Torvalds */
lockref_mark_dead(struct lockref * lockref)160e7d33bb5SLinus Torvalds void lockref_mark_dead(struct lockref *lockref)
161e7d33bb5SLinus Torvalds {
162e7d33bb5SLinus Torvalds assert_spin_locked(&lockref->lock);
163e7d33bb5SLinus Torvalds lockref->count = -128;
164e7d33bb5SLinus Torvalds }
165e66cf161SSteven Whitehouse EXPORT_SYMBOL(lockref_mark_dead);
166e7d33bb5SLinus Torvalds
167e7d33bb5SLinus Torvalds /**
168e7d33bb5SLinus Torvalds * lockref_get_not_dead - Increments count unless the ref is dead
169e7d33bb5SLinus Torvalds * @lockref: pointer to lockref structure
170e7d33bb5SLinus Torvalds * Return: 1 if count updated successfully or 0 if lockref was dead
171e7d33bb5SLinus Torvalds */
lockref_get_not_dead(struct lockref * lockref)172e7d33bb5SLinus Torvalds int lockref_get_not_dead(struct lockref *lockref)
173e7d33bb5SLinus Torvalds {
174e7d33bb5SLinus Torvalds int retval;
175e7d33bb5SLinus Torvalds
176e7d33bb5SLinus Torvalds CMPXCHG_LOOP(
177e7d33bb5SLinus Torvalds new.count++;
178360f5479SLinus Torvalds if (old.count < 0)
179e7d33bb5SLinus Torvalds return 0;
180e7d33bb5SLinus Torvalds ,
181e7d33bb5SLinus Torvalds return 1;
182e7d33bb5SLinus Torvalds );
183e7d33bb5SLinus Torvalds
184e7d33bb5SLinus Torvalds spin_lock(&lockref->lock);
185e7d33bb5SLinus Torvalds retval = 0;
186360f5479SLinus Torvalds if (lockref->count >= 0) {
187e7d33bb5SLinus Torvalds lockref->count++;
188e7d33bb5SLinus Torvalds retval = 1;
189e7d33bb5SLinus Torvalds }
190e7d33bb5SLinus Torvalds spin_unlock(&lockref->lock);
191e7d33bb5SLinus Torvalds return retval;
192e7d33bb5SLinus Torvalds }
193e7d33bb5SLinus Torvalds EXPORT_SYMBOL(lockref_get_not_dead);
194