1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/export.h> 3 #include <linux/lockref.h> 4 5 #if USE_CMPXCHG_LOCKREF 6 7 /* 8 * Note that the "cmpxchg()" reloads the "old" value for the 9 * failure case. 10 */ 11 #define CMPXCHG_LOOP(CODE, SUCCESS) do { \ 12 int retry = 100; \ 13 struct lockref old; \ 14 BUILD_BUG_ON(sizeof(old) != 8); \ 15 old.lock_count = READ_ONCE(lockref->lock_count); \ 16 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \ 17 struct lockref new = old, prev = old; \ 18 CODE \ 19 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \ 20 old.lock_count, \ 21 new.lock_count); \ 22 if (likely(old.lock_count == prev.lock_count)) { \ 23 SUCCESS; \ 24 } \ 25 if (!--retry) \ 26 break; \ 27 cpu_relax(); \ 28 } \ 29 } while (0) 30 31 #else 32 33 #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0) 34 35 #endif 36 37 /** 38 * lockref_get - Increments reference count unconditionally 39 * @lockref: pointer to lockref structure 40 * 41 * This operation is only valid if you already hold a reference 42 * to the object, so you know the count cannot be zero. 43 */ 44 void lockref_get(struct lockref *lockref) 45 { 46 CMPXCHG_LOOP( 47 new.count++; 48 , 49 return; 50 ); 51 52 spin_lock(&lockref->lock); 53 lockref->count++; 54 spin_unlock(&lockref->lock); 55 } 56 EXPORT_SYMBOL(lockref_get); 57 58 /** 59 * lockref_get_not_zero - Increments count unless the count is 0 or dead 60 * @lockref: pointer to lockref structure 61 * Return: 1 if count updated successfully or 0 if count was zero 62 */ 63 int lockref_get_not_zero(struct lockref *lockref) 64 { 65 int retval; 66 67 CMPXCHG_LOOP( 68 new.count++; 69 if (old.count <= 0) 70 return 0; 71 , 72 return 1; 73 ); 74 75 spin_lock(&lockref->lock); 76 retval = 0; 77 if (lockref->count > 0) { 78 lockref->count++; 79 retval = 1; 80 } 81 spin_unlock(&lockref->lock); 82 return retval; 83 } 84 EXPORT_SYMBOL(lockref_get_not_zero); 85 86 /** 87 * lockref_put_not_zero - Decrements count unless count <= 1 before decrement 88 * @lockref: pointer to lockref structure 89 * Return: 1 if count updated successfully or 0 if count would become zero 90 */ 91 int lockref_put_not_zero(struct lockref *lockref) 92 { 93 int retval; 94 95 CMPXCHG_LOOP( 96 new.count--; 97 if (old.count <= 1) 98 return 0; 99 , 100 return 1; 101 ); 102 103 spin_lock(&lockref->lock); 104 retval = 0; 105 if (lockref->count > 1) { 106 lockref->count--; 107 retval = 1; 108 } 109 spin_unlock(&lockref->lock); 110 return retval; 111 } 112 EXPORT_SYMBOL(lockref_put_not_zero); 113 114 /** 115 * lockref_get_or_lock - Increments count unless the count is 0 or dead 116 * @lockref: pointer to lockref structure 117 * Return: 1 if count updated successfully or 0 if count was zero 118 * and we got the lock instead. 119 */ 120 int lockref_get_or_lock(struct lockref *lockref) 121 { 122 CMPXCHG_LOOP( 123 new.count++; 124 if (old.count <= 0) 125 break; 126 , 127 return 1; 128 ); 129 130 spin_lock(&lockref->lock); 131 if (lockref->count <= 0) 132 return 0; 133 lockref->count++; 134 spin_unlock(&lockref->lock); 135 return 1; 136 } 137 EXPORT_SYMBOL(lockref_get_or_lock); 138 139 /** 140 * lockref_put_return - Decrement reference count if possible 141 * @lockref: pointer to lockref structure 142 * 143 * Decrement the reference count and return the new value. 144 * If the lockref was dead or locked, return an error. 145 */ 146 int lockref_put_return(struct lockref *lockref) 147 { 148 CMPXCHG_LOOP( 149 new.count--; 150 if (old.count <= 0) 151 return -1; 152 , 153 return new.count; 154 ); 155 return -1; 156 } 157 EXPORT_SYMBOL(lockref_put_return); 158 159 /** 160 * lockref_put_or_lock - decrements count unless count <= 1 before decrement 161 * @lockref: pointer to lockref structure 162 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken 163 */ 164 int lockref_put_or_lock(struct lockref *lockref) 165 { 166 CMPXCHG_LOOP( 167 new.count--; 168 if (old.count <= 1) 169 break; 170 , 171 return 1; 172 ); 173 174 spin_lock(&lockref->lock); 175 if (lockref->count <= 1) 176 return 0; 177 lockref->count--; 178 spin_unlock(&lockref->lock); 179 return 1; 180 } 181 EXPORT_SYMBOL(lockref_put_or_lock); 182 183 /** 184 * lockref_mark_dead - mark lockref dead 185 * @lockref: pointer to lockref structure 186 */ 187 void lockref_mark_dead(struct lockref *lockref) 188 { 189 assert_spin_locked(&lockref->lock); 190 lockref->count = -128; 191 } 192 EXPORT_SYMBOL(lockref_mark_dead); 193 194 /** 195 * lockref_get_not_dead - Increments count unless the ref is dead 196 * @lockref: pointer to lockref structure 197 * Return: 1 if count updated successfully or 0 if lockref was dead 198 */ 199 int lockref_get_not_dead(struct lockref *lockref) 200 { 201 int retval; 202 203 CMPXCHG_LOOP( 204 new.count++; 205 if (old.count < 0) 206 return 0; 207 , 208 return 1; 209 ); 210 211 spin_lock(&lockref->lock); 212 retval = 0; 213 if (lockref->count >= 0) { 214 lockref->count++; 215 retval = 1; 216 } 217 spin_unlock(&lockref->lock); 218 return retval; 219 } 220 EXPORT_SYMBOL(lockref_get_not_dead); 221