1 #include <linux/compiler.h> 2 #include <linux/module.h> 3 #include <linux/types.h> 4 5 #include <asm/processor.h> 6 #include <asm/cmpxchg.h> 7 #include <asm/atomic.h> 8 9 static noinline u64 cmpxchg8b(u64 *ptr, u64 old, u64 new) 10 { 11 u32 low = new; 12 u32 high = new >> 32; 13 14 asm volatile( 15 LOCK_PREFIX "cmpxchg8b %1\n" 16 : "+A" (old), "+m" (*ptr) 17 : "b" (low), "c" (high) 18 ); 19 return old; 20 } 21 22 u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old_val, u64 new_val) 23 { 24 return cmpxchg8b(&ptr->counter, old_val, new_val); 25 } 26 EXPORT_SYMBOL(atomic64_cmpxchg); 27 28 /** 29 * atomic64_xchg - xchg atomic64 variable 30 * @ptr: pointer to type atomic64_t 31 * @new_val: value to assign 32 * 33 * Atomically xchgs the value of @ptr to @new_val and returns 34 * the old value. 35 */ 36 u64 atomic64_xchg(atomic64_t *ptr, u64 new_val) 37 { 38 /* 39 * Try first with a (possibly incorrect) assumption about 40 * what we have there. We'll do two loops most likely, 41 * but we'll get an ownership MESI transaction straight away 42 * instead of a read transaction followed by a 43 * flush-for-ownership transaction: 44 */ 45 u64 old_val, real_val = 0; 46 47 do { 48 old_val = real_val; 49 50 real_val = atomic64_cmpxchg(ptr, old_val, new_val); 51 52 } while (real_val != old_val); 53 54 return old_val; 55 } 56 EXPORT_SYMBOL(atomic64_xchg); 57 58 /** 59 * atomic64_set - set atomic64 variable 60 * @ptr: pointer to type atomic64_t 61 * @new_val: value to assign 62 * 63 * Atomically sets the value of @ptr to @new_val. 64 */ 65 void atomic64_set(atomic64_t *ptr, u64 new_val) 66 { 67 atomic64_xchg(ptr, new_val); 68 } 69 EXPORT_SYMBOL(atomic64_set); 70 71 /** 72 EXPORT_SYMBOL(atomic64_read); 73 * atomic64_add_return - add and return 74 * @delta: integer value to add 75 * @ptr: pointer to type atomic64_t 76 * 77 * Atomically adds @delta to @ptr and returns @delta + *@ptr 78 */ 79 noinline u64 atomic64_add_return(u64 delta, atomic64_t *ptr) 80 { 81 /* 82 * Try first with a (possibly incorrect) assumption about 83 * what we have there. We'll do two loops most likely, 84 * but we'll get an ownership MESI transaction straight away 85 * instead of a read transaction followed by a 86 * flush-for-ownership transaction: 87 */ 88 u64 old_val, new_val, real_val = 0; 89 90 do { 91 old_val = real_val; 92 new_val = old_val + delta; 93 94 real_val = atomic64_cmpxchg(ptr, old_val, new_val); 95 96 } while (real_val != old_val); 97 98 return new_val; 99 } 100 EXPORT_SYMBOL(atomic64_add_return); 101 102 u64 atomic64_sub_return(u64 delta, atomic64_t *ptr) 103 { 104 return atomic64_add_return(-delta, ptr); 105 } 106 EXPORT_SYMBOL(atomic64_sub_return); 107 108 u64 atomic64_inc_return(atomic64_t *ptr) 109 { 110 return atomic64_add_return(1, ptr); 111 } 112 EXPORT_SYMBOL(atomic64_inc_return); 113 114 u64 atomic64_dec_return(atomic64_t *ptr) 115 { 116 return atomic64_sub_return(1, ptr); 117 } 118 EXPORT_SYMBOL(atomic64_dec_return); 119 120 /** 121 * atomic64_add - add integer to atomic64 variable 122 * @delta: integer value to add 123 * @ptr: pointer to type atomic64_t 124 * 125 * Atomically adds @delta to @ptr. 126 */ 127 void atomic64_add(u64 delta, atomic64_t *ptr) 128 { 129 atomic64_add_return(delta, ptr); 130 } 131 EXPORT_SYMBOL(atomic64_add); 132 133 /** 134 * atomic64_sub - subtract the atomic64 variable 135 * @delta: integer value to subtract 136 * @ptr: pointer to type atomic64_t 137 * 138 * Atomically subtracts @delta from @ptr. 139 */ 140 void atomic64_sub(u64 delta, atomic64_t *ptr) 141 { 142 atomic64_add(-delta, ptr); 143 } 144 EXPORT_SYMBOL(atomic64_sub); 145 146 /** 147 * atomic64_sub_and_test - subtract value from variable and test result 148 * @delta: integer value to subtract 149 * @ptr: pointer to type atomic64_t 150 * 151 * Atomically subtracts @delta from @ptr and returns 152 * true if the result is zero, or false for all 153 * other cases. 154 */ 155 int atomic64_sub_and_test(u64 delta, atomic64_t *ptr) 156 { 157 u64 new_val = atomic64_sub_return(delta, ptr); 158 159 return new_val == 0; 160 } 161 EXPORT_SYMBOL(atomic64_sub_and_test); 162 163 /** 164 * atomic64_inc - increment atomic64 variable 165 * @ptr: pointer to type atomic64_t 166 * 167 * Atomically increments @ptr by 1. 168 */ 169 void atomic64_inc(atomic64_t *ptr) 170 { 171 atomic64_add(1, ptr); 172 } 173 EXPORT_SYMBOL(atomic64_inc); 174 175 /** 176 * atomic64_dec - decrement atomic64 variable 177 * @ptr: pointer to type atomic64_t 178 * 179 * Atomically decrements @ptr by 1. 180 */ 181 void atomic64_dec(atomic64_t *ptr) 182 { 183 atomic64_sub(1, ptr); 184 } 185 EXPORT_SYMBOL(atomic64_dec); 186 187 /** 188 * atomic64_dec_and_test - decrement and test 189 * @ptr: pointer to type atomic64_t 190 * 191 * Atomically decrements @ptr by 1 and 192 * returns true if the result is 0, or false for all other 193 * cases. 194 */ 195 int atomic64_dec_and_test(atomic64_t *ptr) 196 { 197 return atomic64_sub_and_test(1, ptr); 198 } 199 EXPORT_SYMBOL(atomic64_dec_and_test); 200 201 /** 202 * atomic64_inc_and_test - increment and test 203 * @ptr: pointer to type atomic64_t 204 * 205 * Atomically increments @ptr by 1 206 * and returns true if the result is zero, or false for all 207 * other cases. 208 */ 209 int atomic64_inc_and_test(atomic64_t *ptr) 210 { 211 return atomic64_sub_and_test(-1, ptr); 212 } 213 EXPORT_SYMBOL(atomic64_inc_and_test); 214 215 /** 216 * atomic64_add_negative - add and test if negative 217 * @delta: integer value to add 218 * @ptr: pointer to type atomic64_t 219 * 220 * Atomically adds @delta to @ptr and returns true 221 * if the result is negative, or false when 222 * result is greater than or equal to zero. 223 */ 224 int atomic64_add_negative(u64 delta, atomic64_t *ptr) 225 { 226 s64 new_val = atomic64_add_return(delta, ptr); 227 228 return new_val < 0; 229 } 230 EXPORT_SYMBOL(atomic64_add_negative); 231