1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_ATOMIC64_32_H 3 #define _ASM_X86_ATOMIC64_32_H 4 5 #include <linux/compiler.h> 6 #include <linux/types.h> 7 //#include <asm/cmpxchg.h> 8 9 /* An 64bit atomic type */ 10 11 typedef struct { 12 u64 __aligned(8) counter; 13 } atomic64_t; 14 15 #define ATOMIC64_INIT(val) { (val) } 16 17 #define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...) 18 #ifndef ATOMIC64_EXPORT 19 #define ATOMIC64_DECL_ONE __ATOMIC64_DECL 20 #else 21 #define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \ 22 ATOMIC64_EXPORT(atomic64_##sym) 23 #endif 24 25 #ifdef CONFIG_X86_CMPXCHG64 26 #define __alternative_atomic64(f, g, out, in...) \ 27 asm volatile("call %P[func]" \ 28 : out : [func] "i" (atomic64_##g##_cx8), ## in) 29 30 #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8) 31 #else 32 #define __alternative_atomic64(f, g, out, in...) \ 33 alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \ 34 X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in) 35 36 #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \ 37 ATOMIC64_DECL_ONE(sym##_386) 38 39 ATOMIC64_DECL_ONE(add_386); 40 ATOMIC64_DECL_ONE(sub_386); 41 ATOMIC64_DECL_ONE(inc_386); 42 ATOMIC64_DECL_ONE(dec_386); 43 #endif 44 45 #define alternative_atomic64(f, out, in...) \ 46 __alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in) 47 48 ATOMIC64_DECL(read); 49 ATOMIC64_DECL(set); 50 ATOMIC64_DECL(xchg); 51 ATOMIC64_DECL(add_return); 52 ATOMIC64_DECL(sub_return); 53 ATOMIC64_DECL(inc_return); 54 ATOMIC64_DECL(dec_return); 55 ATOMIC64_DECL(dec_if_positive); 56 ATOMIC64_DECL(inc_not_zero); 57 ATOMIC64_DECL(add_unless); 58 59 #undef ATOMIC64_DECL 60 #undef ATOMIC64_DECL_ONE 61 #undef __ATOMIC64_DECL 62 #undef ATOMIC64_EXPORT 63 64 /** 65 * atomic64_cmpxchg - cmpxchg atomic64 variable 66 * @v: pointer to type atomic64_t 67 * @o: expected value 68 * @n: new value 69 * 70 * Atomically sets @v to @n if it was equal to @o and returns 71 * the old value. 72 */ 73 74 static inline long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) 75 { 76 return cmpxchg64(&v->counter, o, n); 77 } 78 79 /** 80 * atomic64_xchg - xchg atomic64 variable 81 * @v: pointer to type atomic64_t 82 * @n: value to assign 83 * 84 * Atomically xchgs the value of @v to @n and returns 85 * the old value. 86 */ 87 static inline long long atomic64_xchg(atomic64_t *v, long long n) 88 { 89 long long o; 90 unsigned high = (unsigned)(n >> 32); 91 unsigned low = (unsigned)n; 92 alternative_atomic64(xchg, "=&A" (o), 93 "S" (v), "b" (low), "c" (high) 94 : "memory"); 95 return o; 96 } 97 98 /** 99 * atomic64_set - set atomic64 variable 100 * @v: pointer to type atomic64_t 101 * @i: value to assign 102 * 103 * Atomically sets the value of @v to @n. 104 */ 105 static inline void atomic64_set(atomic64_t *v, long long i) 106 { 107 unsigned high = (unsigned)(i >> 32); 108 unsigned low = (unsigned)i; 109 alternative_atomic64(set, /* no output */, 110 "S" (v), "b" (low), "c" (high) 111 : "eax", "edx", "memory"); 112 } 113 114 /** 115 * atomic64_read - read atomic64 variable 116 * @v: pointer to type atomic64_t 117 * 118 * Atomically reads the value of @v and returns it. 119 */ 120 static inline long long atomic64_read(const atomic64_t *v) 121 { 122 long long r; 123 alternative_atomic64(read, "=&A" (r), "c" (v) : "memory"); 124 return r; 125 } 126 127 /** 128 * atomic64_add_return - add and return 129 * @i: integer value to add 130 * @v: pointer to type atomic64_t 131 * 132 * Atomically adds @i to @v and returns @i + *@v 133 */ 134 static inline long long atomic64_add_return(long long i, atomic64_t *v) 135 { 136 alternative_atomic64(add_return, 137 ASM_OUTPUT2("+A" (i), "+c" (v)), 138 ASM_NO_INPUT_CLOBBER("memory")); 139 return i; 140 } 141 142 /* 143 * Other variants with different arithmetic operators: 144 */ 145 static inline long long atomic64_sub_return(long long i, atomic64_t *v) 146 { 147 alternative_atomic64(sub_return, 148 ASM_OUTPUT2("+A" (i), "+c" (v)), 149 ASM_NO_INPUT_CLOBBER("memory")); 150 return i; 151 } 152 153 static inline long long atomic64_inc_return(atomic64_t *v) 154 { 155 long long a; 156 alternative_atomic64(inc_return, "=&A" (a), 157 "S" (v) : "memory", "ecx"); 158 return a; 159 } 160 161 static inline long long atomic64_dec_return(atomic64_t *v) 162 { 163 long long a; 164 alternative_atomic64(dec_return, "=&A" (a), 165 "S" (v) : "memory", "ecx"); 166 return a; 167 } 168 169 /** 170 * atomic64_add - add integer to atomic64 variable 171 * @i: integer value to add 172 * @v: pointer to type atomic64_t 173 * 174 * Atomically adds @i to @v. 175 */ 176 static inline long long atomic64_add(long long i, atomic64_t *v) 177 { 178 __alternative_atomic64(add, add_return, 179 ASM_OUTPUT2("+A" (i), "+c" (v)), 180 ASM_NO_INPUT_CLOBBER("memory")); 181 return i; 182 } 183 184 /** 185 * atomic64_sub - subtract the atomic64 variable 186 * @i: integer value to subtract 187 * @v: pointer to type atomic64_t 188 * 189 * Atomically subtracts @i from @v. 190 */ 191 static inline long long atomic64_sub(long long i, atomic64_t *v) 192 { 193 __alternative_atomic64(sub, sub_return, 194 ASM_OUTPUT2("+A" (i), "+c" (v)), 195 ASM_NO_INPUT_CLOBBER("memory")); 196 return i; 197 } 198 199 /** 200 * atomic64_sub_and_test - subtract value from variable and test result 201 * @i: integer value to subtract 202 * @v: pointer to type atomic64_t 203 * 204 * Atomically subtracts @i from @v and returns 205 * true if the result is zero, or false for all 206 * other cases. 207 */ 208 static inline int atomic64_sub_and_test(long long i, atomic64_t *v) 209 { 210 return atomic64_sub_return(i, v) == 0; 211 } 212 213 /** 214 * atomic64_inc - increment atomic64 variable 215 * @v: pointer to type atomic64_t 216 * 217 * Atomically increments @v by 1. 218 */ 219 static inline void atomic64_inc(atomic64_t *v) 220 { 221 __alternative_atomic64(inc, inc_return, /* no output */, 222 "S" (v) : "memory", "eax", "ecx", "edx"); 223 } 224 225 /** 226 * atomic64_dec - decrement atomic64 variable 227 * @v: pointer to type atomic64_t 228 * 229 * Atomically decrements @v by 1. 230 */ 231 static inline void atomic64_dec(atomic64_t *v) 232 { 233 __alternative_atomic64(dec, dec_return, /* no output */, 234 "S" (v) : "memory", "eax", "ecx", "edx"); 235 } 236 237 /** 238 * atomic64_dec_and_test - decrement and test 239 * @v: pointer to type atomic64_t 240 * 241 * Atomically decrements @v by 1 and 242 * returns true if the result is 0, or false for all other 243 * cases. 244 */ 245 static inline int atomic64_dec_and_test(atomic64_t *v) 246 { 247 return atomic64_dec_return(v) == 0; 248 } 249 250 /** 251 * atomic64_inc_and_test - increment and test 252 * @v: pointer to type atomic64_t 253 * 254 * Atomically increments @v by 1 255 * and returns true if the result is zero, or false for all 256 * other cases. 257 */ 258 static inline int atomic64_inc_and_test(atomic64_t *v) 259 { 260 return atomic64_inc_return(v) == 0; 261 } 262 263 /** 264 * atomic64_add_negative - add and test if negative 265 * @i: integer value to add 266 * @v: pointer to type atomic64_t 267 * 268 * Atomically adds @i to @v and returns true 269 * if the result is negative, or false when 270 * result is greater than or equal to zero. 271 */ 272 static inline int atomic64_add_negative(long long i, atomic64_t *v) 273 { 274 return atomic64_add_return(i, v) < 0; 275 } 276 277 /** 278 * atomic64_add_unless - add unless the number is a given value 279 * @v: pointer of type atomic64_t 280 * @a: the amount to add to v... 281 * @u: ...unless v is equal to u. 282 * 283 * Atomically adds @a to @v, so long as it was not @u. 284 * Returns non-zero if the add was done, zero otherwise. 285 */ 286 static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) 287 { 288 unsigned low = (unsigned)u; 289 unsigned high = (unsigned)(u >> 32); 290 alternative_atomic64(add_unless, 291 ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)), 292 "S" (v) : "memory"); 293 return (int)a; 294 } 295 296 297 static inline int atomic64_inc_not_zero(atomic64_t *v) 298 { 299 int r; 300 alternative_atomic64(inc_not_zero, "=&a" (r), 301 "S" (v) : "ecx", "edx", "memory"); 302 return r; 303 } 304 305 static inline long long atomic64_dec_if_positive(atomic64_t *v) 306 { 307 long long r; 308 alternative_atomic64(dec_if_positive, "=&A" (r), 309 "S" (v) : "ecx", "memory"); 310 return r; 311 } 312 313 #undef alternative_atomic64 314 #undef __alternative_atomic64 315 316 static inline void atomic64_and(long long i, atomic64_t *v) 317 { 318 long long old, c = 0; 319 320 while ((old = atomic64_cmpxchg(v, c, c & i)) != c) 321 c = old; 322 } 323 324 static inline long long atomic64_fetch_and(long long i, atomic64_t *v) 325 { 326 long long old, c = 0; 327 328 while ((old = atomic64_cmpxchg(v, c, c & i)) != c) 329 c = old; 330 331 return old; 332 } 333 334 static inline void atomic64_or(long long i, atomic64_t *v) 335 { 336 long long old, c = 0; 337 338 while ((old = atomic64_cmpxchg(v, c, c | i)) != c) 339 c = old; 340 } 341 342 static inline long long atomic64_fetch_or(long long i, atomic64_t *v) 343 { 344 long long old, c = 0; 345 346 while ((old = atomic64_cmpxchg(v, c, c | i)) != c) 347 c = old; 348 349 return old; 350 } 351 352 static inline void atomic64_xor(long long i, atomic64_t *v) 353 { 354 long long old, c = 0; 355 356 while ((old = atomic64_cmpxchg(v, c, c ^ i)) != c) 357 c = old; 358 } 359 360 static inline long long atomic64_fetch_xor(long long i, atomic64_t *v) 361 { 362 long long old, c = 0; 363 364 while ((old = atomic64_cmpxchg(v, c, c ^ i)) != c) 365 c = old; 366 367 return old; 368 } 369 370 static inline long long atomic64_fetch_add(long long i, atomic64_t *v) 371 { 372 long long old, c = 0; 373 374 while ((old = atomic64_cmpxchg(v, c, c + i)) != c) 375 c = old; 376 377 return old; 378 } 379 380 #define atomic64_fetch_sub(i, v) atomic64_fetch_add(-(i), (v)) 381 382 #endif /* _ASM_X86_ATOMIC64_32_H */ 383