1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_ATOMIC64_32_H 3 #define _ASM_X86_ATOMIC64_32_H 4 5 #include <linux/compiler.h> 6 #include <linux/types.h> 7 //#include <asm/cmpxchg.h> 8 9 /* An 64bit atomic type */ 10 11 typedef struct { 12 u64 __aligned(8) counter; 13 } atomic64_t; 14 15 #define ATOMIC64_INIT(val) { (val) } 16 17 #define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...) 18 #ifndef ATOMIC64_EXPORT 19 #define ATOMIC64_DECL_ONE __ATOMIC64_DECL 20 #else 21 #define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \ 22 ATOMIC64_EXPORT(atomic64_##sym) 23 #endif 24 25 #ifdef CONFIG_X86_CMPXCHG64 26 #define __alternative_atomic64(f, g, out, in...) \ 27 asm volatile("call %P[func]" \ 28 : out : [func] "i" (atomic64_##g##_cx8), ## in) 29 30 #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8) 31 #else 32 #define __alternative_atomic64(f, g, out, in...) \ 33 alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \ 34 X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in) 35 36 #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \ 37 ATOMIC64_DECL_ONE(sym##_386) 38 39 ATOMIC64_DECL_ONE(add_386); 40 ATOMIC64_DECL_ONE(sub_386); 41 ATOMIC64_DECL_ONE(inc_386); 42 ATOMIC64_DECL_ONE(dec_386); 43 #endif 44 45 #define alternative_atomic64(f, out, in...) \ 46 __alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in) 47 48 ATOMIC64_DECL(read); 49 ATOMIC64_DECL(set); 50 ATOMIC64_DECL(xchg); 51 ATOMIC64_DECL(add_return); 52 ATOMIC64_DECL(sub_return); 53 ATOMIC64_DECL(inc_return); 54 ATOMIC64_DECL(dec_return); 55 ATOMIC64_DECL(dec_if_positive); 56 ATOMIC64_DECL(inc_not_zero); 57 ATOMIC64_DECL(add_unless); 58 59 #undef ATOMIC64_DECL 60 #undef ATOMIC64_DECL_ONE 61 #undef __ATOMIC64_DECL 62 #undef ATOMIC64_EXPORT 63 64 /** 65 * arch_atomic64_cmpxchg - cmpxchg atomic64 variable 66 * @v: pointer to type atomic64_t 67 * @o: expected value 68 * @n: new value 69 * 70 * Atomically sets @v to @n if it was equal to @o and returns 71 * the old value. 72 */ 73 74 static inline long long arch_atomic64_cmpxchg(atomic64_t *v, long long o, 75 long long n) 76 { 77 return arch_cmpxchg64(&v->counter, o, n); 78 } 79 80 /** 81 * arch_atomic64_xchg - xchg atomic64 variable 82 * @v: pointer to type atomic64_t 83 * @n: value to assign 84 * 85 * Atomically xchgs the value of @v to @n and returns 86 * the old value. 87 */ 88 static inline long long arch_atomic64_xchg(atomic64_t *v, long long n) 89 { 90 long long o; 91 unsigned high = (unsigned)(n >> 32); 92 unsigned low = (unsigned)n; 93 alternative_atomic64(xchg, "=&A" (o), 94 "S" (v), "b" (low), "c" (high) 95 : "memory"); 96 return o; 97 } 98 99 /** 100 * arch_atomic64_set - set atomic64 variable 101 * @v: pointer to type atomic64_t 102 * @i: value to assign 103 * 104 * Atomically sets the value of @v to @n. 105 */ 106 static inline void arch_atomic64_set(atomic64_t *v, long long i) 107 { 108 unsigned high = (unsigned)(i >> 32); 109 unsigned low = (unsigned)i; 110 alternative_atomic64(set, /* no output */, 111 "S" (v), "b" (low), "c" (high) 112 : "eax", "edx", "memory"); 113 } 114 115 /** 116 * arch_atomic64_read - read atomic64 variable 117 * @v: pointer to type atomic64_t 118 * 119 * Atomically reads the value of @v and returns it. 120 */ 121 static inline long long arch_atomic64_read(const atomic64_t *v) 122 { 123 long long r; 124 alternative_atomic64(read, "=&A" (r), "c" (v) : "memory"); 125 return r; 126 } 127 128 /** 129 * arch_atomic64_add_return - add and return 130 * @i: integer value to add 131 * @v: pointer to type atomic64_t 132 * 133 * Atomically adds @i to @v and returns @i + *@v 134 */ 135 static inline long long arch_atomic64_add_return(long long i, atomic64_t *v) 136 { 137 alternative_atomic64(add_return, 138 ASM_OUTPUT2("+A" (i), "+c" (v)), 139 ASM_NO_INPUT_CLOBBER("memory")); 140 return i; 141 } 142 143 /* 144 * Other variants with different arithmetic operators: 145 */ 146 static inline long long arch_atomic64_sub_return(long long i, atomic64_t *v) 147 { 148 alternative_atomic64(sub_return, 149 ASM_OUTPUT2("+A" (i), "+c" (v)), 150 ASM_NO_INPUT_CLOBBER("memory")); 151 return i; 152 } 153 154 static inline long long arch_atomic64_inc_return(atomic64_t *v) 155 { 156 long long a; 157 alternative_atomic64(inc_return, "=&A" (a), 158 "S" (v) : "memory", "ecx"); 159 return a; 160 } 161 162 static inline long long arch_atomic64_dec_return(atomic64_t *v) 163 { 164 long long a; 165 alternative_atomic64(dec_return, "=&A" (a), 166 "S" (v) : "memory", "ecx"); 167 return a; 168 } 169 170 /** 171 * arch_atomic64_add - add integer to atomic64 variable 172 * @i: integer value to add 173 * @v: pointer to type atomic64_t 174 * 175 * Atomically adds @i to @v. 176 */ 177 static inline long long arch_atomic64_add(long long i, atomic64_t *v) 178 { 179 __alternative_atomic64(add, add_return, 180 ASM_OUTPUT2("+A" (i), "+c" (v)), 181 ASM_NO_INPUT_CLOBBER("memory")); 182 return i; 183 } 184 185 /** 186 * arch_atomic64_sub - subtract the atomic64 variable 187 * @i: integer value to subtract 188 * @v: pointer to type atomic64_t 189 * 190 * Atomically subtracts @i from @v. 191 */ 192 static inline long long arch_atomic64_sub(long long i, atomic64_t *v) 193 { 194 __alternative_atomic64(sub, sub_return, 195 ASM_OUTPUT2("+A" (i), "+c" (v)), 196 ASM_NO_INPUT_CLOBBER("memory")); 197 return i; 198 } 199 200 /** 201 * arch_atomic64_sub_and_test - subtract value from variable and test result 202 * @i: integer value to subtract 203 * @v: pointer to type atomic64_t 204 * 205 * Atomically subtracts @i from @v and returns 206 * true if the result is zero, or false for all 207 * other cases. 208 */ 209 static inline int arch_atomic64_sub_and_test(long long i, atomic64_t *v) 210 { 211 return arch_atomic64_sub_return(i, v) == 0; 212 } 213 214 /** 215 * arch_atomic64_inc - increment atomic64 variable 216 * @v: pointer to type atomic64_t 217 * 218 * Atomically increments @v by 1. 219 */ 220 static inline void arch_atomic64_inc(atomic64_t *v) 221 { 222 __alternative_atomic64(inc, inc_return, /* no output */, 223 "S" (v) : "memory", "eax", "ecx", "edx"); 224 } 225 226 /** 227 * arch_atomic64_dec - decrement atomic64 variable 228 * @v: pointer to type atomic64_t 229 * 230 * Atomically decrements @v by 1. 231 */ 232 static inline void arch_atomic64_dec(atomic64_t *v) 233 { 234 __alternative_atomic64(dec, dec_return, /* no output */, 235 "S" (v) : "memory", "eax", "ecx", "edx"); 236 } 237 238 /** 239 * arch_atomic64_dec_and_test - decrement and test 240 * @v: pointer to type atomic64_t 241 * 242 * Atomically decrements @v by 1 and 243 * returns true if the result is 0, or false for all other 244 * cases. 245 */ 246 static inline int arch_atomic64_dec_and_test(atomic64_t *v) 247 { 248 return arch_atomic64_dec_return(v) == 0; 249 } 250 251 /** 252 * atomic64_inc_and_test - increment and test 253 * @v: pointer to type atomic64_t 254 * 255 * Atomically increments @v by 1 256 * and returns true if the result is zero, or false for all 257 * other cases. 258 */ 259 static inline int arch_atomic64_inc_and_test(atomic64_t *v) 260 { 261 return arch_atomic64_inc_return(v) == 0; 262 } 263 264 /** 265 * arch_atomic64_add_negative - add and test if negative 266 * @i: integer value to add 267 * @v: pointer to type atomic64_t 268 * 269 * Atomically adds @i to @v and returns true 270 * if the result is negative, or false when 271 * result is greater than or equal to zero. 272 */ 273 static inline int arch_atomic64_add_negative(long long i, atomic64_t *v) 274 { 275 return arch_atomic64_add_return(i, v) < 0; 276 } 277 278 /** 279 * arch_atomic64_add_unless - add unless the number is a given value 280 * @v: pointer of type atomic64_t 281 * @a: the amount to add to v... 282 * @u: ...unless v is equal to u. 283 * 284 * Atomically adds @a to @v, so long as it was not @u. 285 * Returns non-zero if the add was done, zero otherwise. 286 */ 287 static inline int arch_atomic64_add_unless(atomic64_t *v, long long a, 288 long long u) 289 { 290 unsigned low = (unsigned)u; 291 unsigned high = (unsigned)(u >> 32); 292 alternative_atomic64(add_unless, 293 ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)), 294 "S" (v) : "memory"); 295 return (int)a; 296 } 297 298 299 static inline int arch_atomic64_inc_not_zero(atomic64_t *v) 300 { 301 int r; 302 alternative_atomic64(inc_not_zero, "=&a" (r), 303 "S" (v) : "ecx", "edx", "memory"); 304 return r; 305 } 306 307 static inline long long arch_atomic64_dec_if_positive(atomic64_t *v) 308 { 309 long long r; 310 alternative_atomic64(dec_if_positive, "=&A" (r), 311 "S" (v) : "ecx", "memory"); 312 return r; 313 } 314 315 #undef alternative_atomic64 316 #undef __alternative_atomic64 317 318 static inline void arch_atomic64_and(long long i, atomic64_t *v) 319 { 320 long long old, c = 0; 321 322 while ((old = arch_atomic64_cmpxchg(v, c, c & i)) != c) 323 c = old; 324 } 325 326 static inline long long arch_atomic64_fetch_and(long long i, atomic64_t *v) 327 { 328 long long old, c = 0; 329 330 while ((old = arch_atomic64_cmpxchg(v, c, c & i)) != c) 331 c = old; 332 333 return old; 334 } 335 336 static inline void arch_atomic64_or(long long i, atomic64_t *v) 337 { 338 long long old, c = 0; 339 340 while ((old = arch_atomic64_cmpxchg(v, c, c | i)) != c) 341 c = old; 342 } 343 344 static inline long long arch_atomic64_fetch_or(long long i, atomic64_t *v) 345 { 346 long long old, c = 0; 347 348 while ((old = arch_atomic64_cmpxchg(v, c, c | i)) != c) 349 c = old; 350 351 return old; 352 } 353 354 static inline void arch_atomic64_xor(long long i, atomic64_t *v) 355 { 356 long long old, c = 0; 357 358 while ((old = arch_atomic64_cmpxchg(v, c, c ^ i)) != c) 359 c = old; 360 } 361 362 static inline long long arch_atomic64_fetch_xor(long long i, atomic64_t *v) 363 { 364 long long old, c = 0; 365 366 while ((old = arch_atomic64_cmpxchg(v, c, c ^ i)) != c) 367 c = old; 368 369 return old; 370 } 371 372 static inline long long arch_atomic64_fetch_add(long long i, atomic64_t *v) 373 { 374 long long old, c = 0; 375 376 while ((old = arch_atomic64_cmpxchg(v, c, c + i)) != c) 377 c = old; 378 379 return old; 380 } 381 382 #define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v)) 383 384 #endif /* _ASM_X86_ATOMIC64_32_H */ 385