1 /* SPDX-License-Identifier: GPL-2.0 */
9 /* The 64-bit atomic type */
13 static __always_inline s64 arch_atomic64_read(const atomic64_t *v)  in arch_atomic64_read()  argument
15 	return __READ_ONCE((v)->counter);  in arch_atomic64_read()
18 static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)  in arch_atomic64_set()  argument
20 	__WRITE_ONCE(v->counter, i);  in arch_atomic64_set()
23 static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)  in arch_atomic64_add()  argument
26 		     : "=m" (v->counter)  in arch_atomic64_add()
27 		     : "er" (i), "m" (v->counter) : "memory");  in arch_atomic64_add()
30 static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)  in arch_atomic64_sub()  argument
33 		     : "=m" (v->counter)  in arch_atomic64_sub()
34 		     : "er" (i), "m" (v->counter) : "memory");  in arch_atomic64_sub()
37 static __always_inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v)  in arch_atomic64_sub_and_test()  argument
39 	return GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, e, "er", i);  in arch_atomic64_sub_and_test()
43 static __always_inline void arch_atomic64_inc(atomic64_t *v)  in arch_atomic64_inc()  argument
46 		     : "=m" (v->counter)  in arch_atomic64_inc()
47 		     : "m" (v->counter) : "memory");  in arch_atomic64_inc()
51 static __always_inline void arch_atomic64_dec(atomic64_t *v)  in arch_atomic64_dec()  argument
54 		     : "=m" (v->counter)  in arch_atomic64_dec()
55 		     : "m" (v->counter) : "memory");  in arch_atomic64_dec()
59 static __always_inline bool arch_atomic64_dec_and_test(atomic64_t *v)  in arch_atomic64_dec_and_test()  argument
61 	return GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, e);  in arch_atomic64_dec_and_test()
65 static __always_inline bool arch_atomic64_inc_and_test(atomic64_t *v)  in arch_atomic64_inc_and_test()  argument
67 	return GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, e);  in arch_atomic64_inc_and_test()
71 static __always_inline bool arch_atomic64_add_negative(s64 i, atomic64_t *v)  in arch_atomic64_add_negative()  argument
73 	return GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, s, "er", i);  in arch_atomic64_add_negative()
77 static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)  in arch_atomic64_add_return()  argument
79 	return i + xadd(&v->counter, i);  in arch_atomic64_add_return()
83 static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)  in arch_atomic64_sub_return()  argument
85 	return arch_atomic64_add_return(-i, v);  in arch_atomic64_sub_return()
89 static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)  in arch_atomic64_fetch_add()  argument
91 	return xadd(&v->counter, i);  in arch_atomic64_fetch_add()
95 static __always_inline s64 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)  in arch_atomic64_fetch_sub()  argument
97 	return xadd(&v->counter, -i);  in arch_atomic64_fetch_sub()
101 static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)  in arch_atomic64_cmpxchg()  argument
103 	return arch_cmpxchg(&v->counter, old, new);  in arch_atomic64_cmpxchg()
107 static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)  in arch_atomic64_try_cmpxchg()  argument
109 	return arch_try_cmpxchg(&v->counter, old, new);  in arch_atomic64_try_cmpxchg()
113 static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new)  in arch_atomic64_xchg()  argument
115 	return arch_xchg(&v->counter, new);  in arch_atomic64_xchg()
119 static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v)  in arch_atomic64_and()  argument
122 			: "+m" (v->counter)  in arch_atomic64_and()
127 static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)  in arch_atomic64_fetch_and()  argument
129 	s64 val = arch_atomic64_read(v);  in arch_atomic64_fetch_and()
132 	} while (!arch_atomic64_try_cmpxchg(v, &val, val & i));  in arch_atomic64_fetch_and()
137 static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v)  in arch_atomic64_or()  argument
140 			: "+m" (v->counter)  in arch_atomic64_or()
145 static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)  in arch_atomic64_fetch_or()  argument
147 	s64 val = arch_atomic64_read(v);  in arch_atomic64_fetch_or()
150 	} while (!arch_atomic64_try_cmpxchg(v, &val, val | i));  in arch_atomic64_fetch_or()
155 static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v)  in arch_atomic64_xor()  argument
158 			: "+m" (v->counter)  in arch_atomic64_xor()
163 static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)  in arch_atomic64_fetch_xor()  argument
165 	s64 val = arch_atomic64_read(v);  in arch_atomic64_fetch_xor()
168 	} while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));  in arch_atomic64_fetch_xor()