1 /* 2 * Copyright IBM Corp. 1999, 2009 3 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, 4 * Denis Joseph Barrow, 5 * Arnd Bergmann <arndb@de.ibm.com>, 6 * 7 * Atomic operations that C can't guarantee us. 8 * Useful for resource counting etc. 9 * s390 uses 'Compare And Swap' for atomicity in SMP environment. 10 * 11 */ 12 13 #ifndef __ARCH_S390_ATOMIC__ 14 #define __ARCH_S390_ATOMIC__ 15 16 #include <linux/compiler.h> 17 #include <linux/types.h> 18 #include <asm/cmpxchg.h> 19 20 #define ATOMIC_INIT(i) { (i) } 21 22 #define __CS_LOOP(ptr, op_val, op_string) ({ \ 23 int old_val, new_val; \ 24 asm volatile( \ 25 " l %0,%2\n" \ 26 "0: lr %1,%0\n" \ 27 op_string " %1,%3\n" \ 28 " cs %0,%1,%2\n" \ 29 " jl 0b" \ 30 : "=&d" (old_val), "=&d" (new_val), \ 31 "=Q" (((atomic_t *)(ptr))->counter) \ 32 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \ 33 : "cc", "memory"); \ 34 new_val; \ 35 }) 36 37 static inline int atomic_read(const atomic_t *v) 38 { 39 int c; 40 41 asm volatile( 42 " l %0,%1\n" 43 : "=d" (c) : "Q" (v->counter)); 44 return c; 45 } 46 47 static inline void atomic_set(atomic_t *v, int i) 48 { 49 asm volatile( 50 " st %1,%0\n" 51 : "=Q" (v->counter) : "d" (i)); 52 } 53 54 static inline int atomic_add_return(int i, atomic_t *v) 55 { 56 return __CS_LOOP(v, i, "ar"); 57 } 58 #define atomic_add(_i, _v) atomic_add_return(_i, _v) 59 #define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0) 60 #define atomic_inc(_v) atomic_add_return(1, _v) 61 #define atomic_inc_return(_v) atomic_add_return(1, _v) 62 #define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0) 63 64 static inline int atomic_sub_return(int i, atomic_t *v) 65 { 66 return __CS_LOOP(v, i, "sr"); 67 } 68 #define atomic_sub(_i, _v) atomic_sub_return(_i, _v) 69 #define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0) 70 #define atomic_dec(_v) atomic_sub_return(1, _v) 71 #define atomic_dec_return(_v) atomic_sub_return(1, _v) 72 #define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0) 73 74 static inline void atomic_clear_mask(unsigned long mask, atomic_t *v) 75 { 76 __CS_LOOP(v, ~mask, "nr"); 77 } 78 79 static inline void atomic_set_mask(unsigned long mask, atomic_t *v) 80 { 81 __CS_LOOP(v, mask, "or"); 82 } 83 84 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 85 86 static inline int atomic_cmpxchg(atomic_t *v, int old, int new) 87 { 88 asm volatile( 89 " cs %0,%2,%1" 90 : "+d" (old), "=Q" (v->counter) 91 : "d" (new), "Q" (v->counter) 92 : "cc", "memory"); 93 return old; 94 } 95 96 static inline int __atomic_add_unless(atomic_t *v, int a, int u) 97 { 98 int c, old; 99 c = atomic_read(v); 100 for (;;) { 101 if (unlikely(c == u)) 102 break; 103 old = atomic_cmpxchg(v, c, c + a); 104 if (likely(old == c)) 105 break; 106 c = old; 107 } 108 return c; 109 } 110 111 112 #undef __CS_LOOP 113 114 #define ATOMIC64_INIT(i) { (i) } 115 116 #ifdef CONFIG_64BIT 117 118 #define __CSG_LOOP(ptr, op_val, op_string) ({ \ 119 long long old_val, new_val; \ 120 asm volatile( \ 121 " lg %0,%2\n" \ 122 "0: lgr %1,%0\n" \ 123 op_string " %1,%3\n" \ 124 " csg %0,%1,%2\n" \ 125 " jl 0b" \ 126 : "=&d" (old_val), "=&d" (new_val), \ 127 "=Q" (((atomic_t *)(ptr))->counter) \ 128 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \ 129 : "cc", "memory"); \ 130 new_val; \ 131 }) 132 133 static inline long long atomic64_read(const atomic64_t *v) 134 { 135 long long c; 136 137 asm volatile( 138 " lg %0,%1\n" 139 : "=d" (c) : "Q" (v->counter)); 140 return c; 141 } 142 143 static inline void atomic64_set(atomic64_t *v, long long i) 144 { 145 asm volatile( 146 " stg %1,%0\n" 147 : "=Q" (v->counter) : "d" (i)); 148 } 149 150 static inline long long atomic64_add_return(long long i, atomic64_t *v) 151 { 152 return __CSG_LOOP(v, i, "agr"); 153 } 154 155 static inline long long atomic64_sub_return(long long i, atomic64_t *v) 156 { 157 return __CSG_LOOP(v, i, "sgr"); 158 } 159 160 static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v) 161 { 162 __CSG_LOOP(v, ~mask, "ngr"); 163 } 164 165 static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v) 166 { 167 __CSG_LOOP(v, mask, "ogr"); 168 } 169 170 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 171 172 static inline long long atomic64_cmpxchg(atomic64_t *v, 173 long long old, long long new) 174 { 175 asm volatile( 176 " csg %0,%2,%1" 177 : "+d" (old), "=Q" (v->counter) 178 : "d" (new), "Q" (v->counter) 179 : "cc", "memory"); 180 return old; 181 } 182 183 #undef __CSG_LOOP 184 185 #else /* CONFIG_64BIT */ 186 187 typedef struct { 188 long long counter; 189 } atomic64_t; 190 191 static inline long long atomic64_read(const atomic64_t *v) 192 { 193 register_pair rp; 194 195 asm volatile( 196 " lm %0,%N0,%1" 197 : "=&d" (rp) : "Q" (v->counter) ); 198 return rp.pair; 199 } 200 201 static inline void atomic64_set(atomic64_t *v, long long i) 202 { 203 register_pair rp = {.pair = i}; 204 205 asm volatile( 206 " stm %1,%N1,%0" 207 : "=Q" (v->counter) : "d" (rp) ); 208 } 209 210 static inline long long atomic64_xchg(atomic64_t *v, long long new) 211 { 212 register_pair rp_new = {.pair = new}; 213 register_pair rp_old; 214 215 asm volatile( 216 " lm %0,%N0,%1\n" 217 "0: cds %0,%2,%1\n" 218 " jl 0b\n" 219 : "=&d" (rp_old), "=Q" (v->counter) 220 : "d" (rp_new), "Q" (v->counter) 221 : "cc"); 222 return rp_old.pair; 223 } 224 225 static inline long long atomic64_cmpxchg(atomic64_t *v, 226 long long old, long long new) 227 { 228 register_pair rp_old = {.pair = old}; 229 register_pair rp_new = {.pair = new}; 230 231 asm volatile( 232 " cds %0,%2,%1" 233 : "+&d" (rp_old), "=Q" (v->counter) 234 : "d" (rp_new), "Q" (v->counter) 235 : "cc"); 236 return rp_old.pair; 237 } 238 239 240 static inline long long atomic64_add_return(long long i, atomic64_t *v) 241 { 242 long long old, new; 243 244 do { 245 old = atomic64_read(v); 246 new = old + i; 247 } while (atomic64_cmpxchg(v, old, new) != old); 248 return new; 249 } 250 251 static inline long long atomic64_sub_return(long long i, atomic64_t *v) 252 { 253 long long old, new; 254 255 do { 256 old = atomic64_read(v); 257 new = old - i; 258 } while (atomic64_cmpxchg(v, old, new) != old); 259 return new; 260 } 261 262 static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v) 263 { 264 long long old, new; 265 266 do { 267 old = atomic64_read(v); 268 new = old | mask; 269 } while (atomic64_cmpxchg(v, old, new) != old); 270 } 271 272 static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v) 273 { 274 long long old, new; 275 276 do { 277 old = atomic64_read(v); 278 new = old & mask; 279 } while (atomic64_cmpxchg(v, old, new) != old); 280 } 281 282 #endif /* CONFIG_64BIT */ 283 284 static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) 285 { 286 long long c, old; 287 288 c = atomic64_read(v); 289 for (;;) { 290 if (unlikely(c == u)) 291 break; 292 old = atomic64_cmpxchg(v, c, c + a); 293 if (likely(old == c)) 294 break; 295 c = old; 296 } 297 return c != u; 298 } 299 300 static inline long long atomic64_dec_if_positive(atomic64_t *v) 301 { 302 long long c, old, dec; 303 304 c = atomic64_read(v); 305 for (;;) { 306 dec = c - 1; 307 if (unlikely(dec < 0)) 308 break; 309 old = atomic64_cmpxchg((v), c, dec); 310 if (likely(old == c)) 311 break; 312 c = old; 313 } 314 return dec; 315 } 316 317 #define atomic64_add(_i, _v) atomic64_add_return(_i, _v) 318 #define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0) 319 #define atomic64_inc(_v) atomic64_add_return(1, _v) 320 #define atomic64_inc_return(_v) atomic64_add_return(1, _v) 321 #define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0) 322 #define atomic64_sub(_i, _v) atomic64_sub_return(_i, _v) 323 #define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0) 324 #define atomic64_dec(_v) atomic64_sub_return(1, _v) 325 #define atomic64_dec_return(_v) atomic64_sub_return(1, _v) 326 #define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0) 327 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) 328 329 #define smp_mb__before_atomic_dec() smp_mb() 330 #define smp_mb__after_atomic_dec() smp_mb() 331 #define smp_mb__before_atomic_inc() smp_mb() 332 #define smp_mb__after_atomic_inc() smp_mb() 333 334 #endif /* __ARCH_S390_ATOMIC__ */ 335