1 #ifndef __ARCH_S390_ATOMIC__ 2 #define __ARCH_S390_ATOMIC__ 3 4 /* 5 * Copyright 1999,2009 IBM Corp. 6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, 7 * Denis Joseph Barrow, 8 * Arnd Bergmann <arndb@de.ibm.com>, 9 * 10 * Atomic operations that C can't guarantee us. 11 * Useful for resource counting etc. 12 * s390 uses 'Compare And Swap' for atomicity in SMP enviroment. 13 * 14 */ 15 16 #include <linux/compiler.h> 17 #include <linux/types.h> 18 #include <asm/system.h> 19 20 #define ATOMIC_INIT(i) { (i) } 21 22 #define __CS_LOOP(ptr, op_val, op_string) ({ \ 23 int old_val, new_val; \ 24 asm volatile( \ 25 " l %0,%2\n" \ 26 "0: lr %1,%0\n" \ 27 op_string " %1,%3\n" \ 28 " cs %0,%1,%2\n" \ 29 " jl 0b" \ 30 : "=&d" (old_val), "=&d" (new_val), \ 31 "=Q" (((atomic_t *)(ptr))->counter) \ 32 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \ 33 : "cc", "memory"); \ 34 new_val; \ 35 }) 36 37 static inline int atomic_read(const atomic_t *v) 38 { 39 int c; 40 41 asm volatile( 42 " l %0,%1\n" 43 : "=d" (c) : "Q" (v->counter)); 44 return c; 45 } 46 47 static inline void atomic_set(atomic_t *v, int i) 48 { 49 asm volatile( 50 " st %1,%0\n" 51 : "=Q" (v->counter) : "d" (i)); 52 } 53 54 static inline int atomic_add_return(int i, atomic_t *v) 55 { 56 return __CS_LOOP(v, i, "ar"); 57 } 58 #define atomic_add(_i, _v) atomic_add_return(_i, _v) 59 #define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0) 60 #define atomic_inc(_v) atomic_add_return(1, _v) 61 #define atomic_inc_return(_v) atomic_add_return(1, _v) 62 #define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0) 63 64 static inline int atomic_sub_return(int i, atomic_t *v) 65 { 66 return __CS_LOOP(v, i, "sr"); 67 } 68 #define atomic_sub(_i, _v) atomic_sub_return(_i, _v) 69 #define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0) 70 #define atomic_dec(_v) atomic_sub_return(1, _v) 71 #define atomic_dec_return(_v) atomic_sub_return(1, _v) 72 #define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0) 73 74 static inline void atomic_clear_mask(unsigned long mask, atomic_t *v) 75 { 76 __CS_LOOP(v, ~mask, "nr"); 77 } 78 79 static inline void atomic_set_mask(unsigned long mask, atomic_t *v) 80 { 81 __CS_LOOP(v, mask, "or"); 82 } 83 84 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 85 86 static inline int atomic_cmpxchg(atomic_t *v, int old, int new) 87 { 88 asm volatile( 89 " cs %0,%2,%1" 90 : "+d" (old), "=Q" (v->counter) 91 : "d" (new), "Q" (v->counter) 92 : "cc", "memory"); 93 return old; 94 } 95 96 static inline int atomic_add_unless(atomic_t *v, int a, int u) 97 { 98 int c, old; 99 c = atomic_read(v); 100 for (;;) { 101 if (unlikely(c == u)) 102 break; 103 old = atomic_cmpxchg(v, c, c + a); 104 if (likely(old == c)) 105 break; 106 c = old; 107 } 108 return c != u; 109 } 110 111 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 112 113 #undef __CS_LOOP 114 115 #define ATOMIC64_INIT(i) { (i) } 116 117 #ifdef CONFIG_64BIT 118 119 #define __CSG_LOOP(ptr, op_val, op_string) ({ \ 120 long long old_val, new_val; \ 121 asm volatile( \ 122 " lg %0,%2\n" \ 123 "0: lgr %1,%0\n" \ 124 op_string " %1,%3\n" \ 125 " csg %0,%1,%2\n" \ 126 " jl 0b" \ 127 : "=&d" (old_val), "=&d" (new_val), \ 128 "=Q" (((atomic_t *)(ptr))->counter) \ 129 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \ 130 : "cc", "memory"); \ 131 new_val; \ 132 }) 133 134 static inline long long atomic64_read(const atomic64_t *v) 135 { 136 long long c; 137 138 asm volatile( 139 " lg %0,%1\n" 140 : "=d" (c) : "Q" (v->counter)); 141 return c; 142 } 143 144 static inline void atomic64_set(atomic64_t *v, long long i) 145 { 146 asm volatile( 147 " stg %1,%0\n" 148 : "=Q" (v->counter) : "d" (i)); 149 } 150 151 static inline long long atomic64_add_return(long long i, atomic64_t *v) 152 { 153 return __CSG_LOOP(v, i, "agr"); 154 } 155 156 static inline long long atomic64_sub_return(long long i, atomic64_t *v) 157 { 158 return __CSG_LOOP(v, i, "sgr"); 159 } 160 161 static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v) 162 { 163 __CSG_LOOP(v, ~mask, "ngr"); 164 } 165 166 static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v) 167 { 168 __CSG_LOOP(v, mask, "ogr"); 169 } 170 171 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 172 173 static inline long long atomic64_cmpxchg(atomic64_t *v, 174 long long old, long long new) 175 { 176 asm volatile( 177 " csg %0,%2,%1" 178 : "+d" (old), "=Q" (v->counter) 179 : "d" (new), "Q" (v->counter) 180 : "cc", "memory"); 181 return old; 182 } 183 184 #undef __CSG_LOOP 185 186 #else /* CONFIG_64BIT */ 187 188 typedef struct { 189 long long counter; 190 } atomic64_t; 191 192 static inline long long atomic64_read(const atomic64_t *v) 193 { 194 register_pair rp; 195 196 asm volatile( 197 " lm %0,%N0,%1" 198 : "=&d" (rp) : "Q" (v->counter) ); 199 return rp.pair; 200 } 201 202 static inline void atomic64_set(atomic64_t *v, long long i) 203 { 204 register_pair rp = {.pair = i}; 205 206 asm volatile( 207 " stm %1,%N1,%0" 208 : "=Q" (v->counter) : "d" (rp) ); 209 } 210 211 static inline long long atomic64_xchg(atomic64_t *v, long long new) 212 { 213 register_pair rp_new = {.pair = new}; 214 register_pair rp_old; 215 216 asm volatile( 217 " lm %0,%N0,%1\n" 218 "0: cds %0,%2,%1\n" 219 " jl 0b\n" 220 : "=&d" (rp_old), "=Q" (v->counter) 221 : "d" (rp_new), "Q" (v->counter) 222 : "cc"); 223 return rp_old.pair; 224 } 225 226 static inline long long atomic64_cmpxchg(atomic64_t *v, 227 long long old, long long new) 228 { 229 register_pair rp_old = {.pair = old}; 230 register_pair rp_new = {.pair = new}; 231 232 asm volatile( 233 " cds %0,%2,%1" 234 : "+&d" (rp_old), "=Q" (v->counter) 235 : "d" (rp_new), "Q" (v->counter) 236 : "cc"); 237 return rp_old.pair; 238 } 239 240 241 static inline long long atomic64_add_return(long long i, atomic64_t *v) 242 { 243 long long old, new; 244 245 do { 246 old = atomic64_read(v); 247 new = old + i; 248 } while (atomic64_cmpxchg(v, old, new) != old); 249 return new; 250 } 251 252 static inline long long atomic64_sub_return(long long i, atomic64_t *v) 253 { 254 long long old, new; 255 256 do { 257 old = atomic64_read(v); 258 new = old - i; 259 } while (atomic64_cmpxchg(v, old, new) != old); 260 return new; 261 } 262 263 static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v) 264 { 265 long long old, new; 266 267 do { 268 old = atomic64_read(v); 269 new = old | mask; 270 } while (atomic64_cmpxchg(v, old, new) != old); 271 } 272 273 static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v) 274 { 275 long long old, new; 276 277 do { 278 old = atomic64_read(v); 279 new = old & mask; 280 } while (atomic64_cmpxchg(v, old, new) != old); 281 } 282 283 #endif /* CONFIG_64BIT */ 284 285 static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) 286 { 287 long long c, old; 288 289 c = atomic64_read(v); 290 for (;;) { 291 if (unlikely(c == u)) 292 break; 293 old = atomic64_cmpxchg(v, c, c + a); 294 if (likely(old == c)) 295 break; 296 c = old; 297 } 298 return c != u; 299 } 300 301 static inline long long atomic64_dec_if_positive(atomic64_t *v) 302 { 303 long long c, old, dec; 304 305 c = atomic64_read(v); 306 for (;;) { 307 dec = c - 1; 308 if (unlikely(dec < 0)) 309 break; 310 old = atomic64_cmpxchg((v), c, dec); 311 if (likely(old == c)) 312 break; 313 c = old; 314 } 315 return dec; 316 } 317 318 #define atomic64_add(_i, _v) atomic64_add_return(_i, _v) 319 #define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0) 320 #define atomic64_inc(_v) atomic64_add_return(1, _v) 321 #define atomic64_inc_return(_v) atomic64_add_return(1, _v) 322 #define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0) 323 #define atomic64_sub(_i, _v) atomic64_sub_return(_i, _v) 324 #define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0) 325 #define atomic64_dec(_v) atomic64_sub_return(1, _v) 326 #define atomic64_dec_return(_v) atomic64_sub_return(1, _v) 327 #define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0) 328 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) 329 330 #define smp_mb__before_atomic_dec() smp_mb() 331 #define smp_mb__after_atomic_dec() smp_mb() 332 #define smp_mb__before_atomic_inc() smp_mb() 333 #define smp_mb__after_atomic_inc() smp_mb() 334 335 #include <asm-generic/atomic-long.h> 336 337 #endif /* __ARCH_S390_ATOMIC__ */ 338