1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #if DATA_SIZE == 16 22 # define SUFFIX o 23 # define DATA_TYPE Int128 24 # define BSWAP bswap128 25 #elif DATA_SIZE == 8 26 # define SUFFIX q 27 # define DATA_TYPE uint64_t 28 # define SDATA_TYPE int64_t 29 # define BSWAP bswap64 30 #elif DATA_SIZE == 4 31 # define SUFFIX l 32 # define DATA_TYPE uint32_t 33 # define SDATA_TYPE int32_t 34 # define BSWAP bswap32 35 #elif DATA_SIZE == 2 36 # define SUFFIX w 37 # define DATA_TYPE uint16_t 38 # define SDATA_TYPE int16_t 39 # define BSWAP bswap16 40 #elif DATA_SIZE == 1 41 # define SUFFIX b 42 # define DATA_TYPE uint8_t 43 # define SDATA_TYPE int8_t 44 # define BSWAP 45 #else 46 # error unsupported data size 47 #endif 48 49 #if DATA_SIZE >= 4 50 # define ABI_TYPE DATA_TYPE 51 #else 52 # define ABI_TYPE uint32_t 53 #endif 54 55 /* Define host-endian atomic operations. Note that END is used within 56 the ATOMIC_NAME macro, and redefined below. */ 57 #if DATA_SIZE == 1 58 # define END 59 #elif defined(HOST_WORDS_BIGENDIAN) 60 # define END _be 61 #else 62 # define END _le 63 #endif 64 65 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 66 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 67 { 68 ATOMIC_MMU_DECLS; 69 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 70 DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); 71 ATOMIC_MMU_CLEANUP; 72 return ret; 73 } 74 75 #if DATA_SIZE >= 16 76 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 77 { 78 ATOMIC_MMU_DECLS; 79 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 80 __atomic_load(haddr, &val, __ATOMIC_RELAXED); 81 ATOMIC_MMU_CLEANUP; 82 return val; 83 } 84 85 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 86 ABI_TYPE val EXTRA_ARGS) 87 { 88 ATOMIC_MMU_DECLS; 89 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 90 __atomic_store(haddr, &val, __ATOMIC_RELAXED); 91 ATOMIC_MMU_CLEANUP; 92 } 93 #else 94 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 95 ABI_TYPE val EXTRA_ARGS) 96 { 97 ATOMIC_MMU_DECLS; 98 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 99 DATA_TYPE ret = atomic_xchg__nocheck(haddr, val); 100 ATOMIC_MMU_CLEANUP; 101 return ret; 102 } 103 104 #define GEN_ATOMIC_HELPER(X) \ 105 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 106 ABI_TYPE val EXTRA_ARGS) \ 107 { \ 108 ATOMIC_MMU_DECLS; \ 109 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 110 DATA_TYPE ret = atomic_##X(haddr, val); \ 111 ATOMIC_MMU_CLEANUP; \ 112 return ret; \ 113 } 114 115 GEN_ATOMIC_HELPER(fetch_add) 116 GEN_ATOMIC_HELPER(fetch_and) 117 GEN_ATOMIC_HELPER(fetch_or) 118 GEN_ATOMIC_HELPER(fetch_xor) 119 GEN_ATOMIC_HELPER(add_fetch) 120 GEN_ATOMIC_HELPER(and_fetch) 121 GEN_ATOMIC_HELPER(or_fetch) 122 GEN_ATOMIC_HELPER(xor_fetch) 123 124 #undef GEN_ATOMIC_HELPER 125 126 /* These helpers are, as a whole, full barriers. Within the helper, 127 * the leading barrier is explicit and the trailing barrier is within 128 * cmpxchg primitive. 129 */ 130 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 131 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 132 ABI_TYPE xval EXTRA_ARGS) \ 133 { \ 134 ATOMIC_MMU_DECLS; \ 135 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 136 XDATA_TYPE cmp, old, new, val = xval; \ 137 smp_mb(); \ 138 cmp = atomic_read__nocheck(haddr); \ 139 do { \ 140 old = cmp; new = FN(old, val); \ 141 cmp = atomic_cmpxchg__nocheck(haddr, old, new); \ 142 } while (cmp != old); \ 143 ATOMIC_MMU_CLEANUP; \ 144 return RET; \ 145 } 146 147 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 148 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 149 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 150 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 151 152 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 153 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 154 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 155 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 156 157 #undef GEN_ATOMIC_HELPER_FN 158 #endif /* DATA SIZE >= 16 */ 159 160 #undef END 161 162 #if DATA_SIZE > 1 163 164 /* Define reverse-host-endian atomic operations. Note that END is used 165 within the ATOMIC_NAME macro. */ 166 #ifdef HOST_WORDS_BIGENDIAN 167 # define END _le 168 #else 169 # define END _be 170 #endif 171 172 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 173 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 174 { 175 ATOMIC_MMU_DECLS; 176 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 177 DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 178 ATOMIC_MMU_CLEANUP; 179 return BSWAP(ret); 180 } 181 182 #if DATA_SIZE >= 16 183 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 184 { 185 ATOMIC_MMU_DECLS; 186 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 187 __atomic_load(haddr, &val, __ATOMIC_RELAXED); 188 ATOMIC_MMU_CLEANUP; 189 return BSWAP(val); 190 } 191 192 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 193 ABI_TYPE val EXTRA_ARGS) 194 { 195 ATOMIC_MMU_DECLS; 196 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 197 val = BSWAP(val); 198 __atomic_store(haddr, &val, __ATOMIC_RELAXED); 199 ATOMIC_MMU_CLEANUP; 200 } 201 #else 202 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 203 ABI_TYPE val EXTRA_ARGS) 204 { 205 ATOMIC_MMU_DECLS; 206 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 207 ABI_TYPE ret = atomic_xchg__nocheck(haddr, BSWAP(val)); 208 ATOMIC_MMU_CLEANUP; 209 return BSWAP(ret); 210 } 211 212 #define GEN_ATOMIC_HELPER(X) \ 213 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 214 ABI_TYPE val EXTRA_ARGS) \ 215 { \ 216 ATOMIC_MMU_DECLS; \ 217 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 218 DATA_TYPE ret = atomic_##X(haddr, BSWAP(val)); \ 219 ATOMIC_MMU_CLEANUP; \ 220 return BSWAP(ret); \ 221 } 222 223 GEN_ATOMIC_HELPER(fetch_and) 224 GEN_ATOMIC_HELPER(fetch_or) 225 GEN_ATOMIC_HELPER(fetch_xor) 226 GEN_ATOMIC_HELPER(and_fetch) 227 GEN_ATOMIC_HELPER(or_fetch) 228 GEN_ATOMIC_HELPER(xor_fetch) 229 230 #undef GEN_ATOMIC_HELPER 231 232 /* These helpers are, as a whole, full barriers. Within the helper, 233 * the leading barrier is explicit and the trailing barrier is within 234 * cmpxchg primitive. 235 */ 236 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 237 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 238 ABI_TYPE xval EXTRA_ARGS) \ 239 { \ 240 ATOMIC_MMU_DECLS; \ 241 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 242 XDATA_TYPE ldo, ldn, old, new, val = xval; \ 243 smp_mb(); \ 244 ldn = atomic_read__nocheck(haddr); \ 245 do { \ 246 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 247 ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 248 } while (ldo != ldn); \ 249 ATOMIC_MMU_CLEANUP; \ 250 return RET; \ 251 } 252 253 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 254 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 255 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 256 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 257 258 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 259 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 260 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 261 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 262 263 /* Note that for addition, we need to use a separate cmpxchg loop instead 264 of bswaps for the reverse-host-endian helpers. */ 265 #define ADD(X, Y) (X + Y) 266 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 267 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 268 #undef ADD 269 270 #undef GEN_ATOMIC_HELPER_FN 271 #endif /* DATA_SIZE >= 16 */ 272 273 #undef END 274 #endif /* DATA_SIZE > 1 */ 275 276 #undef BSWAP 277 #undef ABI_TYPE 278 #undef DATA_TYPE 279 #undef SDATA_TYPE 280 #undef SUFFIX 281 #undef DATA_SIZE 282