1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/plugin.h" 22 23 #if DATA_SIZE == 16 24 # define SUFFIX o 25 # define DATA_TYPE Int128 26 # define BSWAP bswap128 27 # define SHIFT 4 28 #elif DATA_SIZE == 8 29 # define SUFFIX q 30 # define DATA_TYPE aligned_uint64_t 31 # define SDATA_TYPE aligned_int64_t 32 # define BSWAP bswap64 33 # define SHIFT 3 34 #elif DATA_SIZE == 4 35 # define SUFFIX l 36 # define DATA_TYPE uint32_t 37 # define SDATA_TYPE int32_t 38 # define BSWAP bswap32 39 # define SHIFT 2 40 #elif DATA_SIZE == 2 41 # define SUFFIX w 42 # define DATA_TYPE uint16_t 43 # define SDATA_TYPE int16_t 44 # define BSWAP bswap16 45 # define SHIFT 1 46 #elif DATA_SIZE == 1 47 # define SUFFIX b 48 # define DATA_TYPE uint8_t 49 # define SDATA_TYPE int8_t 50 # define BSWAP 51 # define SHIFT 0 52 #else 53 # error unsupported data size 54 #endif 55 56 #if DATA_SIZE == 16 57 # define VALUE_LOW(val) int128_getlo(val) 58 # define VALUE_HIGH(val) int128_gethi(val) 59 #else 60 # define VALUE_LOW(val) val 61 # define VALUE_HIGH(val) 0 62 #endif 63 64 #if DATA_SIZE >= 4 65 # define ABI_TYPE DATA_TYPE 66 #else 67 # define ABI_TYPE uint32_t 68 #endif 69 70 /* Define host-endian atomic operations. Note that END is used within 71 the ATOMIC_NAME macro, and redefined below. */ 72 #if DATA_SIZE == 1 73 # define END 74 #elif HOST_BIG_ENDIAN 75 # define END _be 76 #else 77 # define END _le 78 #endif 79 80 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr, 81 ABI_TYPE cmpv, ABI_TYPE newv, 82 MemOpIdx oi, uintptr_t retaddr) 83 { 84 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, 85 DATA_SIZE, retaddr); 86 DATA_TYPE ret; 87 88 #if DATA_SIZE == 16 89 ret = atomic16_cmpxchg(haddr, cmpv, newv); 90 #else 91 ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv); 92 #endif 93 ATOMIC_MMU_CLEANUP; 94 atomic_trace_rmw_post(env, addr, 95 VALUE_LOW(ret), 96 VALUE_HIGH(ret), 97 VALUE_LOW(newv), 98 VALUE_HIGH(newv), 99 oi); 100 return ret; 101 } 102 103 #if DATA_SIZE < 16 104 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val, 105 MemOpIdx oi, uintptr_t retaddr) 106 { 107 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, 108 DATA_SIZE, retaddr); 109 DATA_TYPE ret; 110 111 ret = qatomic_xchg__nocheck(haddr, val); 112 ATOMIC_MMU_CLEANUP; 113 atomic_trace_rmw_post(env, addr, 114 VALUE_LOW(ret), 115 VALUE_HIGH(ret), 116 VALUE_LOW(val), 117 VALUE_HIGH(val), 118 oi); 119 return ret; 120 } 121 122 #define GEN_ATOMIC_HELPER(X) \ 123 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ 124 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ 125 { \ 126 DATA_TYPE *haddr, ret; \ 127 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \ 128 ret = qatomic_##X(haddr, val); \ 129 ATOMIC_MMU_CLEANUP; \ 130 atomic_trace_rmw_post(env, addr, \ 131 VALUE_LOW(ret), \ 132 VALUE_HIGH(ret), \ 133 VALUE_LOW(val), \ 134 VALUE_HIGH(val), \ 135 oi); \ 136 return ret; \ 137 } 138 139 GEN_ATOMIC_HELPER(fetch_add) 140 GEN_ATOMIC_HELPER(fetch_and) 141 GEN_ATOMIC_HELPER(fetch_or) 142 GEN_ATOMIC_HELPER(fetch_xor) 143 GEN_ATOMIC_HELPER(add_fetch) 144 GEN_ATOMIC_HELPER(and_fetch) 145 GEN_ATOMIC_HELPER(or_fetch) 146 GEN_ATOMIC_HELPER(xor_fetch) 147 148 #undef GEN_ATOMIC_HELPER 149 150 /* 151 * These helpers are, as a whole, full barriers. Within the helper, 152 * the leading barrier is explicit and the trailing barrier is within 153 * cmpxchg primitive. 154 * 155 * Trace this load + RMW loop as a single RMW op. This way, regardless 156 * of CF_PARALLEL's value, we'll trace just a read and a write. 157 */ 158 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 159 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ 160 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ 161 { \ 162 XDATA_TYPE *haddr, cmp, old, new, val = xval; \ 163 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \ 164 smp_mb(); \ 165 cmp = qatomic_read__nocheck(haddr); \ 166 do { \ 167 old = cmp; new = FN(old, val); \ 168 cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \ 169 } while (cmp != old); \ 170 ATOMIC_MMU_CLEANUP; \ 171 atomic_trace_rmw_post(env, addr, \ 172 VALUE_LOW(old), \ 173 VALUE_HIGH(old), \ 174 VALUE_LOW(xval), \ 175 VALUE_HIGH(xval), \ 176 oi); \ 177 return RET; \ 178 } 179 180 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 181 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 182 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 183 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 184 185 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 186 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 187 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 188 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 189 190 #undef GEN_ATOMIC_HELPER_FN 191 #endif /* DATA SIZE < 16 */ 192 193 #undef END 194 195 #if DATA_SIZE > 1 196 197 /* Define reverse-host-endian atomic operations. Note that END is used 198 within the ATOMIC_NAME macro. */ 199 #if HOST_BIG_ENDIAN 200 # define END _le 201 #else 202 # define END _be 203 #endif 204 205 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr, 206 ABI_TYPE cmpv, ABI_TYPE newv, 207 MemOpIdx oi, uintptr_t retaddr) 208 { 209 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, 210 DATA_SIZE, retaddr); 211 DATA_TYPE ret; 212 213 #if DATA_SIZE == 16 214 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); 215 #else 216 ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 217 #endif 218 ATOMIC_MMU_CLEANUP; 219 atomic_trace_rmw_post(env, addr, 220 VALUE_LOW(ret), 221 VALUE_HIGH(ret), 222 VALUE_LOW(newv), 223 VALUE_HIGH(newv), 224 oi); 225 return BSWAP(ret); 226 } 227 228 #if DATA_SIZE < 16 229 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val, 230 MemOpIdx oi, uintptr_t retaddr) 231 { 232 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, 233 DATA_SIZE, retaddr); 234 ABI_TYPE ret; 235 236 ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); 237 ATOMIC_MMU_CLEANUP; 238 atomic_trace_rmw_post(env, addr, 239 VALUE_LOW(ret), 240 VALUE_HIGH(ret), 241 VALUE_LOW(val), 242 VALUE_HIGH(val), 243 oi); 244 return BSWAP(ret); 245 } 246 247 #define GEN_ATOMIC_HELPER(X) \ 248 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ 249 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ 250 { \ 251 DATA_TYPE *haddr, ret; \ 252 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \ 253 ret = qatomic_##X(haddr, BSWAP(val)); \ 254 ATOMIC_MMU_CLEANUP; \ 255 atomic_trace_rmw_post(env, addr, \ 256 VALUE_LOW(ret), \ 257 VALUE_HIGH(ret), \ 258 VALUE_LOW(val), \ 259 VALUE_HIGH(val), \ 260 oi); \ 261 return BSWAP(ret); \ 262 } 263 264 GEN_ATOMIC_HELPER(fetch_and) 265 GEN_ATOMIC_HELPER(fetch_or) 266 GEN_ATOMIC_HELPER(fetch_xor) 267 GEN_ATOMIC_HELPER(and_fetch) 268 GEN_ATOMIC_HELPER(or_fetch) 269 GEN_ATOMIC_HELPER(xor_fetch) 270 271 #undef GEN_ATOMIC_HELPER 272 273 /* These helpers are, as a whole, full barriers. Within the helper, 274 * the leading barrier is explicit and the trailing barrier is within 275 * cmpxchg primitive. 276 * 277 * Trace this load + RMW loop as a single RMW op. This way, regardless 278 * of CF_PARALLEL's value, we'll trace just a read and a write. 279 */ 280 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 281 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ 282 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ 283 { \ 284 XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval; \ 285 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \ 286 smp_mb(); \ 287 ldn = qatomic_read__nocheck(haddr); \ 288 do { \ 289 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 290 ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 291 } while (ldo != ldn); \ 292 ATOMIC_MMU_CLEANUP; \ 293 atomic_trace_rmw_post(env, addr, \ 294 VALUE_LOW(old), \ 295 VALUE_HIGH(old), \ 296 VALUE_LOW(xval), \ 297 VALUE_HIGH(xval), \ 298 oi); \ 299 return RET; \ 300 } 301 302 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 303 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 304 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 305 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 306 307 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 308 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 309 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 310 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 311 312 /* Note that for addition, we need to use a separate cmpxchg loop instead 313 of bswaps for the reverse-host-endian helpers. */ 314 #define ADD(X, Y) (X + Y) 315 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 316 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 317 #undef ADD 318 319 #undef GEN_ATOMIC_HELPER_FN 320 #endif /* DATA_SIZE < 16 */ 321 322 #undef END 323 #endif /* DATA_SIZE > 1 */ 324 325 #undef BSWAP 326 #undef ABI_TYPE 327 #undef DATA_TYPE 328 #undef SDATA_TYPE 329 #undef SUFFIX 330 #undef DATA_SIZE 331 #undef SHIFT 332 #undef VALUE_LOW 333 #undef VALUE_HIGH 334