1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/plugin.h" 22 23 #if DATA_SIZE == 16 24 # define SUFFIX o 25 # define DATA_TYPE Int128 26 # define BSWAP bswap128 27 # define SHIFT 4 28 #elif DATA_SIZE == 8 29 # define SUFFIX q 30 # define DATA_TYPE aligned_uint64_t 31 # define SDATA_TYPE aligned_int64_t 32 # define BSWAP bswap64 33 # define SHIFT 3 34 #elif DATA_SIZE == 4 35 # define SUFFIX l 36 # define DATA_TYPE uint32_t 37 # define SDATA_TYPE int32_t 38 # define BSWAP bswap32 39 # define SHIFT 2 40 #elif DATA_SIZE == 2 41 # define SUFFIX w 42 # define DATA_TYPE uint16_t 43 # define SDATA_TYPE int16_t 44 # define BSWAP bswap16 45 # define SHIFT 1 46 #elif DATA_SIZE == 1 47 # define SUFFIX b 48 # define DATA_TYPE uint8_t 49 # define SDATA_TYPE int8_t 50 # define BSWAP 51 # define SHIFT 0 52 #else 53 # error unsupported data size 54 #endif 55 56 #if DATA_SIZE >= 4 57 # define ABI_TYPE DATA_TYPE 58 #else 59 # define ABI_TYPE uint32_t 60 #endif 61 62 /* Define host-endian atomic operations. Note that END is used within 63 the ATOMIC_NAME macro, and redefined below. */ 64 #if DATA_SIZE == 1 65 # define END 66 #elif HOST_BIG_ENDIAN 67 # define END _be 68 #else 69 # define END _le 70 #endif 71 72 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr, 73 ABI_TYPE cmpv, ABI_TYPE newv, 74 MemOpIdx oi, uintptr_t retaddr) 75 { 76 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, 77 DATA_SIZE, retaddr); 78 DATA_TYPE ret; 79 80 #if DATA_SIZE == 16 81 ret = atomic16_cmpxchg(haddr, cmpv, newv); 82 #else 83 ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv); 84 #endif 85 ATOMIC_MMU_CLEANUP; 86 atomic_trace_rmw_post(env, addr, oi); 87 return ret; 88 } 89 90 #if DATA_SIZE < 16 91 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val, 92 MemOpIdx oi, uintptr_t retaddr) 93 { 94 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, 95 DATA_SIZE, retaddr); 96 DATA_TYPE ret; 97 98 ret = qatomic_xchg__nocheck(haddr, val); 99 ATOMIC_MMU_CLEANUP; 100 atomic_trace_rmw_post(env, addr, oi); 101 return ret; 102 } 103 104 #define GEN_ATOMIC_HELPER(X) \ 105 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ 106 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ 107 { \ 108 DATA_TYPE *haddr, ret; \ 109 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \ 110 ret = qatomic_##X(haddr, val); \ 111 ATOMIC_MMU_CLEANUP; \ 112 atomic_trace_rmw_post(env, addr, oi); \ 113 return ret; \ 114 } 115 116 GEN_ATOMIC_HELPER(fetch_add) 117 GEN_ATOMIC_HELPER(fetch_and) 118 GEN_ATOMIC_HELPER(fetch_or) 119 GEN_ATOMIC_HELPER(fetch_xor) 120 GEN_ATOMIC_HELPER(add_fetch) 121 GEN_ATOMIC_HELPER(and_fetch) 122 GEN_ATOMIC_HELPER(or_fetch) 123 GEN_ATOMIC_HELPER(xor_fetch) 124 125 #undef GEN_ATOMIC_HELPER 126 127 /* 128 * These helpers are, as a whole, full barriers. Within the helper, 129 * the leading barrier is explicit and the trailing barrier is within 130 * cmpxchg primitive. 131 * 132 * Trace this load + RMW loop as a single RMW op. This way, regardless 133 * of CF_PARALLEL's value, we'll trace just a read and a write. 134 */ 135 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 136 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ 137 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ 138 { \ 139 XDATA_TYPE *haddr, cmp, old, new, val = xval; \ 140 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \ 141 smp_mb(); \ 142 cmp = qatomic_read__nocheck(haddr); \ 143 do { \ 144 old = cmp; new = FN(old, val); \ 145 cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \ 146 } while (cmp != old); \ 147 ATOMIC_MMU_CLEANUP; \ 148 atomic_trace_rmw_post(env, addr, oi); \ 149 return RET; \ 150 } 151 152 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 153 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 154 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 155 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 156 157 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 158 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 159 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 160 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 161 162 #undef GEN_ATOMIC_HELPER_FN 163 #endif /* DATA SIZE < 16 */ 164 165 #undef END 166 167 #if DATA_SIZE > 1 168 169 /* Define reverse-host-endian atomic operations. Note that END is used 170 within the ATOMIC_NAME macro. */ 171 #if HOST_BIG_ENDIAN 172 # define END _le 173 #else 174 # define END _be 175 #endif 176 177 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr, 178 ABI_TYPE cmpv, ABI_TYPE newv, 179 MemOpIdx oi, uintptr_t retaddr) 180 { 181 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, 182 DATA_SIZE, retaddr); 183 DATA_TYPE ret; 184 185 #if DATA_SIZE == 16 186 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); 187 #else 188 ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 189 #endif 190 ATOMIC_MMU_CLEANUP; 191 atomic_trace_rmw_post(env, addr, oi); 192 return BSWAP(ret); 193 } 194 195 #if DATA_SIZE < 16 196 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val, 197 MemOpIdx oi, uintptr_t retaddr) 198 { 199 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, 200 DATA_SIZE, retaddr); 201 ABI_TYPE ret; 202 203 ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); 204 ATOMIC_MMU_CLEANUP; 205 atomic_trace_rmw_post(env, addr, oi); 206 return BSWAP(ret); 207 } 208 209 #define GEN_ATOMIC_HELPER(X) \ 210 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ 211 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ 212 { \ 213 DATA_TYPE *haddr, ret; \ 214 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \ 215 ret = qatomic_##X(haddr, BSWAP(val)); \ 216 ATOMIC_MMU_CLEANUP; \ 217 atomic_trace_rmw_post(env, addr, oi); \ 218 return BSWAP(ret); \ 219 } 220 221 GEN_ATOMIC_HELPER(fetch_and) 222 GEN_ATOMIC_HELPER(fetch_or) 223 GEN_ATOMIC_HELPER(fetch_xor) 224 GEN_ATOMIC_HELPER(and_fetch) 225 GEN_ATOMIC_HELPER(or_fetch) 226 GEN_ATOMIC_HELPER(xor_fetch) 227 228 #undef GEN_ATOMIC_HELPER 229 230 /* These helpers are, as a whole, full barriers. Within the helper, 231 * the leading barrier is explicit and the trailing barrier is within 232 * cmpxchg primitive. 233 * 234 * Trace this load + RMW loop as a single RMW op. This way, regardless 235 * of CF_PARALLEL's value, we'll trace just a read and a write. 236 */ 237 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 238 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ 239 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ 240 { \ 241 XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval; \ 242 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \ 243 smp_mb(); \ 244 ldn = qatomic_read__nocheck(haddr); \ 245 do { \ 246 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 247 ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 248 } while (ldo != ldn); \ 249 ATOMIC_MMU_CLEANUP; \ 250 atomic_trace_rmw_post(env, addr, oi); \ 251 return RET; \ 252 } 253 254 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 255 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 256 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 257 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 258 259 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 260 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 261 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 262 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 263 264 /* Note that for addition, we need to use a separate cmpxchg loop instead 265 of bswaps for the reverse-host-endian helpers. */ 266 #define ADD(X, Y) (X + Y) 267 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 268 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 269 #undef ADD 270 271 #undef GEN_ATOMIC_HELPER_FN 272 #endif /* DATA_SIZE < 16 */ 273 274 #undef END 275 #endif /* DATA_SIZE > 1 */ 276 277 #undef BSWAP 278 #undef ABI_TYPE 279 #undef DATA_TYPE 280 #undef SDATA_TYPE 281 #undef SUFFIX 282 #undef DATA_SIZE 283 #undef SHIFT 284