1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "trace/mem.h" 22 23 #if DATA_SIZE == 16 24 # define SUFFIX o 25 # define DATA_TYPE Int128 26 # define BSWAP bswap128 27 # define SHIFT 4 28 #elif DATA_SIZE == 8 29 # define SUFFIX q 30 # define DATA_TYPE uint64_t 31 # define SDATA_TYPE int64_t 32 # define BSWAP bswap64 33 # define SHIFT 3 34 #elif DATA_SIZE == 4 35 # define SUFFIX l 36 # define DATA_TYPE uint32_t 37 # define SDATA_TYPE int32_t 38 # define BSWAP bswap32 39 # define SHIFT 2 40 #elif DATA_SIZE == 2 41 # define SUFFIX w 42 # define DATA_TYPE uint16_t 43 # define SDATA_TYPE int16_t 44 # define BSWAP bswap16 45 # define SHIFT 1 46 #elif DATA_SIZE == 1 47 # define SUFFIX b 48 # define DATA_TYPE uint8_t 49 # define SDATA_TYPE int8_t 50 # define BSWAP 51 # define SHIFT 0 52 #else 53 # error unsupported data size 54 #endif 55 56 #if DATA_SIZE >= 4 57 # define ABI_TYPE DATA_TYPE 58 #else 59 # define ABI_TYPE uint32_t 60 #endif 61 62 #define ATOMIC_TRACE_RMW do { \ 63 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \ 64 \ 65 trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \ 66 trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, \ 67 info | TRACE_MEM_ST); \ 68 } while (0) 69 70 #define ATOMIC_TRACE_LD do { \ 71 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \ 72 \ 73 trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \ 74 } while (0) 75 76 # define ATOMIC_TRACE_ST do { \ 77 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \ 78 \ 79 trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \ 80 } while (0) 81 82 /* Define host-endian atomic operations. Note that END is used within 83 the ATOMIC_NAME macro, and redefined below. */ 84 #if DATA_SIZE == 1 85 # define END 86 # define MEND _be /* either le or be would be fine */ 87 #elif defined(HOST_WORDS_BIGENDIAN) 88 # define END _be 89 # define MEND _be 90 #else 91 # define END _le 92 # define MEND _le 93 #endif 94 95 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 96 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 97 { 98 ATOMIC_MMU_DECLS; 99 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 100 DATA_TYPE ret; 101 102 ATOMIC_TRACE_RMW; 103 ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); 104 ATOMIC_MMU_CLEANUP; 105 return ret; 106 } 107 108 #if DATA_SIZE >= 16 109 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 110 { 111 ATOMIC_MMU_DECLS; 112 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 113 114 ATOMIC_TRACE_LD; 115 __atomic_load(haddr, &val, __ATOMIC_RELAXED); 116 ATOMIC_MMU_CLEANUP; 117 return val; 118 } 119 120 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 121 ABI_TYPE val EXTRA_ARGS) 122 { 123 ATOMIC_MMU_DECLS; 124 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 125 126 ATOMIC_TRACE_ST; 127 __atomic_store(haddr, &val, __ATOMIC_RELAXED); 128 ATOMIC_MMU_CLEANUP; 129 } 130 #else 131 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 132 ABI_TYPE val EXTRA_ARGS) 133 { 134 ATOMIC_MMU_DECLS; 135 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 136 DATA_TYPE ret; 137 138 ATOMIC_TRACE_RMW; 139 ret = atomic_xchg__nocheck(haddr, val); 140 ATOMIC_MMU_CLEANUP; 141 return ret; 142 } 143 144 #define GEN_ATOMIC_HELPER(X) \ 145 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 146 ABI_TYPE val EXTRA_ARGS) \ 147 { \ 148 ATOMIC_MMU_DECLS; \ 149 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 150 DATA_TYPE ret; \ 151 \ 152 ATOMIC_TRACE_RMW; \ 153 ret = atomic_##X(haddr, val); \ 154 ATOMIC_MMU_CLEANUP; \ 155 return ret; \ 156 } 157 158 GEN_ATOMIC_HELPER(fetch_add) 159 GEN_ATOMIC_HELPER(fetch_and) 160 GEN_ATOMIC_HELPER(fetch_or) 161 GEN_ATOMIC_HELPER(fetch_xor) 162 GEN_ATOMIC_HELPER(add_fetch) 163 GEN_ATOMIC_HELPER(and_fetch) 164 GEN_ATOMIC_HELPER(or_fetch) 165 GEN_ATOMIC_HELPER(xor_fetch) 166 167 #undef GEN_ATOMIC_HELPER 168 169 /* These helpers are, as a whole, full barriers. Within the helper, 170 * the leading barrier is explicit and the trailing barrier is within 171 * cmpxchg primitive. 172 * 173 * Trace this load + RMW loop as a single RMW op. This way, regardless 174 * of CF_PARALLEL's value, we'll trace just a read and a write. 175 */ 176 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 177 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 178 ABI_TYPE xval EXTRA_ARGS) \ 179 { \ 180 ATOMIC_MMU_DECLS; \ 181 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 182 XDATA_TYPE cmp, old, new, val = xval; \ 183 \ 184 ATOMIC_TRACE_RMW; \ 185 smp_mb(); \ 186 cmp = atomic_read__nocheck(haddr); \ 187 do { \ 188 old = cmp; new = FN(old, val); \ 189 cmp = atomic_cmpxchg__nocheck(haddr, old, new); \ 190 } while (cmp != old); \ 191 ATOMIC_MMU_CLEANUP; \ 192 return RET; \ 193 } 194 195 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 196 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 197 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 198 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 199 200 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 201 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 202 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 203 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 204 205 #undef GEN_ATOMIC_HELPER_FN 206 #endif /* DATA SIZE >= 16 */ 207 208 #undef END 209 #undef MEND 210 211 #if DATA_SIZE > 1 212 213 /* Define reverse-host-endian atomic operations. Note that END is used 214 within the ATOMIC_NAME macro. */ 215 #ifdef HOST_WORDS_BIGENDIAN 216 # define END _le 217 # define MEND _le 218 #else 219 # define END _be 220 # define MEND _be 221 #endif 222 223 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 224 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 225 { 226 ATOMIC_MMU_DECLS; 227 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 228 DATA_TYPE ret; 229 230 ATOMIC_TRACE_RMW; 231 ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 232 ATOMIC_MMU_CLEANUP; 233 return BSWAP(ret); 234 } 235 236 #if DATA_SIZE >= 16 237 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 238 { 239 ATOMIC_MMU_DECLS; 240 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 241 242 ATOMIC_TRACE_LD; 243 __atomic_load(haddr, &val, __ATOMIC_RELAXED); 244 ATOMIC_MMU_CLEANUP; 245 return BSWAP(val); 246 } 247 248 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 249 ABI_TYPE val EXTRA_ARGS) 250 { 251 ATOMIC_MMU_DECLS; 252 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 253 254 ATOMIC_TRACE_ST; 255 val = BSWAP(val); 256 __atomic_store(haddr, &val, __ATOMIC_RELAXED); 257 ATOMIC_MMU_CLEANUP; 258 } 259 #else 260 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 261 ABI_TYPE val EXTRA_ARGS) 262 { 263 ATOMIC_MMU_DECLS; 264 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 265 ABI_TYPE ret; 266 267 ATOMIC_TRACE_RMW; 268 ret = atomic_xchg__nocheck(haddr, BSWAP(val)); 269 ATOMIC_MMU_CLEANUP; 270 return BSWAP(ret); 271 } 272 273 #define GEN_ATOMIC_HELPER(X) \ 274 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 275 ABI_TYPE val EXTRA_ARGS) \ 276 { \ 277 ATOMIC_MMU_DECLS; \ 278 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 279 DATA_TYPE ret; \ 280 \ 281 ATOMIC_TRACE_RMW; \ 282 ret = atomic_##X(haddr, BSWAP(val)); \ 283 ATOMIC_MMU_CLEANUP; \ 284 return BSWAP(ret); \ 285 } 286 287 GEN_ATOMIC_HELPER(fetch_and) 288 GEN_ATOMIC_HELPER(fetch_or) 289 GEN_ATOMIC_HELPER(fetch_xor) 290 GEN_ATOMIC_HELPER(and_fetch) 291 GEN_ATOMIC_HELPER(or_fetch) 292 GEN_ATOMIC_HELPER(xor_fetch) 293 294 #undef GEN_ATOMIC_HELPER 295 296 /* These helpers are, as a whole, full barriers. Within the helper, 297 * the leading barrier is explicit and the trailing barrier is within 298 * cmpxchg primitive. 299 * 300 * Trace this load + RMW loop as a single RMW op. This way, regardless 301 * of CF_PARALLEL's value, we'll trace just a read and a write. 302 */ 303 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 304 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 305 ABI_TYPE xval EXTRA_ARGS) \ 306 { \ 307 ATOMIC_MMU_DECLS; \ 308 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 309 XDATA_TYPE ldo, ldn, old, new, val = xval; \ 310 \ 311 ATOMIC_TRACE_RMW; \ 312 smp_mb(); \ 313 ldn = atomic_read__nocheck(haddr); \ 314 do { \ 315 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 316 ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 317 } while (ldo != ldn); \ 318 ATOMIC_MMU_CLEANUP; \ 319 return RET; \ 320 } 321 322 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 323 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 324 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 325 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 326 327 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 328 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 329 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 330 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 331 332 /* Note that for addition, we need to use a separate cmpxchg loop instead 333 of bswaps for the reverse-host-endian helpers. */ 334 #define ADD(X, Y) (X + Y) 335 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 336 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 337 #undef ADD 338 339 #undef GEN_ATOMIC_HELPER_FN 340 #endif /* DATA_SIZE >= 16 */ 341 342 #undef END 343 #undef MEND 344 #endif /* DATA_SIZE > 1 */ 345 346 #undef ATOMIC_TRACE_ST 347 #undef ATOMIC_TRACE_LD 348 #undef ATOMIC_TRACE_RMW 349 350 #undef BSWAP 351 #undef ABI_TYPE 352 #undef DATA_TYPE 353 #undef SDATA_TYPE 354 #undef SUFFIX 355 #undef DATA_SIZE 356 #undef SHIFT 357