1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/plugin.h" 22 #include "trace/mem.h" 23 24 #if DATA_SIZE == 16 25 # define SUFFIX o 26 # define DATA_TYPE Int128 27 # define BSWAP bswap128 28 # define SHIFT 4 29 #elif DATA_SIZE == 8 30 # define SUFFIX q 31 # define DATA_TYPE uint64_t 32 # define SDATA_TYPE int64_t 33 # define BSWAP bswap64 34 # define SHIFT 3 35 #elif DATA_SIZE == 4 36 # define SUFFIX l 37 # define DATA_TYPE uint32_t 38 # define SDATA_TYPE int32_t 39 # define BSWAP bswap32 40 # define SHIFT 2 41 #elif DATA_SIZE == 2 42 # define SUFFIX w 43 # define DATA_TYPE uint16_t 44 # define SDATA_TYPE int16_t 45 # define BSWAP bswap16 46 # define SHIFT 1 47 #elif DATA_SIZE == 1 48 # define SUFFIX b 49 # define DATA_TYPE uint8_t 50 # define SDATA_TYPE int8_t 51 # define BSWAP 52 # define SHIFT 0 53 #else 54 # error unsupported data size 55 #endif 56 57 #if DATA_SIZE >= 4 58 # define ABI_TYPE DATA_TYPE 59 #else 60 # define ABI_TYPE uint32_t 61 #endif 62 63 /* Define host-endian atomic operations. Note that END is used within 64 the ATOMIC_NAME macro, and redefined below. */ 65 #if DATA_SIZE == 1 66 # define END 67 # define MEND _be /* either le or be would be fine */ 68 #elif defined(HOST_WORDS_BIGENDIAN) 69 # define END _be 70 # define MEND _be 71 #else 72 # define END _le 73 # define MEND _le 74 #endif 75 76 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 77 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 78 { 79 ATOMIC_MMU_DECLS; 80 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 81 DATA_TYPE ret; 82 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false, 83 ATOMIC_MMU_IDX); 84 85 atomic_trace_rmw_pre(env, addr, info); 86 #if DATA_SIZE == 16 87 ret = atomic16_cmpxchg(haddr, cmpv, newv); 88 #else 89 ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); 90 #endif 91 ATOMIC_MMU_CLEANUP; 92 atomic_trace_rmw_post(env, addr, info); 93 return ret; 94 } 95 96 #if DATA_SIZE >= 16 97 #if HAVE_ATOMIC128 98 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 99 { 100 ATOMIC_MMU_DECLS; 101 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 102 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false, 103 ATOMIC_MMU_IDX); 104 105 atomic_trace_ld_pre(env, addr, info); 106 val = atomic16_read(haddr); 107 ATOMIC_MMU_CLEANUP; 108 atomic_trace_ld_post(env, addr, info); 109 return val; 110 } 111 112 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 113 ABI_TYPE val EXTRA_ARGS) 114 { 115 ATOMIC_MMU_DECLS; 116 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 117 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true, 118 ATOMIC_MMU_IDX); 119 120 atomic_trace_st_pre(env, addr, info); 121 atomic16_set(haddr, val); 122 ATOMIC_MMU_CLEANUP; 123 atomic_trace_st_post(env, addr, info); 124 } 125 #endif 126 #else 127 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 128 ABI_TYPE val EXTRA_ARGS) 129 { 130 ATOMIC_MMU_DECLS; 131 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 132 DATA_TYPE ret; 133 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false, 134 ATOMIC_MMU_IDX); 135 136 atomic_trace_rmw_pre(env, addr, info); 137 ret = atomic_xchg__nocheck(haddr, val); 138 ATOMIC_MMU_CLEANUP; 139 atomic_trace_rmw_post(env, addr, info); 140 return ret; 141 } 142 143 #define GEN_ATOMIC_HELPER(X) \ 144 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 145 ABI_TYPE val EXTRA_ARGS) \ 146 { \ 147 ATOMIC_MMU_DECLS; \ 148 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 149 DATA_TYPE ret; \ 150 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \ 151 false, \ 152 ATOMIC_MMU_IDX); \ 153 \ 154 atomic_trace_rmw_pre(env, addr, info); \ 155 ret = atomic_##X(haddr, val); \ 156 ATOMIC_MMU_CLEANUP; \ 157 atomic_trace_rmw_post(env, addr, info); \ 158 return ret; \ 159 } 160 161 GEN_ATOMIC_HELPER(fetch_add) 162 GEN_ATOMIC_HELPER(fetch_and) 163 GEN_ATOMIC_HELPER(fetch_or) 164 GEN_ATOMIC_HELPER(fetch_xor) 165 GEN_ATOMIC_HELPER(add_fetch) 166 GEN_ATOMIC_HELPER(and_fetch) 167 GEN_ATOMIC_HELPER(or_fetch) 168 GEN_ATOMIC_HELPER(xor_fetch) 169 170 #undef GEN_ATOMIC_HELPER 171 172 /* These helpers are, as a whole, full barriers. Within the helper, 173 * the leading barrier is explicit and the trailing barrier is within 174 * cmpxchg primitive. 175 * 176 * Trace this load + RMW loop as a single RMW op. This way, regardless 177 * of CF_PARALLEL's value, we'll trace just a read and a write. 178 */ 179 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 180 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 181 ABI_TYPE xval EXTRA_ARGS) \ 182 { \ 183 ATOMIC_MMU_DECLS; \ 184 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 185 XDATA_TYPE cmp, old, new, val = xval; \ 186 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \ 187 false, \ 188 ATOMIC_MMU_IDX); \ 189 \ 190 atomic_trace_rmw_pre(env, addr, info); \ 191 smp_mb(); \ 192 cmp = atomic_read__nocheck(haddr); \ 193 do { \ 194 old = cmp; new = FN(old, val); \ 195 cmp = atomic_cmpxchg__nocheck(haddr, old, new); \ 196 } while (cmp != old); \ 197 ATOMIC_MMU_CLEANUP; \ 198 atomic_trace_rmw_post(env, addr, info); \ 199 return RET; \ 200 } 201 202 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 203 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 204 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 205 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 206 207 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 208 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 209 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 210 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 211 212 #undef GEN_ATOMIC_HELPER_FN 213 #endif /* DATA SIZE >= 16 */ 214 215 #undef END 216 #undef MEND 217 218 #if DATA_SIZE > 1 219 220 /* Define reverse-host-endian atomic operations. Note that END is used 221 within the ATOMIC_NAME macro. */ 222 #ifdef HOST_WORDS_BIGENDIAN 223 # define END _le 224 # define MEND _le 225 #else 226 # define END _be 227 # define MEND _be 228 #endif 229 230 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 231 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 232 { 233 ATOMIC_MMU_DECLS; 234 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 235 DATA_TYPE ret; 236 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, 237 false, 238 ATOMIC_MMU_IDX); 239 240 atomic_trace_rmw_pre(env, addr, info); 241 #if DATA_SIZE == 16 242 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); 243 #else 244 ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 245 #endif 246 ATOMIC_MMU_CLEANUP; 247 atomic_trace_rmw_post(env, addr, info); 248 return BSWAP(ret); 249 } 250 251 #if DATA_SIZE >= 16 252 #if HAVE_ATOMIC128 253 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 254 { 255 ATOMIC_MMU_DECLS; 256 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 257 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, 258 false, 259 ATOMIC_MMU_IDX); 260 261 atomic_trace_ld_pre(env, addr, info); 262 val = atomic16_read(haddr); 263 ATOMIC_MMU_CLEANUP; 264 atomic_trace_ld_post(env, addr, info); 265 return BSWAP(val); 266 } 267 268 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 269 ABI_TYPE val EXTRA_ARGS) 270 { 271 ATOMIC_MMU_DECLS; 272 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 273 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, 274 true, 275 ATOMIC_MMU_IDX); 276 277 val = BSWAP(val); 278 atomic_trace_st_pre(env, addr, info); 279 val = BSWAP(val); 280 atomic16_set(haddr, val); 281 ATOMIC_MMU_CLEANUP; 282 atomic_trace_st_post(env, addr, info); 283 } 284 #endif 285 #else 286 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 287 ABI_TYPE val EXTRA_ARGS) 288 { 289 ATOMIC_MMU_DECLS; 290 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 291 ABI_TYPE ret; 292 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, 293 false, 294 ATOMIC_MMU_IDX); 295 296 atomic_trace_rmw_pre(env, addr, info); 297 ret = atomic_xchg__nocheck(haddr, BSWAP(val)); 298 ATOMIC_MMU_CLEANUP; 299 atomic_trace_rmw_post(env, addr, info); 300 return BSWAP(ret); 301 } 302 303 #define GEN_ATOMIC_HELPER(X) \ 304 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 305 ABI_TYPE val EXTRA_ARGS) \ 306 { \ 307 ATOMIC_MMU_DECLS; \ 308 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 309 DATA_TYPE ret; \ 310 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \ 311 false, \ 312 ATOMIC_MMU_IDX); \ 313 \ 314 atomic_trace_rmw_pre(env, addr, info); \ 315 ret = atomic_##X(haddr, BSWAP(val)); \ 316 ATOMIC_MMU_CLEANUP; \ 317 atomic_trace_rmw_post(env, addr, info); \ 318 return BSWAP(ret); \ 319 } 320 321 GEN_ATOMIC_HELPER(fetch_and) 322 GEN_ATOMIC_HELPER(fetch_or) 323 GEN_ATOMIC_HELPER(fetch_xor) 324 GEN_ATOMIC_HELPER(and_fetch) 325 GEN_ATOMIC_HELPER(or_fetch) 326 GEN_ATOMIC_HELPER(xor_fetch) 327 328 #undef GEN_ATOMIC_HELPER 329 330 /* These helpers are, as a whole, full barriers. Within the helper, 331 * the leading barrier is explicit and the trailing barrier is within 332 * cmpxchg primitive. 333 * 334 * Trace this load + RMW loop as a single RMW op. This way, regardless 335 * of CF_PARALLEL's value, we'll trace just a read and a write. 336 */ 337 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 338 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 339 ABI_TYPE xval EXTRA_ARGS) \ 340 { \ 341 ATOMIC_MMU_DECLS; \ 342 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 343 XDATA_TYPE ldo, ldn, old, new, val = xval; \ 344 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \ 345 false, \ 346 ATOMIC_MMU_IDX); \ 347 \ 348 atomic_trace_rmw_pre(env, addr, info); \ 349 smp_mb(); \ 350 ldn = atomic_read__nocheck(haddr); \ 351 do { \ 352 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 353 ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 354 } while (ldo != ldn); \ 355 ATOMIC_MMU_CLEANUP; \ 356 atomic_trace_rmw_post(env, addr, info); \ 357 return RET; \ 358 } 359 360 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 361 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 362 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 363 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 364 365 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 366 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 367 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 368 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 369 370 /* Note that for addition, we need to use a separate cmpxchg loop instead 371 of bswaps for the reverse-host-endian helpers. */ 372 #define ADD(X, Y) (X + Y) 373 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 374 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 375 #undef ADD 376 377 #undef GEN_ATOMIC_HELPER_FN 378 #endif /* DATA_SIZE >= 16 */ 379 380 #undef END 381 #undef MEND 382 #endif /* DATA_SIZE > 1 */ 383 384 #undef BSWAP 385 #undef ABI_TYPE 386 #undef DATA_TYPE 387 #undef SDATA_TYPE 388 #undef SUFFIX 389 #undef DATA_SIZE 390 #undef SHIFT 391