1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/plugin.h" 22 #include "trace/mem.h" 23 24 #if DATA_SIZE == 16 25 # define SUFFIX o 26 # define DATA_TYPE Int128 27 # define BSWAP bswap128 28 # define SHIFT 4 29 #elif DATA_SIZE == 8 30 # define SUFFIX q 31 # define DATA_TYPE uint64_t 32 # define SDATA_TYPE int64_t 33 # define BSWAP bswap64 34 # define SHIFT 3 35 #elif DATA_SIZE == 4 36 # define SUFFIX l 37 # define DATA_TYPE uint32_t 38 # define SDATA_TYPE int32_t 39 # define BSWAP bswap32 40 # define SHIFT 2 41 #elif DATA_SIZE == 2 42 # define SUFFIX w 43 # define DATA_TYPE uint16_t 44 # define SDATA_TYPE int16_t 45 # define BSWAP bswap16 46 # define SHIFT 1 47 #elif DATA_SIZE == 1 48 # define SUFFIX b 49 # define DATA_TYPE uint8_t 50 # define SDATA_TYPE int8_t 51 # define BSWAP 52 # define SHIFT 0 53 #else 54 # error unsupported data size 55 #endif 56 57 #if DATA_SIZE >= 4 58 # define ABI_TYPE DATA_TYPE 59 #else 60 # define ABI_TYPE uint32_t 61 #endif 62 63 /* Define host-endian atomic operations. Note that END is used within 64 the ATOMIC_NAME macro, and redefined below. */ 65 #if DATA_SIZE == 1 66 # define END 67 #elif defined(HOST_WORDS_BIGENDIAN) 68 # define END _be 69 #else 70 # define END _le 71 #endif 72 73 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 74 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 75 { 76 ATOMIC_MMU_DECLS; 77 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 78 DATA_TYPE ret; 79 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, 80 ATOMIC_MMU_IDX); 81 82 atomic_trace_rmw_pre(env, addr, info); 83 #if DATA_SIZE == 16 84 ret = atomic16_cmpxchg(haddr, cmpv, newv); 85 #else 86 ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv); 87 #endif 88 ATOMIC_MMU_CLEANUP; 89 atomic_trace_rmw_post(env, addr, info); 90 return ret; 91 } 92 93 #if DATA_SIZE >= 16 94 #if HAVE_ATOMIC128 95 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 96 { 97 ATOMIC_MMU_DECLS; 98 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 99 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, 100 ATOMIC_MMU_IDX); 101 102 atomic_trace_ld_pre(env, addr, info); 103 val = atomic16_read(haddr); 104 ATOMIC_MMU_CLEANUP; 105 atomic_trace_ld_post(env, addr, info); 106 return val; 107 } 108 109 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 110 ABI_TYPE val EXTRA_ARGS) 111 { 112 ATOMIC_MMU_DECLS; 113 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 114 uint16_t info = trace_mem_build_info(SHIFT, false, 0, true, 115 ATOMIC_MMU_IDX); 116 117 atomic_trace_st_pre(env, addr, info); 118 atomic16_set(haddr, val); 119 ATOMIC_MMU_CLEANUP; 120 atomic_trace_st_post(env, addr, info); 121 } 122 #endif 123 #else 124 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 125 ABI_TYPE val EXTRA_ARGS) 126 { 127 ATOMIC_MMU_DECLS; 128 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 129 DATA_TYPE ret; 130 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, 131 ATOMIC_MMU_IDX); 132 133 atomic_trace_rmw_pre(env, addr, info); 134 ret = qatomic_xchg__nocheck(haddr, val); 135 ATOMIC_MMU_CLEANUP; 136 atomic_trace_rmw_post(env, addr, info); 137 return ret; 138 } 139 140 #define GEN_ATOMIC_HELPER(X) \ 141 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 142 ABI_TYPE val EXTRA_ARGS) \ 143 { \ 144 ATOMIC_MMU_DECLS; \ 145 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 146 DATA_TYPE ret; \ 147 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ 148 ATOMIC_MMU_IDX); \ 149 atomic_trace_rmw_pre(env, addr, info); \ 150 ret = qatomic_##X(haddr, val); \ 151 ATOMIC_MMU_CLEANUP; \ 152 atomic_trace_rmw_post(env, addr, info); \ 153 return ret; \ 154 } 155 156 GEN_ATOMIC_HELPER(fetch_add) 157 GEN_ATOMIC_HELPER(fetch_and) 158 GEN_ATOMIC_HELPER(fetch_or) 159 GEN_ATOMIC_HELPER(fetch_xor) 160 GEN_ATOMIC_HELPER(add_fetch) 161 GEN_ATOMIC_HELPER(and_fetch) 162 GEN_ATOMIC_HELPER(or_fetch) 163 GEN_ATOMIC_HELPER(xor_fetch) 164 165 #undef GEN_ATOMIC_HELPER 166 167 /* These helpers are, as a whole, full barriers. Within the helper, 168 * the leading barrier is explicit and the trailing barrier is within 169 * cmpxchg primitive. 170 * 171 * Trace this load + RMW loop as a single RMW op. This way, regardless 172 * of CF_PARALLEL's value, we'll trace just a read and a write. 173 */ 174 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 175 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 176 ABI_TYPE xval EXTRA_ARGS) \ 177 { \ 178 ATOMIC_MMU_DECLS; \ 179 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 180 XDATA_TYPE cmp, old, new, val = xval; \ 181 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ 182 ATOMIC_MMU_IDX); \ 183 atomic_trace_rmw_pre(env, addr, info); \ 184 smp_mb(); \ 185 cmp = qatomic_read__nocheck(haddr); \ 186 do { \ 187 old = cmp; new = FN(old, val); \ 188 cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \ 189 } while (cmp != old); \ 190 ATOMIC_MMU_CLEANUP; \ 191 atomic_trace_rmw_post(env, addr, info); \ 192 return RET; \ 193 } 194 195 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 196 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 197 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 198 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 199 200 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 201 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 202 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 203 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 204 205 #undef GEN_ATOMIC_HELPER_FN 206 #endif /* DATA SIZE >= 16 */ 207 208 #undef END 209 210 #if DATA_SIZE > 1 211 212 /* Define reverse-host-endian atomic operations. Note that END is used 213 within the ATOMIC_NAME macro. */ 214 #ifdef HOST_WORDS_BIGENDIAN 215 # define END _le 216 #else 217 # define END _be 218 #endif 219 220 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 221 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 222 { 223 ATOMIC_MMU_DECLS; 224 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 225 DATA_TYPE ret; 226 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, 227 ATOMIC_MMU_IDX); 228 229 atomic_trace_rmw_pre(env, addr, info); 230 #if DATA_SIZE == 16 231 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); 232 #else 233 ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 234 #endif 235 ATOMIC_MMU_CLEANUP; 236 atomic_trace_rmw_post(env, addr, info); 237 return BSWAP(ret); 238 } 239 240 #if DATA_SIZE >= 16 241 #if HAVE_ATOMIC128 242 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 243 { 244 ATOMIC_MMU_DECLS; 245 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 246 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, 247 ATOMIC_MMU_IDX); 248 249 atomic_trace_ld_pre(env, addr, info); 250 val = atomic16_read(haddr); 251 ATOMIC_MMU_CLEANUP; 252 atomic_trace_ld_post(env, addr, info); 253 return BSWAP(val); 254 } 255 256 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 257 ABI_TYPE val EXTRA_ARGS) 258 { 259 ATOMIC_MMU_DECLS; 260 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 261 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true, 262 ATOMIC_MMU_IDX); 263 264 val = BSWAP(val); 265 atomic_trace_st_pre(env, addr, info); 266 val = BSWAP(val); 267 atomic16_set(haddr, val); 268 ATOMIC_MMU_CLEANUP; 269 atomic_trace_st_post(env, addr, info); 270 } 271 #endif 272 #else 273 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 274 ABI_TYPE val EXTRA_ARGS) 275 { 276 ATOMIC_MMU_DECLS; 277 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 278 ABI_TYPE ret; 279 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, 280 ATOMIC_MMU_IDX); 281 282 atomic_trace_rmw_pre(env, addr, info); 283 ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); 284 ATOMIC_MMU_CLEANUP; 285 atomic_trace_rmw_post(env, addr, info); 286 return BSWAP(ret); 287 } 288 289 #define GEN_ATOMIC_HELPER(X) \ 290 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 291 ABI_TYPE val EXTRA_ARGS) \ 292 { \ 293 ATOMIC_MMU_DECLS; \ 294 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 295 DATA_TYPE ret; \ 296 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ 297 false, ATOMIC_MMU_IDX); \ 298 atomic_trace_rmw_pre(env, addr, info); \ 299 ret = qatomic_##X(haddr, BSWAP(val)); \ 300 ATOMIC_MMU_CLEANUP; \ 301 atomic_trace_rmw_post(env, addr, info); \ 302 return BSWAP(ret); \ 303 } 304 305 GEN_ATOMIC_HELPER(fetch_and) 306 GEN_ATOMIC_HELPER(fetch_or) 307 GEN_ATOMIC_HELPER(fetch_xor) 308 GEN_ATOMIC_HELPER(and_fetch) 309 GEN_ATOMIC_HELPER(or_fetch) 310 GEN_ATOMIC_HELPER(xor_fetch) 311 312 #undef GEN_ATOMIC_HELPER 313 314 /* These helpers are, as a whole, full barriers. Within the helper, 315 * the leading barrier is explicit and the trailing barrier is within 316 * cmpxchg primitive. 317 * 318 * Trace this load + RMW loop as a single RMW op. This way, regardless 319 * of CF_PARALLEL's value, we'll trace just a read and a write. 320 */ 321 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 322 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 323 ABI_TYPE xval EXTRA_ARGS) \ 324 { \ 325 ATOMIC_MMU_DECLS; \ 326 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 327 XDATA_TYPE ldo, ldn, old, new, val = xval; \ 328 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ 329 false, ATOMIC_MMU_IDX); \ 330 atomic_trace_rmw_pre(env, addr, info); \ 331 smp_mb(); \ 332 ldn = qatomic_read__nocheck(haddr); \ 333 do { \ 334 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 335 ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 336 } while (ldo != ldn); \ 337 ATOMIC_MMU_CLEANUP; \ 338 atomic_trace_rmw_post(env, addr, info); \ 339 return RET; \ 340 } 341 342 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 343 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 344 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 345 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 346 347 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 348 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 349 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 350 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 351 352 /* Note that for addition, we need to use a separate cmpxchg loop instead 353 of bswaps for the reverse-host-endian helpers. */ 354 #define ADD(X, Y) (X + Y) 355 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 356 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 357 #undef ADD 358 359 #undef GEN_ATOMIC_HELPER_FN 360 #endif /* DATA_SIZE >= 16 */ 361 362 #undef END 363 #endif /* DATA_SIZE > 1 */ 364 365 #undef BSWAP 366 #undef ABI_TYPE 367 #undef DATA_TYPE 368 #undef SDATA_TYPE 369 #undef SUFFIX 370 #undef DATA_SIZE 371 #undef SHIFT 372