1 /*
2 * Atomic helper templates
3 * Included from tcg-runtime.c and cputlb.c.
4 *
5 * Copyright (c) 2016 Red Hat, Inc
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "qemu/plugin.h"
22
23 #if DATA_SIZE == 16
24 # define SUFFIX o
25 # define DATA_TYPE Int128
26 # define BSWAP bswap128
27 # define SHIFT 4
28 #elif DATA_SIZE == 8
29 # define SUFFIX q
30 # define DATA_TYPE aligned_uint64_t
31 # define SDATA_TYPE aligned_int64_t
32 # define BSWAP bswap64
33 # define SHIFT 3
34 #elif DATA_SIZE == 4
35 # define SUFFIX l
36 # define DATA_TYPE uint32_t
37 # define SDATA_TYPE int32_t
38 # define BSWAP bswap32
39 # define SHIFT 2
40 #elif DATA_SIZE == 2
41 # define SUFFIX w
42 # define DATA_TYPE uint16_t
43 # define SDATA_TYPE int16_t
44 # define BSWAP bswap16
45 # define SHIFT 1
46 #elif DATA_SIZE == 1
47 # define SUFFIX b
48 # define DATA_TYPE uint8_t
49 # define SDATA_TYPE int8_t
50 # define BSWAP
51 # define SHIFT 0
52 #else
53 # error unsupported data size
54 #endif
55
56 #if DATA_SIZE == 16
57 # define VALUE_LOW(val) int128_getlo(val)
58 # define VALUE_HIGH(val) int128_gethi(val)
59 #else
60 # define VALUE_LOW(val) val
61 # define VALUE_HIGH(val) 0
62 #endif
63
64 #if DATA_SIZE >= 4
65 # define ABI_TYPE DATA_TYPE
66 #else
67 # define ABI_TYPE uint32_t
68 #endif
69
70 /* Define host-endian atomic operations. Note that END is used within
71 the ATOMIC_NAME macro, and redefined below. */
72 #if DATA_SIZE == 1
73 # define END
74 #elif HOST_BIG_ENDIAN
75 # define END _be
76 #else
77 # define END _le
78 #endif
79
ATOMIC_NAME(cmpxchg)80 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
81 ABI_TYPE cmpv, ABI_TYPE newv,
82 MemOpIdx oi, uintptr_t retaddr)
83 {
84 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
85 DATA_SIZE, retaddr);
86 DATA_TYPE ret;
87
88 #if DATA_SIZE == 16
89 ret = atomic16_cmpxchg(haddr, cmpv, newv);
90 #else
91 ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
92 #endif
93 ATOMIC_MMU_CLEANUP;
94 atomic_trace_rmw_post(env, addr,
95 VALUE_LOW(ret),
96 VALUE_HIGH(ret),
97 VALUE_LOW(newv),
98 VALUE_HIGH(newv),
99 oi);
100 return ret;
101 }
102
103 #if DATA_SIZE < 16
ATOMIC_NAME(xchg)104 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
105 MemOpIdx oi, uintptr_t retaddr)
106 {
107 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
108 DATA_SIZE, retaddr);
109 DATA_TYPE ret;
110
111 ret = qatomic_xchg__nocheck(haddr, val);
112 ATOMIC_MMU_CLEANUP;
113 atomic_trace_rmw_post(env, addr,
114 VALUE_LOW(ret),
115 VALUE_HIGH(ret),
116 VALUE_LOW(val),
117 VALUE_HIGH(val),
118 oi);
119 return ret;
120 }
121
122 #define GEN_ATOMIC_HELPER(X) \
123 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
124 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
125 { \
126 DATA_TYPE *haddr, ret; \
127 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \
128 ret = qatomic_##X(haddr, val); \
129 ATOMIC_MMU_CLEANUP; \
130 atomic_trace_rmw_post(env, addr, \
131 VALUE_LOW(ret), \
132 VALUE_HIGH(ret), \
133 VALUE_LOW(val), \
134 VALUE_HIGH(val), \
135 oi); \
136 return ret; \
137 }
138
139 GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)140 GEN_ATOMIC_HELPER(fetch_and)
141 GEN_ATOMIC_HELPER(fetch_or)
142 GEN_ATOMIC_HELPER(fetch_xor)
143 GEN_ATOMIC_HELPER(add_fetch)
144 GEN_ATOMIC_HELPER(and_fetch)
145 GEN_ATOMIC_HELPER(or_fetch)
146 GEN_ATOMIC_HELPER(xor_fetch)
147
148 #undef GEN_ATOMIC_HELPER
149
150 /*
151 * These helpers are, as a whole, full barriers. Within the helper,
152 * the leading barrier is explicit and the trailing barrier is within
153 * cmpxchg primitive.
154 *
155 * Trace this load + RMW loop as a single RMW op. This way, regardless
156 * of CF_PARALLEL's value, we'll trace just a read and a write.
157 */
158 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
159 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
160 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
161 { \
162 XDATA_TYPE *haddr, cmp, old, new, val = xval; \
163 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \
164 smp_mb(); \
165 cmp = qatomic_read__nocheck(haddr); \
166 do { \
167 old = cmp; new = FN(old, val); \
168 cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \
169 } while (cmp != old); \
170 ATOMIC_MMU_CLEANUP; \
171 atomic_trace_rmw_post(env, addr, \
172 VALUE_LOW(old), \
173 VALUE_HIGH(old), \
174 VALUE_LOW(xval), \
175 VALUE_HIGH(xval), \
176 oi); \
177 return RET; \
178 }
179
180 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
181 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
182 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
183 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
184
185 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
186 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
187 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
188 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
189
190 #undef GEN_ATOMIC_HELPER_FN
191 #endif /* DATA SIZE < 16 */
192
193 #undef END
194
195 #if DATA_SIZE > 1
196
197 /* Define reverse-host-endian atomic operations. Note that END is used
198 within the ATOMIC_NAME macro. */
199 #if HOST_BIG_ENDIAN
200 # define END _le
201 #else
202 # define END _be
203 #endif
204
205 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
206 ABI_TYPE cmpv, ABI_TYPE newv,
207 MemOpIdx oi, uintptr_t retaddr)
208 {
209 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
210 DATA_SIZE, retaddr);
211 DATA_TYPE ret;
212
213 #if DATA_SIZE == 16
214 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
215 #else
216 ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
217 #endif
218 ATOMIC_MMU_CLEANUP;
219 atomic_trace_rmw_post(env, addr,
220 VALUE_LOW(ret),
221 VALUE_HIGH(ret),
222 VALUE_LOW(newv),
223 VALUE_HIGH(newv),
224 oi);
225 return BSWAP(ret);
226 }
227
228 #if DATA_SIZE < 16
ATOMIC_NAME(xchg)229 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
230 MemOpIdx oi, uintptr_t retaddr)
231 {
232 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
233 DATA_SIZE, retaddr);
234 ABI_TYPE ret;
235
236 ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
237 ATOMIC_MMU_CLEANUP;
238 atomic_trace_rmw_post(env, addr,
239 VALUE_LOW(ret),
240 VALUE_HIGH(ret),
241 VALUE_LOW(val),
242 VALUE_HIGH(val),
243 oi);
244 return BSWAP(ret);
245 }
246
247 #define GEN_ATOMIC_HELPER(X) \
248 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
249 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
250 { \
251 DATA_TYPE *haddr, ret; \
252 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \
253 ret = qatomic_##X(haddr, BSWAP(val)); \
254 ATOMIC_MMU_CLEANUP; \
255 atomic_trace_rmw_post(env, addr, \
256 VALUE_LOW(ret), \
257 VALUE_HIGH(ret), \
258 VALUE_LOW(val), \
259 VALUE_HIGH(val), \
260 oi); \
261 return BSWAP(ret); \
262 }
263
264 GEN_ATOMIC_HELPER(fetch_and)
265 GEN_ATOMIC_HELPER(fetch_or)
266 GEN_ATOMIC_HELPER(fetch_xor)
267 GEN_ATOMIC_HELPER(and_fetch)
268 GEN_ATOMIC_HELPER(or_fetch)
269 GEN_ATOMIC_HELPER(xor_fetch)
270
271 #undef GEN_ATOMIC_HELPER
272
273 /* These helpers are, as a whole, full barriers. Within the helper,
274 * the leading barrier is explicit and the trailing barrier is within
275 * cmpxchg primitive.
276 *
277 * Trace this load + RMW loop as a single RMW op. This way, regardless
278 * of CF_PARALLEL's value, we'll trace just a read and a write.
279 */
280 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
281 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
282 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
283 { \
284 XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval; \
285 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \
286 smp_mb(); \
287 ldn = qatomic_read__nocheck(haddr); \
288 do { \
289 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
290 ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
291 } while (ldo != ldn); \
292 ATOMIC_MMU_CLEANUP; \
293 atomic_trace_rmw_post(env, addr, \
294 VALUE_LOW(old), \
295 VALUE_HIGH(old), \
296 VALUE_LOW(xval), \
297 VALUE_HIGH(xval), \
298 oi); \
299 return RET; \
300 }
301
302 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
303 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
304 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
305 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
306
307 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
308 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
309 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
310 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
311
312 /* Note that for addition, we need to use a separate cmpxchg loop instead
313 of bswaps for the reverse-host-endian helpers. */
314 #define ADD(X, Y) (X + Y)
315 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
316 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
317 #undef ADD
318
319 #undef GEN_ATOMIC_HELPER_FN
320 #endif /* DATA_SIZE < 16 */
321
322 #undef END
323 #endif /* DATA_SIZE > 1 */
324
325 #undef BSWAP
326 #undef ABI_TYPE
327 #undef DATA_TYPE
328 #undef SDATA_TYPE
329 #undef SUFFIX
330 #undef DATA_SIZE
331 #undef SHIFT
332 #undef VALUE_LOW
333 #undef VALUE_HIGH
334