1 /*
2 * Atomic helper templates
3 * Included from tcg-runtime.c and cputlb.c.
4 *
5 * Copyright (c) 2016 Red Hat, Inc
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "qemu/plugin.h"
22
23 #if DATA_SIZE == 16
24 # define SUFFIX o
25 # define DATA_TYPE Int128
26 # define BSWAP bswap128
27 # define SHIFT 4
28 #elif DATA_SIZE == 8
29 # define SUFFIX q
30 # define DATA_TYPE aligned_uint64_t
31 # define SDATA_TYPE aligned_int64_t
32 # define BSWAP bswap64
33 # define SHIFT 3
34 #elif DATA_SIZE == 4
35 # define SUFFIX l
36 # define DATA_TYPE uint32_t
37 # define SDATA_TYPE int32_t
38 # define BSWAP bswap32
39 # define SHIFT 2
40 #elif DATA_SIZE == 2
41 # define SUFFIX w
42 # define DATA_TYPE uint16_t
43 # define SDATA_TYPE int16_t
44 # define BSWAP bswap16
45 # define SHIFT 1
46 #elif DATA_SIZE == 1
47 # define SUFFIX b
48 # define DATA_TYPE uint8_t
49 # define SDATA_TYPE int8_t
50 # define BSWAP
51 # define SHIFT 0
52 #else
53 # error unsupported data size
54 #endif
55
56 #if DATA_SIZE >= 4
57 # define ABI_TYPE DATA_TYPE
58 #else
59 # define ABI_TYPE uint32_t
60 #endif
61
62 /* Define host-endian atomic operations. Note that END is used within
63 the ATOMIC_NAME macro, and redefined below. */
64 #if DATA_SIZE == 1
65 # define END
66 #elif HOST_BIG_ENDIAN
67 # define END _be
68 #else
69 # define END _le
70 #endif
71
ATOMIC_NAME(cmpxchg)72 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
73 ABI_TYPE cmpv, ABI_TYPE newv,
74 MemOpIdx oi, uintptr_t retaddr)
75 {
76 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
77 DATA_SIZE, retaddr);
78 DATA_TYPE ret;
79
80 #if DATA_SIZE == 16
81 ret = atomic16_cmpxchg(haddr, cmpv, newv);
82 #else
83 ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
84 #endif
85 ATOMIC_MMU_CLEANUP;
86 atomic_trace_rmw_post(env, addr, oi);
87 return ret;
88 }
89
90 #if DATA_SIZE < 16
ATOMIC_NAME(xchg)91 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
92 MemOpIdx oi, uintptr_t retaddr)
93 {
94 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
95 DATA_SIZE, retaddr);
96 DATA_TYPE ret;
97
98 ret = qatomic_xchg__nocheck(haddr, val);
99 ATOMIC_MMU_CLEANUP;
100 atomic_trace_rmw_post(env, addr, oi);
101 return ret;
102 }
103
104 #define GEN_ATOMIC_HELPER(X) \
105 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
106 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
107 { \
108 DATA_TYPE *haddr, ret; \
109 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \
110 ret = qatomic_##X(haddr, val); \
111 ATOMIC_MMU_CLEANUP; \
112 atomic_trace_rmw_post(env, addr, oi); \
113 return ret; \
114 }
115
116 GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)117 GEN_ATOMIC_HELPER(fetch_and)
118 GEN_ATOMIC_HELPER(fetch_or)
119 GEN_ATOMIC_HELPER(fetch_xor)
120 GEN_ATOMIC_HELPER(add_fetch)
121 GEN_ATOMIC_HELPER(and_fetch)
122 GEN_ATOMIC_HELPER(or_fetch)
123 GEN_ATOMIC_HELPER(xor_fetch)
124
125 #undef GEN_ATOMIC_HELPER
126
127 /*
128 * These helpers are, as a whole, full barriers. Within the helper,
129 * the leading barrier is explicit and the trailing barrier is within
130 * cmpxchg primitive.
131 *
132 * Trace this load + RMW loop as a single RMW op. This way, regardless
133 * of CF_PARALLEL's value, we'll trace just a read and a write.
134 */
135 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
136 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
137 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
138 { \
139 XDATA_TYPE *haddr, cmp, old, new, val = xval; \
140 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \
141 smp_mb(); \
142 cmp = qatomic_read__nocheck(haddr); \
143 do { \
144 old = cmp; new = FN(old, val); \
145 cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \
146 } while (cmp != old); \
147 ATOMIC_MMU_CLEANUP; \
148 atomic_trace_rmw_post(env, addr, oi); \
149 return RET; \
150 }
151
152 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
153 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
154 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
155 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
156
157 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
158 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
159 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
160 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
161
162 #undef GEN_ATOMIC_HELPER_FN
163 #endif /* DATA SIZE < 16 */
164
165 #undef END
166
167 #if DATA_SIZE > 1
168
169 /* Define reverse-host-endian atomic operations. Note that END is used
170 within the ATOMIC_NAME macro. */
171 #if HOST_BIG_ENDIAN
172 # define END _le
173 #else
174 # define END _be
175 #endif
176
177 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
178 ABI_TYPE cmpv, ABI_TYPE newv,
179 MemOpIdx oi, uintptr_t retaddr)
180 {
181 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
182 DATA_SIZE, retaddr);
183 DATA_TYPE ret;
184
185 #if DATA_SIZE == 16
186 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
187 #else
188 ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
189 #endif
190 ATOMIC_MMU_CLEANUP;
191 atomic_trace_rmw_post(env, addr, oi);
192 return BSWAP(ret);
193 }
194
195 #if DATA_SIZE < 16
ATOMIC_NAME(xchg)196 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
197 MemOpIdx oi, uintptr_t retaddr)
198 {
199 DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
200 DATA_SIZE, retaddr);
201 ABI_TYPE ret;
202
203 ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
204 ATOMIC_MMU_CLEANUP;
205 atomic_trace_rmw_post(env, addr, oi);
206 return BSWAP(ret);
207 }
208
209 #define GEN_ATOMIC_HELPER(X) \
210 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
211 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
212 { \
213 DATA_TYPE *haddr, ret; \
214 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \
215 ret = qatomic_##X(haddr, BSWAP(val)); \
216 ATOMIC_MMU_CLEANUP; \
217 atomic_trace_rmw_post(env, addr, oi); \
218 return BSWAP(ret); \
219 }
220
221 GEN_ATOMIC_HELPER(fetch_and)
222 GEN_ATOMIC_HELPER(fetch_or)
223 GEN_ATOMIC_HELPER(fetch_xor)
224 GEN_ATOMIC_HELPER(and_fetch)
225 GEN_ATOMIC_HELPER(or_fetch)
226 GEN_ATOMIC_HELPER(xor_fetch)
227
228 #undef GEN_ATOMIC_HELPER
229
230 /* These helpers are, as a whole, full barriers. Within the helper,
231 * the leading barrier is explicit and the trailing barrier is within
232 * cmpxchg primitive.
233 *
234 * Trace this load + RMW loop as a single RMW op. This way, regardless
235 * of CF_PARALLEL's value, we'll trace just a read and a write.
236 */
237 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
238 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
239 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
240 { \
241 XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval; \
242 haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr); \
243 smp_mb(); \
244 ldn = qatomic_read__nocheck(haddr); \
245 do { \
246 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
247 ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
248 } while (ldo != ldn); \
249 ATOMIC_MMU_CLEANUP; \
250 atomic_trace_rmw_post(env, addr, oi); \
251 return RET; \
252 }
253
254 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
255 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
256 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
257 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
258
259 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
260 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
261 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
262 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
263
264 /* Note that for addition, we need to use a separate cmpxchg loop instead
265 of bswaps for the reverse-host-endian helpers. */
266 #define ADD(X, Y) (X + Y)
267 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
268 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
269 #undef ADD
270
271 #undef GEN_ATOMIC_HELPER_FN
272 #endif /* DATA_SIZE < 16 */
273
274 #undef END
275 #endif /* DATA_SIZE > 1 */
276
277 #undef BSWAP
278 #undef ABI_TYPE
279 #undef DATA_TYPE
280 #undef SDATA_TYPE
281 #undef SUFFIX
282 #undef DATA_SIZE
283 #undef SHIFT
284