xref: /openbmc/qemu/accel/tcg/atomic_template.h (revision 2e1cacfb)
1 /*
2  * Atomic helper templates
3  * Included from tcg-runtime.c and cputlb.c.
4  *
5  * Copyright (c) 2016 Red Hat, Inc
6  *
7  * This library is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * This library is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19  */
20 
21 #include "qemu/plugin.h"
22 
23 #if DATA_SIZE == 16
24 # define SUFFIX     o
25 # define DATA_TYPE  Int128
26 # define BSWAP      bswap128
27 # define SHIFT      4
28 #elif DATA_SIZE == 8
29 # define SUFFIX     q
30 # define DATA_TYPE  aligned_uint64_t
31 # define SDATA_TYPE aligned_int64_t
32 # define BSWAP      bswap64
33 # define SHIFT      3
34 #elif DATA_SIZE == 4
35 # define SUFFIX     l
36 # define DATA_TYPE  uint32_t
37 # define SDATA_TYPE int32_t
38 # define BSWAP      bswap32
39 # define SHIFT      2
40 #elif DATA_SIZE == 2
41 # define SUFFIX     w
42 # define DATA_TYPE  uint16_t
43 # define SDATA_TYPE int16_t
44 # define BSWAP      bswap16
45 # define SHIFT      1
46 #elif DATA_SIZE == 1
47 # define SUFFIX     b
48 # define DATA_TYPE  uint8_t
49 # define SDATA_TYPE int8_t
50 # define BSWAP
51 # define SHIFT      0
52 #else
53 # error unsupported data size
54 #endif
55 
56 #if DATA_SIZE == 16
57 # define VALUE_LOW(val) int128_getlo(val)
58 # define VALUE_HIGH(val) int128_gethi(val)
59 #else
60 # define VALUE_LOW(val) val
61 # define VALUE_HIGH(val) 0
62 #endif
63 
64 #if DATA_SIZE >= 4
65 # define ABI_TYPE  DATA_TYPE
66 #else
67 # define ABI_TYPE  uint32_t
68 #endif
69 
70 /* Define host-endian atomic operations.  Note that END is used within
71    the ATOMIC_NAME macro, and redefined below.  */
72 #if DATA_SIZE == 1
73 # define END
74 #elif HOST_BIG_ENDIAN
75 # define END  _be
76 #else
77 # define END  _le
78 #endif
79 
80 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
81                               ABI_TYPE cmpv, ABI_TYPE newv,
82                               MemOpIdx oi, uintptr_t retaddr)
83 {
84     DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
85                                          DATA_SIZE, retaddr);
86     DATA_TYPE ret;
87 
88 #if DATA_SIZE == 16
89     ret = atomic16_cmpxchg(haddr, cmpv, newv);
90 #else
91     ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
92 #endif
93     ATOMIC_MMU_CLEANUP;
94     atomic_trace_rmw_post(env, addr,
95                           VALUE_LOW(ret),
96                           VALUE_HIGH(ret),
97                           VALUE_LOW(newv),
98                           VALUE_HIGH(newv),
99                           oi);
100     return ret;
101 }
102 
103 #if DATA_SIZE < 16
104 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
105                            MemOpIdx oi, uintptr_t retaddr)
106 {
107     DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
108                                          DATA_SIZE, retaddr);
109     DATA_TYPE ret;
110 
111     ret = qatomic_xchg__nocheck(haddr, val);
112     ATOMIC_MMU_CLEANUP;
113     atomic_trace_rmw_post(env, addr,
114                           VALUE_LOW(ret),
115                           VALUE_HIGH(ret),
116                           VALUE_LOW(val),
117                           VALUE_HIGH(val),
118                           oi);
119     return ret;
120 }
121 
122 #define GEN_ATOMIC_HELPER(X)                                        \
123 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
124                         ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
125 {                                                                   \
126     DATA_TYPE *haddr, ret;                                          \
127     haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
128     ret = qatomic_##X(haddr, val);                                  \
129     ATOMIC_MMU_CLEANUP;                                             \
130     atomic_trace_rmw_post(env, addr,                                \
131                           VALUE_LOW(ret),                           \
132                           VALUE_HIGH(ret),                          \
133                           VALUE_LOW(val),                           \
134                           VALUE_HIGH(val),                          \
135                           oi);                                      \
136     return ret;                                                     \
137 }
138 
139 GEN_ATOMIC_HELPER(fetch_add)
140 GEN_ATOMIC_HELPER(fetch_and)
141 GEN_ATOMIC_HELPER(fetch_or)
142 GEN_ATOMIC_HELPER(fetch_xor)
143 GEN_ATOMIC_HELPER(add_fetch)
144 GEN_ATOMIC_HELPER(and_fetch)
145 GEN_ATOMIC_HELPER(or_fetch)
146 GEN_ATOMIC_HELPER(xor_fetch)
147 
148 #undef GEN_ATOMIC_HELPER
149 
150 /*
151  * These helpers are, as a whole, full barriers.  Within the helper,
152  * the leading barrier is explicit and the trailing barrier is within
153  * cmpxchg primitive.
154  *
155  * Trace this load + RMW loop as a single RMW op. This way, regardless
156  * of CF_PARALLEL's value, we'll trace just a read and a write.
157  */
158 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
159 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
160                         ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
161 {                                                                   \
162     XDATA_TYPE *haddr, cmp, old, new, val = xval;                   \
163     haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
164     smp_mb();                                                       \
165     cmp = qatomic_read__nocheck(haddr);                             \
166     do {                                                            \
167         old = cmp; new = FN(old, val);                              \
168         cmp = qatomic_cmpxchg__nocheck(haddr, old, new);            \
169     } while (cmp != old);                                           \
170     ATOMIC_MMU_CLEANUP;                                             \
171     atomic_trace_rmw_post(env, addr,                                \
172                           VALUE_LOW(old),                           \
173                           VALUE_HIGH(old),                          \
174                           VALUE_LOW(xval),                          \
175                           VALUE_HIGH(xval),                         \
176                           oi);                                      \
177     return RET;                                                     \
178 }
179 
180 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
181 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
182 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
183 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
184 
185 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
186 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
187 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
188 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
189 
190 #undef GEN_ATOMIC_HELPER_FN
191 #endif /* DATA SIZE < 16 */
192 
193 #undef END
194 
195 #if DATA_SIZE > 1
196 
197 /* Define reverse-host-endian atomic operations.  Note that END is used
198    within the ATOMIC_NAME macro.  */
199 #if HOST_BIG_ENDIAN
200 # define END  _le
201 #else
202 # define END  _be
203 #endif
204 
205 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
206                               ABI_TYPE cmpv, ABI_TYPE newv,
207                               MemOpIdx oi, uintptr_t retaddr)
208 {
209     DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
210                                          DATA_SIZE, retaddr);
211     DATA_TYPE ret;
212 
213 #if DATA_SIZE == 16
214     ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
215 #else
216     ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
217 #endif
218     ATOMIC_MMU_CLEANUP;
219     atomic_trace_rmw_post(env, addr,
220                           VALUE_LOW(ret),
221                           VALUE_HIGH(ret),
222                           VALUE_LOW(newv),
223                           VALUE_HIGH(newv),
224                           oi);
225     return BSWAP(ret);
226 }
227 
228 #if DATA_SIZE < 16
229 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
230                            MemOpIdx oi, uintptr_t retaddr)
231 {
232     DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
233                                          DATA_SIZE, retaddr);
234     ABI_TYPE ret;
235 
236     ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
237     ATOMIC_MMU_CLEANUP;
238     atomic_trace_rmw_post(env, addr,
239                           VALUE_LOW(ret),
240                           VALUE_HIGH(ret),
241                           VALUE_LOW(val),
242                           VALUE_HIGH(val),
243                           oi);
244     return BSWAP(ret);
245 }
246 
247 #define GEN_ATOMIC_HELPER(X)                                        \
248 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
249                         ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
250 {                                                                   \
251     DATA_TYPE *haddr, ret;                                          \
252     haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
253     ret = qatomic_##X(haddr, BSWAP(val));                           \
254     ATOMIC_MMU_CLEANUP;                                             \
255     atomic_trace_rmw_post(env, addr,                                \
256                           VALUE_LOW(ret),                           \
257                           VALUE_HIGH(ret),                          \
258                           VALUE_LOW(val),                           \
259                           VALUE_HIGH(val),                          \
260                           oi);                                      \
261     return BSWAP(ret);                                              \
262 }
263 
264 GEN_ATOMIC_HELPER(fetch_and)
265 GEN_ATOMIC_HELPER(fetch_or)
266 GEN_ATOMIC_HELPER(fetch_xor)
267 GEN_ATOMIC_HELPER(and_fetch)
268 GEN_ATOMIC_HELPER(or_fetch)
269 GEN_ATOMIC_HELPER(xor_fetch)
270 
271 #undef GEN_ATOMIC_HELPER
272 
273 /* These helpers are, as a whole, full barriers.  Within the helper,
274  * the leading barrier is explicit and the trailing barrier is within
275  * cmpxchg primitive.
276  *
277  * Trace this load + RMW loop as a single RMW op. This way, regardless
278  * of CF_PARALLEL's value, we'll trace just a read and a write.
279  */
280 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
281 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
282                         ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
283 {                                                                   \
284     XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval;              \
285     haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
286     smp_mb();                                                       \
287     ldn = qatomic_read__nocheck(haddr);                             \
288     do {                                                            \
289         ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
290         ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));     \
291     } while (ldo != ldn);                                           \
292     ATOMIC_MMU_CLEANUP;                                             \
293     atomic_trace_rmw_post(env, addr,                                \
294                           VALUE_LOW(old),                           \
295                           VALUE_HIGH(old),                          \
296                           VALUE_LOW(xval),                          \
297                           VALUE_HIGH(xval),                         \
298                           oi);                                      \
299     return RET;                                                     \
300 }
301 
302 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
303 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
304 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
305 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
306 
307 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
308 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
309 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
310 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
311 
312 /* Note that for addition, we need to use a separate cmpxchg loop instead
313    of bswaps for the reverse-host-endian helpers.  */
314 #define ADD(X, Y)   (X + Y)
315 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
316 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
317 #undef ADD
318 
319 #undef GEN_ATOMIC_HELPER_FN
320 #endif /* DATA_SIZE < 16 */
321 
322 #undef END
323 #endif /* DATA_SIZE > 1 */
324 
325 #undef BSWAP
326 #undef ABI_TYPE
327 #undef DATA_TYPE
328 #undef SDATA_TYPE
329 #undef SUFFIX
330 #undef DATA_SIZE
331 #undef SHIFT
332 #undef VALUE_LOW
333 #undef VALUE_HIGH
334