xref: /openbmc/qemu/accel/tcg/atomic_template.h (revision d560225f)
1 /*
2  * Atomic helper templates
3  * Included from tcg-runtime.c and cputlb.c.
4  *
5  * Copyright (c) 2016 Red Hat, Inc
6  *
7  * This library is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * This library is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19  */
20 
21 #include "qemu/plugin.h"
22 
23 #if DATA_SIZE == 16
24 # define SUFFIX     o
25 # define DATA_TYPE  Int128
26 # define BSWAP      bswap128
27 # define SHIFT      4
28 #elif DATA_SIZE == 8
29 # define SUFFIX     q
30 # define DATA_TYPE  aligned_uint64_t
31 # define SDATA_TYPE aligned_int64_t
32 # define BSWAP      bswap64
33 # define SHIFT      3
34 #elif DATA_SIZE == 4
35 # define SUFFIX     l
36 # define DATA_TYPE  uint32_t
37 # define SDATA_TYPE int32_t
38 # define BSWAP      bswap32
39 # define SHIFT      2
40 #elif DATA_SIZE == 2
41 # define SUFFIX     w
42 # define DATA_TYPE  uint16_t
43 # define SDATA_TYPE int16_t
44 # define BSWAP      bswap16
45 # define SHIFT      1
46 #elif DATA_SIZE == 1
47 # define SUFFIX     b
48 # define DATA_TYPE  uint8_t
49 # define SDATA_TYPE int8_t
50 # define BSWAP
51 # define SHIFT      0
52 #else
53 # error unsupported data size
54 #endif
55 
56 #if DATA_SIZE >= 4
57 # define ABI_TYPE  DATA_TYPE
58 #else
59 # define ABI_TYPE  uint32_t
60 #endif
61 
62 /* Define host-endian atomic operations.  Note that END is used within
63    the ATOMIC_NAME macro, and redefined below.  */
64 #if DATA_SIZE == 1
65 # define END
66 #elif HOST_BIG_ENDIAN
67 # define END  _be
68 #else
69 # define END  _le
70 #endif
71 
ATOMIC_NAME(cmpxchg)72 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
73                               ABI_TYPE cmpv, ABI_TYPE newv,
74                               MemOpIdx oi, uintptr_t retaddr)
75 {
76     DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
77                                          DATA_SIZE, retaddr);
78     DATA_TYPE ret;
79 
80 #if DATA_SIZE == 16
81     ret = atomic16_cmpxchg(haddr, cmpv, newv);
82 #else
83     ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
84 #endif
85     ATOMIC_MMU_CLEANUP;
86     atomic_trace_rmw_post(env, addr, oi);
87     return ret;
88 }
89 
90 #if DATA_SIZE < 16
ATOMIC_NAME(xchg)91 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
92                            MemOpIdx oi, uintptr_t retaddr)
93 {
94     DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
95                                          DATA_SIZE, retaddr);
96     DATA_TYPE ret;
97 
98     ret = qatomic_xchg__nocheck(haddr, val);
99     ATOMIC_MMU_CLEANUP;
100     atomic_trace_rmw_post(env, addr, oi);
101     return ret;
102 }
103 
104 #define GEN_ATOMIC_HELPER(X)                                        \
105 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
106                         ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
107 {                                                                   \
108     DATA_TYPE *haddr, ret;                                          \
109     haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
110     ret = qatomic_##X(haddr, val);                                  \
111     ATOMIC_MMU_CLEANUP;                                             \
112     atomic_trace_rmw_post(env, addr, oi);                           \
113     return ret;                                                     \
114 }
115 
116 GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)117 GEN_ATOMIC_HELPER(fetch_and)
118 GEN_ATOMIC_HELPER(fetch_or)
119 GEN_ATOMIC_HELPER(fetch_xor)
120 GEN_ATOMIC_HELPER(add_fetch)
121 GEN_ATOMIC_HELPER(and_fetch)
122 GEN_ATOMIC_HELPER(or_fetch)
123 GEN_ATOMIC_HELPER(xor_fetch)
124 
125 #undef GEN_ATOMIC_HELPER
126 
127 /*
128  * These helpers are, as a whole, full barriers.  Within the helper,
129  * the leading barrier is explicit and the trailing barrier is within
130  * cmpxchg primitive.
131  *
132  * Trace this load + RMW loop as a single RMW op. This way, regardless
133  * of CF_PARALLEL's value, we'll trace just a read and a write.
134  */
135 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
136 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
137                         ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
138 {                                                                   \
139     XDATA_TYPE *haddr, cmp, old, new, val = xval;                   \
140     haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
141     smp_mb();                                                       \
142     cmp = qatomic_read__nocheck(haddr);                             \
143     do {                                                            \
144         old = cmp; new = FN(old, val);                              \
145         cmp = qatomic_cmpxchg__nocheck(haddr, old, new);            \
146     } while (cmp != old);                                           \
147     ATOMIC_MMU_CLEANUP;                                             \
148     atomic_trace_rmw_post(env, addr, oi);                           \
149     return RET;                                                     \
150 }
151 
152 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
153 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
154 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
155 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
156 
157 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
158 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
159 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
160 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
161 
162 #undef GEN_ATOMIC_HELPER_FN
163 #endif /* DATA SIZE < 16 */
164 
165 #undef END
166 
167 #if DATA_SIZE > 1
168 
169 /* Define reverse-host-endian atomic operations.  Note that END is used
170    within the ATOMIC_NAME macro.  */
171 #if HOST_BIG_ENDIAN
172 # define END  _le
173 #else
174 # define END  _be
175 #endif
176 
177 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
178                               ABI_TYPE cmpv, ABI_TYPE newv,
179                               MemOpIdx oi, uintptr_t retaddr)
180 {
181     DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
182                                          DATA_SIZE, retaddr);
183     DATA_TYPE ret;
184 
185 #if DATA_SIZE == 16
186     ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
187 #else
188     ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
189 #endif
190     ATOMIC_MMU_CLEANUP;
191     atomic_trace_rmw_post(env, addr, oi);
192     return BSWAP(ret);
193 }
194 
195 #if DATA_SIZE < 16
ATOMIC_NAME(xchg)196 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
197                            MemOpIdx oi, uintptr_t retaddr)
198 {
199     DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
200                                          DATA_SIZE, retaddr);
201     ABI_TYPE ret;
202 
203     ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
204     ATOMIC_MMU_CLEANUP;
205     atomic_trace_rmw_post(env, addr, oi);
206     return BSWAP(ret);
207 }
208 
209 #define GEN_ATOMIC_HELPER(X)                                        \
210 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
211                         ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
212 {                                                                   \
213     DATA_TYPE *haddr, ret;                                          \
214     haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
215     ret = qatomic_##X(haddr, BSWAP(val));                           \
216     ATOMIC_MMU_CLEANUP;                                             \
217     atomic_trace_rmw_post(env, addr, oi);                           \
218     return BSWAP(ret);                                              \
219 }
220 
221 GEN_ATOMIC_HELPER(fetch_and)
222 GEN_ATOMIC_HELPER(fetch_or)
223 GEN_ATOMIC_HELPER(fetch_xor)
224 GEN_ATOMIC_HELPER(and_fetch)
225 GEN_ATOMIC_HELPER(or_fetch)
226 GEN_ATOMIC_HELPER(xor_fetch)
227 
228 #undef GEN_ATOMIC_HELPER
229 
230 /* These helpers are, as a whole, full barriers.  Within the helper,
231  * the leading barrier is explicit and the trailing barrier is within
232  * cmpxchg primitive.
233  *
234  * Trace this load + RMW loop as a single RMW op. This way, regardless
235  * of CF_PARALLEL's value, we'll trace just a read and a write.
236  */
237 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
238 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
239                         ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
240 {                                                                   \
241     XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval;              \
242     haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
243     smp_mb();                                                       \
244     ldn = qatomic_read__nocheck(haddr);                             \
245     do {                                                            \
246         ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
247         ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));     \
248     } while (ldo != ldn);                                           \
249     ATOMIC_MMU_CLEANUP;                                             \
250     atomic_trace_rmw_post(env, addr, oi);                           \
251     return RET;                                                     \
252 }
253 
254 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
255 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
256 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
257 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
258 
259 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
260 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
261 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
262 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
263 
264 /* Note that for addition, we need to use a separate cmpxchg loop instead
265    of bswaps for the reverse-host-endian helpers.  */
266 #define ADD(X, Y)   (X + Y)
267 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
268 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
269 #undef ADD
270 
271 #undef GEN_ATOMIC_HELPER_FN
272 #endif /* DATA_SIZE < 16 */
273 
274 #undef END
275 #endif /* DATA_SIZE > 1 */
276 
277 #undef BSWAP
278 #undef ABI_TYPE
279 #undef DATA_TYPE
280 #undef SDATA_TYPE
281 #undef SUFFIX
282 #undef DATA_SIZE
283 #undef SHIFT
284