1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3 #include "rseq-bits-template.h"
4
5 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
7
8 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)9 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
10 {
11 RSEQ_INJECT_C(9)
12
13 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
14 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
15 #ifdef RSEQ_COMPARE_TWICE
16 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
17 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
18 #endif
19 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
20 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
21 RSEQ_INJECT_ASM(3)
22 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
23 RSEQ_INJECT_ASM(4)
24 #ifdef RSEQ_COMPARE_TWICE
25 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
26 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
27 #endif
28 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
29 RSEQ_INJECT_ASM(5)
30 RSEQ_ASM_DEFINE_ABORT(4, abort)
31 : /* gcc asm goto does not allow outputs */
32 : [cpu_id] "r" (cpu),
33 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
34 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
35 [v] "m" (*v),
36 [expect] "r" (expect),
37 [newv] "r" (newv)
38 RSEQ_INJECT_INPUT
39 : "memory", RSEQ_ASM_TMP_REG_1
40 RSEQ_INJECT_CLOBBER
41 : abort, cmpfail
42 #ifdef RSEQ_COMPARE_TWICE
43 , error1, error2
44 #endif
45 );
46
47 return 0;
48 abort:
49 RSEQ_INJECT_FAILED
50 return -1;
51 cmpfail:
52 return 1;
53 #ifdef RSEQ_COMPARE_TWICE
54 error1:
55 rseq_bug("cpu_id comparison failed");
56 error2:
57 rseq_bug("expected value comparison failed");
58 #endif
59 }
60
61 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)62 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
63 off_t voffp, intptr_t *load, int cpu)
64 {
65 RSEQ_INJECT_C(9)
66
67 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
68 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
69 #ifdef RSEQ_COMPARE_TWICE
70 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
71 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
72 #endif
73 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
74 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
75 RSEQ_INJECT_ASM(3)
76 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
77 RSEQ_INJECT_ASM(4)
78 #ifdef RSEQ_COMPARE_TWICE
79 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
80 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
81 #endif
82 RSEQ_ASM_OP_R_LOAD(v)
83 RSEQ_ASM_OP_R_STORE(load)
84 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
85 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
86 RSEQ_INJECT_ASM(5)
87 RSEQ_ASM_DEFINE_ABORT(4, abort)
88 : /* gcc asm goto does not allow outputs */
89 : [cpu_id] "r" (cpu),
90 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
91 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
92 [v] "m" (*v),
93 [expectnot] "r" (expectnot),
94 [load] "m" (*load),
95 [voffp] "r" (voffp)
96 RSEQ_INJECT_INPUT
97 : "memory", RSEQ_ASM_TMP_REG_1
98 RSEQ_INJECT_CLOBBER
99 : abort, cmpfail
100 #ifdef RSEQ_COMPARE_TWICE
101 , error1, error2
102 #endif
103 );
104 return 0;
105 abort:
106 RSEQ_INJECT_FAILED
107 return -1;
108 cmpfail:
109 return 1;
110 #ifdef RSEQ_COMPARE_TWICE
111 error1:
112 rseq_bug("cpu_id comparison failed");
113 error2:
114 rseq_bug("expected value comparison failed");
115 #endif
116 }
117
118 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)119 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
120 {
121 RSEQ_INJECT_C(9)
122
123 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
124 #ifdef RSEQ_COMPARE_TWICE
125 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
126 #endif
127 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
128 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
129 RSEQ_INJECT_ASM(3)
130 #ifdef RSEQ_COMPARE_TWICE
131 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
132 #endif
133 RSEQ_ASM_OP_R_LOAD(v)
134 RSEQ_ASM_OP_R_ADD(count)
135 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
136 RSEQ_INJECT_ASM(4)
137 RSEQ_ASM_DEFINE_ABORT(4, abort)
138 : /* gcc asm goto does not allow outputs */
139 : [cpu_id] "r" (cpu),
140 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
141 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
142 [v] "m" (*v),
143 [count] "r" (count)
144 RSEQ_INJECT_INPUT
145 : "memory", RSEQ_ASM_TMP_REG_1
146 RSEQ_INJECT_CLOBBER
147 : abort
148 #ifdef RSEQ_COMPARE_TWICE
149 , error1
150 #endif
151 );
152 return 0;
153 abort:
154 RSEQ_INJECT_FAILED
155 return -1;
156 #ifdef RSEQ_COMPARE_TWICE
157 error1:
158 rseq_bug("cpu_id comparison failed");
159 #endif
160 }
161
162 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)163 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
164 intptr_t *v2, intptr_t expect2,
165 intptr_t newv, int cpu)
166 {
167 RSEQ_INJECT_C(9)
168
169 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
170 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
171 #ifdef RSEQ_COMPARE_TWICE
172 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
173 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
174 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
175 #endif
176 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
177 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
178 RSEQ_INJECT_ASM(3)
179 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
180 RSEQ_INJECT_ASM(4)
181 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
182 RSEQ_INJECT_ASM(5)
183 #ifdef RSEQ_COMPARE_TWICE
184 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
185 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
186 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
187 #endif
188 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
189 RSEQ_INJECT_ASM(6)
190 RSEQ_ASM_DEFINE_ABORT(4, abort)
191 : /* gcc asm goto does not allow outputs */
192 : [cpu_id] "r" (cpu),
193 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
194 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
195 [v] "m" (*v),
196 [expect] "r" (expect),
197 [v2] "m" (*v2),
198 [expect2] "r" (expect2),
199 [newv] "r" (newv)
200 RSEQ_INJECT_INPUT
201 : "memory", RSEQ_ASM_TMP_REG_1
202 RSEQ_INJECT_CLOBBER
203 : abort, cmpfail
204 #ifdef RSEQ_COMPARE_TWICE
205 , error1, error2, error3
206 #endif
207 );
208
209 return 0;
210 abort:
211 RSEQ_INJECT_FAILED
212 return -1;
213 cmpfail:
214 return 1;
215 #ifdef RSEQ_COMPARE_TWICE
216 error1:
217 rseq_bug("cpu_id comparison failed");
218 error2:
219 rseq_bug("expected value comparison failed");
220 error3:
221 rseq_bug("2nd expected value comparison failed");
222 #endif
223 }
224
225 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
226
227 /*
228 * pval = *(ptr+off)
229 * *pval += inc;
230 */
231 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)232 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
233 {
234 RSEQ_INJECT_C(9)
235
236 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
237 #ifdef RSEQ_COMPARE_TWICE
238 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
239 #endif
240 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
241 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
242 RSEQ_INJECT_ASM(3)
243 #ifdef RSEQ_COMPARE_TWICE
244 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
245 #endif
246 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
247 RSEQ_INJECT_ASM(4)
248 RSEQ_ASM_DEFINE_ABORT(4, abort)
249 : /* gcc asm goto does not allow outputs */
250 : [cpu_id] "r" (cpu),
251 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
252 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
253 [ptr] "r" (ptr),
254 [off] "er" (off),
255 [inc] "er" (inc)
256 RSEQ_INJECT_INPUT
257 : "memory", RSEQ_ASM_TMP_REG_1
258 RSEQ_INJECT_CLOBBER
259 : abort
260 #ifdef RSEQ_COMPARE_TWICE
261 , error1
262 #endif
263 );
264 return 0;
265 abort:
266 RSEQ_INJECT_FAILED
267 return -1;
268 #ifdef RSEQ_COMPARE_TWICE
269 error1:
270 rseq_bug("cpu_id comparison failed");
271 #endif
272 }
273
274 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
275 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
276
277 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
278 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
279
280 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)281 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
282 intptr_t *v2, intptr_t newv2,
283 intptr_t newv, int cpu)
284 {
285 RSEQ_INJECT_C(9)
286
287 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
288 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
289 #ifdef RSEQ_COMPARE_TWICE
290 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
291 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
292 #endif
293 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
294 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
295 RSEQ_INJECT_ASM(3)
296 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
297 RSEQ_INJECT_ASM(4)
298 #ifdef RSEQ_COMPARE_TWICE
299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
300 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
301 #endif
302 RSEQ_ASM_OP_STORE(newv2, v2)
303 RSEQ_INJECT_ASM(5)
304 #ifdef RSEQ_TEMPLATE_MO_RELEASE
305 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
306 #else
307 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
308 #endif
309 RSEQ_INJECT_ASM(6)
310 RSEQ_ASM_DEFINE_ABORT(4, abort)
311 : /* gcc asm goto does not allow outputs */
312 : [cpu_id] "r" (cpu),
313 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
314 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
315 [expect] "r" (expect),
316 [v] "m" (*v),
317 [newv] "r" (newv),
318 [v2] "m" (*v2),
319 [newv2] "r" (newv2)
320 RSEQ_INJECT_INPUT
321 : "memory", RSEQ_ASM_TMP_REG_1
322 RSEQ_INJECT_CLOBBER
323 : abort, cmpfail
324 #ifdef RSEQ_COMPARE_TWICE
325 , error1, error2
326 #endif
327 );
328
329 return 0;
330 abort:
331 RSEQ_INJECT_FAILED
332 return -1;
333 cmpfail:
334 return 1;
335 #ifdef RSEQ_COMPARE_TWICE
336 error1:
337 rseq_bug("cpu_id comparison failed");
338 error2:
339 rseq_bug("expected value comparison failed");
340 #endif
341 }
342
343 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)344 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
345 void *dst, void *src, size_t len,
346 intptr_t newv, int cpu)
347 {
348 RSEQ_INJECT_C(9)
349 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
350 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
351 #ifdef RSEQ_COMPARE_TWICE
352 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
353 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
354 #endif
355 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
356 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
357 RSEQ_INJECT_ASM(3)
358 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
359 RSEQ_INJECT_ASM(4)
360 #ifdef RSEQ_COMPARE_TWICE
361 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
362 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
363 #endif
364 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
365 RSEQ_INJECT_ASM(5)
366 #ifdef RSEQ_TEMPLATE_MO_RELEASE
367 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
368 #else
369 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
370 #endif
371 RSEQ_INJECT_ASM(6)
372 RSEQ_ASM_DEFINE_ABORT(4, abort)
373 : /* gcc asm goto does not allow outputs */
374 : [cpu_id] "r" (cpu),
375 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
376 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
377 [expect] "r" (expect),
378 [v] "m" (*v),
379 [newv] "r" (newv),
380 [dst] "r" (dst),
381 [src] "r" (src),
382 [len] "r" (len)
383 RSEQ_INJECT_INPUT
384 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
385 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
386 RSEQ_INJECT_CLOBBER
387 : abort, cmpfail
388 #ifdef RSEQ_COMPARE_TWICE
389 , error1, error2
390 #endif
391 );
392
393 return 0;
394 abort:
395 RSEQ_INJECT_FAILED
396 return -1;
397 cmpfail:
398 return 1;
399 #ifdef RSEQ_COMPARE_TWICE
400 error1:
401 rseq_bug("cpu_id comparison failed");
402 error2:
403 rseq_bug("expected value comparison failed");
404 #endif
405 }
406
407 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
408 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
409
410 #include "rseq-bits-reset.h"
411