1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3 #include "rseq-bits-template.h"
4
5 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
7
8 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)9 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
10 {
11 RSEQ_INJECT_C(9)
12
13 __asm__ __volatile__ goto (
14 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
15 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
16 #ifdef RSEQ_COMPARE_TWICE
17 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
18 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
19 #endif
20 /* Start rseq by storing table entry pointer into rseq_cs. */
21 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
22 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
23 RSEQ_INJECT_ASM(3)
24 LONG_CMP " %[expect], %[v]\n\t"
25 "jnz %l[cmpfail]\n\t"
26 RSEQ_INJECT_ASM(4)
27 #ifdef RSEQ_COMPARE_TWICE
28 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
29 LONG_CMP " %[expect], %[v]\n\t"
30 "jnz %l[error2]\n\t"
31 #endif
32 /* final store */
33 LONG_S " %[newv], %[v]\n\t"
34 "2:\n\t"
35 RSEQ_INJECT_ASM(5)
36 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
37 : /* gcc asm goto does not allow outputs */
38 : [cpu_id] "r" (cpu),
39 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
40 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
41 [v] "m" (*v),
42 [expect] "r" (expect),
43 [newv] "r" (newv)
44 RSEQ_INJECT_INPUT
45 : "memory", "cc", "r0"
46 RSEQ_INJECT_CLOBBER
47 : abort, cmpfail
48 #ifdef RSEQ_COMPARE_TWICE
49 , error1, error2
50 #endif
51 );
52 rseq_after_asm_goto();
53 return 0;
54 abort:
55 rseq_after_asm_goto();
56 RSEQ_INJECT_FAILED
57 return -1;
58 cmpfail:
59 rseq_after_asm_goto();
60 return 1;
61 #ifdef RSEQ_COMPARE_TWICE
62 error1:
63 rseq_after_asm_goto();
64 rseq_bug("cpu_id comparison failed");
65 error2:
66 rseq_after_asm_goto();
67 rseq_bug("expected value comparison failed");
68 #endif
69 }
70
71 /*
72 * Compare @v against @expectnot. When it does _not_ match, load @v
73 * into @load, and store the content of *@v + voffp into @v.
74 */
75 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)76 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
77 long voffp, intptr_t *load, int cpu)
78 {
79 RSEQ_INJECT_C(9)
80
81 __asm__ __volatile__ goto (
82 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
83 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
84 #ifdef RSEQ_COMPARE_TWICE
85 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
86 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
87 #endif
88 /* Start rseq by storing table entry pointer into rseq_cs. */
89 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
90 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
91 RSEQ_INJECT_ASM(3)
92 LONG_L " %%r1, %[v]\n\t"
93 LONG_CMP_R " %%r1, %[expectnot]\n\t"
94 "je %l[cmpfail]\n\t"
95 RSEQ_INJECT_ASM(4)
96 #ifdef RSEQ_COMPARE_TWICE
97 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
98 LONG_L " %%r1, %[v]\n\t"
99 LONG_CMP_R " %%r1, %[expectnot]\n\t"
100 "je %l[error2]\n\t"
101 #endif
102 LONG_S " %%r1, %[load]\n\t"
103 LONG_ADD_R " %%r1, %[voffp]\n\t"
104 LONG_L " %%r1, 0(%%r1)\n\t"
105 /* final store */
106 LONG_S " %%r1, %[v]\n\t"
107 "2:\n\t"
108 RSEQ_INJECT_ASM(5)
109 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
110 : /* gcc asm goto does not allow outputs */
111 : [cpu_id] "r" (cpu),
112 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
113 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
114 /* final store input */
115 [v] "m" (*v),
116 [expectnot] "r" (expectnot),
117 [voffp] "r" (voffp),
118 [load] "m" (*load)
119 RSEQ_INJECT_INPUT
120 : "memory", "cc", "r0", "r1"
121 RSEQ_INJECT_CLOBBER
122 : abort, cmpfail
123 #ifdef RSEQ_COMPARE_TWICE
124 , error1, error2
125 #endif
126 );
127 rseq_after_asm_goto();
128 return 0;
129 abort:
130 rseq_after_asm_goto();
131 RSEQ_INJECT_FAILED
132 return -1;
133 cmpfail:
134 rseq_after_asm_goto();
135 return 1;
136 #ifdef RSEQ_COMPARE_TWICE
137 error1:
138 rseq_after_asm_goto();
139 rseq_bug("cpu_id comparison failed");
140 error2:
141 rseq_after_asm_goto();
142 rseq_bug("expected value comparison failed");
143 #endif
144 }
145
146 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)147 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
148 {
149 RSEQ_INJECT_C(9)
150
151 __asm__ __volatile__ goto (
152 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
153 #ifdef RSEQ_COMPARE_TWICE
154 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
155 #endif
156 /* Start rseq by storing table entry pointer into rseq_cs. */
157 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
158 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
159 RSEQ_INJECT_ASM(3)
160 #ifdef RSEQ_COMPARE_TWICE
161 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
162 #endif
163 LONG_L " %%r0, %[v]\n\t"
164 LONG_ADD_R " %%r0, %[count]\n\t"
165 /* final store */
166 LONG_S " %%r0, %[v]\n\t"
167 "2:\n\t"
168 RSEQ_INJECT_ASM(4)
169 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
170 : /* gcc asm goto does not allow outputs */
171 : [cpu_id] "r" (cpu),
172 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
173 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
174 /* final store input */
175 [v] "m" (*v),
176 [count] "r" (count)
177 RSEQ_INJECT_INPUT
178 : "memory", "cc", "r0"
179 RSEQ_INJECT_CLOBBER
180 : abort
181 #ifdef RSEQ_COMPARE_TWICE
182 , error1
183 #endif
184 );
185 rseq_after_asm_goto();
186 return 0;
187 abort:
188 rseq_after_asm_goto();
189 RSEQ_INJECT_FAILED
190 return -1;
191 #ifdef RSEQ_COMPARE_TWICE
192 error1:
193 rseq_after_asm_goto();
194 rseq_bug("cpu_id comparison failed");
195 #endif
196 }
197
198 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)199 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
200 intptr_t *v2, intptr_t expect2,
201 intptr_t newv, int cpu)
202 {
203 RSEQ_INJECT_C(9)
204
205 __asm__ __volatile__ goto (
206 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
207 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
208 #ifdef RSEQ_COMPARE_TWICE
209 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
210 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
211 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
212 #endif
213 /* Start rseq by storing table entry pointer into rseq_cs. */
214 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
215 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
216 RSEQ_INJECT_ASM(3)
217 LONG_CMP " %[expect], %[v]\n\t"
218 "jnz %l[cmpfail]\n\t"
219 RSEQ_INJECT_ASM(4)
220 LONG_CMP " %[expect2], %[v2]\n\t"
221 "jnz %l[cmpfail]\n\t"
222 RSEQ_INJECT_ASM(5)
223 #ifdef RSEQ_COMPARE_TWICE
224 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
225 LONG_CMP " %[expect], %[v]\n\t"
226 "jnz %l[error2]\n\t"
227 LONG_CMP " %[expect2], %[v2]\n\t"
228 "jnz %l[error3]\n\t"
229 #endif
230 /* final store */
231 LONG_S " %[newv], %[v]\n\t"
232 "2:\n\t"
233 RSEQ_INJECT_ASM(6)
234 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
235 : /* gcc asm goto does not allow outputs */
236 : [cpu_id] "r" (cpu),
237 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
238 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
239 /* cmp2 input */
240 [v2] "m" (*v2),
241 [expect2] "r" (expect2),
242 /* final store input */
243 [v] "m" (*v),
244 [expect] "r" (expect),
245 [newv] "r" (newv)
246 RSEQ_INJECT_INPUT
247 : "memory", "cc", "r0"
248 RSEQ_INJECT_CLOBBER
249 : abort, cmpfail
250 #ifdef RSEQ_COMPARE_TWICE
251 , error1, error2, error3
252 #endif
253 );
254 rseq_after_asm_goto();
255 return 0;
256 abort:
257 rseq_after_asm_goto();
258 RSEQ_INJECT_FAILED
259 return -1;
260 cmpfail:
261 rseq_after_asm_goto();
262 return 1;
263 #ifdef RSEQ_COMPARE_TWICE
264 error1:
265 rseq_after_asm_goto();
266 rseq_bug("cpu_id comparison failed");
267 error2:
268 rseq_after_asm_goto();
269 rseq_bug("1st expected value comparison failed");
270 error3:
271 rseq_after_asm_goto();
272 rseq_bug("2nd expected value comparison failed");
273 #endif
274 }
275
276 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
277 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
278
279 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
280 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
281
282 /* s390 is TSO. */
283 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)284 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
285 intptr_t *v2, intptr_t newv2,
286 intptr_t newv, int cpu)
287 {
288 RSEQ_INJECT_C(9)
289
290 __asm__ __volatile__ goto (
291 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
292 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
293 #ifdef RSEQ_COMPARE_TWICE
294 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
295 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
296 #endif
297 /* Start rseq by storing table entry pointer into rseq_cs. */
298 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
300 RSEQ_INJECT_ASM(3)
301 LONG_CMP " %[expect], %[v]\n\t"
302 "jnz %l[cmpfail]\n\t"
303 RSEQ_INJECT_ASM(4)
304 #ifdef RSEQ_COMPARE_TWICE
305 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
306 LONG_CMP " %[expect], %[v]\n\t"
307 "jnz %l[error2]\n\t"
308 #endif
309 /* try store */
310 LONG_S " %[newv2], %[v2]\n\t"
311 RSEQ_INJECT_ASM(5)
312 /* final store */
313 LONG_S " %[newv], %[v]\n\t"
314 "2:\n\t"
315 RSEQ_INJECT_ASM(6)
316 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
317 : /* gcc asm goto does not allow outputs */
318 : [cpu_id] "r" (cpu),
319 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
320 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
321 /* try store input */
322 [v2] "m" (*v2),
323 [newv2] "r" (newv2),
324 /* final store input */
325 [v] "m" (*v),
326 [expect] "r" (expect),
327 [newv] "r" (newv)
328 RSEQ_INJECT_INPUT
329 : "memory", "cc", "r0"
330 RSEQ_INJECT_CLOBBER
331 : abort, cmpfail
332 #ifdef RSEQ_COMPARE_TWICE
333 , error1, error2
334 #endif
335 );
336 rseq_after_asm_goto();
337 return 0;
338 abort:
339 rseq_after_asm_goto();
340 RSEQ_INJECT_FAILED
341 return -1;
342 cmpfail:
343 rseq_after_asm_goto();
344 return 1;
345 #ifdef RSEQ_COMPARE_TWICE
346 error1:
347 rseq_after_asm_goto();
348 rseq_bug("cpu_id comparison failed");
349 error2:
350 rseq_after_asm_goto();
351 rseq_bug("expected value comparison failed");
352 #endif
353 }
354
355 /* s390 is TSO. */
356 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)357 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
358 void *dst, void *src, size_t len,
359 intptr_t newv, int cpu)
360 {
361 uint64_t rseq_scratch[3];
362
363 RSEQ_INJECT_C(9)
364
365 __asm__ __volatile__ goto (
366 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
367 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
368 #ifdef RSEQ_COMPARE_TWICE
369 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
370 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
371 #endif
372 LONG_S " %[src], %[rseq_scratch0]\n\t"
373 LONG_S " %[dst], %[rseq_scratch1]\n\t"
374 LONG_S " %[len], %[rseq_scratch2]\n\t"
375 /* Start rseq by storing table entry pointer into rseq_cs. */
376 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
377 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
378 RSEQ_INJECT_ASM(3)
379 LONG_CMP " %[expect], %[v]\n\t"
380 "jnz 5f\n\t"
381 RSEQ_INJECT_ASM(4)
382 #ifdef RSEQ_COMPARE_TWICE
383 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
384 LONG_CMP " %[expect], %[v]\n\t"
385 "jnz 7f\n\t"
386 #endif
387 /* try memcpy */
388 LONG_LT_R " %[len], %[len]\n\t"
389 "jz 333f\n\t"
390 "222:\n\t"
391 "ic %%r0,0(%[src])\n\t"
392 "stc %%r0,0(%[dst])\n\t"
393 LONG_ADDI " %[src], 1\n\t"
394 LONG_ADDI " %[dst], 1\n\t"
395 LONG_ADDI " %[len], -1\n\t"
396 "jnz 222b\n\t"
397 "333:\n\t"
398 RSEQ_INJECT_ASM(5)
399 /* final store */
400 LONG_S " %[newv], %[v]\n\t"
401 "2:\n\t"
402 RSEQ_INJECT_ASM(6)
403 /* teardown */
404 LONG_L " %[len], %[rseq_scratch2]\n\t"
405 LONG_L " %[dst], %[rseq_scratch1]\n\t"
406 LONG_L " %[src], %[rseq_scratch0]\n\t"
407 RSEQ_ASM_DEFINE_ABORT(4,
408 LONG_L " %[len], %[rseq_scratch2]\n\t"
409 LONG_L " %[dst], %[rseq_scratch1]\n\t"
410 LONG_L " %[src], %[rseq_scratch0]\n\t",
411 abort)
412 RSEQ_ASM_DEFINE_CMPFAIL(5,
413 LONG_L " %[len], %[rseq_scratch2]\n\t"
414 LONG_L " %[dst], %[rseq_scratch1]\n\t"
415 LONG_L " %[src], %[rseq_scratch0]\n\t",
416 cmpfail)
417 #ifdef RSEQ_COMPARE_TWICE
418 RSEQ_ASM_DEFINE_CMPFAIL(6,
419 LONG_L " %[len], %[rseq_scratch2]\n\t"
420 LONG_L " %[dst], %[rseq_scratch1]\n\t"
421 LONG_L " %[src], %[rseq_scratch0]\n\t",
422 error1)
423 RSEQ_ASM_DEFINE_CMPFAIL(7,
424 LONG_L " %[len], %[rseq_scratch2]\n\t"
425 LONG_L " %[dst], %[rseq_scratch1]\n\t"
426 LONG_L " %[src], %[rseq_scratch0]\n\t",
427 error2)
428 #endif
429 : /* gcc asm goto does not allow outputs */
430 : [cpu_id] "r" (cpu),
431 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
432 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
433 /* final store input */
434 [v] "m" (*v),
435 [expect] "r" (expect),
436 [newv] "r" (newv),
437 /* try memcpy input */
438 [dst] "r" (dst),
439 [src] "r" (src),
440 [len] "r" (len),
441 [rseq_scratch0] "m" (rseq_scratch[0]),
442 [rseq_scratch1] "m" (rseq_scratch[1]),
443 [rseq_scratch2] "m" (rseq_scratch[2])
444 RSEQ_INJECT_INPUT
445 : "memory", "cc", "r0"
446 RSEQ_INJECT_CLOBBER
447 : abort, cmpfail
448 #ifdef RSEQ_COMPARE_TWICE
449 , error1, error2
450 #endif
451 );
452 rseq_after_asm_goto();
453 return 0;
454 abort:
455 rseq_after_asm_goto();
456 RSEQ_INJECT_FAILED
457 return -1;
458 cmpfail:
459 rseq_after_asm_goto();
460 return 1;
461 #ifdef RSEQ_COMPARE_TWICE
462 error1:
463 rseq_after_asm_goto();
464 rseq_bug("cpu_id comparison failed");
465 error2:
466 rseq_after_asm_goto();
467 rseq_bug("expected value comparison failed");
468 #endif
469 }
470
471 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
472 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
473
474 #include "rseq-bits-reset.h"
475