1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-ppc-bits.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */
8
9 #include "rseq-bits-template.h"
10
11 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
13
14 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)15 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
16 {
17 RSEQ_INJECT_C(9)
18
19 __asm__ __volatile__ goto (
20 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
21 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
22 #ifdef RSEQ_COMPARE_TWICE
23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
25 #endif
26 /* Start rseq by storing table entry pointer into rseq_cs. */
27 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
28 /* cmp cpuid */
29 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
30 RSEQ_INJECT_ASM(3)
31 /* cmp @v equal to @expect */
32 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
33 RSEQ_INJECT_ASM(4)
34 #ifdef RSEQ_COMPARE_TWICE
35 /* cmp cpuid */
36 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
37 /* cmp @v equal to @expect */
38 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
39 #endif
40 /* final store */
41 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
42 RSEQ_INJECT_ASM(5)
43 RSEQ_ASM_DEFINE_ABORT(4, abort)
44 : /* gcc asm goto does not allow outputs */
45 : [cpu_id] "r" (cpu),
46 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
48 [v] "m" (*v),
49 [expect] "r" (expect),
50 [newv] "r" (newv)
51 RSEQ_INJECT_INPUT
52 : "memory", "cc", "r17"
53 RSEQ_INJECT_CLOBBER
54 : abort, cmpfail
55 #ifdef RSEQ_COMPARE_TWICE
56 , error1, error2
57 #endif
58 );
59 rseq_after_asm_goto();
60 return 0;
61 abort:
62 rseq_after_asm_goto();
63 RSEQ_INJECT_FAILED
64 return -1;
65 cmpfail:
66 rseq_after_asm_goto();
67 return 1;
68 #ifdef RSEQ_COMPARE_TWICE
69 error1:
70 rseq_after_asm_goto();
71 rseq_bug("cpu_id comparison failed");
72 error2:
73 rseq_after_asm_goto();
74 rseq_bug("expected value comparison failed");
75 #endif
76 }
77
78 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)79 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
80 long voffp, intptr_t *load, int cpu)
81 {
82 RSEQ_INJECT_C(9)
83
84 __asm__ __volatile__ goto (
85 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
86 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
87 #ifdef RSEQ_COMPARE_TWICE
88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
89 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
90 #endif
91 /* Start rseq by storing table entry pointer into rseq_cs. */
92 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
93 /* cmp cpuid */
94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
95 RSEQ_INJECT_ASM(3)
96 /* cmp @v not equal to @expectnot */
97 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
98 RSEQ_INJECT_ASM(4)
99 #ifdef RSEQ_COMPARE_TWICE
100 /* cmp cpuid */
101 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
102 /* cmp @v not equal to @expectnot */
103 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
104 #endif
105 /* load the value of @v */
106 RSEQ_ASM_OP_R_LOAD(v)
107 /* store it in @load */
108 RSEQ_ASM_OP_R_STORE(load)
109 /* dereference voffp(v) */
110 RSEQ_ASM_OP_R_LOADX(voffp)
111 /* final store the value at voffp(v) */
112 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
113 RSEQ_INJECT_ASM(5)
114 RSEQ_ASM_DEFINE_ABORT(4, abort)
115 : /* gcc asm goto does not allow outputs */
116 : [cpu_id] "r" (cpu),
117 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
118 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
119 /* final store input */
120 [v] "m" (*v),
121 [expectnot] "r" (expectnot),
122 [voffp] "b" (voffp),
123 [load] "m" (*load)
124 RSEQ_INJECT_INPUT
125 : "memory", "cc", "r17"
126 RSEQ_INJECT_CLOBBER
127 : abort, cmpfail
128 #ifdef RSEQ_COMPARE_TWICE
129 , error1, error2
130 #endif
131 );
132 rseq_after_asm_goto();
133 return 0;
134 abort:
135 rseq_after_asm_goto();
136 RSEQ_INJECT_FAILED
137 return -1;
138 cmpfail:
139 rseq_after_asm_goto();
140 return 1;
141 #ifdef RSEQ_COMPARE_TWICE
142 error1:
143 rseq_after_asm_goto();
144 rseq_bug("cpu_id comparison failed");
145 error2:
146 rseq_after_asm_goto();
147 rseq_bug("expected value comparison failed");
148 #endif
149 }
150
151 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)152 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
153 {
154 RSEQ_INJECT_C(9)
155
156 __asm__ __volatile__ goto (
157 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
158 #ifdef RSEQ_COMPARE_TWICE
159 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
160 #endif
161 /* Start rseq by storing table entry pointer into rseq_cs. */
162 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
163 /* cmp cpuid */
164 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
165 RSEQ_INJECT_ASM(3)
166 #ifdef RSEQ_COMPARE_TWICE
167 /* cmp cpuid */
168 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
169 #endif
170 /* load the value of @v */
171 RSEQ_ASM_OP_R_LOAD(v)
172 /* add @count to it */
173 RSEQ_ASM_OP_R_ADD(count)
174 /* final store */
175 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
176 RSEQ_INJECT_ASM(4)
177 RSEQ_ASM_DEFINE_ABORT(4, abort)
178 : /* gcc asm goto does not allow outputs */
179 : [cpu_id] "r" (cpu),
180 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
181 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
182 /* final store input */
183 [v] "m" (*v),
184 [count] "r" (count)
185 RSEQ_INJECT_INPUT
186 : "memory", "cc", "r17"
187 RSEQ_INJECT_CLOBBER
188 : abort
189 #ifdef RSEQ_COMPARE_TWICE
190 , error1
191 #endif
192 );
193 rseq_after_asm_goto();
194 return 0;
195 abort:
196 rseq_after_asm_goto();
197 RSEQ_INJECT_FAILED
198 return -1;
199 #ifdef RSEQ_COMPARE_TWICE
200 error1:
201 rseq_after_asm_goto();
202 rseq_bug("cpu_id comparison failed");
203 #endif
204 }
205
206 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)207 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
208 intptr_t *v2, intptr_t expect2,
209 intptr_t newv, int cpu)
210 {
211 RSEQ_INJECT_C(9)
212
213 __asm__ __volatile__ goto (
214 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
216 #ifdef RSEQ_COMPARE_TWICE
217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
218 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
220 #endif
221 /* Start rseq by storing table entry pointer into rseq_cs. */
222 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
223 /* cmp cpuid */
224 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
225 RSEQ_INJECT_ASM(3)
226 /* cmp @v equal to @expect */
227 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
228 RSEQ_INJECT_ASM(4)
229 /* cmp @v2 equal to @expct2 */
230 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
231 RSEQ_INJECT_ASM(5)
232 #ifdef RSEQ_COMPARE_TWICE
233 /* cmp cpuid */
234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
235 /* cmp @v equal to @expect */
236 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
237 /* cmp @v2 equal to @expct2 */
238 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
239 #endif
240 /* final store */
241 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
242 RSEQ_INJECT_ASM(6)
243 RSEQ_ASM_DEFINE_ABORT(4, abort)
244 : /* gcc asm goto does not allow outputs */
245 : [cpu_id] "r" (cpu),
246 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
247 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
248 /* cmp2 input */
249 [v2] "m" (*v2),
250 [expect2] "r" (expect2),
251 /* final store input */
252 [v] "m" (*v),
253 [expect] "r" (expect),
254 [newv] "r" (newv)
255 RSEQ_INJECT_INPUT
256 : "memory", "cc", "r17"
257 RSEQ_INJECT_CLOBBER
258 : abort, cmpfail
259 #ifdef RSEQ_COMPARE_TWICE
260 , error1, error2, error3
261 #endif
262 );
263 rseq_after_asm_goto();
264 return 0;
265 abort:
266 rseq_after_asm_goto();
267 RSEQ_INJECT_FAILED
268 return -1;
269 cmpfail:
270 rseq_after_asm_goto();
271 return 1;
272 #ifdef RSEQ_COMPARE_TWICE
273 error1:
274 rseq_after_asm_goto();
275 rseq_bug("cpu_id comparison failed");
276 error2:
277 rseq_after_asm_goto();
278 rseq_bug("1st expected value comparison failed");
279 error3:
280 rseq_after_asm_goto();
281 rseq_bug("2nd expected value comparison failed");
282 #endif
283 }
284
285 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
286 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
287
288 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
289 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
290
291 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)292 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
293 intptr_t *v2, intptr_t newv2,
294 intptr_t newv, int cpu)
295 {
296 RSEQ_INJECT_C(9)
297
298 __asm__ __volatile__ goto (
299 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
300 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
301 #ifdef RSEQ_COMPARE_TWICE
302 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
303 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
304 #endif
305 /* Start rseq by storing table entry pointer into rseq_cs. */
306 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
307 /* cmp cpuid */
308 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
309 RSEQ_INJECT_ASM(3)
310 /* cmp @v equal to @expect */
311 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
312 RSEQ_INJECT_ASM(4)
313 #ifdef RSEQ_COMPARE_TWICE
314 /* cmp cpuid */
315 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
316 /* cmp @v equal to @expect */
317 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
318 #endif
319 /* try store */
320 RSEQ_ASM_OP_STORE(newv2, v2)
321 RSEQ_INJECT_ASM(5)
322 #ifdef RSEQ_TEMPLATE_MO_RELEASE
323 /* for 'release' */
324 "lwsync\n\t"
325 #endif
326 /* final store */
327 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
328 RSEQ_INJECT_ASM(6)
329 RSEQ_ASM_DEFINE_ABORT(4, abort)
330 : /* gcc asm goto does not allow outputs */
331 : [cpu_id] "r" (cpu),
332 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
333 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
334 /* try store input */
335 [v2] "m" (*v2),
336 [newv2] "r" (newv2),
337 /* final store input */
338 [v] "m" (*v),
339 [expect] "r" (expect),
340 [newv] "r" (newv)
341 RSEQ_INJECT_INPUT
342 : "memory", "cc", "r17"
343 RSEQ_INJECT_CLOBBER
344 : abort, cmpfail
345 #ifdef RSEQ_COMPARE_TWICE
346 , error1, error2
347 #endif
348 );
349 rseq_after_asm_goto();
350 return 0;
351 abort:
352 rseq_after_asm_goto();
353 RSEQ_INJECT_FAILED
354 return -1;
355 cmpfail:
356 rseq_after_asm_goto();
357 return 1;
358 #ifdef RSEQ_COMPARE_TWICE
359 error1:
360 rseq_after_asm_goto();
361 rseq_bug("cpu_id comparison failed");
362 error2:
363 rseq_after_asm_goto();
364 rseq_bug("expected value comparison failed");
365 #endif
366 }
367
368 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)369 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
370 void *dst, void *src, size_t len,
371 intptr_t newv, int cpu)
372 {
373 RSEQ_INJECT_C(9)
374
375 __asm__ __volatile__ goto (
376 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
377 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
378 #ifdef RSEQ_COMPARE_TWICE
379 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
380 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
381 #endif
382 /* setup for mempcy */
383 "mr %%r19, %[len]\n\t"
384 "mr %%r20, %[src]\n\t"
385 "mr %%r21, %[dst]\n\t"
386 /* Start rseq by storing table entry pointer into rseq_cs. */
387 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
388 /* cmp cpuid */
389 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
390 RSEQ_INJECT_ASM(3)
391 /* cmp @v equal to @expect */
392 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
393 RSEQ_INJECT_ASM(4)
394 #ifdef RSEQ_COMPARE_TWICE
395 /* cmp cpuid */
396 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
397 /* cmp @v equal to @expect */
398 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
399 #endif
400 /* try memcpy */
401 RSEQ_ASM_OP_R_MEMCPY()
402 RSEQ_INJECT_ASM(5)
403 #ifdef RSEQ_TEMPLATE_MO_RELEASE
404 /* for 'release' */
405 "lwsync\n\t"
406 #endif
407 /* final store */
408 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
409 RSEQ_INJECT_ASM(6)
410 /* teardown */
411 RSEQ_ASM_DEFINE_ABORT(4, abort)
412 : /* gcc asm goto does not allow outputs */
413 : [cpu_id] "r" (cpu),
414 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
415 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
416 /* final store input */
417 [v] "m" (*v),
418 [expect] "r" (expect),
419 [newv] "r" (newv),
420 /* try memcpy input */
421 [dst] "r" (dst),
422 [src] "r" (src),
423 [len] "r" (len)
424 RSEQ_INJECT_INPUT
425 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
426 RSEQ_INJECT_CLOBBER
427 : abort, cmpfail
428 #ifdef RSEQ_COMPARE_TWICE
429 , error1, error2
430 #endif
431 );
432 rseq_after_asm_goto();
433 return 0;
434 abort:
435 rseq_after_asm_goto();
436 RSEQ_INJECT_FAILED
437 return -1;
438 cmpfail:
439 rseq_after_asm_goto();
440 return 1;
441 #ifdef RSEQ_COMPARE_TWICE
442 error1:
443 rseq_after_asm_goto();
444 rseq_bug("cpu_id comparison failed");
445 error2:
446 rseq_after_asm_goto();
447 rseq_bug("expected value comparison failed");
448 #endif
449 }
450
451 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
452 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
453
454 #include "rseq-bits-reset.h"
455