1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * rseq-ppc.h
4  *
5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6  * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7  */
8 
9 /*
10  * RSEQ_SIG is used with the following trap instruction:
11  *
12  * powerpc-be:    0f e5 00 0b           twui   r5,11
13  * powerpc64-le:  0b 00 e5 0f           twui   r5,11
14  * powerpc64-be:  0f e5 00 0b           twui   r5,11
15  */
16 
17 #define RSEQ_SIG	0x0fe5000b
18 
19 #define rseq_smp_mb()		__asm__ __volatile__ ("sync"	::: "memory", "cc")
20 #define rseq_smp_lwsync()	__asm__ __volatile__ ("lwsync"	::: "memory", "cc")
21 #define rseq_smp_rmb()		rseq_smp_lwsync()
22 #define rseq_smp_wmb()		rseq_smp_lwsync()
23 
24 #define rseq_smp_load_acquire(p)					\
25 __extension__ ({							\
26 	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
27 	rseq_smp_lwsync();						\
28 	____p1;								\
29 })
30 
31 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_lwsync()
32 
33 #define rseq_smp_store_release(p, v)					\
34 do {									\
35 	rseq_smp_lwsync();						\
36 	RSEQ_WRITE_ONCE(*p, v);						\
37 } while (0)
38 
39 #ifdef RSEQ_SKIP_FASTPATH
40 #include "rseq-skip.h"
41 #else /* !RSEQ_SKIP_FASTPATH */
42 
43 /*
44  * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
45  * better handle single-stepping through the restartable critical sections.
46  */
47 
48 #ifdef __PPC64__
49 
50 #define STORE_WORD	"std "
51 #define LOAD_WORD	"ld "
52 #define LOADX_WORD	"ldx "
53 #define CMP_WORD	"cmpd "
54 
55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,				\
56 			start_ip, post_commit_offset, abort_ip)			\
57 		".pushsection __rseq_cs, \"aw\"\n\t"				\
58 		".balign 32\n\t"						\
59 		__rseq_str(label) ":\n\t"					\
60 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"	\
61 		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
62 		".popsection\n\t"						\
63 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"			\
64 		".quad " __rseq_str(label) "b\n\t"				\
65 		".popsection\n\t"
66 
67 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
68 		RSEQ_INJECT_ASM(1)						\
69 		"lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t"		\
70 		"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t"	\
71 		"rldicr %%r17, %%r17, 32, 31\n\t"				\
72 		"oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t"	\
73 		"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"		\
74 		"std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"			\
75 		__rseq_str(label) ":\n\t"
76 
77 /*
78  * Exit points of a rseq critical section consist of all instructions outside
79  * of the critical section where a critical section can either branch to or
80  * reach through the normal course of its execution. The abort IP and the
81  * post-commit IP are already part of the __rseq_cs section and should not be
82  * explicitly defined as additional exit points. Knowing all exit points is
83  * useful to assist debuggers stepping over the critical section.
84  */
85 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
86 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
87 		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
88 		".popsection\n\t"
89 
90 #else /* #ifdef __PPC64__ */
91 
92 #define STORE_WORD	"stw "
93 #define LOAD_WORD	"lwz "
94 #define LOADX_WORD	"lwzx "
95 #define CMP_WORD	"cmpw "
96 
97 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,				\
98 			start_ip, post_commit_offset, abort_ip)			\
99 		".pushsection __rseq_cs, \"aw\"\n\t"				\
100 		".balign 32\n\t"						\
101 		__rseq_str(label) ":\n\t"					\
102 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"	\
103 		/* 32-bit only supported on BE */				\
104 		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
105 		".popsection\n\t"					\
106 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
107 		".long 0x0, " __rseq_str(label) "b\n\t"			\
108 		".popsection\n\t"
109 
110 /*
111  * Exit points of a rseq critical section consist of all instructions outside
112  * of the critical section where a critical section can either branch to or
113  * reach through the normal course of its execution. The abort IP and the
114  * post-commit IP are already part of the __rseq_cs section and should not be
115  * explicitly defined as additional exit points. Knowing all exit points is
116  * useful to assist debuggers stepping over the critical section.
117  */
118 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)				\
119 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"		\
120 		/* 32-bit only supported on BE */				\
121 		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t"	\
122 		".popsection\n\t"
123 
124 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
125 		RSEQ_INJECT_ASM(1)						\
126 		"lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"			\
127 		"addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"		\
128 		"stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t"			\
129 		__rseq_str(label) ":\n\t"
130 
131 #endif /* #ifdef __PPC64__ */
132 
133 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)	\
134 		__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
135 					(post_commit_ip - start_ip), abort_ip)
136 
137 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)			\
138 		RSEQ_INJECT_ASM(2)						\
139 		"lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t"		\
140 		"cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"		\
141 		"bne- cr7, " __rseq_str(label) "\n\t"
142 
143 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)				\
144 		".pushsection __rseq_failure, \"ax\"\n\t"			\
145 		".long " __rseq_str(RSEQ_SIG) "\n\t"				\
146 		__rseq_str(label) ":\n\t"					\
147 		"b %l[" __rseq_str(abort_label) "]\n\t"				\
148 		".popsection\n\t"
149 
150 /*
151  * RSEQ_ASM_OPs: asm operations for rseq
152  * 	RSEQ_ASM_OP_R_*: has hard-code registers in it
153  * 	RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
154  */
155 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)					\
156 		LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"			\
157 		CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"		\
158 		"bne- cr7, " __rseq_str(label) "\n\t"
159 
160 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label)				\
161 		LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"			\
162 		CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"		\
163 		"beq- cr7, " __rseq_str(label) "\n\t"
164 
165 #define RSEQ_ASM_OP_STORE(value, var)						\
166 		STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
167 
168 /* Load @var to r17 */
169 #define RSEQ_ASM_OP_R_LOAD(var)							\
170 		LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
171 
172 /* Store r17 to @var */
173 #define RSEQ_ASM_OP_R_STORE(var)						\
174 		STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
175 
176 /* Add @count to r17 */
177 #define RSEQ_ASM_OP_R_ADD(count)						\
178 		"add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
179 
180 /* Load (r17 + voffp) to r17 */
181 #define RSEQ_ASM_OP_R_LOADX(voffp)						\
182 		LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
183 
184 /* TODO: implement a faster memcpy. */
185 #define RSEQ_ASM_OP_R_MEMCPY() \
186 		"cmpdi %%r19, 0\n\t" \
187 		"beq 333f\n\t" \
188 		"addi %%r20, %%r20, -1\n\t" \
189 		"addi %%r21, %%r21, -1\n\t" \
190 		"222:\n\t" \
191 		"lbzu %%r18, 1(%%r20)\n\t" \
192 		"stbu %%r18, 1(%%r21)\n\t" \
193 		"addi %%r19, %%r19, -1\n\t" \
194 		"cmpdi %%r19, 0\n\t" \
195 		"bne 222b\n\t" \
196 		"333:\n\t" \
197 
198 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)			\
199 		STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"			\
200 		__rseq_str(post_commit_label) ":\n\t"
201 
202 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)			\
203 		STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
204 		__rseq_str(post_commit_label) ":\n\t"
205 
206 static inline __attribute__((always_inline))
207 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
208 {
209 	RSEQ_INJECT_C(9)
210 
211 	__asm__ __volatile__ goto (
212 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
213 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
214 #ifdef RSEQ_COMPARE_TWICE
215 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
216 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
217 #endif
218 		/* Start rseq by storing table entry pointer into rseq_cs. */
219 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
220 		/* cmp cpuid */
221 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
222 		RSEQ_INJECT_ASM(3)
223 		/* cmp @v equal to @expect */
224 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
225 		RSEQ_INJECT_ASM(4)
226 #ifdef RSEQ_COMPARE_TWICE
227 		/* cmp cpuid */
228 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
229 		/* cmp @v equal to @expect */
230 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
231 #endif
232 		/* final store */
233 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
234 		RSEQ_INJECT_ASM(5)
235 		RSEQ_ASM_DEFINE_ABORT(4, abort)
236 		: /* gcc asm goto does not allow outputs */
237 		: [cpu_id]		"r" (cpu),
238 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
239 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
240 		  [v]			"m" (*v),
241 		  [expect]		"r" (expect),
242 		  [newv]		"r" (newv)
243 		  RSEQ_INJECT_INPUT
244 		: "memory", "cc", "r17"
245 		  RSEQ_INJECT_CLOBBER
246 		: abort, cmpfail
247 #ifdef RSEQ_COMPARE_TWICE
248 		  , error1, error2
249 #endif
250 	);
251 	return 0;
252 abort:
253 	RSEQ_INJECT_FAILED
254 	return -1;
255 cmpfail:
256 	return 1;
257 #ifdef RSEQ_COMPARE_TWICE
258 error1:
259 	rseq_bug("cpu_id comparison failed");
260 error2:
261 	rseq_bug("expected value comparison failed");
262 #endif
263 }
264 
265 static inline __attribute__((always_inline))
266 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
267 			       off_t voffp, intptr_t *load, int cpu)
268 {
269 	RSEQ_INJECT_C(9)
270 
271 	__asm__ __volatile__ goto (
272 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
273 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
274 #ifdef RSEQ_COMPARE_TWICE
275 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
276 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
277 #endif
278 		/* Start rseq by storing table entry pointer into rseq_cs. */
279 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
280 		/* cmp cpuid */
281 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
282 		RSEQ_INJECT_ASM(3)
283 		/* cmp @v not equal to @expectnot */
284 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
285 		RSEQ_INJECT_ASM(4)
286 #ifdef RSEQ_COMPARE_TWICE
287 		/* cmp cpuid */
288 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
289 		/* cmp @v not equal to @expectnot */
290 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
291 #endif
292 		/* load the value of @v */
293 		RSEQ_ASM_OP_R_LOAD(v)
294 		/* store it in @load */
295 		RSEQ_ASM_OP_R_STORE(load)
296 		/* dereference voffp(v) */
297 		RSEQ_ASM_OP_R_LOADX(voffp)
298 		/* final store the value at voffp(v) */
299 		RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
300 		RSEQ_INJECT_ASM(5)
301 		RSEQ_ASM_DEFINE_ABORT(4, abort)
302 		: /* gcc asm goto does not allow outputs */
303 		: [cpu_id]		"r" (cpu),
304 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
305 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
306 		  /* final store input */
307 		  [v]			"m" (*v),
308 		  [expectnot]		"r" (expectnot),
309 		  [voffp]		"b" (voffp),
310 		  [load]		"m" (*load)
311 		  RSEQ_INJECT_INPUT
312 		: "memory", "cc", "r17"
313 		  RSEQ_INJECT_CLOBBER
314 		: abort, cmpfail
315 #ifdef RSEQ_COMPARE_TWICE
316 		  , error1, error2
317 #endif
318 	);
319 	return 0;
320 abort:
321 	RSEQ_INJECT_FAILED
322 	return -1;
323 cmpfail:
324 	return 1;
325 #ifdef RSEQ_COMPARE_TWICE
326 error1:
327 	rseq_bug("cpu_id comparison failed");
328 error2:
329 	rseq_bug("expected value comparison failed");
330 #endif
331 }
332 
333 static inline __attribute__((always_inline))
334 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
335 {
336 	RSEQ_INJECT_C(9)
337 
338 	__asm__ __volatile__ goto (
339 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
340 #ifdef RSEQ_COMPARE_TWICE
341 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
342 #endif
343 		/* Start rseq by storing table entry pointer into rseq_cs. */
344 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
345 		/* cmp cpuid */
346 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
347 		RSEQ_INJECT_ASM(3)
348 #ifdef RSEQ_COMPARE_TWICE
349 		/* cmp cpuid */
350 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
351 #endif
352 		/* load the value of @v */
353 		RSEQ_ASM_OP_R_LOAD(v)
354 		/* add @count to it */
355 		RSEQ_ASM_OP_R_ADD(count)
356 		/* final store */
357 		RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
358 		RSEQ_INJECT_ASM(4)
359 		RSEQ_ASM_DEFINE_ABORT(4, abort)
360 		: /* gcc asm goto does not allow outputs */
361 		: [cpu_id]		"r" (cpu),
362 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
363 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
364 		  /* final store input */
365 		  [v]			"m" (*v),
366 		  [count]		"r" (count)
367 		  RSEQ_INJECT_INPUT
368 		: "memory", "cc", "r17"
369 		  RSEQ_INJECT_CLOBBER
370 		: abort
371 #ifdef RSEQ_COMPARE_TWICE
372 		  , error1
373 #endif
374 	);
375 	return 0;
376 abort:
377 	RSEQ_INJECT_FAILED
378 	return -1;
379 #ifdef RSEQ_COMPARE_TWICE
380 error1:
381 	rseq_bug("cpu_id comparison failed");
382 #endif
383 }
384 
385 static inline __attribute__((always_inline))
386 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
387 				 intptr_t *v2, intptr_t newv2,
388 				 intptr_t newv, int cpu)
389 {
390 	RSEQ_INJECT_C(9)
391 
392 	__asm__ __volatile__ goto (
393 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
394 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
395 #ifdef RSEQ_COMPARE_TWICE
396 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
397 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
398 #endif
399 		/* Start rseq by storing table entry pointer into rseq_cs. */
400 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
401 		/* cmp cpuid */
402 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
403 		RSEQ_INJECT_ASM(3)
404 		/* cmp @v equal to @expect */
405 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
406 		RSEQ_INJECT_ASM(4)
407 #ifdef RSEQ_COMPARE_TWICE
408 		/* cmp cpuid */
409 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
410 		/* cmp @v equal to @expect */
411 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
412 #endif
413 		/* try store */
414 		RSEQ_ASM_OP_STORE(newv2, v2)
415 		RSEQ_INJECT_ASM(5)
416 		/* final store */
417 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
418 		RSEQ_INJECT_ASM(6)
419 		RSEQ_ASM_DEFINE_ABORT(4, abort)
420 		: /* gcc asm goto does not allow outputs */
421 		: [cpu_id]		"r" (cpu),
422 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
423 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
424 		  /* try store input */
425 		  [v2]			"m" (*v2),
426 		  [newv2]		"r" (newv2),
427 		  /* final store input */
428 		  [v]			"m" (*v),
429 		  [expect]		"r" (expect),
430 		  [newv]		"r" (newv)
431 		  RSEQ_INJECT_INPUT
432 		: "memory", "cc", "r17"
433 		  RSEQ_INJECT_CLOBBER
434 		: abort, cmpfail
435 #ifdef RSEQ_COMPARE_TWICE
436 		  , error1, error2
437 #endif
438 	);
439 	return 0;
440 abort:
441 	RSEQ_INJECT_FAILED
442 	return -1;
443 cmpfail:
444 	return 1;
445 #ifdef RSEQ_COMPARE_TWICE
446 error1:
447 	rseq_bug("cpu_id comparison failed");
448 error2:
449 	rseq_bug("expected value comparison failed");
450 #endif
451 }
452 
453 static inline __attribute__((always_inline))
454 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
455 					 intptr_t *v2, intptr_t newv2,
456 					 intptr_t newv, int cpu)
457 {
458 	RSEQ_INJECT_C(9)
459 
460 	__asm__ __volatile__ goto (
461 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
462 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
463 #ifdef RSEQ_COMPARE_TWICE
464 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
465 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
466 #endif
467 		/* Start rseq by storing table entry pointer into rseq_cs. */
468 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
469 		/* cmp cpuid */
470 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
471 		RSEQ_INJECT_ASM(3)
472 		/* cmp @v equal to @expect */
473 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
474 		RSEQ_INJECT_ASM(4)
475 #ifdef RSEQ_COMPARE_TWICE
476 		/* cmp cpuid */
477 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
478 		/* cmp @v equal to @expect */
479 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
480 #endif
481 		/* try store */
482 		RSEQ_ASM_OP_STORE(newv2, v2)
483 		RSEQ_INJECT_ASM(5)
484 		/* for 'release' */
485 		"lwsync\n\t"
486 		/* final store */
487 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
488 		RSEQ_INJECT_ASM(6)
489 		RSEQ_ASM_DEFINE_ABORT(4, abort)
490 		: /* gcc asm goto does not allow outputs */
491 		: [cpu_id]		"r" (cpu),
492 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
493 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
494 		  /* try store input */
495 		  [v2]			"m" (*v2),
496 		  [newv2]		"r" (newv2),
497 		  /* final store input */
498 		  [v]			"m" (*v),
499 		  [expect]		"r" (expect),
500 		  [newv]		"r" (newv)
501 		  RSEQ_INJECT_INPUT
502 		: "memory", "cc", "r17"
503 		  RSEQ_INJECT_CLOBBER
504 		: abort, cmpfail
505 #ifdef RSEQ_COMPARE_TWICE
506 		  , error1, error2
507 #endif
508 	);
509 	return 0;
510 abort:
511 	RSEQ_INJECT_FAILED
512 	return -1;
513 cmpfail:
514 	return 1;
515 #ifdef RSEQ_COMPARE_TWICE
516 error1:
517 	rseq_bug("cpu_id comparison failed");
518 error2:
519 	rseq_bug("expected value comparison failed");
520 #endif
521 }
522 
523 static inline __attribute__((always_inline))
524 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
525 			      intptr_t *v2, intptr_t expect2,
526 			      intptr_t newv, int cpu)
527 {
528 	RSEQ_INJECT_C(9)
529 
530 	__asm__ __volatile__ goto (
531 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
532 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
533 #ifdef RSEQ_COMPARE_TWICE
534 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
535 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
536 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
537 #endif
538 		/* Start rseq by storing table entry pointer into rseq_cs. */
539 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
540 		/* cmp cpuid */
541 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
542 		RSEQ_INJECT_ASM(3)
543 		/* cmp @v equal to @expect */
544 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
545 		RSEQ_INJECT_ASM(4)
546 		/* cmp @v2 equal to @expct2 */
547 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
548 		RSEQ_INJECT_ASM(5)
549 #ifdef RSEQ_COMPARE_TWICE
550 		/* cmp cpuid */
551 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
552 		/* cmp @v equal to @expect */
553 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
554 		/* cmp @v2 equal to @expct2 */
555 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
556 #endif
557 		/* final store */
558 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
559 		RSEQ_INJECT_ASM(6)
560 		RSEQ_ASM_DEFINE_ABORT(4, abort)
561 		: /* gcc asm goto does not allow outputs */
562 		: [cpu_id]		"r" (cpu),
563 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
564 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
565 		  /* cmp2 input */
566 		  [v2]			"m" (*v2),
567 		  [expect2]		"r" (expect2),
568 		  /* final store input */
569 		  [v]			"m" (*v),
570 		  [expect]		"r" (expect),
571 		  [newv]		"r" (newv)
572 		  RSEQ_INJECT_INPUT
573 		: "memory", "cc", "r17"
574 		  RSEQ_INJECT_CLOBBER
575 		: abort, cmpfail
576 #ifdef RSEQ_COMPARE_TWICE
577 		  , error1, error2, error3
578 #endif
579 	);
580 	return 0;
581 abort:
582 	RSEQ_INJECT_FAILED
583 	return -1;
584 cmpfail:
585 	return 1;
586 #ifdef RSEQ_COMPARE_TWICE
587 error1:
588 	rseq_bug("cpu_id comparison failed");
589 error2:
590 	rseq_bug("1st expected value comparison failed");
591 error3:
592 	rseq_bug("2nd expected value comparison failed");
593 #endif
594 }
595 
596 static inline __attribute__((always_inline))
597 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
598 				 void *dst, void *src, size_t len,
599 				 intptr_t newv, int cpu)
600 {
601 	RSEQ_INJECT_C(9)
602 
603 	__asm__ __volatile__ goto (
604 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
605 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
606 #ifdef RSEQ_COMPARE_TWICE
607 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
608 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
609 #endif
610 		/* setup for mempcy */
611 		"mr %%r19, %[len]\n\t"
612 		"mr %%r20, %[src]\n\t"
613 		"mr %%r21, %[dst]\n\t"
614 		/* Start rseq by storing table entry pointer into rseq_cs. */
615 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
616 		/* cmp cpuid */
617 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
618 		RSEQ_INJECT_ASM(3)
619 		/* cmp @v equal to @expect */
620 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
621 		RSEQ_INJECT_ASM(4)
622 #ifdef RSEQ_COMPARE_TWICE
623 		/* cmp cpuid */
624 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
625 		/* cmp @v equal to @expect */
626 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
627 #endif
628 		/* try memcpy */
629 		RSEQ_ASM_OP_R_MEMCPY()
630 		RSEQ_INJECT_ASM(5)
631 		/* final store */
632 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
633 		RSEQ_INJECT_ASM(6)
634 		/* teardown */
635 		RSEQ_ASM_DEFINE_ABORT(4, abort)
636 		: /* gcc asm goto does not allow outputs */
637 		: [cpu_id]		"r" (cpu),
638 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
639 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
640 		  /* final store input */
641 		  [v]			"m" (*v),
642 		  [expect]		"r" (expect),
643 		  [newv]		"r" (newv),
644 		  /* try memcpy input */
645 		  [dst]			"r" (dst),
646 		  [src]			"r" (src),
647 		  [len]			"r" (len)
648 		  RSEQ_INJECT_INPUT
649 		: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
650 		  RSEQ_INJECT_CLOBBER
651 		: abort, cmpfail
652 #ifdef RSEQ_COMPARE_TWICE
653 		  , error1, error2
654 #endif
655 	);
656 	return 0;
657 abort:
658 	RSEQ_INJECT_FAILED
659 	return -1;
660 cmpfail:
661 	return 1;
662 #ifdef RSEQ_COMPARE_TWICE
663 error1:
664 	rseq_bug("cpu_id comparison failed");
665 error2:
666 	rseq_bug("expected value comparison failed");
667 #endif
668 }
669 
670 static inline __attribute__((always_inline))
671 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
672 					 void *dst, void *src, size_t len,
673 					 intptr_t newv, int cpu)
674 {
675 	RSEQ_INJECT_C(9)
676 
677 	__asm__ __volatile__ goto (
678 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
679 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
680 #ifdef RSEQ_COMPARE_TWICE
681 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
682 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
683 #endif
684 		/* setup for mempcy */
685 		"mr %%r19, %[len]\n\t"
686 		"mr %%r20, %[src]\n\t"
687 		"mr %%r21, %[dst]\n\t"
688 		/* Start rseq by storing table entry pointer into rseq_cs. */
689 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
690 		/* cmp cpuid */
691 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
692 		RSEQ_INJECT_ASM(3)
693 		/* cmp @v equal to @expect */
694 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
695 		RSEQ_INJECT_ASM(4)
696 #ifdef RSEQ_COMPARE_TWICE
697 		/* cmp cpuid */
698 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
699 		/* cmp @v equal to @expect */
700 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
701 #endif
702 		/* try memcpy */
703 		RSEQ_ASM_OP_R_MEMCPY()
704 		RSEQ_INJECT_ASM(5)
705 		/* for 'release' */
706 		"lwsync\n\t"
707 		/* final store */
708 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
709 		RSEQ_INJECT_ASM(6)
710 		/* teardown */
711 		RSEQ_ASM_DEFINE_ABORT(4, abort)
712 		: /* gcc asm goto does not allow outputs */
713 		: [cpu_id]		"r" (cpu),
714 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
715 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
716 		  /* final store input */
717 		  [v]			"m" (*v),
718 		  [expect]		"r" (expect),
719 		  [newv]		"r" (newv),
720 		  /* try memcpy input */
721 		  [dst]			"r" (dst),
722 		  [src]			"r" (src),
723 		  [len]			"r" (len)
724 		  RSEQ_INJECT_INPUT
725 		: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
726 		  RSEQ_INJECT_CLOBBER
727 		: abort, cmpfail
728 #ifdef RSEQ_COMPARE_TWICE
729 		  , error1, error2
730 #endif
731 	);
732 	return 0;
733 abort:
734 	RSEQ_INJECT_FAILED
735 	return -1;
736 cmpfail:
737 	return 1;
738 #ifdef RSEQ_COMPARE_TWICE
739 error1:
740 	rseq_bug("cpu_id comparison failed");
741 error2:
742 	rseq_bug("expected value comparison failed");
743 #endif
744 }
745 
746 #undef STORE_WORD
747 #undef LOAD_WORD
748 #undef LOADX_WORD
749 #undef CMP_WORD
750 
751 #endif /* !RSEQ_SKIP_FASTPATH */
752