1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 
3 /*
4  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
5  * access-register mode nor the linkage stack this instruction will always
6  * cause a special-operation exception (the trap-enabled bit in the DUCT
7  * is and will stay 0). The instruction pattern is
8  *	b2 ff 0f ff	trap4	4095(%r0)
9  */
10 #define RSEQ_SIG	0xB2FF0FFF
11 
12 #define rseq_smp_mb()	__asm__ __volatile__ ("bcr 15,0" ::: "memory")
13 #define rseq_smp_rmb()	rseq_smp_mb()
14 #define rseq_smp_wmb()	rseq_smp_mb()
15 
16 #define rseq_smp_load_acquire(p)					\
17 __extension__ ({							\
18 	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
19 	rseq_barrier();							\
20 	____p1;								\
21 })
22 
23 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
24 
25 #define rseq_smp_store_release(p, v)					\
26 do {									\
27 	rseq_barrier();							\
28 	RSEQ_WRITE_ONCE(*p, v);						\
29 } while (0)
30 
31 #ifdef RSEQ_SKIP_FASTPATH
32 #include "rseq-skip.h"
33 #else /* !RSEQ_SKIP_FASTPATH */
34 
35 #ifdef __s390x__
36 
37 #define LONG_L			"lg"
38 #define LONG_S			"stg"
39 #define LONG_LT_R		"ltgr"
40 #define LONG_CMP		"cg"
41 #define LONG_CMP_R		"cgr"
42 #define LONG_ADDI		"aghi"
43 #define LONG_ADD_R		"agr"
44 
45 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
46 				start_ip, post_commit_offset, abort_ip)	\
47 		".pushsection __rseq_cs, \"aw\"\n\t"			\
48 		".balign 32\n\t"					\
49 		__rseq_str(label) ":\n\t"				\
50 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
51 		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
52 		".popsection\n\t"					\
53 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
54 		".quad " __rseq_str(label) "b\n\t"			\
55 		".popsection\n\t"
56 
57 /*
58  * Exit points of a rseq critical section consist of all instructions outside
59  * of the critical section where a critical section can either branch to or
60  * reach through the normal course of its execution. The abort IP and the
61  * post-commit IP are already part of the __rseq_cs section and should not be
62  * explicitly defined as additional exit points. Knowing all exit points is
63  * useful to assist debuggers stepping over the critical section.
64  */
65 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
66 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
67 		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
68 		".popsection\n\t"
69 
70 #elif __s390__
71 
72 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
73 				start_ip, post_commit_offset, abort_ip)	\
74 		".pushsection __rseq_cs, \"aw\"\n\t"			\
75 		".balign 32\n\t"					\
76 		__rseq_str(label) ":\n\t"				\
77 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
78 		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
79 		".popsection\n\t"					\
80 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
81 		".long 0x0, " __rseq_str(label) "b\n\t"			\
82 		".popsection\n\t"
83 
84 /*
85  * Exit points of a rseq critical section consist of all instructions outside
86  * of the critical section where a critical section can either branch to or
87  * reach through the normal course of its execution. The abort IP and the
88  * post-commit IP are already part of the __rseq_cs section and should not be
89  * explicitly defined as additional exit points. Knowing all exit points is
90  * useful to assist debuggers stepping over the critical section.
91  */
92 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
93 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
94 		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
95 		".popsection\n\t"
96 
97 #define LONG_L			"l"
98 #define LONG_S			"st"
99 #define LONG_LT_R		"ltr"
100 #define LONG_CMP		"c"
101 #define LONG_CMP_R		"cr"
102 #define LONG_ADDI		"ahi"
103 #define LONG_ADD_R		"ar"
104 
105 #endif
106 
107 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
108 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
109 				(post_commit_ip - start_ip), abort_ip)
110 
111 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
112 		RSEQ_INJECT_ASM(1)					\
113 		"larl %%r0, " __rseq_str(cs_label) "\n\t"		\
114 		LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t"		\
115 		__rseq_str(label) ":\n\t"
116 
117 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
118 		RSEQ_INJECT_ASM(2)					\
119 		"c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
120 		"jnz " __rseq_str(label) "\n\t"
121 
122 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
123 		".pushsection __rseq_failure, \"ax\"\n\t"		\
124 		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
125 		__rseq_str(label) ":\n\t"				\
126 		teardown						\
127 		"jg %l[" __rseq_str(abort_label) "]\n\t"		\
128 		".popsection\n\t"
129 
130 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
131 		".pushsection __rseq_failure, \"ax\"\n\t"		\
132 		__rseq_str(label) ":\n\t"				\
133 		teardown						\
134 		"jg %l[" __rseq_str(cmpfail_label) "]\n\t"		\
135 		".popsection\n\t"
136 
137 static inline __attribute__((always_inline))
138 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
139 {
140 	RSEQ_INJECT_C(9)
141 
142 	__asm__ __volatile__ goto (
143 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
144 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
145 #ifdef RSEQ_COMPARE_TWICE
146 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
147 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
148 #endif
149 		/* Start rseq by storing table entry pointer into rseq_cs. */
150 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
151 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
152 		RSEQ_INJECT_ASM(3)
153 		LONG_CMP " %[expect], %[v]\n\t"
154 		"jnz %l[cmpfail]\n\t"
155 		RSEQ_INJECT_ASM(4)
156 #ifdef RSEQ_COMPARE_TWICE
157 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
158 		LONG_CMP " %[expect], %[v]\n\t"
159 		"jnz %l[error2]\n\t"
160 #endif
161 		/* final store */
162 		LONG_S " %[newv], %[v]\n\t"
163 		"2:\n\t"
164 		RSEQ_INJECT_ASM(5)
165 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
166 		: /* gcc asm goto does not allow outputs */
167 		: [cpu_id]		"r" (cpu),
168 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
169 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
170 		  [v]			"m" (*v),
171 		  [expect]		"r" (expect),
172 		  [newv]		"r" (newv)
173 		  RSEQ_INJECT_INPUT
174 		: "memory", "cc", "r0"
175 		  RSEQ_INJECT_CLOBBER
176 		: abort, cmpfail
177 #ifdef RSEQ_COMPARE_TWICE
178 		  , error1, error2
179 #endif
180 	);
181 	return 0;
182 abort:
183 	RSEQ_INJECT_FAILED
184 	return -1;
185 cmpfail:
186 	return 1;
187 #ifdef RSEQ_COMPARE_TWICE
188 error1:
189 	rseq_bug("cpu_id comparison failed");
190 error2:
191 	rseq_bug("expected value comparison failed");
192 #endif
193 }
194 
195 /*
196  * Compare @v against @expectnot. When it does _not_ match, load @v
197  * into @load, and store the content of *@v + voffp into @v.
198  */
199 static inline __attribute__((always_inline))
200 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
201 			       off_t voffp, intptr_t *load, int cpu)
202 {
203 	RSEQ_INJECT_C(9)
204 
205 	__asm__ __volatile__ goto (
206 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
207 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
208 #ifdef RSEQ_COMPARE_TWICE
209 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
210 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
211 #endif
212 		/* Start rseq by storing table entry pointer into rseq_cs. */
213 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
214 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
215 		RSEQ_INJECT_ASM(3)
216 		LONG_L " %%r1, %[v]\n\t"
217 		LONG_CMP_R " %%r1, %[expectnot]\n\t"
218 		"je %l[cmpfail]\n\t"
219 		RSEQ_INJECT_ASM(4)
220 #ifdef RSEQ_COMPARE_TWICE
221 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
222 		LONG_L " %%r1, %[v]\n\t"
223 		LONG_CMP_R " %%r1, %[expectnot]\n\t"
224 		"je %l[error2]\n\t"
225 #endif
226 		LONG_S " %%r1, %[load]\n\t"
227 		LONG_ADD_R " %%r1, %[voffp]\n\t"
228 		LONG_L " %%r1, 0(%%r1)\n\t"
229 		/* final store */
230 		LONG_S " %%r1, %[v]\n\t"
231 		"2:\n\t"
232 		RSEQ_INJECT_ASM(5)
233 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
234 		: /* gcc asm goto does not allow outputs */
235 		: [cpu_id]		"r" (cpu),
236 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
237 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
238 		  /* final store input */
239 		  [v]			"m" (*v),
240 		  [expectnot]		"r" (expectnot),
241 		  [voffp]		"r" (voffp),
242 		  [load]		"m" (*load)
243 		  RSEQ_INJECT_INPUT
244 		: "memory", "cc", "r0", "r1"
245 		  RSEQ_INJECT_CLOBBER
246 		: abort, cmpfail
247 #ifdef RSEQ_COMPARE_TWICE
248 		  , error1, error2
249 #endif
250 	);
251 	return 0;
252 abort:
253 	RSEQ_INJECT_FAILED
254 	return -1;
255 cmpfail:
256 	return 1;
257 #ifdef RSEQ_COMPARE_TWICE
258 error1:
259 	rseq_bug("cpu_id comparison failed");
260 error2:
261 	rseq_bug("expected value comparison failed");
262 #endif
263 }
264 
265 static inline __attribute__((always_inline))
266 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
267 {
268 	RSEQ_INJECT_C(9)
269 
270 	__asm__ __volatile__ goto (
271 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
272 #ifdef RSEQ_COMPARE_TWICE
273 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
274 #endif
275 		/* Start rseq by storing table entry pointer into rseq_cs. */
276 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
277 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
278 		RSEQ_INJECT_ASM(3)
279 #ifdef RSEQ_COMPARE_TWICE
280 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
281 #endif
282 		LONG_L " %%r0, %[v]\n\t"
283 		LONG_ADD_R " %%r0, %[count]\n\t"
284 		/* final store */
285 		LONG_S " %%r0, %[v]\n\t"
286 		"2:\n\t"
287 		RSEQ_INJECT_ASM(4)
288 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
289 		: /* gcc asm goto does not allow outputs */
290 		: [cpu_id]		"r" (cpu),
291 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
292 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
293 		  /* final store input */
294 		  [v]			"m" (*v),
295 		  [count]		"r" (count)
296 		  RSEQ_INJECT_INPUT
297 		: "memory", "cc", "r0"
298 		  RSEQ_INJECT_CLOBBER
299 		: abort
300 #ifdef RSEQ_COMPARE_TWICE
301 		  , error1
302 #endif
303 	);
304 	return 0;
305 abort:
306 	RSEQ_INJECT_FAILED
307 	return -1;
308 #ifdef RSEQ_COMPARE_TWICE
309 error1:
310 	rseq_bug("cpu_id comparison failed");
311 #endif
312 }
313 
314 static inline __attribute__((always_inline))
315 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
316 				 intptr_t *v2, intptr_t newv2,
317 				 intptr_t newv, int cpu)
318 {
319 	RSEQ_INJECT_C(9)
320 
321 	__asm__ __volatile__ goto (
322 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
323 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
324 #ifdef RSEQ_COMPARE_TWICE
325 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
326 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
327 #endif
328 		/* Start rseq by storing table entry pointer into rseq_cs. */
329 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
330 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
331 		RSEQ_INJECT_ASM(3)
332 		LONG_CMP " %[expect], %[v]\n\t"
333 		"jnz %l[cmpfail]\n\t"
334 		RSEQ_INJECT_ASM(4)
335 #ifdef RSEQ_COMPARE_TWICE
336 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
337 		LONG_CMP " %[expect], %[v]\n\t"
338 		"jnz %l[error2]\n\t"
339 #endif
340 		/* try store */
341 		LONG_S " %[newv2], %[v2]\n\t"
342 		RSEQ_INJECT_ASM(5)
343 		/* final store */
344 		LONG_S " %[newv], %[v]\n\t"
345 		"2:\n\t"
346 		RSEQ_INJECT_ASM(6)
347 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
348 		: /* gcc asm goto does not allow outputs */
349 		: [cpu_id]		"r" (cpu),
350 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
351 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
352 		  /* try store input */
353 		  [v2]			"m" (*v2),
354 		  [newv2]		"r" (newv2),
355 		  /* final store input */
356 		  [v]			"m" (*v),
357 		  [expect]		"r" (expect),
358 		  [newv]		"r" (newv)
359 		  RSEQ_INJECT_INPUT
360 		: "memory", "cc", "r0"
361 		  RSEQ_INJECT_CLOBBER
362 		: abort, cmpfail
363 #ifdef RSEQ_COMPARE_TWICE
364 		  , error1, error2
365 #endif
366 	);
367 	return 0;
368 abort:
369 	RSEQ_INJECT_FAILED
370 	return -1;
371 cmpfail:
372 	return 1;
373 #ifdef RSEQ_COMPARE_TWICE
374 error1:
375 	rseq_bug("cpu_id comparison failed");
376 error2:
377 	rseq_bug("expected value comparison failed");
378 #endif
379 }
380 
381 /* s390 is TSO. */
382 static inline __attribute__((always_inline))
383 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
384 					 intptr_t *v2, intptr_t newv2,
385 					 intptr_t newv, int cpu)
386 {
387 	return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
388 }
389 
390 static inline __attribute__((always_inline))
391 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
392 			      intptr_t *v2, intptr_t expect2,
393 			      intptr_t newv, int cpu)
394 {
395 	RSEQ_INJECT_C(9)
396 
397 	__asm__ __volatile__ goto (
398 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
399 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
400 #ifdef RSEQ_COMPARE_TWICE
401 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
402 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
403 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
404 #endif
405 		/* Start rseq by storing table entry pointer into rseq_cs. */
406 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
407 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
408 		RSEQ_INJECT_ASM(3)
409 		LONG_CMP " %[expect], %[v]\n\t"
410 		"jnz %l[cmpfail]\n\t"
411 		RSEQ_INJECT_ASM(4)
412 		LONG_CMP " %[expect2], %[v2]\n\t"
413 		"jnz %l[cmpfail]\n\t"
414 		RSEQ_INJECT_ASM(5)
415 #ifdef RSEQ_COMPARE_TWICE
416 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
417 		LONG_CMP " %[expect], %[v]\n\t"
418 		"jnz %l[error2]\n\t"
419 		LONG_CMP " %[expect2], %[v2]\n\t"
420 		"jnz %l[error3]\n\t"
421 #endif
422 		/* final store */
423 		LONG_S " %[newv], %[v]\n\t"
424 		"2:\n\t"
425 		RSEQ_INJECT_ASM(6)
426 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
427 		: /* gcc asm goto does not allow outputs */
428 		: [cpu_id]		"r" (cpu),
429 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
430 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
431 		  /* cmp2 input */
432 		  [v2]			"m" (*v2),
433 		  [expect2]		"r" (expect2),
434 		  /* final store input */
435 		  [v]			"m" (*v),
436 		  [expect]		"r" (expect),
437 		  [newv]		"r" (newv)
438 		  RSEQ_INJECT_INPUT
439 		: "memory", "cc", "r0"
440 		  RSEQ_INJECT_CLOBBER
441 		: abort, cmpfail
442 #ifdef RSEQ_COMPARE_TWICE
443 		  , error1, error2, error3
444 #endif
445 	);
446 	return 0;
447 abort:
448 	RSEQ_INJECT_FAILED
449 	return -1;
450 cmpfail:
451 	return 1;
452 #ifdef RSEQ_COMPARE_TWICE
453 error1:
454 	rseq_bug("cpu_id comparison failed");
455 error2:
456 	rseq_bug("1st expected value comparison failed");
457 error3:
458 	rseq_bug("2nd expected value comparison failed");
459 #endif
460 }
461 
462 static inline __attribute__((always_inline))
463 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
464 				 void *dst, void *src, size_t len,
465 				 intptr_t newv, int cpu)
466 {
467 	uint64_t rseq_scratch[3];
468 
469 	RSEQ_INJECT_C(9)
470 
471 	__asm__ __volatile__ goto (
472 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
473 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
474 #ifdef RSEQ_COMPARE_TWICE
475 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
476 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
477 #endif
478 		LONG_S " %[src], %[rseq_scratch0]\n\t"
479 		LONG_S " %[dst], %[rseq_scratch1]\n\t"
480 		LONG_S " %[len], %[rseq_scratch2]\n\t"
481 		/* Start rseq by storing table entry pointer into rseq_cs. */
482 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
483 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
484 		RSEQ_INJECT_ASM(3)
485 		LONG_CMP " %[expect], %[v]\n\t"
486 		"jnz 5f\n\t"
487 		RSEQ_INJECT_ASM(4)
488 #ifdef RSEQ_COMPARE_TWICE
489 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
490 		LONG_CMP " %[expect], %[v]\n\t"
491 		"jnz 7f\n\t"
492 #endif
493 		/* try memcpy */
494 		LONG_LT_R " %[len], %[len]\n\t"
495 		"jz 333f\n\t"
496 		"222:\n\t"
497 		"ic %%r0,0(%[src])\n\t"
498 		"stc %%r0,0(%[dst])\n\t"
499 		LONG_ADDI " %[src], 1\n\t"
500 		LONG_ADDI " %[dst], 1\n\t"
501 		LONG_ADDI " %[len], -1\n\t"
502 		"jnz 222b\n\t"
503 		"333:\n\t"
504 		RSEQ_INJECT_ASM(5)
505 		/* final store */
506 		LONG_S " %[newv], %[v]\n\t"
507 		"2:\n\t"
508 		RSEQ_INJECT_ASM(6)
509 		/* teardown */
510 		LONG_L " %[len], %[rseq_scratch2]\n\t"
511 		LONG_L " %[dst], %[rseq_scratch1]\n\t"
512 		LONG_L " %[src], %[rseq_scratch0]\n\t"
513 		RSEQ_ASM_DEFINE_ABORT(4,
514 			LONG_L " %[len], %[rseq_scratch2]\n\t"
515 			LONG_L " %[dst], %[rseq_scratch1]\n\t"
516 			LONG_L " %[src], %[rseq_scratch0]\n\t",
517 			abort)
518 		RSEQ_ASM_DEFINE_CMPFAIL(5,
519 			LONG_L " %[len], %[rseq_scratch2]\n\t"
520 			LONG_L " %[dst], %[rseq_scratch1]\n\t"
521 			LONG_L " %[src], %[rseq_scratch0]\n\t",
522 			cmpfail)
523 #ifdef RSEQ_COMPARE_TWICE
524 		RSEQ_ASM_DEFINE_CMPFAIL(6,
525 			LONG_L " %[len], %[rseq_scratch2]\n\t"
526 			LONG_L " %[dst], %[rseq_scratch1]\n\t"
527 			LONG_L " %[src], %[rseq_scratch0]\n\t",
528 			error1)
529 		RSEQ_ASM_DEFINE_CMPFAIL(7,
530 			LONG_L " %[len], %[rseq_scratch2]\n\t"
531 			LONG_L " %[dst], %[rseq_scratch1]\n\t"
532 			LONG_L " %[src], %[rseq_scratch0]\n\t",
533 			error2)
534 #endif
535 		: /* gcc asm goto does not allow outputs */
536 		: [cpu_id]		"r" (cpu),
537 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
538 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
539 		  /* final store input */
540 		  [v]			"m" (*v),
541 		  [expect]		"r" (expect),
542 		  [newv]		"r" (newv),
543 		  /* try memcpy input */
544 		  [dst]			"r" (dst),
545 		  [src]			"r" (src),
546 		  [len]			"r" (len),
547 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
548 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
549 		  [rseq_scratch2]	"m" (rseq_scratch[2])
550 		  RSEQ_INJECT_INPUT
551 		: "memory", "cc", "r0"
552 		  RSEQ_INJECT_CLOBBER
553 		: abort, cmpfail
554 #ifdef RSEQ_COMPARE_TWICE
555 		  , error1, error2
556 #endif
557 	);
558 	return 0;
559 abort:
560 	RSEQ_INJECT_FAILED
561 	return -1;
562 cmpfail:
563 	return 1;
564 #ifdef RSEQ_COMPARE_TWICE
565 error1:
566 	rseq_bug("cpu_id comparison failed");
567 error2:
568 	rseq_bug("expected value comparison failed");
569 #endif
570 }
571 
572 /* s390 is TSO. */
573 static inline __attribute__((always_inline))
574 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
575 					 void *dst, void *src, size_t len,
576 					 intptr_t newv, int cpu)
577 {
578 	return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
579 					    newv, cpu);
580 }
581 #endif /* !RSEQ_SKIP_FASTPATH */
582