1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * Author: Paul Burton <paul.burton@mips.com>
4  * (C) Copyright 2018 MIPS Tech LLC
5  *
6  * Based on rseq-arm.h:
7  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
8  */
9 
10 /*
11  * RSEQ_SIG uses the break instruction. The instruction pattern is:
12  *
13  * On MIPS:
14  *	0350000d        break     0x350
15  *
16  * On nanoMIPS:
17  *      00100350        break     0x350
18  *
19  * On microMIPS:
20  *      0000d407        break     0x350
21  *
22  * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit
23  * halfwords, so the signature halfwords need to be swapped accordingly for
24  * little-endian.
25  */
26 #if defined(__nanomips__)
27 # ifdef __MIPSEL__
28 #  define RSEQ_SIG	0x03500010
29 # else
30 #  define RSEQ_SIG	0x00100350
31 # endif
32 #elif defined(__mips_micromips)
33 # ifdef __MIPSEL__
34 #  define RSEQ_SIG	0xd4070000
35 # else
36 #  define RSEQ_SIG	0x0000d407
37 # endif
38 #elif defined(__mips__)
39 # define RSEQ_SIG	0x0350000d
40 #else
41 /* Unknown MIPS architecture. */
42 #endif
43 
44 #define rseq_smp_mb()	__asm__ __volatile__ ("sync" ::: "memory")
45 #define rseq_smp_rmb()	rseq_smp_mb()
46 #define rseq_smp_wmb()	rseq_smp_mb()
47 
48 #define rseq_smp_load_acquire(p)					\
49 __extension__ ({							\
50 	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
51 	rseq_smp_mb();							\
52 	____p1;								\
53 })
54 
55 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
56 
57 #define rseq_smp_store_release(p, v)					\
58 do {									\
59 	rseq_smp_mb();							\
60 	RSEQ_WRITE_ONCE(*p, v);						\
61 } while (0)
62 
63 #ifdef RSEQ_SKIP_FASTPATH
64 #include "rseq-skip.h"
65 #else /* !RSEQ_SKIP_FASTPATH */
66 
67 #if _MIPS_SZLONG == 64
68 # define LONG			".dword"
69 # define LONG_LA		"dla"
70 # define LONG_L			"ld"
71 # define LONG_S			"sd"
72 # define LONG_ADDI		"daddiu"
73 # define U32_U64_PAD(x)		x
74 #elif _MIPS_SZLONG == 32
75 # define LONG			".word"
76 # define LONG_LA		"la"
77 # define LONG_L			"lw"
78 # define LONG_S			"sw"
79 # define LONG_ADDI		"addiu"
80 # ifdef __BIG_ENDIAN
81 #  define U32_U64_PAD(x)	"0x0, " x
82 # else
83 #  define U32_U64_PAD(x)	x ", 0x0"
84 # endif
85 #else
86 # error unsupported _MIPS_SZLONG
87 #endif
88 
89 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
90 				post_commit_offset, abort_ip) \
91 		".pushsection __rseq_cs, \"aw\"\n\t" \
92 		".balign 32\n\t" \
93 		__rseq_str(label) ":\n\t"					\
94 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
95 		LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
96 		LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
97 		LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
98 		".popsection\n\t" \
99 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
100 		LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \
101 		".popsection\n\t"
102 
103 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
104 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
105 				(post_commit_ip - start_ip), abort_ip)
106 
107 /*
108  * Exit points of a rseq critical section consist of all instructions outside
109  * of the critical section where a critical section can either branch to or
110  * reach through the normal course of its execution. The abort IP and the
111  * post-commit IP are already part of the __rseq_cs section and should not be
112  * explicitly defined as additional exit points. Knowing all exit points is
113  * useful to assist debuggers stepping over the critical section.
114  */
115 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
116 		".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
117 		LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
118 		LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \
119 		".popsection\n\t"
120 
121 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
122 		RSEQ_INJECT_ASM(1) \
123 		LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
124 		LONG_S  " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
125 		__rseq_str(label) ":\n\t"
126 
127 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
128 		RSEQ_INJECT_ASM(2) \
129 		"lw  $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
130 		"bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
131 
132 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
133 				abort_label, version, flags, \
134 				start_ip, post_commit_offset, abort_ip) \
135 		".balign 32\n\t" \
136 		__rseq_str(table_label) ":\n\t" \
137 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
138 		LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
139 		LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
140 		LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
141 		".word " __rseq_str(RSEQ_SIG) "\n\t" \
142 		__rseq_str(label) ":\n\t" \
143 		teardown \
144 		"b %l[" __rseq_str(abort_label) "]\n\t"
145 
146 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
147 			      start_ip, post_commit_ip, abort_ip) \
148 	__RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
149 				abort_label, 0x0, 0x0, start_ip, \
150 				(post_commit_ip - start_ip), abort_ip)
151 
152 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
153 		__rseq_str(label) ":\n\t" \
154 		teardown \
155 		"b %l[" __rseq_str(cmpfail_label) "]\n\t"
156 
157 static inline __attribute__((always_inline))
158 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
159 {
160 	RSEQ_INJECT_C(9)
161 
162 	__asm__ __volatile__ goto (
163 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
164 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
165 #ifdef RSEQ_COMPARE_TWICE
166 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
167 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
168 #endif
169 		/* Start rseq by storing table entry pointer into rseq_cs. */
170 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
171 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
172 		RSEQ_INJECT_ASM(3)
173 		LONG_L " $4, %[v]\n\t"
174 		"bne $4, %[expect], %l[cmpfail]\n\t"
175 		RSEQ_INJECT_ASM(4)
176 #ifdef RSEQ_COMPARE_TWICE
177 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
178 		LONG_L " $4, %[v]\n\t"
179 		"bne $4, %[expect], %l[error2]\n\t"
180 #endif
181 		/* final store */
182 		LONG_S " %[newv], %[v]\n\t"
183 		"2:\n\t"
184 		RSEQ_INJECT_ASM(5)
185 		"b 5f\n\t"
186 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
187 		"5:\n\t"
188 		: /* gcc asm goto does not allow outputs */
189 		: [cpu_id]		"r" (cpu),
190 		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
191 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
192 		  [v]			"m" (*v),
193 		  [expect]		"r" (expect),
194 		  [newv]		"r" (newv)
195 		  RSEQ_INJECT_INPUT
196 		: "$4", "memory"
197 		  RSEQ_INJECT_CLOBBER
198 		: abort, cmpfail
199 #ifdef RSEQ_COMPARE_TWICE
200 		  , error1, error2
201 #endif
202 	);
203 	return 0;
204 abort:
205 	RSEQ_INJECT_FAILED
206 	return -1;
207 cmpfail:
208 	return 1;
209 #ifdef RSEQ_COMPARE_TWICE
210 error1:
211 	rseq_bug("cpu_id comparison failed");
212 error2:
213 	rseq_bug("expected value comparison failed");
214 #endif
215 }
216 
217 static inline __attribute__((always_inline))
218 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
219 			       long voffp, intptr_t *load, int cpu)
220 {
221 	RSEQ_INJECT_C(9)
222 
223 	__asm__ __volatile__ goto (
224 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
225 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
226 #ifdef RSEQ_COMPARE_TWICE
227 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
228 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
229 #endif
230 		/* Start rseq by storing table entry pointer into rseq_cs. */
231 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
232 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
233 		RSEQ_INJECT_ASM(3)
234 		LONG_L " $4, %[v]\n\t"
235 		"beq $4, %[expectnot], %l[cmpfail]\n\t"
236 		RSEQ_INJECT_ASM(4)
237 #ifdef RSEQ_COMPARE_TWICE
238 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
239 		LONG_L " $4, %[v]\n\t"
240 		"beq $4, %[expectnot], %l[error2]\n\t"
241 #endif
242 		LONG_S " $4, %[load]\n\t"
243 		LONG_ADDI " $4, %[voffp]\n\t"
244 		LONG_L " $4, 0($4)\n\t"
245 		/* final store */
246 		LONG_S " $4, %[v]\n\t"
247 		"2:\n\t"
248 		RSEQ_INJECT_ASM(5)
249 		"b 5f\n\t"
250 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
251 		"5:\n\t"
252 		: /* gcc asm goto does not allow outputs */
253 		: [cpu_id]		"r" (cpu),
254 		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
255 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
256 		  /* final store input */
257 		  [v]			"m" (*v),
258 		  [expectnot]		"r" (expectnot),
259 		  [voffp]		"Ir" (voffp),
260 		  [load]		"m" (*load)
261 		  RSEQ_INJECT_INPUT
262 		: "$4", "memory"
263 		  RSEQ_INJECT_CLOBBER
264 		: abort, cmpfail
265 #ifdef RSEQ_COMPARE_TWICE
266 		  , error1, error2
267 #endif
268 	);
269 	return 0;
270 abort:
271 	RSEQ_INJECT_FAILED
272 	return -1;
273 cmpfail:
274 	return 1;
275 #ifdef RSEQ_COMPARE_TWICE
276 error1:
277 	rseq_bug("cpu_id comparison failed");
278 error2:
279 	rseq_bug("expected value comparison failed");
280 #endif
281 }
282 
283 static inline __attribute__((always_inline))
284 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
285 {
286 	RSEQ_INJECT_C(9)
287 
288 	__asm__ __volatile__ goto (
289 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
290 #ifdef RSEQ_COMPARE_TWICE
291 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
292 #endif
293 		/* Start rseq by storing table entry pointer into rseq_cs. */
294 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
295 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
296 		RSEQ_INJECT_ASM(3)
297 #ifdef RSEQ_COMPARE_TWICE
298 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
299 #endif
300 		LONG_L " $4, %[v]\n\t"
301 		LONG_ADDI " $4, %[count]\n\t"
302 		/* final store */
303 		LONG_S " $4, %[v]\n\t"
304 		"2:\n\t"
305 		RSEQ_INJECT_ASM(4)
306 		"b 5f\n\t"
307 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
308 		"5:\n\t"
309 		: /* gcc asm goto does not allow outputs */
310 		: [cpu_id]		"r" (cpu),
311 		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
312 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
313 		  [v]			"m" (*v),
314 		  [count]		"Ir" (count)
315 		  RSEQ_INJECT_INPUT
316 		: "$4", "memory"
317 		  RSEQ_INJECT_CLOBBER
318 		: abort
319 #ifdef RSEQ_COMPARE_TWICE
320 		  , error1
321 #endif
322 	);
323 	return 0;
324 abort:
325 	RSEQ_INJECT_FAILED
326 	return -1;
327 #ifdef RSEQ_COMPARE_TWICE
328 error1:
329 	rseq_bug("cpu_id comparison failed");
330 #endif
331 }
332 
333 static inline __attribute__((always_inline))
334 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
335 				 intptr_t *v2, intptr_t newv2,
336 				 intptr_t newv, int cpu)
337 {
338 	RSEQ_INJECT_C(9)
339 
340 	__asm__ __volatile__ goto (
341 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
342 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
343 #ifdef RSEQ_COMPARE_TWICE
344 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
345 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
346 #endif
347 		/* Start rseq by storing table entry pointer into rseq_cs. */
348 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
349 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
350 		RSEQ_INJECT_ASM(3)
351 		LONG_L " $4, %[v]\n\t"
352 		"bne $4, %[expect], %l[cmpfail]\n\t"
353 		RSEQ_INJECT_ASM(4)
354 #ifdef RSEQ_COMPARE_TWICE
355 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
356 		LONG_L " $4, %[v]\n\t"
357 		"bne $4, %[expect], %l[error2]\n\t"
358 #endif
359 		/* try store */
360 		LONG_S " %[newv2], %[v2]\n\t"
361 		RSEQ_INJECT_ASM(5)
362 		/* final store */
363 		LONG_S " %[newv], %[v]\n\t"
364 		"2:\n\t"
365 		RSEQ_INJECT_ASM(6)
366 		"b 5f\n\t"
367 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
368 		"5:\n\t"
369 		: /* gcc asm goto does not allow outputs */
370 		: [cpu_id]		"r" (cpu),
371 		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
372 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
373 		  /* try store input */
374 		  [v2]			"m" (*v2),
375 		  [newv2]		"r" (newv2),
376 		  /* final store input */
377 		  [v]			"m" (*v),
378 		  [expect]		"r" (expect),
379 		  [newv]		"r" (newv)
380 		  RSEQ_INJECT_INPUT
381 		: "$4", "memory"
382 		  RSEQ_INJECT_CLOBBER
383 		: abort, cmpfail
384 #ifdef RSEQ_COMPARE_TWICE
385 		  , error1, error2
386 #endif
387 	);
388 	return 0;
389 abort:
390 	RSEQ_INJECT_FAILED
391 	return -1;
392 cmpfail:
393 	return 1;
394 #ifdef RSEQ_COMPARE_TWICE
395 error1:
396 	rseq_bug("cpu_id comparison failed");
397 error2:
398 	rseq_bug("expected value comparison failed");
399 #endif
400 }
401 
402 static inline __attribute__((always_inline))
403 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
404 					 intptr_t *v2, intptr_t newv2,
405 					 intptr_t newv, int cpu)
406 {
407 	RSEQ_INJECT_C(9)
408 
409 	__asm__ __volatile__ goto (
410 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
411 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
412 #ifdef RSEQ_COMPARE_TWICE
413 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
414 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
415 #endif
416 		/* Start rseq by storing table entry pointer into rseq_cs. */
417 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
418 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
419 		RSEQ_INJECT_ASM(3)
420 		LONG_L " $4, %[v]\n\t"
421 		"bne $4, %[expect], %l[cmpfail]\n\t"
422 		RSEQ_INJECT_ASM(4)
423 #ifdef RSEQ_COMPARE_TWICE
424 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
425 		LONG_L " $4, %[v]\n\t"
426 		"bne $4, %[expect], %l[error2]\n\t"
427 #endif
428 		/* try store */
429 		LONG_S " %[newv2], %[v2]\n\t"
430 		RSEQ_INJECT_ASM(5)
431 		"sync\n\t"	/* full sync provides store-release */
432 		/* final store */
433 		LONG_S " %[newv], %[v]\n\t"
434 		"2:\n\t"
435 		RSEQ_INJECT_ASM(6)
436 		"b 5f\n\t"
437 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
438 		"5:\n\t"
439 		: /* gcc asm goto does not allow outputs */
440 		: [cpu_id]		"r" (cpu),
441 		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
442 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
443 		  /* try store input */
444 		  [v2]			"m" (*v2),
445 		  [newv2]		"r" (newv2),
446 		  /* final store input */
447 		  [v]			"m" (*v),
448 		  [expect]		"r" (expect),
449 		  [newv]		"r" (newv)
450 		  RSEQ_INJECT_INPUT
451 		: "$4", "memory"
452 		  RSEQ_INJECT_CLOBBER
453 		: abort, cmpfail
454 #ifdef RSEQ_COMPARE_TWICE
455 		  , error1, error2
456 #endif
457 	);
458 	return 0;
459 abort:
460 	RSEQ_INJECT_FAILED
461 	return -1;
462 cmpfail:
463 	return 1;
464 #ifdef RSEQ_COMPARE_TWICE
465 error1:
466 	rseq_bug("cpu_id comparison failed");
467 error2:
468 	rseq_bug("expected value comparison failed");
469 #endif
470 }
471 
472 static inline __attribute__((always_inline))
473 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
474 			      intptr_t *v2, intptr_t expect2,
475 			      intptr_t newv, int cpu)
476 {
477 	RSEQ_INJECT_C(9)
478 
479 	__asm__ __volatile__ goto (
480 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
481 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
482 #ifdef RSEQ_COMPARE_TWICE
483 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
484 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
485 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
486 #endif
487 		/* Start rseq by storing table entry pointer into rseq_cs. */
488 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
489 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
490 		RSEQ_INJECT_ASM(3)
491 		LONG_L " $4, %[v]\n\t"
492 		"bne $4, %[expect], %l[cmpfail]\n\t"
493 		RSEQ_INJECT_ASM(4)
494 		LONG_L " $4, %[v2]\n\t"
495 		"bne $4, %[expect2], %l[cmpfail]\n\t"
496 		RSEQ_INJECT_ASM(5)
497 #ifdef RSEQ_COMPARE_TWICE
498 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
499 		LONG_L " $4, %[v]\n\t"
500 		"bne $4, %[expect], %l[error2]\n\t"
501 		LONG_L " $4, %[v2]\n\t"
502 		"bne $4, %[expect2], %l[error3]\n\t"
503 #endif
504 		/* final store */
505 		LONG_S " %[newv], %[v]\n\t"
506 		"2:\n\t"
507 		RSEQ_INJECT_ASM(6)
508 		"b 5f\n\t"
509 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
510 		"5:\n\t"
511 		: /* gcc asm goto does not allow outputs */
512 		: [cpu_id]		"r" (cpu),
513 		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
514 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
515 		  /* cmp2 input */
516 		  [v2]			"m" (*v2),
517 		  [expect2]		"r" (expect2),
518 		  /* final store input */
519 		  [v]			"m" (*v),
520 		  [expect]		"r" (expect),
521 		  [newv]		"r" (newv)
522 		  RSEQ_INJECT_INPUT
523 		: "$4", "memory"
524 		  RSEQ_INJECT_CLOBBER
525 		: abort, cmpfail
526 #ifdef RSEQ_COMPARE_TWICE
527 		  , error1, error2, error3
528 #endif
529 	);
530 	return 0;
531 abort:
532 	RSEQ_INJECT_FAILED
533 	return -1;
534 cmpfail:
535 	return 1;
536 #ifdef RSEQ_COMPARE_TWICE
537 error1:
538 	rseq_bug("cpu_id comparison failed");
539 error2:
540 	rseq_bug("1st expected value comparison failed");
541 error3:
542 	rseq_bug("2nd expected value comparison failed");
543 #endif
544 }
545 
546 static inline __attribute__((always_inline))
547 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
548 				 void *dst, void *src, size_t len,
549 				 intptr_t newv, int cpu)
550 {
551 	uintptr_t rseq_scratch[3];
552 
553 	RSEQ_INJECT_C(9)
554 
555 	__asm__ __volatile__ goto (
556 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
557 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
558 #ifdef RSEQ_COMPARE_TWICE
559 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
560 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
561 #endif
562 		LONG_S " %[src], %[rseq_scratch0]\n\t"
563 		LONG_S "  %[dst], %[rseq_scratch1]\n\t"
564 		LONG_S " %[len], %[rseq_scratch2]\n\t"
565 		/* Start rseq by storing table entry pointer into rseq_cs. */
566 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
567 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
568 		RSEQ_INJECT_ASM(3)
569 		LONG_L " $4, %[v]\n\t"
570 		"bne $4, %[expect], 5f\n\t"
571 		RSEQ_INJECT_ASM(4)
572 #ifdef RSEQ_COMPARE_TWICE
573 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
574 		LONG_L " $4, %[v]\n\t"
575 		"bne $4, %[expect], 7f\n\t"
576 #endif
577 		/* try memcpy */
578 		"beqz %[len], 333f\n\t" \
579 		"222:\n\t" \
580 		"lb   $4, 0(%[src])\n\t" \
581 		"sb   $4, 0(%[dst])\n\t" \
582 		LONG_ADDI " %[src], 1\n\t" \
583 		LONG_ADDI " %[dst], 1\n\t" \
584 		LONG_ADDI " %[len], -1\n\t" \
585 		"bnez %[len], 222b\n\t" \
586 		"333:\n\t" \
587 		RSEQ_INJECT_ASM(5)
588 		/* final store */
589 		LONG_S " %[newv], %[v]\n\t"
590 		"2:\n\t"
591 		RSEQ_INJECT_ASM(6)
592 		/* teardown */
593 		LONG_L " %[len], %[rseq_scratch2]\n\t"
594 		LONG_L " %[dst], %[rseq_scratch1]\n\t"
595 		LONG_L " %[src], %[rseq_scratch0]\n\t"
596 		"b 8f\n\t"
597 		RSEQ_ASM_DEFINE_ABORT(3, 4,
598 				      /* teardown */
599 				      LONG_L " %[len], %[rseq_scratch2]\n\t"
600 				      LONG_L " %[dst], %[rseq_scratch1]\n\t"
601 				      LONG_L " %[src], %[rseq_scratch0]\n\t",
602 				      abort, 1b, 2b, 4f)
603 		RSEQ_ASM_DEFINE_CMPFAIL(5,
604 					/* teardown */
605 					LONG_L " %[len], %[rseq_scratch2]\n\t"
606 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
607 					LONG_L " %[src], %[rseq_scratch0]\n\t",
608 					cmpfail)
609 #ifdef RSEQ_COMPARE_TWICE
610 		RSEQ_ASM_DEFINE_CMPFAIL(6,
611 					/* teardown */
612 					LONG_L " %[len], %[rseq_scratch2]\n\t"
613 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
614 					LONG_L " %[src], %[rseq_scratch0]\n\t",
615 					error1)
616 		RSEQ_ASM_DEFINE_CMPFAIL(7,
617 					/* teardown */
618 					LONG_L " %[len], %[rseq_scratch2]\n\t"
619 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
620 					LONG_L " %[src], %[rseq_scratch0]\n\t",
621 					error2)
622 #endif
623 		"8:\n\t"
624 		: /* gcc asm goto does not allow outputs */
625 		: [cpu_id]		"r" (cpu),
626 		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
627 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
628 		  /* final store input */
629 		  [v]			"m" (*v),
630 		  [expect]		"r" (expect),
631 		  [newv]		"r" (newv),
632 		  /* try memcpy input */
633 		  [dst]			"r" (dst),
634 		  [src]			"r" (src),
635 		  [len]			"r" (len),
636 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
637 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
638 		  [rseq_scratch2]	"m" (rseq_scratch[2])
639 		  RSEQ_INJECT_INPUT
640 		: "$4", "memory"
641 		  RSEQ_INJECT_CLOBBER
642 		: abort, cmpfail
643 #ifdef RSEQ_COMPARE_TWICE
644 		  , error1, error2
645 #endif
646 	);
647 	return 0;
648 abort:
649 	RSEQ_INJECT_FAILED
650 	return -1;
651 cmpfail:
652 	return 1;
653 #ifdef RSEQ_COMPARE_TWICE
654 error1:
655 	rseq_bug("cpu_id comparison failed");
656 error2:
657 	rseq_bug("expected value comparison failed");
658 #endif
659 }
660 
661 static inline __attribute__((always_inline))
662 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
663 					 void *dst, void *src, size_t len,
664 					 intptr_t newv, int cpu)
665 {
666 	uintptr_t rseq_scratch[3];
667 
668 	RSEQ_INJECT_C(9)
669 
670 	__asm__ __volatile__ goto (
671 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
672 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
673 #ifdef RSEQ_COMPARE_TWICE
674 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
675 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
676 #endif
677 		LONG_S " %[src], %[rseq_scratch0]\n\t"
678 		LONG_S " %[dst], %[rseq_scratch1]\n\t"
679 		LONG_S " %[len], %[rseq_scratch2]\n\t"
680 		/* Start rseq by storing table entry pointer into rseq_cs. */
681 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
682 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
683 		RSEQ_INJECT_ASM(3)
684 		LONG_L " $4, %[v]\n\t"
685 		"bne $4, %[expect], 5f\n\t"
686 		RSEQ_INJECT_ASM(4)
687 #ifdef RSEQ_COMPARE_TWICE
688 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
689 		LONG_L " $4, %[v]\n\t"
690 		"bne $4, %[expect], 7f\n\t"
691 #endif
692 		/* try memcpy */
693 		"beqz %[len], 333f\n\t" \
694 		"222:\n\t" \
695 		"lb   $4, 0(%[src])\n\t" \
696 		"sb   $4, 0(%[dst])\n\t" \
697 		LONG_ADDI " %[src], 1\n\t" \
698 		LONG_ADDI " %[dst], 1\n\t" \
699 		LONG_ADDI " %[len], -1\n\t" \
700 		"bnez %[len], 222b\n\t" \
701 		"333:\n\t" \
702 		RSEQ_INJECT_ASM(5)
703 		"sync\n\t"	/* full sync provides store-release */
704 		/* final store */
705 		LONG_S " %[newv], %[v]\n\t"
706 		"2:\n\t"
707 		RSEQ_INJECT_ASM(6)
708 		/* teardown */
709 		LONG_L " %[len], %[rseq_scratch2]\n\t"
710 		LONG_L " %[dst], %[rseq_scratch1]\n\t"
711 		LONG_L " %[src], %[rseq_scratch0]\n\t"
712 		"b 8f\n\t"
713 		RSEQ_ASM_DEFINE_ABORT(3, 4,
714 				      /* teardown */
715 				      LONG_L " %[len], %[rseq_scratch2]\n\t"
716 				      LONG_L " %[dst], %[rseq_scratch1]\n\t"
717 				      LONG_L " %[src], %[rseq_scratch0]\n\t",
718 				      abort, 1b, 2b, 4f)
719 		RSEQ_ASM_DEFINE_CMPFAIL(5,
720 					/* teardown */
721 					LONG_L " %[len], %[rseq_scratch2]\n\t"
722 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
723 					LONG_L " %[src], %[rseq_scratch0]\n\t",
724 					cmpfail)
725 #ifdef RSEQ_COMPARE_TWICE
726 		RSEQ_ASM_DEFINE_CMPFAIL(6,
727 					/* teardown */
728 					LONG_L " %[len], %[rseq_scratch2]\n\t"
729 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
730 					LONG_L " %[src], %[rseq_scratch0]\n\t",
731 					error1)
732 		RSEQ_ASM_DEFINE_CMPFAIL(7,
733 					/* teardown */
734 					LONG_L " %[len], %[rseq_scratch2]\n\t"
735 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
736 					LONG_L " %[src], %[rseq_scratch0]\n\t",
737 					error2)
738 #endif
739 		"8:\n\t"
740 		: /* gcc asm goto does not allow outputs */
741 		: [cpu_id]		"r" (cpu),
742 		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
743 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
744 		  /* final store input */
745 		  [v]			"m" (*v),
746 		  [expect]		"r" (expect),
747 		  [newv]		"r" (newv),
748 		  /* try memcpy input */
749 		  [dst]			"r" (dst),
750 		  [src]			"r" (src),
751 		  [len]			"r" (len),
752 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
753 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
754 		  [rseq_scratch2]	"m" (rseq_scratch[2])
755 		  RSEQ_INJECT_INPUT
756 		: "$4", "memory"
757 		  RSEQ_INJECT_CLOBBER
758 		: abort, cmpfail
759 #ifdef RSEQ_COMPARE_TWICE
760 		  , error1, error2
761 #endif
762 	);
763 	return 0;
764 abort:
765 	RSEQ_INJECT_FAILED
766 	return -1;
767 cmpfail:
768 	return 1;
769 #ifdef RSEQ_COMPARE_TWICE
770 error1:
771 	rseq_bug("cpu_id comparison failed");
772 error2:
773 	rseq_bug("expected value comparison failed");
774 #endif
775 }
776 
777 #endif /* !RSEQ_SKIP_FASTPATH */
778