1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * rseq-arm.h
4  *
5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6  */
7 
8 /*
9  * - ARM little endian
10  *
11  * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
12  * value 0x5de3. This traps if user-space reaches this instruction by mistake,
13  * and the uncommon operand ensures the kernel does not move the instruction
14  * pointer to attacker-controlled code on rseq abort.
15  *
16  * The instruction pattern in the A32 instruction set is:
17  *
18  * e7f5def3    udf    #24035    ; 0x5de3
19  *
20  * This translates to the following instruction pattern in the T16 instruction
21  * set:
22  *
23  * little endian:
24  * def3        udf    #243      ; 0xf3
25  * e7f5        b.n    <7f5>
26  *
27  * - ARMv6+ big endian (BE8):
28  *
29  * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
30  * code and big-endian data. The data value of the signature needs to have its
31  * byte order reversed to generate the trap instruction:
32  *
33  * Data: 0xf3def5e7
34  *
35  * Translates to this A32 instruction pattern:
36  *
37  * e7f5def3    udf    #24035    ; 0x5de3
38  *
39  * Translates to this T16 instruction pattern:
40  *
41  * def3        udf    #243      ; 0xf3
42  * e7f5        b.n    <7f5>
43  *
44  * - Prior to ARMv6 big endian (BE32):
45  *
46  * Prior to ARMv6, -mbig-endian generates big-endian code and data
47  * (which match), so the endianness of the data representation of the
48  * signature should not be reversed. However, the choice between BE32
49  * and BE8 is done by the linker, so we cannot know whether code and
50  * data endianness will be mixed before the linker is invoked. So rather
51  * than try to play tricks with the linker, the rseq signature is simply
52  * data (not a trap instruction) prior to ARMv6 on big endian. This is
53  * why the signature is expressed as data (.word) rather than as
54  * instruction (.inst) in assembler.
55  */
56 
57 #ifdef __ARMEB__
58 #define RSEQ_SIG    0xf3def5e7      /* udf    #24035    ; 0x5de3 (ARMv6+) */
59 #else
60 #define RSEQ_SIG    0xe7f5def3      /* udf    #24035    ; 0x5de3 */
61 #endif
62 
63 #define rseq_smp_mb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
64 #define rseq_smp_rmb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
65 #define rseq_smp_wmb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
66 
67 #define rseq_smp_load_acquire(p)					\
68 __extension__ ({							\
69 	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
70 	rseq_smp_mb();							\
71 	____p1;								\
72 })
73 
74 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
75 
76 #define rseq_smp_store_release(p, v)					\
77 do {									\
78 	rseq_smp_mb();							\
79 	RSEQ_WRITE_ONCE(*p, v);						\
80 } while (0)
81 
82 #ifdef RSEQ_SKIP_FASTPATH
83 #include "rseq-skip.h"
84 #else /* !RSEQ_SKIP_FASTPATH */
85 
86 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,	\
87 				post_commit_offset, abort_ip)		\
88 		".pushsection __rseq_cs, \"aw\"\n\t"			\
89 		".balign 32\n\t"					\
90 		__rseq_str(label) ":\n\t"					\
91 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
92 		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
93 		".popsection\n\t"					\
94 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
95 		".word " __rseq_str(label) "b, 0x0\n\t"			\
96 		".popsection\n\t"
97 
98 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
99 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
100 				(post_commit_ip - start_ip), abort_ip)
101 
102 /*
103  * Exit points of a rseq critical section consist of all instructions outside
104  * of the critical section where a critical section can either branch to or
105  * reach through the normal course of its execution. The abort IP and the
106  * post-commit IP are already part of the __rseq_cs section and should not be
107  * explicitly defined as additional exit points. Knowing all exit points is
108  * useful to assist debuggers stepping over the critical section.
109  */
110 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
111 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
112 		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
113 		".popsection\n\t"
114 
115 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
116 		RSEQ_INJECT_ASM(1)					\
117 		"adr r0, " __rseq_str(cs_label) "\n\t"			\
118 		"str r0, %[" __rseq_str(rseq_cs) "]\n\t"		\
119 		__rseq_str(label) ":\n\t"
120 
121 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
122 		RSEQ_INJECT_ASM(2)					\
123 		"ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t"	\
124 		"cmp %[" __rseq_str(cpu_id) "], r0\n\t"		\
125 		"bne " __rseq_str(label) "\n\t"
126 
127 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,		\
128 				abort_label, version, flags,		\
129 				start_ip, post_commit_offset, abort_ip)	\
130 		".balign 32\n\t"					\
131 		__rseq_str(table_label) ":\n\t"				\
132 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
133 		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
134 		".word " __rseq_str(RSEQ_SIG) "\n\t"			\
135 		__rseq_str(label) ":\n\t"				\
136 		teardown						\
137 		"b %l[" __rseq_str(abort_label) "]\n\t"
138 
139 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
140 			      start_ip, post_commit_ip, abort_ip)	\
141 	__RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,		\
142 				abort_label, 0x0, 0x0, start_ip,	\
143 				(post_commit_ip - start_ip), abort_ip)
144 
145 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
146 		__rseq_str(label) ":\n\t"				\
147 		teardown						\
148 		"b %l[" __rseq_str(cmpfail_label) "]\n\t"
149 
150 #define rseq_workaround_gcc_asm_size_guess()	__asm__ __volatile__("")
151 
152 static inline __attribute__((always_inline))
153 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
154 {
155 	RSEQ_INJECT_C(9)
156 
157 	rseq_workaround_gcc_asm_size_guess();
158 	__asm__ __volatile__ goto (
159 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
160 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
161 #ifdef RSEQ_COMPARE_TWICE
162 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
163 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
164 #endif
165 		/* Start rseq by storing table entry pointer into rseq_cs. */
166 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
167 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
168 		RSEQ_INJECT_ASM(3)
169 		"ldr r0, %[v]\n\t"
170 		"cmp %[expect], r0\n\t"
171 		"bne %l[cmpfail]\n\t"
172 		RSEQ_INJECT_ASM(4)
173 #ifdef RSEQ_COMPARE_TWICE
174 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
175 		"ldr r0, %[v]\n\t"
176 		"cmp %[expect], r0\n\t"
177 		"bne %l[error2]\n\t"
178 #endif
179 		/* final store */
180 		"str %[newv], %[v]\n\t"
181 		"2:\n\t"
182 		RSEQ_INJECT_ASM(5)
183 		"b 5f\n\t"
184 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
185 		"5:\n\t"
186 		: /* gcc asm goto does not allow outputs */
187 		: [cpu_id]		"r" (cpu),
188 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
189 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
190 		  [v]			"m" (*v),
191 		  [expect]		"r" (expect),
192 		  [newv]		"r" (newv)
193 		  RSEQ_INJECT_INPUT
194 		: "r0", "memory", "cc"
195 		  RSEQ_INJECT_CLOBBER
196 		: abort, cmpfail
197 #ifdef RSEQ_COMPARE_TWICE
198 		  , error1, error2
199 #endif
200 	);
201 	rseq_workaround_gcc_asm_size_guess();
202 	return 0;
203 abort:
204 	rseq_workaround_gcc_asm_size_guess();
205 	RSEQ_INJECT_FAILED
206 	return -1;
207 cmpfail:
208 	rseq_workaround_gcc_asm_size_guess();
209 	return 1;
210 #ifdef RSEQ_COMPARE_TWICE
211 error1:
212 	rseq_bug("cpu_id comparison failed");
213 error2:
214 	rseq_bug("expected value comparison failed");
215 #endif
216 }
217 
218 static inline __attribute__((always_inline))
219 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
220 			       off_t voffp, intptr_t *load, int cpu)
221 {
222 	RSEQ_INJECT_C(9)
223 
224 	rseq_workaround_gcc_asm_size_guess();
225 	__asm__ __volatile__ goto (
226 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
227 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
228 #ifdef RSEQ_COMPARE_TWICE
229 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
230 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
231 #endif
232 		/* Start rseq by storing table entry pointer into rseq_cs. */
233 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
234 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
235 		RSEQ_INJECT_ASM(3)
236 		"ldr r0, %[v]\n\t"
237 		"cmp %[expectnot], r0\n\t"
238 		"beq %l[cmpfail]\n\t"
239 		RSEQ_INJECT_ASM(4)
240 #ifdef RSEQ_COMPARE_TWICE
241 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
242 		"ldr r0, %[v]\n\t"
243 		"cmp %[expectnot], r0\n\t"
244 		"beq %l[error2]\n\t"
245 #endif
246 		"str r0, %[load]\n\t"
247 		"add r0, %[voffp]\n\t"
248 		"ldr r0, [r0]\n\t"
249 		/* final store */
250 		"str r0, %[v]\n\t"
251 		"2:\n\t"
252 		RSEQ_INJECT_ASM(5)
253 		"b 5f\n\t"
254 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
255 		"5:\n\t"
256 		: /* gcc asm goto does not allow outputs */
257 		: [cpu_id]		"r" (cpu),
258 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
259 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
260 		  /* final store input */
261 		  [v]			"m" (*v),
262 		  [expectnot]		"r" (expectnot),
263 		  [voffp]		"Ir" (voffp),
264 		  [load]		"m" (*load)
265 		  RSEQ_INJECT_INPUT
266 		: "r0", "memory", "cc"
267 		  RSEQ_INJECT_CLOBBER
268 		: abort, cmpfail
269 #ifdef RSEQ_COMPARE_TWICE
270 		  , error1, error2
271 #endif
272 	);
273 	rseq_workaround_gcc_asm_size_guess();
274 	return 0;
275 abort:
276 	rseq_workaround_gcc_asm_size_guess();
277 	RSEQ_INJECT_FAILED
278 	return -1;
279 cmpfail:
280 	rseq_workaround_gcc_asm_size_guess();
281 	return 1;
282 #ifdef RSEQ_COMPARE_TWICE
283 error1:
284 	rseq_bug("cpu_id comparison failed");
285 error2:
286 	rseq_bug("expected value comparison failed");
287 #endif
288 }
289 
290 static inline __attribute__((always_inline))
291 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
292 {
293 	RSEQ_INJECT_C(9)
294 
295 	rseq_workaround_gcc_asm_size_guess();
296 	__asm__ __volatile__ goto (
297 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
298 #ifdef RSEQ_COMPARE_TWICE
299 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
300 #endif
301 		/* Start rseq by storing table entry pointer into rseq_cs. */
302 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
303 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
304 		RSEQ_INJECT_ASM(3)
305 #ifdef RSEQ_COMPARE_TWICE
306 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
307 #endif
308 		"ldr r0, %[v]\n\t"
309 		"add r0, %[count]\n\t"
310 		/* final store */
311 		"str r0, %[v]\n\t"
312 		"2:\n\t"
313 		RSEQ_INJECT_ASM(4)
314 		"b 5f\n\t"
315 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
316 		"5:\n\t"
317 		: /* gcc asm goto does not allow outputs */
318 		: [cpu_id]		"r" (cpu),
319 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
320 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
321 		  [v]			"m" (*v),
322 		  [count]		"Ir" (count)
323 		  RSEQ_INJECT_INPUT
324 		: "r0", "memory", "cc"
325 		  RSEQ_INJECT_CLOBBER
326 		: abort
327 #ifdef RSEQ_COMPARE_TWICE
328 		  , error1
329 #endif
330 	);
331 	rseq_workaround_gcc_asm_size_guess();
332 	return 0;
333 abort:
334 	rseq_workaround_gcc_asm_size_guess();
335 	RSEQ_INJECT_FAILED
336 	return -1;
337 #ifdef RSEQ_COMPARE_TWICE
338 error1:
339 	rseq_bug("cpu_id comparison failed");
340 #endif
341 }
342 
343 static inline __attribute__((always_inline))
344 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
345 				 intptr_t *v2, intptr_t newv2,
346 				 intptr_t newv, int cpu)
347 {
348 	RSEQ_INJECT_C(9)
349 
350 	rseq_workaround_gcc_asm_size_guess();
351 	__asm__ __volatile__ goto (
352 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
353 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
354 #ifdef RSEQ_COMPARE_TWICE
355 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
356 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
357 #endif
358 		/* Start rseq by storing table entry pointer into rseq_cs. */
359 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
360 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
361 		RSEQ_INJECT_ASM(3)
362 		"ldr r0, %[v]\n\t"
363 		"cmp %[expect], r0\n\t"
364 		"bne %l[cmpfail]\n\t"
365 		RSEQ_INJECT_ASM(4)
366 #ifdef RSEQ_COMPARE_TWICE
367 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
368 		"ldr r0, %[v]\n\t"
369 		"cmp %[expect], r0\n\t"
370 		"bne %l[error2]\n\t"
371 #endif
372 		/* try store */
373 		"str %[newv2], %[v2]\n\t"
374 		RSEQ_INJECT_ASM(5)
375 		/* final store */
376 		"str %[newv], %[v]\n\t"
377 		"2:\n\t"
378 		RSEQ_INJECT_ASM(6)
379 		"b 5f\n\t"
380 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
381 		"5:\n\t"
382 		: /* gcc asm goto does not allow outputs */
383 		: [cpu_id]		"r" (cpu),
384 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
385 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
386 		  /* try store input */
387 		  [v2]			"m" (*v2),
388 		  [newv2]		"r" (newv2),
389 		  /* final store input */
390 		  [v]			"m" (*v),
391 		  [expect]		"r" (expect),
392 		  [newv]		"r" (newv)
393 		  RSEQ_INJECT_INPUT
394 		: "r0", "memory", "cc"
395 		  RSEQ_INJECT_CLOBBER
396 		: abort, cmpfail
397 #ifdef RSEQ_COMPARE_TWICE
398 		  , error1, error2
399 #endif
400 	);
401 	rseq_workaround_gcc_asm_size_guess();
402 	return 0;
403 abort:
404 	rseq_workaround_gcc_asm_size_guess();
405 	RSEQ_INJECT_FAILED
406 	return -1;
407 cmpfail:
408 	rseq_workaround_gcc_asm_size_guess();
409 	return 1;
410 #ifdef RSEQ_COMPARE_TWICE
411 error1:
412 	rseq_bug("cpu_id comparison failed");
413 error2:
414 	rseq_bug("expected value comparison failed");
415 #endif
416 }
417 
418 static inline __attribute__((always_inline))
419 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
420 					 intptr_t *v2, intptr_t newv2,
421 					 intptr_t newv, int cpu)
422 {
423 	RSEQ_INJECT_C(9)
424 
425 	rseq_workaround_gcc_asm_size_guess();
426 	__asm__ __volatile__ goto (
427 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
428 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
429 #ifdef RSEQ_COMPARE_TWICE
430 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
431 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
432 #endif
433 		/* Start rseq by storing table entry pointer into rseq_cs. */
434 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
435 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
436 		RSEQ_INJECT_ASM(3)
437 		"ldr r0, %[v]\n\t"
438 		"cmp %[expect], r0\n\t"
439 		"bne %l[cmpfail]\n\t"
440 		RSEQ_INJECT_ASM(4)
441 #ifdef RSEQ_COMPARE_TWICE
442 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
443 		"ldr r0, %[v]\n\t"
444 		"cmp %[expect], r0\n\t"
445 		"bne %l[error2]\n\t"
446 #endif
447 		/* try store */
448 		"str %[newv2], %[v2]\n\t"
449 		RSEQ_INJECT_ASM(5)
450 		"dmb\n\t"	/* full mb provides store-release */
451 		/* final store */
452 		"str %[newv], %[v]\n\t"
453 		"2:\n\t"
454 		RSEQ_INJECT_ASM(6)
455 		"b 5f\n\t"
456 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
457 		"5:\n\t"
458 		: /* gcc asm goto does not allow outputs */
459 		: [cpu_id]		"r" (cpu),
460 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
461 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
462 		  /* try store input */
463 		  [v2]			"m" (*v2),
464 		  [newv2]		"r" (newv2),
465 		  /* final store input */
466 		  [v]			"m" (*v),
467 		  [expect]		"r" (expect),
468 		  [newv]		"r" (newv)
469 		  RSEQ_INJECT_INPUT
470 		: "r0", "memory", "cc"
471 		  RSEQ_INJECT_CLOBBER
472 		: abort, cmpfail
473 #ifdef RSEQ_COMPARE_TWICE
474 		  , error1, error2
475 #endif
476 	);
477 	rseq_workaround_gcc_asm_size_guess();
478 	return 0;
479 abort:
480 	rseq_workaround_gcc_asm_size_guess();
481 	RSEQ_INJECT_FAILED
482 	return -1;
483 cmpfail:
484 	rseq_workaround_gcc_asm_size_guess();
485 	return 1;
486 #ifdef RSEQ_COMPARE_TWICE
487 error1:
488 	rseq_bug("cpu_id comparison failed");
489 error2:
490 	rseq_bug("expected value comparison failed");
491 #endif
492 }
493 
494 static inline __attribute__((always_inline))
495 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
496 			      intptr_t *v2, intptr_t expect2,
497 			      intptr_t newv, int cpu)
498 {
499 	RSEQ_INJECT_C(9)
500 
501 	rseq_workaround_gcc_asm_size_guess();
502 	__asm__ __volatile__ goto (
503 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
504 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
505 #ifdef RSEQ_COMPARE_TWICE
506 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
507 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
508 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
509 #endif
510 		/* Start rseq by storing table entry pointer into rseq_cs. */
511 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
512 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
513 		RSEQ_INJECT_ASM(3)
514 		"ldr r0, %[v]\n\t"
515 		"cmp %[expect], r0\n\t"
516 		"bne %l[cmpfail]\n\t"
517 		RSEQ_INJECT_ASM(4)
518 		"ldr r0, %[v2]\n\t"
519 		"cmp %[expect2], r0\n\t"
520 		"bne %l[cmpfail]\n\t"
521 		RSEQ_INJECT_ASM(5)
522 #ifdef RSEQ_COMPARE_TWICE
523 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
524 		"ldr r0, %[v]\n\t"
525 		"cmp %[expect], r0\n\t"
526 		"bne %l[error2]\n\t"
527 		"ldr r0, %[v2]\n\t"
528 		"cmp %[expect2], r0\n\t"
529 		"bne %l[error3]\n\t"
530 #endif
531 		/* final store */
532 		"str %[newv], %[v]\n\t"
533 		"2:\n\t"
534 		RSEQ_INJECT_ASM(6)
535 		"b 5f\n\t"
536 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
537 		"5:\n\t"
538 		: /* gcc asm goto does not allow outputs */
539 		: [cpu_id]		"r" (cpu),
540 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
541 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
542 		  /* cmp2 input */
543 		  [v2]			"m" (*v2),
544 		  [expect2]		"r" (expect2),
545 		  /* final store input */
546 		  [v]			"m" (*v),
547 		  [expect]		"r" (expect),
548 		  [newv]		"r" (newv)
549 		  RSEQ_INJECT_INPUT
550 		: "r0", "memory", "cc"
551 		  RSEQ_INJECT_CLOBBER
552 		: abort, cmpfail
553 #ifdef RSEQ_COMPARE_TWICE
554 		  , error1, error2, error3
555 #endif
556 	);
557 	rseq_workaround_gcc_asm_size_guess();
558 	return 0;
559 abort:
560 	rseq_workaround_gcc_asm_size_guess();
561 	RSEQ_INJECT_FAILED
562 	return -1;
563 cmpfail:
564 	rseq_workaround_gcc_asm_size_guess();
565 	return 1;
566 #ifdef RSEQ_COMPARE_TWICE
567 error1:
568 	rseq_bug("cpu_id comparison failed");
569 error2:
570 	rseq_bug("1st expected value comparison failed");
571 error3:
572 	rseq_bug("2nd expected value comparison failed");
573 #endif
574 }
575 
576 static inline __attribute__((always_inline))
577 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
578 				 void *dst, void *src, size_t len,
579 				 intptr_t newv, int cpu)
580 {
581 	uint32_t rseq_scratch[3];
582 
583 	RSEQ_INJECT_C(9)
584 
585 	rseq_workaround_gcc_asm_size_guess();
586 	__asm__ __volatile__ goto (
587 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
588 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
589 #ifdef RSEQ_COMPARE_TWICE
590 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
591 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
592 #endif
593 		"str %[src], %[rseq_scratch0]\n\t"
594 		"str %[dst], %[rseq_scratch1]\n\t"
595 		"str %[len], %[rseq_scratch2]\n\t"
596 		/* Start rseq by storing table entry pointer into rseq_cs. */
597 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
598 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
599 		RSEQ_INJECT_ASM(3)
600 		"ldr r0, %[v]\n\t"
601 		"cmp %[expect], r0\n\t"
602 		"bne 5f\n\t"
603 		RSEQ_INJECT_ASM(4)
604 #ifdef RSEQ_COMPARE_TWICE
605 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
606 		"ldr r0, %[v]\n\t"
607 		"cmp %[expect], r0\n\t"
608 		"bne 7f\n\t"
609 #endif
610 		/* try memcpy */
611 		"cmp %[len], #0\n\t" \
612 		"beq 333f\n\t" \
613 		"222:\n\t" \
614 		"ldrb %%r0, [%[src]]\n\t" \
615 		"strb %%r0, [%[dst]]\n\t" \
616 		"adds %[src], #1\n\t" \
617 		"adds %[dst], #1\n\t" \
618 		"subs %[len], #1\n\t" \
619 		"bne 222b\n\t" \
620 		"333:\n\t" \
621 		RSEQ_INJECT_ASM(5)
622 		/* final store */
623 		"str %[newv], %[v]\n\t"
624 		"2:\n\t"
625 		RSEQ_INJECT_ASM(6)
626 		/* teardown */
627 		"ldr %[len], %[rseq_scratch2]\n\t"
628 		"ldr %[dst], %[rseq_scratch1]\n\t"
629 		"ldr %[src], %[rseq_scratch0]\n\t"
630 		"b 8f\n\t"
631 		RSEQ_ASM_DEFINE_ABORT(3, 4,
632 				      /* teardown */
633 				      "ldr %[len], %[rseq_scratch2]\n\t"
634 				      "ldr %[dst], %[rseq_scratch1]\n\t"
635 				      "ldr %[src], %[rseq_scratch0]\n\t",
636 				      abort, 1b, 2b, 4f)
637 		RSEQ_ASM_DEFINE_CMPFAIL(5,
638 					/* teardown */
639 					"ldr %[len], %[rseq_scratch2]\n\t"
640 					"ldr %[dst], %[rseq_scratch1]\n\t"
641 					"ldr %[src], %[rseq_scratch0]\n\t",
642 					cmpfail)
643 #ifdef RSEQ_COMPARE_TWICE
644 		RSEQ_ASM_DEFINE_CMPFAIL(6,
645 					/* teardown */
646 					"ldr %[len], %[rseq_scratch2]\n\t"
647 					"ldr %[dst], %[rseq_scratch1]\n\t"
648 					"ldr %[src], %[rseq_scratch0]\n\t",
649 					error1)
650 		RSEQ_ASM_DEFINE_CMPFAIL(7,
651 					/* teardown */
652 					"ldr %[len], %[rseq_scratch2]\n\t"
653 					"ldr %[dst], %[rseq_scratch1]\n\t"
654 					"ldr %[src], %[rseq_scratch0]\n\t",
655 					error2)
656 #endif
657 		"8:\n\t"
658 		: /* gcc asm goto does not allow outputs */
659 		: [cpu_id]		"r" (cpu),
660 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
661 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
662 		  /* final store input */
663 		  [v]			"m" (*v),
664 		  [expect]		"r" (expect),
665 		  [newv]		"r" (newv),
666 		  /* try memcpy input */
667 		  [dst]			"r" (dst),
668 		  [src]			"r" (src),
669 		  [len]			"r" (len),
670 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
671 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
672 		  [rseq_scratch2]	"m" (rseq_scratch[2])
673 		  RSEQ_INJECT_INPUT
674 		: "r0", "memory", "cc"
675 		  RSEQ_INJECT_CLOBBER
676 		: abort, cmpfail
677 #ifdef RSEQ_COMPARE_TWICE
678 		  , error1, error2
679 #endif
680 	);
681 	rseq_workaround_gcc_asm_size_guess();
682 	return 0;
683 abort:
684 	rseq_workaround_gcc_asm_size_guess();
685 	RSEQ_INJECT_FAILED
686 	return -1;
687 cmpfail:
688 	rseq_workaround_gcc_asm_size_guess();
689 	return 1;
690 #ifdef RSEQ_COMPARE_TWICE
691 error1:
692 	rseq_workaround_gcc_asm_size_guess();
693 	rseq_bug("cpu_id comparison failed");
694 error2:
695 	rseq_workaround_gcc_asm_size_guess();
696 	rseq_bug("expected value comparison failed");
697 #endif
698 }
699 
700 static inline __attribute__((always_inline))
701 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
702 					 void *dst, void *src, size_t len,
703 					 intptr_t newv, int cpu)
704 {
705 	uint32_t rseq_scratch[3];
706 
707 	RSEQ_INJECT_C(9)
708 
709 	rseq_workaround_gcc_asm_size_guess();
710 	__asm__ __volatile__ goto (
711 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
712 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
713 #ifdef RSEQ_COMPARE_TWICE
714 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
715 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
716 #endif
717 		"str %[src], %[rseq_scratch0]\n\t"
718 		"str %[dst], %[rseq_scratch1]\n\t"
719 		"str %[len], %[rseq_scratch2]\n\t"
720 		/* Start rseq by storing table entry pointer into rseq_cs. */
721 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
722 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
723 		RSEQ_INJECT_ASM(3)
724 		"ldr r0, %[v]\n\t"
725 		"cmp %[expect], r0\n\t"
726 		"bne 5f\n\t"
727 		RSEQ_INJECT_ASM(4)
728 #ifdef RSEQ_COMPARE_TWICE
729 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
730 		"ldr r0, %[v]\n\t"
731 		"cmp %[expect], r0\n\t"
732 		"bne 7f\n\t"
733 #endif
734 		/* try memcpy */
735 		"cmp %[len], #0\n\t" \
736 		"beq 333f\n\t" \
737 		"222:\n\t" \
738 		"ldrb %%r0, [%[src]]\n\t" \
739 		"strb %%r0, [%[dst]]\n\t" \
740 		"adds %[src], #1\n\t" \
741 		"adds %[dst], #1\n\t" \
742 		"subs %[len], #1\n\t" \
743 		"bne 222b\n\t" \
744 		"333:\n\t" \
745 		RSEQ_INJECT_ASM(5)
746 		"dmb\n\t"	/* full mb provides store-release */
747 		/* final store */
748 		"str %[newv], %[v]\n\t"
749 		"2:\n\t"
750 		RSEQ_INJECT_ASM(6)
751 		/* teardown */
752 		"ldr %[len], %[rseq_scratch2]\n\t"
753 		"ldr %[dst], %[rseq_scratch1]\n\t"
754 		"ldr %[src], %[rseq_scratch0]\n\t"
755 		"b 8f\n\t"
756 		RSEQ_ASM_DEFINE_ABORT(3, 4,
757 				      /* teardown */
758 				      "ldr %[len], %[rseq_scratch2]\n\t"
759 				      "ldr %[dst], %[rseq_scratch1]\n\t"
760 				      "ldr %[src], %[rseq_scratch0]\n\t",
761 				      abort, 1b, 2b, 4f)
762 		RSEQ_ASM_DEFINE_CMPFAIL(5,
763 					/* teardown */
764 					"ldr %[len], %[rseq_scratch2]\n\t"
765 					"ldr %[dst], %[rseq_scratch1]\n\t"
766 					"ldr %[src], %[rseq_scratch0]\n\t",
767 					cmpfail)
768 #ifdef RSEQ_COMPARE_TWICE
769 		RSEQ_ASM_DEFINE_CMPFAIL(6,
770 					/* teardown */
771 					"ldr %[len], %[rseq_scratch2]\n\t"
772 					"ldr %[dst], %[rseq_scratch1]\n\t"
773 					"ldr %[src], %[rseq_scratch0]\n\t",
774 					error1)
775 		RSEQ_ASM_DEFINE_CMPFAIL(7,
776 					/* teardown */
777 					"ldr %[len], %[rseq_scratch2]\n\t"
778 					"ldr %[dst], %[rseq_scratch1]\n\t"
779 					"ldr %[src], %[rseq_scratch0]\n\t",
780 					error2)
781 #endif
782 		"8:\n\t"
783 		: /* gcc asm goto does not allow outputs */
784 		: [cpu_id]		"r" (cpu),
785 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
786 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
787 		  /* final store input */
788 		  [v]			"m" (*v),
789 		  [expect]		"r" (expect),
790 		  [newv]		"r" (newv),
791 		  /* try memcpy input */
792 		  [dst]			"r" (dst),
793 		  [src]			"r" (src),
794 		  [len]			"r" (len),
795 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
796 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
797 		  [rseq_scratch2]	"m" (rseq_scratch[2])
798 		  RSEQ_INJECT_INPUT
799 		: "r0", "memory", "cc"
800 		  RSEQ_INJECT_CLOBBER
801 		: abort, cmpfail
802 #ifdef RSEQ_COMPARE_TWICE
803 		  , error1, error2
804 #endif
805 	);
806 	rseq_workaround_gcc_asm_size_guess();
807 	return 0;
808 abort:
809 	rseq_workaround_gcc_asm_size_guess();
810 	RSEQ_INJECT_FAILED
811 	return -1;
812 cmpfail:
813 	rseq_workaround_gcc_asm_size_guess();
814 	return 1;
815 #ifdef RSEQ_COMPARE_TWICE
816 error1:
817 	rseq_workaround_gcc_asm_size_guess();
818 	rseq_bug("cpu_id comparison failed");
819 error2:
820 	rseq_workaround_gcc_asm_size_guess();
821 	rseq_bug("expected value comparison failed");
822 #endif
823 }
824 
825 #endif /* !RSEQ_SKIP_FASTPATH */
826