1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * rseq-x86.h
4  *
5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6  */
7 
8 #include <stdint.h>
9 
10 /*
11  * RSEQ_SIG is used with the following reserved undefined instructions, which
12  * trap in user-space:
13  *
14  * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
15  * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
16  */
17 #define RSEQ_SIG	0x53053053
18 
19 /*
20  * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
21  * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
22  * address through a "r" input operand.
23  */
24 
25 /* Offset of cpu_id and rseq_cs fields in struct rseq. */
26 #define RSEQ_CPU_ID_OFFSET	4
27 #define RSEQ_CS_OFFSET		8
28 
29 #ifdef __x86_64__
30 
31 #define rseq_smp_mb()	\
32 	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
33 #define rseq_smp_rmb()	rseq_barrier()
34 #define rseq_smp_wmb()	rseq_barrier()
35 
36 #define rseq_smp_load_acquire(p)					\
37 __extension__ ({							\
38 	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
39 	rseq_barrier();							\
40 	____p1;								\
41 })
42 
43 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
44 
45 #define rseq_smp_store_release(p, v)					\
46 do {									\
47 	rseq_barrier();							\
48 	RSEQ_WRITE_ONCE(*p, v);						\
49 } while (0)
50 
51 #ifdef RSEQ_SKIP_FASTPATH
52 #include "rseq-skip.h"
53 #else /* !RSEQ_SKIP_FASTPATH */
54 
55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
56 				start_ip, post_commit_offset, abort_ip)	\
57 		".pushsection __rseq_cs, \"aw\"\n\t"			\
58 		".balign 32\n\t"					\
59 		__rseq_str(label) ":\n\t"				\
60 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
61 		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
62 		".popsection\n\t"					\
63 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
64 		".quad " __rseq_str(label) "b\n\t"			\
65 		".popsection\n\t"
66 
67 
68 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
69 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
70 				(post_commit_ip - start_ip), abort_ip)
71 
72 /*
73  * Exit points of a rseq critical section consist of all instructions outside
74  * of the critical section where a critical section can either branch to or
75  * reach through the normal course of its execution. The abort IP and the
76  * post-commit IP are already part of the __rseq_cs section and should not be
77  * explicitly defined as additional exit points. Knowing all exit points is
78  * useful to assist debuggers stepping over the critical section.
79  */
80 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
81 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
82 		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
83 		".popsection\n\t"
84 
85 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
86 		RSEQ_INJECT_ASM(1)					\
87 		"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"	\
88 		"movq %%rax, " __rseq_str(rseq_cs) "\n\t"		\
89 		__rseq_str(label) ":\n\t"
90 
91 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
92 		RSEQ_INJECT_ASM(2)					\
93 		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
94 		"jnz " __rseq_str(label) "\n\t"
95 
96 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
97 		".pushsection __rseq_failure, \"ax\"\n\t"		\
98 		/* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
99 		".byte 0x0f, 0xb9, 0x3d\n\t"				\
100 		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
101 		__rseq_str(label) ":\n\t"				\
102 		teardown						\
103 		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
104 		".popsection\n\t"
105 
106 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
107 		".pushsection __rseq_failure, \"ax\"\n\t"		\
108 		__rseq_str(label) ":\n\t"				\
109 		teardown						\
110 		"jmp %l[" __rseq_str(cmpfail_label) "]\n\t"		\
111 		".popsection\n\t"
112 
113 static inline __attribute__((always_inline))
114 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
115 {
116 	RSEQ_INJECT_C(9)
117 
118 	__asm__ __volatile__ goto (
119 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
120 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
121 #ifdef RSEQ_COMPARE_TWICE
122 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
123 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
124 #endif
125 		/* Start rseq by storing table entry pointer into rseq_cs. */
126 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
127 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
128 		RSEQ_INJECT_ASM(3)
129 		"cmpq %[v], %[expect]\n\t"
130 		"jnz %l[cmpfail]\n\t"
131 		RSEQ_INJECT_ASM(4)
132 #ifdef RSEQ_COMPARE_TWICE
133 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
134 		"cmpq %[v], %[expect]\n\t"
135 		"jnz %l[error2]\n\t"
136 #endif
137 		/* final store */
138 		"movq %[newv], %[v]\n\t"
139 		"2:\n\t"
140 		RSEQ_INJECT_ASM(5)
141 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
142 		: /* gcc asm goto does not allow outputs */
143 		: [cpu_id]		"r" (cpu),
144 		  [rseq_abi]		"r" (&__rseq_abi),
145 		  [v]			"m" (*v),
146 		  [expect]		"r" (expect),
147 		  [newv]		"r" (newv)
148 		: "memory", "cc", "rax"
149 		  RSEQ_INJECT_CLOBBER
150 		: abort, cmpfail
151 #ifdef RSEQ_COMPARE_TWICE
152 		  , error1, error2
153 #endif
154 	);
155 	return 0;
156 abort:
157 	RSEQ_INJECT_FAILED
158 	return -1;
159 cmpfail:
160 	return 1;
161 #ifdef RSEQ_COMPARE_TWICE
162 error1:
163 	rseq_bug("cpu_id comparison failed");
164 error2:
165 	rseq_bug("expected value comparison failed");
166 #endif
167 }
168 
169 /*
170  * Compare @v against @expectnot. When it does _not_ match, load @v
171  * into @load, and store the content of *@v + voffp into @v.
172  */
173 static inline __attribute__((always_inline))
174 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
175 			       off_t voffp, intptr_t *load, int cpu)
176 {
177 	RSEQ_INJECT_C(9)
178 
179 	__asm__ __volatile__ goto (
180 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
181 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
182 #ifdef RSEQ_COMPARE_TWICE
183 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
184 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
185 #endif
186 		/* Start rseq by storing table entry pointer into rseq_cs. */
187 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
188 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
189 		RSEQ_INJECT_ASM(3)
190 		"movq %[v], %%rbx\n\t"
191 		"cmpq %%rbx, %[expectnot]\n\t"
192 		"je %l[cmpfail]\n\t"
193 		RSEQ_INJECT_ASM(4)
194 #ifdef RSEQ_COMPARE_TWICE
195 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
196 		"movq %[v], %%rbx\n\t"
197 		"cmpq %%rbx, %[expectnot]\n\t"
198 		"je %l[error2]\n\t"
199 #endif
200 		"movq %%rbx, %[load]\n\t"
201 		"addq %[voffp], %%rbx\n\t"
202 		"movq (%%rbx), %%rbx\n\t"
203 		/* final store */
204 		"movq %%rbx, %[v]\n\t"
205 		"2:\n\t"
206 		RSEQ_INJECT_ASM(5)
207 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
208 		: /* gcc asm goto does not allow outputs */
209 		: [cpu_id]		"r" (cpu),
210 		  [rseq_abi]		"r" (&__rseq_abi),
211 		  /* final store input */
212 		  [v]			"m" (*v),
213 		  [expectnot]		"r" (expectnot),
214 		  [voffp]		"er" (voffp),
215 		  [load]		"m" (*load)
216 		: "memory", "cc", "rax", "rbx"
217 		  RSEQ_INJECT_CLOBBER
218 		: abort, cmpfail
219 #ifdef RSEQ_COMPARE_TWICE
220 		  , error1, error2
221 #endif
222 	);
223 	return 0;
224 abort:
225 	RSEQ_INJECT_FAILED
226 	return -1;
227 cmpfail:
228 	return 1;
229 #ifdef RSEQ_COMPARE_TWICE
230 error1:
231 	rseq_bug("cpu_id comparison failed");
232 error2:
233 	rseq_bug("expected value comparison failed");
234 #endif
235 }
236 
237 static inline __attribute__((always_inline))
238 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
239 {
240 	RSEQ_INJECT_C(9)
241 
242 	__asm__ __volatile__ goto (
243 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
244 #ifdef RSEQ_COMPARE_TWICE
245 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
246 #endif
247 		/* Start rseq by storing table entry pointer into rseq_cs. */
248 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
249 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
250 		RSEQ_INJECT_ASM(3)
251 #ifdef RSEQ_COMPARE_TWICE
252 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
253 #endif
254 		/* final store */
255 		"addq %[count], %[v]\n\t"
256 		"2:\n\t"
257 		RSEQ_INJECT_ASM(4)
258 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
259 		: /* gcc asm goto does not allow outputs */
260 		: [cpu_id]		"r" (cpu),
261 		  [rseq_abi]		"r" (&__rseq_abi),
262 		  /* final store input */
263 		  [v]			"m" (*v),
264 		  [count]		"er" (count)
265 		: "memory", "cc", "rax"
266 		  RSEQ_INJECT_CLOBBER
267 		: abort
268 #ifdef RSEQ_COMPARE_TWICE
269 		  , error1
270 #endif
271 	);
272 	return 0;
273 abort:
274 	RSEQ_INJECT_FAILED
275 	return -1;
276 #ifdef RSEQ_COMPARE_TWICE
277 error1:
278 	rseq_bug("cpu_id comparison failed");
279 #endif
280 }
281 
282 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
283 
284 /*
285  *   pval = *(ptr+off)
286  *  *pval += inc;
287  */
288 static inline __attribute__((always_inline))
289 int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
290 {
291 	RSEQ_INJECT_C(9)
292 
293 	__asm__ __volatile__ goto (
294 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
295 #ifdef RSEQ_COMPARE_TWICE
296 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
297 #endif
298 		/* Start rseq by storing table entry pointer into rseq_cs. */
299 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
300 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
301 		RSEQ_INJECT_ASM(3)
302 #ifdef RSEQ_COMPARE_TWICE
303 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
304 #endif
305 		/* get p+v */
306 		"movq %[ptr], %%rbx\n\t"
307 		"addq %[off], %%rbx\n\t"
308 		/* get pv */
309 		"movq (%%rbx), %%rcx\n\t"
310 		/* *pv += inc */
311 		"addq %[inc], (%%rcx)\n\t"
312 		"2:\n\t"
313 		RSEQ_INJECT_ASM(4)
314 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
315 		: /* gcc asm goto does not allow outputs */
316 		: [cpu_id]		"r" (cpu),
317 		  [rseq_abi]		"r" (&__rseq_abi),
318 		  /* final store input */
319 		  [ptr]			"m" (*ptr),
320 		  [off]			"er" (off),
321 		  [inc]			"er" (inc)
322 		: "memory", "cc", "rax", "rbx", "rcx"
323 		  RSEQ_INJECT_CLOBBER
324 		: abort
325 #ifdef RSEQ_COMPARE_TWICE
326 		  , error1
327 #endif
328 	);
329 	return 0;
330 abort:
331 	RSEQ_INJECT_FAILED
332 	return -1;
333 #ifdef RSEQ_COMPARE_TWICE
334 error1:
335 	rseq_bug("cpu_id comparison failed");
336 #endif
337 }
338 
339 static inline __attribute__((always_inline))
340 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
341 				 intptr_t *v2, intptr_t newv2,
342 				 intptr_t newv, int cpu)
343 {
344 	RSEQ_INJECT_C(9)
345 
346 	__asm__ __volatile__ goto (
347 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
348 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
349 #ifdef RSEQ_COMPARE_TWICE
350 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
351 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
352 #endif
353 		/* Start rseq by storing table entry pointer into rseq_cs. */
354 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
355 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
356 		RSEQ_INJECT_ASM(3)
357 		"cmpq %[v], %[expect]\n\t"
358 		"jnz %l[cmpfail]\n\t"
359 		RSEQ_INJECT_ASM(4)
360 #ifdef RSEQ_COMPARE_TWICE
361 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
362 		"cmpq %[v], %[expect]\n\t"
363 		"jnz %l[error2]\n\t"
364 #endif
365 		/* try store */
366 		"movq %[newv2], %[v2]\n\t"
367 		RSEQ_INJECT_ASM(5)
368 		/* final store */
369 		"movq %[newv], %[v]\n\t"
370 		"2:\n\t"
371 		RSEQ_INJECT_ASM(6)
372 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
373 		: /* gcc asm goto does not allow outputs */
374 		: [cpu_id]		"r" (cpu),
375 		  [rseq_abi]		"r" (&__rseq_abi),
376 		  /* try store input */
377 		  [v2]			"m" (*v2),
378 		  [newv2]		"r" (newv2),
379 		  /* final store input */
380 		  [v]			"m" (*v),
381 		  [expect]		"r" (expect),
382 		  [newv]		"r" (newv)
383 		: "memory", "cc", "rax"
384 		  RSEQ_INJECT_CLOBBER
385 		: abort, cmpfail
386 #ifdef RSEQ_COMPARE_TWICE
387 		  , error1, error2
388 #endif
389 	);
390 	return 0;
391 abort:
392 	RSEQ_INJECT_FAILED
393 	return -1;
394 cmpfail:
395 	return 1;
396 #ifdef RSEQ_COMPARE_TWICE
397 error1:
398 	rseq_bug("cpu_id comparison failed");
399 error2:
400 	rseq_bug("expected value comparison failed");
401 #endif
402 }
403 
404 /* x86-64 is TSO. */
405 static inline __attribute__((always_inline))
406 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
407 					 intptr_t *v2, intptr_t newv2,
408 					 intptr_t newv, int cpu)
409 {
410 	return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
411 }
412 
413 static inline __attribute__((always_inline))
414 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
415 			      intptr_t *v2, intptr_t expect2,
416 			      intptr_t newv, int cpu)
417 {
418 	RSEQ_INJECT_C(9)
419 
420 	__asm__ __volatile__ goto (
421 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
422 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
423 #ifdef RSEQ_COMPARE_TWICE
424 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
425 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
426 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
427 #endif
428 		/* Start rseq by storing table entry pointer into rseq_cs. */
429 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
430 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
431 		RSEQ_INJECT_ASM(3)
432 		"cmpq %[v], %[expect]\n\t"
433 		"jnz %l[cmpfail]\n\t"
434 		RSEQ_INJECT_ASM(4)
435 		"cmpq %[v2], %[expect2]\n\t"
436 		"jnz %l[cmpfail]\n\t"
437 		RSEQ_INJECT_ASM(5)
438 #ifdef RSEQ_COMPARE_TWICE
439 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
440 		"cmpq %[v], %[expect]\n\t"
441 		"jnz %l[error2]\n\t"
442 		"cmpq %[v2], %[expect2]\n\t"
443 		"jnz %l[error3]\n\t"
444 #endif
445 		/* final store */
446 		"movq %[newv], %[v]\n\t"
447 		"2:\n\t"
448 		RSEQ_INJECT_ASM(6)
449 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
450 		: /* gcc asm goto does not allow outputs */
451 		: [cpu_id]		"r" (cpu),
452 		  [rseq_abi]		"r" (&__rseq_abi),
453 		  /* cmp2 input */
454 		  [v2]			"m" (*v2),
455 		  [expect2]		"r" (expect2),
456 		  /* final store input */
457 		  [v]			"m" (*v),
458 		  [expect]		"r" (expect),
459 		  [newv]		"r" (newv)
460 		: "memory", "cc", "rax"
461 		  RSEQ_INJECT_CLOBBER
462 		: abort, cmpfail
463 #ifdef RSEQ_COMPARE_TWICE
464 		  , error1, error2, error3
465 #endif
466 	);
467 	return 0;
468 abort:
469 	RSEQ_INJECT_FAILED
470 	return -1;
471 cmpfail:
472 	return 1;
473 #ifdef RSEQ_COMPARE_TWICE
474 error1:
475 	rseq_bug("cpu_id comparison failed");
476 error2:
477 	rseq_bug("1st expected value comparison failed");
478 error3:
479 	rseq_bug("2nd expected value comparison failed");
480 #endif
481 }
482 
483 static inline __attribute__((always_inline))
484 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
485 				 void *dst, void *src, size_t len,
486 				 intptr_t newv, int cpu)
487 {
488 	uint64_t rseq_scratch[3];
489 
490 	RSEQ_INJECT_C(9)
491 
492 	__asm__ __volatile__ goto (
493 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
494 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
495 #ifdef RSEQ_COMPARE_TWICE
496 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
497 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
498 #endif
499 		"movq %[src], %[rseq_scratch0]\n\t"
500 		"movq %[dst], %[rseq_scratch1]\n\t"
501 		"movq %[len], %[rseq_scratch2]\n\t"
502 		/* Start rseq by storing table entry pointer into rseq_cs. */
503 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
504 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
505 		RSEQ_INJECT_ASM(3)
506 		"cmpq %[v], %[expect]\n\t"
507 		"jnz 5f\n\t"
508 		RSEQ_INJECT_ASM(4)
509 #ifdef RSEQ_COMPARE_TWICE
510 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
511 		"cmpq %[v], %[expect]\n\t"
512 		"jnz 7f\n\t"
513 #endif
514 		/* try memcpy */
515 		"test %[len], %[len]\n\t" \
516 		"jz 333f\n\t" \
517 		"222:\n\t" \
518 		"movb (%[src]), %%al\n\t" \
519 		"movb %%al, (%[dst])\n\t" \
520 		"inc %[src]\n\t" \
521 		"inc %[dst]\n\t" \
522 		"dec %[len]\n\t" \
523 		"jnz 222b\n\t" \
524 		"333:\n\t" \
525 		RSEQ_INJECT_ASM(5)
526 		/* final store */
527 		"movq %[newv], %[v]\n\t"
528 		"2:\n\t"
529 		RSEQ_INJECT_ASM(6)
530 		/* teardown */
531 		"movq %[rseq_scratch2], %[len]\n\t"
532 		"movq %[rseq_scratch1], %[dst]\n\t"
533 		"movq %[rseq_scratch0], %[src]\n\t"
534 		RSEQ_ASM_DEFINE_ABORT(4,
535 			"movq %[rseq_scratch2], %[len]\n\t"
536 			"movq %[rseq_scratch1], %[dst]\n\t"
537 			"movq %[rseq_scratch0], %[src]\n\t",
538 			abort)
539 		RSEQ_ASM_DEFINE_CMPFAIL(5,
540 			"movq %[rseq_scratch2], %[len]\n\t"
541 			"movq %[rseq_scratch1], %[dst]\n\t"
542 			"movq %[rseq_scratch0], %[src]\n\t",
543 			cmpfail)
544 #ifdef RSEQ_COMPARE_TWICE
545 		RSEQ_ASM_DEFINE_CMPFAIL(6,
546 			"movq %[rseq_scratch2], %[len]\n\t"
547 			"movq %[rseq_scratch1], %[dst]\n\t"
548 			"movq %[rseq_scratch0], %[src]\n\t",
549 			error1)
550 		RSEQ_ASM_DEFINE_CMPFAIL(7,
551 			"movq %[rseq_scratch2], %[len]\n\t"
552 			"movq %[rseq_scratch1], %[dst]\n\t"
553 			"movq %[rseq_scratch0], %[src]\n\t",
554 			error2)
555 #endif
556 		: /* gcc asm goto does not allow outputs */
557 		: [cpu_id]		"r" (cpu),
558 		  [rseq_abi]		"r" (&__rseq_abi),
559 		  /* final store input */
560 		  [v]			"m" (*v),
561 		  [expect]		"r" (expect),
562 		  [newv]		"r" (newv),
563 		  /* try memcpy input */
564 		  [dst]			"r" (dst),
565 		  [src]			"r" (src),
566 		  [len]			"r" (len),
567 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
568 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
569 		  [rseq_scratch2]	"m" (rseq_scratch[2])
570 		: "memory", "cc", "rax"
571 		  RSEQ_INJECT_CLOBBER
572 		: abort, cmpfail
573 #ifdef RSEQ_COMPARE_TWICE
574 		  , error1, error2
575 #endif
576 	);
577 	return 0;
578 abort:
579 	RSEQ_INJECT_FAILED
580 	return -1;
581 cmpfail:
582 	return 1;
583 #ifdef RSEQ_COMPARE_TWICE
584 error1:
585 	rseq_bug("cpu_id comparison failed");
586 error2:
587 	rseq_bug("expected value comparison failed");
588 #endif
589 }
590 
591 /* x86-64 is TSO. */
592 static inline __attribute__((always_inline))
593 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
594 					 void *dst, void *src, size_t len,
595 					 intptr_t newv, int cpu)
596 {
597 	return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
598 					    newv, cpu);
599 }
600 
601 #endif /* !RSEQ_SKIP_FASTPATH */
602 
603 #elif __i386__
604 
605 #define rseq_smp_mb()	\
606 	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
607 #define rseq_smp_rmb()	\
608 	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
609 #define rseq_smp_wmb()	\
610 	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
611 
612 #define rseq_smp_load_acquire(p)					\
613 __extension__ ({							\
614 	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
615 	rseq_smp_mb();							\
616 	____p1;								\
617 })
618 
619 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
620 
621 #define rseq_smp_store_release(p, v)					\
622 do {									\
623 	rseq_smp_mb();							\
624 	RSEQ_WRITE_ONCE(*p, v);						\
625 } while (0)
626 
627 #ifdef RSEQ_SKIP_FASTPATH
628 #include "rseq-skip.h"
629 #else /* !RSEQ_SKIP_FASTPATH */
630 
631 /*
632  * Use eax as scratch register and take memory operands as input to
633  * lessen register pressure. Especially needed when compiling in O0.
634  */
635 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
636 				start_ip, post_commit_offset, abort_ip)	\
637 		".pushsection __rseq_cs, \"aw\"\n\t"			\
638 		".balign 32\n\t"					\
639 		__rseq_str(label) ":\n\t"				\
640 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
641 		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
642 		".popsection\n\t"					\
643 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
644 		".long " __rseq_str(label) "b, 0x0\n\t"			\
645 		".popsection\n\t"
646 
647 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
648 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
649 				(post_commit_ip - start_ip), abort_ip)
650 
651 /*
652  * Exit points of a rseq critical section consist of all instructions outside
653  * of the critical section where a critical section can either branch to or
654  * reach through the normal course of its execution. The abort IP and the
655  * post-commit IP are already part of the __rseq_cs section and should not be
656  * explicitly defined as additional exit points. Knowing all exit points is
657  * useful to assist debuggers stepping over the critical section.
658  */
659 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
660 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
661 		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
662 		".popsection\n\t"
663 
664 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
665 		RSEQ_INJECT_ASM(1)					\
666 		"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"	\
667 		__rseq_str(label) ":\n\t"
668 
669 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
670 		RSEQ_INJECT_ASM(2)					\
671 		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
672 		"jnz " __rseq_str(label) "\n\t"
673 
674 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
675 		".pushsection __rseq_failure, \"ax\"\n\t"		\
676 		/* Disassembler-friendly signature: ud1 <sig>,%edi. */	\
677 		".byte 0x0f, 0xb9, 0x3d\n\t"				\
678 		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
679 		__rseq_str(label) ":\n\t"				\
680 		teardown						\
681 		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
682 		".popsection\n\t"
683 
684 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
685 		".pushsection __rseq_failure, \"ax\"\n\t"		\
686 		__rseq_str(label) ":\n\t"				\
687 		teardown						\
688 		"jmp %l[" __rseq_str(cmpfail_label) "]\n\t"		\
689 		".popsection\n\t"
690 
691 static inline __attribute__((always_inline))
692 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
693 {
694 	RSEQ_INJECT_C(9)
695 
696 	__asm__ __volatile__ goto (
697 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
698 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
699 #ifdef RSEQ_COMPARE_TWICE
700 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
701 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
702 #endif
703 		/* Start rseq by storing table entry pointer into rseq_cs. */
704 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
705 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
706 		RSEQ_INJECT_ASM(3)
707 		"cmpl %[v], %[expect]\n\t"
708 		"jnz %l[cmpfail]\n\t"
709 		RSEQ_INJECT_ASM(4)
710 #ifdef RSEQ_COMPARE_TWICE
711 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
712 		"cmpl %[v], %[expect]\n\t"
713 		"jnz %l[error2]\n\t"
714 #endif
715 		/* final store */
716 		"movl %[newv], %[v]\n\t"
717 		"2:\n\t"
718 		RSEQ_INJECT_ASM(5)
719 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
720 		: /* gcc asm goto does not allow outputs */
721 		: [cpu_id]		"r" (cpu),
722 		  [rseq_abi]		"r" (&__rseq_abi),
723 		  [v]			"m" (*v),
724 		  [expect]		"r" (expect),
725 		  [newv]		"r" (newv)
726 		: "memory", "cc", "eax"
727 		  RSEQ_INJECT_CLOBBER
728 		: abort, cmpfail
729 #ifdef RSEQ_COMPARE_TWICE
730 		  , error1, error2
731 #endif
732 	);
733 	return 0;
734 abort:
735 	RSEQ_INJECT_FAILED
736 	return -1;
737 cmpfail:
738 	return 1;
739 #ifdef RSEQ_COMPARE_TWICE
740 error1:
741 	rseq_bug("cpu_id comparison failed");
742 error2:
743 	rseq_bug("expected value comparison failed");
744 #endif
745 }
746 
747 /*
748  * Compare @v against @expectnot. When it does _not_ match, load @v
749  * into @load, and store the content of *@v + voffp into @v.
750  */
751 static inline __attribute__((always_inline))
752 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
753 			       off_t voffp, intptr_t *load, int cpu)
754 {
755 	RSEQ_INJECT_C(9)
756 
757 	__asm__ __volatile__ goto (
758 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
759 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
760 #ifdef RSEQ_COMPARE_TWICE
761 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
762 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
763 #endif
764 		/* Start rseq by storing table entry pointer into rseq_cs. */
765 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
766 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
767 		RSEQ_INJECT_ASM(3)
768 		"movl %[v], %%ebx\n\t"
769 		"cmpl %%ebx, %[expectnot]\n\t"
770 		"je %l[cmpfail]\n\t"
771 		RSEQ_INJECT_ASM(4)
772 #ifdef RSEQ_COMPARE_TWICE
773 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
774 		"movl %[v], %%ebx\n\t"
775 		"cmpl %%ebx, %[expectnot]\n\t"
776 		"je %l[error2]\n\t"
777 #endif
778 		"movl %%ebx, %[load]\n\t"
779 		"addl %[voffp], %%ebx\n\t"
780 		"movl (%%ebx), %%ebx\n\t"
781 		/* final store */
782 		"movl %%ebx, %[v]\n\t"
783 		"2:\n\t"
784 		RSEQ_INJECT_ASM(5)
785 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
786 		: /* gcc asm goto does not allow outputs */
787 		: [cpu_id]		"r" (cpu),
788 		  [rseq_abi]		"r" (&__rseq_abi),
789 		  /* final store input */
790 		  [v]			"m" (*v),
791 		  [expectnot]		"r" (expectnot),
792 		  [voffp]		"ir" (voffp),
793 		  [load]		"m" (*load)
794 		: "memory", "cc", "eax", "ebx"
795 		  RSEQ_INJECT_CLOBBER
796 		: abort, cmpfail
797 #ifdef RSEQ_COMPARE_TWICE
798 		  , error1, error2
799 #endif
800 	);
801 	return 0;
802 abort:
803 	RSEQ_INJECT_FAILED
804 	return -1;
805 cmpfail:
806 	return 1;
807 #ifdef RSEQ_COMPARE_TWICE
808 error1:
809 	rseq_bug("cpu_id comparison failed");
810 error2:
811 	rseq_bug("expected value comparison failed");
812 #endif
813 }
814 
815 static inline __attribute__((always_inline))
816 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
817 {
818 	RSEQ_INJECT_C(9)
819 
820 	__asm__ __volatile__ goto (
821 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
822 #ifdef RSEQ_COMPARE_TWICE
823 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
824 #endif
825 		/* Start rseq by storing table entry pointer into rseq_cs. */
826 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
827 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
828 		RSEQ_INJECT_ASM(3)
829 #ifdef RSEQ_COMPARE_TWICE
830 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
831 #endif
832 		/* final store */
833 		"addl %[count], %[v]\n\t"
834 		"2:\n\t"
835 		RSEQ_INJECT_ASM(4)
836 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
837 		: /* gcc asm goto does not allow outputs */
838 		: [cpu_id]		"r" (cpu),
839 		  [rseq_abi]		"r" (&__rseq_abi),
840 		  /* final store input */
841 		  [v]			"m" (*v),
842 		  [count]		"ir" (count)
843 		: "memory", "cc", "eax"
844 		  RSEQ_INJECT_CLOBBER
845 		: abort
846 #ifdef RSEQ_COMPARE_TWICE
847 		  , error1
848 #endif
849 	);
850 	return 0;
851 abort:
852 	RSEQ_INJECT_FAILED
853 	return -1;
854 #ifdef RSEQ_COMPARE_TWICE
855 error1:
856 	rseq_bug("cpu_id comparison failed");
857 #endif
858 }
859 
860 static inline __attribute__((always_inline))
861 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
862 				 intptr_t *v2, intptr_t newv2,
863 				 intptr_t newv, int cpu)
864 {
865 	RSEQ_INJECT_C(9)
866 
867 	__asm__ __volatile__ goto (
868 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
869 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
870 #ifdef RSEQ_COMPARE_TWICE
871 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
872 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
873 #endif
874 		/* Start rseq by storing table entry pointer into rseq_cs. */
875 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
876 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
877 		RSEQ_INJECT_ASM(3)
878 		"cmpl %[v], %[expect]\n\t"
879 		"jnz %l[cmpfail]\n\t"
880 		RSEQ_INJECT_ASM(4)
881 #ifdef RSEQ_COMPARE_TWICE
882 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
883 		"cmpl %[v], %[expect]\n\t"
884 		"jnz %l[error2]\n\t"
885 #endif
886 		/* try store */
887 		"movl %[newv2], %%eax\n\t"
888 		"movl %%eax, %[v2]\n\t"
889 		RSEQ_INJECT_ASM(5)
890 		/* final store */
891 		"movl %[newv], %[v]\n\t"
892 		"2:\n\t"
893 		RSEQ_INJECT_ASM(6)
894 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
895 		: /* gcc asm goto does not allow outputs */
896 		: [cpu_id]		"r" (cpu),
897 		  [rseq_abi]		"r" (&__rseq_abi),
898 		  /* try store input */
899 		  [v2]			"m" (*v2),
900 		  [newv2]		"m" (newv2),
901 		  /* final store input */
902 		  [v]			"m" (*v),
903 		  [expect]		"r" (expect),
904 		  [newv]		"r" (newv)
905 		: "memory", "cc", "eax"
906 		  RSEQ_INJECT_CLOBBER
907 		: abort, cmpfail
908 #ifdef RSEQ_COMPARE_TWICE
909 		  , error1, error2
910 #endif
911 	);
912 	return 0;
913 abort:
914 	RSEQ_INJECT_FAILED
915 	return -1;
916 cmpfail:
917 	return 1;
918 #ifdef RSEQ_COMPARE_TWICE
919 error1:
920 	rseq_bug("cpu_id comparison failed");
921 error2:
922 	rseq_bug("expected value comparison failed");
923 #endif
924 }
925 
926 static inline __attribute__((always_inline))
927 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
928 					 intptr_t *v2, intptr_t newv2,
929 					 intptr_t newv, int cpu)
930 {
931 	RSEQ_INJECT_C(9)
932 
933 	__asm__ __volatile__ goto (
934 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
935 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
936 #ifdef RSEQ_COMPARE_TWICE
937 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
938 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
939 #endif
940 		/* Start rseq by storing table entry pointer into rseq_cs. */
941 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
942 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
943 		RSEQ_INJECT_ASM(3)
944 		"movl %[expect], %%eax\n\t"
945 		"cmpl %[v], %%eax\n\t"
946 		"jnz %l[cmpfail]\n\t"
947 		RSEQ_INJECT_ASM(4)
948 #ifdef RSEQ_COMPARE_TWICE
949 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
950 		"movl %[expect], %%eax\n\t"
951 		"cmpl %[v], %%eax\n\t"
952 		"jnz %l[error2]\n\t"
953 #endif
954 		/* try store */
955 		"movl %[newv2], %[v2]\n\t"
956 		RSEQ_INJECT_ASM(5)
957 		"lock; addl $0,-128(%%esp)\n\t"
958 		/* final store */
959 		"movl %[newv], %[v]\n\t"
960 		"2:\n\t"
961 		RSEQ_INJECT_ASM(6)
962 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
963 		: /* gcc asm goto does not allow outputs */
964 		: [cpu_id]		"r" (cpu),
965 		  [rseq_abi]		"r" (&__rseq_abi),
966 		  /* try store input */
967 		  [v2]			"m" (*v2),
968 		  [newv2]		"r" (newv2),
969 		  /* final store input */
970 		  [v]			"m" (*v),
971 		  [expect]		"m" (expect),
972 		  [newv]		"r" (newv)
973 		: "memory", "cc", "eax"
974 		  RSEQ_INJECT_CLOBBER
975 		: abort, cmpfail
976 #ifdef RSEQ_COMPARE_TWICE
977 		  , error1, error2
978 #endif
979 	);
980 	return 0;
981 abort:
982 	RSEQ_INJECT_FAILED
983 	return -1;
984 cmpfail:
985 	return 1;
986 #ifdef RSEQ_COMPARE_TWICE
987 error1:
988 	rseq_bug("cpu_id comparison failed");
989 error2:
990 	rseq_bug("expected value comparison failed");
991 #endif
992 
993 }
994 
995 static inline __attribute__((always_inline))
996 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
997 			      intptr_t *v2, intptr_t expect2,
998 			      intptr_t newv, int cpu)
999 {
1000 	RSEQ_INJECT_C(9)
1001 
1002 	__asm__ __volatile__ goto (
1003 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1004 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1005 #ifdef RSEQ_COMPARE_TWICE
1006 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1007 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1008 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1009 #endif
1010 		/* Start rseq by storing table entry pointer into rseq_cs. */
1011 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1012 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1013 		RSEQ_INJECT_ASM(3)
1014 		"cmpl %[v], %[expect]\n\t"
1015 		"jnz %l[cmpfail]\n\t"
1016 		RSEQ_INJECT_ASM(4)
1017 		"cmpl %[expect2], %[v2]\n\t"
1018 		"jnz %l[cmpfail]\n\t"
1019 		RSEQ_INJECT_ASM(5)
1020 #ifdef RSEQ_COMPARE_TWICE
1021 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1022 		"cmpl %[v], %[expect]\n\t"
1023 		"jnz %l[error2]\n\t"
1024 		"cmpl %[expect2], %[v2]\n\t"
1025 		"jnz %l[error3]\n\t"
1026 #endif
1027 		"movl %[newv], %%eax\n\t"
1028 		/* final store */
1029 		"movl %%eax, %[v]\n\t"
1030 		"2:\n\t"
1031 		RSEQ_INJECT_ASM(6)
1032 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1033 		: /* gcc asm goto does not allow outputs */
1034 		: [cpu_id]		"r" (cpu),
1035 		  [rseq_abi]		"r" (&__rseq_abi),
1036 		  /* cmp2 input */
1037 		  [v2]			"m" (*v2),
1038 		  [expect2]		"r" (expect2),
1039 		  /* final store input */
1040 		  [v]			"m" (*v),
1041 		  [expect]		"r" (expect),
1042 		  [newv]		"m" (newv)
1043 		: "memory", "cc", "eax"
1044 		  RSEQ_INJECT_CLOBBER
1045 		: abort, cmpfail
1046 #ifdef RSEQ_COMPARE_TWICE
1047 		  , error1, error2, error3
1048 #endif
1049 	);
1050 	return 0;
1051 abort:
1052 	RSEQ_INJECT_FAILED
1053 	return -1;
1054 cmpfail:
1055 	return 1;
1056 #ifdef RSEQ_COMPARE_TWICE
1057 error1:
1058 	rseq_bug("cpu_id comparison failed");
1059 error2:
1060 	rseq_bug("1st expected value comparison failed");
1061 error3:
1062 	rseq_bug("2nd expected value comparison failed");
1063 #endif
1064 }
1065 
1066 /* TODO: implement a faster memcpy. */
1067 static inline __attribute__((always_inline))
1068 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1069 				 void *dst, void *src, size_t len,
1070 				 intptr_t newv, int cpu)
1071 {
1072 	uint32_t rseq_scratch[3];
1073 
1074 	RSEQ_INJECT_C(9)
1075 
1076 	__asm__ __volatile__ goto (
1077 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1078 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1079 #ifdef RSEQ_COMPARE_TWICE
1080 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1081 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1082 #endif
1083 		"movl %[src], %[rseq_scratch0]\n\t"
1084 		"movl %[dst], %[rseq_scratch1]\n\t"
1085 		"movl %[len], %[rseq_scratch2]\n\t"
1086 		/* Start rseq by storing table entry pointer into rseq_cs. */
1087 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1088 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1089 		RSEQ_INJECT_ASM(3)
1090 		"movl %[expect], %%eax\n\t"
1091 		"cmpl %%eax, %[v]\n\t"
1092 		"jnz 5f\n\t"
1093 		RSEQ_INJECT_ASM(4)
1094 #ifdef RSEQ_COMPARE_TWICE
1095 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1096 		"movl %[expect], %%eax\n\t"
1097 		"cmpl %%eax, %[v]\n\t"
1098 		"jnz 7f\n\t"
1099 #endif
1100 		/* try memcpy */
1101 		"test %[len], %[len]\n\t" \
1102 		"jz 333f\n\t" \
1103 		"222:\n\t" \
1104 		"movb (%[src]), %%al\n\t" \
1105 		"movb %%al, (%[dst])\n\t" \
1106 		"inc %[src]\n\t" \
1107 		"inc %[dst]\n\t" \
1108 		"dec %[len]\n\t" \
1109 		"jnz 222b\n\t" \
1110 		"333:\n\t" \
1111 		RSEQ_INJECT_ASM(5)
1112 		"movl %[newv], %%eax\n\t"
1113 		/* final store */
1114 		"movl %%eax, %[v]\n\t"
1115 		"2:\n\t"
1116 		RSEQ_INJECT_ASM(6)
1117 		/* teardown */
1118 		"movl %[rseq_scratch2], %[len]\n\t"
1119 		"movl %[rseq_scratch1], %[dst]\n\t"
1120 		"movl %[rseq_scratch0], %[src]\n\t"
1121 		RSEQ_ASM_DEFINE_ABORT(4,
1122 			"movl %[rseq_scratch2], %[len]\n\t"
1123 			"movl %[rseq_scratch1], %[dst]\n\t"
1124 			"movl %[rseq_scratch0], %[src]\n\t",
1125 			abort)
1126 		RSEQ_ASM_DEFINE_CMPFAIL(5,
1127 			"movl %[rseq_scratch2], %[len]\n\t"
1128 			"movl %[rseq_scratch1], %[dst]\n\t"
1129 			"movl %[rseq_scratch0], %[src]\n\t",
1130 			cmpfail)
1131 #ifdef RSEQ_COMPARE_TWICE
1132 		RSEQ_ASM_DEFINE_CMPFAIL(6,
1133 			"movl %[rseq_scratch2], %[len]\n\t"
1134 			"movl %[rseq_scratch1], %[dst]\n\t"
1135 			"movl %[rseq_scratch0], %[src]\n\t",
1136 			error1)
1137 		RSEQ_ASM_DEFINE_CMPFAIL(7,
1138 			"movl %[rseq_scratch2], %[len]\n\t"
1139 			"movl %[rseq_scratch1], %[dst]\n\t"
1140 			"movl %[rseq_scratch0], %[src]\n\t",
1141 			error2)
1142 #endif
1143 		: /* gcc asm goto does not allow outputs */
1144 		: [cpu_id]		"r" (cpu),
1145 		  [rseq_abi]		"r" (&__rseq_abi),
1146 		  /* final store input */
1147 		  [v]			"m" (*v),
1148 		  [expect]		"m" (expect),
1149 		  [newv]		"m" (newv),
1150 		  /* try memcpy input */
1151 		  [dst]			"r" (dst),
1152 		  [src]			"r" (src),
1153 		  [len]			"r" (len),
1154 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
1155 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
1156 		  [rseq_scratch2]	"m" (rseq_scratch[2])
1157 		: "memory", "cc", "eax"
1158 		  RSEQ_INJECT_CLOBBER
1159 		: abort, cmpfail
1160 #ifdef RSEQ_COMPARE_TWICE
1161 		  , error1, error2
1162 #endif
1163 	);
1164 	return 0;
1165 abort:
1166 	RSEQ_INJECT_FAILED
1167 	return -1;
1168 cmpfail:
1169 	return 1;
1170 #ifdef RSEQ_COMPARE_TWICE
1171 error1:
1172 	rseq_bug("cpu_id comparison failed");
1173 error2:
1174 	rseq_bug("expected value comparison failed");
1175 #endif
1176 }
1177 
1178 /* TODO: implement a faster memcpy. */
1179 static inline __attribute__((always_inline))
1180 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1181 					 void *dst, void *src, size_t len,
1182 					 intptr_t newv, int cpu)
1183 {
1184 	uint32_t rseq_scratch[3];
1185 
1186 	RSEQ_INJECT_C(9)
1187 
1188 	__asm__ __volatile__ goto (
1189 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1190 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1191 #ifdef RSEQ_COMPARE_TWICE
1192 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1193 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1194 #endif
1195 		"movl %[src], %[rseq_scratch0]\n\t"
1196 		"movl %[dst], %[rseq_scratch1]\n\t"
1197 		"movl %[len], %[rseq_scratch2]\n\t"
1198 		/* Start rseq by storing table entry pointer into rseq_cs. */
1199 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1200 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1201 		RSEQ_INJECT_ASM(3)
1202 		"movl %[expect], %%eax\n\t"
1203 		"cmpl %%eax, %[v]\n\t"
1204 		"jnz 5f\n\t"
1205 		RSEQ_INJECT_ASM(4)
1206 #ifdef RSEQ_COMPARE_TWICE
1207 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1208 		"movl %[expect], %%eax\n\t"
1209 		"cmpl %%eax, %[v]\n\t"
1210 		"jnz 7f\n\t"
1211 #endif
1212 		/* try memcpy */
1213 		"test %[len], %[len]\n\t" \
1214 		"jz 333f\n\t" \
1215 		"222:\n\t" \
1216 		"movb (%[src]), %%al\n\t" \
1217 		"movb %%al, (%[dst])\n\t" \
1218 		"inc %[src]\n\t" \
1219 		"inc %[dst]\n\t" \
1220 		"dec %[len]\n\t" \
1221 		"jnz 222b\n\t" \
1222 		"333:\n\t" \
1223 		RSEQ_INJECT_ASM(5)
1224 		"lock; addl $0,-128(%%esp)\n\t"
1225 		"movl %[newv], %%eax\n\t"
1226 		/* final store */
1227 		"movl %%eax, %[v]\n\t"
1228 		"2:\n\t"
1229 		RSEQ_INJECT_ASM(6)
1230 		/* teardown */
1231 		"movl %[rseq_scratch2], %[len]\n\t"
1232 		"movl %[rseq_scratch1], %[dst]\n\t"
1233 		"movl %[rseq_scratch0], %[src]\n\t"
1234 		RSEQ_ASM_DEFINE_ABORT(4,
1235 			"movl %[rseq_scratch2], %[len]\n\t"
1236 			"movl %[rseq_scratch1], %[dst]\n\t"
1237 			"movl %[rseq_scratch0], %[src]\n\t",
1238 			abort)
1239 		RSEQ_ASM_DEFINE_CMPFAIL(5,
1240 			"movl %[rseq_scratch2], %[len]\n\t"
1241 			"movl %[rseq_scratch1], %[dst]\n\t"
1242 			"movl %[rseq_scratch0], %[src]\n\t",
1243 			cmpfail)
1244 #ifdef RSEQ_COMPARE_TWICE
1245 		RSEQ_ASM_DEFINE_CMPFAIL(6,
1246 			"movl %[rseq_scratch2], %[len]\n\t"
1247 			"movl %[rseq_scratch1], %[dst]\n\t"
1248 			"movl %[rseq_scratch0], %[src]\n\t",
1249 			error1)
1250 		RSEQ_ASM_DEFINE_CMPFAIL(7,
1251 			"movl %[rseq_scratch2], %[len]\n\t"
1252 			"movl %[rseq_scratch1], %[dst]\n\t"
1253 			"movl %[rseq_scratch0], %[src]\n\t",
1254 			error2)
1255 #endif
1256 		: /* gcc asm goto does not allow outputs */
1257 		: [cpu_id]		"r" (cpu),
1258 		  [rseq_abi]		"r" (&__rseq_abi),
1259 		  /* final store input */
1260 		  [v]			"m" (*v),
1261 		  [expect]		"m" (expect),
1262 		  [newv]		"m" (newv),
1263 		  /* try memcpy input */
1264 		  [dst]			"r" (dst),
1265 		  [src]			"r" (src),
1266 		  [len]			"r" (len),
1267 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
1268 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
1269 		  [rseq_scratch2]	"m" (rseq_scratch[2])
1270 		: "memory", "cc", "eax"
1271 		  RSEQ_INJECT_CLOBBER
1272 		: abort, cmpfail
1273 #ifdef RSEQ_COMPARE_TWICE
1274 		  , error1, error2
1275 #endif
1276 	);
1277 	return 0;
1278 abort:
1279 	RSEQ_INJECT_FAILED
1280 	return -1;
1281 cmpfail:
1282 	return 1;
1283 #ifdef RSEQ_COMPARE_TWICE
1284 error1:
1285 	rseq_bug("cpu_id comparison failed");
1286 error2:
1287 	rseq_bug("expected value comparison failed");
1288 #endif
1289 }
1290 
1291 #endif /* !RSEQ_SKIP_FASTPATH */
1292 
1293 #endif
1294