1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * rseq-x86.h
4  *
5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6  */
7 
8 #include <stdint.h>
9 
10 /*
11  * RSEQ_SIG is used with the following reserved undefined instructions, which
12  * trap in user-space:
13  *
14  * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
15  * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
16  */
17 #define RSEQ_SIG	0x53053053
18 
19 /*
20  * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
21  * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
22  * address through a "r" input operand.
23  */
24 
25 /* Offset of cpu_id and rseq_cs fields in struct rseq. */
26 #define RSEQ_CPU_ID_OFFSET	4
27 #define RSEQ_CS_OFFSET		8
28 
29 #ifdef __x86_64__
30 
31 #define RSEQ_ASM_TP_SEGMENT	%%fs
32 
33 #define rseq_smp_mb()	\
34 	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
35 #define rseq_smp_rmb()	rseq_barrier()
36 #define rseq_smp_wmb()	rseq_barrier()
37 
38 #define rseq_smp_load_acquire(p)					\
39 __extension__ ({							\
40 	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
41 	rseq_barrier();							\
42 	____p1;								\
43 })
44 
45 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
46 
47 #define rseq_smp_store_release(p, v)					\
48 do {									\
49 	rseq_barrier();							\
50 	RSEQ_WRITE_ONCE(*p, v);						\
51 } while (0)
52 
53 #ifdef RSEQ_SKIP_FASTPATH
54 #include "rseq-skip.h"
55 #else /* !RSEQ_SKIP_FASTPATH */
56 
57 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
58 				start_ip, post_commit_offset, abort_ip)	\
59 		".pushsection __rseq_cs, \"aw\"\n\t"			\
60 		".balign 32\n\t"					\
61 		__rseq_str(label) ":\n\t"				\
62 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
63 		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
64 		".popsection\n\t"					\
65 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
66 		".quad " __rseq_str(label) "b\n\t"			\
67 		".popsection\n\t"
68 
69 
70 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
71 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
72 				(post_commit_ip - start_ip), abort_ip)
73 
74 /*
75  * Exit points of a rseq critical section consist of all instructions outside
76  * of the critical section where a critical section can either branch to or
77  * reach through the normal course of its execution. The abort IP and the
78  * post-commit IP are already part of the __rseq_cs section and should not be
79  * explicitly defined as additional exit points. Knowing all exit points is
80  * useful to assist debuggers stepping over the critical section.
81  */
82 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
83 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
84 		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
85 		".popsection\n\t"
86 
87 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
88 		RSEQ_INJECT_ASM(1)					\
89 		"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"	\
90 		"movq %%rax, " __rseq_str(rseq_cs) "\n\t"		\
91 		__rseq_str(label) ":\n\t"
92 
93 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
94 		RSEQ_INJECT_ASM(2)					\
95 		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
96 		"jnz " __rseq_str(label) "\n\t"
97 
98 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
99 		".pushsection __rseq_failure, \"ax\"\n\t"		\
100 		/* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
101 		".byte 0x0f, 0xb9, 0x3d\n\t"				\
102 		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
103 		__rseq_str(label) ":\n\t"				\
104 		teardown						\
105 		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
106 		".popsection\n\t"
107 
108 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
109 		".pushsection __rseq_failure, \"ax\"\n\t"		\
110 		__rseq_str(label) ":\n\t"				\
111 		teardown						\
112 		"jmp %l[" __rseq_str(cmpfail_label) "]\n\t"		\
113 		".popsection\n\t"
114 
115 static inline __attribute__((always_inline))
116 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
117 {
118 	RSEQ_INJECT_C(9)
119 
120 	__asm__ __volatile__ goto (
121 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
122 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
123 #ifdef RSEQ_COMPARE_TWICE
124 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
125 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
126 #endif
127 		/* Start rseq by storing table entry pointer into rseq_cs. */
128 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
129 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
130 		RSEQ_INJECT_ASM(3)
131 		"cmpq %[v], %[expect]\n\t"
132 		"jnz %l[cmpfail]\n\t"
133 		RSEQ_INJECT_ASM(4)
134 #ifdef RSEQ_COMPARE_TWICE
135 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
136 		"cmpq %[v], %[expect]\n\t"
137 		"jnz %l[error2]\n\t"
138 #endif
139 		/* final store */
140 		"movq %[newv], %[v]\n\t"
141 		"2:\n\t"
142 		RSEQ_INJECT_ASM(5)
143 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
144 		: /* gcc asm goto does not allow outputs */
145 		: [cpu_id]		"r" (cpu),
146 		  [rseq_offset]		"r" (rseq_offset),
147 		  [v]			"m" (*v),
148 		  [expect]		"r" (expect),
149 		  [newv]		"r" (newv)
150 		: "memory", "cc", "rax"
151 		  RSEQ_INJECT_CLOBBER
152 		: abort, cmpfail
153 #ifdef RSEQ_COMPARE_TWICE
154 		  , error1, error2
155 #endif
156 	);
157 	rseq_after_asm_goto();
158 	return 0;
159 abort:
160 	rseq_after_asm_goto();
161 	RSEQ_INJECT_FAILED
162 	return -1;
163 cmpfail:
164 	rseq_after_asm_goto();
165 	return 1;
166 #ifdef RSEQ_COMPARE_TWICE
167 error1:
168 	rseq_after_asm_goto();
169 	rseq_bug("cpu_id comparison failed");
170 error2:
171 	rseq_after_asm_goto();
172 	rseq_bug("expected value comparison failed");
173 #endif
174 }
175 
176 /*
177  * Compare @v against @expectnot. When it does _not_ match, load @v
178  * into @load, and store the content of *@v + voffp into @v.
179  */
180 static inline __attribute__((always_inline))
181 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
182 			       long voffp, intptr_t *load, int cpu)
183 {
184 	RSEQ_INJECT_C(9)
185 
186 	__asm__ __volatile__ goto (
187 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
188 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
189 #ifdef RSEQ_COMPARE_TWICE
190 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
191 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
192 #endif
193 		/* Start rseq by storing table entry pointer into rseq_cs. */
194 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
195 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
196 		RSEQ_INJECT_ASM(3)
197 		"movq %[v], %%rbx\n\t"
198 		"cmpq %%rbx, %[expectnot]\n\t"
199 		"je %l[cmpfail]\n\t"
200 		RSEQ_INJECT_ASM(4)
201 #ifdef RSEQ_COMPARE_TWICE
202 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
203 		"movq %[v], %%rbx\n\t"
204 		"cmpq %%rbx, %[expectnot]\n\t"
205 		"je %l[error2]\n\t"
206 #endif
207 		"movq %%rbx, %[load]\n\t"
208 		"addq %[voffp], %%rbx\n\t"
209 		"movq (%%rbx), %%rbx\n\t"
210 		/* final store */
211 		"movq %%rbx, %[v]\n\t"
212 		"2:\n\t"
213 		RSEQ_INJECT_ASM(5)
214 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
215 		: /* gcc asm goto does not allow outputs */
216 		: [cpu_id]		"r" (cpu),
217 		  [rseq_offset]		"r" (rseq_offset),
218 		  /* final store input */
219 		  [v]			"m" (*v),
220 		  [expectnot]		"r" (expectnot),
221 		  [voffp]		"er" (voffp),
222 		  [load]		"m" (*load)
223 		: "memory", "cc", "rax", "rbx"
224 		  RSEQ_INJECT_CLOBBER
225 		: abort, cmpfail
226 #ifdef RSEQ_COMPARE_TWICE
227 		  , error1, error2
228 #endif
229 	);
230 	rseq_after_asm_goto();
231 	return 0;
232 abort:
233 	rseq_after_asm_goto();
234 	RSEQ_INJECT_FAILED
235 	return -1;
236 cmpfail:
237 	rseq_after_asm_goto();
238 	return 1;
239 #ifdef RSEQ_COMPARE_TWICE
240 error1:
241 	rseq_after_asm_goto();
242 	rseq_bug("cpu_id comparison failed");
243 error2:
244 	rseq_after_asm_goto();
245 	rseq_bug("expected value comparison failed");
246 #endif
247 }
248 
249 static inline __attribute__((always_inline))
250 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
251 {
252 	RSEQ_INJECT_C(9)
253 
254 	__asm__ __volatile__ goto (
255 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
256 #ifdef RSEQ_COMPARE_TWICE
257 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
258 #endif
259 		/* Start rseq by storing table entry pointer into rseq_cs. */
260 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
261 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
262 		RSEQ_INJECT_ASM(3)
263 #ifdef RSEQ_COMPARE_TWICE
264 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
265 #endif
266 		/* final store */
267 		"addq %[count], %[v]\n\t"
268 		"2:\n\t"
269 		RSEQ_INJECT_ASM(4)
270 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
271 		: /* gcc asm goto does not allow outputs */
272 		: [cpu_id]		"r" (cpu),
273 		  [rseq_offset]		"r" (rseq_offset),
274 		  /* final store input */
275 		  [v]			"m" (*v),
276 		  [count]		"er" (count)
277 		: "memory", "cc", "rax"
278 		  RSEQ_INJECT_CLOBBER
279 		: abort
280 #ifdef RSEQ_COMPARE_TWICE
281 		  , error1
282 #endif
283 	);
284 	rseq_after_asm_goto();
285 	return 0;
286 abort:
287 	rseq_after_asm_goto();
288 	RSEQ_INJECT_FAILED
289 	return -1;
290 #ifdef RSEQ_COMPARE_TWICE
291 error1:
292 	rseq_after_asm_goto();
293 	rseq_bug("cpu_id comparison failed");
294 #endif
295 }
296 
297 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
298 
299 /*
300  *   pval = *(ptr+off)
301  *  *pval += inc;
302  */
303 static inline __attribute__((always_inline))
304 int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
305 {
306 	RSEQ_INJECT_C(9)
307 
308 	__asm__ __volatile__ goto (
309 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
310 #ifdef RSEQ_COMPARE_TWICE
311 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
312 #endif
313 		/* Start rseq by storing table entry pointer into rseq_cs. */
314 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
315 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
316 		RSEQ_INJECT_ASM(3)
317 #ifdef RSEQ_COMPARE_TWICE
318 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
319 #endif
320 		/* get p+v */
321 		"movq %[ptr], %%rbx\n\t"
322 		"addq %[off], %%rbx\n\t"
323 		/* get pv */
324 		"movq (%%rbx), %%rcx\n\t"
325 		/* *pv += inc */
326 		"addq %[inc], (%%rcx)\n\t"
327 		"2:\n\t"
328 		RSEQ_INJECT_ASM(4)
329 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
330 		: /* gcc asm goto does not allow outputs */
331 		: [cpu_id]		"r" (cpu),
332 		  [rseq_offset]		"r" (rseq_offset),
333 		  /* final store input */
334 		  [ptr]			"m" (*ptr),
335 		  [off]			"er" (off),
336 		  [inc]			"er" (inc)
337 		: "memory", "cc", "rax", "rbx", "rcx"
338 		  RSEQ_INJECT_CLOBBER
339 		: abort
340 #ifdef RSEQ_COMPARE_TWICE
341 		  , error1
342 #endif
343 	);
344 	return 0;
345 abort:
346 	RSEQ_INJECT_FAILED
347 	return -1;
348 #ifdef RSEQ_COMPARE_TWICE
349 error1:
350 	rseq_bug("cpu_id comparison failed");
351 #endif
352 }
353 
354 static inline __attribute__((always_inline))
355 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
356 				 intptr_t *v2, intptr_t newv2,
357 				 intptr_t newv, int cpu)
358 {
359 	RSEQ_INJECT_C(9)
360 
361 	__asm__ __volatile__ goto (
362 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
363 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
364 #ifdef RSEQ_COMPARE_TWICE
365 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
366 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
367 #endif
368 		/* Start rseq by storing table entry pointer into rseq_cs. */
369 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
370 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
371 		RSEQ_INJECT_ASM(3)
372 		"cmpq %[v], %[expect]\n\t"
373 		"jnz %l[cmpfail]\n\t"
374 		RSEQ_INJECT_ASM(4)
375 #ifdef RSEQ_COMPARE_TWICE
376 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
377 		"cmpq %[v], %[expect]\n\t"
378 		"jnz %l[error2]\n\t"
379 #endif
380 		/* try store */
381 		"movq %[newv2], %[v2]\n\t"
382 		RSEQ_INJECT_ASM(5)
383 		/* final store */
384 		"movq %[newv], %[v]\n\t"
385 		"2:\n\t"
386 		RSEQ_INJECT_ASM(6)
387 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
388 		: /* gcc asm goto does not allow outputs */
389 		: [cpu_id]		"r" (cpu),
390 		  [rseq_offset]		"r" (rseq_offset),
391 		  /* try store input */
392 		  [v2]			"m" (*v2),
393 		  [newv2]		"r" (newv2),
394 		  /* final store input */
395 		  [v]			"m" (*v),
396 		  [expect]		"r" (expect),
397 		  [newv]		"r" (newv)
398 		: "memory", "cc", "rax"
399 		  RSEQ_INJECT_CLOBBER
400 		: abort, cmpfail
401 #ifdef RSEQ_COMPARE_TWICE
402 		  , error1, error2
403 #endif
404 	);
405 	rseq_after_asm_goto();
406 	return 0;
407 abort:
408 	rseq_after_asm_goto();
409 	RSEQ_INJECT_FAILED
410 	return -1;
411 cmpfail:
412 	rseq_after_asm_goto();
413 	return 1;
414 #ifdef RSEQ_COMPARE_TWICE
415 error1:
416 	rseq_after_asm_goto();
417 	rseq_bug("cpu_id comparison failed");
418 error2:
419 	rseq_after_asm_goto();
420 	rseq_bug("expected value comparison failed");
421 #endif
422 }
423 
424 /* x86-64 is TSO. */
425 static inline __attribute__((always_inline))
426 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
427 					 intptr_t *v2, intptr_t newv2,
428 					 intptr_t newv, int cpu)
429 {
430 	return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
431 }
432 
433 static inline __attribute__((always_inline))
434 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
435 			      intptr_t *v2, intptr_t expect2,
436 			      intptr_t newv, int cpu)
437 {
438 	RSEQ_INJECT_C(9)
439 
440 	__asm__ __volatile__ goto (
441 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
442 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
443 #ifdef RSEQ_COMPARE_TWICE
444 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
445 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
446 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
447 #endif
448 		/* Start rseq by storing table entry pointer into rseq_cs. */
449 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
450 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
451 		RSEQ_INJECT_ASM(3)
452 		"cmpq %[v], %[expect]\n\t"
453 		"jnz %l[cmpfail]\n\t"
454 		RSEQ_INJECT_ASM(4)
455 		"cmpq %[v2], %[expect2]\n\t"
456 		"jnz %l[cmpfail]\n\t"
457 		RSEQ_INJECT_ASM(5)
458 #ifdef RSEQ_COMPARE_TWICE
459 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
460 		"cmpq %[v], %[expect]\n\t"
461 		"jnz %l[error2]\n\t"
462 		"cmpq %[v2], %[expect2]\n\t"
463 		"jnz %l[error3]\n\t"
464 #endif
465 		/* final store */
466 		"movq %[newv], %[v]\n\t"
467 		"2:\n\t"
468 		RSEQ_INJECT_ASM(6)
469 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
470 		: /* gcc asm goto does not allow outputs */
471 		: [cpu_id]		"r" (cpu),
472 		  [rseq_offset]		"r" (rseq_offset),
473 		  /* cmp2 input */
474 		  [v2]			"m" (*v2),
475 		  [expect2]		"r" (expect2),
476 		  /* final store input */
477 		  [v]			"m" (*v),
478 		  [expect]		"r" (expect),
479 		  [newv]		"r" (newv)
480 		: "memory", "cc", "rax"
481 		  RSEQ_INJECT_CLOBBER
482 		: abort, cmpfail
483 #ifdef RSEQ_COMPARE_TWICE
484 		  , error1, error2, error3
485 #endif
486 	);
487 	rseq_after_asm_goto();
488 	return 0;
489 abort:
490 	rseq_after_asm_goto();
491 	RSEQ_INJECT_FAILED
492 	return -1;
493 cmpfail:
494 	rseq_after_asm_goto();
495 	return 1;
496 #ifdef RSEQ_COMPARE_TWICE
497 error1:
498 	rseq_after_asm_goto();
499 	rseq_bug("cpu_id comparison failed");
500 error2:
501 	rseq_after_asm_goto();
502 	rseq_bug("1st expected value comparison failed");
503 error3:
504 	rseq_after_asm_goto();
505 	rseq_bug("2nd expected value comparison failed");
506 #endif
507 }
508 
509 static inline __attribute__((always_inline))
510 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
511 				 void *dst, void *src, size_t len,
512 				 intptr_t newv, int cpu)
513 {
514 	uint64_t rseq_scratch[3];
515 
516 	RSEQ_INJECT_C(9)
517 
518 	__asm__ __volatile__ goto (
519 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
520 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
521 #ifdef RSEQ_COMPARE_TWICE
522 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
523 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
524 #endif
525 		"movq %[src], %[rseq_scratch0]\n\t"
526 		"movq %[dst], %[rseq_scratch1]\n\t"
527 		"movq %[len], %[rseq_scratch2]\n\t"
528 		/* Start rseq by storing table entry pointer into rseq_cs. */
529 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
530 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
531 		RSEQ_INJECT_ASM(3)
532 		"cmpq %[v], %[expect]\n\t"
533 		"jnz 5f\n\t"
534 		RSEQ_INJECT_ASM(4)
535 #ifdef RSEQ_COMPARE_TWICE
536 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
537 		"cmpq %[v], %[expect]\n\t"
538 		"jnz 7f\n\t"
539 #endif
540 		/* try memcpy */
541 		"test %[len], %[len]\n\t" \
542 		"jz 333f\n\t" \
543 		"222:\n\t" \
544 		"movb (%[src]), %%al\n\t" \
545 		"movb %%al, (%[dst])\n\t" \
546 		"inc %[src]\n\t" \
547 		"inc %[dst]\n\t" \
548 		"dec %[len]\n\t" \
549 		"jnz 222b\n\t" \
550 		"333:\n\t" \
551 		RSEQ_INJECT_ASM(5)
552 		/* final store */
553 		"movq %[newv], %[v]\n\t"
554 		"2:\n\t"
555 		RSEQ_INJECT_ASM(6)
556 		/* teardown */
557 		"movq %[rseq_scratch2], %[len]\n\t"
558 		"movq %[rseq_scratch1], %[dst]\n\t"
559 		"movq %[rseq_scratch0], %[src]\n\t"
560 		RSEQ_ASM_DEFINE_ABORT(4,
561 			"movq %[rseq_scratch2], %[len]\n\t"
562 			"movq %[rseq_scratch1], %[dst]\n\t"
563 			"movq %[rseq_scratch0], %[src]\n\t",
564 			abort)
565 		RSEQ_ASM_DEFINE_CMPFAIL(5,
566 			"movq %[rseq_scratch2], %[len]\n\t"
567 			"movq %[rseq_scratch1], %[dst]\n\t"
568 			"movq %[rseq_scratch0], %[src]\n\t",
569 			cmpfail)
570 #ifdef RSEQ_COMPARE_TWICE
571 		RSEQ_ASM_DEFINE_CMPFAIL(6,
572 			"movq %[rseq_scratch2], %[len]\n\t"
573 			"movq %[rseq_scratch1], %[dst]\n\t"
574 			"movq %[rseq_scratch0], %[src]\n\t",
575 			error1)
576 		RSEQ_ASM_DEFINE_CMPFAIL(7,
577 			"movq %[rseq_scratch2], %[len]\n\t"
578 			"movq %[rseq_scratch1], %[dst]\n\t"
579 			"movq %[rseq_scratch0], %[src]\n\t",
580 			error2)
581 #endif
582 		: /* gcc asm goto does not allow outputs */
583 		: [cpu_id]		"r" (cpu),
584 		  [rseq_offset]		"r" (rseq_offset),
585 		  /* final store input */
586 		  [v]			"m" (*v),
587 		  [expect]		"r" (expect),
588 		  [newv]		"r" (newv),
589 		  /* try memcpy input */
590 		  [dst]			"r" (dst),
591 		  [src]			"r" (src),
592 		  [len]			"r" (len),
593 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
594 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
595 		  [rseq_scratch2]	"m" (rseq_scratch[2])
596 		: "memory", "cc", "rax"
597 		  RSEQ_INJECT_CLOBBER
598 		: abort, cmpfail
599 #ifdef RSEQ_COMPARE_TWICE
600 		  , error1, error2
601 #endif
602 	);
603 	rseq_after_asm_goto();
604 	return 0;
605 abort:
606 	rseq_after_asm_goto();
607 	RSEQ_INJECT_FAILED
608 	return -1;
609 cmpfail:
610 	rseq_after_asm_goto();
611 	return 1;
612 #ifdef RSEQ_COMPARE_TWICE
613 error1:
614 	rseq_after_asm_goto();
615 	rseq_bug("cpu_id comparison failed");
616 error2:
617 	rseq_after_asm_goto();
618 	rseq_bug("expected value comparison failed");
619 #endif
620 }
621 
622 /* x86-64 is TSO. */
623 static inline __attribute__((always_inline))
624 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
625 					 void *dst, void *src, size_t len,
626 					 intptr_t newv, int cpu)
627 {
628 	return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
629 					    newv, cpu);
630 }
631 
632 #endif /* !RSEQ_SKIP_FASTPATH */
633 
634 #elif defined(__i386__)
635 
636 #define RSEQ_ASM_TP_SEGMENT	%%gs
637 
638 #define rseq_smp_mb()	\
639 	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
640 #define rseq_smp_rmb()	\
641 	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
642 #define rseq_smp_wmb()	\
643 	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
644 
645 #define rseq_smp_load_acquire(p)					\
646 __extension__ ({							\
647 	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
648 	rseq_smp_mb();							\
649 	____p1;								\
650 })
651 
652 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
653 
654 #define rseq_smp_store_release(p, v)					\
655 do {									\
656 	rseq_smp_mb();							\
657 	RSEQ_WRITE_ONCE(*p, v);						\
658 } while (0)
659 
660 #ifdef RSEQ_SKIP_FASTPATH
661 #include "rseq-skip.h"
662 #else /* !RSEQ_SKIP_FASTPATH */
663 
664 /*
665  * Use eax as scratch register and take memory operands as input to
666  * lessen register pressure. Especially needed when compiling in O0.
667  */
668 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
669 				start_ip, post_commit_offset, abort_ip)	\
670 		".pushsection __rseq_cs, \"aw\"\n\t"			\
671 		".balign 32\n\t"					\
672 		__rseq_str(label) ":\n\t"				\
673 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
674 		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
675 		".popsection\n\t"					\
676 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
677 		".long " __rseq_str(label) "b, 0x0\n\t"			\
678 		".popsection\n\t"
679 
680 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
681 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
682 				(post_commit_ip - start_ip), abort_ip)
683 
684 /*
685  * Exit points of a rseq critical section consist of all instructions outside
686  * of the critical section where a critical section can either branch to or
687  * reach through the normal course of its execution. The abort IP and the
688  * post-commit IP are already part of the __rseq_cs section and should not be
689  * explicitly defined as additional exit points. Knowing all exit points is
690  * useful to assist debuggers stepping over the critical section.
691  */
692 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
693 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
694 		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
695 		".popsection\n\t"
696 
697 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
698 		RSEQ_INJECT_ASM(1)					\
699 		"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"	\
700 		__rseq_str(label) ":\n\t"
701 
702 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
703 		RSEQ_INJECT_ASM(2)					\
704 		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
705 		"jnz " __rseq_str(label) "\n\t"
706 
707 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
708 		".pushsection __rseq_failure, \"ax\"\n\t"		\
709 		/* Disassembler-friendly signature: ud1 <sig>,%edi. */	\
710 		".byte 0x0f, 0xb9, 0x3d\n\t"				\
711 		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
712 		__rseq_str(label) ":\n\t"				\
713 		teardown						\
714 		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
715 		".popsection\n\t"
716 
717 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
718 		".pushsection __rseq_failure, \"ax\"\n\t"		\
719 		__rseq_str(label) ":\n\t"				\
720 		teardown						\
721 		"jmp %l[" __rseq_str(cmpfail_label) "]\n\t"		\
722 		".popsection\n\t"
723 
724 static inline __attribute__((always_inline))
725 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
726 {
727 	RSEQ_INJECT_C(9)
728 
729 	__asm__ __volatile__ goto (
730 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
731 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
732 #ifdef RSEQ_COMPARE_TWICE
733 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
734 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
735 #endif
736 		/* Start rseq by storing table entry pointer into rseq_cs. */
737 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
738 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
739 		RSEQ_INJECT_ASM(3)
740 		"cmpl %[v], %[expect]\n\t"
741 		"jnz %l[cmpfail]\n\t"
742 		RSEQ_INJECT_ASM(4)
743 #ifdef RSEQ_COMPARE_TWICE
744 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
745 		"cmpl %[v], %[expect]\n\t"
746 		"jnz %l[error2]\n\t"
747 #endif
748 		/* final store */
749 		"movl %[newv], %[v]\n\t"
750 		"2:\n\t"
751 		RSEQ_INJECT_ASM(5)
752 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
753 		: /* gcc asm goto does not allow outputs */
754 		: [cpu_id]		"r" (cpu),
755 		  [rseq_offset]		"r" (rseq_offset),
756 		  [v]			"m" (*v),
757 		  [expect]		"r" (expect),
758 		  [newv]		"r" (newv)
759 		: "memory", "cc", "eax"
760 		  RSEQ_INJECT_CLOBBER
761 		: abort, cmpfail
762 #ifdef RSEQ_COMPARE_TWICE
763 		  , error1, error2
764 #endif
765 	);
766 	rseq_after_asm_goto();
767 	return 0;
768 abort:
769 	rseq_after_asm_goto();
770 	RSEQ_INJECT_FAILED
771 	return -1;
772 cmpfail:
773 	rseq_after_asm_goto();
774 	return 1;
775 #ifdef RSEQ_COMPARE_TWICE
776 error1:
777 	rseq_after_asm_goto();
778 	rseq_bug("cpu_id comparison failed");
779 error2:
780 	rseq_after_asm_goto();
781 	rseq_bug("expected value comparison failed");
782 #endif
783 }
784 
785 /*
786  * Compare @v against @expectnot. When it does _not_ match, load @v
787  * into @load, and store the content of *@v + voffp into @v.
788  */
789 static inline __attribute__((always_inline))
790 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
791 			       long voffp, intptr_t *load, int cpu)
792 {
793 	RSEQ_INJECT_C(9)
794 
795 	__asm__ __volatile__ goto (
796 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
797 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
798 #ifdef RSEQ_COMPARE_TWICE
799 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
800 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
801 #endif
802 		/* Start rseq by storing table entry pointer into rseq_cs. */
803 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
804 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
805 		RSEQ_INJECT_ASM(3)
806 		"movl %[v], %%ebx\n\t"
807 		"cmpl %%ebx, %[expectnot]\n\t"
808 		"je %l[cmpfail]\n\t"
809 		RSEQ_INJECT_ASM(4)
810 #ifdef RSEQ_COMPARE_TWICE
811 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
812 		"movl %[v], %%ebx\n\t"
813 		"cmpl %%ebx, %[expectnot]\n\t"
814 		"je %l[error2]\n\t"
815 #endif
816 		"movl %%ebx, %[load]\n\t"
817 		"addl %[voffp], %%ebx\n\t"
818 		"movl (%%ebx), %%ebx\n\t"
819 		/* final store */
820 		"movl %%ebx, %[v]\n\t"
821 		"2:\n\t"
822 		RSEQ_INJECT_ASM(5)
823 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
824 		: /* gcc asm goto does not allow outputs */
825 		: [cpu_id]		"r" (cpu),
826 		  [rseq_offset]		"r" (rseq_offset),
827 		  /* final store input */
828 		  [v]			"m" (*v),
829 		  [expectnot]		"r" (expectnot),
830 		  [voffp]		"ir" (voffp),
831 		  [load]		"m" (*load)
832 		: "memory", "cc", "eax", "ebx"
833 		  RSEQ_INJECT_CLOBBER
834 		: abort, cmpfail
835 #ifdef RSEQ_COMPARE_TWICE
836 		  , error1, error2
837 #endif
838 	);
839 	rseq_after_asm_goto();
840 	return 0;
841 abort:
842 	rseq_after_asm_goto();
843 	RSEQ_INJECT_FAILED
844 	return -1;
845 cmpfail:
846 	rseq_after_asm_goto();
847 	return 1;
848 #ifdef RSEQ_COMPARE_TWICE
849 error1:
850 	rseq_after_asm_goto();
851 	rseq_bug("cpu_id comparison failed");
852 error2:
853 	rseq_after_asm_goto();
854 	rseq_bug("expected value comparison failed");
855 #endif
856 }
857 
858 static inline __attribute__((always_inline))
859 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
860 {
861 	RSEQ_INJECT_C(9)
862 
863 	__asm__ __volatile__ goto (
864 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
865 #ifdef RSEQ_COMPARE_TWICE
866 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
867 #endif
868 		/* Start rseq by storing table entry pointer into rseq_cs. */
869 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
870 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
871 		RSEQ_INJECT_ASM(3)
872 #ifdef RSEQ_COMPARE_TWICE
873 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
874 #endif
875 		/* final store */
876 		"addl %[count], %[v]\n\t"
877 		"2:\n\t"
878 		RSEQ_INJECT_ASM(4)
879 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
880 		: /* gcc asm goto does not allow outputs */
881 		: [cpu_id]		"r" (cpu),
882 		  [rseq_offset]		"r" (rseq_offset),
883 		  /* final store input */
884 		  [v]			"m" (*v),
885 		  [count]		"ir" (count)
886 		: "memory", "cc", "eax"
887 		  RSEQ_INJECT_CLOBBER
888 		: abort
889 #ifdef RSEQ_COMPARE_TWICE
890 		  , error1
891 #endif
892 	);
893 	rseq_after_asm_goto();
894 	return 0;
895 abort:
896 	rseq_after_asm_goto();
897 	RSEQ_INJECT_FAILED
898 	return -1;
899 #ifdef RSEQ_COMPARE_TWICE
900 error1:
901 	rseq_after_asm_goto();
902 	rseq_bug("cpu_id comparison failed");
903 #endif
904 }
905 
906 static inline __attribute__((always_inline))
907 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
908 				 intptr_t *v2, intptr_t newv2,
909 				 intptr_t newv, int cpu)
910 {
911 	RSEQ_INJECT_C(9)
912 
913 	__asm__ __volatile__ goto (
914 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
915 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
916 #ifdef RSEQ_COMPARE_TWICE
917 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
918 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
919 #endif
920 		/* Start rseq by storing table entry pointer into rseq_cs. */
921 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
922 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
923 		RSEQ_INJECT_ASM(3)
924 		"cmpl %[v], %[expect]\n\t"
925 		"jnz %l[cmpfail]\n\t"
926 		RSEQ_INJECT_ASM(4)
927 #ifdef RSEQ_COMPARE_TWICE
928 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
929 		"cmpl %[v], %[expect]\n\t"
930 		"jnz %l[error2]\n\t"
931 #endif
932 		/* try store */
933 		"movl %[newv2], %%eax\n\t"
934 		"movl %%eax, %[v2]\n\t"
935 		RSEQ_INJECT_ASM(5)
936 		/* final store */
937 		"movl %[newv], %[v]\n\t"
938 		"2:\n\t"
939 		RSEQ_INJECT_ASM(6)
940 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
941 		: /* gcc asm goto does not allow outputs */
942 		: [cpu_id]		"r" (cpu),
943 		  [rseq_offset]		"r" (rseq_offset),
944 		  /* try store input */
945 		  [v2]			"m" (*v2),
946 		  [newv2]		"m" (newv2),
947 		  /* final store input */
948 		  [v]			"m" (*v),
949 		  [expect]		"r" (expect),
950 		  [newv]		"r" (newv)
951 		: "memory", "cc", "eax"
952 		  RSEQ_INJECT_CLOBBER
953 		: abort, cmpfail
954 #ifdef RSEQ_COMPARE_TWICE
955 		  , error1, error2
956 #endif
957 	);
958 	rseq_after_asm_goto();
959 	return 0;
960 abort:
961 	rseq_after_asm_goto();
962 	RSEQ_INJECT_FAILED
963 	return -1;
964 cmpfail:
965 	rseq_after_asm_goto();
966 	return 1;
967 #ifdef RSEQ_COMPARE_TWICE
968 error1:
969 	rseq_after_asm_goto();
970 	rseq_bug("cpu_id comparison failed");
971 error2:
972 	rseq_after_asm_goto();
973 	rseq_bug("expected value comparison failed");
974 #endif
975 }
976 
977 static inline __attribute__((always_inline))
978 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
979 					 intptr_t *v2, intptr_t newv2,
980 					 intptr_t newv, int cpu)
981 {
982 	RSEQ_INJECT_C(9)
983 
984 	__asm__ __volatile__ goto (
985 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
986 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
987 #ifdef RSEQ_COMPARE_TWICE
988 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
989 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
990 #endif
991 		/* Start rseq by storing table entry pointer into rseq_cs. */
992 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
993 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
994 		RSEQ_INJECT_ASM(3)
995 		"movl %[expect], %%eax\n\t"
996 		"cmpl %[v], %%eax\n\t"
997 		"jnz %l[cmpfail]\n\t"
998 		RSEQ_INJECT_ASM(4)
999 #ifdef RSEQ_COMPARE_TWICE
1000 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
1001 		"movl %[expect], %%eax\n\t"
1002 		"cmpl %[v], %%eax\n\t"
1003 		"jnz %l[error2]\n\t"
1004 #endif
1005 		/* try store */
1006 		"movl %[newv2], %[v2]\n\t"
1007 		RSEQ_INJECT_ASM(5)
1008 		"lock; addl $0,-128(%%esp)\n\t"
1009 		/* final store */
1010 		"movl %[newv], %[v]\n\t"
1011 		"2:\n\t"
1012 		RSEQ_INJECT_ASM(6)
1013 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1014 		: /* gcc asm goto does not allow outputs */
1015 		: [cpu_id]		"r" (cpu),
1016 		  [rseq_offset]		"r" (rseq_offset),
1017 		  /* try store input */
1018 		  [v2]			"m" (*v2),
1019 		  [newv2]		"r" (newv2),
1020 		  /* final store input */
1021 		  [v]			"m" (*v),
1022 		  [expect]		"m" (expect),
1023 		  [newv]		"r" (newv)
1024 		: "memory", "cc", "eax"
1025 		  RSEQ_INJECT_CLOBBER
1026 		: abort, cmpfail
1027 #ifdef RSEQ_COMPARE_TWICE
1028 		  , error1, error2
1029 #endif
1030 	);
1031 	rseq_after_asm_goto();
1032 	return 0;
1033 abort:
1034 	rseq_after_asm_goto();
1035 	RSEQ_INJECT_FAILED
1036 	return -1;
1037 cmpfail:
1038 	rseq_after_asm_goto();
1039 	return 1;
1040 #ifdef RSEQ_COMPARE_TWICE
1041 error1:
1042 	rseq_after_asm_goto();
1043 	rseq_bug("cpu_id comparison failed");
1044 error2:
1045 	rseq_after_asm_goto();
1046 	rseq_bug("expected value comparison failed");
1047 #endif
1048 
1049 }
1050 
1051 static inline __attribute__((always_inline))
1052 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
1053 			      intptr_t *v2, intptr_t expect2,
1054 			      intptr_t newv, int cpu)
1055 {
1056 	RSEQ_INJECT_C(9)
1057 
1058 	__asm__ __volatile__ goto (
1059 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1060 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1061 #ifdef RSEQ_COMPARE_TWICE
1062 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1063 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1064 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1065 #endif
1066 		/* Start rseq by storing table entry pointer into rseq_cs. */
1067 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1068 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1069 		RSEQ_INJECT_ASM(3)
1070 		"cmpl %[v], %[expect]\n\t"
1071 		"jnz %l[cmpfail]\n\t"
1072 		RSEQ_INJECT_ASM(4)
1073 		"cmpl %[expect2], %[v2]\n\t"
1074 		"jnz %l[cmpfail]\n\t"
1075 		RSEQ_INJECT_ASM(5)
1076 #ifdef RSEQ_COMPARE_TWICE
1077 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
1078 		"cmpl %[v], %[expect]\n\t"
1079 		"jnz %l[error2]\n\t"
1080 		"cmpl %[expect2], %[v2]\n\t"
1081 		"jnz %l[error3]\n\t"
1082 #endif
1083 		"movl %[newv], %%eax\n\t"
1084 		/* final store */
1085 		"movl %%eax, %[v]\n\t"
1086 		"2:\n\t"
1087 		RSEQ_INJECT_ASM(6)
1088 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1089 		: /* gcc asm goto does not allow outputs */
1090 		: [cpu_id]		"r" (cpu),
1091 		  [rseq_offset]		"r" (rseq_offset),
1092 		  /* cmp2 input */
1093 		  [v2]			"m" (*v2),
1094 		  [expect2]		"r" (expect2),
1095 		  /* final store input */
1096 		  [v]			"m" (*v),
1097 		  [expect]		"r" (expect),
1098 		  [newv]		"m" (newv)
1099 		: "memory", "cc", "eax"
1100 		  RSEQ_INJECT_CLOBBER
1101 		: abort, cmpfail
1102 #ifdef RSEQ_COMPARE_TWICE
1103 		  , error1, error2, error3
1104 #endif
1105 	);
1106 	rseq_after_asm_goto();
1107 	return 0;
1108 abort:
1109 	rseq_after_asm_goto();
1110 	RSEQ_INJECT_FAILED
1111 	return -1;
1112 cmpfail:
1113 	rseq_after_asm_goto();
1114 	return 1;
1115 #ifdef RSEQ_COMPARE_TWICE
1116 error1:
1117 	rseq_after_asm_goto();
1118 	rseq_bug("cpu_id comparison failed");
1119 error2:
1120 	rseq_after_asm_goto();
1121 	rseq_bug("1st expected value comparison failed");
1122 error3:
1123 	rseq_after_asm_goto();
1124 	rseq_bug("2nd expected value comparison failed");
1125 #endif
1126 }
1127 
1128 /* TODO: implement a faster memcpy. */
1129 static inline __attribute__((always_inline))
1130 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1131 				 void *dst, void *src, size_t len,
1132 				 intptr_t newv, int cpu)
1133 {
1134 	uint32_t rseq_scratch[3];
1135 
1136 	RSEQ_INJECT_C(9)
1137 
1138 	__asm__ __volatile__ goto (
1139 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1140 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1141 #ifdef RSEQ_COMPARE_TWICE
1142 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1143 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1144 #endif
1145 		"movl %[src], %[rseq_scratch0]\n\t"
1146 		"movl %[dst], %[rseq_scratch1]\n\t"
1147 		"movl %[len], %[rseq_scratch2]\n\t"
1148 		/* Start rseq by storing table entry pointer into rseq_cs. */
1149 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1150 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1151 		RSEQ_INJECT_ASM(3)
1152 		"movl %[expect], %%eax\n\t"
1153 		"cmpl %%eax, %[v]\n\t"
1154 		"jnz 5f\n\t"
1155 		RSEQ_INJECT_ASM(4)
1156 #ifdef RSEQ_COMPARE_TWICE
1157 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
1158 		"movl %[expect], %%eax\n\t"
1159 		"cmpl %%eax, %[v]\n\t"
1160 		"jnz 7f\n\t"
1161 #endif
1162 		/* try memcpy */
1163 		"test %[len], %[len]\n\t" \
1164 		"jz 333f\n\t" \
1165 		"222:\n\t" \
1166 		"movb (%[src]), %%al\n\t" \
1167 		"movb %%al, (%[dst])\n\t" \
1168 		"inc %[src]\n\t" \
1169 		"inc %[dst]\n\t" \
1170 		"dec %[len]\n\t" \
1171 		"jnz 222b\n\t" \
1172 		"333:\n\t" \
1173 		RSEQ_INJECT_ASM(5)
1174 		"movl %[newv], %%eax\n\t"
1175 		/* final store */
1176 		"movl %%eax, %[v]\n\t"
1177 		"2:\n\t"
1178 		RSEQ_INJECT_ASM(6)
1179 		/* teardown */
1180 		"movl %[rseq_scratch2], %[len]\n\t"
1181 		"movl %[rseq_scratch1], %[dst]\n\t"
1182 		"movl %[rseq_scratch0], %[src]\n\t"
1183 		RSEQ_ASM_DEFINE_ABORT(4,
1184 			"movl %[rseq_scratch2], %[len]\n\t"
1185 			"movl %[rseq_scratch1], %[dst]\n\t"
1186 			"movl %[rseq_scratch0], %[src]\n\t",
1187 			abort)
1188 		RSEQ_ASM_DEFINE_CMPFAIL(5,
1189 			"movl %[rseq_scratch2], %[len]\n\t"
1190 			"movl %[rseq_scratch1], %[dst]\n\t"
1191 			"movl %[rseq_scratch0], %[src]\n\t",
1192 			cmpfail)
1193 #ifdef RSEQ_COMPARE_TWICE
1194 		RSEQ_ASM_DEFINE_CMPFAIL(6,
1195 			"movl %[rseq_scratch2], %[len]\n\t"
1196 			"movl %[rseq_scratch1], %[dst]\n\t"
1197 			"movl %[rseq_scratch0], %[src]\n\t",
1198 			error1)
1199 		RSEQ_ASM_DEFINE_CMPFAIL(7,
1200 			"movl %[rseq_scratch2], %[len]\n\t"
1201 			"movl %[rseq_scratch1], %[dst]\n\t"
1202 			"movl %[rseq_scratch0], %[src]\n\t",
1203 			error2)
1204 #endif
1205 		: /* gcc asm goto does not allow outputs */
1206 		: [cpu_id]		"r" (cpu),
1207 		  [rseq_offset]		"r" (rseq_offset),
1208 		  /* final store input */
1209 		  [v]			"m" (*v),
1210 		  [expect]		"m" (expect),
1211 		  [newv]		"m" (newv),
1212 		  /* try memcpy input */
1213 		  [dst]			"r" (dst),
1214 		  [src]			"r" (src),
1215 		  [len]			"r" (len),
1216 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
1217 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
1218 		  [rseq_scratch2]	"m" (rseq_scratch[2])
1219 		: "memory", "cc", "eax"
1220 		  RSEQ_INJECT_CLOBBER
1221 		: abort, cmpfail
1222 #ifdef RSEQ_COMPARE_TWICE
1223 		  , error1, error2
1224 #endif
1225 	);
1226 	rseq_after_asm_goto();
1227 	return 0;
1228 abort:
1229 	rseq_after_asm_goto();
1230 	RSEQ_INJECT_FAILED
1231 	return -1;
1232 cmpfail:
1233 	rseq_after_asm_goto();
1234 	return 1;
1235 #ifdef RSEQ_COMPARE_TWICE
1236 error1:
1237 	rseq_after_asm_goto();
1238 	rseq_bug("cpu_id comparison failed");
1239 error2:
1240 	rseq_after_asm_goto();
1241 	rseq_bug("expected value comparison failed");
1242 #endif
1243 }
1244 
1245 /* TODO: implement a faster memcpy. */
1246 static inline __attribute__((always_inline))
1247 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1248 					 void *dst, void *src, size_t len,
1249 					 intptr_t newv, int cpu)
1250 {
1251 	uint32_t rseq_scratch[3];
1252 
1253 	RSEQ_INJECT_C(9)
1254 
1255 	__asm__ __volatile__ goto (
1256 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1257 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1258 #ifdef RSEQ_COMPARE_TWICE
1259 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1260 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1261 #endif
1262 		"movl %[src], %[rseq_scratch0]\n\t"
1263 		"movl %[dst], %[rseq_scratch1]\n\t"
1264 		"movl %[len], %[rseq_scratch2]\n\t"
1265 		/* Start rseq by storing table entry pointer into rseq_cs. */
1266 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1267 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1268 		RSEQ_INJECT_ASM(3)
1269 		"movl %[expect], %%eax\n\t"
1270 		"cmpl %%eax, %[v]\n\t"
1271 		"jnz 5f\n\t"
1272 		RSEQ_INJECT_ASM(4)
1273 #ifdef RSEQ_COMPARE_TWICE
1274 		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
1275 		"movl %[expect], %%eax\n\t"
1276 		"cmpl %%eax, %[v]\n\t"
1277 		"jnz 7f\n\t"
1278 #endif
1279 		/* try memcpy */
1280 		"test %[len], %[len]\n\t" \
1281 		"jz 333f\n\t" \
1282 		"222:\n\t" \
1283 		"movb (%[src]), %%al\n\t" \
1284 		"movb %%al, (%[dst])\n\t" \
1285 		"inc %[src]\n\t" \
1286 		"inc %[dst]\n\t" \
1287 		"dec %[len]\n\t" \
1288 		"jnz 222b\n\t" \
1289 		"333:\n\t" \
1290 		RSEQ_INJECT_ASM(5)
1291 		"lock; addl $0,-128(%%esp)\n\t"
1292 		"movl %[newv], %%eax\n\t"
1293 		/* final store */
1294 		"movl %%eax, %[v]\n\t"
1295 		"2:\n\t"
1296 		RSEQ_INJECT_ASM(6)
1297 		/* teardown */
1298 		"movl %[rseq_scratch2], %[len]\n\t"
1299 		"movl %[rseq_scratch1], %[dst]\n\t"
1300 		"movl %[rseq_scratch0], %[src]\n\t"
1301 		RSEQ_ASM_DEFINE_ABORT(4,
1302 			"movl %[rseq_scratch2], %[len]\n\t"
1303 			"movl %[rseq_scratch1], %[dst]\n\t"
1304 			"movl %[rseq_scratch0], %[src]\n\t",
1305 			abort)
1306 		RSEQ_ASM_DEFINE_CMPFAIL(5,
1307 			"movl %[rseq_scratch2], %[len]\n\t"
1308 			"movl %[rseq_scratch1], %[dst]\n\t"
1309 			"movl %[rseq_scratch0], %[src]\n\t",
1310 			cmpfail)
1311 #ifdef RSEQ_COMPARE_TWICE
1312 		RSEQ_ASM_DEFINE_CMPFAIL(6,
1313 			"movl %[rseq_scratch2], %[len]\n\t"
1314 			"movl %[rseq_scratch1], %[dst]\n\t"
1315 			"movl %[rseq_scratch0], %[src]\n\t",
1316 			error1)
1317 		RSEQ_ASM_DEFINE_CMPFAIL(7,
1318 			"movl %[rseq_scratch2], %[len]\n\t"
1319 			"movl %[rseq_scratch1], %[dst]\n\t"
1320 			"movl %[rseq_scratch0], %[src]\n\t",
1321 			error2)
1322 #endif
1323 		: /* gcc asm goto does not allow outputs */
1324 		: [cpu_id]		"r" (cpu),
1325 		  [rseq_offset]		"r" (rseq_offset),
1326 		  /* final store input */
1327 		  [v]			"m" (*v),
1328 		  [expect]		"m" (expect),
1329 		  [newv]		"m" (newv),
1330 		  /* try memcpy input */
1331 		  [dst]			"r" (dst),
1332 		  [src]			"r" (src),
1333 		  [len]			"r" (len),
1334 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
1335 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
1336 		  [rseq_scratch2]	"m" (rseq_scratch[2])
1337 		: "memory", "cc", "eax"
1338 		  RSEQ_INJECT_CLOBBER
1339 		: abort, cmpfail
1340 #ifdef RSEQ_COMPARE_TWICE
1341 		  , error1, error2
1342 #endif
1343 	);
1344 	rseq_after_asm_goto();
1345 	return 0;
1346 abort:
1347 	rseq_after_asm_goto();
1348 	RSEQ_INJECT_FAILED
1349 	return -1;
1350 cmpfail:
1351 	rseq_after_asm_goto();
1352 	return 1;
1353 #ifdef RSEQ_COMPARE_TWICE
1354 error1:
1355 	rseq_after_asm_goto();
1356 	rseq_bug("cpu_id comparison failed");
1357 error2:
1358 	rseq_after_asm_goto();
1359 	rseq_bug("expected value comparison failed");
1360 #endif
1361 }
1362 
1363 #endif /* !RSEQ_SKIP_FASTPATH */
1364 
1365 #endif
1366