1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 
3 #include "rseq-bits-template.h"
4 
5 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
6 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
7 
8 static inline __always_inline
9 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
10 {
11 	RSEQ_INJECT_C(9)
12 
13 	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
14 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
15 #ifdef RSEQ_COMPARE_TWICE
16 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
17 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
18 #endif
19 				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
20 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
21 				  RSEQ_INJECT_ASM(3)
22 				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
23 				  RSEQ_INJECT_ASM(4)
24 #ifdef RSEQ_COMPARE_TWICE
25 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
26 				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
27 #endif
28 				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
29 				  RSEQ_INJECT_ASM(5)
30 				  RSEQ_ASM_DEFINE_ABORT(4, abort)
31 				  : /* gcc asm goto does not allow outputs */
32 				  : [cpu_id]		"r" (cpu),
33 				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
34 				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
35 				    [v]			"m" (*v),
36 				    [expect]		"r" (expect),
37 				    [newv]		"r" (newv)
38 				    RSEQ_INJECT_INPUT
39 				  : "memory", RSEQ_ASM_TMP_REG_1
40 				    RSEQ_INJECT_CLOBBER
41 				  : abort, cmpfail
42 #ifdef RSEQ_COMPARE_TWICE
43 				    , error1, error2
44 #endif
45 	);
46 
47 	return 0;
48 abort:
49 	RSEQ_INJECT_FAILED
50 	return -1;
51 cmpfail:
52 	return 1;
53 #ifdef RSEQ_COMPARE_TWICE
54 error1:
55 	rseq_bug("cpu_id comparison failed");
56 error2:
57 	rseq_bug("expected value comparison failed");
58 #endif
59 }
60 
61 static inline __always_inline
62 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
63 			       off_t voffp, intptr_t *load, int cpu)
64 {
65 	RSEQ_INJECT_C(9)
66 
67 	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
68 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
69 #ifdef RSEQ_COMPARE_TWICE
70 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
71 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
72 #endif
73 				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
74 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
75 				  RSEQ_INJECT_ASM(3)
76 				  RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
77 				  RSEQ_INJECT_ASM(4)
78 #ifdef RSEQ_COMPARE_TWICE
79 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
80 				  RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
81 #endif
82 				  RSEQ_ASM_OP_R_LOAD(v)
83 				  RSEQ_ASM_OP_R_STORE(load)
84 				  RSEQ_ASM_OP_R_LOAD_OFF(voffp)
85 				  RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
86 				  RSEQ_INJECT_ASM(5)
87 				  RSEQ_ASM_DEFINE_ABORT(4, abort)
88 				  : /* gcc asm goto does not allow outputs */
89 				  : [cpu_id]		"r" (cpu),
90 				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
91 				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
92 				    [v]			"m" (*v),
93 				    [expectnot]		"r" (expectnot),
94 				    [load]		"m" (*load),
95 				    [voffp]		"r" (voffp)
96 				    RSEQ_INJECT_INPUT
97 				  : "memory", RSEQ_ASM_TMP_REG_1
98 				    RSEQ_INJECT_CLOBBER
99 				  : abort, cmpfail
100 #ifdef RSEQ_COMPARE_TWICE
101 				    , error1, error2
102 #endif
103 	);
104 	return 0;
105 abort:
106 	RSEQ_INJECT_FAILED
107 	return -1;
108 cmpfail:
109 	return 1;
110 #ifdef RSEQ_COMPARE_TWICE
111 error1:
112 	rseq_bug("cpu_id comparison failed");
113 error2:
114 	rseq_bug("expected value comparison failed");
115 #endif
116 }
117 
118 static inline __always_inline
119 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
120 {
121 	RSEQ_INJECT_C(9)
122 
123 	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
124 #ifdef RSEQ_COMPARE_TWICE
125 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
126 #endif
127 				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
128 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
129 				  RSEQ_INJECT_ASM(3)
130 #ifdef RSEQ_COMPARE_TWICE
131 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
132 #endif
133 				  RSEQ_ASM_OP_R_LOAD(v)
134 				  RSEQ_ASM_OP_R_ADD(count)
135 				  RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
136 				  RSEQ_INJECT_ASM(4)
137 				  RSEQ_ASM_DEFINE_ABORT(4, abort)
138 				  : /* gcc asm goto does not allow outputs */
139 				  : [cpu_id]		"r" (cpu),
140 				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
141 				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
142 				    [v]			"m" (*v),
143 				    [count]		"r" (count)
144 				    RSEQ_INJECT_INPUT
145 				  : "memory", RSEQ_ASM_TMP_REG_1
146 				    RSEQ_INJECT_CLOBBER
147 				  : abort
148 #ifdef RSEQ_COMPARE_TWICE
149 				    , error1
150 #endif
151 	);
152 	return 0;
153 abort:
154 	RSEQ_INJECT_FAILED
155 	return -1;
156 #ifdef RSEQ_COMPARE_TWICE
157 error1:
158 	rseq_bug("cpu_id comparison failed");
159 #endif
160 }
161 
162 static inline __always_inline
163 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
164 			      intptr_t *v2, intptr_t expect2,
165 			      intptr_t newv, int cpu)
166 {
167 	RSEQ_INJECT_C(9)
168 
169 	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
170 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
171 #ifdef RSEQ_COMPARE_TWICE
172 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
173 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
174 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
175 #endif
176 				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
177 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
178 				  RSEQ_INJECT_ASM(3)
179 				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
180 				  RSEQ_INJECT_ASM(4)
181 				  RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
182 				  RSEQ_INJECT_ASM(5)
183 #ifdef RSEQ_COMPARE_TWICE
184 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
185 				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
186 				  RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
187 #endif
188 				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
189 				  RSEQ_INJECT_ASM(6)
190 				  RSEQ_ASM_DEFINE_ABORT(4, abort)
191 				  : /* gcc asm goto does not allow outputs */
192 				  : [cpu_id]		"r" (cpu),
193 				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
194 				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
195 				    [v]			"m" (*v),
196 				    [expect]		"r" (expect),
197 				    [v2]			"m" (*v2),
198 				    [expect2]		"r" (expect2),
199 				    [newv]		"r" (newv)
200 				    RSEQ_INJECT_INPUT
201 				  : "memory", RSEQ_ASM_TMP_REG_1
202 				    RSEQ_INJECT_CLOBBER
203 				  : abort, cmpfail
204 #ifdef RSEQ_COMPARE_TWICE
205 				    , error1, error2, error3
206 #endif
207 	);
208 
209 	return 0;
210 abort:
211 	RSEQ_INJECT_FAILED
212 	return -1;
213 cmpfail:
214 	return 1;
215 #ifdef RSEQ_COMPARE_TWICE
216 error1:
217 	rseq_bug("cpu_id comparison failed");
218 error2:
219 	rseq_bug("expected value comparison failed");
220 error3:
221 	rseq_bug("2nd expected value comparison failed");
222 #endif
223 }
224 
225 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
226 
227 /*
228  *   pval = *(ptr+off)
229  *  *pval += inc;
230  */
231 static inline __always_inline
232 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
233 {
234 	RSEQ_INJECT_C(9)
235 
236 	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
237 #ifdef RSEQ_COMPARE_TWICE
238 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
239 #endif
240 				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
241 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
242 				  RSEQ_INJECT_ASM(3)
243 #ifdef RSEQ_COMPARE_TWICE
244 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
245 #endif
246 				  RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
247 				  RSEQ_INJECT_ASM(4)
248 				  RSEQ_ASM_DEFINE_ABORT(4, abort)
249 				  : /* gcc asm goto does not allow outputs */
250 				  : [cpu_id]			"r" (cpu),
251 				    [current_cpu_id]		"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
252 				    [rseq_cs]			"m" (rseq_get_abi()->rseq_cs.arch.ptr),
253 				    [ptr]			"r" (ptr),
254 				    [off]			"er" (off),
255 				    [inc]			"er" (inc)
256 				    RSEQ_INJECT_INPUT
257 				  : "memory", RSEQ_ASM_TMP_REG_1
258 				    RSEQ_INJECT_CLOBBER
259 				  : abort
260 #ifdef RSEQ_COMPARE_TWICE
261 				    , error1
262 #endif
263 	);
264 	return 0;
265 abort:
266 	RSEQ_INJECT_FAILED
267 	return -1;
268 #ifdef RSEQ_COMPARE_TWICE
269 error1:
270 	rseq_bug("cpu_id comparison failed");
271 #endif
272 }
273 
274 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
275 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
276 
277 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
278 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
279 
280 static inline __always_inline
281 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
282 				 intptr_t *v2, intptr_t newv2,
283 				 intptr_t newv, int cpu)
284 {
285 	RSEQ_INJECT_C(9)
286 
287 	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
288 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
289 #ifdef RSEQ_COMPARE_TWICE
290 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
291 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
292 #endif
293 				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
294 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
295 				  RSEQ_INJECT_ASM(3)
296 				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
297 				  RSEQ_INJECT_ASM(4)
298 #ifdef RSEQ_COMPARE_TWICE
299 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
300 				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
301 #endif
302 				  RSEQ_ASM_OP_STORE(newv2, v2)
303 				  RSEQ_INJECT_ASM(5)
304 #ifdef RSEQ_TEMPLATE_MO_RELEASE
305 				  RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
306 #else
307 				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
308 #endif
309 				  RSEQ_INJECT_ASM(6)
310 				  RSEQ_ASM_DEFINE_ABORT(4, abort)
311 				  : /* gcc asm goto does not allow outputs */
312 				  : [cpu_id]			"r" (cpu),
313 				    [current_cpu_id]		"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
314 				    [rseq_cs]			"m" (rseq_get_abi()->rseq_cs.arch.ptr),
315 				    [expect]			"r" (expect),
316 				    [v]				"m" (*v),
317 				    [newv]			"r" (newv),
318 				    [v2]			"m" (*v2),
319 				    [newv2]			"r" (newv2)
320 				    RSEQ_INJECT_INPUT
321 				  : "memory", RSEQ_ASM_TMP_REG_1
322 				    RSEQ_INJECT_CLOBBER
323 				  : abort, cmpfail
324 #ifdef RSEQ_COMPARE_TWICE
325 				    , error1, error2
326 #endif
327 	);
328 
329 	return 0;
330 abort:
331 	RSEQ_INJECT_FAILED
332 	return -1;
333 cmpfail:
334 	return 1;
335 #ifdef RSEQ_COMPARE_TWICE
336 error1:
337 	rseq_bug("cpu_id comparison failed");
338 error2:
339 	rseq_bug("expected value comparison failed");
340 #endif
341 }
342 
343 static inline __always_inline
344 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
345 				 void *dst, void *src, size_t len,
346 				 intptr_t newv, int cpu)
347 {
348 	RSEQ_INJECT_C(9)
349 	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
350 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
351 #ifdef RSEQ_COMPARE_TWICE
352 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
353 				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
354 #endif
355 				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
356 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
357 				  RSEQ_INJECT_ASM(3)
358 				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
359 				  RSEQ_INJECT_ASM(4)
360 #ifdef RSEQ_COMPARE_TWICE
361 				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
362 				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
363 #endif
364 				  RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
365 				  RSEQ_INJECT_ASM(5)
366 #ifdef RSEQ_TEMPLATE_MO_RELEASE
367 				  RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
368 #else
369 				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
370 #endif
371 				  RSEQ_INJECT_ASM(6)
372 				  RSEQ_ASM_DEFINE_ABORT(4, abort)
373 				  : /* gcc asm goto does not allow outputs */
374 				  : [cpu_id]			"r" (cpu),
375 				    [current_cpu_id]		"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
376 				    [rseq_cs]			"m" (rseq_get_abi()->rseq_cs.arch.ptr),
377 				    [expect]			"r" (expect),
378 				    [v]				"m" (*v),
379 				    [newv]			"r" (newv),
380 				    [dst]			"r" (dst),
381 				    [src]			"r" (src),
382 				    [len]			"r" (len)
383 				    RSEQ_INJECT_INPUT
384 				  : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
385 				    RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
386 				    RSEQ_INJECT_CLOBBER
387 				  : abort, cmpfail
388 #ifdef RSEQ_COMPARE_TWICE
389 				    , error1, error2
390 #endif
391 	);
392 
393 	return 0;
394 abort:
395 	RSEQ_INJECT_FAILED
396 	return -1;
397 cmpfail:
398 	return 1;
399 #ifdef RSEQ_COMPARE_TWICE
400 error1:
401 	rseq_bug("cpu_id comparison failed");
402 error2:
403 	rseq_bug("expected value comparison failed");
404 #endif
405 }
406 
407 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
408 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
409 
410 #include "rseq-bits-reset.h"
411