1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * rseq-arm64-bits.h
4  *
5  * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6  * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7  */
8 
9 #include "rseq-bits-template.h"
10 
11 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
12 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
13 
14 static inline __attribute__((always_inline))
15 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
16 {
17 	RSEQ_INJECT_C(9)
18 
19 	__asm__ __volatile__ goto (
20 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
21 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
22 #ifdef RSEQ_COMPARE_TWICE
23 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
24 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
25 #endif
26 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
27 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
28 		RSEQ_INJECT_ASM(3)
29 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
30 		RSEQ_INJECT_ASM(4)
31 #ifdef RSEQ_COMPARE_TWICE
32 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
33 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
34 #endif
35 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
36 		RSEQ_INJECT_ASM(5)
37 		RSEQ_ASM_DEFINE_ABORT(4, abort)
38 		: /* gcc asm goto does not allow outputs */
39 		: [cpu_id]		"r" (cpu),
40 		  [current_cpu_id]	"Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
41 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
42 		  [v]			"Qo" (*v),
43 		  [expect]		"r" (expect),
44 		  [newv]		"r" (newv)
45 		  RSEQ_INJECT_INPUT
46 		: "memory", RSEQ_ASM_TMP_REG
47 		: abort, cmpfail
48 #ifdef RSEQ_COMPARE_TWICE
49 		  , error1, error2
50 #endif
51 	);
52 	rseq_after_asm_goto();
53 	return 0;
54 abort:
55 	rseq_after_asm_goto();
56 	RSEQ_INJECT_FAILED
57 	return -1;
58 cmpfail:
59 	rseq_after_asm_goto();
60 	return 1;
61 #ifdef RSEQ_COMPARE_TWICE
62 error1:
63 	rseq_after_asm_goto();
64 	rseq_bug("cpu_id comparison failed");
65 error2:
66 	rseq_after_asm_goto();
67 	rseq_bug("expected value comparison failed");
68 #endif
69 }
70 
71 static inline __attribute__((always_inline))
72 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
73 			       long voffp, intptr_t *load, int cpu)
74 {
75 	RSEQ_INJECT_C(9)
76 
77 	__asm__ __volatile__ goto (
78 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
79 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
80 #ifdef RSEQ_COMPARE_TWICE
81 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
82 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
83 #endif
84 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
85 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
86 		RSEQ_INJECT_ASM(3)
87 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
88 		RSEQ_INJECT_ASM(4)
89 #ifdef RSEQ_COMPARE_TWICE
90 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
91 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
92 #endif
93 		RSEQ_ASM_OP_R_LOAD(v)
94 		RSEQ_ASM_OP_R_STORE(load)
95 		RSEQ_ASM_OP_R_LOAD_OFF(voffp)
96 		RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
97 		RSEQ_INJECT_ASM(5)
98 		RSEQ_ASM_DEFINE_ABORT(4, abort)
99 		: /* gcc asm goto does not allow outputs */
100 		: [cpu_id]		"r" (cpu),
101 		  [current_cpu_id]	"Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
102 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
103 		  [v]			"Qo" (*v),
104 		  [expectnot]		"r" (expectnot),
105 		  [load]		"Qo" (*load),
106 		  [voffp]		"r" (voffp)
107 		  RSEQ_INJECT_INPUT
108 		: "memory", RSEQ_ASM_TMP_REG
109 		: abort, cmpfail
110 #ifdef RSEQ_COMPARE_TWICE
111 		  , error1, error2
112 #endif
113 	);
114 	rseq_after_asm_goto();
115 	return 0;
116 abort:
117 	rseq_after_asm_goto();
118 	RSEQ_INJECT_FAILED
119 	return -1;
120 cmpfail:
121 	rseq_after_asm_goto();
122 	return 1;
123 #ifdef RSEQ_COMPARE_TWICE
124 error1:
125 	rseq_after_asm_goto();
126 	rseq_bug("cpu_id comparison failed");
127 error2:
128 	rseq_after_asm_goto();
129 	rseq_bug("expected value comparison failed");
130 #endif
131 }
132 
133 static inline __attribute__((always_inline))
134 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
135 {
136 	RSEQ_INJECT_C(9)
137 
138 	__asm__ __volatile__ goto (
139 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
140 #ifdef RSEQ_COMPARE_TWICE
141 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
142 #endif
143 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
144 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
145 		RSEQ_INJECT_ASM(3)
146 #ifdef RSEQ_COMPARE_TWICE
147 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
148 #endif
149 		RSEQ_ASM_OP_R_LOAD(v)
150 		RSEQ_ASM_OP_R_ADD(count)
151 		RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
152 		RSEQ_INJECT_ASM(4)
153 		RSEQ_ASM_DEFINE_ABORT(4, abort)
154 		: /* gcc asm goto does not allow outputs */
155 		: [cpu_id]		"r" (cpu),
156 		  [current_cpu_id]	"Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
157 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
158 		  [v]			"Qo" (*v),
159 		  [count]		"r" (count)
160 		  RSEQ_INJECT_INPUT
161 		: "memory", RSEQ_ASM_TMP_REG
162 		: abort
163 #ifdef RSEQ_COMPARE_TWICE
164 		  , error1
165 #endif
166 	);
167 	rseq_after_asm_goto();
168 	return 0;
169 abort:
170 	rseq_after_asm_goto();
171 	RSEQ_INJECT_FAILED
172 	return -1;
173 #ifdef RSEQ_COMPARE_TWICE
174 error1:
175 	rseq_after_asm_goto();
176 	rseq_bug("cpu_id comparison failed");
177 #endif
178 }
179 
180 static inline __attribute__((always_inline))
181 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
182 			      intptr_t *v2, intptr_t expect2,
183 			      intptr_t newv, int cpu)
184 {
185 	RSEQ_INJECT_C(9)
186 
187 	__asm__ __volatile__ goto (
188 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
189 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
190 #ifdef RSEQ_COMPARE_TWICE
191 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
192 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
193 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
194 #endif
195 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
196 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
197 		RSEQ_INJECT_ASM(3)
198 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
199 		RSEQ_INJECT_ASM(4)
200 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
201 		RSEQ_INJECT_ASM(5)
202 #ifdef RSEQ_COMPARE_TWICE
203 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
204 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
205 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
206 #endif
207 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
208 		RSEQ_INJECT_ASM(6)
209 		RSEQ_ASM_DEFINE_ABORT(4, abort)
210 		: /* gcc asm goto does not allow outputs */
211 		: [cpu_id]		"r" (cpu),
212 		  [current_cpu_id]	"Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
213 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
214 		  [v]			"Qo" (*v),
215 		  [expect]		"r" (expect),
216 		  [v2]			"Qo" (*v2),
217 		  [expect2]		"r" (expect2),
218 		  [newv]		"r" (newv)
219 		  RSEQ_INJECT_INPUT
220 		: "memory", RSEQ_ASM_TMP_REG
221 		: abort, cmpfail
222 #ifdef RSEQ_COMPARE_TWICE
223 		  , error1, error2, error3
224 #endif
225 	);
226 	rseq_after_asm_goto();
227 	return 0;
228 abort:
229 	rseq_after_asm_goto();
230 	RSEQ_INJECT_FAILED
231 	return -1;
232 cmpfail:
233 	rseq_after_asm_goto();
234 	return 1;
235 #ifdef RSEQ_COMPARE_TWICE
236 error1:
237 	rseq_after_asm_goto();
238 	rseq_bug("cpu_id comparison failed");
239 error2:
240 	rseq_after_asm_goto();
241 	rseq_bug("expected value comparison failed");
242 error3:
243 	rseq_after_asm_goto();
244 	rseq_bug("2nd expected value comparison failed");
245 #endif
246 }
247 
248 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
249 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
250 
251 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
252 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
253 
254 static inline __attribute__((always_inline))
255 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
256 				 intptr_t *v2, intptr_t newv2,
257 				 intptr_t newv, int cpu)
258 {
259 	RSEQ_INJECT_C(9)
260 
261 	__asm__ __volatile__ goto (
262 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
263 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
264 #ifdef RSEQ_COMPARE_TWICE
265 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
266 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
267 #endif
268 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
269 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
270 		RSEQ_INJECT_ASM(3)
271 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
272 		RSEQ_INJECT_ASM(4)
273 #ifdef RSEQ_COMPARE_TWICE
274 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
275 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
276 #endif
277 		RSEQ_ASM_OP_STORE(newv2, v2)
278 		RSEQ_INJECT_ASM(5)
279 #ifdef RSEQ_TEMPLATE_MO_RELEASE
280 		RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
281 #else
282 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
283 #endif
284 		RSEQ_INJECT_ASM(6)
285 		RSEQ_ASM_DEFINE_ABORT(4, abort)
286 		: /* gcc asm goto does not allow outputs */
287 		: [cpu_id]		"r" (cpu),
288 		  [current_cpu_id]	"Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
289 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
290 		  [expect]		"r" (expect),
291 		  [v]			"Qo" (*v),
292 		  [newv]		"r" (newv),
293 		  [v2]			"Qo" (*v2),
294 		  [newv2]		"r" (newv2)
295 		  RSEQ_INJECT_INPUT
296 		: "memory", RSEQ_ASM_TMP_REG
297 		: abort, cmpfail
298 #ifdef RSEQ_COMPARE_TWICE
299 		  , error1, error2
300 #endif
301 	);
302 	rseq_after_asm_goto();
303 	return 0;
304 abort:
305 	rseq_after_asm_goto();
306 	RSEQ_INJECT_FAILED
307 	return -1;
308 cmpfail:
309 	rseq_after_asm_goto();
310 	return 1;
311 #ifdef RSEQ_COMPARE_TWICE
312 error1:
313 	rseq_after_asm_goto();
314 	rseq_bug("cpu_id comparison failed");
315 error2:
316 	rseq_after_asm_goto();
317 	rseq_bug("expected value comparison failed");
318 #endif
319 }
320 
321 static inline __attribute__((always_inline))
322 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
323 				 void *dst, void *src, size_t len,
324 				 intptr_t newv, int cpu)
325 {
326 	RSEQ_INJECT_C(9)
327 
328 	__asm__ __volatile__ goto (
329 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
330 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
331 #ifdef RSEQ_COMPARE_TWICE
332 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
333 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
334 #endif
335 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
336 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
337 		RSEQ_INJECT_ASM(3)
338 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
339 		RSEQ_INJECT_ASM(4)
340 #ifdef RSEQ_COMPARE_TWICE
341 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
342 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
343 #endif
344 		RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
345 		RSEQ_INJECT_ASM(5)
346 #ifdef RSEQ_TEMPLATE_MO_RELEASE
347 		RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
348 #else
349 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
350 #endif
351 		RSEQ_INJECT_ASM(6)
352 		RSEQ_ASM_DEFINE_ABORT(4, abort)
353 		: /* gcc asm goto does not allow outputs */
354 		: [cpu_id]		"r" (cpu),
355 		  [current_cpu_id]	"Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
356 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
357 		  [expect]		"r" (expect),
358 		  [v]			"Qo" (*v),
359 		  [newv]		"r" (newv),
360 		  [dst]			"r" (dst),
361 		  [src]			"r" (src),
362 		  [len]			"r" (len)
363 		  RSEQ_INJECT_INPUT
364 		: "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
365 		: abort, cmpfail
366 #ifdef RSEQ_COMPARE_TWICE
367 		  , error1, error2
368 #endif
369 	);
370 	rseq_after_asm_goto();
371 	return 0;
372 abort:
373 	rseq_after_asm_goto();
374 	RSEQ_INJECT_FAILED
375 	return -1;
376 cmpfail:
377 	rseq_after_asm_goto();
378 	return 1;
379 #ifdef RSEQ_COMPARE_TWICE
380 error1:
381 	rseq_after_asm_goto();
382 	rseq_bug("cpu_id comparison failed");
383 error2:
384 	rseq_after_asm_goto();
385 	rseq_bug("expected value comparison failed");
386 #endif
387 }
388 
389 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
390 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
391 
392 #include "rseq-bits-reset.h"
393