1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * rseq-ppc-bits.h
4  *
5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6  * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7  */
8 
9 #include "rseq-bits-template.h"
10 
11 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
12 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
13 
14 static inline __attribute__((always_inline))
15 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
16 {
17 	RSEQ_INJECT_C(9)
18 
19 	__asm__ __volatile__ goto (
20 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
21 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
22 #ifdef RSEQ_COMPARE_TWICE
23 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
24 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
25 #endif
26 		/* Start rseq by storing table entry pointer into rseq_cs. */
27 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
28 		/* cmp cpuid */
29 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
30 		RSEQ_INJECT_ASM(3)
31 		/* cmp @v equal to @expect */
32 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
33 		RSEQ_INJECT_ASM(4)
34 #ifdef RSEQ_COMPARE_TWICE
35 		/* cmp cpuid */
36 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
37 		/* cmp @v equal to @expect */
38 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
39 #endif
40 		/* final store */
41 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
42 		RSEQ_INJECT_ASM(5)
43 		RSEQ_ASM_DEFINE_ABORT(4, abort)
44 		: /* gcc asm goto does not allow outputs */
45 		: [cpu_id]		"r" (cpu),
46 		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
47 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
48 		  [v]			"m" (*v),
49 		  [expect]		"r" (expect),
50 		  [newv]		"r" (newv)
51 		  RSEQ_INJECT_INPUT
52 		: "memory", "cc", "r17"
53 		  RSEQ_INJECT_CLOBBER
54 		: abort, cmpfail
55 #ifdef RSEQ_COMPARE_TWICE
56 		  , error1, error2
57 #endif
58 	);
59 	rseq_after_asm_goto();
60 	return 0;
61 abort:
62 	rseq_after_asm_goto();
63 	RSEQ_INJECT_FAILED
64 	return -1;
65 cmpfail:
66 	rseq_after_asm_goto();
67 	return 1;
68 #ifdef RSEQ_COMPARE_TWICE
69 error1:
70 	rseq_after_asm_goto();
71 	rseq_bug("cpu_id comparison failed");
72 error2:
73 	rseq_after_asm_goto();
74 	rseq_bug("expected value comparison failed");
75 #endif
76 }
77 
78 static inline __attribute__((always_inline))
79 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
80 			       long voffp, intptr_t *load, int cpu)
81 {
82 	RSEQ_INJECT_C(9)
83 
84 	__asm__ __volatile__ goto (
85 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
86 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
87 #ifdef RSEQ_COMPARE_TWICE
88 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
89 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
90 #endif
91 		/* Start rseq by storing table entry pointer into rseq_cs. */
92 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
93 		/* cmp cpuid */
94 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
95 		RSEQ_INJECT_ASM(3)
96 		/* cmp @v not equal to @expectnot */
97 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
98 		RSEQ_INJECT_ASM(4)
99 #ifdef RSEQ_COMPARE_TWICE
100 		/* cmp cpuid */
101 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
102 		/* cmp @v not equal to @expectnot */
103 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
104 #endif
105 		/* load the value of @v */
106 		RSEQ_ASM_OP_R_LOAD(v)
107 		/* store it in @load */
108 		RSEQ_ASM_OP_R_STORE(load)
109 		/* dereference voffp(v) */
110 		RSEQ_ASM_OP_R_LOADX(voffp)
111 		/* final store the value at voffp(v) */
112 		RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
113 		RSEQ_INJECT_ASM(5)
114 		RSEQ_ASM_DEFINE_ABORT(4, abort)
115 		: /* gcc asm goto does not allow outputs */
116 		: [cpu_id]		"r" (cpu),
117 		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
118 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
119 		  /* final store input */
120 		  [v]			"m" (*v),
121 		  [expectnot]		"r" (expectnot),
122 		  [voffp]		"b" (voffp),
123 		  [load]		"m" (*load)
124 		  RSEQ_INJECT_INPUT
125 		: "memory", "cc", "r17"
126 		  RSEQ_INJECT_CLOBBER
127 		: abort, cmpfail
128 #ifdef RSEQ_COMPARE_TWICE
129 		  , error1, error2
130 #endif
131 	);
132 	rseq_after_asm_goto();
133 	return 0;
134 abort:
135 	rseq_after_asm_goto();
136 	RSEQ_INJECT_FAILED
137 	return -1;
138 cmpfail:
139 	rseq_after_asm_goto();
140 	return 1;
141 #ifdef RSEQ_COMPARE_TWICE
142 error1:
143 	rseq_after_asm_goto();
144 	rseq_bug("cpu_id comparison failed");
145 error2:
146 	rseq_after_asm_goto();
147 	rseq_bug("expected value comparison failed");
148 #endif
149 }
150 
151 static inline __attribute__((always_inline))
152 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
153 {
154 	RSEQ_INJECT_C(9)
155 
156 	__asm__ __volatile__ goto (
157 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
158 #ifdef RSEQ_COMPARE_TWICE
159 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
160 #endif
161 		/* Start rseq by storing table entry pointer into rseq_cs. */
162 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
163 		/* cmp cpuid */
164 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
165 		RSEQ_INJECT_ASM(3)
166 #ifdef RSEQ_COMPARE_TWICE
167 		/* cmp cpuid */
168 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
169 #endif
170 		/* load the value of @v */
171 		RSEQ_ASM_OP_R_LOAD(v)
172 		/* add @count to it */
173 		RSEQ_ASM_OP_R_ADD(count)
174 		/* final store */
175 		RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
176 		RSEQ_INJECT_ASM(4)
177 		RSEQ_ASM_DEFINE_ABORT(4, abort)
178 		: /* gcc asm goto does not allow outputs */
179 		: [cpu_id]		"r" (cpu),
180 		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
181 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
182 		  /* final store input */
183 		  [v]			"m" (*v),
184 		  [count]		"r" (count)
185 		  RSEQ_INJECT_INPUT
186 		: "memory", "cc", "r17"
187 		  RSEQ_INJECT_CLOBBER
188 		: abort
189 #ifdef RSEQ_COMPARE_TWICE
190 		  , error1
191 #endif
192 	);
193 	rseq_after_asm_goto();
194 	return 0;
195 abort:
196 	rseq_after_asm_goto();
197 	RSEQ_INJECT_FAILED
198 	return -1;
199 #ifdef RSEQ_COMPARE_TWICE
200 error1:
201 	rseq_after_asm_goto();
202 	rseq_bug("cpu_id comparison failed");
203 #endif
204 }
205 
206 static inline __attribute__((always_inline))
207 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
208 			      intptr_t *v2, intptr_t expect2,
209 			      intptr_t newv, int cpu)
210 {
211 	RSEQ_INJECT_C(9)
212 
213 	__asm__ __volatile__ goto (
214 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
215 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
216 #ifdef RSEQ_COMPARE_TWICE
217 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
218 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
219 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
220 #endif
221 		/* Start rseq by storing table entry pointer into rseq_cs. */
222 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
223 		/* cmp cpuid */
224 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
225 		RSEQ_INJECT_ASM(3)
226 		/* cmp @v equal to @expect */
227 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
228 		RSEQ_INJECT_ASM(4)
229 		/* cmp @v2 equal to @expct2 */
230 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
231 		RSEQ_INJECT_ASM(5)
232 #ifdef RSEQ_COMPARE_TWICE
233 		/* cmp cpuid */
234 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
235 		/* cmp @v equal to @expect */
236 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
237 		/* cmp @v2 equal to @expct2 */
238 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
239 #endif
240 		/* final store */
241 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
242 		RSEQ_INJECT_ASM(6)
243 		RSEQ_ASM_DEFINE_ABORT(4, abort)
244 		: /* gcc asm goto does not allow outputs */
245 		: [cpu_id]		"r" (cpu),
246 		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
247 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
248 		  /* cmp2 input */
249 		  [v2]			"m" (*v2),
250 		  [expect2]		"r" (expect2),
251 		  /* final store input */
252 		  [v]			"m" (*v),
253 		  [expect]		"r" (expect),
254 		  [newv]		"r" (newv)
255 		  RSEQ_INJECT_INPUT
256 		: "memory", "cc", "r17"
257 		  RSEQ_INJECT_CLOBBER
258 		: abort, cmpfail
259 #ifdef RSEQ_COMPARE_TWICE
260 		  , error1, error2, error3
261 #endif
262 	);
263 	rseq_after_asm_goto();
264 	return 0;
265 abort:
266 	rseq_after_asm_goto();
267 	RSEQ_INJECT_FAILED
268 	return -1;
269 cmpfail:
270 	rseq_after_asm_goto();
271 	return 1;
272 #ifdef RSEQ_COMPARE_TWICE
273 error1:
274 	rseq_after_asm_goto();
275 	rseq_bug("cpu_id comparison failed");
276 error2:
277 	rseq_after_asm_goto();
278 	rseq_bug("1st expected value comparison failed");
279 error3:
280 	rseq_after_asm_goto();
281 	rseq_bug("2nd expected value comparison failed");
282 #endif
283 }
284 
285 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
286 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
287 
288 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
289 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
290 
291 static inline __attribute__((always_inline))
292 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
293 				 intptr_t *v2, intptr_t newv2,
294 				 intptr_t newv, int cpu)
295 {
296 	RSEQ_INJECT_C(9)
297 
298 	__asm__ __volatile__ goto (
299 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
300 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
301 #ifdef RSEQ_COMPARE_TWICE
302 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
303 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
304 #endif
305 		/* Start rseq by storing table entry pointer into rseq_cs. */
306 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
307 		/* cmp cpuid */
308 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
309 		RSEQ_INJECT_ASM(3)
310 		/* cmp @v equal to @expect */
311 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
312 		RSEQ_INJECT_ASM(4)
313 #ifdef RSEQ_COMPARE_TWICE
314 		/* cmp cpuid */
315 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
316 		/* cmp @v equal to @expect */
317 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
318 #endif
319 		/* try store */
320 		RSEQ_ASM_OP_STORE(newv2, v2)
321 		RSEQ_INJECT_ASM(5)
322 #ifdef RSEQ_TEMPLATE_MO_RELEASE
323 		/* for 'release' */
324 		"lwsync\n\t"
325 #endif
326 		/* final store */
327 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
328 		RSEQ_INJECT_ASM(6)
329 		RSEQ_ASM_DEFINE_ABORT(4, abort)
330 		: /* gcc asm goto does not allow outputs */
331 		: [cpu_id]		"r" (cpu),
332 		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
333 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
334 		  /* try store input */
335 		  [v2]			"m" (*v2),
336 		  [newv2]		"r" (newv2),
337 		  /* final store input */
338 		  [v]			"m" (*v),
339 		  [expect]		"r" (expect),
340 		  [newv]		"r" (newv)
341 		  RSEQ_INJECT_INPUT
342 		: "memory", "cc", "r17"
343 		  RSEQ_INJECT_CLOBBER
344 		: abort, cmpfail
345 #ifdef RSEQ_COMPARE_TWICE
346 		  , error1, error2
347 #endif
348 	);
349 	rseq_after_asm_goto();
350 	return 0;
351 abort:
352 	rseq_after_asm_goto();
353 	RSEQ_INJECT_FAILED
354 	return -1;
355 cmpfail:
356 	rseq_after_asm_goto();
357 	return 1;
358 #ifdef RSEQ_COMPARE_TWICE
359 error1:
360 	rseq_after_asm_goto();
361 	rseq_bug("cpu_id comparison failed");
362 error2:
363 	rseq_after_asm_goto();
364 	rseq_bug("expected value comparison failed");
365 #endif
366 }
367 
368 static inline __attribute__((always_inline))
369 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
370 				 void *dst, void *src, size_t len,
371 				 intptr_t newv, int cpu)
372 {
373 	RSEQ_INJECT_C(9)
374 
375 	__asm__ __volatile__ goto (
376 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
377 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
378 #ifdef RSEQ_COMPARE_TWICE
379 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
380 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
381 #endif
382 		/* setup for mempcy */
383 		"mr %%r19, %[len]\n\t"
384 		"mr %%r20, %[src]\n\t"
385 		"mr %%r21, %[dst]\n\t"
386 		/* Start rseq by storing table entry pointer into rseq_cs. */
387 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
388 		/* cmp cpuid */
389 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
390 		RSEQ_INJECT_ASM(3)
391 		/* cmp @v equal to @expect */
392 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
393 		RSEQ_INJECT_ASM(4)
394 #ifdef RSEQ_COMPARE_TWICE
395 		/* cmp cpuid */
396 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
397 		/* cmp @v equal to @expect */
398 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
399 #endif
400 		/* try memcpy */
401 		RSEQ_ASM_OP_R_MEMCPY()
402 		RSEQ_INJECT_ASM(5)
403 #ifdef RSEQ_TEMPLATE_MO_RELEASE
404 		/* for 'release' */
405 		"lwsync\n\t"
406 #endif
407 		/* final store */
408 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
409 		RSEQ_INJECT_ASM(6)
410 		/* teardown */
411 		RSEQ_ASM_DEFINE_ABORT(4, abort)
412 		: /* gcc asm goto does not allow outputs */
413 		: [cpu_id]		"r" (cpu),
414 		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
415 		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
416 		  /* final store input */
417 		  [v]			"m" (*v),
418 		  [expect]		"r" (expect),
419 		  [newv]		"r" (newv),
420 		  /* try memcpy input */
421 		  [dst]			"r" (dst),
422 		  [src]			"r" (src),
423 		  [len]			"r" (len)
424 		  RSEQ_INJECT_INPUT
425 		: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
426 		  RSEQ_INJECT_CLOBBER
427 		: abort, cmpfail
428 #ifdef RSEQ_COMPARE_TWICE
429 		  , error1, error2
430 #endif
431 	);
432 	rseq_after_asm_goto();
433 	return 0;
434 abort:
435 	rseq_after_asm_goto();
436 	RSEQ_INJECT_FAILED
437 	return -1;
438 cmpfail:
439 	rseq_after_asm_goto();
440 	return 1;
441 #ifdef RSEQ_COMPARE_TWICE
442 error1:
443 	rseq_after_asm_goto();
444 	rseq_bug("cpu_id comparison failed");
445 error2:
446 	rseq_after_asm_goto();
447 	rseq_bug("expected value comparison failed");
448 #endif
449 }
450 
451 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
452 	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
453 
454 #include "rseq-bits-reset.h"
455