1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * rseq.h
4  *
5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6  */
7 
8 #ifndef RSEQ_H
9 #define RSEQ_H
10 
11 #include <stdint.h>
12 #include <stdbool.h>
13 #include <pthread.h>
14 #include <signal.h>
15 #include <sched.h>
16 #include <errno.h>
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <stddef.h>
20 #include "rseq-abi.h"
21 #include "compiler.h"
22 
23 #ifndef rseq_sizeof_field
24 #define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
25 #endif
26 
27 #ifndef rseq_offsetofend
28 #define rseq_offsetofend(TYPE, MEMBER) \
29 	(offsetof(TYPE, MEMBER)	+ rseq_sizeof_field(TYPE, MEMBER))
30 #endif
31 
32 /*
33  * Empty code injection macros, override when testing.
34  * It is important to consider that the ASM injection macros need to be
35  * fully reentrant (e.g. do not modify the stack).
36  */
37 #ifndef RSEQ_INJECT_ASM
38 #define RSEQ_INJECT_ASM(n)
39 #endif
40 
41 #ifndef RSEQ_INJECT_C
42 #define RSEQ_INJECT_C(n)
43 #endif
44 
45 #ifndef RSEQ_INJECT_INPUT
46 #define RSEQ_INJECT_INPUT
47 #endif
48 
49 #ifndef RSEQ_INJECT_CLOBBER
50 #define RSEQ_INJECT_CLOBBER
51 #endif
52 
53 #ifndef RSEQ_INJECT_FAILED
54 #define RSEQ_INJECT_FAILED
55 #endif
56 
57 #include "rseq-thread-pointer.h"
58 
59 /* Offset from the thread pointer to the rseq area. */
60 extern ptrdiff_t rseq_offset;
61 
62 /*
63  * Size of the registered rseq area. 0 if the registration was
64  * unsuccessful.
65  */
66 extern unsigned int rseq_size;
67 
68 /* Flags used during rseq registration. */
69 extern unsigned int rseq_flags;
70 
71 /*
72  * rseq feature size supported by the kernel. 0 if the registration was
73  * unsuccessful.
74  */
75 extern unsigned int rseq_feature_size;
76 
77 enum rseq_mo {
78 	RSEQ_MO_RELAXED = 0,
79 	RSEQ_MO_CONSUME = 1,	/* Unused */
80 	RSEQ_MO_ACQUIRE = 2,	/* Unused */
81 	RSEQ_MO_RELEASE = 3,
82 	RSEQ_MO_ACQ_REL = 4,	/* Unused */
83 	RSEQ_MO_SEQ_CST = 5,	/* Unused */
84 };
85 
86 enum rseq_percpu_mode {
87 	RSEQ_PERCPU_CPU_ID = 0,
88 	RSEQ_PERCPU_MM_CID = 1,
89 };
90 
91 static inline struct rseq_abi *rseq_get_abi(void)
92 {
93 	return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset);
94 }
95 
96 #define rseq_likely(x)		__builtin_expect(!!(x), 1)
97 #define rseq_unlikely(x)	__builtin_expect(!!(x), 0)
98 #define rseq_barrier()		__asm__ __volatile__("" : : : "memory")
99 
100 #define RSEQ_ACCESS_ONCE(x)	(*(__volatile__  __typeof__(x) *)&(x))
101 #define RSEQ_WRITE_ONCE(x, v)	__extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); })
102 #define RSEQ_READ_ONCE(x)	RSEQ_ACCESS_ONCE(x)
103 
104 #define __rseq_str_1(x)	#x
105 #define __rseq_str(x)		__rseq_str_1(x)
106 
107 #define rseq_log(fmt, args...)						       \
108 	fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \
109 		## args, __func__)
110 
111 #define rseq_bug(fmt, args...)		\
112 	do {				\
113 		rseq_log(fmt, ##args);	\
114 		abort();		\
115 	} while (0)
116 
117 #if defined(__x86_64__) || defined(__i386__)
118 #include <rseq-x86.h>
119 #elif defined(__ARMEL__)
120 #include <rseq-arm.h>
121 #elif defined (__AARCH64EL__)
122 #include <rseq-arm64.h>
123 #elif defined(__PPC__)
124 #include <rseq-ppc.h>
125 #elif defined(__mips__)
126 #include <rseq-mips.h>
127 #elif defined(__s390__)
128 #include <rseq-s390.h>
129 #elif defined(__riscv)
130 #include <rseq-riscv.h>
131 #else
132 #error unsupported target
133 #endif
134 
135 /*
136  * Register rseq for the current thread. This needs to be called once
137  * by any thread which uses restartable sequences, before they start
138  * using restartable sequences, to ensure restartable sequences
139  * succeed. A restartable sequence executed from a non-registered
140  * thread will always fail.
141  */
142 int rseq_register_current_thread(void);
143 
144 /*
145  * Unregister rseq for current thread.
146  */
147 int rseq_unregister_current_thread(void);
148 
149 /*
150  * Restartable sequence fallback for reading the current CPU number.
151  */
152 int32_t rseq_fallback_current_cpu(void);
153 
154 /*
155  * Restartable sequence fallback for reading the current node number.
156  */
157 int32_t rseq_fallback_current_node(void);
158 
159 /*
160  * Values returned can be either the current CPU number, -1 (rseq is
161  * uninitialized), or -2 (rseq initialization has failed).
162  */
163 static inline int32_t rseq_current_cpu_raw(void)
164 {
165 	return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id);
166 }
167 
168 /*
169  * Returns a possible CPU number, which is typically the current CPU.
170  * The returned CPU number can be used to prepare for an rseq critical
171  * section, which will confirm whether the cpu number is indeed the
172  * current one, and whether rseq is initialized.
173  *
174  * The CPU number returned by rseq_cpu_start should always be validated
175  * by passing it to a rseq asm sequence, or by comparing it to the
176  * return value of rseq_current_cpu_raw() if the rseq asm sequence
177  * does not need to be invoked.
178  */
179 static inline uint32_t rseq_cpu_start(void)
180 {
181 	return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start);
182 }
183 
184 static inline uint32_t rseq_current_cpu(void)
185 {
186 	int32_t cpu;
187 
188 	cpu = rseq_current_cpu_raw();
189 	if (rseq_unlikely(cpu < 0))
190 		cpu = rseq_fallback_current_cpu();
191 	return cpu;
192 }
193 
194 static inline bool rseq_node_id_available(void)
195 {
196 	return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, node_id);
197 }
198 
199 /*
200  * Current NUMA node number.
201  */
202 static inline uint32_t rseq_current_node_id(void)
203 {
204 	assert(rseq_node_id_available());
205 	return RSEQ_ACCESS_ONCE(rseq_get_abi()->node_id);
206 }
207 
208 static inline bool rseq_mm_cid_available(void)
209 {
210 	return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, mm_cid);
211 }
212 
213 static inline uint32_t rseq_current_mm_cid(void)
214 {
215 	return RSEQ_ACCESS_ONCE(rseq_get_abi()->mm_cid);
216 }
217 
218 static inline void rseq_clear_rseq_cs(void)
219 {
220 	RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0);
221 }
222 
223 /*
224  * rseq_prepare_unload() should be invoked by each thread executing a rseq
225  * critical section at least once between their last critical section and
226  * library unload of the library defining the rseq critical section (struct
227  * rseq_cs) or the code referred to by the struct rseq_cs start_ip and
228  * post_commit_offset fields. This also applies to use of rseq in code
229  * generated by JIT: rseq_prepare_unload() should be invoked at least once by
230  * each thread executing a rseq critical section before reclaim of the memory
231  * holding the struct rseq_cs or reclaim of the code pointed to by struct
232  * rseq_cs start_ip and post_commit_offset fields.
233  */
234 static inline void rseq_prepare_unload(void)
235 {
236 	rseq_clear_rseq_cs();
237 }
238 
239 static inline __attribute__((always_inline))
240 int rseq_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
241 		       intptr_t *v, intptr_t expect,
242 		       intptr_t newv, int cpu)
243 {
244 	if (rseq_mo != RSEQ_MO_RELAXED)
245 		return -1;
246 	switch (percpu_mode) {
247 	case RSEQ_PERCPU_CPU_ID:
248 		return rseq_cmpeqv_storev_relaxed_cpu_id(v, expect, newv, cpu);
249 	case RSEQ_PERCPU_MM_CID:
250 		return rseq_cmpeqv_storev_relaxed_mm_cid(v, expect, newv, cpu);
251 	}
252 	return -1;
253 }
254 
255 /*
256  * Compare @v against @expectnot. When it does _not_ match, load @v
257  * into @load, and store the content of *@v + voffp into @v.
258  */
259 static inline __attribute__((always_inline))
260 int rseq_cmpnev_storeoffp_load(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
261 			       intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load,
262 			       int cpu)
263 {
264 	if (rseq_mo != RSEQ_MO_RELAXED)
265 		return -1;
266 	switch (percpu_mode) {
267 	case RSEQ_PERCPU_CPU_ID:
268 		return rseq_cmpnev_storeoffp_load_relaxed_cpu_id(v, expectnot, voffp, load, cpu);
269 	case RSEQ_PERCPU_MM_CID:
270 		return rseq_cmpnev_storeoffp_load_relaxed_mm_cid(v, expectnot, voffp, load, cpu);
271 	}
272 	return -1;
273 }
274 
275 static inline __attribute__((always_inline))
276 int rseq_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
277 	      intptr_t *v, intptr_t count, int cpu)
278 {
279 	if (rseq_mo != RSEQ_MO_RELAXED)
280 		return -1;
281 	switch (percpu_mode) {
282 	case RSEQ_PERCPU_CPU_ID:
283 		return rseq_addv_relaxed_cpu_id(v, count, cpu);
284 	case RSEQ_PERCPU_MM_CID:
285 		return rseq_addv_relaxed_mm_cid(v, count, cpu);
286 	}
287 	return -1;
288 }
289 
290 #ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
291 /*
292  *   pval = *(ptr+off)
293  *  *pval += inc;
294  */
295 static inline __attribute__((always_inline))
296 int rseq_offset_deref_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
297 			   intptr_t *ptr, long off, intptr_t inc, int cpu)
298 {
299 	if (rseq_mo != RSEQ_MO_RELAXED)
300 		return -1;
301 	switch (percpu_mode) {
302 	case RSEQ_PERCPU_CPU_ID:
303 		return rseq_offset_deref_addv_relaxed_cpu_id(ptr, off, inc, cpu);
304 	case RSEQ_PERCPU_MM_CID:
305 		return rseq_offset_deref_addv_relaxed_mm_cid(ptr, off, inc, cpu);
306 	}
307 	return -1;
308 }
309 #endif
310 
311 static inline __attribute__((always_inline))
312 int rseq_cmpeqv_trystorev_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
313 				 intptr_t *v, intptr_t expect,
314 				 intptr_t *v2, intptr_t newv2,
315 				 intptr_t newv, int cpu)
316 {
317 	switch (rseq_mo) {
318 	case RSEQ_MO_RELAXED:
319 		switch (percpu_mode) {
320 		case RSEQ_PERCPU_CPU_ID:
321 			return rseq_cmpeqv_trystorev_storev_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu);
322 		case RSEQ_PERCPU_MM_CID:
323 			return rseq_cmpeqv_trystorev_storev_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu);
324 		}
325 		return -1;
326 	case RSEQ_MO_RELEASE:
327 		switch (percpu_mode) {
328 		case RSEQ_PERCPU_CPU_ID:
329 			return rseq_cmpeqv_trystorev_storev_release_cpu_id(v, expect, v2, newv2, newv, cpu);
330 		case RSEQ_PERCPU_MM_CID:
331 			return rseq_cmpeqv_trystorev_storev_release_mm_cid(v, expect, v2, newv2, newv, cpu);
332 		}
333 		return -1;
334 	default:
335 		return -1;
336 	}
337 }
338 
339 static inline __attribute__((always_inline))
340 int rseq_cmpeqv_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
341 			      intptr_t *v, intptr_t expect,
342 			      intptr_t *v2, intptr_t expect2,
343 			      intptr_t newv, int cpu)
344 {
345 	if (rseq_mo != RSEQ_MO_RELAXED)
346 		return -1;
347 	switch (percpu_mode) {
348 	case RSEQ_PERCPU_CPU_ID:
349 		return rseq_cmpeqv_cmpeqv_storev_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu);
350 	case RSEQ_PERCPU_MM_CID:
351 		return rseq_cmpeqv_cmpeqv_storev_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu);
352 	}
353 	return -1;
354 }
355 
356 static inline __attribute__((always_inline))
357 int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
358 				 intptr_t *v, intptr_t expect,
359 				 void *dst, void *src, size_t len,
360 				 intptr_t newv, int cpu)
361 {
362 	switch (rseq_mo) {
363 	case RSEQ_MO_RELAXED:
364 		switch (percpu_mode) {
365 		case RSEQ_PERCPU_CPU_ID:
366 			return rseq_cmpeqv_trymemcpy_storev_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu);
367 		case RSEQ_PERCPU_MM_CID:
368 			return rseq_cmpeqv_trymemcpy_storev_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu);
369 		}
370 		return -1;
371 	case RSEQ_MO_RELEASE:
372 		switch (percpu_mode) {
373 		case RSEQ_PERCPU_CPU_ID:
374 			return rseq_cmpeqv_trymemcpy_storev_release_cpu_id(v, expect, dst, src, len, newv, cpu);
375 		case RSEQ_PERCPU_MM_CID:
376 			return rseq_cmpeqv_trymemcpy_storev_release_mm_cid(v, expect, dst, src, len, newv, cpu);
377 		}
378 		return -1;
379 	default:
380 		return -1;
381 	}
382 }
383 
384 #endif  /* RSEQ_H_ */
385