1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2016 Red Hat, Inc.
4 * Author: Michael S. Tsirkin <mst@redhat.com>
5 *
6 * Common macros and functions for ring benchmarking.
7 */
8 #ifndef MAIN_H
9 #define MAIN_H
10
11 #include <assert.h>
12 #include <stdbool.h>
13
14 extern int param;
15
16 extern bool do_exit;
17
18 #if defined(__x86_64__) || defined(__i386__)
19 #include "x86intrin.h"
20
wait_cycles(unsigned long long cycles)21 static inline void wait_cycles(unsigned long long cycles)
22 {
23 unsigned long long t;
24
25 t = __rdtsc();
26 while (__rdtsc() - t < cycles) {}
27 }
28
29 #define VMEXIT_CYCLES 500
30 #define VMENTRY_CYCLES 500
31
32 #elif defined(__s390x__)
wait_cycles(unsigned long long cycles)33 static inline void wait_cycles(unsigned long long cycles)
34 {
35 asm volatile("0: brctg %0,0b" : : "d" (cycles));
36 }
37
38 /* tweak me */
39 #define VMEXIT_CYCLES 200
40 #define VMENTRY_CYCLES 200
41
42 #else
wait_cycles(unsigned long long cycles)43 static inline void wait_cycles(unsigned long long cycles)
44 {
45 _Exit(5);
46 }
47 #define VMEXIT_CYCLES 0
48 #define VMENTRY_CYCLES 0
49 #endif
50
vmexit(void)51 static inline void vmexit(void)
52 {
53 if (!do_exit)
54 return;
55
56 wait_cycles(VMEXIT_CYCLES);
57 }
vmentry(void)58 static inline void vmentry(void)
59 {
60 if (!do_exit)
61 return;
62
63 wait_cycles(VMENTRY_CYCLES);
64 }
65
66 /* implemented by ring */
67 void alloc_ring(void);
68 /* guest side */
69 int add_inbuf(unsigned, void *, void *);
70 void *get_buf(unsigned *, void **);
71 void disable_call();
72 bool used_empty();
73 bool enable_call();
74 void kick_available();
75 /* host side */
76 void disable_kick();
77 bool avail_empty();
78 bool enable_kick();
79 bool use_buf(unsigned *, void **);
80 void call_used();
81
82 /* implemented by main */
83 extern bool do_sleep;
84 void kick(void);
85 void wait_for_kick(void);
86 void call(void);
87 void wait_for_call(void);
88
89 extern unsigned ring_size;
90
91 /* Compiler barrier - similar to what Linux uses */
92 #define barrier() asm volatile("" ::: "memory")
93
94 /* Is there a portable way to do this? */
95 #if defined(__x86_64__) || defined(__i386__)
96 #define cpu_relax() asm ("rep; nop" ::: "memory")
97 #elif defined(__s390x__)
98 #define cpu_relax() barrier()
99 #elif defined(__aarch64__)
100 #define cpu_relax() asm ("yield" ::: "memory")
101 #else
102 #define cpu_relax() assert(0)
103 #endif
104
105 extern bool do_relax;
106
busy_wait(void)107 static inline void busy_wait(void)
108 {
109 if (do_relax)
110 cpu_relax();
111 else
112 /* prevent compiler from removing busy loops */
113 barrier();
114 }
115
116 #if defined(__x86_64__) || defined(__i386__)
117 #define smp_mb() asm volatile("lock; addl $0,-132(%%rsp)" ::: "memory", "cc")
118 #elif defined(__aarch64__)
119 #define smp_mb() asm volatile("dmb ish" ::: "memory")
120 #else
121 /*
122 * Not using __ATOMIC_SEQ_CST since gcc docs say they are only synchronized
123 * with other __ATOMIC_SEQ_CST calls.
124 */
125 #define smp_mb() __sync_synchronize()
126 #endif
127
128 /*
129 * This abuses the atomic builtins for thread fences, and
130 * adds a compiler barrier.
131 */
132 #define smp_release() do { \
133 barrier(); \
134 __atomic_thread_fence(__ATOMIC_RELEASE); \
135 } while (0)
136
137 #define smp_acquire() do { \
138 __atomic_thread_fence(__ATOMIC_ACQUIRE); \
139 barrier(); \
140 } while (0)
141
142 #if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
143 #define smp_wmb() barrier()
144 #elif defined(__aarch64__)
145 #define smp_wmb() asm volatile("dmb ishst" ::: "memory")
146 #else
147 #define smp_wmb() smp_release()
148 #endif
149
150 #ifndef __always_inline
151 #define __always_inline inline __attribute__((always_inline))
152 #endif
153
154 static __always_inline
__read_once_size(const volatile void * p,void * res,int size)155 void __read_once_size(const volatile void *p, void *res, int size)
156 {
157 switch (size) {
158 case 1: *(unsigned char *)res = *(volatile unsigned char *)p; break;
159 case 2: *(unsigned short *)res = *(volatile unsigned short *)p; break;
160 case 4: *(unsigned int *)res = *(volatile unsigned int *)p; break;
161 case 8: *(unsigned long long *)res = *(volatile unsigned long long *)p; break;
162 default:
163 barrier();
164 __builtin_memcpy((void *)res, (const void *)p, size);
165 barrier();
166 }
167 }
168
__write_once_size(volatile void * p,void * res,int size)169 static __always_inline void __write_once_size(volatile void *p, void *res, int size)
170 {
171 switch (size) {
172 case 1: *(volatile unsigned char *)p = *(unsigned char *)res; break;
173 case 2: *(volatile unsigned short *)p = *(unsigned short *)res; break;
174 case 4: *(volatile unsigned int *)p = *(unsigned int *)res; break;
175 case 8: *(volatile unsigned long long *)p = *(unsigned long long *)res; break;
176 default:
177 barrier();
178 __builtin_memcpy((void *)p, (const void *)res, size);
179 barrier();
180 }
181 }
182
183 #ifdef __alpha__
184 #define READ_ONCE(x) \
185 ({ \
186 union { typeof(x) __val; char __c[1]; } __u; \
187 __read_once_size(&(x), __u.__c, sizeof(x)); \
188 smp_mb(); /* Enforce dependency ordering from x */ \
189 __u.__val; \
190 })
191 #else
192 #define READ_ONCE(x) \
193 ({ \
194 union { typeof(x) __val; char __c[1]; } __u; \
195 __read_once_size(&(x), __u.__c, sizeof(x)); \
196 __u.__val; \
197 })
198 #endif
199
200 #define WRITE_ONCE(x, val) \
201 ({ \
202 union { typeof(x) __val; char __c[1]; } __u = \
203 { .__val = (typeof(x)) (val) }; \
204 __write_once_size(&(x), __u.__c, sizeof(x)); \
205 __u.__val; \
206 })
207
208 #endif
209