1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2016 Red Hat, Inc. 4 * Author: Michael S. Tsirkin <mst@redhat.com> 5 * 6 * Common macros and functions for ring benchmarking. 7 */ 8 #ifndef MAIN_H 9 #define MAIN_H 10 11 #include <stdbool.h> 12 13 extern int param; 14 15 extern bool do_exit; 16 17 #if defined(__x86_64__) || defined(__i386__) 18 #include "x86intrin.h" 19 20 static inline void wait_cycles(unsigned long long cycles) 21 { 22 unsigned long long t; 23 24 t = __rdtsc(); 25 while (__rdtsc() - t < cycles) {} 26 } 27 28 #define VMEXIT_CYCLES 500 29 #define VMENTRY_CYCLES 500 30 31 #elif defined(__s390x__) 32 static inline void wait_cycles(unsigned long long cycles) 33 { 34 asm volatile("0: brctg %0,0b" : : "d" (cycles)); 35 } 36 37 /* tweak me */ 38 #define VMEXIT_CYCLES 200 39 #define VMENTRY_CYCLES 200 40 41 #else 42 static inline void wait_cycles(unsigned long long cycles) 43 { 44 _Exit(5); 45 } 46 #define VMEXIT_CYCLES 0 47 #define VMENTRY_CYCLES 0 48 #endif 49 50 static inline void vmexit(void) 51 { 52 if (!do_exit) 53 return; 54 55 wait_cycles(VMEXIT_CYCLES); 56 } 57 static inline void vmentry(void) 58 { 59 if (!do_exit) 60 return; 61 62 wait_cycles(VMENTRY_CYCLES); 63 } 64 65 /* implemented by ring */ 66 void alloc_ring(void); 67 /* guest side */ 68 int add_inbuf(unsigned, void *, void *); 69 void *get_buf(unsigned *, void **); 70 void disable_call(); 71 bool used_empty(); 72 bool enable_call(); 73 void kick_available(); 74 /* host side */ 75 void disable_kick(); 76 bool avail_empty(); 77 bool enable_kick(); 78 bool use_buf(unsigned *, void **); 79 void call_used(); 80 81 /* implemented by main */ 82 extern bool do_sleep; 83 void kick(void); 84 void wait_for_kick(void); 85 void call(void); 86 void wait_for_call(void); 87 88 extern unsigned ring_size; 89 90 /* Compiler barrier - similar to what Linux uses */ 91 #define barrier() asm volatile("" ::: "memory") 92 93 /* Is there a portable way to do this? */ 94 #if defined(__x86_64__) || defined(__i386__) 95 #define cpu_relax() asm ("rep; nop" ::: "memory") 96 #elif defined(__s390x__) 97 #define cpu_relax() barrier() 98 #else 99 #define cpu_relax() assert(0) 100 #endif 101 102 extern bool do_relax; 103 104 static inline void busy_wait(void) 105 { 106 if (do_relax) 107 cpu_relax(); 108 else 109 /* prevent compiler from removing busy loops */ 110 barrier(); 111 } 112 113 #if defined(__x86_64__) || defined(__i386__) 114 #define smp_mb() asm volatile("lock; addl $0,-132(%%rsp)" ::: "memory", "cc") 115 #else 116 /* 117 * Not using __ATOMIC_SEQ_CST since gcc docs say they are only synchronized 118 * with other __ATOMIC_SEQ_CST calls. 119 */ 120 #define smp_mb() __sync_synchronize() 121 #endif 122 123 /* 124 * This abuses the atomic builtins for thread fences, and 125 * adds a compiler barrier. 126 */ 127 #define smp_release() do { \ 128 barrier(); \ 129 __atomic_thread_fence(__ATOMIC_RELEASE); \ 130 } while (0) 131 132 #define smp_acquire() do { \ 133 __atomic_thread_fence(__ATOMIC_ACQUIRE); \ 134 barrier(); \ 135 } while (0) 136 137 #if defined(__i386__) || defined(__x86_64__) || defined(__s390x__) 138 #define smp_wmb() barrier() 139 #else 140 #define smp_wmb() smp_release() 141 #endif 142 143 static __always_inline 144 void __read_once_size(const volatile void *p, void *res, int size) 145 { 146 switch (size) { 147 case 1: *(unsigned char *)res = *(volatile unsigned char *)p; break; 148 case 2: *(unsigned short *)res = *(volatile unsigned short *)p; break; 149 case 4: *(unsigned int *)res = *(volatile unsigned int *)p; break; 150 case 8: *(unsigned long long *)res = *(volatile unsigned long long *)p; break; 151 default: 152 barrier(); 153 __builtin_memcpy((void *)res, (const void *)p, size); 154 barrier(); 155 } 156 } 157 158 static __always_inline void __write_once_size(volatile void *p, void *res, int size) 159 { 160 switch (size) { 161 case 1: *(volatile unsigned char *)p = *(unsigned char *)res; break; 162 case 2: *(volatile unsigned short *)p = *(unsigned short *)res; break; 163 case 4: *(volatile unsigned int *)p = *(unsigned int *)res; break; 164 case 8: *(volatile unsigned long long *)p = *(unsigned long long *)res; break; 165 default: 166 barrier(); 167 __builtin_memcpy((void *)p, (const void *)res, size); 168 barrier(); 169 } 170 } 171 172 #ifdef __alpha__ 173 #define READ_ONCE(x) \ 174 ({ \ 175 union { typeof(x) __val; char __c[1]; } __u; \ 176 __read_once_size(&(x), __u.__c, sizeof(x)); \ 177 smp_mb(); /* Enforce dependency ordering from x */ \ 178 __u.__val; \ 179 }) 180 #else 181 #define READ_ONCE(x) \ 182 ({ \ 183 union { typeof(x) __val; char __c[1]; } __u; \ 184 __read_once_size(&(x), __u.__c, sizeof(x)); \ 185 __u.__val; \ 186 }) 187 #endif 188 189 #define WRITE_ONCE(x, val) \ 190 ({ \ 191 union { typeof(x) __val; char __c[1]; } __u = \ 192 { .__val = (typeof(x)) (val) }; \ 193 __write_once_size(&(x), __u.__c, sizeof(x)); \ 194 __u.__val; \ 195 }) 196 197 #endif 198