1 /* 2 * ucontext coroutine initialization code 3 * 4 * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws> 5 * Copyright (C) 2011 Kevin Wolf <kwolf@redhat.com> 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.0 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 /* XXX Is there a nicer way to disable glibc's stack check for longjmp? */ 22 #ifdef _FORTIFY_SOURCE 23 #undef _FORTIFY_SOURCE 24 #endif 25 #include "qemu/osdep.h" 26 #include <ucontext.h> 27 #include "qemu/coroutine_int.h" 28 29 #ifdef CONFIG_VALGRIND_H 30 #include <valgrind/valgrind.h> 31 #endif 32 33 #if defined(__SANITIZE_ADDRESS__) || __has_feature(address_sanitizer) 34 #ifdef CONFIG_ASAN_IFACE_FIBER 35 #define CONFIG_ASAN 1 36 #include <sanitizer/asan_interface.h> 37 #endif 38 #endif 39 40 #ifdef CONFIG_TSAN 41 #include <sanitizer/tsan_interface.h> 42 #endif 43 44 typedef struct { 45 Coroutine base; 46 void *stack; 47 size_t stack_size; 48 sigjmp_buf env; 49 50 void *tsan_co_fiber; 51 void *tsan_caller_fiber; 52 53 #ifdef CONFIG_VALGRIND_H 54 unsigned int valgrind_stack_id; 55 #endif 56 57 } CoroutineUContext; 58 59 /** 60 * Per-thread coroutine bookkeeping 61 */ 62 static __thread CoroutineUContext leader; 63 static __thread Coroutine *current; 64 65 /* 66 * va_args to makecontext() must be type 'int', so passing 67 * the pointer we need may require several int args. This 68 * union is a quick hack to let us do that 69 */ 70 union cc_arg { 71 void *p; 72 int i[2]; 73 }; 74 75 /* QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it. */ 76 static inline __attribute__((always_inline)) 77 void on_new_fiber(CoroutineUContext *co) 78 { 79 #ifdef CONFIG_TSAN 80 co->tsan_co_fiber = __tsan_create_fiber(0); /* flags: sync on switch */ 81 co->tsan_caller_fiber = __tsan_get_current_fiber(); 82 #endif 83 } 84 85 static inline __attribute__((always_inline)) 86 void finish_switch_fiber(void *fake_stack_save) 87 { 88 #ifdef CONFIG_ASAN 89 const void *bottom_old; 90 size_t size_old; 91 92 __sanitizer_finish_switch_fiber(fake_stack_save, &bottom_old, &size_old); 93 94 if (!leader.stack) { 95 leader.stack = (void *)bottom_old; 96 leader.stack_size = size_old; 97 } 98 #endif 99 #ifdef CONFIG_TSAN 100 if (fake_stack_save) { 101 __tsan_release(fake_stack_save); 102 __tsan_switch_to_fiber(fake_stack_save, 0); /* 0=synchronize */ 103 } 104 #endif 105 } 106 107 static inline __attribute__((always_inline)) void start_switch_fiber( 108 CoroutineAction action, void **fake_stack_save, 109 const void *bottom, size_t size, void *new_fiber) 110 { 111 #ifdef CONFIG_ASAN 112 __sanitizer_start_switch_fiber( 113 action == COROUTINE_TERMINATE ? NULL : fake_stack_save, 114 bottom, size); 115 #endif 116 #ifdef CONFIG_TSAN 117 void *curr_fiber = 118 __tsan_get_current_fiber(); 119 __tsan_acquire(curr_fiber); 120 121 *fake_stack_save = curr_fiber; 122 __tsan_switch_to_fiber(new_fiber, 0); /* 0=synchronize */ 123 #endif 124 } 125 126 static void coroutine_trampoline(int i0, int i1) 127 { 128 union cc_arg arg; 129 CoroutineUContext *self; 130 Coroutine *co; 131 void *fake_stack_save = NULL; 132 133 finish_switch_fiber(NULL); 134 135 arg.i[0] = i0; 136 arg.i[1] = i1; 137 self = arg.p; 138 co = &self->base; 139 140 /* Initialize longjmp environment and switch back the caller */ 141 if (!sigsetjmp(self->env, 0)) { 142 start_switch_fiber( 143 COROUTINE_YIELD, 144 &fake_stack_save, 145 leader.stack, 146 leader.stack_size, 147 self->tsan_caller_fiber); 148 siglongjmp(*(sigjmp_buf *)co->entry_arg, 1); 149 } 150 151 finish_switch_fiber(fake_stack_save); 152 153 while (true) { 154 co->entry(co->entry_arg); 155 qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE); 156 } 157 } 158 159 Coroutine *qemu_coroutine_new(void) 160 { 161 CoroutineUContext *co; 162 ucontext_t old_uc, uc; 163 sigjmp_buf old_env; 164 union cc_arg arg = {0}; 165 void *fake_stack_save = NULL; 166 167 /* The ucontext functions preserve signal masks which incurs a 168 * system call overhead. sigsetjmp(buf, 0)/siglongjmp() does not 169 * preserve signal masks but only works on the current stack. 170 * Since we need a way to create and switch to a new stack, use 171 * the ucontext functions for that but sigsetjmp()/siglongjmp() for 172 * everything else. 173 */ 174 175 if (getcontext(&uc) == -1) { 176 abort(); 177 } 178 179 co = g_malloc0(sizeof(*co)); 180 co->stack_size = COROUTINE_STACK_SIZE; 181 co->stack = qemu_alloc_stack(&co->stack_size); 182 co->base.entry_arg = &old_env; /* stash away our jmp_buf */ 183 184 uc.uc_link = &old_uc; 185 uc.uc_stack.ss_sp = co->stack; 186 uc.uc_stack.ss_size = co->stack_size; 187 uc.uc_stack.ss_flags = 0; 188 189 #ifdef CONFIG_VALGRIND_H 190 co->valgrind_stack_id = 191 VALGRIND_STACK_REGISTER(co->stack, co->stack + co->stack_size); 192 #endif 193 194 arg.p = co; 195 196 on_new_fiber(co); 197 makecontext(&uc, (void (*)(void))coroutine_trampoline, 198 2, arg.i[0], arg.i[1]); 199 200 /* swapcontext() in, siglongjmp() back out */ 201 if (!sigsetjmp(old_env, 0)) { 202 start_switch_fiber( 203 COROUTINE_YIELD, 204 &fake_stack_save, 205 co->stack, co->stack_size, co->tsan_co_fiber); 206 swapcontext(&old_uc, &uc); 207 } 208 209 finish_switch_fiber(fake_stack_save); 210 211 return &co->base; 212 } 213 214 #ifdef CONFIG_VALGRIND_H 215 #if defined(CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE) && !defined(__clang__) 216 /* Work around an unused variable in the valgrind.h macro... */ 217 #pragma GCC diagnostic push 218 #pragma GCC diagnostic ignored "-Wunused-but-set-variable" 219 #endif 220 static inline void valgrind_stack_deregister(CoroutineUContext *co) 221 { 222 VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id); 223 } 224 #if defined(CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE) && !defined(__clang__) 225 #pragma GCC diagnostic pop 226 #endif 227 #endif 228 229 void qemu_coroutine_delete(Coroutine *co_) 230 { 231 CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_); 232 233 #ifdef CONFIG_VALGRIND_H 234 valgrind_stack_deregister(co); 235 #endif 236 237 qemu_free_stack(co->stack, co->stack_size); 238 g_free(co); 239 } 240 241 /* This function is marked noinline to prevent GCC from inlining it 242 * into coroutine_trampoline(). If we allow it to do that then it 243 * hoists the code to get the address of the TLS variable "current" 244 * out of the while() loop. This is an invalid transformation because 245 * the sigsetjmp() call may be called when running thread A but 246 * return in thread B, and so we might be in a different thread 247 * context each time round the loop. 248 */ 249 CoroutineAction __attribute__((noinline)) 250 qemu_coroutine_switch(Coroutine *from_, Coroutine *to_, 251 CoroutineAction action) 252 { 253 CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_); 254 CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_); 255 int ret; 256 void *fake_stack_save = NULL; 257 258 current = to_; 259 260 ret = sigsetjmp(from->env, 0); 261 if (ret == 0) { 262 start_switch_fiber(action, &fake_stack_save, 263 to->stack, to->stack_size, to->tsan_co_fiber); 264 siglongjmp(to->env, action); 265 } 266 267 finish_switch_fiber(fake_stack_save); 268 269 return ret; 270 } 271 272 Coroutine *qemu_coroutine_self(void) 273 { 274 if (!current) { 275 current = &leader.base; 276 } 277 #ifdef CONFIG_TSAN 278 if (!leader.tsan_co_fiber) { 279 leader.tsan_co_fiber = __tsan_get_current_fiber(); 280 } 281 #endif 282 return current; 283 } 284 285 bool qemu_in_coroutine(void) 286 { 287 return current && current->caller; 288 } 289