1 /* 2 * ucontext coroutine initialization code 3 * 4 * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws> 5 * Copyright (C) 2011 Kevin Wolf <kwolf@redhat.com> 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.0 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 /* XXX Is there a nicer way to disable glibc's stack check for longjmp? */ 22 #ifdef _FORTIFY_SOURCE 23 #undef _FORTIFY_SOURCE 24 #endif 25 #include "qemu/osdep.h" 26 #include <ucontext.h> 27 #include "qemu/coroutine_int.h" 28 29 #ifdef CONFIG_VALGRIND_H 30 #include <valgrind/valgrind.h> 31 #endif 32 33 #if defined(__SANITIZE_ADDRESS__) || __has_feature(address_sanitizer) 34 #ifdef CONFIG_ASAN_IFACE_FIBER 35 #define CONFIG_ASAN 1 36 #include <sanitizer/asan_interface.h> 37 #endif 38 #endif 39 40 #ifdef CONFIG_TSAN 41 #include <sanitizer/tsan_interface.h> 42 #endif 43 44 typedef struct { 45 Coroutine base; 46 void *stack; 47 size_t stack_size; 48 #ifdef CONFIG_SAFESTACK 49 /* Need an unsafe stack for each coroutine */ 50 void *unsafe_stack; 51 size_t unsafe_stack_size; 52 #endif 53 sigjmp_buf env; 54 55 #ifdef CONFIG_TSAN 56 void *tsan_co_fiber; 57 void *tsan_caller_fiber; 58 #endif 59 60 #ifdef CONFIG_VALGRIND_H 61 unsigned int valgrind_stack_id; 62 #endif 63 64 } CoroutineUContext; 65 66 /** 67 * Per-thread coroutine bookkeeping 68 */ 69 static __thread CoroutineUContext leader; 70 static __thread Coroutine *current; 71 72 /* 73 * va_args to makecontext() must be type 'int', so passing 74 * the pointer we need may require several int args. This 75 * union is a quick hack to let us do that 76 */ 77 union cc_arg { 78 void *p; 79 int i[2]; 80 }; 81 82 /* 83 * QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it. 84 * always_inline is required to avoid TSan runtime fatal errors. 85 */ 86 static inline __attribute__((always_inline)) 87 void on_new_fiber(CoroutineUContext *co) 88 { 89 #ifdef CONFIG_TSAN 90 co->tsan_co_fiber = __tsan_create_fiber(0); /* flags: sync on switch */ 91 co->tsan_caller_fiber = __tsan_get_current_fiber(); 92 #endif 93 } 94 95 /* always_inline is required to avoid TSan runtime fatal errors. */ 96 static inline __attribute__((always_inline)) 97 void finish_switch_fiber(void *fake_stack_save) 98 { 99 #ifdef CONFIG_ASAN 100 const void *bottom_old; 101 size_t size_old; 102 103 __sanitizer_finish_switch_fiber(fake_stack_save, &bottom_old, &size_old); 104 105 if (!leader.stack) { 106 leader.stack = (void *)bottom_old; 107 leader.stack_size = size_old; 108 } 109 #endif 110 #ifdef CONFIG_TSAN 111 if (fake_stack_save) { 112 __tsan_release(fake_stack_save); 113 __tsan_switch_to_fiber(fake_stack_save, 0); /* 0=synchronize */ 114 } 115 #endif 116 } 117 118 /* always_inline is required to avoid TSan runtime fatal errors. */ 119 static inline __attribute__((always_inline)) 120 void start_switch_fiber_asan(CoroutineAction action, void **fake_stack_save, 121 const void *bottom, size_t size) 122 { 123 #ifdef CONFIG_ASAN 124 __sanitizer_start_switch_fiber( 125 action == COROUTINE_TERMINATE ? NULL : fake_stack_save, 126 bottom, size); 127 #endif 128 } 129 130 /* always_inline is required to avoid TSan runtime fatal errors. */ 131 static inline __attribute__((always_inline)) 132 void start_switch_fiber_tsan(void **fake_stack_save, 133 CoroutineUContext *co, 134 bool caller) 135 { 136 #ifdef CONFIG_TSAN 137 void *new_fiber = caller ? 138 co->tsan_caller_fiber : 139 co->tsan_co_fiber; 140 void *curr_fiber = __tsan_get_current_fiber(); 141 __tsan_acquire(curr_fiber); 142 143 *fake_stack_save = curr_fiber; 144 __tsan_switch_to_fiber(new_fiber, 0); /* 0=synchronize */ 145 #endif 146 } 147 148 static void coroutine_trampoline(int i0, int i1) 149 { 150 union cc_arg arg; 151 CoroutineUContext *self; 152 Coroutine *co; 153 void *fake_stack_save = NULL; 154 155 finish_switch_fiber(NULL); 156 157 arg.i[0] = i0; 158 arg.i[1] = i1; 159 self = arg.p; 160 co = &self->base; 161 162 /* Initialize longjmp environment and switch back the caller */ 163 if (!sigsetjmp(self->env, 0)) { 164 start_switch_fiber_asan(COROUTINE_YIELD, &fake_stack_save, leader.stack, 165 leader.stack_size); 166 start_switch_fiber_tsan(&fake_stack_save, self, true); /* true=caller */ 167 siglongjmp(*(sigjmp_buf *)co->entry_arg, 1); 168 } 169 170 finish_switch_fiber(fake_stack_save); 171 172 while (true) { 173 co->entry(co->entry_arg); 174 qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE); 175 } 176 } 177 178 Coroutine *qemu_coroutine_new(void) 179 { 180 CoroutineUContext *co; 181 ucontext_t old_uc, uc; 182 sigjmp_buf old_env; 183 union cc_arg arg = {0}; 184 void *fake_stack_save = NULL; 185 186 /* The ucontext functions preserve signal masks which incurs a 187 * system call overhead. sigsetjmp(buf, 0)/siglongjmp() does not 188 * preserve signal masks but only works on the current stack. 189 * Since we need a way to create and switch to a new stack, use 190 * the ucontext functions for that but sigsetjmp()/siglongjmp() for 191 * everything else. 192 */ 193 194 if (getcontext(&uc) == -1) { 195 abort(); 196 } 197 198 co = g_malloc0(sizeof(*co)); 199 co->stack_size = COROUTINE_STACK_SIZE; 200 co->stack = qemu_alloc_stack(&co->stack_size); 201 #ifdef CONFIG_SAFESTACK 202 co->unsafe_stack_size = COROUTINE_STACK_SIZE; 203 co->unsafe_stack = qemu_alloc_stack(&co->unsafe_stack_size); 204 #endif 205 co->base.entry_arg = &old_env; /* stash away our jmp_buf */ 206 207 uc.uc_link = &old_uc; 208 uc.uc_stack.ss_sp = co->stack; 209 uc.uc_stack.ss_size = co->stack_size; 210 uc.uc_stack.ss_flags = 0; 211 212 #ifdef CONFIG_VALGRIND_H 213 co->valgrind_stack_id = 214 VALGRIND_STACK_REGISTER(co->stack, co->stack + co->stack_size); 215 #endif 216 217 arg.p = co; 218 219 on_new_fiber(co); 220 makecontext(&uc, (void (*)(void))coroutine_trampoline, 221 2, arg.i[0], arg.i[1]); 222 223 /* swapcontext() in, siglongjmp() back out */ 224 if (!sigsetjmp(old_env, 0)) { 225 start_switch_fiber_asan(COROUTINE_YIELD, &fake_stack_save, co->stack, 226 co->stack_size); 227 start_switch_fiber_tsan(&fake_stack_save, 228 co, false); /* false=not caller */ 229 230 #ifdef CONFIG_SAFESTACK 231 /* 232 * Before we swap the context, set the new unsafe stack 233 * The unsafe stack grows just like the normal stack, so start from 234 * the last usable location of the memory area. 235 * NOTE: we don't have to re-set the usp afterwards because we are 236 * coming back to this context through a siglongjmp. 237 * The compiler already wrapped the corresponding sigsetjmp call with 238 * code that saves the usp on the (safe) stack before the call, and 239 * restores it right after (which is where we return with siglongjmp). 240 */ 241 void *usp = co->unsafe_stack + co->unsafe_stack_size; 242 __safestack_unsafe_stack_ptr = usp; 243 #endif 244 245 swapcontext(&old_uc, &uc); 246 } 247 248 finish_switch_fiber(fake_stack_save); 249 250 return &co->base; 251 } 252 253 #ifdef CONFIG_VALGRIND_H 254 /* Work around an unused variable in the valgrind.h macro... */ 255 #if !defined(__clang__) 256 #pragma GCC diagnostic push 257 #pragma GCC diagnostic ignored "-Wunused-but-set-variable" 258 #endif 259 static inline void valgrind_stack_deregister(CoroutineUContext *co) 260 { 261 VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id); 262 } 263 #if !defined(__clang__) 264 #pragma GCC diagnostic pop 265 #endif 266 #endif 267 268 void qemu_coroutine_delete(Coroutine *co_) 269 { 270 CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_); 271 272 #ifdef CONFIG_VALGRIND_H 273 valgrind_stack_deregister(co); 274 #endif 275 276 qemu_free_stack(co->stack, co->stack_size); 277 #ifdef CONFIG_SAFESTACK 278 qemu_free_stack(co->unsafe_stack, co->unsafe_stack_size); 279 #endif 280 g_free(co); 281 } 282 283 /* This function is marked noinline to prevent GCC from inlining it 284 * into coroutine_trampoline(). If we allow it to do that then it 285 * hoists the code to get the address of the TLS variable "current" 286 * out of the while() loop. This is an invalid transformation because 287 * the sigsetjmp() call may be called when running thread A but 288 * return in thread B, and so we might be in a different thread 289 * context each time round the loop. 290 */ 291 CoroutineAction __attribute__((noinline)) 292 qemu_coroutine_switch(Coroutine *from_, Coroutine *to_, 293 CoroutineAction action) 294 { 295 CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_); 296 CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_); 297 int ret; 298 void *fake_stack_save = NULL; 299 300 current = to_; 301 302 ret = sigsetjmp(from->env, 0); 303 if (ret == 0) { 304 start_switch_fiber_asan(action, &fake_stack_save, to->stack, 305 to->stack_size); 306 start_switch_fiber_tsan(&fake_stack_save, 307 to, false); /* false=not caller */ 308 siglongjmp(to->env, action); 309 } 310 311 finish_switch_fiber(fake_stack_save); 312 313 return ret; 314 } 315 316 Coroutine *qemu_coroutine_self(void) 317 { 318 if (!current) { 319 current = &leader.base; 320 } 321 #ifdef CONFIG_TSAN 322 if (!leader.tsan_co_fiber) { 323 leader.tsan_co_fiber = __tsan_get_current_fiber(); 324 } 325 #endif 326 return current; 327 } 328 329 bool qemu_in_coroutine(void) 330 { 331 return current && current->caller; 332 } 333