1 /* 2 * ucontext coroutine initialization code 3 * 4 * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws> 5 * Copyright (C) 2011 Kevin Wolf <kwolf@redhat.com> 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.0 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 /* XXX Is there a nicer way to disable glibc's stack check for longjmp? */ 22 #undef _FORTIFY_SOURCE 23 #define _FORTIFY_SOURCE 0 24 25 #include "qemu/osdep.h" 26 #include <ucontext.h> 27 #include "qemu/coroutine_int.h" 28 #include "qemu/coroutine-tls.h" 29 30 #ifdef CONFIG_VALGRIND_H 31 #include <valgrind/valgrind.h> 32 #endif 33 34 #ifdef QEMU_SANITIZE_ADDRESS 35 #ifdef CONFIG_ASAN_IFACE_FIBER 36 #define CONFIG_ASAN 1 37 #include <sanitizer/asan_interface.h> 38 #endif 39 #endif 40 41 #ifdef CONFIG_TSAN 42 #include <sanitizer/tsan_interface.h> 43 #endif 44 45 typedef struct { 46 Coroutine base; 47 void *stack; 48 size_t stack_size; 49 #ifdef CONFIG_SAFESTACK 50 /* Need an unsafe stack for each coroutine */ 51 void *unsafe_stack; 52 size_t unsafe_stack_size; 53 #endif 54 sigjmp_buf env; 55 56 #ifdef CONFIG_TSAN 57 void *tsan_co_fiber; 58 void *tsan_caller_fiber; 59 #endif 60 61 #ifdef CONFIG_VALGRIND_H 62 unsigned int valgrind_stack_id; 63 #endif 64 65 } CoroutineUContext; 66 67 /** 68 * Per-thread coroutine bookkeeping 69 */ 70 QEMU_DEFINE_STATIC_CO_TLS(Coroutine *, current); 71 QEMU_DEFINE_STATIC_CO_TLS(CoroutineUContext, leader); 72 73 /* 74 * va_args to makecontext() must be type 'int', so passing 75 * the pointer we need may require several int args. This 76 * union is a quick hack to let us do that 77 */ 78 union cc_arg { 79 void *p; 80 int i[2]; 81 }; 82 83 /* 84 * QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it. 85 * always_inline is required to avoid TSan runtime fatal errors. 86 */ 87 static inline __attribute__((always_inline)) 88 void on_new_fiber(CoroutineUContext *co) 89 { 90 #ifdef CONFIG_TSAN 91 co->tsan_co_fiber = __tsan_create_fiber(0); /* flags: sync on switch */ 92 co->tsan_caller_fiber = __tsan_get_current_fiber(); 93 #endif 94 } 95 96 /* always_inline is required to avoid TSan runtime fatal errors. */ 97 static inline __attribute__((always_inline)) 98 void finish_switch_fiber(void *fake_stack_save) 99 { 100 #ifdef CONFIG_ASAN 101 CoroutineUContext *leaderp = get_ptr_leader(); 102 const void *bottom_old; 103 size_t size_old; 104 105 __sanitizer_finish_switch_fiber(fake_stack_save, &bottom_old, &size_old); 106 107 if (!leaderp->stack) { 108 leaderp->stack = (void *)bottom_old; 109 leaderp->stack_size = size_old; 110 } 111 #endif 112 #ifdef CONFIG_TSAN 113 if (fake_stack_save) { 114 __tsan_release(fake_stack_save); 115 __tsan_switch_to_fiber(fake_stack_save, 0); /* 0=synchronize */ 116 } 117 #endif 118 } 119 120 /* always_inline is required to avoid TSan runtime fatal errors. */ 121 static inline __attribute__((always_inline)) 122 void start_switch_fiber_asan(CoroutineAction action, void **fake_stack_save, 123 const void *bottom, size_t size) 124 { 125 #ifdef CONFIG_ASAN 126 __sanitizer_start_switch_fiber( 127 action == COROUTINE_TERMINATE ? NULL : fake_stack_save, 128 bottom, size); 129 #endif 130 } 131 132 /* always_inline is required to avoid TSan runtime fatal errors. */ 133 static inline __attribute__((always_inline)) 134 void start_switch_fiber_tsan(void **fake_stack_save, 135 CoroutineUContext *co, 136 bool caller) 137 { 138 #ifdef CONFIG_TSAN 139 void *new_fiber = caller ? 140 co->tsan_caller_fiber : 141 co->tsan_co_fiber; 142 void *curr_fiber = __tsan_get_current_fiber(); 143 __tsan_acquire(curr_fiber); 144 145 *fake_stack_save = curr_fiber; 146 __tsan_switch_to_fiber(new_fiber, 0); /* 0=synchronize */ 147 #endif 148 } 149 150 static void coroutine_trampoline(int i0, int i1) 151 { 152 union cc_arg arg; 153 CoroutineUContext *self; 154 Coroutine *co; 155 void *fake_stack_save = NULL; 156 157 finish_switch_fiber(NULL); 158 159 arg.i[0] = i0; 160 arg.i[1] = i1; 161 self = arg.p; 162 co = &self->base; 163 164 /* Initialize longjmp environment and switch back the caller */ 165 if (!sigsetjmp(self->env, 0)) { 166 CoroutineUContext *leaderp = get_ptr_leader(); 167 168 start_switch_fiber_asan(COROUTINE_YIELD, &fake_stack_save, 169 leaderp->stack, leaderp->stack_size); 170 start_switch_fiber_tsan(&fake_stack_save, self, true); /* true=caller */ 171 siglongjmp(*(sigjmp_buf *)co->entry_arg, 1); 172 } 173 174 finish_switch_fiber(fake_stack_save); 175 176 while (true) { 177 co->entry(co->entry_arg); 178 qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE); 179 } 180 } 181 182 Coroutine *qemu_coroutine_new(void) 183 { 184 CoroutineUContext *co; 185 ucontext_t old_uc, uc; 186 sigjmp_buf old_env; 187 union cc_arg arg = {0}; 188 void *fake_stack_save = NULL; 189 190 /* The ucontext functions preserve signal masks which incurs a 191 * system call overhead. sigsetjmp(buf, 0)/siglongjmp() does not 192 * preserve signal masks but only works on the current stack. 193 * Since we need a way to create and switch to a new stack, use 194 * the ucontext functions for that but sigsetjmp()/siglongjmp() for 195 * everything else. 196 */ 197 198 if (getcontext(&uc) == -1) { 199 abort(); 200 } 201 202 co = g_malloc0(sizeof(*co)); 203 co->stack_size = COROUTINE_STACK_SIZE; 204 co->stack = qemu_alloc_stack(&co->stack_size); 205 #ifdef CONFIG_SAFESTACK 206 co->unsafe_stack_size = COROUTINE_STACK_SIZE; 207 co->unsafe_stack = qemu_alloc_stack(&co->unsafe_stack_size); 208 #endif 209 co->base.entry_arg = &old_env; /* stash away our jmp_buf */ 210 211 uc.uc_link = &old_uc; 212 uc.uc_stack.ss_sp = co->stack; 213 uc.uc_stack.ss_size = co->stack_size; 214 uc.uc_stack.ss_flags = 0; 215 216 #ifdef CONFIG_VALGRIND_H 217 co->valgrind_stack_id = 218 VALGRIND_STACK_REGISTER(co->stack, co->stack + co->stack_size); 219 #endif 220 221 arg.p = co; 222 223 on_new_fiber(co); 224 makecontext(&uc, (void (*)(void))coroutine_trampoline, 225 2, arg.i[0], arg.i[1]); 226 227 /* swapcontext() in, siglongjmp() back out */ 228 if (!sigsetjmp(old_env, 0)) { 229 start_switch_fiber_asan(COROUTINE_YIELD, &fake_stack_save, co->stack, 230 co->stack_size); 231 start_switch_fiber_tsan(&fake_stack_save, 232 co, false); /* false=not caller */ 233 234 #ifdef CONFIG_SAFESTACK 235 /* 236 * Before we swap the context, set the new unsafe stack 237 * The unsafe stack grows just like the normal stack, so start from 238 * the last usable location of the memory area. 239 * NOTE: we don't have to re-set the usp afterwards because we are 240 * coming back to this context through a siglongjmp. 241 * The compiler already wrapped the corresponding sigsetjmp call with 242 * code that saves the usp on the (safe) stack before the call, and 243 * restores it right after (which is where we return with siglongjmp). 244 */ 245 void *usp = co->unsafe_stack + co->unsafe_stack_size; 246 __safestack_unsafe_stack_ptr = usp; 247 #endif 248 249 swapcontext(&old_uc, &uc); 250 } 251 252 finish_switch_fiber(fake_stack_save); 253 254 return &co->base; 255 } 256 257 #ifdef CONFIG_VALGRIND_H 258 /* Work around an unused variable in the valgrind.h macro... */ 259 #if !defined(__clang__) 260 #pragma GCC diagnostic push 261 #pragma GCC diagnostic ignored "-Wunused-but-set-variable" 262 #endif 263 static inline void valgrind_stack_deregister(CoroutineUContext *co) 264 { 265 VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id); 266 } 267 #if !defined(__clang__) 268 #pragma GCC diagnostic pop 269 #endif 270 #endif 271 272 void qemu_coroutine_delete(Coroutine *co_) 273 { 274 CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_); 275 276 #ifdef CONFIG_VALGRIND_H 277 valgrind_stack_deregister(co); 278 #endif 279 280 qemu_free_stack(co->stack, co->stack_size); 281 #ifdef CONFIG_SAFESTACK 282 qemu_free_stack(co->unsafe_stack, co->unsafe_stack_size); 283 #endif 284 g_free(co); 285 } 286 287 /* This function is marked noinline to prevent GCC from inlining it 288 * into coroutine_trampoline(). If we allow it to do that then it 289 * hoists the code to get the address of the TLS variable "current" 290 * out of the while() loop. This is an invalid transformation because 291 * the sigsetjmp() call may be called when running thread A but 292 * return in thread B, and so we might be in a different thread 293 * context each time round the loop. 294 */ 295 CoroutineAction __attribute__((noinline)) 296 qemu_coroutine_switch(Coroutine *from_, Coroutine *to_, 297 CoroutineAction action) 298 { 299 CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_); 300 CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_); 301 int ret; 302 void *fake_stack_save = NULL; 303 304 set_current(to_); 305 306 ret = sigsetjmp(from->env, 0); 307 if (ret == 0) { 308 start_switch_fiber_asan(action, &fake_stack_save, to->stack, 309 to->stack_size); 310 start_switch_fiber_tsan(&fake_stack_save, 311 to, false); /* false=not caller */ 312 siglongjmp(to->env, action); 313 } 314 315 finish_switch_fiber(fake_stack_save); 316 317 return ret; 318 } 319 320 Coroutine *qemu_coroutine_self(void) 321 { 322 Coroutine *self = get_current(); 323 CoroutineUContext *leaderp = get_ptr_leader(); 324 325 if (!self) { 326 self = &leaderp->base; 327 set_current(self); 328 } 329 #ifdef CONFIG_TSAN 330 if (!leaderp->tsan_co_fiber) { 331 leaderp->tsan_co_fiber = __tsan_get_current_fiber(); 332 } 333 #endif 334 return self; 335 } 336 337 bool qemu_in_coroutine(void) 338 { 339 Coroutine *self = get_current(); 340 341 return self && self->caller; 342 } 343