xref: /openbmc/qemu/util/coroutine-ucontext.c (revision f413f9fcc923083a7db038e32964148cf74134e6)
1  /*
2   * ucontext coroutine initialization code
3   *
4   * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
5   * Copyright (C) 2011  Kevin Wolf <kwolf@redhat.com>
6   *
7   * This library is free software; you can redistribute it and/or
8   * modify it under the terms of the GNU Lesser General Public
9   * License as published by the Free Software Foundation; either
10   * version 2.0 of the License, or (at your option) any later version.
11   *
12   * This library is distributed in the hope that it will be useful,
13   * but WITHOUT ANY WARRANTY; without even the implied warranty of
14   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15   * Lesser General Public License for more details.
16   *
17   * You should have received a copy of the GNU Lesser General Public
18   * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19   */
20  
21  /* XXX Is there a nicer way to disable glibc's stack check for longjmp? */
22  #undef _FORTIFY_SOURCE
23  #define _FORTIFY_SOURCE 0
24  
25  #include "qemu/osdep.h"
26  #include <ucontext.h>
27  #include "qemu/coroutine_int.h"
28  #include "qemu/coroutine-tls.h"
29  
30  #ifdef CONFIG_VALGRIND_H
31  #include <valgrind/valgrind.h>
32  #endif
33  
34  #ifdef QEMU_SANITIZE_ADDRESS
35  #ifdef CONFIG_ASAN_IFACE_FIBER
36  #define CONFIG_ASAN 1
37  #include <sanitizer/asan_interface.h>
38  #endif
39  #endif
40  
41  #ifdef CONFIG_TSAN
42  #include <sanitizer/tsan_interface.h>
43  #endif
44  
45  typedef struct {
46      Coroutine base;
47      void *stack;
48      size_t stack_size;
49  #ifdef CONFIG_SAFESTACK
50      /* Need an unsafe stack for each coroutine */
51      void *unsafe_stack;
52      size_t unsafe_stack_size;
53  #endif
54      sigjmp_buf env;
55  
56  #ifdef CONFIG_TSAN
57      void *tsan_co_fiber;
58      void *tsan_caller_fiber;
59  #endif
60  
61  #ifdef CONFIG_VALGRIND_H
62      unsigned int valgrind_stack_id;
63  #endif
64  
65  } CoroutineUContext;
66  
67  /**
68   * Per-thread coroutine bookkeeping
69   */
70  QEMU_DEFINE_STATIC_CO_TLS(Coroutine *, current);
71  QEMU_DEFINE_STATIC_CO_TLS(CoroutineUContext, leader);
72  
73  /*
74   * va_args to makecontext() must be type 'int', so passing
75   * the pointer we need may require several int args. This
76   * union is a quick hack to let us do that
77   */
78  union cc_arg {
79      void *p;
80      int i[2];
81  };
82  
83  /*
84   * QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it.
85   * always_inline is required to avoid TSan runtime fatal errors.
86   */
87  static inline __attribute__((always_inline))
on_new_fiber(CoroutineUContext * co)88  void on_new_fiber(CoroutineUContext *co)
89  {
90  #ifdef CONFIG_TSAN
91      co->tsan_co_fiber = __tsan_create_fiber(0); /* flags: sync on switch */
92      co->tsan_caller_fiber = __tsan_get_current_fiber();
93  #endif
94  }
95  
96  /* always_inline is required to avoid TSan runtime fatal errors. */
97  static inline __attribute__((always_inline))
finish_switch_fiber(void * fake_stack_save)98  void finish_switch_fiber(void *fake_stack_save)
99  {
100  #ifdef CONFIG_ASAN
101      CoroutineUContext *leaderp = get_ptr_leader();
102      const void *bottom_old;
103      size_t size_old;
104  
105      __sanitizer_finish_switch_fiber(fake_stack_save, &bottom_old, &size_old);
106  
107      if (!leaderp->stack) {
108          leaderp->stack = (void *)bottom_old;
109          leaderp->stack_size = size_old;
110      }
111  #endif
112  #ifdef CONFIG_TSAN
113      if (fake_stack_save) {
114          __tsan_release(fake_stack_save);
115          __tsan_switch_to_fiber(fake_stack_save, 0);  /* 0=synchronize */
116      }
117  #endif
118  }
119  
120  /* always_inline is required to avoid TSan runtime fatal errors. */
121  static inline __attribute__((always_inline))
start_switch_fiber_asan(void ** fake_stack_save,const void * bottom,size_t size)122  void start_switch_fiber_asan(void **fake_stack_save,
123                               const void *bottom, size_t size)
124  {
125  #ifdef CONFIG_ASAN
126      __sanitizer_start_switch_fiber(fake_stack_save, bottom, size);
127  #endif
128  }
129  
130  /* always_inline is required to avoid TSan runtime fatal errors. */
131  static inline __attribute__((always_inline))
start_switch_fiber_tsan(void ** fake_stack_save,CoroutineUContext * co,bool caller)132  void start_switch_fiber_tsan(void **fake_stack_save,
133                               CoroutineUContext *co,
134                               bool caller)
135  {
136  #ifdef CONFIG_TSAN
137      void *new_fiber = caller ?
138                        co->tsan_caller_fiber :
139                        co->tsan_co_fiber;
140      void *curr_fiber = __tsan_get_current_fiber();
141      __tsan_acquire(curr_fiber);
142  
143      *fake_stack_save = curr_fiber;
144      __tsan_switch_to_fiber(new_fiber, 0);  /* 0=synchronize */
145  #endif
146  }
147  
coroutine_trampoline(int i0,int i1)148  static void coroutine_trampoline(int i0, int i1)
149  {
150      union cc_arg arg;
151      CoroutineUContext *self;
152      Coroutine *co;
153      void *fake_stack_save = NULL;
154  
155      finish_switch_fiber(NULL);
156  
157      arg.i[0] = i0;
158      arg.i[1] = i1;
159      self = arg.p;
160      co = &self->base;
161  
162      /* Initialize longjmp environment and switch back the caller */
163      if (!sigsetjmp(self->env, 0)) {
164          CoroutineUContext *leaderp = get_ptr_leader();
165  
166          start_switch_fiber_asan(&fake_stack_save,
167                                  leaderp->stack, leaderp->stack_size);
168          start_switch_fiber_tsan(&fake_stack_save, self, true); /* true=caller */
169          siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
170      }
171  
172      finish_switch_fiber(fake_stack_save);
173  
174      while (true) {
175          co->entry(co->entry_arg);
176          qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
177      }
178  }
179  
qemu_coroutine_new(void)180  Coroutine *qemu_coroutine_new(void)
181  {
182      CoroutineUContext *co;
183      ucontext_t old_uc, uc;
184      sigjmp_buf old_env;
185      union cc_arg arg = {0};
186      void *fake_stack_save = NULL;
187  
188      /* The ucontext functions preserve signal masks which incurs a
189       * system call overhead.  sigsetjmp(buf, 0)/siglongjmp() does not
190       * preserve signal masks but only works on the current stack.
191       * Since we need a way to create and switch to a new stack, use
192       * the ucontext functions for that but sigsetjmp()/siglongjmp() for
193       * everything else.
194       */
195  
196      if (getcontext(&uc) == -1) {
197          abort();
198      }
199  
200      co = g_malloc0(sizeof(*co));
201      co->stack_size = COROUTINE_STACK_SIZE;
202      co->stack = qemu_alloc_stack(&co->stack_size);
203  #ifdef CONFIG_SAFESTACK
204      co->unsafe_stack_size = COROUTINE_STACK_SIZE;
205      co->unsafe_stack = qemu_alloc_stack(&co->unsafe_stack_size);
206  #endif
207      co->base.entry_arg = &old_env; /* stash away our jmp_buf */
208  
209      uc.uc_link = &old_uc;
210      uc.uc_stack.ss_sp = co->stack;
211      uc.uc_stack.ss_size = co->stack_size;
212      uc.uc_stack.ss_flags = 0;
213  
214  #ifdef CONFIG_VALGRIND_H
215      co->valgrind_stack_id =
216          VALGRIND_STACK_REGISTER(co->stack, co->stack + co->stack_size);
217  #endif
218  
219      arg.p = co;
220  
221      on_new_fiber(co);
222      makecontext(&uc, (void (*)(void))coroutine_trampoline,
223                  2, arg.i[0], arg.i[1]);
224  
225      /* swapcontext() in, siglongjmp() back out */
226      if (!sigsetjmp(old_env, 0)) {
227          start_switch_fiber_asan(&fake_stack_save, co->stack, co->stack_size);
228          start_switch_fiber_tsan(&fake_stack_save,
229                                  co, false); /* false=not caller */
230  
231  #ifdef CONFIG_SAFESTACK
232          /*
233           * Before we swap the context, set the new unsafe stack
234           * The unsafe stack grows just like the normal stack, so start from
235           * the last usable location of the memory area.
236           * NOTE: we don't have to re-set the usp afterwards because we are
237           * coming back to this context through a siglongjmp.
238           * The compiler already wrapped the corresponding sigsetjmp call with
239           * code that saves the usp on the (safe) stack before the call, and
240           * restores it right after (which is where we return with siglongjmp).
241           */
242          void *usp = co->unsafe_stack + co->unsafe_stack_size;
243          __safestack_unsafe_stack_ptr = usp;
244  #endif
245  
246          swapcontext(&old_uc, &uc);
247      }
248  
249      finish_switch_fiber(fake_stack_save);
250  
251      return &co->base;
252  }
253  
254  #ifdef CONFIG_VALGRIND_H
255  /* Work around an unused variable in the valgrind.h macro... */
256  #if !defined(__clang__)
257  #pragma GCC diagnostic push
258  #pragma GCC diagnostic ignored "-Wunused-but-set-variable"
259  #endif
valgrind_stack_deregister(CoroutineUContext * co)260  static inline void valgrind_stack_deregister(CoroutineUContext *co)
261  {
262      VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
263  }
264  #if !defined(__clang__)
265  #pragma GCC diagnostic pop
266  #endif
267  #endif
268  
269  #if defined(CONFIG_ASAN) && defined(CONFIG_COROUTINE_POOL)
terminate_asan(void * opaque)270  static void coroutine_fn terminate_asan(void *opaque)
271  {
272      CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, opaque);
273  
274      set_current(opaque);
275      start_switch_fiber_asan(NULL, to->stack, to->stack_size);
276      G_STATIC_ASSERT(!IS_ENABLED(CONFIG_TSAN));
277      siglongjmp(to->env, COROUTINE_ENTER);
278  }
279  #endif
280  
qemu_coroutine_delete(Coroutine * co_)281  void qemu_coroutine_delete(Coroutine *co_)
282  {
283      CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
284  
285  #if defined(CONFIG_ASAN) && defined(CONFIG_COROUTINE_POOL)
286      co_->entry_arg = qemu_coroutine_self();
287      co_->entry = terminate_asan;
288      qemu_coroutine_switch(co_->entry_arg, co_, COROUTINE_ENTER);
289  #endif
290  
291  #ifdef CONFIG_VALGRIND_H
292      valgrind_stack_deregister(co);
293  #endif
294  
295      qemu_free_stack(co->stack, co->stack_size);
296  #ifdef CONFIG_SAFESTACK
297      qemu_free_stack(co->unsafe_stack, co->unsafe_stack_size);
298  #endif
299      g_free(co);
300  }
301  
302  /* This function is marked noinline to prevent GCC from inlining it
303   * into coroutine_trampoline(). If we allow it to do that then it
304   * hoists the code to get the address of the TLS variable "current"
305   * out of the while() loop. This is an invalid transformation because
306   * the sigsetjmp() call may be called when running thread A but
307   * return in thread B, and so we might be in a different thread
308   * context each time round the loop.
309   */
310  CoroutineAction __attribute__((noinline))
qemu_coroutine_switch(Coroutine * from_,Coroutine * to_,CoroutineAction action)311  qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
312                        CoroutineAction action)
313  {
314      CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
315      CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
316      int ret;
317      void *fake_stack_save = NULL;
318  
319      set_current(to_);
320  
321      ret = sigsetjmp(from->env, 0);
322      if (ret == 0) {
323          start_switch_fiber_asan(IS_ENABLED(CONFIG_COROUTINE_POOL) ||
324                                  action != COROUTINE_TERMINATE ?
325                                      &fake_stack_save : NULL,
326                                  to->stack, to->stack_size);
327          start_switch_fiber_tsan(&fake_stack_save,
328                                  to, false); /* false=not caller */
329          siglongjmp(to->env, action);
330      }
331  
332      finish_switch_fiber(fake_stack_save);
333  
334      return ret;
335  }
336  
qemu_coroutine_self(void)337  Coroutine *qemu_coroutine_self(void)
338  {
339      Coroutine *self = get_current();
340      CoroutineUContext *leaderp = get_ptr_leader();
341  
342      if (!self) {
343          self = &leaderp->base;
344          set_current(self);
345      }
346  #ifdef CONFIG_TSAN
347      if (!leaderp->tsan_co_fiber) {
348          leaderp->tsan_co_fiber = __tsan_get_current_fiber();
349      }
350  #endif
351      return self;
352  }
353  
qemu_in_coroutine(void)354  bool qemu_in_coroutine(void)
355  {
356      Coroutine *self = get_current();
357  
358      return self && self->caller;
359  }
360