110817bf0SDaniel P. Berrange /*
210817bf0SDaniel P. Berrange * QEMU coroutines
310817bf0SDaniel P. Berrange *
410817bf0SDaniel P. Berrange * Copyright IBM, Corp. 2011
510817bf0SDaniel P. Berrange *
610817bf0SDaniel P. Berrange * Authors:
710817bf0SDaniel P. Berrange * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
810817bf0SDaniel P. Berrange * Kevin Wolf <kwolf@redhat.com>
910817bf0SDaniel P. Berrange *
1010817bf0SDaniel P. Berrange * This work is licensed under the terms of the GNU LGPL, version 2 or later.
1110817bf0SDaniel P. Berrange * See the COPYING.LIB file in the top-level directory.
1210817bf0SDaniel P. Berrange *
1310817bf0SDaniel P. Berrange */
1410817bf0SDaniel P. Berrange
15aafd7584SPeter Maydell #include "qemu/osdep.h"
1610817bf0SDaniel P. Berrange #include "trace.h"
1710817bf0SDaniel P. Berrange #include "qemu/thread.h"
1810817bf0SDaniel P. Berrange #include "qemu/atomic.h"
1910817bf0SDaniel P. Berrange #include "qemu/coroutine_int.h"
20ac387a08SStefan Hajnoczi #include "qemu/coroutine-tls.h"
210c330a73SPaolo Bonzini #include "block/aio.h"
2210817bf0SDaniel P. Berrange
239ec7a59bSKevin Wolf /**
249ec7a59bSKevin Wolf * The minimal batch size is always 64, coroutines from the release_pool are
259ec7a59bSKevin Wolf * reused as soon as there are 64 coroutines in it. The maximum pool size starts
269ec7a59bSKevin Wolf * with 64 and is increased on demand so that coroutines are not deleted even if
279ec7a59bSKevin Wolf * they are not immediately reused.
289ec7a59bSKevin Wolf */
2910817bf0SDaniel P. Berrange enum {
309ec7a59bSKevin Wolf POOL_MIN_BATCH_SIZE = 64,
319ec7a59bSKevin Wolf POOL_INITIAL_MAX_SIZE = 64,
3210817bf0SDaniel P. Berrange };
3310817bf0SDaniel P. Berrange
3410817bf0SDaniel P. Berrange /** Free list to speed up creation */
3510817bf0SDaniel P. Berrange static QSLIST_HEAD(, Coroutine) release_pool = QSLIST_HEAD_INITIALIZER(pool);
369ec7a59bSKevin Wolf static unsigned int pool_max_size = POOL_INITIAL_MAX_SIZE;
3710817bf0SDaniel P. Berrange static unsigned int release_pool_size;
38ac387a08SStefan Hajnoczi
39ac387a08SStefan Hajnoczi typedef QSLIST_HEAD(, Coroutine) CoroutineQSList;
40ac387a08SStefan Hajnoczi QEMU_DEFINE_STATIC_CO_TLS(CoroutineQSList, alloc_pool);
41ac387a08SStefan Hajnoczi QEMU_DEFINE_STATIC_CO_TLS(unsigned int, alloc_pool_size);
42ac387a08SStefan Hajnoczi QEMU_DEFINE_STATIC_CO_TLS(Notifier, coroutine_pool_cleanup_notifier);
4310817bf0SDaniel P. Berrange
coroutine_pool_cleanup(Notifier * n,void * value)4410817bf0SDaniel P. Berrange static void coroutine_pool_cleanup(Notifier *n, void *value)
4510817bf0SDaniel P. Berrange {
4610817bf0SDaniel P. Berrange Coroutine *co;
4710817bf0SDaniel P. Berrange Coroutine *tmp;
48ac387a08SStefan Hajnoczi CoroutineQSList *alloc_pool = get_ptr_alloc_pool();
4910817bf0SDaniel P. Berrange
50ac387a08SStefan Hajnoczi QSLIST_FOREACH_SAFE(co, alloc_pool, pool_next, tmp) {
51ac387a08SStefan Hajnoczi QSLIST_REMOVE_HEAD(alloc_pool, pool_next);
5210817bf0SDaniel P. Berrange qemu_coroutine_delete(co);
5310817bf0SDaniel P. Berrange }
5410817bf0SDaniel P. Berrange }
5510817bf0SDaniel P. Berrange
qemu_coroutine_create(CoroutineEntry * entry,void * opaque)560b8b8753SPaolo Bonzini Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque)
5710817bf0SDaniel P. Berrange {
5810817bf0SDaniel P. Berrange Coroutine *co = NULL;
5910817bf0SDaniel P. Berrange
60*230f6e06SPaolo Bonzini if (IS_ENABLED(CONFIG_COROUTINE_POOL)) {
61ac387a08SStefan Hajnoczi CoroutineQSList *alloc_pool = get_ptr_alloc_pool();
62ac387a08SStefan Hajnoczi
63ac387a08SStefan Hajnoczi co = QSLIST_FIRST(alloc_pool);
6410817bf0SDaniel P. Berrange if (!co) {
659ec7a59bSKevin Wolf if (release_pool_size > POOL_MIN_BATCH_SIZE) {
6610817bf0SDaniel P. Berrange /* Slow path; a good place to register the destructor, too. */
67ac387a08SStefan Hajnoczi Notifier *notifier = get_ptr_coroutine_pool_cleanup_notifier();
68ac387a08SStefan Hajnoczi if (!notifier->notify) {
69ac387a08SStefan Hajnoczi notifier->notify = coroutine_pool_cleanup;
70ac387a08SStefan Hajnoczi qemu_thread_atexit_add(notifier);
7110817bf0SDaniel P. Berrange }
7210817bf0SDaniel P. Berrange
7310817bf0SDaniel P. Berrange /* This is not exact; there could be a little skew between
7410817bf0SDaniel P. Berrange * release_pool_size and the actual size of release_pool. But
7510817bf0SDaniel P. Berrange * it is just a heuristic, it does not need to be perfect.
7610817bf0SDaniel P. Berrange */
77ac387a08SStefan Hajnoczi set_alloc_pool_size(qatomic_xchg(&release_pool_size, 0));
78ac387a08SStefan Hajnoczi QSLIST_MOVE_ATOMIC(alloc_pool, &release_pool);
79ac387a08SStefan Hajnoczi co = QSLIST_FIRST(alloc_pool);
8010817bf0SDaniel P. Berrange }
8110817bf0SDaniel P. Berrange }
8210817bf0SDaniel P. Berrange if (co) {
83ac387a08SStefan Hajnoczi QSLIST_REMOVE_HEAD(alloc_pool, pool_next);
84ac387a08SStefan Hajnoczi set_alloc_pool_size(get_alloc_pool_size() - 1);
8510817bf0SDaniel P. Berrange }
8610817bf0SDaniel P. Berrange }
8710817bf0SDaniel P. Berrange
8810817bf0SDaniel P. Berrange if (!co) {
8910817bf0SDaniel P. Berrange co = qemu_coroutine_new();
9010817bf0SDaniel P. Berrange }
9110817bf0SDaniel P. Berrange
9210817bf0SDaniel P. Berrange co->entry = entry;
930b8b8753SPaolo Bonzini co->entry_arg = opaque;
947d9c8581SPaolo Bonzini QSIMPLEQ_INIT(&co->co_queue_wakeup);
9510817bf0SDaniel P. Berrange return co;
9610817bf0SDaniel P. Berrange }
9710817bf0SDaniel P. Berrange
coroutine_delete(Coroutine * co)9810817bf0SDaniel P. Berrange static void coroutine_delete(Coroutine *co)
9910817bf0SDaniel P. Berrange {
10010817bf0SDaniel P. Berrange co->caller = NULL;
10110817bf0SDaniel P. Berrange
102*230f6e06SPaolo Bonzini if (IS_ENABLED(CONFIG_COROUTINE_POOL)) {
1039ec7a59bSKevin Wolf if (release_pool_size < qatomic_read(&pool_max_size) * 2) {
10410817bf0SDaniel P. Berrange QSLIST_INSERT_HEAD_ATOMIC(&release_pool, co, pool_next);
105d73415a3SStefan Hajnoczi qatomic_inc(&release_pool_size);
10610817bf0SDaniel P. Berrange return;
10710817bf0SDaniel P. Berrange }
1089ec7a59bSKevin Wolf if (get_alloc_pool_size() < qatomic_read(&pool_max_size)) {
109ac387a08SStefan Hajnoczi QSLIST_INSERT_HEAD(get_ptr_alloc_pool(), co, pool_next);
110ac387a08SStefan Hajnoczi set_alloc_pool_size(get_alloc_pool_size() + 1);
11110817bf0SDaniel P. Berrange return;
11210817bf0SDaniel P. Berrange }
11310817bf0SDaniel P. Berrange }
11410817bf0SDaniel P. Berrange
11510817bf0SDaniel P. Berrange qemu_coroutine_delete(co);
11610817bf0SDaniel P. Berrange }
11710817bf0SDaniel P. Berrange
qemu_aio_coroutine_enter(AioContext * ctx,Coroutine * co)118ba9e75ceSFam Zheng void qemu_aio_coroutine_enter(AioContext *ctx, Coroutine *co)
11910817bf0SDaniel P. Berrange {
120c40a2545SStefan Hajnoczi QSIMPLEQ_HEAD(, Coroutine) pending = QSIMPLEQ_HEAD_INITIALIZER(pending);
121c40a2545SStefan Hajnoczi Coroutine *from = qemu_coroutine_self();
122c40a2545SStefan Hajnoczi
123c40a2545SStefan Hajnoczi QSIMPLEQ_INSERT_TAIL(&pending, co, co_queue_next);
124c40a2545SStefan Hajnoczi
125c40a2545SStefan Hajnoczi /* Run co and any queued coroutines */
126c40a2545SStefan Hajnoczi while (!QSIMPLEQ_EMPTY(&pending)) {
127c40a2545SStefan Hajnoczi Coroutine *to = QSIMPLEQ_FIRST(&pending);
12810817bf0SDaniel P. Berrange CoroutineAction ret;
12910817bf0SDaniel P. Berrange
130512c90c9SPaolo Bonzini /*
131512c90c9SPaolo Bonzini * Read to before to->scheduled; pairs with qatomic_cmpxchg in
132512c90c9SPaolo Bonzini * qemu_co_sleep(), aio_co_schedule() etc.
133512c90c9SPaolo Bonzini */
134512c90c9SPaolo Bonzini smp_read_barrier_depends();
135512c90c9SPaolo Bonzini
136512c90c9SPaolo Bonzini const char *scheduled = qatomic_read(&to->scheduled);
1376133b39fSJeff Cody
138c40a2545SStefan Hajnoczi QSIMPLEQ_REMOVE_HEAD(&pending, co_queue_next);
139c40a2545SStefan Hajnoczi
140c40a2545SStefan Hajnoczi trace_qemu_aio_coroutine_enter(ctx, from, to, to->entry_arg);
14110817bf0SDaniel P. Berrange
1426133b39fSJeff Cody /* if the Coroutine has already been scheduled, entering it again will
1436133b39fSJeff Cody * cause us to enter it twice, potentially even after the coroutine has
1446133b39fSJeff Cody * been deleted */
1456133b39fSJeff Cody if (scheduled) {
1466133b39fSJeff Cody fprintf(stderr,
1476133b39fSJeff Cody "%s: Co-routine was already scheduled in '%s'\n",
1486133b39fSJeff Cody __func__, scheduled);
1496133b39fSJeff Cody abort();
1506133b39fSJeff Cody }
1516133b39fSJeff Cody
152c40a2545SStefan Hajnoczi if (to->caller) {
15310817bf0SDaniel P. Berrange fprintf(stderr, "Co-routine re-entered recursively\n");
15410817bf0SDaniel P. Berrange abort();
15510817bf0SDaniel P. Berrange }
15610817bf0SDaniel P. Berrange
157c40a2545SStefan Hajnoczi to->caller = from;
158c40a2545SStefan Hajnoczi to->ctx = ctx;
1590c330a73SPaolo Bonzini
160c40a2545SStefan Hajnoczi /* Store to->ctx before anything that stores to. Matches
161480cff63SPaolo Bonzini * barrier in aio_co_wake and qemu_co_mutex_wake.
1620c330a73SPaolo Bonzini */
1630c330a73SPaolo Bonzini smp_wmb();
1640c330a73SPaolo Bonzini
165c40a2545SStefan Hajnoczi ret = qemu_coroutine_switch(from, to, COROUTINE_ENTER);
16610817bf0SDaniel P. Berrange
167c40a2545SStefan Hajnoczi /* Queued coroutines are run depth-first; previously pending coroutines
168c40a2545SStefan Hajnoczi * run after those queued more recently.
169528f449fSRoman Pen */
170c40a2545SStefan Hajnoczi QSIMPLEQ_PREPEND(&pending, &to->co_queue_wakeup);
171528f449fSRoman Pen
17210817bf0SDaniel P. Berrange switch (ret) {
17310817bf0SDaniel P. Berrange case COROUTINE_YIELD:
174c40a2545SStefan Hajnoczi break;
17510817bf0SDaniel P. Berrange case COROUTINE_TERMINATE:
176c40a2545SStefan Hajnoczi assert(!to->locks_held);
177c40a2545SStefan Hajnoczi trace_qemu_coroutine_terminate(to);
178c40a2545SStefan Hajnoczi coroutine_delete(to);
179c40a2545SStefan Hajnoczi break;
18010817bf0SDaniel P. Berrange default:
18110817bf0SDaniel P. Berrange abort();
18210817bf0SDaniel P. Berrange }
18310817bf0SDaniel P. Berrange }
184c40a2545SStefan Hajnoczi }
18510817bf0SDaniel P. Berrange
qemu_coroutine_enter(Coroutine * co)186ba9e75ceSFam Zheng void qemu_coroutine_enter(Coroutine *co)
187ba9e75ceSFam Zheng {
188ba9e75ceSFam Zheng qemu_aio_coroutine_enter(qemu_get_current_aio_context(), co);
189ba9e75ceSFam Zheng }
190ba9e75ceSFam Zheng
qemu_coroutine_enter_if_inactive(Coroutine * co)191536fca7fSKevin Wolf void qemu_coroutine_enter_if_inactive(Coroutine *co)
192536fca7fSKevin Wolf {
193536fca7fSKevin Wolf if (!qemu_coroutine_entered(co)) {
194536fca7fSKevin Wolf qemu_coroutine_enter(co);
195536fca7fSKevin Wolf }
196536fca7fSKevin Wolf }
197536fca7fSKevin Wolf
qemu_coroutine_yield(void)19810817bf0SDaniel P. Berrange void coroutine_fn qemu_coroutine_yield(void)
19910817bf0SDaniel P. Berrange {
20010817bf0SDaniel P. Berrange Coroutine *self = qemu_coroutine_self();
20110817bf0SDaniel P. Berrange Coroutine *to = self->caller;
20210817bf0SDaniel P. Berrange
20310817bf0SDaniel P. Berrange trace_qemu_coroutine_yield(self, to);
20410817bf0SDaniel P. Berrange
20510817bf0SDaniel P. Berrange if (!to) {
20610817bf0SDaniel P. Berrange fprintf(stderr, "Co-routine is yielding to no one\n");
20710817bf0SDaniel P. Berrange abort();
20810817bf0SDaniel P. Berrange }
20910817bf0SDaniel P. Berrange
21010817bf0SDaniel P. Berrange self->caller = NULL;
21110817bf0SDaniel P. Berrange qemu_coroutine_switch(self, to, COROUTINE_YIELD);
21210817bf0SDaniel P. Berrange }
213f643e469SStefan Hajnoczi
qemu_coroutine_entered(Coroutine * co)214f643e469SStefan Hajnoczi bool qemu_coroutine_entered(Coroutine *co)
215f643e469SStefan Hajnoczi {
216f643e469SStefan Hajnoczi return co->caller;
217f643e469SStefan Hajnoczi }
218aa1361d5SKevin Wolf
qemu_coroutine_get_aio_context(Coroutine * co)219a248b856SPaolo Bonzini AioContext *qemu_coroutine_get_aio_context(Coroutine *co)
220aa1361d5SKevin Wolf {
221aa1361d5SKevin Wolf return co->ctx;
222aa1361d5SKevin Wolf }
2234c41c69eSHiroki Narukawa
qemu_coroutine_inc_pool_size(unsigned int additional_pool_size)22498e3ab35SKevin Wolf void qemu_coroutine_inc_pool_size(unsigned int additional_pool_size)
2254c41c69eSHiroki Narukawa {
2269ec7a59bSKevin Wolf qatomic_add(&pool_max_size, additional_pool_size);
2274c41c69eSHiroki Narukawa }
2284c41c69eSHiroki Narukawa
qemu_coroutine_dec_pool_size(unsigned int removing_pool_size)22998e3ab35SKevin Wolf void qemu_coroutine_dec_pool_size(unsigned int removing_pool_size)
2304c41c69eSHiroki Narukawa {
2319ec7a59bSKevin Wolf qatomic_sub(&pool_max_size, removing_pool_size);
2324c41c69eSHiroki Narukawa }
233