1c2b38b27SPaolo Bonzini /*
2c2b38b27SPaolo Bonzini * QEMU aio implementation
3c2b38b27SPaolo Bonzini *
4c2b38b27SPaolo Bonzini * Copyright IBM, Corp. 2008
5c2b38b27SPaolo Bonzini *
6c2b38b27SPaolo Bonzini * Authors:
7c2b38b27SPaolo Bonzini * Anthony Liguori <aliguori@us.ibm.com>
8c2b38b27SPaolo Bonzini *
9c2b38b27SPaolo Bonzini * This work is licensed under the terms of the GNU GPL, version 2. See
10c2b38b27SPaolo Bonzini * the COPYING file in the top-level directory.
11c2b38b27SPaolo Bonzini *
12c2b38b27SPaolo Bonzini * Contributions after 2012-01-13 are licensed under the terms of the
13c2b38b27SPaolo Bonzini * GNU GPL, version 2 or (at your option) any later version.
14c2b38b27SPaolo Bonzini */
15c2b38b27SPaolo Bonzini
16c2b38b27SPaolo Bonzini #include "qemu/osdep.h"
17c2b38b27SPaolo Bonzini #include "block/block.h"
1871ad4713SNicolas Saenz Julienne #include "block/thread-pool.h"
199ce44e2cSKevin Wolf #include "qemu/main-loop.h"
20f25c0b54SStefan Hajnoczi #include "qemu/rcu.h"
21c2b38b27SPaolo Bonzini #include "qemu/rcu_queue.h"
22c2b38b27SPaolo Bonzini #include "qemu/sockets.h"
23c2b38b27SPaolo Bonzini #include "qemu/cutils.h"
24c2b38b27SPaolo Bonzini #include "trace.h"
251f050a46SStefan Hajnoczi #include "aio-posix.h"
26c2b38b27SPaolo Bonzini
27d37d0e36SStefan Hajnoczi /* Stop userspace polling on a handler if it isn't active for some time */
28d37d0e36SStefan Hajnoczi #define POLL_IDLE_INTERVAL_NS (7 * NANOSECONDS_PER_SECOND)
29d37d0e36SStefan Hajnoczi
aio_poll_disabled(AioContext * ctx)30aa38e19fSStefan Hajnoczi bool aio_poll_disabled(AioContext *ctx)
31aa38e19fSStefan Hajnoczi {
32d73415a3SStefan Hajnoczi return qatomic_read(&ctx->poll_disable_cnt);
33aa38e19fSStefan Hajnoczi }
34aa38e19fSStefan Hajnoczi
aio_add_ready_handler(AioHandlerList * ready_list,AioHandler * node,int revents)351f050a46SStefan Hajnoczi void aio_add_ready_handler(AioHandlerList *ready_list,
367391d34cSStefan Hajnoczi AioHandler *node,
377391d34cSStefan Hajnoczi int revents)
387391d34cSStefan Hajnoczi {
397391d34cSStefan Hajnoczi QLIST_SAFE_REMOVE(node, node_ready); /* remove from nested parent's list */
407391d34cSStefan Hajnoczi node->pfd.revents = revents;
417391d34cSStefan Hajnoczi QLIST_INSERT_HEAD(ready_list, node, node_ready);
427391d34cSStefan Hajnoczi }
437391d34cSStefan Hajnoczi
aio_add_poll_ready_handler(AioHandlerList * ready_list,AioHandler * node)44fc879646SStefan Hajnoczi static void aio_add_poll_ready_handler(AioHandlerList *ready_list,
45fc879646SStefan Hajnoczi AioHandler *node)
46fc879646SStefan Hajnoczi {
47fc879646SStefan Hajnoczi QLIST_SAFE_REMOVE(node, node_ready); /* remove from nested parent's list */
48fc879646SStefan Hajnoczi node->poll_ready = true;
49fc879646SStefan Hajnoczi QLIST_INSERT_HEAD(ready_list, node, node_ready);
50fc879646SStefan Hajnoczi }
51fc879646SStefan Hajnoczi
find_aio_handler(AioContext * ctx,int fd)52c2b38b27SPaolo Bonzini static AioHandler *find_aio_handler(AioContext *ctx, int fd)
53c2b38b27SPaolo Bonzini {
54c2b38b27SPaolo Bonzini AioHandler *node;
55c2b38b27SPaolo Bonzini
56c2b38b27SPaolo Bonzini QLIST_FOREACH(node, &ctx->aio_handlers, node) {
574749079cSStefan Hajnoczi if (node->pfd.fd == fd) {
584749079cSStefan Hajnoczi if (!QLIST_IS_INSERTED(node, node_deleted)) {
59c2b38b27SPaolo Bonzini return node;
60c2b38b27SPaolo Bonzini }
614749079cSStefan Hajnoczi }
624749079cSStefan Hajnoczi }
63c2b38b27SPaolo Bonzini
64c2b38b27SPaolo Bonzini return NULL;
65c2b38b27SPaolo Bonzini }
66c2b38b27SPaolo Bonzini
aio_remove_fd_handler(AioContext * ctx,AioHandler * node)67fef16601SRemy Noel static bool aio_remove_fd_handler(AioContext *ctx, AioHandler *node)
68fef16601SRemy Noel {
69fef16601SRemy Noel /* If the GSource is in the process of being destroyed then
70fef16601SRemy Noel * g_source_remove_poll() causes an assertion failure. Skip
71fef16601SRemy Noel * removal in that case, because glib cleans up its state during
72fef16601SRemy Noel * destruction anyway.
73fef16601SRemy Noel */
74fef16601SRemy Noel if (!g_source_is_destroyed(&ctx->source)) {
75fef16601SRemy Noel g_source_remove_poll(&ctx->source, &node->pfd);
76fef16601SRemy Noel }
77fef16601SRemy Noel
7873fd282eSStefan Hajnoczi node->pfd.revents = 0;
79fc879646SStefan Hajnoczi node->poll_ready = false;
8073fd282eSStefan Hajnoczi
8173fd282eSStefan Hajnoczi /* If the fd monitor has already marked it deleted, leave it alone */
8273fd282eSStefan Hajnoczi if (QLIST_IS_INSERTED(node, node_deleted)) {
8373fd282eSStefan Hajnoczi return false;
8473fd282eSStefan Hajnoczi }
8573fd282eSStefan Hajnoczi
86fef16601SRemy Noel /* If a read is in progress, just mark the node as deleted */
87fef16601SRemy Noel if (qemu_lockcnt_count(&ctx->list_lock)) {
884749079cSStefan Hajnoczi QLIST_INSERT_HEAD_RCU(&ctx->deleted_aio_handlers, node, node_deleted);
89fef16601SRemy Noel return false;
90fef16601SRemy Noel }
91fef16601SRemy Noel /* Otherwise, delete it for real. We can't just mark it as
92fef16601SRemy Noel * deleted because deleted nodes are only cleaned up while
93fef16601SRemy Noel * no one is walking the handlers list.
94fef16601SRemy Noel */
95d37d0e36SStefan Hajnoczi QLIST_SAFE_REMOVE(node, node_poll);
96fef16601SRemy Noel QLIST_REMOVE(node, node);
97fef16601SRemy Noel return true;
98fef16601SRemy Noel }
99fef16601SRemy Noel
aio_set_fd_handler(AioContext * ctx,int fd,IOHandler * io_read,IOHandler * io_write,AioPollFn * io_poll,IOHandler * io_poll_ready,void * opaque)100c2b38b27SPaolo Bonzini void aio_set_fd_handler(AioContext *ctx,
101c2b38b27SPaolo Bonzini int fd,
102c2b38b27SPaolo Bonzini IOHandler *io_read,
103c2b38b27SPaolo Bonzini IOHandler *io_write,
104c2b38b27SPaolo Bonzini AioPollFn *io_poll,
105826cc324SStefan Hajnoczi IOHandler *io_poll_ready,
106c2b38b27SPaolo Bonzini void *opaque)
107c2b38b27SPaolo Bonzini {
108c2b38b27SPaolo Bonzini AioHandler *node;
109fef16601SRemy Noel AioHandler *new_node = NULL;
110c2b38b27SPaolo Bonzini bool is_new = false;
111c2b38b27SPaolo Bonzini bool deleted = false;
112d7be5dd1SPaolo Bonzini int poll_disable_change;
113c2b38b27SPaolo Bonzini
114826cc324SStefan Hajnoczi if (io_poll && !io_poll_ready) {
115826cc324SStefan Hajnoczi io_poll = NULL; /* polling only makes sense if there is a handler */
116826cc324SStefan Hajnoczi }
117826cc324SStefan Hajnoczi
118c2b38b27SPaolo Bonzini qemu_lockcnt_lock(&ctx->list_lock);
119c2b38b27SPaolo Bonzini
120c2b38b27SPaolo Bonzini node = find_aio_handler(ctx, fd);
121c2b38b27SPaolo Bonzini
122c2b38b27SPaolo Bonzini /* Are we deleting the fd handler? */
123c2b38b27SPaolo Bonzini if (!io_read && !io_write && !io_poll) {
124c2b38b27SPaolo Bonzini if (node == NULL) {
125c2b38b27SPaolo Bonzini qemu_lockcnt_unlock(&ctx->list_lock);
126c2b38b27SPaolo Bonzini return;
127c2b38b27SPaolo Bonzini }
1288821b34aSRemy Noel /* Clean events in order to unregister fd from the ctx epoll. */
1298821b34aSRemy Noel node->pfd.events = 0;
1308821b34aSRemy Noel
131d7be5dd1SPaolo Bonzini poll_disable_change = -!node->io_poll;
132c2b38b27SPaolo Bonzini } else {
133d7be5dd1SPaolo Bonzini poll_disable_change = !io_poll - (node && !node->io_poll);
134c2b38b27SPaolo Bonzini if (node == NULL) {
135c2b38b27SPaolo Bonzini is_new = true;
136c2b38b27SPaolo Bonzini }
137fef16601SRemy Noel /* Alloc and insert if it's not already there */
138fef16601SRemy Noel new_node = g_new0(AioHandler, 1);
139c2b38b27SPaolo Bonzini
140c2b38b27SPaolo Bonzini /* Update handler with latest information */
141fef16601SRemy Noel new_node->io_read = io_read;
142fef16601SRemy Noel new_node->io_write = io_write;
143fef16601SRemy Noel new_node->io_poll = io_poll;
144826cc324SStefan Hajnoczi new_node->io_poll_ready = io_poll_ready;
145fef16601SRemy Noel new_node->opaque = opaque;
146c2b38b27SPaolo Bonzini
147fef16601SRemy Noel if (is_new) {
148fef16601SRemy Noel new_node->pfd.fd = fd;
149fef16601SRemy Noel } else {
150fef16601SRemy Noel new_node->pfd = node->pfd;
151fef16601SRemy Noel }
152fef16601SRemy Noel g_source_add_poll(&ctx->source, &new_node->pfd);
153fef16601SRemy Noel
154fef16601SRemy Noel new_node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0);
155fef16601SRemy Noel new_node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0);
156fef16601SRemy Noel
157fef16601SRemy Noel QLIST_INSERT_HEAD_RCU(&ctx->aio_handlers, new_node, node);
158fef16601SRemy Noel }
159c2b38b27SPaolo Bonzini
160d7be5dd1SPaolo Bonzini /* No need to order poll_disable_cnt writes against other updates;
161d7be5dd1SPaolo Bonzini * the counter is only used to avoid wasting time and latency on
162d7be5dd1SPaolo Bonzini * iterated polling when the system call will be ultimately necessary.
163d7be5dd1SPaolo Bonzini * Changing handlers is a rare event, and a little wasted polling until
164d7be5dd1SPaolo Bonzini * the aio_notify below is not an issue.
165d7be5dd1SPaolo Bonzini */
166d73415a3SStefan Hajnoczi qatomic_set(&ctx->poll_disable_cnt,
167d73415a3SStefan Hajnoczi qatomic_read(&ctx->poll_disable_cnt) + poll_disable_change);
168d7be5dd1SPaolo Bonzini
169b321051cSStefan Hajnoczi ctx->fdmon_ops->update(ctx, node, new_node);
17073fd282eSStefan Hajnoczi if (node) {
17173fd282eSStefan Hajnoczi deleted = aio_remove_fd_handler(ctx, node);
17273fd282eSStefan Hajnoczi }
173c2b38b27SPaolo Bonzini qemu_lockcnt_unlock(&ctx->list_lock);
174c2b38b27SPaolo Bonzini aio_notify(ctx);
175c2b38b27SPaolo Bonzini
176c2b38b27SPaolo Bonzini if (deleted) {
177c2b38b27SPaolo Bonzini g_free(node);
178c2b38b27SPaolo Bonzini }
179c2b38b27SPaolo Bonzini }
180c2b38b27SPaolo Bonzini
aio_set_fd_poll(AioContext * ctx,int fd,IOHandler * io_poll_begin,IOHandler * io_poll_end)1816eeef447SMarc-André Lureau static void aio_set_fd_poll(AioContext *ctx, int fd,
182c2b38b27SPaolo Bonzini IOHandler *io_poll_begin,
183c2b38b27SPaolo Bonzini IOHandler *io_poll_end)
184c2b38b27SPaolo Bonzini {
185c2b38b27SPaolo Bonzini AioHandler *node = find_aio_handler(ctx, fd);
186c2b38b27SPaolo Bonzini
187c2b38b27SPaolo Bonzini if (!node) {
188c2b38b27SPaolo Bonzini return;
189c2b38b27SPaolo Bonzini }
190c2b38b27SPaolo Bonzini
191c2b38b27SPaolo Bonzini node->io_poll_begin = io_poll_begin;
192c2b38b27SPaolo Bonzini node->io_poll_end = io_poll_end;
193c2b38b27SPaolo Bonzini }
194c2b38b27SPaolo Bonzini
aio_set_event_notifier(AioContext * ctx,EventNotifier * notifier,EventNotifierHandler * io_read,AioPollFn * io_poll,EventNotifierHandler * io_poll_ready)195c2b38b27SPaolo Bonzini void aio_set_event_notifier(AioContext *ctx,
196c2b38b27SPaolo Bonzini EventNotifier *notifier,
197c2b38b27SPaolo Bonzini EventNotifierHandler *io_read,
198826cc324SStefan Hajnoczi AioPollFn *io_poll,
199826cc324SStefan Hajnoczi EventNotifierHandler *io_poll_ready)
200c2b38b27SPaolo Bonzini {
201*60f782b6SStefan Hajnoczi aio_set_fd_handler(ctx, event_notifier_get_fd(notifier),
202826cc324SStefan Hajnoczi (IOHandler *)io_read, NULL, io_poll,
203826cc324SStefan Hajnoczi (IOHandler *)io_poll_ready, notifier);
204c2b38b27SPaolo Bonzini }
205c2b38b27SPaolo Bonzini
aio_set_event_notifier_poll(AioContext * ctx,EventNotifier * notifier,EventNotifierHandler * io_poll_begin,EventNotifierHandler * io_poll_end)206c2b38b27SPaolo Bonzini void aio_set_event_notifier_poll(AioContext *ctx,
207c2b38b27SPaolo Bonzini EventNotifier *notifier,
208c2b38b27SPaolo Bonzini EventNotifierHandler *io_poll_begin,
209c2b38b27SPaolo Bonzini EventNotifierHandler *io_poll_end)
210c2b38b27SPaolo Bonzini {
211c2b38b27SPaolo Bonzini aio_set_fd_poll(ctx, event_notifier_get_fd(notifier),
212c2b38b27SPaolo Bonzini (IOHandler *)io_poll_begin,
213c2b38b27SPaolo Bonzini (IOHandler *)io_poll_end);
214c2b38b27SPaolo Bonzini }
215c2b38b27SPaolo Bonzini
poll_set_started(AioContext * ctx,AioHandlerList * ready_list,bool started)216826cc324SStefan Hajnoczi static bool poll_set_started(AioContext *ctx, AioHandlerList *ready_list,
217826cc324SStefan Hajnoczi bool started)
218c2b38b27SPaolo Bonzini {
219c2b38b27SPaolo Bonzini AioHandler *node;
220e4346192SStefan Hajnoczi bool progress = false;
221c2b38b27SPaolo Bonzini
222c2b38b27SPaolo Bonzini if (started == ctx->poll_started) {
223e4346192SStefan Hajnoczi return false;
224c2b38b27SPaolo Bonzini }
225c2b38b27SPaolo Bonzini
226c2b38b27SPaolo Bonzini ctx->poll_started = started;
227c2b38b27SPaolo Bonzini
228c2b38b27SPaolo Bonzini qemu_lockcnt_inc(&ctx->list_lock);
229d37d0e36SStefan Hajnoczi QLIST_FOREACH(node, &ctx->poll_aio_handlers, node_poll) {
230c2b38b27SPaolo Bonzini IOHandler *fn;
231c2b38b27SPaolo Bonzini
2324749079cSStefan Hajnoczi if (QLIST_IS_INSERTED(node, node_deleted)) {
233c2b38b27SPaolo Bonzini continue;
234c2b38b27SPaolo Bonzini }
235c2b38b27SPaolo Bonzini
236c2b38b27SPaolo Bonzini if (started) {
237c2b38b27SPaolo Bonzini fn = node->io_poll_begin;
238c2b38b27SPaolo Bonzini } else {
239c2b38b27SPaolo Bonzini fn = node->io_poll_end;
240c2b38b27SPaolo Bonzini }
241c2b38b27SPaolo Bonzini
242c2b38b27SPaolo Bonzini if (fn) {
243c2b38b27SPaolo Bonzini fn(node->opaque);
244c2b38b27SPaolo Bonzini }
245e4346192SStefan Hajnoczi
246e4346192SStefan Hajnoczi /* Poll one last time in case ->io_poll_end() raced with the event */
247826cc324SStefan Hajnoczi if (!started && node->io_poll(node->opaque)) {
248fc879646SStefan Hajnoczi aio_add_poll_ready_handler(ready_list, node);
249826cc324SStefan Hajnoczi progress = true;
250e4346192SStefan Hajnoczi }
251c2b38b27SPaolo Bonzini }
252c2b38b27SPaolo Bonzini qemu_lockcnt_dec(&ctx->list_lock);
253e4346192SStefan Hajnoczi
254e4346192SStefan Hajnoczi return progress;
255c2b38b27SPaolo Bonzini }
256c2b38b27SPaolo Bonzini
257c2b38b27SPaolo Bonzini
aio_prepare(AioContext * ctx)258c2b38b27SPaolo Bonzini bool aio_prepare(AioContext *ctx)
259c2b38b27SPaolo Bonzini {
260826cc324SStefan Hajnoczi AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);
261826cc324SStefan Hajnoczi
262c2b38b27SPaolo Bonzini /* Poll mode cannot be used with glib's event loop, disable it. */
263826cc324SStefan Hajnoczi poll_set_started(ctx, &ready_list, false);
264826cc324SStefan Hajnoczi /* TODO what to do with this list? */
265c2b38b27SPaolo Bonzini
266c2b38b27SPaolo Bonzini return false;
267c2b38b27SPaolo Bonzini }
268c2b38b27SPaolo Bonzini
aio_pending(AioContext * ctx)269c2b38b27SPaolo Bonzini bool aio_pending(AioContext *ctx)
270c2b38b27SPaolo Bonzini {
271c2b38b27SPaolo Bonzini AioHandler *node;
272c2b38b27SPaolo Bonzini bool result = false;
273c2b38b27SPaolo Bonzini
274c2b38b27SPaolo Bonzini /*
275c2b38b27SPaolo Bonzini * We have to walk very carefully in case aio_set_fd_handler is
276c2b38b27SPaolo Bonzini * called while we're walking.
277c2b38b27SPaolo Bonzini */
278c2b38b27SPaolo Bonzini qemu_lockcnt_inc(&ctx->list_lock);
279c2b38b27SPaolo Bonzini
280c2b38b27SPaolo Bonzini QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) {
281c2b38b27SPaolo Bonzini int revents;
282c2b38b27SPaolo Bonzini
283fc879646SStefan Hajnoczi /* TODO should this check poll ready? */
284c2b38b27SPaolo Bonzini revents = node->pfd.revents & node->pfd.events;
285*60f782b6SStefan Hajnoczi if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read) {
286c2b38b27SPaolo Bonzini result = true;
287c2b38b27SPaolo Bonzini break;
288c2b38b27SPaolo Bonzini }
289*60f782b6SStefan Hajnoczi if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write) {
290c2b38b27SPaolo Bonzini result = true;
291c2b38b27SPaolo Bonzini break;
292c2b38b27SPaolo Bonzini }
293c2b38b27SPaolo Bonzini }
294c2b38b27SPaolo Bonzini qemu_lockcnt_dec(&ctx->list_lock);
295c2b38b27SPaolo Bonzini
296c2b38b27SPaolo Bonzini return result;
297c2b38b27SPaolo Bonzini }
298c2b38b27SPaolo Bonzini
aio_free_deleted_handlers(AioContext * ctx)2994749079cSStefan Hajnoczi static void aio_free_deleted_handlers(AioContext *ctx)
3004749079cSStefan Hajnoczi {
3014749079cSStefan Hajnoczi AioHandler *node;
3024749079cSStefan Hajnoczi
3034749079cSStefan Hajnoczi if (QLIST_EMPTY_RCU(&ctx->deleted_aio_handlers)) {
3044749079cSStefan Hajnoczi return;
3054749079cSStefan Hajnoczi }
3064749079cSStefan Hajnoczi if (!qemu_lockcnt_dec_if_lock(&ctx->list_lock)) {
3074749079cSStefan Hajnoczi return; /* we are nested, let the parent do the freeing */
3084749079cSStefan Hajnoczi }
3094749079cSStefan Hajnoczi
3104749079cSStefan Hajnoczi while ((node = QLIST_FIRST_RCU(&ctx->deleted_aio_handlers))) {
3114749079cSStefan Hajnoczi QLIST_REMOVE(node, node);
3124749079cSStefan Hajnoczi QLIST_REMOVE(node, node_deleted);
313d37d0e36SStefan Hajnoczi QLIST_SAFE_REMOVE(node, node_poll);
3144749079cSStefan Hajnoczi g_free(node);
3154749079cSStefan Hajnoczi }
3164749079cSStefan Hajnoczi
3174749079cSStefan Hajnoczi qemu_lockcnt_inc_and_unlock(&ctx->list_lock);
3184749079cSStefan Hajnoczi }
3194749079cSStefan Hajnoczi
aio_dispatch_handler(AioContext * ctx,AioHandler * node)3207391d34cSStefan Hajnoczi static bool aio_dispatch_handler(AioContext *ctx, AioHandler *node)
321c2b38b27SPaolo Bonzini {
322c2b38b27SPaolo Bonzini bool progress = false;
323fc879646SStefan Hajnoczi bool poll_ready;
324c2b38b27SPaolo Bonzini int revents;
325c2b38b27SPaolo Bonzini
326c2b38b27SPaolo Bonzini revents = node->pfd.revents & node->pfd.events;
327c2b38b27SPaolo Bonzini node->pfd.revents = 0;
328c2b38b27SPaolo Bonzini
329fc879646SStefan Hajnoczi poll_ready = node->poll_ready;
330fc879646SStefan Hajnoczi node->poll_ready = false;
331fc879646SStefan Hajnoczi
332d37d0e36SStefan Hajnoczi /*
333d37d0e36SStefan Hajnoczi * Start polling AioHandlers when they become ready because activity is
334d37d0e36SStefan Hajnoczi * likely to continue. Note that starvation is theoretically possible when
335d37d0e36SStefan Hajnoczi * fdmon_supports_polling(), but only until the fd fires for the first
336d37d0e36SStefan Hajnoczi * time.
337d37d0e36SStefan Hajnoczi */
338d37d0e36SStefan Hajnoczi if (!QLIST_IS_INSERTED(node, node_deleted) &&
339d37d0e36SStefan Hajnoczi !QLIST_IS_INSERTED(node, node_poll) &&
340d37d0e36SStefan Hajnoczi node->io_poll) {
341d37d0e36SStefan Hajnoczi trace_poll_add(ctx, node, node->pfd.fd, revents);
342d37d0e36SStefan Hajnoczi if (ctx->poll_started && node->io_poll_begin) {
343d37d0e36SStefan Hajnoczi node->io_poll_begin(node->opaque);
344d37d0e36SStefan Hajnoczi }
345d37d0e36SStefan Hajnoczi QLIST_INSERT_HEAD(&ctx->poll_aio_handlers, node, node_poll);
346d37d0e36SStefan Hajnoczi }
347826cc324SStefan Hajnoczi if (!QLIST_IS_INSERTED(node, node_deleted) &&
348*60f782b6SStefan Hajnoczi poll_ready && revents == 0 && node->io_poll_ready) {
3496d740fb0SStefan Hajnoczi /*
3506d740fb0SStefan Hajnoczi * Remove temporarily to avoid infinite loops when ->io_poll_ready()
3516d740fb0SStefan Hajnoczi * calls aio_poll() before clearing the condition that made the poll
3526d740fb0SStefan Hajnoczi * handler become ready.
3536d740fb0SStefan Hajnoczi */
3546d740fb0SStefan Hajnoczi QLIST_SAFE_REMOVE(node, node_poll);
3556d740fb0SStefan Hajnoczi
356826cc324SStefan Hajnoczi node->io_poll_ready(node->opaque);
357826cc324SStefan Hajnoczi
3586d740fb0SStefan Hajnoczi if (!QLIST_IS_INSERTED(node, node_poll)) {
3596d740fb0SStefan Hajnoczi QLIST_INSERT_HEAD(&ctx->poll_aio_handlers, node, node_poll);
3606d740fb0SStefan Hajnoczi }
3616d740fb0SStefan Hajnoczi
362826cc324SStefan Hajnoczi /*
363826cc324SStefan Hajnoczi * Return early since revents was zero. aio_notify() does not count as
364826cc324SStefan Hajnoczi * progress.
365826cc324SStefan Hajnoczi */
366826cc324SStefan Hajnoczi return node->opaque != &ctx->notifier;
367826cc324SStefan Hajnoczi }
368d37d0e36SStefan Hajnoczi
3694749079cSStefan Hajnoczi if (!QLIST_IS_INSERTED(node, node_deleted) &&
370c2b38b27SPaolo Bonzini (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) &&
371c2b38b27SPaolo Bonzini node->io_read) {
372c2b38b27SPaolo Bonzini node->io_read(node->opaque);
373c2b38b27SPaolo Bonzini
374c2b38b27SPaolo Bonzini /* aio_notify() does not count as progress */
375c2b38b27SPaolo Bonzini if (node->opaque != &ctx->notifier) {
376c2b38b27SPaolo Bonzini progress = true;
377c2b38b27SPaolo Bonzini }
378c2b38b27SPaolo Bonzini }
3794749079cSStefan Hajnoczi if (!QLIST_IS_INSERTED(node, node_deleted) &&
380c2b38b27SPaolo Bonzini (revents & (G_IO_OUT | G_IO_ERR)) &&
381c2b38b27SPaolo Bonzini node->io_write) {
382c2b38b27SPaolo Bonzini node->io_write(node->opaque);
383c2b38b27SPaolo Bonzini progress = true;
384c2b38b27SPaolo Bonzini }
3857391d34cSStefan Hajnoczi
3867391d34cSStefan Hajnoczi return progress;
3877391d34cSStefan Hajnoczi }
3887391d34cSStefan Hajnoczi
3897391d34cSStefan Hajnoczi /*
3907391d34cSStefan Hajnoczi * If we have a list of ready handlers then this is more efficient than
3917391d34cSStefan Hajnoczi * scanning all handlers with aio_dispatch_handlers().
3927391d34cSStefan Hajnoczi */
aio_dispatch_ready_handlers(AioContext * ctx,AioHandlerList * ready_list)3937391d34cSStefan Hajnoczi static bool aio_dispatch_ready_handlers(AioContext *ctx,
3947391d34cSStefan Hajnoczi AioHandlerList *ready_list)
3957391d34cSStefan Hajnoczi {
3967391d34cSStefan Hajnoczi bool progress = false;
3977391d34cSStefan Hajnoczi AioHandler *node;
3987391d34cSStefan Hajnoczi
3997391d34cSStefan Hajnoczi while ((node = QLIST_FIRST(ready_list))) {
400c39cbedbSStefan Hajnoczi QLIST_REMOVE(node, node_ready);
4017391d34cSStefan Hajnoczi progress = aio_dispatch_handler(ctx, node) || progress;
4027391d34cSStefan Hajnoczi }
4037391d34cSStefan Hajnoczi
4047391d34cSStefan Hajnoczi return progress;
4057391d34cSStefan Hajnoczi }
4067391d34cSStefan Hajnoczi
4077391d34cSStefan Hajnoczi /* Slower than aio_dispatch_ready_handlers() but only used via glib */
aio_dispatch_handlers(AioContext * ctx)4087391d34cSStefan Hajnoczi static bool aio_dispatch_handlers(AioContext *ctx)
4097391d34cSStefan Hajnoczi {
4107391d34cSStefan Hajnoczi AioHandler *node, *tmp;
4117391d34cSStefan Hajnoczi bool progress = false;
4127391d34cSStefan Hajnoczi
4137391d34cSStefan Hajnoczi QLIST_FOREACH_SAFE_RCU(node, &ctx->aio_handlers, node, tmp) {
4147391d34cSStefan Hajnoczi progress = aio_dispatch_handler(ctx, node) || progress;
415c2b38b27SPaolo Bonzini }
416c2b38b27SPaolo Bonzini
417c2b38b27SPaolo Bonzini return progress;
418c2b38b27SPaolo Bonzini }
419c2b38b27SPaolo Bonzini
aio_dispatch(AioContext * ctx)420a153bf52SPaolo Bonzini void aio_dispatch(AioContext *ctx)
421c2b38b27SPaolo Bonzini {
422a153bf52SPaolo Bonzini qemu_lockcnt_inc(&ctx->list_lock);
423bd451435SPaolo Bonzini aio_bh_poll(ctx);
424a153bf52SPaolo Bonzini aio_dispatch_handlers(ctx);
4254749079cSStefan Hajnoczi aio_free_deleted_handlers(ctx);
426a153bf52SPaolo Bonzini qemu_lockcnt_dec(&ctx->list_lock);
427c2b38b27SPaolo Bonzini
428a153bf52SPaolo Bonzini timerlistgroup_run_timers(&ctx->tlg);
429c2b38b27SPaolo Bonzini }
430c2b38b27SPaolo Bonzini
run_poll_handlers_once(AioContext * ctx,AioHandlerList * ready_list,int64_t now,int64_t * timeout)431d37d0e36SStefan Hajnoczi static bool run_poll_handlers_once(AioContext *ctx,
432826cc324SStefan Hajnoczi AioHandlerList *ready_list,
433d37d0e36SStefan Hajnoczi int64_t now,
434d37d0e36SStefan Hajnoczi int64_t *timeout)
435c2b38b27SPaolo Bonzini {
436c2b38b27SPaolo Bonzini bool progress = false;
437c2b38b27SPaolo Bonzini AioHandler *node;
438d37d0e36SStefan Hajnoczi AioHandler *tmp;
439c2b38b27SPaolo Bonzini
440d37d0e36SStefan Hajnoczi QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) {
441*60f782b6SStefan Hajnoczi if (node->io_poll(node->opaque)) {
442fc879646SStefan Hajnoczi aio_add_poll_ready_handler(ready_list, node);
443826cc324SStefan Hajnoczi
444d37d0e36SStefan Hajnoczi node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS;
445d37d0e36SStefan Hajnoczi
446993ed89fSPaolo Bonzini /*
447993ed89fSPaolo Bonzini * Polling was successful, exit try_poll_mode immediately
448993ed89fSPaolo Bonzini * to adjust the next polling time.
449993ed89fSPaolo Bonzini */
450e30cffa0SPaolo Bonzini *timeout = 0;
451cfeb35d6SPaolo Bonzini if (node->opaque != &ctx->notifier) {
452c2b38b27SPaolo Bonzini progress = true;
453c2b38b27SPaolo Bonzini }
454cfeb35d6SPaolo Bonzini }
455c2b38b27SPaolo Bonzini
456c2b38b27SPaolo Bonzini /* Caller handles freeing deleted nodes. Don't do it here. */
457c2b38b27SPaolo Bonzini }
458c2b38b27SPaolo Bonzini
459c2b38b27SPaolo Bonzini return progress;
460c2b38b27SPaolo Bonzini }
461c2b38b27SPaolo Bonzini
fdmon_supports_polling(AioContext * ctx)462d37d0e36SStefan Hajnoczi static bool fdmon_supports_polling(AioContext *ctx)
463d37d0e36SStefan Hajnoczi {
464d37d0e36SStefan Hajnoczi return ctx->fdmon_ops->need_wait != aio_poll_disabled;
465d37d0e36SStefan Hajnoczi }
466d37d0e36SStefan Hajnoczi
remove_idle_poll_handlers(AioContext * ctx,AioHandlerList * ready_list,int64_t now)467826cc324SStefan Hajnoczi static bool remove_idle_poll_handlers(AioContext *ctx,
468826cc324SStefan Hajnoczi AioHandlerList *ready_list,
469826cc324SStefan Hajnoczi int64_t now)
470d37d0e36SStefan Hajnoczi {
471d37d0e36SStefan Hajnoczi AioHandler *node;
472d37d0e36SStefan Hajnoczi AioHandler *tmp;
473d37d0e36SStefan Hajnoczi bool progress = false;
474d37d0e36SStefan Hajnoczi
475d37d0e36SStefan Hajnoczi /*
476d37d0e36SStefan Hajnoczi * File descriptor monitoring implementations without userspace polling
477d37d0e36SStefan Hajnoczi * support suffer from starvation when a subset of handlers is polled
478d37d0e36SStefan Hajnoczi * because fds will not be processed in a timely fashion. Don't remove
479d37d0e36SStefan Hajnoczi * idle poll handlers.
480d37d0e36SStefan Hajnoczi */
481d37d0e36SStefan Hajnoczi if (!fdmon_supports_polling(ctx)) {
482d37d0e36SStefan Hajnoczi return false;
483d37d0e36SStefan Hajnoczi }
484d37d0e36SStefan Hajnoczi
485d37d0e36SStefan Hajnoczi QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) {
486d37d0e36SStefan Hajnoczi if (node->poll_idle_timeout == 0LL) {
487d37d0e36SStefan Hajnoczi node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS;
488d37d0e36SStefan Hajnoczi } else if (now >= node->poll_idle_timeout) {
489d37d0e36SStefan Hajnoczi trace_poll_remove(ctx, node, node->pfd.fd);
490d37d0e36SStefan Hajnoczi node->poll_idle_timeout = 0LL;
491d37d0e36SStefan Hajnoczi QLIST_SAFE_REMOVE(node, node_poll);
492d37d0e36SStefan Hajnoczi if (ctx->poll_started && node->io_poll_end) {
493d37d0e36SStefan Hajnoczi node->io_poll_end(node->opaque);
494d37d0e36SStefan Hajnoczi
495d37d0e36SStefan Hajnoczi /*
496d37d0e36SStefan Hajnoczi * Final poll in case ->io_poll_end() races with an event.
497d37d0e36SStefan Hajnoczi * Nevermind about re-adding the handler in the rare case where
498d37d0e36SStefan Hajnoczi * this causes progress.
499d37d0e36SStefan Hajnoczi */
500826cc324SStefan Hajnoczi if (node->io_poll(node->opaque)) {
501fc879646SStefan Hajnoczi aio_add_poll_ready_handler(ready_list, node);
502826cc324SStefan Hajnoczi progress = true;
503826cc324SStefan Hajnoczi }
504d37d0e36SStefan Hajnoczi }
505d37d0e36SStefan Hajnoczi }
506d37d0e36SStefan Hajnoczi }
507d37d0e36SStefan Hajnoczi
508d37d0e36SStefan Hajnoczi return progress;
509d37d0e36SStefan Hajnoczi }
510d37d0e36SStefan Hajnoczi
511c2b38b27SPaolo Bonzini /* run_poll_handlers:
512c2b38b27SPaolo Bonzini * @ctx: the AioContext
513826cc324SStefan Hajnoczi * @ready_list: the list to place ready handlers on
514c2b38b27SPaolo Bonzini * @max_ns: maximum time to poll for, in nanoseconds
515c2b38b27SPaolo Bonzini *
516c2b38b27SPaolo Bonzini * Polls for a given time.
517c2b38b27SPaolo Bonzini *
518c2b38b27SPaolo Bonzini * Note that the caller must have incremented ctx->list_lock.
519c2b38b27SPaolo Bonzini *
520c2b38b27SPaolo Bonzini * Returns: true if progress was made, false otherwise
521c2b38b27SPaolo Bonzini */
run_poll_handlers(AioContext * ctx,AioHandlerList * ready_list,int64_t max_ns,int64_t * timeout)522826cc324SStefan Hajnoczi static bool run_poll_handlers(AioContext *ctx, AioHandlerList *ready_list,
523826cc324SStefan Hajnoczi int64_t max_ns, int64_t *timeout)
524c2b38b27SPaolo Bonzini {
525c2b38b27SPaolo Bonzini bool progress;
526e30cffa0SPaolo Bonzini int64_t start_time, elapsed_time;
527c2b38b27SPaolo Bonzini
528c2b38b27SPaolo Bonzini assert(qemu_lockcnt_count(&ctx->list_lock) > 0);
529c2b38b27SPaolo Bonzini
530e30cffa0SPaolo Bonzini trace_run_poll_handlers_begin(ctx, max_ns, *timeout);
531c2b38b27SPaolo Bonzini
5323aa221b3SStefan Hajnoczi /*
5333aa221b3SStefan Hajnoczi * Optimization: ->io_poll() handlers often contain RCU read critical
5343aa221b3SStefan Hajnoczi * sections and we therefore see many rcu_read_lock() -> rcu_read_unlock()
5353aa221b3SStefan Hajnoczi * -> rcu_read_lock() -> ... sequences with expensive memory
5363aa221b3SStefan Hajnoczi * synchronization primitives. Make the entire polling loop an RCU
5373aa221b3SStefan Hajnoczi * critical section because nested rcu_read_lock()/rcu_read_unlock() calls
5383aa221b3SStefan Hajnoczi * are cheap.
5393aa221b3SStefan Hajnoczi */
5403aa221b3SStefan Hajnoczi RCU_READ_LOCK_GUARD();
5413aa221b3SStefan Hajnoczi
542e30cffa0SPaolo Bonzini start_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
543c2b38b27SPaolo Bonzini do {
544826cc324SStefan Hajnoczi progress = run_poll_handlers_once(ctx, ready_list,
545826cc324SStefan Hajnoczi start_time, timeout);
546e30cffa0SPaolo Bonzini elapsed_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time;
547993ed89fSPaolo Bonzini max_ns = qemu_soonest_timeout(*timeout, max_ns);
548993ed89fSPaolo Bonzini assert(!(max_ns && progress));
549aa38e19fSStefan Hajnoczi } while (elapsed_time < max_ns && !ctx->fdmon_ops->need_wait(ctx));
550c2b38b27SPaolo Bonzini
551826cc324SStefan Hajnoczi if (remove_idle_poll_handlers(ctx, ready_list,
552826cc324SStefan Hajnoczi start_time + elapsed_time)) {
553d37d0e36SStefan Hajnoczi *timeout = 0;
554d37d0e36SStefan Hajnoczi progress = true;
555d37d0e36SStefan Hajnoczi }
556d37d0e36SStefan Hajnoczi
557e30cffa0SPaolo Bonzini /* If time has passed with no successful polling, adjust *timeout to
558e30cffa0SPaolo Bonzini * keep the same ending time.
559e30cffa0SPaolo Bonzini */
560e30cffa0SPaolo Bonzini if (*timeout != -1) {
561e30cffa0SPaolo Bonzini *timeout -= MIN(*timeout, elapsed_time);
562e30cffa0SPaolo Bonzini }
563c2b38b27SPaolo Bonzini
564e30cffa0SPaolo Bonzini trace_run_poll_handlers_end(ctx, progress, *timeout);
565c2b38b27SPaolo Bonzini return progress;
566c2b38b27SPaolo Bonzini }
567c2b38b27SPaolo Bonzini
568c2b38b27SPaolo Bonzini /* try_poll_mode:
569c2b38b27SPaolo Bonzini * @ctx: the AioContext
570826cc324SStefan Hajnoczi * @ready_list: list to add handlers that need to be run
571e30cffa0SPaolo Bonzini * @timeout: timeout for blocking wait, computed by the caller and updated if
572e30cffa0SPaolo Bonzini * polling succeeds.
573c2b38b27SPaolo Bonzini *
574c2b38b27SPaolo Bonzini * Note that the caller must have incremented ctx->list_lock.
575c2b38b27SPaolo Bonzini *
576c2b38b27SPaolo Bonzini * Returns: true if progress was made, false otherwise
577c2b38b27SPaolo Bonzini */
try_poll_mode(AioContext * ctx,AioHandlerList * ready_list,int64_t * timeout)578826cc324SStefan Hajnoczi static bool try_poll_mode(AioContext *ctx, AioHandlerList *ready_list,
579826cc324SStefan Hajnoczi int64_t *timeout)
580c2b38b27SPaolo Bonzini {
581d37d0e36SStefan Hajnoczi int64_t max_ns;
582c2b38b27SPaolo Bonzini
583d37d0e36SStefan Hajnoczi if (QLIST_EMPTY_RCU(&ctx->poll_aio_handlers)) {
584d37d0e36SStefan Hajnoczi return false;
585d37d0e36SStefan Hajnoczi }
586d37d0e36SStefan Hajnoczi
587d37d0e36SStefan Hajnoczi max_ns = qemu_soonest_timeout(*timeout, ctx->poll_ns);
588aa38e19fSStefan Hajnoczi if (max_ns && !ctx->fdmon_ops->need_wait(ctx)) {
589816a430cSChao Gao /*
590816a430cSChao Gao * Enable poll mode. It pairs with the poll_set_started() in
591816a430cSChao Gao * aio_poll() which disables poll mode.
592816a430cSChao Gao */
593826cc324SStefan Hajnoczi poll_set_started(ctx, ready_list, true);
594c2b38b27SPaolo Bonzini
595826cc324SStefan Hajnoczi if (run_poll_handlers(ctx, ready_list, max_ns, timeout)) {
596c2b38b27SPaolo Bonzini return true;
597c2b38b27SPaolo Bonzini }
598c2b38b27SPaolo Bonzini }
599e4346192SStefan Hajnoczi return false;
600c2b38b27SPaolo Bonzini }
601c2b38b27SPaolo Bonzini
aio_poll(AioContext * ctx,bool blocking)602c2b38b27SPaolo Bonzini bool aio_poll(AioContext *ctx, bool blocking)
603c2b38b27SPaolo Bonzini {
6047391d34cSStefan Hajnoczi AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);
605c2b38b27SPaolo Bonzini bool progress;
60644277bf9SStefan Hajnoczi bool use_notify_me;
607c2b38b27SPaolo Bonzini int64_t timeout;
608c2b38b27SPaolo Bonzini int64_t start = 0;
609c2b38b27SPaolo Bonzini
6105710a3e0SPaolo Bonzini /*
6115710a3e0SPaolo Bonzini * There cannot be two concurrent aio_poll calls for the same AioContext (or
6125710a3e0SPaolo Bonzini * an aio_poll concurrent with a GSource prepare/check/dispatch callback).
6135710a3e0SPaolo Bonzini * We rely on this below to avoid slow locked accesses to ctx->notify_me.
6149ce44e2cSKevin Wolf *
6159ce44e2cSKevin Wolf * aio_poll() may only be called in the AioContext's thread. iohandler_ctx
6169ce44e2cSKevin Wolf * is special in that it runs in the main thread, but that thread's context
6179ce44e2cSKevin Wolf * is qemu_aio_context.
6185710a3e0SPaolo Bonzini */
6199ce44e2cSKevin Wolf assert(in_aio_context_home_thread(ctx == iohandler_get_aio_context() ?
6209ce44e2cSKevin Wolf qemu_get_aio_context() : ctx));
6210dc165c1SKevin Wolf
622c2b38b27SPaolo Bonzini qemu_lockcnt_inc(&ctx->list_lock);
623c2b38b27SPaolo Bonzini
624c2b38b27SPaolo Bonzini if (ctx->poll_max_ns) {
625c2b38b27SPaolo Bonzini start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
626c2b38b27SPaolo Bonzini }
627c2b38b27SPaolo Bonzini
628e30cffa0SPaolo Bonzini timeout = blocking ? aio_compute_timeout(ctx) : 0;
629826cc324SStefan Hajnoczi progress = try_poll_mode(ctx, &ready_list, &timeout);
630e30cffa0SPaolo Bonzini assert(!(timeout && progress));
631e30cffa0SPaolo Bonzini
63244277bf9SStefan Hajnoczi /*
63344277bf9SStefan Hajnoczi * aio_notify can avoid the expensive event_notifier_set if
63444277bf9SStefan Hajnoczi * everything (file descriptors, bottom halves, timers) will
63544277bf9SStefan Hajnoczi * be re-evaluated before the next blocking poll(). This is
63644277bf9SStefan Hajnoczi * already true when aio_poll is called with blocking == false;
63744277bf9SStefan Hajnoczi * if blocking == true, it is only true after poll() returns,
63844277bf9SStefan Hajnoczi * so disable the optimization now.
63944277bf9SStefan Hajnoczi */
64044277bf9SStefan Hajnoczi use_notify_me = timeout != 0;
64144277bf9SStefan Hajnoczi if (use_notify_me) {
642d73415a3SStefan Hajnoczi qatomic_set(&ctx->notify_me, qatomic_read(&ctx->notify_me) + 2);
64344277bf9SStefan Hajnoczi /*
64444277bf9SStefan Hajnoczi * Write ctx->notify_me before reading ctx->notified. Pairs with
64544277bf9SStefan Hajnoczi * smp_mb in aio_notify().
64644277bf9SStefan Hajnoczi */
64744277bf9SStefan Hajnoczi smp_mb();
64844277bf9SStefan Hajnoczi
64944277bf9SStefan Hajnoczi /* Don't block if aio_notify() was called */
650d73415a3SStefan Hajnoczi if (qatomic_read(&ctx->notified)) {
65144277bf9SStefan Hajnoczi timeout = 0;
65244277bf9SStefan Hajnoczi }
65344277bf9SStefan Hajnoczi }
65444277bf9SStefan Hajnoczi
655e30cffa0SPaolo Bonzini /* If polling is allowed, non-blocking aio_poll does not need the
656e30cffa0SPaolo Bonzini * system call---a single round of run_poll_handlers_once suffices.
657e30cffa0SPaolo Bonzini */
658aa38e19fSStefan Hajnoczi if (timeout || ctx->fdmon_ops->need_wait(ctx)) {
659816a430cSChao Gao /*
660816a430cSChao Gao * Disable poll mode. poll mode should be disabled before the call
661816a430cSChao Gao * of ctx->fdmon_ops->wait() so that guest's notification can wake
662816a430cSChao Gao * up IO threads when some work becomes pending. It is essential to
663816a430cSChao Gao * avoid hangs or unnecessary latency.
664816a430cSChao Gao */
665816a430cSChao Gao if (poll_set_started(ctx, &ready_list, false)) {
666816a430cSChao Gao timeout = 0;
667816a430cSChao Gao progress = true;
668816a430cSChao Gao }
669816a430cSChao Gao
670826cc324SStefan Hajnoczi ctx->fdmon_ops->wait(ctx, &ready_list, timeout);
671c2b38b27SPaolo Bonzini }
672c2b38b27SPaolo Bonzini
67344277bf9SStefan Hajnoczi if (use_notify_me) {
6745710a3e0SPaolo Bonzini /* Finish the poll before clearing the flag. */
675d73415a3SStefan Hajnoczi qatomic_store_release(&ctx->notify_me,
676d73415a3SStefan Hajnoczi qatomic_read(&ctx->notify_me) - 2);
677c2b38b27SPaolo Bonzini }
678c2b38b27SPaolo Bonzini
67944277bf9SStefan Hajnoczi aio_notify_accept(ctx);
68044277bf9SStefan Hajnoczi
681c2b38b27SPaolo Bonzini /* Adjust polling time */
682c2b38b27SPaolo Bonzini if (ctx->poll_max_ns) {
683c2b38b27SPaolo Bonzini int64_t block_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start;
684c2b38b27SPaolo Bonzini
685c2b38b27SPaolo Bonzini if (block_ns <= ctx->poll_ns) {
686c2b38b27SPaolo Bonzini /* This is the sweet spot, no adjustment needed */
687c2b38b27SPaolo Bonzini } else if (block_ns > ctx->poll_max_ns) {
688c2b38b27SPaolo Bonzini /* We'd have to poll for too long, poll less */
689c2b38b27SPaolo Bonzini int64_t old = ctx->poll_ns;
690c2b38b27SPaolo Bonzini
691c2b38b27SPaolo Bonzini if (ctx->poll_shrink) {
692c2b38b27SPaolo Bonzini ctx->poll_ns /= ctx->poll_shrink;
693c2b38b27SPaolo Bonzini } else {
694c2b38b27SPaolo Bonzini ctx->poll_ns = 0;
695c2b38b27SPaolo Bonzini }
696c2b38b27SPaolo Bonzini
697c2b38b27SPaolo Bonzini trace_poll_shrink(ctx, old, ctx->poll_ns);
698c2b38b27SPaolo Bonzini } else if (ctx->poll_ns < ctx->poll_max_ns &&
699c2b38b27SPaolo Bonzini block_ns < ctx->poll_max_ns) {
700c2b38b27SPaolo Bonzini /* There is room to grow, poll longer */
701c2b38b27SPaolo Bonzini int64_t old = ctx->poll_ns;
702c2b38b27SPaolo Bonzini int64_t grow = ctx->poll_grow;
703c2b38b27SPaolo Bonzini
704c2b38b27SPaolo Bonzini if (grow == 0) {
705c2b38b27SPaolo Bonzini grow = 2;
706c2b38b27SPaolo Bonzini }
707c2b38b27SPaolo Bonzini
708c2b38b27SPaolo Bonzini if (ctx->poll_ns) {
709c2b38b27SPaolo Bonzini ctx->poll_ns *= grow;
710c2b38b27SPaolo Bonzini } else {
711c2b38b27SPaolo Bonzini ctx->poll_ns = 4000; /* start polling at 4 microseconds */
712c2b38b27SPaolo Bonzini }
713c2b38b27SPaolo Bonzini
714c2b38b27SPaolo Bonzini if (ctx->poll_ns > ctx->poll_max_ns) {
715c2b38b27SPaolo Bonzini ctx->poll_ns = ctx->poll_max_ns;
716c2b38b27SPaolo Bonzini }
717c2b38b27SPaolo Bonzini
718c2b38b27SPaolo Bonzini trace_poll_grow(ctx, old, ctx->poll_ns);
719c2b38b27SPaolo Bonzini }
720c2b38b27SPaolo Bonzini }
721c2b38b27SPaolo Bonzini
722a153bf52SPaolo Bonzini progress |= aio_bh_poll(ctx);
7237391d34cSStefan Hajnoczi progress |= aio_dispatch_ready_handlers(ctx, &ready_list);
724c2b38b27SPaolo Bonzini
7254749079cSStefan Hajnoczi aio_free_deleted_handlers(ctx);
7264749079cSStefan Hajnoczi
727bd451435SPaolo Bonzini qemu_lockcnt_dec(&ctx->list_lock);
728bd451435SPaolo Bonzini
729a153bf52SPaolo Bonzini progress |= timerlistgroup_run_timers(&ctx->tlg);
730a153bf52SPaolo Bonzini
731c2b38b27SPaolo Bonzini return progress;
732c2b38b27SPaolo Bonzini }
733c2b38b27SPaolo Bonzini
aio_context_setup(AioContext * ctx)734c2b38b27SPaolo Bonzini void aio_context_setup(AioContext *ctx)
735c2b38b27SPaolo Bonzini {
7361f050a46SStefan Hajnoczi ctx->fdmon_ops = &fdmon_poll_ops;
7371f050a46SStefan Hajnoczi ctx->epollfd = -1;
7381f050a46SStefan Hajnoczi
73973fd282eSStefan Hajnoczi /* Use the fastest fd monitoring implementation if available */
74073fd282eSStefan Hajnoczi if (fdmon_io_uring_setup(ctx)) {
74173fd282eSStefan Hajnoczi return;
74273fd282eSStefan Hajnoczi }
74373fd282eSStefan Hajnoczi
7441f050a46SStefan Hajnoczi fdmon_epoll_setup(ctx);
745c2b38b27SPaolo Bonzini }
746c2b38b27SPaolo Bonzini
aio_context_destroy(AioContext * ctx)747cd0a6d2bSJie Wang void aio_context_destroy(AioContext *ctx)
748cd0a6d2bSJie Wang {
74973fd282eSStefan Hajnoczi fdmon_io_uring_destroy(ctx);
7501f050a46SStefan Hajnoczi fdmon_epoll_disable(ctx);
751de137e44SStefan Hajnoczi aio_free_deleted_handlers(ctx);
752cd0a6d2bSJie Wang }
753cd0a6d2bSJie Wang
aio_context_use_g_source(AioContext * ctx)754ba607ca8SStefan Hajnoczi void aio_context_use_g_source(AioContext *ctx)
755ba607ca8SStefan Hajnoczi {
756ba607ca8SStefan Hajnoczi /*
757ba607ca8SStefan Hajnoczi * Disable io_uring when the glib main loop is used because it doesn't
758ba607ca8SStefan Hajnoczi * support mixed glib/aio_poll() usage. It relies on aio_poll() being
759ba607ca8SStefan Hajnoczi * called regularly so that changes to the monitored file descriptors are
760ba607ca8SStefan Hajnoczi * submitted, otherwise a list of pending fd handlers builds up.
761ba607ca8SStefan Hajnoczi */
762ba607ca8SStefan Hajnoczi fdmon_io_uring_destroy(ctx);
763ba607ca8SStefan Hajnoczi aio_free_deleted_handlers(ctx);
764ba607ca8SStefan Hajnoczi }
765ba607ca8SStefan Hajnoczi
aio_context_set_poll_params(AioContext * ctx,int64_t max_ns,int64_t grow,int64_t shrink,Error ** errp)766c2b38b27SPaolo Bonzini void aio_context_set_poll_params(AioContext *ctx, int64_t max_ns,
767c2b38b27SPaolo Bonzini int64_t grow, int64_t shrink, Error **errp)
768c2b38b27SPaolo Bonzini {
769c2b38b27SPaolo Bonzini /* No thread synchronization here, it doesn't matter if an incorrect value
770c2b38b27SPaolo Bonzini * is used once.
771c2b38b27SPaolo Bonzini */
772c2b38b27SPaolo Bonzini ctx->poll_max_ns = max_ns;
773c2b38b27SPaolo Bonzini ctx->poll_ns = 0;
774c2b38b27SPaolo Bonzini ctx->poll_grow = grow;
775c2b38b27SPaolo Bonzini ctx->poll_shrink = shrink;
776c2b38b27SPaolo Bonzini
777c2b38b27SPaolo Bonzini aio_notify(ctx);
778c2b38b27SPaolo Bonzini }
7791793ad02SStefano Garzarella
aio_context_set_aio_params(AioContext * ctx,int64_t max_batch,Error ** errp)7801793ad02SStefano Garzarella void aio_context_set_aio_params(AioContext *ctx, int64_t max_batch,
7811793ad02SStefano Garzarella Error **errp)
7821793ad02SStefano Garzarella {
7831793ad02SStefano Garzarella /*
7841793ad02SStefano Garzarella * No thread synchronization here, it doesn't matter if an incorrect value
7851793ad02SStefano Garzarella * is used once.
7861793ad02SStefano Garzarella */
7871793ad02SStefano Garzarella ctx->aio_max_batch = max_batch;
7881793ad02SStefano Garzarella
7891793ad02SStefano Garzarella aio_notify(ctx);
7901793ad02SStefano Garzarella }
791