1 /*
2 * Migration support for VFIO devices
3 *
4 * Copyright NVIDIA, Inc. 2020
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2. See
7 * the COPYING file in the top-level directory.
8 */
9
10 #include "qemu/osdep.h"
11 #include "qemu/main-loop.h"
12 #include "qemu/cutils.h"
13 #include "qemu/units.h"
14 #include "qemu/error-report.h"
15 #include <linux/vfio.h>
16 #include <sys/ioctl.h>
17
18 #include "system/runstate.h"
19 #include "hw/vfio/vfio-device.h"
20 #include "hw/vfio/vfio-migration.h"
21 #include "migration/misc.h"
22 #include "migration/savevm.h"
23 #include "migration/vmstate.h"
24 #include "migration/qemu-file.h"
25 #include "migration/register.h"
26 #include "migration/blocker.h"
27 #include "migration-multifd.h"
28 #include "qapi/error.h"
29 #include "qapi/qapi-events-vfio.h"
30 #include "exec/ramlist.h"
31 #include "pci.h"
32 #include "trace.h"
33 #include "hw/hw.h"
34 #include "vfio-migration-internal.h"
35
36 /*
37 * This is an arbitrary size based on migration of mlx5 devices, where typically
38 * total device migration size is on the order of 100s of MB. Testing with
39 * larger values, e.g. 128MB and 1GB, did not show a performance improvement.
40 */
41 #define VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE (1 * MiB)
42
43 static unsigned long bytes_transferred;
44
mig_state_to_str(enum vfio_device_mig_state state)45 static const char *mig_state_to_str(enum vfio_device_mig_state state)
46 {
47 switch (state) {
48 case VFIO_DEVICE_STATE_ERROR:
49 return "ERROR";
50 case VFIO_DEVICE_STATE_STOP:
51 return "STOP";
52 case VFIO_DEVICE_STATE_RUNNING:
53 return "RUNNING";
54 case VFIO_DEVICE_STATE_STOP_COPY:
55 return "STOP_COPY";
56 case VFIO_DEVICE_STATE_RESUMING:
57 return "RESUMING";
58 case VFIO_DEVICE_STATE_RUNNING_P2P:
59 return "RUNNING_P2P";
60 case VFIO_DEVICE_STATE_PRE_COPY:
61 return "PRE_COPY";
62 case VFIO_DEVICE_STATE_PRE_COPY_P2P:
63 return "PRE_COPY_P2P";
64 default:
65 return "UNKNOWN STATE";
66 }
67 }
68
69 static QapiVfioMigrationState
mig_state_to_qapi_state(enum vfio_device_mig_state state)70 mig_state_to_qapi_state(enum vfio_device_mig_state state)
71 {
72 switch (state) {
73 case VFIO_DEVICE_STATE_STOP:
74 return QAPI_VFIO_MIGRATION_STATE_STOP;
75 case VFIO_DEVICE_STATE_RUNNING:
76 return QAPI_VFIO_MIGRATION_STATE_RUNNING;
77 case VFIO_DEVICE_STATE_STOP_COPY:
78 return QAPI_VFIO_MIGRATION_STATE_STOP_COPY;
79 case VFIO_DEVICE_STATE_RESUMING:
80 return QAPI_VFIO_MIGRATION_STATE_RESUMING;
81 case VFIO_DEVICE_STATE_RUNNING_P2P:
82 return QAPI_VFIO_MIGRATION_STATE_RUNNING_P2P;
83 case VFIO_DEVICE_STATE_PRE_COPY:
84 return QAPI_VFIO_MIGRATION_STATE_PRE_COPY;
85 case VFIO_DEVICE_STATE_PRE_COPY_P2P:
86 return QAPI_VFIO_MIGRATION_STATE_PRE_COPY_P2P;
87 default:
88 g_assert_not_reached();
89 }
90 }
91
vfio_migration_send_event(VFIODevice * vbasedev)92 static void vfio_migration_send_event(VFIODevice *vbasedev)
93 {
94 VFIOMigration *migration = vbasedev->migration;
95 DeviceState *dev = vbasedev->dev;
96 g_autofree char *qom_path = NULL;
97 Object *obj;
98
99 if (!vbasedev->migration_events) {
100 return;
101 }
102
103 g_assert(vbasedev->ops->vfio_get_object);
104 obj = vbasedev->ops->vfio_get_object(vbasedev);
105 g_assert(obj);
106 qom_path = object_get_canonical_path(obj);
107
108 qapi_event_send_vfio_migration(
109 dev->id, qom_path, mig_state_to_qapi_state(migration->device_state));
110 }
111
vfio_migration_set_device_state(VFIODevice * vbasedev,enum vfio_device_mig_state state)112 static void vfio_migration_set_device_state(VFIODevice *vbasedev,
113 enum vfio_device_mig_state state)
114 {
115 VFIOMigration *migration = vbasedev->migration;
116
117 trace_vfio_migration_set_device_state(vbasedev->name,
118 mig_state_to_str(state));
119
120 migration->device_state = state;
121 vfio_migration_send_event(vbasedev);
122 }
123
vfio_migration_set_state(VFIODevice * vbasedev,enum vfio_device_mig_state new_state,enum vfio_device_mig_state recover_state,Error ** errp)124 int vfio_migration_set_state(VFIODevice *vbasedev,
125 enum vfio_device_mig_state new_state,
126 enum vfio_device_mig_state recover_state,
127 Error **errp)
128 {
129 VFIOMigration *migration = vbasedev->migration;
130 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
131 sizeof(struct vfio_device_feature_mig_state),
132 sizeof(uint64_t))] = {};
133 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
134 struct vfio_device_feature_mig_state *mig_state =
135 (struct vfio_device_feature_mig_state *)feature->data;
136 int ret;
137 g_autofree char *error_prefix =
138 g_strdup_printf("%s: Failed setting device state to %s.",
139 vbasedev->name, mig_state_to_str(new_state));
140
141 trace_vfio_migration_set_state(vbasedev->name, mig_state_to_str(new_state),
142 mig_state_to_str(recover_state));
143
144 if (new_state == migration->device_state) {
145 return 0;
146 }
147
148 feature->argsz = sizeof(buf);
149 feature->flags =
150 VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE;
151 mig_state->device_state = new_state;
152 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
153 /* Try to set the device in some good state */
154 ret = -errno;
155
156 if (recover_state == VFIO_DEVICE_STATE_ERROR) {
157 error_setg_errno(errp, errno,
158 "%s Recover state is ERROR. Resetting device",
159 error_prefix);
160
161 goto reset_device;
162 }
163
164 error_setg_errno(errp, errno,
165 "%s Setting device in recover state %s",
166 error_prefix, mig_state_to_str(recover_state));
167
168 mig_state->device_state = recover_state;
169 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
170 ret = -errno;
171 /*
172 * If setting the device in recover state fails, report
173 * the error here and propagate the first error.
174 */
175 error_report(
176 "%s: Failed setting device in recover state, err: %s. Resetting device",
177 vbasedev->name, strerror(errno));
178
179 goto reset_device;
180 }
181
182 vfio_migration_set_device_state(vbasedev, recover_state);
183
184 return ret;
185 }
186
187 vfio_migration_set_device_state(vbasedev, new_state);
188 if (mig_state->data_fd != -1) {
189 if (migration->data_fd != -1) {
190 /*
191 * This can happen if the device is asynchronously reset and
192 * terminates a data transfer.
193 */
194 error_setg(errp, "%s: data_fd out of sync", vbasedev->name);
195 close(mig_state->data_fd);
196
197 return -EBADF;
198 }
199
200 migration->data_fd = mig_state->data_fd;
201 }
202
203 return 0;
204
205 reset_device:
206 if (ioctl(vbasedev->fd, VFIO_DEVICE_RESET)) {
207 hw_error("%s: Failed resetting device, err: %s", vbasedev->name,
208 strerror(errno));
209 }
210
211 vfio_migration_set_device_state(vbasedev, VFIO_DEVICE_STATE_RUNNING);
212
213 return ret;
214 }
215
216 /*
217 * Some device state transitions require resetting the device if they fail.
218 * This function sets the device in new_state and resets the device if that
219 * fails. Reset is done by using ERROR as the recover state.
220 */
221 static int
vfio_migration_set_state_or_reset(VFIODevice * vbasedev,enum vfio_device_mig_state new_state,Error ** errp)222 vfio_migration_set_state_or_reset(VFIODevice *vbasedev,
223 enum vfio_device_mig_state new_state,
224 Error **errp)
225 {
226 return vfio_migration_set_state(vbasedev, new_state,
227 VFIO_DEVICE_STATE_ERROR, errp);
228 }
229
vfio_load_buffer(QEMUFile * f,VFIODevice * vbasedev,uint64_t data_size)230 static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev,
231 uint64_t data_size)
232 {
233 VFIOMigration *migration = vbasedev->migration;
234 int ret;
235
236 ret = qemu_file_get_to_fd(f, migration->data_fd, data_size);
237 trace_vfio_load_state_device_data(vbasedev->name, data_size, ret);
238
239 return ret;
240 }
241
vfio_save_device_config_state(QEMUFile * f,void * opaque,Error ** errp)242 int vfio_save_device_config_state(QEMUFile *f, void *opaque, Error **errp)
243 {
244 VFIODevice *vbasedev = opaque;
245 int ret;
246
247 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_STATE);
248
249 if (vbasedev->ops && vbasedev->ops->vfio_save_config) {
250 ret = vbasedev->ops->vfio_save_config(vbasedev, f, errp);
251 if (ret) {
252 return ret;
253 }
254 }
255
256 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
257
258 trace_vfio_save_device_config_state(vbasedev->name);
259
260 ret = qemu_file_get_error(f);
261 if (ret < 0) {
262 error_setg_errno(errp, -ret, "Failed to save state");
263 }
264 return ret;
265 }
266
vfio_load_device_config_state(QEMUFile * f,void * opaque)267 int vfio_load_device_config_state(QEMUFile *f, void *opaque)
268 {
269 VFIODevice *vbasedev = opaque;
270 uint64_t data;
271
272 trace_vfio_load_device_config_state_start(vbasedev->name);
273
274 if (vbasedev->ops && vbasedev->ops->vfio_load_config) {
275 int ret;
276
277 ret = vbasedev->ops->vfio_load_config(vbasedev, f);
278 if (ret) {
279 error_report("%s: Failed to load device config space",
280 vbasedev->name);
281 return ret;
282 }
283 }
284
285 data = qemu_get_be64(f);
286 if (data != VFIO_MIG_FLAG_END_OF_STATE) {
287 error_report("%s: Failed loading device config space, "
288 "end flag incorrect 0x%"PRIx64, vbasedev->name, data);
289 return -EINVAL;
290 }
291
292 trace_vfio_load_device_config_state_end(vbasedev->name);
293 return qemu_file_get_error(f);
294 }
295
vfio_migration_cleanup(VFIODevice * vbasedev)296 static void vfio_migration_cleanup(VFIODevice *vbasedev)
297 {
298 VFIOMigration *migration = vbasedev->migration;
299
300 close(migration->data_fd);
301 migration->data_fd = -1;
302 }
303
vfio_query_stop_copy_size(VFIODevice * vbasedev,uint64_t * stop_copy_size)304 static int vfio_query_stop_copy_size(VFIODevice *vbasedev,
305 uint64_t *stop_copy_size)
306 {
307 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
308 sizeof(struct vfio_device_feature_mig_data_size),
309 sizeof(uint64_t))] = {};
310 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
311 struct vfio_device_feature_mig_data_size *mig_data_size =
312 (struct vfio_device_feature_mig_data_size *)feature->data;
313
314 feature->argsz = sizeof(buf);
315 feature->flags =
316 VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIG_DATA_SIZE;
317
318 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
319 return -errno;
320 }
321
322 *stop_copy_size = mig_data_size->stop_copy_length;
323
324 return 0;
325 }
326
vfio_query_precopy_size(VFIOMigration * migration)327 static int vfio_query_precopy_size(VFIOMigration *migration)
328 {
329 struct vfio_precopy_info precopy = {
330 .argsz = sizeof(precopy),
331 };
332
333 migration->precopy_init_size = 0;
334 migration->precopy_dirty_size = 0;
335
336 if (ioctl(migration->data_fd, VFIO_MIG_GET_PRECOPY_INFO, &precopy)) {
337 return -errno;
338 }
339
340 migration->precopy_init_size = precopy.initial_bytes;
341 migration->precopy_dirty_size = precopy.dirty_bytes;
342
343 return 0;
344 }
345
346 /* Returns the size of saved data on success and -errno on error */
vfio_save_block(QEMUFile * f,VFIOMigration * migration)347 static ssize_t vfio_save_block(QEMUFile *f, VFIOMigration *migration)
348 {
349 ssize_t data_size;
350
351 data_size = read(migration->data_fd, migration->data_buffer,
352 migration->data_buffer_size);
353 if (data_size < 0) {
354 /*
355 * Pre-copy emptied all the device state for now. For more information,
356 * please refer to the Linux kernel VFIO uAPI.
357 */
358 if (errno == ENOMSG) {
359 if (!migration->event_precopy_empty_hit) {
360 trace_vfio_save_block_precopy_empty_hit(migration->vbasedev->name);
361 migration->event_precopy_empty_hit = true;
362 }
363 return 0;
364 }
365
366 return -errno;
367 }
368 if (data_size == 0) {
369 return 0;
370 }
371
372 /* Non-empty read: re-arm the trace event */
373 migration->event_precopy_empty_hit = false;
374
375 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE);
376 qemu_put_be64(f, data_size);
377 qemu_put_buffer(f, migration->data_buffer, data_size);
378 vfio_migration_add_bytes_transferred(data_size);
379
380 trace_vfio_save_block(migration->vbasedev->name, data_size);
381
382 return qemu_file_get_error(f) ?: data_size;
383 }
384
vfio_update_estimated_pending_data(VFIOMigration * migration,uint64_t data_size)385 static void vfio_update_estimated_pending_data(VFIOMigration *migration,
386 uint64_t data_size)
387 {
388 if (!data_size) {
389 /*
390 * Pre-copy emptied all the device state for now, update estimated sizes
391 * accordingly.
392 */
393 migration->precopy_init_size = 0;
394 migration->precopy_dirty_size = 0;
395
396 return;
397 }
398
399 if (migration->precopy_init_size) {
400 uint64_t init_size = MIN(migration->precopy_init_size, data_size);
401
402 migration->precopy_init_size -= init_size;
403 data_size -= init_size;
404 }
405
406 migration->precopy_dirty_size -= MIN(migration->precopy_dirty_size,
407 data_size);
408 }
409
vfio_precopy_supported(VFIODevice * vbasedev)410 static bool vfio_precopy_supported(VFIODevice *vbasedev)
411 {
412 VFIOMigration *migration = vbasedev->migration;
413
414 return migration->mig_flags & VFIO_MIGRATION_PRE_COPY;
415 }
416
417 /* ---------------------------------------------------------------------- */
418
vfio_save_prepare(void * opaque,Error ** errp)419 static int vfio_save_prepare(void *opaque, Error **errp)
420 {
421 VFIODevice *vbasedev = opaque;
422
423 /*
424 * Snapshot doesn't use postcopy nor background snapshot, so allow snapshot
425 * even if they are on.
426 */
427 if (runstate_check(RUN_STATE_SAVE_VM)) {
428 return 0;
429 }
430
431 if (migrate_postcopy_ram()) {
432 error_setg(
433 errp, "%s: VFIO migration is not supported with postcopy migration",
434 vbasedev->name);
435 return -EOPNOTSUPP;
436 }
437
438 if (migrate_background_snapshot()) {
439 error_setg(
440 errp,
441 "%s: VFIO migration is not supported with background snapshot",
442 vbasedev->name);
443 return -EOPNOTSUPP;
444 }
445
446 return 0;
447 }
448
vfio_save_setup(QEMUFile * f,void * opaque,Error ** errp)449 static int vfio_save_setup(QEMUFile *f, void *opaque, Error **errp)
450 {
451 VFIODevice *vbasedev = opaque;
452 VFIOMigration *migration = vbasedev->migration;
453 uint64_t stop_copy_size = VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE;
454 int ret;
455
456 if (!vfio_multifd_setup(vbasedev, false, errp)) {
457 return -EINVAL;
458 }
459
460 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_SETUP_STATE);
461
462 vfio_query_stop_copy_size(vbasedev, &stop_copy_size);
463 migration->data_buffer_size = MIN(VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE,
464 stop_copy_size);
465 migration->data_buffer = g_try_malloc0(migration->data_buffer_size);
466 if (!migration->data_buffer) {
467 error_setg(errp, "%s: Failed to allocate migration data buffer",
468 vbasedev->name);
469 return -ENOMEM;
470 }
471
472 migration->event_save_iterate_started = false;
473 migration->event_precopy_empty_hit = false;
474
475 if (vfio_precopy_supported(vbasedev)) {
476 switch (migration->device_state) {
477 case VFIO_DEVICE_STATE_RUNNING:
478 ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_PRE_COPY,
479 VFIO_DEVICE_STATE_RUNNING, errp);
480 if (ret) {
481 return ret;
482 }
483
484 vfio_query_precopy_size(migration);
485
486 break;
487 case VFIO_DEVICE_STATE_STOP:
488 /* vfio_save_complete_precopy() will go to STOP_COPY */
489 break;
490 default:
491 error_setg(errp, "%s: Invalid device state %d", vbasedev->name,
492 migration->device_state);
493 return -EINVAL;
494 }
495 }
496
497 trace_vfio_save_setup(vbasedev->name, migration->data_buffer_size);
498
499 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
500
501 ret = qemu_file_get_error(f);
502 if (ret < 0) {
503 error_setg_errno(errp, -ret, "%s: save setup failed", vbasedev->name);
504 }
505
506 return ret;
507 }
508
vfio_save_cleanup(void * opaque)509 static void vfio_save_cleanup(void *opaque)
510 {
511 VFIODevice *vbasedev = opaque;
512 VFIOMigration *migration = vbasedev->migration;
513 Error *local_err = NULL;
514 int ret;
515
516 /* Currently a NOP, done for symmetry with load_cleanup() */
517 vfio_multifd_cleanup(vbasedev);
518
519 /*
520 * Changing device state from STOP_COPY to STOP can take time. Do it here,
521 * after migration has completed, so it won't increase downtime.
522 */
523 if (migration->device_state == VFIO_DEVICE_STATE_STOP_COPY) {
524 ret = vfio_migration_set_state_or_reset(vbasedev,
525 VFIO_DEVICE_STATE_STOP,
526 &local_err);
527 if (ret) {
528 error_report_err(local_err);
529 }
530 }
531
532 g_free(migration->data_buffer);
533 migration->data_buffer = NULL;
534 migration->precopy_init_size = 0;
535 migration->precopy_dirty_size = 0;
536 migration->initial_data_sent = false;
537 vfio_migration_cleanup(vbasedev);
538 trace_vfio_save_cleanup(vbasedev->name);
539 }
540
vfio_state_pending_estimate(void * opaque,uint64_t * must_precopy,uint64_t * can_postcopy)541 static void vfio_state_pending_estimate(void *opaque, uint64_t *must_precopy,
542 uint64_t *can_postcopy)
543 {
544 VFIODevice *vbasedev = opaque;
545 VFIOMigration *migration = vbasedev->migration;
546
547 if (!vfio_device_state_is_precopy(vbasedev)) {
548 return;
549 }
550
551 *must_precopy +=
552 migration->precopy_init_size + migration->precopy_dirty_size;
553
554 trace_vfio_state_pending_estimate(vbasedev->name, *must_precopy,
555 *can_postcopy,
556 migration->precopy_init_size,
557 migration->precopy_dirty_size);
558 }
559
560 /*
561 * Migration size of VFIO devices can be as little as a few KBs or as big as
562 * many GBs. This value should be big enough to cover the worst case.
563 */
564 #define VFIO_MIG_STOP_COPY_SIZE (100 * GiB)
565
vfio_state_pending_exact(void * opaque,uint64_t * must_precopy,uint64_t * can_postcopy)566 static void vfio_state_pending_exact(void *opaque, uint64_t *must_precopy,
567 uint64_t *can_postcopy)
568 {
569 VFIODevice *vbasedev = opaque;
570 VFIOMigration *migration = vbasedev->migration;
571 uint64_t stop_copy_size = VFIO_MIG_STOP_COPY_SIZE;
572
573 /*
574 * If getting pending migration size fails, VFIO_MIG_STOP_COPY_SIZE is
575 * reported so downtime limit won't be violated.
576 */
577 vfio_query_stop_copy_size(vbasedev, &stop_copy_size);
578 *must_precopy += stop_copy_size;
579
580 if (vfio_device_state_is_precopy(vbasedev)) {
581 vfio_query_precopy_size(migration);
582 }
583
584 trace_vfio_state_pending_exact(vbasedev->name, *must_precopy, *can_postcopy,
585 stop_copy_size, migration->precopy_init_size,
586 migration->precopy_dirty_size);
587 }
588
vfio_is_active_iterate(void * opaque)589 static bool vfio_is_active_iterate(void *opaque)
590 {
591 VFIODevice *vbasedev = opaque;
592
593 return vfio_device_state_is_precopy(vbasedev);
594 }
595
596 /*
597 * Note about migration rate limiting: VFIO migration buffer size is currently
598 * limited to 1MB, so there is no need to check if migration rate exceeded (as
599 * in the worst case it will exceed by 1MB). However, if the buffer size is
600 * later changed to a bigger value, migration rate should be enforced here.
601 */
vfio_save_iterate(QEMUFile * f,void * opaque)602 static int vfio_save_iterate(QEMUFile *f, void *opaque)
603 {
604 VFIODevice *vbasedev = opaque;
605 VFIOMigration *migration = vbasedev->migration;
606 ssize_t data_size;
607
608 if (!migration->event_save_iterate_started) {
609 trace_vfio_save_iterate_start(vbasedev->name);
610 migration->event_save_iterate_started = true;
611 }
612
613 data_size = vfio_save_block(f, migration);
614 if (data_size < 0) {
615 return data_size;
616 }
617
618 vfio_update_estimated_pending_data(migration, data_size);
619
620 if (migrate_switchover_ack() && !migration->precopy_init_size &&
621 !migration->initial_data_sent) {
622 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_INIT_DATA_SENT);
623 migration->initial_data_sent = true;
624 } else {
625 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
626 }
627
628 trace_vfio_save_iterate(vbasedev->name, migration->precopy_init_size,
629 migration->precopy_dirty_size);
630
631 return !migration->precopy_init_size && !migration->precopy_dirty_size;
632 }
633
vfio_save_complete_precopy(QEMUFile * f,void * opaque)634 static int vfio_save_complete_precopy(QEMUFile *f, void *opaque)
635 {
636 VFIODevice *vbasedev = opaque;
637 ssize_t data_size;
638 int ret;
639 Error *local_err = NULL;
640
641 if (vfio_multifd_transfer_enabled(vbasedev)) {
642 vfio_multifd_emit_dummy_eos(vbasedev, f);
643 return 0;
644 }
645
646 trace_vfio_save_complete_precopy_start(vbasedev->name);
647
648 /* We reach here with device state STOP or STOP_COPY only */
649 ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_STOP_COPY,
650 VFIO_DEVICE_STATE_STOP, &local_err);
651 if (ret) {
652 error_report_err(local_err);
653 return ret;
654 }
655
656 do {
657 data_size = vfio_save_block(f, vbasedev->migration);
658 if (data_size < 0) {
659 return data_size;
660 }
661 } while (data_size);
662
663 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
664 ret = qemu_file_get_error(f);
665
666 trace_vfio_save_complete_precopy(vbasedev->name, ret);
667
668 return ret;
669 }
670
vfio_save_state(QEMUFile * f,void * opaque)671 static void vfio_save_state(QEMUFile *f, void *opaque)
672 {
673 VFIODevice *vbasedev = opaque;
674 Error *local_err = NULL;
675 int ret;
676
677 if (vfio_multifd_transfer_enabled(vbasedev)) {
678 if (vfio_load_config_after_iter(vbasedev)) {
679 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_LOAD_READY);
680 } else {
681 vfio_multifd_emit_dummy_eos(vbasedev, f);
682 }
683 return;
684 }
685
686 ret = vfio_save_device_config_state(f, opaque, &local_err);
687 if (ret) {
688 error_prepend(&local_err,
689 "vfio: Failed to save device config space of %s - ",
690 vbasedev->name);
691 qemu_file_set_error_obj(f, ret, local_err);
692 }
693 }
694
vfio_load_setup(QEMUFile * f,void * opaque,Error ** errp)695 static int vfio_load_setup(QEMUFile *f, void *opaque, Error **errp)
696 {
697 VFIODevice *vbasedev = opaque;
698 VFIOMigration *migration = vbasedev->migration;
699 int ret;
700
701 if (!vfio_multifd_setup(vbasedev, true, errp)) {
702 return -EINVAL;
703 }
704
705 ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_RESUMING,
706 migration->device_state, errp);
707 if (ret) {
708 return ret;
709 }
710
711 return 0;
712 }
713
vfio_load_cleanup(void * opaque)714 static int vfio_load_cleanup(void *opaque)
715 {
716 VFIODevice *vbasedev = opaque;
717
718 vfio_multifd_cleanup(vbasedev);
719
720 vfio_migration_cleanup(vbasedev);
721 trace_vfio_load_cleanup(vbasedev->name);
722
723 return 0;
724 }
725
vfio_load_state(QEMUFile * f,void * opaque,int version_id)726 static int vfio_load_state(QEMUFile *f, void *opaque, int version_id)
727 {
728 VFIODevice *vbasedev = opaque;
729 int ret = 0;
730 uint64_t data;
731
732 data = qemu_get_be64(f);
733 while (data != VFIO_MIG_FLAG_END_OF_STATE) {
734
735 trace_vfio_load_state(vbasedev->name, data);
736
737 switch (data) {
738 case VFIO_MIG_FLAG_DEV_CONFIG_STATE:
739 {
740 if (vfio_multifd_transfer_enabled(vbasedev)) {
741 error_report("%s: got DEV_CONFIG_STATE in main migration "
742 "channel but doing multifd transfer",
743 vbasedev->name);
744 return -EINVAL;
745 }
746
747 return vfio_load_device_config_state(f, opaque);
748 }
749 case VFIO_MIG_FLAG_DEV_SETUP_STATE:
750 {
751 data = qemu_get_be64(f);
752 if (data == VFIO_MIG_FLAG_END_OF_STATE) {
753 return ret;
754 } else {
755 error_report("%s: SETUP STATE: EOS not found 0x%"PRIx64,
756 vbasedev->name, data);
757 return -EINVAL;
758 }
759 break;
760 }
761 case VFIO_MIG_FLAG_DEV_DATA_STATE:
762 {
763 uint64_t data_size = qemu_get_be64(f);
764
765 if (data_size) {
766 ret = vfio_load_buffer(f, vbasedev, data_size);
767 if (ret < 0) {
768 return ret;
769 }
770 }
771 break;
772 }
773 case VFIO_MIG_FLAG_DEV_INIT_DATA_SENT:
774 {
775 if (!vfio_precopy_supported(vbasedev) ||
776 !migrate_switchover_ack()) {
777 error_report("%s: Received INIT_DATA_SENT but switchover ack "
778 "is not used", vbasedev->name);
779 return -EINVAL;
780 }
781
782 ret = qemu_loadvm_approve_switchover();
783 if (ret) {
784 error_report(
785 "%s: qemu_loadvm_approve_switchover failed, err=%d (%s)",
786 vbasedev->name, ret, strerror(-ret));
787 }
788
789 return ret;
790 }
791 case VFIO_MIG_FLAG_DEV_CONFIG_LOAD_READY:
792 {
793 return vfio_load_state_config_load_ready(vbasedev);
794 }
795 default:
796 error_report("%s: Unknown tag 0x%"PRIx64, vbasedev->name, data);
797 return -EINVAL;
798 }
799
800 data = qemu_get_be64(f);
801 ret = qemu_file_get_error(f);
802 if (ret) {
803 return ret;
804 }
805 }
806 return ret;
807 }
808
vfio_switchover_ack_needed(void * opaque)809 static bool vfio_switchover_ack_needed(void *opaque)
810 {
811 VFIODevice *vbasedev = opaque;
812
813 return vfio_precopy_supported(vbasedev);
814 }
815
vfio_switchover_start(void * opaque)816 static int vfio_switchover_start(void *opaque)
817 {
818 VFIODevice *vbasedev = opaque;
819
820 if (vfio_multifd_transfer_enabled(vbasedev)) {
821 return vfio_multifd_switchover_start(vbasedev);
822 }
823
824 return 0;
825 }
826
827 static const SaveVMHandlers savevm_vfio_handlers = {
828 .save_prepare = vfio_save_prepare,
829 .save_setup = vfio_save_setup,
830 .save_cleanup = vfio_save_cleanup,
831 .state_pending_estimate = vfio_state_pending_estimate,
832 .state_pending_exact = vfio_state_pending_exact,
833 .is_active_iterate = vfio_is_active_iterate,
834 .save_live_iterate = vfio_save_iterate,
835 .save_complete = vfio_save_complete_precopy,
836 .save_state = vfio_save_state,
837 .load_setup = vfio_load_setup,
838 .load_cleanup = vfio_load_cleanup,
839 .load_state = vfio_load_state,
840 .switchover_ack_needed = vfio_switchover_ack_needed,
841 /*
842 * Multifd support
843 */
844 .load_state_buffer = vfio_multifd_load_state_buffer,
845 .switchover_start = vfio_switchover_start,
846 .save_complete_precopy_thread = vfio_multifd_save_complete_precopy_thread,
847 };
848
849 /* ---------------------------------------------------------------------- */
850
vfio_vmstate_change_prepare(void * opaque,bool running,RunState state)851 static void vfio_vmstate_change_prepare(void *opaque, bool running,
852 RunState state)
853 {
854 VFIODevice *vbasedev = opaque;
855 VFIOMigration *migration = vbasedev->migration;
856 enum vfio_device_mig_state new_state;
857 Error *local_err = NULL;
858 int ret;
859
860 new_state = migration->device_state == VFIO_DEVICE_STATE_PRE_COPY ?
861 VFIO_DEVICE_STATE_PRE_COPY_P2P :
862 VFIO_DEVICE_STATE_RUNNING_P2P;
863
864 ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err);
865 if (ret) {
866 /*
867 * Migration should be aborted in this case, but vm_state_notify()
868 * currently does not support reporting failures.
869 */
870 migration_file_set_error(ret, local_err);
871 }
872
873 trace_vfio_vmstate_change_prepare(vbasedev->name, running,
874 RunState_str(state),
875 mig_state_to_str(new_state));
876 }
877
vfio_vmstate_change(void * opaque,bool running,RunState state)878 static void vfio_vmstate_change(void *opaque, bool running, RunState state)
879 {
880 VFIODevice *vbasedev = opaque;
881 enum vfio_device_mig_state new_state;
882 Error *local_err = NULL;
883 int ret;
884
885 if (running) {
886 new_state = VFIO_DEVICE_STATE_RUNNING;
887 } else {
888 new_state =
889 (vfio_device_state_is_precopy(vbasedev) &&
890 (state == RUN_STATE_FINISH_MIGRATE || state == RUN_STATE_PAUSED)) ?
891 VFIO_DEVICE_STATE_STOP_COPY :
892 VFIO_DEVICE_STATE_STOP;
893 }
894
895 ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err);
896 if (ret) {
897 /*
898 * Migration should be aborted in this case, but vm_state_notify()
899 * currently does not support reporting failures.
900 */
901 migration_file_set_error(ret, local_err);
902 }
903
904 trace_vfio_vmstate_change(vbasedev->name, running, RunState_str(state),
905 mig_state_to_str(new_state));
906 }
907
vfio_migration_state_notifier(NotifierWithReturn * notifier,MigrationEvent * e,Error ** errp)908 static int vfio_migration_state_notifier(NotifierWithReturn *notifier,
909 MigrationEvent *e, Error **errp)
910 {
911 VFIOMigration *migration = container_of(notifier, VFIOMigration,
912 migration_state);
913 VFIODevice *vbasedev = migration->vbasedev;
914 Error *local_err = NULL;
915 int ret;
916
917 trace_vfio_migration_state_notifier(vbasedev->name, e->type);
918
919 if (e->type == MIG_EVENT_PRECOPY_FAILED) {
920 /*
921 * MigrationNotifyFunc may not return an error code and an Error
922 * object for MIG_EVENT_PRECOPY_FAILED. Hence, report the error
923 * locally and ignore the errp argument.
924 */
925 ret = vfio_migration_set_state_or_reset(vbasedev,
926 VFIO_DEVICE_STATE_RUNNING,
927 &local_err);
928 if (ret) {
929 error_report_err(local_err);
930 }
931 }
932 return 0;
933 }
934
vfio_migration_free(VFIODevice * vbasedev)935 static void vfio_migration_free(VFIODevice *vbasedev)
936 {
937 g_free(vbasedev->migration);
938 vbasedev->migration = NULL;
939 }
940
vfio_migration_query_flags(VFIODevice * vbasedev,uint64_t * mig_flags)941 static int vfio_migration_query_flags(VFIODevice *vbasedev, uint64_t *mig_flags)
942 {
943 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
944 sizeof(struct vfio_device_feature_migration),
945 sizeof(uint64_t))] = {};
946 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
947 struct vfio_device_feature_migration *mig =
948 (struct vfio_device_feature_migration *)feature->data;
949
950 feature->argsz = sizeof(buf);
951 feature->flags = VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIGRATION;
952 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
953 return -errno;
954 }
955
956 *mig_flags = mig->flags;
957
958 return 0;
959 }
960
vfio_dma_logging_supported(VFIODevice * vbasedev)961 static bool vfio_dma_logging_supported(VFIODevice *vbasedev)
962 {
963 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature),
964 sizeof(uint64_t))] = {};
965 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
966
967 feature->argsz = sizeof(buf);
968 feature->flags = VFIO_DEVICE_FEATURE_PROBE |
969 VFIO_DEVICE_FEATURE_DMA_LOGGING_START;
970
971 return !ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature);
972 }
973
vfio_migration_init(VFIODevice * vbasedev)974 static int vfio_migration_init(VFIODevice *vbasedev)
975 {
976 int ret;
977 Object *obj;
978 VFIOMigration *migration;
979 char id[256] = "";
980 g_autofree char *path = NULL, *oid = NULL;
981 uint64_t mig_flags = 0;
982 VMChangeStateHandler *prepare_cb;
983
984 if (!vbasedev->ops->vfio_get_object) {
985 return -EINVAL;
986 }
987
988 obj = vbasedev->ops->vfio_get_object(vbasedev);
989 if (!obj) {
990 return -EINVAL;
991 }
992
993 ret = vfio_migration_query_flags(vbasedev, &mig_flags);
994 if (ret) {
995 return ret;
996 }
997
998 /* Basic migration functionality must be supported */
999 if (!(mig_flags & VFIO_MIGRATION_STOP_COPY)) {
1000 return -EOPNOTSUPP;
1001 }
1002
1003 vbasedev->migration = g_new0(VFIOMigration, 1);
1004 migration = vbasedev->migration;
1005 migration->vbasedev = vbasedev;
1006 migration->device_state = VFIO_DEVICE_STATE_RUNNING;
1007 migration->data_fd = -1;
1008 migration->mig_flags = mig_flags;
1009
1010 vbasedev->dirty_pages_supported = vfio_dma_logging_supported(vbasedev);
1011
1012 oid = vmstate_if_get_id(VMSTATE_IF(DEVICE(obj)));
1013 if (oid) {
1014 path = g_strdup_printf("%s/vfio", oid);
1015 } else {
1016 path = g_strdup("vfio");
1017 }
1018 strpadcpy(id, sizeof(id), path, '\0');
1019
1020 register_savevm_live(id, VMSTATE_INSTANCE_ID_ANY, 1, &savevm_vfio_handlers,
1021 vbasedev);
1022
1023 prepare_cb = migration->mig_flags & VFIO_MIGRATION_P2P ?
1024 vfio_vmstate_change_prepare :
1025 NULL;
1026 migration->vm_state = qdev_add_vm_change_state_handler_full(
1027 vbasedev->dev, vfio_vmstate_change, prepare_cb, NULL, vbasedev);
1028 migration_add_notifier(&migration->migration_state,
1029 vfio_migration_state_notifier);
1030
1031 return 0;
1032 }
1033
1034 static Error *multiple_devices_migration_blocker;
1035
1036 /*
1037 * Multiple devices migration is allowed only if all devices support P2P
1038 * migration. Single device migration is allowed regardless of P2P migration
1039 * support.
1040 */
vfio_multiple_devices_migration_is_supported(void)1041 static bool vfio_multiple_devices_migration_is_supported(void)
1042 {
1043 VFIODevice *vbasedev;
1044 unsigned int device_num = 0;
1045 bool all_support_p2p = true;
1046
1047 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) {
1048 if (vbasedev->migration) {
1049 device_num++;
1050
1051 if (!(vbasedev->migration->mig_flags & VFIO_MIGRATION_P2P)) {
1052 all_support_p2p = false;
1053 }
1054 }
1055 }
1056
1057 return all_support_p2p || device_num <= 1;
1058 }
1059
vfio_block_multiple_devices_migration(VFIODevice * vbasedev,Error ** errp)1060 static int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp)
1061 {
1062 if (vfio_multiple_devices_migration_is_supported()) {
1063 return 0;
1064 }
1065
1066 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) {
1067 error_setg(errp, "Multiple VFIO devices migration is supported only if "
1068 "all of them support P2P migration");
1069 return -EINVAL;
1070 }
1071
1072 if (multiple_devices_migration_blocker) {
1073 return 0;
1074 }
1075
1076 error_setg(&multiple_devices_migration_blocker,
1077 "Multiple VFIO devices migration is supported only if all of "
1078 "them support P2P migration");
1079 return migrate_add_blocker_normal(&multiple_devices_migration_blocker,
1080 errp);
1081 }
1082
vfio_unblock_multiple_devices_migration(void)1083 static void vfio_unblock_multiple_devices_migration(void)
1084 {
1085 if (!multiple_devices_migration_blocker ||
1086 !vfio_multiple_devices_migration_is_supported()) {
1087 return;
1088 }
1089
1090 migrate_del_blocker(&multiple_devices_migration_blocker);
1091 }
1092
vfio_migration_deinit(VFIODevice * vbasedev)1093 static void vfio_migration_deinit(VFIODevice *vbasedev)
1094 {
1095 VFIOMigration *migration = vbasedev->migration;
1096
1097 migration_remove_notifier(&migration->migration_state);
1098 qemu_del_vm_change_state_handler(migration->vm_state);
1099 unregister_savevm(VMSTATE_IF(vbasedev->dev), "vfio", vbasedev);
1100 vfio_migration_free(vbasedev);
1101 vfio_unblock_multiple_devices_migration();
1102 }
1103
vfio_block_migration(VFIODevice * vbasedev,Error * err,Error ** errp)1104 static int vfio_block_migration(VFIODevice *vbasedev, Error *err, Error **errp)
1105 {
1106 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) {
1107 error_propagate(errp, err);
1108 return -EINVAL;
1109 }
1110
1111 vbasedev->migration_blocker = error_copy(err);
1112 error_free(err);
1113
1114 return migrate_add_blocker_normal(&vbasedev->migration_blocker, errp);
1115 }
1116
1117 /* ---------------------------------------------------------------------- */
1118
vfio_migration_bytes_transferred(void)1119 int64_t vfio_migration_bytes_transferred(void)
1120 {
1121 return MIN(qatomic_read(&bytes_transferred), INT64_MAX);
1122 }
1123
vfio_migration_reset_bytes_transferred(void)1124 void vfio_migration_reset_bytes_transferred(void)
1125 {
1126 qatomic_set(&bytes_transferred, 0);
1127 }
1128
vfio_migration_add_bytes_transferred(unsigned long val)1129 void vfio_migration_add_bytes_transferred(unsigned long val)
1130 {
1131 qatomic_add(&bytes_transferred, val);
1132 }
1133
vfio_migration_active(void)1134 bool vfio_migration_active(void)
1135 {
1136 VFIODevice *vbasedev;
1137
1138 if (QLIST_EMPTY(&vfio_device_list)) {
1139 return false;
1140 }
1141
1142 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) {
1143 if (vbasedev->migration_blocker) {
1144 return false;
1145 }
1146 }
1147 return true;
1148 }
1149
vfio_viommu_preset(VFIODevice * vbasedev)1150 static bool vfio_viommu_preset(VFIODevice *vbasedev)
1151 {
1152 return vbasedev->bcontainer->space->as != &address_space_memory;
1153 }
1154
1155 /*
1156 * Return true when either migration initialized or blocker registered.
1157 * Currently only return false when adding blocker fails which will
1158 * de-register vfio device.
1159 */
vfio_migration_realize(VFIODevice * vbasedev,Error ** errp)1160 bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
1161 {
1162 Error *err = NULL;
1163 int ret;
1164
1165 if (vbasedev->enable_migration == ON_OFF_AUTO_OFF) {
1166 error_setg(&err, "%s: Migration is disabled for VFIO device",
1167 vbasedev->name);
1168 return !vfio_block_migration(vbasedev, err, errp);
1169 }
1170
1171 ret = vfio_migration_init(vbasedev);
1172 if (ret) {
1173 if (ret == -ENOTTY) {
1174 error_setg(&err, "%s: VFIO migration is not supported in kernel",
1175 vbasedev->name);
1176 } else {
1177 error_setg(&err,
1178 "%s: Migration couldn't be initialized for VFIO device, "
1179 "err: %d (%s)",
1180 vbasedev->name, ret, strerror(-ret));
1181 }
1182
1183 return !vfio_block_migration(vbasedev, err, errp);
1184 }
1185
1186 if ((!vbasedev->dirty_pages_supported ||
1187 vbasedev->device_dirty_page_tracking == ON_OFF_AUTO_OFF) &&
1188 !vbasedev->iommu_dirty_tracking) {
1189 if (vbasedev->enable_migration == ON_OFF_AUTO_AUTO) {
1190 error_setg(&err,
1191 "%s: VFIO device doesn't support device and "
1192 "IOMMU dirty tracking", vbasedev->name);
1193 goto add_blocker;
1194 }
1195
1196 warn_report("%s: VFIO device doesn't support device and "
1197 "IOMMU dirty tracking", vbasedev->name);
1198 }
1199
1200 ret = vfio_block_multiple_devices_migration(vbasedev, errp);
1201 if (ret) {
1202 goto out_deinit;
1203 }
1204
1205 if (vfio_viommu_preset(vbasedev)) {
1206 error_setg(&err, "%s: Migration is currently not supported "
1207 "with vIOMMU enabled", vbasedev->name);
1208 goto add_blocker;
1209 }
1210
1211 trace_vfio_migration_realize(vbasedev->name);
1212 return true;
1213
1214 add_blocker:
1215 ret = vfio_block_migration(vbasedev, err, errp);
1216 out_deinit:
1217 if (ret) {
1218 vfio_migration_deinit(vbasedev);
1219 }
1220 return !ret;
1221 }
1222
vfio_migration_exit(VFIODevice * vbasedev)1223 void vfio_migration_exit(VFIODevice *vbasedev)
1224 {
1225 if (vbasedev->migration) {
1226 vfio_migration_deinit(vbasedev);
1227 }
1228
1229 migrate_del_blocker(&vbasedev->migration_blocker);
1230 }
1231
vfio_device_state_is_running(VFIODevice * vbasedev)1232 bool vfio_device_state_is_running(VFIODevice *vbasedev)
1233 {
1234 VFIOMigration *migration = vbasedev->migration;
1235
1236 return migration->device_state == VFIO_DEVICE_STATE_RUNNING ||
1237 migration->device_state == VFIO_DEVICE_STATE_RUNNING_P2P;
1238 }
1239
vfio_device_state_is_precopy(VFIODevice * vbasedev)1240 bool vfio_device_state_is_precopy(VFIODevice *vbasedev)
1241 {
1242 VFIOMigration *migration = vbasedev->migration;
1243
1244 return migration->device_state == VFIO_DEVICE_STATE_PRE_COPY ||
1245 migration->device_state == VFIO_DEVICE_STATE_PRE_COPY_P2P;
1246 }
1247