1 /*
2 * Migration support for VFIO devices
3 *
4 * Copyright NVIDIA, Inc. 2020
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2. See
7 * the COPYING file in the top-level directory.
8 */
9
10 #include "qemu/osdep.h"
11 #include "qemu/main-loop.h"
12 #include "qemu/cutils.h"
13 #include "qemu/units.h"
14 #include "qemu/error-report.h"
15 #include <linux/vfio.h>
16 #include <sys/ioctl.h>
17
18 #include "sysemu/runstate.h"
19 #include "hw/vfio/vfio-common.h"
20 #include "migration/misc.h"
21 #include "migration/savevm.h"
22 #include "migration/vmstate.h"
23 #include "migration/qemu-file.h"
24 #include "migration/register.h"
25 #include "migration/blocker.h"
26 #include "qapi/error.h"
27 #include "qapi/qapi-events-vfio.h"
28 #include "exec/ramlist.h"
29 #include "exec/ram_addr.h"
30 #include "pci.h"
31 #include "trace.h"
32 #include "hw/hw.h"
33
34 /*
35 * Flags to be used as unique delimiters for VFIO devices in the migration
36 * stream. These flags are composed as:
37 * 0xffffffff => MSB 32-bit all 1s
38 * 0xef10 => Magic ID, represents emulated (virtual) function IO
39 * 0x0000 => 16-bits reserved for flags
40 *
41 * The beginning of state information is marked by _DEV_CONFIG_STATE,
42 * _DEV_SETUP_STATE, or _DEV_DATA_STATE, respectively. The end of a
43 * certain state information is marked by _END_OF_STATE.
44 */
45 #define VFIO_MIG_FLAG_END_OF_STATE (0xffffffffef100001ULL)
46 #define VFIO_MIG_FLAG_DEV_CONFIG_STATE (0xffffffffef100002ULL)
47 #define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL)
48 #define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL)
49 #define VFIO_MIG_FLAG_DEV_INIT_DATA_SENT (0xffffffffef100005ULL)
50
51 /*
52 * This is an arbitrary size based on migration of mlx5 devices, where typically
53 * total device migration size is on the order of 100s of MB. Testing with
54 * larger values, e.g. 128MB and 1GB, did not show a performance improvement.
55 */
56 #define VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE (1 * MiB)
57
58 static int64_t bytes_transferred;
59
mig_state_to_str(enum vfio_device_mig_state state)60 static const char *mig_state_to_str(enum vfio_device_mig_state state)
61 {
62 switch (state) {
63 case VFIO_DEVICE_STATE_ERROR:
64 return "ERROR";
65 case VFIO_DEVICE_STATE_STOP:
66 return "STOP";
67 case VFIO_DEVICE_STATE_RUNNING:
68 return "RUNNING";
69 case VFIO_DEVICE_STATE_STOP_COPY:
70 return "STOP_COPY";
71 case VFIO_DEVICE_STATE_RESUMING:
72 return "RESUMING";
73 case VFIO_DEVICE_STATE_RUNNING_P2P:
74 return "RUNNING_P2P";
75 case VFIO_DEVICE_STATE_PRE_COPY:
76 return "PRE_COPY";
77 case VFIO_DEVICE_STATE_PRE_COPY_P2P:
78 return "PRE_COPY_P2P";
79 default:
80 return "UNKNOWN STATE";
81 }
82 }
83
84 static QapiVfioMigrationState
mig_state_to_qapi_state(enum vfio_device_mig_state state)85 mig_state_to_qapi_state(enum vfio_device_mig_state state)
86 {
87 switch (state) {
88 case VFIO_DEVICE_STATE_STOP:
89 return QAPI_VFIO_MIGRATION_STATE_STOP;
90 case VFIO_DEVICE_STATE_RUNNING:
91 return QAPI_VFIO_MIGRATION_STATE_RUNNING;
92 case VFIO_DEVICE_STATE_STOP_COPY:
93 return QAPI_VFIO_MIGRATION_STATE_STOP_COPY;
94 case VFIO_DEVICE_STATE_RESUMING:
95 return QAPI_VFIO_MIGRATION_STATE_RESUMING;
96 case VFIO_DEVICE_STATE_RUNNING_P2P:
97 return QAPI_VFIO_MIGRATION_STATE_RUNNING_P2P;
98 case VFIO_DEVICE_STATE_PRE_COPY:
99 return QAPI_VFIO_MIGRATION_STATE_PRE_COPY;
100 case VFIO_DEVICE_STATE_PRE_COPY_P2P:
101 return QAPI_VFIO_MIGRATION_STATE_PRE_COPY_P2P;
102 default:
103 g_assert_not_reached();
104 }
105 }
106
vfio_migration_send_event(VFIODevice * vbasedev)107 static void vfio_migration_send_event(VFIODevice *vbasedev)
108 {
109 VFIOMigration *migration = vbasedev->migration;
110 DeviceState *dev = vbasedev->dev;
111 g_autofree char *qom_path = NULL;
112 Object *obj;
113
114 if (!vbasedev->migration_events) {
115 return;
116 }
117
118 g_assert(vbasedev->ops->vfio_get_object);
119 obj = vbasedev->ops->vfio_get_object(vbasedev);
120 g_assert(obj);
121 qom_path = object_get_canonical_path(obj);
122
123 qapi_event_send_vfio_migration(
124 dev->id, qom_path, mig_state_to_qapi_state(migration->device_state));
125 }
126
vfio_migration_set_device_state(VFIODevice * vbasedev,enum vfio_device_mig_state state)127 static void vfio_migration_set_device_state(VFIODevice *vbasedev,
128 enum vfio_device_mig_state state)
129 {
130 VFIOMigration *migration = vbasedev->migration;
131
132 trace_vfio_migration_set_device_state(vbasedev->name,
133 mig_state_to_str(state));
134
135 migration->device_state = state;
136 vfio_migration_send_event(vbasedev);
137 }
138
vfio_migration_set_state(VFIODevice * vbasedev,enum vfio_device_mig_state new_state,enum vfio_device_mig_state recover_state,Error ** errp)139 static int vfio_migration_set_state(VFIODevice *vbasedev,
140 enum vfio_device_mig_state new_state,
141 enum vfio_device_mig_state recover_state,
142 Error **errp)
143 {
144 VFIOMigration *migration = vbasedev->migration;
145 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
146 sizeof(struct vfio_device_feature_mig_state),
147 sizeof(uint64_t))] = {};
148 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
149 struct vfio_device_feature_mig_state *mig_state =
150 (struct vfio_device_feature_mig_state *)feature->data;
151 int ret;
152 g_autofree char *error_prefix =
153 g_strdup_printf("%s: Failed setting device state to %s.",
154 vbasedev->name, mig_state_to_str(new_state));
155
156 trace_vfio_migration_set_state(vbasedev->name, mig_state_to_str(new_state),
157 mig_state_to_str(recover_state));
158
159 if (new_state == migration->device_state) {
160 return 0;
161 }
162
163 feature->argsz = sizeof(buf);
164 feature->flags =
165 VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE;
166 mig_state->device_state = new_state;
167 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
168 /* Try to set the device in some good state */
169 ret = -errno;
170
171 if (recover_state == VFIO_DEVICE_STATE_ERROR) {
172 error_setg_errno(errp, errno,
173 "%s Recover state is ERROR. Resetting device",
174 error_prefix);
175
176 goto reset_device;
177 }
178
179 error_setg_errno(errp, errno,
180 "%s Setting device in recover state %s",
181 error_prefix, mig_state_to_str(recover_state));
182
183 mig_state->device_state = recover_state;
184 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
185 ret = -errno;
186 /*
187 * If setting the device in recover state fails, report
188 * the error here and propagate the first error.
189 */
190 error_report(
191 "%s: Failed setting device in recover state, err: %s. Resetting device",
192 vbasedev->name, strerror(errno));
193
194 goto reset_device;
195 }
196
197 vfio_migration_set_device_state(vbasedev, recover_state);
198
199 return ret;
200 }
201
202 vfio_migration_set_device_state(vbasedev, new_state);
203 if (mig_state->data_fd != -1) {
204 if (migration->data_fd != -1) {
205 /*
206 * This can happen if the device is asynchronously reset and
207 * terminates a data transfer.
208 */
209 error_setg(errp, "%s: data_fd out of sync", vbasedev->name);
210 close(mig_state->data_fd);
211
212 return -EBADF;
213 }
214
215 migration->data_fd = mig_state->data_fd;
216 }
217
218 return 0;
219
220 reset_device:
221 if (ioctl(vbasedev->fd, VFIO_DEVICE_RESET)) {
222 hw_error("%s: Failed resetting device, err: %s", vbasedev->name,
223 strerror(errno));
224 }
225
226 vfio_migration_set_device_state(vbasedev, VFIO_DEVICE_STATE_RUNNING);
227
228 return ret;
229 }
230
231 /*
232 * Some device state transitions require resetting the device if they fail.
233 * This function sets the device in new_state and resets the device if that
234 * fails. Reset is done by using ERROR as the recover state.
235 */
236 static int
vfio_migration_set_state_or_reset(VFIODevice * vbasedev,enum vfio_device_mig_state new_state,Error ** errp)237 vfio_migration_set_state_or_reset(VFIODevice *vbasedev,
238 enum vfio_device_mig_state new_state,
239 Error **errp)
240 {
241 return vfio_migration_set_state(vbasedev, new_state,
242 VFIO_DEVICE_STATE_ERROR, errp);
243 }
244
vfio_load_buffer(QEMUFile * f,VFIODevice * vbasedev,uint64_t data_size)245 static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev,
246 uint64_t data_size)
247 {
248 VFIOMigration *migration = vbasedev->migration;
249 int ret;
250
251 ret = qemu_file_get_to_fd(f, migration->data_fd, data_size);
252 trace_vfio_load_state_device_data(vbasedev->name, data_size, ret);
253
254 return ret;
255 }
256
vfio_save_device_config_state(QEMUFile * f,void * opaque,Error ** errp)257 static int vfio_save_device_config_state(QEMUFile *f, void *opaque,
258 Error **errp)
259 {
260 VFIODevice *vbasedev = opaque;
261 int ret;
262
263 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_STATE);
264
265 if (vbasedev->ops && vbasedev->ops->vfio_save_config) {
266 ret = vbasedev->ops->vfio_save_config(vbasedev, f, errp);
267 if (ret) {
268 return ret;
269 }
270 }
271
272 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
273
274 trace_vfio_save_device_config_state(vbasedev->name);
275
276 ret = qemu_file_get_error(f);
277 if (ret < 0) {
278 error_setg_errno(errp, -ret, "Failed to save state");
279 }
280 return ret;
281 }
282
vfio_load_device_config_state(QEMUFile * f,void * opaque)283 static int vfio_load_device_config_state(QEMUFile *f, void *opaque)
284 {
285 VFIODevice *vbasedev = opaque;
286 uint64_t data;
287
288 if (vbasedev->ops && vbasedev->ops->vfio_load_config) {
289 int ret;
290
291 ret = vbasedev->ops->vfio_load_config(vbasedev, f);
292 if (ret) {
293 error_report("%s: Failed to load device config space",
294 vbasedev->name);
295 return ret;
296 }
297 }
298
299 data = qemu_get_be64(f);
300 if (data != VFIO_MIG_FLAG_END_OF_STATE) {
301 error_report("%s: Failed loading device config space, "
302 "end flag incorrect 0x%"PRIx64, vbasedev->name, data);
303 return -EINVAL;
304 }
305
306 trace_vfio_load_device_config_state(vbasedev->name);
307 return qemu_file_get_error(f);
308 }
309
vfio_migration_cleanup(VFIODevice * vbasedev)310 static void vfio_migration_cleanup(VFIODevice *vbasedev)
311 {
312 VFIOMigration *migration = vbasedev->migration;
313
314 close(migration->data_fd);
315 migration->data_fd = -1;
316 }
317
vfio_query_stop_copy_size(VFIODevice * vbasedev,uint64_t * stop_copy_size)318 static int vfio_query_stop_copy_size(VFIODevice *vbasedev,
319 uint64_t *stop_copy_size)
320 {
321 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
322 sizeof(struct vfio_device_feature_mig_data_size),
323 sizeof(uint64_t))] = {};
324 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
325 struct vfio_device_feature_mig_data_size *mig_data_size =
326 (struct vfio_device_feature_mig_data_size *)feature->data;
327
328 feature->argsz = sizeof(buf);
329 feature->flags =
330 VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIG_DATA_SIZE;
331
332 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
333 return -errno;
334 }
335
336 *stop_copy_size = mig_data_size->stop_copy_length;
337
338 return 0;
339 }
340
vfio_query_precopy_size(VFIOMigration * migration)341 static int vfio_query_precopy_size(VFIOMigration *migration)
342 {
343 struct vfio_precopy_info precopy = {
344 .argsz = sizeof(precopy),
345 };
346
347 migration->precopy_init_size = 0;
348 migration->precopy_dirty_size = 0;
349
350 if (ioctl(migration->data_fd, VFIO_MIG_GET_PRECOPY_INFO, &precopy)) {
351 return -errno;
352 }
353
354 migration->precopy_init_size = precopy.initial_bytes;
355 migration->precopy_dirty_size = precopy.dirty_bytes;
356
357 return 0;
358 }
359
360 /* Returns the size of saved data on success and -errno on error */
vfio_save_block(QEMUFile * f,VFIOMigration * migration)361 static ssize_t vfio_save_block(QEMUFile *f, VFIOMigration *migration)
362 {
363 ssize_t data_size;
364
365 data_size = read(migration->data_fd, migration->data_buffer,
366 migration->data_buffer_size);
367 if (data_size < 0) {
368 /*
369 * Pre-copy emptied all the device state for now. For more information,
370 * please refer to the Linux kernel VFIO uAPI.
371 */
372 if (errno == ENOMSG) {
373 if (!migration->event_precopy_empty_hit) {
374 trace_vfio_save_block_precopy_empty_hit(migration->vbasedev->name);
375 migration->event_precopy_empty_hit = true;
376 }
377 return 0;
378 }
379
380 return -errno;
381 }
382 if (data_size == 0) {
383 return 0;
384 }
385
386 /* Non-empty read: re-arm the trace event */
387 migration->event_precopy_empty_hit = false;
388
389 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE);
390 qemu_put_be64(f, data_size);
391 qemu_put_buffer(f, migration->data_buffer, data_size);
392 bytes_transferred += data_size;
393
394 trace_vfio_save_block(migration->vbasedev->name, data_size);
395
396 return qemu_file_get_error(f) ?: data_size;
397 }
398
vfio_update_estimated_pending_data(VFIOMigration * migration,uint64_t data_size)399 static void vfio_update_estimated_pending_data(VFIOMigration *migration,
400 uint64_t data_size)
401 {
402 if (!data_size) {
403 /*
404 * Pre-copy emptied all the device state for now, update estimated sizes
405 * accordingly.
406 */
407 migration->precopy_init_size = 0;
408 migration->precopy_dirty_size = 0;
409
410 return;
411 }
412
413 if (migration->precopy_init_size) {
414 uint64_t init_size = MIN(migration->precopy_init_size, data_size);
415
416 migration->precopy_init_size -= init_size;
417 data_size -= init_size;
418 }
419
420 migration->precopy_dirty_size -= MIN(migration->precopy_dirty_size,
421 data_size);
422 }
423
vfio_precopy_supported(VFIODevice * vbasedev)424 static bool vfio_precopy_supported(VFIODevice *vbasedev)
425 {
426 VFIOMigration *migration = vbasedev->migration;
427
428 return migration->mig_flags & VFIO_MIGRATION_PRE_COPY;
429 }
430
431 /* ---------------------------------------------------------------------- */
432
vfio_save_prepare(void * opaque,Error ** errp)433 static int vfio_save_prepare(void *opaque, Error **errp)
434 {
435 VFIODevice *vbasedev = opaque;
436
437 /*
438 * Snapshot doesn't use postcopy nor background snapshot, so allow snapshot
439 * even if they are on.
440 */
441 if (runstate_check(RUN_STATE_SAVE_VM)) {
442 return 0;
443 }
444
445 if (migrate_postcopy_ram()) {
446 error_setg(
447 errp, "%s: VFIO migration is not supported with postcopy migration",
448 vbasedev->name);
449 return -EOPNOTSUPP;
450 }
451
452 if (migrate_background_snapshot()) {
453 error_setg(
454 errp,
455 "%s: VFIO migration is not supported with background snapshot",
456 vbasedev->name);
457 return -EOPNOTSUPP;
458 }
459
460 return 0;
461 }
462
vfio_save_setup(QEMUFile * f,void * opaque,Error ** errp)463 static int vfio_save_setup(QEMUFile *f, void *opaque, Error **errp)
464 {
465 VFIODevice *vbasedev = opaque;
466 VFIOMigration *migration = vbasedev->migration;
467 uint64_t stop_copy_size = VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE;
468 int ret;
469
470 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_SETUP_STATE);
471
472 vfio_query_stop_copy_size(vbasedev, &stop_copy_size);
473 migration->data_buffer_size = MIN(VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE,
474 stop_copy_size);
475 migration->data_buffer = g_try_malloc0(migration->data_buffer_size);
476 if (!migration->data_buffer) {
477 error_setg(errp, "%s: Failed to allocate migration data buffer",
478 vbasedev->name);
479 return -ENOMEM;
480 }
481
482 migration->event_save_iterate_started = false;
483 migration->event_precopy_empty_hit = false;
484
485 if (vfio_precopy_supported(vbasedev)) {
486 switch (migration->device_state) {
487 case VFIO_DEVICE_STATE_RUNNING:
488 ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_PRE_COPY,
489 VFIO_DEVICE_STATE_RUNNING, errp);
490 if (ret) {
491 return ret;
492 }
493
494 vfio_query_precopy_size(migration);
495
496 break;
497 case VFIO_DEVICE_STATE_STOP:
498 /* vfio_save_complete_precopy() will go to STOP_COPY */
499 break;
500 default:
501 error_setg(errp, "%s: Invalid device state %d", vbasedev->name,
502 migration->device_state);
503 return -EINVAL;
504 }
505 }
506
507 trace_vfio_save_setup(vbasedev->name, migration->data_buffer_size);
508
509 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
510
511 ret = qemu_file_get_error(f);
512 if (ret < 0) {
513 error_setg_errno(errp, -ret, "%s: save setup failed", vbasedev->name);
514 }
515
516 return ret;
517 }
518
vfio_save_cleanup(void * opaque)519 static void vfio_save_cleanup(void *opaque)
520 {
521 VFIODevice *vbasedev = opaque;
522 VFIOMigration *migration = vbasedev->migration;
523 Error *local_err = NULL;
524 int ret;
525
526 /*
527 * Changing device state from STOP_COPY to STOP can take time. Do it here,
528 * after migration has completed, so it won't increase downtime.
529 */
530 if (migration->device_state == VFIO_DEVICE_STATE_STOP_COPY) {
531 ret = vfio_migration_set_state_or_reset(vbasedev,
532 VFIO_DEVICE_STATE_STOP,
533 &local_err);
534 if (ret) {
535 error_report_err(local_err);
536 }
537 }
538
539 g_free(migration->data_buffer);
540 migration->data_buffer = NULL;
541 migration->precopy_init_size = 0;
542 migration->precopy_dirty_size = 0;
543 migration->initial_data_sent = false;
544 vfio_migration_cleanup(vbasedev);
545 trace_vfio_save_cleanup(vbasedev->name);
546 }
547
vfio_state_pending_estimate(void * opaque,uint64_t * must_precopy,uint64_t * can_postcopy)548 static void vfio_state_pending_estimate(void *opaque, uint64_t *must_precopy,
549 uint64_t *can_postcopy)
550 {
551 VFIODevice *vbasedev = opaque;
552 VFIOMigration *migration = vbasedev->migration;
553
554 if (!vfio_device_state_is_precopy(vbasedev)) {
555 return;
556 }
557
558 *must_precopy +=
559 migration->precopy_init_size + migration->precopy_dirty_size;
560
561 trace_vfio_state_pending_estimate(vbasedev->name, *must_precopy,
562 *can_postcopy,
563 migration->precopy_init_size,
564 migration->precopy_dirty_size);
565 }
566
567 /*
568 * Migration size of VFIO devices can be as little as a few KBs or as big as
569 * many GBs. This value should be big enough to cover the worst case.
570 */
571 #define VFIO_MIG_STOP_COPY_SIZE (100 * GiB)
572
vfio_state_pending_exact(void * opaque,uint64_t * must_precopy,uint64_t * can_postcopy)573 static void vfio_state_pending_exact(void *opaque, uint64_t *must_precopy,
574 uint64_t *can_postcopy)
575 {
576 VFIODevice *vbasedev = opaque;
577 VFIOMigration *migration = vbasedev->migration;
578 uint64_t stop_copy_size = VFIO_MIG_STOP_COPY_SIZE;
579
580 /*
581 * If getting pending migration size fails, VFIO_MIG_STOP_COPY_SIZE is
582 * reported so downtime limit won't be violated.
583 */
584 vfio_query_stop_copy_size(vbasedev, &stop_copy_size);
585 *must_precopy += stop_copy_size;
586
587 if (vfio_device_state_is_precopy(vbasedev)) {
588 vfio_query_precopy_size(migration);
589 }
590
591 trace_vfio_state_pending_exact(vbasedev->name, *must_precopy, *can_postcopy,
592 stop_copy_size, migration->precopy_init_size,
593 migration->precopy_dirty_size);
594 }
595
vfio_is_active_iterate(void * opaque)596 static bool vfio_is_active_iterate(void *opaque)
597 {
598 VFIODevice *vbasedev = opaque;
599
600 return vfio_device_state_is_precopy(vbasedev);
601 }
602
603 /*
604 * Note about migration rate limiting: VFIO migration buffer size is currently
605 * limited to 1MB, so there is no need to check if migration rate exceeded (as
606 * in the worst case it will exceed by 1MB). However, if the buffer size is
607 * later changed to a bigger value, migration rate should be enforced here.
608 */
vfio_save_iterate(QEMUFile * f,void * opaque)609 static int vfio_save_iterate(QEMUFile *f, void *opaque)
610 {
611 VFIODevice *vbasedev = opaque;
612 VFIOMigration *migration = vbasedev->migration;
613 ssize_t data_size;
614
615 if (!migration->event_save_iterate_started) {
616 trace_vfio_save_iterate_start(vbasedev->name);
617 migration->event_save_iterate_started = true;
618 }
619
620 data_size = vfio_save_block(f, migration);
621 if (data_size < 0) {
622 return data_size;
623 }
624
625 vfio_update_estimated_pending_data(migration, data_size);
626
627 if (migrate_switchover_ack() && !migration->precopy_init_size &&
628 !migration->initial_data_sent) {
629 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_INIT_DATA_SENT);
630 migration->initial_data_sent = true;
631 } else {
632 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
633 }
634
635 trace_vfio_save_iterate(vbasedev->name, migration->precopy_init_size,
636 migration->precopy_dirty_size);
637
638 return !migration->precopy_init_size && !migration->precopy_dirty_size;
639 }
640
vfio_save_complete_precopy(QEMUFile * f,void * opaque)641 static int vfio_save_complete_precopy(QEMUFile *f, void *opaque)
642 {
643 VFIODevice *vbasedev = opaque;
644 ssize_t data_size;
645 int ret;
646 Error *local_err = NULL;
647
648 trace_vfio_save_complete_precopy_start(vbasedev->name);
649
650 /* We reach here with device state STOP or STOP_COPY only */
651 ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_STOP_COPY,
652 VFIO_DEVICE_STATE_STOP, &local_err);
653 if (ret) {
654 error_report_err(local_err);
655 return ret;
656 }
657
658 do {
659 data_size = vfio_save_block(f, vbasedev->migration);
660 if (data_size < 0) {
661 return data_size;
662 }
663 } while (data_size);
664
665 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
666 ret = qemu_file_get_error(f);
667
668 trace_vfio_save_complete_precopy(vbasedev->name, ret);
669
670 return ret;
671 }
672
vfio_save_state(QEMUFile * f,void * opaque)673 static void vfio_save_state(QEMUFile *f, void *opaque)
674 {
675 VFIODevice *vbasedev = opaque;
676 Error *local_err = NULL;
677 int ret;
678
679 ret = vfio_save_device_config_state(f, opaque, &local_err);
680 if (ret) {
681 error_prepend(&local_err,
682 "vfio: Failed to save device config space of %s - ",
683 vbasedev->name);
684 qemu_file_set_error_obj(f, ret, local_err);
685 }
686 }
687
vfio_load_setup(QEMUFile * f,void * opaque,Error ** errp)688 static int vfio_load_setup(QEMUFile *f, void *opaque, Error **errp)
689 {
690 VFIODevice *vbasedev = opaque;
691
692 return vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_RESUMING,
693 vbasedev->migration->device_state, errp);
694 }
695
vfio_load_cleanup(void * opaque)696 static int vfio_load_cleanup(void *opaque)
697 {
698 VFIODevice *vbasedev = opaque;
699
700 vfio_migration_cleanup(vbasedev);
701 trace_vfio_load_cleanup(vbasedev->name);
702
703 return 0;
704 }
705
vfio_load_state(QEMUFile * f,void * opaque,int version_id)706 static int vfio_load_state(QEMUFile *f, void *opaque, int version_id)
707 {
708 VFIODevice *vbasedev = opaque;
709 int ret = 0;
710 uint64_t data;
711
712 data = qemu_get_be64(f);
713 while (data != VFIO_MIG_FLAG_END_OF_STATE) {
714
715 trace_vfio_load_state(vbasedev->name, data);
716
717 switch (data) {
718 case VFIO_MIG_FLAG_DEV_CONFIG_STATE:
719 {
720 return vfio_load_device_config_state(f, opaque);
721 }
722 case VFIO_MIG_FLAG_DEV_SETUP_STATE:
723 {
724 data = qemu_get_be64(f);
725 if (data == VFIO_MIG_FLAG_END_OF_STATE) {
726 return ret;
727 } else {
728 error_report("%s: SETUP STATE: EOS not found 0x%"PRIx64,
729 vbasedev->name, data);
730 return -EINVAL;
731 }
732 break;
733 }
734 case VFIO_MIG_FLAG_DEV_DATA_STATE:
735 {
736 uint64_t data_size = qemu_get_be64(f);
737
738 if (data_size) {
739 ret = vfio_load_buffer(f, vbasedev, data_size);
740 if (ret < 0) {
741 return ret;
742 }
743 }
744 break;
745 }
746 case VFIO_MIG_FLAG_DEV_INIT_DATA_SENT:
747 {
748 if (!vfio_precopy_supported(vbasedev) ||
749 !migrate_switchover_ack()) {
750 error_report("%s: Received INIT_DATA_SENT but switchover ack "
751 "is not used", vbasedev->name);
752 return -EINVAL;
753 }
754
755 ret = qemu_loadvm_approve_switchover();
756 if (ret) {
757 error_report(
758 "%s: qemu_loadvm_approve_switchover failed, err=%d (%s)",
759 vbasedev->name, ret, strerror(-ret));
760 }
761
762 return ret;
763 }
764 default:
765 error_report("%s: Unknown tag 0x%"PRIx64, vbasedev->name, data);
766 return -EINVAL;
767 }
768
769 data = qemu_get_be64(f);
770 ret = qemu_file_get_error(f);
771 if (ret) {
772 return ret;
773 }
774 }
775 return ret;
776 }
777
vfio_switchover_ack_needed(void * opaque)778 static bool vfio_switchover_ack_needed(void *opaque)
779 {
780 VFIODevice *vbasedev = opaque;
781
782 return vfio_precopy_supported(vbasedev);
783 }
784
785 static const SaveVMHandlers savevm_vfio_handlers = {
786 .save_prepare = vfio_save_prepare,
787 .save_setup = vfio_save_setup,
788 .save_cleanup = vfio_save_cleanup,
789 .state_pending_estimate = vfio_state_pending_estimate,
790 .state_pending_exact = vfio_state_pending_exact,
791 .is_active_iterate = vfio_is_active_iterate,
792 .save_live_iterate = vfio_save_iterate,
793 .save_live_complete_precopy = vfio_save_complete_precopy,
794 .save_state = vfio_save_state,
795 .load_setup = vfio_load_setup,
796 .load_cleanup = vfio_load_cleanup,
797 .load_state = vfio_load_state,
798 .switchover_ack_needed = vfio_switchover_ack_needed,
799 };
800
801 /* ---------------------------------------------------------------------- */
802
vfio_vmstate_change_prepare(void * opaque,bool running,RunState state)803 static void vfio_vmstate_change_prepare(void *opaque, bool running,
804 RunState state)
805 {
806 VFIODevice *vbasedev = opaque;
807 VFIOMigration *migration = vbasedev->migration;
808 enum vfio_device_mig_state new_state;
809 Error *local_err = NULL;
810 int ret;
811
812 new_state = migration->device_state == VFIO_DEVICE_STATE_PRE_COPY ?
813 VFIO_DEVICE_STATE_PRE_COPY_P2P :
814 VFIO_DEVICE_STATE_RUNNING_P2P;
815
816 ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err);
817 if (ret) {
818 /*
819 * Migration should be aborted in this case, but vm_state_notify()
820 * currently does not support reporting failures.
821 */
822 migration_file_set_error(ret, local_err);
823 }
824
825 trace_vfio_vmstate_change_prepare(vbasedev->name, running,
826 RunState_str(state),
827 mig_state_to_str(new_state));
828 }
829
vfio_vmstate_change(void * opaque,bool running,RunState state)830 static void vfio_vmstate_change(void *opaque, bool running, RunState state)
831 {
832 VFIODevice *vbasedev = opaque;
833 enum vfio_device_mig_state new_state;
834 Error *local_err = NULL;
835 int ret;
836
837 if (running) {
838 new_state = VFIO_DEVICE_STATE_RUNNING;
839 } else {
840 new_state =
841 (vfio_device_state_is_precopy(vbasedev) &&
842 (state == RUN_STATE_FINISH_MIGRATE || state == RUN_STATE_PAUSED)) ?
843 VFIO_DEVICE_STATE_STOP_COPY :
844 VFIO_DEVICE_STATE_STOP;
845 }
846
847 ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err);
848 if (ret) {
849 /*
850 * Migration should be aborted in this case, but vm_state_notify()
851 * currently does not support reporting failures.
852 */
853 migration_file_set_error(ret, local_err);
854 }
855
856 trace_vfio_vmstate_change(vbasedev->name, running, RunState_str(state),
857 mig_state_to_str(new_state));
858 }
859
vfio_migration_state_notifier(NotifierWithReturn * notifier,MigrationEvent * e,Error ** errp)860 static int vfio_migration_state_notifier(NotifierWithReturn *notifier,
861 MigrationEvent *e, Error **errp)
862 {
863 VFIOMigration *migration = container_of(notifier, VFIOMigration,
864 migration_state);
865 VFIODevice *vbasedev = migration->vbasedev;
866 Error *local_err = NULL;
867 int ret;
868
869 trace_vfio_migration_state_notifier(vbasedev->name, e->type);
870
871 if (e->type == MIG_EVENT_PRECOPY_FAILED) {
872 /*
873 * MigrationNotifyFunc may not return an error code and an Error
874 * object for MIG_EVENT_PRECOPY_FAILED. Hence, report the error
875 * locally and ignore the errp argument.
876 */
877 ret = vfio_migration_set_state_or_reset(vbasedev,
878 VFIO_DEVICE_STATE_RUNNING,
879 &local_err);
880 if (ret) {
881 error_report_err(local_err);
882 }
883 }
884 return 0;
885 }
886
vfio_migration_free(VFIODevice * vbasedev)887 static void vfio_migration_free(VFIODevice *vbasedev)
888 {
889 g_free(vbasedev->migration);
890 vbasedev->migration = NULL;
891 }
892
vfio_migration_query_flags(VFIODevice * vbasedev,uint64_t * mig_flags)893 static int vfio_migration_query_flags(VFIODevice *vbasedev, uint64_t *mig_flags)
894 {
895 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
896 sizeof(struct vfio_device_feature_migration),
897 sizeof(uint64_t))] = {};
898 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
899 struct vfio_device_feature_migration *mig =
900 (struct vfio_device_feature_migration *)feature->data;
901
902 feature->argsz = sizeof(buf);
903 feature->flags = VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIGRATION;
904 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
905 return -errno;
906 }
907
908 *mig_flags = mig->flags;
909
910 return 0;
911 }
912
vfio_dma_logging_supported(VFIODevice * vbasedev)913 static bool vfio_dma_logging_supported(VFIODevice *vbasedev)
914 {
915 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature),
916 sizeof(uint64_t))] = {};
917 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
918
919 feature->argsz = sizeof(buf);
920 feature->flags = VFIO_DEVICE_FEATURE_PROBE |
921 VFIO_DEVICE_FEATURE_DMA_LOGGING_START;
922
923 return !ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature);
924 }
925
vfio_migration_init(VFIODevice * vbasedev)926 static int vfio_migration_init(VFIODevice *vbasedev)
927 {
928 int ret;
929 Object *obj;
930 VFIOMigration *migration;
931 char id[256] = "";
932 g_autofree char *path = NULL, *oid = NULL;
933 uint64_t mig_flags = 0;
934 VMChangeStateHandler *prepare_cb;
935
936 if (!vbasedev->ops->vfio_get_object) {
937 return -EINVAL;
938 }
939
940 obj = vbasedev->ops->vfio_get_object(vbasedev);
941 if (!obj) {
942 return -EINVAL;
943 }
944
945 ret = vfio_migration_query_flags(vbasedev, &mig_flags);
946 if (ret) {
947 return ret;
948 }
949
950 /* Basic migration functionality must be supported */
951 if (!(mig_flags & VFIO_MIGRATION_STOP_COPY)) {
952 return -EOPNOTSUPP;
953 }
954
955 vbasedev->migration = g_new0(VFIOMigration, 1);
956 migration = vbasedev->migration;
957 migration->vbasedev = vbasedev;
958 migration->device_state = VFIO_DEVICE_STATE_RUNNING;
959 migration->data_fd = -1;
960 migration->mig_flags = mig_flags;
961
962 vbasedev->dirty_pages_supported = vfio_dma_logging_supported(vbasedev);
963
964 oid = vmstate_if_get_id(VMSTATE_IF(DEVICE(obj)));
965 if (oid) {
966 path = g_strdup_printf("%s/vfio", oid);
967 } else {
968 path = g_strdup("vfio");
969 }
970 strpadcpy(id, sizeof(id), path, '\0');
971
972 register_savevm_live(id, VMSTATE_INSTANCE_ID_ANY, 1, &savevm_vfio_handlers,
973 vbasedev);
974
975 prepare_cb = migration->mig_flags & VFIO_MIGRATION_P2P ?
976 vfio_vmstate_change_prepare :
977 NULL;
978 migration->vm_state = qdev_add_vm_change_state_handler_full(
979 vbasedev->dev, vfio_vmstate_change, prepare_cb, vbasedev);
980 migration_add_notifier(&migration->migration_state,
981 vfio_migration_state_notifier);
982
983 return 0;
984 }
985
vfio_migration_deinit(VFIODevice * vbasedev)986 static void vfio_migration_deinit(VFIODevice *vbasedev)
987 {
988 VFIOMigration *migration = vbasedev->migration;
989
990 migration_remove_notifier(&migration->migration_state);
991 qemu_del_vm_change_state_handler(migration->vm_state);
992 unregister_savevm(VMSTATE_IF(vbasedev->dev), "vfio", vbasedev);
993 vfio_migration_free(vbasedev);
994 vfio_unblock_multiple_devices_migration();
995 }
996
vfio_block_migration(VFIODevice * vbasedev,Error * err,Error ** errp)997 static int vfio_block_migration(VFIODevice *vbasedev, Error *err, Error **errp)
998 {
999 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) {
1000 error_propagate(errp, err);
1001 return -EINVAL;
1002 }
1003
1004 vbasedev->migration_blocker = error_copy(err);
1005 error_free(err);
1006
1007 return migrate_add_blocker_normal(&vbasedev->migration_blocker, errp);
1008 }
1009
1010 /* ---------------------------------------------------------------------- */
1011
vfio_mig_bytes_transferred(void)1012 int64_t vfio_mig_bytes_transferred(void)
1013 {
1014 return bytes_transferred;
1015 }
1016
vfio_reset_bytes_transferred(void)1017 void vfio_reset_bytes_transferred(void)
1018 {
1019 bytes_transferred = 0;
1020 }
1021
1022 /*
1023 * Return true when either migration initialized or blocker registered.
1024 * Currently only return false when adding blocker fails which will
1025 * de-register vfio device.
1026 */
vfio_migration_realize(VFIODevice * vbasedev,Error ** errp)1027 bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
1028 {
1029 Error *err = NULL;
1030 int ret;
1031
1032 if (vbasedev->enable_migration == ON_OFF_AUTO_OFF) {
1033 error_setg(&err, "%s: Migration is disabled for VFIO device",
1034 vbasedev->name);
1035 return !vfio_block_migration(vbasedev, err, errp);
1036 }
1037
1038 ret = vfio_migration_init(vbasedev);
1039 if (ret) {
1040 if (ret == -ENOTTY) {
1041 error_setg(&err, "%s: VFIO migration is not supported in kernel",
1042 vbasedev->name);
1043 } else {
1044 error_setg(&err,
1045 "%s: Migration couldn't be initialized for VFIO device, "
1046 "err: %d (%s)",
1047 vbasedev->name, ret, strerror(-ret));
1048 }
1049
1050 return !vfio_block_migration(vbasedev, err, errp);
1051 }
1052
1053 if ((!vbasedev->dirty_pages_supported ||
1054 vbasedev->device_dirty_page_tracking == ON_OFF_AUTO_OFF) &&
1055 !vbasedev->iommu_dirty_tracking) {
1056 if (vbasedev->enable_migration == ON_OFF_AUTO_AUTO) {
1057 error_setg(&err,
1058 "%s: VFIO device doesn't support device and "
1059 "IOMMU dirty tracking", vbasedev->name);
1060 goto add_blocker;
1061 }
1062
1063 warn_report("%s: VFIO device doesn't support device and "
1064 "IOMMU dirty tracking", vbasedev->name);
1065 }
1066
1067 ret = vfio_block_multiple_devices_migration(vbasedev, errp);
1068 if (ret) {
1069 goto out_deinit;
1070 }
1071
1072 if (vfio_viommu_preset(vbasedev)) {
1073 error_setg(&err, "%s: Migration is currently not supported "
1074 "with vIOMMU enabled", vbasedev->name);
1075 goto add_blocker;
1076 }
1077
1078 trace_vfio_migration_realize(vbasedev->name);
1079 return true;
1080
1081 add_blocker:
1082 ret = vfio_block_migration(vbasedev, err, errp);
1083 out_deinit:
1084 if (ret) {
1085 vfio_migration_deinit(vbasedev);
1086 }
1087 return !ret;
1088 }
1089
vfio_migration_exit(VFIODevice * vbasedev)1090 void vfio_migration_exit(VFIODevice *vbasedev)
1091 {
1092 if (vbasedev->migration) {
1093 vfio_migration_deinit(vbasedev);
1094 }
1095
1096 migrate_del_blocker(&vbasedev->migration_blocker);
1097 }
1098