xref: /openbmc/qemu/block/qed.c (revision 1141159c)
1 /*
2  * QEMU Enhanced Disk Format
3  *
4  * Copyright IBM, Corp. 2010
5  *
6  * Authors:
7  *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
8  *  Anthony Liguori   <aliguori@us.ibm.com>
9  *
10  * This work is licensed under the terms of the GNU LGPL, version 2 or later.
11  * See the COPYING.LIB file in the top-level directory.
12  *
13  */
14 
15 #include "qemu/osdep.h"
16 #include "block/qdict.h"
17 #include "qapi/error.h"
18 #include "qemu/timer.h"
19 #include "qemu/bswap.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "qemu/option.h"
23 #include "qemu/memalign.h"
24 #include "trace.h"
25 #include "qed.h"
26 #include "sysemu/block-backend.h"
27 #include "qapi/qmp/qdict.h"
28 #include "qapi/qobject-input-visitor.h"
29 #include "qapi/qapi-visit-block-core.h"
30 
31 static QemuOptsList qed_create_opts;
32 
33 static int bdrv_qed_probe(const uint8_t *buf, int buf_size,
34                           const char *filename)
35 {
36     const QEDHeader *header = (const QEDHeader *)buf;
37 
38     if (buf_size < sizeof(*header)) {
39         return 0;
40     }
41     if (le32_to_cpu(header->magic) != QED_MAGIC) {
42         return 0;
43     }
44     return 100;
45 }
46 
47 /**
48  * Check whether an image format is raw
49  *
50  * @fmt:    Backing file format, may be NULL
51  */
52 static bool qed_fmt_is_raw(const char *fmt)
53 {
54     return fmt && strcmp(fmt, "raw") == 0;
55 }
56 
57 static void qed_header_le_to_cpu(const QEDHeader *le, QEDHeader *cpu)
58 {
59     cpu->magic = le32_to_cpu(le->magic);
60     cpu->cluster_size = le32_to_cpu(le->cluster_size);
61     cpu->table_size = le32_to_cpu(le->table_size);
62     cpu->header_size = le32_to_cpu(le->header_size);
63     cpu->features = le64_to_cpu(le->features);
64     cpu->compat_features = le64_to_cpu(le->compat_features);
65     cpu->autoclear_features = le64_to_cpu(le->autoclear_features);
66     cpu->l1_table_offset = le64_to_cpu(le->l1_table_offset);
67     cpu->image_size = le64_to_cpu(le->image_size);
68     cpu->backing_filename_offset = le32_to_cpu(le->backing_filename_offset);
69     cpu->backing_filename_size = le32_to_cpu(le->backing_filename_size);
70 }
71 
72 static void qed_header_cpu_to_le(const QEDHeader *cpu, QEDHeader *le)
73 {
74     le->magic = cpu_to_le32(cpu->magic);
75     le->cluster_size = cpu_to_le32(cpu->cluster_size);
76     le->table_size = cpu_to_le32(cpu->table_size);
77     le->header_size = cpu_to_le32(cpu->header_size);
78     le->features = cpu_to_le64(cpu->features);
79     le->compat_features = cpu_to_le64(cpu->compat_features);
80     le->autoclear_features = cpu_to_le64(cpu->autoclear_features);
81     le->l1_table_offset = cpu_to_le64(cpu->l1_table_offset);
82     le->image_size = cpu_to_le64(cpu->image_size);
83     le->backing_filename_offset = cpu_to_le32(cpu->backing_filename_offset);
84     le->backing_filename_size = cpu_to_le32(cpu->backing_filename_size);
85 }
86 
87 int qed_write_header_sync(BDRVQEDState *s)
88 {
89     QEDHeader le;
90 
91     qed_header_cpu_to_le(&s->header, &le);
92     return bdrv_pwrite(s->bs->file, 0, sizeof(le), &le, 0);
93 }
94 
95 /**
96  * Update header in-place (does not rewrite backing filename or other strings)
97  *
98  * This function only updates known header fields in-place and does not affect
99  * extra data after the QED header.
100  *
101  * No new allocating reqs can start while this function runs.
102  */
103 static int coroutine_fn GRAPH_RDLOCK qed_write_header(BDRVQEDState *s)
104 {
105     /* We must write full sectors for O_DIRECT but cannot necessarily generate
106      * the data following the header if an unrecognized compat feature is
107      * active.  Therefore, first read the sectors containing the header, update
108      * them, and write back.
109      */
110 
111     int nsectors = DIV_ROUND_UP(sizeof(QEDHeader), BDRV_SECTOR_SIZE);
112     size_t len = nsectors * BDRV_SECTOR_SIZE;
113     uint8_t *buf;
114     int ret;
115 
116     assert(s->allocating_acb || s->allocating_write_reqs_plugged);
117 
118     buf = qemu_blockalign(s->bs, len);
119 
120     ret = bdrv_co_pread(s->bs->file, 0, len, buf, 0);
121     if (ret < 0) {
122         goto out;
123     }
124 
125     /* Update header */
126     qed_header_cpu_to_le(&s->header, (QEDHeader *) buf);
127 
128     ret = bdrv_co_pwrite(s->bs->file, 0, len,  buf, 0);
129     if (ret < 0) {
130         goto out;
131     }
132 
133     ret = 0;
134 out:
135     qemu_vfree(buf);
136     return ret;
137 }
138 
139 static uint64_t qed_max_image_size(uint32_t cluster_size, uint32_t table_size)
140 {
141     uint64_t table_entries;
142     uint64_t l2_size;
143 
144     table_entries = (table_size * cluster_size) / sizeof(uint64_t);
145     l2_size = table_entries * cluster_size;
146 
147     return l2_size * table_entries;
148 }
149 
150 static bool qed_is_cluster_size_valid(uint32_t cluster_size)
151 {
152     if (cluster_size < QED_MIN_CLUSTER_SIZE ||
153         cluster_size > QED_MAX_CLUSTER_SIZE) {
154         return false;
155     }
156     if (cluster_size & (cluster_size - 1)) {
157         return false; /* not power of 2 */
158     }
159     return true;
160 }
161 
162 static bool qed_is_table_size_valid(uint32_t table_size)
163 {
164     if (table_size < QED_MIN_TABLE_SIZE ||
165         table_size > QED_MAX_TABLE_SIZE) {
166         return false;
167     }
168     if (table_size & (table_size - 1)) {
169         return false; /* not power of 2 */
170     }
171     return true;
172 }
173 
174 static bool qed_is_image_size_valid(uint64_t image_size, uint32_t cluster_size,
175                                     uint32_t table_size)
176 {
177     if (image_size % BDRV_SECTOR_SIZE != 0) {
178         return false; /* not multiple of sector size */
179     }
180     if (image_size > qed_max_image_size(cluster_size, table_size)) {
181         return false; /* image is too large */
182     }
183     return true;
184 }
185 
186 /**
187  * Read a string of known length from the image file
188  *
189  * @file:       Image file
190  * @offset:     File offset to start of string, in bytes
191  * @n:          String length in bytes
192  * @buf:        Destination buffer
193  * @buflen:     Destination buffer length in bytes
194  * @ret:        0 on success, -errno on failure
195  *
196  * The string is NUL-terminated.
197  */
198 static int qed_read_string(BdrvChild *file, uint64_t offset, size_t n,
199                            char *buf, size_t buflen)
200 {
201     int ret;
202     if (n >= buflen) {
203         return -EINVAL;
204     }
205     ret = bdrv_pread(file, offset, n, buf, 0);
206     if (ret < 0) {
207         return ret;
208     }
209     buf[n] = '\0';
210     return 0;
211 }
212 
213 /**
214  * Allocate new clusters
215  *
216  * @s:          QED state
217  * @n:          Number of contiguous clusters to allocate
218  * @ret:        Offset of first allocated cluster
219  *
220  * This function only produces the offset where the new clusters should be
221  * written.  It updates BDRVQEDState but does not make any changes to the image
222  * file.
223  *
224  * Called with table_lock held.
225  */
226 static uint64_t qed_alloc_clusters(BDRVQEDState *s, unsigned int n)
227 {
228     uint64_t offset = s->file_size;
229     s->file_size += n * s->header.cluster_size;
230     return offset;
231 }
232 
233 QEDTable *qed_alloc_table(BDRVQEDState *s)
234 {
235     /* Honor O_DIRECT memory alignment requirements */
236     return qemu_blockalign(s->bs,
237                            s->header.cluster_size * s->header.table_size);
238 }
239 
240 /**
241  * Allocate a new zeroed L2 table
242  *
243  * Called with table_lock held.
244  */
245 static CachedL2Table *qed_new_l2_table(BDRVQEDState *s)
246 {
247     CachedL2Table *l2_table = qed_alloc_l2_cache_entry(&s->l2_cache);
248 
249     l2_table->table = qed_alloc_table(s);
250     l2_table->offset = qed_alloc_clusters(s, s->header.table_size);
251 
252     memset(l2_table->table->offsets, 0,
253            s->header.cluster_size * s->header.table_size);
254     return l2_table;
255 }
256 
257 static bool coroutine_fn qed_plug_allocating_write_reqs(BDRVQEDState *s)
258 {
259     qemu_co_mutex_lock(&s->table_lock);
260 
261     /* No reentrancy is allowed.  */
262     assert(!s->allocating_write_reqs_plugged);
263     if (s->allocating_acb != NULL) {
264         /* Another allocating write came concurrently.  This cannot happen
265          * from bdrv_qed_drain_begin, but it can happen when the timer runs.
266          */
267         qemu_co_mutex_unlock(&s->table_lock);
268         return false;
269     }
270 
271     s->allocating_write_reqs_plugged = true;
272     qemu_co_mutex_unlock(&s->table_lock);
273     return true;
274 }
275 
276 static void coroutine_fn qed_unplug_allocating_write_reqs(BDRVQEDState *s)
277 {
278     qemu_co_mutex_lock(&s->table_lock);
279     assert(s->allocating_write_reqs_plugged);
280     s->allocating_write_reqs_plugged = false;
281     qemu_co_queue_next(&s->allocating_write_reqs);
282     qemu_co_mutex_unlock(&s->table_lock);
283 }
284 
285 static void coroutine_fn GRAPH_RDLOCK qed_need_check_timer(BDRVQEDState *s)
286 {
287     int ret;
288 
289     trace_qed_need_check_timer_cb(s);
290     assert_bdrv_graph_readable();
291 
292     if (!qed_plug_allocating_write_reqs(s)) {
293         return;
294     }
295 
296     /* Ensure writes are on disk before clearing flag */
297     ret = bdrv_co_flush(s->bs->file->bs);
298     if (ret < 0) {
299         qed_unplug_allocating_write_reqs(s);
300         return;
301     }
302 
303     s->header.features &= ~QED_F_NEED_CHECK;
304     ret = qed_write_header(s);
305     (void) ret;
306 
307     qed_unplug_allocating_write_reqs(s);
308 
309     ret = bdrv_co_flush(s->bs);
310     (void) ret;
311 }
312 
313 static void coroutine_fn qed_need_check_timer_entry(void *opaque)
314 {
315     BDRVQEDState *s = opaque;
316     GRAPH_RDLOCK_GUARD();
317 
318     qed_need_check_timer(opaque);
319     bdrv_dec_in_flight(s->bs);
320 }
321 
322 static void qed_need_check_timer_cb(void *opaque)
323 {
324     BDRVQEDState *s = opaque;
325     Coroutine *co = qemu_coroutine_create(qed_need_check_timer_entry, opaque);
326 
327     bdrv_inc_in_flight(s->bs);
328     qemu_coroutine_enter(co);
329 }
330 
331 static void qed_start_need_check_timer(BDRVQEDState *s)
332 {
333     trace_qed_start_need_check_timer(s);
334 
335     /* Use QEMU_CLOCK_VIRTUAL so we don't alter the image file while suspended for
336      * migration.
337      */
338     timer_mod(s->need_check_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
339                    NANOSECONDS_PER_SECOND * QED_NEED_CHECK_TIMEOUT);
340 }
341 
342 /* It's okay to call this multiple times or when no timer is started */
343 static void qed_cancel_need_check_timer(BDRVQEDState *s)
344 {
345     trace_qed_cancel_need_check_timer(s);
346     timer_del(s->need_check_timer);
347 }
348 
349 static void bdrv_qed_detach_aio_context(BlockDriverState *bs)
350 {
351     BDRVQEDState *s = bs->opaque;
352 
353     qed_cancel_need_check_timer(s);
354     timer_free(s->need_check_timer);
355 }
356 
357 static void bdrv_qed_attach_aio_context(BlockDriverState *bs,
358                                         AioContext *new_context)
359 {
360     BDRVQEDState *s = bs->opaque;
361 
362     s->need_check_timer = aio_timer_new(new_context,
363                                         QEMU_CLOCK_VIRTUAL, SCALE_NS,
364                                         qed_need_check_timer_cb, s);
365     if (s->header.features & QED_F_NEED_CHECK) {
366         qed_start_need_check_timer(s);
367     }
368 }
369 
370 static void bdrv_qed_drain_begin(BlockDriverState *bs)
371 {
372     BDRVQEDState *s = bs->opaque;
373 
374     /* Fire the timer immediately in order to start doing I/O as soon as the
375      * header is flushed.
376      */
377     if (s->need_check_timer && timer_pending(s->need_check_timer)) {
378         Coroutine *co;
379 
380         qed_cancel_need_check_timer(s);
381         co = qemu_coroutine_create(qed_need_check_timer_entry, s);
382         bdrv_inc_in_flight(bs);
383         aio_co_enter(bdrv_get_aio_context(bs), co);
384     }
385 }
386 
387 static void bdrv_qed_init_state(BlockDriverState *bs)
388 {
389     BDRVQEDState *s = bs->opaque;
390 
391     memset(s, 0, sizeof(BDRVQEDState));
392     s->bs = bs;
393     qemu_co_mutex_init(&s->table_lock);
394     qemu_co_queue_init(&s->allocating_write_reqs);
395 }
396 
397 /* Called with table_lock held.  */
398 static int coroutine_fn GRAPH_RDLOCK
399 bdrv_qed_do_open(BlockDriverState *bs, QDict *options, int flags, Error **errp)
400 {
401     BDRVQEDState *s = bs->opaque;
402     QEDHeader le_header;
403     int64_t file_size;
404     int ret;
405 
406     ret = bdrv_co_pread(bs->file, 0, sizeof(le_header), &le_header, 0);
407     if (ret < 0) {
408         error_setg(errp, "Failed to read QED header");
409         return ret;
410     }
411     qed_header_le_to_cpu(&le_header, &s->header);
412 
413     if (s->header.magic != QED_MAGIC) {
414         error_setg(errp, "Image not in QED format");
415         return -EINVAL;
416     }
417     if (s->header.features & ~QED_FEATURE_MASK) {
418         /* image uses unsupported feature bits */
419         error_setg(errp, "Unsupported QED features: %" PRIx64,
420                    s->header.features & ~QED_FEATURE_MASK);
421         return -ENOTSUP;
422     }
423     if (!qed_is_cluster_size_valid(s->header.cluster_size)) {
424         error_setg(errp, "QED cluster size is invalid");
425         return -EINVAL;
426     }
427 
428     /* Round down file size to the last cluster */
429     file_size = bdrv_co_getlength(bs->file->bs);
430     if (file_size < 0) {
431         error_setg(errp, "Failed to get file length");
432         return file_size;
433     }
434     s->file_size = qed_start_of_cluster(s, file_size);
435 
436     if (!qed_is_table_size_valid(s->header.table_size)) {
437         error_setg(errp, "QED table size is invalid");
438         return -EINVAL;
439     }
440     if (!qed_is_image_size_valid(s->header.image_size,
441                                  s->header.cluster_size,
442                                  s->header.table_size)) {
443         error_setg(errp, "QED image size is invalid");
444         return -EINVAL;
445     }
446     if (!qed_check_table_offset(s, s->header.l1_table_offset)) {
447         error_setg(errp, "QED table offset is invalid");
448         return -EINVAL;
449     }
450 
451     s->table_nelems = (s->header.cluster_size * s->header.table_size) /
452                       sizeof(uint64_t);
453     s->l2_shift = ctz32(s->header.cluster_size);
454     s->l2_mask = s->table_nelems - 1;
455     s->l1_shift = s->l2_shift + ctz32(s->table_nelems);
456 
457     /* Header size calculation must not overflow uint32_t */
458     if (s->header.header_size > UINT32_MAX / s->header.cluster_size) {
459         error_setg(errp, "QED header size is too large");
460         return -EINVAL;
461     }
462 
463     if ((s->header.features & QED_F_BACKING_FILE)) {
464         g_autofree char *backing_file_str = NULL;
465 
466         if ((uint64_t)s->header.backing_filename_offset +
467             s->header.backing_filename_size >
468             s->header.cluster_size * s->header.header_size) {
469             error_setg(errp, "QED backing filename offset is invalid");
470             return -EINVAL;
471         }
472 
473         backing_file_str = g_malloc(sizeof(bs->backing_file));
474         ret = qed_read_string(bs->file, s->header.backing_filename_offset,
475                               s->header.backing_filename_size,
476                               backing_file_str, sizeof(bs->backing_file));
477         if (ret < 0) {
478             error_setg(errp, "Failed to read backing filename");
479             return ret;
480         }
481 
482         if (!g_str_equal(backing_file_str, bs->backing_file)) {
483             pstrcpy(bs->backing_file, sizeof(bs->backing_file),
484                     backing_file_str);
485             pstrcpy(bs->auto_backing_file, sizeof(bs->auto_backing_file),
486                     backing_file_str);
487         }
488 
489         if (s->header.features & QED_F_BACKING_FORMAT_NO_PROBE) {
490             pstrcpy(bs->backing_format, sizeof(bs->backing_format), "raw");
491         }
492     }
493 
494     /* Reset unknown autoclear feature bits.  This is a backwards
495      * compatibility mechanism that allows images to be opened by older
496      * programs, which "knock out" unknown feature bits.  When an image is
497      * opened by a newer program again it can detect that the autoclear
498      * feature is no longer valid.
499      */
500     if ((s->header.autoclear_features & ~QED_AUTOCLEAR_FEATURE_MASK) != 0 &&
501         !bdrv_is_read_only(bs->file->bs) && !(flags & BDRV_O_INACTIVE)) {
502         s->header.autoclear_features &= QED_AUTOCLEAR_FEATURE_MASK;
503 
504         ret = qed_write_header_sync(s);
505         if (ret) {
506             error_setg(errp, "Failed to update header");
507             return ret;
508         }
509 
510         /* From here on only known autoclear feature bits are valid */
511         bdrv_co_flush(bs->file->bs);
512     }
513 
514     s->l1_table = qed_alloc_table(s);
515     qed_init_l2_cache(&s->l2_cache);
516 
517     ret = qed_read_l1_table_sync(s);
518     if (ret) {
519         error_setg(errp, "Failed to read L1 table");
520         goto out;
521     }
522 
523     /* If image was not closed cleanly, check consistency */
524     if (!(flags & BDRV_O_CHECK) && (s->header.features & QED_F_NEED_CHECK)) {
525         /* Read-only images cannot be fixed.  There is no risk of corruption
526          * since write operations are not possible.  Therefore, allow
527          * potentially inconsistent images to be opened read-only.  This can
528          * aid data recovery from an otherwise inconsistent image.
529          */
530         if (!bdrv_is_read_only(bs->file->bs) &&
531             !(flags & BDRV_O_INACTIVE)) {
532             BdrvCheckResult result = {0};
533 
534             ret = qed_check(s, &result, true);
535             if (ret) {
536                 error_setg(errp, "Image corrupted");
537                 goto out;
538             }
539         }
540     }
541 
542     bdrv_qed_attach_aio_context(bs, bdrv_get_aio_context(bs));
543 
544 out:
545     if (ret) {
546         qed_free_l2_cache(&s->l2_cache);
547         qemu_vfree(s->l1_table);
548     }
549     return ret;
550 }
551 
552 typedef struct QEDOpenCo {
553     BlockDriverState *bs;
554     QDict *options;
555     int flags;
556     Error **errp;
557     int ret;
558 } QEDOpenCo;
559 
560 static void coroutine_fn bdrv_qed_open_entry(void *opaque)
561 {
562     QEDOpenCo *qoc = opaque;
563     BDRVQEDState *s = qoc->bs->opaque;
564 
565     GRAPH_RDLOCK_GUARD();
566 
567     qemu_co_mutex_lock(&s->table_lock);
568     qoc->ret = bdrv_qed_do_open(qoc->bs, qoc->options, qoc->flags, qoc->errp);
569     qemu_co_mutex_unlock(&s->table_lock);
570 }
571 
572 static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
573                          Error **errp)
574 {
575     QEDOpenCo qoc = {
576         .bs = bs,
577         .options = options,
578         .flags = flags,
579         .errp = errp,
580         .ret = -EINPROGRESS
581     };
582     int ret;
583 
584     ret = bdrv_open_file_child(NULL, options, "file", bs, errp);
585     if (ret < 0) {
586         return ret;
587     }
588 
589     bdrv_qed_init_state(bs);
590     assert(!qemu_in_coroutine());
591     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
592     qemu_coroutine_enter(qemu_coroutine_create(bdrv_qed_open_entry, &qoc));
593     BDRV_POLL_WHILE(bs, qoc.ret == -EINPROGRESS);
594 
595     return qoc.ret;
596 }
597 
598 static void bdrv_qed_refresh_limits(BlockDriverState *bs, Error **errp)
599 {
600     BDRVQEDState *s = bs->opaque;
601 
602     bs->bl.pwrite_zeroes_alignment = s->header.cluster_size;
603     bs->bl.max_pwrite_zeroes = QEMU_ALIGN_DOWN(INT_MAX, s->header.cluster_size);
604 }
605 
606 /* We have nothing to do for QED reopen, stubs just return
607  * success */
608 static int bdrv_qed_reopen_prepare(BDRVReopenState *state,
609                                    BlockReopenQueue *queue, Error **errp)
610 {
611     return 0;
612 }
613 
614 static void bdrv_qed_close(BlockDriverState *bs)
615 {
616     BDRVQEDState *s = bs->opaque;
617 
618     bdrv_qed_detach_aio_context(bs);
619 
620     /* Ensure writes reach stable storage */
621     bdrv_flush(bs->file->bs);
622 
623     /* Clean shutdown, no check required on next open */
624     if (s->header.features & QED_F_NEED_CHECK) {
625         s->header.features &= ~QED_F_NEED_CHECK;
626         qed_write_header_sync(s);
627     }
628 
629     qed_free_l2_cache(&s->l2_cache);
630     qemu_vfree(s->l1_table);
631 }
632 
633 static int coroutine_fn bdrv_qed_co_create(BlockdevCreateOptions *opts,
634                                            Error **errp)
635 {
636     BlockdevCreateOptionsQed *qed_opts;
637     BlockBackend *blk = NULL;
638     BlockDriverState *bs = NULL;
639 
640     QEDHeader header;
641     QEDHeader le_header;
642     uint8_t *l1_table = NULL;
643     size_t l1_size;
644     int ret = 0;
645 
646     assert(opts->driver == BLOCKDEV_DRIVER_QED);
647     qed_opts = &opts->u.qed;
648 
649     /* Validate options and set default values */
650     if (!qed_opts->has_cluster_size) {
651         qed_opts->cluster_size = QED_DEFAULT_CLUSTER_SIZE;
652     }
653     if (!qed_opts->has_table_size) {
654         qed_opts->table_size = QED_DEFAULT_TABLE_SIZE;
655     }
656 
657     if (!qed_is_cluster_size_valid(qed_opts->cluster_size)) {
658         error_setg(errp, "QED cluster size must be within range [%u, %u] "
659                          "and power of 2",
660                    QED_MIN_CLUSTER_SIZE, QED_MAX_CLUSTER_SIZE);
661         return -EINVAL;
662     }
663     if (!qed_is_table_size_valid(qed_opts->table_size)) {
664         error_setg(errp, "QED table size must be within range [%u, %u] "
665                          "and power of 2",
666                    QED_MIN_TABLE_SIZE, QED_MAX_TABLE_SIZE);
667         return -EINVAL;
668     }
669     if (!qed_is_image_size_valid(qed_opts->size, qed_opts->cluster_size,
670                                  qed_opts->table_size))
671     {
672         error_setg(errp, "QED image size must be a non-zero multiple of "
673                          "cluster size and less than %" PRIu64 " bytes",
674                    qed_max_image_size(qed_opts->cluster_size,
675                                       qed_opts->table_size));
676         return -EINVAL;
677     }
678 
679     /* Create BlockBackend to write to the image */
680     bs = bdrv_co_open_blockdev_ref(qed_opts->file, errp);
681     if (bs == NULL) {
682         return -EIO;
683     }
684 
685     blk = blk_co_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
686                              errp);
687     if (!blk) {
688         ret = -EPERM;
689         goto out;
690     }
691     blk_set_allow_write_beyond_eof(blk, true);
692 
693     /* Prepare image format */
694     header = (QEDHeader) {
695         .magic = QED_MAGIC,
696         .cluster_size = qed_opts->cluster_size,
697         .table_size = qed_opts->table_size,
698         .header_size = 1,
699         .features = 0,
700         .compat_features = 0,
701         .l1_table_offset = qed_opts->cluster_size,
702         .image_size = qed_opts->size,
703     };
704 
705     l1_size = header.cluster_size * header.table_size;
706 
707     /*
708      * The QED format associates file length with allocation status,
709      * so a new file (which is empty) must have a length of 0.
710      */
711     ret = blk_co_truncate(blk, 0, true, PREALLOC_MODE_OFF, 0, errp);
712     if (ret < 0) {
713         goto out;
714     }
715 
716     if (qed_opts->backing_file) {
717         header.features |= QED_F_BACKING_FILE;
718         header.backing_filename_offset = sizeof(le_header);
719         header.backing_filename_size = strlen(qed_opts->backing_file);
720 
721         if (qed_opts->has_backing_fmt) {
722             const char *backing_fmt = BlockdevDriver_str(qed_opts->backing_fmt);
723             if (qed_fmt_is_raw(backing_fmt)) {
724                 header.features |= QED_F_BACKING_FORMAT_NO_PROBE;
725             }
726         }
727     }
728 
729     qed_header_cpu_to_le(&header, &le_header);
730     ret = blk_co_pwrite(blk, 0, sizeof(le_header), &le_header, 0);
731     if (ret < 0) {
732         goto out;
733     }
734     ret = blk_co_pwrite(blk, sizeof(le_header), header.backing_filename_size,
735                      qed_opts->backing_file, 0);
736     if (ret < 0) {
737         goto out;
738     }
739 
740     l1_table = g_malloc0(l1_size);
741     ret = blk_co_pwrite(blk, header.l1_table_offset, l1_size, l1_table, 0);
742     if (ret < 0) {
743         goto out;
744     }
745 
746     ret = 0; /* success */
747 out:
748     g_free(l1_table);
749     blk_co_unref(blk);
750     bdrv_co_unref(bs);
751     return ret;
752 }
753 
754 static int coroutine_fn GRAPH_RDLOCK
755 bdrv_qed_co_create_opts(BlockDriver *drv, const char *filename,
756                         QemuOpts *opts, Error **errp)
757 {
758     BlockdevCreateOptions *create_options = NULL;
759     QDict *qdict;
760     Visitor *v;
761     BlockDriverState *bs = NULL;
762     int ret;
763 
764     static const QDictRenames opt_renames[] = {
765         { BLOCK_OPT_BACKING_FILE,       "backing-file" },
766         { BLOCK_OPT_BACKING_FMT,        "backing-fmt" },
767         { BLOCK_OPT_CLUSTER_SIZE,       "cluster-size" },
768         { BLOCK_OPT_TABLE_SIZE,         "table-size" },
769         { NULL, NULL },
770     };
771 
772     /* Parse options and convert legacy syntax */
773     qdict = qemu_opts_to_qdict_filtered(opts, NULL, &qed_create_opts, true);
774 
775     if (!qdict_rename_keys(qdict, opt_renames, errp)) {
776         ret = -EINVAL;
777         goto fail;
778     }
779 
780     /* Create and open the file (protocol layer) */
781     ret = bdrv_co_create_file(filename, opts, errp);
782     if (ret < 0) {
783         goto fail;
784     }
785 
786     bs = bdrv_co_open(filename, NULL, NULL,
787                       BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_PROTOCOL, errp);
788     if (bs == NULL) {
789         ret = -EIO;
790         goto fail;
791     }
792 
793     /* Now get the QAPI type BlockdevCreateOptions */
794     qdict_put_str(qdict, "driver", "qed");
795     qdict_put_str(qdict, "file", bs->node_name);
796 
797     v = qobject_input_visitor_new_flat_confused(qdict, errp);
798     if (!v) {
799         ret = -EINVAL;
800         goto fail;
801     }
802 
803     visit_type_BlockdevCreateOptions(v, NULL, &create_options, errp);
804     visit_free(v);
805     if (!create_options) {
806         ret = -EINVAL;
807         goto fail;
808     }
809 
810     /* Silently round up size */
811     assert(create_options->driver == BLOCKDEV_DRIVER_QED);
812     create_options->u.qed.size =
813         ROUND_UP(create_options->u.qed.size, BDRV_SECTOR_SIZE);
814 
815     /* Create the qed image (format layer) */
816     ret = bdrv_qed_co_create(create_options, errp);
817 
818 fail:
819     qobject_unref(qdict);
820     bdrv_co_unref(bs);
821     qapi_free_BlockdevCreateOptions(create_options);
822     return ret;
823 }
824 
825 static int coroutine_fn GRAPH_RDLOCK
826 bdrv_qed_co_block_status(BlockDriverState *bs, bool want_zero, int64_t pos,
827                          int64_t bytes, int64_t *pnum, int64_t *map,
828                          BlockDriverState **file)
829 {
830     BDRVQEDState *s = bs->opaque;
831     size_t len = MIN(bytes, SIZE_MAX);
832     int status;
833     QEDRequest request = { .l2_table = NULL };
834     uint64_t offset;
835     int ret;
836 
837     qemu_co_mutex_lock(&s->table_lock);
838     ret = qed_find_cluster(s, &request, pos, &len, &offset);
839 
840     *pnum = len;
841     switch (ret) {
842     case QED_CLUSTER_FOUND:
843         *map = offset | qed_offset_into_cluster(s, pos);
844         status = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
845         *file = bs->file->bs;
846         break;
847     case QED_CLUSTER_ZERO:
848         status = BDRV_BLOCK_ZERO;
849         break;
850     case QED_CLUSTER_L2:
851     case QED_CLUSTER_L1:
852         status = 0;
853         break;
854     default:
855         assert(ret < 0);
856         status = ret;
857         break;
858     }
859 
860     qed_unref_l2_cache_entry(request.l2_table);
861     qemu_co_mutex_unlock(&s->table_lock);
862 
863     return status;
864 }
865 
866 static BDRVQEDState *acb_to_s(QEDAIOCB *acb)
867 {
868     return acb->bs->opaque;
869 }
870 
871 /**
872  * Read from the backing file or zero-fill if no backing file
873  *
874  * @s:              QED state
875  * @pos:            Byte position in device
876  * @qiov:           Destination I/O vector
877  *
878  * This function reads qiov->size bytes starting at pos from the backing file.
879  * If there is no backing file then zeroes are read.
880  */
881 static int coroutine_fn GRAPH_RDLOCK
882 qed_read_backing_file(BDRVQEDState *s, uint64_t pos, QEMUIOVector *qiov)
883 {
884     if (s->bs->backing) {
885         BLKDBG_EVENT(s->bs->file, BLKDBG_READ_BACKING_AIO);
886         return bdrv_co_preadv(s->bs->backing, pos, qiov->size, qiov, 0);
887     }
888     qemu_iovec_memset(qiov, 0, 0, qiov->size);
889     return 0;
890 }
891 
892 /**
893  * Copy data from backing file into the image
894  *
895  * @s:          QED state
896  * @pos:        Byte position in device
897  * @len:        Number of bytes
898  * @offset:     Byte offset in image file
899  */
900 static int coroutine_fn GRAPH_RDLOCK
901 qed_copy_from_backing_file(BDRVQEDState *s, uint64_t pos, uint64_t len,
902                            uint64_t offset)
903 {
904     QEMUIOVector qiov;
905     int ret;
906 
907     /* Skip copy entirely if there is no work to do */
908     if (len == 0) {
909         return 0;
910     }
911 
912     qemu_iovec_init_buf(&qiov, qemu_blockalign(s->bs, len), len);
913 
914     ret = qed_read_backing_file(s, pos, &qiov);
915 
916     if (ret) {
917         goto out;
918     }
919 
920     BLKDBG_EVENT(s->bs->file, BLKDBG_COW_WRITE);
921     ret = bdrv_co_pwritev(s->bs->file, offset, qiov.size, &qiov, 0);
922     if (ret < 0) {
923         goto out;
924     }
925     ret = 0;
926 out:
927     qemu_vfree(qemu_iovec_buf(&qiov));
928     return ret;
929 }
930 
931 /**
932  * Link one or more contiguous clusters into a table
933  *
934  * @s:              QED state
935  * @table:          L2 table
936  * @index:          First cluster index
937  * @n:              Number of contiguous clusters
938  * @cluster:        First cluster offset
939  *
940  * The cluster offset may be an allocated byte offset in the image file, the
941  * zero cluster marker, or the unallocated cluster marker.
942  *
943  * Called with table_lock held.
944  */
945 static void coroutine_fn qed_update_l2_table(BDRVQEDState *s, QEDTable *table,
946                                              int index, unsigned int n,
947                                              uint64_t cluster)
948 {
949     int i;
950     for (i = index; i < index + n; i++) {
951         table->offsets[i] = cluster;
952         if (!qed_offset_is_unalloc_cluster(cluster) &&
953             !qed_offset_is_zero_cluster(cluster)) {
954             cluster += s->header.cluster_size;
955         }
956     }
957 }
958 
959 /* Called with table_lock held.  */
960 static void coroutine_fn qed_aio_complete(QEDAIOCB *acb)
961 {
962     BDRVQEDState *s = acb_to_s(acb);
963 
964     /* Free resources */
965     qemu_iovec_destroy(&acb->cur_qiov);
966     qed_unref_l2_cache_entry(acb->request.l2_table);
967 
968     /* Free the buffer we may have allocated for zero writes */
969     if (acb->flags & QED_AIOCB_ZERO) {
970         qemu_vfree(acb->qiov->iov[0].iov_base);
971         acb->qiov->iov[0].iov_base = NULL;
972     }
973 
974     /* Start next allocating write request waiting behind this one.  Note that
975      * requests enqueue themselves when they first hit an unallocated cluster
976      * but they wait until the entire request is finished before waking up the
977      * next request in the queue.  This ensures that we don't cycle through
978      * requests multiple times but rather finish one at a time completely.
979      */
980     if (acb == s->allocating_acb) {
981         s->allocating_acb = NULL;
982         if (!qemu_co_queue_empty(&s->allocating_write_reqs)) {
983             qemu_co_queue_next(&s->allocating_write_reqs);
984         } else if (s->header.features & QED_F_NEED_CHECK) {
985             qed_start_need_check_timer(s);
986         }
987     }
988 }
989 
990 /**
991  * Update L1 table with new L2 table offset and write it out
992  *
993  * Called with table_lock held.
994  */
995 static int coroutine_fn GRAPH_RDLOCK qed_aio_write_l1_update(QEDAIOCB *acb)
996 {
997     BDRVQEDState *s = acb_to_s(acb);
998     CachedL2Table *l2_table = acb->request.l2_table;
999     uint64_t l2_offset = l2_table->offset;
1000     int index, ret;
1001 
1002     index = qed_l1_index(s, acb->cur_pos);
1003     s->l1_table->offsets[index] = l2_table->offset;
1004 
1005     ret = qed_write_l1_table(s, index, 1);
1006 
1007     /* Commit the current L2 table to the cache */
1008     qed_commit_l2_cache_entry(&s->l2_cache, l2_table);
1009 
1010     /* This is guaranteed to succeed because we just committed the entry to the
1011      * cache.
1012      */
1013     acb->request.l2_table = qed_find_l2_cache_entry(&s->l2_cache, l2_offset);
1014     assert(acb->request.l2_table != NULL);
1015 
1016     return ret;
1017 }
1018 
1019 
1020 /**
1021  * Update L2 table with new cluster offsets and write them out
1022  *
1023  * Called with table_lock held.
1024  */
1025 static int coroutine_fn GRAPH_RDLOCK
1026 qed_aio_write_l2_update(QEDAIOCB *acb, uint64_t offset)
1027 {
1028     BDRVQEDState *s = acb_to_s(acb);
1029     bool need_alloc = acb->find_cluster_ret == QED_CLUSTER_L1;
1030     int index, ret;
1031 
1032     if (need_alloc) {
1033         qed_unref_l2_cache_entry(acb->request.l2_table);
1034         acb->request.l2_table = qed_new_l2_table(s);
1035     }
1036 
1037     index = qed_l2_index(s, acb->cur_pos);
1038     qed_update_l2_table(s, acb->request.l2_table->table, index, acb->cur_nclusters,
1039                          offset);
1040 
1041     if (need_alloc) {
1042         /* Write out the whole new L2 table */
1043         ret = qed_write_l2_table(s, &acb->request, 0, s->table_nelems, true);
1044         if (ret) {
1045             return ret;
1046         }
1047         return qed_aio_write_l1_update(acb);
1048     } else {
1049         /* Write out only the updated part of the L2 table */
1050         ret = qed_write_l2_table(s, &acb->request, index, acb->cur_nclusters,
1051                                  false);
1052         if (ret) {
1053             return ret;
1054         }
1055     }
1056     return 0;
1057 }
1058 
1059 /**
1060  * Write data to the image file
1061  *
1062  * Called with table_lock *not* held.
1063  */
1064 static int coroutine_fn GRAPH_RDLOCK qed_aio_write_main(QEDAIOCB *acb)
1065 {
1066     BDRVQEDState *s = acb_to_s(acb);
1067     uint64_t offset = acb->cur_cluster +
1068                       qed_offset_into_cluster(s, acb->cur_pos);
1069 
1070     trace_qed_aio_write_main(s, acb, 0, offset, acb->cur_qiov.size);
1071 
1072     BLKDBG_EVENT(s->bs->file, BLKDBG_WRITE_AIO);
1073     return bdrv_co_pwritev(s->bs->file, offset, acb->cur_qiov.size,
1074                            &acb->cur_qiov, 0);
1075 }
1076 
1077 /**
1078  * Populate untouched regions of new data cluster
1079  *
1080  * Called with table_lock held.
1081  */
1082 static int coroutine_fn GRAPH_RDLOCK qed_aio_write_cow(QEDAIOCB *acb)
1083 {
1084     BDRVQEDState *s = acb_to_s(acb);
1085     uint64_t start, len, offset;
1086     int ret;
1087 
1088     qemu_co_mutex_unlock(&s->table_lock);
1089 
1090     /* Populate front untouched region of new data cluster */
1091     start = qed_start_of_cluster(s, acb->cur_pos);
1092     len = qed_offset_into_cluster(s, acb->cur_pos);
1093 
1094     trace_qed_aio_write_prefill(s, acb, start, len, acb->cur_cluster);
1095     ret = qed_copy_from_backing_file(s, start, len, acb->cur_cluster);
1096     if (ret < 0) {
1097         goto out;
1098     }
1099 
1100     /* Populate back untouched region of new data cluster */
1101     start = acb->cur_pos + acb->cur_qiov.size;
1102     len = qed_start_of_cluster(s, start + s->header.cluster_size - 1) - start;
1103     offset = acb->cur_cluster +
1104              qed_offset_into_cluster(s, acb->cur_pos) +
1105              acb->cur_qiov.size;
1106 
1107     trace_qed_aio_write_postfill(s, acb, start, len, offset);
1108     ret = qed_copy_from_backing_file(s, start, len, offset);
1109     if (ret < 0) {
1110         goto out;
1111     }
1112 
1113     ret = qed_aio_write_main(acb);
1114     if (ret < 0) {
1115         goto out;
1116     }
1117 
1118     if (s->bs->backing) {
1119         /*
1120          * Flush new data clusters before updating the L2 table
1121          *
1122          * This flush is necessary when a backing file is in use.  A crash
1123          * during an allocating write could result in empty clusters in the
1124          * image.  If the write only touched a subregion of the cluster,
1125          * then backing image sectors have been lost in the untouched
1126          * region.  The solution is to flush after writing a new data
1127          * cluster and before updating the L2 table.
1128          */
1129         ret = bdrv_co_flush(s->bs->file->bs);
1130     }
1131 
1132 out:
1133     qemu_co_mutex_lock(&s->table_lock);
1134     return ret;
1135 }
1136 
1137 /**
1138  * Check if the QED_F_NEED_CHECK bit should be set during allocating write
1139  */
1140 static bool qed_should_set_need_check(BDRVQEDState *s)
1141 {
1142     /* The flush before L2 update path ensures consistency */
1143     if (s->bs->backing) {
1144         return false;
1145     }
1146 
1147     return !(s->header.features & QED_F_NEED_CHECK);
1148 }
1149 
1150 /**
1151  * Write new data cluster
1152  *
1153  * @acb:        Write request
1154  * @len:        Length in bytes
1155  *
1156  * This path is taken when writing to previously unallocated clusters.
1157  *
1158  * Called with table_lock held.
1159  */
1160 static int coroutine_fn GRAPH_RDLOCK
1161 qed_aio_write_alloc(QEDAIOCB *acb, size_t len)
1162 {
1163     BDRVQEDState *s = acb_to_s(acb);
1164     int ret;
1165 
1166     /* Cancel timer when the first allocating request comes in */
1167     if (s->allocating_acb == NULL) {
1168         qed_cancel_need_check_timer(s);
1169     }
1170 
1171     /* Freeze this request if another allocating write is in progress */
1172     if (s->allocating_acb != acb || s->allocating_write_reqs_plugged) {
1173         if (s->allocating_acb != NULL) {
1174             qemu_co_queue_wait(&s->allocating_write_reqs, &s->table_lock);
1175             assert(s->allocating_acb == NULL);
1176         }
1177         s->allocating_acb = acb;
1178         return -EAGAIN; /* start over with looking up table entries */
1179     }
1180 
1181     acb->cur_nclusters = qed_bytes_to_clusters(s,
1182             qed_offset_into_cluster(s, acb->cur_pos) + len);
1183     qemu_iovec_concat(&acb->cur_qiov, acb->qiov, acb->qiov_offset, len);
1184 
1185     if (acb->flags & QED_AIOCB_ZERO) {
1186         /* Skip ahead if the clusters are already zero */
1187         if (acb->find_cluster_ret == QED_CLUSTER_ZERO) {
1188             return 0;
1189         }
1190         acb->cur_cluster = 1;
1191     } else {
1192         acb->cur_cluster = qed_alloc_clusters(s, acb->cur_nclusters);
1193     }
1194 
1195     if (qed_should_set_need_check(s)) {
1196         s->header.features |= QED_F_NEED_CHECK;
1197         ret = qed_write_header(s);
1198         if (ret < 0) {
1199             return ret;
1200         }
1201     }
1202 
1203     if (!(acb->flags & QED_AIOCB_ZERO)) {
1204         ret = qed_aio_write_cow(acb);
1205         if (ret < 0) {
1206             return ret;
1207         }
1208     }
1209 
1210     return qed_aio_write_l2_update(acb, acb->cur_cluster);
1211 }
1212 
1213 /**
1214  * Write data cluster in place
1215  *
1216  * @acb:        Write request
1217  * @offset:     Cluster offset in bytes
1218  * @len:        Length in bytes
1219  *
1220  * This path is taken when writing to already allocated clusters.
1221  *
1222  * Called with table_lock held.
1223  */
1224 static int coroutine_fn GRAPH_RDLOCK
1225 qed_aio_write_inplace(QEDAIOCB *acb, uint64_t offset, size_t len)
1226 {
1227     BDRVQEDState *s = acb_to_s(acb);
1228     int r;
1229 
1230     qemu_co_mutex_unlock(&s->table_lock);
1231 
1232     /* Allocate buffer for zero writes */
1233     if (acb->flags & QED_AIOCB_ZERO) {
1234         struct iovec *iov = acb->qiov->iov;
1235 
1236         if (!iov->iov_base) {
1237             iov->iov_base = qemu_try_blockalign(acb->bs, iov->iov_len);
1238             if (iov->iov_base == NULL) {
1239                 r = -ENOMEM;
1240                 goto out;
1241             }
1242             memset(iov->iov_base, 0, iov->iov_len);
1243         }
1244     }
1245 
1246     /* Calculate the I/O vector */
1247     acb->cur_cluster = offset;
1248     qemu_iovec_concat(&acb->cur_qiov, acb->qiov, acb->qiov_offset, len);
1249 
1250     /* Do the actual write.  */
1251     r = qed_aio_write_main(acb);
1252 out:
1253     qemu_co_mutex_lock(&s->table_lock);
1254     return r;
1255 }
1256 
1257 /**
1258  * Write data cluster
1259  *
1260  * @opaque:     Write request
1261  * @ret:        QED_CLUSTER_FOUND, QED_CLUSTER_L2 or QED_CLUSTER_L1
1262  * @offset:     Cluster offset in bytes
1263  * @len:        Length in bytes
1264  *
1265  * Called with table_lock held.
1266  */
1267 static int coroutine_fn GRAPH_RDLOCK
1268 qed_aio_write_data(void *opaque, int ret, uint64_t offset, size_t len)
1269 {
1270     QEDAIOCB *acb = opaque;
1271 
1272     trace_qed_aio_write_data(acb_to_s(acb), acb, ret, offset, len);
1273 
1274     acb->find_cluster_ret = ret;
1275 
1276     switch (ret) {
1277     case QED_CLUSTER_FOUND:
1278         return qed_aio_write_inplace(acb, offset, len);
1279 
1280     case QED_CLUSTER_L2:
1281     case QED_CLUSTER_L1:
1282     case QED_CLUSTER_ZERO:
1283         return qed_aio_write_alloc(acb, len);
1284 
1285     default:
1286         g_assert_not_reached();
1287     }
1288 }
1289 
1290 /**
1291  * Read data cluster
1292  *
1293  * @opaque:     Read request
1294  * @ret:        QED_CLUSTER_FOUND, QED_CLUSTER_L2 or QED_CLUSTER_L1
1295  * @offset:     Cluster offset in bytes
1296  * @len:        Length in bytes
1297  *
1298  * Called with table_lock held.
1299  */
1300 static int coroutine_fn GRAPH_RDLOCK
1301 qed_aio_read_data(void *opaque, int ret, uint64_t offset, size_t len)
1302 {
1303     QEDAIOCB *acb = opaque;
1304     BDRVQEDState *s = acb_to_s(acb);
1305     BlockDriverState *bs = acb->bs;
1306     int r;
1307 
1308     qemu_co_mutex_unlock(&s->table_lock);
1309 
1310     /* Adjust offset into cluster */
1311     offset += qed_offset_into_cluster(s, acb->cur_pos);
1312 
1313     trace_qed_aio_read_data(s, acb, ret, offset, len);
1314 
1315     qemu_iovec_concat(&acb->cur_qiov, acb->qiov, acb->qiov_offset, len);
1316 
1317     /* Handle zero cluster and backing file reads, otherwise read
1318      * data cluster directly.
1319      */
1320     if (ret == QED_CLUSTER_ZERO) {
1321         qemu_iovec_memset(&acb->cur_qiov, 0, 0, acb->cur_qiov.size);
1322         r = 0;
1323     } else if (ret != QED_CLUSTER_FOUND) {
1324         r = qed_read_backing_file(s, acb->cur_pos, &acb->cur_qiov);
1325     } else {
1326         BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
1327         r = bdrv_co_preadv(bs->file, offset, acb->cur_qiov.size,
1328                            &acb->cur_qiov, 0);
1329     }
1330 
1331     qemu_co_mutex_lock(&s->table_lock);
1332     return r;
1333 }
1334 
1335 /**
1336  * Begin next I/O or complete the request
1337  */
1338 static int coroutine_fn GRAPH_RDLOCK qed_aio_next_io(QEDAIOCB *acb)
1339 {
1340     BDRVQEDState *s = acb_to_s(acb);
1341     uint64_t offset;
1342     size_t len;
1343     int ret;
1344 
1345     qemu_co_mutex_lock(&s->table_lock);
1346     while (1) {
1347         trace_qed_aio_next_io(s, acb, 0, acb->cur_pos + acb->cur_qiov.size);
1348 
1349         acb->qiov_offset += acb->cur_qiov.size;
1350         acb->cur_pos += acb->cur_qiov.size;
1351         qemu_iovec_reset(&acb->cur_qiov);
1352 
1353         /* Complete request */
1354         if (acb->cur_pos >= acb->end_pos) {
1355             ret = 0;
1356             break;
1357         }
1358 
1359         /* Find next cluster and start I/O */
1360         len = acb->end_pos - acb->cur_pos;
1361         ret = qed_find_cluster(s, &acb->request, acb->cur_pos, &len, &offset);
1362         if (ret < 0) {
1363             break;
1364         }
1365 
1366         if (acb->flags & QED_AIOCB_WRITE) {
1367             ret = qed_aio_write_data(acb, ret, offset, len);
1368         } else {
1369             ret = qed_aio_read_data(acb, ret, offset, len);
1370         }
1371 
1372         if (ret < 0 && ret != -EAGAIN) {
1373             break;
1374         }
1375     }
1376 
1377     trace_qed_aio_complete(s, acb, ret);
1378     qed_aio_complete(acb);
1379     qemu_co_mutex_unlock(&s->table_lock);
1380     return ret;
1381 }
1382 
1383 static int coroutine_fn GRAPH_RDLOCK
1384 qed_co_request(BlockDriverState *bs, int64_t sector_num, QEMUIOVector *qiov,
1385                int nb_sectors, int flags)
1386 {
1387     QEDAIOCB acb = {
1388         .bs         = bs,
1389         .cur_pos    = (uint64_t) sector_num * BDRV_SECTOR_SIZE,
1390         .end_pos    = (sector_num + nb_sectors) * BDRV_SECTOR_SIZE,
1391         .qiov       = qiov,
1392         .flags      = flags,
1393     };
1394     qemu_iovec_init(&acb.cur_qiov, qiov->niov);
1395 
1396     trace_qed_aio_setup(bs->opaque, &acb, sector_num, nb_sectors, NULL, flags);
1397 
1398     /* Start request */
1399     return qed_aio_next_io(&acb);
1400 }
1401 
1402 static int coroutine_fn GRAPH_RDLOCK
1403 bdrv_qed_co_readv(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
1404                   QEMUIOVector *qiov)
1405 {
1406     return qed_co_request(bs, sector_num, qiov, nb_sectors, 0);
1407 }
1408 
1409 static int coroutine_fn GRAPH_RDLOCK
1410 bdrv_qed_co_writev(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
1411                    QEMUIOVector *qiov, int flags)
1412 {
1413     return qed_co_request(bs, sector_num, qiov, nb_sectors, QED_AIOCB_WRITE);
1414 }
1415 
1416 static int coroutine_fn GRAPH_RDLOCK
1417 bdrv_qed_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int64_t bytes,
1418                           BdrvRequestFlags flags)
1419 {
1420     BDRVQEDState *s = bs->opaque;
1421 
1422     /*
1423      * Zero writes start without an I/O buffer.  If a buffer becomes necessary
1424      * then it will be allocated during request processing.
1425      */
1426     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, NULL, bytes);
1427 
1428     /*
1429      * QED is not prepared for 63bit write-zero requests, so rely on
1430      * max_pwrite_zeroes.
1431      */
1432     assert(bytes <= INT_MAX);
1433 
1434     /* Fall back if the request is not aligned */
1435     if (qed_offset_into_cluster(s, offset) ||
1436         qed_offset_into_cluster(s, bytes)) {
1437         return -ENOTSUP;
1438     }
1439 
1440     return qed_co_request(bs, offset >> BDRV_SECTOR_BITS, &qiov,
1441                           bytes >> BDRV_SECTOR_BITS,
1442                           QED_AIOCB_WRITE | QED_AIOCB_ZERO);
1443 }
1444 
1445 static int coroutine_fn bdrv_qed_co_truncate(BlockDriverState *bs,
1446                                              int64_t offset,
1447                                              bool exact,
1448                                              PreallocMode prealloc,
1449                                              BdrvRequestFlags flags,
1450                                              Error **errp)
1451 {
1452     BDRVQEDState *s = bs->opaque;
1453     uint64_t old_image_size;
1454     int ret;
1455 
1456     if (prealloc != PREALLOC_MODE_OFF) {
1457         error_setg(errp, "Unsupported preallocation mode '%s'",
1458                    PreallocMode_str(prealloc));
1459         return -ENOTSUP;
1460     }
1461 
1462     if (!qed_is_image_size_valid(offset, s->header.cluster_size,
1463                                  s->header.table_size)) {
1464         error_setg(errp, "Invalid image size specified");
1465         return -EINVAL;
1466     }
1467 
1468     if ((uint64_t)offset < s->header.image_size) {
1469         error_setg(errp, "Shrinking images is currently not supported");
1470         return -ENOTSUP;
1471     }
1472 
1473     old_image_size = s->header.image_size;
1474     s->header.image_size = offset;
1475     ret = qed_write_header_sync(s);
1476     if (ret < 0) {
1477         s->header.image_size = old_image_size;
1478         error_setg_errno(errp, -ret, "Failed to update the image size");
1479     }
1480     return ret;
1481 }
1482 
1483 static int64_t coroutine_fn bdrv_qed_co_getlength(BlockDriverState *bs)
1484 {
1485     BDRVQEDState *s = bs->opaque;
1486     return s->header.image_size;
1487 }
1488 
1489 static int coroutine_fn
1490 bdrv_qed_co_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
1491 {
1492     BDRVQEDState *s = bs->opaque;
1493 
1494     memset(bdi, 0, sizeof(*bdi));
1495     bdi->cluster_size = s->header.cluster_size;
1496     bdi->is_dirty = s->header.features & QED_F_NEED_CHECK;
1497     return 0;
1498 }
1499 
1500 static int bdrv_qed_change_backing_file(BlockDriverState *bs,
1501                                         const char *backing_file,
1502                                         const char *backing_fmt)
1503 {
1504     BDRVQEDState *s = bs->opaque;
1505     QEDHeader new_header, le_header;
1506     void *buffer;
1507     size_t buffer_len, backing_file_len;
1508     int ret;
1509 
1510     /* Refuse to set backing filename if unknown compat feature bits are
1511      * active.  If the image uses an unknown compat feature then we may not
1512      * know the layout of data following the header structure and cannot safely
1513      * add a new string.
1514      */
1515     if (backing_file && (s->header.compat_features &
1516                          ~QED_COMPAT_FEATURE_MASK)) {
1517         return -ENOTSUP;
1518     }
1519 
1520     memcpy(&new_header, &s->header, sizeof(new_header));
1521 
1522     new_header.features &= ~(QED_F_BACKING_FILE |
1523                              QED_F_BACKING_FORMAT_NO_PROBE);
1524 
1525     /* Adjust feature flags */
1526     if (backing_file) {
1527         new_header.features |= QED_F_BACKING_FILE;
1528 
1529         if (qed_fmt_is_raw(backing_fmt)) {
1530             new_header.features |= QED_F_BACKING_FORMAT_NO_PROBE;
1531         }
1532     }
1533 
1534     /* Calculate new header size */
1535     backing_file_len = 0;
1536 
1537     if (backing_file) {
1538         backing_file_len = strlen(backing_file);
1539     }
1540 
1541     buffer_len = sizeof(new_header);
1542     new_header.backing_filename_offset = buffer_len;
1543     new_header.backing_filename_size = backing_file_len;
1544     buffer_len += backing_file_len;
1545 
1546     /* Make sure we can rewrite header without failing */
1547     if (buffer_len > new_header.header_size * new_header.cluster_size) {
1548         return -ENOSPC;
1549     }
1550 
1551     /* Prepare new header */
1552     buffer = g_malloc(buffer_len);
1553 
1554     qed_header_cpu_to_le(&new_header, &le_header);
1555     memcpy(buffer, &le_header, sizeof(le_header));
1556     buffer_len = sizeof(le_header);
1557 
1558     if (backing_file) {
1559         memcpy(buffer + buffer_len, backing_file, backing_file_len);
1560         buffer_len += backing_file_len;
1561     }
1562 
1563     /* Write new header */
1564     ret = bdrv_pwrite_sync(bs->file, 0, buffer_len, buffer, 0);
1565     g_free(buffer);
1566     if (ret == 0) {
1567         memcpy(&s->header, &new_header, sizeof(new_header));
1568     }
1569     return ret;
1570 }
1571 
1572 static void coroutine_fn GRAPH_RDLOCK
1573 bdrv_qed_co_invalidate_cache(BlockDriverState *bs, Error **errp)
1574 {
1575     BDRVQEDState *s = bs->opaque;
1576     int ret;
1577 
1578     bdrv_qed_close(bs);
1579 
1580     bdrv_qed_init_state(bs);
1581     qemu_co_mutex_lock(&s->table_lock);
1582     ret = bdrv_qed_do_open(bs, NULL, bs->open_flags, errp);
1583     qemu_co_mutex_unlock(&s->table_lock);
1584     if (ret < 0) {
1585         error_prepend(errp, "Could not reopen qed layer: ");
1586     }
1587 }
1588 
1589 static int coroutine_fn GRAPH_RDLOCK
1590 bdrv_qed_co_check(BlockDriverState *bs, BdrvCheckResult *result,
1591                   BdrvCheckMode fix)
1592 {
1593     BDRVQEDState *s = bs->opaque;
1594     int ret;
1595 
1596     qemu_co_mutex_lock(&s->table_lock);
1597     ret = qed_check(s, result, !!fix);
1598     qemu_co_mutex_unlock(&s->table_lock);
1599 
1600     return ret;
1601 }
1602 
1603 static QemuOptsList qed_create_opts = {
1604     .name = "qed-create-opts",
1605     .head = QTAILQ_HEAD_INITIALIZER(qed_create_opts.head),
1606     .desc = {
1607         {
1608             .name = BLOCK_OPT_SIZE,
1609             .type = QEMU_OPT_SIZE,
1610             .help = "Virtual disk size"
1611         },
1612         {
1613             .name = BLOCK_OPT_BACKING_FILE,
1614             .type = QEMU_OPT_STRING,
1615             .help = "File name of a base image"
1616         },
1617         {
1618             .name = BLOCK_OPT_BACKING_FMT,
1619             .type = QEMU_OPT_STRING,
1620             .help = "Image format of the base image"
1621         },
1622         {
1623             .name = BLOCK_OPT_CLUSTER_SIZE,
1624             .type = QEMU_OPT_SIZE,
1625             .help = "Cluster size (in bytes)",
1626             .def_value_str = stringify(QED_DEFAULT_CLUSTER_SIZE)
1627         },
1628         {
1629             .name = BLOCK_OPT_TABLE_SIZE,
1630             .type = QEMU_OPT_SIZE,
1631             .help = "L1/L2 table size (in clusters)"
1632         },
1633         { /* end of list */ }
1634     }
1635 };
1636 
1637 static BlockDriver bdrv_qed = {
1638     .format_name              = "qed",
1639     .instance_size            = sizeof(BDRVQEDState),
1640     .create_opts              = &qed_create_opts,
1641     .is_format                = true,
1642     .supports_backing         = true,
1643 
1644     .bdrv_probe               = bdrv_qed_probe,
1645     .bdrv_open                = bdrv_qed_open,
1646     .bdrv_close               = bdrv_qed_close,
1647     .bdrv_reopen_prepare      = bdrv_qed_reopen_prepare,
1648     .bdrv_child_perm          = bdrv_default_perms,
1649     .bdrv_co_create           = bdrv_qed_co_create,
1650     .bdrv_co_create_opts      = bdrv_qed_co_create_opts,
1651     .bdrv_has_zero_init       = bdrv_has_zero_init_1,
1652     .bdrv_co_block_status     = bdrv_qed_co_block_status,
1653     .bdrv_co_readv            = bdrv_qed_co_readv,
1654     .bdrv_co_writev           = bdrv_qed_co_writev,
1655     .bdrv_co_pwrite_zeroes    = bdrv_qed_co_pwrite_zeroes,
1656     .bdrv_co_truncate         = bdrv_qed_co_truncate,
1657     .bdrv_co_getlength        = bdrv_qed_co_getlength,
1658     .bdrv_co_get_info         = bdrv_qed_co_get_info,
1659     .bdrv_refresh_limits      = bdrv_qed_refresh_limits,
1660     .bdrv_change_backing_file = bdrv_qed_change_backing_file,
1661     .bdrv_co_invalidate_cache = bdrv_qed_co_invalidate_cache,
1662     .bdrv_co_check            = bdrv_qed_co_check,
1663     .bdrv_detach_aio_context  = bdrv_qed_detach_aio_context,
1664     .bdrv_attach_aio_context  = bdrv_qed_attach_aio_context,
1665     .bdrv_drain_begin         = bdrv_qed_drain_begin,
1666 };
1667 
1668 static void bdrv_qed_init(void)
1669 {
1670     bdrv_register(&bdrv_qed);
1671 }
1672 
1673 block_init(bdrv_qed_init);
1674