1 /*
2 * copy-before-write filter driver
3 *
4 * The driver performs Copy-Before-Write (CBW) operation: it is injected above
5 * some node, and before each write it copies _old_ data to the target node.
6 *
7 * Copyright (c) 2018-2021 Virtuozzo International GmbH.
8 *
9 * Author:
10 * Sementsov-Ogievskiy Vladimir <vsementsov@virtuozzo.com>
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program. If not, see <http://www.gnu.org/licenses/>.
24 */
25
26 #include "qemu/osdep.h"
27 #include "qapi/qmp/qjson.h"
28
29 #include "sysemu/block-backend.h"
30 #include "qemu/cutils.h"
31 #include "qapi/error.h"
32 #include "block/block_int.h"
33 #include "block/qdict.h"
34 #include "block/block-copy.h"
35 #include "block/dirty-bitmap.h"
36
37 #include "block/copy-before-write.h"
38 #include "block/reqlist.h"
39
40 #include "qapi/qapi-visit-block-core.h"
41
42 typedef struct BDRVCopyBeforeWriteState {
43 BlockCopyState *bcs;
44 BdrvChild *target;
45 OnCbwError on_cbw_error;
46 uint64_t cbw_timeout_ns;
47 bool discard_source;
48
49 /*
50 * @lock: protects access to @access_bitmap, @done_bitmap and
51 * @frozen_read_reqs
52 */
53 CoMutex lock;
54
55 /*
56 * @access_bitmap: represents areas allowed for reading by fleecing user.
57 * Reading from non-dirty areas leads to -EACCES.
58 */
59 BdrvDirtyBitmap *access_bitmap;
60
61 /*
62 * @done_bitmap: represents areas that was successfully copied to @target by
63 * copy-before-write operations.
64 */
65 BdrvDirtyBitmap *done_bitmap;
66
67 /*
68 * @frozen_read_reqs: current read requests for fleecing user in bs->file
69 * node. These areas must not be rewritten by guest.
70 */
71 BlockReqList frozen_read_reqs;
72
73 /*
74 * @snapshot_error is normally zero. But on first copy-before-write failure
75 * when @on_cbw_error == ON_CBW_ERROR_BREAK_SNAPSHOT, @snapshot_error takes
76 * value of this error (<0). After that all in-flight and further
77 * snapshot-API requests will fail with that error.
78 */
79 int snapshot_error;
80 } BDRVCopyBeforeWriteState;
81
82 static int coroutine_fn GRAPH_RDLOCK
cbw_co_preadv(BlockDriverState * bs,int64_t offset,int64_t bytes,QEMUIOVector * qiov,BdrvRequestFlags flags)83 cbw_co_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes,
84 QEMUIOVector *qiov, BdrvRequestFlags flags)
85 {
86 return bdrv_co_preadv(bs->file, offset, bytes, qiov, flags);
87 }
88
block_copy_cb(void * opaque)89 static void block_copy_cb(void *opaque)
90 {
91 BlockDriverState *bs = opaque;
92
93 bdrv_dec_in_flight(bs);
94 }
95
96 /*
97 * Do copy-before-write operation.
98 *
99 * On failure guest request must be failed too.
100 *
101 * On success, we also wait for all in-flight fleecing read requests in source
102 * node, and it's guaranteed that after cbw_do_copy_before_write() successful
103 * return there are no such requests and they will never appear.
104 */
cbw_do_copy_before_write(BlockDriverState * bs,uint64_t offset,uint64_t bytes,BdrvRequestFlags flags)105 static coroutine_fn int cbw_do_copy_before_write(BlockDriverState *bs,
106 uint64_t offset, uint64_t bytes, BdrvRequestFlags flags)
107 {
108 BDRVCopyBeforeWriteState *s = bs->opaque;
109 int ret;
110 uint64_t off, end;
111 int64_t cluster_size = block_copy_cluster_size(s->bcs);
112
113 if (flags & BDRV_REQ_WRITE_UNCHANGED) {
114 return 0;
115 }
116
117 if (s->snapshot_error) {
118 return 0;
119 }
120
121 off = QEMU_ALIGN_DOWN(offset, cluster_size);
122 end = QEMU_ALIGN_UP(offset + bytes, cluster_size);
123
124 /*
125 * Increase in_flight, so that in case of timed-out block-copy, the
126 * remaining background block_copy() request (which can't be immediately
127 * cancelled by timeout) is presented in bs->in_flight. This way we are
128 * sure that on bs close() we'll previously wait for all timed-out but yet
129 * running block_copy calls.
130 */
131 bdrv_inc_in_flight(bs);
132 ret = block_copy(s->bcs, off, end - off, true, s->cbw_timeout_ns,
133 block_copy_cb, bs);
134 if (ret < 0 && s->on_cbw_error == ON_CBW_ERROR_BREAK_GUEST_WRITE) {
135 return ret;
136 }
137
138 WITH_QEMU_LOCK_GUARD(&s->lock) {
139 if (ret < 0) {
140 assert(s->on_cbw_error == ON_CBW_ERROR_BREAK_SNAPSHOT);
141 if (!s->snapshot_error) {
142 s->snapshot_error = ret;
143 }
144 } else {
145 bdrv_set_dirty_bitmap(s->done_bitmap, off, end - off);
146 }
147 reqlist_wait_all(&s->frozen_read_reqs, off, end - off, &s->lock);
148 }
149
150 return 0;
151 }
152
153 static int coroutine_fn GRAPH_RDLOCK
cbw_co_pdiscard(BlockDriverState * bs,int64_t offset,int64_t bytes)154 cbw_co_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes)
155 {
156 int ret = cbw_do_copy_before_write(bs, offset, bytes, 0);
157 if (ret < 0) {
158 return ret;
159 }
160
161 return bdrv_co_pdiscard(bs->file, offset, bytes);
162 }
163
164 static int coroutine_fn GRAPH_RDLOCK
cbw_co_pwrite_zeroes(BlockDriverState * bs,int64_t offset,int64_t bytes,BdrvRequestFlags flags)165 cbw_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int64_t bytes,
166 BdrvRequestFlags flags)
167 {
168 int ret = cbw_do_copy_before_write(bs, offset, bytes, flags);
169 if (ret < 0) {
170 return ret;
171 }
172
173 return bdrv_co_pwrite_zeroes(bs->file, offset, bytes, flags);
174 }
175
176 static coroutine_fn GRAPH_RDLOCK
cbw_co_pwritev(BlockDriverState * bs,int64_t offset,int64_t bytes,QEMUIOVector * qiov,BdrvRequestFlags flags)177 int cbw_co_pwritev(BlockDriverState *bs, int64_t offset, int64_t bytes,
178 QEMUIOVector *qiov, BdrvRequestFlags flags)
179 {
180 int ret = cbw_do_copy_before_write(bs, offset, bytes, flags);
181 if (ret < 0) {
182 return ret;
183 }
184
185 return bdrv_co_pwritev(bs->file, offset, bytes, qiov, flags);
186 }
187
cbw_co_flush(BlockDriverState * bs)188 static int coroutine_fn GRAPH_RDLOCK cbw_co_flush(BlockDriverState *bs)
189 {
190 if (!bs->file) {
191 return 0;
192 }
193
194 return bdrv_co_flush(bs->file->bs);
195 }
196
197 /*
198 * If @offset not accessible - return NULL.
199 *
200 * Otherwise, set @pnum to some bytes that accessible from @file (@file is set
201 * to bs->file or to s->target). Return newly allocated BlockReq object that
202 * should be than passed to cbw_snapshot_read_unlock().
203 *
204 * It's guaranteed that guest writes will not interact in the region until
205 * cbw_snapshot_read_unlock() called.
206 */
207 static BlockReq * coroutine_fn GRAPH_RDLOCK
cbw_snapshot_read_lock(BlockDriverState * bs,int64_t offset,int64_t bytes,int64_t * pnum,BdrvChild ** file)208 cbw_snapshot_read_lock(BlockDriverState *bs, int64_t offset, int64_t bytes,
209 int64_t *pnum, BdrvChild **file)
210 {
211 BDRVCopyBeforeWriteState *s = bs->opaque;
212 BlockReq *req = g_new(BlockReq, 1);
213 bool done;
214
215 QEMU_LOCK_GUARD(&s->lock);
216
217 if (s->snapshot_error) {
218 g_free(req);
219 return NULL;
220 }
221
222 if (bdrv_dirty_bitmap_next_zero(s->access_bitmap, offset, bytes) != -1) {
223 g_free(req);
224 return NULL;
225 }
226
227 done = bdrv_dirty_bitmap_status(s->done_bitmap, offset, bytes, pnum);
228 if (done) {
229 /*
230 * Special invalid BlockReq, that is handled in
231 * cbw_snapshot_read_unlock(). We don't need to lock something to read
232 * from s->target.
233 */
234 *req = (BlockReq) {.offset = -1, .bytes = -1};
235 *file = s->target;
236 } else {
237 reqlist_init_req(&s->frozen_read_reqs, req, offset, bytes);
238 *file = bs->file;
239 }
240
241 return req;
242 }
243
244 static coroutine_fn void
cbw_snapshot_read_unlock(BlockDriverState * bs,BlockReq * req)245 cbw_snapshot_read_unlock(BlockDriverState *bs, BlockReq *req)
246 {
247 BDRVCopyBeforeWriteState *s = bs->opaque;
248
249 if (req->offset == -1 && req->bytes == -1) {
250 g_free(req);
251 return;
252 }
253
254 QEMU_LOCK_GUARD(&s->lock);
255
256 reqlist_remove_req(req);
257 g_free(req);
258 }
259
260 static int coroutine_fn GRAPH_RDLOCK
cbw_co_preadv_snapshot(BlockDriverState * bs,int64_t offset,int64_t bytes,QEMUIOVector * qiov,size_t qiov_offset)261 cbw_co_preadv_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes,
262 QEMUIOVector *qiov, size_t qiov_offset)
263 {
264 BlockReq *req;
265 BdrvChild *file;
266 int ret;
267
268 /* TODO: upgrade to async loop using AioTask */
269 while (bytes) {
270 int64_t cur_bytes;
271
272 req = cbw_snapshot_read_lock(bs, offset, bytes, &cur_bytes, &file);
273 if (!req) {
274 return -EACCES;
275 }
276
277 ret = bdrv_co_preadv_part(file, offset, cur_bytes,
278 qiov, qiov_offset, 0);
279 cbw_snapshot_read_unlock(bs, req);
280 if (ret < 0) {
281 return ret;
282 }
283
284 bytes -= cur_bytes;
285 offset += cur_bytes;
286 qiov_offset += cur_bytes;
287 }
288
289 return 0;
290 }
291
292 static int coroutine_fn GRAPH_RDLOCK
cbw_co_snapshot_block_status(BlockDriverState * bs,bool want_zero,int64_t offset,int64_t bytes,int64_t * pnum,int64_t * map,BlockDriverState ** file)293 cbw_co_snapshot_block_status(BlockDriverState *bs,
294 bool want_zero, int64_t offset, int64_t bytes,
295 int64_t *pnum, int64_t *map,
296 BlockDriverState **file)
297 {
298 BDRVCopyBeforeWriteState *s = bs->opaque;
299 BlockReq *req;
300 int ret;
301 int64_t cur_bytes;
302 BdrvChild *child;
303
304 req = cbw_snapshot_read_lock(bs, offset, bytes, &cur_bytes, &child);
305 if (!req) {
306 return -EACCES;
307 }
308
309 ret = bdrv_co_block_status(child->bs, offset, cur_bytes, pnum, map, file);
310 if (child == s->target) {
311 /*
312 * We refer to s->target only for areas that we've written to it.
313 * And we can not report unallocated blocks in s->target: this will
314 * break generic block-status-above logic, that will go to
315 * copy-before-write filtered child in this case.
316 */
317 assert(ret & BDRV_BLOCK_ALLOCATED);
318 }
319
320 cbw_snapshot_read_unlock(bs, req);
321
322 return ret;
323 }
324
325 static int coroutine_fn GRAPH_RDLOCK
cbw_co_pdiscard_snapshot(BlockDriverState * bs,int64_t offset,int64_t bytes)326 cbw_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes)
327 {
328 BDRVCopyBeforeWriteState *s = bs->opaque;
329 uint32_t cluster_size = block_copy_cluster_size(s->bcs);
330 int64_t aligned_offset = QEMU_ALIGN_UP(offset, cluster_size);
331 int64_t aligned_end = QEMU_ALIGN_DOWN(offset + bytes, cluster_size);
332 int64_t aligned_bytes;
333
334 if (aligned_end <= aligned_offset) {
335 return 0;
336 }
337 aligned_bytes = aligned_end - aligned_offset;
338
339 WITH_QEMU_LOCK_GUARD(&s->lock) {
340 bdrv_reset_dirty_bitmap(s->access_bitmap, aligned_offset,
341 aligned_bytes);
342 }
343
344 block_copy_reset(s->bcs, aligned_offset, aligned_bytes);
345
346 return bdrv_co_pdiscard(s->target, aligned_offset, aligned_bytes);
347 }
348
cbw_refresh_filename(BlockDriverState * bs)349 static void GRAPH_RDLOCK cbw_refresh_filename(BlockDriverState *bs)
350 {
351 pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
352 bs->file->bs->filename);
353 }
354
355 static void GRAPH_RDLOCK
cbw_child_perm(BlockDriverState * bs,BdrvChild * c,BdrvChildRole role,BlockReopenQueue * reopen_queue,uint64_t perm,uint64_t shared,uint64_t * nperm,uint64_t * nshared)356 cbw_child_perm(BlockDriverState *bs, BdrvChild *c, BdrvChildRole role,
357 BlockReopenQueue *reopen_queue,
358 uint64_t perm, uint64_t shared,
359 uint64_t *nperm, uint64_t *nshared)
360 {
361 BDRVCopyBeforeWriteState *s = bs->opaque;
362
363 if (!(role & BDRV_CHILD_FILTERED)) {
364 /*
365 * Target child
366 *
367 * Share write to target (child_file), to not interfere
368 * with guest writes to its disk which may be in target backing chain.
369 * Can't resize during a backup block job because we check the size
370 * only upfront.
371 */
372 *nshared = BLK_PERM_ALL & ~BLK_PERM_RESIZE;
373 *nperm = BLK_PERM_WRITE;
374 } else {
375 /* Source child */
376 bdrv_default_perms(bs, c, role, reopen_queue,
377 perm, shared, nperm, nshared);
378
379 if (!QLIST_EMPTY(&bs->parents)) {
380 /*
381 * Note, that source child may be shared with backup job. Backup job
382 * does create own blk parent on copy-before-write node, so this
383 * works even if source node does not have any parents before backup
384 * start
385 */
386 *nperm = *nperm | BLK_PERM_CONSISTENT_READ;
387 if (s->discard_source) {
388 *nperm = *nperm | BLK_PERM_WRITE;
389 }
390
391 *nshared &= ~(BLK_PERM_WRITE | BLK_PERM_RESIZE);
392 }
393 }
394 }
395
cbw_parse_options(QDict * options,Error ** errp)396 static BlockdevOptions *cbw_parse_options(QDict *options, Error **errp)
397 {
398 BlockdevOptions *opts = NULL;
399 Visitor *v = NULL;
400
401 qdict_put_str(options, "driver", "copy-before-write");
402
403 v = qobject_input_visitor_new_flat_confused(options, errp);
404 if (!v) {
405 goto out;
406 }
407
408 visit_type_BlockdevOptions(v, NULL, &opts, errp);
409 if (!opts) {
410 goto out;
411 }
412
413 /*
414 * Delete options which we are going to parse through BlockdevOptions
415 * object for original options.
416 */
417 qdict_extract_subqdict(options, NULL, "bitmap");
418 qdict_del(options, "on-cbw-error");
419 qdict_del(options, "cbw-timeout");
420
421 out:
422 visit_free(v);
423 qdict_del(options, "driver");
424
425 return opts;
426 }
427
cbw_open(BlockDriverState * bs,QDict * options,int flags,Error ** errp)428 static int cbw_open(BlockDriverState *bs, QDict *options, int flags,
429 Error **errp)
430 {
431 ERRP_GUARD();
432 BDRVCopyBeforeWriteState *s = bs->opaque;
433 BdrvDirtyBitmap *bitmap = NULL;
434 int64_t cluster_size;
435 g_autoptr(BlockdevOptions) full_opts = NULL;
436 BlockdevOptionsCbw *opts;
437 int ret;
438
439 full_opts = cbw_parse_options(options, errp);
440 if (!full_opts) {
441 return -EINVAL;
442 }
443 assert(full_opts->driver == BLOCKDEV_DRIVER_COPY_BEFORE_WRITE);
444 opts = &full_opts->u.copy_before_write;
445
446 ret = bdrv_open_file_child(NULL, options, "file", bs, errp);
447 if (ret < 0) {
448 return ret;
449 }
450
451 s->target = bdrv_open_child(NULL, options, "target", bs, &child_of_bds,
452 BDRV_CHILD_DATA, false, errp);
453 if (!s->target) {
454 return -EINVAL;
455 }
456
457 GRAPH_RDLOCK_GUARD_MAINLOOP();
458
459 if (opts->bitmap) {
460 bitmap = block_dirty_bitmap_lookup(opts->bitmap->node,
461 opts->bitmap->name, NULL, errp);
462 if (!bitmap) {
463 return -EINVAL;
464 }
465 }
466 s->on_cbw_error = opts->has_on_cbw_error ? opts->on_cbw_error :
467 ON_CBW_ERROR_BREAK_GUEST_WRITE;
468 s->cbw_timeout_ns = opts->has_cbw_timeout ?
469 opts->cbw_timeout * NANOSECONDS_PER_SECOND : 0;
470
471 bs->total_sectors = bs->file->bs->total_sectors;
472 bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED |
473 (BDRV_REQ_FUA & bs->file->bs->supported_write_flags);
474 bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
475 ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK) &
476 bs->file->bs->supported_zero_flags);
477
478 s->discard_source = flags & BDRV_O_CBW_DISCARD_SOURCE;
479 s->bcs = block_copy_state_new(bs->file, s->target, bs, bitmap,
480 flags & BDRV_O_CBW_DISCARD_SOURCE, errp);
481 if (!s->bcs) {
482 error_prepend(errp, "Cannot create block-copy-state: ");
483 return -EINVAL;
484 }
485
486 cluster_size = block_copy_cluster_size(s->bcs);
487
488 s->done_bitmap = bdrv_create_dirty_bitmap(bs, cluster_size, NULL, errp);
489 if (!s->done_bitmap) {
490 return -EINVAL;
491 }
492 bdrv_disable_dirty_bitmap(s->done_bitmap);
493
494 /* s->access_bitmap starts equal to bcs bitmap */
495 s->access_bitmap = bdrv_create_dirty_bitmap(bs, cluster_size, NULL, errp);
496 if (!s->access_bitmap) {
497 return -EINVAL;
498 }
499 bdrv_disable_dirty_bitmap(s->access_bitmap);
500 bdrv_dirty_bitmap_merge_internal(s->access_bitmap,
501 block_copy_dirty_bitmap(s->bcs), NULL,
502 true);
503
504 qemu_co_mutex_init(&s->lock);
505 QLIST_INIT(&s->frozen_read_reqs);
506 return 0;
507 }
508
cbw_close(BlockDriverState * bs)509 static void cbw_close(BlockDriverState *bs)
510 {
511 BDRVCopyBeforeWriteState *s = bs->opaque;
512
513 bdrv_release_dirty_bitmap(s->access_bitmap);
514 bdrv_release_dirty_bitmap(s->done_bitmap);
515
516 block_copy_state_free(s->bcs);
517 s->bcs = NULL;
518 }
519
520 static BlockDriver bdrv_cbw_filter = {
521 .format_name = "copy-before-write",
522 .instance_size = sizeof(BDRVCopyBeforeWriteState),
523
524 .bdrv_open = cbw_open,
525 .bdrv_close = cbw_close,
526
527 .bdrv_co_preadv = cbw_co_preadv,
528 .bdrv_co_pwritev = cbw_co_pwritev,
529 .bdrv_co_pwrite_zeroes = cbw_co_pwrite_zeroes,
530 .bdrv_co_pdiscard = cbw_co_pdiscard,
531 .bdrv_co_flush = cbw_co_flush,
532
533 .bdrv_co_preadv_snapshot = cbw_co_preadv_snapshot,
534 .bdrv_co_pdiscard_snapshot = cbw_co_pdiscard_snapshot,
535 .bdrv_co_snapshot_block_status = cbw_co_snapshot_block_status,
536
537 .bdrv_refresh_filename = cbw_refresh_filename,
538
539 .bdrv_child_perm = cbw_child_perm,
540
541 .is_filter = true,
542 };
543
bdrv_cbw_append(BlockDriverState * source,BlockDriverState * target,const char * filter_node_name,bool discard_source,BlockCopyState ** bcs,Error ** errp)544 BlockDriverState *bdrv_cbw_append(BlockDriverState *source,
545 BlockDriverState *target,
546 const char *filter_node_name,
547 bool discard_source,
548 BlockCopyState **bcs,
549 Error **errp)
550 {
551 BDRVCopyBeforeWriteState *state;
552 BlockDriverState *top;
553 QDict *opts;
554 int flags = BDRV_O_RDWR | (discard_source ? BDRV_O_CBW_DISCARD_SOURCE : 0);
555
556 assert(source->total_sectors == target->total_sectors);
557 GLOBAL_STATE_CODE();
558
559 opts = qdict_new();
560 qdict_put_str(opts, "driver", "copy-before-write");
561 if (filter_node_name) {
562 qdict_put_str(opts, "node-name", filter_node_name);
563 }
564 qdict_put_str(opts, "file", bdrv_get_node_name(source));
565 qdict_put_str(opts, "target", bdrv_get_node_name(target));
566
567 top = bdrv_insert_node(source, opts, flags, errp);
568 if (!top) {
569 return NULL;
570 }
571
572 state = top->opaque;
573 *bcs = state->bcs;
574
575 return top;
576 }
577
bdrv_cbw_drop(BlockDriverState * bs)578 void bdrv_cbw_drop(BlockDriverState *bs)
579 {
580 GLOBAL_STATE_CODE();
581 bdrv_drop_filter(bs, &error_abort);
582 bdrv_unref(bs);
583 }
584
cbw_init(void)585 static void cbw_init(void)
586 {
587 bdrv_register(&bdrv_cbw_filter);
588 }
589
590 block_init(cbw_init);
591