1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright 2022 Red Hat, Inc. 4 */ 5 6 #include <linux/bio.h> 7 #include <linux/blk-crypto.h> 8 #include <linux/blk-integrity.h> 9 10 #include "dm-core.h" 11 12 static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv, 13 struct bvec_iter *iter, 14 unsigned int bytes) 15 { 16 int idx; 17 18 iter->bi_size += bytes; 19 if (bytes <= iter->bi_bvec_done) { 20 iter->bi_bvec_done -= bytes; 21 return true; 22 } 23 24 bytes -= iter->bi_bvec_done; 25 idx = iter->bi_idx - 1; 26 27 while (idx >= 0 && bytes && bytes > bv[idx].bv_len) { 28 bytes -= bv[idx].bv_len; 29 idx--; 30 } 31 32 if (WARN_ONCE(idx < 0 && bytes, 33 "Attempted to rewind iter beyond bvec's boundaries\n")) { 34 iter->bi_size -= bytes; 35 iter->bi_bvec_done = 0; 36 iter->bi_idx = 0; 37 return false; 38 } 39 40 iter->bi_idx = idx; 41 iter->bi_bvec_done = bv[idx].bv_len - bytes; 42 return true; 43 } 44 45 #if defined(CONFIG_BLK_DEV_INTEGRITY) 46 47 /** 48 * dm_bio_integrity_rewind - Rewind integrity vector 49 * @bio: bio whose integrity vector to update 50 * @bytes_done: number of data bytes to rewind 51 * 52 * Description: This function calculates how many integrity bytes the 53 * number of completed data bytes correspond to and rewind the 54 * integrity vector accordingly. 55 */ 56 static void dm_bio_integrity_rewind(struct bio *bio, unsigned int bytes_done) 57 { 58 struct bio_integrity_payload *bip = bio_integrity(bio); 59 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 60 unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); 61 62 bip->bip_iter.bi_sector -= bio_integrity_intervals(bi, bytes_done >> 9); 63 dm_bvec_iter_rewind(bip->bip_vec, &bip->bip_iter, bytes); 64 } 65 66 #else /* CONFIG_BLK_DEV_INTEGRITY */ 67 68 static inline void dm_bio_integrity_rewind(struct bio *bio, 69 unsigned int bytes_done) 70 { 71 return; 72 } 73 74 #endif 75 76 #if defined(CONFIG_BLK_INLINE_ENCRYPTION) 77 78 /* Decrements @dun by @dec, treating @dun as a multi-limb integer. */ 79 static void dm_bio_crypt_dun_decrement(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], 80 unsigned int dec) 81 { 82 int i; 83 84 for (i = 0; dec && i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) { 85 u64 prev = dun[i]; 86 87 dun[i] -= dec; 88 if (dun[i] > prev) 89 dec = 1; 90 else 91 dec = 0; 92 } 93 } 94 95 static void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes) 96 { 97 struct bio_crypt_ctx *bc = bio->bi_crypt_context; 98 99 dm_bio_crypt_dun_decrement(bc->bc_dun, 100 bytes >> bc->bc_key->data_unit_size_bits); 101 } 102 103 #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 104 105 static inline void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes) 106 { 107 return; 108 } 109 110 #endif 111 112 static inline void dm_bio_rewind_iter(const struct bio *bio, 113 struct bvec_iter *iter, unsigned int bytes) 114 { 115 iter->bi_sector -= bytes >> 9; 116 117 /* No advance means no rewind */ 118 if (bio_no_advance_iter(bio)) 119 iter->bi_size += bytes; 120 else 121 dm_bvec_iter_rewind(bio->bi_io_vec, iter, bytes); 122 } 123 124 /** 125 * dm_bio_rewind - update ->bi_iter of @bio by rewinding @bytes. 126 * @bio: bio to rewind 127 * @bytes: how many bytes to rewind 128 * 129 * WARNING: 130 * Caller must ensure that @bio has a fixed end sector, to allow 131 * rewinding from end of bio and restoring its original position. 132 * Caller is also responsibile for restoring bio's size. 133 */ 134 static void dm_bio_rewind(struct bio *bio, unsigned bytes) 135 { 136 if (bio_integrity(bio)) 137 dm_bio_integrity_rewind(bio, bytes); 138 139 if (bio_has_crypt_ctx(bio)) 140 dm_bio_crypt_rewind(bio, bytes); 141 142 dm_bio_rewind_iter(bio, &bio->bi_iter, bytes); 143 } 144 145 void dm_io_rewind(struct dm_io *io, struct bio_set *bs) 146 { 147 struct bio *orig = io->orig_bio; 148 struct bio *new_orig = bio_alloc_clone(orig->bi_bdev, orig, 149 GFP_NOIO, bs); 150 /* 151 * dm_bio_rewind can restore to previous position since the 152 * end sector is fixed for original bio, but we still need 153 * to restore bio's size manually (using io->sectors). 154 */ 155 dm_bio_rewind(new_orig, ((io->sector_offset << 9) - 156 orig->bi_iter.bi_size)); 157 bio_trim(new_orig, 0, io->sectors); 158 159 bio_chain(new_orig, orig); 160 /* 161 * __bi_remaining was increased (by dm_split_and_process_bio), 162 * so must drop the one added in bio_chain. 163 */ 164 atomic_dec(&orig->__bi_remaining); 165 io->orig_bio = new_orig; 166 } 167