1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 36cbd5570SChris Mason * Copyright (C) 2007 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6065631f6SChris Mason #include <linux/bio.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8065631f6SChris Mason #include <linux/pagemap.h> 9065631f6SChris Mason #include <linux/highmem.h> 10a3d46aeaSNikolay Borisov #include <linux/sched/mm.h> 11d5178578SJohannes Thumshirn #include <crypto/hash.h> 129b569ea0SJosef Bacik #include "messages.h" 13cea62800SJohannes Thumshirn #include "misc.h" 141e1d2701SChris Mason #include "ctree.h" 15dee26a9fSChris Mason #include "disk-io.h" 169f5fae2fSChris Mason #include "transaction.h" 17*103c1972SChristoph Hellwig #include "bio.h" 181de037a4SChris Mason #include "print-tree.h" 19ebb8765bSAnand Jain #include "compression.h" 20c7f13d42SJosef Bacik #include "fs.h" 2107e81dc9SJosef Bacik #include "accessors.h" 227c8ede16SJosef Bacik #include "file-item.h" 237f0add25SJosef Bacik #include "super.h" 241e1d2701SChris Mason 2542049bf6SChris Mason #define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \ 2642049bf6SChris Mason sizeof(struct btrfs_item) * 2) / \ 2742049bf6SChris Mason size) - 1)) 2807d400a6SYan Zheng 29221b8318SZach Brown #define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \ 3009cbfeafSKirill A. Shutemov PAGE_SIZE)) 317ca4be45SChris Mason 3243dd529aSDavid Sterba /* 3343dd529aSDavid Sterba * Set inode's size according to filesystem options. 34ca4207aeSNikolay Borisov * 35ca4207aeSNikolay Borisov * @inode: inode we want to update the disk_i_size for 36ca4207aeSNikolay Borisov * @new_i_size: i_size we want to set to, 0 if we use i_size 3741a2ee75SJosef Bacik * 3841a2ee75SJosef Bacik * With NO_HOLES set this simply sets the disk_is_size to whatever i_size_read() 3941a2ee75SJosef Bacik * returns as it is perfectly fine with a file that has holes without hole file 4041a2ee75SJosef Bacik * extent items. 4141a2ee75SJosef Bacik * 4241a2ee75SJosef Bacik * However without NO_HOLES we need to only return the area that is contiguous 4341a2ee75SJosef Bacik * from the 0 offset of the file. Otherwise we could end up adjust i_size up 4441a2ee75SJosef Bacik * to an extent that has a gap in between. 4541a2ee75SJosef Bacik * 4641a2ee75SJosef Bacik * Finally new_i_size should only be set in the case of truncate where we're not 4741a2ee75SJosef Bacik * ready to use i_size_read() as the limiter yet. 4841a2ee75SJosef Bacik */ 4976aea537SNikolay Borisov void btrfs_inode_safe_disk_i_size_write(struct btrfs_inode *inode, u64 new_i_size) 5041a2ee75SJosef Bacik { 5176aea537SNikolay Borisov struct btrfs_fs_info *fs_info = inode->root->fs_info; 5241a2ee75SJosef Bacik u64 start, end, i_size; 5341a2ee75SJosef Bacik int ret; 5441a2ee75SJosef Bacik 5576aea537SNikolay Borisov i_size = new_i_size ?: i_size_read(&inode->vfs_inode); 5641a2ee75SJosef Bacik if (btrfs_fs_incompat(fs_info, NO_HOLES)) { 5776aea537SNikolay Borisov inode->disk_i_size = i_size; 5841a2ee75SJosef Bacik return; 5941a2ee75SJosef Bacik } 6041a2ee75SJosef Bacik 6176aea537SNikolay Borisov spin_lock(&inode->lock); 6276aea537SNikolay Borisov ret = find_contiguous_extent_bit(&inode->file_extent_tree, 0, &start, 6376aea537SNikolay Borisov &end, EXTENT_DIRTY); 6441a2ee75SJosef Bacik if (!ret && start == 0) 6541a2ee75SJosef Bacik i_size = min(i_size, end + 1); 6641a2ee75SJosef Bacik else 6741a2ee75SJosef Bacik i_size = 0; 6876aea537SNikolay Borisov inode->disk_i_size = i_size; 6976aea537SNikolay Borisov spin_unlock(&inode->lock); 7041a2ee75SJosef Bacik } 7141a2ee75SJosef Bacik 7243dd529aSDavid Sterba /* 7343dd529aSDavid Sterba * Mark range within a file as having a new extent inserted. 74ca4207aeSNikolay Borisov * 75ca4207aeSNikolay Borisov * @inode: inode being modified 76ca4207aeSNikolay Borisov * @start: start file offset of the file extent we've inserted 77ca4207aeSNikolay Borisov * @len: logical length of the file extent item 7841a2ee75SJosef Bacik * 7941a2ee75SJosef Bacik * Call when we are inserting a new file extent where there was none before. 8041a2ee75SJosef Bacik * Does not need to call this in the case where we're replacing an existing file 8141a2ee75SJosef Bacik * extent, however if not sure it's fine to call this multiple times. 8241a2ee75SJosef Bacik * 8341a2ee75SJosef Bacik * The start and len must match the file extent item, so thus must be sectorsize 8441a2ee75SJosef Bacik * aligned. 8541a2ee75SJosef Bacik */ 8641a2ee75SJosef Bacik int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start, 8741a2ee75SJosef Bacik u64 len) 8841a2ee75SJosef Bacik { 8941a2ee75SJosef Bacik if (len == 0) 9041a2ee75SJosef Bacik return 0; 9141a2ee75SJosef Bacik 9241a2ee75SJosef Bacik ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize)); 9341a2ee75SJosef Bacik 9441a2ee75SJosef Bacik if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES)) 9541a2ee75SJosef Bacik return 0; 9641a2ee75SJosef Bacik return set_extent_bits(&inode->file_extent_tree, start, start + len - 1, 9741a2ee75SJosef Bacik EXTENT_DIRTY); 9841a2ee75SJosef Bacik } 9941a2ee75SJosef Bacik 10043dd529aSDavid Sterba /* 10143dd529aSDavid Sterba * Mark an inode range as not having a backing extent. 102ca4207aeSNikolay Borisov * 103ca4207aeSNikolay Borisov * @inode: inode being modified 104ca4207aeSNikolay Borisov * @start: start file offset of the file extent we've inserted 105ca4207aeSNikolay Borisov * @len: logical length of the file extent item 10641a2ee75SJosef Bacik * 10741a2ee75SJosef Bacik * Called when we drop a file extent, for example when we truncate. Doesn't 10841a2ee75SJosef Bacik * need to be called for cases where we're replacing a file extent, like when 10941a2ee75SJosef Bacik * we've COWed a file extent. 11041a2ee75SJosef Bacik * 11141a2ee75SJosef Bacik * The start and len must match the file extent item, so thus must be sectorsize 11241a2ee75SJosef Bacik * aligned. 11341a2ee75SJosef Bacik */ 11441a2ee75SJosef Bacik int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start, 11541a2ee75SJosef Bacik u64 len) 11641a2ee75SJosef Bacik { 11741a2ee75SJosef Bacik if (len == 0) 11841a2ee75SJosef Bacik return 0; 11941a2ee75SJosef Bacik 12041a2ee75SJosef Bacik ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize) || 12141a2ee75SJosef Bacik len == (u64)-1); 12241a2ee75SJosef Bacik 12341a2ee75SJosef Bacik if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES)) 12441a2ee75SJosef Bacik return 0; 12541a2ee75SJosef Bacik return clear_extent_bit(&inode->file_extent_tree, start, 126bd015294SJosef Bacik start + len - 1, EXTENT_DIRTY, NULL); 12741a2ee75SJosef Bacik } 12841a2ee75SJosef Bacik 129cb649e81SQu Wenruo static size_t bytes_to_csum_size(const struct btrfs_fs_info *fs_info, u32 bytes) 1301e25a2e3SJohannes Thumshirn { 131cb649e81SQu Wenruo ASSERT(IS_ALIGNED(bytes, fs_info->sectorsize)); 1321e25a2e3SJohannes Thumshirn 133cb649e81SQu Wenruo return (bytes >> fs_info->sectorsize_bits) * fs_info->csum_size; 134cb649e81SQu Wenruo } 135cb649e81SQu Wenruo 136cb649e81SQu Wenruo static size_t csum_size_to_bytes(const struct btrfs_fs_info *fs_info, u32 csum_size) 137cb649e81SQu Wenruo { 138cb649e81SQu Wenruo ASSERT(IS_ALIGNED(csum_size, fs_info->csum_size)); 139cb649e81SQu Wenruo 140cb649e81SQu Wenruo return (csum_size / fs_info->csum_size) << fs_info->sectorsize_bits; 141cb649e81SQu Wenruo } 142cb649e81SQu Wenruo 143cb649e81SQu Wenruo static inline u32 max_ordered_sum_bytes(const struct btrfs_fs_info *fs_info) 144cb649e81SQu Wenruo { 145cb649e81SQu Wenruo u32 max_csum_size = round_down(PAGE_SIZE - sizeof(struct btrfs_ordered_sum), 146cb649e81SQu Wenruo fs_info->csum_size); 147cb649e81SQu Wenruo 148cb649e81SQu Wenruo return csum_size_to_bytes(fs_info, max_csum_size); 1491e25a2e3SJohannes Thumshirn } 15007d400a6SYan Zheng 1512b6433c7SJosef Bacik /* 1522b6433c7SJosef Bacik * Calculate the total size needed to allocate for an ordered sum structure 1532b6433c7SJosef Bacik * spanning @bytes in the file. 1542b6433c7SJosef Bacik */ 1552b6433c7SJosef Bacik static int btrfs_ordered_sum_size(struct btrfs_fs_info *fs_info, unsigned long bytes) 1562b6433c7SJosef Bacik { 157cb649e81SQu Wenruo return sizeof(struct btrfs_ordered_sum) + bytes_to_csum_size(fs_info, bytes); 1582b6433c7SJosef Bacik } 1592b6433c7SJosef Bacik 160d1f68ba0SOmar Sandoval int btrfs_insert_hole_extent(struct btrfs_trans_handle *trans, 161dee26a9fSChris Mason struct btrfs_root *root, 162d1f68ba0SOmar Sandoval u64 objectid, u64 pos, u64 num_bytes) 1639f5fae2fSChris Mason { 164dee26a9fSChris Mason int ret = 0; 165dee26a9fSChris Mason struct btrfs_file_extent_item *item; 166dee26a9fSChris Mason struct btrfs_key file_key; 1675caf2a00SChris Mason struct btrfs_path *path; 1685f39d397SChris Mason struct extent_buffer *leaf; 169dee26a9fSChris Mason 1705caf2a00SChris Mason path = btrfs_alloc_path(); 171db5b493aSTsutomu Itoh if (!path) 172db5b493aSTsutomu Itoh return -ENOMEM; 173dee26a9fSChris Mason file_key.objectid = objectid; 174b18c6685SChris Mason file_key.offset = pos; 175962a298fSDavid Sterba file_key.type = BTRFS_EXTENT_DATA_KEY; 176dee26a9fSChris Mason 1775caf2a00SChris Mason ret = btrfs_insert_empty_item(trans, root, path, &file_key, 178dee26a9fSChris Mason sizeof(*item)); 17954aa1f4dSChris Mason if (ret < 0) 18054aa1f4dSChris Mason goto out; 18179787eaaSJeff Mahoney BUG_ON(ret); /* Can't happen */ 1825f39d397SChris Mason leaf = path->nodes[0]; 1835f39d397SChris Mason item = btrfs_item_ptr(leaf, path->slots[0], 184dee26a9fSChris Mason struct btrfs_file_extent_item); 185d1f68ba0SOmar Sandoval btrfs_set_file_extent_disk_bytenr(leaf, item, 0); 186d1f68ba0SOmar Sandoval btrfs_set_file_extent_disk_num_bytes(leaf, item, 0); 187d1f68ba0SOmar Sandoval btrfs_set_file_extent_offset(leaf, item, 0); 188db94535dSChris Mason btrfs_set_file_extent_num_bytes(leaf, item, num_bytes); 189d1f68ba0SOmar Sandoval btrfs_set_file_extent_ram_bytes(leaf, item, num_bytes); 1905f39d397SChris Mason btrfs_set_file_extent_generation(leaf, item, trans->transid); 1915f39d397SChris Mason btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); 192d1f68ba0SOmar Sandoval btrfs_set_file_extent_compression(leaf, item, 0); 193d1f68ba0SOmar Sandoval btrfs_set_file_extent_encryption(leaf, item, 0); 194d1f68ba0SOmar Sandoval btrfs_set_file_extent_other_encoding(leaf, item, 0); 195c8b97818SChris Mason 1965f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 19754aa1f4dSChris Mason out: 1985caf2a00SChris Mason btrfs_free_path(path); 19954aa1f4dSChris Mason return ret; 2009f5fae2fSChris Mason } 201dee26a9fSChris Mason 20248a3b636SEric Sandeen static struct btrfs_csum_item * 20348a3b636SEric Sandeen btrfs_lookup_csum(struct btrfs_trans_handle *trans, 204b18c6685SChris Mason struct btrfs_root *root, 2056567e837SChris Mason struct btrfs_path *path, 206d20f7043SChris Mason u64 bytenr, int cow) 2076567e837SChris Mason { 2080b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2096567e837SChris Mason int ret; 2106567e837SChris Mason struct btrfs_key file_key; 2116567e837SChris Mason struct btrfs_key found_key; 2126567e837SChris Mason struct btrfs_csum_item *item; 2135f39d397SChris Mason struct extent_buffer *leaf; 2146567e837SChris Mason u64 csum_offset = 0; 215223486c2SDavid Sterba const u32 csum_size = fs_info->csum_size; 216a429e513SChris Mason int csums_in_item; 2176567e837SChris Mason 218d20f7043SChris Mason file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 219d20f7043SChris Mason file_key.offset = bytenr; 220962a298fSDavid Sterba file_key.type = BTRFS_EXTENT_CSUM_KEY; 221b18c6685SChris Mason ret = btrfs_search_slot(trans, root, &file_key, path, 0, cow); 2226567e837SChris Mason if (ret < 0) 2236567e837SChris Mason goto fail; 2245f39d397SChris Mason leaf = path->nodes[0]; 2256567e837SChris Mason if (ret > 0) { 2266567e837SChris Mason ret = 1; 22770b2befdSChris Mason if (path->slots[0] == 0) 2286567e837SChris Mason goto fail; 2296567e837SChris Mason path->slots[0]--; 2305f39d397SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 231962a298fSDavid Sterba if (found_key.type != BTRFS_EXTENT_CSUM_KEY) 2326567e837SChris Mason goto fail; 233d20f7043SChris Mason 234d20f7043SChris Mason csum_offset = (bytenr - found_key.offset) >> 235265fdfa6SDavid Sterba fs_info->sectorsize_bits; 2363212fa14SJosef Bacik csums_in_item = btrfs_item_size(leaf, path->slots[0]); 237607d432dSJosef Bacik csums_in_item /= csum_size; 238a429e513SChris Mason 23982d130ffSMiao Xie if (csum_offset == csums_in_item) { 240a429e513SChris Mason ret = -EFBIG; 2416567e837SChris Mason goto fail; 24282d130ffSMiao Xie } else if (csum_offset > csums_in_item) { 24382d130ffSMiao Xie goto fail; 2446567e837SChris Mason } 2456567e837SChris Mason } 2466567e837SChris Mason item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item); 247509659cdSChris Mason item = (struct btrfs_csum_item *)((unsigned char *)item + 248607d432dSJosef Bacik csum_offset * csum_size); 2496567e837SChris Mason return item; 2506567e837SChris Mason fail: 2516567e837SChris Mason if (ret > 0) 252b18c6685SChris Mason ret = -ENOENT; 2536567e837SChris Mason return ERR_PTR(ret); 2546567e837SChris Mason } 2556567e837SChris Mason 256dee26a9fSChris Mason int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, 257dee26a9fSChris Mason struct btrfs_root *root, 258dee26a9fSChris Mason struct btrfs_path *path, u64 objectid, 2599773a788SChris Mason u64 offset, int mod) 260dee26a9fSChris Mason { 261dee26a9fSChris Mason struct btrfs_key file_key; 262dee26a9fSChris Mason int ins_len = mod < 0 ? -1 : 0; 263dee26a9fSChris Mason int cow = mod != 0; 264dee26a9fSChris Mason 265dee26a9fSChris Mason file_key.objectid = objectid; 26670b2befdSChris Mason file_key.offset = offset; 267962a298fSDavid Sterba file_key.type = BTRFS_EXTENT_DATA_KEY; 268f8ee80deSMarcos Paulo de Souza 269f8ee80deSMarcos Paulo de Souza return btrfs_search_slot(trans, root, &file_key, path, ins_len, cow); 270dee26a9fSChris Mason } 271f254e52cSChris Mason 2726275193eSQu Wenruo /* 2736275193eSQu Wenruo * Find checksums for logical bytenr range [disk_bytenr, disk_bytenr + len) and 27443dd529aSDavid Sterba * store the result to @dst. 2756275193eSQu Wenruo * 2766275193eSQu Wenruo * Return >0 for the number of sectors we found. 2776275193eSQu Wenruo * Return 0 for the range [disk_bytenr, disk_bytenr + sectorsize) has no csum 2786275193eSQu Wenruo * for it. Caller may want to try next sector until one range is hit. 2796275193eSQu Wenruo * Return <0 for fatal error. 2806275193eSQu Wenruo */ 2816275193eSQu Wenruo static int search_csum_tree(struct btrfs_fs_info *fs_info, 2826275193eSQu Wenruo struct btrfs_path *path, u64 disk_bytenr, 2836275193eSQu Wenruo u64 len, u8 *dst) 2846275193eSQu Wenruo { 285fc28b25eSJosef Bacik struct btrfs_root *csum_root; 2866275193eSQu Wenruo struct btrfs_csum_item *item = NULL; 2876275193eSQu Wenruo struct btrfs_key key; 2886275193eSQu Wenruo const u32 sectorsize = fs_info->sectorsize; 2896275193eSQu Wenruo const u32 csum_size = fs_info->csum_size; 2906275193eSQu Wenruo u32 itemsize; 2916275193eSQu Wenruo int ret; 2926275193eSQu Wenruo u64 csum_start; 2936275193eSQu Wenruo u64 csum_len; 2946275193eSQu Wenruo 2956275193eSQu Wenruo ASSERT(IS_ALIGNED(disk_bytenr, sectorsize) && 2966275193eSQu Wenruo IS_ALIGNED(len, sectorsize)); 2976275193eSQu Wenruo 2986275193eSQu Wenruo /* Check if the current csum item covers disk_bytenr */ 2996275193eSQu Wenruo if (path->nodes[0]) { 3006275193eSQu Wenruo item = btrfs_item_ptr(path->nodes[0], path->slots[0], 3016275193eSQu Wenruo struct btrfs_csum_item); 3026275193eSQu Wenruo btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); 3033212fa14SJosef Bacik itemsize = btrfs_item_size(path->nodes[0], path->slots[0]); 3046275193eSQu Wenruo 3056275193eSQu Wenruo csum_start = key.offset; 3066275193eSQu Wenruo csum_len = (itemsize / csum_size) * sectorsize; 3076275193eSQu Wenruo 3086275193eSQu Wenruo if (in_range(disk_bytenr, csum_start, csum_len)) 3096275193eSQu Wenruo goto found; 3106275193eSQu Wenruo } 3116275193eSQu Wenruo 3126275193eSQu Wenruo /* Current item doesn't contain the desired range, search again */ 3136275193eSQu Wenruo btrfs_release_path(path); 314fc28b25eSJosef Bacik csum_root = btrfs_csum_root(fs_info, disk_bytenr); 315fc28b25eSJosef Bacik item = btrfs_lookup_csum(NULL, csum_root, path, disk_bytenr, 0); 3166275193eSQu Wenruo if (IS_ERR(item)) { 3176275193eSQu Wenruo ret = PTR_ERR(item); 3186275193eSQu Wenruo goto out; 3196275193eSQu Wenruo } 3206275193eSQu Wenruo btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); 3213212fa14SJosef Bacik itemsize = btrfs_item_size(path->nodes[0], path->slots[0]); 3226275193eSQu Wenruo 3236275193eSQu Wenruo csum_start = key.offset; 3246275193eSQu Wenruo csum_len = (itemsize / csum_size) * sectorsize; 3256275193eSQu Wenruo ASSERT(in_range(disk_bytenr, csum_start, csum_len)); 3266275193eSQu Wenruo 3276275193eSQu Wenruo found: 3286275193eSQu Wenruo ret = (min(csum_start + csum_len, disk_bytenr + len) - 3296275193eSQu Wenruo disk_bytenr) >> fs_info->sectorsize_bits; 3306275193eSQu Wenruo read_extent_buffer(path->nodes[0], dst, (unsigned long)item, 3316275193eSQu Wenruo ret * csum_size); 3326275193eSQu Wenruo out: 33303ddb19dSJosef Bacik if (ret == -ENOENT || ret == -EFBIG) 3346275193eSQu Wenruo ret = 0; 3356275193eSQu Wenruo return ret; 3366275193eSQu Wenruo } 3376275193eSQu Wenruo 3386275193eSQu Wenruo /* 3396275193eSQu Wenruo * Locate the file_offset of @cur_disk_bytenr of a @bio. 3406275193eSQu Wenruo * 3416275193eSQu Wenruo * Bio of btrfs represents read range of 3426275193eSQu Wenruo * [bi_sector << 9, bi_sector << 9 + bi_size). 3436275193eSQu Wenruo * Knowing this, we can iterate through each bvec to locate the page belong to 3446275193eSQu Wenruo * @cur_disk_bytenr and get the file offset. 3456275193eSQu Wenruo * 3466275193eSQu Wenruo * @inode is used to determine if the bvec page really belongs to @inode. 3476275193eSQu Wenruo * 3486275193eSQu Wenruo * Return 0 if we can't find the file offset 3496275193eSQu Wenruo * Return >0 if we find the file offset and restore it to @file_offset_ret 3506275193eSQu Wenruo */ 3516275193eSQu Wenruo static int search_file_offset_in_bio(struct bio *bio, struct inode *inode, 3526275193eSQu Wenruo u64 disk_bytenr, u64 *file_offset_ret) 3536275193eSQu Wenruo { 3546275193eSQu Wenruo struct bvec_iter iter; 3556275193eSQu Wenruo struct bio_vec bvec; 3566275193eSQu Wenruo u64 cur = bio->bi_iter.bi_sector << SECTOR_SHIFT; 3576275193eSQu Wenruo int ret = 0; 3586275193eSQu Wenruo 3596275193eSQu Wenruo bio_for_each_segment(bvec, bio, iter) { 3606275193eSQu Wenruo struct page *page = bvec.bv_page; 3616275193eSQu Wenruo 3626275193eSQu Wenruo if (cur > disk_bytenr) 3636275193eSQu Wenruo break; 3646275193eSQu Wenruo if (cur + bvec.bv_len <= disk_bytenr) { 3656275193eSQu Wenruo cur += bvec.bv_len; 3666275193eSQu Wenruo continue; 3676275193eSQu Wenruo } 3686275193eSQu Wenruo ASSERT(in_range(disk_bytenr, cur, bvec.bv_len)); 3696275193eSQu Wenruo if (page->mapping && page->mapping->host && 3706275193eSQu Wenruo page->mapping->host == inode) { 3716275193eSQu Wenruo ret = 1; 3726275193eSQu Wenruo *file_offset_ret = page_offset(page) + bvec.bv_offset + 3736275193eSQu Wenruo disk_bytenr - cur; 3746275193eSQu Wenruo break; 3756275193eSQu Wenruo } 3766275193eSQu Wenruo } 3776275193eSQu Wenruo return ret; 3786275193eSQu Wenruo } 3796275193eSQu Wenruo 38043dd529aSDavid Sterba /* 3816275193eSQu Wenruo * Lookup the checksum for the read bio in csum tree. 3829e46458aSQu Wenruo * 383e62958fcSOmar Sandoval * @inode: inode that the bio is for. 384fb30f470SOmar Sandoval * @bio: bio to look up. 385fb30f470SOmar Sandoval * @dst: Buffer of size nblocks * btrfs_super_csum_size() used to return 386fb30f470SOmar Sandoval * checksum (nblocks = bio->bi_iter.bi_size / fs_info->sectorsize). If 387fb30f470SOmar Sandoval * NULL, the checksum buffer is allocated and returned in 388c3a3b19bSQu Wenruo * btrfs_bio(bio)->csum instead. 389e62958fcSOmar Sandoval * 390e62958fcSOmar Sandoval * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise. 391e62958fcSOmar Sandoval */ 3926275193eSQu Wenruo blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst) 39361b49440SChris Mason { 3940b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 395facc8a22SMiao Xie struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 3961784b7d5SJosef Bacik struct btrfs_bio *bbio = NULL; 397facc8a22SMiao Xie struct btrfs_path *path; 3986275193eSQu Wenruo const u32 sectorsize = fs_info->sectorsize; 399223486c2SDavid Sterba const u32 csum_size = fs_info->csum_size; 4006275193eSQu Wenruo u32 orig_len = bio->bi_iter.bi_size; 4016275193eSQu Wenruo u64 orig_disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT; 4026275193eSQu Wenruo u64 cur_disk_bytenr; 4036275193eSQu Wenruo u8 *csum; 4046275193eSQu Wenruo const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits; 4056275193eSQu Wenruo int count = 0; 4061784b7d5SJosef Bacik blk_status_t ret = BLK_STS_OK; 40761b49440SChris Mason 408056c8311SJosef Bacik if ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) || 409056c8311SJosef Bacik test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state)) 410334c16d8SJosef Bacik return BLK_STS_OK; 411334c16d8SJosef Bacik 4129e46458aSQu Wenruo /* 4139e46458aSQu Wenruo * This function is only called for read bio. 4149e46458aSQu Wenruo * 4159e46458aSQu Wenruo * This means two things: 4169e46458aSQu Wenruo * - All our csums should only be in csum tree 4179e46458aSQu Wenruo * No ordered extents csums, as ordered extents are only for write 4189e46458aSQu Wenruo * path. 4196275193eSQu Wenruo * - No need to bother any other info from bvec 4206275193eSQu Wenruo * Since we're looking up csums, the only important info is the 4216275193eSQu Wenruo * disk_bytenr and the length, which can be extracted from bi_iter 4226275193eSQu Wenruo * directly. 4239e46458aSQu Wenruo */ 4249e46458aSQu Wenruo ASSERT(bio_op(bio) == REQ_OP_READ); 42561b49440SChris Mason path = btrfs_alloc_path(); 426c2db1073STsutomu Itoh if (!path) 4274e4cbee9SChristoph Hellwig return BLK_STS_RESOURCE; 428facc8a22SMiao Xie 429facc8a22SMiao Xie if (!dst) { 4301784b7d5SJosef Bacik bbio = btrfs_bio(bio); 431fb30f470SOmar Sandoval 432facc8a22SMiao Xie if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) { 433c3a3b19bSQu Wenruo bbio->csum = kmalloc_array(nblocks, csum_size, GFP_NOFS); 434c3a3b19bSQu Wenruo if (!bbio->csum) { 435facc8a22SMiao Xie btrfs_free_path(path); 4364e4cbee9SChristoph Hellwig return BLK_STS_RESOURCE; 437facc8a22SMiao Xie } 438facc8a22SMiao Xie } else { 439c3a3b19bSQu Wenruo bbio->csum = bbio->csum_inline; 440facc8a22SMiao Xie } 441c3a3b19bSQu Wenruo csum = bbio->csum; 442facc8a22SMiao Xie } else { 44310fe6ca8SJohannes Thumshirn csum = dst; 444facc8a22SMiao Xie } 445facc8a22SMiao Xie 44635478d05SQu Wenruo /* 44735478d05SQu Wenruo * If requested number of sectors is larger than one leaf can contain, 44835478d05SQu Wenruo * kick the readahead for csum tree. 44935478d05SQu Wenruo */ 45035478d05SQu Wenruo if (nblocks > fs_info->csums_per_leaf) 451e4058b54SDavid Sterba path->reada = READA_FORWARD; 45261b49440SChris Mason 4532cf8572dSChris Mason /* 4542cf8572dSChris Mason * the free space stuff is only read when it hasn't been 4552cf8572dSChris Mason * updated in the current transaction. So, we can safely 4562cf8572dSChris Mason * read from the commit root and sidestep a nasty deadlock 4572cf8572dSChris Mason * between reading the free space cache and updating the csum tree. 4582cf8572dSChris Mason */ 45970ddc553SNikolay Borisov if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 4602cf8572dSChris Mason path->search_commit_root = 1; 461ddf23b3fSJosef Bacik path->skip_locking = 1; 462ddf23b3fSJosef Bacik } 4632cf8572dSChris Mason 4646275193eSQu Wenruo for (cur_disk_bytenr = orig_disk_bytenr; 4656275193eSQu Wenruo cur_disk_bytenr < orig_disk_bytenr + orig_len; 4666275193eSQu Wenruo cur_disk_bytenr += (count * sectorsize)) { 4676275193eSQu Wenruo u64 search_len = orig_disk_bytenr + orig_len - cur_disk_bytenr; 4686275193eSQu Wenruo unsigned int sector_offset; 4696275193eSQu Wenruo u8 *csum_dst; 470c40a3d38SChandan Rajendra 4716275193eSQu Wenruo /* 4726275193eSQu Wenruo * Although both cur_disk_bytenr and orig_disk_bytenr is u64, 4736275193eSQu Wenruo * we're calculating the offset to the bio start. 4746275193eSQu Wenruo * 4756275193eSQu Wenruo * Bio size is limited to UINT_MAX, thus unsigned int is large 4766275193eSQu Wenruo * enough to contain the raw result, not to mention the right 4776275193eSQu Wenruo * shifted result. 4786275193eSQu Wenruo */ 4796275193eSQu Wenruo ASSERT(cur_disk_bytenr - orig_disk_bytenr < UINT_MAX); 4806275193eSQu Wenruo sector_offset = (cur_disk_bytenr - orig_disk_bytenr) >> 4816275193eSQu Wenruo fs_info->sectorsize_bits; 4826275193eSQu Wenruo csum_dst = csum + sector_offset * csum_size; 4834989d277SChristoph Hellwig 4846275193eSQu Wenruo count = search_csum_tree(fs_info, path, cur_disk_bytenr, 4856275193eSQu Wenruo search_len, csum_dst); 4861784b7d5SJosef Bacik if (count < 0) { 4871784b7d5SJosef Bacik ret = errno_to_blk_status(count); 4881784b7d5SJosef Bacik if (bbio) 4891784b7d5SJosef Bacik btrfs_bio_free_csum(bbio); 4901784b7d5SJosef Bacik break; 4911784b7d5SJosef Bacik } 4921784b7d5SJosef Bacik 4936275193eSQu Wenruo /* 4941784b7d5SJosef Bacik * We didn't find a csum for this range. We need to make sure 4951784b7d5SJosef Bacik * we complain loudly about this, because we are not NODATASUM. 4961784b7d5SJosef Bacik * 4971784b7d5SJosef Bacik * However for the DATA_RELOC inode we could potentially be 4981784b7d5SJosef Bacik * relocating data extents for a NODATASUM inode, so the inode 4991784b7d5SJosef Bacik * itself won't be marked with NODATASUM, but the extent we're 5001784b7d5SJosef Bacik * copying is in fact NODATASUM. If we don't find a csum we 5011784b7d5SJosef Bacik * assume this is the case. 5026275193eSQu Wenruo */ 5031784b7d5SJosef Bacik if (count == 0) { 5046275193eSQu Wenruo memset(csum_dst, 0, csum_size); 505e4100d98SMiao Xie count = 1; 5066275193eSQu Wenruo 50717d217feSYan Zheng if (BTRFS_I(inode)->root->root_key.objectid == 50817d217feSYan Zheng BTRFS_DATA_RELOC_TREE_OBJECTID) { 5096275193eSQu Wenruo u64 file_offset; 5106275193eSQu Wenruo int ret; 5116275193eSQu Wenruo 5126275193eSQu Wenruo ret = search_file_offset_in_bio(bio, inode, 5136275193eSQu Wenruo cur_disk_bytenr, &file_offset); 5146275193eSQu Wenruo if (ret) 5156275193eSQu Wenruo set_extent_bits(io_tree, file_offset, 5166275193eSQu Wenruo file_offset + sectorsize - 1, 517ceeb0ae7SDavid Sterba EXTENT_NODATASUM); 51817d217feSYan Zheng } else { 5196275193eSQu Wenruo btrfs_warn_rl(fs_info, 5206275193eSQu Wenruo "csum hole found for disk bytenr range [%llu, %llu)", 5216275193eSQu Wenruo cur_disk_bytenr, cur_disk_bytenr + sectorsize); 52217d217feSYan Zheng } 5234989d277SChristoph Hellwig } 5244989d277SChristoph Hellwig } 5254989d277SChristoph Hellwig 52661b49440SChris Mason btrfs_free_path(path); 5271784b7d5SJosef Bacik return ret; 5284b46fce2SJosef Bacik } 5294b46fce2SJosef Bacik 53097e38239SQu Wenruo int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end, 53126ce9114SJosef Bacik struct list_head *list, int search_commit, 53226ce9114SJosef Bacik bool nowait) 53317d217feSYan Zheng { 5340b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 53517d217feSYan Zheng struct btrfs_key key; 53617d217feSYan Zheng struct btrfs_path *path; 53717d217feSYan Zheng struct extent_buffer *leaf; 53817d217feSYan Zheng struct btrfs_ordered_sum *sums; 53917d217feSYan Zheng struct btrfs_csum_item *item; 5400678b618SMark Fasheh LIST_HEAD(tmplist); 54117d217feSYan Zheng int ret; 54217d217feSYan Zheng 5430b246afaSJeff Mahoney ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 5440b246afaSJeff Mahoney IS_ALIGNED(end + 1, fs_info->sectorsize)); 5454277a9c3SJosef Bacik 54617d217feSYan Zheng path = btrfs_alloc_path(); 547d8926bb3SMark Fasheh if (!path) 548d8926bb3SMark Fasheh return -ENOMEM; 54917d217feSYan Zheng 55026ce9114SJosef Bacik path->nowait = nowait; 551a2de733cSArne Jansen if (search_commit) { 552a2de733cSArne Jansen path->skip_locking = 1; 553e4058b54SDavid Sterba path->reada = READA_FORWARD; 554a2de733cSArne Jansen path->search_commit_root = 1; 555a2de733cSArne Jansen } 556a2de733cSArne Jansen 55717d217feSYan Zheng key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 55817d217feSYan Zheng key.offset = start; 55917d217feSYan Zheng key.type = BTRFS_EXTENT_CSUM_KEY; 56017d217feSYan Zheng 56107d400a6SYan Zheng ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 56217d217feSYan Zheng if (ret < 0) 56317d217feSYan Zheng goto fail; 56417d217feSYan Zheng if (ret > 0 && path->slots[0] > 0) { 56517d217feSYan Zheng leaf = path->nodes[0]; 56617d217feSYan Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1); 567cb649e81SQu Wenruo 568cb649e81SQu Wenruo /* 569cb649e81SQu Wenruo * There are two cases we can hit here for the previous csum 570cb649e81SQu Wenruo * item: 571cb649e81SQu Wenruo * 572cb649e81SQu Wenruo * |<- search range ->| 573cb649e81SQu Wenruo * |<- csum item ->| 574cb649e81SQu Wenruo * 575cb649e81SQu Wenruo * Or 576cb649e81SQu Wenruo * |<- search range ->| 577cb649e81SQu Wenruo * |<- csum item ->| 578cb649e81SQu Wenruo * 579cb649e81SQu Wenruo * Check if the previous csum item covers the leading part of 580cb649e81SQu Wenruo * the search range. If so we have to start from previous csum 581cb649e81SQu Wenruo * item. 582cb649e81SQu Wenruo */ 58317d217feSYan Zheng if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID && 58417d217feSYan Zheng key.type == BTRFS_EXTENT_CSUM_KEY) { 585cb649e81SQu Wenruo if (bytes_to_csum_size(fs_info, start - key.offset) < 5863212fa14SJosef Bacik btrfs_item_size(leaf, path->slots[0] - 1)) 58717d217feSYan Zheng path->slots[0]--; 58817d217feSYan Zheng } 58917d217feSYan Zheng } 59017d217feSYan Zheng 59117d217feSYan Zheng while (start <= end) { 592cb649e81SQu Wenruo u64 csum_end; 593cb649e81SQu Wenruo 59417d217feSYan Zheng leaf = path->nodes[0]; 59517d217feSYan Zheng if (path->slots[0] >= btrfs_header_nritems(leaf)) { 59607d400a6SYan Zheng ret = btrfs_next_leaf(root, path); 59717d217feSYan Zheng if (ret < 0) 59817d217feSYan Zheng goto fail; 59917d217feSYan Zheng if (ret > 0) 60017d217feSYan Zheng break; 60117d217feSYan Zheng leaf = path->nodes[0]; 60217d217feSYan Zheng } 60317d217feSYan Zheng 60417d217feSYan Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 60517d217feSYan Zheng if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || 606628c8282SZhi Yong Wu key.type != BTRFS_EXTENT_CSUM_KEY || 607628c8282SZhi Yong Wu key.offset > end) 60817d217feSYan Zheng break; 60917d217feSYan Zheng 61017d217feSYan Zheng if (key.offset > start) 61117d217feSYan Zheng start = key.offset; 61217d217feSYan Zheng 613cb649e81SQu Wenruo csum_end = key.offset + csum_size_to_bytes(fs_info, 614cb649e81SQu Wenruo btrfs_item_size(leaf, path->slots[0])); 61587b29b20SYan Zheng if (csum_end <= start) { 61687b29b20SYan Zheng path->slots[0]++; 61787b29b20SYan Zheng continue; 61887b29b20SYan Zheng } 61917d217feSYan Zheng 62007d400a6SYan Zheng csum_end = min(csum_end, end + 1); 62107d400a6SYan Zheng item = btrfs_item_ptr(path->nodes[0], path->slots[0], 62207d400a6SYan Zheng struct btrfs_csum_item); 62307d400a6SYan Zheng while (start < csum_end) { 624cb649e81SQu Wenruo unsigned long offset; 625cb649e81SQu Wenruo size_t size; 626cb649e81SQu Wenruo 62707d400a6SYan Zheng size = min_t(size_t, csum_end - start, 628cb649e81SQu Wenruo max_ordered_sum_bytes(fs_info)); 6290b246afaSJeff Mahoney sums = kzalloc(btrfs_ordered_sum_size(fs_info, size), 63007d400a6SYan Zheng GFP_NOFS); 6310678b618SMark Fasheh if (!sums) { 6320678b618SMark Fasheh ret = -ENOMEM; 6330678b618SMark Fasheh goto fail; 6340678b618SMark Fasheh } 63517d217feSYan Zheng 63617d217feSYan Zheng sums->bytenr = start; 637f51a4a18SMiao Xie sums->len = (int)size; 63817d217feSYan Zheng 639cb649e81SQu Wenruo offset = bytes_to_csum_size(fs_info, start - key.offset); 64017d217feSYan Zheng 64107d400a6SYan Zheng read_extent_buffer(path->nodes[0], 642f51a4a18SMiao Xie sums->sums, 643f51a4a18SMiao Xie ((unsigned long)item) + offset, 644cb649e81SQu Wenruo bytes_to_csum_size(fs_info, size)); 64517d217feSYan Zheng 646cb649e81SQu Wenruo start += size; 6470678b618SMark Fasheh list_add_tail(&sums->list, &tmplist); 64807d400a6SYan Zheng } 64917d217feSYan Zheng path->slots[0]++; 65017d217feSYan Zheng } 65117d217feSYan Zheng ret = 0; 65217d217feSYan Zheng fail: 6530678b618SMark Fasheh while (ret < 0 && !list_empty(&tmplist)) { 6546e5aafb2SChris Mason sums = list_entry(tmplist.next, struct btrfs_ordered_sum, list); 6550678b618SMark Fasheh list_del(&sums->list); 6560678b618SMark Fasheh kfree(sums); 6570678b618SMark Fasheh } 6580678b618SMark Fasheh list_splice_tail(&tmplist, list); 6590678b618SMark Fasheh 66017d217feSYan Zheng btrfs_free_path(path); 66117d217feSYan Zheng return ret; 66217d217feSYan Zheng } 66317d217feSYan Zheng 66443dd529aSDavid Sterba /* 66597e38239SQu Wenruo * Do the same work as btrfs_lookup_csums_list(), the difference is in how 66697e38239SQu Wenruo * we return the result. 66797e38239SQu Wenruo * 66897e38239SQu Wenruo * This version will set the corresponding bits in @csum_bitmap to represent 66997e38239SQu Wenruo * that there is a csum found. 67097e38239SQu Wenruo * Each bit represents a sector. Thus caller should ensure @csum_buf passed 67197e38239SQu Wenruo * in is large enough to contain all csums. 67297e38239SQu Wenruo */ 67397e38239SQu Wenruo int btrfs_lookup_csums_bitmap(struct btrfs_root *root, u64 start, u64 end, 67497e38239SQu Wenruo u8 *csum_buf, unsigned long *csum_bitmap) 67597e38239SQu Wenruo { 67697e38239SQu Wenruo struct btrfs_fs_info *fs_info = root->fs_info; 67797e38239SQu Wenruo struct btrfs_key key; 67897e38239SQu Wenruo struct btrfs_path *path; 67997e38239SQu Wenruo struct extent_buffer *leaf; 68097e38239SQu Wenruo struct btrfs_csum_item *item; 68197e38239SQu Wenruo const u64 orig_start = start; 68297e38239SQu Wenruo int ret; 68397e38239SQu Wenruo 68497e38239SQu Wenruo ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 68597e38239SQu Wenruo IS_ALIGNED(end + 1, fs_info->sectorsize)); 68697e38239SQu Wenruo 68797e38239SQu Wenruo path = btrfs_alloc_path(); 68897e38239SQu Wenruo if (!path) 68997e38239SQu Wenruo return -ENOMEM; 69097e38239SQu Wenruo 69197e38239SQu Wenruo key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 69297e38239SQu Wenruo key.type = BTRFS_EXTENT_CSUM_KEY; 69397e38239SQu Wenruo key.offset = start; 69497e38239SQu Wenruo 69597e38239SQu Wenruo ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 69697e38239SQu Wenruo if (ret < 0) 69797e38239SQu Wenruo goto fail; 69897e38239SQu Wenruo if (ret > 0 && path->slots[0] > 0) { 69997e38239SQu Wenruo leaf = path->nodes[0]; 70097e38239SQu Wenruo btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1); 70197e38239SQu Wenruo 70297e38239SQu Wenruo /* 70397e38239SQu Wenruo * There are two cases we can hit here for the previous csum 70497e38239SQu Wenruo * item: 70597e38239SQu Wenruo * 70697e38239SQu Wenruo * |<- search range ->| 70797e38239SQu Wenruo * |<- csum item ->| 70897e38239SQu Wenruo * 70997e38239SQu Wenruo * Or 71097e38239SQu Wenruo * |<- search range ->| 71197e38239SQu Wenruo * |<- csum item ->| 71297e38239SQu Wenruo * 71397e38239SQu Wenruo * Check if the previous csum item covers the leading part of 71497e38239SQu Wenruo * the search range. If so we have to start from previous csum 71597e38239SQu Wenruo * item. 71697e38239SQu Wenruo */ 71797e38239SQu Wenruo if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID && 71897e38239SQu Wenruo key.type == BTRFS_EXTENT_CSUM_KEY) { 71997e38239SQu Wenruo if (bytes_to_csum_size(fs_info, start - key.offset) < 72097e38239SQu Wenruo btrfs_item_size(leaf, path->slots[0] - 1)) 72197e38239SQu Wenruo path->slots[0]--; 72297e38239SQu Wenruo } 72397e38239SQu Wenruo } 72497e38239SQu Wenruo 72597e38239SQu Wenruo while (start <= end) { 72697e38239SQu Wenruo u64 csum_end; 72797e38239SQu Wenruo 72897e38239SQu Wenruo leaf = path->nodes[0]; 72997e38239SQu Wenruo if (path->slots[0] >= btrfs_header_nritems(leaf)) { 73097e38239SQu Wenruo ret = btrfs_next_leaf(root, path); 73197e38239SQu Wenruo if (ret < 0) 73297e38239SQu Wenruo goto fail; 73397e38239SQu Wenruo if (ret > 0) 73497e38239SQu Wenruo break; 73597e38239SQu Wenruo leaf = path->nodes[0]; 73697e38239SQu Wenruo } 73797e38239SQu Wenruo 73897e38239SQu Wenruo btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 73997e38239SQu Wenruo if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || 74097e38239SQu Wenruo key.type != BTRFS_EXTENT_CSUM_KEY || 74197e38239SQu Wenruo key.offset > end) 74297e38239SQu Wenruo break; 74397e38239SQu Wenruo 74497e38239SQu Wenruo if (key.offset > start) 74597e38239SQu Wenruo start = key.offset; 74697e38239SQu Wenruo 74797e38239SQu Wenruo csum_end = key.offset + csum_size_to_bytes(fs_info, 74897e38239SQu Wenruo btrfs_item_size(leaf, path->slots[0])); 74997e38239SQu Wenruo if (csum_end <= start) { 75097e38239SQu Wenruo path->slots[0]++; 75197e38239SQu Wenruo continue; 75297e38239SQu Wenruo } 75397e38239SQu Wenruo 75497e38239SQu Wenruo csum_end = min(csum_end, end + 1); 75597e38239SQu Wenruo item = btrfs_item_ptr(path->nodes[0], path->slots[0], 75697e38239SQu Wenruo struct btrfs_csum_item); 75797e38239SQu Wenruo while (start < csum_end) { 75897e38239SQu Wenruo unsigned long offset; 75997e38239SQu Wenruo size_t size; 76097e38239SQu Wenruo u8 *csum_dest = csum_buf + bytes_to_csum_size(fs_info, 76197e38239SQu Wenruo start - orig_start); 76297e38239SQu Wenruo 76397e38239SQu Wenruo size = min_t(size_t, csum_end - start, end + 1 - start); 76497e38239SQu Wenruo 76597e38239SQu Wenruo offset = bytes_to_csum_size(fs_info, start - key.offset); 76697e38239SQu Wenruo 76797e38239SQu Wenruo read_extent_buffer(path->nodes[0], csum_dest, 76897e38239SQu Wenruo ((unsigned long)item) + offset, 76997e38239SQu Wenruo bytes_to_csum_size(fs_info, size)); 77097e38239SQu Wenruo 77197e38239SQu Wenruo bitmap_set(csum_bitmap, 77297e38239SQu Wenruo (start - orig_start) >> fs_info->sectorsize_bits, 77397e38239SQu Wenruo size >> fs_info->sectorsize_bits); 77497e38239SQu Wenruo 77597e38239SQu Wenruo start += size; 77697e38239SQu Wenruo } 77797e38239SQu Wenruo path->slots[0]++; 77897e38239SQu Wenruo } 77997e38239SQu Wenruo ret = 0; 78097e38239SQu Wenruo fail: 78197e38239SQu Wenruo btrfs_free_path(path); 78297e38239SQu Wenruo return ret; 78397e38239SQu Wenruo } 78497e38239SQu Wenruo 78597e38239SQu Wenruo /* 78643dd529aSDavid Sterba * Calculate checksums of the data contained inside a bio. 787e331f6b1SOmar Sandoval * 78851d470aeSNikolay Borisov * @inode: Owner of the data inside the bio 78951d470aeSNikolay Borisov * @bio: Contains the data to be checksummed 790e331f6b1SOmar Sandoval * @offset: If (u64)-1, @bio may contain discontiguous bio vecs, so the 791e331f6b1SOmar Sandoval * file offsets are determined from the page offsets in the bio. 792e331f6b1SOmar Sandoval * Otherwise, this is the starting file offset of the bio vecs in 793e331f6b1SOmar Sandoval * @bio, which must be contiguous. 794e331f6b1SOmar Sandoval * @one_ordered: If true, @bio only refers to one ordered extent. 79551d470aeSNikolay Borisov */ 796bd242a08SNikolay Borisov blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, 797e331f6b1SOmar Sandoval u64 offset, bool one_ordered) 798e015640fSChris Mason { 799c3504372SNikolay Borisov struct btrfs_fs_info *fs_info = inode->root->fs_info; 800d5178578SJohannes Thumshirn SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); 801e6dcd2dcSChris Mason struct btrfs_ordered_sum *sums; 8026cd7ce49SChristoph Hellwig struct btrfs_ordered_extent *ordered = NULL; 803e331f6b1SOmar Sandoval const bool use_page_offsets = (offset == (u64)-1); 804e015640fSChris Mason char *data; 80517347cecSLiu Bo struct bvec_iter iter; 80617347cecSLiu Bo struct bio_vec bvec; 807f51a4a18SMiao Xie int index; 808e331f6b1SOmar Sandoval unsigned int blockcount; 8093edf7d33SChris Mason unsigned long total_bytes = 0; 8103edf7d33SChris Mason unsigned long this_sum_bytes = 0; 81117347cecSLiu Bo int i; 812a3d46aeaSNikolay Borisov unsigned nofs_flag; 813e015640fSChris Mason 814a3d46aeaSNikolay Borisov nofs_flag = memalloc_nofs_save(); 815a3d46aeaSNikolay Borisov sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size), 816a3d46aeaSNikolay Borisov GFP_KERNEL); 817a3d46aeaSNikolay Borisov memalloc_nofs_restore(nofs_flag); 818a3d46aeaSNikolay Borisov 819e015640fSChris Mason if (!sums) 8204e4cbee9SChristoph Hellwig return BLK_STS_RESOURCE; 8213edf7d33SChris Mason 8224f024f37SKent Overstreet sums->len = bio->bi_iter.bi_size; 823e6dcd2dcSChris Mason INIT_LIST_HEAD(&sums->list); 824d20f7043SChris Mason 8251201b58bSDavid Sterba sums->bytenr = bio->bi_iter.bi_sector << 9; 826f51a4a18SMiao Xie index = 0; 827e015640fSChris Mason 828d5178578SJohannes Thumshirn shash->tfm = fs_info->csum_shash; 829d5178578SJohannes Thumshirn 83017347cecSLiu Bo bio_for_each_segment(bvec, bio, iter) { 831e331f6b1SOmar Sandoval if (use_page_offsets) 83217347cecSLiu Bo offset = page_offset(bvec.bv_page) + bvec.bv_offset; 833d20f7043SChris Mason 8346cd7ce49SChristoph Hellwig if (!ordered) { 8356cd7ce49SChristoph Hellwig ordered = btrfs_lookup_ordered_extent(inode, offset); 836bbc9a6ebSQu Wenruo /* 837bbc9a6ebSQu Wenruo * The bio range is not covered by any ordered extent, 838bbc9a6ebSQu Wenruo * must be a code logic error. 839bbc9a6ebSQu Wenruo */ 840bbc9a6ebSQu Wenruo if (unlikely(!ordered)) { 841bbc9a6ebSQu Wenruo WARN(1, KERN_WARNING 842bbc9a6ebSQu Wenruo "no ordered extent for root %llu ino %llu offset %llu\n", 843bbc9a6ebSQu Wenruo inode->root->root_key.objectid, 844bbc9a6ebSQu Wenruo btrfs_ino(inode), offset); 845bbc9a6ebSQu Wenruo kvfree(sums); 846bbc9a6ebSQu Wenruo return BLK_STS_IOERR; 847bbc9a6ebSQu Wenruo } 8486cd7ce49SChristoph Hellwig } 8496cd7ce49SChristoph Hellwig 850e331f6b1SOmar Sandoval blockcount = BTRFS_BYTES_TO_BLKS(fs_info, 85117347cecSLiu Bo bvec.bv_len + fs_info->sectorsize 852c40a3d38SChandan Rajendra - 1); 853c40a3d38SChandan Rajendra 854e331f6b1SOmar Sandoval for (i = 0; i < blockcount; i++) { 855e331f6b1SOmar Sandoval if (!one_ordered && 856e331f6b1SOmar Sandoval !in_range(offset, ordered->file_offset, 857e331f6b1SOmar Sandoval ordered->num_bytes)) { 8583edf7d33SChris Mason unsigned long bytes_left; 859c40a3d38SChandan Rajendra 8603edf7d33SChris Mason sums->len = this_sum_bytes; 8613edf7d33SChris Mason this_sum_bytes = 0; 862f9756261SNikolay Borisov btrfs_add_ordered_sum(ordered, sums); 8633edf7d33SChris Mason btrfs_put_ordered_extent(ordered); 8643edf7d33SChris Mason 8654f024f37SKent Overstreet bytes_left = bio->bi_iter.bi_size - total_bytes; 8663edf7d33SChris Mason 867a3d46aeaSNikolay Borisov nofs_flag = memalloc_nofs_save(); 868a3d46aeaSNikolay Borisov sums = kvzalloc(btrfs_ordered_sum_size(fs_info, 869a3d46aeaSNikolay Borisov bytes_left), GFP_KERNEL); 870a3d46aeaSNikolay Borisov memalloc_nofs_restore(nofs_flag); 87179787eaaSJeff Mahoney BUG_ON(!sums); /* -ENOMEM */ 8723edf7d33SChris Mason sums->len = bytes_left; 873c40a3d38SChandan Rajendra ordered = btrfs_lookup_ordered_extent(inode, 874c40a3d38SChandan Rajendra offset); 875c40a3d38SChandan Rajendra ASSERT(ordered); /* Logic error */ 8761201b58bSDavid Sterba sums->bytenr = (bio->bi_iter.bi_sector << 9) 877c40a3d38SChandan Rajendra + total_bytes; 878f51a4a18SMiao Xie index = 0; 879c40a3d38SChandan Rajendra } 880c40a3d38SChandan Rajendra 8813dcfbcceSChristoph Hellwig data = bvec_kmap_local(&bvec); 8823dcfbcceSChristoph Hellwig crypto_shash_digest(shash, 8833dcfbcceSChristoph Hellwig data + (i * fs_info->sectorsize), 884fd08001fSEric Biggers fs_info->sectorsize, 885fd08001fSEric Biggers sums->sums + index); 8863dcfbcceSChristoph Hellwig kunmap_local(data); 887713cebfbSDavid Sterba index += fs_info->csum_size; 8880b246afaSJeff Mahoney offset += fs_info->sectorsize; 8890b246afaSJeff Mahoney this_sum_bytes += fs_info->sectorsize; 8900b246afaSJeff Mahoney total_bytes += fs_info->sectorsize; 891c40a3d38SChandan Rajendra } 892c40a3d38SChandan Rajendra 893e015640fSChris Mason } 894ed98b56aSChris Mason this_sum_bytes = 0; 895f9756261SNikolay Borisov btrfs_add_ordered_sum(ordered, sums); 8963edf7d33SChris Mason btrfs_put_ordered_extent(ordered); 897e015640fSChris Mason return 0; 898e015640fSChris Mason } 899e015640fSChris Mason 900459931ecSChris Mason /* 90143dd529aSDavid Sterba * Remove one checksum overlapping a range. 902459931ecSChris Mason * 90343dd529aSDavid Sterba * This expects the key to describe the csum pointed to by the path, and it 90443dd529aSDavid Sterba * expects the csum to overlap the range [bytenr, len] 905459931ecSChris Mason * 90643dd529aSDavid Sterba * The csum should not be entirely contained in the range and the range should 90743dd529aSDavid Sterba * not be entirely contained in the csum. 90843dd529aSDavid Sterba * 90943dd529aSDavid Sterba * This calls btrfs_truncate_item with the correct args based on the overlap, 91043dd529aSDavid Sterba * and fixes up the key as required. 911459931ecSChris Mason */ 9122ff7e61eSJeff Mahoney static noinline void truncate_one_csum(struct btrfs_fs_info *fs_info, 913459931ecSChris Mason struct btrfs_path *path, 914459931ecSChris Mason struct btrfs_key *key, 915459931ecSChris Mason u64 bytenr, u64 len) 916459931ecSChris Mason { 917459931ecSChris Mason struct extent_buffer *leaf; 918223486c2SDavid Sterba const u32 csum_size = fs_info->csum_size; 919459931ecSChris Mason u64 csum_end; 920459931ecSChris Mason u64 end_byte = bytenr + len; 921265fdfa6SDavid Sterba u32 blocksize_bits = fs_info->sectorsize_bits; 922459931ecSChris Mason 923459931ecSChris Mason leaf = path->nodes[0]; 9243212fa14SJosef Bacik csum_end = btrfs_item_size(leaf, path->slots[0]) / csum_size; 925265fdfa6SDavid Sterba csum_end <<= blocksize_bits; 926459931ecSChris Mason csum_end += key->offset; 927459931ecSChris Mason 928459931ecSChris Mason if (key->offset < bytenr && csum_end <= end_byte) { 929459931ecSChris Mason /* 930459931ecSChris Mason * [ bytenr - len ] 931459931ecSChris Mason * [ ] 932459931ecSChris Mason * [csum ] 933459931ecSChris Mason * A simple truncate off the end of the item 934459931ecSChris Mason */ 935459931ecSChris Mason u32 new_size = (bytenr - key->offset) >> blocksize_bits; 936459931ecSChris Mason new_size *= csum_size; 93778ac4f9eSDavid Sterba btrfs_truncate_item(path, new_size, 1); 938459931ecSChris Mason } else if (key->offset >= bytenr && csum_end > end_byte && 939459931ecSChris Mason end_byte > key->offset) { 940459931ecSChris Mason /* 941459931ecSChris Mason * [ bytenr - len ] 942459931ecSChris Mason * [ ] 943459931ecSChris Mason * [csum ] 944459931ecSChris Mason * we need to truncate from the beginning of the csum 945459931ecSChris Mason */ 946459931ecSChris Mason u32 new_size = (csum_end - end_byte) >> blocksize_bits; 947459931ecSChris Mason new_size *= csum_size; 948459931ecSChris Mason 94978ac4f9eSDavid Sterba btrfs_truncate_item(path, new_size, 0); 950459931ecSChris Mason 951459931ecSChris Mason key->offset = end_byte; 9520b246afaSJeff Mahoney btrfs_set_item_key_safe(fs_info, path, key); 953459931ecSChris Mason } else { 954459931ecSChris Mason BUG(); 955459931ecSChris Mason } 956459931ecSChris Mason } 957459931ecSChris Mason 958459931ecSChris Mason /* 95943dd529aSDavid Sterba * Delete the csum items from the csum tree for a given range of bytes. 960459931ecSChris Mason */ 961459931ecSChris Mason int btrfs_del_csums(struct btrfs_trans_handle *trans, 96240e046acSFilipe Manana struct btrfs_root *root, u64 bytenr, u64 len) 963459931ecSChris Mason { 96440e046acSFilipe Manana struct btrfs_fs_info *fs_info = trans->fs_info; 965459931ecSChris Mason struct btrfs_path *path; 966459931ecSChris Mason struct btrfs_key key; 967459931ecSChris Mason u64 end_byte = bytenr + len; 968459931ecSChris Mason u64 csum_end; 969459931ecSChris Mason struct extent_buffer *leaf; 970b86652beSJosef Bacik int ret = 0; 971223486c2SDavid Sterba const u32 csum_size = fs_info->csum_size; 972265fdfa6SDavid Sterba u32 blocksize_bits = fs_info->sectorsize_bits; 973459931ecSChris Mason 97484d2d6c7SJosef Bacik ASSERT(root->root_key.objectid == BTRFS_CSUM_TREE_OBJECTID || 97540e046acSFilipe Manana root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID); 97640e046acSFilipe Manana 977459931ecSChris Mason path = btrfs_alloc_path(); 9782a29edc6Sliubo if (!path) 9792a29edc6Sliubo return -ENOMEM; 980459931ecSChris Mason 981459931ecSChris Mason while (1) { 982459931ecSChris Mason key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 983459931ecSChris Mason key.offset = end_byte - 1; 984459931ecSChris Mason key.type = BTRFS_EXTENT_CSUM_KEY; 985459931ecSChris Mason 986459931ecSChris Mason ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 987459931ecSChris Mason if (ret > 0) { 988b86652beSJosef Bacik ret = 0; 989459931ecSChris Mason if (path->slots[0] == 0) 99065a246c5STsutomu Itoh break; 991459931ecSChris Mason path->slots[0]--; 992ad0397a7SJosef Bacik } else if (ret < 0) { 99365a246c5STsutomu Itoh break; 994459931ecSChris Mason } 995ad0397a7SJosef Bacik 996459931ecSChris Mason leaf = path->nodes[0]; 997459931ecSChris Mason btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 998459931ecSChris Mason 999459931ecSChris Mason if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || 1000459931ecSChris Mason key.type != BTRFS_EXTENT_CSUM_KEY) { 1001459931ecSChris Mason break; 1002459931ecSChris Mason } 1003459931ecSChris Mason 1004459931ecSChris Mason if (key.offset >= end_byte) 1005459931ecSChris Mason break; 1006459931ecSChris Mason 10073212fa14SJosef Bacik csum_end = btrfs_item_size(leaf, path->slots[0]) / csum_size; 1008459931ecSChris Mason csum_end <<= blocksize_bits; 1009459931ecSChris Mason csum_end += key.offset; 1010459931ecSChris Mason 1011459931ecSChris Mason /* this csum ends before we start, we're done */ 1012459931ecSChris Mason if (csum_end <= bytenr) 1013459931ecSChris Mason break; 1014459931ecSChris Mason 1015459931ecSChris Mason /* delete the entire item, it is inside our range */ 1016459931ecSChris Mason if (key.offset >= bytenr && csum_end <= end_byte) { 10176f546216SFilipe Manana int del_nr = 1; 10186f546216SFilipe Manana 10196f546216SFilipe Manana /* 10206f546216SFilipe Manana * Check how many csum items preceding this one in this 10216f546216SFilipe Manana * leaf correspond to our range and then delete them all 10226f546216SFilipe Manana * at once. 10236f546216SFilipe Manana */ 10246f546216SFilipe Manana if (key.offset > bytenr && path->slots[0] > 0) { 10256f546216SFilipe Manana int slot = path->slots[0] - 1; 10266f546216SFilipe Manana 10276f546216SFilipe Manana while (slot >= 0) { 10286f546216SFilipe Manana struct btrfs_key pk; 10296f546216SFilipe Manana 10306f546216SFilipe Manana btrfs_item_key_to_cpu(leaf, &pk, slot); 10316f546216SFilipe Manana if (pk.offset < bytenr || 10326f546216SFilipe Manana pk.type != BTRFS_EXTENT_CSUM_KEY || 10336f546216SFilipe Manana pk.objectid != 10346f546216SFilipe Manana BTRFS_EXTENT_CSUM_OBJECTID) 10356f546216SFilipe Manana break; 10366f546216SFilipe Manana path->slots[0] = slot; 10376f546216SFilipe Manana del_nr++; 10386f546216SFilipe Manana key.offset = pk.offset; 10396f546216SFilipe Manana slot--; 10406f546216SFilipe Manana } 10416f546216SFilipe Manana } 10426f546216SFilipe Manana ret = btrfs_del_items(trans, root, path, 10436f546216SFilipe Manana path->slots[0], del_nr); 104465a246c5STsutomu Itoh if (ret) 1045b86652beSJosef Bacik break; 1046dcbdd4dcSChris Mason if (key.offset == bytenr) 1047dcbdd4dcSChris Mason break; 1048459931ecSChris Mason } else if (key.offset < bytenr && csum_end > end_byte) { 1049459931ecSChris Mason unsigned long offset; 1050459931ecSChris Mason unsigned long shift_len; 1051459931ecSChris Mason unsigned long item_offset; 1052459931ecSChris Mason /* 1053459931ecSChris Mason * [ bytenr - len ] 1054459931ecSChris Mason * [csum ] 1055459931ecSChris Mason * 1056459931ecSChris Mason * Our bytes are in the middle of the csum, 1057459931ecSChris Mason * we need to split this item and insert a new one. 1058459931ecSChris Mason * 1059459931ecSChris Mason * But we can't drop the path because the 1060459931ecSChris Mason * csum could change, get removed, extended etc. 1061459931ecSChris Mason * 1062459931ecSChris Mason * The trick here is the max size of a csum item leaves 1063459931ecSChris Mason * enough room in the tree block for a single 1064459931ecSChris Mason * item header. So, we split the item in place, 1065459931ecSChris Mason * adding a new header pointing to the existing 1066459931ecSChris Mason * bytes. Then we loop around again and we have 1067459931ecSChris Mason * a nicely formed csum item that we can neatly 1068459931ecSChris Mason * truncate. 1069459931ecSChris Mason */ 1070459931ecSChris Mason offset = (bytenr - key.offset) >> blocksize_bits; 1071459931ecSChris Mason offset *= csum_size; 1072459931ecSChris Mason 1073459931ecSChris Mason shift_len = (len >> blocksize_bits) * csum_size; 1074459931ecSChris Mason 1075459931ecSChris Mason item_offset = btrfs_item_ptr_offset(leaf, 1076459931ecSChris Mason path->slots[0]); 1077459931ecSChris Mason 1078b159fa28SDavid Sterba memzero_extent_buffer(leaf, item_offset + offset, 1079459931ecSChris Mason shift_len); 1080459931ecSChris Mason key.offset = bytenr; 1081459931ecSChris Mason 1082459931ecSChris Mason /* 1083459931ecSChris Mason * btrfs_split_item returns -EAGAIN when the 1084459931ecSChris Mason * item changed size or key 1085459931ecSChris Mason */ 1086459931ecSChris Mason ret = btrfs_split_item(trans, root, path, &key, offset); 108779787eaaSJeff Mahoney if (ret && ret != -EAGAIN) { 108866642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 1089b86652beSJosef Bacik break; 109079787eaaSJeff Mahoney } 1091b86652beSJosef Bacik ret = 0; 1092459931ecSChris Mason 1093459931ecSChris Mason key.offset = end_byte - 1; 1094459931ecSChris Mason } else { 10952ff7e61eSJeff Mahoney truncate_one_csum(fs_info, path, &key, bytenr, len); 1096dcbdd4dcSChris Mason if (key.offset < bytenr) 1097dcbdd4dcSChris Mason break; 1098459931ecSChris Mason } 1099b3b4aa74SDavid Sterba btrfs_release_path(path); 1100459931ecSChris Mason } 1101459931ecSChris Mason btrfs_free_path(path); 110265a246c5STsutomu Itoh return ret; 1103459931ecSChris Mason } 1104459931ecSChris Mason 1105ea7036deSFilipe Manana static int find_next_csum_offset(struct btrfs_root *root, 1106ea7036deSFilipe Manana struct btrfs_path *path, 1107ea7036deSFilipe Manana u64 *next_offset) 1108ea7036deSFilipe Manana { 1109ea7036deSFilipe Manana const u32 nritems = btrfs_header_nritems(path->nodes[0]); 1110ea7036deSFilipe Manana struct btrfs_key found_key; 1111ea7036deSFilipe Manana int slot = path->slots[0] + 1; 1112ea7036deSFilipe Manana int ret; 1113ea7036deSFilipe Manana 1114ea7036deSFilipe Manana if (nritems == 0 || slot >= nritems) { 1115ea7036deSFilipe Manana ret = btrfs_next_leaf(root, path); 1116ea7036deSFilipe Manana if (ret < 0) { 1117ea7036deSFilipe Manana return ret; 1118ea7036deSFilipe Manana } else if (ret > 0) { 1119ea7036deSFilipe Manana *next_offset = (u64)-1; 1120ea7036deSFilipe Manana return 0; 1121ea7036deSFilipe Manana } 1122ea7036deSFilipe Manana slot = path->slots[0]; 1123ea7036deSFilipe Manana } 1124ea7036deSFilipe Manana 1125ea7036deSFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot); 1126ea7036deSFilipe Manana 1127ea7036deSFilipe Manana if (found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || 1128ea7036deSFilipe Manana found_key.type != BTRFS_EXTENT_CSUM_KEY) 1129ea7036deSFilipe Manana *next_offset = (u64)-1; 1130ea7036deSFilipe Manana else 1131ea7036deSFilipe Manana *next_offset = found_key.offset; 1132ea7036deSFilipe Manana 1133ea7036deSFilipe Manana return 0; 1134ea7036deSFilipe Manana } 1135ea7036deSFilipe Manana 1136065631f6SChris Mason int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, 1137d20f7043SChris Mason struct btrfs_root *root, 1138e6dcd2dcSChris Mason struct btrfs_ordered_sum *sums) 1139f254e52cSChris Mason { 11400b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1141f254e52cSChris Mason struct btrfs_key file_key; 11426567e837SChris Mason struct btrfs_key found_key; 11435caf2a00SChris Mason struct btrfs_path *path; 1144f254e52cSChris Mason struct btrfs_csum_item *item; 1145065631f6SChris Mason struct btrfs_csum_item *item_end; 1146ff79f819SChris Mason struct extent_buffer *leaf = NULL; 1147f51a4a18SMiao Xie u64 next_offset; 1148f51a4a18SMiao Xie u64 total_bytes = 0; 11496567e837SChris Mason u64 csum_offset; 1150f51a4a18SMiao Xie u64 bytenr; 1151f578d4bdSChris Mason u32 ins_size; 1152f51a4a18SMiao Xie int index = 0; 1153f51a4a18SMiao Xie int found_next; 1154f51a4a18SMiao Xie int ret; 1155223486c2SDavid Sterba const u32 csum_size = fs_info->csum_size; 11566e92f5e6SChris Mason 11575caf2a00SChris Mason path = btrfs_alloc_path(); 1158d8926bb3SMark Fasheh if (!path) 1159d8926bb3SMark Fasheh return -ENOMEM; 1160065631f6SChris Mason again: 1161065631f6SChris Mason next_offset = (u64)-1; 1162065631f6SChris Mason found_next = 0; 1163f51a4a18SMiao Xie bytenr = sums->bytenr + total_bytes; 1164d20f7043SChris Mason file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 1165f51a4a18SMiao Xie file_key.offset = bytenr; 1166962a298fSDavid Sterba file_key.type = BTRFS_EXTENT_CSUM_KEY; 1167a429e513SChris Mason 1168f51a4a18SMiao Xie item = btrfs_lookup_csum(trans, root, path, bytenr, 1); 1169ff79f819SChris Mason if (!IS_ERR(item)) { 1170639cb586SChris Mason ret = 0; 1171f51a4a18SMiao Xie leaf = path->nodes[0]; 1172f51a4a18SMiao Xie item_end = btrfs_item_ptr(leaf, path->slots[0], 1173f51a4a18SMiao Xie struct btrfs_csum_item); 1174f51a4a18SMiao Xie item_end = (struct btrfs_csum_item *)((char *)item_end + 11753212fa14SJosef Bacik btrfs_item_size(leaf, path->slots[0])); 1176a429e513SChris Mason goto found; 1177ff79f819SChris Mason } 1178a429e513SChris Mason ret = PTR_ERR(item); 11794a500fd1SYan, Zheng if (ret != -EFBIG && ret != -ENOENT) 1180918cdf44SFilipe Manana goto out; 11814a500fd1SYan, Zheng 1182a429e513SChris Mason if (ret == -EFBIG) { 1183a429e513SChris Mason u32 item_size; 1184a429e513SChris Mason /* we found one, but it isn't big enough yet */ 11855f39d397SChris Mason leaf = path->nodes[0]; 11863212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 1187607d432dSJosef Bacik if ((item_size / csum_size) >= 11880b246afaSJeff Mahoney MAX_CSUM_ITEMS(fs_info, csum_size)) { 1189a429e513SChris Mason /* already at max size, make a new one */ 1190a429e513SChris Mason goto insert; 1191a429e513SChris Mason } 1192a429e513SChris Mason } else { 1193ea7036deSFilipe Manana /* We didn't find a csum item, insert one. */ 1194ea7036deSFilipe Manana ret = find_next_csum_offset(root, path, &next_offset); 1195ea7036deSFilipe Manana if (ret < 0) 11967e4a3f7eSFilipe Manana goto out; 1197f578d4bdSChris Mason found_next = 1; 1198a429e513SChris Mason goto insert; 1199a429e513SChris Mason } 1200a429e513SChris Mason 1201a429e513SChris Mason /* 1202cc14600cSFilipe Manana * At this point, we know the tree has a checksum item that ends at an 1203cc14600cSFilipe Manana * offset matching the start of the checksum range we want to insert. 1204cc14600cSFilipe Manana * We try to extend that item as much as possible and then add as many 1205cc14600cSFilipe Manana * checksums to it as they fit. 1206cc14600cSFilipe Manana * 1207cc14600cSFilipe Manana * First check if the leaf has enough free space for at least one 1208cc14600cSFilipe Manana * checksum. If it has go directly to the item extension code, otherwise 1209cc14600cSFilipe Manana * release the path and do a search for insertion before the extension. 1210a429e513SChris Mason */ 1211cc14600cSFilipe Manana if (btrfs_leaf_free_space(leaf) >= csum_size) { 1212cc14600cSFilipe Manana btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1213cc14600cSFilipe Manana csum_offset = (bytenr - found_key.offset) >> 1214265fdfa6SDavid Sterba fs_info->sectorsize_bits; 1215cc14600cSFilipe Manana goto extend_csum; 1216cc14600cSFilipe Manana } 1217cc14600cSFilipe Manana 1218b3b4aa74SDavid Sterba btrfs_release_path(path); 12199a664971Sethanwu path->search_for_extension = 1; 12206567e837SChris Mason ret = btrfs_search_slot(trans, root, &file_key, path, 1221607d432dSJosef Bacik csum_size, 1); 12229a664971Sethanwu path->search_for_extension = 0; 12236567e837SChris Mason if (ret < 0) 1224918cdf44SFilipe Manana goto out; 1225459931ecSChris Mason 1226459931ecSChris Mason if (ret > 0) { 1227459931ecSChris Mason if (path->slots[0] == 0) 12286567e837SChris Mason goto insert; 12296567e837SChris Mason path->slots[0]--; 1230459931ecSChris Mason } 1231459931ecSChris Mason 12325f39d397SChris Mason leaf = path->nodes[0]; 12335f39d397SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1234265fdfa6SDavid Sterba csum_offset = (bytenr - found_key.offset) >> fs_info->sectorsize_bits; 1235459931ecSChris Mason 1236962a298fSDavid Sterba if (found_key.type != BTRFS_EXTENT_CSUM_KEY || 1237d20f7043SChris Mason found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || 12380b246afaSJeff Mahoney csum_offset >= MAX_CSUM_ITEMS(fs_info, csum_size)) { 12396567e837SChris Mason goto insert; 12406567e837SChris Mason } 1241459931ecSChris Mason 1242cc14600cSFilipe Manana extend_csum: 12433212fa14SJosef Bacik if (csum_offset == btrfs_item_size(leaf, path->slots[0]) / 1244607d432dSJosef Bacik csum_size) { 12452f697dc6SLiu Bo int extend_nr; 12462f697dc6SLiu Bo u64 tmp; 12472f697dc6SLiu Bo u32 diff; 1248459931ecSChris Mason 1249f51a4a18SMiao Xie tmp = sums->len - total_bytes; 1250265fdfa6SDavid Sterba tmp >>= fs_info->sectorsize_bits; 12512f697dc6SLiu Bo WARN_ON(tmp < 1); 1252ea7036deSFilipe Manana extend_nr = max_t(int, 1, tmp); 12532f697dc6SLiu Bo 1254ea7036deSFilipe Manana /* 1255ea7036deSFilipe Manana * A log tree can already have checksum items with a subset of 1256ea7036deSFilipe Manana * the checksums we are trying to log. This can happen after 1257ea7036deSFilipe Manana * doing a sequence of partial writes into prealloc extents and 1258ea7036deSFilipe Manana * fsyncs in between, with a full fsync logging a larger subrange 1259ea7036deSFilipe Manana * of an extent for which a previous fast fsync logged a smaller 1260ea7036deSFilipe Manana * subrange. And this happens in particular due to merging file 1261ea7036deSFilipe Manana * extent items when we complete an ordered extent for a range 1262ea7036deSFilipe Manana * covered by a prealloc extent - this is done at 1263ea7036deSFilipe Manana * btrfs_mark_extent_written(). 1264ea7036deSFilipe Manana * 1265ea7036deSFilipe Manana * So if we try to extend the previous checksum item, which has 1266ea7036deSFilipe Manana * a range that ends at the start of the range we want to insert, 1267ea7036deSFilipe Manana * make sure we don't extend beyond the start offset of the next 1268ea7036deSFilipe Manana * checksum item. If we are at the last item in the leaf, then 1269ea7036deSFilipe Manana * forget the optimization of extending and add a new checksum 1270ea7036deSFilipe Manana * item - it is not worth the complexity of releasing the path, 1271ea7036deSFilipe Manana * getting the first key for the next leaf, repeat the btree 1272ea7036deSFilipe Manana * search, etc, because log trees are temporary anyway and it 1273ea7036deSFilipe Manana * would only save a few bytes of leaf space. 1274ea7036deSFilipe Manana */ 1275ea7036deSFilipe Manana if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) { 1276ea7036deSFilipe Manana if (path->slots[0] + 1 >= 1277ea7036deSFilipe Manana btrfs_header_nritems(path->nodes[0])) { 1278ea7036deSFilipe Manana ret = find_next_csum_offset(root, path, &next_offset); 1279ea7036deSFilipe Manana if (ret < 0) 1280ea7036deSFilipe Manana goto out; 1281ea7036deSFilipe Manana found_next = 1; 1282ea7036deSFilipe Manana goto insert; 1283ea7036deSFilipe Manana } 1284ea7036deSFilipe Manana 1285ea7036deSFilipe Manana ret = find_next_csum_offset(root, path, &next_offset); 1286ea7036deSFilipe Manana if (ret < 0) 1287ea7036deSFilipe Manana goto out; 1288ea7036deSFilipe Manana 1289ea7036deSFilipe Manana tmp = (next_offset - bytenr) >> fs_info->sectorsize_bits; 1290ea7036deSFilipe Manana if (tmp <= INT_MAX) 1291ea7036deSFilipe Manana extend_nr = min_t(int, extend_nr, tmp); 1292ea7036deSFilipe Manana } 1293ea7036deSFilipe Manana 12942f697dc6SLiu Bo diff = (csum_offset + extend_nr) * csum_size; 12950b246afaSJeff Mahoney diff = min(diff, 12960b246afaSJeff Mahoney MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size); 1297459931ecSChris Mason 12983212fa14SJosef Bacik diff = diff - btrfs_item_size(leaf, path->slots[0]); 1299cc14600cSFilipe Manana diff = min_t(u32, btrfs_leaf_free_space(leaf), diff); 13002f697dc6SLiu Bo diff /= csum_size; 13012f697dc6SLiu Bo diff *= csum_size; 1302459931ecSChris Mason 1303c71dd880SDavid Sterba btrfs_extend_item(path, diff); 1304f51a4a18SMiao Xie ret = 0; 13056567e837SChris Mason goto csum; 13066567e837SChris Mason } 13076567e837SChris Mason 13086567e837SChris Mason insert: 1309b3b4aa74SDavid Sterba btrfs_release_path(path); 13106567e837SChris Mason csum_offset = 0; 1311f578d4bdSChris Mason if (found_next) { 13122f697dc6SLiu Bo u64 tmp; 1313d20f7043SChris Mason 1314f51a4a18SMiao Xie tmp = sums->len - total_bytes; 1315265fdfa6SDavid Sterba tmp >>= fs_info->sectorsize_bits; 13162f697dc6SLiu Bo tmp = min(tmp, (next_offset - file_key.offset) >> 1317265fdfa6SDavid Sterba fs_info->sectorsize_bits); 13182f697dc6SLiu Bo 131950d0446eSSeraphime Kirkovski tmp = max_t(u64, 1, tmp); 132050d0446eSSeraphime Kirkovski tmp = min_t(u64, tmp, MAX_CSUM_ITEMS(fs_info, csum_size)); 1321607d432dSJosef Bacik ins_size = csum_size * tmp; 1322f578d4bdSChris Mason } else { 1323607d432dSJosef Bacik ins_size = csum_size; 1324f578d4bdSChris Mason } 13255caf2a00SChris Mason ret = btrfs_insert_empty_item(trans, root, path, &file_key, 1326f578d4bdSChris Mason ins_size); 132754aa1f4dSChris Mason if (ret < 0) 1328918cdf44SFilipe Manana goto out; 1329fae7f21cSDulshani Gunawardhana if (WARN_ON(ret != 0)) 1330918cdf44SFilipe Manana goto out; 13315f39d397SChris Mason leaf = path->nodes[0]; 1332f51a4a18SMiao Xie csum: 13335f39d397SChris Mason item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item); 1334f51a4a18SMiao Xie item_end = (struct btrfs_csum_item *)((unsigned char *)item + 13353212fa14SJosef Bacik btrfs_item_size(leaf, path->slots[0])); 1336509659cdSChris Mason item = (struct btrfs_csum_item *)((unsigned char *)item + 1337607d432dSJosef Bacik csum_offset * csum_size); 1338b18c6685SChris Mason found: 1339265fdfa6SDavid Sterba ins_size = (u32)(sums->len - total_bytes) >> fs_info->sectorsize_bits; 1340f51a4a18SMiao Xie ins_size *= csum_size; 1341f51a4a18SMiao Xie ins_size = min_t(u32, (unsigned long)item_end - (unsigned long)item, 1342f51a4a18SMiao Xie ins_size); 1343f51a4a18SMiao Xie write_extent_buffer(leaf, sums->sums + index, (unsigned long)item, 1344f51a4a18SMiao Xie ins_size); 1345aadfeb6eSChris Mason 13461e25a2e3SJohannes Thumshirn index += ins_size; 1347f51a4a18SMiao Xie ins_size /= csum_size; 13480b246afaSJeff Mahoney total_bytes += ins_size * fs_info->sectorsize; 1349a6591715SChris Mason 13505caf2a00SChris Mason btrfs_mark_buffer_dirty(path->nodes[0]); 1351e6dcd2dcSChris Mason if (total_bytes < sums->len) { 1352b3b4aa74SDavid Sterba btrfs_release_path(path); 1353b9473439SChris Mason cond_resched(); 1354065631f6SChris Mason goto again; 1355065631f6SChris Mason } 135653863232SChris Mason out: 13575caf2a00SChris Mason btrfs_free_path(path); 1358f254e52cSChris Mason return ret; 1359f254e52cSChris Mason } 13607ffbb598SFilipe Manana 13619cdc5124SNikolay Borisov void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode, 13627ffbb598SFilipe Manana const struct btrfs_path *path, 13637ffbb598SFilipe Manana struct btrfs_file_extent_item *fi, 13647ffbb598SFilipe Manana struct extent_map *em) 13657ffbb598SFilipe Manana { 13663ffbd68cSDavid Sterba struct btrfs_fs_info *fs_info = inode->root->fs_info; 13679cdc5124SNikolay Borisov struct btrfs_root *root = inode->root; 13687ffbb598SFilipe Manana struct extent_buffer *leaf = path->nodes[0]; 13697ffbb598SFilipe Manana const int slot = path->slots[0]; 13707ffbb598SFilipe Manana struct btrfs_key key; 13717ffbb598SFilipe Manana u64 extent_start, extent_end; 13727ffbb598SFilipe Manana u64 bytenr; 13737ffbb598SFilipe Manana u8 type = btrfs_file_extent_type(leaf, fi); 13747ffbb598SFilipe Manana int compress_type = btrfs_file_extent_compression(leaf, fi); 13757ffbb598SFilipe Manana 13767ffbb598SFilipe Manana btrfs_item_key_to_cpu(leaf, &key, slot); 13777ffbb598SFilipe Manana extent_start = key.offset; 1378a5eeb3d1SFilipe Manana extent_end = btrfs_file_extent_end(path); 13797ffbb598SFilipe Manana em->ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 138040e7efe0SQu Wenruo em->generation = btrfs_file_extent_generation(leaf, fi); 13817ffbb598SFilipe Manana if (type == BTRFS_FILE_EXTENT_REG || 13827ffbb598SFilipe Manana type == BTRFS_FILE_EXTENT_PREALLOC) { 13837ffbb598SFilipe Manana em->start = extent_start; 13847ffbb598SFilipe Manana em->len = extent_end - extent_start; 13857ffbb598SFilipe Manana em->orig_start = extent_start - 13867ffbb598SFilipe Manana btrfs_file_extent_offset(leaf, fi); 13877ffbb598SFilipe Manana em->orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi); 13887ffbb598SFilipe Manana bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 13897ffbb598SFilipe Manana if (bytenr == 0) { 13907ffbb598SFilipe Manana em->block_start = EXTENT_MAP_HOLE; 13917ffbb598SFilipe Manana return; 13927ffbb598SFilipe Manana } 13937ffbb598SFilipe Manana if (compress_type != BTRFS_COMPRESS_NONE) { 13947ffbb598SFilipe Manana set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 13957ffbb598SFilipe Manana em->compress_type = compress_type; 13967ffbb598SFilipe Manana em->block_start = bytenr; 13977ffbb598SFilipe Manana em->block_len = em->orig_block_len; 13987ffbb598SFilipe Manana } else { 13997ffbb598SFilipe Manana bytenr += btrfs_file_extent_offset(leaf, fi); 14007ffbb598SFilipe Manana em->block_start = bytenr; 14017ffbb598SFilipe Manana em->block_len = em->len; 14027ffbb598SFilipe Manana if (type == BTRFS_FILE_EXTENT_PREALLOC) 14037ffbb598SFilipe Manana set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 14047ffbb598SFilipe Manana } 14057ffbb598SFilipe Manana } else if (type == BTRFS_FILE_EXTENT_INLINE) { 14067ffbb598SFilipe Manana em->block_start = EXTENT_MAP_INLINE; 14077ffbb598SFilipe Manana em->start = extent_start; 14087ffbb598SFilipe Manana em->len = extent_end - extent_start; 14097ffbb598SFilipe Manana /* 14107ffbb598SFilipe Manana * Initialize orig_start and block_len with the same values 14117ffbb598SFilipe Manana * as in inode.c:btrfs_get_extent(). 14127ffbb598SFilipe Manana */ 14137ffbb598SFilipe Manana em->orig_start = EXTENT_MAP_HOLE; 14147ffbb598SFilipe Manana em->block_len = (u64)-1; 14157ffbb598SFilipe Manana em->compress_type = compress_type; 1416280f15cbSQu Wenruo if (compress_type != BTRFS_COMPRESS_NONE) 1417280f15cbSQu Wenruo set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 14187ffbb598SFilipe Manana } else { 14190b246afaSJeff Mahoney btrfs_err(fs_info, 14209cdc5124SNikolay Borisov "unknown file extent item type %d, inode %llu, offset %llu, " 14219cdc5124SNikolay Borisov "root %llu", type, btrfs_ino(inode), extent_start, 14227ffbb598SFilipe Manana root->root_key.objectid); 14237ffbb598SFilipe Manana } 14247ffbb598SFilipe Manana } 1425a5eeb3d1SFilipe Manana 1426a5eeb3d1SFilipe Manana /* 1427a5eeb3d1SFilipe Manana * Returns the end offset (non inclusive) of the file extent item the given path 1428a5eeb3d1SFilipe Manana * points to. If it points to an inline extent, the returned offset is rounded 1429a5eeb3d1SFilipe Manana * up to the sector size. 1430a5eeb3d1SFilipe Manana */ 1431a5eeb3d1SFilipe Manana u64 btrfs_file_extent_end(const struct btrfs_path *path) 1432a5eeb3d1SFilipe Manana { 1433a5eeb3d1SFilipe Manana const struct extent_buffer *leaf = path->nodes[0]; 1434a5eeb3d1SFilipe Manana const int slot = path->slots[0]; 1435a5eeb3d1SFilipe Manana struct btrfs_file_extent_item *fi; 1436a5eeb3d1SFilipe Manana struct btrfs_key key; 1437a5eeb3d1SFilipe Manana u64 end; 1438a5eeb3d1SFilipe Manana 1439a5eeb3d1SFilipe Manana btrfs_item_key_to_cpu(leaf, &key, slot); 1440a5eeb3d1SFilipe Manana ASSERT(key.type == BTRFS_EXTENT_DATA_KEY); 1441a5eeb3d1SFilipe Manana fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 1442a5eeb3d1SFilipe Manana 1443a5eeb3d1SFilipe Manana if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) { 1444a5eeb3d1SFilipe Manana end = btrfs_file_extent_ram_bytes(leaf, fi); 1445a5eeb3d1SFilipe Manana end = ALIGN(key.offset + end, leaf->fs_info->sectorsize); 1446a5eeb3d1SFilipe Manana } else { 1447a5eeb3d1SFilipe Manana end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); 1448a5eeb3d1SFilipe Manana } 1449a5eeb3d1SFilipe Manana 1450a5eeb3d1SFilipe Manana return end; 1451a5eeb3d1SFilipe Manana } 1452