1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007 Oracle. All rights reserved. 4 */ 5 6 #include <linux/bio.h> 7 #include <linux/slab.h> 8 #include <linux/pagemap.h> 9 #include <linux/highmem.h> 10 #include <linux/sched/mm.h> 11 #include <crypto/hash.h> 12 #include "ctree.h" 13 #include "disk-io.h" 14 #include "transaction.h" 15 #include "volumes.h" 16 #include "print-tree.h" 17 #include "compression.h" 18 19 #define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \ 20 sizeof(struct btrfs_item) * 2) / \ 21 size) - 1)) 22 23 #define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \ 24 PAGE_SIZE)) 25 26 /** 27 * @inode - the inode we want to update the disk_i_size for 28 * @new_i_size - the i_size we want to set to, 0 if we use i_size 29 * 30 * With NO_HOLES set this simply sets the disk_is_size to whatever i_size_read() 31 * returns as it is perfectly fine with a file that has holes without hole file 32 * extent items. 33 * 34 * However without NO_HOLES we need to only return the area that is contiguous 35 * from the 0 offset of the file. Otherwise we could end up adjust i_size up 36 * to an extent that has a gap in between. 37 * 38 * Finally new_i_size should only be set in the case of truncate where we're not 39 * ready to use i_size_read() as the limiter yet. 40 */ 41 void btrfs_inode_safe_disk_i_size_write(struct inode *inode, u64 new_i_size) 42 { 43 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 44 u64 start, end, i_size; 45 int ret; 46 47 i_size = new_i_size ?: i_size_read(inode); 48 if (btrfs_fs_incompat(fs_info, NO_HOLES)) { 49 BTRFS_I(inode)->disk_i_size = i_size; 50 return; 51 } 52 53 spin_lock(&BTRFS_I(inode)->lock); 54 ret = find_contiguous_extent_bit(&BTRFS_I(inode)->file_extent_tree, 0, 55 &start, &end, EXTENT_DIRTY); 56 if (!ret && start == 0) 57 i_size = min(i_size, end + 1); 58 else 59 i_size = 0; 60 BTRFS_I(inode)->disk_i_size = i_size; 61 spin_unlock(&BTRFS_I(inode)->lock); 62 } 63 64 /** 65 * @inode - the inode we're modifying 66 * @start - the start file offset of the file extent we've inserted 67 * @len - the logical length of the file extent item 68 * 69 * Call when we are inserting a new file extent where there was none before. 70 * Does not need to call this in the case where we're replacing an existing file 71 * extent, however if not sure it's fine to call this multiple times. 72 * 73 * The start and len must match the file extent item, so thus must be sectorsize 74 * aligned. 75 */ 76 int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start, 77 u64 len) 78 { 79 if (len == 0) 80 return 0; 81 82 ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize)); 83 84 if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES)) 85 return 0; 86 return set_extent_bits(&inode->file_extent_tree, start, start + len - 1, 87 EXTENT_DIRTY); 88 } 89 90 /** 91 * @inode - the inode we're modifying 92 * @start - the start file offset of the file extent we've inserted 93 * @len - the logical length of the file extent item 94 * 95 * Called when we drop a file extent, for example when we truncate. Doesn't 96 * need to be called for cases where we're replacing a file extent, like when 97 * we've COWed a file extent. 98 * 99 * The start and len must match the file extent item, so thus must be sectorsize 100 * aligned. 101 */ 102 int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start, 103 u64 len) 104 { 105 if (len == 0) 106 return 0; 107 108 ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize) || 109 len == (u64)-1); 110 111 if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES)) 112 return 0; 113 return clear_extent_bit(&inode->file_extent_tree, start, 114 start + len - 1, EXTENT_DIRTY, 0, 0, NULL); 115 } 116 117 static inline u32 max_ordered_sum_bytes(struct btrfs_fs_info *fs_info, 118 u16 csum_size) 119 { 120 u32 ncsums = (PAGE_SIZE - sizeof(struct btrfs_ordered_sum)) / csum_size; 121 122 return ncsums * fs_info->sectorsize; 123 } 124 125 int btrfs_insert_file_extent(struct btrfs_trans_handle *trans, 126 struct btrfs_root *root, 127 u64 objectid, u64 pos, 128 u64 disk_offset, u64 disk_num_bytes, 129 u64 num_bytes, u64 offset, u64 ram_bytes, 130 u8 compression, u8 encryption, u16 other_encoding) 131 { 132 int ret = 0; 133 struct btrfs_file_extent_item *item; 134 struct btrfs_key file_key; 135 struct btrfs_path *path; 136 struct extent_buffer *leaf; 137 138 path = btrfs_alloc_path(); 139 if (!path) 140 return -ENOMEM; 141 file_key.objectid = objectid; 142 file_key.offset = pos; 143 file_key.type = BTRFS_EXTENT_DATA_KEY; 144 145 path->leave_spinning = 1; 146 ret = btrfs_insert_empty_item(trans, root, path, &file_key, 147 sizeof(*item)); 148 if (ret < 0) 149 goto out; 150 BUG_ON(ret); /* Can't happen */ 151 leaf = path->nodes[0]; 152 item = btrfs_item_ptr(leaf, path->slots[0], 153 struct btrfs_file_extent_item); 154 btrfs_set_file_extent_disk_bytenr(leaf, item, disk_offset); 155 btrfs_set_file_extent_disk_num_bytes(leaf, item, disk_num_bytes); 156 btrfs_set_file_extent_offset(leaf, item, offset); 157 btrfs_set_file_extent_num_bytes(leaf, item, num_bytes); 158 btrfs_set_file_extent_ram_bytes(leaf, item, ram_bytes); 159 btrfs_set_file_extent_generation(leaf, item, trans->transid); 160 btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); 161 btrfs_set_file_extent_compression(leaf, item, compression); 162 btrfs_set_file_extent_encryption(leaf, item, encryption); 163 btrfs_set_file_extent_other_encoding(leaf, item, other_encoding); 164 165 btrfs_mark_buffer_dirty(leaf); 166 out: 167 btrfs_free_path(path); 168 return ret; 169 } 170 171 static struct btrfs_csum_item * 172 btrfs_lookup_csum(struct btrfs_trans_handle *trans, 173 struct btrfs_root *root, 174 struct btrfs_path *path, 175 u64 bytenr, int cow) 176 { 177 struct btrfs_fs_info *fs_info = root->fs_info; 178 int ret; 179 struct btrfs_key file_key; 180 struct btrfs_key found_key; 181 struct btrfs_csum_item *item; 182 struct extent_buffer *leaf; 183 u64 csum_offset = 0; 184 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 185 int csums_in_item; 186 187 file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 188 file_key.offset = bytenr; 189 file_key.type = BTRFS_EXTENT_CSUM_KEY; 190 ret = btrfs_search_slot(trans, root, &file_key, path, 0, cow); 191 if (ret < 0) 192 goto fail; 193 leaf = path->nodes[0]; 194 if (ret > 0) { 195 ret = 1; 196 if (path->slots[0] == 0) 197 goto fail; 198 path->slots[0]--; 199 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 200 if (found_key.type != BTRFS_EXTENT_CSUM_KEY) 201 goto fail; 202 203 csum_offset = (bytenr - found_key.offset) >> 204 fs_info->sb->s_blocksize_bits; 205 csums_in_item = btrfs_item_size_nr(leaf, path->slots[0]); 206 csums_in_item /= csum_size; 207 208 if (csum_offset == csums_in_item) { 209 ret = -EFBIG; 210 goto fail; 211 } else if (csum_offset > csums_in_item) { 212 goto fail; 213 } 214 } 215 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item); 216 item = (struct btrfs_csum_item *)((unsigned char *)item + 217 csum_offset * csum_size); 218 return item; 219 fail: 220 if (ret > 0) 221 ret = -ENOENT; 222 return ERR_PTR(ret); 223 } 224 225 int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, 226 struct btrfs_root *root, 227 struct btrfs_path *path, u64 objectid, 228 u64 offset, int mod) 229 { 230 int ret; 231 struct btrfs_key file_key; 232 int ins_len = mod < 0 ? -1 : 0; 233 int cow = mod != 0; 234 235 file_key.objectid = objectid; 236 file_key.offset = offset; 237 file_key.type = BTRFS_EXTENT_DATA_KEY; 238 ret = btrfs_search_slot(trans, root, &file_key, path, ins_len, cow); 239 return ret; 240 } 241 242 /** 243 * btrfs_lookup_bio_sums - Look up checksums for a bio. 244 * @inode: inode that the bio is for. 245 * @bio: bio to look up. 246 * @offset: Unless (u64)-1, look up checksums for this offset in the file. 247 * If (u64)-1, use the page offsets from the bio instead. 248 * @dst: Buffer of size nblocks * btrfs_super_csum_size() used to return 249 * checksum (nblocks = bio->bi_iter.bi_size / fs_info->sectorsize). If 250 * NULL, the checksum buffer is allocated and returned in 251 * btrfs_io_bio(bio)->csum instead. 252 * 253 * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise. 254 */ 255 blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, 256 u64 offset, u8 *dst) 257 { 258 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 259 struct bio_vec bvec; 260 struct bvec_iter iter; 261 struct btrfs_csum_item *item = NULL; 262 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 263 struct btrfs_path *path; 264 const bool page_offsets = (offset == (u64)-1); 265 u8 *csum; 266 u64 item_start_offset = 0; 267 u64 item_last_offset = 0; 268 u64 disk_bytenr; 269 u64 page_bytes_left; 270 u32 diff; 271 int nblocks; 272 int count = 0; 273 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 274 275 path = btrfs_alloc_path(); 276 if (!path) 277 return BLK_STS_RESOURCE; 278 279 nblocks = bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits; 280 if (!dst) { 281 struct btrfs_io_bio *btrfs_bio = btrfs_io_bio(bio); 282 283 if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) { 284 btrfs_bio->csum = kmalloc_array(nblocks, csum_size, 285 GFP_NOFS); 286 if (!btrfs_bio->csum) { 287 btrfs_free_path(path); 288 return BLK_STS_RESOURCE; 289 } 290 } else { 291 btrfs_bio->csum = btrfs_bio->csum_inline; 292 } 293 csum = btrfs_bio->csum; 294 } else { 295 csum = dst; 296 } 297 298 if (bio->bi_iter.bi_size > PAGE_SIZE * 8) 299 path->reada = READA_FORWARD; 300 301 /* 302 * the free space stuff is only read when it hasn't been 303 * updated in the current transaction. So, we can safely 304 * read from the commit root and sidestep a nasty deadlock 305 * between reading the free space cache and updating the csum tree. 306 */ 307 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 308 path->search_commit_root = 1; 309 path->skip_locking = 1; 310 } 311 312 disk_bytenr = (u64)bio->bi_iter.bi_sector << 9; 313 314 bio_for_each_segment(bvec, bio, iter) { 315 page_bytes_left = bvec.bv_len; 316 if (count) 317 goto next; 318 319 if (page_offsets) 320 offset = page_offset(bvec.bv_page) + bvec.bv_offset; 321 count = btrfs_find_ordered_sum(BTRFS_I(inode), offset, 322 disk_bytenr, csum, nblocks); 323 if (count) 324 goto found; 325 326 if (!item || disk_bytenr < item_start_offset || 327 disk_bytenr >= item_last_offset) { 328 struct btrfs_key found_key; 329 u32 item_size; 330 331 if (item) 332 btrfs_release_path(path); 333 item = btrfs_lookup_csum(NULL, fs_info->csum_root, 334 path, disk_bytenr, 0); 335 if (IS_ERR(item)) { 336 count = 1; 337 memset(csum, 0, csum_size); 338 if (BTRFS_I(inode)->root->root_key.objectid == 339 BTRFS_DATA_RELOC_TREE_OBJECTID) { 340 set_extent_bits(io_tree, offset, 341 offset + fs_info->sectorsize - 1, 342 EXTENT_NODATASUM); 343 } else { 344 btrfs_info_rl(fs_info, 345 "no csum found for inode %llu start %llu", 346 btrfs_ino(BTRFS_I(inode)), offset); 347 } 348 item = NULL; 349 btrfs_release_path(path); 350 goto found; 351 } 352 btrfs_item_key_to_cpu(path->nodes[0], &found_key, 353 path->slots[0]); 354 355 item_start_offset = found_key.offset; 356 item_size = btrfs_item_size_nr(path->nodes[0], 357 path->slots[0]); 358 item_last_offset = item_start_offset + 359 (item_size / csum_size) * 360 fs_info->sectorsize; 361 item = btrfs_item_ptr(path->nodes[0], path->slots[0], 362 struct btrfs_csum_item); 363 } 364 /* 365 * this byte range must be able to fit inside 366 * a single leaf so it will also fit inside a u32 367 */ 368 diff = disk_bytenr - item_start_offset; 369 diff = diff / fs_info->sectorsize; 370 diff = diff * csum_size; 371 count = min_t(int, nblocks, (item_last_offset - disk_bytenr) >> 372 inode->i_sb->s_blocksize_bits); 373 read_extent_buffer(path->nodes[0], csum, 374 ((unsigned long)item) + diff, 375 csum_size * count); 376 found: 377 csum += count * csum_size; 378 nblocks -= count; 379 next: 380 while (count > 0) { 381 count--; 382 disk_bytenr += fs_info->sectorsize; 383 offset += fs_info->sectorsize; 384 page_bytes_left -= fs_info->sectorsize; 385 if (!page_bytes_left) 386 break; /* move to next bio */ 387 } 388 } 389 390 WARN_ON_ONCE(count); 391 btrfs_free_path(path); 392 return BLK_STS_OK; 393 } 394 395 int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, 396 struct list_head *list, int search_commit) 397 { 398 struct btrfs_fs_info *fs_info = root->fs_info; 399 struct btrfs_key key; 400 struct btrfs_path *path; 401 struct extent_buffer *leaf; 402 struct btrfs_ordered_sum *sums; 403 struct btrfs_csum_item *item; 404 LIST_HEAD(tmplist); 405 unsigned long offset; 406 int ret; 407 size_t size; 408 u64 csum_end; 409 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 410 411 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 412 IS_ALIGNED(end + 1, fs_info->sectorsize)); 413 414 path = btrfs_alloc_path(); 415 if (!path) 416 return -ENOMEM; 417 418 if (search_commit) { 419 path->skip_locking = 1; 420 path->reada = READA_FORWARD; 421 path->search_commit_root = 1; 422 } 423 424 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 425 key.offset = start; 426 key.type = BTRFS_EXTENT_CSUM_KEY; 427 428 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 429 if (ret < 0) 430 goto fail; 431 if (ret > 0 && path->slots[0] > 0) { 432 leaf = path->nodes[0]; 433 btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1); 434 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID && 435 key.type == BTRFS_EXTENT_CSUM_KEY) { 436 offset = (start - key.offset) >> 437 fs_info->sb->s_blocksize_bits; 438 if (offset * csum_size < 439 btrfs_item_size_nr(leaf, path->slots[0] - 1)) 440 path->slots[0]--; 441 } 442 } 443 444 while (start <= end) { 445 leaf = path->nodes[0]; 446 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 447 ret = btrfs_next_leaf(root, path); 448 if (ret < 0) 449 goto fail; 450 if (ret > 0) 451 break; 452 leaf = path->nodes[0]; 453 } 454 455 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 456 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || 457 key.type != BTRFS_EXTENT_CSUM_KEY || 458 key.offset > end) 459 break; 460 461 if (key.offset > start) 462 start = key.offset; 463 464 size = btrfs_item_size_nr(leaf, path->slots[0]); 465 csum_end = key.offset + (size / csum_size) * fs_info->sectorsize; 466 if (csum_end <= start) { 467 path->slots[0]++; 468 continue; 469 } 470 471 csum_end = min(csum_end, end + 1); 472 item = btrfs_item_ptr(path->nodes[0], path->slots[0], 473 struct btrfs_csum_item); 474 while (start < csum_end) { 475 size = min_t(size_t, csum_end - start, 476 max_ordered_sum_bytes(fs_info, csum_size)); 477 sums = kzalloc(btrfs_ordered_sum_size(fs_info, size), 478 GFP_NOFS); 479 if (!sums) { 480 ret = -ENOMEM; 481 goto fail; 482 } 483 484 sums->bytenr = start; 485 sums->len = (int)size; 486 487 offset = (start - key.offset) >> 488 fs_info->sb->s_blocksize_bits; 489 offset *= csum_size; 490 size >>= fs_info->sb->s_blocksize_bits; 491 492 read_extent_buffer(path->nodes[0], 493 sums->sums, 494 ((unsigned long)item) + offset, 495 csum_size * size); 496 497 start += fs_info->sectorsize * size; 498 list_add_tail(&sums->list, &tmplist); 499 } 500 path->slots[0]++; 501 } 502 ret = 0; 503 fail: 504 while (ret < 0 && !list_empty(&tmplist)) { 505 sums = list_entry(tmplist.next, struct btrfs_ordered_sum, list); 506 list_del(&sums->list); 507 kfree(sums); 508 } 509 list_splice_tail(&tmplist, list); 510 511 btrfs_free_path(path); 512 return ret; 513 } 514 515 /* 516 * btrfs_csum_one_bio - Calculates checksums of the data contained inside a bio 517 * @inode: Owner of the data inside the bio 518 * @bio: Contains the data to be checksummed 519 * @file_start: offset in file this bio begins to describe 520 * @contig: Boolean. If true/1 means all bio vecs in this bio are 521 * contiguous and they begin at @file_start in the file. False/0 522 * means this bio can contains potentially discontigous bio vecs 523 * so the logical offset of each should be calculated separately. 524 */ 525 blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, 526 u64 file_start, int contig) 527 { 528 struct btrfs_fs_info *fs_info = inode->root->fs_info; 529 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); 530 struct btrfs_ordered_sum *sums; 531 struct btrfs_ordered_extent *ordered = NULL; 532 char *data; 533 struct bvec_iter iter; 534 struct bio_vec bvec; 535 int index; 536 int nr_sectors; 537 unsigned long total_bytes = 0; 538 unsigned long this_sum_bytes = 0; 539 int i; 540 u64 offset; 541 unsigned nofs_flag; 542 const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 543 544 nofs_flag = memalloc_nofs_save(); 545 sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size), 546 GFP_KERNEL); 547 memalloc_nofs_restore(nofs_flag); 548 549 if (!sums) 550 return BLK_STS_RESOURCE; 551 552 sums->len = bio->bi_iter.bi_size; 553 INIT_LIST_HEAD(&sums->list); 554 555 if (contig) 556 offset = file_start; 557 else 558 offset = 0; /* shut up gcc */ 559 560 sums->bytenr = (u64)bio->bi_iter.bi_sector << 9; 561 index = 0; 562 563 shash->tfm = fs_info->csum_shash; 564 565 bio_for_each_segment(bvec, bio, iter) { 566 if (!contig) 567 offset = page_offset(bvec.bv_page) + bvec.bv_offset; 568 569 if (!ordered) { 570 ordered = btrfs_lookup_ordered_extent(inode, offset); 571 BUG_ON(!ordered); /* Logic error */ 572 } 573 574 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, 575 bvec.bv_len + fs_info->sectorsize 576 - 1); 577 578 for (i = 0; i < nr_sectors; i++) { 579 if (offset >= ordered->file_offset + ordered->num_bytes || 580 offset < ordered->file_offset) { 581 unsigned long bytes_left; 582 583 sums->len = this_sum_bytes; 584 this_sum_bytes = 0; 585 btrfs_add_ordered_sum(ordered, sums); 586 btrfs_put_ordered_extent(ordered); 587 588 bytes_left = bio->bi_iter.bi_size - total_bytes; 589 590 nofs_flag = memalloc_nofs_save(); 591 sums = kvzalloc(btrfs_ordered_sum_size(fs_info, 592 bytes_left), GFP_KERNEL); 593 memalloc_nofs_restore(nofs_flag); 594 BUG_ON(!sums); /* -ENOMEM */ 595 sums->len = bytes_left; 596 ordered = btrfs_lookup_ordered_extent(inode, 597 offset); 598 ASSERT(ordered); /* Logic error */ 599 sums->bytenr = ((u64)bio->bi_iter.bi_sector << 9) 600 + total_bytes; 601 index = 0; 602 } 603 604 data = kmap_atomic(bvec.bv_page); 605 crypto_shash_digest(shash, data + bvec.bv_offset 606 + (i * fs_info->sectorsize), 607 fs_info->sectorsize, 608 sums->sums + index); 609 kunmap_atomic(data); 610 index += csum_size; 611 offset += fs_info->sectorsize; 612 this_sum_bytes += fs_info->sectorsize; 613 total_bytes += fs_info->sectorsize; 614 } 615 616 } 617 this_sum_bytes = 0; 618 btrfs_add_ordered_sum(ordered, sums); 619 btrfs_put_ordered_extent(ordered); 620 return 0; 621 } 622 623 /* 624 * helper function for csum removal, this expects the 625 * key to describe the csum pointed to by the path, and it expects 626 * the csum to overlap the range [bytenr, len] 627 * 628 * The csum should not be entirely contained in the range and the 629 * range should not be entirely contained in the csum. 630 * 631 * This calls btrfs_truncate_item with the correct args based on the 632 * overlap, and fixes up the key as required. 633 */ 634 static noinline void truncate_one_csum(struct btrfs_fs_info *fs_info, 635 struct btrfs_path *path, 636 struct btrfs_key *key, 637 u64 bytenr, u64 len) 638 { 639 struct extent_buffer *leaf; 640 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 641 u64 csum_end; 642 u64 end_byte = bytenr + len; 643 u32 blocksize_bits = fs_info->sb->s_blocksize_bits; 644 645 leaf = path->nodes[0]; 646 csum_end = btrfs_item_size_nr(leaf, path->slots[0]) / csum_size; 647 csum_end <<= fs_info->sb->s_blocksize_bits; 648 csum_end += key->offset; 649 650 if (key->offset < bytenr && csum_end <= end_byte) { 651 /* 652 * [ bytenr - len ] 653 * [ ] 654 * [csum ] 655 * A simple truncate off the end of the item 656 */ 657 u32 new_size = (bytenr - key->offset) >> blocksize_bits; 658 new_size *= csum_size; 659 btrfs_truncate_item(path, new_size, 1); 660 } else if (key->offset >= bytenr && csum_end > end_byte && 661 end_byte > key->offset) { 662 /* 663 * [ bytenr - len ] 664 * [ ] 665 * [csum ] 666 * we need to truncate from the beginning of the csum 667 */ 668 u32 new_size = (csum_end - end_byte) >> blocksize_bits; 669 new_size *= csum_size; 670 671 btrfs_truncate_item(path, new_size, 0); 672 673 key->offset = end_byte; 674 btrfs_set_item_key_safe(fs_info, path, key); 675 } else { 676 BUG(); 677 } 678 } 679 680 /* 681 * deletes the csum items from the csum tree for a given 682 * range of bytes. 683 */ 684 int btrfs_del_csums(struct btrfs_trans_handle *trans, 685 struct btrfs_root *root, u64 bytenr, u64 len) 686 { 687 struct btrfs_fs_info *fs_info = trans->fs_info; 688 struct btrfs_path *path; 689 struct btrfs_key key; 690 u64 end_byte = bytenr + len; 691 u64 csum_end; 692 struct extent_buffer *leaf; 693 int ret; 694 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 695 int blocksize_bits = fs_info->sb->s_blocksize_bits; 696 697 ASSERT(root == fs_info->csum_root || 698 root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID); 699 700 path = btrfs_alloc_path(); 701 if (!path) 702 return -ENOMEM; 703 704 while (1) { 705 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 706 key.offset = end_byte - 1; 707 key.type = BTRFS_EXTENT_CSUM_KEY; 708 709 path->leave_spinning = 1; 710 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 711 if (ret > 0) { 712 if (path->slots[0] == 0) 713 break; 714 path->slots[0]--; 715 } else if (ret < 0) { 716 break; 717 } 718 719 leaf = path->nodes[0]; 720 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 721 722 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || 723 key.type != BTRFS_EXTENT_CSUM_KEY) { 724 break; 725 } 726 727 if (key.offset >= end_byte) 728 break; 729 730 csum_end = btrfs_item_size_nr(leaf, path->slots[0]) / csum_size; 731 csum_end <<= blocksize_bits; 732 csum_end += key.offset; 733 734 /* this csum ends before we start, we're done */ 735 if (csum_end <= bytenr) 736 break; 737 738 /* delete the entire item, it is inside our range */ 739 if (key.offset >= bytenr && csum_end <= end_byte) { 740 int del_nr = 1; 741 742 /* 743 * Check how many csum items preceding this one in this 744 * leaf correspond to our range and then delete them all 745 * at once. 746 */ 747 if (key.offset > bytenr && path->slots[0] > 0) { 748 int slot = path->slots[0] - 1; 749 750 while (slot >= 0) { 751 struct btrfs_key pk; 752 753 btrfs_item_key_to_cpu(leaf, &pk, slot); 754 if (pk.offset < bytenr || 755 pk.type != BTRFS_EXTENT_CSUM_KEY || 756 pk.objectid != 757 BTRFS_EXTENT_CSUM_OBJECTID) 758 break; 759 path->slots[0] = slot; 760 del_nr++; 761 key.offset = pk.offset; 762 slot--; 763 } 764 } 765 ret = btrfs_del_items(trans, root, path, 766 path->slots[0], del_nr); 767 if (ret) 768 goto out; 769 if (key.offset == bytenr) 770 break; 771 } else if (key.offset < bytenr && csum_end > end_byte) { 772 unsigned long offset; 773 unsigned long shift_len; 774 unsigned long item_offset; 775 /* 776 * [ bytenr - len ] 777 * [csum ] 778 * 779 * Our bytes are in the middle of the csum, 780 * we need to split this item and insert a new one. 781 * 782 * But we can't drop the path because the 783 * csum could change, get removed, extended etc. 784 * 785 * The trick here is the max size of a csum item leaves 786 * enough room in the tree block for a single 787 * item header. So, we split the item in place, 788 * adding a new header pointing to the existing 789 * bytes. Then we loop around again and we have 790 * a nicely formed csum item that we can neatly 791 * truncate. 792 */ 793 offset = (bytenr - key.offset) >> blocksize_bits; 794 offset *= csum_size; 795 796 shift_len = (len >> blocksize_bits) * csum_size; 797 798 item_offset = btrfs_item_ptr_offset(leaf, 799 path->slots[0]); 800 801 memzero_extent_buffer(leaf, item_offset + offset, 802 shift_len); 803 key.offset = bytenr; 804 805 /* 806 * btrfs_split_item returns -EAGAIN when the 807 * item changed size or key 808 */ 809 ret = btrfs_split_item(trans, root, path, &key, offset); 810 if (ret && ret != -EAGAIN) { 811 btrfs_abort_transaction(trans, ret); 812 goto out; 813 } 814 815 key.offset = end_byte - 1; 816 } else { 817 truncate_one_csum(fs_info, path, &key, bytenr, len); 818 if (key.offset < bytenr) 819 break; 820 } 821 btrfs_release_path(path); 822 } 823 ret = 0; 824 out: 825 btrfs_free_path(path); 826 return ret; 827 } 828 829 int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, 830 struct btrfs_root *root, 831 struct btrfs_ordered_sum *sums) 832 { 833 struct btrfs_fs_info *fs_info = root->fs_info; 834 struct btrfs_key file_key; 835 struct btrfs_key found_key; 836 struct btrfs_path *path; 837 struct btrfs_csum_item *item; 838 struct btrfs_csum_item *item_end; 839 struct extent_buffer *leaf = NULL; 840 u64 next_offset; 841 u64 total_bytes = 0; 842 u64 csum_offset; 843 u64 bytenr; 844 u32 nritems; 845 u32 ins_size; 846 int index = 0; 847 int found_next; 848 int ret; 849 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 850 851 path = btrfs_alloc_path(); 852 if (!path) 853 return -ENOMEM; 854 again: 855 next_offset = (u64)-1; 856 found_next = 0; 857 bytenr = sums->bytenr + total_bytes; 858 file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 859 file_key.offset = bytenr; 860 file_key.type = BTRFS_EXTENT_CSUM_KEY; 861 862 item = btrfs_lookup_csum(trans, root, path, bytenr, 1); 863 if (!IS_ERR(item)) { 864 ret = 0; 865 leaf = path->nodes[0]; 866 item_end = btrfs_item_ptr(leaf, path->slots[0], 867 struct btrfs_csum_item); 868 item_end = (struct btrfs_csum_item *)((char *)item_end + 869 btrfs_item_size_nr(leaf, path->slots[0])); 870 goto found; 871 } 872 ret = PTR_ERR(item); 873 if (ret != -EFBIG && ret != -ENOENT) 874 goto out; 875 876 if (ret == -EFBIG) { 877 u32 item_size; 878 /* we found one, but it isn't big enough yet */ 879 leaf = path->nodes[0]; 880 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 881 if ((item_size / csum_size) >= 882 MAX_CSUM_ITEMS(fs_info, csum_size)) { 883 /* already at max size, make a new one */ 884 goto insert; 885 } 886 } else { 887 int slot = path->slots[0] + 1; 888 /* we didn't find a csum item, insert one */ 889 nritems = btrfs_header_nritems(path->nodes[0]); 890 if (!nritems || (path->slots[0] >= nritems - 1)) { 891 ret = btrfs_next_leaf(root, path); 892 if (ret < 0) { 893 goto out; 894 } else if (ret > 0) { 895 found_next = 1; 896 goto insert; 897 } 898 slot = path->slots[0]; 899 } 900 btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot); 901 if (found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || 902 found_key.type != BTRFS_EXTENT_CSUM_KEY) { 903 found_next = 1; 904 goto insert; 905 } 906 next_offset = found_key.offset; 907 found_next = 1; 908 goto insert; 909 } 910 911 /* 912 * At this point, we know the tree has a checksum item that ends at an 913 * offset matching the start of the checksum range we want to insert. 914 * We try to extend that item as much as possible and then add as many 915 * checksums to it as they fit. 916 * 917 * First check if the leaf has enough free space for at least one 918 * checksum. If it has go directly to the item extension code, otherwise 919 * release the path and do a search for insertion before the extension. 920 */ 921 if (btrfs_leaf_free_space(leaf) >= csum_size) { 922 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 923 csum_offset = (bytenr - found_key.offset) >> 924 fs_info->sb->s_blocksize_bits; 925 goto extend_csum; 926 } 927 928 btrfs_release_path(path); 929 ret = btrfs_search_slot(trans, root, &file_key, path, 930 csum_size, 1); 931 if (ret < 0) 932 goto out; 933 934 if (ret > 0) { 935 if (path->slots[0] == 0) 936 goto insert; 937 path->slots[0]--; 938 } 939 940 leaf = path->nodes[0]; 941 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 942 csum_offset = (bytenr - found_key.offset) >> 943 fs_info->sb->s_blocksize_bits; 944 945 if (found_key.type != BTRFS_EXTENT_CSUM_KEY || 946 found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || 947 csum_offset >= MAX_CSUM_ITEMS(fs_info, csum_size)) { 948 goto insert; 949 } 950 951 extend_csum: 952 if (csum_offset == btrfs_item_size_nr(leaf, path->slots[0]) / 953 csum_size) { 954 int extend_nr; 955 u64 tmp; 956 u32 diff; 957 958 tmp = sums->len - total_bytes; 959 tmp >>= fs_info->sb->s_blocksize_bits; 960 WARN_ON(tmp < 1); 961 962 extend_nr = max_t(int, 1, (int)tmp); 963 diff = (csum_offset + extend_nr) * csum_size; 964 diff = min(diff, 965 MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size); 966 967 diff = diff - btrfs_item_size_nr(leaf, path->slots[0]); 968 diff = min_t(u32, btrfs_leaf_free_space(leaf), diff); 969 diff /= csum_size; 970 diff *= csum_size; 971 972 btrfs_extend_item(path, diff); 973 ret = 0; 974 goto csum; 975 } 976 977 insert: 978 btrfs_release_path(path); 979 csum_offset = 0; 980 if (found_next) { 981 u64 tmp; 982 983 tmp = sums->len - total_bytes; 984 tmp >>= fs_info->sb->s_blocksize_bits; 985 tmp = min(tmp, (next_offset - file_key.offset) >> 986 fs_info->sb->s_blocksize_bits); 987 988 tmp = max_t(u64, 1, tmp); 989 tmp = min_t(u64, tmp, MAX_CSUM_ITEMS(fs_info, csum_size)); 990 ins_size = csum_size * tmp; 991 } else { 992 ins_size = csum_size; 993 } 994 path->leave_spinning = 1; 995 ret = btrfs_insert_empty_item(trans, root, path, &file_key, 996 ins_size); 997 path->leave_spinning = 0; 998 if (ret < 0) 999 goto out; 1000 if (WARN_ON(ret != 0)) 1001 goto out; 1002 leaf = path->nodes[0]; 1003 csum: 1004 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item); 1005 item_end = (struct btrfs_csum_item *)((unsigned char *)item + 1006 btrfs_item_size_nr(leaf, path->slots[0])); 1007 item = (struct btrfs_csum_item *)((unsigned char *)item + 1008 csum_offset * csum_size); 1009 found: 1010 ins_size = (u32)(sums->len - total_bytes) >> 1011 fs_info->sb->s_blocksize_bits; 1012 ins_size *= csum_size; 1013 ins_size = min_t(u32, (unsigned long)item_end - (unsigned long)item, 1014 ins_size); 1015 write_extent_buffer(leaf, sums->sums + index, (unsigned long)item, 1016 ins_size); 1017 1018 index += ins_size; 1019 ins_size /= csum_size; 1020 total_bytes += ins_size * fs_info->sectorsize; 1021 1022 btrfs_mark_buffer_dirty(path->nodes[0]); 1023 if (total_bytes < sums->len) { 1024 btrfs_release_path(path); 1025 cond_resched(); 1026 goto again; 1027 } 1028 out: 1029 btrfs_free_path(path); 1030 return ret; 1031 } 1032 1033 void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode, 1034 const struct btrfs_path *path, 1035 struct btrfs_file_extent_item *fi, 1036 const bool new_inline, 1037 struct extent_map *em) 1038 { 1039 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1040 struct btrfs_root *root = inode->root; 1041 struct extent_buffer *leaf = path->nodes[0]; 1042 const int slot = path->slots[0]; 1043 struct btrfs_key key; 1044 u64 extent_start, extent_end; 1045 u64 bytenr; 1046 u8 type = btrfs_file_extent_type(leaf, fi); 1047 int compress_type = btrfs_file_extent_compression(leaf, fi); 1048 1049 btrfs_item_key_to_cpu(leaf, &key, slot); 1050 extent_start = key.offset; 1051 extent_end = btrfs_file_extent_end(path); 1052 em->ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 1053 if (type == BTRFS_FILE_EXTENT_REG || 1054 type == BTRFS_FILE_EXTENT_PREALLOC) { 1055 em->start = extent_start; 1056 em->len = extent_end - extent_start; 1057 em->orig_start = extent_start - 1058 btrfs_file_extent_offset(leaf, fi); 1059 em->orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi); 1060 bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1061 if (bytenr == 0) { 1062 em->block_start = EXTENT_MAP_HOLE; 1063 return; 1064 } 1065 if (compress_type != BTRFS_COMPRESS_NONE) { 1066 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 1067 em->compress_type = compress_type; 1068 em->block_start = bytenr; 1069 em->block_len = em->orig_block_len; 1070 } else { 1071 bytenr += btrfs_file_extent_offset(leaf, fi); 1072 em->block_start = bytenr; 1073 em->block_len = em->len; 1074 if (type == BTRFS_FILE_EXTENT_PREALLOC) 1075 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 1076 } 1077 } else if (type == BTRFS_FILE_EXTENT_INLINE) { 1078 em->block_start = EXTENT_MAP_INLINE; 1079 em->start = extent_start; 1080 em->len = extent_end - extent_start; 1081 /* 1082 * Initialize orig_start and block_len with the same values 1083 * as in inode.c:btrfs_get_extent(). 1084 */ 1085 em->orig_start = EXTENT_MAP_HOLE; 1086 em->block_len = (u64)-1; 1087 if (!new_inline && compress_type != BTRFS_COMPRESS_NONE) { 1088 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 1089 em->compress_type = compress_type; 1090 } 1091 } else { 1092 btrfs_err(fs_info, 1093 "unknown file extent item type %d, inode %llu, offset %llu, " 1094 "root %llu", type, btrfs_ino(inode), extent_start, 1095 root->root_key.objectid); 1096 } 1097 } 1098 1099 /* 1100 * Returns the end offset (non inclusive) of the file extent item the given path 1101 * points to. If it points to an inline extent, the returned offset is rounded 1102 * up to the sector size. 1103 */ 1104 u64 btrfs_file_extent_end(const struct btrfs_path *path) 1105 { 1106 const struct extent_buffer *leaf = path->nodes[0]; 1107 const int slot = path->slots[0]; 1108 struct btrfs_file_extent_item *fi; 1109 struct btrfs_key key; 1110 u64 end; 1111 1112 btrfs_item_key_to_cpu(leaf, &key, slot); 1113 ASSERT(key.type == BTRFS_EXTENT_DATA_KEY); 1114 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 1115 1116 if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) { 1117 end = btrfs_file_extent_ram_bytes(leaf, fi); 1118 end = ALIGN(key.offset + end, leaf->fs_info->sectorsize); 1119 } else { 1120 end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); 1121 } 1122 1123 return end; 1124 } 1125