1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2016 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <darrick.wong@oracle.com> 5 */ 6 #include "xfs.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_log_format.h" 11 #include "xfs_trans_resv.h" 12 #include "xfs_mount.h" 13 #include "xfs_defer.h" 14 #include "xfs_inode.h" 15 #include "xfs_trans.h" 16 #include "xfs_bmap.h" 17 #include "xfs_bmap_util.h" 18 #include "xfs_trace.h" 19 #include "xfs_icache.h" 20 #include "xfs_btree.h" 21 #include "xfs_refcount_btree.h" 22 #include "xfs_refcount.h" 23 #include "xfs_bmap_btree.h" 24 #include "xfs_trans_space.h" 25 #include "xfs_bit.h" 26 #include "xfs_alloc.h" 27 #include "xfs_quota.h" 28 #include "xfs_reflink.h" 29 #include "xfs_iomap.h" 30 #include "xfs_sb.h" 31 #include "xfs_ag_resv.h" 32 33 /* 34 * Copy on Write of Shared Blocks 35 * 36 * XFS must preserve "the usual" file semantics even when two files share 37 * the same physical blocks. This means that a write to one file must not 38 * alter the blocks in a different file; the way that we'll do that is 39 * through the use of a copy-on-write mechanism. At a high level, that 40 * means that when we want to write to a shared block, we allocate a new 41 * block, write the data to the new block, and if that succeeds we map the 42 * new block into the file. 43 * 44 * XFS provides a "delayed allocation" mechanism that defers the allocation 45 * of disk blocks to dirty-but-not-yet-mapped file blocks as long as 46 * possible. This reduces fragmentation by enabling the filesystem to ask 47 * for bigger chunks less often, which is exactly what we want for CoW. 48 * 49 * The delalloc mechanism begins when the kernel wants to make a block 50 * writable (write_begin or page_mkwrite). If the offset is not mapped, we 51 * create a delalloc mapping, which is a regular in-core extent, but without 52 * a real startblock. (For delalloc mappings, the startblock encodes both 53 * a flag that this is a delalloc mapping, and a worst-case estimate of how 54 * many blocks might be required to put the mapping into the BMBT.) delalloc 55 * mappings are a reservation against the free space in the filesystem; 56 * adjacent mappings can also be combined into fewer larger mappings. 57 * 58 * As an optimization, the CoW extent size hint (cowextsz) creates 59 * outsized aligned delalloc reservations in the hope of landing out of 60 * order nearby CoW writes in a single extent on disk, thereby reducing 61 * fragmentation and improving future performance. 62 * 63 * D: --RRRRRRSSSRRRRRRRR--- (data fork) 64 * C: ------DDDDDDD--------- (CoW fork) 65 * 66 * When dirty pages are being written out (typically in writepage), the 67 * delalloc reservations are converted into unwritten mappings by 68 * allocating blocks and replacing the delalloc mapping with real ones. 69 * A delalloc mapping can be replaced by several unwritten ones if the 70 * free space is fragmented. 71 * 72 * D: --RRRRRRSSSRRRRRRRR--- 73 * C: ------UUUUUUU--------- 74 * 75 * We want to adapt the delalloc mechanism for copy-on-write, since the 76 * write paths are similar. The first two steps (creating the reservation 77 * and allocating the blocks) are exactly the same as delalloc except that 78 * the mappings must be stored in a separate CoW fork because we do not want 79 * to disturb the mapping in the data fork until we're sure that the write 80 * succeeded. IO completion in this case is the process of removing the old 81 * mapping from the data fork and moving the new mapping from the CoW fork to 82 * the data fork. This will be discussed shortly. 83 * 84 * For now, unaligned directio writes will be bounced back to the page cache. 85 * Block-aligned directio writes will use the same mechanism as buffered 86 * writes. 87 * 88 * Just prior to submitting the actual disk write requests, we convert 89 * the extents representing the range of the file actually being written 90 * (as opposed to extra pieces created for the cowextsize hint) to real 91 * extents. This will become important in the next step: 92 * 93 * D: --RRRRRRSSSRRRRRRRR--- 94 * C: ------UUrrUUU--------- 95 * 96 * CoW remapping must be done after the data block write completes, 97 * because we don't want to destroy the old data fork map until we're sure 98 * the new block has been written. Since the new mappings are kept in a 99 * separate fork, we can simply iterate these mappings to find the ones 100 * that cover the file blocks that we just CoW'd. For each extent, simply 101 * unmap the corresponding range in the data fork, map the new range into 102 * the data fork, and remove the extent from the CoW fork. Because of 103 * the presence of the cowextsize hint, however, we must be careful 104 * only to remap the blocks that we've actually written out -- we must 105 * never remap delalloc reservations nor CoW staging blocks that have 106 * yet to be written. This corresponds exactly to the real extents in 107 * the CoW fork: 108 * 109 * D: --RRRRRRrrSRRRRRRRR--- 110 * C: ------UU--UUU--------- 111 * 112 * Since the remapping operation can be applied to an arbitrary file 113 * range, we record the need for the remap step as a flag in the ioend 114 * instead of declaring a new IO type. This is required for direct io 115 * because we only have ioend for the whole dio, and we have to be able to 116 * remember the presence of unwritten blocks and CoW blocks with a single 117 * ioend structure. Better yet, the more ground we can cover with one 118 * ioend, the better. 119 */ 120 121 /* 122 * Given an AG extent, find the lowest-numbered run of shared blocks 123 * within that range and return the range in fbno/flen. If 124 * find_end_of_shared is true, return the longest contiguous extent of 125 * shared blocks. If there are no shared extents, fbno and flen will 126 * be set to NULLAGBLOCK and 0, respectively. 127 */ 128 int 129 xfs_reflink_find_shared( 130 struct xfs_mount *mp, 131 struct xfs_trans *tp, 132 xfs_agnumber_t agno, 133 xfs_agblock_t agbno, 134 xfs_extlen_t aglen, 135 xfs_agblock_t *fbno, 136 xfs_extlen_t *flen, 137 bool find_end_of_shared) 138 { 139 struct xfs_buf *agbp; 140 struct xfs_btree_cur *cur; 141 int error; 142 143 error = xfs_alloc_read_agf(mp, tp, agno, 0, &agbp); 144 if (error) 145 return error; 146 147 cur = xfs_refcountbt_init_cursor(mp, tp, agbp, agno); 148 149 error = xfs_refcount_find_shared(cur, agbno, aglen, fbno, flen, 150 find_end_of_shared); 151 152 xfs_btree_del_cursor(cur, error); 153 154 xfs_trans_brelse(tp, agbp); 155 return error; 156 } 157 158 /* 159 * Trim the mapping to the next block where there's a change in the 160 * shared/unshared status. More specifically, this means that we 161 * find the lowest-numbered extent of shared blocks that coincides with 162 * the given block mapping. If the shared extent overlaps the start of 163 * the mapping, trim the mapping to the end of the shared extent. If 164 * the shared region intersects the mapping, trim the mapping to the 165 * start of the shared extent. If there are no shared regions that 166 * overlap, just return the original extent. 167 */ 168 int 169 xfs_reflink_trim_around_shared( 170 struct xfs_inode *ip, 171 struct xfs_bmbt_irec *irec, 172 bool *shared) 173 { 174 xfs_agnumber_t agno; 175 xfs_agblock_t agbno; 176 xfs_extlen_t aglen; 177 xfs_agblock_t fbno; 178 xfs_extlen_t flen; 179 int error = 0; 180 181 /* Holes, unwritten, and delalloc extents cannot be shared */ 182 if (!xfs_is_cow_inode(ip) || !xfs_bmap_is_written_extent(irec)) { 183 *shared = false; 184 return 0; 185 } 186 187 trace_xfs_reflink_trim_around_shared(ip, irec); 188 189 agno = XFS_FSB_TO_AGNO(ip->i_mount, irec->br_startblock); 190 agbno = XFS_FSB_TO_AGBNO(ip->i_mount, irec->br_startblock); 191 aglen = irec->br_blockcount; 192 193 error = xfs_reflink_find_shared(ip->i_mount, NULL, agno, agbno, 194 aglen, &fbno, &flen, true); 195 if (error) 196 return error; 197 198 *shared = false; 199 if (fbno == NULLAGBLOCK) { 200 /* No shared blocks at all. */ 201 return 0; 202 } else if (fbno == agbno) { 203 /* 204 * The start of this extent is shared. Truncate the 205 * mapping at the end of the shared region so that a 206 * subsequent iteration starts at the start of the 207 * unshared region. 208 */ 209 irec->br_blockcount = flen; 210 *shared = true; 211 return 0; 212 } else { 213 /* 214 * There's a shared extent midway through this extent. 215 * Truncate the mapping at the start of the shared 216 * extent so that a subsequent iteration starts at the 217 * start of the shared region. 218 */ 219 irec->br_blockcount = fbno - agbno; 220 return 0; 221 } 222 } 223 224 int 225 xfs_bmap_trim_cow( 226 struct xfs_inode *ip, 227 struct xfs_bmbt_irec *imap, 228 bool *shared) 229 { 230 /* We can't update any real extents in always COW mode. */ 231 if (xfs_is_always_cow_inode(ip) && 232 !isnullstartblock(imap->br_startblock)) { 233 *shared = true; 234 return 0; 235 } 236 237 /* Trim the mapping to the nearest shared extent boundary. */ 238 return xfs_reflink_trim_around_shared(ip, imap, shared); 239 } 240 241 static int 242 xfs_reflink_convert_cow_locked( 243 struct xfs_inode *ip, 244 xfs_fileoff_t offset_fsb, 245 xfs_filblks_t count_fsb) 246 { 247 struct xfs_iext_cursor icur; 248 struct xfs_bmbt_irec got; 249 struct xfs_btree_cur *dummy_cur = NULL; 250 int dummy_logflags; 251 int error = 0; 252 253 if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got)) 254 return 0; 255 256 do { 257 if (got.br_startoff >= offset_fsb + count_fsb) 258 break; 259 if (got.br_state == XFS_EXT_NORM) 260 continue; 261 if (WARN_ON_ONCE(isnullstartblock(got.br_startblock))) 262 return -EIO; 263 264 xfs_trim_extent(&got, offset_fsb, count_fsb); 265 if (!got.br_blockcount) 266 continue; 267 268 got.br_state = XFS_EXT_NORM; 269 error = xfs_bmap_add_extent_unwritten_real(NULL, ip, 270 XFS_COW_FORK, &icur, &dummy_cur, &got, 271 &dummy_logflags); 272 if (error) 273 return error; 274 } while (xfs_iext_next_extent(ip->i_cowfp, &icur, &got)); 275 276 return error; 277 } 278 279 /* Convert all of the unwritten CoW extents in a file's range to real ones. */ 280 int 281 xfs_reflink_convert_cow( 282 struct xfs_inode *ip, 283 xfs_off_t offset, 284 xfs_off_t count) 285 { 286 struct xfs_mount *mp = ip->i_mount; 287 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 288 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); 289 xfs_filblks_t count_fsb = end_fsb - offset_fsb; 290 int error; 291 292 ASSERT(count != 0); 293 294 xfs_ilock(ip, XFS_ILOCK_EXCL); 295 error = xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb); 296 xfs_iunlock(ip, XFS_ILOCK_EXCL); 297 return error; 298 } 299 300 /* 301 * Find the extent that maps the given range in the COW fork. Even if the extent 302 * is not shared we might have a preallocation for it in the COW fork. If so we 303 * use it that rather than trigger a new allocation. 304 */ 305 static int 306 xfs_find_trim_cow_extent( 307 struct xfs_inode *ip, 308 struct xfs_bmbt_irec *imap, 309 struct xfs_bmbt_irec *cmap, 310 bool *shared, 311 bool *found) 312 { 313 xfs_fileoff_t offset_fsb = imap->br_startoff; 314 xfs_filblks_t count_fsb = imap->br_blockcount; 315 struct xfs_iext_cursor icur; 316 317 *found = false; 318 319 /* 320 * If we don't find an overlapping extent, trim the range we need to 321 * allocate to fit the hole we found. 322 */ 323 if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, cmap)) 324 cmap->br_startoff = offset_fsb + count_fsb; 325 if (cmap->br_startoff > offset_fsb) { 326 xfs_trim_extent(imap, imap->br_startoff, 327 cmap->br_startoff - imap->br_startoff); 328 return xfs_bmap_trim_cow(ip, imap, shared); 329 } 330 331 *shared = true; 332 if (isnullstartblock(cmap->br_startblock)) { 333 xfs_trim_extent(imap, cmap->br_startoff, cmap->br_blockcount); 334 return 0; 335 } 336 337 /* real extent found - no need to allocate */ 338 xfs_trim_extent(cmap, offset_fsb, count_fsb); 339 *found = true; 340 return 0; 341 } 342 343 /* Allocate all CoW reservations covering a range of blocks in a file. */ 344 int 345 xfs_reflink_allocate_cow( 346 struct xfs_inode *ip, 347 struct xfs_bmbt_irec *imap, 348 struct xfs_bmbt_irec *cmap, 349 bool *shared, 350 uint *lockmode, 351 bool convert_now) 352 { 353 struct xfs_mount *mp = ip->i_mount; 354 xfs_fileoff_t offset_fsb = imap->br_startoff; 355 xfs_filblks_t count_fsb = imap->br_blockcount; 356 struct xfs_trans *tp; 357 int nimaps, error = 0; 358 bool found; 359 xfs_filblks_t resaligned; 360 xfs_extlen_t resblks = 0; 361 362 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); 363 if (!ip->i_cowfp) { 364 ASSERT(!xfs_is_reflink_inode(ip)); 365 xfs_ifork_init_cow(ip); 366 } 367 368 error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); 369 if (error || !*shared) 370 return error; 371 if (found) 372 goto convert; 373 374 resaligned = xfs_aligned_fsb_count(imap->br_startoff, 375 imap->br_blockcount, xfs_get_cowextsz_hint(ip)); 376 resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); 377 378 xfs_iunlock(ip, *lockmode); 379 *lockmode = 0; 380 381 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 0, 382 false, &tp); 383 if (error) 384 return error; 385 386 *lockmode = XFS_ILOCK_EXCL; 387 388 /* 389 * Check for an overlapping extent again now that we dropped the ilock. 390 */ 391 error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); 392 if (error || !*shared) 393 goto out_trans_cancel; 394 if (found) { 395 xfs_trans_cancel(tp); 396 goto convert; 397 } 398 399 /* Allocate the entire reservation as unwritten blocks. */ 400 nimaps = 1; 401 error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount, 402 XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, 0, cmap, 403 &nimaps); 404 if (error) 405 goto out_trans_cancel; 406 407 xfs_inode_set_cowblocks_tag(ip); 408 error = xfs_trans_commit(tp); 409 if (error) 410 return error; 411 412 /* 413 * Allocation succeeded but the requested range was not even partially 414 * satisfied? Bail out! 415 */ 416 if (nimaps == 0) 417 return -ENOSPC; 418 convert: 419 xfs_trim_extent(cmap, offset_fsb, count_fsb); 420 /* 421 * COW fork extents are supposed to remain unwritten until we're ready 422 * to initiate a disk write. For direct I/O we are going to write the 423 * data and need the conversion, but for buffered writes we're done. 424 */ 425 if (!convert_now || cmap->br_state == XFS_EXT_NORM) 426 return 0; 427 trace_xfs_reflink_convert_cow(ip, cmap); 428 return xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb); 429 430 out_trans_cancel: 431 xfs_trans_cancel(tp); 432 return error; 433 } 434 435 /* 436 * Cancel CoW reservations for some block range of an inode. 437 * 438 * If cancel_real is true this function cancels all COW fork extents for the 439 * inode; if cancel_real is false, real extents are not cleared. 440 * 441 * Caller must have already joined the inode to the current transaction. The 442 * inode will be joined to the transaction returned to the caller. 443 */ 444 int 445 xfs_reflink_cancel_cow_blocks( 446 struct xfs_inode *ip, 447 struct xfs_trans **tpp, 448 xfs_fileoff_t offset_fsb, 449 xfs_fileoff_t end_fsb, 450 bool cancel_real) 451 { 452 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 453 struct xfs_bmbt_irec got, del; 454 struct xfs_iext_cursor icur; 455 int error = 0; 456 457 if (!xfs_inode_has_cow_data(ip)) 458 return 0; 459 if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) 460 return 0; 461 462 /* Walk backwards until we're out of the I/O range... */ 463 while (got.br_startoff + got.br_blockcount > offset_fsb) { 464 del = got; 465 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 466 467 /* Extent delete may have bumped ext forward */ 468 if (!del.br_blockcount) { 469 xfs_iext_prev(ifp, &icur); 470 goto next_extent; 471 } 472 473 trace_xfs_reflink_cancel_cow(ip, &del); 474 475 if (isnullstartblock(del.br_startblock)) { 476 error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, 477 &icur, &got, &del); 478 if (error) 479 break; 480 } else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) { 481 ASSERT((*tpp)->t_firstblock == NULLFSBLOCK); 482 483 /* Free the CoW orphan record. */ 484 xfs_refcount_free_cow_extent(*tpp, del.br_startblock, 485 del.br_blockcount); 486 487 xfs_bmap_add_free(*tpp, del.br_startblock, 488 del.br_blockcount, NULL); 489 490 /* Roll the transaction */ 491 error = xfs_defer_finish(tpp); 492 if (error) 493 break; 494 495 /* Remove the mapping from the CoW fork. */ 496 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 497 498 /* Remove the quota reservation */ 499 error = xfs_quota_unreserve_blkres(ip, 500 del.br_blockcount); 501 if (error) 502 break; 503 } else { 504 /* Didn't do anything, push cursor back. */ 505 xfs_iext_prev(ifp, &icur); 506 } 507 next_extent: 508 if (!xfs_iext_get_extent(ifp, &icur, &got)) 509 break; 510 } 511 512 /* clear tag if cow fork is emptied */ 513 if (!ifp->if_bytes) 514 xfs_inode_clear_cowblocks_tag(ip); 515 return error; 516 } 517 518 /* 519 * Cancel CoW reservations for some byte range of an inode. 520 * 521 * If cancel_real is true this function cancels all COW fork extents for the 522 * inode; if cancel_real is false, real extents are not cleared. 523 */ 524 int 525 xfs_reflink_cancel_cow_range( 526 struct xfs_inode *ip, 527 xfs_off_t offset, 528 xfs_off_t count, 529 bool cancel_real) 530 { 531 struct xfs_trans *tp; 532 xfs_fileoff_t offset_fsb; 533 xfs_fileoff_t end_fsb; 534 int error; 535 536 trace_xfs_reflink_cancel_cow_range(ip, offset, count); 537 ASSERT(ip->i_cowfp); 538 539 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 540 if (count == NULLFILEOFF) 541 end_fsb = NULLFILEOFF; 542 else 543 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 544 545 /* Start a rolling transaction to remove the mappings */ 546 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 547 0, 0, 0, &tp); 548 if (error) 549 goto out; 550 551 xfs_ilock(ip, XFS_ILOCK_EXCL); 552 xfs_trans_ijoin(tp, ip, 0); 553 554 /* Scrape out the old CoW reservations */ 555 error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb, 556 cancel_real); 557 if (error) 558 goto out_cancel; 559 560 error = xfs_trans_commit(tp); 561 562 xfs_iunlock(ip, XFS_ILOCK_EXCL); 563 return error; 564 565 out_cancel: 566 xfs_trans_cancel(tp); 567 xfs_iunlock(ip, XFS_ILOCK_EXCL); 568 out: 569 trace_xfs_reflink_cancel_cow_range_error(ip, error, _RET_IP_); 570 return error; 571 } 572 573 /* 574 * Remap part of the CoW fork into the data fork. 575 * 576 * We aim to remap the range starting at @offset_fsb and ending at @end_fsb 577 * into the data fork; this function will remap what it can (at the end of the 578 * range) and update @end_fsb appropriately. Each remap gets its own 579 * transaction because we can end up merging and splitting bmbt blocks for 580 * every remap operation and we'd like to keep the block reservation 581 * requirements as low as possible. 582 */ 583 STATIC int 584 xfs_reflink_end_cow_extent( 585 struct xfs_inode *ip, 586 xfs_fileoff_t offset_fsb, 587 xfs_fileoff_t *end_fsb) 588 { 589 struct xfs_bmbt_irec got, del; 590 struct xfs_iext_cursor icur; 591 struct xfs_mount *mp = ip->i_mount; 592 struct xfs_trans *tp; 593 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 594 xfs_filblks_t rlen; 595 unsigned int resblks; 596 int error; 597 598 /* No COW extents? That's easy! */ 599 if (ifp->if_bytes == 0) { 600 *end_fsb = offset_fsb; 601 return 0; 602 } 603 604 resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); 605 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 606 XFS_TRANS_RESERVE, &tp); 607 if (error) 608 return error; 609 610 /* 611 * Lock the inode. We have to ijoin without automatic unlock because 612 * the lead transaction is the refcountbt record deletion; the data 613 * fork update follows as a deferred log item. 614 */ 615 xfs_ilock(ip, XFS_ILOCK_EXCL); 616 xfs_trans_ijoin(tp, ip, 0); 617 618 error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, 619 XFS_IEXT_REFLINK_END_COW_CNT); 620 if (error) 621 goto out_cancel; 622 623 /* 624 * In case of racing, overlapping AIO writes no COW extents might be 625 * left by the time I/O completes for the loser of the race. In that 626 * case we are done. 627 */ 628 if (!xfs_iext_lookup_extent_before(ip, ifp, end_fsb, &icur, &got) || 629 got.br_startoff + got.br_blockcount <= offset_fsb) { 630 *end_fsb = offset_fsb; 631 goto out_cancel; 632 } 633 634 /* 635 * Structure copy @got into @del, then trim @del to the range that we 636 * were asked to remap. We preserve @got for the eventual CoW fork 637 * deletion; from now on @del represents the mapping that we're 638 * actually remapping. 639 */ 640 del = got; 641 xfs_trim_extent(&del, offset_fsb, *end_fsb - offset_fsb); 642 643 ASSERT(del.br_blockcount > 0); 644 645 /* 646 * Only remap real extents that contain data. With AIO, speculative 647 * preallocations can leak into the range we are called upon, and we 648 * need to skip them. 649 */ 650 if (!xfs_bmap_is_written_extent(&got)) { 651 *end_fsb = del.br_startoff; 652 goto out_cancel; 653 } 654 655 /* Unmap the old blocks in the data fork. */ 656 rlen = del.br_blockcount; 657 error = __xfs_bunmapi(tp, ip, del.br_startoff, &rlen, 0, 1); 658 if (error) 659 goto out_cancel; 660 661 /* Trim the extent to whatever got unmapped. */ 662 xfs_trim_extent(&del, del.br_startoff + rlen, del.br_blockcount - rlen); 663 trace_xfs_reflink_cow_remap(ip, &del); 664 665 /* Free the CoW orphan record. */ 666 xfs_refcount_free_cow_extent(tp, del.br_startblock, del.br_blockcount); 667 668 /* Map the new blocks into the data fork. */ 669 xfs_bmap_map_extent(tp, ip, &del); 670 671 /* Charge this new data fork mapping to the on-disk quota. */ 672 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_DELBCOUNT, 673 (long)del.br_blockcount); 674 675 /* Remove the mapping from the CoW fork. */ 676 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 677 678 error = xfs_trans_commit(tp); 679 xfs_iunlock(ip, XFS_ILOCK_EXCL); 680 if (error) 681 return error; 682 683 /* Update the caller about how much progress we made. */ 684 *end_fsb = del.br_startoff; 685 return 0; 686 687 out_cancel: 688 xfs_trans_cancel(tp); 689 xfs_iunlock(ip, XFS_ILOCK_EXCL); 690 return error; 691 } 692 693 /* 694 * Remap parts of a file's data fork after a successful CoW. 695 */ 696 int 697 xfs_reflink_end_cow( 698 struct xfs_inode *ip, 699 xfs_off_t offset, 700 xfs_off_t count) 701 { 702 xfs_fileoff_t offset_fsb; 703 xfs_fileoff_t end_fsb; 704 int error = 0; 705 706 trace_xfs_reflink_end_cow(ip, offset, count); 707 708 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 709 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 710 711 /* 712 * Walk backwards until we're out of the I/O range. The loop function 713 * repeatedly cycles the ILOCK to allocate one transaction per remapped 714 * extent. 715 * 716 * If we're being called by writeback then the pages will still 717 * have PageWriteback set, which prevents races with reflink remapping 718 * and truncate. Reflink remapping prevents races with writeback by 719 * taking the iolock and mmaplock before flushing the pages and 720 * remapping, which means there won't be any further writeback or page 721 * cache dirtying until the reflink completes. 722 * 723 * We should never have two threads issuing writeback for the same file 724 * region. There are also have post-eof checks in the writeback 725 * preparation code so that we don't bother writing out pages that are 726 * about to be truncated. 727 * 728 * If we're being called as part of directio write completion, the dio 729 * count is still elevated, which reflink and truncate will wait for. 730 * Reflink remapping takes the iolock and mmaplock and waits for 731 * pending dio to finish, which should prevent any directio until the 732 * remap completes. Multiple concurrent directio writes to the same 733 * region are handled by end_cow processing only occurring for the 734 * threads which succeed; the outcome of multiple overlapping direct 735 * writes is not well defined anyway. 736 * 737 * It's possible that a buffered write and a direct write could collide 738 * here (the buffered write stumbles in after the dio flushes and 739 * invalidates the page cache and immediately queues writeback), but we 740 * have never supported this 100%. If either disk write succeeds the 741 * blocks will be remapped. 742 */ 743 while (end_fsb > offset_fsb && !error) 744 error = xfs_reflink_end_cow_extent(ip, offset_fsb, &end_fsb); 745 746 if (error) 747 trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_); 748 return error; 749 } 750 751 /* 752 * Free leftover CoW reservations that didn't get cleaned out. 753 */ 754 int 755 xfs_reflink_recover_cow( 756 struct xfs_mount *mp) 757 { 758 xfs_agnumber_t agno; 759 int error = 0; 760 761 if (!xfs_sb_version_hasreflink(&mp->m_sb)) 762 return 0; 763 764 for (agno = 0; agno < mp->m_sb.sb_agcount; agno++) { 765 error = xfs_refcount_recover_cow_leftovers(mp, agno); 766 if (error) 767 break; 768 } 769 770 return error; 771 } 772 773 /* 774 * Reflinking (Block) Ranges of Two Files Together 775 * 776 * First, ensure that the reflink flag is set on both inodes. The flag is an 777 * optimization to avoid unnecessary refcount btree lookups in the write path. 778 * 779 * Now we can iteratively remap the range of extents (and holes) in src to the 780 * corresponding ranges in dest. Let drange and srange denote the ranges of 781 * logical blocks in dest and src touched by the reflink operation. 782 * 783 * While the length of drange is greater than zero, 784 * - Read src's bmbt at the start of srange ("imap") 785 * - If imap doesn't exist, make imap appear to start at the end of srange 786 * with zero length. 787 * - If imap starts before srange, advance imap to start at srange. 788 * - If imap goes beyond srange, truncate imap to end at the end of srange. 789 * - Punch (imap start - srange start + imap len) blocks from dest at 790 * offset (drange start). 791 * - If imap points to a real range of pblks, 792 * > Increase the refcount of the imap's pblks 793 * > Map imap's pblks into dest at the offset 794 * (drange start + imap start - srange start) 795 * - Advance drange and srange by (imap start - srange start + imap len) 796 * 797 * Finally, if the reflink made dest longer, update both the in-core and 798 * on-disk file sizes. 799 * 800 * ASCII Art Demonstration: 801 * 802 * Let's say we want to reflink this source file: 803 * 804 * ----SSSSSSS-SSSSS----SSSSSS (src file) 805 * <--------------------> 806 * 807 * into this destination file: 808 * 809 * --DDDDDDDDDDDDDDDDDDD--DDD (dest file) 810 * <--------------------> 811 * '-' means a hole, and 'S' and 'D' are written blocks in the src and dest. 812 * Observe that the range has different logical offsets in either file. 813 * 814 * Consider that the first extent in the source file doesn't line up with our 815 * reflink range. Unmapping and remapping are separate operations, so we can 816 * unmap more blocks from the destination file than we remap. 817 * 818 * ----SSSSSSS-SSSSS----SSSSSS 819 * <-------> 820 * --DDDDD---------DDDDD--DDD 821 * <-------> 822 * 823 * Now remap the source extent into the destination file: 824 * 825 * ----SSSSSSS-SSSSS----SSSSSS 826 * <-------> 827 * --DDDDD--SSSSSSSDDDDD--DDD 828 * <-------> 829 * 830 * Do likewise with the second hole and extent in our range. Holes in the 831 * unmap range don't affect our operation. 832 * 833 * ----SSSSSSS-SSSSS----SSSSSS 834 * <----> 835 * --DDDDD--SSSSSSS-SSSSS-DDD 836 * <----> 837 * 838 * Finally, unmap and remap part of the third extent. This will increase the 839 * size of the destination file. 840 * 841 * ----SSSSSSS-SSSSS----SSSSSS 842 * <-----> 843 * --DDDDD--SSSSSSS-SSSSS----SSS 844 * <-----> 845 * 846 * Once we update the destination file's i_size, we're done. 847 */ 848 849 /* 850 * Ensure the reflink bit is set in both inodes. 851 */ 852 STATIC int 853 xfs_reflink_set_inode_flag( 854 struct xfs_inode *src, 855 struct xfs_inode *dest) 856 { 857 struct xfs_mount *mp = src->i_mount; 858 int error; 859 struct xfs_trans *tp; 860 861 if (xfs_is_reflink_inode(src) && xfs_is_reflink_inode(dest)) 862 return 0; 863 864 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 865 if (error) 866 goto out_error; 867 868 /* Lock both files against IO */ 869 if (src->i_ino == dest->i_ino) 870 xfs_ilock(src, XFS_ILOCK_EXCL); 871 else 872 xfs_lock_two_inodes(src, XFS_ILOCK_EXCL, dest, XFS_ILOCK_EXCL); 873 874 if (!xfs_is_reflink_inode(src)) { 875 trace_xfs_reflink_set_inode_flag(src); 876 xfs_trans_ijoin(tp, src, XFS_ILOCK_EXCL); 877 src->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 878 xfs_trans_log_inode(tp, src, XFS_ILOG_CORE); 879 xfs_ifork_init_cow(src); 880 } else 881 xfs_iunlock(src, XFS_ILOCK_EXCL); 882 883 if (src->i_ino == dest->i_ino) 884 goto commit_flags; 885 886 if (!xfs_is_reflink_inode(dest)) { 887 trace_xfs_reflink_set_inode_flag(dest); 888 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 889 dest->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 890 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 891 xfs_ifork_init_cow(dest); 892 } else 893 xfs_iunlock(dest, XFS_ILOCK_EXCL); 894 895 commit_flags: 896 error = xfs_trans_commit(tp); 897 if (error) 898 goto out_error; 899 return error; 900 901 out_error: 902 trace_xfs_reflink_set_inode_flag_error(dest, error, _RET_IP_); 903 return error; 904 } 905 906 /* 907 * Update destination inode size & cowextsize hint, if necessary. 908 */ 909 int 910 xfs_reflink_update_dest( 911 struct xfs_inode *dest, 912 xfs_off_t newlen, 913 xfs_extlen_t cowextsize, 914 unsigned int remap_flags) 915 { 916 struct xfs_mount *mp = dest->i_mount; 917 struct xfs_trans *tp; 918 int error; 919 920 if (newlen <= i_size_read(VFS_I(dest)) && cowextsize == 0) 921 return 0; 922 923 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 924 if (error) 925 goto out_error; 926 927 xfs_ilock(dest, XFS_ILOCK_EXCL); 928 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 929 930 if (newlen > i_size_read(VFS_I(dest))) { 931 trace_xfs_reflink_update_inode_size(dest, newlen); 932 i_size_write(VFS_I(dest), newlen); 933 dest->i_d.di_size = newlen; 934 } 935 936 if (cowextsize) { 937 dest->i_d.di_cowextsize = cowextsize; 938 dest->i_d.di_flags2 |= XFS_DIFLAG2_COWEXTSIZE; 939 } 940 941 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 942 943 error = xfs_trans_commit(tp); 944 if (error) 945 goto out_error; 946 return error; 947 948 out_error: 949 trace_xfs_reflink_update_inode_size_error(dest, error, _RET_IP_); 950 return error; 951 } 952 953 /* 954 * Do we have enough reserve in this AG to handle a reflink? The refcount 955 * btree already reserved all the space it needs, but the rmap btree can grow 956 * infinitely, so we won't allow more reflinks when the AG is down to the 957 * btree reserves. 958 */ 959 static int 960 xfs_reflink_ag_has_free_space( 961 struct xfs_mount *mp, 962 xfs_agnumber_t agno) 963 { 964 struct xfs_perag *pag; 965 int error = 0; 966 967 if (!xfs_sb_version_hasrmapbt(&mp->m_sb)) 968 return 0; 969 970 pag = xfs_perag_get(mp, agno); 971 if (xfs_ag_resv_critical(pag, XFS_AG_RESV_RMAPBT) || 972 xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA)) 973 error = -ENOSPC; 974 xfs_perag_put(pag); 975 return error; 976 } 977 978 /* 979 * Remap the given extent into the file. The dmap blockcount will be set to 980 * the number of blocks that were actually remapped. 981 */ 982 STATIC int 983 xfs_reflink_remap_extent( 984 struct xfs_inode *ip, 985 struct xfs_bmbt_irec *dmap, 986 xfs_off_t new_isize) 987 { 988 struct xfs_bmbt_irec smap; 989 struct xfs_mount *mp = ip->i_mount; 990 struct xfs_trans *tp; 991 xfs_off_t newlen; 992 int64_t qdelta = 0; 993 unsigned int resblks; 994 bool quota_reserved = true; 995 bool smap_real; 996 bool dmap_written = xfs_bmap_is_written_extent(dmap); 997 int iext_delta = 0; 998 int nimaps; 999 int error; 1000 1001 /* 1002 * Start a rolling transaction to switch the mappings. 1003 * 1004 * Adding a written extent to the extent map can cause a bmbt split, 1005 * and removing a mapped extent from the extent can cause a bmbt split. 1006 * The two operations cannot both cause a split since they operate on 1007 * the same index in the bmap btree, so we only need a reservation for 1008 * one bmbt split if either thing is happening. However, we haven't 1009 * locked the inode yet, so we reserve assuming this is the case. 1010 * 1011 * The first allocation call tries to reserve enough space to handle 1012 * mapping dmap into a sparse part of the file plus the bmbt split. We 1013 * haven't locked the inode or read the existing mapping yet, so we do 1014 * not know for sure that we need the space. This should succeed most 1015 * of the time. 1016 * 1017 * If the first attempt fails, try again but reserving only enough 1018 * space to handle a bmbt split. This is the hard minimum requirement, 1019 * and we revisit quota reservations later when we know more about what 1020 * we're remapping. 1021 */ 1022 resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); 1023 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, 1024 resblks + dmap->br_blockcount, 0, false, &tp); 1025 if (error == -EDQUOT || error == -ENOSPC) { 1026 quota_reserved = false; 1027 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, 1028 resblks, 0, false, &tp); 1029 } 1030 if (error) 1031 goto out; 1032 1033 /* 1034 * Read what's currently mapped in the destination file into smap. 1035 * If smap isn't a hole, we will have to remove it before we can add 1036 * dmap to the destination file. 1037 */ 1038 nimaps = 1; 1039 error = xfs_bmapi_read(ip, dmap->br_startoff, dmap->br_blockcount, 1040 &smap, &nimaps, 0); 1041 if (error) 1042 goto out_cancel; 1043 ASSERT(nimaps == 1 && smap.br_startoff == dmap->br_startoff); 1044 smap_real = xfs_bmap_is_real_extent(&smap); 1045 1046 /* 1047 * We can only remap as many blocks as the smaller of the two extent 1048 * maps, because we can only remap one extent at a time. 1049 */ 1050 dmap->br_blockcount = min(dmap->br_blockcount, smap.br_blockcount); 1051 ASSERT(dmap->br_blockcount == smap.br_blockcount); 1052 1053 trace_xfs_reflink_remap_extent_dest(ip, &smap); 1054 1055 /* 1056 * Two extents mapped to the same physical block must not have 1057 * different states; that's filesystem corruption. Move on to the next 1058 * extent if they're both holes or both the same physical extent. 1059 */ 1060 if (dmap->br_startblock == smap.br_startblock) { 1061 if (dmap->br_state != smap.br_state) 1062 error = -EFSCORRUPTED; 1063 goto out_cancel; 1064 } 1065 1066 /* If both extents are unwritten, leave them alone. */ 1067 if (dmap->br_state == XFS_EXT_UNWRITTEN && 1068 smap.br_state == XFS_EXT_UNWRITTEN) 1069 goto out_cancel; 1070 1071 /* No reflinking if the AG of the dest mapping is low on space. */ 1072 if (dmap_written) { 1073 error = xfs_reflink_ag_has_free_space(mp, 1074 XFS_FSB_TO_AGNO(mp, dmap->br_startblock)); 1075 if (error) 1076 goto out_cancel; 1077 } 1078 1079 /* 1080 * Increase quota reservation if we think the quota block counter for 1081 * this file could increase. 1082 * 1083 * If we are mapping a written extent into the file, we need to have 1084 * enough quota block count reservation to handle the blocks in that 1085 * extent. We log only the delta to the quota block counts, so if the 1086 * extent we're unmapping also has blocks allocated to it, we don't 1087 * need a quota reservation for the extent itself. 1088 * 1089 * Note that if we're replacing a delalloc reservation with a written 1090 * extent, we have to take the full quota reservation because removing 1091 * the delalloc reservation gives the block count back to the quota 1092 * count. This is suboptimal, but the VFS flushed the dest range 1093 * before we started. That should have removed all the delalloc 1094 * reservations, but we code defensively. 1095 * 1096 * xfs_trans_alloc_inode above already tried to grab an even larger 1097 * quota reservation, and kicked off a blockgc scan if it couldn't. 1098 * If we can't get a potentially smaller quota reservation now, we're 1099 * done. 1100 */ 1101 if (!quota_reserved && !smap_real && dmap_written) { 1102 error = xfs_trans_reserve_quota_nblks(tp, ip, 1103 dmap->br_blockcount, 0, false); 1104 if (error) 1105 goto out_cancel; 1106 } 1107 1108 if (smap_real) 1109 ++iext_delta; 1110 1111 if (dmap_written) 1112 ++iext_delta; 1113 1114 error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, iext_delta); 1115 if (error) 1116 goto out_cancel; 1117 1118 if (smap_real) { 1119 /* 1120 * If the extent we're unmapping is backed by storage (written 1121 * or not), unmap the extent and drop its refcount. 1122 */ 1123 xfs_bmap_unmap_extent(tp, ip, &smap); 1124 xfs_refcount_decrease_extent(tp, &smap); 1125 qdelta -= smap.br_blockcount; 1126 } else if (smap.br_startblock == DELAYSTARTBLOCK) { 1127 xfs_filblks_t len = smap.br_blockcount; 1128 1129 /* 1130 * If the extent we're unmapping is a delalloc reservation, 1131 * we can use the regular bunmapi function to release the 1132 * incore state. Dropping the delalloc reservation takes care 1133 * of the quota reservation for us. 1134 */ 1135 error = __xfs_bunmapi(NULL, ip, smap.br_startoff, &len, 0, 1); 1136 if (error) 1137 goto out_cancel; 1138 ASSERT(len == 0); 1139 } 1140 1141 /* 1142 * If the extent we're sharing is backed by written storage, increase 1143 * its refcount and map it into the file. 1144 */ 1145 if (dmap_written) { 1146 xfs_refcount_increase_extent(tp, dmap); 1147 xfs_bmap_map_extent(tp, ip, dmap); 1148 qdelta += dmap->br_blockcount; 1149 } 1150 1151 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, qdelta); 1152 1153 /* Update dest isize if needed. */ 1154 newlen = XFS_FSB_TO_B(mp, dmap->br_startoff + dmap->br_blockcount); 1155 newlen = min_t(xfs_off_t, newlen, new_isize); 1156 if (newlen > i_size_read(VFS_I(ip))) { 1157 trace_xfs_reflink_update_inode_size(ip, newlen); 1158 i_size_write(VFS_I(ip), newlen); 1159 ip->i_d.di_size = newlen; 1160 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1161 } 1162 1163 /* Commit everything and unlock. */ 1164 error = xfs_trans_commit(tp); 1165 goto out_unlock; 1166 1167 out_cancel: 1168 xfs_trans_cancel(tp); 1169 out_unlock: 1170 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1171 out: 1172 if (error) 1173 trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_); 1174 return error; 1175 } 1176 1177 /* Remap a range of one file to the other. */ 1178 int 1179 xfs_reflink_remap_blocks( 1180 struct xfs_inode *src, 1181 loff_t pos_in, 1182 struct xfs_inode *dest, 1183 loff_t pos_out, 1184 loff_t remap_len, 1185 loff_t *remapped) 1186 { 1187 struct xfs_bmbt_irec imap; 1188 struct xfs_mount *mp = src->i_mount; 1189 xfs_fileoff_t srcoff = XFS_B_TO_FSBT(mp, pos_in); 1190 xfs_fileoff_t destoff = XFS_B_TO_FSBT(mp, pos_out); 1191 xfs_filblks_t len; 1192 xfs_filblks_t remapped_len = 0; 1193 xfs_off_t new_isize = pos_out + remap_len; 1194 int nimaps; 1195 int error = 0; 1196 1197 len = min_t(xfs_filblks_t, XFS_B_TO_FSB(mp, remap_len), 1198 XFS_MAX_FILEOFF); 1199 1200 trace_xfs_reflink_remap_blocks(src, srcoff, len, dest, destoff); 1201 1202 while (len > 0) { 1203 unsigned int lock_mode; 1204 1205 /* Read extent from the source file */ 1206 nimaps = 1; 1207 lock_mode = xfs_ilock_data_map_shared(src); 1208 error = xfs_bmapi_read(src, srcoff, len, &imap, &nimaps, 0); 1209 xfs_iunlock(src, lock_mode); 1210 if (error) 1211 break; 1212 /* 1213 * The caller supposedly flushed all dirty pages in the source 1214 * file range, which means that writeback should have allocated 1215 * or deleted all delalloc reservations in that range. If we 1216 * find one, that's a good sign that something is seriously 1217 * wrong here. 1218 */ 1219 ASSERT(nimaps == 1 && imap.br_startoff == srcoff); 1220 if (imap.br_startblock == DELAYSTARTBLOCK) { 1221 ASSERT(imap.br_startblock != DELAYSTARTBLOCK); 1222 error = -EFSCORRUPTED; 1223 break; 1224 } 1225 1226 trace_xfs_reflink_remap_extent_src(src, &imap); 1227 1228 /* Remap into the destination file at the given offset. */ 1229 imap.br_startoff = destoff; 1230 error = xfs_reflink_remap_extent(dest, &imap, new_isize); 1231 if (error) 1232 break; 1233 1234 if (fatal_signal_pending(current)) { 1235 error = -EINTR; 1236 break; 1237 } 1238 1239 /* Advance drange/srange */ 1240 srcoff += imap.br_blockcount; 1241 destoff += imap.br_blockcount; 1242 len -= imap.br_blockcount; 1243 remapped_len += imap.br_blockcount; 1244 } 1245 1246 if (error) 1247 trace_xfs_reflink_remap_blocks_error(dest, error, _RET_IP_); 1248 *remapped = min_t(loff_t, remap_len, 1249 XFS_FSB_TO_B(src->i_mount, remapped_len)); 1250 return error; 1251 } 1252 1253 /* 1254 * If we're reflinking to a point past the destination file's EOF, we must 1255 * zero any speculative post-EOF preallocations that sit between the old EOF 1256 * and the destination file offset. 1257 */ 1258 static int 1259 xfs_reflink_zero_posteof( 1260 struct xfs_inode *ip, 1261 loff_t pos) 1262 { 1263 loff_t isize = i_size_read(VFS_I(ip)); 1264 1265 if (pos <= isize) 1266 return 0; 1267 1268 trace_xfs_zero_eof(ip, isize, pos - isize); 1269 return iomap_zero_range(VFS_I(ip), isize, pos - isize, NULL, 1270 &xfs_buffered_write_iomap_ops); 1271 } 1272 1273 /* 1274 * Prepare two files for range cloning. Upon a successful return both inodes 1275 * will have the iolock and mmaplock held, the page cache of the out file will 1276 * be truncated, and any leases on the out file will have been broken. This 1277 * function borrows heavily from xfs_file_aio_write_checks. 1278 * 1279 * The VFS allows partial EOF blocks to "match" for dedupe even though it hasn't 1280 * checked that the bytes beyond EOF physically match. Hence we cannot use the 1281 * EOF block in the source dedupe range because it's not a complete block match, 1282 * hence can introduce a corruption into the file that has it's block replaced. 1283 * 1284 * In similar fashion, the VFS file cloning also allows partial EOF blocks to be 1285 * "block aligned" for the purposes of cloning entire files. However, if the 1286 * source file range includes the EOF block and it lands within the existing EOF 1287 * of the destination file, then we can expose stale data from beyond the source 1288 * file EOF in the destination file. 1289 * 1290 * XFS doesn't support partial block sharing, so in both cases we have check 1291 * these cases ourselves. For dedupe, we can simply round the length to dedupe 1292 * down to the previous whole block and ignore the partial EOF block. While this 1293 * means we can't dedupe the last block of a file, this is an acceptible 1294 * tradeoff for simplicity on implementation. 1295 * 1296 * For cloning, we want to share the partial EOF block if it is also the new EOF 1297 * block of the destination file. If the partial EOF block lies inside the 1298 * existing destination EOF, then we have to abort the clone to avoid exposing 1299 * stale data in the destination file. Hence we reject these clone attempts with 1300 * -EINVAL in this case. 1301 */ 1302 int 1303 xfs_reflink_remap_prep( 1304 struct file *file_in, 1305 loff_t pos_in, 1306 struct file *file_out, 1307 loff_t pos_out, 1308 loff_t *len, 1309 unsigned int remap_flags) 1310 { 1311 struct inode *inode_in = file_inode(file_in); 1312 struct xfs_inode *src = XFS_I(inode_in); 1313 struct inode *inode_out = file_inode(file_out); 1314 struct xfs_inode *dest = XFS_I(inode_out); 1315 int ret; 1316 1317 /* Lock both files against IO */ 1318 ret = xfs_ilock2_io_mmap(src, dest); 1319 if (ret) 1320 return ret; 1321 1322 /* Check file eligibility and prepare for block sharing. */ 1323 ret = -EINVAL; 1324 /* Don't reflink realtime inodes */ 1325 if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest)) 1326 goto out_unlock; 1327 1328 /* Don't share DAX file data for now. */ 1329 if (IS_DAX(inode_in) || IS_DAX(inode_out)) 1330 goto out_unlock; 1331 1332 ret = generic_remap_file_range_prep(file_in, pos_in, file_out, pos_out, 1333 len, remap_flags); 1334 if (ret || *len == 0) 1335 goto out_unlock; 1336 1337 /* Attach dquots to dest inode before changing block map */ 1338 ret = xfs_qm_dqattach(dest); 1339 if (ret) 1340 goto out_unlock; 1341 1342 /* 1343 * Zero existing post-eof speculative preallocations in the destination 1344 * file. 1345 */ 1346 ret = xfs_reflink_zero_posteof(dest, pos_out); 1347 if (ret) 1348 goto out_unlock; 1349 1350 /* Set flags and remap blocks. */ 1351 ret = xfs_reflink_set_inode_flag(src, dest); 1352 if (ret) 1353 goto out_unlock; 1354 1355 /* 1356 * If pos_out > EOF, we may have dirtied blocks between EOF and 1357 * pos_out. In that case, we need to extend the flush and unmap to cover 1358 * from EOF to the end of the copy length. 1359 */ 1360 if (pos_out > XFS_ISIZE(dest)) { 1361 loff_t flen = *len + (pos_out - XFS_ISIZE(dest)); 1362 ret = xfs_flush_unmap_range(dest, XFS_ISIZE(dest), flen); 1363 } else { 1364 ret = xfs_flush_unmap_range(dest, pos_out, *len); 1365 } 1366 if (ret) 1367 goto out_unlock; 1368 1369 return 0; 1370 out_unlock: 1371 xfs_iunlock2_io_mmap(src, dest); 1372 return ret; 1373 } 1374 1375 /* Does this inode need the reflink flag? */ 1376 int 1377 xfs_reflink_inode_has_shared_extents( 1378 struct xfs_trans *tp, 1379 struct xfs_inode *ip, 1380 bool *has_shared) 1381 { 1382 struct xfs_bmbt_irec got; 1383 struct xfs_mount *mp = ip->i_mount; 1384 struct xfs_ifork *ifp; 1385 xfs_agnumber_t agno; 1386 xfs_agblock_t agbno; 1387 xfs_extlen_t aglen; 1388 xfs_agblock_t rbno; 1389 xfs_extlen_t rlen; 1390 struct xfs_iext_cursor icur; 1391 bool found; 1392 int error; 1393 1394 ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); 1395 if (!(ifp->if_flags & XFS_IFEXTENTS)) { 1396 error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); 1397 if (error) 1398 return error; 1399 } 1400 1401 *has_shared = false; 1402 found = xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got); 1403 while (found) { 1404 if (isnullstartblock(got.br_startblock) || 1405 got.br_state != XFS_EXT_NORM) 1406 goto next; 1407 agno = XFS_FSB_TO_AGNO(mp, got.br_startblock); 1408 agbno = XFS_FSB_TO_AGBNO(mp, got.br_startblock); 1409 aglen = got.br_blockcount; 1410 1411 error = xfs_reflink_find_shared(mp, tp, agno, agbno, aglen, 1412 &rbno, &rlen, false); 1413 if (error) 1414 return error; 1415 /* Is there still a shared block here? */ 1416 if (rbno != NULLAGBLOCK) { 1417 *has_shared = true; 1418 return 0; 1419 } 1420 next: 1421 found = xfs_iext_next_extent(ifp, &icur, &got); 1422 } 1423 1424 return 0; 1425 } 1426 1427 /* 1428 * Clear the inode reflink flag if there are no shared extents. 1429 * 1430 * The caller is responsible for joining the inode to the transaction passed in. 1431 * The inode will be joined to the transaction that is returned to the caller. 1432 */ 1433 int 1434 xfs_reflink_clear_inode_flag( 1435 struct xfs_inode *ip, 1436 struct xfs_trans **tpp) 1437 { 1438 bool needs_flag; 1439 int error = 0; 1440 1441 ASSERT(xfs_is_reflink_inode(ip)); 1442 1443 error = xfs_reflink_inode_has_shared_extents(*tpp, ip, &needs_flag); 1444 if (error || needs_flag) 1445 return error; 1446 1447 /* 1448 * We didn't find any shared blocks so turn off the reflink flag. 1449 * First, get rid of any leftover CoW mappings. 1450 */ 1451 error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, XFS_MAX_FILEOFF, 1452 true); 1453 if (error) 1454 return error; 1455 1456 /* Clear the inode flag. */ 1457 trace_xfs_reflink_unset_inode_flag(ip); 1458 ip->i_d.di_flags2 &= ~XFS_DIFLAG2_REFLINK; 1459 xfs_inode_clear_cowblocks_tag(ip); 1460 xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE); 1461 1462 return error; 1463 } 1464 1465 /* 1466 * Clear the inode reflink flag if there are no shared extents and the size 1467 * hasn't changed. 1468 */ 1469 STATIC int 1470 xfs_reflink_try_clear_inode_flag( 1471 struct xfs_inode *ip) 1472 { 1473 struct xfs_mount *mp = ip->i_mount; 1474 struct xfs_trans *tp; 1475 int error = 0; 1476 1477 /* Start a rolling transaction to remove the mappings */ 1478 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); 1479 if (error) 1480 return error; 1481 1482 xfs_ilock(ip, XFS_ILOCK_EXCL); 1483 xfs_trans_ijoin(tp, ip, 0); 1484 1485 error = xfs_reflink_clear_inode_flag(ip, &tp); 1486 if (error) 1487 goto cancel; 1488 1489 error = xfs_trans_commit(tp); 1490 if (error) 1491 goto out; 1492 1493 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1494 return 0; 1495 cancel: 1496 xfs_trans_cancel(tp); 1497 out: 1498 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1499 return error; 1500 } 1501 1502 /* 1503 * Pre-COW all shared blocks within a given byte range of a file and turn off 1504 * the reflink flag if we unshare all of the file's blocks. 1505 */ 1506 int 1507 xfs_reflink_unshare( 1508 struct xfs_inode *ip, 1509 xfs_off_t offset, 1510 xfs_off_t len) 1511 { 1512 struct inode *inode = VFS_I(ip); 1513 int error; 1514 1515 if (!xfs_is_reflink_inode(ip)) 1516 return 0; 1517 1518 trace_xfs_reflink_unshare(ip, offset, len); 1519 1520 inode_dio_wait(inode); 1521 1522 error = iomap_file_unshare(inode, offset, len, 1523 &xfs_buffered_write_iomap_ops); 1524 if (error) 1525 goto out; 1526 1527 error = filemap_write_and_wait_range(inode->i_mapping, offset, len); 1528 if (error) 1529 goto out; 1530 1531 /* Turn off the reflink flag if possible. */ 1532 error = xfs_reflink_try_clear_inode_flag(ip); 1533 if (error) 1534 goto out; 1535 return 0; 1536 1537 out: 1538 trace_xfs_reflink_unshare_error(ip, error, _RET_IP_); 1539 return error; 1540 } 1541