1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2016 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <darrick.wong@oracle.com> 5 */ 6 #include "xfs.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_log_format.h" 11 #include "xfs_trans_resv.h" 12 #include "xfs_mount.h" 13 #include "xfs_defer.h" 14 #include "xfs_inode.h" 15 #include "xfs_trans.h" 16 #include "xfs_bmap.h" 17 #include "xfs_bmap_util.h" 18 #include "xfs_trace.h" 19 #include "xfs_icache.h" 20 #include "xfs_btree.h" 21 #include "xfs_refcount_btree.h" 22 #include "xfs_refcount.h" 23 #include "xfs_bmap_btree.h" 24 #include "xfs_trans_space.h" 25 #include "xfs_bit.h" 26 #include "xfs_alloc.h" 27 #include "xfs_quota.h" 28 #include "xfs_reflink.h" 29 #include "xfs_iomap.h" 30 #include "xfs_ag.h" 31 #include "xfs_ag_resv.h" 32 33 /* 34 * Copy on Write of Shared Blocks 35 * 36 * XFS must preserve "the usual" file semantics even when two files share 37 * the same physical blocks. This means that a write to one file must not 38 * alter the blocks in a different file; the way that we'll do that is 39 * through the use of a copy-on-write mechanism. At a high level, that 40 * means that when we want to write to a shared block, we allocate a new 41 * block, write the data to the new block, and if that succeeds we map the 42 * new block into the file. 43 * 44 * XFS provides a "delayed allocation" mechanism that defers the allocation 45 * of disk blocks to dirty-but-not-yet-mapped file blocks as long as 46 * possible. This reduces fragmentation by enabling the filesystem to ask 47 * for bigger chunks less often, which is exactly what we want for CoW. 48 * 49 * The delalloc mechanism begins when the kernel wants to make a block 50 * writable (write_begin or page_mkwrite). If the offset is not mapped, we 51 * create a delalloc mapping, which is a regular in-core extent, but without 52 * a real startblock. (For delalloc mappings, the startblock encodes both 53 * a flag that this is a delalloc mapping, and a worst-case estimate of how 54 * many blocks might be required to put the mapping into the BMBT.) delalloc 55 * mappings are a reservation against the free space in the filesystem; 56 * adjacent mappings can also be combined into fewer larger mappings. 57 * 58 * As an optimization, the CoW extent size hint (cowextsz) creates 59 * outsized aligned delalloc reservations in the hope of landing out of 60 * order nearby CoW writes in a single extent on disk, thereby reducing 61 * fragmentation and improving future performance. 62 * 63 * D: --RRRRRRSSSRRRRRRRR--- (data fork) 64 * C: ------DDDDDDD--------- (CoW fork) 65 * 66 * When dirty pages are being written out (typically in writepage), the 67 * delalloc reservations are converted into unwritten mappings by 68 * allocating blocks and replacing the delalloc mapping with real ones. 69 * A delalloc mapping can be replaced by several unwritten ones if the 70 * free space is fragmented. 71 * 72 * D: --RRRRRRSSSRRRRRRRR--- 73 * C: ------UUUUUUU--------- 74 * 75 * We want to adapt the delalloc mechanism for copy-on-write, since the 76 * write paths are similar. The first two steps (creating the reservation 77 * and allocating the blocks) are exactly the same as delalloc except that 78 * the mappings must be stored in a separate CoW fork because we do not want 79 * to disturb the mapping in the data fork until we're sure that the write 80 * succeeded. IO completion in this case is the process of removing the old 81 * mapping from the data fork and moving the new mapping from the CoW fork to 82 * the data fork. This will be discussed shortly. 83 * 84 * For now, unaligned directio writes will be bounced back to the page cache. 85 * Block-aligned directio writes will use the same mechanism as buffered 86 * writes. 87 * 88 * Just prior to submitting the actual disk write requests, we convert 89 * the extents representing the range of the file actually being written 90 * (as opposed to extra pieces created for the cowextsize hint) to real 91 * extents. This will become important in the next step: 92 * 93 * D: --RRRRRRSSSRRRRRRRR--- 94 * C: ------UUrrUUU--------- 95 * 96 * CoW remapping must be done after the data block write completes, 97 * because we don't want to destroy the old data fork map until we're sure 98 * the new block has been written. Since the new mappings are kept in a 99 * separate fork, we can simply iterate these mappings to find the ones 100 * that cover the file blocks that we just CoW'd. For each extent, simply 101 * unmap the corresponding range in the data fork, map the new range into 102 * the data fork, and remove the extent from the CoW fork. Because of 103 * the presence of the cowextsize hint, however, we must be careful 104 * only to remap the blocks that we've actually written out -- we must 105 * never remap delalloc reservations nor CoW staging blocks that have 106 * yet to be written. This corresponds exactly to the real extents in 107 * the CoW fork: 108 * 109 * D: --RRRRRRrrSRRRRRRRR--- 110 * C: ------UU--UUU--------- 111 * 112 * Since the remapping operation can be applied to an arbitrary file 113 * range, we record the need for the remap step as a flag in the ioend 114 * instead of declaring a new IO type. This is required for direct io 115 * because we only have ioend for the whole dio, and we have to be able to 116 * remember the presence of unwritten blocks and CoW blocks with a single 117 * ioend structure. Better yet, the more ground we can cover with one 118 * ioend, the better. 119 */ 120 121 /* 122 * Given an AG extent, find the lowest-numbered run of shared blocks 123 * within that range and return the range in fbno/flen. If 124 * find_end_of_shared is true, return the longest contiguous extent of 125 * shared blocks. If there are no shared extents, fbno and flen will 126 * be set to NULLAGBLOCK and 0, respectively. 127 */ 128 int 129 xfs_reflink_find_shared( 130 struct xfs_mount *mp, 131 struct xfs_trans *tp, 132 xfs_agnumber_t agno, 133 xfs_agblock_t agbno, 134 xfs_extlen_t aglen, 135 xfs_agblock_t *fbno, 136 xfs_extlen_t *flen, 137 bool find_end_of_shared) 138 { 139 struct xfs_buf *agbp; 140 struct xfs_btree_cur *cur; 141 int error; 142 143 error = xfs_alloc_read_agf(mp, tp, agno, 0, &agbp); 144 if (error) 145 return error; 146 147 cur = xfs_refcountbt_init_cursor(mp, tp, agbp, agbp->b_pag); 148 149 error = xfs_refcount_find_shared(cur, agbno, aglen, fbno, flen, 150 find_end_of_shared); 151 152 xfs_btree_del_cursor(cur, error); 153 154 xfs_trans_brelse(tp, agbp); 155 return error; 156 } 157 158 /* 159 * Trim the mapping to the next block where there's a change in the 160 * shared/unshared status. More specifically, this means that we 161 * find the lowest-numbered extent of shared blocks that coincides with 162 * the given block mapping. If the shared extent overlaps the start of 163 * the mapping, trim the mapping to the end of the shared extent. If 164 * the shared region intersects the mapping, trim the mapping to the 165 * start of the shared extent. If there are no shared regions that 166 * overlap, just return the original extent. 167 */ 168 int 169 xfs_reflink_trim_around_shared( 170 struct xfs_inode *ip, 171 struct xfs_bmbt_irec *irec, 172 bool *shared) 173 { 174 xfs_agnumber_t agno; 175 xfs_agblock_t agbno; 176 xfs_extlen_t aglen; 177 xfs_agblock_t fbno; 178 xfs_extlen_t flen; 179 int error = 0; 180 181 /* Holes, unwritten, and delalloc extents cannot be shared */ 182 if (!xfs_is_cow_inode(ip) || !xfs_bmap_is_written_extent(irec)) { 183 *shared = false; 184 return 0; 185 } 186 187 trace_xfs_reflink_trim_around_shared(ip, irec); 188 189 agno = XFS_FSB_TO_AGNO(ip->i_mount, irec->br_startblock); 190 agbno = XFS_FSB_TO_AGBNO(ip->i_mount, irec->br_startblock); 191 aglen = irec->br_blockcount; 192 193 error = xfs_reflink_find_shared(ip->i_mount, NULL, agno, agbno, 194 aglen, &fbno, &flen, true); 195 if (error) 196 return error; 197 198 *shared = false; 199 if (fbno == NULLAGBLOCK) { 200 /* No shared blocks at all. */ 201 return 0; 202 } else if (fbno == agbno) { 203 /* 204 * The start of this extent is shared. Truncate the 205 * mapping at the end of the shared region so that a 206 * subsequent iteration starts at the start of the 207 * unshared region. 208 */ 209 irec->br_blockcount = flen; 210 *shared = true; 211 return 0; 212 } else { 213 /* 214 * There's a shared extent midway through this extent. 215 * Truncate the mapping at the start of the shared 216 * extent so that a subsequent iteration starts at the 217 * start of the shared region. 218 */ 219 irec->br_blockcount = fbno - agbno; 220 return 0; 221 } 222 } 223 224 int 225 xfs_bmap_trim_cow( 226 struct xfs_inode *ip, 227 struct xfs_bmbt_irec *imap, 228 bool *shared) 229 { 230 /* We can't update any real extents in always COW mode. */ 231 if (xfs_is_always_cow_inode(ip) && 232 !isnullstartblock(imap->br_startblock)) { 233 *shared = true; 234 return 0; 235 } 236 237 /* Trim the mapping to the nearest shared extent boundary. */ 238 return xfs_reflink_trim_around_shared(ip, imap, shared); 239 } 240 241 static int 242 xfs_reflink_convert_cow_locked( 243 struct xfs_inode *ip, 244 xfs_fileoff_t offset_fsb, 245 xfs_filblks_t count_fsb) 246 { 247 struct xfs_iext_cursor icur; 248 struct xfs_bmbt_irec got; 249 struct xfs_btree_cur *dummy_cur = NULL; 250 int dummy_logflags; 251 int error = 0; 252 253 if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got)) 254 return 0; 255 256 do { 257 if (got.br_startoff >= offset_fsb + count_fsb) 258 break; 259 if (got.br_state == XFS_EXT_NORM) 260 continue; 261 if (WARN_ON_ONCE(isnullstartblock(got.br_startblock))) 262 return -EIO; 263 264 xfs_trim_extent(&got, offset_fsb, count_fsb); 265 if (!got.br_blockcount) 266 continue; 267 268 got.br_state = XFS_EXT_NORM; 269 error = xfs_bmap_add_extent_unwritten_real(NULL, ip, 270 XFS_COW_FORK, &icur, &dummy_cur, &got, 271 &dummy_logflags); 272 if (error) 273 return error; 274 } while (xfs_iext_next_extent(ip->i_cowfp, &icur, &got)); 275 276 return error; 277 } 278 279 /* Convert all of the unwritten CoW extents in a file's range to real ones. */ 280 int 281 xfs_reflink_convert_cow( 282 struct xfs_inode *ip, 283 xfs_off_t offset, 284 xfs_off_t count) 285 { 286 struct xfs_mount *mp = ip->i_mount; 287 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 288 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); 289 xfs_filblks_t count_fsb = end_fsb - offset_fsb; 290 int error; 291 292 ASSERT(count != 0); 293 294 xfs_ilock(ip, XFS_ILOCK_EXCL); 295 error = xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb); 296 xfs_iunlock(ip, XFS_ILOCK_EXCL); 297 return error; 298 } 299 300 /* 301 * Find the extent that maps the given range in the COW fork. Even if the extent 302 * is not shared we might have a preallocation for it in the COW fork. If so we 303 * use it that rather than trigger a new allocation. 304 */ 305 static int 306 xfs_find_trim_cow_extent( 307 struct xfs_inode *ip, 308 struct xfs_bmbt_irec *imap, 309 struct xfs_bmbt_irec *cmap, 310 bool *shared, 311 bool *found) 312 { 313 xfs_fileoff_t offset_fsb = imap->br_startoff; 314 xfs_filblks_t count_fsb = imap->br_blockcount; 315 struct xfs_iext_cursor icur; 316 317 *found = false; 318 319 /* 320 * If we don't find an overlapping extent, trim the range we need to 321 * allocate to fit the hole we found. 322 */ 323 if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, cmap)) 324 cmap->br_startoff = offset_fsb + count_fsb; 325 if (cmap->br_startoff > offset_fsb) { 326 xfs_trim_extent(imap, imap->br_startoff, 327 cmap->br_startoff - imap->br_startoff); 328 return xfs_bmap_trim_cow(ip, imap, shared); 329 } 330 331 *shared = true; 332 if (isnullstartblock(cmap->br_startblock)) { 333 xfs_trim_extent(imap, cmap->br_startoff, cmap->br_blockcount); 334 return 0; 335 } 336 337 /* real extent found - no need to allocate */ 338 xfs_trim_extent(cmap, offset_fsb, count_fsb); 339 *found = true; 340 return 0; 341 } 342 343 /* Allocate all CoW reservations covering a range of blocks in a file. */ 344 int 345 xfs_reflink_allocate_cow( 346 struct xfs_inode *ip, 347 struct xfs_bmbt_irec *imap, 348 struct xfs_bmbt_irec *cmap, 349 bool *shared, 350 uint *lockmode, 351 bool convert_now) 352 { 353 struct xfs_mount *mp = ip->i_mount; 354 xfs_fileoff_t offset_fsb = imap->br_startoff; 355 xfs_filblks_t count_fsb = imap->br_blockcount; 356 struct xfs_trans *tp; 357 int nimaps, error = 0; 358 bool found; 359 xfs_filblks_t resaligned; 360 xfs_extlen_t resblks = 0; 361 362 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); 363 if (!ip->i_cowfp) { 364 ASSERT(!xfs_is_reflink_inode(ip)); 365 xfs_ifork_init_cow(ip); 366 } 367 368 error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); 369 if (error || !*shared) 370 return error; 371 if (found) 372 goto convert; 373 374 resaligned = xfs_aligned_fsb_count(imap->br_startoff, 375 imap->br_blockcount, xfs_get_cowextsz_hint(ip)); 376 resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); 377 378 xfs_iunlock(ip, *lockmode); 379 *lockmode = 0; 380 381 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 0, 382 false, &tp); 383 if (error) 384 return error; 385 386 *lockmode = XFS_ILOCK_EXCL; 387 388 /* 389 * Check for an overlapping extent again now that we dropped the ilock. 390 */ 391 error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); 392 if (error || !*shared) 393 goto out_trans_cancel; 394 if (found) { 395 xfs_trans_cancel(tp); 396 goto convert; 397 } 398 399 /* Allocate the entire reservation as unwritten blocks. */ 400 nimaps = 1; 401 error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount, 402 XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, 0, cmap, 403 &nimaps); 404 if (error) 405 goto out_trans_cancel; 406 407 xfs_inode_set_cowblocks_tag(ip); 408 error = xfs_trans_commit(tp); 409 if (error) 410 return error; 411 412 /* 413 * Allocation succeeded but the requested range was not even partially 414 * satisfied? Bail out! 415 */ 416 if (nimaps == 0) 417 return -ENOSPC; 418 convert: 419 xfs_trim_extent(cmap, offset_fsb, count_fsb); 420 /* 421 * COW fork extents are supposed to remain unwritten until we're ready 422 * to initiate a disk write. For direct I/O we are going to write the 423 * data and need the conversion, but for buffered writes we're done. 424 */ 425 if (!convert_now || cmap->br_state == XFS_EXT_NORM) 426 return 0; 427 trace_xfs_reflink_convert_cow(ip, cmap); 428 return xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb); 429 430 out_trans_cancel: 431 xfs_trans_cancel(tp); 432 return error; 433 } 434 435 /* 436 * Cancel CoW reservations for some block range of an inode. 437 * 438 * If cancel_real is true this function cancels all COW fork extents for the 439 * inode; if cancel_real is false, real extents are not cleared. 440 * 441 * Caller must have already joined the inode to the current transaction. The 442 * inode will be joined to the transaction returned to the caller. 443 */ 444 int 445 xfs_reflink_cancel_cow_blocks( 446 struct xfs_inode *ip, 447 struct xfs_trans **tpp, 448 xfs_fileoff_t offset_fsb, 449 xfs_fileoff_t end_fsb, 450 bool cancel_real) 451 { 452 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 453 struct xfs_bmbt_irec got, del; 454 struct xfs_iext_cursor icur; 455 int error = 0; 456 457 if (!xfs_inode_has_cow_data(ip)) 458 return 0; 459 if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) 460 return 0; 461 462 /* Walk backwards until we're out of the I/O range... */ 463 while (got.br_startoff + got.br_blockcount > offset_fsb) { 464 del = got; 465 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 466 467 /* Extent delete may have bumped ext forward */ 468 if (!del.br_blockcount) { 469 xfs_iext_prev(ifp, &icur); 470 goto next_extent; 471 } 472 473 trace_xfs_reflink_cancel_cow(ip, &del); 474 475 if (isnullstartblock(del.br_startblock)) { 476 error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, 477 &icur, &got, &del); 478 if (error) 479 break; 480 } else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) { 481 ASSERT((*tpp)->t_firstblock == NULLFSBLOCK); 482 483 /* Free the CoW orphan record. */ 484 xfs_refcount_free_cow_extent(*tpp, del.br_startblock, 485 del.br_blockcount); 486 487 xfs_free_extent_later(*tpp, del.br_startblock, 488 del.br_blockcount, NULL); 489 490 /* Roll the transaction */ 491 error = xfs_defer_finish(tpp); 492 if (error) 493 break; 494 495 /* Remove the mapping from the CoW fork. */ 496 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 497 498 /* Remove the quota reservation */ 499 error = xfs_quota_unreserve_blkres(ip, 500 del.br_blockcount); 501 if (error) 502 break; 503 } else { 504 /* Didn't do anything, push cursor back. */ 505 xfs_iext_prev(ifp, &icur); 506 } 507 next_extent: 508 if (!xfs_iext_get_extent(ifp, &icur, &got)) 509 break; 510 } 511 512 /* clear tag if cow fork is emptied */ 513 if (!ifp->if_bytes) 514 xfs_inode_clear_cowblocks_tag(ip); 515 return error; 516 } 517 518 /* 519 * Cancel CoW reservations for some byte range of an inode. 520 * 521 * If cancel_real is true this function cancels all COW fork extents for the 522 * inode; if cancel_real is false, real extents are not cleared. 523 */ 524 int 525 xfs_reflink_cancel_cow_range( 526 struct xfs_inode *ip, 527 xfs_off_t offset, 528 xfs_off_t count, 529 bool cancel_real) 530 { 531 struct xfs_trans *tp; 532 xfs_fileoff_t offset_fsb; 533 xfs_fileoff_t end_fsb; 534 int error; 535 536 trace_xfs_reflink_cancel_cow_range(ip, offset, count); 537 ASSERT(ip->i_cowfp); 538 539 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 540 if (count == NULLFILEOFF) 541 end_fsb = NULLFILEOFF; 542 else 543 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 544 545 /* Start a rolling transaction to remove the mappings */ 546 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 547 0, 0, 0, &tp); 548 if (error) 549 goto out; 550 551 xfs_ilock(ip, XFS_ILOCK_EXCL); 552 xfs_trans_ijoin(tp, ip, 0); 553 554 /* Scrape out the old CoW reservations */ 555 error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb, 556 cancel_real); 557 if (error) 558 goto out_cancel; 559 560 error = xfs_trans_commit(tp); 561 562 xfs_iunlock(ip, XFS_ILOCK_EXCL); 563 return error; 564 565 out_cancel: 566 xfs_trans_cancel(tp); 567 xfs_iunlock(ip, XFS_ILOCK_EXCL); 568 out: 569 trace_xfs_reflink_cancel_cow_range_error(ip, error, _RET_IP_); 570 return error; 571 } 572 573 /* 574 * Remap part of the CoW fork into the data fork. 575 * 576 * We aim to remap the range starting at @offset_fsb and ending at @end_fsb 577 * into the data fork; this function will remap what it can (at the end of the 578 * range) and update @end_fsb appropriately. Each remap gets its own 579 * transaction because we can end up merging and splitting bmbt blocks for 580 * every remap operation and we'd like to keep the block reservation 581 * requirements as low as possible. 582 */ 583 STATIC int 584 xfs_reflink_end_cow_extent( 585 struct xfs_inode *ip, 586 xfs_fileoff_t offset_fsb, 587 xfs_fileoff_t *end_fsb) 588 { 589 struct xfs_bmbt_irec got, del; 590 struct xfs_iext_cursor icur; 591 struct xfs_mount *mp = ip->i_mount; 592 struct xfs_trans *tp; 593 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 594 xfs_filblks_t rlen; 595 unsigned int resblks; 596 int error; 597 598 /* No COW extents? That's easy! */ 599 if (ifp->if_bytes == 0) { 600 *end_fsb = offset_fsb; 601 return 0; 602 } 603 604 resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); 605 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 606 XFS_TRANS_RESERVE, &tp); 607 if (error) 608 return error; 609 610 /* 611 * Lock the inode. We have to ijoin without automatic unlock because 612 * the lead transaction is the refcountbt record deletion; the data 613 * fork update follows as a deferred log item. 614 */ 615 xfs_ilock(ip, XFS_ILOCK_EXCL); 616 xfs_trans_ijoin(tp, ip, 0); 617 618 error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, 619 XFS_IEXT_REFLINK_END_COW_CNT); 620 if (error) 621 goto out_cancel; 622 623 /* 624 * In case of racing, overlapping AIO writes no COW extents might be 625 * left by the time I/O completes for the loser of the race. In that 626 * case we are done. 627 */ 628 if (!xfs_iext_lookup_extent_before(ip, ifp, end_fsb, &icur, &got) || 629 got.br_startoff + got.br_blockcount <= offset_fsb) { 630 *end_fsb = offset_fsb; 631 goto out_cancel; 632 } 633 634 /* 635 * Structure copy @got into @del, then trim @del to the range that we 636 * were asked to remap. We preserve @got for the eventual CoW fork 637 * deletion; from now on @del represents the mapping that we're 638 * actually remapping. 639 */ 640 del = got; 641 xfs_trim_extent(&del, offset_fsb, *end_fsb - offset_fsb); 642 643 ASSERT(del.br_blockcount > 0); 644 645 /* 646 * Only remap real extents that contain data. With AIO, speculative 647 * preallocations can leak into the range we are called upon, and we 648 * need to skip them. 649 */ 650 if (!xfs_bmap_is_written_extent(&got)) { 651 *end_fsb = del.br_startoff; 652 goto out_cancel; 653 } 654 655 /* Unmap the old blocks in the data fork. */ 656 rlen = del.br_blockcount; 657 error = __xfs_bunmapi(tp, ip, del.br_startoff, &rlen, 0, 1); 658 if (error) 659 goto out_cancel; 660 661 /* Trim the extent to whatever got unmapped. */ 662 xfs_trim_extent(&del, del.br_startoff + rlen, del.br_blockcount - rlen); 663 trace_xfs_reflink_cow_remap(ip, &del); 664 665 /* Free the CoW orphan record. */ 666 xfs_refcount_free_cow_extent(tp, del.br_startblock, del.br_blockcount); 667 668 /* Map the new blocks into the data fork. */ 669 xfs_bmap_map_extent(tp, ip, &del); 670 671 /* Charge this new data fork mapping to the on-disk quota. */ 672 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_DELBCOUNT, 673 (long)del.br_blockcount); 674 675 /* Remove the mapping from the CoW fork. */ 676 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 677 678 error = xfs_trans_commit(tp); 679 xfs_iunlock(ip, XFS_ILOCK_EXCL); 680 if (error) 681 return error; 682 683 /* Update the caller about how much progress we made. */ 684 *end_fsb = del.br_startoff; 685 return 0; 686 687 out_cancel: 688 xfs_trans_cancel(tp); 689 xfs_iunlock(ip, XFS_ILOCK_EXCL); 690 return error; 691 } 692 693 /* 694 * Remap parts of a file's data fork after a successful CoW. 695 */ 696 int 697 xfs_reflink_end_cow( 698 struct xfs_inode *ip, 699 xfs_off_t offset, 700 xfs_off_t count) 701 { 702 xfs_fileoff_t offset_fsb; 703 xfs_fileoff_t end_fsb; 704 int error = 0; 705 706 trace_xfs_reflink_end_cow(ip, offset, count); 707 708 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 709 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 710 711 /* 712 * Walk backwards until we're out of the I/O range. The loop function 713 * repeatedly cycles the ILOCK to allocate one transaction per remapped 714 * extent. 715 * 716 * If we're being called by writeback then the pages will still 717 * have PageWriteback set, which prevents races with reflink remapping 718 * and truncate. Reflink remapping prevents races with writeback by 719 * taking the iolock and mmaplock before flushing the pages and 720 * remapping, which means there won't be any further writeback or page 721 * cache dirtying until the reflink completes. 722 * 723 * We should never have two threads issuing writeback for the same file 724 * region. There are also have post-eof checks in the writeback 725 * preparation code so that we don't bother writing out pages that are 726 * about to be truncated. 727 * 728 * If we're being called as part of directio write completion, the dio 729 * count is still elevated, which reflink and truncate will wait for. 730 * Reflink remapping takes the iolock and mmaplock and waits for 731 * pending dio to finish, which should prevent any directio until the 732 * remap completes. Multiple concurrent directio writes to the same 733 * region are handled by end_cow processing only occurring for the 734 * threads which succeed; the outcome of multiple overlapping direct 735 * writes is not well defined anyway. 736 * 737 * It's possible that a buffered write and a direct write could collide 738 * here (the buffered write stumbles in after the dio flushes and 739 * invalidates the page cache and immediately queues writeback), but we 740 * have never supported this 100%. If either disk write succeeds the 741 * blocks will be remapped. 742 */ 743 while (end_fsb > offset_fsb && !error) 744 error = xfs_reflink_end_cow_extent(ip, offset_fsb, &end_fsb); 745 746 if (error) 747 trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_); 748 return error; 749 } 750 751 /* 752 * Free all CoW staging blocks that are still referenced by the ondisk refcount 753 * metadata. The ondisk metadata does not track which inode created the 754 * staging extent, so callers must ensure that there are no cached inodes with 755 * live CoW staging extents. 756 */ 757 int 758 xfs_reflink_recover_cow( 759 struct xfs_mount *mp) 760 { 761 struct xfs_perag *pag; 762 xfs_agnumber_t agno; 763 int error = 0; 764 765 if (!xfs_has_reflink(mp)) 766 return 0; 767 768 for_each_perag(mp, agno, pag) { 769 error = xfs_refcount_recover_cow_leftovers(mp, pag); 770 if (error) { 771 xfs_perag_put(pag); 772 break; 773 } 774 } 775 776 return error; 777 } 778 779 /* 780 * Reflinking (Block) Ranges of Two Files Together 781 * 782 * First, ensure that the reflink flag is set on both inodes. The flag is an 783 * optimization to avoid unnecessary refcount btree lookups in the write path. 784 * 785 * Now we can iteratively remap the range of extents (and holes) in src to the 786 * corresponding ranges in dest. Let drange and srange denote the ranges of 787 * logical blocks in dest and src touched by the reflink operation. 788 * 789 * While the length of drange is greater than zero, 790 * - Read src's bmbt at the start of srange ("imap") 791 * - If imap doesn't exist, make imap appear to start at the end of srange 792 * with zero length. 793 * - If imap starts before srange, advance imap to start at srange. 794 * - If imap goes beyond srange, truncate imap to end at the end of srange. 795 * - Punch (imap start - srange start + imap len) blocks from dest at 796 * offset (drange start). 797 * - If imap points to a real range of pblks, 798 * > Increase the refcount of the imap's pblks 799 * > Map imap's pblks into dest at the offset 800 * (drange start + imap start - srange start) 801 * - Advance drange and srange by (imap start - srange start + imap len) 802 * 803 * Finally, if the reflink made dest longer, update both the in-core and 804 * on-disk file sizes. 805 * 806 * ASCII Art Demonstration: 807 * 808 * Let's say we want to reflink this source file: 809 * 810 * ----SSSSSSS-SSSSS----SSSSSS (src file) 811 * <--------------------> 812 * 813 * into this destination file: 814 * 815 * --DDDDDDDDDDDDDDDDDDD--DDD (dest file) 816 * <--------------------> 817 * '-' means a hole, and 'S' and 'D' are written blocks in the src and dest. 818 * Observe that the range has different logical offsets in either file. 819 * 820 * Consider that the first extent in the source file doesn't line up with our 821 * reflink range. Unmapping and remapping are separate operations, so we can 822 * unmap more blocks from the destination file than we remap. 823 * 824 * ----SSSSSSS-SSSSS----SSSSSS 825 * <-------> 826 * --DDDDD---------DDDDD--DDD 827 * <-------> 828 * 829 * Now remap the source extent into the destination file: 830 * 831 * ----SSSSSSS-SSSSS----SSSSSS 832 * <-------> 833 * --DDDDD--SSSSSSSDDDDD--DDD 834 * <-------> 835 * 836 * Do likewise with the second hole and extent in our range. Holes in the 837 * unmap range don't affect our operation. 838 * 839 * ----SSSSSSS-SSSSS----SSSSSS 840 * <----> 841 * --DDDDD--SSSSSSS-SSSSS-DDD 842 * <----> 843 * 844 * Finally, unmap and remap part of the third extent. This will increase the 845 * size of the destination file. 846 * 847 * ----SSSSSSS-SSSSS----SSSSSS 848 * <-----> 849 * --DDDDD--SSSSSSS-SSSSS----SSS 850 * <-----> 851 * 852 * Once we update the destination file's i_size, we're done. 853 */ 854 855 /* 856 * Ensure the reflink bit is set in both inodes. 857 */ 858 STATIC int 859 xfs_reflink_set_inode_flag( 860 struct xfs_inode *src, 861 struct xfs_inode *dest) 862 { 863 struct xfs_mount *mp = src->i_mount; 864 int error; 865 struct xfs_trans *tp; 866 867 if (xfs_is_reflink_inode(src) && xfs_is_reflink_inode(dest)) 868 return 0; 869 870 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 871 if (error) 872 goto out_error; 873 874 /* Lock both files against IO */ 875 if (src->i_ino == dest->i_ino) 876 xfs_ilock(src, XFS_ILOCK_EXCL); 877 else 878 xfs_lock_two_inodes(src, XFS_ILOCK_EXCL, dest, XFS_ILOCK_EXCL); 879 880 if (!xfs_is_reflink_inode(src)) { 881 trace_xfs_reflink_set_inode_flag(src); 882 xfs_trans_ijoin(tp, src, XFS_ILOCK_EXCL); 883 src->i_diflags2 |= XFS_DIFLAG2_REFLINK; 884 xfs_trans_log_inode(tp, src, XFS_ILOG_CORE); 885 xfs_ifork_init_cow(src); 886 } else 887 xfs_iunlock(src, XFS_ILOCK_EXCL); 888 889 if (src->i_ino == dest->i_ino) 890 goto commit_flags; 891 892 if (!xfs_is_reflink_inode(dest)) { 893 trace_xfs_reflink_set_inode_flag(dest); 894 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 895 dest->i_diflags2 |= XFS_DIFLAG2_REFLINK; 896 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 897 xfs_ifork_init_cow(dest); 898 } else 899 xfs_iunlock(dest, XFS_ILOCK_EXCL); 900 901 commit_flags: 902 error = xfs_trans_commit(tp); 903 if (error) 904 goto out_error; 905 return error; 906 907 out_error: 908 trace_xfs_reflink_set_inode_flag_error(dest, error, _RET_IP_); 909 return error; 910 } 911 912 /* 913 * Update destination inode size & cowextsize hint, if necessary. 914 */ 915 int 916 xfs_reflink_update_dest( 917 struct xfs_inode *dest, 918 xfs_off_t newlen, 919 xfs_extlen_t cowextsize, 920 unsigned int remap_flags) 921 { 922 struct xfs_mount *mp = dest->i_mount; 923 struct xfs_trans *tp; 924 int error; 925 926 if (newlen <= i_size_read(VFS_I(dest)) && cowextsize == 0) 927 return 0; 928 929 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 930 if (error) 931 goto out_error; 932 933 xfs_ilock(dest, XFS_ILOCK_EXCL); 934 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 935 936 if (newlen > i_size_read(VFS_I(dest))) { 937 trace_xfs_reflink_update_inode_size(dest, newlen); 938 i_size_write(VFS_I(dest), newlen); 939 dest->i_disk_size = newlen; 940 } 941 942 if (cowextsize) { 943 dest->i_cowextsize = cowextsize; 944 dest->i_diflags2 |= XFS_DIFLAG2_COWEXTSIZE; 945 } 946 947 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 948 949 error = xfs_trans_commit(tp); 950 if (error) 951 goto out_error; 952 return error; 953 954 out_error: 955 trace_xfs_reflink_update_inode_size_error(dest, error, _RET_IP_); 956 return error; 957 } 958 959 /* 960 * Do we have enough reserve in this AG to handle a reflink? The refcount 961 * btree already reserved all the space it needs, but the rmap btree can grow 962 * infinitely, so we won't allow more reflinks when the AG is down to the 963 * btree reserves. 964 */ 965 static int 966 xfs_reflink_ag_has_free_space( 967 struct xfs_mount *mp, 968 xfs_agnumber_t agno) 969 { 970 struct xfs_perag *pag; 971 int error = 0; 972 973 if (!xfs_has_rmapbt(mp)) 974 return 0; 975 976 pag = xfs_perag_get(mp, agno); 977 if (xfs_ag_resv_critical(pag, XFS_AG_RESV_RMAPBT) || 978 xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA)) 979 error = -ENOSPC; 980 xfs_perag_put(pag); 981 return error; 982 } 983 984 /* 985 * Remap the given extent into the file. The dmap blockcount will be set to 986 * the number of blocks that were actually remapped. 987 */ 988 STATIC int 989 xfs_reflink_remap_extent( 990 struct xfs_inode *ip, 991 struct xfs_bmbt_irec *dmap, 992 xfs_off_t new_isize) 993 { 994 struct xfs_bmbt_irec smap; 995 struct xfs_mount *mp = ip->i_mount; 996 struct xfs_trans *tp; 997 xfs_off_t newlen; 998 int64_t qdelta = 0; 999 unsigned int resblks; 1000 bool quota_reserved = true; 1001 bool smap_real; 1002 bool dmap_written = xfs_bmap_is_written_extent(dmap); 1003 int iext_delta = 0; 1004 int nimaps; 1005 int error; 1006 1007 /* 1008 * Start a rolling transaction to switch the mappings. 1009 * 1010 * Adding a written extent to the extent map can cause a bmbt split, 1011 * and removing a mapped extent from the extent can cause a bmbt split. 1012 * The two operations cannot both cause a split since they operate on 1013 * the same index in the bmap btree, so we only need a reservation for 1014 * one bmbt split if either thing is happening. However, we haven't 1015 * locked the inode yet, so we reserve assuming this is the case. 1016 * 1017 * The first allocation call tries to reserve enough space to handle 1018 * mapping dmap into a sparse part of the file plus the bmbt split. We 1019 * haven't locked the inode or read the existing mapping yet, so we do 1020 * not know for sure that we need the space. This should succeed most 1021 * of the time. 1022 * 1023 * If the first attempt fails, try again but reserving only enough 1024 * space to handle a bmbt split. This is the hard minimum requirement, 1025 * and we revisit quota reservations later when we know more about what 1026 * we're remapping. 1027 */ 1028 resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); 1029 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, 1030 resblks + dmap->br_blockcount, 0, false, &tp); 1031 if (error == -EDQUOT || error == -ENOSPC) { 1032 quota_reserved = false; 1033 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, 1034 resblks, 0, false, &tp); 1035 } 1036 if (error) 1037 goto out; 1038 1039 /* 1040 * Read what's currently mapped in the destination file into smap. 1041 * If smap isn't a hole, we will have to remove it before we can add 1042 * dmap to the destination file. 1043 */ 1044 nimaps = 1; 1045 error = xfs_bmapi_read(ip, dmap->br_startoff, dmap->br_blockcount, 1046 &smap, &nimaps, 0); 1047 if (error) 1048 goto out_cancel; 1049 ASSERT(nimaps == 1 && smap.br_startoff == dmap->br_startoff); 1050 smap_real = xfs_bmap_is_real_extent(&smap); 1051 1052 /* 1053 * We can only remap as many blocks as the smaller of the two extent 1054 * maps, because we can only remap one extent at a time. 1055 */ 1056 dmap->br_blockcount = min(dmap->br_blockcount, smap.br_blockcount); 1057 ASSERT(dmap->br_blockcount == smap.br_blockcount); 1058 1059 trace_xfs_reflink_remap_extent_dest(ip, &smap); 1060 1061 /* 1062 * Two extents mapped to the same physical block must not have 1063 * different states; that's filesystem corruption. Move on to the next 1064 * extent if they're both holes or both the same physical extent. 1065 */ 1066 if (dmap->br_startblock == smap.br_startblock) { 1067 if (dmap->br_state != smap.br_state) 1068 error = -EFSCORRUPTED; 1069 goto out_cancel; 1070 } 1071 1072 /* If both extents are unwritten, leave them alone. */ 1073 if (dmap->br_state == XFS_EXT_UNWRITTEN && 1074 smap.br_state == XFS_EXT_UNWRITTEN) 1075 goto out_cancel; 1076 1077 /* No reflinking if the AG of the dest mapping is low on space. */ 1078 if (dmap_written) { 1079 error = xfs_reflink_ag_has_free_space(mp, 1080 XFS_FSB_TO_AGNO(mp, dmap->br_startblock)); 1081 if (error) 1082 goto out_cancel; 1083 } 1084 1085 /* 1086 * Increase quota reservation if we think the quota block counter for 1087 * this file could increase. 1088 * 1089 * If we are mapping a written extent into the file, we need to have 1090 * enough quota block count reservation to handle the blocks in that 1091 * extent. We log only the delta to the quota block counts, so if the 1092 * extent we're unmapping also has blocks allocated to it, we don't 1093 * need a quota reservation for the extent itself. 1094 * 1095 * Note that if we're replacing a delalloc reservation with a written 1096 * extent, we have to take the full quota reservation because removing 1097 * the delalloc reservation gives the block count back to the quota 1098 * count. This is suboptimal, but the VFS flushed the dest range 1099 * before we started. That should have removed all the delalloc 1100 * reservations, but we code defensively. 1101 * 1102 * xfs_trans_alloc_inode above already tried to grab an even larger 1103 * quota reservation, and kicked off a blockgc scan if it couldn't. 1104 * If we can't get a potentially smaller quota reservation now, we're 1105 * done. 1106 */ 1107 if (!quota_reserved && !smap_real && dmap_written) { 1108 error = xfs_trans_reserve_quota_nblks(tp, ip, 1109 dmap->br_blockcount, 0, false); 1110 if (error) 1111 goto out_cancel; 1112 } 1113 1114 if (smap_real) 1115 ++iext_delta; 1116 1117 if (dmap_written) 1118 ++iext_delta; 1119 1120 error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, iext_delta); 1121 if (error) 1122 goto out_cancel; 1123 1124 if (smap_real) { 1125 /* 1126 * If the extent we're unmapping is backed by storage (written 1127 * or not), unmap the extent and drop its refcount. 1128 */ 1129 xfs_bmap_unmap_extent(tp, ip, &smap); 1130 xfs_refcount_decrease_extent(tp, &smap); 1131 qdelta -= smap.br_blockcount; 1132 } else if (smap.br_startblock == DELAYSTARTBLOCK) { 1133 xfs_filblks_t len = smap.br_blockcount; 1134 1135 /* 1136 * If the extent we're unmapping is a delalloc reservation, 1137 * we can use the regular bunmapi function to release the 1138 * incore state. Dropping the delalloc reservation takes care 1139 * of the quota reservation for us. 1140 */ 1141 error = __xfs_bunmapi(NULL, ip, smap.br_startoff, &len, 0, 1); 1142 if (error) 1143 goto out_cancel; 1144 ASSERT(len == 0); 1145 } 1146 1147 /* 1148 * If the extent we're sharing is backed by written storage, increase 1149 * its refcount and map it into the file. 1150 */ 1151 if (dmap_written) { 1152 xfs_refcount_increase_extent(tp, dmap); 1153 xfs_bmap_map_extent(tp, ip, dmap); 1154 qdelta += dmap->br_blockcount; 1155 } 1156 1157 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, qdelta); 1158 1159 /* Update dest isize if needed. */ 1160 newlen = XFS_FSB_TO_B(mp, dmap->br_startoff + dmap->br_blockcount); 1161 newlen = min_t(xfs_off_t, newlen, new_isize); 1162 if (newlen > i_size_read(VFS_I(ip))) { 1163 trace_xfs_reflink_update_inode_size(ip, newlen); 1164 i_size_write(VFS_I(ip), newlen); 1165 ip->i_disk_size = newlen; 1166 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1167 } 1168 1169 /* Commit everything and unlock. */ 1170 error = xfs_trans_commit(tp); 1171 goto out_unlock; 1172 1173 out_cancel: 1174 xfs_trans_cancel(tp); 1175 out_unlock: 1176 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1177 out: 1178 if (error) 1179 trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_); 1180 return error; 1181 } 1182 1183 /* Remap a range of one file to the other. */ 1184 int 1185 xfs_reflink_remap_blocks( 1186 struct xfs_inode *src, 1187 loff_t pos_in, 1188 struct xfs_inode *dest, 1189 loff_t pos_out, 1190 loff_t remap_len, 1191 loff_t *remapped) 1192 { 1193 struct xfs_bmbt_irec imap; 1194 struct xfs_mount *mp = src->i_mount; 1195 xfs_fileoff_t srcoff = XFS_B_TO_FSBT(mp, pos_in); 1196 xfs_fileoff_t destoff = XFS_B_TO_FSBT(mp, pos_out); 1197 xfs_filblks_t len; 1198 xfs_filblks_t remapped_len = 0; 1199 xfs_off_t new_isize = pos_out + remap_len; 1200 int nimaps; 1201 int error = 0; 1202 1203 len = min_t(xfs_filblks_t, XFS_B_TO_FSB(mp, remap_len), 1204 XFS_MAX_FILEOFF); 1205 1206 trace_xfs_reflink_remap_blocks(src, srcoff, len, dest, destoff); 1207 1208 while (len > 0) { 1209 unsigned int lock_mode; 1210 1211 /* Read extent from the source file */ 1212 nimaps = 1; 1213 lock_mode = xfs_ilock_data_map_shared(src); 1214 error = xfs_bmapi_read(src, srcoff, len, &imap, &nimaps, 0); 1215 xfs_iunlock(src, lock_mode); 1216 if (error) 1217 break; 1218 /* 1219 * The caller supposedly flushed all dirty pages in the source 1220 * file range, which means that writeback should have allocated 1221 * or deleted all delalloc reservations in that range. If we 1222 * find one, that's a good sign that something is seriously 1223 * wrong here. 1224 */ 1225 ASSERT(nimaps == 1 && imap.br_startoff == srcoff); 1226 if (imap.br_startblock == DELAYSTARTBLOCK) { 1227 ASSERT(imap.br_startblock != DELAYSTARTBLOCK); 1228 error = -EFSCORRUPTED; 1229 break; 1230 } 1231 1232 trace_xfs_reflink_remap_extent_src(src, &imap); 1233 1234 /* Remap into the destination file at the given offset. */ 1235 imap.br_startoff = destoff; 1236 error = xfs_reflink_remap_extent(dest, &imap, new_isize); 1237 if (error) 1238 break; 1239 1240 if (fatal_signal_pending(current)) { 1241 error = -EINTR; 1242 break; 1243 } 1244 1245 /* Advance drange/srange */ 1246 srcoff += imap.br_blockcount; 1247 destoff += imap.br_blockcount; 1248 len -= imap.br_blockcount; 1249 remapped_len += imap.br_blockcount; 1250 } 1251 1252 if (error) 1253 trace_xfs_reflink_remap_blocks_error(dest, error, _RET_IP_); 1254 *remapped = min_t(loff_t, remap_len, 1255 XFS_FSB_TO_B(src->i_mount, remapped_len)); 1256 return error; 1257 } 1258 1259 /* 1260 * If we're reflinking to a point past the destination file's EOF, we must 1261 * zero any speculative post-EOF preallocations that sit between the old EOF 1262 * and the destination file offset. 1263 */ 1264 static int 1265 xfs_reflink_zero_posteof( 1266 struct xfs_inode *ip, 1267 loff_t pos) 1268 { 1269 loff_t isize = i_size_read(VFS_I(ip)); 1270 1271 if (pos <= isize) 1272 return 0; 1273 1274 trace_xfs_zero_eof(ip, isize, pos - isize); 1275 return xfs_zero_range(ip, isize, pos - isize, NULL); 1276 } 1277 1278 /* 1279 * Prepare two files for range cloning. Upon a successful return both inodes 1280 * will have the iolock and mmaplock held, the page cache of the out file will 1281 * be truncated, and any leases on the out file will have been broken. This 1282 * function borrows heavily from xfs_file_aio_write_checks. 1283 * 1284 * The VFS allows partial EOF blocks to "match" for dedupe even though it hasn't 1285 * checked that the bytes beyond EOF physically match. Hence we cannot use the 1286 * EOF block in the source dedupe range because it's not a complete block match, 1287 * hence can introduce a corruption into the file that has it's block replaced. 1288 * 1289 * In similar fashion, the VFS file cloning also allows partial EOF blocks to be 1290 * "block aligned" for the purposes of cloning entire files. However, if the 1291 * source file range includes the EOF block and it lands within the existing EOF 1292 * of the destination file, then we can expose stale data from beyond the source 1293 * file EOF in the destination file. 1294 * 1295 * XFS doesn't support partial block sharing, so in both cases we have check 1296 * these cases ourselves. For dedupe, we can simply round the length to dedupe 1297 * down to the previous whole block and ignore the partial EOF block. While this 1298 * means we can't dedupe the last block of a file, this is an acceptible 1299 * tradeoff for simplicity on implementation. 1300 * 1301 * For cloning, we want to share the partial EOF block if it is also the new EOF 1302 * block of the destination file. If the partial EOF block lies inside the 1303 * existing destination EOF, then we have to abort the clone to avoid exposing 1304 * stale data in the destination file. Hence we reject these clone attempts with 1305 * -EINVAL in this case. 1306 */ 1307 int 1308 xfs_reflink_remap_prep( 1309 struct file *file_in, 1310 loff_t pos_in, 1311 struct file *file_out, 1312 loff_t pos_out, 1313 loff_t *len, 1314 unsigned int remap_flags) 1315 { 1316 struct inode *inode_in = file_inode(file_in); 1317 struct xfs_inode *src = XFS_I(inode_in); 1318 struct inode *inode_out = file_inode(file_out); 1319 struct xfs_inode *dest = XFS_I(inode_out); 1320 int ret; 1321 1322 /* Lock both files against IO */ 1323 ret = xfs_ilock2_io_mmap(src, dest); 1324 if (ret) 1325 return ret; 1326 1327 /* Check file eligibility and prepare for block sharing. */ 1328 ret = -EINVAL; 1329 /* Don't reflink realtime inodes */ 1330 if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest)) 1331 goto out_unlock; 1332 1333 /* Don't share DAX file data for now. */ 1334 if (IS_DAX(inode_in) || IS_DAX(inode_out)) 1335 goto out_unlock; 1336 1337 ret = generic_remap_file_range_prep(file_in, pos_in, file_out, pos_out, 1338 len, remap_flags); 1339 if (ret || *len == 0) 1340 goto out_unlock; 1341 1342 /* Attach dquots to dest inode before changing block map */ 1343 ret = xfs_qm_dqattach(dest); 1344 if (ret) 1345 goto out_unlock; 1346 1347 /* 1348 * Zero existing post-eof speculative preallocations in the destination 1349 * file. 1350 */ 1351 ret = xfs_reflink_zero_posteof(dest, pos_out); 1352 if (ret) 1353 goto out_unlock; 1354 1355 /* Set flags and remap blocks. */ 1356 ret = xfs_reflink_set_inode_flag(src, dest); 1357 if (ret) 1358 goto out_unlock; 1359 1360 /* 1361 * If pos_out > EOF, we may have dirtied blocks between EOF and 1362 * pos_out. In that case, we need to extend the flush and unmap to cover 1363 * from EOF to the end of the copy length. 1364 */ 1365 if (pos_out > XFS_ISIZE(dest)) { 1366 loff_t flen = *len + (pos_out - XFS_ISIZE(dest)); 1367 ret = xfs_flush_unmap_range(dest, XFS_ISIZE(dest), flen); 1368 } else { 1369 ret = xfs_flush_unmap_range(dest, pos_out, *len); 1370 } 1371 if (ret) 1372 goto out_unlock; 1373 1374 return 0; 1375 out_unlock: 1376 xfs_iunlock2_io_mmap(src, dest); 1377 return ret; 1378 } 1379 1380 /* Does this inode need the reflink flag? */ 1381 int 1382 xfs_reflink_inode_has_shared_extents( 1383 struct xfs_trans *tp, 1384 struct xfs_inode *ip, 1385 bool *has_shared) 1386 { 1387 struct xfs_bmbt_irec got; 1388 struct xfs_mount *mp = ip->i_mount; 1389 struct xfs_ifork *ifp; 1390 xfs_agnumber_t agno; 1391 xfs_agblock_t agbno; 1392 xfs_extlen_t aglen; 1393 xfs_agblock_t rbno; 1394 xfs_extlen_t rlen; 1395 struct xfs_iext_cursor icur; 1396 bool found; 1397 int error; 1398 1399 ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); 1400 error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); 1401 if (error) 1402 return error; 1403 1404 *has_shared = false; 1405 found = xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got); 1406 while (found) { 1407 if (isnullstartblock(got.br_startblock) || 1408 got.br_state != XFS_EXT_NORM) 1409 goto next; 1410 agno = XFS_FSB_TO_AGNO(mp, got.br_startblock); 1411 agbno = XFS_FSB_TO_AGBNO(mp, got.br_startblock); 1412 aglen = got.br_blockcount; 1413 1414 error = xfs_reflink_find_shared(mp, tp, agno, agbno, aglen, 1415 &rbno, &rlen, false); 1416 if (error) 1417 return error; 1418 /* Is there still a shared block here? */ 1419 if (rbno != NULLAGBLOCK) { 1420 *has_shared = true; 1421 return 0; 1422 } 1423 next: 1424 found = xfs_iext_next_extent(ifp, &icur, &got); 1425 } 1426 1427 return 0; 1428 } 1429 1430 /* 1431 * Clear the inode reflink flag if there are no shared extents. 1432 * 1433 * The caller is responsible for joining the inode to the transaction passed in. 1434 * The inode will be joined to the transaction that is returned to the caller. 1435 */ 1436 int 1437 xfs_reflink_clear_inode_flag( 1438 struct xfs_inode *ip, 1439 struct xfs_trans **tpp) 1440 { 1441 bool needs_flag; 1442 int error = 0; 1443 1444 ASSERT(xfs_is_reflink_inode(ip)); 1445 1446 error = xfs_reflink_inode_has_shared_extents(*tpp, ip, &needs_flag); 1447 if (error || needs_flag) 1448 return error; 1449 1450 /* 1451 * We didn't find any shared blocks so turn off the reflink flag. 1452 * First, get rid of any leftover CoW mappings. 1453 */ 1454 error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, XFS_MAX_FILEOFF, 1455 true); 1456 if (error) 1457 return error; 1458 1459 /* Clear the inode flag. */ 1460 trace_xfs_reflink_unset_inode_flag(ip); 1461 ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; 1462 xfs_inode_clear_cowblocks_tag(ip); 1463 xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE); 1464 1465 return error; 1466 } 1467 1468 /* 1469 * Clear the inode reflink flag if there are no shared extents and the size 1470 * hasn't changed. 1471 */ 1472 STATIC int 1473 xfs_reflink_try_clear_inode_flag( 1474 struct xfs_inode *ip) 1475 { 1476 struct xfs_mount *mp = ip->i_mount; 1477 struct xfs_trans *tp; 1478 int error = 0; 1479 1480 /* Start a rolling transaction to remove the mappings */ 1481 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); 1482 if (error) 1483 return error; 1484 1485 xfs_ilock(ip, XFS_ILOCK_EXCL); 1486 xfs_trans_ijoin(tp, ip, 0); 1487 1488 error = xfs_reflink_clear_inode_flag(ip, &tp); 1489 if (error) 1490 goto cancel; 1491 1492 error = xfs_trans_commit(tp); 1493 if (error) 1494 goto out; 1495 1496 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1497 return 0; 1498 cancel: 1499 xfs_trans_cancel(tp); 1500 out: 1501 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1502 return error; 1503 } 1504 1505 /* 1506 * Pre-COW all shared blocks within a given byte range of a file and turn off 1507 * the reflink flag if we unshare all of the file's blocks. 1508 */ 1509 int 1510 xfs_reflink_unshare( 1511 struct xfs_inode *ip, 1512 xfs_off_t offset, 1513 xfs_off_t len) 1514 { 1515 struct inode *inode = VFS_I(ip); 1516 int error; 1517 1518 if (!xfs_is_reflink_inode(ip)) 1519 return 0; 1520 1521 trace_xfs_reflink_unshare(ip, offset, len); 1522 1523 inode_dio_wait(inode); 1524 1525 error = iomap_file_unshare(inode, offset, len, 1526 &xfs_buffered_write_iomap_ops); 1527 if (error) 1528 goto out; 1529 1530 error = filemap_write_and_wait_range(inode->i_mapping, offset, 1531 offset + len - 1); 1532 if (error) 1533 goto out; 1534 1535 /* Turn off the reflink flag if possible. */ 1536 error = xfs_reflink_try_clear_inode_flag(ip); 1537 if (error) 1538 goto out; 1539 return 0; 1540 1541 out: 1542 trace_xfs_reflink_unshare_error(ip, error, _RET_IP_); 1543 return error; 1544 } 1545