1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2016 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <darrick.wong@oracle.com> 5 */ 6 #include "xfs.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_log_format.h" 11 #include "xfs_trans_resv.h" 12 #include "xfs_mount.h" 13 #include "xfs_defer.h" 14 #include "xfs_da_format.h" 15 #include "xfs_da_btree.h" 16 #include "xfs_inode.h" 17 #include "xfs_trans.h" 18 #include "xfs_inode_item.h" 19 #include "xfs_bmap.h" 20 #include "xfs_bmap_util.h" 21 #include "xfs_error.h" 22 #include "xfs_dir2.h" 23 #include "xfs_dir2_priv.h" 24 #include "xfs_ioctl.h" 25 #include "xfs_trace.h" 26 #include "xfs_log.h" 27 #include "xfs_icache.h" 28 #include "xfs_pnfs.h" 29 #include "xfs_btree.h" 30 #include "xfs_refcount_btree.h" 31 #include "xfs_refcount.h" 32 #include "xfs_bmap_btree.h" 33 #include "xfs_trans_space.h" 34 #include "xfs_bit.h" 35 #include "xfs_alloc.h" 36 #include "xfs_quota_defs.h" 37 #include "xfs_quota.h" 38 #include "xfs_reflink.h" 39 #include "xfs_iomap.h" 40 #include "xfs_rmap_btree.h" 41 #include "xfs_sb.h" 42 #include "xfs_ag_resv.h" 43 44 /* 45 * Copy on Write of Shared Blocks 46 * 47 * XFS must preserve "the usual" file semantics even when two files share 48 * the same physical blocks. This means that a write to one file must not 49 * alter the blocks in a different file; the way that we'll do that is 50 * through the use of a copy-on-write mechanism. At a high level, that 51 * means that when we want to write to a shared block, we allocate a new 52 * block, write the data to the new block, and if that succeeds we map the 53 * new block into the file. 54 * 55 * XFS provides a "delayed allocation" mechanism that defers the allocation 56 * of disk blocks to dirty-but-not-yet-mapped file blocks as long as 57 * possible. This reduces fragmentation by enabling the filesystem to ask 58 * for bigger chunks less often, which is exactly what we want for CoW. 59 * 60 * The delalloc mechanism begins when the kernel wants to make a block 61 * writable (write_begin or page_mkwrite). If the offset is not mapped, we 62 * create a delalloc mapping, which is a regular in-core extent, but without 63 * a real startblock. (For delalloc mappings, the startblock encodes both 64 * a flag that this is a delalloc mapping, and a worst-case estimate of how 65 * many blocks might be required to put the mapping into the BMBT.) delalloc 66 * mappings are a reservation against the free space in the filesystem; 67 * adjacent mappings can also be combined into fewer larger mappings. 68 * 69 * As an optimization, the CoW extent size hint (cowextsz) creates 70 * outsized aligned delalloc reservations in the hope of landing out of 71 * order nearby CoW writes in a single extent on disk, thereby reducing 72 * fragmentation and improving future performance. 73 * 74 * D: --RRRRRRSSSRRRRRRRR--- (data fork) 75 * C: ------DDDDDDD--------- (CoW fork) 76 * 77 * When dirty pages are being written out (typically in writepage), the 78 * delalloc reservations are converted into unwritten mappings by 79 * allocating blocks and replacing the delalloc mapping with real ones. 80 * A delalloc mapping can be replaced by several unwritten ones if the 81 * free space is fragmented. 82 * 83 * D: --RRRRRRSSSRRRRRRRR--- 84 * C: ------UUUUUUU--------- 85 * 86 * We want to adapt the delalloc mechanism for copy-on-write, since the 87 * write paths are similar. The first two steps (creating the reservation 88 * and allocating the blocks) are exactly the same as delalloc except that 89 * the mappings must be stored in a separate CoW fork because we do not want 90 * to disturb the mapping in the data fork until we're sure that the write 91 * succeeded. IO completion in this case is the process of removing the old 92 * mapping from the data fork and moving the new mapping from the CoW fork to 93 * the data fork. This will be discussed shortly. 94 * 95 * For now, unaligned directio writes will be bounced back to the page cache. 96 * Block-aligned directio writes will use the same mechanism as buffered 97 * writes. 98 * 99 * Just prior to submitting the actual disk write requests, we convert 100 * the extents representing the range of the file actually being written 101 * (as opposed to extra pieces created for the cowextsize hint) to real 102 * extents. This will become important in the next step: 103 * 104 * D: --RRRRRRSSSRRRRRRRR--- 105 * C: ------UUrrUUU--------- 106 * 107 * CoW remapping must be done after the data block write completes, 108 * because we don't want to destroy the old data fork map until we're sure 109 * the new block has been written. Since the new mappings are kept in a 110 * separate fork, we can simply iterate these mappings to find the ones 111 * that cover the file blocks that we just CoW'd. For each extent, simply 112 * unmap the corresponding range in the data fork, map the new range into 113 * the data fork, and remove the extent from the CoW fork. Because of 114 * the presence of the cowextsize hint, however, we must be careful 115 * only to remap the blocks that we've actually written out -- we must 116 * never remap delalloc reservations nor CoW staging blocks that have 117 * yet to be written. This corresponds exactly to the real extents in 118 * the CoW fork: 119 * 120 * D: --RRRRRRrrSRRRRRRRR--- 121 * C: ------UU--UUU--------- 122 * 123 * Since the remapping operation can be applied to an arbitrary file 124 * range, we record the need for the remap step as a flag in the ioend 125 * instead of declaring a new IO type. This is required for direct io 126 * because we only have ioend for the whole dio, and we have to be able to 127 * remember the presence of unwritten blocks and CoW blocks with a single 128 * ioend structure. Better yet, the more ground we can cover with one 129 * ioend, the better. 130 */ 131 132 /* 133 * Given an AG extent, find the lowest-numbered run of shared blocks 134 * within that range and return the range in fbno/flen. If 135 * find_end_of_shared is true, return the longest contiguous extent of 136 * shared blocks. If there are no shared extents, fbno and flen will 137 * be set to NULLAGBLOCK and 0, respectively. 138 */ 139 int 140 xfs_reflink_find_shared( 141 struct xfs_mount *mp, 142 struct xfs_trans *tp, 143 xfs_agnumber_t agno, 144 xfs_agblock_t agbno, 145 xfs_extlen_t aglen, 146 xfs_agblock_t *fbno, 147 xfs_extlen_t *flen, 148 bool find_end_of_shared) 149 { 150 struct xfs_buf *agbp; 151 struct xfs_btree_cur *cur; 152 int error; 153 154 error = xfs_alloc_read_agf(mp, tp, agno, 0, &agbp); 155 if (error) 156 return error; 157 if (!agbp) 158 return -ENOMEM; 159 160 cur = xfs_refcountbt_init_cursor(mp, tp, agbp, agno); 161 162 error = xfs_refcount_find_shared(cur, agbno, aglen, fbno, flen, 163 find_end_of_shared); 164 165 xfs_btree_del_cursor(cur, error); 166 167 xfs_trans_brelse(tp, agbp); 168 return error; 169 } 170 171 /* 172 * Trim the mapping to the next block where there's a change in the 173 * shared/unshared status. More specifically, this means that we 174 * find the lowest-numbered extent of shared blocks that coincides with 175 * the given block mapping. If the shared extent overlaps the start of 176 * the mapping, trim the mapping to the end of the shared extent. If 177 * the shared region intersects the mapping, trim the mapping to the 178 * start of the shared extent. If there are no shared regions that 179 * overlap, just return the original extent. 180 */ 181 int 182 xfs_reflink_trim_around_shared( 183 struct xfs_inode *ip, 184 struct xfs_bmbt_irec *irec, 185 bool *shared, 186 bool *trimmed) 187 { 188 xfs_agnumber_t agno; 189 xfs_agblock_t agbno; 190 xfs_extlen_t aglen; 191 xfs_agblock_t fbno; 192 xfs_extlen_t flen; 193 int error = 0; 194 195 /* Holes, unwritten, and delalloc extents cannot be shared */ 196 if (!xfs_is_reflink_inode(ip) || !xfs_bmap_is_real_extent(irec)) { 197 *shared = false; 198 return 0; 199 } 200 201 trace_xfs_reflink_trim_around_shared(ip, irec); 202 203 agno = XFS_FSB_TO_AGNO(ip->i_mount, irec->br_startblock); 204 agbno = XFS_FSB_TO_AGBNO(ip->i_mount, irec->br_startblock); 205 aglen = irec->br_blockcount; 206 207 error = xfs_reflink_find_shared(ip->i_mount, NULL, agno, agbno, 208 aglen, &fbno, &flen, true); 209 if (error) 210 return error; 211 212 *shared = *trimmed = false; 213 if (fbno == NULLAGBLOCK) { 214 /* No shared blocks at all. */ 215 return 0; 216 } else if (fbno == agbno) { 217 /* 218 * The start of this extent is shared. Truncate the 219 * mapping at the end of the shared region so that a 220 * subsequent iteration starts at the start of the 221 * unshared region. 222 */ 223 irec->br_blockcount = flen; 224 *shared = true; 225 if (flen != aglen) 226 *trimmed = true; 227 return 0; 228 } else { 229 /* 230 * There's a shared extent midway through this extent. 231 * Truncate the mapping at the start of the shared 232 * extent so that a subsequent iteration starts at the 233 * start of the shared region. 234 */ 235 irec->br_blockcount = fbno - agbno; 236 *trimmed = true; 237 return 0; 238 } 239 } 240 241 /* 242 * Trim the passed in imap to the next shared/unshared extent boundary, and 243 * if imap->br_startoff points to a shared extent reserve space for it in the 244 * COW fork. In this case *shared is set to true, else to false. 245 * 246 * Note that imap will always contain the block numbers for the existing blocks 247 * in the data fork, as the upper layers need them for read-modify-write 248 * operations. 249 */ 250 int 251 xfs_reflink_reserve_cow( 252 struct xfs_inode *ip, 253 struct xfs_bmbt_irec *imap, 254 bool *shared) 255 { 256 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 257 struct xfs_bmbt_irec got; 258 int error = 0; 259 bool eof = false, trimmed; 260 struct xfs_iext_cursor icur; 261 262 /* 263 * Search the COW fork extent list first. This serves two purposes: 264 * first this implement the speculative preallocation using cowextisze, 265 * so that we also unshared block adjacent to shared blocks instead 266 * of just the shared blocks themselves. Second the lookup in the 267 * extent list is generally faster than going out to the shared extent 268 * tree. 269 */ 270 271 if (!xfs_iext_lookup_extent(ip, ifp, imap->br_startoff, &icur, &got)) 272 eof = true; 273 if (!eof && got.br_startoff <= imap->br_startoff) { 274 trace_xfs_reflink_cow_found(ip, imap); 275 xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); 276 277 *shared = true; 278 return 0; 279 } 280 281 /* Trim the mapping to the nearest shared extent boundary. */ 282 error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed); 283 if (error) 284 return error; 285 286 /* Not shared? Just report the (potentially capped) extent. */ 287 if (!*shared) 288 return 0; 289 290 /* 291 * Fork all the shared blocks from our write offset until the end of 292 * the extent. 293 */ 294 error = xfs_qm_dqattach_locked(ip, false); 295 if (error) 296 return error; 297 298 error = xfs_bmapi_reserve_delalloc(ip, XFS_COW_FORK, imap->br_startoff, 299 imap->br_blockcount, 0, &got, &icur, eof); 300 if (error == -ENOSPC || error == -EDQUOT) 301 trace_xfs_reflink_cow_enospc(ip, imap); 302 if (error) 303 return error; 304 305 trace_xfs_reflink_cow_alloc(ip, &got); 306 return 0; 307 } 308 309 /* Convert part of an unwritten CoW extent to a real one. */ 310 STATIC int 311 xfs_reflink_convert_cow_extent( 312 struct xfs_inode *ip, 313 struct xfs_bmbt_irec *imap, 314 xfs_fileoff_t offset_fsb, 315 xfs_filblks_t count_fsb) 316 { 317 int nimaps = 1; 318 319 if (imap->br_state == XFS_EXT_NORM) 320 return 0; 321 322 xfs_trim_extent(imap, offset_fsb, count_fsb); 323 trace_xfs_reflink_convert_cow(ip, imap); 324 if (imap->br_blockcount == 0) 325 return 0; 326 return xfs_bmapi_write(NULL, ip, imap->br_startoff, imap->br_blockcount, 327 XFS_BMAPI_COWFORK | XFS_BMAPI_CONVERT, 0, imap, 328 &nimaps); 329 } 330 331 /* Convert all of the unwritten CoW extents in a file's range to real ones. */ 332 int 333 xfs_reflink_convert_cow( 334 struct xfs_inode *ip, 335 xfs_off_t offset, 336 xfs_off_t count) 337 { 338 struct xfs_mount *mp = ip->i_mount; 339 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 340 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); 341 xfs_filblks_t count_fsb = end_fsb - offset_fsb; 342 struct xfs_bmbt_irec imap; 343 int nimaps = 1, error = 0; 344 345 ASSERT(count != 0); 346 347 xfs_ilock(ip, XFS_ILOCK_EXCL); 348 error = xfs_bmapi_write(NULL, ip, offset_fsb, count_fsb, 349 XFS_BMAPI_COWFORK | XFS_BMAPI_CONVERT | 350 XFS_BMAPI_CONVERT_ONLY, 0, &imap, &nimaps); 351 xfs_iunlock(ip, XFS_ILOCK_EXCL); 352 return error; 353 } 354 355 /* Allocate all CoW reservations covering a range of blocks in a file. */ 356 int 357 xfs_reflink_allocate_cow( 358 struct xfs_inode *ip, 359 struct xfs_bmbt_irec *imap, 360 bool *shared, 361 uint *lockmode) 362 { 363 struct xfs_mount *mp = ip->i_mount; 364 xfs_fileoff_t offset_fsb = imap->br_startoff; 365 xfs_filblks_t count_fsb = imap->br_blockcount; 366 struct xfs_bmbt_irec got; 367 struct xfs_trans *tp = NULL; 368 int nimaps, error = 0; 369 bool trimmed; 370 xfs_filblks_t resaligned; 371 xfs_extlen_t resblks = 0; 372 struct xfs_iext_cursor icur; 373 374 retry: 375 ASSERT(xfs_is_reflink_inode(ip)); 376 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); 377 378 /* 379 * Even if the extent is not shared we might have a preallocation for 380 * it in the COW fork. If so use it. 381 */ 382 if (xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got) && 383 got.br_startoff <= offset_fsb) { 384 *shared = true; 385 386 /* If we have a real allocation in the COW fork we're done. */ 387 if (!isnullstartblock(got.br_startblock)) { 388 xfs_trim_extent(&got, offset_fsb, count_fsb); 389 *imap = got; 390 goto convert; 391 } 392 393 xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); 394 } else { 395 error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed); 396 if (error || !*shared) 397 goto out; 398 } 399 400 if (!tp) { 401 resaligned = xfs_aligned_fsb_count(imap->br_startoff, 402 imap->br_blockcount, xfs_get_cowextsz_hint(ip)); 403 resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); 404 405 xfs_iunlock(ip, *lockmode); 406 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); 407 *lockmode = XFS_ILOCK_EXCL; 408 xfs_ilock(ip, *lockmode); 409 410 if (error) 411 return error; 412 413 error = xfs_qm_dqattach_locked(ip, false); 414 if (error) 415 goto out; 416 goto retry; 417 } 418 419 error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, 420 XFS_QMOPT_RES_REGBLKS); 421 if (error) 422 goto out; 423 424 xfs_trans_ijoin(tp, ip, 0); 425 426 nimaps = 1; 427 428 /* Allocate the entire reservation as unwritten blocks. */ 429 error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount, 430 XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, 431 resblks, imap, &nimaps); 432 if (error) 433 goto out_trans_cancel; 434 435 xfs_inode_set_cowblocks_tag(ip); 436 437 /* Finish up. */ 438 error = xfs_trans_commit(tp); 439 if (error) 440 return error; 441 442 /* 443 * Allocation succeeded but the requested range was not even partially 444 * satisfied? Bail out! 445 */ 446 if (nimaps == 0) 447 return -ENOSPC; 448 convert: 449 return xfs_reflink_convert_cow_extent(ip, imap, offset_fsb, count_fsb); 450 out_trans_cancel: 451 xfs_trans_unreserve_quota_nblks(tp, ip, (long)resblks, 0, 452 XFS_QMOPT_RES_REGBLKS); 453 out: 454 if (tp) 455 xfs_trans_cancel(tp); 456 return error; 457 } 458 459 /* 460 * Cancel CoW reservations for some block range of an inode. 461 * 462 * If cancel_real is true this function cancels all COW fork extents for the 463 * inode; if cancel_real is false, real extents are not cleared. 464 * 465 * Caller must have already joined the inode to the current transaction. The 466 * inode will be joined to the transaction returned to the caller. 467 */ 468 int 469 xfs_reflink_cancel_cow_blocks( 470 struct xfs_inode *ip, 471 struct xfs_trans **tpp, 472 xfs_fileoff_t offset_fsb, 473 xfs_fileoff_t end_fsb, 474 bool cancel_real) 475 { 476 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 477 struct xfs_bmbt_irec got, del; 478 struct xfs_iext_cursor icur; 479 int error = 0; 480 481 if (!xfs_inode_has_cow_data(ip)) 482 return 0; 483 if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) 484 return 0; 485 486 /* Walk backwards until we're out of the I/O range... */ 487 while (got.br_startoff + got.br_blockcount > offset_fsb) { 488 del = got; 489 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 490 491 /* Extent delete may have bumped ext forward */ 492 if (!del.br_blockcount) { 493 xfs_iext_prev(ifp, &icur); 494 goto next_extent; 495 } 496 497 trace_xfs_reflink_cancel_cow(ip, &del); 498 499 if (isnullstartblock(del.br_startblock)) { 500 error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, 501 &icur, &got, &del); 502 if (error) 503 break; 504 } else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) { 505 ASSERT((*tpp)->t_firstblock == NULLFSBLOCK); 506 507 /* Free the CoW orphan record. */ 508 error = xfs_refcount_free_cow_extent(*tpp, 509 del.br_startblock, del.br_blockcount); 510 if (error) 511 break; 512 513 xfs_bmap_add_free(*tpp, del.br_startblock, 514 del.br_blockcount, NULL); 515 516 /* Roll the transaction */ 517 error = xfs_defer_finish(tpp); 518 if (error) 519 break; 520 521 /* Remove the mapping from the CoW fork. */ 522 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 523 524 /* Remove the quota reservation */ 525 error = xfs_trans_reserve_quota_nblks(NULL, ip, 526 -(long)del.br_blockcount, 0, 527 XFS_QMOPT_RES_REGBLKS); 528 if (error) 529 break; 530 } else { 531 /* Didn't do anything, push cursor back. */ 532 xfs_iext_prev(ifp, &icur); 533 } 534 next_extent: 535 if (!xfs_iext_get_extent(ifp, &icur, &got)) 536 break; 537 } 538 539 /* clear tag if cow fork is emptied */ 540 if (!ifp->if_bytes) 541 xfs_inode_clear_cowblocks_tag(ip); 542 return error; 543 } 544 545 /* 546 * Cancel CoW reservations for some byte range of an inode. 547 * 548 * If cancel_real is true this function cancels all COW fork extents for the 549 * inode; if cancel_real is false, real extents are not cleared. 550 */ 551 int 552 xfs_reflink_cancel_cow_range( 553 struct xfs_inode *ip, 554 xfs_off_t offset, 555 xfs_off_t count, 556 bool cancel_real) 557 { 558 struct xfs_trans *tp; 559 xfs_fileoff_t offset_fsb; 560 xfs_fileoff_t end_fsb; 561 int error; 562 563 trace_xfs_reflink_cancel_cow_range(ip, offset, count); 564 ASSERT(xfs_is_reflink_inode(ip)); 565 566 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 567 if (count == NULLFILEOFF) 568 end_fsb = NULLFILEOFF; 569 else 570 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 571 572 /* Start a rolling transaction to remove the mappings */ 573 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 574 0, 0, XFS_TRANS_NOFS, &tp); 575 if (error) 576 goto out; 577 578 xfs_ilock(ip, XFS_ILOCK_EXCL); 579 xfs_trans_ijoin(tp, ip, 0); 580 581 /* Scrape out the old CoW reservations */ 582 error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb, 583 cancel_real); 584 if (error) 585 goto out_cancel; 586 587 error = xfs_trans_commit(tp); 588 589 xfs_iunlock(ip, XFS_ILOCK_EXCL); 590 return error; 591 592 out_cancel: 593 xfs_trans_cancel(tp); 594 xfs_iunlock(ip, XFS_ILOCK_EXCL); 595 out: 596 trace_xfs_reflink_cancel_cow_range_error(ip, error, _RET_IP_); 597 return error; 598 } 599 600 /* 601 * Remap parts of a file's data fork after a successful CoW. 602 */ 603 int 604 xfs_reflink_end_cow( 605 struct xfs_inode *ip, 606 xfs_off_t offset, 607 xfs_off_t count) 608 { 609 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 610 struct xfs_bmbt_irec got, del; 611 struct xfs_trans *tp; 612 xfs_fileoff_t offset_fsb; 613 xfs_fileoff_t end_fsb; 614 int error; 615 unsigned int resblks; 616 xfs_filblks_t rlen; 617 struct xfs_iext_cursor icur; 618 619 trace_xfs_reflink_end_cow(ip, offset, count); 620 621 /* No COW extents? That's easy! */ 622 if (ifp->if_bytes == 0) 623 return 0; 624 625 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 626 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 627 628 /* 629 * Start a rolling transaction to switch the mappings. We're 630 * unlikely ever to have to remap 16T worth of single-block 631 * extents, so just cap the worst case extent count to 2^32-1. 632 * Stick a warning in just in case, and avoid 64-bit division. 633 */ 634 BUILD_BUG_ON(MAX_RW_COUNT > UINT_MAX); 635 if (end_fsb - offset_fsb > UINT_MAX) { 636 error = -EFSCORRUPTED; 637 xfs_force_shutdown(ip->i_mount, SHUTDOWN_CORRUPT_INCORE); 638 ASSERT(0); 639 goto out; 640 } 641 resblks = XFS_NEXTENTADD_SPACE_RES(ip->i_mount, 642 (unsigned int)(end_fsb - offset_fsb), 643 XFS_DATA_FORK); 644 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 645 resblks, 0, XFS_TRANS_RESERVE | XFS_TRANS_NOFS, &tp); 646 if (error) 647 goto out; 648 649 xfs_ilock(ip, XFS_ILOCK_EXCL); 650 xfs_trans_ijoin(tp, ip, 0); 651 652 /* 653 * In case of racing, overlapping AIO writes no COW extents might be 654 * left by the time I/O completes for the loser of the race. In that 655 * case we are done. 656 */ 657 if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) 658 goto out_cancel; 659 660 /* Walk backwards until we're out of the I/O range... */ 661 while (got.br_startoff + got.br_blockcount > offset_fsb) { 662 del = got; 663 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 664 665 /* Extent delete may have bumped ext forward */ 666 if (!del.br_blockcount) 667 goto prev_extent; 668 669 ASSERT(!isnullstartblock(got.br_startblock)); 670 671 /* 672 * Don't remap unwritten extents; these are 673 * speculatively preallocated CoW extents that have been 674 * allocated but have not yet been involved in a write. 675 */ 676 if (got.br_state == XFS_EXT_UNWRITTEN) 677 goto prev_extent; 678 679 /* Unmap the old blocks in the data fork. */ 680 ASSERT(tp->t_firstblock == NULLFSBLOCK); 681 rlen = del.br_blockcount; 682 error = __xfs_bunmapi(tp, ip, del.br_startoff, &rlen, 0, 1); 683 if (error) 684 goto out_cancel; 685 686 /* Trim the extent to whatever got unmapped. */ 687 if (rlen) { 688 xfs_trim_extent(&del, del.br_startoff + rlen, 689 del.br_blockcount - rlen); 690 } 691 trace_xfs_reflink_cow_remap(ip, &del); 692 693 /* Free the CoW orphan record. */ 694 error = xfs_refcount_free_cow_extent(tp, del.br_startblock, 695 del.br_blockcount); 696 if (error) 697 goto out_cancel; 698 699 /* Map the new blocks into the data fork. */ 700 error = xfs_bmap_map_extent(tp, ip, &del); 701 if (error) 702 goto out_cancel; 703 704 /* Charge this new data fork mapping to the on-disk quota. */ 705 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_DELBCOUNT, 706 (long)del.br_blockcount); 707 708 /* Remove the mapping from the CoW fork. */ 709 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 710 711 error = xfs_defer_finish(&tp); 712 if (error) 713 goto out_cancel; 714 if (!xfs_iext_get_extent(ifp, &icur, &got)) 715 break; 716 continue; 717 prev_extent: 718 if (!xfs_iext_prev_extent(ifp, &icur, &got)) 719 break; 720 } 721 722 error = xfs_trans_commit(tp); 723 xfs_iunlock(ip, XFS_ILOCK_EXCL); 724 if (error) 725 goto out; 726 return 0; 727 728 out_cancel: 729 xfs_trans_cancel(tp); 730 xfs_iunlock(ip, XFS_ILOCK_EXCL); 731 out: 732 trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_); 733 return error; 734 } 735 736 /* 737 * Free leftover CoW reservations that didn't get cleaned out. 738 */ 739 int 740 xfs_reflink_recover_cow( 741 struct xfs_mount *mp) 742 { 743 xfs_agnumber_t agno; 744 int error = 0; 745 746 if (!xfs_sb_version_hasreflink(&mp->m_sb)) 747 return 0; 748 749 for (agno = 0; agno < mp->m_sb.sb_agcount; agno++) { 750 error = xfs_refcount_recover_cow_leftovers(mp, agno); 751 if (error) 752 break; 753 } 754 755 return error; 756 } 757 758 /* 759 * Reflinking (Block) Ranges of Two Files Together 760 * 761 * First, ensure that the reflink flag is set on both inodes. The flag is an 762 * optimization to avoid unnecessary refcount btree lookups in the write path. 763 * 764 * Now we can iteratively remap the range of extents (and holes) in src to the 765 * corresponding ranges in dest. Let drange and srange denote the ranges of 766 * logical blocks in dest and src touched by the reflink operation. 767 * 768 * While the length of drange is greater than zero, 769 * - Read src's bmbt at the start of srange ("imap") 770 * - If imap doesn't exist, make imap appear to start at the end of srange 771 * with zero length. 772 * - If imap starts before srange, advance imap to start at srange. 773 * - If imap goes beyond srange, truncate imap to end at the end of srange. 774 * - Punch (imap start - srange start + imap len) blocks from dest at 775 * offset (drange start). 776 * - If imap points to a real range of pblks, 777 * > Increase the refcount of the imap's pblks 778 * > Map imap's pblks into dest at the offset 779 * (drange start + imap start - srange start) 780 * - Advance drange and srange by (imap start - srange start + imap len) 781 * 782 * Finally, if the reflink made dest longer, update both the in-core and 783 * on-disk file sizes. 784 * 785 * ASCII Art Demonstration: 786 * 787 * Let's say we want to reflink this source file: 788 * 789 * ----SSSSSSS-SSSSS----SSSSSS (src file) 790 * <--------------------> 791 * 792 * into this destination file: 793 * 794 * --DDDDDDDDDDDDDDDDDDD--DDD (dest file) 795 * <--------------------> 796 * '-' means a hole, and 'S' and 'D' are written blocks in the src and dest. 797 * Observe that the range has different logical offsets in either file. 798 * 799 * Consider that the first extent in the source file doesn't line up with our 800 * reflink range. Unmapping and remapping are separate operations, so we can 801 * unmap more blocks from the destination file than we remap. 802 * 803 * ----SSSSSSS-SSSSS----SSSSSS 804 * <-------> 805 * --DDDDD---------DDDDD--DDD 806 * <-------> 807 * 808 * Now remap the source extent into the destination file: 809 * 810 * ----SSSSSSS-SSSSS----SSSSSS 811 * <-------> 812 * --DDDDD--SSSSSSSDDDDD--DDD 813 * <-------> 814 * 815 * Do likewise with the second hole and extent in our range. Holes in the 816 * unmap range don't affect our operation. 817 * 818 * ----SSSSSSS-SSSSS----SSSSSS 819 * <----> 820 * --DDDDD--SSSSSSS-SSSSS-DDD 821 * <----> 822 * 823 * Finally, unmap and remap part of the third extent. This will increase the 824 * size of the destination file. 825 * 826 * ----SSSSSSS-SSSSS----SSSSSS 827 * <-----> 828 * --DDDDD--SSSSSSS-SSSSS----SSS 829 * <-----> 830 * 831 * Once we update the destination file's i_size, we're done. 832 */ 833 834 /* 835 * Ensure the reflink bit is set in both inodes. 836 */ 837 STATIC int 838 xfs_reflink_set_inode_flag( 839 struct xfs_inode *src, 840 struct xfs_inode *dest) 841 { 842 struct xfs_mount *mp = src->i_mount; 843 int error; 844 struct xfs_trans *tp; 845 846 if (xfs_is_reflink_inode(src) && xfs_is_reflink_inode(dest)) 847 return 0; 848 849 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 850 if (error) 851 goto out_error; 852 853 /* Lock both files against IO */ 854 if (src->i_ino == dest->i_ino) 855 xfs_ilock(src, XFS_ILOCK_EXCL); 856 else 857 xfs_lock_two_inodes(src, XFS_ILOCK_EXCL, dest, XFS_ILOCK_EXCL); 858 859 if (!xfs_is_reflink_inode(src)) { 860 trace_xfs_reflink_set_inode_flag(src); 861 xfs_trans_ijoin(tp, src, XFS_ILOCK_EXCL); 862 src->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 863 xfs_trans_log_inode(tp, src, XFS_ILOG_CORE); 864 xfs_ifork_init_cow(src); 865 } else 866 xfs_iunlock(src, XFS_ILOCK_EXCL); 867 868 if (src->i_ino == dest->i_ino) 869 goto commit_flags; 870 871 if (!xfs_is_reflink_inode(dest)) { 872 trace_xfs_reflink_set_inode_flag(dest); 873 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 874 dest->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 875 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 876 xfs_ifork_init_cow(dest); 877 } else 878 xfs_iunlock(dest, XFS_ILOCK_EXCL); 879 880 commit_flags: 881 error = xfs_trans_commit(tp); 882 if (error) 883 goto out_error; 884 return error; 885 886 out_error: 887 trace_xfs_reflink_set_inode_flag_error(dest, error, _RET_IP_); 888 return error; 889 } 890 891 /* 892 * Update destination inode size & cowextsize hint, if necessary. 893 */ 894 STATIC int 895 xfs_reflink_update_dest( 896 struct xfs_inode *dest, 897 xfs_off_t newlen, 898 xfs_extlen_t cowextsize, 899 bool is_dedupe) 900 { 901 struct xfs_mount *mp = dest->i_mount; 902 struct xfs_trans *tp; 903 int error; 904 905 if (is_dedupe && newlen <= i_size_read(VFS_I(dest)) && cowextsize == 0) 906 return 0; 907 908 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 909 if (error) 910 goto out_error; 911 912 xfs_ilock(dest, XFS_ILOCK_EXCL); 913 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 914 915 if (newlen > i_size_read(VFS_I(dest))) { 916 trace_xfs_reflink_update_inode_size(dest, newlen); 917 i_size_write(VFS_I(dest), newlen); 918 dest->i_d.di_size = newlen; 919 } 920 921 if (cowextsize) { 922 dest->i_d.di_cowextsize = cowextsize; 923 dest->i_d.di_flags2 |= XFS_DIFLAG2_COWEXTSIZE; 924 } 925 926 if (!is_dedupe) { 927 xfs_trans_ichgtime(tp, dest, 928 XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); 929 } 930 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 931 932 error = xfs_trans_commit(tp); 933 if (error) 934 goto out_error; 935 return error; 936 937 out_error: 938 trace_xfs_reflink_update_inode_size_error(dest, error, _RET_IP_); 939 return error; 940 } 941 942 /* 943 * Do we have enough reserve in this AG to handle a reflink? The refcount 944 * btree already reserved all the space it needs, but the rmap btree can grow 945 * infinitely, so we won't allow more reflinks when the AG is down to the 946 * btree reserves. 947 */ 948 static int 949 xfs_reflink_ag_has_free_space( 950 struct xfs_mount *mp, 951 xfs_agnumber_t agno) 952 { 953 struct xfs_perag *pag; 954 int error = 0; 955 956 if (!xfs_sb_version_hasrmapbt(&mp->m_sb)) 957 return 0; 958 959 pag = xfs_perag_get(mp, agno); 960 if (xfs_ag_resv_critical(pag, XFS_AG_RESV_RMAPBT) || 961 xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA)) 962 error = -ENOSPC; 963 xfs_perag_put(pag); 964 return error; 965 } 966 967 /* 968 * Unmap a range of blocks from a file, then map other blocks into the hole. 969 * The range to unmap is (destoff : destoff + srcioff + irec->br_blockcount). 970 * The extent irec is mapped into dest at irec->br_startoff. 971 */ 972 STATIC int 973 xfs_reflink_remap_extent( 974 struct xfs_inode *ip, 975 struct xfs_bmbt_irec *irec, 976 xfs_fileoff_t destoff, 977 xfs_off_t new_isize) 978 { 979 struct xfs_mount *mp = ip->i_mount; 980 bool real_extent = xfs_bmap_is_real_extent(irec); 981 struct xfs_trans *tp; 982 unsigned int resblks; 983 struct xfs_bmbt_irec uirec; 984 xfs_filblks_t rlen; 985 xfs_filblks_t unmap_len; 986 xfs_off_t newlen; 987 int error; 988 989 unmap_len = irec->br_startoff + irec->br_blockcount - destoff; 990 trace_xfs_reflink_punch_range(ip, destoff, unmap_len); 991 992 /* No reflinking if we're low on space */ 993 if (real_extent) { 994 error = xfs_reflink_ag_has_free_space(mp, 995 XFS_FSB_TO_AGNO(mp, irec->br_startblock)); 996 if (error) 997 goto out; 998 } 999 1000 /* Start a rolling transaction to switch the mappings */ 1001 resblks = XFS_EXTENTADD_SPACE_RES(ip->i_mount, XFS_DATA_FORK); 1002 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); 1003 if (error) 1004 goto out; 1005 1006 xfs_ilock(ip, XFS_ILOCK_EXCL); 1007 xfs_trans_ijoin(tp, ip, 0); 1008 1009 /* If we're not just clearing space, then do we have enough quota? */ 1010 if (real_extent) { 1011 error = xfs_trans_reserve_quota_nblks(tp, ip, 1012 irec->br_blockcount, 0, XFS_QMOPT_RES_REGBLKS); 1013 if (error) 1014 goto out_cancel; 1015 } 1016 1017 trace_xfs_reflink_remap(ip, irec->br_startoff, 1018 irec->br_blockcount, irec->br_startblock); 1019 1020 /* Unmap the old blocks in the data fork. */ 1021 rlen = unmap_len; 1022 while (rlen) { 1023 ASSERT(tp->t_firstblock == NULLFSBLOCK); 1024 error = __xfs_bunmapi(tp, ip, destoff, &rlen, 0, 1); 1025 if (error) 1026 goto out_cancel; 1027 1028 /* 1029 * Trim the extent to whatever got unmapped. 1030 * Remember, bunmapi works backwards. 1031 */ 1032 uirec.br_startblock = irec->br_startblock + rlen; 1033 uirec.br_startoff = irec->br_startoff + rlen; 1034 uirec.br_blockcount = unmap_len - rlen; 1035 unmap_len = rlen; 1036 1037 /* If this isn't a real mapping, we're done. */ 1038 if (!real_extent || uirec.br_blockcount == 0) 1039 goto next_extent; 1040 1041 trace_xfs_reflink_remap(ip, uirec.br_startoff, 1042 uirec.br_blockcount, uirec.br_startblock); 1043 1044 /* Update the refcount tree */ 1045 error = xfs_refcount_increase_extent(tp, &uirec); 1046 if (error) 1047 goto out_cancel; 1048 1049 /* Map the new blocks into the data fork. */ 1050 error = xfs_bmap_map_extent(tp, ip, &uirec); 1051 if (error) 1052 goto out_cancel; 1053 1054 /* Update quota accounting. */ 1055 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1056 uirec.br_blockcount); 1057 1058 /* Update dest isize if needed. */ 1059 newlen = XFS_FSB_TO_B(mp, 1060 uirec.br_startoff + uirec.br_blockcount); 1061 newlen = min_t(xfs_off_t, newlen, new_isize); 1062 if (newlen > i_size_read(VFS_I(ip))) { 1063 trace_xfs_reflink_update_inode_size(ip, newlen); 1064 i_size_write(VFS_I(ip), newlen); 1065 ip->i_d.di_size = newlen; 1066 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1067 } 1068 1069 next_extent: 1070 /* Process all the deferred stuff. */ 1071 error = xfs_defer_finish(&tp); 1072 if (error) 1073 goto out_cancel; 1074 } 1075 1076 error = xfs_trans_commit(tp); 1077 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1078 if (error) 1079 goto out; 1080 return 0; 1081 1082 out_cancel: 1083 xfs_trans_cancel(tp); 1084 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1085 out: 1086 trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_); 1087 return error; 1088 } 1089 1090 /* 1091 * Iteratively remap one file's extents (and holes) to another's. 1092 */ 1093 STATIC int 1094 xfs_reflink_remap_blocks( 1095 struct xfs_inode *src, 1096 xfs_fileoff_t srcoff, 1097 struct xfs_inode *dest, 1098 xfs_fileoff_t destoff, 1099 xfs_filblks_t len, 1100 xfs_off_t new_isize) 1101 { 1102 struct xfs_bmbt_irec imap; 1103 int nimaps; 1104 int error = 0; 1105 xfs_filblks_t range_len; 1106 1107 /* drange = (destoff, destoff + len); srange = (srcoff, srcoff + len) */ 1108 while (len) { 1109 uint lock_mode; 1110 1111 trace_xfs_reflink_remap_blocks_loop(src, srcoff, len, 1112 dest, destoff); 1113 1114 /* Read extent from the source file */ 1115 nimaps = 1; 1116 lock_mode = xfs_ilock_data_map_shared(src); 1117 error = xfs_bmapi_read(src, srcoff, len, &imap, &nimaps, 0); 1118 xfs_iunlock(src, lock_mode); 1119 if (error) 1120 goto err; 1121 ASSERT(nimaps == 1); 1122 1123 trace_xfs_reflink_remap_imap(src, srcoff, len, XFS_IO_OVERWRITE, 1124 &imap); 1125 1126 /* Translate imap into the destination file. */ 1127 range_len = imap.br_startoff + imap.br_blockcount - srcoff; 1128 imap.br_startoff += destoff - srcoff; 1129 1130 /* Clear dest from destoff to the end of imap and map it in. */ 1131 error = xfs_reflink_remap_extent(dest, &imap, destoff, 1132 new_isize); 1133 if (error) 1134 goto err; 1135 1136 if (fatal_signal_pending(current)) { 1137 error = -EINTR; 1138 goto err; 1139 } 1140 1141 /* Advance drange/srange */ 1142 srcoff += range_len; 1143 destoff += range_len; 1144 len -= range_len; 1145 } 1146 1147 return 0; 1148 1149 err: 1150 trace_xfs_reflink_remap_blocks_error(dest, error, _RET_IP_); 1151 return error; 1152 } 1153 1154 /* 1155 * Grab the exclusive iolock for a data copy from src to dest, making 1156 * sure to abide vfs locking order (lowest pointer value goes first) and 1157 * breaking the pnfs layout leases on dest before proceeding. The loop 1158 * is needed because we cannot call the blocking break_layout() with the 1159 * src iolock held, and therefore have to back out both locks. 1160 */ 1161 static int 1162 xfs_iolock_two_inodes_and_break_layout( 1163 struct inode *src, 1164 struct inode *dest) 1165 { 1166 int error; 1167 1168 retry: 1169 if (src < dest) { 1170 inode_lock_shared(src); 1171 inode_lock_nested(dest, I_MUTEX_NONDIR2); 1172 } else { 1173 /* src >= dest */ 1174 inode_lock(dest); 1175 } 1176 1177 error = break_layout(dest, false); 1178 if (error == -EWOULDBLOCK) { 1179 inode_unlock(dest); 1180 if (src < dest) 1181 inode_unlock_shared(src); 1182 error = break_layout(dest, true); 1183 if (error) 1184 return error; 1185 goto retry; 1186 } 1187 if (error) { 1188 inode_unlock(dest); 1189 if (src < dest) 1190 inode_unlock_shared(src); 1191 return error; 1192 } 1193 if (src > dest) 1194 inode_lock_shared_nested(src, I_MUTEX_NONDIR2); 1195 return 0; 1196 } 1197 1198 /* 1199 * Link a range of blocks from one file to another. 1200 */ 1201 int 1202 xfs_reflink_remap_range( 1203 struct file *file_in, 1204 loff_t pos_in, 1205 struct file *file_out, 1206 loff_t pos_out, 1207 u64 len, 1208 bool is_dedupe) 1209 { 1210 struct inode *inode_in = file_inode(file_in); 1211 struct xfs_inode *src = XFS_I(inode_in); 1212 struct inode *inode_out = file_inode(file_out); 1213 struct xfs_inode *dest = XFS_I(inode_out); 1214 struct xfs_mount *mp = src->i_mount; 1215 bool same_inode = (inode_in == inode_out); 1216 xfs_fileoff_t sfsbno, dfsbno; 1217 xfs_filblks_t fsblen; 1218 xfs_extlen_t cowextsize; 1219 ssize_t ret; 1220 1221 if (!xfs_sb_version_hasreflink(&mp->m_sb)) 1222 return -EOPNOTSUPP; 1223 1224 if (XFS_FORCED_SHUTDOWN(mp)) 1225 return -EIO; 1226 1227 /* Lock both files against IO */ 1228 ret = xfs_iolock_two_inodes_and_break_layout(inode_in, inode_out); 1229 if (ret) 1230 return ret; 1231 if (same_inode) 1232 xfs_ilock(src, XFS_MMAPLOCK_EXCL); 1233 else 1234 xfs_lock_two_inodes(src, XFS_MMAPLOCK_SHARED, dest, 1235 XFS_MMAPLOCK_EXCL); 1236 1237 /* Check file eligibility and prepare for block sharing. */ 1238 ret = -EINVAL; 1239 /* Don't reflink realtime inodes */ 1240 if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest)) 1241 goto out_unlock; 1242 1243 /* Don't share DAX file data for now. */ 1244 if (IS_DAX(inode_in) || IS_DAX(inode_out)) 1245 goto out_unlock; 1246 1247 ret = vfs_clone_file_prep_inodes(inode_in, pos_in, inode_out, pos_out, 1248 &len, is_dedupe); 1249 if (ret <= 0) 1250 goto out_unlock; 1251 1252 /* Attach dquots to dest inode before changing block map */ 1253 ret = xfs_qm_dqattach(dest); 1254 if (ret) 1255 goto out_unlock; 1256 1257 trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out); 1258 1259 /* 1260 * Clear out post-eof preallocations because we don't have page cache 1261 * backing the delayed allocations and they'll never get freed on 1262 * their own. 1263 */ 1264 if (xfs_can_free_eofblocks(dest, true)) { 1265 ret = xfs_free_eofblocks(dest); 1266 if (ret) 1267 goto out_unlock; 1268 } 1269 1270 /* Set flags and remap blocks. */ 1271 ret = xfs_reflink_set_inode_flag(src, dest); 1272 if (ret) 1273 goto out_unlock; 1274 1275 dfsbno = XFS_B_TO_FSBT(mp, pos_out); 1276 sfsbno = XFS_B_TO_FSBT(mp, pos_in); 1277 fsblen = XFS_B_TO_FSB(mp, len); 1278 ret = xfs_reflink_remap_blocks(src, sfsbno, dest, dfsbno, fsblen, 1279 pos_out + len); 1280 if (ret) 1281 goto out_unlock; 1282 1283 /* Zap any page cache for the destination file's range. */ 1284 truncate_inode_pages_range(&inode_out->i_data, pos_out, 1285 PAGE_ALIGN(pos_out + len) - 1); 1286 1287 /* 1288 * Carry the cowextsize hint from src to dest if we're sharing the 1289 * entire source file to the entire destination file, the source file 1290 * has a cowextsize hint, and the destination file does not. 1291 */ 1292 cowextsize = 0; 1293 if (pos_in == 0 && len == i_size_read(inode_in) && 1294 (src->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE) && 1295 pos_out == 0 && len >= i_size_read(inode_out) && 1296 !(dest->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE)) 1297 cowextsize = src->i_d.di_cowextsize; 1298 1299 ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize, 1300 is_dedupe); 1301 1302 out_unlock: 1303 xfs_iunlock(dest, XFS_MMAPLOCK_EXCL); 1304 if (!same_inode) 1305 xfs_iunlock(src, XFS_MMAPLOCK_SHARED); 1306 inode_unlock(inode_out); 1307 if (!same_inode) 1308 inode_unlock_shared(inode_in); 1309 if (ret) 1310 trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_); 1311 return ret; 1312 } 1313 1314 /* 1315 * The user wants to preemptively CoW all shared blocks in this file, 1316 * which enables us to turn off the reflink flag. Iterate all 1317 * extents which are not prealloc/delalloc to see which ranges are 1318 * mentioned in the refcount tree, then read those blocks into the 1319 * pagecache, dirty them, fsync them back out, and then we can update 1320 * the inode flag. What happens if we run out of memory? :) 1321 */ 1322 STATIC int 1323 xfs_reflink_dirty_extents( 1324 struct xfs_inode *ip, 1325 xfs_fileoff_t fbno, 1326 xfs_filblks_t end, 1327 xfs_off_t isize) 1328 { 1329 struct xfs_mount *mp = ip->i_mount; 1330 xfs_agnumber_t agno; 1331 xfs_agblock_t agbno; 1332 xfs_extlen_t aglen; 1333 xfs_agblock_t rbno; 1334 xfs_extlen_t rlen; 1335 xfs_off_t fpos; 1336 xfs_off_t flen; 1337 struct xfs_bmbt_irec map[2]; 1338 int nmaps; 1339 int error = 0; 1340 1341 while (end - fbno > 0) { 1342 nmaps = 1; 1343 /* 1344 * Look for extents in the file. Skip holes, delalloc, or 1345 * unwritten extents; they can't be reflinked. 1346 */ 1347 error = xfs_bmapi_read(ip, fbno, end - fbno, map, &nmaps, 0); 1348 if (error) 1349 goto out; 1350 if (nmaps == 0) 1351 break; 1352 if (!xfs_bmap_is_real_extent(&map[0])) 1353 goto next; 1354 1355 map[1] = map[0]; 1356 while (map[1].br_blockcount) { 1357 agno = XFS_FSB_TO_AGNO(mp, map[1].br_startblock); 1358 agbno = XFS_FSB_TO_AGBNO(mp, map[1].br_startblock); 1359 aglen = map[1].br_blockcount; 1360 1361 error = xfs_reflink_find_shared(mp, NULL, agno, agbno, 1362 aglen, &rbno, &rlen, true); 1363 if (error) 1364 goto out; 1365 if (rbno == NULLAGBLOCK) 1366 break; 1367 1368 /* Dirty the pages */ 1369 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1370 fpos = XFS_FSB_TO_B(mp, map[1].br_startoff + 1371 (rbno - agbno)); 1372 flen = XFS_FSB_TO_B(mp, rlen); 1373 if (fpos + flen > isize) 1374 flen = isize - fpos; 1375 error = iomap_file_dirty(VFS_I(ip), fpos, flen, 1376 &xfs_iomap_ops); 1377 xfs_ilock(ip, XFS_ILOCK_EXCL); 1378 if (error) 1379 goto out; 1380 1381 map[1].br_blockcount -= (rbno - agbno + rlen); 1382 map[1].br_startoff += (rbno - agbno + rlen); 1383 map[1].br_startblock += (rbno - agbno + rlen); 1384 } 1385 1386 next: 1387 fbno = map[0].br_startoff + map[0].br_blockcount; 1388 } 1389 out: 1390 return error; 1391 } 1392 1393 /* Does this inode need the reflink flag? */ 1394 int 1395 xfs_reflink_inode_has_shared_extents( 1396 struct xfs_trans *tp, 1397 struct xfs_inode *ip, 1398 bool *has_shared) 1399 { 1400 struct xfs_bmbt_irec got; 1401 struct xfs_mount *mp = ip->i_mount; 1402 struct xfs_ifork *ifp; 1403 xfs_agnumber_t agno; 1404 xfs_agblock_t agbno; 1405 xfs_extlen_t aglen; 1406 xfs_agblock_t rbno; 1407 xfs_extlen_t rlen; 1408 struct xfs_iext_cursor icur; 1409 bool found; 1410 int error; 1411 1412 ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); 1413 if (!(ifp->if_flags & XFS_IFEXTENTS)) { 1414 error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); 1415 if (error) 1416 return error; 1417 } 1418 1419 *has_shared = false; 1420 found = xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got); 1421 while (found) { 1422 if (isnullstartblock(got.br_startblock) || 1423 got.br_state != XFS_EXT_NORM) 1424 goto next; 1425 agno = XFS_FSB_TO_AGNO(mp, got.br_startblock); 1426 agbno = XFS_FSB_TO_AGBNO(mp, got.br_startblock); 1427 aglen = got.br_blockcount; 1428 1429 error = xfs_reflink_find_shared(mp, tp, agno, agbno, aglen, 1430 &rbno, &rlen, false); 1431 if (error) 1432 return error; 1433 /* Is there still a shared block here? */ 1434 if (rbno != NULLAGBLOCK) { 1435 *has_shared = true; 1436 return 0; 1437 } 1438 next: 1439 found = xfs_iext_next_extent(ifp, &icur, &got); 1440 } 1441 1442 return 0; 1443 } 1444 1445 /* 1446 * Clear the inode reflink flag if there are no shared extents. 1447 * 1448 * The caller is responsible for joining the inode to the transaction passed in. 1449 * The inode will be joined to the transaction that is returned to the caller. 1450 */ 1451 int 1452 xfs_reflink_clear_inode_flag( 1453 struct xfs_inode *ip, 1454 struct xfs_trans **tpp) 1455 { 1456 bool needs_flag; 1457 int error = 0; 1458 1459 ASSERT(xfs_is_reflink_inode(ip)); 1460 1461 error = xfs_reflink_inode_has_shared_extents(*tpp, ip, &needs_flag); 1462 if (error || needs_flag) 1463 return error; 1464 1465 /* 1466 * We didn't find any shared blocks so turn off the reflink flag. 1467 * First, get rid of any leftover CoW mappings. 1468 */ 1469 error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, NULLFILEOFF, true); 1470 if (error) 1471 return error; 1472 1473 /* Clear the inode flag. */ 1474 trace_xfs_reflink_unset_inode_flag(ip); 1475 ip->i_d.di_flags2 &= ~XFS_DIFLAG2_REFLINK; 1476 xfs_inode_clear_cowblocks_tag(ip); 1477 xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE); 1478 1479 return error; 1480 } 1481 1482 /* 1483 * Clear the inode reflink flag if there are no shared extents and the size 1484 * hasn't changed. 1485 */ 1486 STATIC int 1487 xfs_reflink_try_clear_inode_flag( 1488 struct xfs_inode *ip) 1489 { 1490 struct xfs_mount *mp = ip->i_mount; 1491 struct xfs_trans *tp; 1492 int error = 0; 1493 1494 /* Start a rolling transaction to remove the mappings */ 1495 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); 1496 if (error) 1497 return error; 1498 1499 xfs_ilock(ip, XFS_ILOCK_EXCL); 1500 xfs_trans_ijoin(tp, ip, 0); 1501 1502 error = xfs_reflink_clear_inode_flag(ip, &tp); 1503 if (error) 1504 goto cancel; 1505 1506 error = xfs_trans_commit(tp); 1507 if (error) 1508 goto out; 1509 1510 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1511 return 0; 1512 cancel: 1513 xfs_trans_cancel(tp); 1514 out: 1515 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1516 return error; 1517 } 1518 1519 /* 1520 * Pre-COW all shared blocks within a given byte range of a file and turn off 1521 * the reflink flag if we unshare all of the file's blocks. 1522 */ 1523 int 1524 xfs_reflink_unshare( 1525 struct xfs_inode *ip, 1526 xfs_off_t offset, 1527 xfs_off_t len) 1528 { 1529 struct xfs_mount *mp = ip->i_mount; 1530 xfs_fileoff_t fbno; 1531 xfs_filblks_t end; 1532 xfs_off_t isize; 1533 int error; 1534 1535 if (!xfs_is_reflink_inode(ip)) 1536 return 0; 1537 1538 trace_xfs_reflink_unshare(ip, offset, len); 1539 1540 inode_dio_wait(VFS_I(ip)); 1541 1542 /* Try to CoW the selected ranges */ 1543 xfs_ilock(ip, XFS_ILOCK_EXCL); 1544 fbno = XFS_B_TO_FSBT(mp, offset); 1545 isize = i_size_read(VFS_I(ip)); 1546 end = XFS_B_TO_FSB(mp, offset + len); 1547 error = xfs_reflink_dirty_extents(ip, fbno, end, isize); 1548 if (error) 1549 goto out_unlock; 1550 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1551 1552 /* Wait for the IO to finish */ 1553 error = filemap_write_and_wait(VFS_I(ip)->i_mapping); 1554 if (error) 1555 goto out; 1556 1557 /* Turn off the reflink flag if possible. */ 1558 error = xfs_reflink_try_clear_inode_flag(ip); 1559 if (error) 1560 goto out; 1561 1562 return 0; 1563 1564 out_unlock: 1565 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1566 out: 1567 trace_xfs_reflink_unshare_error(ip, error, _RET_IP_); 1568 return error; 1569 } 1570