1 /* 2 * Copyright (C) 2016 Oracle. All Rights Reserved. 3 * 4 * Author: Darrick J. Wong <darrick.wong@oracle.com> 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 2 9 * of the License, or (at your option) any later version. 10 * 11 * This program is distributed in the hope that it would be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. 19 */ 20 #include "xfs.h" 21 #include "xfs_fs.h" 22 #include "xfs_shared.h" 23 #include "xfs_format.h" 24 #include "xfs_log_format.h" 25 #include "xfs_trans_resv.h" 26 #include "xfs_mount.h" 27 #include "xfs_defer.h" 28 #include "xfs_da_format.h" 29 #include "xfs_da_btree.h" 30 #include "xfs_inode.h" 31 #include "xfs_trans.h" 32 #include "xfs_inode_item.h" 33 #include "xfs_bmap.h" 34 #include "xfs_bmap_util.h" 35 #include "xfs_error.h" 36 #include "xfs_dir2.h" 37 #include "xfs_dir2_priv.h" 38 #include "xfs_ioctl.h" 39 #include "xfs_trace.h" 40 #include "xfs_log.h" 41 #include "xfs_icache.h" 42 #include "xfs_pnfs.h" 43 #include "xfs_btree.h" 44 #include "xfs_refcount_btree.h" 45 #include "xfs_refcount.h" 46 #include "xfs_bmap_btree.h" 47 #include "xfs_trans_space.h" 48 #include "xfs_bit.h" 49 #include "xfs_alloc.h" 50 #include "xfs_quota_defs.h" 51 #include "xfs_quota.h" 52 #include "xfs_btree.h" 53 #include "xfs_bmap_btree.h" 54 #include "xfs_reflink.h" 55 #include "xfs_iomap.h" 56 #include "xfs_rmap_btree.h" 57 #include "xfs_sb.h" 58 #include "xfs_ag_resv.h" 59 60 /* 61 * Copy on Write of Shared Blocks 62 * 63 * XFS must preserve "the usual" file semantics even when two files share 64 * the same physical blocks. This means that a write to one file must not 65 * alter the blocks in a different file; the way that we'll do that is 66 * through the use of a copy-on-write mechanism. At a high level, that 67 * means that when we want to write to a shared block, we allocate a new 68 * block, write the data to the new block, and if that succeeds we map the 69 * new block into the file. 70 * 71 * XFS provides a "delayed allocation" mechanism that defers the allocation 72 * of disk blocks to dirty-but-not-yet-mapped file blocks as long as 73 * possible. This reduces fragmentation by enabling the filesystem to ask 74 * for bigger chunks less often, which is exactly what we want for CoW. 75 * 76 * The delalloc mechanism begins when the kernel wants to make a block 77 * writable (write_begin or page_mkwrite). If the offset is not mapped, we 78 * create a delalloc mapping, which is a regular in-core extent, but without 79 * a real startblock. (For delalloc mappings, the startblock encodes both 80 * a flag that this is a delalloc mapping, and a worst-case estimate of how 81 * many blocks might be required to put the mapping into the BMBT.) delalloc 82 * mappings are a reservation against the free space in the filesystem; 83 * adjacent mappings can also be combined into fewer larger mappings. 84 * 85 * As an optimization, the CoW extent size hint (cowextsz) creates 86 * outsized aligned delalloc reservations in the hope of landing out of 87 * order nearby CoW writes in a single extent on disk, thereby reducing 88 * fragmentation and improving future performance. 89 * 90 * D: --RRRRRRSSSRRRRRRRR--- (data fork) 91 * C: ------DDDDDDD--------- (CoW fork) 92 * 93 * When dirty pages are being written out (typically in writepage), the 94 * delalloc reservations are converted into unwritten mappings by 95 * allocating blocks and replacing the delalloc mapping with real ones. 96 * A delalloc mapping can be replaced by several unwritten ones if the 97 * free space is fragmented. 98 * 99 * D: --RRRRRRSSSRRRRRRRR--- 100 * C: ------UUUUUUU--------- 101 * 102 * We want to adapt the delalloc mechanism for copy-on-write, since the 103 * write paths are similar. The first two steps (creating the reservation 104 * and allocating the blocks) are exactly the same as delalloc except that 105 * the mappings must be stored in a separate CoW fork because we do not want 106 * to disturb the mapping in the data fork until we're sure that the write 107 * succeeded. IO completion in this case is the process of removing the old 108 * mapping from the data fork and moving the new mapping from the CoW fork to 109 * the data fork. This will be discussed shortly. 110 * 111 * For now, unaligned directio writes will be bounced back to the page cache. 112 * Block-aligned directio writes will use the same mechanism as buffered 113 * writes. 114 * 115 * Just prior to submitting the actual disk write requests, we convert 116 * the extents representing the range of the file actually being written 117 * (as opposed to extra pieces created for the cowextsize hint) to real 118 * extents. This will become important in the next step: 119 * 120 * D: --RRRRRRSSSRRRRRRRR--- 121 * C: ------UUrrUUU--------- 122 * 123 * CoW remapping must be done after the data block write completes, 124 * because we don't want to destroy the old data fork map until we're sure 125 * the new block has been written. Since the new mappings are kept in a 126 * separate fork, we can simply iterate these mappings to find the ones 127 * that cover the file blocks that we just CoW'd. For each extent, simply 128 * unmap the corresponding range in the data fork, map the new range into 129 * the data fork, and remove the extent from the CoW fork. Because of 130 * the presence of the cowextsize hint, however, we must be careful 131 * only to remap the blocks that we've actually written out -- we must 132 * never remap delalloc reservations nor CoW staging blocks that have 133 * yet to be written. This corresponds exactly to the real extents in 134 * the CoW fork: 135 * 136 * D: --RRRRRRrrSRRRRRRRR--- 137 * C: ------UU--UUU--------- 138 * 139 * Since the remapping operation can be applied to an arbitrary file 140 * range, we record the need for the remap step as a flag in the ioend 141 * instead of declaring a new IO type. This is required for direct io 142 * because we only have ioend for the whole dio, and we have to be able to 143 * remember the presence of unwritten blocks and CoW blocks with a single 144 * ioend structure. Better yet, the more ground we can cover with one 145 * ioend, the better. 146 */ 147 148 /* 149 * Given an AG extent, find the lowest-numbered run of shared blocks 150 * within that range and return the range in fbno/flen. If 151 * find_end_of_shared is true, return the longest contiguous extent of 152 * shared blocks. If there are no shared extents, fbno and flen will 153 * be set to NULLAGBLOCK and 0, respectively. 154 */ 155 int 156 xfs_reflink_find_shared( 157 struct xfs_mount *mp, 158 struct xfs_trans *tp, 159 xfs_agnumber_t agno, 160 xfs_agblock_t agbno, 161 xfs_extlen_t aglen, 162 xfs_agblock_t *fbno, 163 xfs_extlen_t *flen, 164 bool find_end_of_shared) 165 { 166 struct xfs_buf *agbp; 167 struct xfs_btree_cur *cur; 168 int error; 169 170 error = xfs_alloc_read_agf(mp, tp, agno, 0, &agbp); 171 if (error) 172 return error; 173 if (!agbp) 174 return -ENOMEM; 175 176 cur = xfs_refcountbt_init_cursor(mp, tp, agbp, agno, NULL); 177 178 error = xfs_refcount_find_shared(cur, agbno, aglen, fbno, flen, 179 find_end_of_shared); 180 181 xfs_btree_del_cursor(cur, error ? XFS_BTREE_ERROR : XFS_BTREE_NOERROR); 182 183 xfs_trans_brelse(tp, agbp); 184 return error; 185 } 186 187 /* 188 * Trim the mapping to the next block where there's a change in the 189 * shared/unshared status. More specifically, this means that we 190 * find the lowest-numbered extent of shared blocks that coincides with 191 * the given block mapping. If the shared extent overlaps the start of 192 * the mapping, trim the mapping to the end of the shared extent. If 193 * the shared region intersects the mapping, trim the mapping to the 194 * start of the shared extent. If there are no shared regions that 195 * overlap, just return the original extent. 196 */ 197 int 198 xfs_reflink_trim_around_shared( 199 struct xfs_inode *ip, 200 struct xfs_bmbt_irec *irec, 201 bool *shared, 202 bool *trimmed) 203 { 204 xfs_agnumber_t agno; 205 xfs_agblock_t agbno; 206 xfs_extlen_t aglen; 207 xfs_agblock_t fbno; 208 xfs_extlen_t flen; 209 int error = 0; 210 211 /* Holes, unwritten, and delalloc extents cannot be shared */ 212 if (!xfs_is_reflink_inode(ip) || !xfs_bmap_is_real_extent(irec)) { 213 *shared = false; 214 return 0; 215 } 216 217 trace_xfs_reflink_trim_around_shared(ip, irec); 218 219 agno = XFS_FSB_TO_AGNO(ip->i_mount, irec->br_startblock); 220 agbno = XFS_FSB_TO_AGBNO(ip->i_mount, irec->br_startblock); 221 aglen = irec->br_blockcount; 222 223 error = xfs_reflink_find_shared(ip->i_mount, NULL, agno, agbno, 224 aglen, &fbno, &flen, true); 225 if (error) 226 return error; 227 228 *shared = *trimmed = false; 229 if (fbno == NULLAGBLOCK) { 230 /* No shared blocks at all. */ 231 return 0; 232 } else if (fbno == agbno) { 233 /* 234 * The start of this extent is shared. Truncate the 235 * mapping at the end of the shared region so that a 236 * subsequent iteration starts at the start of the 237 * unshared region. 238 */ 239 irec->br_blockcount = flen; 240 *shared = true; 241 if (flen != aglen) 242 *trimmed = true; 243 return 0; 244 } else { 245 /* 246 * There's a shared extent midway through this extent. 247 * Truncate the mapping at the start of the shared 248 * extent so that a subsequent iteration starts at the 249 * start of the shared region. 250 */ 251 irec->br_blockcount = fbno - agbno; 252 *trimmed = true; 253 return 0; 254 } 255 } 256 257 /* 258 * Trim the passed in imap to the next shared/unshared extent boundary, and 259 * if imap->br_startoff points to a shared extent reserve space for it in the 260 * COW fork. In this case *shared is set to true, else to false. 261 * 262 * Note that imap will always contain the block numbers for the existing blocks 263 * in the data fork, as the upper layers need them for read-modify-write 264 * operations. 265 */ 266 int 267 xfs_reflink_reserve_cow( 268 struct xfs_inode *ip, 269 struct xfs_bmbt_irec *imap, 270 bool *shared) 271 { 272 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 273 struct xfs_bmbt_irec got; 274 int error = 0; 275 bool eof = false, trimmed; 276 xfs_extnum_t idx; 277 278 /* 279 * Search the COW fork extent list first. This serves two purposes: 280 * first this implement the speculative preallocation using cowextisze, 281 * so that we also unshared block adjacent to shared blocks instead 282 * of just the shared blocks themselves. Second the lookup in the 283 * extent list is generally faster than going out to the shared extent 284 * tree. 285 */ 286 287 if (!xfs_iext_lookup_extent(ip, ifp, imap->br_startoff, &idx, &got)) 288 eof = true; 289 if (!eof && got.br_startoff <= imap->br_startoff) { 290 trace_xfs_reflink_cow_found(ip, imap); 291 xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); 292 293 *shared = true; 294 return 0; 295 } 296 297 /* Trim the mapping to the nearest shared extent boundary. */ 298 error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed); 299 if (error) 300 return error; 301 302 /* Not shared? Just report the (potentially capped) extent. */ 303 if (!*shared) 304 return 0; 305 306 /* 307 * Fork all the shared blocks from our write offset until the end of 308 * the extent. 309 */ 310 error = xfs_qm_dqattach_locked(ip, 0); 311 if (error) 312 return error; 313 314 error = xfs_bmapi_reserve_delalloc(ip, XFS_COW_FORK, imap->br_startoff, 315 imap->br_blockcount, 0, &got, &idx, eof); 316 if (error == -ENOSPC || error == -EDQUOT) 317 trace_xfs_reflink_cow_enospc(ip, imap); 318 if (error) 319 return error; 320 321 trace_xfs_reflink_cow_alloc(ip, &got); 322 return 0; 323 } 324 325 /* Convert part of an unwritten CoW extent to a real one. */ 326 STATIC int 327 xfs_reflink_convert_cow_extent( 328 struct xfs_inode *ip, 329 struct xfs_bmbt_irec *imap, 330 xfs_fileoff_t offset_fsb, 331 xfs_filblks_t count_fsb, 332 struct xfs_defer_ops *dfops) 333 { 334 xfs_fsblock_t first_block = NULLFSBLOCK; 335 int nimaps = 1; 336 337 if (imap->br_state == XFS_EXT_NORM) 338 return 0; 339 340 xfs_trim_extent(imap, offset_fsb, count_fsb); 341 trace_xfs_reflink_convert_cow(ip, imap); 342 if (imap->br_blockcount == 0) 343 return 0; 344 return xfs_bmapi_write(NULL, ip, imap->br_startoff, imap->br_blockcount, 345 XFS_BMAPI_COWFORK | XFS_BMAPI_CONVERT, &first_block, 346 0, imap, &nimaps, dfops); 347 } 348 349 /* Convert all of the unwritten CoW extents in a file's range to real ones. */ 350 int 351 xfs_reflink_convert_cow( 352 struct xfs_inode *ip, 353 xfs_off_t offset, 354 xfs_off_t count) 355 { 356 struct xfs_bmbt_irec got; 357 struct xfs_defer_ops dfops; 358 struct xfs_mount *mp = ip->i_mount; 359 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 360 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 361 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); 362 xfs_extnum_t idx; 363 bool found; 364 int error = 0; 365 366 xfs_ilock(ip, XFS_ILOCK_EXCL); 367 368 /* Convert all the extents to real from unwritten. */ 369 for (found = xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got); 370 found && got.br_startoff < end_fsb; 371 found = xfs_iext_get_extent(ifp, ++idx, &got)) { 372 error = xfs_reflink_convert_cow_extent(ip, &got, offset_fsb, 373 end_fsb - offset_fsb, &dfops); 374 if (error) 375 break; 376 } 377 378 /* Finish up. */ 379 xfs_iunlock(ip, XFS_ILOCK_EXCL); 380 return error; 381 } 382 383 /* Allocate all CoW reservations covering a range of blocks in a file. */ 384 int 385 xfs_reflink_allocate_cow( 386 struct xfs_inode *ip, 387 struct xfs_bmbt_irec *imap, 388 bool *shared, 389 uint *lockmode) 390 { 391 struct xfs_mount *mp = ip->i_mount; 392 xfs_fileoff_t offset_fsb = imap->br_startoff; 393 xfs_filblks_t count_fsb = imap->br_blockcount; 394 struct xfs_bmbt_irec got; 395 struct xfs_defer_ops dfops; 396 struct xfs_trans *tp = NULL; 397 xfs_fsblock_t first_block; 398 int nimaps, error = 0; 399 bool trimmed; 400 xfs_filblks_t resaligned; 401 xfs_extlen_t resblks = 0; 402 xfs_extnum_t idx; 403 404 retry: 405 ASSERT(xfs_is_reflink_inode(ip)); 406 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED)); 407 408 /* 409 * Even if the extent is not shared we might have a preallocation for 410 * it in the COW fork. If so use it. 411 */ 412 if (xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &idx, &got) && 413 got.br_startoff <= offset_fsb) { 414 *shared = true; 415 416 /* If we have a real allocation in the COW fork we're done. */ 417 if (!isnullstartblock(got.br_startblock)) { 418 xfs_trim_extent(&got, offset_fsb, count_fsb); 419 *imap = got; 420 goto convert; 421 } 422 423 xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); 424 } else { 425 error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed); 426 if (error || !*shared) 427 goto out; 428 } 429 430 if (!tp) { 431 resaligned = xfs_aligned_fsb_count(imap->br_startoff, 432 imap->br_blockcount, xfs_get_cowextsz_hint(ip)); 433 resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); 434 435 xfs_iunlock(ip, *lockmode); 436 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); 437 *lockmode = XFS_ILOCK_EXCL; 438 xfs_ilock(ip, *lockmode); 439 440 if (error) 441 return error; 442 443 error = xfs_qm_dqattach_locked(ip, 0); 444 if (error) 445 goto out; 446 goto retry; 447 } 448 449 error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, 450 XFS_QMOPT_RES_REGBLKS); 451 if (error) 452 goto out; 453 454 xfs_trans_ijoin(tp, ip, 0); 455 456 xfs_defer_init(&dfops, &first_block); 457 nimaps = 1; 458 459 /* Allocate the entire reservation as unwritten blocks. */ 460 error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount, 461 XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, &first_block, 462 resblks, imap, &nimaps, &dfops); 463 if (error) 464 goto out_bmap_cancel; 465 466 /* Finish up. */ 467 error = xfs_defer_finish(&tp, &dfops, NULL); 468 if (error) 469 goto out_bmap_cancel; 470 471 error = xfs_trans_commit(tp); 472 if (error) 473 return error; 474 convert: 475 return xfs_reflink_convert_cow_extent(ip, imap, offset_fsb, count_fsb, 476 &dfops); 477 out_bmap_cancel: 478 xfs_defer_cancel(&dfops); 479 xfs_trans_unreserve_quota_nblks(tp, ip, (long)resblks, 0, 480 XFS_QMOPT_RES_REGBLKS); 481 out: 482 if (tp) 483 xfs_trans_cancel(tp); 484 return error; 485 } 486 487 /* 488 * Find the CoW reservation for a given byte offset of a file. 489 */ 490 bool 491 xfs_reflink_find_cow_mapping( 492 struct xfs_inode *ip, 493 xfs_off_t offset, 494 struct xfs_bmbt_irec *imap) 495 { 496 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 497 xfs_fileoff_t offset_fsb; 498 struct xfs_bmbt_irec got; 499 xfs_extnum_t idx; 500 501 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED)); 502 ASSERT(xfs_is_reflink_inode(ip)); 503 504 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 505 if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got)) 506 return false; 507 if (got.br_startoff > offset_fsb) 508 return false; 509 510 trace_xfs_reflink_find_cow_mapping(ip, offset, 1, XFS_IO_OVERWRITE, 511 &got); 512 *imap = got; 513 return true; 514 } 515 516 /* 517 * Trim an extent to end at the next CoW reservation past offset_fsb. 518 */ 519 void 520 xfs_reflink_trim_irec_to_next_cow( 521 struct xfs_inode *ip, 522 xfs_fileoff_t offset_fsb, 523 struct xfs_bmbt_irec *imap) 524 { 525 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 526 struct xfs_bmbt_irec got; 527 xfs_extnum_t idx; 528 529 if (!xfs_is_reflink_inode(ip)) 530 return; 531 532 /* Find the extent in the CoW fork. */ 533 if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got)) 534 return; 535 536 /* This is the extent before; try sliding up one. */ 537 if (got.br_startoff < offset_fsb) { 538 if (!xfs_iext_get_extent(ifp, idx + 1, &got)) 539 return; 540 } 541 542 if (got.br_startoff >= imap->br_startoff + imap->br_blockcount) 543 return; 544 545 imap->br_blockcount = got.br_startoff - imap->br_startoff; 546 trace_xfs_reflink_trim_irec(ip, imap); 547 } 548 549 /* 550 * Cancel CoW reservations for some block range of an inode. 551 * 552 * If cancel_real is true this function cancels all COW fork extents for the 553 * inode; if cancel_real is false, real extents are not cleared. 554 */ 555 int 556 xfs_reflink_cancel_cow_blocks( 557 struct xfs_inode *ip, 558 struct xfs_trans **tpp, 559 xfs_fileoff_t offset_fsb, 560 xfs_fileoff_t end_fsb, 561 bool cancel_real) 562 { 563 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 564 struct xfs_bmbt_irec got, del; 565 xfs_extnum_t idx; 566 xfs_fsblock_t firstfsb; 567 struct xfs_defer_ops dfops; 568 int error = 0; 569 570 if (!xfs_is_reflink_inode(ip)) 571 return 0; 572 if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got)) 573 return 0; 574 575 while (got.br_startoff < end_fsb) { 576 del = got; 577 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 578 trace_xfs_reflink_cancel_cow(ip, &del); 579 580 if (isnullstartblock(del.br_startblock)) { 581 error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, 582 &idx, &got, &del); 583 if (error) 584 break; 585 } else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) { 586 xfs_trans_ijoin(*tpp, ip, 0); 587 xfs_defer_init(&dfops, &firstfsb); 588 589 /* Free the CoW orphan record. */ 590 error = xfs_refcount_free_cow_extent(ip->i_mount, 591 &dfops, del.br_startblock, 592 del.br_blockcount); 593 if (error) 594 break; 595 596 xfs_bmap_add_free(ip->i_mount, &dfops, 597 del.br_startblock, del.br_blockcount, 598 NULL); 599 600 /* Update quota accounting */ 601 xfs_trans_mod_dquot_byino(*tpp, ip, XFS_TRANS_DQ_BCOUNT, 602 -(long)del.br_blockcount); 603 604 /* Roll the transaction */ 605 error = xfs_defer_finish(tpp, &dfops, ip); 606 if (error) { 607 xfs_defer_cancel(&dfops); 608 break; 609 } 610 611 /* Remove the mapping from the CoW fork. */ 612 xfs_bmap_del_extent_cow(ip, &idx, &got, &del); 613 } 614 615 if (!xfs_iext_get_extent(ifp, ++idx, &got)) 616 break; 617 } 618 619 /* clear tag if cow fork is emptied */ 620 if (!ifp->if_bytes) 621 xfs_inode_clear_cowblocks_tag(ip); 622 623 return error; 624 } 625 626 /* 627 * Cancel CoW reservations for some byte range of an inode. 628 * 629 * If cancel_real is true this function cancels all COW fork extents for the 630 * inode; if cancel_real is false, real extents are not cleared. 631 */ 632 int 633 xfs_reflink_cancel_cow_range( 634 struct xfs_inode *ip, 635 xfs_off_t offset, 636 xfs_off_t count, 637 bool cancel_real) 638 { 639 struct xfs_trans *tp; 640 xfs_fileoff_t offset_fsb; 641 xfs_fileoff_t end_fsb; 642 int error; 643 644 trace_xfs_reflink_cancel_cow_range(ip, offset, count); 645 ASSERT(xfs_is_reflink_inode(ip)); 646 647 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 648 if (count == NULLFILEOFF) 649 end_fsb = NULLFILEOFF; 650 else 651 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 652 653 /* Start a rolling transaction to remove the mappings */ 654 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 655 0, 0, 0, &tp); 656 if (error) 657 goto out; 658 659 xfs_ilock(ip, XFS_ILOCK_EXCL); 660 xfs_trans_ijoin(tp, ip, 0); 661 662 /* Scrape out the old CoW reservations */ 663 error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb, 664 cancel_real); 665 if (error) 666 goto out_cancel; 667 668 error = xfs_trans_commit(tp); 669 670 xfs_iunlock(ip, XFS_ILOCK_EXCL); 671 return error; 672 673 out_cancel: 674 xfs_trans_cancel(tp); 675 xfs_iunlock(ip, XFS_ILOCK_EXCL); 676 out: 677 trace_xfs_reflink_cancel_cow_range_error(ip, error, _RET_IP_); 678 return error; 679 } 680 681 /* 682 * Remap parts of a file's data fork after a successful CoW. 683 */ 684 int 685 xfs_reflink_end_cow( 686 struct xfs_inode *ip, 687 xfs_off_t offset, 688 xfs_off_t count) 689 { 690 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 691 struct xfs_bmbt_irec got, del; 692 struct xfs_trans *tp; 693 xfs_fileoff_t offset_fsb; 694 xfs_fileoff_t end_fsb; 695 xfs_fsblock_t firstfsb; 696 struct xfs_defer_ops dfops; 697 int error; 698 unsigned int resblks; 699 xfs_filblks_t rlen; 700 xfs_extnum_t idx; 701 702 trace_xfs_reflink_end_cow(ip, offset, count); 703 704 /* No COW extents? That's easy! */ 705 if (ifp->if_bytes == 0) 706 return 0; 707 708 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 709 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 710 711 /* 712 * Start a rolling transaction to switch the mappings. We're 713 * unlikely ever to have to remap 16T worth of single-block 714 * extents, so just cap the worst case extent count to 2^32-1. 715 * Stick a warning in just in case, and avoid 64-bit division. 716 */ 717 BUILD_BUG_ON(MAX_RW_COUNT > UINT_MAX); 718 if (end_fsb - offset_fsb > UINT_MAX) { 719 error = -EFSCORRUPTED; 720 xfs_force_shutdown(ip->i_mount, SHUTDOWN_CORRUPT_INCORE); 721 ASSERT(0); 722 goto out; 723 } 724 resblks = XFS_NEXTENTADD_SPACE_RES(ip->i_mount, 725 (unsigned int)(end_fsb - offset_fsb), 726 XFS_DATA_FORK); 727 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 728 resblks, 0, 0, &tp); 729 if (error) 730 goto out; 731 732 xfs_ilock(ip, XFS_ILOCK_EXCL); 733 xfs_trans_ijoin(tp, ip, 0); 734 735 /* If there is a hole at end_fsb - 1 go to the previous extent */ 736 if (!xfs_iext_lookup_extent(ip, ifp, end_fsb - 1, &idx, &got) || 737 got.br_startoff > end_fsb) { 738 ASSERT(idx > 0); 739 xfs_iext_get_extent(ifp, --idx, &got); 740 } 741 742 /* Walk backwards until we're out of the I/O range... */ 743 while (got.br_startoff + got.br_blockcount > offset_fsb) { 744 del = got; 745 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 746 747 /* Extent delete may have bumped idx forward */ 748 if (!del.br_blockcount) { 749 idx--; 750 goto next_extent; 751 } 752 753 ASSERT(!isnullstartblock(got.br_startblock)); 754 755 /* 756 * Don't remap unwritten extents; these are 757 * speculatively preallocated CoW extents that have been 758 * allocated but have not yet been involved in a write. 759 */ 760 if (got.br_state == XFS_EXT_UNWRITTEN) { 761 idx--; 762 goto next_extent; 763 } 764 765 /* Unmap the old blocks in the data fork. */ 766 xfs_defer_init(&dfops, &firstfsb); 767 rlen = del.br_blockcount; 768 error = __xfs_bunmapi(tp, ip, del.br_startoff, &rlen, 0, 1, 769 &firstfsb, &dfops); 770 if (error) 771 goto out_defer; 772 773 /* Trim the extent to whatever got unmapped. */ 774 if (rlen) { 775 xfs_trim_extent(&del, del.br_startoff + rlen, 776 del.br_blockcount - rlen); 777 } 778 trace_xfs_reflink_cow_remap(ip, &del); 779 780 /* Free the CoW orphan record. */ 781 error = xfs_refcount_free_cow_extent(tp->t_mountp, &dfops, 782 del.br_startblock, del.br_blockcount); 783 if (error) 784 goto out_defer; 785 786 /* Map the new blocks into the data fork. */ 787 error = xfs_bmap_map_extent(tp->t_mountp, &dfops, ip, &del); 788 if (error) 789 goto out_defer; 790 791 /* Remove the mapping from the CoW fork. */ 792 xfs_bmap_del_extent_cow(ip, &idx, &got, &del); 793 794 error = xfs_defer_finish(&tp, &dfops, ip); 795 if (error) 796 goto out_defer; 797 next_extent: 798 if (!xfs_iext_get_extent(ifp, idx, &got)) 799 break; 800 } 801 802 error = xfs_trans_commit(tp); 803 xfs_iunlock(ip, XFS_ILOCK_EXCL); 804 if (error) 805 goto out; 806 return 0; 807 808 out_defer: 809 xfs_defer_cancel(&dfops); 810 xfs_trans_cancel(tp); 811 xfs_iunlock(ip, XFS_ILOCK_EXCL); 812 out: 813 trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_); 814 return error; 815 } 816 817 /* 818 * Free leftover CoW reservations that didn't get cleaned out. 819 */ 820 int 821 xfs_reflink_recover_cow( 822 struct xfs_mount *mp) 823 { 824 xfs_agnumber_t agno; 825 int error = 0; 826 827 if (!xfs_sb_version_hasreflink(&mp->m_sb)) 828 return 0; 829 830 for (agno = 0; agno < mp->m_sb.sb_agcount; agno++) { 831 error = xfs_refcount_recover_cow_leftovers(mp, agno); 832 if (error) 833 break; 834 } 835 836 return error; 837 } 838 839 /* 840 * Reflinking (Block) Ranges of Two Files Together 841 * 842 * First, ensure that the reflink flag is set on both inodes. The flag is an 843 * optimization to avoid unnecessary refcount btree lookups in the write path. 844 * 845 * Now we can iteratively remap the range of extents (and holes) in src to the 846 * corresponding ranges in dest. Let drange and srange denote the ranges of 847 * logical blocks in dest and src touched by the reflink operation. 848 * 849 * While the length of drange is greater than zero, 850 * - Read src's bmbt at the start of srange ("imap") 851 * - If imap doesn't exist, make imap appear to start at the end of srange 852 * with zero length. 853 * - If imap starts before srange, advance imap to start at srange. 854 * - If imap goes beyond srange, truncate imap to end at the end of srange. 855 * - Punch (imap start - srange start + imap len) blocks from dest at 856 * offset (drange start). 857 * - If imap points to a real range of pblks, 858 * > Increase the refcount of the imap's pblks 859 * > Map imap's pblks into dest at the offset 860 * (drange start + imap start - srange start) 861 * - Advance drange and srange by (imap start - srange start + imap len) 862 * 863 * Finally, if the reflink made dest longer, update both the in-core and 864 * on-disk file sizes. 865 * 866 * ASCII Art Demonstration: 867 * 868 * Let's say we want to reflink this source file: 869 * 870 * ----SSSSSSS-SSSSS----SSSSSS (src file) 871 * <--------------------> 872 * 873 * into this destination file: 874 * 875 * --DDDDDDDDDDDDDDDDDDD--DDD (dest file) 876 * <--------------------> 877 * '-' means a hole, and 'S' and 'D' are written blocks in the src and dest. 878 * Observe that the range has different logical offsets in either file. 879 * 880 * Consider that the first extent in the source file doesn't line up with our 881 * reflink range. Unmapping and remapping are separate operations, so we can 882 * unmap more blocks from the destination file than we remap. 883 * 884 * ----SSSSSSS-SSSSS----SSSSSS 885 * <-------> 886 * --DDDDD---------DDDDD--DDD 887 * <-------> 888 * 889 * Now remap the source extent into the destination file: 890 * 891 * ----SSSSSSS-SSSSS----SSSSSS 892 * <-------> 893 * --DDDDD--SSSSSSSDDDDD--DDD 894 * <-------> 895 * 896 * Do likewise with the second hole and extent in our range. Holes in the 897 * unmap range don't affect our operation. 898 * 899 * ----SSSSSSS-SSSSS----SSSSSS 900 * <----> 901 * --DDDDD--SSSSSSS-SSSSS-DDD 902 * <----> 903 * 904 * Finally, unmap and remap part of the third extent. This will increase the 905 * size of the destination file. 906 * 907 * ----SSSSSSS-SSSSS----SSSSSS 908 * <-----> 909 * --DDDDD--SSSSSSS-SSSSS----SSS 910 * <-----> 911 * 912 * Once we update the destination file's i_size, we're done. 913 */ 914 915 /* 916 * Ensure the reflink bit is set in both inodes. 917 */ 918 STATIC int 919 xfs_reflink_set_inode_flag( 920 struct xfs_inode *src, 921 struct xfs_inode *dest) 922 { 923 struct xfs_mount *mp = src->i_mount; 924 int error; 925 struct xfs_trans *tp; 926 927 if (xfs_is_reflink_inode(src) && xfs_is_reflink_inode(dest)) 928 return 0; 929 930 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 931 if (error) 932 goto out_error; 933 934 /* Lock both files against IO */ 935 if (src->i_ino == dest->i_ino) 936 xfs_ilock(src, XFS_ILOCK_EXCL); 937 else 938 xfs_lock_two_inodes(src, dest, XFS_ILOCK_EXCL); 939 940 if (!xfs_is_reflink_inode(src)) { 941 trace_xfs_reflink_set_inode_flag(src); 942 xfs_trans_ijoin(tp, src, XFS_ILOCK_EXCL); 943 src->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 944 xfs_trans_log_inode(tp, src, XFS_ILOG_CORE); 945 xfs_ifork_init_cow(src); 946 } else 947 xfs_iunlock(src, XFS_ILOCK_EXCL); 948 949 if (src->i_ino == dest->i_ino) 950 goto commit_flags; 951 952 if (!xfs_is_reflink_inode(dest)) { 953 trace_xfs_reflink_set_inode_flag(dest); 954 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 955 dest->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 956 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 957 xfs_ifork_init_cow(dest); 958 } else 959 xfs_iunlock(dest, XFS_ILOCK_EXCL); 960 961 commit_flags: 962 error = xfs_trans_commit(tp); 963 if (error) 964 goto out_error; 965 return error; 966 967 out_error: 968 trace_xfs_reflink_set_inode_flag_error(dest, error, _RET_IP_); 969 return error; 970 } 971 972 /* 973 * Update destination inode size & cowextsize hint, if necessary. 974 */ 975 STATIC int 976 xfs_reflink_update_dest( 977 struct xfs_inode *dest, 978 xfs_off_t newlen, 979 xfs_extlen_t cowextsize, 980 bool is_dedupe) 981 { 982 struct xfs_mount *mp = dest->i_mount; 983 struct xfs_trans *tp; 984 int error; 985 986 if (is_dedupe && newlen <= i_size_read(VFS_I(dest)) && cowextsize == 0) 987 return 0; 988 989 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 990 if (error) 991 goto out_error; 992 993 xfs_ilock(dest, XFS_ILOCK_EXCL); 994 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 995 996 if (newlen > i_size_read(VFS_I(dest))) { 997 trace_xfs_reflink_update_inode_size(dest, newlen); 998 i_size_write(VFS_I(dest), newlen); 999 dest->i_d.di_size = newlen; 1000 } 1001 1002 if (cowextsize) { 1003 dest->i_d.di_cowextsize = cowextsize; 1004 dest->i_d.di_flags2 |= XFS_DIFLAG2_COWEXTSIZE; 1005 } 1006 1007 if (!is_dedupe) { 1008 xfs_trans_ichgtime(tp, dest, 1009 XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); 1010 } 1011 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 1012 1013 error = xfs_trans_commit(tp); 1014 if (error) 1015 goto out_error; 1016 return error; 1017 1018 out_error: 1019 trace_xfs_reflink_update_inode_size_error(dest, error, _RET_IP_); 1020 return error; 1021 } 1022 1023 /* 1024 * Do we have enough reserve in this AG to handle a reflink? The refcount 1025 * btree already reserved all the space it needs, but the rmap btree can grow 1026 * infinitely, so we won't allow more reflinks when the AG is down to the 1027 * btree reserves. 1028 */ 1029 static int 1030 xfs_reflink_ag_has_free_space( 1031 struct xfs_mount *mp, 1032 xfs_agnumber_t agno) 1033 { 1034 struct xfs_perag *pag; 1035 int error = 0; 1036 1037 if (!xfs_sb_version_hasrmapbt(&mp->m_sb)) 1038 return 0; 1039 1040 pag = xfs_perag_get(mp, agno); 1041 if (xfs_ag_resv_critical(pag, XFS_AG_RESV_AGFL) || 1042 xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA)) 1043 error = -ENOSPC; 1044 xfs_perag_put(pag); 1045 return error; 1046 } 1047 1048 /* 1049 * Unmap a range of blocks from a file, then map other blocks into the hole. 1050 * The range to unmap is (destoff : destoff + srcioff + irec->br_blockcount). 1051 * The extent irec is mapped into dest at irec->br_startoff. 1052 */ 1053 STATIC int 1054 xfs_reflink_remap_extent( 1055 struct xfs_inode *ip, 1056 struct xfs_bmbt_irec *irec, 1057 xfs_fileoff_t destoff, 1058 xfs_off_t new_isize) 1059 { 1060 struct xfs_mount *mp = ip->i_mount; 1061 bool real_extent = xfs_bmap_is_real_extent(irec); 1062 struct xfs_trans *tp; 1063 xfs_fsblock_t firstfsb; 1064 unsigned int resblks; 1065 struct xfs_defer_ops dfops; 1066 struct xfs_bmbt_irec uirec; 1067 xfs_filblks_t rlen; 1068 xfs_filblks_t unmap_len; 1069 xfs_off_t newlen; 1070 int error; 1071 1072 unmap_len = irec->br_startoff + irec->br_blockcount - destoff; 1073 trace_xfs_reflink_punch_range(ip, destoff, unmap_len); 1074 1075 /* No reflinking if we're low on space */ 1076 if (real_extent) { 1077 error = xfs_reflink_ag_has_free_space(mp, 1078 XFS_FSB_TO_AGNO(mp, irec->br_startblock)); 1079 if (error) 1080 goto out; 1081 } 1082 1083 /* Start a rolling transaction to switch the mappings */ 1084 resblks = XFS_EXTENTADD_SPACE_RES(ip->i_mount, XFS_DATA_FORK); 1085 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); 1086 if (error) 1087 goto out; 1088 1089 xfs_ilock(ip, XFS_ILOCK_EXCL); 1090 xfs_trans_ijoin(tp, ip, 0); 1091 1092 /* If we're not just clearing space, then do we have enough quota? */ 1093 if (real_extent) { 1094 error = xfs_trans_reserve_quota_nblks(tp, ip, 1095 irec->br_blockcount, 0, XFS_QMOPT_RES_REGBLKS); 1096 if (error) 1097 goto out_cancel; 1098 } 1099 1100 trace_xfs_reflink_remap(ip, irec->br_startoff, 1101 irec->br_blockcount, irec->br_startblock); 1102 1103 /* Unmap the old blocks in the data fork. */ 1104 rlen = unmap_len; 1105 while (rlen) { 1106 xfs_defer_init(&dfops, &firstfsb); 1107 error = __xfs_bunmapi(tp, ip, destoff, &rlen, 0, 1, 1108 &firstfsb, &dfops); 1109 if (error) 1110 goto out_defer; 1111 1112 /* 1113 * Trim the extent to whatever got unmapped. 1114 * Remember, bunmapi works backwards. 1115 */ 1116 uirec.br_startblock = irec->br_startblock + rlen; 1117 uirec.br_startoff = irec->br_startoff + rlen; 1118 uirec.br_blockcount = unmap_len - rlen; 1119 unmap_len = rlen; 1120 1121 /* If this isn't a real mapping, we're done. */ 1122 if (!real_extent || uirec.br_blockcount == 0) 1123 goto next_extent; 1124 1125 trace_xfs_reflink_remap(ip, uirec.br_startoff, 1126 uirec.br_blockcount, uirec.br_startblock); 1127 1128 /* Update the refcount tree */ 1129 error = xfs_refcount_increase_extent(mp, &dfops, &uirec); 1130 if (error) 1131 goto out_defer; 1132 1133 /* Map the new blocks into the data fork. */ 1134 error = xfs_bmap_map_extent(mp, &dfops, ip, &uirec); 1135 if (error) 1136 goto out_defer; 1137 1138 /* Update quota accounting. */ 1139 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1140 uirec.br_blockcount); 1141 1142 /* Update dest isize if needed. */ 1143 newlen = XFS_FSB_TO_B(mp, 1144 uirec.br_startoff + uirec.br_blockcount); 1145 newlen = min_t(xfs_off_t, newlen, new_isize); 1146 if (newlen > i_size_read(VFS_I(ip))) { 1147 trace_xfs_reflink_update_inode_size(ip, newlen); 1148 i_size_write(VFS_I(ip), newlen); 1149 ip->i_d.di_size = newlen; 1150 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1151 } 1152 1153 next_extent: 1154 /* Process all the deferred stuff. */ 1155 error = xfs_defer_finish(&tp, &dfops, ip); 1156 if (error) 1157 goto out_defer; 1158 } 1159 1160 error = xfs_trans_commit(tp); 1161 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1162 if (error) 1163 goto out; 1164 return 0; 1165 1166 out_defer: 1167 xfs_defer_cancel(&dfops); 1168 out_cancel: 1169 xfs_trans_cancel(tp); 1170 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1171 out: 1172 trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_); 1173 return error; 1174 } 1175 1176 /* 1177 * Iteratively remap one file's extents (and holes) to another's. 1178 */ 1179 STATIC int 1180 xfs_reflink_remap_blocks( 1181 struct xfs_inode *src, 1182 xfs_fileoff_t srcoff, 1183 struct xfs_inode *dest, 1184 xfs_fileoff_t destoff, 1185 xfs_filblks_t len, 1186 xfs_off_t new_isize) 1187 { 1188 struct xfs_bmbt_irec imap; 1189 int nimaps; 1190 int error = 0; 1191 xfs_filblks_t range_len; 1192 1193 /* drange = (destoff, destoff + len); srange = (srcoff, srcoff + len) */ 1194 while (len) { 1195 trace_xfs_reflink_remap_blocks_loop(src, srcoff, len, 1196 dest, destoff); 1197 /* Read extent from the source file */ 1198 nimaps = 1; 1199 xfs_ilock(src, XFS_ILOCK_EXCL); 1200 error = xfs_bmapi_read(src, srcoff, len, &imap, &nimaps, 0); 1201 xfs_iunlock(src, XFS_ILOCK_EXCL); 1202 if (error) 1203 goto err; 1204 ASSERT(nimaps == 1); 1205 1206 trace_xfs_reflink_remap_imap(src, srcoff, len, XFS_IO_OVERWRITE, 1207 &imap); 1208 1209 /* Translate imap into the destination file. */ 1210 range_len = imap.br_startoff + imap.br_blockcount - srcoff; 1211 imap.br_startoff += destoff - srcoff; 1212 1213 /* Clear dest from destoff to the end of imap and map it in. */ 1214 error = xfs_reflink_remap_extent(dest, &imap, destoff, 1215 new_isize); 1216 if (error) 1217 goto err; 1218 1219 if (fatal_signal_pending(current)) { 1220 error = -EINTR; 1221 goto err; 1222 } 1223 1224 /* Advance drange/srange */ 1225 srcoff += range_len; 1226 destoff += range_len; 1227 len -= range_len; 1228 } 1229 1230 return 0; 1231 1232 err: 1233 trace_xfs_reflink_remap_blocks_error(dest, error, _RET_IP_); 1234 return error; 1235 } 1236 1237 /* 1238 * Link a range of blocks from one file to another. 1239 */ 1240 int 1241 xfs_reflink_remap_range( 1242 struct file *file_in, 1243 loff_t pos_in, 1244 struct file *file_out, 1245 loff_t pos_out, 1246 u64 len, 1247 bool is_dedupe) 1248 { 1249 struct inode *inode_in = file_inode(file_in); 1250 struct xfs_inode *src = XFS_I(inode_in); 1251 struct inode *inode_out = file_inode(file_out); 1252 struct xfs_inode *dest = XFS_I(inode_out); 1253 struct xfs_mount *mp = src->i_mount; 1254 bool same_inode = (inode_in == inode_out); 1255 xfs_fileoff_t sfsbno, dfsbno; 1256 xfs_filblks_t fsblen; 1257 xfs_extlen_t cowextsize; 1258 ssize_t ret; 1259 1260 if (!xfs_sb_version_hasreflink(&mp->m_sb)) 1261 return -EOPNOTSUPP; 1262 1263 if (XFS_FORCED_SHUTDOWN(mp)) 1264 return -EIO; 1265 1266 /* Lock both files against IO */ 1267 lock_two_nondirectories(inode_in, inode_out); 1268 if (same_inode) 1269 xfs_ilock(src, XFS_MMAPLOCK_EXCL); 1270 else 1271 xfs_lock_two_inodes(src, dest, XFS_MMAPLOCK_EXCL); 1272 1273 /* Check file eligibility and prepare for block sharing. */ 1274 ret = -EINVAL; 1275 /* Don't reflink realtime inodes */ 1276 if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest)) 1277 goto out_unlock; 1278 1279 /* Don't share DAX file data for now. */ 1280 if (IS_DAX(inode_in) || IS_DAX(inode_out)) 1281 goto out_unlock; 1282 1283 ret = vfs_clone_file_prep_inodes(inode_in, pos_in, inode_out, pos_out, 1284 &len, is_dedupe); 1285 if (ret <= 0) 1286 goto out_unlock; 1287 1288 trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out); 1289 1290 /* Set flags and remap blocks. */ 1291 ret = xfs_reflink_set_inode_flag(src, dest); 1292 if (ret) 1293 goto out_unlock; 1294 1295 dfsbno = XFS_B_TO_FSBT(mp, pos_out); 1296 sfsbno = XFS_B_TO_FSBT(mp, pos_in); 1297 fsblen = XFS_B_TO_FSB(mp, len); 1298 ret = xfs_reflink_remap_blocks(src, sfsbno, dest, dfsbno, fsblen, 1299 pos_out + len); 1300 if (ret) 1301 goto out_unlock; 1302 1303 /* Zap any page cache for the destination file's range. */ 1304 truncate_inode_pages_range(&inode_out->i_data, pos_out, 1305 PAGE_ALIGN(pos_out + len) - 1); 1306 1307 /* 1308 * Carry the cowextsize hint from src to dest if we're sharing the 1309 * entire source file to the entire destination file, the source file 1310 * has a cowextsize hint, and the destination file does not. 1311 */ 1312 cowextsize = 0; 1313 if (pos_in == 0 && len == i_size_read(inode_in) && 1314 (src->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE) && 1315 pos_out == 0 && len >= i_size_read(inode_out) && 1316 !(dest->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE)) 1317 cowextsize = src->i_d.di_cowextsize; 1318 1319 ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize, 1320 is_dedupe); 1321 1322 out_unlock: 1323 xfs_iunlock(src, XFS_MMAPLOCK_EXCL); 1324 if (!same_inode) 1325 xfs_iunlock(dest, XFS_MMAPLOCK_EXCL); 1326 unlock_two_nondirectories(inode_in, inode_out); 1327 if (ret) 1328 trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_); 1329 return ret; 1330 } 1331 1332 /* 1333 * The user wants to preemptively CoW all shared blocks in this file, 1334 * which enables us to turn off the reflink flag. Iterate all 1335 * extents which are not prealloc/delalloc to see which ranges are 1336 * mentioned in the refcount tree, then read those blocks into the 1337 * pagecache, dirty them, fsync them back out, and then we can update 1338 * the inode flag. What happens if we run out of memory? :) 1339 */ 1340 STATIC int 1341 xfs_reflink_dirty_extents( 1342 struct xfs_inode *ip, 1343 xfs_fileoff_t fbno, 1344 xfs_filblks_t end, 1345 xfs_off_t isize) 1346 { 1347 struct xfs_mount *mp = ip->i_mount; 1348 xfs_agnumber_t agno; 1349 xfs_agblock_t agbno; 1350 xfs_extlen_t aglen; 1351 xfs_agblock_t rbno; 1352 xfs_extlen_t rlen; 1353 xfs_off_t fpos; 1354 xfs_off_t flen; 1355 struct xfs_bmbt_irec map[2]; 1356 int nmaps; 1357 int error = 0; 1358 1359 while (end - fbno > 0) { 1360 nmaps = 1; 1361 /* 1362 * Look for extents in the file. Skip holes, delalloc, or 1363 * unwritten extents; they can't be reflinked. 1364 */ 1365 error = xfs_bmapi_read(ip, fbno, end - fbno, map, &nmaps, 0); 1366 if (error) 1367 goto out; 1368 if (nmaps == 0) 1369 break; 1370 if (!xfs_bmap_is_real_extent(&map[0])) 1371 goto next; 1372 1373 map[1] = map[0]; 1374 while (map[1].br_blockcount) { 1375 agno = XFS_FSB_TO_AGNO(mp, map[1].br_startblock); 1376 agbno = XFS_FSB_TO_AGBNO(mp, map[1].br_startblock); 1377 aglen = map[1].br_blockcount; 1378 1379 error = xfs_reflink_find_shared(mp, NULL, agno, agbno, 1380 aglen, &rbno, &rlen, true); 1381 if (error) 1382 goto out; 1383 if (rbno == NULLAGBLOCK) 1384 break; 1385 1386 /* Dirty the pages */ 1387 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1388 fpos = XFS_FSB_TO_B(mp, map[1].br_startoff + 1389 (rbno - agbno)); 1390 flen = XFS_FSB_TO_B(mp, rlen); 1391 if (fpos + flen > isize) 1392 flen = isize - fpos; 1393 error = iomap_file_dirty(VFS_I(ip), fpos, flen, 1394 &xfs_iomap_ops); 1395 xfs_ilock(ip, XFS_ILOCK_EXCL); 1396 if (error) 1397 goto out; 1398 1399 map[1].br_blockcount -= (rbno - agbno + rlen); 1400 map[1].br_startoff += (rbno - agbno + rlen); 1401 map[1].br_startblock += (rbno - agbno + rlen); 1402 } 1403 1404 next: 1405 fbno = map[0].br_startoff + map[0].br_blockcount; 1406 } 1407 out: 1408 return error; 1409 } 1410 1411 /* Does this inode need the reflink flag? */ 1412 int 1413 xfs_reflink_inode_has_shared_extents( 1414 struct xfs_trans *tp, 1415 struct xfs_inode *ip, 1416 bool *has_shared) 1417 { 1418 struct xfs_bmbt_irec got; 1419 struct xfs_mount *mp = ip->i_mount; 1420 struct xfs_ifork *ifp; 1421 xfs_agnumber_t agno; 1422 xfs_agblock_t agbno; 1423 xfs_extlen_t aglen; 1424 xfs_agblock_t rbno; 1425 xfs_extlen_t rlen; 1426 xfs_extnum_t idx; 1427 bool found; 1428 int error; 1429 1430 ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); 1431 if (!(ifp->if_flags & XFS_IFEXTENTS)) { 1432 error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); 1433 if (error) 1434 return error; 1435 } 1436 1437 *has_shared = false; 1438 found = xfs_iext_lookup_extent(ip, ifp, 0, &idx, &got); 1439 while (found) { 1440 if (isnullstartblock(got.br_startblock) || 1441 got.br_state != XFS_EXT_NORM) 1442 goto next; 1443 agno = XFS_FSB_TO_AGNO(mp, got.br_startblock); 1444 agbno = XFS_FSB_TO_AGBNO(mp, got.br_startblock); 1445 aglen = got.br_blockcount; 1446 1447 error = xfs_reflink_find_shared(mp, tp, agno, agbno, aglen, 1448 &rbno, &rlen, false); 1449 if (error) 1450 return error; 1451 /* Is there still a shared block here? */ 1452 if (rbno != NULLAGBLOCK) { 1453 *has_shared = true; 1454 return 0; 1455 } 1456 next: 1457 found = xfs_iext_get_extent(ifp, ++idx, &got); 1458 } 1459 1460 return 0; 1461 } 1462 1463 /* Clear the inode reflink flag if there are no shared extents. */ 1464 int 1465 xfs_reflink_clear_inode_flag( 1466 struct xfs_inode *ip, 1467 struct xfs_trans **tpp) 1468 { 1469 bool needs_flag; 1470 int error = 0; 1471 1472 ASSERT(xfs_is_reflink_inode(ip)); 1473 1474 error = xfs_reflink_inode_has_shared_extents(*tpp, ip, &needs_flag); 1475 if (error || needs_flag) 1476 return error; 1477 1478 /* 1479 * We didn't find any shared blocks so turn off the reflink flag. 1480 * First, get rid of any leftover CoW mappings. 1481 */ 1482 error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, NULLFILEOFF, true); 1483 if (error) 1484 return error; 1485 1486 /* Clear the inode flag. */ 1487 trace_xfs_reflink_unset_inode_flag(ip); 1488 ip->i_d.di_flags2 &= ~XFS_DIFLAG2_REFLINK; 1489 xfs_inode_clear_cowblocks_tag(ip); 1490 xfs_trans_ijoin(*tpp, ip, 0); 1491 xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE); 1492 1493 return error; 1494 } 1495 1496 /* 1497 * Clear the inode reflink flag if there are no shared extents and the size 1498 * hasn't changed. 1499 */ 1500 STATIC int 1501 xfs_reflink_try_clear_inode_flag( 1502 struct xfs_inode *ip) 1503 { 1504 struct xfs_mount *mp = ip->i_mount; 1505 struct xfs_trans *tp; 1506 int error = 0; 1507 1508 /* Start a rolling transaction to remove the mappings */ 1509 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); 1510 if (error) 1511 return error; 1512 1513 xfs_ilock(ip, XFS_ILOCK_EXCL); 1514 xfs_trans_ijoin(tp, ip, 0); 1515 1516 error = xfs_reflink_clear_inode_flag(ip, &tp); 1517 if (error) 1518 goto cancel; 1519 1520 error = xfs_trans_commit(tp); 1521 if (error) 1522 goto out; 1523 1524 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1525 return 0; 1526 cancel: 1527 xfs_trans_cancel(tp); 1528 out: 1529 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1530 return error; 1531 } 1532 1533 /* 1534 * Pre-COW all shared blocks within a given byte range of a file and turn off 1535 * the reflink flag if we unshare all of the file's blocks. 1536 */ 1537 int 1538 xfs_reflink_unshare( 1539 struct xfs_inode *ip, 1540 xfs_off_t offset, 1541 xfs_off_t len) 1542 { 1543 struct xfs_mount *mp = ip->i_mount; 1544 xfs_fileoff_t fbno; 1545 xfs_filblks_t end; 1546 xfs_off_t isize; 1547 int error; 1548 1549 if (!xfs_is_reflink_inode(ip)) 1550 return 0; 1551 1552 trace_xfs_reflink_unshare(ip, offset, len); 1553 1554 inode_dio_wait(VFS_I(ip)); 1555 1556 /* Try to CoW the selected ranges */ 1557 xfs_ilock(ip, XFS_ILOCK_EXCL); 1558 fbno = XFS_B_TO_FSBT(mp, offset); 1559 isize = i_size_read(VFS_I(ip)); 1560 end = XFS_B_TO_FSB(mp, offset + len); 1561 error = xfs_reflink_dirty_extents(ip, fbno, end, isize); 1562 if (error) 1563 goto out_unlock; 1564 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1565 1566 /* Wait for the IO to finish */ 1567 error = filemap_write_and_wait(VFS_I(ip)->i_mapping); 1568 if (error) 1569 goto out; 1570 1571 /* Turn off the reflink flag if possible. */ 1572 error = xfs_reflink_try_clear_inode_flag(ip); 1573 if (error) 1574 goto out; 1575 1576 return 0; 1577 1578 out_unlock: 1579 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1580 out: 1581 trace_xfs_reflink_unshare_error(ip, error, _RET_IP_); 1582 return error; 1583 } 1584