1 /* 2 * Copyright (C) 2016 Oracle. All Rights Reserved. 3 * 4 * Author: Darrick J. Wong <darrick.wong@oracle.com> 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 2 9 * of the License, or (at your option) any later version. 10 * 11 * This program is distributed in the hope that it would be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. 19 */ 20 #include "xfs.h" 21 #include "xfs_fs.h" 22 #include "xfs_shared.h" 23 #include "xfs_format.h" 24 #include "xfs_log_format.h" 25 #include "xfs_trans_resv.h" 26 #include "xfs_mount.h" 27 #include "xfs_defer.h" 28 #include "xfs_da_format.h" 29 #include "xfs_da_btree.h" 30 #include "xfs_inode.h" 31 #include "xfs_trans.h" 32 #include "xfs_inode_item.h" 33 #include "xfs_bmap.h" 34 #include "xfs_bmap_util.h" 35 #include "xfs_error.h" 36 #include "xfs_dir2.h" 37 #include "xfs_dir2_priv.h" 38 #include "xfs_ioctl.h" 39 #include "xfs_trace.h" 40 #include "xfs_log.h" 41 #include "xfs_icache.h" 42 #include "xfs_pnfs.h" 43 #include "xfs_btree.h" 44 #include "xfs_refcount_btree.h" 45 #include "xfs_refcount.h" 46 #include "xfs_bmap_btree.h" 47 #include "xfs_trans_space.h" 48 #include "xfs_bit.h" 49 #include "xfs_alloc.h" 50 #include "xfs_quota_defs.h" 51 #include "xfs_quota.h" 52 #include "xfs_reflink.h" 53 #include "xfs_iomap.h" 54 #include "xfs_rmap_btree.h" 55 #include "xfs_sb.h" 56 #include "xfs_ag_resv.h" 57 58 /* 59 * Copy on Write of Shared Blocks 60 * 61 * XFS must preserve "the usual" file semantics even when two files share 62 * the same physical blocks. This means that a write to one file must not 63 * alter the blocks in a different file; the way that we'll do that is 64 * through the use of a copy-on-write mechanism. At a high level, that 65 * means that when we want to write to a shared block, we allocate a new 66 * block, write the data to the new block, and if that succeeds we map the 67 * new block into the file. 68 * 69 * XFS provides a "delayed allocation" mechanism that defers the allocation 70 * of disk blocks to dirty-but-not-yet-mapped file blocks as long as 71 * possible. This reduces fragmentation by enabling the filesystem to ask 72 * for bigger chunks less often, which is exactly what we want for CoW. 73 * 74 * The delalloc mechanism begins when the kernel wants to make a block 75 * writable (write_begin or page_mkwrite). If the offset is not mapped, we 76 * create a delalloc mapping, which is a regular in-core extent, but without 77 * a real startblock. (For delalloc mappings, the startblock encodes both 78 * a flag that this is a delalloc mapping, and a worst-case estimate of how 79 * many blocks might be required to put the mapping into the BMBT.) delalloc 80 * mappings are a reservation against the free space in the filesystem; 81 * adjacent mappings can also be combined into fewer larger mappings. 82 * 83 * As an optimization, the CoW extent size hint (cowextsz) creates 84 * outsized aligned delalloc reservations in the hope of landing out of 85 * order nearby CoW writes in a single extent on disk, thereby reducing 86 * fragmentation and improving future performance. 87 * 88 * D: --RRRRRRSSSRRRRRRRR--- (data fork) 89 * C: ------DDDDDDD--------- (CoW fork) 90 * 91 * When dirty pages are being written out (typically in writepage), the 92 * delalloc reservations are converted into unwritten mappings by 93 * allocating blocks and replacing the delalloc mapping with real ones. 94 * A delalloc mapping can be replaced by several unwritten ones if the 95 * free space is fragmented. 96 * 97 * D: --RRRRRRSSSRRRRRRRR--- 98 * C: ------UUUUUUU--------- 99 * 100 * We want to adapt the delalloc mechanism for copy-on-write, since the 101 * write paths are similar. The first two steps (creating the reservation 102 * and allocating the blocks) are exactly the same as delalloc except that 103 * the mappings must be stored in a separate CoW fork because we do not want 104 * to disturb the mapping in the data fork until we're sure that the write 105 * succeeded. IO completion in this case is the process of removing the old 106 * mapping from the data fork and moving the new mapping from the CoW fork to 107 * the data fork. This will be discussed shortly. 108 * 109 * For now, unaligned directio writes will be bounced back to the page cache. 110 * Block-aligned directio writes will use the same mechanism as buffered 111 * writes. 112 * 113 * Just prior to submitting the actual disk write requests, we convert 114 * the extents representing the range of the file actually being written 115 * (as opposed to extra pieces created for the cowextsize hint) to real 116 * extents. This will become important in the next step: 117 * 118 * D: --RRRRRRSSSRRRRRRRR--- 119 * C: ------UUrrUUU--------- 120 * 121 * CoW remapping must be done after the data block write completes, 122 * because we don't want to destroy the old data fork map until we're sure 123 * the new block has been written. Since the new mappings are kept in a 124 * separate fork, we can simply iterate these mappings to find the ones 125 * that cover the file blocks that we just CoW'd. For each extent, simply 126 * unmap the corresponding range in the data fork, map the new range into 127 * the data fork, and remove the extent from the CoW fork. Because of 128 * the presence of the cowextsize hint, however, we must be careful 129 * only to remap the blocks that we've actually written out -- we must 130 * never remap delalloc reservations nor CoW staging blocks that have 131 * yet to be written. This corresponds exactly to the real extents in 132 * the CoW fork: 133 * 134 * D: --RRRRRRrrSRRRRRRRR--- 135 * C: ------UU--UUU--------- 136 * 137 * Since the remapping operation can be applied to an arbitrary file 138 * range, we record the need for the remap step as a flag in the ioend 139 * instead of declaring a new IO type. This is required for direct io 140 * because we only have ioend for the whole dio, and we have to be able to 141 * remember the presence of unwritten blocks and CoW blocks with a single 142 * ioend structure. Better yet, the more ground we can cover with one 143 * ioend, the better. 144 */ 145 146 /* 147 * Given an AG extent, find the lowest-numbered run of shared blocks 148 * within that range and return the range in fbno/flen. If 149 * find_end_of_shared is true, return the longest contiguous extent of 150 * shared blocks. If there are no shared extents, fbno and flen will 151 * be set to NULLAGBLOCK and 0, respectively. 152 */ 153 int 154 xfs_reflink_find_shared( 155 struct xfs_mount *mp, 156 struct xfs_trans *tp, 157 xfs_agnumber_t agno, 158 xfs_agblock_t agbno, 159 xfs_extlen_t aglen, 160 xfs_agblock_t *fbno, 161 xfs_extlen_t *flen, 162 bool find_end_of_shared) 163 { 164 struct xfs_buf *agbp; 165 struct xfs_btree_cur *cur; 166 int error; 167 168 error = xfs_alloc_read_agf(mp, tp, agno, 0, &agbp); 169 if (error) 170 return error; 171 if (!agbp) 172 return -ENOMEM; 173 174 cur = xfs_refcountbt_init_cursor(mp, tp, agbp, agno, NULL); 175 176 error = xfs_refcount_find_shared(cur, agbno, aglen, fbno, flen, 177 find_end_of_shared); 178 179 xfs_btree_del_cursor(cur, error ? XFS_BTREE_ERROR : XFS_BTREE_NOERROR); 180 181 xfs_trans_brelse(tp, agbp); 182 return error; 183 } 184 185 /* 186 * Trim the mapping to the next block where there's a change in the 187 * shared/unshared status. More specifically, this means that we 188 * find the lowest-numbered extent of shared blocks that coincides with 189 * the given block mapping. If the shared extent overlaps the start of 190 * the mapping, trim the mapping to the end of the shared extent. If 191 * the shared region intersects the mapping, trim the mapping to the 192 * start of the shared extent. If there are no shared regions that 193 * overlap, just return the original extent. 194 */ 195 int 196 xfs_reflink_trim_around_shared( 197 struct xfs_inode *ip, 198 struct xfs_bmbt_irec *irec, 199 bool *shared, 200 bool *trimmed) 201 { 202 xfs_agnumber_t agno; 203 xfs_agblock_t agbno; 204 xfs_extlen_t aglen; 205 xfs_agblock_t fbno; 206 xfs_extlen_t flen; 207 int error = 0; 208 209 /* Holes, unwritten, and delalloc extents cannot be shared */ 210 if (!xfs_is_reflink_inode(ip) || !xfs_bmap_is_real_extent(irec)) { 211 *shared = false; 212 return 0; 213 } 214 215 trace_xfs_reflink_trim_around_shared(ip, irec); 216 217 agno = XFS_FSB_TO_AGNO(ip->i_mount, irec->br_startblock); 218 agbno = XFS_FSB_TO_AGBNO(ip->i_mount, irec->br_startblock); 219 aglen = irec->br_blockcount; 220 221 error = xfs_reflink_find_shared(ip->i_mount, NULL, agno, agbno, 222 aglen, &fbno, &flen, true); 223 if (error) 224 return error; 225 226 *shared = *trimmed = false; 227 if (fbno == NULLAGBLOCK) { 228 /* No shared blocks at all. */ 229 return 0; 230 } else if (fbno == agbno) { 231 /* 232 * The start of this extent is shared. Truncate the 233 * mapping at the end of the shared region so that a 234 * subsequent iteration starts at the start of the 235 * unshared region. 236 */ 237 irec->br_blockcount = flen; 238 *shared = true; 239 if (flen != aglen) 240 *trimmed = true; 241 return 0; 242 } else { 243 /* 244 * There's a shared extent midway through this extent. 245 * Truncate the mapping at the start of the shared 246 * extent so that a subsequent iteration starts at the 247 * start of the shared region. 248 */ 249 irec->br_blockcount = fbno - agbno; 250 *trimmed = true; 251 return 0; 252 } 253 } 254 255 /* 256 * Trim the passed in imap to the next shared/unshared extent boundary, and 257 * if imap->br_startoff points to a shared extent reserve space for it in the 258 * COW fork. In this case *shared is set to true, else to false. 259 * 260 * Note that imap will always contain the block numbers for the existing blocks 261 * in the data fork, as the upper layers need them for read-modify-write 262 * operations. 263 */ 264 int 265 xfs_reflink_reserve_cow( 266 struct xfs_inode *ip, 267 struct xfs_bmbt_irec *imap, 268 bool *shared) 269 { 270 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 271 struct xfs_bmbt_irec got; 272 int error = 0; 273 bool eof = false, trimmed; 274 struct xfs_iext_cursor icur; 275 276 /* 277 * Search the COW fork extent list first. This serves two purposes: 278 * first this implement the speculative preallocation using cowextisze, 279 * so that we also unshared block adjacent to shared blocks instead 280 * of just the shared blocks themselves. Second the lookup in the 281 * extent list is generally faster than going out to the shared extent 282 * tree. 283 */ 284 285 if (!xfs_iext_lookup_extent(ip, ifp, imap->br_startoff, &icur, &got)) 286 eof = true; 287 if (!eof && got.br_startoff <= imap->br_startoff) { 288 trace_xfs_reflink_cow_found(ip, imap); 289 xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); 290 291 *shared = true; 292 return 0; 293 } 294 295 /* Trim the mapping to the nearest shared extent boundary. */ 296 error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed); 297 if (error) 298 return error; 299 300 /* Not shared? Just report the (potentially capped) extent. */ 301 if (!*shared) 302 return 0; 303 304 /* 305 * Fork all the shared blocks from our write offset until the end of 306 * the extent. 307 */ 308 error = xfs_qm_dqattach_locked(ip, false); 309 if (error) 310 return error; 311 312 error = xfs_bmapi_reserve_delalloc(ip, XFS_COW_FORK, imap->br_startoff, 313 imap->br_blockcount, 0, &got, &icur, eof); 314 if (error == -ENOSPC || error == -EDQUOT) 315 trace_xfs_reflink_cow_enospc(ip, imap); 316 if (error) 317 return error; 318 319 trace_xfs_reflink_cow_alloc(ip, &got); 320 return 0; 321 } 322 323 /* Convert part of an unwritten CoW extent to a real one. */ 324 STATIC int 325 xfs_reflink_convert_cow_extent( 326 struct xfs_inode *ip, 327 struct xfs_bmbt_irec *imap, 328 xfs_fileoff_t offset_fsb, 329 xfs_filblks_t count_fsb, 330 struct xfs_defer_ops *dfops) 331 { 332 xfs_fsblock_t first_block = NULLFSBLOCK; 333 int nimaps = 1; 334 335 if (imap->br_state == XFS_EXT_NORM) 336 return 0; 337 338 xfs_trim_extent(imap, offset_fsb, count_fsb); 339 trace_xfs_reflink_convert_cow(ip, imap); 340 if (imap->br_blockcount == 0) 341 return 0; 342 return xfs_bmapi_write(NULL, ip, imap->br_startoff, imap->br_blockcount, 343 XFS_BMAPI_COWFORK | XFS_BMAPI_CONVERT, &first_block, 344 0, imap, &nimaps, dfops); 345 } 346 347 /* Convert all of the unwritten CoW extents in a file's range to real ones. */ 348 int 349 xfs_reflink_convert_cow( 350 struct xfs_inode *ip, 351 xfs_off_t offset, 352 xfs_off_t count) 353 { 354 struct xfs_mount *mp = ip->i_mount; 355 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 356 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); 357 xfs_filblks_t count_fsb = end_fsb - offset_fsb; 358 struct xfs_bmbt_irec imap; 359 struct xfs_defer_ops dfops; 360 xfs_fsblock_t first_block = NULLFSBLOCK; 361 int nimaps = 1, error = 0; 362 363 ASSERT(count != 0); 364 365 xfs_ilock(ip, XFS_ILOCK_EXCL); 366 error = xfs_bmapi_write(NULL, ip, offset_fsb, count_fsb, 367 XFS_BMAPI_COWFORK | XFS_BMAPI_CONVERT | 368 XFS_BMAPI_CONVERT_ONLY, &first_block, 0, &imap, &nimaps, 369 &dfops); 370 xfs_iunlock(ip, XFS_ILOCK_EXCL); 371 return error; 372 } 373 374 /* Allocate all CoW reservations covering a range of blocks in a file. */ 375 int 376 xfs_reflink_allocate_cow( 377 struct xfs_inode *ip, 378 struct xfs_bmbt_irec *imap, 379 bool *shared, 380 uint *lockmode) 381 { 382 struct xfs_mount *mp = ip->i_mount; 383 xfs_fileoff_t offset_fsb = imap->br_startoff; 384 xfs_filblks_t count_fsb = imap->br_blockcount; 385 struct xfs_bmbt_irec got; 386 struct xfs_defer_ops dfops; 387 struct xfs_trans *tp = NULL; 388 xfs_fsblock_t first_block; 389 int nimaps, error = 0; 390 bool trimmed; 391 xfs_filblks_t resaligned; 392 xfs_extlen_t resblks = 0; 393 struct xfs_iext_cursor icur; 394 395 retry: 396 ASSERT(xfs_is_reflink_inode(ip)); 397 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); 398 399 /* 400 * Even if the extent is not shared we might have a preallocation for 401 * it in the COW fork. If so use it. 402 */ 403 if (xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got) && 404 got.br_startoff <= offset_fsb) { 405 *shared = true; 406 407 /* If we have a real allocation in the COW fork we're done. */ 408 if (!isnullstartblock(got.br_startblock)) { 409 xfs_trim_extent(&got, offset_fsb, count_fsb); 410 *imap = got; 411 goto convert; 412 } 413 414 xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); 415 } else { 416 error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed); 417 if (error || !*shared) 418 goto out; 419 } 420 421 if (!tp) { 422 resaligned = xfs_aligned_fsb_count(imap->br_startoff, 423 imap->br_blockcount, xfs_get_cowextsz_hint(ip)); 424 resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); 425 426 xfs_iunlock(ip, *lockmode); 427 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); 428 *lockmode = XFS_ILOCK_EXCL; 429 xfs_ilock(ip, *lockmode); 430 431 if (error) 432 return error; 433 434 error = xfs_qm_dqattach_locked(ip, false); 435 if (error) 436 goto out; 437 goto retry; 438 } 439 440 error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, 441 XFS_QMOPT_RES_REGBLKS); 442 if (error) 443 goto out; 444 445 xfs_trans_ijoin(tp, ip, 0); 446 447 xfs_defer_init(&dfops, &first_block); 448 nimaps = 1; 449 450 /* Allocate the entire reservation as unwritten blocks. */ 451 error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount, 452 XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, &first_block, 453 resblks, imap, &nimaps, &dfops); 454 if (error) 455 goto out_bmap_cancel; 456 457 xfs_inode_set_cowblocks_tag(ip); 458 459 /* Finish up. */ 460 error = xfs_defer_finish(&tp, &dfops); 461 if (error) 462 goto out_bmap_cancel; 463 464 error = xfs_trans_commit(tp); 465 if (error) 466 return error; 467 468 /* 469 * Allocation succeeded but the requested range was not even partially 470 * satisfied? Bail out! 471 */ 472 if (nimaps == 0) 473 return -ENOSPC; 474 convert: 475 return xfs_reflink_convert_cow_extent(ip, imap, offset_fsb, count_fsb, 476 &dfops); 477 out_bmap_cancel: 478 xfs_defer_cancel(&dfops); 479 xfs_trans_unreserve_quota_nblks(tp, ip, (long)resblks, 0, 480 XFS_QMOPT_RES_REGBLKS); 481 out: 482 if (tp) 483 xfs_trans_cancel(tp); 484 return error; 485 } 486 487 /* 488 * Find the CoW reservation for a given byte offset of a file. 489 */ 490 bool 491 xfs_reflink_find_cow_mapping( 492 struct xfs_inode *ip, 493 xfs_off_t offset, 494 struct xfs_bmbt_irec *imap) 495 { 496 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 497 xfs_fileoff_t offset_fsb; 498 struct xfs_bmbt_irec got; 499 struct xfs_iext_cursor icur; 500 501 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED)); 502 503 if (!xfs_is_reflink_inode(ip)) 504 return false; 505 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 506 if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &icur, &got)) 507 return false; 508 if (got.br_startoff > offset_fsb) 509 return false; 510 511 trace_xfs_reflink_find_cow_mapping(ip, offset, 1, XFS_IO_OVERWRITE, 512 &got); 513 *imap = got; 514 return true; 515 } 516 517 /* 518 * Trim an extent to end at the next CoW reservation past offset_fsb. 519 */ 520 void 521 xfs_reflink_trim_irec_to_next_cow( 522 struct xfs_inode *ip, 523 xfs_fileoff_t offset_fsb, 524 struct xfs_bmbt_irec *imap) 525 { 526 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 527 struct xfs_bmbt_irec got; 528 struct xfs_iext_cursor icur; 529 530 if (!xfs_is_reflink_inode(ip)) 531 return; 532 533 /* Find the extent in the CoW fork. */ 534 if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &icur, &got)) 535 return; 536 537 /* This is the extent before; try sliding up one. */ 538 if (got.br_startoff < offset_fsb) { 539 if (!xfs_iext_next_extent(ifp, &icur, &got)) 540 return; 541 } 542 543 if (got.br_startoff >= imap->br_startoff + imap->br_blockcount) 544 return; 545 546 imap->br_blockcount = got.br_startoff - imap->br_startoff; 547 trace_xfs_reflink_trim_irec(ip, imap); 548 } 549 550 /* 551 * Cancel CoW reservations for some block range of an inode. 552 * 553 * If cancel_real is true this function cancels all COW fork extents for the 554 * inode; if cancel_real is false, real extents are not cleared. 555 * 556 * Caller must have already joined the inode to the current transaction. The 557 * inode will be joined to the transaction returned to the caller. 558 */ 559 int 560 xfs_reflink_cancel_cow_blocks( 561 struct xfs_inode *ip, 562 struct xfs_trans **tpp, 563 xfs_fileoff_t offset_fsb, 564 xfs_fileoff_t end_fsb, 565 bool cancel_real) 566 { 567 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 568 struct xfs_bmbt_irec got, del; 569 struct xfs_iext_cursor icur; 570 xfs_fsblock_t firstfsb; 571 struct xfs_defer_ops dfops; 572 int error = 0; 573 574 if (!xfs_is_reflink_inode(ip)) 575 return 0; 576 if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) 577 return 0; 578 579 /* Walk backwards until we're out of the I/O range... */ 580 while (got.br_startoff + got.br_blockcount > offset_fsb) { 581 del = got; 582 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 583 584 /* Extent delete may have bumped ext forward */ 585 if (!del.br_blockcount) { 586 xfs_iext_prev(ifp, &icur); 587 goto next_extent; 588 } 589 590 trace_xfs_reflink_cancel_cow(ip, &del); 591 592 if (isnullstartblock(del.br_startblock)) { 593 error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, 594 &icur, &got, &del); 595 if (error) 596 break; 597 } else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) { 598 xfs_defer_init(&dfops, &firstfsb); 599 600 /* Free the CoW orphan record. */ 601 error = xfs_refcount_free_cow_extent(ip->i_mount, 602 &dfops, del.br_startblock, 603 del.br_blockcount); 604 if (error) 605 break; 606 607 xfs_bmap_add_free(ip->i_mount, &dfops, 608 del.br_startblock, del.br_blockcount, 609 NULL); 610 611 /* Roll the transaction */ 612 xfs_defer_ijoin(&dfops, ip); 613 error = xfs_defer_finish(tpp, &dfops); 614 if (error) { 615 xfs_defer_cancel(&dfops); 616 break; 617 } 618 619 /* Remove the mapping from the CoW fork. */ 620 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 621 622 /* Remove the quota reservation */ 623 error = xfs_trans_reserve_quota_nblks(NULL, ip, 624 -(long)del.br_blockcount, 0, 625 XFS_QMOPT_RES_REGBLKS); 626 if (error) 627 break; 628 } else { 629 /* Didn't do anything, push cursor back. */ 630 xfs_iext_prev(ifp, &icur); 631 } 632 next_extent: 633 if (!xfs_iext_get_extent(ifp, &icur, &got)) 634 break; 635 } 636 637 /* clear tag if cow fork is emptied */ 638 if (!ifp->if_bytes) 639 xfs_inode_clear_cowblocks_tag(ip); 640 641 return error; 642 } 643 644 /* 645 * Cancel CoW reservations for some byte range of an inode. 646 * 647 * If cancel_real is true this function cancels all COW fork extents for the 648 * inode; if cancel_real is false, real extents are not cleared. 649 */ 650 int 651 xfs_reflink_cancel_cow_range( 652 struct xfs_inode *ip, 653 xfs_off_t offset, 654 xfs_off_t count, 655 bool cancel_real) 656 { 657 struct xfs_trans *tp; 658 xfs_fileoff_t offset_fsb; 659 xfs_fileoff_t end_fsb; 660 int error; 661 662 trace_xfs_reflink_cancel_cow_range(ip, offset, count); 663 ASSERT(xfs_is_reflink_inode(ip)); 664 665 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 666 if (count == NULLFILEOFF) 667 end_fsb = NULLFILEOFF; 668 else 669 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 670 671 /* Start a rolling transaction to remove the mappings */ 672 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 673 0, 0, XFS_TRANS_NOFS, &tp); 674 if (error) 675 goto out; 676 677 xfs_ilock(ip, XFS_ILOCK_EXCL); 678 xfs_trans_ijoin(tp, ip, 0); 679 680 /* Scrape out the old CoW reservations */ 681 error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb, 682 cancel_real); 683 if (error) 684 goto out_cancel; 685 686 error = xfs_trans_commit(tp); 687 688 xfs_iunlock(ip, XFS_ILOCK_EXCL); 689 return error; 690 691 out_cancel: 692 xfs_trans_cancel(tp); 693 xfs_iunlock(ip, XFS_ILOCK_EXCL); 694 out: 695 trace_xfs_reflink_cancel_cow_range_error(ip, error, _RET_IP_); 696 return error; 697 } 698 699 /* 700 * Remap parts of a file's data fork after a successful CoW. 701 */ 702 int 703 xfs_reflink_end_cow( 704 struct xfs_inode *ip, 705 xfs_off_t offset, 706 xfs_off_t count) 707 { 708 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 709 struct xfs_bmbt_irec got, del; 710 struct xfs_trans *tp; 711 xfs_fileoff_t offset_fsb; 712 xfs_fileoff_t end_fsb; 713 xfs_fsblock_t firstfsb; 714 struct xfs_defer_ops dfops; 715 int error; 716 unsigned int resblks; 717 xfs_filblks_t rlen; 718 struct xfs_iext_cursor icur; 719 720 trace_xfs_reflink_end_cow(ip, offset, count); 721 722 /* No COW extents? That's easy! */ 723 if (ifp->if_bytes == 0) 724 return 0; 725 726 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 727 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 728 729 /* 730 * Start a rolling transaction to switch the mappings. We're 731 * unlikely ever to have to remap 16T worth of single-block 732 * extents, so just cap the worst case extent count to 2^32-1. 733 * Stick a warning in just in case, and avoid 64-bit division. 734 */ 735 BUILD_BUG_ON(MAX_RW_COUNT > UINT_MAX); 736 if (end_fsb - offset_fsb > UINT_MAX) { 737 error = -EFSCORRUPTED; 738 xfs_force_shutdown(ip->i_mount, SHUTDOWN_CORRUPT_INCORE); 739 ASSERT(0); 740 goto out; 741 } 742 resblks = XFS_NEXTENTADD_SPACE_RES(ip->i_mount, 743 (unsigned int)(end_fsb - offset_fsb), 744 XFS_DATA_FORK); 745 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 746 resblks, 0, XFS_TRANS_RESERVE | XFS_TRANS_NOFS, &tp); 747 if (error) 748 goto out; 749 750 xfs_ilock(ip, XFS_ILOCK_EXCL); 751 xfs_trans_ijoin(tp, ip, 0); 752 753 /* 754 * In case of racing, overlapping AIO writes no COW extents might be 755 * left by the time I/O completes for the loser of the race. In that 756 * case we are done. 757 */ 758 if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) 759 goto out_cancel; 760 761 /* Walk backwards until we're out of the I/O range... */ 762 while (got.br_startoff + got.br_blockcount > offset_fsb) { 763 del = got; 764 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 765 766 /* Extent delete may have bumped ext forward */ 767 if (!del.br_blockcount) 768 goto prev_extent; 769 770 ASSERT(!isnullstartblock(got.br_startblock)); 771 772 /* 773 * Don't remap unwritten extents; these are 774 * speculatively preallocated CoW extents that have been 775 * allocated but have not yet been involved in a write. 776 */ 777 if (got.br_state == XFS_EXT_UNWRITTEN) 778 goto prev_extent; 779 780 /* Unmap the old blocks in the data fork. */ 781 xfs_defer_init(&dfops, &firstfsb); 782 rlen = del.br_blockcount; 783 error = __xfs_bunmapi(tp, ip, del.br_startoff, &rlen, 0, 1, 784 &firstfsb, &dfops); 785 if (error) 786 goto out_defer; 787 788 /* Trim the extent to whatever got unmapped. */ 789 if (rlen) { 790 xfs_trim_extent(&del, del.br_startoff + rlen, 791 del.br_blockcount - rlen); 792 } 793 trace_xfs_reflink_cow_remap(ip, &del); 794 795 /* Free the CoW orphan record. */ 796 error = xfs_refcount_free_cow_extent(tp->t_mountp, &dfops, 797 del.br_startblock, del.br_blockcount); 798 if (error) 799 goto out_defer; 800 801 /* Map the new blocks into the data fork. */ 802 error = xfs_bmap_map_extent(tp->t_mountp, &dfops, ip, &del); 803 if (error) 804 goto out_defer; 805 806 /* Charge this new data fork mapping to the on-disk quota. */ 807 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_DELBCOUNT, 808 (long)del.br_blockcount); 809 810 /* Remove the mapping from the CoW fork. */ 811 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 812 813 xfs_defer_ijoin(&dfops, ip); 814 error = xfs_defer_finish(&tp, &dfops); 815 if (error) 816 goto out_defer; 817 if (!xfs_iext_get_extent(ifp, &icur, &got)) 818 break; 819 continue; 820 prev_extent: 821 if (!xfs_iext_prev_extent(ifp, &icur, &got)) 822 break; 823 } 824 825 error = xfs_trans_commit(tp); 826 xfs_iunlock(ip, XFS_ILOCK_EXCL); 827 if (error) 828 goto out; 829 return 0; 830 831 out_defer: 832 xfs_defer_cancel(&dfops); 833 out_cancel: 834 xfs_trans_cancel(tp); 835 xfs_iunlock(ip, XFS_ILOCK_EXCL); 836 out: 837 trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_); 838 return error; 839 } 840 841 /* 842 * Free leftover CoW reservations that didn't get cleaned out. 843 */ 844 int 845 xfs_reflink_recover_cow( 846 struct xfs_mount *mp) 847 { 848 xfs_agnumber_t agno; 849 int error = 0; 850 851 if (!xfs_sb_version_hasreflink(&mp->m_sb)) 852 return 0; 853 854 for (agno = 0; agno < mp->m_sb.sb_agcount; agno++) { 855 error = xfs_refcount_recover_cow_leftovers(mp, agno); 856 if (error) 857 break; 858 } 859 860 return error; 861 } 862 863 /* 864 * Reflinking (Block) Ranges of Two Files Together 865 * 866 * First, ensure that the reflink flag is set on both inodes. The flag is an 867 * optimization to avoid unnecessary refcount btree lookups in the write path. 868 * 869 * Now we can iteratively remap the range of extents (and holes) in src to the 870 * corresponding ranges in dest. Let drange and srange denote the ranges of 871 * logical blocks in dest and src touched by the reflink operation. 872 * 873 * While the length of drange is greater than zero, 874 * - Read src's bmbt at the start of srange ("imap") 875 * - If imap doesn't exist, make imap appear to start at the end of srange 876 * with zero length. 877 * - If imap starts before srange, advance imap to start at srange. 878 * - If imap goes beyond srange, truncate imap to end at the end of srange. 879 * - Punch (imap start - srange start + imap len) blocks from dest at 880 * offset (drange start). 881 * - If imap points to a real range of pblks, 882 * > Increase the refcount of the imap's pblks 883 * > Map imap's pblks into dest at the offset 884 * (drange start + imap start - srange start) 885 * - Advance drange and srange by (imap start - srange start + imap len) 886 * 887 * Finally, if the reflink made dest longer, update both the in-core and 888 * on-disk file sizes. 889 * 890 * ASCII Art Demonstration: 891 * 892 * Let's say we want to reflink this source file: 893 * 894 * ----SSSSSSS-SSSSS----SSSSSS (src file) 895 * <--------------------> 896 * 897 * into this destination file: 898 * 899 * --DDDDDDDDDDDDDDDDDDD--DDD (dest file) 900 * <--------------------> 901 * '-' means a hole, and 'S' and 'D' are written blocks in the src and dest. 902 * Observe that the range has different logical offsets in either file. 903 * 904 * Consider that the first extent in the source file doesn't line up with our 905 * reflink range. Unmapping and remapping are separate operations, so we can 906 * unmap more blocks from the destination file than we remap. 907 * 908 * ----SSSSSSS-SSSSS----SSSSSS 909 * <-------> 910 * --DDDDD---------DDDDD--DDD 911 * <-------> 912 * 913 * Now remap the source extent into the destination file: 914 * 915 * ----SSSSSSS-SSSSS----SSSSSS 916 * <-------> 917 * --DDDDD--SSSSSSSDDDDD--DDD 918 * <-------> 919 * 920 * Do likewise with the second hole and extent in our range. Holes in the 921 * unmap range don't affect our operation. 922 * 923 * ----SSSSSSS-SSSSS----SSSSSS 924 * <----> 925 * --DDDDD--SSSSSSS-SSSSS-DDD 926 * <----> 927 * 928 * Finally, unmap and remap part of the third extent. This will increase the 929 * size of the destination file. 930 * 931 * ----SSSSSSS-SSSSS----SSSSSS 932 * <-----> 933 * --DDDDD--SSSSSSS-SSSSS----SSS 934 * <-----> 935 * 936 * Once we update the destination file's i_size, we're done. 937 */ 938 939 /* 940 * Ensure the reflink bit is set in both inodes. 941 */ 942 STATIC int 943 xfs_reflink_set_inode_flag( 944 struct xfs_inode *src, 945 struct xfs_inode *dest) 946 { 947 struct xfs_mount *mp = src->i_mount; 948 int error; 949 struct xfs_trans *tp; 950 951 if (xfs_is_reflink_inode(src) && xfs_is_reflink_inode(dest)) 952 return 0; 953 954 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 955 if (error) 956 goto out_error; 957 958 /* Lock both files against IO */ 959 if (src->i_ino == dest->i_ino) 960 xfs_ilock(src, XFS_ILOCK_EXCL); 961 else 962 xfs_lock_two_inodes(src, XFS_ILOCK_EXCL, dest, XFS_ILOCK_EXCL); 963 964 if (!xfs_is_reflink_inode(src)) { 965 trace_xfs_reflink_set_inode_flag(src); 966 xfs_trans_ijoin(tp, src, XFS_ILOCK_EXCL); 967 src->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 968 xfs_trans_log_inode(tp, src, XFS_ILOG_CORE); 969 xfs_ifork_init_cow(src); 970 } else 971 xfs_iunlock(src, XFS_ILOCK_EXCL); 972 973 if (src->i_ino == dest->i_ino) 974 goto commit_flags; 975 976 if (!xfs_is_reflink_inode(dest)) { 977 trace_xfs_reflink_set_inode_flag(dest); 978 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 979 dest->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 980 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 981 xfs_ifork_init_cow(dest); 982 } else 983 xfs_iunlock(dest, XFS_ILOCK_EXCL); 984 985 commit_flags: 986 error = xfs_trans_commit(tp); 987 if (error) 988 goto out_error; 989 return error; 990 991 out_error: 992 trace_xfs_reflink_set_inode_flag_error(dest, error, _RET_IP_); 993 return error; 994 } 995 996 /* 997 * Update destination inode size & cowextsize hint, if necessary. 998 */ 999 STATIC int 1000 xfs_reflink_update_dest( 1001 struct xfs_inode *dest, 1002 xfs_off_t newlen, 1003 xfs_extlen_t cowextsize, 1004 bool is_dedupe) 1005 { 1006 struct xfs_mount *mp = dest->i_mount; 1007 struct xfs_trans *tp; 1008 int error; 1009 1010 if (is_dedupe && newlen <= i_size_read(VFS_I(dest)) && cowextsize == 0) 1011 return 0; 1012 1013 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 1014 if (error) 1015 goto out_error; 1016 1017 xfs_ilock(dest, XFS_ILOCK_EXCL); 1018 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 1019 1020 if (newlen > i_size_read(VFS_I(dest))) { 1021 trace_xfs_reflink_update_inode_size(dest, newlen); 1022 i_size_write(VFS_I(dest), newlen); 1023 dest->i_d.di_size = newlen; 1024 } 1025 1026 if (cowextsize) { 1027 dest->i_d.di_cowextsize = cowextsize; 1028 dest->i_d.di_flags2 |= XFS_DIFLAG2_COWEXTSIZE; 1029 } 1030 1031 if (!is_dedupe) { 1032 xfs_trans_ichgtime(tp, dest, 1033 XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); 1034 } 1035 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 1036 1037 error = xfs_trans_commit(tp); 1038 if (error) 1039 goto out_error; 1040 return error; 1041 1042 out_error: 1043 trace_xfs_reflink_update_inode_size_error(dest, error, _RET_IP_); 1044 return error; 1045 } 1046 1047 /* 1048 * Do we have enough reserve in this AG to handle a reflink? The refcount 1049 * btree already reserved all the space it needs, but the rmap btree can grow 1050 * infinitely, so we won't allow more reflinks when the AG is down to the 1051 * btree reserves. 1052 */ 1053 static int 1054 xfs_reflink_ag_has_free_space( 1055 struct xfs_mount *mp, 1056 xfs_agnumber_t agno) 1057 { 1058 struct xfs_perag *pag; 1059 int error = 0; 1060 1061 if (!xfs_sb_version_hasrmapbt(&mp->m_sb)) 1062 return 0; 1063 1064 pag = xfs_perag_get(mp, agno); 1065 if (xfs_ag_resv_critical(pag, XFS_AG_RESV_RMAPBT) || 1066 xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA)) 1067 error = -ENOSPC; 1068 xfs_perag_put(pag); 1069 return error; 1070 } 1071 1072 /* 1073 * Unmap a range of blocks from a file, then map other blocks into the hole. 1074 * The range to unmap is (destoff : destoff + srcioff + irec->br_blockcount). 1075 * The extent irec is mapped into dest at irec->br_startoff. 1076 */ 1077 STATIC int 1078 xfs_reflink_remap_extent( 1079 struct xfs_inode *ip, 1080 struct xfs_bmbt_irec *irec, 1081 xfs_fileoff_t destoff, 1082 xfs_off_t new_isize) 1083 { 1084 struct xfs_mount *mp = ip->i_mount; 1085 bool real_extent = xfs_bmap_is_real_extent(irec); 1086 struct xfs_trans *tp; 1087 xfs_fsblock_t firstfsb; 1088 unsigned int resblks; 1089 struct xfs_defer_ops dfops; 1090 struct xfs_bmbt_irec uirec; 1091 xfs_filblks_t rlen; 1092 xfs_filblks_t unmap_len; 1093 xfs_off_t newlen; 1094 int error; 1095 1096 unmap_len = irec->br_startoff + irec->br_blockcount - destoff; 1097 trace_xfs_reflink_punch_range(ip, destoff, unmap_len); 1098 1099 /* No reflinking if we're low on space */ 1100 if (real_extent) { 1101 error = xfs_reflink_ag_has_free_space(mp, 1102 XFS_FSB_TO_AGNO(mp, irec->br_startblock)); 1103 if (error) 1104 goto out; 1105 } 1106 1107 /* Start a rolling transaction to switch the mappings */ 1108 resblks = XFS_EXTENTADD_SPACE_RES(ip->i_mount, XFS_DATA_FORK); 1109 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); 1110 if (error) 1111 goto out; 1112 1113 xfs_ilock(ip, XFS_ILOCK_EXCL); 1114 xfs_trans_ijoin(tp, ip, 0); 1115 1116 /* If we're not just clearing space, then do we have enough quota? */ 1117 if (real_extent) { 1118 error = xfs_trans_reserve_quota_nblks(tp, ip, 1119 irec->br_blockcount, 0, XFS_QMOPT_RES_REGBLKS); 1120 if (error) 1121 goto out_cancel; 1122 } 1123 1124 trace_xfs_reflink_remap(ip, irec->br_startoff, 1125 irec->br_blockcount, irec->br_startblock); 1126 1127 /* Unmap the old blocks in the data fork. */ 1128 rlen = unmap_len; 1129 while (rlen) { 1130 xfs_defer_init(&dfops, &firstfsb); 1131 error = __xfs_bunmapi(tp, ip, destoff, &rlen, 0, 1, 1132 &firstfsb, &dfops); 1133 if (error) 1134 goto out_defer; 1135 1136 /* 1137 * Trim the extent to whatever got unmapped. 1138 * Remember, bunmapi works backwards. 1139 */ 1140 uirec.br_startblock = irec->br_startblock + rlen; 1141 uirec.br_startoff = irec->br_startoff + rlen; 1142 uirec.br_blockcount = unmap_len - rlen; 1143 unmap_len = rlen; 1144 1145 /* If this isn't a real mapping, we're done. */ 1146 if (!real_extent || uirec.br_blockcount == 0) 1147 goto next_extent; 1148 1149 trace_xfs_reflink_remap(ip, uirec.br_startoff, 1150 uirec.br_blockcount, uirec.br_startblock); 1151 1152 /* Update the refcount tree */ 1153 error = xfs_refcount_increase_extent(mp, &dfops, &uirec); 1154 if (error) 1155 goto out_defer; 1156 1157 /* Map the new blocks into the data fork. */ 1158 error = xfs_bmap_map_extent(mp, &dfops, ip, &uirec); 1159 if (error) 1160 goto out_defer; 1161 1162 /* Update quota accounting. */ 1163 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1164 uirec.br_blockcount); 1165 1166 /* Update dest isize if needed. */ 1167 newlen = XFS_FSB_TO_B(mp, 1168 uirec.br_startoff + uirec.br_blockcount); 1169 newlen = min_t(xfs_off_t, newlen, new_isize); 1170 if (newlen > i_size_read(VFS_I(ip))) { 1171 trace_xfs_reflink_update_inode_size(ip, newlen); 1172 i_size_write(VFS_I(ip), newlen); 1173 ip->i_d.di_size = newlen; 1174 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1175 } 1176 1177 next_extent: 1178 /* Process all the deferred stuff. */ 1179 xfs_defer_ijoin(&dfops, ip); 1180 error = xfs_defer_finish(&tp, &dfops); 1181 if (error) 1182 goto out_defer; 1183 } 1184 1185 error = xfs_trans_commit(tp); 1186 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1187 if (error) 1188 goto out; 1189 return 0; 1190 1191 out_defer: 1192 xfs_defer_cancel(&dfops); 1193 out_cancel: 1194 xfs_trans_cancel(tp); 1195 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1196 out: 1197 trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_); 1198 return error; 1199 } 1200 1201 /* 1202 * Iteratively remap one file's extents (and holes) to another's. 1203 */ 1204 STATIC int 1205 xfs_reflink_remap_blocks( 1206 struct xfs_inode *src, 1207 xfs_fileoff_t srcoff, 1208 struct xfs_inode *dest, 1209 xfs_fileoff_t destoff, 1210 xfs_filblks_t len, 1211 xfs_off_t new_isize) 1212 { 1213 struct xfs_bmbt_irec imap; 1214 int nimaps; 1215 int error = 0; 1216 xfs_filblks_t range_len; 1217 1218 /* drange = (destoff, destoff + len); srange = (srcoff, srcoff + len) */ 1219 while (len) { 1220 uint lock_mode; 1221 1222 trace_xfs_reflink_remap_blocks_loop(src, srcoff, len, 1223 dest, destoff); 1224 1225 /* Read extent from the source file */ 1226 nimaps = 1; 1227 lock_mode = xfs_ilock_data_map_shared(src); 1228 error = xfs_bmapi_read(src, srcoff, len, &imap, &nimaps, 0); 1229 xfs_iunlock(src, lock_mode); 1230 if (error) 1231 goto err; 1232 ASSERT(nimaps == 1); 1233 1234 trace_xfs_reflink_remap_imap(src, srcoff, len, XFS_IO_OVERWRITE, 1235 &imap); 1236 1237 /* Translate imap into the destination file. */ 1238 range_len = imap.br_startoff + imap.br_blockcount - srcoff; 1239 imap.br_startoff += destoff - srcoff; 1240 1241 /* Clear dest from destoff to the end of imap and map it in. */ 1242 error = xfs_reflink_remap_extent(dest, &imap, destoff, 1243 new_isize); 1244 if (error) 1245 goto err; 1246 1247 if (fatal_signal_pending(current)) { 1248 error = -EINTR; 1249 goto err; 1250 } 1251 1252 /* Advance drange/srange */ 1253 srcoff += range_len; 1254 destoff += range_len; 1255 len -= range_len; 1256 } 1257 1258 return 0; 1259 1260 err: 1261 trace_xfs_reflink_remap_blocks_error(dest, error, _RET_IP_); 1262 return error; 1263 } 1264 1265 /* 1266 * Grab the exclusive iolock for a data copy from src to dest, making 1267 * sure to abide vfs locking order (lowest pointer value goes first) and 1268 * breaking the pnfs layout leases on dest before proceeding. The loop 1269 * is needed because we cannot call the blocking break_layout() with the 1270 * src iolock held, and therefore have to back out both locks. 1271 */ 1272 static int 1273 xfs_iolock_two_inodes_and_break_layout( 1274 struct inode *src, 1275 struct inode *dest) 1276 { 1277 int error; 1278 1279 retry: 1280 if (src < dest) { 1281 inode_lock_shared(src); 1282 inode_lock_nested(dest, I_MUTEX_NONDIR2); 1283 } else { 1284 /* src >= dest */ 1285 inode_lock(dest); 1286 } 1287 1288 error = break_layout(dest, false); 1289 if (error == -EWOULDBLOCK) { 1290 inode_unlock(dest); 1291 if (src < dest) 1292 inode_unlock_shared(src); 1293 error = break_layout(dest, true); 1294 if (error) 1295 return error; 1296 goto retry; 1297 } 1298 if (error) { 1299 inode_unlock(dest); 1300 if (src < dest) 1301 inode_unlock_shared(src); 1302 return error; 1303 } 1304 if (src > dest) 1305 inode_lock_shared_nested(src, I_MUTEX_NONDIR2); 1306 return 0; 1307 } 1308 1309 /* 1310 * Link a range of blocks from one file to another. 1311 */ 1312 int 1313 xfs_reflink_remap_range( 1314 struct file *file_in, 1315 loff_t pos_in, 1316 struct file *file_out, 1317 loff_t pos_out, 1318 u64 len, 1319 bool is_dedupe) 1320 { 1321 struct inode *inode_in = file_inode(file_in); 1322 struct xfs_inode *src = XFS_I(inode_in); 1323 struct inode *inode_out = file_inode(file_out); 1324 struct xfs_inode *dest = XFS_I(inode_out); 1325 struct xfs_mount *mp = src->i_mount; 1326 bool same_inode = (inode_in == inode_out); 1327 xfs_fileoff_t sfsbno, dfsbno; 1328 xfs_filblks_t fsblen; 1329 xfs_extlen_t cowextsize; 1330 ssize_t ret; 1331 1332 if (!xfs_sb_version_hasreflink(&mp->m_sb)) 1333 return -EOPNOTSUPP; 1334 1335 if (XFS_FORCED_SHUTDOWN(mp)) 1336 return -EIO; 1337 1338 /* Lock both files against IO */ 1339 ret = xfs_iolock_two_inodes_and_break_layout(inode_in, inode_out); 1340 if (ret) 1341 return ret; 1342 if (same_inode) 1343 xfs_ilock(src, XFS_MMAPLOCK_EXCL); 1344 else 1345 xfs_lock_two_inodes(src, XFS_MMAPLOCK_SHARED, dest, 1346 XFS_MMAPLOCK_EXCL); 1347 1348 /* Check file eligibility and prepare for block sharing. */ 1349 ret = -EINVAL; 1350 /* Don't reflink realtime inodes */ 1351 if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest)) 1352 goto out_unlock; 1353 1354 /* Don't share DAX file data for now. */ 1355 if (IS_DAX(inode_in) || IS_DAX(inode_out)) 1356 goto out_unlock; 1357 1358 ret = vfs_clone_file_prep_inodes(inode_in, pos_in, inode_out, pos_out, 1359 &len, is_dedupe); 1360 if (ret <= 0) 1361 goto out_unlock; 1362 1363 /* Attach dquots to dest inode before changing block map */ 1364 ret = xfs_qm_dqattach(dest); 1365 if (ret) 1366 goto out_unlock; 1367 1368 trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out); 1369 1370 /* 1371 * Clear out post-eof preallocations because we don't have page cache 1372 * backing the delayed allocations and they'll never get freed on 1373 * their own. 1374 */ 1375 if (xfs_can_free_eofblocks(dest, true)) { 1376 ret = xfs_free_eofblocks(dest); 1377 if (ret) 1378 goto out_unlock; 1379 } 1380 1381 /* Set flags and remap blocks. */ 1382 ret = xfs_reflink_set_inode_flag(src, dest); 1383 if (ret) 1384 goto out_unlock; 1385 1386 dfsbno = XFS_B_TO_FSBT(mp, pos_out); 1387 sfsbno = XFS_B_TO_FSBT(mp, pos_in); 1388 fsblen = XFS_B_TO_FSB(mp, len); 1389 ret = xfs_reflink_remap_blocks(src, sfsbno, dest, dfsbno, fsblen, 1390 pos_out + len); 1391 if (ret) 1392 goto out_unlock; 1393 1394 /* Zap any page cache for the destination file's range. */ 1395 truncate_inode_pages_range(&inode_out->i_data, pos_out, 1396 PAGE_ALIGN(pos_out + len) - 1); 1397 1398 /* 1399 * Carry the cowextsize hint from src to dest if we're sharing the 1400 * entire source file to the entire destination file, the source file 1401 * has a cowextsize hint, and the destination file does not. 1402 */ 1403 cowextsize = 0; 1404 if (pos_in == 0 && len == i_size_read(inode_in) && 1405 (src->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE) && 1406 pos_out == 0 && len >= i_size_read(inode_out) && 1407 !(dest->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE)) 1408 cowextsize = src->i_d.di_cowextsize; 1409 1410 ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize, 1411 is_dedupe); 1412 1413 out_unlock: 1414 xfs_iunlock(dest, XFS_MMAPLOCK_EXCL); 1415 if (!same_inode) 1416 xfs_iunlock(src, XFS_MMAPLOCK_SHARED); 1417 inode_unlock(inode_out); 1418 if (!same_inode) 1419 inode_unlock_shared(inode_in); 1420 if (ret) 1421 trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_); 1422 return ret; 1423 } 1424 1425 /* 1426 * The user wants to preemptively CoW all shared blocks in this file, 1427 * which enables us to turn off the reflink flag. Iterate all 1428 * extents which are not prealloc/delalloc to see which ranges are 1429 * mentioned in the refcount tree, then read those blocks into the 1430 * pagecache, dirty them, fsync them back out, and then we can update 1431 * the inode flag. What happens if we run out of memory? :) 1432 */ 1433 STATIC int 1434 xfs_reflink_dirty_extents( 1435 struct xfs_inode *ip, 1436 xfs_fileoff_t fbno, 1437 xfs_filblks_t end, 1438 xfs_off_t isize) 1439 { 1440 struct xfs_mount *mp = ip->i_mount; 1441 xfs_agnumber_t agno; 1442 xfs_agblock_t agbno; 1443 xfs_extlen_t aglen; 1444 xfs_agblock_t rbno; 1445 xfs_extlen_t rlen; 1446 xfs_off_t fpos; 1447 xfs_off_t flen; 1448 struct xfs_bmbt_irec map[2]; 1449 int nmaps; 1450 int error = 0; 1451 1452 while (end - fbno > 0) { 1453 nmaps = 1; 1454 /* 1455 * Look for extents in the file. Skip holes, delalloc, or 1456 * unwritten extents; they can't be reflinked. 1457 */ 1458 error = xfs_bmapi_read(ip, fbno, end - fbno, map, &nmaps, 0); 1459 if (error) 1460 goto out; 1461 if (nmaps == 0) 1462 break; 1463 if (!xfs_bmap_is_real_extent(&map[0])) 1464 goto next; 1465 1466 map[1] = map[0]; 1467 while (map[1].br_blockcount) { 1468 agno = XFS_FSB_TO_AGNO(mp, map[1].br_startblock); 1469 agbno = XFS_FSB_TO_AGBNO(mp, map[1].br_startblock); 1470 aglen = map[1].br_blockcount; 1471 1472 error = xfs_reflink_find_shared(mp, NULL, agno, agbno, 1473 aglen, &rbno, &rlen, true); 1474 if (error) 1475 goto out; 1476 if (rbno == NULLAGBLOCK) 1477 break; 1478 1479 /* Dirty the pages */ 1480 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1481 fpos = XFS_FSB_TO_B(mp, map[1].br_startoff + 1482 (rbno - agbno)); 1483 flen = XFS_FSB_TO_B(mp, rlen); 1484 if (fpos + flen > isize) 1485 flen = isize - fpos; 1486 error = iomap_file_dirty(VFS_I(ip), fpos, flen, 1487 &xfs_iomap_ops); 1488 xfs_ilock(ip, XFS_ILOCK_EXCL); 1489 if (error) 1490 goto out; 1491 1492 map[1].br_blockcount -= (rbno - agbno + rlen); 1493 map[1].br_startoff += (rbno - agbno + rlen); 1494 map[1].br_startblock += (rbno - agbno + rlen); 1495 } 1496 1497 next: 1498 fbno = map[0].br_startoff + map[0].br_blockcount; 1499 } 1500 out: 1501 return error; 1502 } 1503 1504 /* Does this inode need the reflink flag? */ 1505 int 1506 xfs_reflink_inode_has_shared_extents( 1507 struct xfs_trans *tp, 1508 struct xfs_inode *ip, 1509 bool *has_shared) 1510 { 1511 struct xfs_bmbt_irec got; 1512 struct xfs_mount *mp = ip->i_mount; 1513 struct xfs_ifork *ifp; 1514 xfs_agnumber_t agno; 1515 xfs_agblock_t agbno; 1516 xfs_extlen_t aglen; 1517 xfs_agblock_t rbno; 1518 xfs_extlen_t rlen; 1519 struct xfs_iext_cursor icur; 1520 bool found; 1521 int error; 1522 1523 ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); 1524 if (!(ifp->if_flags & XFS_IFEXTENTS)) { 1525 error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); 1526 if (error) 1527 return error; 1528 } 1529 1530 *has_shared = false; 1531 found = xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got); 1532 while (found) { 1533 if (isnullstartblock(got.br_startblock) || 1534 got.br_state != XFS_EXT_NORM) 1535 goto next; 1536 agno = XFS_FSB_TO_AGNO(mp, got.br_startblock); 1537 agbno = XFS_FSB_TO_AGBNO(mp, got.br_startblock); 1538 aglen = got.br_blockcount; 1539 1540 error = xfs_reflink_find_shared(mp, tp, agno, agbno, aglen, 1541 &rbno, &rlen, false); 1542 if (error) 1543 return error; 1544 /* Is there still a shared block here? */ 1545 if (rbno != NULLAGBLOCK) { 1546 *has_shared = true; 1547 return 0; 1548 } 1549 next: 1550 found = xfs_iext_next_extent(ifp, &icur, &got); 1551 } 1552 1553 return 0; 1554 } 1555 1556 /* 1557 * Clear the inode reflink flag if there are no shared extents. 1558 * 1559 * The caller is responsible for joining the inode to the transaction passed in. 1560 * The inode will be joined to the transaction that is returned to the caller. 1561 */ 1562 int 1563 xfs_reflink_clear_inode_flag( 1564 struct xfs_inode *ip, 1565 struct xfs_trans **tpp) 1566 { 1567 bool needs_flag; 1568 int error = 0; 1569 1570 ASSERT(xfs_is_reflink_inode(ip)); 1571 1572 error = xfs_reflink_inode_has_shared_extents(*tpp, ip, &needs_flag); 1573 if (error || needs_flag) 1574 return error; 1575 1576 /* 1577 * We didn't find any shared blocks so turn off the reflink flag. 1578 * First, get rid of any leftover CoW mappings. 1579 */ 1580 error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, NULLFILEOFF, true); 1581 if (error) 1582 return error; 1583 1584 /* Clear the inode flag. */ 1585 trace_xfs_reflink_unset_inode_flag(ip); 1586 ip->i_d.di_flags2 &= ~XFS_DIFLAG2_REFLINK; 1587 xfs_inode_clear_cowblocks_tag(ip); 1588 xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE); 1589 1590 return error; 1591 } 1592 1593 /* 1594 * Clear the inode reflink flag if there are no shared extents and the size 1595 * hasn't changed. 1596 */ 1597 STATIC int 1598 xfs_reflink_try_clear_inode_flag( 1599 struct xfs_inode *ip) 1600 { 1601 struct xfs_mount *mp = ip->i_mount; 1602 struct xfs_trans *tp; 1603 int error = 0; 1604 1605 /* Start a rolling transaction to remove the mappings */ 1606 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); 1607 if (error) 1608 return error; 1609 1610 xfs_ilock(ip, XFS_ILOCK_EXCL); 1611 xfs_trans_ijoin(tp, ip, 0); 1612 1613 error = xfs_reflink_clear_inode_flag(ip, &tp); 1614 if (error) 1615 goto cancel; 1616 1617 error = xfs_trans_commit(tp); 1618 if (error) 1619 goto out; 1620 1621 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1622 return 0; 1623 cancel: 1624 xfs_trans_cancel(tp); 1625 out: 1626 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1627 return error; 1628 } 1629 1630 /* 1631 * Pre-COW all shared blocks within a given byte range of a file and turn off 1632 * the reflink flag if we unshare all of the file's blocks. 1633 */ 1634 int 1635 xfs_reflink_unshare( 1636 struct xfs_inode *ip, 1637 xfs_off_t offset, 1638 xfs_off_t len) 1639 { 1640 struct xfs_mount *mp = ip->i_mount; 1641 xfs_fileoff_t fbno; 1642 xfs_filblks_t end; 1643 xfs_off_t isize; 1644 int error; 1645 1646 if (!xfs_is_reflink_inode(ip)) 1647 return 0; 1648 1649 trace_xfs_reflink_unshare(ip, offset, len); 1650 1651 inode_dio_wait(VFS_I(ip)); 1652 1653 /* Try to CoW the selected ranges */ 1654 xfs_ilock(ip, XFS_ILOCK_EXCL); 1655 fbno = XFS_B_TO_FSBT(mp, offset); 1656 isize = i_size_read(VFS_I(ip)); 1657 end = XFS_B_TO_FSB(mp, offset + len); 1658 error = xfs_reflink_dirty_extents(ip, fbno, end, isize); 1659 if (error) 1660 goto out_unlock; 1661 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1662 1663 /* Wait for the IO to finish */ 1664 error = filemap_write_and_wait(VFS_I(ip)->i_mapping); 1665 if (error) 1666 goto out; 1667 1668 /* Turn off the reflink flag if possible. */ 1669 error = xfs_reflink_try_clear_inode_flag(ip); 1670 if (error) 1671 goto out; 1672 1673 return 0; 1674 1675 out_unlock: 1676 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1677 out: 1678 trace_xfs_reflink_unshare_error(ip, error, _RET_IP_); 1679 return error; 1680 } 1681