1 /* 2 * Copyright (C) 2016 Oracle. All Rights Reserved. 3 * 4 * Author: Darrick J. Wong <darrick.wong@oracle.com> 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 2 9 * of the License, or (at your option) any later version. 10 * 11 * This program is distributed in the hope that it would be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. 19 */ 20 #include "xfs.h" 21 #include "xfs_fs.h" 22 #include "xfs_shared.h" 23 #include "xfs_format.h" 24 #include "xfs_log_format.h" 25 #include "xfs_trans_resv.h" 26 #include "xfs_mount.h" 27 #include "xfs_defer.h" 28 #include "xfs_da_format.h" 29 #include "xfs_da_btree.h" 30 #include "xfs_inode.h" 31 #include "xfs_trans.h" 32 #include "xfs_inode_item.h" 33 #include "xfs_bmap.h" 34 #include "xfs_bmap_util.h" 35 #include "xfs_error.h" 36 #include "xfs_dir2.h" 37 #include "xfs_dir2_priv.h" 38 #include "xfs_ioctl.h" 39 #include "xfs_trace.h" 40 #include "xfs_log.h" 41 #include "xfs_icache.h" 42 #include "xfs_pnfs.h" 43 #include "xfs_btree.h" 44 #include "xfs_refcount_btree.h" 45 #include "xfs_refcount.h" 46 #include "xfs_bmap_btree.h" 47 #include "xfs_trans_space.h" 48 #include "xfs_bit.h" 49 #include "xfs_alloc.h" 50 #include "xfs_quota_defs.h" 51 #include "xfs_quota.h" 52 #include "xfs_reflink.h" 53 #include "xfs_iomap.h" 54 #include "xfs_rmap_btree.h" 55 #include "xfs_sb.h" 56 #include "xfs_ag_resv.h" 57 58 /* 59 * Copy on Write of Shared Blocks 60 * 61 * XFS must preserve "the usual" file semantics even when two files share 62 * the same physical blocks. This means that a write to one file must not 63 * alter the blocks in a different file; the way that we'll do that is 64 * through the use of a copy-on-write mechanism. At a high level, that 65 * means that when we want to write to a shared block, we allocate a new 66 * block, write the data to the new block, and if that succeeds we map the 67 * new block into the file. 68 * 69 * XFS provides a "delayed allocation" mechanism that defers the allocation 70 * of disk blocks to dirty-but-not-yet-mapped file blocks as long as 71 * possible. This reduces fragmentation by enabling the filesystem to ask 72 * for bigger chunks less often, which is exactly what we want for CoW. 73 * 74 * The delalloc mechanism begins when the kernel wants to make a block 75 * writable (write_begin or page_mkwrite). If the offset is not mapped, we 76 * create a delalloc mapping, which is a regular in-core extent, but without 77 * a real startblock. (For delalloc mappings, the startblock encodes both 78 * a flag that this is a delalloc mapping, and a worst-case estimate of how 79 * many blocks might be required to put the mapping into the BMBT.) delalloc 80 * mappings are a reservation against the free space in the filesystem; 81 * adjacent mappings can also be combined into fewer larger mappings. 82 * 83 * As an optimization, the CoW extent size hint (cowextsz) creates 84 * outsized aligned delalloc reservations in the hope of landing out of 85 * order nearby CoW writes in a single extent on disk, thereby reducing 86 * fragmentation and improving future performance. 87 * 88 * D: --RRRRRRSSSRRRRRRRR--- (data fork) 89 * C: ------DDDDDDD--------- (CoW fork) 90 * 91 * When dirty pages are being written out (typically in writepage), the 92 * delalloc reservations are converted into unwritten mappings by 93 * allocating blocks and replacing the delalloc mapping with real ones. 94 * A delalloc mapping can be replaced by several unwritten ones if the 95 * free space is fragmented. 96 * 97 * D: --RRRRRRSSSRRRRRRRR--- 98 * C: ------UUUUUUU--------- 99 * 100 * We want to adapt the delalloc mechanism for copy-on-write, since the 101 * write paths are similar. The first two steps (creating the reservation 102 * and allocating the blocks) are exactly the same as delalloc except that 103 * the mappings must be stored in a separate CoW fork because we do not want 104 * to disturb the mapping in the data fork until we're sure that the write 105 * succeeded. IO completion in this case is the process of removing the old 106 * mapping from the data fork and moving the new mapping from the CoW fork to 107 * the data fork. This will be discussed shortly. 108 * 109 * For now, unaligned directio writes will be bounced back to the page cache. 110 * Block-aligned directio writes will use the same mechanism as buffered 111 * writes. 112 * 113 * Just prior to submitting the actual disk write requests, we convert 114 * the extents representing the range of the file actually being written 115 * (as opposed to extra pieces created for the cowextsize hint) to real 116 * extents. This will become important in the next step: 117 * 118 * D: --RRRRRRSSSRRRRRRRR--- 119 * C: ------UUrrUUU--------- 120 * 121 * CoW remapping must be done after the data block write completes, 122 * because we don't want to destroy the old data fork map until we're sure 123 * the new block has been written. Since the new mappings are kept in a 124 * separate fork, we can simply iterate these mappings to find the ones 125 * that cover the file blocks that we just CoW'd. For each extent, simply 126 * unmap the corresponding range in the data fork, map the new range into 127 * the data fork, and remove the extent from the CoW fork. Because of 128 * the presence of the cowextsize hint, however, we must be careful 129 * only to remap the blocks that we've actually written out -- we must 130 * never remap delalloc reservations nor CoW staging blocks that have 131 * yet to be written. This corresponds exactly to the real extents in 132 * the CoW fork: 133 * 134 * D: --RRRRRRrrSRRRRRRRR--- 135 * C: ------UU--UUU--------- 136 * 137 * Since the remapping operation can be applied to an arbitrary file 138 * range, we record the need for the remap step as a flag in the ioend 139 * instead of declaring a new IO type. This is required for direct io 140 * because we only have ioend for the whole dio, and we have to be able to 141 * remember the presence of unwritten blocks and CoW blocks with a single 142 * ioend structure. Better yet, the more ground we can cover with one 143 * ioend, the better. 144 */ 145 146 /* 147 * Given an AG extent, find the lowest-numbered run of shared blocks 148 * within that range and return the range in fbno/flen. If 149 * find_end_of_shared is true, return the longest contiguous extent of 150 * shared blocks. If there are no shared extents, fbno and flen will 151 * be set to NULLAGBLOCK and 0, respectively. 152 */ 153 int 154 xfs_reflink_find_shared( 155 struct xfs_mount *mp, 156 struct xfs_trans *tp, 157 xfs_agnumber_t agno, 158 xfs_agblock_t agbno, 159 xfs_extlen_t aglen, 160 xfs_agblock_t *fbno, 161 xfs_extlen_t *flen, 162 bool find_end_of_shared) 163 { 164 struct xfs_buf *agbp; 165 struct xfs_btree_cur *cur; 166 int error; 167 168 error = xfs_alloc_read_agf(mp, tp, agno, 0, &agbp); 169 if (error) 170 return error; 171 if (!agbp) 172 return -ENOMEM; 173 174 cur = xfs_refcountbt_init_cursor(mp, tp, agbp, agno, NULL); 175 176 error = xfs_refcount_find_shared(cur, agbno, aglen, fbno, flen, 177 find_end_of_shared); 178 179 xfs_btree_del_cursor(cur, error ? XFS_BTREE_ERROR : XFS_BTREE_NOERROR); 180 181 xfs_trans_brelse(tp, agbp); 182 return error; 183 } 184 185 /* 186 * Trim the mapping to the next block where there's a change in the 187 * shared/unshared status. More specifically, this means that we 188 * find the lowest-numbered extent of shared blocks that coincides with 189 * the given block mapping. If the shared extent overlaps the start of 190 * the mapping, trim the mapping to the end of the shared extent. If 191 * the shared region intersects the mapping, trim the mapping to the 192 * start of the shared extent. If there are no shared regions that 193 * overlap, just return the original extent. 194 */ 195 int 196 xfs_reflink_trim_around_shared( 197 struct xfs_inode *ip, 198 struct xfs_bmbt_irec *irec, 199 bool *shared, 200 bool *trimmed) 201 { 202 xfs_agnumber_t agno; 203 xfs_agblock_t agbno; 204 xfs_extlen_t aglen; 205 xfs_agblock_t fbno; 206 xfs_extlen_t flen; 207 int error = 0; 208 209 /* Holes, unwritten, and delalloc extents cannot be shared */ 210 if (!xfs_is_reflink_inode(ip) || !xfs_bmap_is_real_extent(irec)) { 211 *shared = false; 212 return 0; 213 } 214 215 trace_xfs_reflink_trim_around_shared(ip, irec); 216 217 agno = XFS_FSB_TO_AGNO(ip->i_mount, irec->br_startblock); 218 agbno = XFS_FSB_TO_AGBNO(ip->i_mount, irec->br_startblock); 219 aglen = irec->br_blockcount; 220 221 error = xfs_reflink_find_shared(ip->i_mount, NULL, agno, agbno, 222 aglen, &fbno, &flen, true); 223 if (error) 224 return error; 225 226 *shared = *trimmed = false; 227 if (fbno == NULLAGBLOCK) { 228 /* No shared blocks at all. */ 229 return 0; 230 } else if (fbno == agbno) { 231 /* 232 * The start of this extent is shared. Truncate the 233 * mapping at the end of the shared region so that a 234 * subsequent iteration starts at the start of the 235 * unshared region. 236 */ 237 irec->br_blockcount = flen; 238 *shared = true; 239 if (flen != aglen) 240 *trimmed = true; 241 return 0; 242 } else { 243 /* 244 * There's a shared extent midway through this extent. 245 * Truncate the mapping at the start of the shared 246 * extent so that a subsequent iteration starts at the 247 * start of the shared region. 248 */ 249 irec->br_blockcount = fbno - agbno; 250 *trimmed = true; 251 return 0; 252 } 253 } 254 255 /* 256 * Trim the passed in imap to the next shared/unshared extent boundary, and 257 * if imap->br_startoff points to a shared extent reserve space for it in the 258 * COW fork. In this case *shared is set to true, else to false. 259 * 260 * Note that imap will always contain the block numbers for the existing blocks 261 * in the data fork, as the upper layers need them for read-modify-write 262 * operations. 263 */ 264 int 265 xfs_reflink_reserve_cow( 266 struct xfs_inode *ip, 267 struct xfs_bmbt_irec *imap, 268 bool *shared) 269 { 270 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 271 struct xfs_bmbt_irec got; 272 int error = 0; 273 bool eof = false, trimmed; 274 struct xfs_iext_cursor icur; 275 276 /* 277 * Search the COW fork extent list first. This serves two purposes: 278 * first this implement the speculative preallocation using cowextisze, 279 * so that we also unshared block adjacent to shared blocks instead 280 * of just the shared blocks themselves. Second the lookup in the 281 * extent list is generally faster than going out to the shared extent 282 * tree. 283 */ 284 285 if (!xfs_iext_lookup_extent(ip, ifp, imap->br_startoff, &icur, &got)) 286 eof = true; 287 if (!eof && got.br_startoff <= imap->br_startoff) { 288 trace_xfs_reflink_cow_found(ip, imap); 289 xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); 290 291 *shared = true; 292 return 0; 293 } 294 295 /* Trim the mapping to the nearest shared extent boundary. */ 296 error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed); 297 if (error) 298 return error; 299 300 /* Not shared? Just report the (potentially capped) extent. */ 301 if (!*shared) 302 return 0; 303 304 /* 305 * Fork all the shared blocks from our write offset until the end of 306 * the extent. 307 */ 308 error = xfs_qm_dqattach_locked(ip, 0); 309 if (error) 310 return error; 311 312 error = xfs_bmapi_reserve_delalloc(ip, XFS_COW_FORK, imap->br_startoff, 313 imap->br_blockcount, 0, &got, &icur, eof); 314 if (error == -ENOSPC || error == -EDQUOT) 315 trace_xfs_reflink_cow_enospc(ip, imap); 316 if (error) 317 return error; 318 319 trace_xfs_reflink_cow_alloc(ip, &got); 320 return 0; 321 } 322 323 /* Convert part of an unwritten CoW extent to a real one. */ 324 STATIC int 325 xfs_reflink_convert_cow_extent( 326 struct xfs_inode *ip, 327 struct xfs_bmbt_irec *imap, 328 xfs_fileoff_t offset_fsb, 329 xfs_filblks_t count_fsb, 330 struct xfs_defer_ops *dfops) 331 { 332 xfs_fsblock_t first_block = NULLFSBLOCK; 333 int nimaps = 1; 334 335 if (imap->br_state == XFS_EXT_NORM) 336 return 0; 337 338 xfs_trim_extent(imap, offset_fsb, count_fsb); 339 trace_xfs_reflink_convert_cow(ip, imap); 340 if (imap->br_blockcount == 0) 341 return 0; 342 return xfs_bmapi_write(NULL, ip, imap->br_startoff, imap->br_blockcount, 343 XFS_BMAPI_COWFORK | XFS_BMAPI_CONVERT, &first_block, 344 0, imap, &nimaps, dfops); 345 } 346 347 /* Convert all of the unwritten CoW extents in a file's range to real ones. */ 348 int 349 xfs_reflink_convert_cow( 350 struct xfs_inode *ip, 351 xfs_off_t offset, 352 xfs_off_t count) 353 { 354 struct xfs_mount *mp = ip->i_mount; 355 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 356 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); 357 xfs_filblks_t count_fsb = end_fsb - offset_fsb; 358 struct xfs_bmbt_irec imap; 359 struct xfs_defer_ops dfops; 360 xfs_fsblock_t first_block = NULLFSBLOCK; 361 int nimaps = 1, error = 0; 362 363 ASSERT(count != 0); 364 365 xfs_ilock(ip, XFS_ILOCK_EXCL); 366 error = xfs_bmapi_write(NULL, ip, offset_fsb, count_fsb, 367 XFS_BMAPI_COWFORK | XFS_BMAPI_CONVERT | 368 XFS_BMAPI_CONVERT_ONLY, &first_block, 0, &imap, &nimaps, 369 &dfops); 370 xfs_iunlock(ip, XFS_ILOCK_EXCL); 371 return error; 372 } 373 374 /* Allocate all CoW reservations covering a range of blocks in a file. */ 375 int 376 xfs_reflink_allocate_cow( 377 struct xfs_inode *ip, 378 struct xfs_bmbt_irec *imap, 379 bool *shared, 380 uint *lockmode) 381 { 382 struct xfs_mount *mp = ip->i_mount; 383 xfs_fileoff_t offset_fsb = imap->br_startoff; 384 xfs_filblks_t count_fsb = imap->br_blockcount; 385 struct xfs_bmbt_irec got; 386 struct xfs_defer_ops dfops; 387 struct xfs_trans *tp = NULL; 388 xfs_fsblock_t first_block; 389 int nimaps, error = 0; 390 bool trimmed; 391 xfs_filblks_t resaligned; 392 xfs_extlen_t resblks = 0; 393 struct xfs_iext_cursor icur; 394 395 retry: 396 ASSERT(xfs_is_reflink_inode(ip)); 397 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED)); 398 399 /* 400 * Even if the extent is not shared we might have a preallocation for 401 * it in the COW fork. If so use it. 402 */ 403 if (xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got) && 404 got.br_startoff <= offset_fsb) { 405 *shared = true; 406 407 /* If we have a real allocation in the COW fork we're done. */ 408 if (!isnullstartblock(got.br_startblock)) { 409 xfs_trim_extent(&got, offset_fsb, count_fsb); 410 *imap = got; 411 goto convert; 412 } 413 414 xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); 415 } else { 416 error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed); 417 if (error || !*shared) 418 goto out; 419 } 420 421 if (!tp) { 422 resaligned = xfs_aligned_fsb_count(imap->br_startoff, 423 imap->br_blockcount, xfs_get_cowextsz_hint(ip)); 424 resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); 425 426 xfs_iunlock(ip, *lockmode); 427 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); 428 *lockmode = XFS_ILOCK_EXCL; 429 xfs_ilock(ip, *lockmode); 430 431 if (error) 432 return error; 433 434 error = xfs_qm_dqattach_locked(ip, 0); 435 if (error) 436 goto out; 437 goto retry; 438 } 439 440 error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, 441 XFS_QMOPT_RES_REGBLKS); 442 if (error) 443 goto out; 444 445 xfs_trans_ijoin(tp, ip, 0); 446 447 xfs_defer_init(&dfops, &first_block); 448 nimaps = 1; 449 450 /* Allocate the entire reservation as unwritten blocks. */ 451 error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount, 452 XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, &first_block, 453 resblks, imap, &nimaps, &dfops); 454 if (error) 455 goto out_bmap_cancel; 456 457 xfs_inode_set_cowblocks_tag(ip); 458 459 /* Finish up. */ 460 error = xfs_defer_finish(&tp, &dfops); 461 if (error) 462 goto out_bmap_cancel; 463 464 error = xfs_trans_commit(tp); 465 if (error) 466 return error; 467 468 /* 469 * Allocation succeeded but the requested range was not even partially 470 * satisfied? Bail out! 471 */ 472 if (nimaps == 0) 473 return -ENOSPC; 474 convert: 475 return xfs_reflink_convert_cow_extent(ip, imap, offset_fsb, count_fsb, 476 &dfops); 477 out_bmap_cancel: 478 xfs_defer_cancel(&dfops); 479 xfs_trans_unreserve_quota_nblks(tp, ip, (long)resblks, 0, 480 XFS_QMOPT_RES_REGBLKS); 481 out: 482 if (tp) 483 xfs_trans_cancel(tp); 484 return error; 485 } 486 487 /* 488 * Find the CoW reservation for a given byte offset of a file. 489 */ 490 bool 491 xfs_reflink_find_cow_mapping( 492 struct xfs_inode *ip, 493 xfs_off_t offset, 494 struct xfs_bmbt_irec *imap) 495 { 496 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 497 xfs_fileoff_t offset_fsb; 498 struct xfs_bmbt_irec got; 499 struct xfs_iext_cursor icur; 500 501 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED)); 502 503 if (!xfs_is_reflink_inode(ip)) 504 return false; 505 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 506 if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &icur, &got)) 507 return false; 508 if (got.br_startoff > offset_fsb) 509 return false; 510 511 trace_xfs_reflink_find_cow_mapping(ip, offset, 1, XFS_IO_OVERWRITE, 512 &got); 513 *imap = got; 514 return true; 515 } 516 517 /* 518 * Trim an extent to end at the next CoW reservation past offset_fsb. 519 */ 520 void 521 xfs_reflink_trim_irec_to_next_cow( 522 struct xfs_inode *ip, 523 xfs_fileoff_t offset_fsb, 524 struct xfs_bmbt_irec *imap) 525 { 526 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 527 struct xfs_bmbt_irec got; 528 struct xfs_iext_cursor icur; 529 530 if (!xfs_is_reflink_inode(ip)) 531 return; 532 533 /* Find the extent in the CoW fork. */ 534 if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &icur, &got)) 535 return; 536 537 /* This is the extent before; try sliding up one. */ 538 if (got.br_startoff < offset_fsb) { 539 if (!xfs_iext_next_extent(ifp, &icur, &got)) 540 return; 541 } 542 543 if (got.br_startoff >= imap->br_startoff + imap->br_blockcount) 544 return; 545 546 imap->br_blockcount = got.br_startoff - imap->br_startoff; 547 trace_xfs_reflink_trim_irec(ip, imap); 548 } 549 550 /* 551 * Cancel CoW reservations for some block range of an inode. 552 * 553 * If cancel_real is true this function cancels all COW fork extents for the 554 * inode; if cancel_real is false, real extents are not cleared. 555 */ 556 int 557 xfs_reflink_cancel_cow_blocks( 558 struct xfs_inode *ip, 559 struct xfs_trans **tpp, 560 xfs_fileoff_t offset_fsb, 561 xfs_fileoff_t end_fsb, 562 bool cancel_real) 563 { 564 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 565 struct xfs_bmbt_irec got, del; 566 struct xfs_iext_cursor icur; 567 xfs_fsblock_t firstfsb; 568 struct xfs_defer_ops dfops; 569 int error = 0; 570 571 if (!xfs_is_reflink_inode(ip)) 572 return 0; 573 if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) 574 return 0; 575 576 /* Walk backwards until we're out of the I/O range... */ 577 while (got.br_startoff + got.br_blockcount > offset_fsb) { 578 del = got; 579 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 580 581 /* Extent delete may have bumped ext forward */ 582 if (!del.br_blockcount) { 583 xfs_iext_prev(ifp, &icur); 584 goto next_extent; 585 } 586 587 trace_xfs_reflink_cancel_cow(ip, &del); 588 589 if (isnullstartblock(del.br_startblock)) { 590 error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, 591 &icur, &got, &del); 592 if (error) 593 break; 594 } else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) { 595 xfs_trans_ijoin(*tpp, ip, 0); 596 xfs_defer_init(&dfops, &firstfsb); 597 598 /* Free the CoW orphan record. */ 599 error = xfs_refcount_free_cow_extent(ip->i_mount, 600 &dfops, del.br_startblock, 601 del.br_blockcount); 602 if (error) 603 break; 604 605 xfs_bmap_add_free(ip->i_mount, &dfops, 606 del.br_startblock, del.br_blockcount, 607 NULL); 608 609 /* Roll the transaction */ 610 xfs_defer_ijoin(&dfops, ip); 611 error = xfs_defer_finish(tpp, &dfops); 612 if (error) { 613 xfs_defer_cancel(&dfops); 614 break; 615 } 616 617 /* Remove the mapping from the CoW fork. */ 618 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 619 620 /* Remove the quota reservation */ 621 error = xfs_trans_reserve_quota_nblks(NULL, ip, 622 -(long)del.br_blockcount, 0, 623 XFS_QMOPT_RES_REGBLKS); 624 if (error) 625 break; 626 } else { 627 /* Didn't do anything, push cursor back. */ 628 xfs_iext_prev(ifp, &icur); 629 } 630 next_extent: 631 if (!xfs_iext_get_extent(ifp, &icur, &got)) 632 break; 633 } 634 635 /* clear tag if cow fork is emptied */ 636 if (!ifp->if_bytes) 637 xfs_inode_clear_cowblocks_tag(ip); 638 639 return error; 640 } 641 642 /* 643 * Cancel CoW reservations for some byte range of an inode. 644 * 645 * If cancel_real is true this function cancels all COW fork extents for the 646 * inode; if cancel_real is false, real extents are not cleared. 647 */ 648 int 649 xfs_reflink_cancel_cow_range( 650 struct xfs_inode *ip, 651 xfs_off_t offset, 652 xfs_off_t count, 653 bool cancel_real) 654 { 655 struct xfs_trans *tp; 656 xfs_fileoff_t offset_fsb; 657 xfs_fileoff_t end_fsb; 658 int error; 659 660 trace_xfs_reflink_cancel_cow_range(ip, offset, count); 661 ASSERT(xfs_is_reflink_inode(ip)); 662 663 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 664 if (count == NULLFILEOFF) 665 end_fsb = NULLFILEOFF; 666 else 667 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 668 669 /* Start a rolling transaction to remove the mappings */ 670 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 671 0, 0, 0, &tp); 672 if (error) 673 goto out; 674 675 xfs_ilock(ip, XFS_ILOCK_EXCL); 676 xfs_trans_ijoin(tp, ip, 0); 677 678 /* Scrape out the old CoW reservations */ 679 error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb, 680 cancel_real); 681 if (error) 682 goto out_cancel; 683 684 error = xfs_trans_commit(tp); 685 686 xfs_iunlock(ip, XFS_ILOCK_EXCL); 687 return error; 688 689 out_cancel: 690 xfs_trans_cancel(tp); 691 xfs_iunlock(ip, XFS_ILOCK_EXCL); 692 out: 693 trace_xfs_reflink_cancel_cow_range_error(ip, error, _RET_IP_); 694 return error; 695 } 696 697 /* 698 * Remap parts of a file's data fork after a successful CoW. 699 */ 700 int 701 xfs_reflink_end_cow( 702 struct xfs_inode *ip, 703 xfs_off_t offset, 704 xfs_off_t count) 705 { 706 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); 707 struct xfs_bmbt_irec got, del; 708 struct xfs_trans *tp; 709 xfs_fileoff_t offset_fsb; 710 xfs_fileoff_t end_fsb; 711 xfs_fsblock_t firstfsb; 712 struct xfs_defer_ops dfops; 713 int error; 714 unsigned int resblks; 715 xfs_filblks_t rlen; 716 struct xfs_iext_cursor icur; 717 718 trace_xfs_reflink_end_cow(ip, offset, count); 719 720 /* No COW extents? That's easy! */ 721 if (ifp->if_bytes == 0) 722 return 0; 723 724 offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 725 end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); 726 727 /* 728 * Start a rolling transaction to switch the mappings. We're 729 * unlikely ever to have to remap 16T worth of single-block 730 * extents, so just cap the worst case extent count to 2^32-1. 731 * Stick a warning in just in case, and avoid 64-bit division. 732 */ 733 BUILD_BUG_ON(MAX_RW_COUNT > UINT_MAX); 734 if (end_fsb - offset_fsb > UINT_MAX) { 735 error = -EFSCORRUPTED; 736 xfs_force_shutdown(ip->i_mount, SHUTDOWN_CORRUPT_INCORE); 737 ASSERT(0); 738 goto out; 739 } 740 resblks = XFS_NEXTENTADD_SPACE_RES(ip->i_mount, 741 (unsigned int)(end_fsb - offset_fsb), 742 XFS_DATA_FORK); 743 error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, 744 resblks, 0, XFS_TRANS_RESERVE, &tp); 745 if (error) 746 goto out; 747 748 xfs_ilock(ip, XFS_ILOCK_EXCL); 749 xfs_trans_ijoin(tp, ip, 0); 750 751 /* 752 * In case of racing, overlapping AIO writes no COW extents might be 753 * left by the time I/O completes for the loser of the race. In that 754 * case we are done. 755 */ 756 if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) 757 goto out_cancel; 758 759 /* Walk backwards until we're out of the I/O range... */ 760 while (got.br_startoff + got.br_blockcount > offset_fsb) { 761 del = got; 762 xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); 763 764 /* Extent delete may have bumped ext forward */ 765 if (!del.br_blockcount) { 766 xfs_iext_prev(ifp, &icur); 767 goto next_extent; 768 } 769 770 ASSERT(!isnullstartblock(got.br_startblock)); 771 772 /* 773 * Don't remap unwritten extents; these are 774 * speculatively preallocated CoW extents that have been 775 * allocated but have not yet been involved in a write. 776 */ 777 if (got.br_state == XFS_EXT_UNWRITTEN) { 778 xfs_iext_prev(ifp, &icur); 779 goto next_extent; 780 } 781 782 /* Unmap the old blocks in the data fork. */ 783 xfs_defer_init(&dfops, &firstfsb); 784 rlen = del.br_blockcount; 785 error = __xfs_bunmapi(tp, ip, del.br_startoff, &rlen, 0, 1, 786 &firstfsb, &dfops); 787 if (error) 788 goto out_defer; 789 790 /* Trim the extent to whatever got unmapped. */ 791 if (rlen) { 792 xfs_trim_extent(&del, del.br_startoff + rlen, 793 del.br_blockcount - rlen); 794 } 795 trace_xfs_reflink_cow_remap(ip, &del); 796 797 /* Free the CoW orphan record. */ 798 error = xfs_refcount_free_cow_extent(tp->t_mountp, &dfops, 799 del.br_startblock, del.br_blockcount); 800 if (error) 801 goto out_defer; 802 803 /* Map the new blocks into the data fork. */ 804 error = xfs_bmap_map_extent(tp->t_mountp, &dfops, ip, &del); 805 if (error) 806 goto out_defer; 807 808 /* Charge this new data fork mapping to the on-disk quota. */ 809 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_DELBCOUNT, 810 (long)del.br_blockcount); 811 812 /* Remove the mapping from the CoW fork. */ 813 xfs_bmap_del_extent_cow(ip, &icur, &got, &del); 814 815 xfs_defer_ijoin(&dfops, ip); 816 error = xfs_defer_finish(&tp, &dfops); 817 if (error) 818 goto out_defer; 819 next_extent: 820 if (!xfs_iext_get_extent(ifp, &icur, &got)) 821 break; 822 } 823 824 error = xfs_trans_commit(tp); 825 xfs_iunlock(ip, XFS_ILOCK_EXCL); 826 if (error) 827 goto out; 828 return 0; 829 830 out_defer: 831 xfs_defer_cancel(&dfops); 832 out_cancel: 833 xfs_trans_cancel(tp); 834 xfs_iunlock(ip, XFS_ILOCK_EXCL); 835 out: 836 trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_); 837 return error; 838 } 839 840 /* 841 * Free leftover CoW reservations that didn't get cleaned out. 842 */ 843 int 844 xfs_reflink_recover_cow( 845 struct xfs_mount *mp) 846 { 847 xfs_agnumber_t agno; 848 int error = 0; 849 850 if (!xfs_sb_version_hasreflink(&mp->m_sb)) 851 return 0; 852 853 for (agno = 0; agno < mp->m_sb.sb_agcount; agno++) { 854 error = xfs_refcount_recover_cow_leftovers(mp, agno); 855 if (error) 856 break; 857 } 858 859 return error; 860 } 861 862 /* 863 * Reflinking (Block) Ranges of Two Files Together 864 * 865 * First, ensure that the reflink flag is set on both inodes. The flag is an 866 * optimization to avoid unnecessary refcount btree lookups in the write path. 867 * 868 * Now we can iteratively remap the range of extents (and holes) in src to the 869 * corresponding ranges in dest. Let drange and srange denote the ranges of 870 * logical blocks in dest and src touched by the reflink operation. 871 * 872 * While the length of drange is greater than zero, 873 * - Read src's bmbt at the start of srange ("imap") 874 * - If imap doesn't exist, make imap appear to start at the end of srange 875 * with zero length. 876 * - If imap starts before srange, advance imap to start at srange. 877 * - If imap goes beyond srange, truncate imap to end at the end of srange. 878 * - Punch (imap start - srange start + imap len) blocks from dest at 879 * offset (drange start). 880 * - If imap points to a real range of pblks, 881 * > Increase the refcount of the imap's pblks 882 * > Map imap's pblks into dest at the offset 883 * (drange start + imap start - srange start) 884 * - Advance drange and srange by (imap start - srange start + imap len) 885 * 886 * Finally, if the reflink made dest longer, update both the in-core and 887 * on-disk file sizes. 888 * 889 * ASCII Art Demonstration: 890 * 891 * Let's say we want to reflink this source file: 892 * 893 * ----SSSSSSS-SSSSS----SSSSSS (src file) 894 * <--------------------> 895 * 896 * into this destination file: 897 * 898 * --DDDDDDDDDDDDDDDDDDD--DDD (dest file) 899 * <--------------------> 900 * '-' means a hole, and 'S' and 'D' are written blocks in the src and dest. 901 * Observe that the range has different logical offsets in either file. 902 * 903 * Consider that the first extent in the source file doesn't line up with our 904 * reflink range. Unmapping and remapping are separate operations, so we can 905 * unmap more blocks from the destination file than we remap. 906 * 907 * ----SSSSSSS-SSSSS----SSSSSS 908 * <-------> 909 * --DDDDD---------DDDDD--DDD 910 * <-------> 911 * 912 * Now remap the source extent into the destination file: 913 * 914 * ----SSSSSSS-SSSSS----SSSSSS 915 * <-------> 916 * --DDDDD--SSSSSSSDDDDD--DDD 917 * <-------> 918 * 919 * Do likewise with the second hole and extent in our range. Holes in the 920 * unmap range don't affect our operation. 921 * 922 * ----SSSSSSS-SSSSS----SSSSSS 923 * <----> 924 * --DDDDD--SSSSSSS-SSSSS-DDD 925 * <----> 926 * 927 * Finally, unmap and remap part of the third extent. This will increase the 928 * size of the destination file. 929 * 930 * ----SSSSSSS-SSSSS----SSSSSS 931 * <-----> 932 * --DDDDD--SSSSSSS-SSSSS----SSS 933 * <-----> 934 * 935 * Once we update the destination file's i_size, we're done. 936 */ 937 938 /* 939 * Ensure the reflink bit is set in both inodes. 940 */ 941 STATIC int 942 xfs_reflink_set_inode_flag( 943 struct xfs_inode *src, 944 struct xfs_inode *dest) 945 { 946 struct xfs_mount *mp = src->i_mount; 947 int error; 948 struct xfs_trans *tp; 949 950 if (xfs_is_reflink_inode(src) && xfs_is_reflink_inode(dest)) 951 return 0; 952 953 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 954 if (error) 955 goto out_error; 956 957 /* Lock both files against IO */ 958 if (src->i_ino == dest->i_ino) 959 xfs_ilock(src, XFS_ILOCK_EXCL); 960 else 961 xfs_lock_two_inodes(src, XFS_ILOCK_EXCL, dest, XFS_ILOCK_EXCL); 962 963 if (!xfs_is_reflink_inode(src)) { 964 trace_xfs_reflink_set_inode_flag(src); 965 xfs_trans_ijoin(tp, src, XFS_ILOCK_EXCL); 966 src->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 967 xfs_trans_log_inode(tp, src, XFS_ILOG_CORE); 968 xfs_ifork_init_cow(src); 969 } else 970 xfs_iunlock(src, XFS_ILOCK_EXCL); 971 972 if (src->i_ino == dest->i_ino) 973 goto commit_flags; 974 975 if (!xfs_is_reflink_inode(dest)) { 976 trace_xfs_reflink_set_inode_flag(dest); 977 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 978 dest->i_d.di_flags2 |= XFS_DIFLAG2_REFLINK; 979 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 980 xfs_ifork_init_cow(dest); 981 } else 982 xfs_iunlock(dest, XFS_ILOCK_EXCL); 983 984 commit_flags: 985 error = xfs_trans_commit(tp); 986 if (error) 987 goto out_error; 988 return error; 989 990 out_error: 991 trace_xfs_reflink_set_inode_flag_error(dest, error, _RET_IP_); 992 return error; 993 } 994 995 /* 996 * Update destination inode size & cowextsize hint, if necessary. 997 */ 998 STATIC int 999 xfs_reflink_update_dest( 1000 struct xfs_inode *dest, 1001 xfs_off_t newlen, 1002 xfs_extlen_t cowextsize, 1003 bool is_dedupe) 1004 { 1005 struct xfs_mount *mp = dest->i_mount; 1006 struct xfs_trans *tp; 1007 int error; 1008 1009 if (is_dedupe && newlen <= i_size_read(VFS_I(dest)) && cowextsize == 0) 1010 return 0; 1011 1012 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); 1013 if (error) 1014 goto out_error; 1015 1016 xfs_ilock(dest, XFS_ILOCK_EXCL); 1017 xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); 1018 1019 if (newlen > i_size_read(VFS_I(dest))) { 1020 trace_xfs_reflink_update_inode_size(dest, newlen); 1021 i_size_write(VFS_I(dest), newlen); 1022 dest->i_d.di_size = newlen; 1023 } 1024 1025 if (cowextsize) { 1026 dest->i_d.di_cowextsize = cowextsize; 1027 dest->i_d.di_flags2 |= XFS_DIFLAG2_COWEXTSIZE; 1028 } 1029 1030 if (!is_dedupe) { 1031 xfs_trans_ichgtime(tp, dest, 1032 XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); 1033 } 1034 xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); 1035 1036 error = xfs_trans_commit(tp); 1037 if (error) 1038 goto out_error; 1039 return error; 1040 1041 out_error: 1042 trace_xfs_reflink_update_inode_size_error(dest, error, _RET_IP_); 1043 return error; 1044 } 1045 1046 /* 1047 * Do we have enough reserve in this AG to handle a reflink? The refcount 1048 * btree already reserved all the space it needs, but the rmap btree can grow 1049 * infinitely, so we won't allow more reflinks when the AG is down to the 1050 * btree reserves. 1051 */ 1052 static int 1053 xfs_reflink_ag_has_free_space( 1054 struct xfs_mount *mp, 1055 xfs_agnumber_t agno) 1056 { 1057 struct xfs_perag *pag; 1058 int error = 0; 1059 1060 if (!xfs_sb_version_hasrmapbt(&mp->m_sb)) 1061 return 0; 1062 1063 pag = xfs_perag_get(mp, agno); 1064 if (xfs_ag_resv_critical(pag, XFS_AG_RESV_AGFL) || 1065 xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA)) 1066 error = -ENOSPC; 1067 xfs_perag_put(pag); 1068 return error; 1069 } 1070 1071 /* 1072 * Unmap a range of blocks from a file, then map other blocks into the hole. 1073 * The range to unmap is (destoff : destoff + srcioff + irec->br_blockcount). 1074 * The extent irec is mapped into dest at irec->br_startoff. 1075 */ 1076 STATIC int 1077 xfs_reflink_remap_extent( 1078 struct xfs_inode *ip, 1079 struct xfs_bmbt_irec *irec, 1080 xfs_fileoff_t destoff, 1081 xfs_off_t new_isize) 1082 { 1083 struct xfs_mount *mp = ip->i_mount; 1084 bool real_extent = xfs_bmap_is_real_extent(irec); 1085 struct xfs_trans *tp; 1086 xfs_fsblock_t firstfsb; 1087 unsigned int resblks; 1088 struct xfs_defer_ops dfops; 1089 struct xfs_bmbt_irec uirec; 1090 xfs_filblks_t rlen; 1091 xfs_filblks_t unmap_len; 1092 xfs_off_t newlen; 1093 int error; 1094 1095 unmap_len = irec->br_startoff + irec->br_blockcount - destoff; 1096 trace_xfs_reflink_punch_range(ip, destoff, unmap_len); 1097 1098 /* No reflinking if we're low on space */ 1099 if (real_extent) { 1100 error = xfs_reflink_ag_has_free_space(mp, 1101 XFS_FSB_TO_AGNO(mp, irec->br_startblock)); 1102 if (error) 1103 goto out; 1104 } 1105 1106 /* Start a rolling transaction to switch the mappings */ 1107 resblks = XFS_EXTENTADD_SPACE_RES(ip->i_mount, XFS_DATA_FORK); 1108 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); 1109 if (error) 1110 goto out; 1111 1112 xfs_ilock(ip, XFS_ILOCK_EXCL); 1113 xfs_trans_ijoin(tp, ip, 0); 1114 1115 /* If we're not just clearing space, then do we have enough quota? */ 1116 if (real_extent) { 1117 error = xfs_trans_reserve_quota_nblks(tp, ip, 1118 irec->br_blockcount, 0, XFS_QMOPT_RES_REGBLKS); 1119 if (error) 1120 goto out_cancel; 1121 } 1122 1123 trace_xfs_reflink_remap(ip, irec->br_startoff, 1124 irec->br_blockcount, irec->br_startblock); 1125 1126 /* Unmap the old blocks in the data fork. */ 1127 rlen = unmap_len; 1128 while (rlen) { 1129 xfs_defer_init(&dfops, &firstfsb); 1130 error = __xfs_bunmapi(tp, ip, destoff, &rlen, 0, 1, 1131 &firstfsb, &dfops); 1132 if (error) 1133 goto out_defer; 1134 1135 /* 1136 * Trim the extent to whatever got unmapped. 1137 * Remember, bunmapi works backwards. 1138 */ 1139 uirec.br_startblock = irec->br_startblock + rlen; 1140 uirec.br_startoff = irec->br_startoff + rlen; 1141 uirec.br_blockcount = unmap_len - rlen; 1142 unmap_len = rlen; 1143 1144 /* If this isn't a real mapping, we're done. */ 1145 if (!real_extent || uirec.br_blockcount == 0) 1146 goto next_extent; 1147 1148 trace_xfs_reflink_remap(ip, uirec.br_startoff, 1149 uirec.br_blockcount, uirec.br_startblock); 1150 1151 /* Update the refcount tree */ 1152 error = xfs_refcount_increase_extent(mp, &dfops, &uirec); 1153 if (error) 1154 goto out_defer; 1155 1156 /* Map the new blocks into the data fork. */ 1157 error = xfs_bmap_map_extent(mp, &dfops, ip, &uirec); 1158 if (error) 1159 goto out_defer; 1160 1161 /* Update quota accounting. */ 1162 xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1163 uirec.br_blockcount); 1164 1165 /* Update dest isize if needed. */ 1166 newlen = XFS_FSB_TO_B(mp, 1167 uirec.br_startoff + uirec.br_blockcount); 1168 newlen = min_t(xfs_off_t, newlen, new_isize); 1169 if (newlen > i_size_read(VFS_I(ip))) { 1170 trace_xfs_reflink_update_inode_size(ip, newlen); 1171 i_size_write(VFS_I(ip), newlen); 1172 ip->i_d.di_size = newlen; 1173 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1174 } 1175 1176 next_extent: 1177 /* Process all the deferred stuff. */ 1178 xfs_defer_ijoin(&dfops, ip); 1179 error = xfs_defer_finish(&tp, &dfops); 1180 if (error) 1181 goto out_defer; 1182 } 1183 1184 error = xfs_trans_commit(tp); 1185 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1186 if (error) 1187 goto out; 1188 return 0; 1189 1190 out_defer: 1191 xfs_defer_cancel(&dfops); 1192 out_cancel: 1193 xfs_trans_cancel(tp); 1194 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1195 out: 1196 trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_); 1197 return error; 1198 } 1199 1200 /* 1201 * Iteratively remap one file's extents (and holes) to another's. 1202 */ 1203 STATIC int 1204 xfs_reflink_remap_blocks( 1205 struct xfs_inode *src, 1206 xfs_fileoff_t srcoff, 1207 struct xfs_inode *dest, 1208 xfs_fileoff_t destoff, 1209 xfs_filblks_t len, 1210 xfs_off_t new_isize) 1211 { 1212 struct xfs_bmbt_irec imap; 1213 int nimaps; 1214 int error = 0; 1215 xfs_filblks_t range_len; 1216 1217 /* drange = (destoff, destoff + len); srange = (srcoff, srcoff + len) */ 1218 while (len) { 1219 uint lock_mode; 1220 1221 trace_xfs_reflink_remap_blocks_loop(src, srcoff, len, 1222 dest, destoff); 1223 1224 /* Read extent from the source file */ 1225 nimaps = 1; 1226 lock_mode = xfs_ilock_data_map_shared(src); 1227 error = xfs_bmapi_read(src, srcoff, len, &imap, &nimaps, 0); 1228 xfs_iunlock(src, lock_mode); 1229 if (error) 1230 goto err; 1231 ASSERT(nimaps == 1); 1232 1233 trace_xfs_reflink_remap_imap(src, srcoff, len, XFS_IO_OVERWRITE, 1234 &imap); 1235 1236 /* Translate imap into the destination file. */ 1237 range_len = imap.br_startoff + imap.br_blockcount - srcoff; 1238 imap.br_startoff += destoff - srcoff; 1239 1240 /* Clear dest from destoff to the end of imap and map it in. */ 1241 error = xfs_reflink_remap_extent(dest, &imap, destoff, 1242 new_isize); 1243 if (error) 1244 goto err; 1245 1246 if (fatal_signal_pending(current)) { 1247 error = -EINTR; 1248 goto err; 1249 } 1250 1251 /* Advance drange/srange */ 1252 srcoff += range_len; 1253 destoff += range_len; 1254 len -= range_len; 1255 } 1256 1257 return 0; 1258 1259 err: 1260 trace_xfs_reflink_remap_blocks_error(dest, error, _RET_IP_); 1261 return error; 1262 } 1263 1264 /* 1265 * Grab the exclusive iolock for a data copy from src to dest, making 1266 * sure to abide vfs locking order (lowest pointer value goes first) and 1267 * breaking the pnfs layout leases on dest before proceeding. The loop 1268 * is needed because we cannot call the blocking break_layout() with the 1269 * src iolock held, and therefore have to back out both locks. 1270 */ 1271 static int 1272 xfs_iolock_two_inodes_and_break_layout( 1273 struct inode *src, 1274 struct inode *dest) 1275 { 1276 int error; 1277 1278 retry: 1279 if (src < dest) { 1280 inode_lock_shared(src); 1281 inode_lock_nested(dest, I_MUTEX_NONDIR2); 1282 } else { 1283 /* src >= dest */ 1284 inode_lock(dest); 1285 } 1286 1287 error = break_layout(dest, false); 1288 if (error == -EWOULDBLOCK) { 1289 inode_unlock(dest); 1290 if (src < dest) 1291 inode_unlock_shared(src); 1292 error = break_layout(dest, true); 1293 if (error) 1294 return error; 1295 goto retry; 1296 } 1297 if (error) { 1298 inode_unlock(dest); 1299 if (src < dest) 1300 inode_unlock_shared(src); 1301 return error; 1302 } 1303 if (src > dest) 1304 inode_lock_shared_nested(src, I_MUTEX_NONDIR2); 1305 return 0; 1306 } 1307 1308 /* 1309 * Link a range of blocks from one file to another. 1310 */ 1311 int 1312 xfs_reflink_remap_range( 1313 struct file *file_in, 1314 loff_t pos_in, 1315 struct file *file_out, 1316 loff_t pos_out, 1317 u64 len, 1318 bool is_dedupe) 1319 { 1320 struct inode *inode_in = file_inode(file_in); 1321 struct xfs_inode *src = XFS_I(inode_in); 1322 struct inode *inode_out = file_inode(file_out); 1323 struct xfs_inode *dest = XFS_I(inode_out); 1324 struct xfs_mount *mp = src->i_mount; 1325 bool same_inode = (inode_in == inode_out); 1326 xfs_fileoff_t sfsbno, dfsbno; 1327 xfs_filblks_t fsblen; 1328 xfs_extlen_t cowextsize; 1329 ssize_t ret; 1330 1331 if (!xfs_sb_version_hasreflink(&mp->m_sb)) 1332 return -EOPNOTSUPP; 1333 1334 if (XFS_FORCED_SHUTDOWN(mp)) 1335 return -EIO; 1336 1337 /* Lock both files against IO */ 1338 ret = xfs_iolock_two_inodes_and_break_layout(inode_in, inode_out); 1339 if (ret) 1340 return ret; 1341 if (same_inode) 1342 xfs_ilock(src, XFS_MMAPLOCK_EXCL); 1343 else 1344 xfs_lock_two_inodes(src, XFS_MMAPLOCK_SHARED, dest, 1345 XFS_MMAPLOCK_EXCL); 1346 1347 /* Check file eligibility and prepare for block sharing. */ 1348 ret = -EINVAL; 1349 /* Don't reflink realtime inodes */ 1350 if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest)) 1351 goto out_unlock; 1352 1353 /* Don't share DAX file data for now. */ 1354 if (IS_DAX(inode_in) || IS_DAX(inode_out)) 1355 goto out_unlock; 1356 1357 ret = vfs_clone_file_prep_inodes(inode_in, pos_in, inode_out, pos_out, 1358 &len, is_dedupe); 1359 if (ret <= 0) 1360 goto out_unlock; 1361 1362 /* Attach dquots to dest inode before changing block map */ 1363 ret = xfs_qm_dqattach(dest, 0); 1364 if (ret) 1365 goto out_unlock; 1366 1367 trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out); 1368 1369 /* 1370 * Clear out post-eof preallocations because we don't have page cache 1371 * backing the delayed allocations and they'll never get freed on 1372 * their own. 1373 */ 1374 if (xfs_can_free_eofblocks(dest, true)) { 1375 ret = xfs_free_eofblocks(dest); 1376 if (ret) 1377 goto out_unlock; 1378 } 1379 1380 /* Set flags and remap blocks. */ 1381 ret = xfs_reflink_set_inode_flag(src, dest); 1382 if (ret) 1383 goto out_unlock; 1384 1385 dfsbno = XFS_B_TO_FSBT(mp, pos_out); 1386 sfsbno = XFS_B_TO_FSBT(mp, pos_in); 1387 fsblen = XFS_B_TO_FSB(mp, len); 1388 ret = xfs_reflink_remap_blocks(src, sfsbno, dest, dfsbno, fsblen, 1389 pos_out + len); 1390 if (ret) 1391 goto out_unlock; 1392 1393 /* Zap any page cache for the destination file's range. */ 1394 truncate_inode_pages_range(&inode_out->i_data, pos_out, 1395 PAGE_ALIGN(pos_out + len) - 1); 1396 1397 /* 1398 * Carry the cowextsize hint from src to dest if we're sharing the 1399 * entire source file to the entire destination file, the source file 1400 * has a cowextsize hint, and the destination file does not. 1401 */ 1402 cowextsize = 0; 1403 if (pos_in == 0 && len == i_size_read(inode_in) && 1404 (src->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE) && 1405 pos_out == 0 && len >= i_size_read(inode_out) && 1406 !(dest->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE)) 1407 cowextsize = src->i_d.di_cowextsize; 1408 1409 ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize, 1410 is_dedupe); 1411 1412 out_unlock: 1413 xfs_iunlock(dest, XFS_MMAPLOCK_EXCL); 1414 if (!same_inode) 1415 xfs_iunlock(src, XFS_MMAPLOCK_SHARED); 1416 inode_unlock(inode_out); 1417 if (!same_inode) 1418 inode_unlock_shared(inode_in); 1419 if (ret) 1420 trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_); 1421 return ret; 1422 } 1423 1424 /* 1425 * The user wants to preemptively CoW all shared blocks in this file, 1426 * which enables us to turn off the reflink flag. Iterate all 1427 * extents which are not prealloc/delalloc to see which ranges are 1428 * mentioned in the refcount tree, then read those blocks into the 1429 * pagecache, dirty them, fsync them back out, and then we can update 1430 * the inode flag. What happens if we run out of memory? :) 1431 */ 1432 STATIC int 1433 xfs_reflink_dirty_extents( 1434 struct xfs_inode *ip, 1435 xfs_fileoff_t fbno, 1436 xfs_filblks_t end, 1437 xfs_off_t isize) 1438 { 1439 struct xfs_mount *mp = ip->i_mount; 1440 xfs_agnumber_t agno; 1441 xfs_agblock_t agbno; 1442 xfs_extlen_t aglen; 1443 xfs_agblock_t rbno; 1444 xfs_extlen_t rlen; 1445 xfs_off_t fpos; 1446 xfs_off_t flen; 1447 struct xfs_bmbt_irec map[2]; 1448 int nmaps; 1449 int error = 0; 1450 1451 while (end - fbno > 0) { 1452 nmaps = 1; 1453 /* 1454 * Look for extents in the file. Skip holes, delalloc, or 1455 * unwritten extents; they can't be reflinked. 1456 */ 1457 error = xfs_bmapi_read(ip, fbno, end - fbno, map, &nmaps, 0); 1458 if (error) 1459 goto out; 1460 if (nmaps == 0) 1461 break; 1462 if (!xfs_bmap_is_real_extent(&map[0])) 1463 goto next; 1464 1465 map[1] = map[0]; 1466 while (map[1].br_blockcount) { 1467 agno = XFS_FSB_TO_AGNO(mp, map[1].br_startblock); 1468 agbno = XFS_FSB_TO_AGBNO(mp, map[1].br_startblock); 1469 aglen = map[1].br_blockcount; 1470 1471 error = xfs_reflink_find_shared(mp, NULL, agno, agbno, 1472 aglen, &rbno, &rlen, true); 1473 if (error) 1474 goto out; 1475 if (rbno == NULLAGBLOCK) 1476 break; 1477 1478 /* Dirty the pages */ 1479 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1480 fpos = XFS_FSB_TO_B(mp, map[1].br_startoff + 1481 (rbno - agbno)); 1482 flen = XFS_FSB_TO_B(mp, rlen); 1483 if (fpos + flen > isize) 1484 flen = isize - fpos; 1485 error = iomap_file_dirty(VFS_I(ip), fpos, flen, 1486 &xfs_iomap_ops); 1487 xfs_ilock(ip, XFS_ILOCK_EXCL); 1488 if (error) 1489 goto out; 1490 1491 map[1].br_blockcount -= (rbno - agbno + rlen); 1492 map[1].br_startoff += (rbno - agbno + rlen); 1493 map[1].br_startblock += (rbno - agbno + rlen); 1494 } 1495 1496 next: 1497 fbno = map[0].br_startoff + map[0].br_blockcount; 1498 } 1499 out: 1500 return error; 1501 } 1502 1503 /* Does this inode need the reflink flag? */ 1504 int 1505 xfs_reflink_inode_has_shared_extents( 1506 struct xfs_trans *tp, 1507 struct xfs_inode *ip, 1508 bool *has_shared) 1509 { 1510 struct xfs_bmbt_irec got; 1511 struct xfs_mount *mp = ip->i_mount; 1512 struct xfs_ifork *ifp; 1513 xfs_agnumber_t agno; 1514 xfs_agblock_t agbno; 1515 xfs_extlen_t aglen; 1516 xfs_agblock_t rbno; 1517 xfs_extlen_t rlen; 1518 struct xfs_iext_cursor icur; 1519 bool found; 1520 int error; 1521 1522 ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); 1523 if (!(ifp->if_flags & XFS_IFEXTENTS)) { 1524 error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); 1525 if (error) 1526 return error; 1527 } 1528 1529 *has_shared = false; 1530 found = xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got); 1531 while (found) { 1532 if (isnullstartblock(got.br_startblock) || 1533 got.br_state != XFS_EXT_NORM) 1534 goto next; 1535 agno = XFS_FSB_TO_AGNO(mp, got.br_startblock); 1536 agbno = XFS_FSB_TO_AGBNO(mp, got.br_startblock); 1537 aglen = got.br_blockcount; 1538 1539 error = xfs_reflink_find_shared(mp, tp, agno, agbno, aglen, 1540 &rbno, &rlen, false); 1541 if (error) 1542 return error; 1543 /* Is there still a shared block here? */ 1544 if (rbno != NULLAGBLOCK) { 1545 *has_shared = true; 1546 return 0; 1547 } 1548 next: 1549 found = xfs_iext_next_extent(ifp, &icur, &got); 1550 } 1551 1552 return 0; 1553 } 1554 1555 /* Clear the inode reflink flag if there are no shared extents. */ 1556 int 1557 xfs_reflink_clear_inode_flag( 1558 struct xfs_inode *ip, 1559 struct xfs_trans **tpp) 1560 { 1561 bool needs_flag; 1562 int error = 0; 1563 1564 ASSERT(xfs_is_reflink_inode(ip)); 1565 1566 error = xfs_reflink_inode_has_shared_extents(*tpp, ip, &needs_flag); 1567 if (error || needs_flag) 1568 return error; 1569 1570 /* 1571 * We didn't find any shared blocks so turn off the reflink flag. 1572 * First, get rid of any leftover CoW mappings. 1573 */ 1574 error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, NULLFILEOFF, true); 1575 if (error) 1576 return error; 1577 1578 /* Clear the inode flag. */ 1579 trace_xfs_reflink_unset_inode_flag(ip); 1580 ip->i_d.di_flags2 &= ~XFS_DIFLAG2_REFLINK; 1581 xfs_inode_clear_cowblocks_tag(ip); 1582 xfs_trans_ijoin(*tpp, ip, 0); 1583 xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE); 1584 1585 return error; 1586 } 1587 1588 /* 1589 * Clear the inode reflink flag if there are no shared extents and the size 1590 * hasn't changed. 1591 */ 1592 STATIC int 1593 xfs_reflink_try_clear_inode_flag( 1594 struct xfs_inode *ip) 1595 { 1596 struct xfs_mount *mp = ip->i_mount; 1597 struct xfs_trans *tp; 1598 int error = 0; 1599 1600 /* Start a rolling transaction to remove the mappings */ 1601 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); 1602 if (error) 1603 return error; 1604 1605 xfs_ilock(ip, XFS_ILOCK_EXCL); 1606 xfs_trans_ijoin(tp, ip, 0); 1607 1608 error = xfs_reflink_clear_inode_flag(ip, &tp); 1609 if (error) 1610 goto cancel; 1611 1612 error = xfs_trans_commit(tp); 1613 if (error) 1614 goto out; 1615 1616 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1617 return 0; 1618 cancel: 1619 xfs_trans_cancel(tp); 1620 out: 1621 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1622 return error; 1623 } 1624 1625 /* 1626 * Pre-COW all shared blocks within a given byte range of a file and turn off 1627 * the reflink flag if we unshare all of the file's blocks. 1628 */ 1629 int 1630 xfs_reflink_unshare( 1631 struct xfs_inode *ip, 1632 xfs_off_t offset, 1633 xfs_off_t len) 1634 { 1635 struct xfs_mount *mp = ip->i_mount; 1636 xfs_fileoff_t fbno; 1637 xfs_filblks_t end; 1638 xfs_off_t isize; 1639 int error; 1640 1641 if (!xfs_is_reflink_inode(ip)) 1642 return 0; 1643 1644 trace_xfs_reflink_unshare(ip, offset, len); 1645 1646 inode_dio_wait(VFS_I(ip)); 1647 1648 /* Try to CoW the selected ranges */ 1649 xfs_ilock(ip, XFS_ILOCK_EXCL); 1650 fbno = XFS_B_TO_FSBT(mp, offset); 1651 isize = i_size_read(VFS_I(ip)); 1652 end = XFS_B_TO_FSB(mp, offset + len); 1653 error = xfs_reflink_dirty_extents(ip, fbno, end, isize); 1654 if (error) 1655 goto out_unlock; 1656 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1657 1658 /* Wait for the IO to finish */ 1659 error = filemap_write_and_wait(VFS_I(ip)->i_mapping); 1660 if (error) 1661 goto out; 1662 1663 /* Turn off the reflink flag if possible. */ 1664 error = xfs_reflink_try_clear_inode_flag(ip); 1665 if (error) 1666 goto out; 1667 1668 return 0; 1669 1670 out_unlock: 1671 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1672 out: 1673 trace_xfs_reflink_unshare_error(ip, error, _RET_IP_); 1674 return error; 1675 } 1676