1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2016 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <darrick.wong@oracle.com> 5 */ 6 #include "xfs.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_log_format.h" 11 #include "xfs_trans_resv.h" 12 #include "xfs_bit.h" 13 #include "xfs_sb.h" 14 #include "xfs_mount.h" 15 #include "xfs_defer.h" 16 #include "xfs_trans.h" 17 #include "xfs_buf_item.h" 18 #include "xfs_inode.h" 19 #include "xfs_inode_item.h" 20 #include "xfs_trace.h" 21 22 /* 23 * Deferred Operations in XFS 24 * 25 * Due to the way locking rules work in XFS, certain transactions (block 26 * mapping and unmapping, typically) have permanent reservations so that 27 * we can roll the transaction to adhere to AG locking order rules and 28 * to unlock buffers between metadata updates. Prior to rmap/reflink, 29 * the mapping code had a mechanism to perform these deferrals for 30 * extents that were going to be freed; this code makes that facility 31 * more generic. 32 * 33 * When adding the reverse mapping and reflink features, it became 34 * necessary to perform complex remapping multi-transactions to comply 35 * with AG locking order rules, and to be able to spread a single 36 * refcount update operation (an operation on an n-block extent can 37 * update as many as n records!) among multiple transactions. XFS can 38 * roll a transaction to facilitate this, but using this facility 39 * requires us to log "intent" items in case log recovery needs to 40 * redo the operation, and to log "done" items to indicate that redo 41 * is not necessary. 42 * 43 * Deferred work is tracked in xfs_defer_pending items. Each pending 44 * item tracks one type of deferred work. Incoming work items (which 45 * have not yet had an intent logged) are attached to a pending item 46 * on the dop_intake list, where they wait for the caller to finish 47 * the deferred operations. 48 * 49 * Finishing a set of deferred operations is an involved process. To 50 * start, we define "rolling a deferred-op transaction" as follows: 51 * 52 * > For each xfs_defer_pending item on the dop_intake list, 53 * - Sort the work items in AG order. XFS locking 54 * order rules require us to lock buffers in AG order. 55 * - Create a log intent item for that type. 56 * - Attach it to the pending item. 57 * - Move the pending item from the dop_intake list to the 58 * dop_pending list. 59 * > Roll the transaction. 60 * 61 * NOTE: To avoid exceeding the transaction reservation, we limit the 62 * number of items that we attach to a given xfs_defer_pending. 63 * 64 * The actual finishing process looks like this: 65 * 66 * > For each xfs_defer_pending in the dop_pending list, 67 * - Roll the deferred-op transaction as above. 68 * - Create a log done item for that type, and attach it to the 69 * log intent item. 70 * - For each work item attached to the log intent item, 71 * * Perform the described action. 72 * * Attach the work item to the log done item. 73 * * If the result of doing the work was -EAGAIN, ->finish work 74 * wants a new transaction. See the "Requesting a Fresh 75 * Transaction while Finishing Deferred Work" section below for 76 * details. 77 * 78 * The key here is that we must log an intent item for all pending 79 * work items every time we roll the transaction, and that we must log 80 * a done item as soon as the work is completed. With this mechanism 81 * we can perform complex remapping operations, chaining intent items 82 * as needed. 83 * 84 * Requesting a Fresh Transaction while Finishing Deferred Work 85 * 86 * If ->finish_item decides that it needs a fresh transaction to 87 * finish the work, it must ask its caller (xfs_defer_finish) for a 88 * continuation. The most likely cause of this circumstance are the 89 * refcount adjust functions deciding that they've logged enough items 90 * to be at risk of exceeding the transaction reservation. 91 * 92 * To get a fresh transaction, we want to log the existing log done 93 * item to prevent the log intent item from replaying, immediately log 94 * a new log intent item with the unfinished work items, roll the 95 * transaction, and re-call ->finish_item wherever it left off. The 96 * log done item and the new log intent item must be in the same 97 * transaction or atomicity cannot be guaranteed; defer_finish ensures 98 * that this happens. 99 * 100 * This requires some coordination between ->finish_item and 101 * defer_finish. Upon deciding to request a new transaction, 102 * ->finish_item should update the current work item to reflect the 103 * unfinished work. Next, it should reset the log done item's list 104 * count to the number of items finished, and return -EAGAIN. 105 * defer_finish sees the -EAGAIN, logs the new log intent item 106 * with the remaining work items, and leaves the xfs_defer_pending 107 * item at the head of the dop_work queue. Then it rolls the 108 * transaction and picks up processing where it left off. It is 109 * required that ->finish_item must be careful to leave enough 110 * transaction reservation to fit the new log intent item. 111 * 112 * This is an example of remapping the extent (E, E+B) into file X at 113 * offset A and dealing with the extent (C, C+B) already being mapped 114 * there: 115 * +-------------------------------------------------+ 116 * | Unmap file X startblock C offset A length B | t0 117 * | Intent to reduce refcount for extent (C, B) | 118 * | Intent to remove rmap (X, C, A, B) | 119 * | Intent to free extent (D, 1) (bmbt block) | 120 * | Intent to map (X, A, B) at startblock E | 121 * +-------------------------------------------------+ 122 * | Map file X startblock E offset A length B | t1 123 * | Done mapping (X, E, A, B) | 124 * | Intent to increase refcount for extent (E, B) | 125 * | Intent to add rmap (X, E, A, B) | 126 * +-------------------------------------------------+ 127 * | Reduce refcount for extent (C, B) | t2 128 * | Done reducing refcount for extent (C, 9) | 129 * | Intent to reduce refcount for extent (C+9, B-9) | 130 * | (ran out of space after 9 refcount updates) | 131 * +-------------------------------------------------+ 132 * | Reduce refcount for extent (C+9, B+9) | t3 133 * | Done reducing refcount for extent (C+9, B-9) | 134 * | Increase refcount for extent (E, B) | 135 * | Done increasing refcount for extent (E, B) | 136 * | Intent to free extent (C, B) | 137 * | Intent to free extent (F, 1) (refcountbt block) | 138 * | Intent to remove rmap (F, 1, REFC) | 139 * +-------------------------------------------------+ 140 * | Remove rmap (X, C, A, B) | t4 141 * | Done removing rmap (X, C, A, B) | 142 * | Add rmap (X, E, A, B) | 143 * | Done adding rmap (X, E, A, B) | 144 * | Remove rmap (F, 1, REFC) | 145 * | Done removing rmap (F, 1, REFC) | 146 * +-------------------------------------------------+ 147 * | Free extent (C, B) | t5 148 * | Done freeing extent (C, B) | 149 * | Free extent (D, 1) | 150 * | Done freeing extent (D, 1) | 151 * | Free extent (F, 1) | 152 * | Done freeing extent (F, 1) | 153 * +-------------------------------------------------+ 154 * 155 * If we should crash before t2 commits, log recovery replays 156 * the following intent items: 157 * 158 * - Intent to reduce refcount for extent (C, B) 159 * - Intent to remove rmap (X, C, A, B) 160 * - Intent to free extent (D, 1) (bmbt block) 161 * - Intent to increase refcount for extent (E, B) 162 * - Intent to add rmap (X, E, A, B) 163 * 164 * In the process of recovering, it should also generate and take care 165 * of these intent items: 166 * 167 * - Intent to free extent (C, B) 168 * - Intent to free extent (F, 1) (refcountbt block) 169 * - Intent to remove rmap (F, 1, REFC) 170 * 171 * Note that the continuation requested between t2 and t3 is likely to 172 * reoccur. 173 */ 174 175 static const struct xfs_defer_op_type *defer_op_types[XFS_DEFER_OPS_TYPE_MAX]; 176 177 /* 178 * For each pending item in the intake list, log its intent item and the 179 * associated extents, then add the entire intake list to the end of 180 * the pending list. 181 */ 182 STATIC void 183 xfs_defer_intake_work( 184 struct xfs_trans *tp) 185 { 186 struct xfs_defer_ops *dop = tp->t_dfops; 187 struct list_head *li; 188 struct xfs_defer_pending *dfp; 189 190 list_for_each_entry(dfp, &dop->dop_intake, dfp_list) { 191 dfp->dfp_intent = dfp->dfp_type->create_intent(tp, 192 dfp->dfp_count); 193 trace_xfs_defer_intake_work(tp->t_mountp, dfp); 194 list_sort(tp->t_mountp, &dfp->dfp_work, 195 dfp->dfp_type->diff_items); 196 list_for_each(li, &dfp->dfp_work) 197 dfp->dfp_type->log_item(tp, dfp->dfp_intent, li); 198 } 199 200 list_splice_tail_init(&dop->dop_intake, &dop->dop_pending); 201 } 202 203 /* Abort all the intents that were committed. */ 204 STATIC void 205 xfs_defer_trans_abort( 206 struct xfs_trans *tp, 207 int error) 208 { 209 struct xfs_defer_ops *dop = tp->t_dfops; 210 struct xfs_defer_pending *dfp; 211 212 trace_xfs_defer_trans_abort(tp->t_mountp, dop, _RET_IP_); 213 214 /* Abort intent items that don't have a done item. */ 215 list_for_each_entry(dfp, &dop->dop_pending, dfp_list) { 216 trace_xfs_defer_pending_abort(tp->t_mountp, dfp); 217 if (dfp->dfp_intent && !dfp->dfp_done) { 218 dfp->dfp_type->abort_intent(dfp->dfp_intent); 219 dfp->dfp_intent = NULL; 220 } 221 } 222 223 /* Shut down FS. */ 224 xfs_force_shutdown(tp->t_mountp, (error == -EFSCORRUPTED) ? 225 SHUTDOWN_CORRUPT_INCORE : SHUTDOWN_META_IO_ERROR); 226 } 227 228 /* Roll a transaction so we can do some deferred op processing. */ 229 STATIC int 230 xfs_defer_trans_roll( 231 struct xfs_trans **tp) 232 { 233 struct xfs_buf_log_item *bli; 234 struct xfs_inode_log_item *ili; 235 struct xfs_log_item *lip; 236 struct xfs_buf *bplist[XFS_DEFER_OPS_NR_BUFS]; 237 struct xfs_inode *iplist[XFS_DEFER_OPS_NR_INODES]; 238 int bpcount = 0, ipcount = 0; 239 int i; 240 int error; 241 242 list_for_each_entry(lip, &(*tp)->t_items, li_trans) { 243 switch (lip->li_type) { 244 case XFS_LI_BUF: 245 bli = container_of(lip, struct xfs_buf_log_item, 246 bli_item); 247 if (bli->bli_flags & XFS_BLI_HOLD) { 248 if (bpcount >= XFS_DEFER_OPS_NR_BUFS) { 249 ASSERT(0); 250 return -EFSCORRUPTED; 251 } 252 xfs_trans_dirty_buf(*tp, bli->bli_buf); 253 bplist[bpcount++] = bli->bli_buf; 254 } 255 break; 256 case XFS_LI_INODE: 257 ili = container_of(lip, struct xfs_inode_log_item, 258 ili_item); 259 if (ili->ili_lock_flags == 0) { 260 if (ipcount >= XFS_DEFER_OPS_NR_INODES) { 261 ASSERT(0); 262 return -EFSCORRUPTED; 263 } 264 xfs_trans_log_inode(*tp, ili->ili_inode, 265 XFS_ILOG_CORE); 266 iplist[ipcount++] = ili->ili_inode; 267 } 268 break; 269 default: 270 break; 271 } 272 } 273 274 trace_xfs_defer_trans_roll((*tp)->t_mountp, (*tp)->t_dfops, _RET_IP_); 275 276 /* Roll the transaction. */ 277 error = xfs_trans_roll(tp); 278 if (error) { 279 trace_xfs_defer_trans_roll_error((*tp)->t_mountp, 280 (*tp)->t_dfops, error); 281 xfs_defer_trans_abort(*tp, error); 282 return error; 283 } 284 285 /* Rejoin the joined inodes. */ 286 for (i = 0; i < ipcount; i++) 287 xfs_trans_ijoin(*tp, iplist[i], 0); 288 289 /* Rejoin the buffers and dirty them so the log moves forward. */ 290 for (i = 0; i < bpcount; i++) { 291 xfs_trans_bjoin(*tp, bplist[i]); 292 xfs_trans_bhold(*tp, bplist[i]); 293 } 294 295 return error; 296 } 297 298 /* Do we have any work items to finish? */ 299 bool 300 xfs_defer_has_unfinished_work( 301 struct xfs_trans *tp) 302 { 303 return !list_empty(&tp->t_dfops->dop_pending) || 304 !list_empty(&tp->t_dfops->dop_intake); 305 } 306 307 /* 308 * Reset an already used dfops after finish. 309 */ 310 static void 311 xfs_defer_reset( 312 struct xfs_trans *tp) 313 { 314 ASSERT(!xfs_defer_has_unfinished_work(tp)); 315 316 /* 317 * Low mode state transfers across transaction rolls to mirror dfops 318 * lifetime. Clear it now that dfops is reset. 319 */ 320 tp->t_flags &= ~XFS_TRANS_LOWMODE; 321 } 322 323 /* 324 * Finish all the pending work. This involves logging intent items for 325 * any work items that wandered in since the last transaction roll (if 326 * one has even happened), rolling the transaction, and finishing the 327 * work items in the first item on the logged-and-pending list. 328 * 329 * If an inode is provided, relog it to the new transaction. 330 */ 331 int 332 xfs_defer_finish_noroll( 333 struct xfs_trans **tp) 334 { 335 struct xfs_defer_pending *dfp; 336 struct list_head *li; 337 struct list_head *n; 338 void *state; 339 int error = 0; 340 void (*cleanup_fn)(struct xfs_trans *, void *, int); 341 342 ASSERT((*tp)->t_flags & XFS_TRANS_PERM_LOG_RES); 343 344 trace_xfs_defer_finish((*tp)->t_mountp, (*tp)->t_dfops, _RET_IP_); 345 346 /* Until we run out of pending work to finish... */ 347 while (xfs_defer_has_unfinished_work(*tp)) { 348 /* Log intents for work items sitting in the intake. */ 349 xfs_defer_intake_work(*tp); 350 351 /* 352 * Roll the transaction. 353 */ 354 error = xfs_defer_trans_roll(tp); 355 if (error) 356 goto out; 357 358 /* Log an intent-done item for the first pending item. */ 359 dfp = list_first_entry(&(*tp)->t_dfops->dop_pending, 360 struct xfs_defer_pending, dfp_list); 361 trace_xfs_defer_pending_finish((*tp)->t_mountp, dfp); 362 dfp->dfp_done = dfp->dfp_type->create_done(*tp, dfp->dfp_intent, 363 dfp->dfp_count); 364 cleanup_fn = dfp->dfp_type->finish_cleanup; 365 366 /* Finish the work items. */ 367 state = NULL; 368 list_for_each_safe(li, n, &dfp->dfp_work) { 369 list_del(li); 370 dfp->dfp_count--; 371 error = dfp->dfp_type->finish_item(*tp, li, 372 dfp->dfp_done, &state); 373 if (error == -EAGAIN) { 374 /* 375 * Caller wants a fresh transaction; 376 * put the work item back on the list 377 * and jump out. 378 */ 379 list_add(li, &dfp->dfp_work); 380 dfp->dfp_count++; 381 break; 382 } else if (error) { 383 /* 384 * Clean up after ourselves and jump out. 385 * xfs_defer_cancel will take care of freeing 386 * all these lists and stuff. 387 */ 388 if (cleanup_fn) 389 cleanup_fn(*tp, state, error); 390 xfs_defer_trans_abort(*tp, error); 391 goto out; 392 } 393 } 394 if (error == -EAGAIN) { 395 /* 396 * Caller wants a fresh transaction, so log a 397 * new log intent item to replace the old one 398 * and roll the transaction. See "Requesting 399 * a Fresh Transaction while Finishing 400 * Deferred Work" above. 401 */ 402 dfp->dfp_intent = dfp->dfp_type->create_intent(*tp, 403 dfp->dfp_count); 404 dfp->dfp_done = NULL; 405 list_for_each(li, &dfp->dfp_work) 406 dfp->dfp_type->log_item(*tp, dfp->dfp_intent, 407 li); 408 } else { 409 /* Done with the dfp, free it. */ 410 list_del(&dfp->dfp_list); 411 kmem_free(dfp); 412 } 413 414 if (cleanup_fn) 415 cleanup_fn(*tp, state, error); 416 } 417 418 out: 419 if (error) { 420 trace_xfs_defer_finish_error((*tp)->t_mountp, (*tp)->t_dfops, 421 error); 422 xfs_defer_cancel(*tp); 423 return error; 424 } 425 426 trace_xfs_defer_finish_done((*tp)->t_mountp, (*tp)->t_dfops, _RET_IP_); 427 return 0; 428 } 429 430 int 431 xfs_defer_finish( 432 struct xfs_trans **tp) 433 { 434 int error; 435 436 /* 437 * Finish and roll the transaction once more to avoid returning to the 438 * caller with a dirty transaction. 439 */ 440 error = xfs_defer_finish_noroll(tp); 441 if (error) 442 return error; 443 if ((*tp)->t_flags & XFS_TRANS_DIRTY) { 444 error = xfs_defer_trans_roll(tp); 445 if (error) 446 return error; 447 } 448 xfs_defer_reset(*tp); 449 return 0; 450 } 451 452 /* 453 * Free up any items left in the list. 454 */ 455 void 456 xfs_defer_cancel( 457 struct xfs_trans *tp) 458 { 459 struct xfs_defer_ops *dop = tp->t_dfops; 460 struct xfs_defer_pending *dfp; 461 struct xfs_defer_pending *pli; 462 struct list_head *pwi; 463 struct list_head *n; 464 465 trace_xfs_defer_cancel(NULL, dop, _RET_IP_); 466 467 /* 468 * Free the pending items. Caller should already have arranged 469 * for the intent items to be released. 470 */ 471 list_for_each_entry_safe(dfp, pli, &dop->dop_intake, dfp_list) { 472 trace_xfs_defer_intake_cancel(NULL, dfp); 473 list_del(&dfp->dfp_list); 474 list_for_each_safe(pwi, n, &dfp->dfp_work) { 475 list_del(pwi); 476 dfp->dfp_count--; 477 dfp->dfp_type->cancel_item(pwi); 478 } 479 ASSERT(dfp->dfp_count == 0); 480 kmem_free(dfp); 481 } 482 list_for_each_entry_safe(dfp, pli, &dop->dop_pending, dfp_list) { 483 trace_xfs_defer_pending_cancel(NULL, dfp); 484 list_del(&dfp->dfp_list); 485 list_for_each_safe(pwi, n, &dfp->dfp_work) { 486 list_del(pwi); 487 dfp->dfp_count--; 488 dfp->dfp_type->cancel_item(pwi); 489 } 490 ASSERT(dfp->dfp_count == 0); 491 kmem_free(dfp); 492 } 493 } 494 495 /* Add an item for later deferred processing. */ 496 void 497 xfs_defer_add( 498 struct xfs_defer_ops *dop, 499 enum xfs_defer_ops_type type, 500 struct list_head *li) 501 { 502 struct xfs_defer_pending *dfp = NULL; 503 504 /* 505 * Add the item to a pending item at the end of the intake list. 506 * If the last pending item has the same type, reuse it. Else, 507 * create a new pending item at the end of the intake list. 508 */ 509 if (!list_empty(&dop->dop_intake)) { 510 dfp = list_last_entry(&dop->dop_intake, 511 struct xfs_defer_pending, dfp_list); 512 if (dfp->dfp_type->type != type || 513 (dfp->dfp_type->max_items && 514 dfp->dfp_count >= dfp->dfp_type->max_items)) 515 dfp = NULL; 516 } 517 if (!dfp) { 518 dfp = kmem_alloc(sizeof(struct xfs_defer_pending), 519 KM_SLEEP | KM_NOFS); 520 dfp->dfp_type = defer_op_types[type]; 521 dfp->dfp_intent = NULL; 522 dfp->dfp_done = NULL; 523 dfp->dfp_count = 0; 524 INIT_LIST_HEAD(&dfp->dfp_work); 525 list_add_tail(&dfp->dfp_list, &dop->dop_intake); 526 } 527 528 list_add_tail(li, &dfp->dfp_work); 529 dfp->dfp_count++; 530 } 531 532 /* Initialize a deferred operation list. */ 533 void 534 xfs_defer_init_op_type( 535 const struct xfs_defer_op_type *type) 536 { 537 defer_op_types[type->type] = type; 538 } 539 540 /* Initialize a deferred operation. */ 541 void 542 xfs_defer_init( 543 struct xfs_trans *tp, 544 struct xfs_defer_ops *dop) 545 { 546 struct xfs_mount *mp = NULL; 547 548 memset(dop, 0, sizeof(struct xfs_defer_ops)); 549 INIT_LIST_HEAD(&dop->dop_intake); 550 INIT_LIST_HEAD(&dop->dop_pending); 551 if (tp) { 552 ASSERT(tp->t_firstblock == NULLFSBLOCK); 553 tp->t_dfops = dop; 554 mp = tp->t_mountp; 555 } 556 trace_xfs_defer_init(mp, dop, _RET_IP_); 557 } 558 559 /* 560 * Move state from one xfs_defer_ops to another and reset the source to initial 561 * state. This is primarily used to carry state forward across transaction rolls 562 * with internal dfops. 563 */ 564 void 565 xfs_defer_move( 566 struct xfs_trans *dtp, 567 struct xfs_trans *stp) 568 { 569 struct xfs_defer_ops *dst = dtp->t_dfops; 570 struct xfs_defer_ops *src = stp->t_dfops; 571 ASSERT(dst != src); 572 573 list_splice_init(&src->dop_intake, &dst->dop_intake); 574 list_splice_init(&src->dop_pending, &dst->dop_pending); 575 576 /* 577 * Low free space mode was historically controlled by a dfops field. 578 * This meant that low mode state potentially carried across multiple 579 * transaction rolls. Transfer low mode on a dfops move to preserve 580 * that behavior. 581 */ 582 dtp->t_flags |= (stp->t_flags & XFS_TRANS_LOWMODE); 583 584 xfs_defer_reset(stp); 585 } 586