1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2016 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <darrick.wong@oracle.com> 5 */ 6 #include "xfs.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_log_format.h" 11 #include "xfs_trans_resv.h" 12 #include "xfs_bit.h" 13 #include "xfs_sb.h" 14 #include "xfs_mount.h" 15 #include "xfs_defer.h" 16 #include "xfs_trans.h" 17 #include "xfs_buf_item.h" 18 #include "xfs_inode.h" 19 #include "xfs_inode_item.h" 20 #include "xfs_trace.h" 21 22 /* 23 * Deferred Operations in XFS 24 * 25 * Due to the way locking rules work in XFS, certain transactions (block 26 * mapping and unmapping, typically) have permanent reservations so that 27 * we can roll the transaction to adhere to AG locking order rules and 28 * to unlock buffers between metadata updates. Prior to rmap/reflink, 29 * the mapping code had a mechanism to perform these deferrals for 30 * extents that were going to be freed; this code makes that facility 31 * more generic. 32 * 33 * When adding the reverse mapping and reflink features, it became 34 * necessary to perform complex remapping multi-transactions to comply 35 * with AG locking order rules, and to be able to spread a single 36 * refcount update operation (an operation on an n-block extent can 37 * update as many as n records!) among multiple transactions. XFS can 38 * roll a transaction to facilitate this, but using this facility 39 * requires us to log "intent" items in case log recovery needs to 40 * redo the operation, and to log "done" items to indicate that redo 41 * is not necessary. 42 * 43 * Deferred work is tracked in xfs_defer_pending items. Each pending 44 * item tracks one type of deferred work. Incoming work items (which 45 * have not yet had an intent logged) are attached to a pending item 46 * on the dop_intake list, where they wait for the caller to finish 47 * the deferred operations. 48 * 49 * Finishing a set of deferred operations is an involved process. To 50 * start, we define "rolling a deferred-op transaction" as follows: 51 * 52 * > For each xfs_defer_pending item on the dop_intake list, 53 * - Sort the work items in AG order. XFS locking 54 * order rules require us to lock buffers in AG order. 55 * - Create a log intent item for that type. 56 * - Attach it to the pending item. 57 * - Move the pending item from the dop_intake list to the 58 * dop_pending list. 59 * > Roll the transaction. 60 * 61 * NOTE: To avoid exceeding the transaction reservation, we limit the 62 * number of items that we attach to a given xfs_defer_pending. 63 * 64 * The actual finishing process looks like this: 65 * 66 * > For each xfs_defer_pending in the dop_pending list, 67 * - Roll the deferred-op transaction as above. 68 * - Create a log done item for that type, and attach it to the 69 * log intent item. 70 * - For each work item attached to the log intent item, 71 * * Perform the described action. 72 * * Attach the work item to the log done item. 73 * * If the result of doing the work was -EAGAIN, ->finish work 74 * wants a new transaction. See the "Requesting a Fresh 75 * Transaction while Finishing Deferred Work" section below for 76 * details. 77 * 78 * The key here is that we must log an intent item for all pending 79 * work items every time we roll the transaction, and that we must log 80 * a done item as soon as the work is completed. With this mechanism 81 * we can perform complex remapping operations, chaining intent items 82 * as needed. 83 * 84 * Requesting a Fresh Transaction while Finishing Deferred Work 85 * 86 * If ->finish_item decides that it needs a fresh transaction to 87 * finish the work, it must ask its caller (xfs_defer_finish) for a 88 * continuation. The most likely cause of this circumstance are the 89 * refcount adjust functions deciding that they've logged enough items 90 * to be at risk of exceeding the transaction reservation. 91 * 92 * To get a fresh transaction, we want to log the existing log done 93 * item to prevent the log intent item from replaying, immediately log 94 * a new log intent item with the unfinished work items, roll the 95 * transaction, and re-call ->finish_item wherever it left off. The 96 * log done item and the new log intent item must be in the same 97 * transaction or atomicity cannot be guaranteed; defer_finish ensures 98 * that this happens. 99 * 100 * This requires some coordination between ->finish_item and 101 * defer_finish. Upon deciding to request a new transaction, 102 * ->finish_item should update the current work item to reflect the 103 * unfinished work. Next, it should reset the log done item's list 104 * count to the number of items finished, and return -EAGAIN. 105 * defer_finish sees the -EAGAIN, logs the new log intent item 106 * with the remaining work items, and leaves the xfs_defer_pending 107 * item at the head of the dop_work queue. Then it rolls the 108 * transaction and picks up processing where it left off. It is 109 * required that ->finish_item must be careful to leave enough 110 * transaction reservation to fit the new log intent item. 111 * 112 * This is an example of remapping the extent (E, E+B) into file X at 113 * offset A and dealing with the extent (C, C+B) already being mapped 114 * there: 115 * +-------------------------------------------------+ 116 * | Unmap file X startblock C offset A length B | t0 117 * | Intent to reduce refcount for extent (C, B) | 118 * | Intent to remove rmap (X, C, A, B) | 119 * | Intent to free extent (D, 1) (bmbt block) | 120 * | Intent to map (X, A, B) at startblock E | 121 * +-------------------------------------------------+ 122 * | Map file X startblock E offset A length B | t1 123 * | Done mapping (X, E, A, B) | 124 * | Intent to increase refcount for extent (E, B) | 125 * | Intent to add rmap (X, E, A, B) | 126 * +-------------------------------------------------+ 127 * | Reduce refcount for extent (C, B) | t2 128 * | Done reducing refcount for extent (C, 9) | 129 * | Intent to reduce refcount for extent (C+9, B-9) | 130 * | (ran out of space after 9 refcount updates) | 131 * +-------------------------------------------------+ 132 * | Reduce refcount for extent (C+9, B+9) | t3 133 * | Done reducing refcount for extent (C+9, B-9) | 134 * | Increase refcount for extent (E, B) | 135 * | Done increasing refcount for extent (E, B) | 136 * | Intent to free extent (C, B) | 137 * | Intent to free extent (F, 1) (refcountbt block) | 138 * | Intent to remove rmap (F, 1, REFC) | 139 * +-------------------------------------------------+ 140 * | Remove rmap (X, C, A, B) | t4 141 * | Done removing rmap (X, C, A, B) | 142 * | Add rmap (X, E, A, B) | 143 * | Done adding rmap (X, E, A, B) | 144 * | Remove rmap (F, 1, REFC) | 145 * | Done removing rmap (F, 1, REFC) | 146 * +-------------------------------------------------+ 147 * | Free extent (C, B) | t5 148 * | Done freeing extent (C, B) | 149 * | Free extent (D, 1) | 150 * | Done freeing extent (D, 1) | 151 * | Free extent (F, 1) | 152 * | Done freeing extent (F, 1) | 153 * +-------------------------------------------------+ 154 * 155 * If we should crash before t2 commits, log recovery replays 156 * the following intent items: 157 * 158 * - Intent to reduce refcount for extent (C, B) 159 * - Intent to remove rmap (X, C, A, B) 160 * - Intent to free extent (D, 1) (bmbt block) 161 * - Intent to increase refcount for extent (E, B) 162 * - Intent to add rmap (X, E, A, B) 163 * 164 * In the process of recovering, it should also generate and take care 165 * of these intent items: 166 * 167 * - Intent to free extent (C, B) 168 * - Intent to free extent (F, 1) (refcountbt block) 169 * - Intent to remove rmap (F, 1, REFC) 170 * 171 * Note that the continuation requested between t2 and t3 is likely to 172 * reoccur. 173 */ 174 175 static const struct xfs_defer_op_type *defer_op_types[XFS_DEFER_OPS_TYPE_MAX]; 176 177 /* 178 * For each pending item in the intake list, log its intent item and the 179 * associated extents, then add the entire intake list to the end of 180 * the pending list. 181 */ 182 STATIC void 183 xfs_defer_create_intents( 184 struct xfs_trans *tp) 185 { 186 struct xfs_defer_ops *dop = tp->t_dfops; 187 struct list_head *li; 188 struct xfs_defer_pending *dfp; 189 190 list_for_each_entry(dfp, &dop->dop_intake, dfp_list) { 191 dfp->dfp_intent = dfp->dfp_type->create_intent(tp, 192 dfp->dfp_count); 193 trace_xfs_defer_create_intent(tp->t_mountp, dfp); 194 list_sort(tp->t_mountp, &dfp->dfp_work, 195 dfp->dfp_type->diff_items); 196 list_for_each(li, &dfp->dfp_work) 197 dfp->dfp_type->log_item(tp, dfp->dfp_intent, li); 198 } 199 } 200 201 /* Abort all the intents that were committed. */ 202 STATIC void 203 xfs_defer_trans_abort( 204 struct xfs_trans *tp, 205 struct list_head *dop_pending) 206 { 207 struct xfs_defer_ops *dop = tp->t_dfops; 208 struct xfs_defer_pending *dfp; 209 210 trace_xfs_defer_trans_abort(tp->t_mountp, dop, _RET_IP_); 211 212 /* Abort intent items that don't have a done item. */ 213 list_for_each_entry(dfp, dop_pending, dfp_list) { 214 trace_xfs_defer_pending_abort(tp->t_mountp, dfp); 215 if (dfp->dfp_intent && !dfp->dfp_done) { 216 dfp->dfp_type->abort_intent(dfp->dfp_intent); 217 dfp->dfp_intent = NULL; 218 } 219 } 220 } 221 222 /* Roll a transaction so we can do some deferred op processing. */ 223 STATIC int 224 xfs_defer_trans_roll( 225 struct xfs_trans **tpp) 226 { 227 struct xfs_trans *tp = *tpp; 228 struct xfs_buf_log_item *bli; 229 struct xfs_inode_log_item *ili; 230 struct xfs_log_item *lip; 231 struct xfs_buf *bplist[XFS_DEFER_OPS_NR_BUFS]; 232 struct xfs_inode *iplist[XFS_DEFER_OPS_NR_INODES]; 233 int bpcount = 0, ipcount = 0; 234 int i; 235 int error; 236 237 list_for_each_entry(lip, &tp->t_items, li_trans) { 238 switch (lip->li_type) { 239 case XFS_LI_BUF: 240 bli = container_of(lip, struct xfs_buf_log_item, 241 bli_item); 242 if (bli->bli_flags & XFS_BLI_HOLD) { 243 if (bpcount >= XFS_DEFER_OPS_NR_BUFS) { 244 ASSERT(0); 245 return -EFSCORRUPTED; 246 } 247 xfs_trans_dirty_buf(tp, bli->bli_buf); 248 bplist[bpcount++] = bli->bli_buf; 249 } 250 break; 251 case XFS_LI_INODE: 252 ili = container_of(lip, struct xfs_inode_log_item, 253 ili_item); 254 if (ili->ili_lock_flags == 0) { 255 if (ipcount >= XFS_DEFER_OPS_NR_INODES) { 256 ASSERT(0); 257 return -EFSCORRUPTED; 258 } 259 xfs_trans_log_inode(tp, ili->ili_inode, 260 XFS_ILOG_CORE); 261 iplist[ipcount++] = ili->ili_inode; 262 } 263 break; 264 default: 265 break; 266 } 267 } 268 269 trace_xfs_defer_trans_roll(tp->t_mountp, tp->t_dfops, _RET_IP_); 270 271 /* Roll the transaction. */ 272 error = xfs_trans_roll(tpp); 273 tp = *tpp; 274 if (error) { 275 trace_xfs_defer_trans_roll_error(tp->t_mountp, 276 tp->t_dfops, error); 277 return error; 278 } 279 280 /* Rejoin the joined inodes. */ 281 for (i = 0; i < ipcount; i++) 282 xfs_trans_ijoin(tp, iplist[i], 0); 283 284 /* Rejoin the buffers and dirty them so the log moves forward. */ 285 for (i = 0; i < bpcount; i++) { 286 xfs_trans_bjoin(tp, bplist[i]); 287 xfs_trans_bhold(tp, bplist[i]); 288 } 289 290 return error; 291 } 292 293 /* 294 * Reset an already used dfops after finish. 295 */ 296 static void 297 xfs_defer_reset( 298 struct xfs_trans *tp) 299 { 300 ASSERT(list_empty(&tp->t_dfops->dop_intake)); 301 302 /* 303 * Low mode state transfers across transaction rolls to mirror dfops 304 * lifetime. Clear it now that dfops is reset. 305 */ 306 tp->t_flags &= ~XFS_TRANS_LOWMODE; 307 } 308 309 /* 310 * Free up any items left in the list. 311 */ 312 static void 313 xfs_defer_cancel_list( 314 struct xfs_mount *mp, 315 struct list_head *dop_list) 316 { 317 struct xfs_defer_pending *dfp; 318 struct xfs_defer_pending *pli; 319 struct list_head *pwi; 320 struct list_head *n; 321 322 /* 323 * Free the pending items. Caller should already have arranged 324 * for the intent items to be released. 325 */ 326 list_for_each_entry_safe(dfp, pli, dop_list, dfp_list) { 327 trace_xfs_defer_cancel_list(mp, dfp); 328 list_del(&dfp->dfp_list); 329 list_for_each_safe(pwi, n, &dfp->dfp_work) { 330 list_del(pwi); 331 dfp->dfp_count--; 332 dfp->dfp_type->cancel_item(pwi); 333 } 334 ASSERT(dfp->dfp_count == 0); 335 kmem_free(dfp); 336 } 337 } 338 339 /* 340 * Finish all the pending work. This involves logging intent items for 341 * any work items that wandered in since the last transaction roll (if 342 * one has even happened), rolling the transaction, and finishing the 343 * work items in the first item on the logged-and-pending list. 344 * 345 * If an inode is provided, relog it to the new transaction. 346 */ 347 int 348 xfs_defer_finish_noroll( 349 struct xfs_trans **tp) 350 { 351 struct xfs_defer_pending *dfp; 352 struct list_head *li; 353 struct list_head *n; 354 void *state; 355 int error = 0; 356 void (*cleanup_fn)(struct xfs_trans *, void *, int); 357 LIST_HEAD(dop_pending); 358 359 ASSERT((*tp)->t_flags & XFS_TRANS_PERM_LOG_RES); 360 361 trace_xfs_defer_finish((*tp)->t_mountp, (*tp)->t_dfops, _RET_IP_); 362 363 /* Until we run out of pending work to finish... */ 364 while (!list_empty(&dop_pending) || 365 !list_empty(&(*tp)->t_dfops->dop_intake)) { 366 /* log intents and pull in intake items */ 367 xfs_defer_create_intents(*tp); 368 list_splice_tail_init(&(*tp)->t_dfops->dop_intake, 369 &dop_pending); 370 371 /* 372 * Roll the transaction. 373 */ 374 error = xfs_defer_trans_roll(tp); 375 if (error) 376 goto out; 377 378 /* Log an intent-done item for the first pending item. */ 379 dfp = list_first_entry(&dop_pending, struct xfs_defer_pending, 380 dfp_list); 381 trace_xfs_defer_pending_finish((*tp)->t_mountp, dfp); 382 dfp->dfp_done = dfp->dfp_type->create_done(*tp, dfp->dfp_intent, 383 dfp->dfp_count); 384 cleanup_fn = dfp->dfp_type->finish_cleanup; 385 386 /* Finish the work items. */ 387 state = NULL; 388 list_for_each_safe(li, n, &dfp->dfp_work) { 389 list_del(li); 390 dfp->dfp_count--; 391 error = dfp->dfp_type->finish_item(*tp, li, 392 dfp->dfp_done, &state); 393 if (error == -EAGAIN) { 394 /* 395 * Caller wants a fresh transaction; 396 * put the work item back on the list 397 * and jump out. 398 */ 399 list_add(li, &dfp->dfp_work); 400 dfp->dfp_count++; 401 break; 402 } else if (error) { 403 /* 404 * Clean up after ourselves and jump out. 405 * xfs_defer_cancel will take care of freeing 406 * all these lists and stuff. 407 */ 408 if (cleanup_fn) 409 cleanup_fn(*tp, state, error); 410 goto out; 411 } 412 } 413 if (error == -EAGAIN) { 414 /* 415 * Caller wants a fresh transaction, so log a 416 * new log intent item to replace the old one 417 * and roll the transaction. See "Requesting 418 * a Fresh Transaction while Finishing 419 * Deferred Work" above. 420 */ 421 dfp->dfp_intent = dfp->dfp_type->create_intent(*tp, 422 dfp->dfp_count); 423 dfp->dfp_done = NULL; 424 list_for_each(li, &dfp->dfp_work) 425 dfp->dfp_type->log_item(*tp, dfp->dfp_intent, 426 li); 427 } else { 428 /* Done with the dfp, free it. */ 429 list_del(&dfp->dfp_list); 430 kmem_free(dfp); 431 } 432 433 if (cleanup_fn) 434 cleanup_fn(*tp, state, error); 435 } 436 437 out: 438 if (error) { 439 xfs_defer_trans_abort(*tp, &dop_pending); 440 xfs_force_shutdown((*tp)->t_mountp, SHUTDOWN_CORRUPT_INCORE); 441 trace_xfs_defer_finish_error((*tp)->t_mountp, (*tp)->t_dfops, 442 error); 443 xfs_defer_cancel_list((*tp)->t_mountp, &dop_pending); 444 xfs_defer_cancel(*tp); 445 return error; 446 } 447 448 trace_xfs_defer_finish_done((*tp)->t_mountp, (*tp)->t_dfops, _RET_IP_); 449 return 0; 450 } 451 452 int 453 xfs_defer_finish( 454 struct xfs_trans **tp) 455 { 456 int error; 457 458 /* 459 * Finish and roll the transaction once more to avoid returning to the 460 * caller with a dirty transaction. 461 */ 462 error = xfs_defer_finish_noroll(tp); 463 if (error) 464 return error; 465 if ((*tp)->t_flags & XFS_TRANS_DIRTY) { 466 error = xfs_defer_trans_roll(tp); 467 if (error) { 468 xfs_force_shutdown((*tp)->t_mountp, 469 SHUTDOWN_CORRUPT_INCORE); 470 return error; 471 } 472 } 473 xfs_defer_reset(*tp); 474 return 0; 475 } 476 477 void 478 xfs_defer_cancel( 479 struct xfs_trans *tp) 480 { 481 struct xfs_mount *mp = tp->t_mountp; 482 483 trace_xfs_defer_cancel(mp, tp->t_dfops, _RET_IP_); 484 xfs_defer_cancel_list(mp, &tp->t_dfops->dop_intake); 485 } 486 487 /* Add an item for later deferred processing. */ 488 void 489 xfs_defer_add( 490 struct xfs_defer_ops *dop, 491 enum xfs_defer_ops_type type, 492 struct list_head *li) 493 { 494 struct xfs_defer_pending *dfp = NULL; 495 496 /* 497 * Add the item to a pending item at the end of the intake list. 498 * If the last pending item has the same type, reuse it. Else, 499 * create a new pending item at the end of the intake list. 500 */ 501 if (!list_empty(&dop->dop_intake)) { 502 dfp = list_last_entry(&dop->dop_intake, 503 struct xfs_defer_pending, dfp_list); 504 if (dfp->dfp_type->type != type || 505 (dfp->dfp_type->max_items && 506 dfp->dfp_count >= dfp->dfp_type->max_items)) 507 dfp = NULL; 508 } 509 if (!dfp) { 510 dfp = kmem_alloc(sizeof(struct xfs_defer_pending), 511 KM_SLEEP | KM_NOFS); 512 dfp->dfp_type = defer_op_types[type]; 513 dfp->dfp_intent = NULL; 514 dfp->dfp_done = NULL; 515 dfp->dfp_count = 0; 516 INIT_LIST_HEAD(&dfp->dfp_work); 517 list_add_tail(&dfp->dfp_list, &dop->dop_intake); 518 } 519 520 list_add_tail(li, &dfp->dfp_work); 521 dfp->dfp_count++; 522 } 523 524 /* Initialize a deferred operation list. */ 525 void 526 xfs_defer_init_op_type( 527 const struct xfs_defer_op_type *type) 528 { 529 defer_op_types[type->type] = type; 530 } 531 532 /* Initialize a deferred operation. */ 533 void 534 xfs_defer_init( 535 struct xfs_trans *tp, 536 struct xfs_defer_ops *dop) 537 { 538 struct xfs_mount *mp = NULL; 539 540 memset(dop, 0, sizeof(struct xfs_defer_ops)); 541 INIT_LIST_HEAD(&dop->dop_intake); 542 if (tp) { 543 ASSERT(tp->t_firstblock == NULLFSBLOCK); 544 tp->t_dfops = dop; 545 mp = tp->t_mountp; 546 } 547 trace_xfs_defer_init(mp, dop, _RET_IP_); 548 } 549 550 /* 551 * Move state from one xfs_defer_ops to another and reset the source to initial 552 * state. This is primarily used to carry state forward across transaction rolls 553 * with internal dfops. 554 */ 555 void 556 xfs_defer_move( 557 struct xfs_trans *dtp, 558 struct xfs_trans *stp) 559 { 560 struct xfs_defer_ops *dst = dtp->t_dfops; 561 struct xfs_defer_ops *src = stp->t_dfops; 562 ASSERT(dst != src); 563 564 list_splice_init(&src->dop_intake, &dst->dop_intake); 565 566 /* 567 * Low free space mode was historically controlled by a dfops field. 568 * This meant that low mode state potentially carried across multiple 569 * transaction rolls. Transfer low mode on a dfops move to preserve 570 * that behavior. 571 */ 572 dtp->t_flags |= (stp->t_flags & XFS_TRANS_LOWMODE); 573 574 xfs_defer_reset(stp); 575 } 576