1 /* 2 * Copyright (c) 2000-2005 Silicon Graphics, Inc. 3 * All Rights Reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it would be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 #include "xfs.h" 19 #include "xfs_fs.h" 20 #include "xfs_log_format.h" 21 #include "xfs_trans_resv.h" 22 #include "xfs_bit.h" 23 #include "xfs_sb.h" 24 #include "xfs_ag.h" 25 #include "xfs_mount.h" 26 #include "xfs_trans.h" 27 #include "xfs_buf_item.h" 28 #include "xfs_trans_priv.h" 29 #include "xfs_error.h" 30 #include "xfs_trace.h" 31 #include "xfs_log.h" 32 33 34 kmem_zone_t *xfs_buf_item_zone; 35 36 static inline struct xfs_buf_log_item *BUF_ITEM(struct xfs_log_item *lip) 37 { 38 return container_of(lip, struct xfs_buf_log_item, bli_item); 39 } 40 41 STATIC void xfs_buf_do_callbacks(struct xfs_buf *bp); 42 43 static inline int 44 xfs_buf_log_format_size( 45 struct xfs_buf_log_format *blfp) 46 { 47 return offsetof(struct xfs_buf_log_format, blf_data_map) + 48 (blfp->blf_map_size * sizeof(blfp->blf_data_map[0])); 49 } 50 51 /* 52 * This returns the number of log iovecs needed to log the 53 * given buf log item. 54 * 55 * It calculates this as 1 iovec for the buf log format structure 56 * and 1 for each stretch of non-contiguous chunks to be logged. 57 * Contiguous chunks are logged in a single iovec. 58 * 59 * If the XFS_BLI_STALE flag has been set, then log nothing. 60 */ 61 STATIC void 62 xfs_buf_item_size_segment( 63 struct xfs_buf_log_item *bip, 64 struct xfs_buf_log_format *blfp, 65 int *nvecs, 66 int *nbytes) 67 { 68 struct xfs_buf *bp = bip->bli_buf; 69 int next_bit; 70 int last_bit; 71 72 last_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0); 73 if (last_bit == -1) 74 return; 75 76 /* 77 * initial count for a dirty buffer is 2 vectors - the format structure 78 * and the first dirty region. 79 */ 80 *nvecs += 2; 81 *nbytes += xfs_buf_log_format_size(blfp) + XFS_BLF_CHUNK; 82 83 while (last_bit != -1) { 84 /* 85 * This takes the bit number to start looking from and 86 * returns the next set bit from there. It returns -1 87 * if there are no more bits set or the start bit is 88 * beyond the end of the bitmap. 89 */ 90 next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 91 last_bit + 1); 92 /* 93 * If we run out of bits, leave the loop, 94 * else if we find a new set of bits bump the number of vecs, 95 * else keep scanning the current set of bits. 96 */ 97 if (next_bit == -1) { 98 break; 99 } else if (next_bit != last_bit + 1) { 100 last_bit = next_bit; 101 (*nvecs)++; 102 } else if (xfs_buf_offset(bp, next_bit * XFS_BLF_CHUNK) != 103 (xfs_buf_offset(bp, last_bit * XFS_BLF_CHUNK) + 104 XFS_BLF_CHUNK)) { 105 last_bit = next_bit; 106 (*nvecs)++; 107 } else { 108 last_bit++; 109 } 110 *nbytes += XFS_BLF_CHUNK; 111 } 112 } 113 114 /* 115 * This returns the number of log iovecs needed to log the given buf log item. 116 * 117 * It calculates this as 1 iovec for the buf log format structure and 1 for each 118 * stretch of non-contiguous chunks to be logged. Contiguous chunks are logged 119 * in a single iovec. 120 * 121 * Discontiguous buffers need a format structure per region that that is being 122 * logged. This makes the changes in the buffer appear to log recovery as though 123 * they came from separate buffers, just like would occur if multiple buffers 124 * were used instead of a single discontiguous buffer. This enables 125 * discontiguous buffers to be in-memory constructs, completely transparent to 126 * what ends up on disk. 127 * 128 * If the XFS_BLI_STALE flag has been set, then log nothing but the buf log 129 * format structures. 130 */ 131 STATIC void 132 xfs_buf_item_size( 133 struct xfs_log_item *lip, 134 int *nvecs, 135 int *nbytes) 136 { 137 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 138 int i; 139 140 ASSERT(atomic_read(&bip->bli_refcount) > 0); 141 if (bip->bli_flags & XFS_BLI_STALE) { 142 /* 143 * The buffer is stale, so all we need to log 144 * is the buf log format structure with the 145 * cancel flag in it. 146 */ 147 trace_xfs_buf_item_size_stale(bip); 148 ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL); 149 *nvecs += bip->bli_format_count; 150 for (i = 0; i < bip->bli_format_count; i++) { 151 *nbytes += xfs_buf_log_format_size(&bip->bli_formats[i]); 152 } 153 return; 154 } 155 156 ASSERT(bip->bli_flags & XFS_BLI_LOGGED); 157 158 if (bip->bli_flags & XFS_BLI_ORDERED) { 159 /* 160 * The buffer has been logged just to order it. 161 * It is not being included in the transaction 162 * commit, so no vectors are used at all. 163 */ 164 trace_xfs_buf_item_size_ordered(bip); 165 *nvecs = XFS_LOG_VEC_ORDERED; 166 return; 167 } 168 169 /* 170 * the vector count is based on the number of buffer vectors we have 171 * dirty bits in. This will only be greater than one when we have a 172 * compound buffer with more than one segment dirty. Hence for compound 173 * buffers we need to track which segment the dirty bits correspond to, 174 * and when we move from one segment to the next increment the vector 175 * count for the extra buf log format structure that will need to be 176 * written. 177 */ 178 for (i = 0; i < bip->bli_format_count; i++) { 179 xfs_buf_item_size_segment(bip, &bip->bli_formats[i], 180 nvecs, nbytes); 181 } 182 trace_xfs_buf_item_size(bip); 183 } 184 185 static struct xfs_log_iovec * 186 xfs_buf_item_format_segment( 187 struct xfs_buf_log_item *bip, 188 struct xfs_log_iovec *vecp, 189 uint offset, 190 struct xfs_buf_log_format *blfp) 191 { 192 struct xfs_buf *bp = bip->bli_buf; 193 uint base_size; 194 uint nvecs; 195 int first_bit; 196 int last_bit; 197 int next_bit; 198 uint nbits; 199 uint buffer_offset; 200 201 /* copy the flags across from the base format item */ 202 blfp->blf_flags = bip->__bli_format.blf_flags; 203 204 /* 205 * Base size is the actual size of the ondisk structure - it reflects 206 * the actual size of the dirty bitmap rather than the size of the in 207 * memory structure. 208 */ 209 base_size = xfs_buf_log_format_size(blfp); 210 211 nvecs = 0; 212 first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0); 213 if (!(bip->bli_flags & XFS_BLI_STALE) && first_bit == -1) { 214 /* 215 * If the map is not be dirty in the transaction, mark 216 * the size as zero and do not advance the vector pointer. 217 */ 218 goto out; 219 } 220 221 vecp->i_addr = blfp; 222 vecp->i_len = base_size; 223 vecp->i_type = XLOG_REG_TYPE_BFORMAT; 224 vecp++; 225 nvecs = 1; 226 227 if (bip->bli_flags & XFS_BLI_STALE) { 228 /* 229 * The buffer is stale, so all we need to log 230 * is the buf log format structure with the 231 * cancel flag in it. 232 */ 233 trace_xfs_buf_item_format_stale(bip); 234 ASSERT(blfp->blf_flags & XFS_BLF_CANCEL); 235 goto out; 236 } 237 238 239 /* 240 * Fill in an iovec for each set of contiguous chunks. 241 */ 242 243 last_bit = first_bit; 244 nbits = 1; 245 for (;;) { 246 /* 247 * This takes the bit number to start looking from and 248 * returns the next set bit from there. It returns -1 249 * if there are no more bits set or the start bit is 250 * beyond the end of the bitmap. 251 */ 252 next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 253 (uint)last_bit + 1); 254 /* 255 * If we run out of bits fill in the last iovec and get 256 * out of the loop. 257 * Else if we start a new set of bits then fill in the 258 * iovec for the series we were looking at and start 259 * counting the bits in the new one. 260 * Else we're still in the same set of bits so just 261 * keep counting and scanning. 262 */ 263 if (next_bit == -1) { 264 buffer_offset = offset + first_bit * XFS_BLF_CHUNK; 265 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 266 vecp->i_len = nbits * XFS_BLF_CHUNK; 267 vecp->i_type = XLOG_REG_TYPE_BCHUNK; 268 nvecs++; 269 break; 270 } else if (next_bit != last_bit + 1) { 271 buffer_offset = offset + first_bit * XFS_BLF_CHUNK; 272 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 273 vecp->i_len = nbits * XFS_BLF_CHUNK; 274 vecp->i_type = XLOG_REG_TYPE_BCHUNK; 275 nvecs++; 276 vecp++; 277 first_bit = next_bit; 278 last_bit = next_bit; 279 nbits = 1; 280 } else if (xfs_buf_offset(bp, offset + 281 (next_bit << XFS_BLF_SHIFT)) != 282 (xfs_buf_offset(bp, offset + 283 (last_bit << XFS_BLF_SHIFT)) + 284 XFS_BLF_CHUNK)) { 285 buffer_offset = offset + first_bit * XFS_BLF_CHUNK; 286 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 287 vecp->i_len = nbits * XFS_BLF_CHUNK; 288 vecp->i_type = XLOG_REG_TYPE_BCHUNK; 289 nvecs++; 290 vecp++; 291 first_bit = next_bit; 292 last_bit = next_bit; 293 nbits = 1; 294 } else { 295 last_bit++; 296 nbits++; 297 } 298 } 299 out: 300 blfp->blf_size = nvecs; 301 return vecp; 302 } 303 304 /* 305 * This is called to fill in the vector of log iovecs for the 306 * given log buf item. It fills the first entry with a buf log 307 * format structure, and the rest point to contiguous chunks 308 * within the buffer. 309 */ 310 STATIC void 311 xfs_buf_item_format( 312 struct xfs_log_item *lip, 313 struct xfs_log_iovec *vecp) 314 { 315 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 316 struct xfs_buf *bp = bip->bli_buf; 317 uint offset = 0; 318 int i; 319 320 ASSERT(atomic_read(&bip->bli_refcount) > 0); 321 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || 322 (bip->bli_flags & XFS_BLI_STALE)); 323 324 /* 325 * If it is an inode buffer, transfer the in-memory state to the 326 * format flags and clear the in-memory state. 327 * 328 * For buffer based inode allocation, we do not transfer 329 * this state if the inode buffer allocation has not yet been committed 330 * to the log as setting the XFS_BLI_INODE_BUF flag will prevent 331 * correct replay of the inode allocation. 332 * 333 * For icreate item based inode allocation, the buffers aren't written 334 * to the journal during allocation, and hence we should always tag the 335 * buffer as an inode buffer so that the correct unlinked list replay 336 * occurs during recovery. 337 */ 338 if (bip->bli_flags & XFS_BLI_INODE_BUF) { 339 if (xfs_sb_version_hascrc(&lip->li_mountp->m_sb) || 340 !((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && 341 xfs_log_item_in_current_chkpt(lip))) 342 bip->__bli_format.blf_flags |= XFS_BLF_INODE_BUF; 343 bip->bli_flags &= ~XFS_BLI_INODE_BUF; 344 } 345 346 if ((bip->bli_flags & (XFS_BLI_ORDERED|XFS_BLI_STALE)) == 347 XFS_BLI_ORDERED) { 348 /* 349 * The buffer has been logged just to order it. It is not being 350 * included in the transaction commit, so don't format it. 351 */ 352 trace_xfs_buf_item_format_ordered(bip); 353 return; 354 } 355 356 for (i = 0; i < bip->bli_format_count; i++) { 357 vecp = xfs_buf_item_format_segment(bip, vecp, offset, 358 &bip->bli_formats[i]); 359 offset += bp->b_maps[i].bm_len; 360 } 361 362 /* 363 * Check to make sure everything is consistent. 364 */ 365 trace_xfs_buf_item_format(bip); 366 } 367 368 /* 369 * This is called to pin the buffer associated with the buf log item in memory 370 * so it cannot be written out. 371 * 372 * We also always take a reference to the buffer log item here so that the bli 373 * is held while the item is pinned in memory. This means that we can 374 * unconditionally drop the reference count a transaction holds when the 375 * transaction is completed. 376 */ 377 STATIC void 378 xfs_buf_item_pin( 379 struct xfs_log_item *lip) 380 { 381 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 382 383 ASSERT(atomic_read(&bip->bli_refcount) > 0); 384 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || 385 (bip->bli_flags & XFS_BLI_ORDERED) || 386 (bip->bli_flags & XFS_BLI_STALE)); 387 388 trace_xfs_buf_item_pin(bip); 389 390 atomic_inc(&bip->bli_refcount); 391 atomic_inc(&bip->bli_buf->b_pin_count); 392 } 393 394 /* 395 * This is called to unpin the buffer associated with the buf log 396 * item which was previously pinned with a call to xfs_buf_item_pin(). 397 * 398 * Also drop the reference to the buf item for the current transaction. 399 * If the XFS_BLI_STALE flag is set and we are the last reference, 400 * then free up the buf log item and unlock the buffer. 401 * 402 * If the remove flag is set we are called from uncommit in the 403 * forced-shutdown path. If that is true and the reference count on 404 * the log item is going to drop to zero we need to free the item's 405 * descriptor in the transaction. 406 */ 407 STATIC void 408 xfs_buf_item_unpin( 409 struct xfs_log_item *lip, 410 int remove) 411 { 412 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 413 xfs_buf_t *bp = bip->bli_buf; 414 struct xfs_ail *ailp = lip->li_ailp; 415 int stale = bip->bli_flags & XFS_BLI_STALE; 416 int freed; 417 418 ASSERT(bp->b_fspriv == bip); 419 ASSERT(atomic_read(&bip->bli_refcount) > 0); 420 421 trace_xfs_buf_item_unpin(bip); 422 423 freed = atomic_dec_and_test(&bip->bli_refcount); 424 425 if (atomic_dec_and_test(&bp->b_pin_count)) 426 wake_up_all(&bp->b_waiters); 427 428 if (freed && stale) { 429 ASSERT(bip->bli_flags & XFS_BLI_STALE); 430 ASSERT(xfs_buf_islocked(bp)); 431 ASSERT(XFS_BUF_ISSTALE(bp)); 432 ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL); 433 434 trace_xfs_buf_item_unpin_stale(bip); 435 436 if (remove) { 437 /* 438 * If we are in a transaction context, we have to 439 * remove the log item from the transaction as we are 440 * about to release our reference to the buffer. If we 441 * don't, the unlock that occurs later in 442 * xfs_trans_uncommit() will try to reference the 443 * buffer which we no longer have a hold on. 444 */ 445 if (lip->li_desc) 446 xfs_trans_del_item(lip); 447 448 /* 449 * Since the transaction no longer refers to the buffer, 450 * the buffer should no longer refer to the transaction. 451 */ 452 bp->b_transp = NULL; 453 } 454 455 /* 456 * If we get called here because of an IO error, we may 457 * or may not have the item on the AIL. xfs_trans_ail_delete() 458 * will take care of that situation. 459 * xfs_trans_ail_delete() drops the AIL lock. 460 */ 461 if (bip->bli_flags & XFS_BLI_STALE_INODE) { 462 xfs_buf_do_callbacks(bp); 463 bp->b_fspriv = NULL; 464 bp->b_iodone = NULL; 465 } else { 466 spin_lock(&ailp->xa_lock); 467 xfs_trans_ail_delete(ailp, lip, SHUTDOWN_LOG_IO_ERROR); 468 xfs_buf_item_relse(bp); 469 ASSERT(bp->b_fspriv == NULL); 470 } 471 xfs_buf_relse(bp); 472 } else if (freed && remove) { 473 /* 474 * There are currently two references to the buffer - the active 475 * LRU reference and the buf log item. What we are about to do 476 * here - simulate a failed IO completion - requires 3 477 * references. 478 * 479 * The LRU reference is removed by the xfs_buf_stale() call. The 480 * buf item reference is removed by the xfs_buf_iodone() 481 * callback that is run by xfs_buf_do_callbacks() during ioend 482 * processing (via the bp->b_iodone callback), and then finally 483 * the ioend processing will drop the IO reference if the buffer 484 * is marked XBF_ASYNC. 485 * 486 * Hence we need to take an additional reference here so that IO 487 * completion processing doesn't free the buffer prematurely. 488 */ 489 xfs_buf_lock(bp); 490 xfs_buf_hold(bp); 491 bp->b_flags |= XBF_ASYNC; 492 xfs_buf_ioerror(bp, EIO); 493 XFS_BUF_UNDONE(bp); 494 xfs_buf_stale(bp); 495 xfs_buf_ioend(bp, 0); 496 } 497 } 498 499 /* 500 * Buffer IO error rate limiting. Limit it to no more than 10 messages per 30 501 * seconds so as to not spam logs too much on repeated detection of the same 502 * buffer being bad.. 503 */ 504 505 DEFINE_RATELIMIT_STATE(xfs_buf_write_fail_rl_state, 30 * HZ, 10); 506 507 STATIC uint 508 xfs_buf_item_push( 509 struct xfs_log_item *lip, 510 struct list_head *buffer_list) 511 { 512 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 513 struct xfs_buf *bp = bip->bli_buf; 514 uint rval = XFS_ITEM_SUCCESS; 515 516 if (xfs_buf_ispinned(bp)) 517 return XFS_ITEM_PINNED; 518 if (!xfs_buf_trylock(bp)) { 519 /* 520 * If we have just raced with a buffer being pinned and it has 521 * been marked stale, we could end up stalling until someone else 522 * issues a log force to unpin the stale buffer. Check for the 523 * race condition here so xfsaild recognizes the buffer is pinned 524 * and queues a log force to move it along. 525 */ 526 if (xfs_buf_ispinned(bp)) 527 return XFS_ITEM_PINNED; 528 return XFS_ITEM_LOCKED; 529 } 530 531 ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); 532 533 trace_xfs_buf_item_push(bip); 534 535 /* has a previous flush failed due to IO errors? */ 536 if ((bp->b_flags & XBF_WRITE_FAIL) && 537 ___ratelimit(&xfs_buf_write_fail_rl_state, "XFS:")) { 538 xfs_warn(bp->b_target->bt_mount, 539 "Detected failing async write on buffer block 0x%llx. Retrying async write.\n", 540 (long long)bp->b_bn); 541 } 542 543 if (!xfs_buf_delwri_queue(bp, buffer_list)) 544 rval = XFS_ITEM_FLUSHING; 545 xfs_buf_unlock(bp); 546 return rval; 547 } 548 549 /* 550 * Release the buffer associated with the buf log item. If there is no dirty 551 * logged data associated with the buffer recorded in the buf log item, then 552 * free the buf log item and remove the reference to it in the buffer. 553 * 554 * This call ignores the recursion count. It is only called when the buffer 555 * should REALLY be unlocked, regardless of the recursion count. 556 * 557 * We unconditionally drop the transaction's reference to the log item. If the 558 * item was logged, then another reference was taken when it was pinned, so we 559 * can safely drop the transaction reference now. This also allows us to avoid 560 * potential races with the unpin code freeing the bli by not referencing the 561 * bli after we've dropped the reference count. 562 * 563 * If the XFS_BLI_HOLD flag is set in the buf log item, then free the log item 564 * if necessary but do not unlock the buffer. This is for support of 565 * xfs_trans_bhold(). Make sure the XFS_BLI_HOLD field is cleared if we don't 566 * free the item. 567 */ 568 STATIC void 569 xfs_buf_item_unlock( 570 struct xfs_log_item *lip) 571 { 572 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 573 struct xfs_buf *bp = bip->bli_buf; 574 bool clean; 575 bool aborted; 576 int flags; 577 578 /* Clear the buffer's association with this transaction. */ 579 bp->b_transp = NULL; 580 581 /* 582 * If this is a transaction abort, don't return early. Instead, allow 583 * the brelse to happen. Normally it would be done for stale 584 * (cancelled) buffers at unpin time, but we'll never go through the 585 * pin/unpin cycle if we abort inside commit. 586 */ 587 aborted = (lip->li_flags & XFS_LI_ABORTED) ? true : false; 588 /* 589 * Before possibly freeing the buf item, copy the per-transaction state 590 * so we can reference it safely later after clearing it from the 591 * buffer log item. 592 */ 593 flags = bip->bli_flags; 594 bip->bli_flags &= ~(XFS_BLI_LOGGED | XFS_BLI_HOLD | XFS_BLI_ORDERED); 595 596 /* 597 * If the buf item is marked stale, then don't do anything. We'll 598 * unlock the buffer and free the buf item when the buffer is unpinned 599 * for the last time. 600 */ 601 if (flags & XFS_BLI_STALE) { 602 trace_xfs_buf_item_unlock_stale(bip); 603 ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL); 604 if (!aborted) { 605 atomic_dec(&bip->bli_refcount); 606 return; 607 } 608 } 609 610 trace_xfs_buf_item_unlock(bip); 611 612 /* 613 * If the buf item isn't tracking any data, free it, otherwise drop the 614 * reference we hold to it. If we are aborting the transaction, this may 615 * be the only reference to the buf item, so we free it anyway 616 * regardless of whether it is dirty or not. A dirty abort implies a 617 * shutdown, anyway. 618 * 619 * Ordered buffers are dirty but may have no recorded changes, so ensure 620 * we only release clean items here. 621 */ 622 clean = (flags & XFS_BLI_DIRTY) ? false : true; 623 if (clean) { 624 int i; 625 for (i = 0; i < bip->bli_format_count; i++) { 626 if (!xfs_bitmap_empty(bip->bli_formats[i].blf_data_map, 627 bip->bli_formats[i].blf_map_size)) { 628 clean = false; 629 break; 630 } 631 } 632 } 633 634 /* 635 * Clean buffers, by definition, cannot be in the AIL. However, aborted 636 * buffers may be dirty and hence in the AIL. Therefore if we are 637 * aborting a buffer and we've just taken the last refernce away, we 638 * have to check if it is in the AIL before freeing it. We need to free 639 * it in this case, because an aborted transaction has already shut the 640 * filesystem down and this is the last chance we will have to do so. 641 */ 642 if (atomic_dec_and_test(&bip->bli_refcount)) { 643 if (clean) 644 xfs_buf_item_relse(bp); 645 else if (aborted) { 646 ASSERT(XFS_FORCED_SHUTDOWN(lip->li_mountp)); 647 if (lip->li_flags & XFS_LI_IN_AIL) { 648 spin_lock(&lip->li_ailp->xa_lock); 649 xfs_trans_ail_delete(lip->li_ailp, lip, 650 SHUTDOWN_LOG_IO_ERROR); 651 } 652 xfs_buf_item_relse(bp); 653 } 654 } 655 656 if (!(flags & XFS_BLI_HOLD)) 657 xfs_buf_relse(bp); 658 } 659 660 /* 661 * This is called to find out where the oldest active copy of the 662 * buf log item in the on disk log resides now that the last log 663 * write of it completed at the given lsn. 664 * We always re-log all the dirty data in a buffer, so usually the 665 * latest copy in the on disk log is the only one that matters. For 666 * those cases we simply return the given lsn. 667 * 668 * The one exception to this is for buffers full of newly allocated 669 * inodes. These buffers are only relogged with the XFS_BLI_INODE_BUF 670 * flag set, indicating that only the di_next_unlinked fields from the 671 * inodes in the buffers will be replayed during recovery. If the 672 * original newly allocated inode images have not yet been flushed 673 * when the buffer is so relogged, then we need to make sure that we 674 * keep the old images in the 'active' portion of the log. We do this 675 * by returning the original lsn of that transaction here rather than 676 * the current one. 677 */ 678 STATIC xfs_lsn_t 679 xfs_buf_item_committed( 680 struct xfs_log_item *lip, 681 xfs_lsn_t lsn) 682 { 683 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 684 685 trace_xfs_buf_item_committed(bip); 686 687 if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && lip->li_lsn != 0) 688 return lip->li_lsn; 689 return lsn; 690 } 691 692 STATIC void 693 xfs_buf_item_committing( 694 struct xfs_log_item *lip, 695 xfs_lsn_t commit_lsn) 696 { 697 } 698 699 /* 700 * This is the ops vector shared by all buf log items. 701 */ 702 static const struct xfs_item_ops xfs_buf_item_ops = { 703 .iop_size = xfs_buf_item_size, 704 .iop_format = xfs_buf_item_format, 705 .iop_pin = xfs_buf_item_pin, 706 .iop_unpin = xfs_buf_item_unpin, 707 .iop_unlock = xfs_buf_item_unlock, 708 .iop_committed = xfs_buf_item_committed, 709 .iop_push = xfs_buf_item_push, 710 .iop_committing = xfs_buf_item_committing 711 }; 712 713 STATIC int 714 xfs_buf_item_get_format( 715 struct xfs_buf_log_item *bip, 716 int count) 717 { 718 ASSERT(bip->bli_formats == NULL); 719 bip->bli_format_count = count; 720 721 if (count == 1) { 722 bip->bli_formats = &bip->__bli_format; 723 return 0; 724 } 725 726 bip->bli_formats = kmem_zalloc(count * sizeof(struct xfs_buf_log_format), 727 KM_SLEEP); 728 if (!bip->bli_formats) 729 return ENOMEM; 730 return 0; 731 } 732 733 STATIC void 734 xfs_buf_item_free_format( 735 struct xfs_buf_log_item *bip) 736 { 737 if (bip->bli_formats != &bip->__bli_format) { 738 kmem_free(bip->bli_formats); 739 bip->bli_formats = NULL; 740 } 741 } 742 743 /* 744 * Allocate a new buf log item to go with the given buffer. 745 * Set the buffer's b_fsprivate field to point to the new 746 * buf log item. If there are other item's attached to the 747 * buffer (see xfs_buf_attach_iodone() below), then put the 748 * buf log item at the front. 749 */ 750 void 751 xfs_buf_item_init( 752 xfs_buf_t *bp, 753 xfs_mount_t *mp) 754 { 755 xfs_log_item_t *lip = bp->b_fspriv; 756 xfs_buf_log_item_t *bip; 757 int chunks; 758 int map_size; 759 int error; 760 int i; 761 762 /* 763 * Check to see if there is already a buf log item for 764 * this buffer. If there is, it is guaranteed to be 765 * the first. If we do already have one, there is 766 * nothing to do here so return. 767 */ 768 ASSERT(bp->b_target->bt_mount == mp); 769 if (lip != NULL && lip->li_type == XFS_LI_BUF) 770 return; 771 772 bip = kmem_zone_zalloc(xfs_buf_item_zone, KM_SLEEP); 773 xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops); 774 bip->bli_buf = bp; 775 xfs_buf_hold(bp); 776 777 /* 778 * chunks is the number of XFS_BLF_CHUNK size pieces the buffer 779 * can be divided into. Make sure not to truncate any pieces. 780 * map_size is the size of the bitmap needed to describe the 781 * chunks of the buffer. 782 * 783 * Discontiguous buffer support follows the layout of the underlying 784 * buffer. This makes the implementation as simple as possible. 785 */ 786 error = xfs_buf_item_get_format(bip, bp->b_map_count); 787 ASSERT(error == 0); 788 789 for (i = 0; i < bip->bli_format_count; i++) { 790 chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len), 791 XFS_BLF_CHUNK); 792 map_size = DIV_ROUND_UP(chunks, NBWORD); 793 794 bip->bli_formats[i].blf_type = XFS_LI_BUF; 795 bip->bli_formats[i].blf_blkno = bp->b_maps[i].bm_bn; 796 bip->bli_formats[i].blf_len = bp->b_maps[i].bm_len; 797 bip->bli_formats[i].blf_map_size = map_size; 798 } 799 800 #ifdef XFS_TRANS_DEBUG 801 /* 802 * Allocate the arrays for tracking what needs to be logged 803 * and what our callers request to be logged. bli_orig 804 * holds a copy of the original, clean buffer for comparison 805 * against, and bli_logged keeps a 1 bit flag per byte in 806 * the buffer to indicate which bytes the callers have asked 807 * to have logged. 808 */ 809 bip->bli_orig = kmem_alloc(BBTOB(bp->b_length), KM_SLEEP); 810 memcpy(bip->bli_orig, bp->b_addr, BBTOB(bp->b_length)); 811 bip->bli_logged = kmem_zalloc(BBTOB(bp->b_length) / NBBY, KM_SLEEP); 812 #endif 813 814 /* 815 * Put the buf item into the list of items attached to the 816 * buffer at the front. 817 */ 818 if (bp->b_fspriv) 819 bip->bli_item.li_bio_list = bp->b_fspriv; 820 bp->b_fspriv = bip; 821 } 822 823 824 /* 825 * Mark bytes first through last inclusive as dirty in the buf 826 * item's bitmap. 827 */ 828 static void 829 xfs_buf_item_log_segment( 830 struct xfs_buf_log_item *bip, 831 uint first, 832 uint last, 833 uint *map) 834 { 835 uint first_bit; 836 uint last_bit; 837 uint bits_to_set; 838 uint bits_set; 839 uint word_num; 840 uint *wordp; 841 uint bit; 842 uint end_bit; 843 uint mask; 844 845 /* 846 * Convert byte offsets to bit numbers. 847 */ 848 first_bit = first >> XFS_BLF_SHIFT; 849 last_bit = last >> XFS_BLF_SHIFT; 850 851 /* 852 * Calculate the total number of bits to be set. 853 */ 854 bits_to_set = last_bit - first_bit + 1; 855 856 /* 857 * Get a pointer to the first word in the bitmap 858 * to set a bit in. 859 */ 860 word_num = first_bit >> BIT_TO_WORD_SHIFT; 861 wordp = &map[word_num]; 862 863 /* 864 * Calculate the starting bit in the first word. 865 */ 866 bit = first_bit & (uint)(NBWORD - 1); 867 868 /* 869 * First set any bits in the first word of our range. 870 * If it starts at bit 0 of the word, it will be 871 * set below rather than here. That is what the variable 872 * bit tells us. The variable bits_set tracks the number 873 * of bits that have been set so far. End_bit is the number 874 * of the last bit to be set in this word plus one. 875 */ 876 if (bit) { 877 end_bit = MIN(bit + bits_to_set, (uint)NBWORD); 878 mask = ((1 << (end_bit - bit)) - 1) << bit; 879 *wordp |= mask; 880 wordp++; 881 bits_set = end_bit - bit; 882 } else { 883 bits_set = 0; 884 } 885 886 /* 887 * Now set bits a whole word at a time that are between 888 * first_bit and last_bit. 889 */ 890 while ((bits_to_set - bits_set) >= NBWORD) { 891 *wordp |= 0xffffffff; 892 bits_set += NBWORD; 893 wordp++; 894 } 895 896 /* 897 * Finally, set any bits left to be set in one last partial word. 898 */ 899 end_bit = bits_to_set - bits_set; 900 if (end_bit) { 901 mask = (1 << end_bit) - 1; 902 *wordp |= mask; 903 } 904 } 905 906 /* 907 * Mark bytes first through last inclusive as dirty in the buf 908 * item's bitmap. 909 */ 910 void 911 xfs_buf_item_log( 912 xfs_buf_log_item_t *bip, 913 uint first, 914 uint last) 915 { 916 int i; 917 uint start; 918 uint end; 919 struct xfs_buf *bp = bip->bli_buf; 920 921 /* 922 * walk each buffer segment and mark them dirty appropriately. 923 */ 924 start = 0; 925 for (i = 0; i < bip->bli_format_count; i++) { 926 if (start > last) 927 break; 928 end = start + BBTOB(bp->b_maps[i].bm_len); 929 if (first > end) { 930 start += BBTOB(bp->b_maps[i].bm_len); 931 continue; 932 } 933 if (first < start) 934 first = start; 935 if (end > last) 936 end = last; 937 938 xfs_buf_item_log_segment(bip, first, end, 939 &bip->bli_formats[i].blf_data_map[0]); 940 941 start += bp->b_maps[i].bm_len; 942 } 943 } 944 945 946 /* 947 * Return 1 if the buffer has been logged or ordered in a transaction (at any 948 * point, not just the current transaction) and 0 if not. 949 */ 950 uint 951 xfs_buf_item_dirty( 952 xfs_buf_log_item_t *bip) 953 { 954 return (bip->bli_flags & XFS_BLI_DIRTY); 955 } 956 957 STATIC void 958 xfs_buf_item_free( 959 xfs_buf_log_item_t *bip) 960 { 961 #ifdef XFS_TRANS_DEBUG 962 kmem_free(bip->bli_orig); 963 kmem_free(bip->bli_logged); 964 #endif /* XFS_TRANS_DEBUG */ 965 966 xfs_buf_item_free_format(bip); 967 kmem_zone_free(xfs_buf_item_zone, bip); 968 } 969 970 /* 971 * This is called when the buf log item is no longer needed. It should 972 * free the buf log item associated with the given buffer and clear 973 * the buffer's pointer to the buf log item. If there are no more 974 * items in the list, clear the b_iodone field of the buffer (see 975 * xfs_buf_attach_iodone() below). 976 */ 977 void 978 xfs_buf_item_relse( 979 xfs_buf_t *bp) 980 { 981 xfs_buf_log_item_t *bip = bp->b_fspriv; 982 983 trace_xfs_buf_item_relse(bp, _RET_IP_); 984 ASSERT(!(bip->bli_item.li_flags & XFS_LI_IN_AIL)); 985 986 bp->b_fspriv = bip->bli_item.li_bio_list; 987 if (bp->b_fspriv == NULL) 988 bp->b_iodone = NULL; 989 990 xfs_buf_rele(bp); 991 xfs_buf_item_free(bip); 992 } 993 994 995 /* 996 * Add the given log item with its callback to the list of callbacks 997 * to be called when the buffer's I/O completes. If it is not set 998 * already, set the buffer's b_iodone() routine to be 999 * xfs_buf_iodone_callbacks() and link the log item into the list of 1000 * items rooted at b_fsprivate. Items are always added as the second 1001 * entry in the list if there is a first, because the buf item code 1002 * assumes that the buf log item is first. 1003 */ 1004 void 1005 xfs_buf_attach_iodone( 1006 xfs_buf_t *bp, 1007 void (*cb)(xfs_buf_t *, xfs_log_item_t *), 1008 xfs_log_item_t *lip) 1009 { 1010 xfs_log_item_t *head_lip; 1011 1012 ASSERT(xfs_buf_islocked(bp)); 1013 1014 lip->li_cb = cb; 1015 head_lip = bp->b_fspriv; 1016 if (head_lip) { 1017 lip->li_bio_list = head_lip->li_bio_list; 1018 head_lip->li_bio_list = lip; 1019 } else { 1020 bp->b_fspriv = lip; 1021 } 1022 1023 ASSERT(bp->b_iodone == NULL || 1024 bp->b_iodone == xfs_buf_iodone_callbacks); 1025 bp->b_iodone = xfs_buf_iodone_callbacks; 1026 } 1027 1028 /* 1029 * We can have many callbacks on a buffer. Running the callbacks individually 1030 * can cause a lot of contention on the AIL lock, so we allow for a single 1031 * callback to be able to scan the remaining lip->li_bio_list for other items 1032 * of the same type and callback to be processed in the first call. 1033 * 1034 * As a result, the loop walking the callback list below will also modify the 1035 * list. it removes the first item from the list and then runs the callback. 1036 * The loop then restarts from the new head of the list. This allows the 1037 * callback to scan and modify the list attached to the buffer and we don't 1038 * have to care about maintaining a next item pointer. 1039 */ 1040 STATIC void 1041 xfs_buf_do_callbacks( 1042 struct xfs_buf *bp) 1043 { 1044 struct xfs_log_item *lip; 1045 1046 while ((lip = bp->b_fspriv) != NULL) { 1047 bp->b_fspriv = lip->li_bio_list; 1048 ASSERT(lip->li_cb != NULL); 1049 /* 1050 * Clear the next pointer so we don't have any 1051 * confusion if the item is added to another buf. 1052 * Don't touch the log item after calling its 1053 * callback, because it could have freed itself. 1054 */ 1055 lip->li_bio_list = NULL; 1056 lip->li_cb(bp, lip); 1057 } 1058 } 1059 1060 /* 1061 * This is the iodone() function for buffers which have had callbacks 1062 * attached to them by xfs_buf_attach_iodone(). It should remove each 1063 * log item from the buffer's list and call the callback of each in turn. 1064 * When done, the buffer's fsprivate field is set to NULL and the buffer 1065 * is unlocked with a call to iodone(). 1066 */ 1067 void 1068 xfs_buf_iodone_callbacks( 1069 struct xfs_buf *bp) 1070 { 1071 struct xfs_log_item *lip = bp->b_fspriv; 1072 struct xfs_mount *mp = lip->li_mountp; 1073 static ulong lasttime; 1074 static xfs_buftarg_t *lasttarg; 1075 1076 if (likely(!xfs_buf_geterror(bp))) 1077 goto do_callbacks; 1078 1079 /* 1080 * If we've already decided to shutdown the filesystem because of 1081 * I/O errors, there's no point in giving this a retry. 1082 */ 1083 if (XFS_FORCED_SHUTDOWN(mp)) { 1084 xfs_buf_stale(bp); 1085 XFS_BUF_DONE(bp); 1086 trace_xfs_buf_item_iodone(bp, _RET_IP_); 1087 goto do_callbacks; 1088 } 1089 1090 if (bp->b_target != lasttarg || 1091 time_after(jiffies, (lasttime + 5*HZ))) { 1092 lasttime = jiffies; 1093 xfs_buf_ioerror_alert(bp, __func__); 1094 } 1095 lasttarg = bp->b_target; 1096 1097 /* 1098 * If the write was asynchronous then no one will be looking for the 1099 * error. Clear the error state and write the buffer out again. 1100 * 1101 * XXX: This helps against transient write errors, but we need to find 1102 * a way to shut the filesystem down if the writes keep failing. 1103 * 1104 * In practice we'll shut the filesystem down soon as non-transient 1105 * erorrs tend to affect the whole device and a failing log write 1106 * will make us give up. But we really ought to do better here. 1107 */ 1108 if (XFS_BUF_ISASYNC(bp)) { 1109 ASSERT(bp->b_iodone != NULL); 1110 1111 trace_xfs_buf_item_iodone_async(bp, _RET_IP_); 1112 1113 xfs_buf_ioerror(bp, 0); /* errno of 0 unsets the flag */ 1114 1115 if (!(bp->b_flags & (XBF_STALE|XBF_WRITE_FAIL))) { 1116 bp->b_flags |= XBF_WRITE | XBF_ASYNC | 1117 XBF_DONE | XBF_WRITE_FAIL; 1118 xfs_buf_iorequest(bp); 1119 } else { 1120 xfs_buf_relse(bp); 1121 } 1122 1123 return; 1124 } 1125 1126 /* 1127 * If the write of the buffer was synchronous, we want to make 1128 * sure to return the error to the caller of xfs_bwrite(). 1129 */ 1130 xfs_buf_stale(bp); 1131 XFS_BUF_DONE(bp); 1132 1133 trace_xfs_buf_error_relse(bp, _RET_IP_); 1134 1135 do_callbacks: 1136 xfs_buf_do_callbacks(bp); 1137 bp->b_fspriv = NULL; 1138 bp->b_iodone = NULL; 1139 xfs_buf_ioend(bp, 0); 1140 } 1141 1142 /* 1143 * This is the iodone() function for buffers which have been 1144 * logged. It is called when they are eventually flushed out. 1145 * It should remove the buf item from the AIL, and free the buf item. 1146 * It is called by xfs_buf_iodone_callbacks() above which will take 1147 * care of cleaning up the buffer itself. 1148 */ 1149 void 1150 xfs_buf_iodone( 1151 struct xfs_buf *bp, 1152 struct xfs_log_item *lip) 1153 { 1154 struct xfs_ail *ailp = lip->li_ailp; 1155 1156 ASSERT(BUF_ITEM(lip)->bli_buf == bp); 1157 1158 xfs_buf_rele(bp); 1159 1160 /* 1161 * If we are forcibly shutting down, this may well be 1162 * off the AIL already. That's because we simulate the 1163 * log-committed callbacks to unpin these buffers. Or we may never 1164 * have put this item on AIL because of the transaction was 1165 * aborted forcibly. xfs_trans_ail_delete() takes care of these. 1166 * 1167 * Either way, AIL is useless if we're forcing a shutdown. 1168 */ 1169 spin_lock(&ailp->xa_lock); 1170 xfs_trans_ail_delete(ailp, lip, SHUTDOWN_CORRUPT_INCORE); 1171 xfs_buf_item_free(BUF_ITEM(lip)); 1172 } 1173