1 /* 2 * Copyright (c) 2000-2005 Silicon Graphics, Inc. 3 * All Rights Reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it would be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 #include "xfs.h" 19 #include "xfs_fs.h" 20 #include "xfs_types.h" 21 #include "xfs_bit.h" 22 #include "xfs_log.h" 23 #include "xfs_inum.h" 24 #include "xfs_trans.h" 25 #include "xfs_sb.h" 26 #include "xfs_ag.h" 27 #include "xfs_mount.h" 28 #include "xfs_buf_item.h" 29 #include "xfs_trans_priv.h" 30 #include "xfs_error.h" 31 #include "xfs_trace.h" 32 33 34 kmem_zone_t *xfs_buf_item_zone; 35 36 static inline struct xfs_buf_log_item *BUF_ITEM(struct xfs_log_item *lip) 37 { 38 return container_of(lip, struct xfs_buf_log_item, bli_item); 39 } 40 41 42 #ifdef XFS_TRANS_DEBUG 43 /* 44 * This function uses an alternate strategy for tracking the bytes 45 * that the user requests to be logged. This can then be used 46 * in conjunction with the bli_orig array in the buf log item to 47 * catch bugs in our callers' code. 48 * 49 * We also double check the bits set in xfs_buf_item_log using a 50 * simple algorithm to check that every byte is accounted for. 51 */ 52 STATIC void 53 xfs_buf_item_log_debug( 54 xfs_buf_log_item_t *bip, 55 uint first, 56 uint last) 57 { 58 uint x; 59 uint byte; 60 uint nbytes; 61 uint chunk_num; 62 uint word_num; 63 uint bit_num; 64 uint bit_set; 65 uint *wordp; 66 67 ASSERT(bip->bli_logged != NULL); 68 byte = first; 69 nbytes = last - first + 1; 70 bfset(bip->bli_logged, first, nbytes); 71 for (x = 0; x < nbytes; x++) { 72 chunk_num = byte >> XFS_BLF_SHIFT; 73 word_num = chunk_num >> BIT_TO_WORD_SHIFT; 74 bit_num = chunk_num & (NBWORD - 1); 75 wordp = &(bip->bli_format.blf_data_map[word_num]); 76 bit_set = *wordp & (1 << bit_num); 77 ASSERT(bit_set); 78 byte++; 79 } 80 } 81 82 /* 83 * This function is called when we flush something into a buffer without 84 * logging it. This happens for things like inodes which are logged 85 * separately from the buffer. 86 */ 87 void 88 xfs_buf_item_flush_log_debug( 89 xfs_buf_t *bp, 90 uint first, 91 uint last) 92 { 93 xfs_buf_log_item_t *bip; 94 uint nbytes; 95 96 bip = XFS_BUF_FSPRIVATE(bp, xfs_buf_log_item_t*); 97 if ((bip == NULL) || (bip->bli_item.li_type != XFS_LI_BUF)) { 98 return; 99 } 100 101 ASSERT(bip->bli_logged != NULL); 102 nbytes = last - first + 1; 103 bfset(bip->bli_logged, first, nbytes); 104 } 105 106 /* 107 * This function is called to verify that our callers have logged 108 * all the bytes that they changed. 109 * 110 * It does this by comparing the original copy of the buffer stored in 111 * the buf log item's bli_orig array to the current copy of the buffer 112 * and ensuring that all bytes which mismatch are set in the bli_logged 113 * array of the buf log item. 114 */ 115 STATIC void 116 xfs_buf_item_log_check( 117 xfs_buf_log_item_t *bip) 118 { 119 char *orig; 120 char *buffer; 121 int x; 122 xfs_buf_t *bp; 123 124 ASSERT(bip->bli_orig != NULL); 125 ASSERT(bip->bli_logged != NULL); 126 127 bp = bip->bli_buf; 128 ASSERT(XFS_BUF_COUNT(bp) > 0); 129 ASSERT(XFS_BUF_PTR(bp) != NULL); 130 orig = bip->bli_orig; 131 buffer = XFS_BUF_PTR(bp); 132 for (x = 0; x < XFS_BUF_COUNT(bp); x++) { 133 if (orig[x] != buffer[x] && !btst(bip->bli_logged, x)) 134 cmn_err(CE_PANIC, 135 "xfs_buf_item_log_check bip %x buffer %x orig %x index %d", 136 bip, bp, orig, x); 137 } 138 } 139 #else 140 #define xfs_buf_item_log_debug(x,y,z) 141 #define xfs_buf_item_log_check(x) 142 #endif 143 144 STATIC void xfs_buf_error_relse(xfs_buf_t *bp); 145 STATIC void xfs_buf_do_callbacks(xfs_buf_t *bp, xfs_log_item_t *lip); 146 147 /* 148 * This returns the number of log iovecs needed to log the 149 * given buf log item. 150 * 151 * It calculates this as 1 iovec for the buf log format structure 152 * and 1 for each stretch of non-contiguous chunks to be logged. 153 * Contiguous chunks are logged in a single iovec. 154 * 155 * If the XFS_BLI_STALE flag has been set, then log nothing. 156 */ 157 STATIC uint 158 xfs_buf_item_size( 159 struct xfs_log_item *lip) 160 { 161 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 162 struct xfs_buf *bp = bip->bli_buf; 163 uint nvecs; 164 int next_bit; 165 int last_bit; 166 167 ASSERT(atomic_read(&bip->bli_refcount) > 0); 168 if (bip->bli_flags & XFS_BLI_STALE) { 169 /* 170 * The buffer is stale, so all we need to log 171 * is the buf log format structure with the 172 * cancel flag in it. 173 */ 174 trace_xfs_buf_item_size_stale(bip); 175 ASSERT(bip->bli_format.blf_flags & XFS_BLF_CANCEL); 176 return 1; 177 } 178 179 ASSERT(bip->bli_flags & XFS_BLI_LOGGED); 180 nvecs = 1; 181 last_bit = xfs_next_bit(bip->bli_format.blf_data_map, 182 bip->bli_format.blf_map_size, 0); 183 ASSERT(last_bit != -1); 184 nvecs++; 185 while (last_bit != -1) { 186 /* 187 * This takes the bit number to start looking from and 188 * returns the next set bit from there. It returns -1 189 * if there are no more bits set or the start bit is 190 * beyond the end of the bitmap. 191 */ 192 next_bit = xfs_next_bit(bip->bli_format.blf_data_map, 193 bip->bli_format.blf_map_size, 194 last_bit + 1); 195 /* 196 * If we run out of bits, leave the loop, 197 * else if we find a new set of bits bump the number of vecs, 198 * else keep scanning the current set of bits. 199 */ 200 if (next_bit == -1) { 201 last_bit = -1; 202 } else if (next_bit != last_bit + 1) { 203 last_bit = next_bit; 204 nvecs++; 205 } else if (xfs_buf_offset(bp, next_bit * XFS_BLF_CHUNK) != 206 (xfs_buf_offset(bp, last_bit * XFS_BLF_CHUNK) + 207 XFS_BLF_CHUNK)) { 208 last_bit = next_bit; 209 nvecs++; 210 } else { 211 last_bit++; 212 } 213 } 214 215 trace_xfs_buf_item_size(bip); 216 return nvecs; 217 } 218 219 /* 220 * This is called to fill in the vector of log iovecs for the 221 * given log buf item. It fills the first entry with a buf log 222 * format structure, and the rest point to contiguous chunks 223 * within the buffer. 224 */ 225 STATIC void 226 xfs_buf_item_format( 227 struct xfs_log_item *lip, 228 struct xfs_log_iovec *vecp) 229 { 230 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 231 struct xfs_buf *bp = bip->bli_buf; 232 uint base_size; 233 uint nvecs; 234 int first_bit; 235 int last_bit; 236 int next_bit; 237 uint nbits; 238 uint buffer_offset; 239 240 ASSERT(atomic_read(&bip->bli_refcount) > 0); 241 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || 242 (bip->bli_flags & XFS_BLI_STALE)); 243 244 /* 245 * The size of the base structure is the size of the 246 * declared structure plus the space for the extra words 247 * of the bitmap. We subtract one from the map size, because 248 * the first element of the bitmap is accounted for in the 249 * size of the base structure. 250 */ 251 base_size = 252 (uint)(sizeof(xfs_buf_log_format_t) + 253 ((bip->bli_format.blf_map_size - 1) * sizeof(uint))); 254 vecp->i_addr = &bip->bli_format; 255 vecp->i_len = base_size; 256 vecp->i_type = XLOG_REG_TYPE_BFORMAT; 257 vecp++; 258 nvecs = 1; 259 260 /* 261 * If it is an inode buffer, transfer the in-memory state to the 262 * format flags and clear the in-memory state. We do not transfer 263 * this state if the inode buffer allocation has not yet been committed 264 * to the log as setting the XFS_BLI_INODE_BUF flag will prevent 265 * correct replay of the inode allocation. 266 */ 267 if (bip->bli_flags & XFS_BLI_INODE_BUF) { 268 if (!((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && 269 xfs_log_item_in_current_chkpt(lip))) 270 bip->bli_format.blf_flags |= XFS_BLF_INODE_BUF; 271 bip->bli_flags &= ~XFS_BLI_INODE_BUF; 272 } 273 274 if (bip->bli_flags & XFS_BLI_STALE) { 275 /* 276 * The buffer is stale, so all we need to log 277 * is the buf log format structure with the 278 * cancel flag in it. 279 */ 280 trace_xfs_buf_item_format_stale(bip); 281 ASSERT(bip->bli_format.blf_flags & XFS_BLF_CANCEL); 282 bip->bli_format.blf_size = nvecs; 283 return; 284 } 285 286 /* 287 * Fill in an iovec for each set of contiguous chunks. 288 */ 289 first_bit = xfs_next_bit(bip->bli_format.blf_data_map, 290 bip->bli_format.blf_map_size, 0); 291 ASSERT(first_bit != -1); 292 last_bit = first_bit; 293 nbits = 1; 294 for (;;) { 295 /* 296 * This takes the bit number to start looking from and 297 * returns the next set bit from there. It returns -1 298 * if there are no more bits set or the start bit is 299 * beyond the end of the bitmap. 300 */ 301 next_bit = xfs_next_bit(bip->bli_format.blf_data_map, 302 bip->bli_format.blf_map_size, 303 (uint)last_bit + 1); 304 /* 305 * If we run out of bits fill in the last iovec and get 306 * out of the loop. 307 * Else if we start a new set of bits then fill in the 308 * iovec for the series we were looking at and start 309 * counting the bits in the new one. 310 * Else we're still in the same set of bits so just 311 * keep counting and scanning. 312 */ 313 if (next_bit == -1) { 314 buffer_offset = first_bit * XFS_BLF_CHUNK; 315 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 316 vecp->i_len = nbits * XFS_BLF_CHUNK; 317 vecp->i_type = XLOG_REG_TYPE_BCHUNK; 318 nvecs++; 319 break; 320 } else if (next_bit != last_bit + 1) { 321 buffer_offset = first_bit * XFS_BLF_CHUNK; 322 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 323 vecp->i_len = nbits * XFS_BLF_CHUNK; 324 vecp->i_type = XLOG_REG_TYPE_BCHUNK; 325 nvecs++; 326 vecp++; 327 first_bit = next_bit; 328 last_bit = next_bit; 329 nbits = 1; 330 } else if (xfs_buf_offset(bp, next_bit << XFS_BLF_SHIFT) != 331 (xfs_buf_offset(bp, last_bit << XFS_BLF_SHIFT) + 332 XFS_BLF_CHUNK)) { 333 buffer_offset = first_bit * XFS_BLF_CHUNK; 334 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 335 vecp->i_len = nbits * XFS_BLF_CHUNK; 336 vecp->i_type = XLOG_REG_TYPE_BCHUNK; 337 /* You would think we need to bump the nvecs here too, but we do not 338 * this number is used by recovery, and it gets confused by the boundary 339 * split here 340 * nvecs++; 341 */ 342 vecp++; 343 first_bit = next_bit; 344 last_bit = next_bit; 345 nbits = 1; 346 } else { 347 last_bit++; 348 nbits++; 349 } 350 } 351 bip->bli_format.blf_size = nvecs; 352 353 /* 354 * Check to make sure everything is consistent. 355 */ 356 trace_xfs_buf_item_format(bip); 357 xfs_buf_item_log_check(bip); 358 } 359 360 /* 361 * This is called to pin the buffer associated with the buf log item in memory 362 * so it cannot be written out. 363 * 364 * We also always take a reference to the buffer log item here so that the bli 365 * is held while the item is pinned in memory. This means that we can 366 * unconditionally drop the reference count a transaction holds when the 367 * transaction is completed. 368 */ 369 STATIC void 370 xfs_buf_item_pin( 371 struct xfs_log_item *lip) 372 { 373 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 374 375 ASSERT(XFS_BUF_ISBUSY(bip->bli_buf)); 376 ASSERT(atomic_read(&bip->bli_refcount) > 0); 377 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || 378 (bip->bli_flags & XFS_BLI_STALE)); 379 380 trace_xfs_buf_item_pin(bip); 381 382 atomic_inc(&bip->bli_refcount); 383 atomic_inc(&bip->bli_buf->b_pin_count); 384 } 385 386 /* 387 * This is called to unpin the buffer associated with the buf log 388 * item which was previously pinned with a call to xfs_buf_item_pin(). 389 * 390 * Also drop the reference to the buf item for the current transaction. 391 * If the XFS_BLI_STALE flag is set and we are the last reference, 392 * then free up the buf log item and unlock the buffer. 393 * 394 * If the remove flag is set we are called from uncommit in the 395 * forced-shutdown path. If that is true and the reference count on 396 * the log item is going to drop to zero we need to free the item's 397 * descriptor in the transaction. 398 */ 399 STATIC void 400 xfs_buf_item_unpin( 401 struct xfs_log_item *lip, 402 int remove) 403 { 404 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 405 xfs_buf_t *bp = bip->bli_buf; 406 struct xfs_ail *ailp = lip->li_ailp; 407 int stale = bip->bli_flags & XFS_BLI_STALE; 408 int freed; 409 410 ASSERT(XFS_BUF_FSPRIVATE(bp, xfs_buf_log_item_t *) == bip); 411 ASSERT(atomic_read(&bip->bli_refcount) > 0); 412 413 trace_xfs_buf_item_unpin(bip); 414 415 freed = atomic_dec_and_test(&bip->bli_refcount); 416 417 if (atomic_dec_and_test(&bp->b_pin_count)) 418 wake_up_all(&bp->b_waiters); 419 420 if (freed && stale) { 421 ASSERT(bip->bli_flags & XFS_BLI_STALE); 422 ASSERT(XFS_BUF_VALUSEMA(bp) <= 0); 423 ASSERT(!(XFS_BUF_ISDELAYWRITE(bp))); 424 ASSERT(XFS_BUF_ISSTALE(bp)); 425 ASSERT(bip->bli_format.blf_flags & XFS_BLF_CANCEL); 426 427 trace_xfs_buf_item_unpin_stale(bip); 428 429 if (remove) { 430 /* 431 * We have to remove the log item from the transaction 432 * as we are about to release our reference to the 433 * buffer. If we don't, the unlock that occurs later 434 * in xfs_trans_uncommit() will ry to reference the 435 * buffer which we no longer have a hold on. 436 */ 437 xfs_trans_del_item(lip); 438 439 /* 440 * Since the transaction no longer refers to the buffer, 441 * the buffer should no longer refer to the transaction. 442 */ 443 XFS_BUF_SET_FSPRIVATE2(bp, NULL); 444 } 445 446 /* 447 * If we get called here because of an IO error, we may 448 * or may not have the item on the AIL. xfs_trans_ail_delete() 449 * will take care of that situation. 450 * xfs_trans_ail_delete() drops the AIL lock. 451 */ 452 if (bip->bli_flags & XFS_BLI_STALE_INODE) { 453 xfs_buf_do_callbacks(bp, (xfs_log_item_t *)bip); 454 XFS_BUF_SET_FSPRIVATE(bp, NULL); 455 XFS_BUF_CLR_IODONE_FUNC(bp); 456 } else { 457 spin_lock(&ailp->xa_lock); 458 xfs_trans_ail_delete(ailp, (xfs_log_item_t *)bip); 459 xfs_buf_item_relse(bp); 460 ASSERT(XFS_BUF_FSPRIVATE(bp, void *) == NULL); 461 } 462 xfs_buf_relse(bp); 463 } 464 } 465 466 /* 467 * This is called to attempt to lock the buffer associated with this 468 * buf log item. Don't sleep on the buffer lock. If we can't get 469 * the lock right away, return 0. If we can get the lock, take a 470 * reference to the buffer. If this is a delayed write buffer that 471 * needs AIL help to be written back, invoke the pushbuf routine 472 * rather than the normal success path. 473 */ 474 STATIC uint 475 xfs_buf_item_trylock( 476 struct xfs_log_item *lip) 477 { 478 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 479 struct xfs_buf *bp = bip->bli_buf; 480 481 if (XFS_BUF_ISPINNED(bp)) 482 return XFS_ITEM_PINNED; 483 if (!XFS_BUF_CPSEMA(bp)) 484 return XFS_ITEM_LOCKED; 485 486 /* take a reference to the buffer. */ 487 XFS_BUF_HOLD(bp); 488 489 ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); 490 trace_xfs_buf_item_trylock(bip); 491 if (XFS_BUF_ISDELAYWRITE(bp)) 492 return XFS_ITEM_PUSHBUF; 493 return XFS_ITEM_SUCCESS; 494 } 495 496 /* 497 * Release the buffer associated with the buf log item. If there is no dirty 498 * logged data associated with the buffer recorded in the buf log item, then 499 * free the buf log item and remove the reference to it in the buffer. 500 * 501 * This call ignores the recursion count. It is only called when the buffer 502 * should REALLY be unlocked, regardless of the recursion count. 503 * 504 * We unconditionally drop the transaction's reference to the log item. If the 505 * item was logged, then another reference was taken when it was pinned, so we 506 * can safely drop the transaction reference now. This also allows us to avoid 507 * potential races with the unpin code freeing the bli by not referencing the 508 * bli after we've dropped the reference count. 509 * 510 * If the XFS_BLI_HOLD flag is set in the buf log item, then free the log item 511 * if necessary but do not unlock the buffer. This is for support of 512 * xfs_trans_bhold(). Make sure the XFS_BLI_HOLD field is cleared if we don't 513 * free the item. 514 */ 515 STATIC void 516 xfs_buf_item_unlock( 517 struct xfs_log_item *lip) 518 { 519 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 520 struct xfs_buf *bp = bip->bli_buf; 521 int aborted; 522 uint hold; 523 524 /* Clear the buffer's association with this transaction. */ 525 XFS_BUF_SET_FSPRIVATE2(bp, NULL); 526 527 /* 528 * If this is a transaction abort, don't return early. Instead, allow 529 * the brelse to happen. Normally it would be done for stale 530 * (cancelled) buffers at unpin time, but we'll never go through the 531 * pin/unpin cycle if we abort inside commit. 532 */ 533 aborted = (lip->li_flags & XFS_LI_ABORTED) != 0; 534 535 /* 536 * Before possibly freeing the buf item, determine if we should 537 * release the buffer at the end of this routine. 538 */ 539 hold = bip->bli_flags & XFS_BLI_HOLD; 540 541 /* Clear the per transaction state. */ 542 bip->bli_flags &= ~(XFS_BLI_LOGGED | XFS_BLI_HOLD); 543 544 /* 545 * If the buf item is marked stale, then don't do anything. We'll 546 * unlock the buffer and free the buf item when the buffer is unpinned 547 * for the last time. 548 */ 549 if (bip->bli_flags & XFS_BLI_STALE) { 550 trace_xfs_buf_item_unlock_stale(bip); 551 ASSERT(bip->bli_format.blf_flags & XFS_BLF_CANCEL); 552 if (!aborted) { 553 atomic_dec(&bip->bli_refcount); 554 return; 555 } 556 } 557 558 trace_xfs_buf_item_unlock(bip); 559 560 /* 561 * If the buf item isn't tracking any data, free it, otherwise drop the 562 * reference we hold to it. 563 */ 564 if (xfs_bitmap_empty(bip->bli_format.blf_data_map, 565 bip->bli_format.blf_map_size)) 566 xfs_buf_item_relse(bp); 567 else 568 atomic_dec(&bip->bli_refcount); 569 570 if (!hold) 571 xfs_buf_relse(bp); 572 } 573 574 /* 575 * This is called to find out where the oldest active copy of the 576 * buf log item in the on disk log resides now that the last log 577 * write of it completed at the given lsn. 578 * We always re-log all the dirty data in a buffer, so usually the 579 * latest copy in the on disk log is the only one that matters. For 580 * those cases we simply return the given lsn. 581 * 582 * The one exception to this is for buffers full of newly allocated 583 * inodes. These buffers are only relogged with the XFS_BLI_INODE_BUF 584 * flag set, indicating that only the di_next_unlinked fields from the 585 * inodes in the buffers will be replayed during recovery. If the 586 * original newly allocated inode images have not yet been flushed 587 * when the buffer is so relogged, then we need to make sure that we 588 * keep the old images in the 'active' portion of the log. We do this 589 * by returning the original lsn of that transaction here rather than 590 * the current one. 591 */ 592 STATIC xfs_lsn_t 593 xfs_buf_item_committed( 594 struct xfs_log_item *lip, 595 xfs_lsn_t lsn) 596 { 597 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 598 599 trace_xfs_buf_item_committed(bip); 600 601 if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && lip->li_lsn != 0) 602 return lip->li_lsn; 603 return lsn; 604 } 605 606 /* 607 * The buffer is locked, but is not a delayed write buffer. This happens 608 * if we race with IO completion and hence we don't want to try to write it 609 * again. Just release the buffer. 610 */ 611 STATIC void 612 xfs_buf_item_push( 613 struct xfs_log_item *lip) 614 { 615 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 616 struct xfs_buf *bp = bip->bli_buf; 617 618 ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); 619 ASSERT(!XFS_BUF_ISDELAYWRITE(bp)); 620 621 trace_xfs_buf_item_push(bip); 622 623 xfs_buf_relse(bp); 624 } 625 626 /* 627 * The buffer is locked and is a delayed write buffer. Promote the buffer 628 * in the delayed write queue as the caller knows that they must invoke 629 * the xfsbufd to get this buffer written. We have to unlock the buffer 630 * to allow the xfsbufd to write it, too. 631 */ 632 STATIC void 633 xfs_buf_item_pushbuf( 634 struct xfs_log_item *lip) 635 { 636 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 637 struct xfs_buf *bp = bip->bli_buf; 638 639 ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); 640 ASSERT(XFS_BUF_ISDELAYWRITE(bp)); 641 642 trace_xfs_buf_item_pushbuf(bip); 643 644 xfs_buf_delwri_promote(bp); 645 xfs_buf_relse(bp); 646 } 647 648 STATIC void 649 xfs_buf_item_committing( 650 struct xfs_log_item *lip, 651 xfs_lsn_t commit_lsn) 652 { 653 } 654 655 /* 656 * This is the ops vector shared by all buf log items. 657 */ 658 static struct xfs_item_ops xfs_buf_item_ops = { 659 .iop_size = xfs_buf_item_size, 660 .iop_format = xfs_buf_item_format, 661 .iop_pin = xfs_buf_item_pin, 662 .iop_unpin = xfs_buf_item_unpin, 663 .iop_trylock = xfs_buf_item_trylock, 664 .iop_unlock = xfs_buf_item_unlock, 665 .iop_committed = xfs_buf_item_committed, 666 .iop_push = xfs_buf_item_push, 667 .iop_pushbuf = xfs_buf_item_pushbuf, 668 .iop_committing = xfs_buf_item_committing 669 }; 670 671 672 /* 673 * Allocate a new buf log item to go with the given buffer. 674 * Set the buffer's b_fsprivate field to point to the new 675 * buf log item. If there are other item's attached to the 676 * buffer (see xfs_buf_attach_iodone() below), then put the 677 * buf log item at the front. 678 */ 679 void 680 xfs_buf_item_init( 681 xfs_buf_t *bp, 682 xfs_mount_t *mp) 683 { 684 xfs_log_item_t *lip; 685 xfs_buf_log_item_t *bip; 686 int chunks; 687 int map_size; 688 689 /* 690 * Check to see if there is already a buf log item for 691 * this buffer. If there is, it is guaranteed to be 692 * the first. If we do already have one, there is 693 * nothing to do here so return. 694 */ 695 ASSERT(bp->b_target->bt_mount == mp); 696 if (XFS_BUF_FSPRIVATE(bp, void *) != NULL) { 697 lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 698 if (lip->li_type == XFS_LI_BUF) { 699 return; 700 } 701 } 702 703 /* 704 * chunks is the number of XFS_BLF_CHUNK size pieces 705 * the buffer can be divided into. Make sure not to 706 * truncate any pieces. map_size is the size of the 707 * bitmap needed to describe the chunks of the buffer. 708 */ 709 chunks = (int)((XFS_BUF_COUNT(bp) + (XFS_BLF_CHUNK - 1)) >> XFS_BLF_SHIFT); 710 map_size = (int)((chunks + NBWORD) >> BIT_TO_WORD_SHIFT); 711 712 bip = (xfs_buf_log_item_t*)kmem_zone_zalloc(xfs_buf_item_zone, 713 KM_SLEEP); 714 xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops); 715 bip->bli_buf = bp; 716 xfs_buf_hold(bp); 717 bip->bli_format.blf_type = XFS_LI_BUF; 718 bip->bli_format.blf_blkno = (__int64_t)XFS_BUF_ADDR(bp); 719 bip->bli_format.blf_len = (ushort)BTOBB(XFS_BUF_COUNT(bp)); 720 bip->bli_format.blf_map_size = map_size; 721 722 #ifdef XFS_TRANS_DEBUG 723 /* 724 * Allocate the arrays for tracking what needs to be logged 725 * and what our callers request to be logged. bli_orig 726 * holds a copy of the original, clean buffer for comparison 727 * against, and bli_logged keeps a 1 bit flag per byte in 728 * the buffer to indicate which bytes the callers have asked 729 * to have logged. 730 */ 731 bip->bli_orig = (char *)kmem_alloc(XFS_BUF_COUNT(bp), KM_SLEEP); 732 memcpy(bip->bli_orig, XFS_BUF_PTR(bp), XFS_BUF_COUNT(bp)); 733 bip->bli_logged = (char *)kmem_zalloc(XFS_BUF_COUNT(bp) / NBBY, KM_SLEEP); 734 #endif 735 736 /* 737 * Put the buf item into the list of items attached to the 738 * buffer at the front. 739 */ 740 if (XFS_BUF_FSPRIVATE(bp, void *) != NULL) { 741 bip->bli_item.li_bio_list = 742 XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 743 } 744 XFS_BUF_SET_FSPRIVATE(bp, bip); 745 } 746 747 748 /* 749 * Mark bytes first through last inclusive as dirty in the buf 750 * item's bitmap. 751 */ 752 void 753 xfs_buf_item_log( 754 xfs_buf_log_item_t *bip, 755 uint first, 756 uint last) 757 { 758 uint first_bit; 759 uint last_bit; 760 uint bits_to_set; 761 uint bits_set; 762 uint word_num; 763 uint *wordp; 764 uint bit; 765 uint end_bit; 766 uint mask; 767 768 /* 769 * Mark the item as having some dirty data for 770 * quick reference in xfs_buf_item_dirty. 771 */ 772 bip->bli_flags |= XFS_BLI_DIRTY; 773 774 /* 775 * Convert byte offsets to bit numbers. 776 */ 777 first_bit = first >> XFS_BLF_SHIFT; 778 last_bit = last >> XFS_BLF_SHIFT; 779 780 /* 781 * Calculate the total number of bits to be set. 782 */ 783 bits_to_set = last_bit - first_bit + 1; 784 785 /* 786 * Get a pointer to the first word in the bitmap 787 * to set a bit in. 788 */ 789 word_num = first_bit >> BIT_TO_WORD_SHIFT; 790 wordp = &(bip->bli_format.blf_data_map[word_num]); 791 792 /* 793 * Calculate the starting bit in the first word. 794 */ 795 bit = first_bit & (uint)(NBWORD - 1); 796 797 /* 798 * First set any bits in the first word of our range. 799 * If it starts at bit 0 of the word, it will be 800 * set below rather than here. That is what the variable 801 * bit tells us. The variable bits_set tracks the number 802 * of bits that have been set so far. End_bit is the number 803 * of the last bit to be set in this word plus one. 804 */ 805 if (bit) { 806 end_bit = MIN(bit + bits_to_set, (uint)NBWORD); 807 mask = ((1 << (end_bit - bit)) - 1) << bit; 808 *wordp |= mask; 809 wordp++; 810 bits_set = end_bit - bit; 811 } else { 812 bits_set = 0; 813 } 814 815 /* 816 * Now set bits a whole word at a time that are between 817 * first_bit and last_bit. 818 */ 819 while ((bits_to_set - bits_set) >= NBWORD) { 820 *wordp |= 0xffffffff; 821 bits_set += NBWORD; 822 wordp++; 823 } 824 825 /* 826 * Finally, set any bits left to be set in one last partial word. 827 */ 828 end_bit = bits_to_set - bits_set; 829 if (end_bit) { 830 mask = (1 << end_bit) - 1; 831 *wordp |= mask; 832 } 833 834 xfs_buf_item_log_debug(bip, first, last); 835 } 836 837 838 /* 839 * Return 1 if the buffer has some data that has been logged (at any 840 * point, not just the current transaction) and 0 if not. 841 */ 842 uint 843 xfs_buf_item_dirty( 844 xfs_buf_log_item_t *bip) 845 { 846 return (bip->bli_flags & XFS_BLI_DIRTY); 847 } 848 849 STATIC void 850 xfs_buf_item_free( 851 xfs_buf_log_item_t *bip) 852 { 853 #ifdef XFS_TRANS_DEBUG 854 kmem_free(bip->bli_orig); 855 kmem_free(bip->bli_logged); 856 #endif /* XFS_TRANS_DEBUG */ 857 858 kmem_zone_free(xfs_buf_item_zone, bip); 859 } 860 861 /* 862 * This is called when the buf log item is no longer needed. It should 863 * free the buf log item associated with the given buffer and clear 864 * the buffer's pointer to the buf log item. If there are no more 865 * items in the list, clear the b_iodone field of the buffer (see 866 * xfs_buf_attach_iodone() below). 867 */ 868 void 869 xfs_buf_item_relse( 870 xfs_buf_t *bp) 871 { 872 xfs_buf_log_item_t *bip; 873 874 trace_xfs_buf_item_relse(bp, _RET_IP_); 875 876 bip = XFS_BUF_FSPRIVATE(bp, xfs_buf_log_item_t*); 877 XFS_BUF_SET_FSPRIVATE(bp, bip->bli_item.li_bio_list); 878 if ((XFS_BUF_FSPRIVATE(bp, void *) == NULL) && 879 (XFS_BUF_IODONE_FUNC(bp) != NULL)) { 880 XFS_BUF_CLR_IODONE_FUNC(bp); 881 } 882 xfs_buf_rele(bp); 883 xfs_buf_item_free(bip); 884 } 885 886 887 /* 888 * Add the given log item with its callback to the list of callbacks 889 * to be called when the buffer's I/O completes. If it is not set 890 * already, set the buffer's b_iodone() routine to be 891 * xfs_buf_iodone_callbacks() and link the log item into the list of 892 * items rooted at b_fsprivate. Items are always added as the second 893 * entry in the list if there is a first, because the buf item code 894 * assumes that the buf log item is first. 895 */ 896 void 897 xfs_buf_attach_iodone( 898 xfs_buf_t *bp, 899 void (*cb)(xfs_buf_t *, xfs_log_item_t *), 900 xfs_log_item_t *lip) 901 { 902 xfs_log_item_t *head_lip; 903 904 ASSERT(XFS_BUF_ISBUSY(bp)); 905 ASSERT(XFS_BUF_VALUSEMA(bp) <= 0); 906 907 lip->li_cb = cb; 908 if (XFS_BUF_FSPRIVATE(bp, void *) != NULL) { 909 head_lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 910 lip->li_bio_list = head_lip->li_bio_list; 911 head_lip->li_bio_list = lip; 912 } else { 913 XFS_BUF_SET_FSPRIVATE(bp, lip); 914 } 915 916 ASSERT((XFS_BUF_IODONE_FUNC(bp) == xfs_buf_iodone_callbacks) || 917 (XFS_BUF_IODONE_FUNC(bp) == NULL)); 918 XFS_BUF_SET_IODONE_FUNC(bp, xfs_buf_iodone_callbacks); 919 } 920 921 STATIC void 922 xfs_buf_do_callbacks( 923 xfs_buf_t *bp, 924 xfs_log_item_t *lip) 925 { 926 xfs_log_item_t *nlip; 927 928 while (lip != NULL) { 929 nlip = lip->li_bio_list; 930 ASSERT(lip->li_cb != NULL); 931 /* 932 * Clear the next pointer so we don't have any 933 * confusion if the item is added to another buf. 934 * Don't touch the log item after calling its 935 * callback, because it could have freed itself. 936 */ 937 lip->li_bio_list = NULL; 938 lip->li_cb(bp, lip); 939 lip = nlip; 940 } 941 } 942 943 /* 944 * This is the iodone() function for buffers which have had callbacks 945 * attached to them by xfs_buf_attach_iodone(). It should remove each 946 * log item from the buffer's list and call the callback of each in turn. 947 * When done, the buffer's fsprivate field is set to NULL and the buffer 948 * is unlocked with a call to iodone(). 949 */ 950 void 951 xfs_buf_iodone_callbacks( 952 xfs_buf_t *bp) 953 { 954 xfs_log_item_t *lip; 955 static ulong lasttime; 956 static xfs_buftarg_t *lasttarg; 957 xfs_mount_t *mp; 958 959 ASSERT(XFS_BUF_FSPRIVATE(bp, void *) != NULL); 960 lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 961 962 if (XFS_BUF_GETERROR(bp) != 0) { 963 /* 964 * If we've already decided to shutdown the filesystem 965 * because of IO errors, there's no point in giving this 966 * a retry. 967 */ 968 mp = lip->li_mountp; 969 if (XFS_FORCED_SHUTDOWN(mp)) { 970 ASSERT(XFS_BUF_TARGET(bp) == mp->m_ddev_targp); 971 XFS_BUF_SUPER_STALE(bp); 972 trace_xfs_buf_item_iodone(bp, _RET_IP_); 973 xfs_buf_do_callbacks(bp, lip); 974 XFS_BUF_SET_FSPRIVATE(bp, NULL); 975 XFS_BUF_CLR_IODONE_FUNC(bp); 976 xfs_buf_ioend(bp, 0); 977 return; 978 } 979 980 if ((XFS_BUF_TARGET(bp) != lasttarg) || 981 (time_after(jiffies, (lasttime + 5*HZ)))) { 982 lasttime = jiffies; 983 cmn_err(CE_ALERT, "Device %s, XFS metadata write error" 984 " block 0x%llx in %s", 985 XFS_BUFTARG_NAME(XFS_BUF_TARGET(bp)), 986 (__uint64_t)XFS_BUF_ADDR(bp), mp->m_fsname); 987 } 988 lasttarg = XFS_BUF_TARGET(bp); 989 990 if (XFS_BUF_ISASYNC(bp)) { 991 /* 992 * If the write was asynchronous then noone will be 993 * looking for the error. Clear the error state 994 * and write the buffer out again delayed write. 995 * 996 * XXXsup This is OK, so long as we catch these 997 * before we start the umount; we don't want these 998 * DELWRI metadata bufs to be hanging around. 999 */ 1000 XFS_BUF_ERROR(bp,0); /* errno of 0 unsets the flag */ 1001 1002 if (!(XFS_BUF_ISSTALE(bp))) { 1003 XFS_BUF_DELAYWRITE(bp); 1004 XFS_BUF_DONE(bp); 1005 XFS_BUF_SET_START(bp); 1006 } 1007 ASSERT(XFS_BUF_IODONE_FUNC(bp)); 1008 trace_xfs_buf_item_iodone_async(bp, _RET_IP_); 1009 xfs_buf_relse(bp); 1010 } else { 1011 /* 1012 * If the write of the buffer was not asynchronous, 1013 * then we want to make sure to return the error 1014 * to the caller of bwrite(). Because of this we 1015 * cannot clear the B_ERROR state at this point. 1016 * Instead we install a callback function that 1017 * will be called when the buffer is released, and 1018 * that routine will clear the error state and 1019 * set the buffer to be written out again after 1020 * some delay. 1021 */ 1022 /* We actually overwrite the existing b-relse 1023 function at times, but we're gonna be shutting down 1024 anyway. */ 1025 XFS_BUF_SET_BRELSE_FUNC(bp,xfs_buf_error_relse); 1026 XFS_BUF_DONE(bp); 1027 XFS_BUF_FINISH_IOWAIT(bp); 1028 } 1029 return; 1030 } 1031 1032 xfs_buf_do_callbacks(bp, lip); 1033 XFS_BUF_SET_FSPRIVATE(bp, NULL); 1034 XFS_BUF_CLR_IODONE_FUNC(bp); 1035 xfs_buf_ioend(bp, 0); 1036 } 1037 1038 /* 1039 * This is a callback routine attached to a buffer which gets an error 1040 * when being written out synchronously. 1041 */ 1042 STATIC void 1043 xfs_buf_error_relse( 1044 xfs_buf_t *bp) 1045 { 1046 xfs_log_item_t *lip; 1047 xfs_mount_t *mp; 1048 1049 lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 1050 mp = (xfs_mount_t *)lip->li_mountp; 1051 ASSERT(XFS_BUF_TARGET(bp) == mp->m_ddev_targp); 1052 1053 XFS_BUF_STALE(bp); 1054 XFS_BUF_DONE(bp); 1055 XFS_BUF_UNDELAYWRITE(bp); 1056 XFS_BUF_ERROR(bp,0); 1057 1058 trace_xfs_buf_error_relse(bp, _RET_IP_); 1059 1060 if (! XFS_FORCED_SHUTDOWN(mp)) 1061 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); 1062 /* 1063 * We have to unpin the pinned buffers so do the 1064 * callbacks. 1065 */ 1066 xfs_buf_do_callbacks(bp, lip); 1067 XFS_BUF_SET_FSPRIVATE(bp, NULL); 1068 XFS_BUF_CLR_IODONE_FUNC(bp); 1069 XFS_BUF_SET_BRELSE_FUNC(bp,NULL); 1070 xfs_buf_relse(bp); 1071 } 1072 1073 1074 /* 1075 * This is the iodone() function for buffers which have been 1076 * logged. It is called when they are eventually flushed out. 1077 * It should remove the buf item from the AIL, and free the buf item. 1078 * It is called by xfs_buf_iodone_callbacks() above which will take 1079 * care of cleaning up the buffer itself. 1080 */ 1081 void 1082 xfs_buf_iodone( 1083 struct xfs_buf *bp, 1084 struct xfs_log_item *lip) 1085 { 1086 struct xfs_ail *ailp = lip->li_ailp; 1087 1088 ASSERT(BUF_ITEM(lip)->bli_buf == bp); 1089 1090 xfs_buf_rele(bp); 1091 1092 /* 1093 * If we are forcibly shutting down, this may well be 1094 * off the AIL already. That's because we simulate the 1095 * log-committed callbacks to unpin these buffers. Or we may never 1096 * have put this item on AIL because of the transaction was 1097 * aborted forcibly. xfs_trans_ail_delete() takes care of these. 1098 * 1099 * Either way, AIL is useless if we're forcing a shutdown. 1100 */ 1101 spin_lock(&ailp->xa_lock); 1102 xfs_trans_ail_delete(ailp, lip); 1103 xfs_buf_item_free(BUF_ITEM(lip)); 1104 } 1105