1 /* 2 * Copyright (c) 2000-2005 Silicon Graphics, Inc. 3 * All Rights Reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it would be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 #include "xfs.h" 19 #include "xfs_fs.h" 20 #include "xfs_types.h" 21 #include "xfs_bit.h" 22 #include "xfs_log.h" 23 #include "xfs_inum.h" 24 #include "xfs_trans.h" 25 #include "xfs_sb.h" 26 #include "xfs_ag.h" 27 #include "xfs_mount.h" 28 #include "xfs_buf_item.h" 29 #include "xfs_trans_priv.h" 30 #include "xfs_error.h" 31 #include "xfs_trace.h" 32 33 34 kmem_zone_t *xfs_buf_item_zone; 35 36 static inline struct xfs_buf_log_item *BUF_ITEM(struct xfs_log_item *lip) 37 { 38 return container_of(lip, struct xfs_buf_log_item, bli_item); 39 } 40 41 42 #ifdef XFS_TRANS_DEBUG 43 /* 44 * This function uses an alternate strategy for tracking the bytes 45 * that the user requests to be logged. This can then be used 46 * in conjunction with the bli_orig array in the buf log item to 47 * catch bugs in our callers' code. 48 * 49 * We also double check the bits set in xfs_buf_item_log using a 50 * simple algorithm to check that every byte is accounted for. 51 */ 52 STATIC void 53 xfs_buf_item_log_debug( 54 xfs_buf_log_item_t *bip, 55 uint first, 56 uint last) 57 { 58 uint x; 59 uint byte; 60 uint nbytes; 61 uint chunk_num; 62 uint word_num; 63 uint bit_num; 64 uint bit_set; 65 uint *wordp; 66 67 ASSERT(bip->bli_logged != NULL); 68 byte = first; 69 nbytes = last - first + 1; 70 bfset(bip->bli_logged, first, nbytes); 71 for (x = 0; x < nbytes; x++) { 72 chunk_num = byte >> XFS_BLF_SHIFT; 73 word_num = chunk_num >> BIT_TO_WORD_SHIFT; 74 bit_num = chunk_num & (NBWORD - 1); 75 wordp = &(bip->bli_format.blf_data_map[word_num]); 76 bit_set = *wordp & (1 << bit_num); 77 ASSERT(bit_set); 78 byte++; 79 } 80 } 81 82 /* 83 * This function is called when we flush something into a buffer without 84 * logging it. This happens for things like inodes which are logged 85 * separately from the buffer. 86 */ 87 void 88 xfs_buf_item_flush_log_debug( 89 xfs_buf_t *bp, 90 uint first, 91 uint last) 92 { 93 xfs_buf_log_item_t *bip = bp->b_fspriv; 94 uint nbytes; 95 96 if (bip == NULL || (bip->bli_item.li_type != XFS_LI_BUF)) 97 return; 98 99 ASSERT(bip->bli_logged != NULL); 100 nbytes = last - first + 1; 101 bfset(bip->bli_logged, first, nbytes); 102 } 103 104 /* 105 * This function is called to verify that our callers have logged 106 * all the bytes that they changed. 107 * 108 * It does this by comparing the original copy of the buffer stored in 109 * the buf log item's bli_orig array to the current copy of the buffer 110 * and ensuring that all bytes which mismatch are set in the bli_logged 111 * array of the buf log item. 112 */ 113 STATIC void 114 xfs_buf_item_log_check( 115 xfs_buf_log_item_t *bip) 116 { 117 char *orig; 118 char *buffer; 119 int x; 120 xfs_buf_t *bp; 121 122 ASSERT(bip->bli_orig != NULL); 123 ASSERT(bip->bli_logged != NULL); 124 125 bp = bip->bli_buf; 126 ASSERT(XFS_BUF_COUNT(bp) > 0); 127 ASSERT(bp->b_addr != NULL); 128 orig = bip->bli_orig; 129 buffer = bp->b_addr; 130 for (x = 0; x < XFS_BUF_COUNT(bp); x++) { 131 if (orig[x] != buffer[x] && !btst(bip->bli_logged, x)) { 132 xfs_emerg(bp->b_mount, 133 "%s: bip %x buffer %x orig %x index %d", 134 __func__, bip, bp, orig, x); 135 ASSERT(0); 136 } 137 } 138 } 139 #else 140 #define xfs_buf_item_log_debug(x,y,z) 141 #define xfs_buf_item_log_check(x) 142 #endif 143 144 STATIC void xfs_buf_do_callbacks(struct xfs_buf *bp); 145 146 /* 147 * This returns the number of log iovecs needed to log the 148 * given buf log item. 149 * 150 * It calculates this as 1 iovec for the buf log format structure 151 * and 1 for each stretch of non-contiguous chunks to be logged. 152 * Contiguous chunks are logged in a single iovec. 153 * 154 * If the XFS_BLI_STALE flag has been set, then log nothing. 155 */ 156 STATIC uint 157 xfs_buf_item_size( 158 struct xfs_log_item *lip) 159 { 160 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 161 struct xfs_buf *bp = bip->bli_buf; 162 uint nvecs; 163 int next_bit; 164 int last_bit; 165 166 ASSERT(atomic_read(&bip->bli_refcount) > 0); 167 if (bip->bli_flags & XFS_BLI_STALE) { 168 /* 169 * The buffer is stale, so all we need to log 170 * is the buf log format structure with the 171 * cancel flag in it. 172 */ 173 trace_xfs_buf_item_size_stale(bip); 174 ASSERT(bip->bli_format.blf_flags & XFS_BLF_CANCEL); 175 return 1; 176 } 177 178 ASSERT(bip->bli_flags & XFS_BLI_LOGGED); 179 nvecs = 1; 180 last_bit = xfs_next_bit(bip->bli_format.blf_data_map, 181 bip->bli_format.blf_map_size, 0); 182 ASSERT(last_bit != -1); 183 nvecs++; 184 while (last_bit != -1) { 185 /* 186 * This takes the bit number to start looking from and 187 * returns the next set bit from there. It returns -1 188 * if there are no more bits set or the start bit is 189 * beyond the end of the bitmap. 190 */ 191 next_bit = xfs_next_bit(bip->bli_format.blf_data_map, 192 bip->bli_format.blf_map_size, 193 last_bit + 1); 194 /* 195 * If we run out of bits, leave the loop, 196 * else if we find a new set of bits bump the number of vecs, 197 * else keep scanning the current set of bits. 198 */ 199 if (next_bit == -1) { 200 last_bit = -1; 201 } else if (next_bit != last_bit + 1) { 202 last_bit = next_bit; 203 nvecs++; 204 } else if (xfs_buf_offset(bp, next_bit * XFS_BLF_CHUNK) != 205 (xfs_buf_offset(bp, last_bit * XFS_BLF_CHUNK) + 206 XFS_BLF_CHUNK)) { 207 last_bit = next_bit; 208 nvecs++; 209 } else { 210 last_bit++; 211 } 212 } 213 214 trace_xfs_buf_item_size(bip); 215 return nvecs; 216 } 217 218 /* 219 * This is called to fill in the vector of log iovecs for the 220 * given log buf item. It fills the first entry with a buf log 221 * format structure, and the rest point to contiguous chunks 222 * within the buffer. 223 */ 224 STATIC void 225 xfs_buf_item_format( 226 struct xfs_log_item *lip, 227 struct xfs_log_iovec *vecp) 228 { 229 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 230 struct xfs_buf *bp = bip->bli_buf; 231 uint base_size; 232 uint nvecs; 233 int first_bit; 234 int last_bit; 235 int next_bit; 236 uint nbits; 237 uint buffer_offset; 238 239 ASSERT(atomic_read(&bip->bli_refcount) > 0); 240 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || 241 (bip->bli_flags & XFS_BLI_STALE)); 242 243 /* 244 * The size of the base structure is the size of the 245 * declared structure plus the space for the extra words 246 * of the bitmap. We subtract one from the map size, because 247 * the first element of the bitmap is accounted for in the 248 * size of the base structure. 249 */ 250 base_size = 251 (uint)(sizeof(xfs_buf_log_format_t) + 252 ((bip->bli_format.blf_map_size - 1) * sizeof(uint))); 253 vecp->i_addr = &bip->bli_format; 254 vecp->i_len = base_size; 255 vecp->i_type = XLOG_REG_TYPE_BFORMAT; 256 vecp++; 257 nvecs = 1; 258 259 /* 260 * If it is an inode buffer, transfer the in-memory state to the 261 * format flags and clear the in-memory state. We do not transfer 262 * this state if the inode buffer allocation has not yet been committed 263 * to the log as setting the XFS_BLI_INODE_BUF flag will prevent 264 * correct replay of the inode allocation. 265 */ 266 if (bip->bli_flags & XFS_BLI_INODE_BUF) { 267 if (!((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && 268 xfs_log_item_in_current_chkpt(lip))) 269 bip->bli_format.blf_flags |= XFS_BLF_INODE_BUF; 270 bip->bli_flags &= ~XFS_BLI_INODE_BUF; 271 } 272 273 if (bip->bli_flags & XFS_BLI_STALE) { 274 /* 275 * The buffer is stale, so all we need to log 276 * is the buf log format structure with the 277 * cancel flag in it. 278 */ 279 trace_xfs_buf_item_format_stale(bip); 280 ASSERT(bip->bli_format.blf_flags & XFS_BLF_CANCEL); 281 bip->bli_format.blf_size = nvecs; 282 return; 283 } 284 285 /* 286 * Fill in an iovec for each set of contiguous chunks. 287 */ 288 first_bit = xfs_next_bit(bip->bli_format.blf_data_map, 289 bip->bli_format.blf_map_size, 0); 290 ASSERT(first_bit != -1); 291 last_bit = first_bit; 292 nbits = 1; 293 for (;;) { 294 /* 295 * This takes the bit number to start looking from and 296 * returns the next set bit from there. It returns -1 297 * if there are no more bits set or the start bit is 298 * beyond the end of the bitmap. 299 */ 300 next_bit = xfs_next_bit(bip->bli_format.blf_data_map, 301 bip->bli_format.blf_map_size, 302 (uint)last_bit + 1); 303 /* 304 * If we run out of bits fill in the last iovec and get 305 * out of the loop. 306 * Else if we start a new set of bits then fill in the 307 * iovec for the series we were looking at and start 308 * counting the bits in the new one. 309 * Else we're still in the same set of bits so just 310 * keep counting and scanning. 311 */ 312 if (next_bit == -1) { 313 buffer_offset = first_bit * XFS_BLF_CHUNK; 314 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 315 vecp->i_len = nbits * XFS_BLF_CHUNK; 316 vecp->i_type = XLOG_REG_TYPE_BCHUNK; 317 nvecs++; 318 break; 319 } else if (next_bit != last_bit + 1) { 320 buffer_offset = first_bit * XFS_BLF_CHUNK; 321 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 322 vecp->i_len = nbits * XFS_BLF_CHUNK; 323 vecp->i_type = XLOG_REG_TYPE_BCHUNK; 324 nvecs++; 325 vecp++; 326 first_bit = next_bit; 327 last_bit = next_bit; 328 nbits = 1; 329 } else if (xfs_buf_offset(bp, next_bit << XFS_BLF_SHIFT) != 330 (xfs_buf_offset(bp, last_bit << XFS_BLF_SHIFT) + 331 XFS_BLF_CHUNK)) { 332 buffer_offset = first_bit * XFS_BLF_CHUNK; 333 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 334 vecp->i_len = nbits * XFS_BLF_CHUNK; 335 vecp->i_type = XLOG_REG_TYPE_BCHUNK; 336 /* You would think we need to bump the nvecs here too, but we do not 337 * this number is used by recovery, and it gets confused by the boundary 338 * split here 339 * nvecs++; 340 */ 341 vecp++; 342 first_bit = next_bit; 343 last_bit = next_bit; 344 nbits = 1; 345 } else { 346 last_bit++; 347 nbits++; 348 } 349 } 350 bip->bli_format.blf_size = nvecs; 351 352 /* 353 * Check to make sure everything is consistent. 354 */ 355 trace_xfs_buf_item_format(bip); 356 xfs_buf_item_log_check(bip); 357 } 358 359 /* 360 * This is called to pin the buffer associated with the buf log item in memory 361 * so it cannot be written out. 362 * 363 * We also always take a reference to the buffer log item here so that the bli 364 * is held while the item is pinned in memory. This means that we can 365 * unconditionally drop the reference count a transaction holds when the 366 * transaction is completed. 367 */ 368 STATIC void 369 xfs_buf_item_pin( 370 struct xfs_log_item *lip) 371 { 372 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 373 374 ASSERT(atomic_read(&bip->bli_refcount) > 0); 375 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || 376 (bip->bli_flags & XFS_BLI_STALE)); 377 378 trace_xfs_buf_item_pin(bip); 379 380 atomic_inc(&bip->bli_refcount); 381 atomic_inc(&bip->bli_buf->b_pin_count); 382 } 383 384 /* 385 * This is called to unpin the buffer associated with the buf log 386 * item which was previously pinned with a call to xfs_buf_item_pin(). 387 * 388 * Also drop the reference to the buf item for the current transaction. 389 * If the XFS_BLI_STALE flag is set and we are the last reference, 390 * then free up the buf log item and unlock the buffer. 391 * 392 * If the remove flag is set we are called from uncommit in the 393 * forced-shutdown path. If that is true and the reference count on 394 * the log item is going to drop to zero we need to free the item's 395 * descriptor in the transaction. 396 */ 397 STATIC void 398 xfs_buf_item_unpin( 399 struct xfs_log_item *lip, 400 int remove) 401 { 402 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 403 xfs_buf_t *bp = bip->bli_buf; 404 struct xfs_ail *ailp = lip->li_ailp; 405 int stale = bip->bli_flags & XFS_BLI_STALE; 406 int freed; 407 408 ASSERT(bp->b_fspriv == bip); 409 ASSERT(atomic_read(&bip->bli_refcount) > 0); 410 411 trace_xfs_buf_item_unpin(bip); 412 413 freed = atomic_dec_and_test(&bip->bli_refcount); 414 415 if (atomic_dec_and_test(&bp->b_pin_count)) 416 wake_up_all(&bp->b_waiters); 417 418 if (freed && stale) { 419 ASSERT(bip->bli_flags & XFS_BLI_STALE); 420 ASSERT(xfs_buf_islocked(bp)); 421 ASSERT(!(XFS_BUF_ISDELAYWRITE(bp))); 422 ASSERT(XFS_BUF_ISSTALE(bp)); 423 ASSERT(bip->bli_format.blf_flags & XFS_BLF_CANCEL); 424 425 trace_xfs_buf_item_unpin_stale(bip); 426 427 if (remove) { 428 /* 429 * If we are in a transaction context, we have to 430 * remove the log item from the transaction as we are 431 * about to release our reference to the buffer. If we 432 * don't, the unlock that occurs later in 433 * xfs_trans_uncommit() will try to reference the 434 * buffer which we no longer have a hold on. 435 */ 436 if (lip->li_desc) 437 xfs_trans_del_item(lip); 438 439 /* 440 * Since the transaction no longer refers to the buffer, 441 * the buffer should no longer refer to the transaction. 442 */ 443 bp->b_transp = NULL; 444 } 445 446 /* 447 * If we get called here because of an IO error, we may 448 * or may not have the item on the AIL. xfs_trans_ail_delete() 449 * will take care of that situation. 450 * xfs_trans_ail_delete() drops the AIL lock. 451 */ 452 if (bip->bli_flags & XFS_BLI_STALE_INODE) { 453 xfs_buf_do_callbacks(bp); 454 bp->b_fspriv = NULL; 455 bp->b_iodone = NULL; 456 } else { 457 spin_lock(&ailp->xa_lock); 458 xfs_trans_ail_delete(ailp, (xfs_log_item_t *)bip); 459 xfs_buf_item_relse(bp); 460 ASSERT(bp->b_fspriv == NULL); 461 } 462 xfs_buf_relse(bp); 463 } 464 } 465 466 /* 467 * This is called to attempt to lock the buffer associated with this 468 * buf log item. Don't sleep on the buffer lock. If we can't get 469 * the lock right away, return 0. If we can get the lock, take a 470 * reference to the buffer. If this is a delayed write buffer that 471 * needs AIL help to be written back, invoke the pushbuf routine 472 * rather than the normal success path. 473 */ 474 STATIC uint 475 xfs_buf_item_trylock( 476 struct xfs_log_item *lip) 477 { 478 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 479 struct xfs_buf *bp = bip->bli_buf; 480 481 if (xfs_buf_ispinned(bp)) 482 return XFS_ITEM_PINNED; 483 if (!xfs_buf_trylock(bp)) 484 return XFS_ITEM_LOCKED; 485 486 /* take a reference to the buffer. */ 487 xfs_buf_hold(bp); 488 489 ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); 490 trace_xfs_buf_item_trylock(bip); 491 if (XFS_BUF_ISDELAYWRITE(bp)) 492 return XFS_ITEM_PUSHBUF; 493 return XFS_ITEM_SUCCESS; 494 } 495 496 /* 497 * Release the buffer associated with the buf log item. If there is no dirty 498 * logged data associated with the buffer recorded in the buf log item, then 499 * free the buf log item and remove the reference to it in the buffer. 500 * 501 * This call ignores the recursion count. It is only called when the buffer 502 * should REALLY be unlocked, regardless of the recursion count. 503 * 504 * We unconditionally drop the transaction's reference to the log item. If the 505 * item was logged, then another reference was taken when it was pinned, so we 506 * can safely drop the transaction reference now. This also allows us to avoid 507 * potential races with the unpin code freeing the bli by not referencing the 508 * bli after we've dropped the reference count. 509 * 510 * If the XFS_BLI_HOLD flag is set in the buf log item, then free the log item 511 * if necessary but do not unlock the buffer. This is for support of 512 * xfs_trans_bhold(). Make sure the XFS_BLI_HOLD field is cleared if we don't 513 * free the item. 514 */ 515 STATIC void 516 xfs_buf_item_unlock( 517 struct xfs_log_item *lip) 518 { 519 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 520 struct xfs_buf *bp = bip->bli_buf; 521 int aborted; 522 uint hold; 523 524 /* Clear the buffer's association with this transaction. */ 525 bp->b_transp = NULL; 526 527 /* 528 * If this is a transaction abort, don't return early. Instead, allow 529 * the brelse to happen. Normally it would be done for stale 530 * (cancelled) buffers at unpin time, but we'll never go through the 531 * pin/unpin cycle if we abort inside commit. 532 */ 533 aborted = (lip->li_flags & XFS_LI_ABORTED) != 0; 534 535 /* 536 * Before possibly freeing the buf item, determine if we should 537 * release the buffer at the end of this routine. 538 */ 539 hold = bip->bli_flags & XFS_BLI_HOLD; 540 541 /* Clear the per transaction state. */ 542 bip->bli_flags &= ~(XFS_BLI_LOGGED | XFS_BLI_HOLD); 543 544 /* 545 * If the buf item is marked stale, then don't do anything. We'll 546 * unlock the buffer and free the buf item when the buffer is unpinned 547 * for the last time. 548 */ 549 if (bip->bli_flags & XFS_BLI_STALE) { 550 trace_xfs_buf_item_unlock_stale(bip); 551 ASSERT(bip->bli_format.blf_flags & XFS_BLF_CANCEL); 552 if (!aborted) { 553 atomic_dec(&bip->bli_refcount); 554 return; 555 } 556 } 557 558 trace_xfs_buf_item_unlock(bip); 559 560 /* 561 * If the buf item isn't tracking any data, free it, otherwise drop the 562 * reference we hold to it. 563 */ 564 if (xfs_bitmap_empty(bip->bli_format.blf_data_map, 565 bip->bli_format.blf_map_size)) 566 xfs_buf_item_relse(bp); 567 else 568 atomic_dec(&bip->bli_refcount); 569 570 if (!hold) 571 xfs_buf_relse(bp); 572 } 573 574 /* 575 * This is called to find out where the oldest active copy of the 576 * buf log item in the on disk log resides now that the last log 577 * write of it completed at the given lsn. 578 * We always re-log all the dirty data in a buffer, so usually the 579 * latest copy in the on disk log is the only one that matters. For 580 * those cases we simply return the given lsn. 581 * 582 * The one exception to this is for buffers full of newly allocated 583 * inodes. These buffers are only relogged with the XFS_BLI_INODE_BUF 584 * flag set, indicating that only the di_next_unlinked fields from the 585 * inodes in the buffers will be replayed during recovery. If the 586 * original newly allocated inode images have not yet been flushed 587 * when the buffer is so relogged, then we need to make sure that we 588 * keep the old images in the 'active' portion of the log. We do this 589 * by returning the original lsn of that transaction here rather than 590 * the current one. 591 */ 592 STATIC xfs_lsn_t 593 xfs_buf_item_committed( 594 struct xfs_log_item *lip, 595 xfs_lsn_t lsn) 596 { 597 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 598 599 trace_xfs_buf_item_committed(bip); 600 601 if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && lip->li_lsn != 0) 602 return lip->li_lsn; 603 return lsn; 604 } 605 606 /* 607 * The buffer is locked, but is not a delayed write buffer. This happens 608 * if we race with IO completion and hence we don't want to try to write it 609 * again. Just release the buffer. 610 */ 611 STATIC void 612 xfs_buf_item_push( 613 struct xfs_log_item *lip) 614 { 615 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 616 struct xfs_buf *bp = bip->bli_buf; 617 618 ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); 619 ASSERT(!XFS_BUF_ISDELAYWRITE(bp)); 620 621 trace_xfs_buf_item_push(bip); 622 623 xfs_buf_relse(bp); 624 } 625 626 /* 627 * The buffer is locked and is a delayed write buffer. Promote the buffer 628 * in the delayed write queue as the caller knows that they must invoke 629 * the xfsbufd to get this buffer written. We have to unlock the buffer 630 * to allow the xfsbufd to write it, too. 631 */ 632 STATIC bool 633 xfs_buf_item_pushbuf( 634 struct xfs_log_item *lip) 635 { 636 struct xfs_buf_log_item *bip = BUF_ITEM(lip); 637 struct xfs_buf *bp = bip->bli_buf; 638 639 ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); 640 ASSERT(XFS_BUF_ISDELAYWRITE(bp)); 641 642 trace_xfs_buf_item_pushbuf(bip); 643 644 xfs_buf_delwri_promote(bp); 645 xfs_buf_relse(bp); 646 return true; 647 } 648 649 STATIC void 650 xfs_buf_item_committing( 651 struct xfs_log_item *lip, 652 xfs_lsn_t commit_lsn) 653 { 654 } 655 656 /* 657 * This is the ops vector shared by all buf log items. 658 */ 659 static const struct xfs_item_ops xfs_buf_item_ops = { 660 .iop_size = xfs_buf_item_size, 661 .iop_format = xfs_buf_item_format, 662 .iop_pin = xfs_buf_item_pin, 663 .iop_unpin = xfs_buf_item_unpin, 664 .iop_trylock = xfs_buf_item_trylock, 665 .iop_unlock = xfs_buf_item_unlock, 666 .iop_committed = xfs_buf_item_committed, 667 .iop_push = xfs_buf_item_push, 668 .iop_pushbuf = xfs_buf_item_pushbuf, 669 .iop_committing = xfs_buf_item_committing 670 }; 671 672 673 /* 674 * Allocate a new buf log item to go with the given buffer. 675 * Set the buffer's b_fsprivate field to point to the new 676 * buf log item. If there are other item's attached to the 677 * buffer (see xfs_buf_attach_iodone() below), then put the 678 * buf log item at the front. 679 */ 680 void 681 xfs_buf_item_init( 682 xfs_buf_t *bp, 683 xfs_mount_t *mp) 684 { 685 xfs_log_item_t *lip = bp->b_fspriv; 686 xfs_buf_log_item_t *bip; 687 int chunks; 688 int map_size; 689 690 /* 691 * Check to see if there is already a buf log item for 692 * this buffer. If there is, it is guaranteed to be 693 * the first. If we do already have one, there is 694 * nothing to do here so return. 695 */ 696 ASSERT(bp->b_target->bt_mount == mp); 697 if (lip != NULL && lip->li_type == XFS_LI_BUF) 698 return; 699 700 /* 701 * chunks is the number of XFS_BLF_CHUNK size pieces 702 * the buffer can be divided into. Make sure not to 703 * truncate any pieces. map_size is the size of the 704 * bitmap needed to describe the chunks of the buffer. 705 */ 706 chunks = (int)((XFS_BUF_COUNT(bp) + (XFS_BLF_CHUNK - 1)) >> XFS_BLF_SHIFT); 707 map_size = (int)((chunks + NBWORD) >> BIT_TO_WORD_SHIFT); 708 709 bip = (xfs_buf_log_item_t*)kmem_zone_zalloc(xfs_buf_item_zone, 710 KM_SLEEP); 711 xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops); 712 bip->bli_buf = bp; 713 xfs_buf_hold(bp); 714 bip->bli_format.blf_type = XFS_LI_BUF; 715 bip->bli_format.blf_blkno = (__int64_t)XFS_BUF_ADDR(bp); 716 bip->bli_format.blf_len = (ushort)BTOBB(XFS_BUF_COUNT(bp)); 717 bip->bli_format.blf_map_size = map_size; 718 719 #ifdef XFS_TRANS_DEBUG 720 /* 721 * Allocate the arrays for tracking what needs to be logged 722 * and what our callers request to be logged. bli_orig 723 * holds a copy of the original, clean buffer for comparison 724 * against, and bli_logged keeps a 1 bit flag per byte in 725 * the buffer to indicate which bytes the callers have asked 726 * to have logged. 727 */ 728 bip->bli_orig = (char *)kmem_alloc(XFS_BUF_COUNT(bp), KM_SLEEP); 729 memcpy(bip->bli_orig, bp->b_addr, XFS_BUF_COUNT(bp)); 730 bip->bli_logged = (char *)kmem_zalloc(XFS_BUF_COUNT(bp) / NBBY, KM_SLEEP); 731 #endif 732 733 /* 734 * Put the buf item into the list of items attached to the 735 * buffer at the front. 736 */ 737 if (bp->b_fspriv) 738 bip->bli_item.li_bio_list = bp->b_fspriv; 739 bp->b_fspriv = bip; 740 } 741 742 743 /* 744 * Mark bytes first through last inclusive as dirty in the buf 745 * item's bitmap. 746 */ 747 void 748 xfs_buf_item_log( 749 xfs_buf_log_item_t *bip, 750 uint first, 751 uint last) 752 { 753 uint first_bit; 754 uint last_bit; 755 uint bits_to_set; 756 uint bits_set; 757 uint word_num; 758 uint *wordp; 759 uint bit; 760 uint end_bit; 761 uint mask; 762 763 /* 764 * Mark the item as having some dirty data for 765 * quick reference in xfs_buf_item_dirty. 766 */ 767 bip->bli_flags |= XFS_BLI_DIRTY; 768 769 /* 770 * Convert byte offsets to bit numbers. 771 */ 772 first_bit = first >> XFS_BLF_SHIFT; 773 last_bit = last >> XFS_BLF_SHIFT; 774 775 /* 776 * Calculate the total number of bits to be set. 777 */ 778 bits_to_set = last_bit - first_bit + 1; 779 780 /* 781 * Get a pointer to the first word in the bitmap 782 * to set a bit in. 783 */ 784 word_num = first_bit >> BIT_TO_WORD_SHIFT; 785 wordp = &(bip->bli_format.blf_data_map[word_num]); 786 787 /* 788 * Calculate the starting bit in the first word. 789 */ 790 bit = first_bit & (uint)(NBWORD - 1); 791 792 /* 793 * First set any bits in the first word of our range. 794 * If it starts at bit 0 of the word, it will be 795 * set below rather than here. That is what the variable 796 * bit tells us. The variable bits_set tracks the number 797 * of bits that have been set so far. End_bit is the number 798 * of the last bit to be set in this word plus one. 799 */ 800 if (bit) { 801 end_bit = MIN(bit + bits_to_set, (uint)NBWORD); 802 mask = ((1 << (end_bit - bit)) - 1) << bit; 803 *wordp |= mask; 804 wordp++; 805 bits_set = end_bit - bit; 806 } else { 807 bits_set = 0; 808 } 809 810 /* 811 * Now set bits a whole word at a time that are between 812 * first_bit and last_bit. 813 */ 814 while ((bits_to_set - bits_set) >= NBWORD) { 815 *wordp |= 0xffffffff; 816 bits_set += NBWORD; 817 wordp++; 818 } 819 820 /* 821 * Finally, set any bits left to be set in one last partial word. 822 */ 823 end_bit = bits_to_set - bits_set; 824 if (end_bit) { 825 mask = (1 << end_bit) - 1; 826 *wordp |= mask; 827 } 828 829 xfs_buf_item_log_debug(bip, first, last); 830 } 831 832 833 /* 834 * Return 1 if the buffer has some data that has been logged (at any 835 * point, not just the current transaction) and 0 if not. 836 */ 837 uint 838 xfs_buf_item_dirty( 839 xfs_buf_log_item_t *bip) 840 { 841 return (bip->bli_flags & XFS_BLI_DIRTY); 842 } 843 844 STATIC void 845 xfs_buf_item_free( 846 xfs_buf_log_item_t *bip) 847 { 848 #ifdef XFS_TRANS_DEBUG 849 kmem_free(bip->bli_orig); 850 kmem_free(bip->bli_logged); 851 #endif /* XFS_TRANS_DEBUG */ 852 853 kmem_zone_free(xfs_buf_item_zone, bip); 854 } 855 856 /* 857 * This is called when the buf log item is no longer needed. It should 858 * free the buf log item associated with the given buffer and clear 859 * the buffer's pointer to the buf log item. If there are no more 860 * items in the list, clear the b_iodone field of the buffer (see 861 * xfs_buf_attach_iodone() below). 862 */ 863 void 864 xfs_buf_item_relse( 865 xfs_buf_t *bp) 866 { 867 xfs_buf_log_item_t *bip; 868 869 trace_xfs_buf_item_relse(bp, _RET_IP_); 870 871 bip = bp->b_fspriv; 872 bp->b_fspriv = bip->bli_item.li_bio_list; 873 if (bp->b_fspriv == NULL) 874 bp->b_iodone = NULL; 875 876 xfs_buf_rele(bp); 877 xfs_buf_item_free(bip); 878 } 879 880 881 /* 882 * Add the given log item with its callback to the list of callbacks 883 * to be called when the buffer's I/O completes. If it is not set 884 * already, set the buffer's b_iodone() routine to be 885 * xfs_buf_iodone_callbacks() and link the log item into the list of 886 * items rooted at b_fsprivate. Items are always added as the second 887 * entry in the list if there is a first, because the buf item code 888 * assumes that the buf log item is first. 889 */ 890 void 891 xfs_buf_attach_iodone( 892 xfs_buf_t *bp, 893 void (*cb)(xfs_buf_t *, xfs_log_item_t *), 894 xfs_log_item_t *lip) 895 { 896 xfs_log_item_t *head_lip; 897 898 ASSERT(xfs_buf_islocked(bp)); 899 900 lip->li_cb = cb; 901 head_lip = bp->b_fspriv; 902 if (head_lip) { 903 lip->li_bio_list = head_lip->li_bio_list; 904 head_lip->li_bio_list = lip; 905 } else { 906 bp->b_fspriv = lip; 907 } 908 909 ASSERT(bp->b_iodone == NULL || 910 bp->b_iodone == xfs_buf_iodone_callbacks); 911 bp->b_iodone = xfs_buf_iodone_callbacks; 912 } 913 914 /* 915 * We can have many callbacks on a buffer. Running the callbacks individually 916 * can cause a lot of contention on the AIL lock, so we allow for a single 917 * callback to be able to scan the remaining lip->li_bio_list for other items 918 * of the same type and callback to be processed in the first call. 919 * 920 * As a result, the loop walking the callback list below will also modify the 921 * list. it removes the first item from the list and then runs the callback. 922 * The loop then restarts from the new head of the list. This allows the 923 * callback to scan and modify the list attached to the buffer and we don't 924 * have to care about maintaining a next item pointer. 925 */ 926 STATIC void 927 xfs_buf_do_callbacks( 928 struct xfs_buf *bp) 929 { 930 struct xfs_log_item *lip; 931 932 while ((lip = bp->b_fspriv) != NULL) { 933 bp->b_fspriv = lip->li_bio_list; 934 ASSERT(lip->li_cb != NULL); 935 /* 936 * Clear the next pointer so we don't have any 937 * confusion if the item is added to another buf. 938 * Don't touch the log item after calling its 939 * callback, because it could have freed itself. 940 */ 941 lip->li_bio_list = NULL; 942 lip->li_cb(bp, lip); 943 } 944 } 945 946 /* 947 * This is the iodone() function for buffers which have had callbacks 948 * attached to them by xfs_buf_attach_iodone(). It should remove each 949 * log item from the buffer's list and call the callback of each in turn. 950 * When done, the buffer's fsprivate field is set to NULL and the buffer 951 * is unlocked with a call to iodone(). 952 */ 953 void 954 xfs_buf_iodone_callbacks( 955 struct xfs_buf *bp) 956 { 957 struct xfs_log_item *lip = bp->b_fspriv; 958 struct xfs_mount *mp = lip->li_mountp; 959 static ulong lasttime; 960 static xfs_buftarg_t *lasttarg; 961 962 if (likely(!xfs_buf_geterror(bp))) 963 goto do_callbacks; 964 965 /* 966 * If we've already decided to shutdown the filesystem because of 967 * I/O errors, there's no point in giving this a retry. 968 */ 969 if (XFS_FORCED_SHUTDOWN(mp)) { 970 xfs_buf_stale(bp); 971 XFS_BUF_DONE(bp); 972 trace_xfs_buf_item_iodone(bp, _RET_IP_); 973 goto do_callbacks; 974 } 975 976 if (bp->b_target != lasttarg || 977 time_after(jiffies, (lasttime + 5*HZ))) { 978 lasttime = jiffies; 979 xfs_buf_ioerror_alert(bp, __func__); 980 } 981 lasttarg = bp->b_target; 982 983 /* 984 * If the write was asynchronous then no one will be looking for the 985 * error. Clear the error state and write the buffer out again. 986 * 987 * During sync or umount we'll write all pending buffers again 988 * synchronous, which will catch these errors if they keep hanging 989 * around. 990 */ 991 if (XFS_BUF_ISASYNC(bp)) { 992 xfs_buf_ioerror(bp, 0); /* errno of 0 unsets the flag */ 993 994 if (!XFS_BUF_ISSTALE(bp)) { 995 xfs_buf_delwri_queue(bp); 996 XFS_BUF_DONE(bp); 997 } 998 ASSERT(bp->b_iodone != NULL); 999 trace_xfs_buf_item_iodone_async(bp, _RET_IP_); 1000 xfs_buf_relse(bp); 1001 return; 1002 } 1003 1004 /* 1005 * If the write of the buffer was synchronous, we want to make 1006 * sure to return the error to the caller of xfs_bwrite(). 1007 */ 1008 xfs_buf_stale(bp); 1009 XFS_BUF_DONE(bp); 1010 1011 trace_xfs_buf_error_relse(bp, _RET_IP_); 1012 1013 do_callbacks: 1014 xfs_buf_do_callbacks(bp); 1015 bp->b_fspriv = NULL; 1016 bp->b_iodone = NULL; 1017 xfs_buf_ioend(bp, 0); 1018 } 1019 1020 /* 1021 * This is the iodone() function for buffers which have been 1022 * logged. It is called when they are eventually flushed out. 1023 * It should remove the buf item from the AIL, and free the buf item. 1024 * It is called by xfs_buf_iodone_callbacks() above which will take 1025 * care of cleaning up the buffer itself. 1026 */ 1027 void 1028 xfs_buf_iodone( 1029 struct xfs_buf *bp, 1030 struct xfs_log_item *lip) 1031 { 1032 struct xfs_ail *ailp = lip->li_ailp; 1033 1034 ASSERT(BUF_ITEM(lip)->bli_buf == bp); 1035 1036 xfs_buf_rele(bp); 1037 1038 /* 1039 * If we are forcibly shutting down, this may well be 1040 * off the AIL already. That's because we simulate the 1041 * log-committed callbacks to unpin these buffers. Or we may never 1042 * have put this item on AIL because of the transaction was 1043 * aborted forcibly. xfs_trans_ail_delete() takes care of these. 1044 * 1045 * Either way, AIL is useless if we're forcing a shutdown. 1046 */ 1047 spin_lock(&ailp->xa_lock); 1048 xfs_trans_ail_delete(ailp, lip); 1049 xfs_buf_item_free(BUF_ITEM(lip)); 1050 } 1051