1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * linux/fs/jbd2/commit.c 4 * 5 * Written by Stephen C. Tweedie <sct@redhat.com>, 1998 6 * 7 * Copyright 1998 Red Hat corp --- All Rights Reserved 8 * 9 * Journal commit routines for the generic filesystem journaling code; 10 * part of the ext2fs journaling system. 11 */ 12 13 #include <linux/time.h> 14 #include <linux/fs.h> 15 #include <linux/jbd2.h> 16 #include <linux/errno.h> 17 #include <linux/slab.h> 18 #include <linux/mm.h> 19 #include <linux/pagemap.h> 20 #include <linux/jiffies.h> 21 #include <linux/crc32.h> 22 #include <linux/writeback.h> 23 #include <linux/backing-dev.h> 24 #include <linux/bio.h> 25 #include <linux/blkdev.h> 26 #include <linux/bitops.h> 27 #include <trace/events/jbd2.h> 28 29 /* 30 * IO end handler for temporary buffer_heads handling writes to the journal. 31 */ 32 static void journal_end_buffer_io_sync(struct buffer_head *bh, int uptodate) 33 { 34 struct buffer_head *orig_bh = bh->b_private; 35 36 BUFFER_TRACE(bh, ""); 37 if (uptodate) 38 set_buffer_uptodate(bh); 39 else 40 clear_buffer_uptodate(bh); 41 if (orig_bh) { 42 clear_bit_unlock(BH_Shadow, &orig_bh->b_state); 43 smp_mb__after_atomic(); 44 wake_up_bit(&orig_bh->b_state, BH_Shadow); 45 } 46 unlock_buffer(bh); 47 } 48 49 /* 50 * When an ext4 file is truncated, it is possible that some pages are not 51 * successfully freed, because they are attached to a committing transaction. 52 * After the transaction commits, these pages are left on the LRU, with no 53 * ->mapping, and with attached buffers. These pages are trivially reclaimable 54 * by the VM, but their apparent absence upsets the VM accounting, and it makes 55 * the numbers in /proc/meminfo look odd. 56 * 57 * So here, we have a buffer which has just come off the forget list. Look to 58 * see if we can strip all buffers from the backing page. 59 * 60 * Called under lock_journal(), and possibly under journal_datalist_lock. The 61 * caller provided us with a ref against the buffer, and we drop that here. 62 */ 63 static void release_buffer_page(struct buffer_head *bh) 64 { 65 struct page *page; 66 67 if (buffer_dirty(bh)) 68 goto nope; 69 if (atomic_read(&bh->b_count) != 1) 70 goto nope; 71 page = bh->b_page; 72 if (!page) 73 goto nope; 74 if (page->mapping) 75 goto nope; 76 77 /* OK, it's a truncated page */ 78 if (!trylock_page(page)) 79 goto nope; 80 81 get_page(page); 82 __brelse(bh); 83 try_to_free_buffers(page); 84 unlock_page(page); 85 put_page(page); 86 return; 87 88 nope: 89 __brelse(bh); 90 } 91 92 static void jbd2_commit_block_csum_set(journal_t *j, struct buffer_head *bh) 93 { 94 struct commit_header *h; 95 __u32 csum; 96 97 if (!jbd2_journal_has_csum_v2or3(j)) 98 return; 99 100 h = (struct commit_header *)(bh->b_data); 101 h->h_chksum_type = 0; 102 h->h_chksum_size = 0; 103 h->h_chksum[0] = 0; 104 csum = jbd2_chksum(j, j->j_csum_seed, bh->b_data, j->j_blocksize); 105 h->h_chksum[0] = cpu_to_be32(csum); 106 } 107 108 /* 109 * Done it all: now submit the commit record. We should have 110 * cleaned up our previous buffers by now, so if we are in abort 111 * mode we can now just skip the rest of the journal write 112 * entirely. 113 * 114 * Returns 1 if the journal needs to be aborted or 0 on success 115 */ 116 static int journal_submit_commit_record(journal_t *journal, 117 transaction_t *commit_transaction, 118 struct buffer_head **cbh, 119 __u32 crc32_sum) 120 { 121 struct commit_header *tmp; 122 struct buffer_head *bh; 123 int ret; 124 struct timespec64 now; 125 126 *cbh = NULL; 127 128 if (is_journal_aborted(journal)) 129 return 0; 130 131 bh = jbd2_journal_get_descriptor_buffer(commit_transaction, 132 JBD2_COMMIT_BLOCK); 133 if (!bh) 134 return 1; 135 136 tmp = (struct commit_header *)bh->b_data; 137 ktime_get_coarse_real_ts64(&now); 138 tmp->h_commit_sec = cpu_to_be64(now.tv_sec); 139 tmp->h_commit_nsec = cpu_to_be32(now.tv_nsec); 140 141 if (jbd2_has_feature_checksum(journal)) { 142 tmp->h_chksum_type = JBD2_CRC32_CHKSUM; 143 tmp->h_chksum_size = JBD2_CRC32_CHKSUM_SIZE; 144 tmp->h_chksum[0] = cpu_to_be32(crc32_sum); 145 } 146 jbd2_commit_block_csum_set(journal, bh); 147 148 BUFFER_TRACE(bh, "submit commit block"); 149 lock_buffer(bh); 150 clear_buffer_dirty(bh); 151 set_buffer_uptodate(bh); 152 bh->b_end_io = journal_end_buffer_io_sync; 153 154 if (journal->j_flags & JBD2_BARRIER && 155 !jbd2_has_feature_async_commit(journal)) 156 ret = submit_bh(REQ_OP_WRITE, 157 REQ_SYNC | REQ_PREFLUSH | REQ_FUA, bh); 158 else 159 ret = submit_bh(REQ_OP_WRITE, REQ_SYNC, bh); 160 161 *cbh = bh; 162 return ret; 163 } 164 165 /* 166 * This function along with journal_submit_commit_record 167 * allows to write the commit record asynchronously. 168 */ 169 static int journal_wait_on_commit_record(journal_t *journal, 170 struct buffer_head *bh) 171 { 172 int ret = 0; 173 174 clear_buffer_dirty(bh); 175 wait_on_buffer(bh); 176 177 if (unlikely(!buffer_uptodate(bh))) 178 ret = -EIO; 179 put_bh(bh); /* One for getblk() */ 180 181 return ret; 182 } 183 184 /* 185 * write the filemap data using writepage() address_space_operations. 186 * We don't do block allocation here even for delalloc. We don't 187 * use writepages() because with delayed allocation we may be doing 188 * block allocation in writepages(). 189 */ 190 int jbd2_journal_submit_inode_data_buffers(struct jbd2_inode *jinode) 191 { 192 struct address_space *mapping = jinode->i_vfs_inode->i_mapping; 193 struct writeback_control wbc = { 194 .sync_mode = WB_SYNC_ALL, 195 .nr_to_write = mapping->nrpages * 2, 196 .range_start = jinode->i_dirty_start, 197 .range_end = jinode->i_dirty_end, 198 }; 199 200 /* 201 * submit the inode data buffers. We use writepage 202 * instead of writepages. Because writepages can do 203 * block allocation with delalloc. We need to write 204 * only allocated blocks here. 205 */ 206 return generic_writepages(mapping, &wbc); 207 } 208 209 /* Send all the data buffers related to an inode */ 210 int jbd2_submit_inode_data(struct jbd2_inode *jinode) 211 { 212 213 if (!jinode || !(jinode->i_flags & JI_WRITE_DATA)) 214 return 0; 215 216 trace_jbd2_submit_inode_data(jinode->i_vfs_inode); 217 return jbd2_journal_submit_inode_data_buffers(jinode); 218 219 } 220 EXPORT_SYMBOL(jbd2_submit_inode_data); 221 222 int jbd2_wait_inode_data(journal_t *journal, struct jbd2_inode *jinode) 223 { 224 if (!jinode || !(jinode->i_flags & JI_WAIT_DATA) || 225 !jinode->i_vfs_inode || !jinode->i_vfs_inode->i_mapping) 226 return 0; 227 return filemap_fdatawait_range_keep_errors( 228 jinode->i_vfs_inode->i_mapping, jinode->i_dirty_start, 229 jinode->i_dirty_end); 230 } 231 EXPORT_SYMBOL(jbd2_wait_inode_data); 232 233 /* 234 * Submit all the data buffers of inode associated with the transaction to 235 * disk. 236 * 237 * We are in a committing transaction. Therefore no new inode can be added to 238 * our inode list. We use JI_COMMIT_RUNNING flag to protect inode we currently 239 * operate on from being released while we write out pages. 240 */ 241 static int journal_submit_data_buffers(journal_t *journal, 242 transaction_t *commit_transaction) 243 { 244 struct jbd2_inode *jinode; 245 int err, ret = 0; 246 247 spin_lock(&journal->j_list_lock); 248 list_for_each_entry(jinode, &commit_transaction->t_inode_list, i_list) { 249 if (!(jinode->i_flags & JI_WRITE_DATA)) 250 continue; 251 jinode->i_flags |= JI_COMMIT_RUNNING; 252 spin_unlock(&journal->j_list_lock); 253 /* submit the inode data buffers. */ 254 trace_jbd2_submit_inode_data(jinode->i_vfs_inode); 255 if (journal->j_submit_inode_data_buffers) { 256 err = journal->j_submit_inode_data_buffers(jinode); 257 if (!ret) 258 ret = err; 259 } 260 spin_lock(&journal->j_list_lock); 261 J_ASSERT(jinode->i_transaction == commit_transaction); 262 jinode->i_flags &= ~JI_COMMIT_RUNNING; 263 smp_mb(); 264 wake_up_bit(&jinode->i_flags, __JI_COMMIT_RUNNING); 265 } 266 spin_unlock(&journal->j_list_lock); 267 return ret; 268 } 269 270 int jbd2_journal_finish_inode_data_buffers(struct jbd2_inode *jinode) 271 { 272 struct address_space *mapping = jinode->i_vfs_inode->i_mapping; 273 274 return filemap_fdatawait_range_keep_errors(mapping, 275 jinode->i_dirty_start, 276 jinode->i_dirty_end); 277 } 278 279 /* 280 * Wait for data submitted for writeout, refile inodes to proper 281 * transaction if needed. 282 * 283 */ 284 static int journal_finish_inode_data_buffers(journal_t *journal, 285 transaction_t *commit_transaction) 286 { 287 struct jbd2_inode *jinode, *next_i; 288 int err, ret = 0; 289 290 /* For locking, see the comment in journal_submit_data_buffers() */ 291 spin_lock(&journal->j_list_lock); 292 list_for_each_entry(jinode, &commit_transaction->t_inode_list, i_list) { 293 if (!(jinode->i_flags & JI_WAIT_DATA)) 294 continue; 295 jinode->i_flags |= JI_COMMIT_RUNNING; 296 spin_unlock(&journal->j_list_lock); 297 /* wait for the inode data buffers writeout. */ 298 if (journal->j_finish_inode_data_buffers) { 299 err = journal->j_finish_inode_data_buffers(jinode); 300 if (!ret) 301 ret = err; 302 } 303 spin_lock(&journal->j_list_lock); 304 jinode->i_flags &= ~JI_COMMIT_RUNNING; 305 smp_mb(); 306 wake_up_bit(&jinode->i_flags, __JI_COMMIT_RUNNING); 307 } 308 309 /* Now refile inode to proper lists */ 310 list_for_each_entry_safe(jinode, next_i, 311 &commit_transaction->t_inode_list, i_list) { 312 list_del(&jinode->i_list); 313 if (jinode->i_next_transaction) { 314 jinode->i_transaction = jinode->i_next_transaction; 315 jinode->i_next_transaction = NULL; 316 list_add(&jinode->i_list, 317 &jinode->i_transaction->t_inode_list); 318 } else { 319 jinode->i_transaction = NULL; 320 jinode->i_dirty_start = 0; 321 jinode->i_dirty_end = 0; 322 } 323 } 324 spin_unlock(&journal->j_list_lock); 325 326 return ret; 327 } 328 329 static __u32 jbd2_checksum_data(__u32 crc32_sum, struct buffer_head *bh) 330 { 331 struct page *page = bh->b_page; 332 char *addr; 333 __u32 checksum; 334 335 addr = kmap_atomic(page); 336 checksum = crc32_be(crc32_sum, 337 (void *)(addr + offset_in_page(bh->b_data)), bh->b_size); 338 kunmap_atomic(addr); 339 340 return checksum; 341 } 342 343 static void write_tag_block(journal_t *j, journal_block_tag_t *tag, 344 unsigned long long block) 345 { 346 tag->t_blocknr = cpu_to_be32(block & (u32)~0); 347 if (jbd2_has_feature_64bit(j)) 348 tag->t_blocknr_high = cpu_to_be32((block >> 31) >> 1); 349 } 350 351 static void jbd2_block_tag_csum_set(journal_t *j, journal_block_tag_t *tag, 352 struct buffer_head *bh, __u32 sequence) 353 { 354 journal_block_tag3_t *tag3 = (journal_block_tag3_t *)tag; 355 struct page *page = bh->b_page; 356 __u8 *addr; 357 __u32 csum32; 358 __be32 seq; 359 360 if (!jbd2_journal_has_csum_v2or3(j)) 361 return; 362 363 seq = cpu_to_be32(sequence); 364 addr = kmap_atomic(page); 365 csum32 = jbd2_chksum(j, j->j_csum_seed, (__u8 *)&seq, sizeof(seq)); 366 csum32 = jbd2_chksum(j, csum32, addr + offset_in_page(bh->b_data), 367 bh->b_size); 368 kunmap_atomic(addr); 369 370 if (jbd2_has_feature_csum3(j)) 371 tag3->t_checksum = cpu_to_be32(csum32); 372 else 373 tag->t_checksum = cpu_to_be16(csum32); 374 } 375 /* 376 * jbd2_journal_commit_transaction 377 * 378 * The primary function for committing a transaction to the log. This 379 * function is called by the journal thread to begin a complete commit. 380 */ 381 void jbd2_journal_commit_transaction(journal_t *journal) 382 { 383 struct transaction_stats_s stats; 384 transaction_t *commit_transaction; 385 struct journal_head *jh; 386 struct buffer_head *descriptor; 387 struct buffer_head **wbuf = journal->j_wbuf; 388 int bufs; 389 int flags; 390 int err; 391 unsigned long long blocknr; 392 ktime_t start_time; 393 u64 commit_time; 394 char *tagp = NULL; 395 journal_block_tag_t *tag = NULL; 396 int space_left = 0; 397 int first_tag = 0; 398 int tag_flag; 399 int i; 400 int tag_bytes = journal_tag_bytes(journal); 401 struct buffer_head *cbh = NULL; /* For transactional checksums */ 402 __u32 crc32_sum = ~0; 403 struct blk_plug plug; 404 /* Tail of the journal */ 405 unsigned long first_block; 406 tid_t first_tid; 407 int update_tail; 408 int csum_size = 0; 409 LIST_HEAD(io_bufs); 410 LIST_HEAD(log_bufs); 411 412 if (jbd2_journal_has_csum_v2or3(journal)) 413 csum_size = sizeof(struct jbd2_journal_block_tail); 414 415 /* 416 * First job: lock down the current transaction and wait for 417 * all outstanding updates to complete. 418 */ 419 420 /* Do we need to erase the effects of a prior jbd2_journal_flush? */ 421 if (journal->j_flags & JBD2_FLUSHED) { 422 jbd_debug(3, "super block updated\n"); 423 mutex_lock_io(&journal->j_checkpoint_mutex); 424 /* 425 * We hold j_checkpoint_mutex so tail cannot change under us. 426 * We don't need any special data guarantees for writing sb 427 * since journal is empty and it is ok for write to be 428 * flushed only with transaction commit. 429 */ 430 jbd2_journal_update_sb_log_tail(journal, 431 journal->j_tail_sequence, 432 journal->j_tail, 433 REQ_SYNC); 434 mutex_unlock(&journal->j_checkpoint_mutex); 435 } else { 436 jbd_debug(3, "superblock not updated\n"); 437 } 438 439 J_ASSERT(journal->j_running_transaction != NULL); 440 J_ASSERT(journal->j_committing_transaction == NULL); 441 442 write_lock(&journal->j_state_lock); 443 journal->j_flags |= JBD2_FULL_COMMIT_ONGOING; 444 while (journal->j_flags & JBD2_FAST_COMMIT_ONGOING) { 445 DEFINE_WAIT(wait); 446 447 prepare_to_wait(&journal->j_fc_wait, &wait, 448 TASK_UNINTERRUPTIBLE); 449 write_unlock(&journal->j_state_lock); 450 schedule(); 451 write_lock(&journal->j_state_lock); 452 finish_wait(&journal->j_fc_wait, &wait); 453 } 454 write_unlock(&journal->j_state_lock); 455 456 commit_transaction = journal->j_running_transaction; 457 458 trace_jbd2_start_commit(journal, commit_transaction); 459 jbd_debug(1, "JBD2: starting commit of transaction %d\n", 460 commit_transaction->t_tid); 461 462 write_lock(&journal->j_state_lock); 463 journal->j_fc_off = 0; 464 J_ASSERT(commit_transaction->t_state == T_RUNNING); 465 commit_transaction->t_state = T_LOCKED; 466 467 trace_jbd2_commit_locking(journal, commit_transaction); 468 stats.run.rs_wait = commit_transaction->t_max_wait; 469 stats.run.rs_request_delay = 0; 470 stats.run.rs_locked = jiffies; 471 if (commit_transaction->t_requested) 472 stats.run.rs_request_delay = 473 jbd2_time_diff(commit_transaction->t_requested, 474 stats.run.rs_locked); 475 stats.run.rs_running = jbd2_time_diff(commit_transaction->t_start, 476 stats.run.rs_locked); 477 478 spin_lock(&commit_transaction->t_handle_lock); 479 while (atomic_read(&commit_transaction->t_updates)) { 480 DEFINE_WAIT(wait); 481 482 prepare_to_wait(&journal->j_wait_updates, &wait, 483 TASK_UNINTERRUPTIBLE); 484 if (atomic_read(&commit_transaction->t_updates)) { 485 spin_unlock(&commit_transaction->t_handle_lock); 486 write_unlock(&journal->j_state_lock); 487 schedule(); 488 write_lock(&journal->j_state_lock); 489 spin_lock(&commit_transaction->t_handle_lock); 490 } 491 finish_wait(&journal->j_wait_updates, &wait); 492 } 493 spin_unlock(&commit_transaction->t_handle_lock); 494 commit_transaction->t_state = T_SWITCH; 495 write_unlock(&journal->j_state_lock); 496 497 J_ASSERT (atomic_read(&commit_transaction->t_outstanding_credits) <= 498 journal->j_max_transaction_buffers); 499 500 /* 501 * First thing we are allowed to do is to discard any remaining 502 * BJ_Reserved buffers. Note, it is _not_ permissible to assume 503 * that there are no such buffers: if a large filesystem 504 * operation like a truncate needs to split itself over multiple 505 * transactions, then it may try to do a jbd2_journal_restart() while 506 * there are still BJ_Reserved buffers outstanding. These must 507 * be released cleanly from the current transaction. 508 * 509 * In this case, the filesystem must still reserve write access 510 * again before modifying the buffer in the new transaction, but 511 * we do not require it to remember exactly which old buffers it 512 * has reserved. This is consistent with the existing behaviour 513 * that multiple jbd2_journal_get_write_access() calls to the same 514 * buffer are perfectly permissible. 515 */ 516 while (commit_transaction->t_reserved_list) { 517 jh = commit_transaction->t_reserved_list; 518 JBUFFER_TRACE(jh, "reserved, unused: refile"); 519 /* 520 * A jbd2_journal_get_undo_access()+jbd2_journal_release_buffer() may 521 * leave undo-committed data. 522 */ 523 if (jh->b_committed_data) { 524 struct buffer_head *bh = jh2bh(jh); 525 526 spin_lock(&jh->b_state_lock); 527 jbd2_free(jh->b_committed_data, bh->b_size); 528 jh->b_committed_data = NULL; 529 spin_unlock(&jh->b_state_lock); 530 } 531 jbd2_journal_refile_buffer(journal, jh); 532 } 533 534 /* 535 * Now try to drop any written-back buffers from the journal's 536 * checkpoint lists. We do this *before* commit because it potentially 537 * frees some memory 538 */ 539 spin_lock(&journal->j_list_lock); 540 __jbd2_journal_clean_checkpoint_list(journal, false); 541 spin_unlock(&journal->j_list_lock); 542 543 jbd_debug(3, "JBD2: commit phase 1\n"); 544 545 /* 546 * Clear revoked flag to reflect there is no revoked buffers 547 * in the next transaction which is going to be started. 548 */ 549 jbd2_clear_buffer_revoked_flags(journal); 550 551 /* 552 * Switch to a new revoke table. 553 */ 554 jbd2_journal_switch_revoke_table(journal); 555 556 /* 557 * Reserved credits cannot be claimed anymore, free them 558 */ 559 atomic_sub(atomic_read(&journal->j_reserved_credits), 560 &commit_transaction->t_outstanding_credits); 561 562 write_lock(&journal->j_state_lock); 563 trace_jbd2_commit_flushing(journal, commit_transaction); 564 stats.run.rs_flushing = jiffies; 565 stats.run.rs_locked = jbd2_time_diff(stats.run.rs_locked, 566 stats.run.rs_flushing); 567 568 commit_transaction->t_state = T_FLUSH; 569 journal->j_committing_transaction = commit_transaction; 570 journal->j_running_transaction = NULL; 571 start_time = ktime_get(); 572 commit_transaction->t_log_start = journal->j_head; 573 wake_up(&journal->j_wait_transaction_locked); 574 write_unlock(&journal->j_state_lock); 575 576 jbd_debug(3, "JBD2: commit phase 2a\n"); 577 578 /* 579 * Now start flushing things to disk, in the order they appear 580 * on the transaction lists. Data blocks go first. 581 */ 582 err = journal_submit_data_buffers(journal, commit_transaction); 583 if (err) 584 jbd2_journal_abort(journal, err); 585 586 blk_start_plug(&plug); 587 jbd2_journal_write_revoke_records(commit_transaction, &log_bufs); 588 589 jbd_debug(3, "JBD2: commit phase 2b\n"); 590 591 /* 592 * Way to go: we have now written out all of the data for a 593 * transaction! Now comes the tricky part: we need to write out 594 * metadata. Loop over the transaction's entire buffer list: 595 */ 596 write_lock(&journal->j_state_lock); 597 commit_transaction->t_state = T_COMMIT; 598 write_unlock(&journal->j_state_lock); 599 600 trace_jbd2_commit_logging(journal, commit_transaction); 601 stats.run.rs_logging = jiffies; 602 stats.run.rs_flushing = jbd2_time_diff(stats.run.rs_flushing, 603 stats.run.rs_logging); 604 stats.run.rs_blocks = commit_transaction->t_nr_buffers; 605 stats.run.rs_blocks_logged = 0; 606 607 J_ASSERT(commit_transaction->t_nr_buffers <= 608 atomic_read(&commit_transaction->t_outstanding_credits)); 609 610 err = 0; 611 bufs = 0; 612 descriptor = NULL; 613 while (commit_transaction->t_buffers) { 614 615 /* Find the next buffer to be journaled... */ 616 617 jh = commit_transaction->t_buffers; 618 619 /* If we're in abort mode, we just un-journal the buffer and 620 release it. */ 621 622 if (is_journal_aborted(journal)) { 623 clear_buffer_jbddirty(jh2bh(jh)); 624 JBUFFER_TRACE(jh, "journal is aborting: refile"); 625 jbd2_buffer_abort_trigger(jh, 626 jh->b_frozen_data ? 627 jh->b_frozen_triggers : 628 jh->b_triggers); 629 jbd2_journal_refile_buffer(journal, jh); 630 /* If that was the last one, we need to clean up 631 * any descriptor buffers which may have been 632 * already allocated, even if we are now 633 * aborting. */ 634 if (!commit_transaction->t_buffers) 635 goto start_journal_io; 636 continue; 637 } 638 639 /* Make sure we have a descriptor block in which to 640 record the metadata buffer. */ 641 642 if (!descriptor) { 643 J_ASSERT (bufs == 0); 644 645 jbd_debug(4, "JBD2: get descriptor\n"); 646 647 descriptor = jbd2_journal_get_descriptor_buffer( 648 commit_transaction, 649 JBD2_DESCRIPTOR_BLOCK); 650 if (!descriptor) { 651 jbd2_journal_abort(journal, -EIO); 652 continue; 653 } 654 655 jbd_debug(4, "JBD2: got buffer %llu (%p)\n", 656 (unsigned long long)descriptor->b_blocknr, 657 descriptor->b_data); 658 tagp = &descriptor->b_data[sizeof(journal_header_t)]; 659 space_left = descriptor->b_size - 660 sizeof(journal_header_t); 661 first_tag = 1; 662 set_buffer_jwrite(descriptor); 663 set_buffer_dirty(descriptor); 664 wbuf[bufs++] = descriptor; 665 666 /* Record it so that we can wait for IO 667 completion later */ 668 BUFFER_TRACE(descriptor, "ph3: file as descriptor"); 669 jbd2_file_log_bh(&log_bufs, descriptor); 670 } 671 672 /* Where is the buffer to be written? */ 673 674 err = jbd2_journal_next_log_block(journal, &blocknr); 675 /* If the block mapping failed, just abandon the buffer 676 and repeat this loop: we'll fall into the 677 refile-on-abort condition above. */ 678 if (err) { 679 jbd2_journal_abort(journal, err); 680 continue; 681 } 682 683 /* 684 * start_this_handle() uses t_outstanding_credits to determine 685 * the free space in the log. 686 */ 687 atomic_dec(&commit_transaction->t_outstanding_credits); 688 689 /* Bump b_count to prevent truncate from stumbling over 690 the shadowed buffer! @@@ This can go if we ever get 691 rid of the shadow pairing of buffers. */ 692 atomic_inc(&jh2bh(jh)->b_count); 693 694 /* 695 * Make a temporary IO buffer with which to write it out 696 * (this will requeue the metadata buffer to BJ_Shadow). 697 */ 698 set_bit(BH_JWrite, &jh2bh(jh)->b_state); 699 JBUFFER_TRACE(jh, "ph3: write metadata"); 700 flags = jbd2_journal_write_metadata_buffer(commit_transaction, 701 jh, &wbuf[bufs], blocknr); 702 if (flags < 0) { 703 jbd2_journal_abort(journal, flags); 704 continue; 705 } 706 jbd2_file_log_bh(&io_bufs, wbuf[bufs]); 707 708 /* Record the new block's tag in the current descriptor 709 buffer */ 710 711 tag_flag = 0; 712 if (flags & 1) 713 tag_flag |= JBD2_FLAG_ESCAPE; 714 if (!first_tag) 715 tag_flag |= JBD2_FLAG_SAME_UUID; 716 717 tag = (journal_block_tag_t *) tagp; 718 write_tag_block(journal, tag, jh2bh(jh)->b_blocknr); 719 tag->t_flags = cpu_to_be16(tag_flag); 720 jbd2_block_tag_csum_set(journal, tag, wbuf[bufs], 721 commit_transaction->t_tid); 722 tagp += tag_bytes; 723 space_left -= tag_bytes; 724 bufs++; 725 726 if (first_tag) { 727 memcpy (tagp, journal->j_uuid, 16); 728 tagp += 16; 729 space_left -= 16; 730 first_tag = 0; 731 } 732 733 /* If there's no more to do, or if the descriptor is full, 734 let the IO rip! */ 735 736 if (bufs == journal->j_wbufsize || 737 commit_transaction->t_buffers == NULL || 738 space_left < tag_bytes + 16 + csum_size) { 739 740 jbd_debug(4, "JBD2: Submit %d IOs\n", bufs); 741 742 /* Write an end-of-descriptor marker before 743 submitting the IOs. "tag" still points to 744 the last tag we set up. */ 745 746 tag->t_flags |= cpu_to_be16(JBD2_FLAG_LAST_TAG); 747 start_journal_io: 748 if (descriptor) 749 jbd2_descriptor_block_csum_set(journal, 750 descriptor); 751 752 for (i = 0; i < bufs; i++) { 753 struct buffer_head *bh = wbuf[i]; 754 /* 755 * Compute checksum. 756 */ 757 if (jbd2_has_feature_checksum(journal)) { 758 crc32_sum = 759 jbd2_checksum_data(crc32_sum, bh); 760 } 761 762 lock_buffer(bh); 763 clear_buffer_dirty(bh); 764 set_buffer_uptodate(bh); 765 bh->b_end_io = journal_end_buffer_io_sync; 766 submit_bh(REQ_OP_WRITE, REQ_SYNC, bh); 767 } 768 cond_resched(); 769 770 /* Force a new descriptor to be generated next 771 time round the loop. */ 772 descriptor = NULL; 773 bufs = 0; 774 } 775 } 776 777 err = journal_finish_inode_data_buffers(journal, commit_transaction); 778 if (err) { 779 printk(KERN_WARNING 780 "JBD2: Detected IO errors while flushing file data " 781 "on %s\n", journal->j_devname); 782 if (journal->j_flags & JBD2_ABORT_ON_SYNCDATA_ERR) 783 jbd2_journal_abort(journal, err); 784 err = 0; 785 } 786 787 /* 788 * Get current oldest transaction in the log before we issue flush 789 * to the filesystem device. After the flush we can be sure that 790 * blocks of all older transactions are checkpointed to persistent 791 * storage and we will be safe to update journal start in the 792 * superblock with the numbers we get here. 793 */ 794 update_tail = 795 jbd2_journal_get_log_tail(journal, &first_tid, &first_block); 796 797 write_lock(&journal->j_state_lock); 798 if (update_tail) { 799 long freed = first_block - journal->j_tail; 800 801 if (first_block < journal->j_tail) 802 freed += journal->j_last - journal->j_first; 803 /* Update tail only if we free significant amount of space */ 804 if (freed < journal->j_maxlen / 4) 805 update_tail = 0; 806 } 807 J_ASSERT(commit_transaction->t_state == T_COMMIT); 808 commit_transaction->t_state = T_COMMIT_DFLUSH; 809 write_unlock(&journal->j_state_lock); 810 811 /* 812 * If the journal is not located on the file system device, 813 * then we must flush the file system device before we issue 814 * the commit record 815 */ 816 if (commit_transaction->t_need_data_flush && 817 (journal->j_fs_dev != journal->j_dev) && 818 (journal->j_flags & JBD2_BARRIER)) 819 blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS); 820 821 /* Done it all: now write the commit record asynchronously. */ 822 if (jbd2_has_feature_async_commit(journal)) { 823 err = journal_submit_commit_record(journal, commit_transaction, 824 &cbh, crc32_sum); 825 if (err) 826 jbd2_journal_abort(journal, err); 827 } 828 829 blk_finish_plug(&plug); 830 831 /* Lo and behold: we have just managed to send a transaction to 832 the log. Before we can commit it, wait for the IO so far to 833 complete. Control buffers being written are on the 834 transaction's t_log_list queue, and metadata buffers are on 835 the io_bufs list. 836 837 Wait for the buffers in reverse order. That way we are 838 less likely to be woken up until all IOs have completed, and 839 so we incur less scheduling load. 840 */ 841 842 jbd_debug(3, "JBD2: commit phase 3\n"); 843 844 while (!list_empty(&io_bufs)) { 845 struct buffer_head *bh = list_entry(io_bufs.prev, 846 struct buffer_head, 847 b_assoc_buffers); 848 849 wait_on_buffer(bh); 850 cond_resched(); 851 852 if (unlikely(!buffer_uptodate(bh))) 853 err = -EIO; 854 jbd2_unfile_log_bh(bh); 855 stats.run.rs_blocks_logged++; 856 857 /* 858 * The list contains temporary buffer heads created by 859 * jbd2_journal_write_metadata_buffer(). 860 */ 861 BUFFER_TRACE(bh, "dumping temporary bh"); 862 __brelse(bh); 863 J_ASSERT_BH(bh, atomic_read(&bh->b_count) == 0); 864 free_buffer_head(bh); 865 866 /* We also have to refile the corresponding shadowed buffer */ 867 jh = commit_transaction->t_shadow_list->b_tprev; 868 bh = jh2bh(jh); 869 clear_buffer_jwrite(bh); 870 J_ASSERT_BH(bh, buffer_jbddirty(bh)); 871 J_ASSERT_BH(bh, !buffer_shadow(bh)); 872 873 /* The metadata is now released for reuse, but we need 874 to remember it against this transaction so that when 875 we finally commit, we can do any checkpointing 876 required. */ 877 JBUFFER_TRACE(jh, "file as BJ_Forget"); 878 jbd2_journal_file_buffer(jh, commit_transaction, BJ_Forget); 879 JBUFFER_TRACE(jh, "brelse shadowed buffer"); 880 __brelse(bh); 881 } 882 883 J_ASSERT (commit_transaction->t_shadow_list == NULL); 884 885 jbd_debug(3, "JBD2: commit phase 4\n"); 886 887 /* Here we wait for the revoke record and descriptor record buffers */ 888 while (!list_empty(&log_bufs)) { 889 struct buffer_head *bh; 890 891 bh = list_entry(log_bufs.prev, struct buffer_head, b_assoc_buffers); 892 wait_on_buffer(bh); 893 cond_resched(); 894 895 if (unlikely(!buffer_uptodate(bh))) 896 err = -EIO; 897 898 BUFFER_TRACE(bh, "ph5: control buffer writeout done: unfile"); 899 clear_buffer_jwrite(bh); 900 jbd2_unfile_log_bh(bh); 901 stats.run.rs_blocks_logged++; 902 __brelse(bh); /* One for getblk */ 903 /* AKPM: bforget here */ 904 } 905 906 if (err) 907 jbd2_journal_abort(journal, err); 908 909 jbd_debug(3, "JBD2: commit phase 5\n"); 910 write_lock(&journal->j_state_lock); 911 J_ASSERT(commit_transaction->t_state == T_COMMIT_DFLUSH); 912 commit_transaction->t_state = T_COMMIT_JFLUSH; 913 write_unlock(&journal->j_state_lock); 914 915 if (!jbd2_has_feature_async_commit(journal)) { 916 err = journal_submit_commit_record(journal, commit_transaction, 917 &cbh, crc32_sum); 918 if (err) 919 jbd2_journal_abort(journal, err); 920 } 921 if (cbh) 922 err = journal_wait_on_commit_record(journal, cbh); 923 stats.run.rs_blocks_logged++; 924 if (jbd2_has_feature_async_commit(journal) && 925 journal->j_flags & JBD2_BARRIER) { 926 blkdev_issue_flush(journal->j_dev, GFP_NOFS); 927 } 928 929 if (err) 930 jbd2_journal_abort(journal, err); 931 932 WARN_ON_ONCE( 933 atomic_read(&commit_transaction->t_outstanding_credits) < 0); 934 935 /* 936 * Now disk caches for filesystem device are flushed so we are safe to 937 * erase checkpointed transactions from the log by updating journal 938 * superblock. 939 */ 940 if (update_tail) 941 jbd2_update_log_tail(journal, first_tid, first_block); 942 943 /* End of a transaction! Finally, we can do checkpoint 944 processing: any buffers committed as a result of this 945 transaction can be removed from any checkpoint list it was on 946 before. */ 947 948 jbd_debug(3, "JBD2: commit phase 6\n"); 949 950 J_ASSERT(list_empty(&commit_transaction->t_inode_list)); 951 J_ASSERT(commit_transaction->t_buffers == NULL); 952 J_ASSERT(commit_transaction->t_checkpoint_list == NULL); 953 J_ASSERT(commit_transaction->t_shadow_list == NULL); 954 955 restart_loop: 956 /* 957 * As there are other places (journal_unmap_buffer()) adding buffers 958 * to this list we have to be careful and hold the j_list_lock. 959 */ 960 spin_lock(&journal->j_list_lock); 961 while (commit_transaction->t_forget) { 962 transaction_t *cp_transaction; 963 struct buffer_head *bh; 964 int try_to_free = 0; 965 bool drop_ref; 966 967 jh = commit_transaction->t_forget; 968 spin_unlock(&journal->j_list_lock); 969 bh = jh2bh(jh); 970 /* 971 * Get a reference so that bh cannot be freed before we are 972 * done with it. 973 */ 974 get_bh(bh); 975 spin_lock(&jh->b_state_lock); 976 J_ASSERT_JH(jh, jh->b_transaction == commit_transaction); 977 978 /* 979 * If there is undo-protected committed data against 980 * this buffer, then we can remove it now. If it is a 981 * buffer needing such protection, the old frozen_data 982 * field now points to a committed version of the 983 * buffer, so rotate that field to the new committed 984 * data. 985 * 986 * Otherwise, we can just throw away the frozen data now. 987 * 988 * We also know that the frozen data has already fired 989 * its triggers if they exist, so we can clear that too. 990 */ 991 if (jh->b_committed_data) { 992 jbd2_free(jh->b_committed_data, bh->b_size); 993 jh->b_committed_data = NULL; 994 if (jh->b_frozen_data) { 995 jh->b_committed_data = jh->b_frozen_data; 996 jh->b_frozen_data = NULL; 997 jh->b_frozen_triggers = NULL; 998 } 999 } else if (jh->b_frozen_data) { 1000 jbd2_free(jh->b_frozen_data, bh->b_size); 1001 jh->b_frozen_data = NULL; 1002 jh->b_frozen_triggers = NULL; 1003 } 1004 1005 spin_lock(&journal->j_list_lock); 1006 cp_transaction = jh->b_cp_transaction; 1007 if (cp_transaction) { 1008 JBUFFER_TRACE(jh, "remove from old cp transaction"); 1009 cp_transaction->t_chp_stats.cs_dropped++; 1010 __jbd2_journal_remove_checkpoint(jh); 1011 } 1012 1013 /* Only re-checkpoint the buffer_head if it is marked 1014 * dirty. If the buffer was added to the BJ_Forget list 1015 * by jbd2_journal_forget, it may no longer be dirty and 1016 * there's no point in keeping a checkpoint record for 1017 * it. */ 1018 1019 /* 1020 * A buffer which has been freed while still being journaled 1021 * by a previous transaction, refile the buffer to BJ_Forget of 1022 * the running transaction. If the just committed transaction 1023 * contains "add to orphan" operation, we can completely 1024 * invalidate the buffer now. We are rather through in that 1025 * since the buffer may be still accessible when blocksize < 1026 * pagesize and it is attached to the last partial page. 1027 */ 1028 if (buffer_freed(bh) && !jh->b_next_transaction) { 1029 struct address_space *mapping; 1030 1031 clear_buffer_freed(bh); 1032 clear_buffer_jbddirty(bh); 1033 1034 /* 1035 * Block device buffers need to stay mapped all the 1036 * time, so it is enough to clear buffer_jbddirty and 1037 * buffer_freed bits. For the file mapping buffers (i.e. 1038 * journalled data) we need to unmap buffer and clear 1039 * more bits. We also need to be careful about the check 1040 * because the data page mapping can get cleared under 1041 * our hands. Note that if mapping == NULL, we don't 1042 * need to make buffer unmapped because the page is 1043 * already detached from the mapping and buffers cannot 1044 * get reused. 1045 */ 1046 mapping = READ_ONCE(bh->b_page->mapping); 1047 if (mapping && !sb_is_blkdev_sb(mapping->host->i_sb)) { 1048 clear_buffer_mapped(bh); 1049 clear_buffer_new(bh); 1050 clear_buffer_req(bh); 1051 bh->b_bdev = NULL; 1052 } 1053 } 1054 1055 if (buffer_jbddirty(bh)) { 1056 JBUFFER_TRACE(jh, "add to new checkpointing trans"); 1057 __jbd2_journal_insert_checkpoint(jh, commit_transaction); 1058 if (is_journal_aborted(journal)) 1059 clear_buffer_jbddirty(bh); 1060 } else { 1061 J_ASSERT_BH(bh, !buffer_dirty(bh)); 1062 /* 1063 * The buffer on BJ_Forget list and not jbddirty means 1064 * it has been freed by this transaction and hence it 1065 * could not have been reallocated until this 1066 * transaction has committed. *BUT* it could be 1067 * reallocated once we have written all the data to 1068 * disk and before we process the buffer on BJ_Forget 1069 * list. 1070 */ 1071 if (!jh->b_next_transaction) 1072 try_to_free = 1; 1073 } 1074 JBUFFER_TRACE(jh, "refile or unfile buffer"); 1075 drop_ref = __jbd2_journal_refile_buffer(jh); 1076 spin_unlock(&jh->b_state_lock); 1077 if (drop_ref) 1078 jbd2_journal_put_journal_head(jh); 1079 if (try_to_free) 1080 release_buffer_page(bh); /* Drops bh reference */ 1081 else 1082 __brelse(bh); 1083 cond_resched_lock(&journal->j_list_lock); 1084 } 1085 spin_unlock(&journal->j_list_lock); 1086 /* 1087 * This is a bit sleazy. We use j_list_lock to protect transition 1088 * of a transaction into T_FINISHED state and calling 1089 * __jbd2_journal_drop_transaction(). Otherwise we could race with 1090 * other checkpointing code processing the transaction... 1091 */ 1092 write_lock(&journal->j_state_lock); 1093 spin_lock(&journal->j_list_lock); 1094 /* 1095 * Now recheck if some buffers did not get attached to the transaction 1096 * while the lock was dropped... 1097 */ 1098 if (commit_transaction->t_forget) { 1099 spin_unlock(&journal->j_list_lock); 1100 write_unlock(&journal->j_state_lock); 1101 goto restart_loop; 1102 } 1103 1104 /* Add the transaction to the checkpoint list 1105 * __journal_remove_checkpoint() can not destroy transaction 1106 * under us because it is not marked as T_FINISHED yet */ 1107 if (journal->j_checkpoint_transactions == NULL) { 1108 journal->j_checkpoint_transactions = commit_transaction; 1109 commit_transaction->t_cpnext = commit_transaction; 1110 commit_transaction->t_cpprev = commit_transaction; 1111 } else { 1112 commit_transaction->t_cpnext = 1113 journal->j_checkpoint_transactions; 1114 commit_transaction->t_cpprev = 1115 commit_transaction->t_cpnext->t_cpprev; 1116 commit_transaction->t_cpnext->t_cpprev = 1117 commit_transaction; 1118 commit_transaction->t_cpprev->t_cpnext = 1119 commit_transaction; 1120 } 1121 spin_unlock(&journal->j_list_lock); 1122 1123 /* Done with this transaction! */ 1124 1125 jbd_debug(3, "JBD2: commit phase 7\n"); 1126 1127 J_ASSERT(commit_transaction->t_state == T_COMMIT_JFLUSH); 1128 1129 commit_transaction->t_start = jiffies; 1130 stats.run.rs_logging = jbd2_time_diff(stats.run.rs_logging, 1131 commit_transaction->t_start); 1132 1133 /* 1134 * File the transaction statistics 1135 */ 1136 stats.ts_tid = commit_transaction->t_tid; 1137 stats.run.rs_handle_count = 1138 atomic_read(&commit_transaction->t_handle_count); 1139 trace_jbd2_run_stats(journal->j_fs_dev->bd_dev, 1140 commit_transaction->t_tid, &stats.run); 1141 stats.ts_requested = (commit_transaction->t_requested) ? 1 : 0; 1142 1143 commit_transaction->t_state = T_COMMIT_CALLBACK; 1144 J_ASSERT(commit_transaction == journal->j_committing_transaction); 1145 journal->j_commit_sequence = commit_transaction->t_tid; 1146 journal->j_committing_transaction = NULL; 1147 commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time)); 1148 1149 /* 1150 * weight the commit time higher than the average time so we don't 1151 * react too strongly to vast changes in the commit time 1152 */ 1153 if (likely(journal->j_average_commit_time)) 1154 journal->j_average_commit_time = (commit_time + 1155 journal->j_average_commit_time*3) / 4; 1156 else 1157 journal->j_average_commit_time = commit_time; 1158 1159 write_unlock(&journal->j_state_lock); 1160 1161 if (journal->j_commit_callback) 1162 journal->j_commit_callback(journal, commit_transaction); 1163 if (journal->j_fc_cleanup_callback) 1164 journal->j_fc_cleanup_callback(journal, 1); 1165 1166 trace_jbd2_end_commit(journal, commit_transaction); 1167 jbd_debug(1, "JBD2: commit %d complete, head %d\n", 1168 journal->j_commit_sequence, journal->j_tail_sequence); 1169 1170 write_lock(&journal->j_state_lock); 1171 journal->j_flags &= ~JBD2_FULL_COMMIT_ONGOING; 1172 journal->j_flags &= ~JBD2_FAST_COMMIT_ONGOING; 1173 spin_lock(&journal->j_list_lock); 1174 commit_transaction->t_state = T_FINISHED; 1175 /* Check if the transaction can be dropped now that we are finished */ 1176 if (commit_transaction->t_checkpoint_list == NULL && 1177 commit_transaction->t_checkpoint_io_list == NULL) { 1178 __jbd2_journal_drop_transaction(journal, commit_transaction); 1179 jbd2_journal_free_transaction(commit_transaction); 1180 } 1181 spin_unlock(&journal->j_list_lock); 1182 write_unlock(&journal->j_state_lock); 1183 wake_up(&journal->j_wait_done_commit); 1184 wake_up(&journal->j_fc_wait); 1185 1186 /* 1187 * Calculate overall stats 1188 */ 1189 spin_lock(&journal->j_history_lock); 1190 journal->j_stats.ts_tid++; 1191 journal->j_stats.ts_requested += stats.ts_requested; 1192 journal->j_stats.run.rs_wait += stats.run.rs_wait; 1193 journal->j_stats.run.rs_request_delay += stats.run.rs_request_delay; 1194 journal->j_stats.run.rs_running += stats.run.rs_running; 1195 journal->j_stats.run.rs_locked += stats.run.rs_locked; 1196 journal->j_stats.run.rs_flushing += stats.run.rs_flushing; 1197 journal->j_stats.run.rs_logging += stats.run.rs_logging; 1198 journal->j_stats.run.rs_handle_count += stats.run.rs_handle_count; 1199 journal->j_stats.run.rs_blocks += stats.run.rs_blocks; 1200 journal->j_stats.run.rs_blocks_logged += stats.run.rs_blocks_logged; 1201 spin_unlock(&journal->j_history_lock); 1202 } 1203