16866d7b3SHarshad Shirwadkar // SPDX-License-Identifier: GPL-2.0 26866d7b3SHarshad Shirwadkar 36866d7b3SHarshad Shirwadkar /* 46866d7b3SHarshad Shirwadkar * fs/ext4/fast_commit.c 56866d7b3SHarshad Shirwadkar * 66866d7b3SHarshad Shirwadkar * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com> 76866d7b3SHarshad Shirwadkar * 86866d7b3SHarshad Shirwadkar * Ext4 fast commits routines. 96866d7b3SHarshad Shirwadkar */ 10aa75f4d3SHarshad Shirwadkar #include "ext4.h" 116866d7b3SHarshad Shirwadkar #include "ext4_jbd2.h" 12aa75f4d3SHarshad Shirwadkar #include "ext4_extents.h" 13aa75f4d3SHarshad Shirwadkar #include "mballoc.h" 14aa75f4d3SHarshad Shirwadkar 15aa75f4d3SHarshad Shirwadkar /* 16aa75f4d3SHarshad Shirwadkar * Ext4 Fast Commits 17aa75f4d3SHarshad Shirwadkar * ----------------- 18aa75f4d3SHarshad Shirwadkar * 19aa75f4d3SHarshad Shirwadkar * Ext4 fast commits implement fine grained journalling for Ext4. 20aa75f4d3SHarshad Shirwadkar * 21aa75f4d3SHarshad Shirwadkar * Fast commits are organized as a log of tag-length-value (TLV) structs. (See 22aa75f4d3SHarshad Shirwadkar * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by 23aa75f4d3SHarshad Shirwadkar * TLV during the recovery phase. For the scenarios for which we currently 24aa75f4d3SHarshad Shirwadkar * don't have replay code, fast commit falls back to full commits. 25aa75f4d3SHarshad Shirwadkar * Fast commits record delta in one of the following three categories. 26aa75f4d3SHarshad Shirwadkar * 27aa75f4d3SHarshad Shirwadkar * (A) Directory entry updates: 28aa75f4d3SHarshad Shirwadkar * 29aa75f4d3SHarshad Shirwadkar * - EXT4_FC_TAG_UNLINK - records directory entry unlink 30aa75f4d3SHarshad Shirwadkar * - EXT4_FC_TAG_LINK - records directory entry link 31aa75f4d3SHarshad Shirwadkar * - EXT4_FC_TAG_CREAT - records inode and directory entry creation 32aa75f4d3SHarshad Shirwadkar * 33aa75f4d3SHarshad Shirwadkar * (B) File specific data range updates: 34aa75f4d3SHarshad Shirwadkar * 35aa75f4d3SHarshad Shirwadkar * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode 36aa75f4d3SHarshad Shirwadkar * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode 37aa75f4d3SHarshad Shirwadkar * 38aa75f4d3SHarshad Shirwadkar * (C) Inode metadata (mtime / ctime etc): 39aa75f4d3SHarshad Shirwadkar * 40aa75f4d3SHarshad Shirwadkar * - EXT4_FC_TAG_INODE - record the inode that should be replayed 41aa75f4d3SHarshad Shirwadkar * during recovery. Note that iblocks field is 42aa75f4d3SHarshad Shirwadkar * not replayed and instead derived during 43aa75f4d3SHarshad Shirwadkar * replay. 44aa75f4d3SHarshad Shirwadkar * Commit Operation 45aa75f4d3SHarshad Shirwadkar * ---------------- 46aa75f4d3SHarshad Shirwadkar * With fast commits, we maintain all the directory entry operations in the 47aa75f4d3SHarshad Shirwadkar * order in which they are issued in an in-memory queue. This queue is flushed 48aa75f4d3SHarshad Shirwadkar * to disk during the commit operation. We also maintain a list of inodes 49aa75f4d3SHarshad Shirwadkar * that need to be committed during a fast commit in another in memory queue of 50aa75f4d3SHarshad Shirwadkar * inodes. During the commit operation, we commit in the following order: 51aa75f4d3SHarshad Shirwadkar * 52aa75f4d3SHarshad Shirwadkar * [1] Lock inodes for any further data updates by setting COMMITTING state 53aa75f4d3SHarshad Shirwadkar * [2] Submit data buffers of all the inodes 54aa75f4d3SHarshad Shirwadkar * [3] Wait for [2] to complete 55aa75f4d3SHarshad Shirwadkar * [4] Commit all the directory entry updates in the fast commit space 56aa75f4d3SHarshad Shirwadkar * [5] Commit all the changed inode structures 57aa75f4d3SHarshad Shirwadkar * [6] Write tail tag (this tag ensures the atomicity, please read the following 58aa75f4d3SHarshad Shirwadkar * section for more details). 59aa75f4d3SHarshad Shirwadkar * [7] Wait for [4], [5] and [6] to complete. 60aa75f4d3SHarshad Shirwadkar * 61aa75f4d3SHarshad Shirwadkar * All the inode updates must call ext4_fc_start_update() before starting an 62aa75f4d3SHarshad Shirwadkar * update. If such an ongoing update is present, fast commit waits for it to 63aa75f4d3SHarshad Shirwadkar * complete. The completion of such an update is marked by 64aa75f4d3SHarshad Shirwadkar * ext4_fc_stop_update(). 65aa75f4d3SHarshad Shirwadkar * 66aa75f4d3SHarshad Shirwadkar * Fast Commit Ineligibility 67aa75f4d3SHarshad Shirwadkar * ------------------------- 68aa75f4d3SHarshad Shirwadkar * Not all operations are supported by fast commits today (e.g extended 69aa75f4d3SHarshad Shirwadkar * attributes). Fast commit ineligiblity is marked by calling one of the 70aa75f4d3SHarshad Shirwadkar * two following functions: 71aa75f4d3SHarshad Shirwadkar * 72aa75f4d3SHarshad Shirwadkar * - ext4_fc_mark_ineligible(): This makes next fast commit operation to fall 73aa75f4d3SHarshad Shirwadkar * back to full commit. This is useful in case of transient errors. 74aa75f4d3SHarshad Shirwadkar * 75aa75f4d3SHarshad Shirwadkar * - ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() - This makes all 76aa75f4d3SHarshad Shirwadkar * the fast commits happening between ext4_fc_start_ineligible() and 77aa75f4d3SHarshad Shirwadkar * ext4_fc_stop_ineligible() and one fast commit after the call to 78aa75f4d3SHarshad Shirwadkar * ext4_fc_stop_ineligible() to fall back to full commits. It is important to 79aa75f4d3SHarshad Shirwadkar * make one more fast commit to fall back to full commit after stop call so 80aa75f4d3SHarshad Shirwadkar * that it guaranteed that the fast commit ineligible operation contained 81aa75f4d3SHarshad Shirwadkar * within ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() is 82aa75f4d3SHarshad Shirwadkar * followed by at least 1 full commit. 83aa75f4d3SHarshad Shirwadkar * 84aa75f4d3SHarshad Shirwadkar * Atomicity of commits 85aa75f4d3SHarshad Shirwadkar * -------------------- 86aa75f4d3SHarshad Shirwadkar * In order to gaurantee atomicity during the commit operation, fast commit 87aa75f4d3SHarshad Shirwadkar * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail 88aa75f4d3SHarshad Shirwadkar * tag contains CRC of the contents and TID of the transaction after which 89aa75f4d3SHarshad Shirwadkar * this fast commit should be applied. Recovery code replays fast commit 90aa75f4d3SHarshad Shirwadkar * logs only if there's at least 1 valid tail present. For every fast commit 91aa75f4d3SHarshad Shirwadkar * operation, there is 1 tail. This means, we may end up with multiple tails 92aa75f4d3SHarshad Shirwadkar * in the fast commit space. Here's an example: 93aa75f4d3SHarshad Shirwadkar * 94aa75f4d3SHarshad Shirwadkar * - Create a new file A and remove existing file B 95aa75f4d3SHarshad Shirwadkar * - fsync() 96aa75f4d3SHarshad Shirwadkar * - Append contents to file A 97aa75f4d3SHarshad Shirwadkar * - Truncate file A 98aa75f4d3SHarshad Shirwadkar * - fsync() 99aa75f4d3SHarshad Shirwadkar * 100aa75f4d3SHarshad Shirwadkar * The fast commit space at the end of above operations would look like this: 101aa75f4d3SHarshad Shirwadkar * [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL] 102aa75f4d3SHarshad Shirwadkar * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->| 103aa75f4d3SHarshad Shirwadkar * 104aa75f4d3SHarshad Shirwadkar * Replay code should thus check for all the valid tails in the FC area. 105aa75f4d3SHarshad Shirwadkar * 106aa75f4d3SHarshad Shirwadkar * TODOs 107aa75f4d3SHarshad Shirwadkar * ----- 108aa75f4d3SHarshad Shirwadkar * 1) Make fast commit atomic updates more fine grained. Today, a fast commit 109aa75f4d3SHarshad Shirwadkar * eligible update must be protected within ext4_fc_start_update() and 110aa75f4d3SHarshad Shirwadkar * ext4_fc_stop_update(). These routines are called at much higher 111aa75f4d3SHarshad Shirwadkar * routines. This can be made more fine grained by combining with 112aa75f4d3SHarshad Shirwadkar * ext4_journal_start(). 113aa75f4d3SHarshad Shirwadkar * 114aa75f4d3SHarshad Shirwadkar * 2) Same above for ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() 115aa75f4d3SHarshad Shirwadkar * 116aa75f4d3SHarshad Shirwadkar * 3) Handle more ineligible cases. 117aa75f4d3SHarshad Shirwadkar */ 118aa75f4d3SHarshad Shirwadkar 119aa75f4d3SHarshad Shirwadkar #include <trace/events/ext4.h> 120aa75f4d3SHarshad Shirwadkar static struct kmem_cache *ext4_fc_dentry_cachep; 121aa75f4d3SHarshad Shirwadkar 122aa75f4d3SHarshad Shirwadkar static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate) 123aa75f4d3SHarshad Shirwadkar { 124aa75f4d3SHarshad Shirwadkar BUFFER_TRACE(bh, ""); 125aa75f4d3SHarshad Shirwadkar if (uptodate) { 126aa75f4d3SHarshad Shirwadkar ext4_debug("%s: Block %lld up-to-date", 127aa75f4d3SHarshad Shirwadkar __func__, bh->b_blocknr); 128aa75f4d3SHarshad Shirwadkar set_buffer_uptodate(bh); 129aa75f4d3SHarshad Shirwadkar } else { 130aa75f4d3SHarshad Shirwadkar ext4_debug("%s: Block %lld not up-to-date", 131aa75f4d3SHarshad Shirwadkar __func__, bh->b_blocknr); 132aa75f4d3SHarshad Shirwadkar clear_buffer_uptodate(bh); 133aa75f4d3SHarshad Shirwadkar } 134aa75f4d3SHarshad Shirwadkar 135aa75f4d3SHarshad Shirwadkar unlock_buffer(bh); 136aa75f4d3SHarshad Shirwadkar } 137aa75f4d3SHarshad Shirwadkar 138aa75f4d3SHarshad Shirwadkar static inline void ext4_fc_reset_inode(struct inode *inode) 139aa75f4d3SHarshad Shirwadkar { 140aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 141aa75f4d3SHarshad Shirwadkar 142aa75f4d3SHarshad Shirwadkar ei->i_fc_lblk_start = 0; 143aa75f4d3SHarshad Shirwadkar ei->i_fc_lblk_len = 0; 144aa75f4d3SHarshad Shirwadkar } 145aa75f4d3SHarshad Shirwadkar 146aa75f4d3SHarshad Shirwadkar void ext4_fc_init_inode(struct inode *inode) 147aa75f4d3SHarshad Shirwadkar { 148aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 149aa75f4d3SHarshad Shirwadkar 150aa75f4d3SHarshad Shirwadkar ext4_fc_reset_inode(inode); 151aa75f4d3SHarshad Shirwadkar ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING); 152aa75f4d3SHarshad Shirwadkar INIT_LIST_HEAD(&ei->i_fc_list); 153aa75f4d3SHarshad Shirwadkar init_waitqueue_head(&ei->i_fc_wait); 154aa75f4d3SHarshad Shirwadkar atomic_set(&ei->i_fc_updates, 0); 155aa75f4d3SHarshad Shirwadkar ei->i_fc_committed_subtid = 0; 156aa75f4d3SHarshad Shirwadkar } 157aa75f4d3SHarshad Shirwadkar 158aa75f4d3SHarshad Shirwadkar /* 159aa75f4d3SHarshad Shirwadkar * Inform Ext4's fast about start of an inode update 160aa75f4d3SHarshad Shirwadkar * 161aa75f4d3SHarshad Shirwadkar * This function is called by the high level call VFS callbacks before 162aa75f4d3SHarshad Shirwadkar * performing any inode update. This function blocks if there's an ongoing 163aa75f4d3SHarshad Shirwadkar * fast commit on the inode in question. 164aa75f4d3SHarshad Shirwadkar */ 165aa75f4d3SHarshad Shirwadkar void ext4_fc_start_update(struct inode *inode) 166aa75f4d3SHarshad Shirwadkar { 167aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 168aa75f4d3SHarshad Shirwadkar 169aa75f4d3SHarshad Shirwadkar if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT)) 170aa75f4d3SHarshad Shirwadkar return; 171aa75f4d3SHarshad Shirwadkar 172aa75f4d3SHarshad Shirwadkar restart: 173aa75f4d3SHarshad Shirwadkar spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock); 174aa75f4d3SHarshad Shirwadkar if (list_empty(&ei->i_fc_list)) 175aa75f4d3SHarshad Shirwadkar goto out; 176aa75f4d3SHarshad Shirwadkar 177aa75f4d3SHarshad Shirwadkar if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) { 178aa75f4d3SHarshad Shirwadkar wait_queue_head_t *wq; 179aa75f4d3SHarshad Shirwadkar #if (BITS_PER_LONG < 64) 180aa75f4d3SHarshad Shirwadkar DEFINE_WAIT_BIT(wait, &ei->i_state_flags, 181aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING); 182aa75f4d3SHarshad Shirwadkar wq = bit_waitqueue(&ei->i_state_flags, 183aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING); 184aa75f4d3SHarshad Shirwadkar #else 185aa75f4d3SHarshad Shirwadkar DEFINE_WAIT_BIT(wait, &ei->i_flags, 186aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING); 187aa75f4d3SHarshad Shirwadkar wq = bit_waitqueue(&ei->i_flags, 188aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING); 189aa75f4d3SHarshad Shirwadkar #endif 190aa75f4d3SHarshad Shirwadkar prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); 191aa75f4d3SHarshad Shirwadkar spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); 192aa75f4d3SHarshad Shirwadkar schedule(); 193aa75f4d3SHarshad Shirwadkar finish_wait(wq, &wait.wq_entry); 194aa75f4d3SHarshad Shirwadkar goto restart; 195aa75f4d3SHarshad Shirwadkar } 196aa75f4d3SHarshad Shirwadkar out: 197aa75f4d3SHarshad Shirwadkar atomic_inc(&ei->i_fc_updates); 198aa75f4d3SHarshad Shirwadkar spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); 199aa75f4d3SHarshad Shirwadkar } 200aa75f4d3SHarshad Shirwadkar 201aa75f4d3SHarshad Shirwadkar /* 202aa75f4d3SHarshad Shirwadkar * Stop inode update and wake up waiting fast commits if any. 203aa75f4d3SHarshad Shirwadkar */ 204aa75f4d3SHarshad Shirwadkar void ext4_fc_stop_update(struct inode *inode) 205aa75f4d3SHarshad Shirwadkar { 206aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 207aa75f4d3SHarshad Shirwadkar 208aa75f4d3SHarshad Shirwadkar if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT)) 209aa75f4d3SHarshad Shirwadkar return; 210aa75f4d3SHarshad Shirwadkar 211aa75f4d3SHarshad Shirwadkar if (atomic_dec_and_test(&ei->i_fc_updates)) 212aa75f4d3SHarshad Shirwadkar wake_up_all(&ei->i_fc_wait); 213aa75f4d3SHarshad Shirwadkar } 214aa75f4d3SHarshad Shirwadkar 215aa75f4d3SHarshad Shirwadkar /* 216aa75f4d3SHarshad Shirwadkar * Remove inode from fast commit list. If the inode is being committed 217aa75f4d3SHarshad Shirwadkar * we wait until inode commit is done. 218aa75f4d3SHarshad Shirwadkar */ 219aa75f4d3SHarshad Shirwadkar void ext4_fc_del(struct inode *inode) 220aa75f4d3SHarshad Shirwadkar { 221aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 222aa75f4d3SHarshad Shirwadkar 223aa75f4d3SHarshad Shirwadkar if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT)) 224aa75f4d3SHarshad Shirwadkar return; 225aa75f4d3SHarshad Shirwadkar 226aa75f4d3SHarshad Shirwadkar 227aa75f4d3SHarshad Shirwadkar if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT)) 228aa75f4d3SHarshad Shirwadkar return; 229aa75f4d3SHarshad Shirwadkar 230aa75f4d3SHarshad Shirwadkar restart: 231aa75f4d3SHarshad Shirwadkar spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock); 232aa75f4d3SHarshad Shirwadkar if (list_empty(&ei->i_fc_list)) { 233aa75f4d3SHarshad Shirwadkar spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); 234aa75f4d3SHarshad Shirwadkar return; 235aa75f4d3SHarshad Shirwadkar } 236aa75f4d3SHarshad Shirwadkar 237aa75f4d3SHarshad Shirwadkar if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) { 238aa75f4d3SHarshad Shirwadkar wait_queue_head_t *wq; 239aa75f4d3SHarshad Shirwadkar #if (BITS_PER_LONG < 64) 240aa75f4d3SHarshad Shirwadkar DEFINE_WAIT_BIT(wait, &ei->i_state_flags, 241aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING); 242aa75f4d3SHarshad Shirwadkar wq = bit_waitqueue(&ei->i_state_flags, 243aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING); 244aa75f4d3SHarshad Shirwadkar #else 245aa75f4d3SHarshad Shirwadkar DEFINE_WAIT_BIT(wait, &ei->i_flags, 246aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING); 247aa75f4d3SHarshad Shirwadkar wq = bit_waitqueue(&ei->i_flags, 248aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING); 249aa75f4d3SHarshad Shirwadkar #endif 250aa75f4d3SHarshad Shirwadkar prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); 251aa75f4d3SHarshad Shirwadkar spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); 252aa75f4d3SHarshad Shirwadkar schedule(); 253aa75f4d3SHarshad Shirwadkar finish_wait(wq, &wait.wq_entry); 254aa75f4d3SHarshad Shirwadkar goto restart; 255aa75f4d3SHarshad Shirwadkar } 256aa75f4d3SHarshad Shirwadkar if (!list_empty(&ei->i_fc_list)) 257aa75f4d3SHarshad Shirwadkar list_del_init(&ei->i_fc_list); 258aa75f4d3SHarshad Shirwadkar spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); 259aa75f4d3SHarshad Shirwadkar } 260aa75f4d3SHarshad Shirwadkar 261aa75f4d3SHarshad Shirwadkar /* 262aa75f4d3SHarshad Shirwadkar * Mark file system as fast commit ineligible. This means that next commit 263aa75f4d3SHarshad Shirwadkar * operation would result in a full jbd2 commit. 264aa75f4d3SHarshad Shirwadkar */ 265aa75f4d3SHarshad Shirwadkar void ext4_fc_mark_ineligible(struct super_block *sb, int reason) 266aa75f4d3SHarshad Shirwadkar { 267aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 268aa75f4d3SHarshad Shirwadkar 269aa75f4d3SHarshad Shirwadkar sbi->s_mount_state |= EXT4_FC_INELIGIBLE; 270aa75f4d3SHarshad Shirwadkar WARN_ON(reason >= EXT4_FC_REASON_MAX); 271aa75f4d3SHarshad Shirwadkar sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; 272aa75f4d3SHarshad Shirwadkar } 273aa75f4d3SHarshad Shirwadkar 274aa75f4d3SHarshad Shirwadkar /* 275aa75f4d3SHarshad Shirwadkar * Start a fast commit ineligible update. Any commits that happen while 276aa75f4d3SHarshad Shirwadkar * such an operation is in progress fall back to full commits. 277aa75f4d3SHarshad Shirwadkar */ 278aa75f4d3SHarshad Shirwadkar void ext4_fc_start_ineligible(struct super_block *sb, int reason) 279aa75f4d3SHarshad Shirwadkar { 280aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 281aa75f4d3SHarshad Shirwadkar 282aa75f4d3SHarshad Shirwadkar WARN_ON(reason >= EXT4_FC_REASON_MAX); 283aa75f4d3SHarshad Shirwadkar sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; 284aa75f4d3SHarshad Shirwadkar atomic_inc(&sbi->s_fc_ineligible_updates); 285aa75f4d3SHarshad Shirwadkar } 286aa75f4d3SHarshad Shirwadkar 287aa75f4d3SHarshad Shirwadkar /* 288aa75f4d3SHarshad Shirwadkar * Stop a fast commit ineligible update. We set EXT4_FC_INELIGIBLE flag here 289aa75f4d3SHarshad Shirwadkar * to ensure that after stopping the ineligible update, at least one full 290aa75f4d3SHarshad Shirwadkar * commit takes place. 291aa75f4d3SHarshad Shirwadkar */ 292aa75f4d3SHarshad Shirwadkar void ext4_fc_stop_ineligible(struct super_block *sb) 293aa75f4d3SHarshad Shirwadkar { 294aa75f4d3SHarshad Shirwadkar EXT4_SB(sb)->s_mount_state |= EXT4_FC_INELIGIBLE; 295aa75f4d3SHarshad Shirwadkar atomic_dec(&EXT4_SB(sb)->s_fc_ineligible_updates); 296aa75f4d3SHarshad Shirwadkar } 297aa75f4d3SHarshad Shirwadkar 298aa75f4d3SHarshad Shirwadkar static inline int ext4_fc_is_ineligible(struct super_block *sb) 299aa75f4d3SHarshad Shirwadkar { 300aa75f4d3SHarshad Shirwadkar return (EXT4_SB(sb)->s_mount_state & EXT4_FC_INELIGIBLE) || 301aa75f4d3SHarshad Shirwadkar atomic_read(&EXT4_SB(sb)->s_fc_ineligible_updates); 302aa75f4d3SHarshad Shirwadkar } 303aa75f4d3SHarshad Shirwadkar 304aa75f4d3SHarshad Shirwadkar /* 305aa75f4d3SHarshad Shirwadkar * Generic fast commit tracking function. If this is the first time this we are 306aa75f4d3SHarshad Shirwadkar * called after a full commit, we initialize fast commit fields and then call 307aa75f4d3SHarshad Shirwadkar * __fc_track_fn() with update = 0. If we have already been called after a full 308aa75f4d3SHarshad Shirwadkar * commit, we pass update = 1. Based on that, the track function can determine 309aa75f4d3SHarshad Shirwadkar * if it needs to track a field for the first time or if it needs to just 310aa75f4d3SHarshad Shirwadkar * update the previously tracked value. 311aa75f4d3SHarshad Shirwadkar * 312aa75f4d3SHarshad Shirwadkar * If enqueue is set, this function enqueues the inode in fast commit list. 313aa75f4d3SHarshad Shirwadkar */ 314aa75f4d3SHarshad Shirwadkar static int ext4_fc_track_template( 315aa75f4d3SHarshad Shirwadkar struct inode *inode, int (*__fc_track_fn)(struct inode *, void *, bool), 316aa75f4d3SHarshad Shirwadkar void *args, int enqueue) 317aa75f4d3SHarshad Shirwadkar { 318aa75f4d3SHarshad Shirwadkar tid_t running_txn_tid; 319aa75f4d3SHarshad Shirwadkar bool update = false; 320aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 321aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 322aa75f4d3SHarshad Shirwadkar int ret; 323aa75f4d3SHarshad Shirwadkar 324aa75f4d3SHarshad Shirwadkar if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT)) 325aa75f4d3SHarshad Shirwadkar return -EOPNOTSUPP; 326aa75f4d3SHarshad Shirwadkar 327aa75f4d3SHarshad Shirwadkar if (ext4_fc_is_ineligible(inode->i_sb)) 328aa75f4d3SHarshad Shirwadkar return -EINVAL; 329aa75f4d3SHarshad Shirwadkar 330aa75f4d3SHarshad Shirwadkar running_txn_tid = sbi->s_journal ? 331aa75f4d3SHarshad Shirwadkar sbi->s_journal->j_commit_sequence + 1 : 0; 332aa75f4d3SHarshad Shirwadkar 333aa75f4d3SHarshad Shirwadkar mutex_lock(&ei->i_fc_lock); 334aa75f4d3SHarshad Shirwadkar if (running_txn_tid == ei->i_sync_tid) { 335aa75f4d3SHarshad Shirwadkar update = true; 336aa75f4d3SHarshad Shirwadkar } else { 337aa75f4d3SHarshad Shirwadkar ext4_fc_reset_inode(inode); 338aa75f4d3SHarshad Shirwadkar ei->i_sync_tid = running_txn_tid; 339aa75f4d3SHarshad Shirwadkar } 340aa75f4d3SHarshad Shirwadkar ret = __fc_track_fn(inode, args, update); 341aa75f4d3SHarshad Shirwadkar mutex_unlock(&ei->i_fc_lock); 342aa75f4d3SHarshad Shirwadkar 343aa75f4d3SHarshad Shirwadkar if (!enqueue) 344aa75f4d3SHarshad Shirwadkar return ret; 345aa75f4d3SHarshad Shirwadkar 346aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 347aa75f4d3SHarshad Shirwadkar if (list_empty(&EXT4_I(inode)->i_fc_list)) 348aa75f4d3SHarshad Shirwadkar list_add_tail(&EXT4_I(inode)->i_fc_list, 349aa75f4d3SHarshad Shirwadkar (sbi->s_mount_state & EXT4_FC_COMMITTING) ? 350aa75f4d3SHarshad Shirwadkar &sbi->s_fc_q[FC_Q_STAGING] : 351aa75f4d3SHarshad Shirwadkar &sbi->s_fc_q[FC_Q_MAIN]); 352aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 353aa75f4d3SHarshad Shirwadkar 354aa75f4d3SHarshad Shirwadkar return ret; 355aa75f4d3SHarshad Shirwadkar } 356aa75f4d3SHarshad Shirwadkar 357aa75f4d3SHarshad Shirwadkar struct __track_dentry_update_args { 358aa75f4d3SHarshad Shirwadkar struct dentry *dentry; 359aa75f4d3SHarshad Shirwadkar int op; 360aa75f4d3SHarshad Shirwadkar }; 361aa75f4d3SHarshad Shirwadkar 362aa75f4d3SHarshad Shirwadkar /* __track_fn for directory entry updates. Called with ei->i_fc_lock. */ 363aa75f4d3SHarshad Shirwadkar static int __track_dentry_update(struct inode *inode, void *arg, bool update) 364aa75f4d3SHarshad Shirwadkar { 365aa75f4d3SHarshad Shirwadkar struct ext4_fc_dentry_update *node; 366aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 367aa75f4d3SHarshad Shirwadkar struct __track_dentry_update_args *dentry_update = 368aa75f4d3SHarshad Shirwadkar (struct __track_dentry_update_args *)arg; 369aa75f4d3SHarshad Shirwadkar struct dentry *dentry = dentry_update->dentry; 370aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 371aa75f4d3SHarshad Shirwadkar 372aa75f4d3SHarshad Shirwadkar mutex_unlock(&ei->i_fc_lock); 373aa75f4d3SHarshad Shirwadkar node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS); 374aa75f4d3SHarshad Shirwadkar if (!node) { 375aa75f4d3SHarshad Shirwadkar ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_MEM); 376aa75f4d3SHarshad Shirwadkar mutex_lock(&ei->i_fc_lock); 377aa75f4d3SHarshad Shirwadkar return -ENOMEM; 378aa75f4d3SHarshad Shirwadkar } 379aa75f4d3SHarshad Shirwadkar 380aa75f4d3SHarshad Shirwadkar node->fcd_op = dentry_update->op; 381aa75f4d3SHarshad Shirwadkar node->fcd_parent = dentry->d_parent->d_inode->i_ino; 382aa75f4d3SHarshad Shirwadkar node->fcd_ino = inode->i_ino; 383aa75f4d3SHarshad Shirwadkar if (dentry->d_name.len > DNAME_INLINE_LEN) { 384aa75f4d3SHarshad Shirwadkar node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS); 385aa75f4d3SHarshad Shirwadkar if (!node->fcd_name.name) { 386aa75f4d3SHarshad Shirwadkar kmem_cache_free(ext4_fc_dentry_cachep, node); 387aa75f4d3SHarshad Shirwadkar ext4_fc_mark_ineligible(inode->i_sb, 388aa75f4d3SHarshad Shirwadkar EXT4_FC_REASON_MEM); 389aa75f4d3SHarshad Shirwadkar mutex_lock(&ei->i_fc_lock); 390aa75f4d3SHarshad Shirwadkar return -ENOMEM; 391aa75f4d3SHarshad Shirwadkar } 392aa75f4d3SHarshad Shirwadkar memcpy((u8 *)node->fcd_name.name, dentry->d_name.name, 393aa75f4d3SHarshad Shirwadkar dentry->d_name.len); 394aa75f4d3SHarshad Shirwadkar } else { 395aa75f4d3SHarshad Shirwadkar memcpy(node->fcd_iname, dentry->d_name.name, 396aa75f4d3SHarshad Shirwadkar dentry->d_name.len); 397aa75f4d3SHarshad Shirwadkar node->fcd_name.name = node->fcd_iname; 398aa75f4d3SHarshad Shirwadkar } 399aa75f4d3SHarshad Shirwadkar node->fcd_name.len = dentry->d_name.len; 400aa75f4d3SHarshad Shirwadkar 401aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 402aa75f4d3SHarshad Shirwadkar if (sbi->s_mount_state & EXT4_FC_COMMITTING) 403aa75f4d3SHarshad Shirwadkar list_add_tail(&node->fcd_list, 404aa75f4d3SHarshad Shirwadkar &sbi->s_fc_dentry_q[FC_Q_STAGING]); 405aa75f4d3SHarshad Shirwadkar else 406aa75f4d3SHarshad Shirwadkar list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]); 407aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 408aa75f4d3SHarshad Shirwadkar mutex_lock(&ei->i_fc_lock); 409aa75f4d3SHarshad Shirwadkar 410aa75f4d3SHarshad Shirwadkar return 0; 411aa75f4d3SHarshad Shirwadkar } 412aa75f4d3SHarshad Shirwadkar 413aa75f4d3SHarshad Shirwadkar void ext4_fc_track_unlink(struct inode *inode, struct dentry *dentry) 414aa75f4d3SHarshad Shirwadkar { 415aa75f4d3SHarshad Shirwadkar struct __track_dentry_update_args args; 416aa75f4d3SHarshad Shirwadkar int ret; 417aa75f4d3SHarshad Shirwadkar 418aa75f4d3SHarshad Shirwadkar args.dentry = dentry; 419aa75f4d3SHarshad Shirwadkar args.op = EXT4_FC_TAG_UNLINK; 420aa75f4d3SHarshad Shirwadkar 421aa75f4d3SHarshad Shirwadkar ret = ext4_fc_track_template(inode, __track_dentry_update, 422aa75f4d3SHarshad Shirwadkar (void *)&args, 0); 423aa75f4d3SHarshad Shirwadkar trace_ext4_fc_track_unlink(inode, dentry, ret); 424aa75f4d3SHarshad Shirwadkar } 425aa75f4d3SHarshad Shirwadkar 426aa75f4d3SHarshad Shirwadkar void ext4_fc_track_link(struct inode *inode, struct dentry *dentry) 427aa75f4d3SHarshad Shirwadkar { 428aa75f4d3SHarshad Shirwadkar struct __track_dentry_update_args args; 429aa75f4d3SHarshad Shirwadkar int ret; 430aa75f4d3SHarshad Shirwadkar 431aa75f4d3SHarshad Shirwadkar args.dentry = dentry; 432aa75f4d3SHarshad Shirwadkar args.op = EXT4_FC_TAG_LINK; 433aa75f4d3SHarshad Shirwadkar 434aa75f4d3SHarshad Shirwadkar ret = ext4_fc_track_template(inode, __track_dentry_update, 435aa75f4d3SHarshad Shirwadkar (void *)&args, 0); 436aa75f4d3SHarshad Shirwadkar trace_ext4_fc_track_link(inode, dentry, ret); 437aa75f4d3SHarshad Shirwadkar } 438aa75f4d3SHarshad Shirwadkar 439aa75f4d3SHarshad Shirwadkar void ext4_fc_track_create(struct inode *inode, struct dentry *dentry) 440aa75f4d3SHarshad Shirwadkar { 441aa75f4d3SHarshad Shirwadkar struct __track_dentry_update_args args; 442aa75f4d3SHarshad Shirwadkar int ret; 443aa75f4d3SHarshad Shirwadkar 444aa75f4d3SHarshad Shirwadkar args.dentry = dentry; 445aa75f4d3SHarshad Shirwadkar args.op = EXT4_FC_TAG_CREAT; 446aa75f4d3SHarshad Shirwadkar 447aa75f4d3SHarshad Shirwadkar ret = ext4_fc_track_template(inode, __track_dentry_update, 448aa75f4d3SHarshad Shirwadkar (void *)&args, 0); 449aa75f4d3SHarshad Shirwadkar trace_ext4_fc_track_create(inode, dentry, ret); 450aa75f4d3SHarshad Shirwadkar } 451aa75f4d3SHarshad Shirwadkar 452aa75f4d3SHarshad Shirwadkar /* __track_fn for inode tracking */ 453aa75f4d3SHarshad Shirwadkar static int __track_inode(struct inode *inode, void *arg, bool update) 454aa75f4d3SHarshad Shirwadkar { 455aa75f4d3SHarshad Shirwadkar if (update) 456aa75f4d3SHarshad Shirwadkar return -EEXIST; 457aa75f4d3SHarshad Shirwadkar 458aa75f4d3SHarshad Shirwadkar EXT4_I(inode)->i_fc_lblk_len = 0; 459aa75f4d3SHarshad Shirwadkar 460aa75f4d3SHarshad Shirwadkar return 0; 461aa75f4d3SHarshad Shirwadkar } 462aa75f4d3SHarshad Shirwadkar 463aa75f4d3SHarshad Shirwadkar void ext4_fc_track_inode(struct inode *inode) 464aa75f4d3SHarshad Shirwadkar { 465aa75f4d3SHarshad Shirwadkar int ret; 466aa75f4d3SHarshad Shirwadkar 467aa75f4d3SHarshad Shirwadkar if (S_ISDIR(inode->i_mode)) 468aa75f4d3SHarshad Shirwadkar return; 469aa75f4d3SHarshad Shirwadkar 470aa75f4d3SHarshad Shirwadkar ret = ext4_fc_track_template(inode, __track_inode, NULL, 1); 471aa75f4d3SHarshad Shirwadkar trace_ext4_fc_track_inode(inode, ret); 472aa75f4d3SHarshad Shirwadkar } 473aa75f4d3SHarshad Shirwadkar 474aa75f4d3SHarshad Shirwadkar struct __track_range_args { 475aa75f4d3SHarshad Shirwadkar ext4_lblk_t start, end; 476aa75f4d3SHarshad Shirwadkar }; 477aa75f4d3SHarshad Shirwadkar 478aa75f4d3SHarshad Shirwadkar /* __track_fn for tracking data updates */ 479aa75f4d3SHarshad Shirwadkar static int __track_range(struct inode *inode, void *arg, bool update) 480aa75f4d3SHarshad Shirwadkar { 481aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 482aa75f4d3SHarshad Shirwadkar ext4_lblk_t oldstart; 483aa75f4d3SHarshad Shirwadkar struct __track_range_args *__arg = 484aa75f4d3SHarshad Shirwadkar (struct __track_range_args *)arg; 485aa75f4d3SHarshad Shirwadkar 486aa75f4d3SHarshad Shirwadkar if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) { 487aa75f4d3SHarshad Shirwadkar ext4_debug("Special inode %ld being modified\n", inode->i_ino); 488aa75f4d3SHarshad Shirwadkar return -ECANCELED; 489aa75f4d3SHarshad Shirwadkar } 490aa75f4d3SHarshad Shirwadkar 491aa75f4d3SHarshad Shirwadkar oldstart = ei->i_fc_lblk_start; 492aa75f4d3SHarshad Shirwadkar 493aa75f4d3SHarshad Shirwadkar if (update && ei->i_fc_lblk_len > 0) { 494aa75f4d3SHarshad Shirwadkar ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start); 495aa75f4d3SHarshad Shirwadkar ei->i_fc_lblk_len = 496aa75f4d3SHarshad Shirwadkar max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) - 497aa75f4d3SHarshad Shirwadkar ei->i_fc_lblk_start + 1; 498aa75f4d3SHarshad Shirwadkar } else { 499aa75f4d3SHarshad Shirwadkar ei->i_fc_lblk_start = __arg->start; 500aa75f4d3SHarshad Shirwadkar ei->i_fc_lblk_len = __arg->end - __arg->start + 1; 501aa75f4d3SHarshad Shirwadkar } 502aa75f4d3SHarshad Shirwadkar 503aa75f4d3SHarshad Shirwadkar return 0; 504aa75f4d3SHarshad Shirwadkar } 505aa75f4d3SHarshad Shirwadkar 506aa75f4d3SHarshad Shirwadkar void ext4_fc_track_range(struct inode *inode, ext4_lblk_t start, 507aa75f4d3SHarshad Shirwadkar ext4_lblk_t end) 508aa75f4d3SHarshad Shirwadkar { 509aa75f4d3SHarshad Shirwadkar struct __track_range_args args; 510aa75f4d3SHarshad Shirwadkar int ret; 511aa75f4d3SHarshad Shirwadkar 512aa75f4d3SHarshad Shirwadkar if (S_ISDIR(inode->i_mode)) 513aa75f4d3SHarshad Shirwadkar return; 514aa75f4d3SHarshad Shirwadkar 515aa75f4d3SHarshad Shirwadkar args.start = start; 516aa75f4d3SHarshad Shirwadkar args.end = end; 517aa75f4d3SHarshad Shirwadkar 518aa75f4d3SHarshad Shirwadkar ret = ext4_fc_track_template(inode, __track_range, &args, 1); 519aa75f4d3SHarshad Shirwadkar 520aa75f4d3SHarshad Shirwadkar trace_ext4_fc_track_range(inode, start, end, ret); 521aa75f4d3SHarshad Shirwadkar } 522aa75f4d3SHarshad Shirwadkar 523aa75f4d3SHarshad Shirwadkar static void ext4_fc_submit_bh(struct super_block *sb) 524aa75f4d3SHarshad Shirwadkar { 525aa75f4d3SHarshad Shirwadkar int write_flags = REQ_SYNC; 526aa75f4d3SHarshad Shirwadkar struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh; 527aa75f4d3SHarshad Shirwadkar 528aa75f4d3SHarshad Shirwadkar if (test_opt(sb, BARRIER)) 529aa75f4d3SHarshad Shirwadkar write_flags |= REQ_FUA | REQ_PREFLUSH; 530aa75f4d3SHarshad Shirwadkar lock_buffer(bh); 531aa75f4d3SHarshad Shirwadkar clear_buffer_dirty(bh); 532aa75f4d3SHarshad Shirwadkar set_buffer_uptodate(bh); 533aa75f4d3SHarshad Shirwadkar bh->b_end_io = ext4_end_buffer_io_sync; 534aa75f4d3SHarshad Shirwadkar submit_bh(REQ_OP_WRITE, write_flags, bh); 535aa75f4d3SHarshad Shirwadkar EXT4_SB(sb)->s_fc_bh = NULL; 536aa75f4d3SHarshad Shirwadkar } 537aa75f4d3SHarshad Shirwadkar 538aa75f4d3SHarshad Shirwadkar /* Ext4 commit path routines */ 539aa75f4d3SHarshad Shirwadkar 540aa75f4d3SHarshad Shirwadkar /* memzero and update CRC */ 541aa75f4d3SHarshad Shirwadkar static void *ext4_fc_memzero(struct super_block *sb, void *dst, int len, 542aa75f4d3SHarshad Shirwadkar u32 *crc) 543aa75f4d3SHarshad Shirwadkar { 544aa75f4d3SHarshad Shirwadkar void *ret; 545aa75f4d3SHarshad Shirwadkar 546aa75f4d3SHarshad Shirwadkar ret = memset(dst, 0, len); 547aa75f4d3SHarshad Shirwadkar if (crc) 548aa75f4d3SHarshad Shirwadkar *crc = ext4_chksum(EXT4_SB(sb), *crc, dst, len); 549aa75f4d3SHarshad Shirwadkar return ret; 550aa75f4d3SHarshad Shirwadkar } 551aa75f4d3SHarshad Shirwadkar 552aa75f4d3SHarshad Shirwadkar /* 553aa75f4d3SHarshad Shirwadkar * Allocate len bytes on a fast commit buffer. 554aa75f4d3SHarshad Shirwadkar * 555aa75f4d3SHarshad Shirwadkar * During the commit time this function is used to manage fast commit 556aa75f4d3SHarshad Shirwadkar * block space. We don't split a fast commit log onto different 557aa75f4d3SHarshad Shirwadkar * blocks. So this function makes sure that if there's not enough space 558aa75f4d3SHarshad Shirwadkar * on the current block, the remaining space in the current block is 559aa75f4d3SHarshad Shirwadkar * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case, 560aa75f4d3SHarshad Shirwadkar * new block is from jbd2 and CRC is updated to reflect the padding 561aa75f4d3SHarshad Shirwadkar * we added. 562aa75f4d3SHarshad Shirwadkar */ 563aa75f4d3SHarshad Shirwadkar static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc) 564aa75f4d3SHarshad Shirwadkar { 565aa75f4d3SHarshad Shirwadkar struct ext4_fc_tl *tl; 566aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 567aa75f4d3SHarshad Shirwadkar struct buffer_head *bh; 568aa75f4d3SHarshad Shirwadkar int bsize = sbi->s_journal->j_blocksize; 569aa75f4d3SHarshad Shirwadkar int ret, off = sbi->s_fc_bytes % bsize; 570aa75f4d3SHarshad Shirwadkar int pad_len; 571aa75f4d3SHarshad Shirwadkar 572aa75f4d3SHarshad Shirwadkar /* 573aa75f4d3SHarshad Shirwadkar * After allocating len, we should have space at least for a 0 byte 574aa75f4d3SHarshad Shirwadkar * padding. 575aa75f4d3SHarshad Shirwadkar */ 576aa75f4d3SHarshad Shirwadkar if (len + sizeof(struct ext4_fc_tl) > bsize) 577aa75f4d3SHarshad Shirwadkar return NULL; 578aa75f4d3SHarshad Shirwadkar 579aa75f4d3SHarshad Shirwadkar if (bsize - off - 1 > len + sizeof(struct ext4_fc_tl)) { 580aa75f4d3SHarshad Shirwadkar /* 581aa75f4d3SHarshad Shirwadkar * Only allocate from current buffer if we have enough space for 582aa75f4d3SHarshad Shirwadkar * this request AND we have space to add a zero byte padding. 583aa75f4d3SHarshad Shirwadkar */ 584aa75f4d3SHarshad Shirwadkar if (!sbi->s_fc_bh) { 585aa75f4d3SHarshad Shirwadkar ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); 586aa75f4d3SHarshad Shirwadkar if (ret) 587aa75f4d3SHarshad Shirwadkar return NULL; 588aa75f4d3SHarshad Shirwadkar sbi->s_fc_bh = bh; 589aa75f4d3SHarshad Shirwadkar } 590aa75f4d3SHarshad Shirwadkar sbi->s_fc_bytes += len; 591aa75f4d3SHarshad Shirwadkar return sbi->s_fc_bh->b_data + off; 592aa75f4d3SHarshad Shirwadkar } 593aa75f4d3SHarshad Shirwadkar /* Need to add PAD tag */ 594aa75f4d3SHarshad Shirwadkar tl = (struct ext4_fc_tl *)(sbi->s_fc_bh->b_data + off); 595aa75f4d3SHarshad Shirwadkar tl->fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD); 596aa75f4d3SHarshad Shirwadkar pad_len = bsize - off - 1 - sizeof(struct ext4_fc_tl); 597aa75f4d3SHarshad Shirwadkar tl->fc_len = cpu_to_le16(pad_len); 598aa75f4d3SHarshad Shirwadkar if (crc) 599aa75f4d3SHarshad Shirwadkar *crc = ext4_chksum(sbi, *crc, tl, sizeof(*tl)); 600aa75f4d3SHarshad Shirwadkar if (pad_len > 0) 601aa75f4d3SHarshad Shirwadkar ext4_fc_memzero(sb, tl + 1, pad_len, crc); 602aa75f4d3SHarshad Shirwadkar ext4_fc_submit_bh(sb); 603aa75f4d3SHarshad Shirwadkar 604aa75f4d3SHarshad Shirwadkar ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); 605aa75f4d3SHarshad Shirwadkar if (ret) 606aa75f4d3SHarshad Shirwadkar return NULL; 607aa75f4d3SHarshad Shirwadkar sbi->s_fc_bh = bh; 608aa75f4d3SHarshad Shirwadkar sbi->s_fc_bytes = (sbi->s_fc_bytes / bsize + 1) * bsize + len; 609aa75f4d3SHarshad Shirwadkar return sbi->s_fc_bh->b_data; 610aa75f4d3SHarshad Shirwadkar } 611aa75f4d3SHarshad Shirwadkar 612aa75f4d3SHarshad Shirwadkar /* memcpy to fc reserved space and update CRC */ 613aa75f4d3SHarshad Shirwadkar static void *ext4_fc_memcpy(struct super_block *sb, void *dst, const void *src, 614aa75f4d3SHarshad Shirwadkar int len, u32 *crc) 615aa75f4d3SHarshad Shirwadkar { 616aa75f4d3SHarshad Shirwadkar if (crc) 617aa75f4d3SHarshad Shirwadkar *crc = ext4_chksum(EXT4_SB(sb), *crc, src, len); 618aa75f4d3SHarshad Shirwadkar return memcpy(dst, src, len); 619aa75f4d3SHarshad Shirwadkar } 620aa75f4d3SHarshad Shirwadkar 621aa75f4d3SHarshad Shirwadkar /* 622aa75f4d3SHarshad Shirwadkar * Complete a fast commit by writing tail tag. 623aa75f4d3SHarshad Shirwadkar * 624aa75f4d3SHarshad Shirwadkar * Writing tail tag marks the end of a fast commit. In order to guarantee 625aa75f4d3SHarshad Shirwadkar * atomicity, after writing tail tag, even if there's space remaining 626aa75f4d3SHarshad Shirwadkar * in the block, next commit shouldn't use it. That's why tail tag 627aa75f4d3SHarshad Shirwadkar * has the length as that of the remaining space on the block. 628aa75f4d3SHarshad Shirwadkar */ 629aa75f4d3SHarshad Shirwadkar static int ext4_fc_write_tail(struct super_block *sb, u32 crc) 630aa75f4d3SHarshad Shirwadkar { 631aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 632aa75f4d3SHarshad Shirwadkar struct ext4_fc_tl tl; 633aa75f4d3SHarshad Shirwadkar struct ext4_fc_tail tail; 634aa75f4d3SHarshad Shirwadkar int off, bsize = sbi->s_journal->j_blocksize; 635aa75f4d3SHarshad Shirwadkar u8 *dst; 636aa75f4d3SHarshad Shirwadkar 637aa75f4d3SHarshad Shirwadkar /* 638aa75f4d3SHarshad Shirwadkar * ext4_fc_reserve_space takes care of allocating an extra block if 639aa75f4d3SHarshad Shirwadkar * there's no enough space on this block for accommodating this tail. 640aa75f4d3SHarshad Shirwadkar */ 641aa75f4d3SHarshad Shirwadkar dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(tail), &crc); 642aa75f4d3SHarshad Shirwadkar if (!dst) 643aa75f4d3SHarshad Shirwadkar return -ENOSPC; 644aa75f4d3SHarshad Shirwadkar 645aa75f4d3SHarshad Shirwadkar off = sbi->s_fc_bytes % bsize; 646aa75f4d3SHarshad Shirwadkar 647aa75f4d3SHarshad Shirwadkar tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL); 648aa75f4d3SHarshad Shirwadkar tl.fc_len = cpu_to_le16(bsize - off - 1 + sizeof(struct ext4_fc_tail)); 649aa75f4d3SHarshad Shirwadkar sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize); 650aa75f4d3SHarshad Shirwadkar 651aa75f4d3SHarshad Shirwadkar ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), &crc); 652aa75f4d3SHarshad Shirwadkar dst += sizeof(tl); 653aa75f4d3SHarshad Shirwadkar tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid); 654aa75f4d3SHarshad Shirwadkar ext4_fc_memcpy(sb, dst, &tail.fc_tid, sizeof(tail.fc_tid), &crc); 655aa75f4d3SHarshad Shirwadkar dst += sizeof(tail.fc_tid); 656aa75f4d3SHarshad Shirwadkar tail.fc_crc = cpu_to_le32(crc); 657aa75f4d3SHarshad Shirwadkar ext4_fc_memcpy(sb, dst, &tail.fc_crc, sizeof(tail.fc_crc), NULL); 658aa75f4d3SHarshad Shirwadkar 659aa75f4d3SHarshad Shirwadkar ext4_fc_submit_bh(sb); 660aa75f4d3SHarshad Shirwadkar 661aa75f4d3SHarshad Shirwadkar return 0; 662aa75f4d3SHarshad Shirwadkar } 663aa75f4d3SHarshad Shirwadkar 664aa75f4d3SHarshad Shirwadkar /* 665aa75f4d3SHarshad Shirwadkar * Adds tag, length, value and updates CRC. Returns true if tlv was added. 666aa75f4d3SHarshad Shirwadkar * Returns false if there's not enough space. 667aa75f4d3SHarshad Shirwadkar */ 668aa75f4d3SHarshad Shirwadkar static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val, 669aa75f4d3SHarshad Shirwadkar u32 *crc) 670aa75f4d3SHarshad Shirwadkar { 671aa75f4d3SHarshad Shirwadkar struct ext4_fc_tl tl; 672aa75f4d3SHarshad Shirwadkar u8 *dst; 673aa75f4d3SHarshad Shirwadkar 674aa75f4d3SHarshad Shirwadkar dst = ext4_fc_reserve_space(sb, sizeof(tl) + len, crc); 675aa75f4d3SHarshad Shirwadkar if (!dst) 676aa75f4d3SHarshad Shirwadkar return false; 677aa75f4d3SHarshad Shirwadkar 678aa75f4d3SHarshad Shirwadkar tl.fc_tag = cpu_to_le16(tag); 679aa75f4d3SHarshad Shirwadkar tl.fc_len = cpu_to_le16(len); 680aa75f4d3SHarshad Shirwadkar 681aa75f4d3SHarshad Shirwadkar ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc); 682aa75f4d3SHarshad Shirwadkar ext4_fc_memcpy(sb, dst + sizeof(tl), val, len, crc); 683aa75f4d3SHarshad Shirwadkar 684aa75f4d3SHarshad Shirwadkar return true; 685aa75f4d3SHarshad Shirwadkar } 686aa75f4d3SHarshad Shirwadkar 687aa75f4d3SHarshad Shirwadkar /* Same as above, but adds dentry tlv. */ 688aa75f4d3SHarshad Shirwadkar static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u16 tag, 689aa75f4d3SHarshad Shirwadkar int parent_ino, int ino, int dlen, 690aa75f4d3SHarshad Shirwadkar const unsigned char *dname, 691aa75f4d3SHarshad Shirwadkar u32 *crc) 692aa75f4d3SHarshad Shirwadkar { 693aa75f4d3SHarshad Shirwadkar struct ext4_fc_dentry_info fcd; 694aa75f4d3SHarshad Shirwadkar struct ext4_fc_tl tl; 695aa75f4d3SHarshad Shirwadkar u8 *dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(fcd) + dlen, 696aa75f4d3SHarshad Shirwadkar crc); 697aa75f4d3SHarshad Shirwadkar 698aa75f4d3SHarshad Shirwadkar if (!dst) 699aa75f4d3SHarshad Shirwadkar return false; 700aa75f4d3SHarshad Shirwadkar 701aa75f4d3SHarshad Shirwadkar fcd.fc_parent_ino = cpu_to_le32(parent_ino); 702aa75f4d3SHarshad Shirwadkar fcd.fc_ino = cpu_to_le32(ino); 703aa75f4d3SHarshad Shirwadkar tl.fc_tag = cpu_to_le16(tag); 704aa75f4d3SHarshad Shirwadkar tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen); 705aa75f4d3SHarshad Shirwadkar ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc); 706aa75f4d3SHarshad Shirwadkar dst += sizeof(tl); 707aa75f4d3SHarshad Shirwadkar ext4_fc_memcpy(sb, dst, &fcd, sizeof(fcd), crc); 708aa75f4d3SHarshad Shirwadkar dst += sizeof(fcd); 709aa75f4d3SHarshad Shirwadkar ext4_fc_memcpy(sb, dst, dname, dlen, crc); 710aa75f4d3SHarshad Shirwadkar dst += dlen; 711aa75f4d3SHarshad Shirwadkar 712aa75f4d3SHarshad Shirwadkar return true; 713aa75f4d3SHarshad Shirwadkar } 714aa75f4d3SHarshad Shirwadkar 715aa75f4d3SHarshad Shirwadkar /* 716aa75f4d3SHarshad Shirwadkar * Writes inode in the fast commit space under TLV with tag @tag. 717aa75f4d3SHarshad Shirwadkar * Returns 0 on success, error on failure. 718aa75f4d3SHarshad Shirwadkar */ 719aa75f4d3SHarshad Shirwadkar static int ext4_fc_write_inode(struct inode *inode, u32 *crc) 720aa75f4d3SHarshad Shirwadkar { 721aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 722aa75f4d3SHarshad Shirwadkar int inode_len = EXT4_GOOD_OLD_INODE_SIZE; 723aa75f4d3SHarshad Shirwadkar int ret; 724aa75f4d3SHarshad Shirwadkar struct ext4_iloc iloc; 725aa75f4d3SHarshad Shirwadkar struct ext4_fc_inode fc_inode; 726aa75f4d3SHarshad Shirwadkar struct ext4_fc_tl tl; 727aa75f4d3SHarshad Shirwadkar u8 *dst; 728aa75f4d3SHarshad Shirwadkar 729aa75f4d3SHarshad Shirwadkar ret = ext4_get_inode_loc(inode, &iloc); 730aa75f4d3SHarshad Shirwadkar if (ret) 731aa75f4d3SHarshad Shirwadkar return ret; 732aa75f4d3SHarshad Shirwadkar 733aa75f4d3SHarshad Shirwadkar if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) 734aa75f4d3SHarshad Shirwadkar inode_len += ei->i_extra_isize; 735aa75f4d3SHarshad Shirwadkar 736aa75f4d3SHarshad Shirwadkar fc_inode.fc_ino = cpu_to_le32(inode->i_ino); 737aa75f4d3SHarshad Shirwadkar tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE); 738aa75f4d3SHarshad Shirwadkar tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino)); 739aa75f4d3SHarshad Shirwadkar 740aa75f4d3SHarshad Shirwadkar dst = ext4_fc_reserve_space(inode->i_sb, 741aa75f4d3SHarshad Shirwadkar sizeof(tl) + inode_len + sizeof(fc_inode.fc_ino), crc); 742aa75f4d3SHarshad Shirwadkar if (!dst) 743aa75f4d3SHarshad Shirwadkar return -ECANCELED; 744aa75f4d3SHarshad Shirwadkar 745aa75f4d3SHarshad Shirwadkar if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, sizeof(tl), crc)) 746aa75f4d3SHarshad Shirwadkar return -ECANCELED; 747aa75f4d3SHarshad Shirwadkar dst += sizeof(tl); 748aa75f4d3SHarshad Shirwadkar if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc)) 749aa75f4d3SHarshad Shirwadkar return -ECANCELED; 750aa75f4d3SHarshad Shirwadkar dst += sizeof(fc_inode); 751aa75f4d3SHarshad Shirwadkar if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc), 752aa75f4d3SHarshad Shirwadkar inode_len, crc)) 753aa75f4d3SHarshad Shirwadkar return -ECANCELED; 754aa75f4d3SHarshad Shirwadkar 755aa75f4d3SHarshad Shirwadkar return 0; 756aa75f4d3SHarshad Shirwadkar } 757aa75f4d3SHarshad Shirwadkar 758aa75f4d3SHarshad Shirwadkar /* 759aa75f4d3SHarshad Shirwadkar * Writes updated data ranges for the inode in question. Updates CRC. 760aa75f4d3SHarshad Shirwadkar * Returns 0 on success, error otherwise. 761aa75f4d3SHarshad Shirwadkar */ 762aa75f4d3SHarshad Shirwadkar static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc) 763aa75f4d3SHarshad Shirwadkar { 764aa75f4d3SHarshad Shirwadkar ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size; 765aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei = EXT4_I(inode); 766aa75f4d3SHarshad Shirwadkar struct ext4_map_blocks map; 767aa75f4d3SHarshad Shirwadkar struct ext4_fc_add_range fc_ext; 768aa75f4d3SHarshad Shirwadkar struct ext4_fc_del_range lrange; 769aa75f4d3SHarshad Shirwadkar struct ext4_extent *ex; 770aa75f4d3SHarshad Shirwadkar int ret; 771aa75f4d3SHarshad Shirwadkar 772aa75f4d3SHarshad Shirwadkar mutex_lock(&ei->i_fc_lock); 773aa75f4d3SHarshad Shirwadkar if (ei->i_fc_lblk_len == 0) { 774aa75f4d3SHarshad Shirwadkar mutex_unlock(&ei->i_fc_lock); 775aa75f4d3SHarshad Shirwadkar return 0; 776aa75f4d3SHarshad Shirwadkar } 777aa75f4d3SHarshad Shirwadkar old_blk_size = ei->i_fc_lblk_start; 778aa75f4d3SHarshad Shirwadkar new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; 779aa75f4d3SHarshad Shirwadkar ei->i_fc_lblk_len = 0; 780aa75f4d3SHarshad Shirwadkar mutex_unlock(&ei->i_fc_lock); 781aa75f4d3SHarshad Shirwadkar 782aa75f4d3SHarshad Shirwadkar cur_lblk_off = old_blk_size; 783aa75f4d3SHarshad Shirwadkar jbd_debug(1, "%s: will try writing %d to %d for inode %ld\n", 784aa75f4d3SHarshad Shirwadkar __func__, cur_lblk_off, new_blk_size, inode->i_ino); 785aa75f4d3SHarshad Shirwadkar 786aa75f4d3SHarshad Shirwadkar while (cur_lblk_off <= new_blk_size) { 787aa75f4d3SHarshad Shirwadkar map.m_lblk = cur_lblk_off; 788aa75f4d3SHarshad Shirwadkar map.m_len = new_blk_size - cur_lblk_off + 1; 789aa75f4d3SHarshad Shirwadkar ret = ext4_map_blocks(NULL, inode, &map, 0); 790aa75f4d3SHarshad Shirwadkar if (ret < 0) 791aa75f4d3SHarshad Shirwadkar return -ECANCELED; 792aa75f4d3SHarshad Shirwadkar 793aa75f4d3SHarshad Shirwadkar if (map.m_len == 0) { 794aa75f4d3SHarshad Shirwadkar cur_lblk_off++; 795aa75f4d3SHarshad Shirwadkar continue; 796aa75f4d3SHarshad Shirwadkar } 797aa75f4d3SHarshad Shirwadkar 798aa75f4d3SHarshad Shirwadkar if (ret == 0) { 799aa75f4d3SHarshad Shirwadkar lrange.fc_ino = cpu_to_le32(inode->i_ino); 800aa75f4d3SHarshad Shirwadkar lrange.fc_lblk = cpu_to_le32(map.m_lblk); 801aa75f4d3SHarshad Shirwadkar lrange.fc_len = cpu_to_le32(map.m_len); 802aa75f4d3SHarshad Shirwadkar if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, 803aa75f4d3SHarshad Shirwadkar sizeof(lrange), (u8 *)&lrange, crc)) 804aa75f4d3SHarshad Shirwadkar return -ENOSPC; 805aa75f4d3SHarshad Shirwadkar } else { 806aa75f4d3SHarshad Shirwadkar fc_ext.fc_ino = cpu_to_le32(inode->i_ino); 807aa75f4d3SHarshad Shirwadkar ex = (struct ext4_extent *)&fc_ext.fc_ex; 808aa75f4d3SHarshad Shirwadkar ex->ee_block = cpu_to_le32(map.m_lblk); 809aa75f4d3SHarshad Shirwadkar ex->ee_len = cpu_to_le16(map.m_len); 810aa75f4d3SHarshad Shirwadkar ext4_ext_store_pblock(ex, map.m_pblk); 811aa75f4d3SHarshad Shirwadkar if (map.m_flags & EXT4_MAP_UNWRITTEN) 812aa75f4d3SHarshad Shirwadkar ext4_ext_mark_unwritten(ex); 813aa75f4d3SHarshad Shirwadkar else 814aa75f4d3SHarshad Shirwadkar ext4_ext_mark_initialized(ex); 815aa75f4d3SHarshad Shirwadkar if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, 816aa75f4d3SHarshad Shirwadkar sizeof(fc_ext), (u8 *)&fc_ext, crc)) 817aa75f4d3SHarshad Shirwadkar return -ENOSPC; 818aa75f4d3SHarshad Shirwadkar } 819aa75f4d3SHarshad Shirwadkar 820aa75f4d3SHarshad Shirwadkar cur_lblk_off += map.m_len; 821aa75f4d3SHarshad Shirwadkar } 822aa75f4d3SHarshad Shirwadkar 823aa75f4d3SHarshad Shirwadkar return 0; 824aa75f4d3SHarshad Shirwadkar } 825aa75f4d3SHarshad Shirwadkar 826aa75f4d3SHarshad Shirwadkar 827aa75f4d3SHarshad Shirwadkar /* Submit data for all the fast commit inodes */ 828aa75f4d3SHarshad Shirwadkar static int ext4_fc_submit_inode_data_all(journal_t *journal) 829aa75f4d3SHarshad Shirwadkar { 830aa75f4d3SHarshad Shirwadkar struct super_block *sb = (struct super_block *)(journal->j_private); 831aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 832aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei; 833aa75f4d3SHarshad Shirwadkar struct list_head *pos; 834aa75f4d3SHarshad Shirwadkar int ret = 0; 835aa75f4d3SHarshad Shirwadkar 836aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 837aa75f4d3SHarshad Shirwadkar sbi->s_mount_state |= EXT4_FC_COMMITTING; 838aa75f4d3SHarshad Shirwadkar list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) { 839aa75f4d3SHarshad Shirwadkar ei = list_entry(pos, struct ext4_inode_info, i_fc_list); 840aa75f4d3SHarshad Shirwadkar ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); 841aa75f4d3SHarshad Shirwadkar while (atomic_read(&ei->i_fc_updates)) { 842aa75f4d3SHarshad Shirwadkar DEFINE_WAIT(wait); 843aa75f4d3SHarshad Shirwadkar 844aa75f4d3SHarshad Shirwadkar prepare_to_wait(&ei->i_fc_wait, &wait, 845aa75f4d3SHarshad Shirwadkar TASK_UNINTERRUPTIBLE); 846aa75f4d3SHarshad Shirwadkar if (atomic_read(&ei->i_fc_updates)) { 847aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 848aa75f4d3SHarshad Shirwadkar schedule(); 849aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 850aa75f4d3SHarshad Shirwadkar } 851aa75f4d3SHarshad Shirwadkar finish_wait(&ei->i_fc_wait, &wait); 852aa75f4d3SHarshad Shirwadkar } 853aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 854aa75f4d3SHarshad Shirwadkar ret = jbd2_submit_inode_data(ei->jinode); 855aa75f4d3SHarshad Shirwadkar if (ret) 856aa75f4d3SHarshad Shirwadkar return ret; 857aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 858aa75f4d3SHarshad Shirwadkar } 859aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 860aa75f4d3SHarshad Shirwadkar 861aa75f4d3SHarshad Shirwadkar return ret; 862aa75f4d3SHarshad Shirwadkar } 863aa75f4d3SHarshad Shirwadkar 864aa75f4d3SHarshad Shirwadkar /* Wait for completion of data for all the fast commit inodes */ 865aa75f4d3SHarshad Shirwadkar static int ext4_fc_wait_inode_data_all(journal_t *journal) 866aa75f4d3SHarshad Shirwadkar { 867aa75f4d3SHarshad Shirwadkar struct super_block *sb = (struct super_block *)(journal->j_private); 868aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 869aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *pos, *n; 870aa75f4d3SHarshad Shirwadkar int ret = 0; 871aa75f4d3SHarshad Shirwadkar 872aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 873aa75f4d3SHarshad Shirwadkar list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { 874aa75f4d3SHarshad Shirwadkar if (!ext4_test_inode_state(&pos->vfs_inode, 875aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING)) 876aa75f4d3SHarshad Shirwadkar continue; 877aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 878aa75f4d3SHarshad Shirwadkar 879aa75f4d3SHarshad Shirwadkar ret = jbd2_wait_inode_data(journal, pos->jinode); 880aa75f4d3SHarshad Shirwadkar if (ret) 881aa75f4d3SHarshad Shirwadkar return ret; 882aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 883aa75f4d3SHarshad Shirwadkar } 884aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 885aa75f4d3SHarshad Shirwadkar 886aa75f4d3SHarshad Shirwadkar return 0; 887aa75f4d3SHarshad Shirwadkar } 888aa75f4d3SHarshad Shirwadkar 889aa75f4d3SHarshad Shirwadkar /* Commit all the directory entry updates */ 890aa75f4d3SHarshad Shirwadkar static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc) 891aa75f4d3SHarshad Shirwadkar { 892aa75f4d3SHarshad Shirwadkar struct super_block *sb = (struct super_block *)(journal->j_private); 893aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 894aa75f4d3SHarshad Shirwadkar struct ext4_fc_dentry_update *fc_dentry; 895aa75f4d3SHarshad Shirwadkar struct inode *inode; 896aa75f4d3SHarshad Shirwadkar struct list_head *pos, *n, *fcd_pos, *fcd_n; 897aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *ei; 898aa75f4d3SHarshad Shirwadkar int ret; 899aa75f4d3SHarshad Shirwadkar 900aa75f4d3SHarshad Shirwadkar if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) 901aa75f4d3SHarshad Shirwadkar return 0; 902aa75f4d3SHarshad Shirwadkar list_for_each_safe(fcd_pos, fcd_n, &sbi->s_fc_dentry_q[FC_Q_MAIN]) { 903aa75f4d3SHarshad Shirwadkar fc_dentry = list_entry(fcd_pos, struct ext4_fc_dentry_update, 904aa75f4d3SHarshad Shirwadkar fcd_list); 905aa75f4d3SHarshad Shirwadkar if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) { 906aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 907aa75f4d3SHarshad Shirwadkar if (!ext4_fc_add_dentry_tlv( 908aa75f4d3SHarshad Shirwadkar sb, fc_dentry->fcd_op, 909aa75f4d3SHarshad Shirwadkar fc_dentry->fcd_parent, fc_dentry->fcd_ino, 910aa75f4d3SHarshad Shirwadkar fc_dentry->fcd_name.len, 911aa75f4d3SHarshad Shirwadkar fc_dentry->fcd_name.name, crc)) { 912aa75f4d3SHarshad Shirwadkar ret = -ENOSPC; 913aa75f4d3SHarshad Shirwadkar goto lock_and_exit; 914aa75f4d3SHarshad Shirwadkar } 915aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 916aa75f4d3SHarshad Shirwadkar continue; 917aa75f4d3SHarshad Shirwadkar } 918aa75f4d3SHarshad Shirwadkar 919aa75f4d3SHarshad Shirwadkar inode = NULL; 920aa75f4d3SHarshad Shirwadkar list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) { 921aa75f4d3SHarshad Shirwadkar ei = list_entry(pos, struct ext4_inode_info, i_fc_list); 922aa75f4d3SHarshad Shirwadkar if (ei->vfs_inode.i_ino == fc_dentry->fcd_ino) { 923aa75f4d3SHarshad Shirwadkar inode = &ei->vfs_inode; 924aa75f4d3SHarshad Shirwadkar break; 925aa75f4d3SHarshad Shirwadkar } 926aa75f4d3SHarshad Shirwadkar } 927aa75f4d3SHarshad Shirwadkar /* 928aa75f4d3SHarshad Shirwadkar * If we don't find inode in our list, then it was deleted, 929aa75f4d3SHarshad Shirwadkar * in which case, we don't need to record it's create tag. 930aa75f4d3SHarshad Shirwadkar */ 931aa75f4d3SHarshad Shirwadkar if (!inode) 932aa75f4d3SHarshad Shirwadkar continue; 933aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 934aa75f4d3SHarshad Shirwadkar 935aa75f4d3SHarshad Shirwadkar /* 936aa75f4d3SHarshad Shirwadkar * We first write the inode and then the create dirent. This 937aa75f4d3SHarshad Shirwadkar * allows the recovery code to create an unnamed inode first 938aa75f4d3SHarshad Shirwadkar * and then link it to a directory entry. This allows us 939aa75f4d3SHarshad Shirwadkar * to use namei.c routines almost as is and simplifies 940aa75f4d3SHarshad Shirwadkar * the recovery code. 941aa75f4d3SHarshad Shirwadkar */ 942aa75f4d3SHarshad Shirwadkar ret = ext4_fc_write_inode(inode, crc); 943aa75f4d3SHarshad Shirwadkar if (ret) 944aa75f4d3SHarshad Shirwadkar goto lock_and_exit; 945aa75f4d3SHarshad Shirwadkar 946aa75f4d3SHarshad Shirwadkar ret = ext4_fc_write_inode_data(inode, crc); 947aa75f4d3SHarshad Shirwadkar if (ret) 948aa75f4d3SHarshad Shirwadkar goto lock_and_exit; 949aa75f4d3SHarshad Shirwadkar 950aa75f4d3SHarshad Shirwadkar if (!ext4_fc_add_dentry_tlv( 951aa75f4d3SHarshad Shirwadkar sb, fc_dentry->fcd_op, 952aa75f4d3SHarshad Shirwadkar fc_dentry->fcd_parent, fc_dentry->fcd_ino, 953aa75f4d3SHarshad Shirwadkar fc_dentry->fcd_name.len, 954aa75f4d3SHarshad Shirwadkar fc_dentry->fcd_name.name, crc)) { 955aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 956aa75f4d3SHarshad Shirwadkar ret = -ENOSPC; 957aa75f4d3SHarshad Shirwadkar goto lock_and_exit; 958aa75f4d3SHarshad Shirwadkar } 959aa75f4d3SHarshad Shirwadkar 960aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 961aa75f4d3SHarshad Shirwadkar } 962aa75f4d3SHarshad Shirwadkar return 0; 963aa75f4d3SHarshad Shirwadkar lock_and_exit: 964aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 965aa75f4d3SHarshad Shirwadkar return ret; 966aa75f4d3SHarshad Shirwadkar } 967aa75f4d3SHarshad Shirwadkar 968aa75f4d3SHarshad Shirwadkar static int ext4_fc_perform_commit(journal_t *journal) 969aa75f4d3SHarshad Shirwadkar { 970aa75f4d3SHarshad Shirwadkar struct super_block *sb = (struct super_block *)(journal->j_private); 971aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 972aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *iter; 973aa75f4d3SHarshad Shirwadkar struct ext4_fc_head head; 974aa75f4d3SHarshad Shirwadkar struct list_head *pos; 975aa75f4d3SHarshad Shirwadkar struct inode *inode; 976aa75f4d3SHarshad Shirwadkar struct blk_plug plug; 977aa75f4d3SHarshad Shirwadkar int ret = 0; 978aa75f4d3SHarshad Shirwadkar u32 crc = 0; 979aa75f4d3SHarshad Shirwadkar 980aa75f4d3SHarshad Shirwadkar ret = ext4_fc_submit_inode_data_all(journal); 981aa75f4d3SHarshad Shirwadkar if (ret) 982aa75f4d3SHarshad Shirwadkar return ret; 983aa75f4d3SHarshad Shirwadkar 984aa75f4d3SHarshad Shirwadkar ret = ext4_fc_wait_inode_data_all(journal); 985aa75f4d3SHarshad Shirwadkar if (ret) 986aa75f4d3SHarshad Shirwadkar return ret; 987aa75f4d3SHarshad Shirwadkar 988aa75f4d3SHarshad Shirwadkar blk_start_plug(&plug); 989aa75f4d3SHarshad Shirwadkar if (sbi->s_fc_bytes == 0) { 990aa75f4d3SHarshad Shirwadkar /* 991aa75f4d3SHarshad Shirwadkar * Add a head tag only if this is the first fast commit 992aa75f4d3SHarshad Shirwadkar * in this TID. 993aa75f4d3SHarshad Shirwadkar */ 994aa75f4d3SHarshad Shirwadkar head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES); 995aa75f4d3SHarshad Shirwadkar head.fc_tid = cpu_to_le32( 996aa75f4d3SHarshad Shirwadkar sbi->s_journal->j_running_transaction->t_tid); 997aa75f4d3SHarshad Shirwadkar if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head), 998aa75f4d3SHarshad Shirwadkar (u8 *)&head, &crc)) 999aa75f4d3SHarshad Shirwadkar goto out; 1000aa75f4d3SHarshad Shirwadkar } 1001aa75f4d3SHarshad Shirwadkar 1002aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 1003aa75f4d3SHarshad Shirwadkar ret = ext4_fc_commit_dentry_updates(journal, &crc); 1004aa75f4d3SHarshad Shirwadkar if (ret) { 1005aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 1006aa75f4d3SHarshad Shirwadkar goto out; 1007aa75f4d3SHarshad Shirwadkar } 1008aa75f4d3SHarshad Shirwadkar 1009aa75f4d3SHarshad Shirwadkar list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) { 1010aa75f4d3SHarshad Shirwadkar iter = list_entry(pos, struct ext4_inode_info, i_fc_list); 1011aa75f4d3SHarshad Shirwadkar inode = &iter->vfs_inode; 1012aa75f4d3SHarshad Shirwadkar if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) 1013aa75f4d3SHarshad Shirwadkar continue; 1014aa75f4d3SHarshad Shirwadkar 1015aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 1016aa75f4d3SHarshad Shirwadkar ret = ext4_fc_write_inode_data(inode, &crc); 1017aa75f4d3SHarshad Shirwadkar if (ret) 1018aa75f4d3SHarshad Shirwadkar goto out; 1019aa75f4d3SHarshad Shirwadkar ret = ext4_fc_write_inode(inode, &crc); 1020aa75f4d3SHarshad Shirwadkar if (ret) 1021aa75f4d3SHarshad Shirwadkar goto out; 1022aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 1023aa75f4d3SHarshad Shirwadkar EXT4_I(inode)->i_fc_committed_subtid = 1024aa75f4d3SHarshad Shirwadkar atomic_read(&sbi->s_fc_subtid); 1025aa75f4d3SHarshad Shirwadkar } 1026aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 1027aa75f4d3SHarshad Shirwadkar 1028aa75f4d3SHarshad Shirwadkar ret = ext4_fc_write_tail(sb, crc); 1029aa75f4d3SHarshad Shirwadkar 1030aa75f4d3SHarshad Shirwadkar out: 1031aa75f4d3SHarshad Shirwadkar blk_finish_plug(&plug); 1032aa75f4d3SHarshad Shirwadkar return ret; 1033aa75f4d3SHarshad Shirwadkar } 1034aa75f4d3SHarshad Shirwadkar 1035aa75f4d3SHarshad Shirwadkar /* 1036aa75f4d3SHarshad Shirwadkar * The main commit entry point. Performs a fast commit for transaction 1037aa75f4d3SHarshad Shirwadkar * commit_tid if needed. If it's not possible to perform a fast commit 1038aa75f4d3SHarshad Shirwadkar * due to various reasons, we fall back to full commit. Returns 0 1039aa75f4d3SHarshad Shirwadkar * on success, error otherwise. 1040aa75f4d3SHarshad Shirwadkar */ 1041aa75f4d3SHarshad Shirwadkar int ext4_fc_commit(journal_t *journal, tid_t commit_tid) 1042aa75f4d3SHarshad Shirwadkar { 1043aa75f4d3SHarshad Shirwadkar struct super_block *sb = (struct super_block *)(journal->j_private); 1044aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 1045aa75f4d3SHarshad Shirwadkar int nblks = 0, ret, bsize = journal->j_blocksize; 1046aa75f4d3SHarshad Shirwadkar int subtid = atomic_read(&sbi->s_fc_subtid); 1047aa75f4d3SHarshad Shirwadkar int reason = EXT4_FC_REASON_OK, fc_bufs_before = 0; 1048aa75f4d3SHarshad Shirwadkar ktime_t start_time, commit_time; 1049aa75f4d3SHarshad Shirwadkar 1050aa75f4d3SHarshad Shirwadkar trace_ext4_fc_commit_start(sb); 1051aa75f4d3SHarshad Shirwadkar 1052aa75f4d3SHarshad Shirwadkar start_time = ktime_get(); 1053aa75f4d3SHarshad Shirwadkar 1054aa75f4d3SHarshad Shirwadkar if (!test_opt2(sb, JOURNAL_FAST_COMMIT) || 1055aa75f4d3SHarshad Shirwadkar (ext4_fc_is_ineligible(sb))) { 1056aa75f4d3SHarshad Shirwadkar reason = EXT4_FC_REASON_INELIGIBLE; 1057aa75f4d3SHarshad Shirwadkar goto out; 1058aa75f4d3SHarshad Shirwadkar } 1059aa75f4d3SHarshad Shirwadkar 1060aa75f4d3SHarshad Shirwadkar restart_fc: 1061aa75f4d3SHarshad Shirwadkar ret = jbd2_fc_begin_commit(journal, commit_tid); 1062aa75f4d3SHarshad Shirwadkar if (ret == -EALREADY) { 1063aa75f4d3SHarshad Shirwadkar /* There was an ongoing commit, check if we need to restart */ 1064aa75f4d3SHarshad Shirwadkar if (atomic_read(&sbi->s_fc_subtid) <= subtid && 1065aa75f4d3SHarshad Shirwadkar commit_tid > journal->j_commit_sequence) 1066aa75f4d3SHarshad Shirwadkar goto restart_fc; 1067aa75f4d3SHarshad Shirwadkar reason = EXT4_FC_REASON_ALREADY_COMMITTED; 1068aa75f4d3SHarshad Shirwadkar goto out; 1069aa75f4d3SHarshad Shirwadkar } else if (ret) { 1070aa75f4d3SHarshad Shirwadkar sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++; 1071aa75f4d3SHarshad Shirwadkar reason = EXT4_FC_REASON_FC_START_FAILED; 1072aa75f4d3SHarshad Shirwadkar goto out; 1073aa75f4d3SHarshad Shirwadkar } 1074aa75f4d3SHarshad Shirwadkar 1075aa75f4d3SHarshad Shirwadkar fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize; 1076aa75f4d3SHarshad Shirwadkar ret = ext4_fc_perform_commit(journal); 1077aa75f4d3SHarshad Shirwadkar if (ret < 0) { 1078aa75f4d3SHarshad Shirwadkar sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++; 1079aa75f4d3SHarshad Shirwadkar reason = EXT4_FC_REASON_FC_FAILED; 1080aa75f4d3SHarshad Shirwadkar goto out; 1081aa75f4d3SHarshad Shirwadkar } 1082aa75f4d3SHarshad Shirwadkar nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before; 1083aa75f4d3SHarshad Shirwadkar ret = jbd2_fc_wait_bufs(journal, nblks); 1084aa75f4d3SHarshad Shirwadkar if (ret < 0) { 1085aa75f4d3SHarshad Shirwadkar sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++; 1086aa75f4d3SHarshad Shirwadkar reason = EXT4_FC_REASON_FC_FAILED; 1087aa75f4d3SHarshad Shirwadkar goto out; 1088aa75f4d3SHarshad Shirwadkar } 1089aa75f4d3SHarshad Shirwadkar atomic_inc(&sbi->s_fc_subtid); 1090aa75f4d3SHarshad Shirwadkar jbd2_fc_end_commit(journal); 1091aa75f4d3SHarshad Shirwadkar out: 1092aa75f4d3SHarshad Shirwadkar /* Has any ineligible update happened since we started? */ 1093aa75f4d3SHarshad Shirwadkar if (reason == EXT4_FC_REASON_OK && ext4_fc_is_ineligible(sb)) { 1094aa75f4d3SHarshad Shirwadkar sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++; 1095aa75f4d3SHarshad Shirwadkar reason = EXT4_FC_REASON_INELIGIBLE; 1096aa75f4d3SHarshad Shirwadkar } 1097aa75f4d3SHarshad Shirwadkar 1098aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 1099aa75f4d3SHarshad Shirwadkar if (reason != EXT4_FC_REASON_OK && 1100aa75f4d3SHarshad Shirwadkar reason != EXT4_FC_REASON_ALREADY_COMMITTED) { 1101aa75f4d3SHarshad Shirwadkar sbi->s_fc_stats.fc_ineligible_commits++; 1102aa75f4d3SHarshad Shirwadkar } else { 1103aa75f4d3SHarshad Shirwadkar sbi->s_fc_stats.fc_num_commits++; 1104aa75f4d3SHarshad Shirwadkar sbi->s_fc_stats.fc_numblks += nblks; 1105aa75f4d3SHarshad Shirwadkar } 1106aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 1107aa75f4d3SHarshad Shirwadkar nblks = (reason == EXT4_FC_REASON_OK) ? nblks : 0; 1108aa75f4d3SHarshad Shirwadkar trace_ext4_fc_commit_stop(sb, nblks, reason); 1109aa75f4d3SHarshad Shirwadkar commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time)); 1110aa75f4d3SHarshad Shirwadkar /* 1111aa75f4d3SHarshad Shirwadkar * weight the commit time higher than the average time so we don't 1112aa75f4d3SHarshad Shirwadkar * react too strongly to vast changes in the commit time 1113aa75f4d3SHarshad Shirwadkar */ 1114aa75f4d3SHarshad Shirwadkar if (likely(sbi->s_fc_avg_commit_time)) 1115aa75f4d3SHarshad Shirwadkar sbi->s_fc_avg_commit_time = (commit_time + 1116aa75f4d3SHarshad Shirwadkar sbi->s_fc_avg_commit_time * 3) / 4; 1117aa75f4d3SHarshad Shirwadkar else 1118aa75f4d3SHarshad Shirwadkar sbi->s_fc_avg_commit_time = commit_time; 1119aa75f4d3SHarshad Shirwadkar jbd_debug(1, 1120aa75f4d3SHarshad Shirwadkar "Fast commit ended with blks = %d, reason = %d, subtid - %d", 1121aa75f4d3SHarshad Shirwadkar nblks, reason, subtid); 1122aa75f4d3SHarshad Shirwadkar if (reason == EXT4_FC_REASON_FC_FAILED) 1123aa75f4d3SHarshad Shirwadkar return jbd2_fc_end_commit_fallback(journal, commit_tid); 1124aa75f4d3SHarshad Shirwadkar if (reason == EXT4_FC_REASON_FC_START_FAILED || 1125aa75f4d3SHarshad Shirwadkar reason == EXT4_FC_REASON_INELIGIBLE) 1126aa75f4d3SHarshad Shirwadkar return jbd2_complete_transaction(journal, commit_tid); 1127aa75f4d3SHarshad Shirwadkar return 0; 1128aa75f4d3SHarshad Shirwadkar } 1129aa75f4d3SHarshad Shirwadkar 1130ff780b91SHarshad Shirwadkar /* 1131ff780b91SHarshad Shirwadkar * Fast commit cleanup routine. This is called after every fast commit and 1132ff780b91SHarshad Shirwadkar * full commit. full is true if we are called after a full commit. 1133ff780b91SHarshad Shirwadkar */ 1134ff780b91SHarshad Shirwadkar static void ext4_fc_cleanup(journal_t *journal, int full) 1135ff780b91SHarshad Shirwadkar { 1136aa75f4d3SHarshad Shirwadkar struct super_block *sb = journal->j_private; 1137aa75f4d3SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 1138aa75f4d3SHarshad Shirwadkar struct ext4_inode_info *iter; 1139aa75f4d3SHarshad Shirwadkar struct ext4_fc_dentry_update *fc_dentry; 1140aa75f4d3SHarshad Shirwadkar struct list_head *pos, *n; 1141aa75f4d3SHarshad Shirwadkar 1142aa75f4d3SHarshad Shirwadkar if (full && sbi->s_fc_bh) 1143aa75f4d3SHarshad Shirwadkar sbi->s_fc_bh = NULL; 1144aa75f4d3SHarshad Shirwadkar 1145aa75f4d3SHarshad Shirwadkar jbd2_fc_release_bufs(journal); 1146aa75f4d3SHarshad Shirwadkar 1147aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 1148aa75f4d3SHarshad Shirwadkar list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) { 1149aa75f4d3SHarshad Shirwadkar iter = list_entry(pos, struct ext4_inode_info, i_fc_list); 1150aa75f4d3SHarshad Shirwadkar list_del_init(&iter->i_fc_list); 1151aa75f4d3SHarshad Shirwadkar ext4_clear_inode_state(&iter->vfs_inode, 1152aa75f4d3SHarshad Shirwadkar EXT4_STATE_FC_COMMITTING); 1153aa75f4d3SHarshad Shirwadkar ext4_fc_reset_inode(&iter->vfs_inode); 1154aa75f4d3SHarshad Shirwadkar /* Make sure EXT4_STATE_FC_COMMITTING bit is clear */ 1155aa75f4d3SHarshad Shirwadkar smp_mb(); 1156aa75f4d3SHarshad Shirwadkar #if (BITS_PER_LONG < 64) 1157aa75f4d3SHarshad Shirwadkar wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING); 1158aa75f4d3SHarshad Shirwadkar #else 1159aa75f4d3SHarshad Shirwadkar wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING); 1160aa75f4d3SHarshad Shirwadkar #endif 1161aa75f4d3SHarshad Shirwadkar } 1162aa75f4d3SHarshad Shirwadkar 1163aa75f4d3SHarshad Shirwadkar while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) { 1164aa75f4d3SHarshad Shirwadkar fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN], 1165aa75f4d3SHarshad Shirwadkar struct ext4_fc_dentry_update, 1166aa75f4d3SHarshad Shirwadkar fcd_list); 1167aa75f4d3SHarshad Shirwadkar list_del_init(&fc_dentry->fcd_list); 1168aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 1169aa75f4d3SHarshad Shirwadkar 1170aa75f4d3SHarshad Shirwadkar if (fc_dentry->fcd_name.name && 1171aa75f4d3SHarshad Shirwadkar fc_dentry->fcd_name.len > DNAME_INLINE_LEN) 1172aa75f4d3SHarshad Shirwadkar kfree(fc_dentry->fcd_name.name); 1173aa75f4d3SHarshad Shirwadkar kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); 1174aa75f4d3SHarshad Shirwadkar spin_lock(&sbi->s_fc_lock); 1175aa75f4d3SHarshad Shirwadkar } 1176aa75f4d3SHarshad Shirwadkar 1177aa75f4d3SHarshad Shirwadkar list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING], 1178aa75f4d3SHarshad Shirwadkar &sbi->s_fc_dentry_q[FC_Q_MAIN]); 1179aa75f4d3SHarshad Shirwadkar list_splice_init(&sbi->s_fc_q[FC_Q_STAGING], 1180aa75f4d3SHarshad Shirwadkar &sbi->s_fc_q[FC_Q_STAGING]); 1181aa75f4d3SHarshad Shirwadkar 1182aa75f4d3SHarshad Shirwadkar sbi->s_mount_state &= ~EXT4_FC_COMMITTING; 1183aa75f4d3SHarshad Shirwadkar sbi->s_mount_state &= ~EXT4_FC_INELIGIBLE; 1184aa75f4d3SHarshad Shirwadkar 1185aa75f4d3SHarshad Shirwadkar if (full) 1186aa75f4d3SHarshad Shirwadkar sbi->s_fc_bytes = 0; 1187aa75f4d3SHarshad Shirwadkar spin_unlock(&sbi->s_fc_lock); 1188aa75f4d3SHarshad Shirwadkar trace_ext4_fc_stats(sb); 1189ff780b91SHarshad Shirwadkar } 11906866d7b3SHarshad Shirwadkar 1191*5b849b5fSHarshad Shirwadkar /* 1192*5b849b5fSHarshad Shirwadkar * Main recovery path entry point. 1193*5b849b5fSHarshad Shirwadkar */ 1194*5b849b5fSHarshad Shirwadkar static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh, 1195*5b849b5fSHarshad Shirwadkar enum passtype pass, int off, tid_t expected_tid) 1196*5b849b5fSHarshad Shirwadkar { 1197*5b849b5fSHarshad Shirwadkar return 0; 1198*5b849b5fSHarshad Shirwadkar } 1199*5b849b5fSHarshad Shirwadkar 12006866d7b3SHarshad Shirwadkar void ext4_fc_init(struct super_block *sb, journal_t *journal) 12016866d7b3SHarshad Shirwadkar { 1202*5b849b5fSHarshad Shirwadkar /* 1203*5b849b5fSHarshad Shirwadkar * We set replay callback even if fast commit disabled because we may 1204*5b849b5fSHarshad Shirwadkar * could still have fast commit blocks that need to be replayed even if 1205*5b849b5fSHarshad Shirwadkar * fast commit has now been turned off. 1206*5b849b5fSHarshad Shirwadkar */ 1207*5b849b5fSHarshad Shirwadkar journal->j_fc_replay_callback = ext4_fc_replay; 12086866d7b3SHarshad Shirwadkar if (!test_opt2(sb, JOURNAL_FAST_COMMIT)) 12096866d7b3SHarshad Shirwadkar return; 1210ff780b91SHarshad Shirwadkar journal->j_fc_cleanup_callback = ext4_fc_cleanup; 12116866d7b3SHarshad Shirwadkar if (jbd2_fc_init(journal, EXT4_NUM_FC_BLKS)) { 12126866d7b3SHarshad Shirwadkar pr_warn("Error while enabling fast commits, turning off."); 12136866d7b3SHarshad Shirwadkar ext4_clear_feature_fast_commit(sb); 12146866d7b3SHarshad Shirwadkar } 12156866d7b3SHarshad Shirwadkar } 1216aa75f4d3SHarshad Shirwadkar 1217aa75f4d3SHarshad Shirwadkar int __init ext4_fc_init_dentry_cache(void) 1218aa75f4d3SHarshad Shirwadkar { 1219aa75f4d3SHarshad Shirwadkar ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update, 1220aa75f4d3SHarshad Shirwadkar SLAB_RECLAIM_ACCOUNT); 1221aa75f4d3SHarshad Shirwadkar 1222aa75f4d3SHarshad Shirwadkar if (ext4_fc_dentry_cachep == NULL) 1223aa75f4d3SHarshad Shirwadkar return -ENOMEM; 1224aa75f4d3SHarshad Shirwadkar 1225aa75f4d3SHarshad Shirwadkar return 0; 1226aa75f4d3SHarshad Shirwadkar } 1227