10b61f8a4SDave Chinner // SPDX-License-Identifier: GPL-2.0 21da177e4SLinus Torvalds /* 37b718769SNathan Scott * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. 4c7e8f268SDavid Chinner * Copyright (c) 2008 Dave Chinner 57b718769SNathan Scott * All Rights Reserved. 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds #include "xfs.h" 8a844f451SNathan Scott #include "xfs_fs.h" 95467b34bSDarrick J. Wong #include "xfs_shared.h" 104fb6e8adSChristoph Hellwig #include "xfs_format.h" 11239880efSDave Chinner #include "xfs_log_format.h" 12239880efSDave Chinner #include "xfs_trans_resv.h" 131da177e4SLinus Torvalds #include "xfs_mount.h" 14239880efSDave Chinner #include "xfs_trans.h" 151da177e4SLinus Torvalds #include "xfs_trans_priv.h" 169e4c109aSChristoph Hellwig #include "xfs_trace.h" 17e9e899a2SDarrick J. Wong #include "xfs_errortag.h" 181da177e4SLinus Torvalds #include "xfs_error.h" 19239880efSDave Chinner #include "xfs_log.h" 201da177e4SLinus Torvalds 211da177e4SLinus Torvalds #ifdef DEBUG 22cd4a3c50SDave Chinner /* 23cd4a3c50SDave Chinner * Check that the list is sorted as it should be. 24d686d12dSDave Chinner * 25d686d12dSDave Chinner * Called with the ail lock held, but we don't want to assert fail with it 26d686d12dSDave Chinner * held otherwise we'll lock everything up and won't be able to debug the 27d686d12dSDave Chinner * cause. Hence we sample and check the state under the AIL lock and return if 28d686d12dSDave Chinner * everything is fine, otherwise we drop the lock and run the ASSERT checks. 29d686d12dSDave Chinner * Asserts may not be fatal, so pick the lock back up and continue onwards. 30cd4a3c50SDave Chinner */ 31cd4a3c50SDave Chinner STATIC void 32cd4a3c50SDave Chinner xfs_ail_check( 33cd4a3c50SDave Chinner struct xfs_ail *ailp, 34d686d12dSDave Chinner struct xfs_log_item *lip) 35daebba1bSJules Irenge __must_hold(&ailp->ail_lock) 36cd4a3c50SDave Chinner { 37d686d12dSDave Chinner struct xfs_log_item *prev_lip; 38d686d12dSDave Chinner struct xfs_log_item *next_lip; 39d686d12dSDave Chinner xfs_lsn_t prev_lsn = NULLCOMMITLSN; 40d686d12dSDave Chinner xfs_lsn_t next_lsn = NULLCOMMITLSN; 41d686d12dSDave Chinner xfs_lsn_t lsn; 42d686d12dSDave Chinner bool in_ail; 43d686d12dSDave Chinner 44cd4a3c50SDave Chinner 4557e80956SMatthew Wilcox if (list_empty(&ailp->ail_head)) 46cd4a3c50SDave Chinner return; 47cd4a3c50SDave Chinner 48cd4a3c50SDave Chinner /* 49d686d12dSDave Chinner * Sample then check the next and previous entries are valid. 50cd4a3c50SDave Chinner */ 51d686d12dSDave Chinner in_ail = test_bit(XFS_LI_IN_AIL, &lip->li_flags); 52d686d12dSDave Chinner prev_lip = list_entry(lip->li_ail.prev, struct xfs_log_item, li_ail); 5357e80956SMatthew Wilcox if (&prev_lip->li_ail != &ailp->ail_head) 54d686d12dSDave Chinner prev_lsn = prev_lip->li_lsn; 55d686d12dSDave Chinner next_lip = list_entry(lip->li_ail.next, struct xfs_log_item, li_ail); 56d686d12dSDave Chinner if (&next_lip->li_ail != &ailp->ail_head) 57d686d12dSDave Chinner next_lsn = next_lip->li_lsn; 58d686d12dSDave Chinner lsn = lip->li_lsn; 59cd4a3c50SDave Chinner 60d686d12dSDave Chinner if (in_ail && 61d686d12dSDave Chinner (prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0) && 62d686d12dSDave Chinner (next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0)) 63d686d12dSDave Chinner return; 64cd4a3c50SDave Chinner 65d686d12dSDave Chinner spin_unlock(&ailp->ail_lock); 66d686d12dSDave Chinner ASSERT(in_ail); 67d686d12dSDave Chinner ASSERT(prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0); 68d686d12dSDave Chinner ASSERT(next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0); 69d686d12dSDave Chinner spin_lock(&ailp->ail_lock); 70cd4a3c50SDave Chinner } 71cd4a3c50SDave Chinner #else /* !DEBUG */ 72de08dbc1SDavid Chinner #define xfs_ail_check(a,l) 731da177e4SLinus Torvalds #endif /* DEBUG */ 741da177e4SLinus Torvalds 75cd4a3c50SDave Chinner /* 76fd074841SDave Chinner * Return a pointer to the last item in the AIL. If the AIL is empty, then 77fd074841SDave Chinner * return NULL. 78fd074841SDave Chinner */ 79efe2330fSChristoph Hellwig static struct xfs_log_item * 80fd074841SDave Chinner xfs_ail_max( 81fd074841SDave Chinner struct xfs_ail *ailp) 82fd074841SDave Chinner { 8357e80956SMatthew Wilcox if (list_empty(&ailp->ail_head)) 84fd074841SDave Chinner return NULL; 85fd074841SDave Chinner 86efe2330fSChristoph Hellwig return list_entry(ailp->ail_head.prev, struct xfs_log_item, li_ail); 87fd074841SDave Chinner } 88fd074841SDave Chinner 89fd074841SDave Chinner /* 90cd4a3c50SDave Chinner * Return a pointer to the item which follows the given item in the AIL. If 91cd4a3c50SDave Chinner * the given item is the last item in the list, then return NULL. 92cd4a3c50SDave Chinner */ 93efe2330fSChristoph Hellwig static struct xfs_log_item * 94cd4a3c50SDave Chinner xfs_ail_next( 95cd4a3c50SDave Chinner struct xfs_ail *ailp, 96efe2330fSChristoph Hellwig struct xfs_log_item *lip) 97cd4a3c50SDave Chinner { 9857e80956SMatthew Wilcox if (lip->li_ail.next == &ailp->ail_head) 99cd4a3c50SDave Chinner return NULL; 100cd4a3c50SDave Chinner 101efe2330fSChristoph Hellwig return list_first_entry(&lip->li_ail, struct xfs_log_item, li_ail); 102cd4a3c50SDave Chinner } 103cd4a3c50SDave Chinner 104cd4a3c50SDave Chinner /* 105cd4a3c50SDave Chinner * This is called by the log manager code to determine the LSN of the tail of 106cd4a3c50SDave Chinner * the log. This is exactly the LSN of the first item in the AIL. If the AIL 107cd4a3c50SDave Chinner * is empty, then this function returns 0. 1081da177e4SLinus Torvalds * 109cd4a3c50SDave Chinner * We need the AIL lock in order to get a coherent read of the lsn of the last 110cd4a3c50SDave Chinner * item in the AIL. 1111da177e4SLinus Torvalds */ 1128eb807bdSDave Chinner static xfs_lsn_t 1138eb807bdSDave Chinner __xfs_ail_min_lsn( 1148eb807bdSDave Chinner struct xfs_ail *ailp) 1158eb807bdSDave Chinner { 1168eb807bdSDave Chinner struct xfs_log_item *lip = xfs_ail_min(ailp); 1178eb807bdSDave Chinner 1188eb807bdSDave Chinner if (lip) 1198eb807bdSDave Chinner return lip->li_lsn; 1208eb807bdSDave Chinner return 0; 1218eb807bdSDave Chinner } 1228eb807bdSDave Chinner 1231da177e4SLinus Torvalds xfs_lsn_t 124fd074841SDave Chinner xfs_ail_min_lsn( 1255b00f14fSDavid Chinner struct xfs_ail *ailp) 1261da177e4SLinus Torvalds { 1278eb807bdSDave Chinner xfs_lsn_t lsn; 1281da177e4SLinus Torvalds 12957e80956SMatthew Wilcox spin_lock(&ailp->ail_lock); 1308eb807bdSDave Chinner lsn = __xfs_ail_min_lsn(ailp); 13157e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock); 1321da177e4SLinus Torvalds 1331da177e4SLinus Torvalds return lsn; 1341da177e4SLinus Torvalds } 1351da177e4SLinus Torvalds 1361da177e4SLinus Torvalds /* 137fd074841SDave Chinner * Return the maximum lsn held in the AIL, or zero if the AIL is empty. 138fd074841SDave Chinner */ 139fd074841SDave Chinner static xfs_lsn_t 140fd074841SDave Chinner xfs_ail_max_lsn( 141fd074841SDave Chinner struct xfs_ail *ailp) 142fd074841SDave Chinner { 143fd074841SDave Chinner xfs_lsn_t lsn = 0; 144efe2330fSChristoph Hellwig struct xfs_log_item *lip; 145fd074841SDave Chinner 14657e80956SMatthew Wilcox spin_lock(&ailp->ail_lock); 147fd074841SDave Chinner lip = xfs_ail_max(ailp); 148fd074841SDave Chinner if (lip) 149fd074841SDave Chinner lsn = lip->li_lsn; 15057e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock); 151fd074841SDave Chinner 152fd074841SDave Chinner return lsn; 153fd074841SDave Chinner } 154fd074841SDave Chinner 155fd074841SDave Chinner /* 156af3e4022SDave Chinner * The cursor keeps track of where our current traversal is up to by tracking 157af3e4022SDave Chinner * the next item in the list for us. However, for this to be safe, removing an 158af3e4022SDave Chinner * object from the AIL needs to invalidate any cursor that points to it. hence 159af3e4022SDave Chinner * the traversal cursor needs to be linked to the struct xfs_ail so that 160af3e4022SDave Chinner * deletion can search all the active cursors for invalidation. 16127d8d5feSDavid Chinner */ 1625b00f14fSDavid Chinner STATIC void 16327d8d5feSDavid Chinner xfs_trans_ail_cursor_init( 16427d8d5feSDavid Chinner struct xfs_ail *ailp, 16527d8d5feSDavid Chinner struct xfs_ail_cursor *cur) 16627d8d5feSDavid Chinner { 16727d8d5feSDavid Chinner cur->item = NULL; 16857e80956SMatthew Wilcox list_add_tail(&cur->list, &ailp->ail_cursors); 16927d8d5feSDavid Chinner } 17027d8d5feSDavid Chinner 17127d8d5feSDavid Chinner /* 172af3e4022SDave Chinner * Get the next item in the traversal and advance the cursor. If the cursor 173af3e4022SDave Chinner * was invalidated (indicated by a lip of 1), restart the traversal. 17427d8d5feSDavid Chinner */ 1755b00f14fSDavid Chinner struct xfs_log_item * 17627d8d5feSDavid Chinner xfs_trans_ail_cursor_next( 17727d8d5feSDavid Chinner struct xfs_ail *ailp, 17827d8d5feSDavid Chinner struct xfs_ail_cursor *cur) 17927d8d5feSDavid Chinner { 18027d8d5feSDavid Chinner struct xfs_log_item *lip = cur->item; 18127d8d5feSDavid Chinner 182db9d67d6SChristoph Hellwig if ((uintptr_t)lip & 1) 18327d8d5feSDavid Chinner lip = xfs_ail_min(ailp); 18416b59029SDave Chinner if (lip) 18516b59029SDave Chinner cur->item = xfs_ail_next(ailp, lip); 18627d8d5feSDavid Chinner return lip; 18727d8d5feSDavid Chinner } 18827d8d5feSDavid Chinner 18927d8d5feSDavid Chinner /* 190af3e4022SDave Chinner * When the traversal is complete, we need to remove the cursor from the list 191af3e4022SDave Chinner * of traversing cursors. 19227d8d5feSDavid Chinner */ 19327d8d5feSDavid Chinner void 19427d8d5feSDavid Chinner xfs_trans_ail_cursor_done( 195af3e4022SDave Chinner struct xfs_ail_cursor *cur) 19627d8d5feSDavid Chinner { 197af3e4022SDave Chinner cur->item = NULL; 198af3e4022SDave Chinner list_del_init(&cur->list); 19927d8d5feSDavid Chinner } 20027d8d5feSDavid Chinner 20127d8d5feSDavid Chinner /* 202af3e4022SDave Chinner * Invalidate any cursor that is pointing to this item. This is called when an 203af3e4022SDave Chinner * item is removed from the AIL. Any cursor pointing to this object is now 204af3e4022SDave Chinner * invalid and the traversal needs to be terminated so it doesn't reference a 205af3e4022SDave Chinner * freed object. We set the low bit of the cursor item pointer so we can 206af3e4022SDave Chinner * distinguish between an invalidation and the end of the list when getting the 207af3e4022SDave Chinner * next item from the cursor. 2085b00f14fSDavid Chinner */ 2095b00f14fSDavid Chinner STATIC void 2105b00f14fSDavid Chinner xfs_trans_ail_cursor_clear( 2115b00f14fSDavid Chinner struct xfs_ail *ailp, 2125b00f14fSDavid Chinner struct xfs_log_item *lip) 2135b00f14fSDavid Chinner { 2145b00f14fSDavid Chinner struct xfs_ail_cursor *cur; 2155b00f14fSDavid Chinner 21657e80956SMatthew Wilcox list_for_each_entry(cur, &ailp->ail_cursors, list) { 2175b00f14fSDavid Chinner if (cur->item == lip) 2185b00f14fSDavid Chinner cur->item = (struct xfs_log_item *) 219db9d67d6SChristoph Hellwig ((uintptr_t)cur->item | 1); 2205b00f14fSDavid Chinner } 2215b00f14fSDavid Chinner } 2225b00f14fSDavid Chinner 2235b00f14fSDavid Chinner /* 22416b59029SDave Chinner * Find the first item in the AIL with the given @lsn by searching in ascending 22516b59029SDave Chinner * LSN order and initialise the cursor to point to the next item for a 22616b59029SDave Chinner * ascending traversal. Pass a @lsn of zero to initialise the cursor to the 22716b59029SDave Chinner * first item in the AIL. Returns NULL if the list is empty. 228249a8c11SDavid Chinner */ 229efe2330fSChristoph Hellwig struct xfs_log_item * 2305b00f14fSDavid Chinner xfs_trans_ail_cursor_first( 23127d8d5feSDavid Chinner struct xfs_ail *ailp, 23227d8d5feSDavid Chinner struct xfs_ail_cursor *cur, 233249a8c11SDavid Chinner xfs_lsn_t lsn) 234249a8c11SDavid Chinner { 235efe2330fSChristoph Hellwig struct xfs_log_item *lip; 236249a8c11SDavid Chinner 2375b00f14fSDavid Chinner xfs_trans_ail_cursor_init(ailp, cur); 23816b59029SDave Chinner 23916b59029SDave Chinner if (lsn == 0) { 24027d8d5feSDavid Chinner lip = xfs_ail_min(ailp); 2415b00f14fSDavid Chinner goto out; 24216b59029SDave Chinner } 243249a8c11SDavid Chinner 24457e80956SMatthew Wilcox list_for_each_entry(lip, &ailp->ail_head, li_ail) { 2455b00f14fSDavid Chinner if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0) 2467ee49acfSDavid Chinner goto out; 2475b00f14fSDavid Chinner } 24816b59029SDave Chinner return NULL; 24916b59029SDave Chinner 2505b00f14fSDavid Chinner out: 25116b59029SDave Chinner if (lip) 25216b59029SDave Chinner cur->item = xfs_ail_next(ailp, lip); 253249a8c11SDavid Chinner return lip; 254249a8c11SDavid Chinner } 255535f6b37SJosef 'Jeff' Sipek 2561d8c95a3SDave Chinner static struct xfs_log_item * 2571d8c95a3SDave Chinner __xfs_trans_ail_cursor_last( 2581d8c95a3SDave Chinner struct xfs_ail *ailp, 2591d8c95a3SDave Chinner xfs_lsn_t lsn) 2601d8c95a3SDave Chinner { 261efe2330fSChristoph Hellwig struct xfs_log_item *lip; 2621d8c95a3SDave Chinner 26357e80956SMatthew Wilcox list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) { 2641d8c95a3SDave Chinner if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0) 2651d8c95a3SDave Chinner return lip; 2661d8c95a3SDave Chinner } 2671d8c95a3SDave Chinner return NULL; 2681d8c95a3SDave Chinner } 2691d8c95a3SDave Chinner 2701d8c95a3SDave Chinner /* 27116b59029SDave Chinner * Find the last item in the AIL with the given @lsn by searching in descending 27216b59029SDave Chinner * LSN order and initialise the cursor to point to that item. If there is no 27316b59029SDave Chinner * item with the value of @lsn, then it sets the cursor to the last item with an 27416b59029SDave Chinner * LSN lower than @lsn. Returns NULL if the list is empty. 2751d8c95a3SDave Chinner */ 2761d8c95a3SDave Chinner struct xfs_log_item * 2771d8c95a3SDave Chinner xfs_trans_ail_cursor_last( 2781d8c95a3SDave Chinner struct xfs_ail *ailp, 2791d8c95a3SDave Chinner struct xfs_ail_cursor *cur, 2801d8c95a3SDave Chinner xfs_lsn_t lsn) 2811d8c95a3SDave Chinner { 2821d8c95a3SDave Chinner xfs_trans_ail_cursor_init(ailp, cur); 2831d8c95a3SDave Chinner cur->item = __xfs_trans_ail_cursor_last(ailp, lsn); 2841d8c95a3SDave Chinner return cur->item; 2851d8c95a3SDave Chinner } 2861d8c95a3SDave Chinner 2871d8c95a3SDave Chinner /* 28816b59029SDave Chinner * Splice the log item list into the AIL at the given LSN. We splice to the 2891d8c95a3SDave Chinner * tail of the given LSN to maintain insert order for push traversals. The 2901d8c95a3SDave Chinner * cursor is optional, allowing repeated updates to the same LSN to avoid 291e44f4112SAlex Elder * repeated traversals. This should not be called with an empty list. 292cd4a3c50SDave Chinner */ 293cd4a3c50SDave Chinner static void 294cd4a3c50SDave Chinner xfs_ail_splice( 295cd4a3c50SDave Chinner struct xfs_ail *ailp, 2961d8c95a3SDave Chinner struct xfs_ail_cursor *cur, 297cd4a3c50SDave Chinner struct list_head *list, 298cd4a3c50SDave Chinner xfs_lsn_t lsn) 299cd4a3c50SDave Chinner { 300e44f4112SAlex Elder struct xfs_log_item *lip; 301e44f4112SAlex Elder 302e44f4112SAlex Elder ASSERT(!list_empty(list)); 303cd4a3c50SDave Chinner 3041d8c95a3SDave Chinner /* 305e44f4112SAlex Elder * Use the cursor to determine the insertion point if one is 306e44f4112SAlex Elder * provided. If not, or if the one we got is not valid, 307e44f4112SAlex Elder * find the place in the AIL where the items belong. 3081d8c95a3SDave Chinner */ 309e44f4112SAlex Elder lip = cur ? cur->item : NULL; 310db9d67d6SChristoph Hellwig if (!lip || (uintptr_t)lip & 1) 3111d8c95a3SDave Chinner lip = __xfs_trans_ail_cursor_last(ailp, lsn); 3121d8c95a3SDave Chinner 313e44f4112SAlex Elder /* 314e44f4112SAlex Elder * If a cursor is provided, we know we're processing the AIL 315e44f4112SAlex Elder * in lsn order, and future items to be spliced in will 316e44f4112SAlex Elder * follow the last one being inserted now. Update the 317e44f4112SAlex Elder * cursor to point to that last item, now while we have a 318e44f4112SAlex Elder * reliable pointer to it. 319e44f4112SAlex Elder */ 3201d8c95a3SDave Chinner if (cur) 321e44f4112SAlex Elder cur->item = list_entry(list->prev, struct xfs_log_item, li_ail); 322cd4a3c50SDave Chinner 3231d8c95a3SDave Chinner /* 324e44f4112SAlex Elder * Finally perform the splice. Unless the AIL was empty, 325e44f4112SAlex Elder * lip points to the item in the AIL _after_ which the new 326e44f4112SAlex Elder * items should go. If lip is null the AIL was empty, so 327e44f4112SAlex Elder * the new items go at the head of the AIL. 3281d8c95a3SDave Chinner */ 329e44f4112SAlex Elder if (lip) 3301d8c95a3SDave Chinner list_splice(list, &lip->li_ail); 331e44f4112SAlex Elder else 33257e80956SMatthew Wilcox list_splice(list, &ailp->ail_head); 333cd4a3c50SDave Chinner } 334cd4a3c50SDave Chinner 335cd4a3c50SDave Chinner /* 336cd4a3c50SDave Chinner * Delete the given item from the AIL. Return a pointer to the item. 337cd4a3c50SDave Chinner */ 338cd4a3c50SDave Chinner static void 339cd4a3c50SDave Chinner xfs_ail_delete( 340cd4a3c50SDave Chinner struct xfs_ail *ailp, 341efe2330fSChristoph Hellwig struct xfs_log_item *lip) 342cd4a3c50SDave Chinner { 343cd4a3c50SDave Chinner xfs_ail_check(ailp, lip); 344cd4a3c50SDave Chinner list_del(&lip->li_ail); 345cd4a3c50SDave Chinner xfs_trans_ail_cursor_clear(ailp, lip); 346cd4a3c50SDave Chinner } 347cd4a3c50SDave Chinner 3487f4d01f3SBrian Foster static inline uint 3497f4d01f3SBrian Foster xfsaild_push_item( 3507f4d01f3SBrian Foster struct xfs_ail *ailp, 3517f4d01f3SBrian Foster struct xfs_log_item *lip) 3527f4d01f3SBrian Foster { 3537f4d01f3SBrian Foster /* 3547f4d01f3SBrian Foster * If log item pinning is enabled, skip the push and track the item as 3557f4d01f3SBrian Foster * pinned. This can help induce head-behind-tail conditions. 3567f4d01f3SBrian Foster */ 35757e80956SMatthew Wilcox if (XFS_TEST_ERROR(false, ailp->ail_mount, XFS_ERRTAG_LOG_ITEM_PIN)) 3587f4d01f3SBrian Foster return XFS_ITEM_PINNED; 3597f4d01f3SBrian Foster 360e8b78db7SChristoph Hellwig /* 361e8b78db7SChristoph Hellwig * Consider the item pinned if a push callback is not defined so the 362e8b78db7SChristoph Hellwig * caller will force the log. This should only happen for intent items 363e8b78db7SChristoph Hellwig * as they are unpinned once the associated done item is committed to 364e8b78db7SChristoph Hellwig * the on-disk log. 365e8b78db7SChristoph Hellwig */ 366e8b78db7SChristoph Hellwig if (!lip->li_ops->iop_push) 367e8b78db7SChristoph Hellwig return XFS_ITEM_PINNED; 36857e80956SMatthew Wilcox return lip->li_ops->iop_push(lip, &ailp->ail_buf_list); 3697f4d01f3SBrian Foster } 3707f4d01f3SBrian Foster 3710030807cSChristoph Hellwig static long 3720030807cSChristoph Hellwig xfsaild_push( 3730030807cSChristoph Hellwig struct xfs_ail *ailp) 374249a8c11SDavid Chinner { 37557e80956SMatthew Wilcox xfs_mount_t *mp = ailp->ail_mount; 376af3e4022SDave Chinner struct xfs_ail_cursor cur; 377efe2330fSChristoph Hellwig struct xfs_log_item *lip; 3789e7004e7SDave Chinner xfs_lsn_t lsn; 379fe0da767SDave Chinner xfs_lsn_t target; 38043ff2122SChristoph Hellwig long tout; 3819e7004e7SDave Chinner int stuck = 0; 38243ff2122SChristoph Hellwig int flushing = 0; 3839e7004e7SDave Chinner int count = 0; 3841da177e4SLinus Torvalds 385670ce93fSDave Chinner /* 38643ff2122SChristoph Hellwig * If we encountered pinned items or did not finish writing out all 38743ff2122SChristoph Hellwig * buffers the last time we ran, force the log first and wait for it 38843ff2122SChristoph Hellwig * before pushing again. 389670ce93fSDave Chinner */ 39057e80956SMatthew Wilcox if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 && 39157e80956SMatthew Wilcox (!list_empty_careful(&ailp->ail_buf_list) || 39243ff2122SChristoph Hellwig xfs_ail_min_lsn(ailp))) { 39357e80956SMatthew Wilcox ailp->ail_log_flush = 0; 39443ff2122SChristoph Hellwig 395ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_flush); 396670ce93fSDave Chinner xfs_log_force(mp, XFS_LOG_SYNC); 397670ce93fSDave Chinner } 398670ce93fSDave Chinner 39957e80956SMatthew Wilcox spin_lock(&ailp->ail_lock); 4008375f922SBrian Foster 40157e80956SMatthew Wilcox /* barrier matches the ail_target update in xfs_ail_push() */ 4028375f922SBrian Foster smp_rmb(); 40357e80956SMatthew Wilcox target = ailp->ail_target; 40457e80956SMatthew Wilcox ailp->ail_target_prev = target; 4058375f922SBrian Foster 40657e80956SMatthew Wilcox lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn); 407211e4d43SChristoph Hellwig if (!lip) { 4081da177e4SLinus Torvalds /* 40943ff2122SChristoph Hellwig * If the AIL is empty or our push has reached the end we are 41043ff2122SChristoph Hellwig * done now. 4111da177e4SLinus Torvalds */ 412e4a1e29cSEric Sandeen xfs_trans_ail_cursor_done(&cur); 41357e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock); 4149e7004e7SDave Chinner goto out_done; 4151da177e4SLinus Torvalds } 4161da177e4SLinus Torvalds 417ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail); 4181da177e4SLinus Torvalds 419249a8c11SDavid Chinner lsn = lip->li_lsn; 42050e86686SDave Chinner while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) { 421249a8c11SDavid Chinner int lock_result; 42243ff2122SChristoph Hellwig 423249a8c11SDavid Chinner /* 424904c17e6SDave Chinner * Note that iop_push may unlock and reacquire the AIL lock. We 42543ff2122SChristoph Hellwig * rely on the AIL cursor implementation to be able to deal with 42643ff2122SChristoph Hellwig * the dropped lock. 4271da177e4SLinus Torvalds */ 4287f4d01f3SBrian Foster lock_result = xfsaild_push_item(ailp, lip); 4291da177e4SLinus Torvalds switch (lock_result) { 4301da177e4SLinus Torvalds case XFS_ITEM_SUCCESS: 431ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_success); 4329e4c109aSChristoph Hellwig trace_xfs_ail_push(lip); 4339e4c109aSChristoph Hellwig 43457e80956SMatthew Wilcox ailp->ail_last_pushed_lsn = lsn; 4351da177e4SLinus Torvalds break; 4361da177e4SLinus Torvalds 43743ff2122SChristoph Hellwig case XFS_ITEM_FLUSHING: 43843ff2122SChristoph Hellwig /* 439cf085a1bSJoe Perches * The item or its backing buffer is already being 44043ff2122SChristoph Hellwig * flushed. The typical reason for that is that an 44143ff2122SChristoph Hellwig * inode buffer is locked because we already pushed the 44243ff2122SChristoph Hellwig * updates to it as part of inode clustering. 44343ff2122SChristoph Hellwig * 44443ff2122SChristoph Hellwig * We do not want to to stop flushing just because lots 445cf085a1bSJoe Perches * of items are already being flushed, but we need to 44643ff2122SChristoph Hellwig * re-try the flushing relatively soon if most of the 447cf085a1bSJoe Perches * AIL is being flushed. 44843ff2122SChristoph Hellwig */ 449ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_flushing); 45043ff2122SChristoph Hellwig trace_xfs_ail_flushing(lip); 45117b38471SChristoph Hellwig 45243ff2122SChristoph Hellwig flushing++; 45357e80956SMatthew Wilcox ailp->ail_last_pushed_lsn = lsn; 4541da177e4SLinus Torvalds break; 4551da177e4SLinus Torvalds 4561da177e4SLinus Torvalds case XFS_ITEM_PINNED: 457ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_pinned); 4589e4c109aSChristoph Hellwig trace_xfs_ail_pinned(lip); 4599e4c109aSChristoph Hellwig 460249a8c11SDavid Chinner stuck++; 46157e80956SMatthew Wilcox ailp->ail_log_flush++; 4621da177e4SLinus Torvalds break; 4631da177e4SLinus Torvalds case XFS_ITEM_LOCKED: 464ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_locked); 4659e4c109aSChristoph Hellwig trace_xfs_ail_locked(lip); 46643ff2122SChristoph Hellwig 467249a8c11SDavid Chinner stuck++; 4681da177e4SLinus Torvalds break; 4691da177e4SLinus Torvalds default: 4701da177e4SLinus Torvalds ASSERT(0); 4711da177e4SLinus Torvalds break; 4721da177e4SLinus Torvalds } 4731da177e4SLinus Torvalds 474249a8c11SDavid Chinner count++; 475249a8c11SDavid Chinner 476249a8c11SDavid Chinner /* 477249a8c11SDavid Chinner * Are there too many items we can't do anything with? 47843ff2122SChristoph Hellwig * 479249a8c11SDavid Chinner * If we we are skipping too many items because we can't flush 480249a8c11SDavid Chinner * them or they are already being flushed, we back off and 481249a8c11SDavid Chinner * given them time to complete whatever operation is being 482249a8c11SDavid Chinner * done. i.e. remove pressure from the AIL while we can't make 483249a8c11SDavid Chinner * progress so traversals don't slow down further inserts and 484249a8c11SDavid Chinner * removals to/from the AIL. 485249a8c11SDavid Chinner * 486249a8c11SDavid Chinner * The value of 100 is an arbitrary magic number based on 487249a8c11SDavid Chinner * observation. 488249a8c11SDavid Chinner */ 489249a8c11SDavid Chinner if (stuck > 100) 490249a8c11SDavid Chinner break; 491249a8c11SDavid Chinner 492af3e4022SDave Chinner lip = xfs_trans_ail_cursor_next(ailp, &cur); 493249a8c11SDavid Chinner if (lip == NULL) 494249a8c11SDavid Chinner break; 495249a8c11SDavid Chinner lsn = lip->li_lsn; 4961da177e4SLinus Torvalds } 497e4a1e29cSEric Sandeen xfs_trans_ail_cursor_done(&cur); 49857e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock); 4991da177e4SLinus Torvalds 50057e80956SMatthew Wilcox if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list)) 50157e80956SMatthew Wilcox ailp->ail_log_flush++; 502d808f617SDave Chinner 50343ff2122SChristoph Hellwig if (!count || XFS_LSN_CMP(lsn, target) >= 0) { 5049e7004e7SDave Chinner out_done: 505249a8c11SDavid Chinner /* 50643ff2122SChristoph Hellwig * We reached the target or the AIL is empty, so wait a bit 50743ff2122SChristoph Hellwig * longer for I/O to complete and remove pushed items from the 50843ff2122SChristoph Hellwig * AIL before we start the next scan from the start of the AIL. 509249a8c11SDavid Chinner */ 510453eac8aSDave Chinner tout = 50; 51157e80956SMatthew Wilcox ailp->ail_last_pushed_lsn = 0; 51243ff2122SChristoph Hellwig } else if (((stuck + flushing) * 100) / count > 90) { 513249a8c11SDavid Chinner /* 51443ff2122SChristoph Hellwig * Either there is a lot of contention on the AIL or we are 51543ff2122SChristoph Hellwig * stuck due to operations in progress. "Stuck" in this case 51643ff2122SChristoph Hellwig * is defined as >90% of the items we tried to push were stuck. 517249a8c11SDavid Chinner * 518249a8c11SDavid Chinner * Backoff a bit more to allow some I/O to complete before 51943ff2122SChristoph Hellwig * restarting from the start of the AIL. This prevents us from 52043ff2122SChristoph Hellwig * spinning on the same items, and if they are pinned will all 52143ff2122SChristoph Hellwig * the restart to issue a log force to unpin the stuck items. 522249a8c11SDavid Chinner */ 523453eac8aSDave Chinner tout = 20; 52457e80956SMatthew Wilcox ailp->ail_last_pushed_lsn = 0; 52543ff2122SChristoph Hellwig } else { 52643ff2122SChristoph Hellwig /* 52743ff2122SChristoph Hellwig * Assume we have more work to do in a short while. 52843ff2122SChristoph Hellwig */ 52943ff2122SChristoph Hellwig tout = 10; 530453eac8aSDave Chinner } 5311da177e4SLinus Torvalds 5320030807cSChristoph Hellwig return tout; 5330030807cSChristoph Hellwig } 5340030807cSChristoph Hellwig 5350030807cSChristoph Hellwig static int 5360030807cSChristoph Hellwig xfsaild( 5370030807cSChristoph Hellwig void *data) 5380030807cSChristoph Hellwig { 5390030807cSChristoph Hellwig struct xfs_ail *ailp = data; 5400030807cSChristoph Hellwig long tout = 0; /* milliseconds */ 54110a98cb1SEric Biggers unsigned int noreclaim_flag; 5420030807cSChristoph Hellwig 54310a98cb1SEric Biggers noreclaim_flag = memalloc_noreclaim_save(); 54418f1df4eSMichal Hocko set_freezable(); 54543ff2122SChristoph Hellwig 5460bd89676SHou Tao while (1) { 5470030807cSChristoph Hellwig if (tout && tout <= 20) 5480bd89676SHou Tao set_current_state(TASK_KILLABLE); 5490030807cSChristoph Hellwig else 5500bd89676SHou Tao set_current_state(TASK_INTERRUPTIBLE); 5510bd89676SHou Tao 5520bd89676SHou Tao /* 553efc3289cSBrian Foster * Check kthread_should_stop() after we set the task state to 554efc3289cSBrian Foster * guarantee that we either see the stop bit and exit or the 555efc3289cSBrian Foster * task state is reset to runnable such that it's not scheduled 556efc3289cSBrian Foster * out indefinitely and detects the stop bit at next iteration. 5570bd89676SHou Tao * A memory barrier is included in above task state set to 5580bd89676SHou Tao * serialize again kthread_stop(). 5590bd89676SHou Tao */ 5600bd89676SHou Tao if (kthread_should_stop()) { 5610bd89676SHou Tao __set_current_state(TASK_RUNNING); 562efc3289cSBrian Foster 563efc3289cSBrian Foster /* 564efc3289cSBrian Foster * The caller forces out the AIL before stopping the 565efc3289cSBrian Foster * thread in the common case, which means the delwri 566efc3289cSBrian Foster * queue is drained. In the shutdown case, the queue may 567efc3289cSBrian Foster * still hold relogged buffers that haven't been 568efc3289cSBrian Foster * submitted because they were pinned since added to the 569efc3289cSBrian Foster * queue. 570efc3289cSBrian Foster * 571efc3289cSBrian Foster * Log I/O error processing stales the underlying buffer 572efc3289cSBrian Foster * and clears the delwri state, expecting the buf to be 573efc3289cSBrian Foster * removed on the next submission attempt. That won't 574efc3289cSBrian Foster * happen if we're shutting down, so this is the last 575efc3289cSBrian Foster * opportunity to release such buffers from the queue. 576efc3289cSBrian Foster */ 577efc3289cSBrian Foster ASSERT(list_empty(&ailp->ail_buf_list) || 578efc3289cSBrian Foster XFS_FORCED_SHUTDOWN(ailp->ail_mount)); 579efc3289cSBrian Foster xfs_buf_delwri_cancel(&ailp->ail_buf_list); 5800bd89676SHou Tao break; 5810bd89676SHou Tao } 5828375f922SBrian Foster 58357e80956SMatthew Wilcox spin_lock(&ailp->ail_lock); 5848375f922SBrian Foster 5858375f922SBrian Foster /* 5868375f922SBrian Foster * Idle if the AIL is empty and we are not racing with a target 5878375f922SBrian Foster * update. We check the AIL after we set the task to a sleep 58857e80956SMatthew Wilcox * state to guarantee that we either catch an ail_target update 5898375f922SBrian Foster * or that a wake_up resets the state to TASK_RUNNING. 5908375f922SBrian Foster * Otherwise, we run the risk of sleeping indefinitely. 5918375f922SBrian Foster * 59257e80956SMatthew Wilcox * The barrier matches the ail_target update in xfs_ail_push(). 5938375f922SBrian Foster */ 5948375f922SBrian Foster smp_rmb(); 5958375f922SBrian Foster if (!xfs_ail_min(ailp) && 59657e80956SMatthew Wilcox ailp->ail_target == ailp->ail_target_prev) { 59757e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock); 59818f1df4eSMichal Hocko freezable_schedule(); 5998375f922SBrian Foster tout = 0; 6008375f922SBrian Foster continue; 6018375f922SBrian Foster } 60257e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock); 6038375f922SBrian Foster 6048375f922SBrian Foster if (tout) 60518f1df4eSMichal Hocko freezable_schedule_timeout(msecs_to_jiffies(tout)); 6068375f922SBrian Foster 6078375f922SBrian Foster __set_current_state(TASK_RUNNING); 6080030807cSChristoph Hellwig 6090030807cSChristoph Hellwig try_to_freeze(); 6100030807cSChristoph Hellwig 6110030807cSChristoph Hellwig tout = xfsaild_push(ailp); 6120030807cSChristoph Hellwig } 6130030807cSChristoph Hellwig 61410a98cb1SEric Biggers memalloc_noreclaim_restore(noreclaim_flag); 6150030807cSChristoph Hellwig return 0; 6160bf6a5bdSDave Chinner } 6170bf6a5bdSDave Chinner 6180bf6a5bdSDave Chinner /* 6190bf6a5bdSDave Chinner * This routine is called to move the tail of the AIL forward. It does this by 6200bf6a5bdSDave Chinner * trying to flush items in the AIL whose lsns are below the given 6210bf6a5bdSDave Chinner * threshold_lsn. 6220bf6a5bdSDave Chinner * 6230bf6a5bdSDave Chinner * The push is run asynchronously in a workqueue, which means the caller needs 6240bf6a5bdSDave Chinner * to handle waiting on the async flush for space to become available. 6250bf6a5bdSDave Chinner * We don't want to interrupt any push that is in progress, hence we only queue 626cf085a1bSJoe Perches * work if we set the pushing bit appropriately. 6270bf6a5bdSDave Chinner * 6280bf6a5bdSDave Chinner * We do this unlocked - we only need to know whether there is anything in the 6290bf6a5bdSDave Chinner * AIL at the time we are called. We don't need to access the contents of 6300bf6a5bdSDave Chinner * any of the objects, so the lock is not needed. 6310bf6a5bdSDave Chinner */ 6320bf6a5bdSDave Chinner void 633fd074841SDave Chinner xfs_ail_push( 6340bf6a5bdSDave Chinner struct xfs_ail *ailp, 6350bf6a5bdSDave Chinner xfs_lsn_t threshold_lsn) 6360bf6a5bdSDave Chinner { 637efe2330fSChristoph Hellwig struct xfs_log_item *lip; 6380bf6a5bdSDave Chinner 6390bf6a5bdSDave Chinner lip = xfs_ail_min(ailp); 64057e80956SMatthew Wilcox if (!lip || XFS_FORCED_SHUTDOWN(ailp->ail_mount) || 64157e80956SMatthew Wilcox XFS_LSN_CMP(threshold_lsn, ailp->ail_target) <= 0) 6420bf6a5bdSDave Chinner return; 6430bf6a5bdSDave Chinner 6440bf6a5bdSDave Chinner /* 6450bf6a5bdSDave Chinner * Ensure that the new target is noticed in push code before it clears 6460bf6a5bdSDave Chinner * the XFS_AIL_PUSHING_BIT. 6470bf6a5bdSDave Chinner */ 6480bf6a5bdSDave Chinner smp_wmb(); 64957e80956SMatthew Wilcox xfs_trans_ail_copy_lsn(ailp, &ailp->ail_target, &threshold_lsn); 6500030807cSChristoph Hellwig smp_wmb(); 6510030807cSChristoph Hellwig 65257e80956SMatthew Wilcox wake_up_process(ailp->ail_task); 6530bf6a5bdSDave Chinner } 6541da177e4SLinus Torvalds 6551da177e4SLinus Torvalds /* 656fd074841SDave Chinner * Push out all items in the AIL immediately 657fd074841SDave Chinner */ 658fd074841SDave Chinner void 659fd074841SDave Chinner xfs_ail_push_all( 660fd074841SDave Chinner struct xfs_ail *ailp) 661fd074841SDave Chinner { 662fd074841SDave Chinner xfs_lsn_t threshold_lsn = xfs_ail_max_lsn(ailp); 663fd074841SDave Chinner 664fd074841SDave Chinner if (threshold_lsn) 665fd074841SDave Chinner xfs_ail_push(ailp, threshold_lsn); 666fd074841SDave Chinner } 667fd074841SDave Chinner 668fd074841SDave Chinner /* 669211e4d43SChristoph Hellwig * Push out all items in the AIL immediately and wait until the AIL is empty. 670211e4d43SChristoph Hellwig */ 671211e4d43SChristoph Hellwig void 672211e4d43SChristoph Hellwig xfs_ail_push_all_sync( 673211e4d43SChristoph Hellwig struct xfs_ail *ailp) 674211e4d43SChristoph Hellwig { 675211e4d43SChristoph Hellwig struct xfs_log_item *lip; 676211e4d43SChristoph Hellwig DEFINE_WAIT(wait); 677211e4d43SChristoph Hellwig 67857e80956SMatthew Wilcox spin_lock(&ailp->ail_lock); 679211e4d43SChristoph Hellwig while ((lip = xfs_ail_max(ailp)) != NULL) { 68057e80956SMatthew Wilcox prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE); 68157e80956SMatthew Wilcox ailp->ail_target = lip->li_lsn; 68257e80956SMatthew Wilcox wake_up_process(ailp->ail_task); 68357e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock); 684211e4d43SChristoph Hellwig schedule(); 68557e80956SMatthew Wilcox spin_lock(&ailp->ail_lock); 686211e4d43SChristoph Hellwig } 68757e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock); 688211e4d43SChristoph Hellwig 68957e80956SMatthew Wilcox finish_wait(&ailp->ail_empty, &wait); 690211e4d43SChristoph Hellwig } 691211e4d43SChristoph Hellwig 6924165994aSDave Chinner void 6934165994aSDave Chinner xfs_ail_update_finish( 6944165994aSDave Chinner struct xfs_ail *ailp, 6958eb807bdSDave Chinner xfs_lsn_t old_lsn) __releases(ailp->ail_lock) 6964165994aSDave Chinner { 6974165994aSDave Chinner struct xfs_mount *mp = ailp->ail_mount; 6984165994aSDave Chinner 6998eb807bdSDave Chinner /* if the tail lsn hasn't changed, don't do updates or wakeups. */ 7008eb807bdSDave Chinner if (!old_lsn || old_lsn == __xfs_ail_min_lsn(ailp)) { 7014165994aSDave Chinner spin_unlock(&ailp->ail_lock); 7024165994aSDave Chinner return; 7034165994aSDave Chinner } 7044165994aSDave Chinner 7054165994aSDave Chinner if (!XFS_FORCED_SHUTDOWN(mp)) 7064165994aSDave Chinner xlog_assign_tail_lsn_locked(mp); 7074165994aSDave Chinner 7084165994aSDave Chinner if (list_empty(&ailp->ail_head)) 7094165994aSDave Chinner wake_up_all(&ailp->ail_empty); 7104165994aSDave Chinner spin_unlock(&ailp->ail_lock); 7114165994aSDave Chinner xfs_log_space_wake(mp); 7124165994aSDave Chinner } 7134165994aSDave Chinner 714211e4d43SChristoph Hellwig /* 7150e57f6a3SDave Chinner * xfs_trans_ail_update - bulk AIL insertion operation. 7160e57f6a3SDave Chinner * 7170e57f6a3SDave Chinner * @xfs_trans_ail_update takes an array of log items that all need to be 7180e57f6a3SDave Chinner * positioned at the same LSN in the AIL. If an item is not in the AIL, it will 7190e57f6a3SDave Chinner * be added. Otherwise, it will be repositioned by removing it and re-adding 7200e57f6a3SDave Chinner * it to the AIL. If we move the first item in the AIL, update the log tail to 7210e57f6a3SDave Chinner * match the new minimum LSN in the AIL. 7220e57f6a3SDave Chinner * 7230e57f6a3SDave Chinner * This function takes the AIL lock once to execute the update operations on 7240e57f6a3SDave Chinner * all the items in the array, and as such should not be called with the AIL 7250e57f6a3SDave Chinner * lock held. As a result, once we have the AIL lock, we need to check each log 7260e57f6a3SDave Chinner * item LSN to confirm it needs to be moved forward in the AIL. 7270e57f6a3SDave Chinner * 7280e57f6a3SDave Chinner * To optimise the insert operation, we delete all the items from the AIL in 7290e57f6a3SDave Chinner * the first pass, moving them into a temporary list, then splice the temporary 7300e57f6a3SDave Chinner * list into the correct position in the AIL. This avoids needing to do an 7310e57f6a3SDave Chinner * insert operation on every item. 7320e57f6a3SDave Chinner * 7330e57f6a3SDave Chinner * This function must be called with the AIL lock held. The lock is dropped 7340e57f6a3SDave Chinner * before returning. 7350e57f6a3SDave Chinner */ 7360e57f6a3SDave Chinner void 7370e57f6a3SDave Chinner xfs_trans_ail_update_bulk( 7380e57f6a3SDave Chinner struct xfs_ail *ailp, 7391d8c95a3SDave Chinner struct xfs_ail_cursor *cur, 7400e57f6a3SDave Chinner struct xfs_log_item **log_items, 7410e57f6a3SDave Chinner int nr_items, 74257e80956SMatthew Wilcox xfs_lsn_t lsn) __releases(ailp->ail_lock) 7430e57f6a3SDave Chinner { 744efe2330fSChristoph Hellwig struct xfs_log_item *mlip; 7458eb807bdSDave Chinner xfs_lsn_t tail_lsn = 0; 7460e57f6a3SDave Chinner int i; 7470e57f6a3SDave Chinner LIST_HEAD(tmp); 7480e57f6a3SDave Chinner 749e44f4112SAlex Elder ASSERT(nr_items > 0); /* Not required, but true. */ 7500e57f6a3SDave Chinner mlip = xfs_ail_min(ailp); 7510e57f6a3SDave Chinner 7520e57f6a3SDave Chinner for (i = 0; i < nr_items; i++) { 7530e57f6a3SDave Chinner struct xfs_log_item *lip = log_items[i]; 75422525c17SDave Chinner if (test_and_set_bit(XFS_LI_IN_AIL, &lip->li_flags)) { 7550e57f6a3SDave Chinner /* check if we really need to move the item */ 7560e57f6a3SDave Chinner if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0) 7570e57f6a3SDave Chinner continue; 7580e57f6a3SDave Chinner 759750b9c90SDave Chinner trace_xfs_ail_move(lip, lip->li_lsn, lsn); 7608eb807bdSDave Chinner if (mlip == lip && !tail_lsn) 7618eb807bdSDave Chinner tail_lsn = lip->li_lsn; 7628eb807bdSDave Chinner 7630e57f6a3SDave Chinner xfs_ail_delete(ailp, lip); 7640e57f6a3SDave Chinner } else { 765750b9c90SDave Chinner trace_xfs_ail_insert(lip, 0, lsn); 7660e57f6a3SDave Chinner } 7670e57f6a3SDave Chinner lip->li_lsn = lsn; 7680e57f6a3SDave Chinner list_add(&lip->li_ail, &tmp); 7690e57f6a3SDave Chinner } 7700e57f6a3SDave Chinner 771e44f4112SAlex Elder if (!list_empty(&tmp)) 7721d8c95a3SDave Chinner xfs_ail_splice(ailp, cur, &tmp, lsn); 7731c304625SChristoph Hellwig 7748eb807bdSDave Chinner xfs_ail_update_finish(ailp, tail_lsn); 7750e57f6a3SDave Chinner } 7760e57f6a3SDave Chinner 7778eb807bdSDave Chinner /* 7788eb807bdSDave Chinner * Delete one log item from the AIL. 7798eb807bdSDave Chinner * 7808eb807bdSDave Chinner * If this item was at the tail of the AIL, return the LSN of the log item so 7818eb807bdSDave Chinner * that we can use it to check if the LSN of the tail of the log has moved 7828eb807bdSDave Chinner * when finishing up the AIL delete process in xfs_ail_update_finish(). 7838eb807bdSDave Chinner */ 7848eb807bdSDave Chinner xfs_lsn_t 78527af1bbfSChristoph Hellwig xfs_ail_delete_one( 78627af1bbfSChristoph Hellwig struct xfs_ail *ailp, 78727af1bbfSChristoph Hellwig struct xfs_log_item *lip) 78827af1bbfSChristoph Hellwig { 78927af1bbfSChristoph Hellwig struct xfs_log_item *mlip = xfs_ail_min(ailp); 7908eb807bdSDave Chinner xfs_lsn_t lsn = lip->li_lsn; 79127af1bbfSChristoph Hellwig 79227af1bbfSChristoph Hellwig trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn); 79327af1bbfSChristoph Hellwig xfs_ail_delete(ailp, lip); 794d3a304b6SCarlos Maiolino xfs_clear_li_failed(lip); 79522525c17SDave Chinner clear_bit(XFS_LI_IN_AIL, &lip->li_flags); 79627af1bbfSChristoph Hellwig lip->li_lsn = 0; 79727af1bbfSChristoph Hellwig 7988eb807bdSDave Chinner if (mlip == lip) 7998eb807bdSDave Chinner return lsn; 8008eb807bdSDave Chinner return 0; 80127af1bbfSChristoph Hellwig } 80227af1bbfSChristoph Hellwig 80327af1bbfSChristoph Hellwig /** 80427af1bbfSChristoph Hellwig * Remove a log items from the AIL 80530136832SDave Chinner * 80630136832SDave Chinner * @xfs_trans_ail_delete_bulk takes an array of log items that all need to 80730136832SDave Chinner * removed from the AIL. The caller is already holding the AIL lock, and done 80830136832SDave Chinner * all the checks necessary to ensure the items passed in via @log_items are 80930136832SDave Chinner * ready for deletion. This includes checking that the items are in the AIL. 81030136832SDave Chinner * 81130136832SDave Chinner * For each log item to be removed, unlink it from the AIL, clear the IN_AIL 81230136832SDave Chinner * flag from the item and reset the item's lsn to 0. If we remove the first 81330136832SDave Chinner * item in the AIL, update the log tail to match the new minimum LSN in the 81430136832SDave Chinner * AIL. 81530136832SDave Chinner * 81630136832SDave Chinner * This function will not drop the AIL lock until all items are removed from 81730136832SDave Chinner * the AIL to minimise the amount of lock traffic on the AIL. This does not 81830136832SDave Chinner * greatly increase the AIL hold time, but does significantly reduce the amount 81930136832SDave Chinner * of traffic on the lock, especially during IO completion. 82030136832SDave Chinner * 82130136832SDave Chinner * This function must be called with the AIL lock held. The lock is dropped 82230136832SDave Chinner * before returning. 82330136832SDave Chinner */ 82430136832SDave Chinner void 82527af1bbfSChristoph Hellwig xfs_trans_ail_delete( 82630136832SDave Chinner struct xfs_ail *ailp, 82727af1bbfSChristoph Hellwig struct xfs_log_item *lip, 8284165994aSDave Chinner int shutdown_type) 82930136832SDave Chinner { 83057e80956SMatthew Wilcox struct xfs_mount *mp = ailp->ail_mount; 8318eb807bdSDave Chinner xfs_lsn_t tail_lsn; 83230136832SDave Chinner 83322525c17SDave Chinner if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { 83457e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock); 83530136832SDave Chinner if (!XFS_FORCED_SHUTDOWN(mp)) { 8366a19d939SDave Chinner xfs_alert_tag(mp, XFS_PTAG_AILDELETE, 83730136832SDave Chinner "%s: attempting to delete a log item that is not in the AIL", 83830136832SDave Chinner __func__); 83904913fddSDave Chinner xfs_force_shutdown(mp, shutdown_type); 84030136832SDave Chinner } 84130136832SDave Chinner return; 84230136832SDave Chinner } 84330136832SDave Chinner 8448eb807bdSDave Chinner tail_lsn = xfs_ail_delete_one(ailp, lip); 8458eb807bdSDave Chinner xfs_ail_update_finish(ailp, tail_lsn); 84630136832SDave Chinner } 8471da177e4SLinus Torvalds 848249a8c11SDavid Chinner int 8491da177e4SLinus Torvalds xfs_trans_ail_init( 8501da177e4SLinus Torvalds xfs_mount_t *mp) 8511da177e4SLinus Torvalds { 85282fa9012SDavid Chinner struct xfs_ail *ailp; 85382fa9012SDavid Chinner 85482fa9012SDavid Chinner ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL); 85582fa9012SDavid Chinner if (!ailp) 8562451337dSDave Chinner return -ENOMEM; 85782fa9012SDavid Chinner 85857e80956SMatthew Wilcox ailp->ail_mount = mp; 85957e80956SMatthew Wilcox INIT_LIST_HEAD(&ailp->ail_head); 86057e80956SMatthew Wilcox INIT_LIST_HEAD(&ailp->ail_cursors); 86157e80956SMatthew Wilcox spin_lock_init(&ailp->ail_lock); 86257e80956SMatthew Wilcox INIT_LIST_HEAD(&ailp->ail_buf_list); 86357e80956SMatthew Wilcox init_waitqueue_head(&ailp->ail_empty); 8640030807cSChristoph Hellwig 86557e80956SMatthew Wilcox ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s", 866e1d3d218SIan Kent ailp->ail_mount->m_super->s_id); 86757e80956SMatthew Wilcox if (IS_ERR(ailp->ail_task)) 8680030807cSChristoph Hellwig goto out_free_ailp; 8690030807cSChristoph Hellwig 87027d8d5feSDavid Chinner mp->m_ail = ailp; 87127d8d5feSDavid Chinner return 0; 8720030807cSChristoph Hellwig 8730030807cSChristoph Hellwig out_free_ailp: 8740030807cSChristoph Hellwig kmem_free(ailp); 8752451337dSDave Chinner return -ENOMEM; 876249a8c11SDavid Chinner } 877249a8c11SDavid Chinner 878249a8c11SDavid Chinner void 879249a8c11SDavid Chinner xfs_trans_ail_destroy( 880249a8c11SDavid Chinner xfs_mount_t *mp) 881249a8c11SDavid Chinner { 88282fa9012SDavid Chinner struct xfs_ail *ailp = mp->m_ail; 88382fa9012SDavid Chinner 88457e80956SMatthew Wilcox kthread_stop(ailp->ail_task); 88582fa9012SDavid Chinner kmem_free(ailp); 8861da177e4SLinus Torvalds } 887