10b61f8a4SDave Chinner // SPDX-License-Identifier: GPL-2.0
21da177e4SLinus Torvalds /*
37b718769SNathan Scott * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
4c7e8f268SDavid Chinner * Copyright (c) 2008 Dave Chinner
57b718769SNathan Scott * All Rights Reserved.
61da177e4SLinus Torvalds */
71da177e4SLinus Torvalds #include "xfs.h"
8a844f451SNathan Scott #include "xfs_fs.h"
95467b34bSDarrick J. Wong #include "xfs_shared.h"
104fb6e8adSChristoph Hellwig #include "xfs_format.h"
11239880efSDave Chinner #include "xfs_log_format.h"
12239880efSDave Chinner #include "xfs_trans_resv.h"
131da177e4SLinus Torvalds #include "xfs_mount.h"
14239880efSDave Chinner #include "xfs_trans.h"
151da177e4SLinus Torvalds #include "xfs_trans_priv.h"
169e4c109aSChristoph Hellwig #include "xfs_trace.h"
17e9e899a2SDarrick J. Wong #include "xfs_errortag.h"
181da177e4SLinus Torvalds #include "xfs_error.h"
19239880efSDave Chinner #include "xfs_log.h"
200020a190SDave Chinner #include "xfs_log_priv.h"
211da177e4SLinus Torvalds
221da177e4SLinus Torvalds #ifdef DEBUG
23cd4a3c50SDave Chinner /*
24cd4a3c50SDave Chinner * Check that the list is sorted as it should be.
25d686d12dSDave Chinner *
26d686d12dSDave Chinner * Called with the ail lock held, but we don't want to assert fail with it
27d686d12dSDave Chinner * held otherwise we'll lock everything up and won't be able to debug the
28d686d12dSDave Chinner * cause. Hence we sample and check the state under the AIL lock and return if
29d686d12dSDave Chinner * everything is fine, otherwise we drop the lock and run the ASSERT checks.
30d686d12dSDave Chinner * Asserts may not be fatal, so pick the lock back up and continue onwards.
31cd4a3c50SDave Chinner */
32cd4a3c50SDave Chinner STATIC void
xfs_ail_check(struct xfs_ail * ailp,struct xfs_log_item * lip)33cd4a3c50SDave Chinner xfs_ail_check(
34cd4a3c50SDave Chinner struct xfs_ail *ailp,
35d686d12dSDave Chinner struct xfs_log_item *lip)
36daebba1bSJules Irenge __must_hold(&ailp->ail_lock)
37cd4a3c50SDave Chinner {
38d686d12dSDave Chinner struct xfs_log_item *prev_lip;
39d686d12dSDave Chinner struct xfs_log_item *next_lip;
40d686d12dSDave Chinner xfs_lsn_t prev_lsn = NULLCOMMITLSN;
41d686d12dSDave Chinner xfs_lsn_t next_lsn = NULLCOMMITLSN;
42d686d12dSDave Chinner xfs_lsn_t lsn;
43d686d12dSDave Chinner bool in_ail;
44d686d12dSDave Chinner
45cd4a3c50SDave Chinner
4657e80956SMatthew Wilcox if (list_empty(&ailp->ail_head))
47cd4a3c50SDave Chinner return;
48cd4a3c50SDave Chinner
49cd4a3c50SDave Chinner /*
50d686d12dSDave Chinner * Sample then check the next and previous entries are valid.
51cd4a3c50SDave Chinner */
52d686d12dSDave Chinner in_ail = test_bit(XFS_LI_IN_AIL, &lip->li_flags);
53d686d12dSDave Chinner prev_lip = list_entry(lip->li_ail.prev, struct xfs_log_item, li_ail);
5457e80956SMatthew Wilcox if (&prev_lip->li_ail != &ailp->ail_head)
55d686d12dSDave Chinner prev_lsn = prev_lip->li_lsn;
56d686d12dSDave Chinner next_lip = list_entry(lip->li_ail.next, struct xfs_log_item, li_ail);
57d686d12dSDave Chinner if (&next_lip->li_ail != &ailp->ail_head)
58d686d12dSDave Chinner next_lsn = next_lip->li_lsn;
59d686d12dSDave Chinner lsn = lip->li_lsn;
60cd4a3c50SDave Chinner
61d686d12dSDave Chinner if (in_ail &&
62d686d12dSDave Chinner (prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0) &&
63d686d12dSDave Chinner (next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0))
64d686d12dSDave Chinner return;
65cd4a3c50SDave Chinner
66d686d12dSDave Chinner spin_unlock(&ailp->ail_lock);
67d686d12dSDave Chinner ASSERT(in_ail);
68d686d12dSDave Chinner ASSERT(prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0);
69d686d12dSDave Chinner ASSERT(next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0);
70d686d12dSDave Chinner spin_lock(&ailp->ail_lock);
71cd4a3c50SDave Chinner }
72cd4a3c50SDave Chinner #else /* !DEBUG */
73de08dbc1SDavid Chinner #define xfs_ail_check(a,l)
741da177e4SLinus Torvalds #endif /* DEBUG */
751da177e4SLinus Torvalds
76cd4a3c50SDave Chinner /*
77fd074841SDave Chinner * Return a pointer to the last item in the AIL. If the AIL is empty, then
78fd074841SDave Chinner * return NULL.
79fd074841SDave Chinner */
80efe2330fSChristoph Hellwig static struct xfs_log_item *
xfs_ail_max(struct xfs_ail * ailp)81fd074841SDave Chinner xfs_ail_max(
82fd074841SDave Chinner struct xfs_ail *ailp)
83fd074841SDave Chinner {
8457e80956SMatthew Wilcox if (list_empty(&ailp->ail_head))
85fd074841SDave Chinner return NULL;
86fd074841SDave Chinner
87efe2330fSChristoph Hellwig return list_entry(ailp->ail_head.prev, struct xfs_log_item, li_ail);
88fd074841SDave Chinner }
89fd074841SDave Chinner
90fd074841SDave Chinner /*
91cd4a3c50SDave Chinner * Return a pointer to the item which follows the given item in the AIL. If
92cd4a3c50SDave Chinner * the given item is the last item in the list, then return NULL.
93cd4a3c50SDave Chinner */
94efe2330fSChristoph Hellwig static struct xfs_log_item *
xfs_ail_next(struct xfs_ail * ailp,struct xfs_log_item * lip)95cd4a3c50SDave Chinner xfs_ail_next(
96cd4a3c50SDave Chinner struct xfs_ail *ailp,
97efe2330fSChristoph Hellwig struct xfs_log_item *lip)
98cd4a3c50SDave Chinner {
9957e80956SMatthew Wilcox if (lip->li_ail.next == &ailp->ail_head)
100cd4a3c50SDave Chinner return NULL;
101cd4a3c50SDave Chinner
102efe2330fSChristoph Hellwig return list_first_entry(&lip->li_ail, struct xfs_log_item, li_ail);
103cd4a3c50SDave Chinner }
104cd4a3c50SDave Chinner
105cd4a3c50SDave Chinner /*
106cd4a3c50SDave Chinner * This is called by the log manager code to determine the LSN of the tail of
107cd4a3c50SDave Chinner * the log. This is exactly the LSN of the first item in the AIL. If the AIL
108cd4a3c50SDave Chinner * is empty, then this function returns 0.
1091da177e4SLinus Torvalds *
110cd4a3c50SDave Chinner * We need the AIL lock in order to get a coherent read of the lsn of the last
111cd4a3c50SDave Chinner * item in the AIL.
1121da177e4SLinus Torvalds */
1138eb807bdSDave Chinner static xfs_lsn_t
__xfs_ail_min_lsn(struct xfs_ail * ailp)1148eb807bdSDave Chinner __xfs_ail_min_lsn(
1158eb807bdSDave Chinner struct xfs_ail *ailp)
1168eb807bdSDave Chinner {
1178eb807bdSDave Chinner struct xfs_log_item *lip = xfs_ail_min(ailp);
1188eb807bdSDave Chinner
1198eb807bdSDave Chinner if (lip)
1208eb807bdSDave Chinner return lip->li_lsn;
1218eb807bdSDave Chinner return 0;
1228eb807bdSDave Chinner }
1238eb807bdSDave Chinner
1241da177e4SLinus Torvalds xfs_lsn_t
xfs_ail_min_lsn(struct xfs_ail * ailp)125fd074841SDave Chinner xfs_ail_min_lsn(
1265b00f14fSDavid Chinner struct xfs_ail *ailp)
1271da177e4SLinus Torvalds {
1288eb807bdSDave Chinner xfs_lsn_t lsn;
1291da177e4SLinus Torvalds
13057e80956SMatthew Wilcox spin_lock(&ailp->ail_lock);
1318eb807bdSDave Chinner lsn = __xfs_ail_min_lsn(ailp);
13257e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock);
1331da177e4SLinus Torvalds
1341da177e4SLinus Torvalds return lsn;
1351da177e4SLinus Torvalds }
1361da177e4SLinus Torvalds
1371da177e4SLinus Torvalds /*
138fd074841SDave Chinner * Return the maximum lsn held in the AIL, or zero if the AIL is empty.
139fd074841SDave Chinner */
140fd074841SDave Chinner static xfs_lsn_t
xfs_ail_max_lsn(struct xfs_ail * ailp)141fd074841SDave Chinner xfs_ail_max_lsn(
142fd074841SDave Chinner struct xfs_ail *ailp)
143fd074841SDave Chinner {
144fd074841SDave Chinner xfs_lsn_t lsn = 0;
145efe2330fSChristoph Hellwig struct xfs_log_item *lip;
146fd074841SDave Chinner
14757e80956SMatthew Wilcox spin_lock(&ailp->ail_lock);
148fd074841SDave Chinner lip = xfs_ail_max(ailp);
149fd074841SDave Chinner if (lip)
150fd074841SDave Chinner lsn = lip->li_lsn;
15157e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock);
152fd074841SDave Chinner
153fd074841SDave Chinner return lsn;
154fd074841SDave Chinner }
155fd074841SDave Chinner
156fd074841SDave Chinner /*
157af3e4022SDave Chinner * The cursor keeps track of where our current traversal is up to by tracking
158af3e4022SDave Chinner * the next item in the list for us. However, for this to be safe, removing an
159af3e4022SDave Chinner * object from the AIL needs to invalidate any cursor that points to it. hence
160af3e4022SDave Chinner * the traversal cursor needs to be linked to the struct xfs_ail so that
161af3e4022SDave Chinner * deletion can search all the active cursors for invalidation.
16227d8d5feSDavid Chinner */
1635b00f14fSDavid Chinner STATIC void
xfs_trans_ail_cursor_init(struct xfs_ail * ailp,struct xfs_ail_cursor * cur)16427d8d5feSDavid Chinner xfs_trans_ail_cursor_init(
16527d8d5feSDavid Chinner struct xfs_ail *ailp,
16627d8d5feSDavid Chinner struct xfs_ail_cursor *cur)
16727d8d5feSDavid Chinner {
16827d8d5feSDavid Chinner cur->item = NULL;
16957e80956SMatthew Wilcox list_add_tail(&cur->list, &ailp->ail_cursors);
17027d8d5feSDavid Chinner }
17127d8d5feSDavid Chinner
17227d8d5feSDavid Chinner /*
173af3e4022SDave Chinner * Get the next item in the traversal and advance the cursor. If the cursor
174af3e4022SDave Chinner * was invalidated (indicated by a lip of 1), restart the traversal.
17527d8d5feSDavid Chinner */
1765b00f14fSDavid Chinner struct xfs_log_item *
xfs_trans_ail_cursor_next(struct xfs_ail * ailp,struct xfs_ail_cursor * cur)17727d8d5feSDavid Chinner xfs_trans_ail_cursor_next(
17827d8d5feSDavid Chinner struct xfs_ail *ailp,
17927d8d5feSDavid Chinner struct xfs_ail_cursor *cur)
18027d8d5feSDavid Chinner {
18127d8d5feSDavid Chinner struct xfs_log_item *lip = cur->item;
18227d8d5feSDavid Chinner
183db9d67d6SChristoph Hellwig if ((uintptr_t)lip & 1)
18427d8d5feSDavid Chinner lip = xfs_ail_min(ailp);
18516b59029SDave Chinner if (lip)
18616b59029SDave Chinner cur->item = xfs_ail_next(ailp, lip);
18727d8d5feSDavid Chinner return lip;
18827d8d5feSDavid Chinner }
18927d8d5feSDavid Chinner
19027d8d5feSDavid Chinner /*
191af3e4022SDave Chinner * When the traversal is complete, we need to remove the cursor from the list
192af3e4022SDave Chinner * of traversing cursors.
19327d8d5feSDavid Chinner */
19427d8d5feSDavid Chinner void
xfs_trans_ail_cursor_done(struct xfs_ail_cursor * cur)19527d8d5feSDavid Chinner xfs_trans_ail_cursor_done(
196af3e4022SDave Chinner struct xfs_ail_cursor *cur)
19727d8d5feSDavid Chinner {
198af3e4022SDave Chinner cur->item = NULL;
199af3e4022SDave Chinner list_del_init(&cur->list);
20027d8d5feSDavid Chinner }
20127d8d5feSDavid Chinner
20227d8d5feSDavid Chinner /*
203af3e4022SDave Chinner * Invalidate any cursor that is pointing to this item. This is called when an
204af3e4022SDave Chinner * item is removed from the AIL. Any cursor pointing to this object is now
205af3e4022SDave Chinner * invalid and the traversal needs to be terminated so it doesn't reference a
206af3e4022SDave Chinner * freed object. We set the low bit of the cursor item pointer so we can
207af3e4022SDave Chinner * distinguish between an invalidation and the end of the list when getting the
208af3e4022SDave Chinner * next item from the cursor.
2095b00f14fSDavid Chinner */
2105b00f14fSDavid Chinner STATIC void
xfs_trans_ail_cursor_clear(struct xfs_ail * ailp,struct xfs_log_item * lip)2115b00f14fSDavid Chinner xfs_trans_ail_cursor_clear(
2125b00f14fSDavid Chinner struct xfs_ail *ailp,
2135b00f14fSDavid Chinner struct xfs_log_item *lip)
2145b00f14fSDavid Chinner {
2155b00f14fSDavid Chinner struct xfs_ail_cursor *cur;
2165b00f14fSDavid Chinner
21757e80956SMatthew Wilcox list_for_each_entry(cur, &ailp->ail_cursors, list) {
2185b00f14fSDavid Chinner if (cur->item == lip)
2195b00f14fSDavid Chinner cur->item = (struct xfs_log_item *)
220db9d67d6SChristoph Hellwig ((uintptr_t)cur->item | 1);
2215b00f14fSDavid Chinner }
2225b00f14fSDavid Chinner }
2235b00f14fSDavid Chinner
2245b00f14fSDavid Chinner /*
22516b59029SDave Chinner * Find the first item in the AIL with the given @lsn by searching in ascending
22616b59029SDave Chinner * LSN order and initialise the cursor to point to the next item for a
22716b59029SDave Chinner * ascending traversal. Pass a @lsn of zero to initialise the cursor to the
22816b59029SDave Chinner * first item in the AIL. Returns NULL if the list is empty.
229249a8c11SDavid Chinner */
230efe2330fSChristoph Hellwig struct xfs_log_item *
xfs_trans_ail_cursor_first(struct xfs_ail * ailp,struct xfs_ail_cursor * cur,xfs_lsn_t lsn)2315b00f14fSDavid Chinner xfs_trans_ail_cursor_first(
23227d8d5feSDavid Chinner struct xfs_ail *ailp,
23327d8d5feSDavid Chinner struct xfs_ail_cursor *cur,
234249a8c11SDavid Chinner xfs_lsn_t lsn)
235249a8c11SDavid Chinner {
236efe2330fSChristoph Hellwig struct xfs_log_item *lip;
237249a8c11SDavid Chinner
2385b00f14fSDavid Chinner xfs_trans_ail_cursor_init(ailp, cur);
23916b59029SDave Chinner
24016b59029SDave Chinner if (lsn == 0) {
24127d8d5feSDavid Chinner lip = xfs_ail_min(ailp);
2425b00f14fSDavid Chinner goto out;
24316b59029SDave Chinner }
244249a8c11SDavid Chinner
24557e80956SMatthew Wilcox list_for_each_entry(lip, &ailp->ail_head, li_ail) {
2465b00f14fSDavid Chinner if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0)
2477ee49acfSDavid Chinner goto out;
2485b00f14fSDavid Chinner }
24916b59029SDave Chinner return NULL;
25016b59029SDave Chinner
2515b00f14fSDavid Chinner out:
25216b59029SDave Chinner if (lip)
25316b59029SDave Chinner cur->item = xfs_ail_next(ailp, lip);
254249a8c11SDavid Chinner return lip;
255249a8c11SDavid Chinner }
256535f6b37SJosef 'Jeff' Sipek
2571d8c95a3SDave Chinner static struct xfs_log_item *
__xfs_trans_ail_cursor_last(struct xfs_ail * ailp,xfs_lsn_t lsn)2581d8c95a3SDave Chinner __xfs_trans_ail_cursor_last(
2591d8c95a3SDave Chinner struct xfs_ail *ailp,
2601d8c95a3SDave Chinner xfs_lsn_t lsn)
2611d8c95a3SDave Chinner {
262efe2330fSChristoph Hellwig struct xfs_log_item *lip;
2631d8c95a3SDave Chinner
26457e80956SMatthew Wilcox list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) {
2651d8c95a3SDave Chinner if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0)
2661d8c95a3SDave Chinner return lip;
2671d8c95a3SDave Chinner }
2681d8c95a3SDave Chinner return NULL;
2691d8c95a3SDave Chinner }
2701d8c95a3SDave Chinner
2711d8c95a3SDave Chinner /*
27216b59029SDave Chinner * Find the last item in the AIL with the given @lsn by searching in descending
27316b59029SDave Chinner * LSN order and initialise the cursor to point to that item. If there is no
27416b59029SDave Chinner * item with the value of @lsn, then it sets the cursor to the last item with an
27516b59029SDave Chinner * LSN lower than @lsn. Returns NULL if the list is empty.
2761d8c95a3SDave Chinner */
2771d8c95a3SDave Chinner struct xfs_log_item *
xfs_trans_ail_cursor_last(struct xfs_ail * ailp,struct xfs_ail_cursor * cur,xfs_lsn_t lsn)2781d8c95a3SDave Chinner xfs_trans_ail_cursor_last(
2791d8c95a3SDave Chinner struct xfs_ail *ailp,
2801d8c95a3SDave Chinner struct xfs_ail_cursor *cur,
2811d8c95a3SDave Chinner xfs_lsn_t lsn)
2821d8c95a3SDave Chinner {
2831d8c95a3SDave Chinner xfs_trans_ail_cursor_init(ailp, cur);
2841d8c95a3SDave Chinner cur->item = __xfs_trans_ail_cursor_last(ailp, lsn);
2851d8c95a3SDave Chinner return cur->item;
2861d8c95a3SDave Chinner }
2871d8c95a3SDave Chinner
2881d8c95a3SDave Chinner /*
28916b59029SDave Chinner * Splice the log item list into the AIL at the given LSN. We splice to the
2901d8c95a3SDave Chinner * tail of the given LSN to maintain insert order for push traversals. The
2911d8c95a3SDave Chinner * cursor is optional, allowing repeated updates to the same LSN to avoid
292e44f4112SAlex Elder * repeated traversals. This should not be called with an empty list.
293cd4a3c50SDave Chinner */
294cd4a3c50SDave Chinner static void
xfs_ail_splice(struct xfs_ail * ailp,struct xfs_ail_cursor * cur,struct list_head * list,xfs_lsn_t lsn)295cd4a3c50SDave Chinner xfs_ail_splice(
296cd4a3c50SDave Chinner struct xfs_ail *ailp,
2971d8c95a3SDave Chinner struct xfs_ail_cursor *cur,
298cd4a3c50SDave Chinner struct list_head *list,
299cd4a3c50SDave Chinner xfs_lsn_t lsn)
300cd4a3c50SDave Chinner {
301e44f4112SAlex Elder struct xfs_log_item *lip;
302e44f4112SAlex Elder
303e44f4112SAlex Elder ASSERT(!list_empty(list));
304cd4a3c50SDave Chinner
3051d8c95a3SDave Chinner /*
306e44f4112SAlex Elder * Use the cursor to determine the insertion point if one is
307e44f4112SAlex Elder * provided. If not, or if the one we got is not valid,
308e44f4112SAlex Elder * find the place in the AIL where the items belong.
3091d8c95a3SDave Chinner */
310e44f4112SAlex Elder lip = cur ? cur->item : NULL;
311db9d67d6SChristoph Hellwig if (!lip || (uintptr_t)lip & 1)
3121d8c95a3SDave Chinner lip = __xfs_trans_ail_cursor_last(ailp, lsn);
3131d8c95a3SDave Chinner
314e44f4112SAlex Elder /*
315e44f4112SAlex Elder * If a cursor is provided, we know we're processing the AIL
316e44f4112SAlex Elder * in lsn order, and future items to be spliced in will
317e44f4112SAlex Elder * follow the last one being inserted now. Update the
318e44f4112SAlex Elder * cursor to point to that last item, now while we have a
319e44f4112SAlex Elder * reliable pointer to it.
320e44f4112SAlex Elder */
3211d8c95a3SDave Chinner if (cur)
322e44f4112SAlex Elder cur->item = list_entry(list->prev, struct xfs_log_item, li_ail);
323cd4a3c50SDave Chinner
3241d8c95a3SDave Chinner /*
325e44f4112SAlex Elder * Finally perform the splice. Unless the AIL was empty,
326e44f4112SAlex Elder * lip points to the item in the AIL _after_ which the new
327e44f4112SAlex Elder * items should go. If lip is null the AIL was empty, so
328e44f4112SAlex Elder * the new items go at the head of the AIL.
3291d8c95a3SDave Chinner */
330e44f4112SAlex Elder if (lip)
3311d8c95a3SDave Chinner list_splice(list, &lip->li_ail);
332e44f4112SAlex Elder else
33357e80956SMatthew Wilcox list_splice(list, &ailp->ail_head);
334cd4a3c50SDave Chinner }
335cd4a3c50SDave Chinner
336cd4a3c50SDave Chinner /*
337cd4a3c50SDave Chinner * Delete the given item from the AIL. Return a pointer to the item.
338cd4a3c50SDave Chinner */
339cd4a3c50SDave Chinner static void
xfs_ail_delete(struct xfs_ail * ailp,struct xfs_log_item * lip)340cd4a3c50SDave Chinner xfs_ail_delete(
341cd4a3c50SDave Chinner struct xfs_ail *ailp,
342efe2330fSChristoph Hellwig struct xfs_log_item *lip)
343cd4a3c50SDave Chinner {
344cd4a3c50SDave Chinner xfs_ail_check(ailp, lip);
345cd4a3c50SDave Chinner list_del(&lip->li_ail);
346cd4a3c50SDave Chinner xfs_trans_ail_cursor_clear(ailp, lip);
347cd4a3c50SDave Chinner }
348cd4a3c50SDave Chinner
349cb6ad099SBrian Foster /*
350cb6ad099SBrian Foster * Requeue a failed buffer for writeback.
351cb6ad099SBrian Foster *
352cb6ad099SBrian Foster * We clear the log item failed state here as well, but we have to be careful
353cb6ad099SBrian Foster * about reference counts because the only active reference counts on the buffer
354cb6ad099SBrian Foster * may be the failed log items. Hence if we clear the log item failed state
355cb6ad099SBrian Foster * before queuing the buffer for IO we can release all active references to
356cb6ad099SBrian Foster * the buffer and free it, leading to use after free problems in
357cb6ad099SBrian Foster * xfs_buf_delwri_queue. It makes no difference to the buffer or log items which
358cb6ad099SBrian Foster * order we process them in - the buffer is locked, and we own the buffer list
359cb6ad099SBrian Foster * so nothing on them is going to change while we are performing this action.
360cb6ad099SBrian Foster *
361cb6ad099SBrian Foster * Hence we can safely queue the buffer for IO before we clear the failed log
362cb6ad099SBrian Foster * item state, therefore always having an active reference to the buffer and
363cb6ad099SBrian Foster * avoiding the transient zero-reference state that leads to use-after-free.
364cb6ad099SBrian Foster */
365cb6ad099SBrian Foster static inline int
xfsaild_resubmit_item(struct xfs_log_item * lip,struct list_head * buffer_list)366cb6ad099SBrian Foster xfsaild_resubmit_item(
367cb6ad099SBrian Foster struct xfs_log_item *lip,
368cb6ad099SBrian Foster struct list_head *buffer_list)
369cb6ad099SBrian Foster {
370cb6ad099SBrian Foster struct xfs_buf *bp = lip->li_buf;
371cb6ad099SBrian Foster
372cb6ad099SBrian Foster if (!xfs_buf_trylock(bp))
373cb6ad099SBrian Foster return XFS_ITEM_LOCKED;
374cb6ad099SBrian Foster
375cb6ad099SBrian Foster if (!xfs_buf_delwri_queue(bp, buffer_list)) {
376cb6ad099SBrian Foster xfs_buf_unlock(bp);
377cb6ad099SBrian Foster return XFS_ITEM_FLUSHING;
378cb6ad099SBrian Foster }
379cb6ad099SBrian Foster
380cb6ad099SBrian Foster /* protected by ail_lock */
381298f7becSDave Chinner list_for_each_entry(lip, &bp->b_li_list, li_bio_list) {
382298f7becSDave Chinner if (bp->b_flags & _XBF_INODES)
383298f7becSDave Chinner clear_bit(XFS_LI_FAILED, &lip->li_flags);
384298f7becSDave Chinner else
385cb6ad099SBrian Foster xfs_clear_li_failed(lip);
386298f7becSDave Chinner }
387cb6ad099SBrian Foster
388cb6ad099SBrian Foster xfs_buf_unlock(bp);
389cb6ad099SBrian Foster return XFS_ITEM_SUCCESS;
390cb6ad099SBrian Foster }
391cb6ad099SBrian Foster
3927f4d01f3SBrian Foster static inline uint
xfsaild_push_item(struct xfs_ail * ailp,struct xfs_log_item * lip)3937f4d01f3SBrian Foster xfsaild_push_item(
3947f4d01f3SBrian Foster struct xfs_ail *ailp,
3957f4d01f3SBrian Foster struct xfs_log_item *lip)
3967f4d01f3SBrian Foster {
3977f4d01f3SBrian Foster /*
3987f4d01f3SBrian Foster * If log item pinning is enabled, skip the push and track the item as
3997f4d01f3SBrian Foster * pinned. This can help induce head-behind-tail conditions.
4007f4d01f3SBrian Foster */
4018eda8721SDave Chinner if (XFS_TEST_ERROR(false, ailp->ail_log->l_mp, XFS_ERRTAG_LOG_ITEM_PIN))
4027f4d01f3SBrian Foster return XFS_ITEM_PINNED;
4037f4d01f3SBrian Foster
404e8b78db7SChristoph Hellwig /*
405e8b78db7SChristoph Hellwig * Consider the item pinned if a push callback is not defined so the
406e8b78db7SChristoph Hellwig * caller will force the log. This should only happen for intent items
407e8b78db7SChristoph Hellwig * as they are unpinned once the associated done item is committed to
408e8b78db7SChristoph Hellwig * the on-disk log.
409e8b78db7SChristoph Hellwig */
410e8b78db7SChristoph Hellwig if (!lip->li_ops->iop_push)
411e8b78db7SChristoph Hellwig return XFS_ITEM_PINNED;
412cb6ad099SBrian Foster if (test_bit(XFS_LI_FAILED, &lip->li_flags))
413cb6ad099SBrian Foster return xfsaild_resubmit_item(lip, &ailp->ail_buf_list);
41457e80956SMatthew Wilcox return lip->li_ops->iop_push(lip, &ailp->ail_buf_list);
4157f4d01f3SBrian Foster }
4167f4d01f3SBrian Foster
4170030807cSChristoph Hellwig static long
xfsaild_push(struct xfs_ail * ailp)4180030807cSChristoph Hellwig xfsaild_push(
4190030807cSChristoph Hellwig struct xfs_ail *ailp)
420249a8c11SDavid Chinner {
4218eda8721SDave Chinner struct xfs_mount *mp = ailp->ail_log->l_mp;
422af3e4022SDave Chinner struct xfs_ail_cursor cur;
423efe2330fSChristoph Hellwig struct xfs_log_item *lip;
4249e7004e7SDave Chinner xfs_lsn_t lsn;
425e5827a00SDarrick J. Wong xfs_lsn_t target = NULLCOMMITLSN;
42643ff2122SChristoph Hellwig long tout;
4279e7004e7SDave Chinner int stuck = 0;
42843ff2122SChristoph Hellwig int flushing = 0;
4299e7004e7SDave Chinner int count = 0;
4301da177e4SLinus Torvalds
431670ce93fSDave Chinner /*
43243ff2122SChristoph Hellwig * If we encountered pinned items or did not finish writing out all
4330020a190SDave Chinner * buffers the last time we ran, force a background CIL push to get the
4340020a190SDave Chinner * items unpinned in the near future. We do not wait on the CIL push as
4350020a190SDave Chinner * that could stall us for seconds if there is enough background IO
4360020a190SDave Chinner * load. Stalling for that long when the tail of the log is pinned and
4370020a190SDave Chinner * needs flushing will hard stop the transaction subsystem when log
4380020a190SDave Chinner * space runs out.
439670ce93fSDave Chinner */
44057e80956SMatthew Wilcox if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 &&
44157e80956SMatthew Wilcox (!list_empty_careful(&ailp->ail_buf_list) ||
44243ff2122SChristoph Hellwig xfs_ail_min_lsn(ailp))) {
44357e80956SMatthew Wilcox ailp->ail_log_flush = 0;
44443ff2122SChristoph Hellwig
445ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_flush);
4468eda8721SDave Chinner xlog_cil_flush(ailp->ail_log);
447670ce93fSDave Chinner }
448670ce93fSDave Chinner
44957e80956SMatthew Wilcox spin_lock(&ailp->ail_lock);
4508375f922SBrian Foster
451941fbdfdSDave Chinner /*
452941fbdfdSDave Chinner * If we have a sync push waiter, we always have to push till the AIL is
453941fbdfdSDave Chinner * empty. Update the target to point to the end of the AIL so that
454941fbdfdSDave Chinner * capture updates that occur after the sync push waiter has gone to
455941fbdfdSDave Chinner * sleep.
456941fbdfdSDave Chinner */
457941fbdfdSDave Chinner if (waitqueue_active(&ailp->ail_empty)) {
458941fbdfdSDave Chinner lip = xfs_ail_max(ailp);
459941fbdfdSDave Chinner if (lip)
460941fbdfdSDave Chinner target = lip->li_lsn;
461941fbdfdSDave Chinner } else {
46257e80956SMatthew Wilcox /* barrier matches the ail_target update in xfs_ail_push() */
4638375f922SBrian Foster smp_rmb();
46457e80956SMatthew Wilcox target = ailp->ail_target;
46557e80956SMatthew Wilcox ailp->ail_target_prev = target;
466941fbdfdSDave Chinner }
4678375f922SBrian Foster
468f376b45eSBrian Foster /* we're done if the AIL is empty or our push has reached the end */
46957e80956SMatthew Wilcox lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn);
470f376b45eSBrian Foster if (!lip)
4719e7004e7SDave Chinner goto out_done;
4721da177e4SLinus Torvalds
473ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail);
4741da177e4SLinus Torvalds
475e5827a00SDarrick J. Wong ASSERT(target != NULLCOMMITLSN);
476e5827a00SDarrick J. Wong
477249a8c11SDavid Chinner lsn = lip->li_lsn;
47850e86686SDave Chinner while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) {
479249a8c11SDavid Chinner int lock_result;
48043ff2122SChristoph Hellwig
481249a8c11SDavid Chinner /*
482904c17e6SDave Chinner * Note that iop_push may unlock and reacquire the AIL lock. We
48343ff2122SChristoph Hellwig * rely on the AIL cursor implementation to be able to deal with
48443ff2122SChristoph Hellwig * the dropped lock.
4851da177e4SLinus Torvalds */
4867f4d01f3SBrian Foster lock_result = xfsaild_push_item(ailp, lip);
4871da177e4SLinus Torvalds switch (lock_result) {
4881da177e4SLinus Torvalds case XFS_ITEM_SUCCESS:
489ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_success);
4909e4c109aSChristoph Hellwig trace_xfs_ail_push(lip);
4919e4c109aSChristoph Hellwig
49257e80956SMatthew Wilcox ailp->ail_last_pushed_lsn = lsn;
4931da177e4SLinus Torvalds break;
4941da177e4SLinus Torvalds
49543ff2122SChristoph Hellwig case XFS_ITEM_FLUSHING:
49643ff2122SChristoph Hellwig /*
497cf085a1bSJoe Perches * The item or its backing buffer is already being
49843ff2122SChristoph Hellwig * flushed. The typical reason for that is that an
49943ff2122SChristoph Hellwig * inode buffer is locked because we already pushed the
50043ff2122SChristoph Hellwig * updates to it as part of inode clustering.
50143ff2122SChristoph Hellwig *
502b63da6c8SRandy Dunlap * We do not want to stop flushing just because lots
503cf085a1bSJoe Perches * of items are already being flushed, but we need to
50443ff2122SChristoph Hellwig * re-try the flushing relatively soon if most of the
505cf085a1bSJoe Perches * AIL is being flushed.
50643ff2122SChristoph Hellwig */
507ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_flushing);
50843ff2122SChristoph Hellwig trace_xfs_ail_flushing(lip);
50917b38471SChristoph Hellwig
51043ff2122SChristoph Hellwig flushing++;
51157e80956SMatthew Wilcox ailp->ail_last_pushed_lsn = lsn;
5121da177e4SLinus Torvalds break;
5131da177e4SLinus Torvalds
5141da177e4SLinus Torvalds case XFS_ITEM_PINNED:
515ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_pinned);
5169e4c109aSChristoph Hellwig trace_xfs_ail_pinned(lip);
5179e4c109aSChristoph Hellwig
518249a8c11SDavid Chinner stuck++;
51957e80956SMatthew Wilcox ailp->ail_log_flush++;
5201da177e4SLinus Torvalds break;
5211da177e4SLinus Torvalds case XFS_ITEM_LOCKED:
522ff6d6af2SBill O'Donnell XFS_STATS_INC(mp, xs_push_ail_locked);
5239e4c109aSChristoph Hellwig trace_xfs_ail_locked(lip);
52443ff2122SChristoph Hellwig
525249a8c11SDavid Chinner stuck++;
5261da177e4SLinus Torvalds break;
5271da177e4SLinus Torvalds default:
5281da177e4SLinus Torvalds ASSERT(0);
5291da177e4SLinus Torvalds break;
5301da177e4SLinus Torvalds }
5311da177e4SLinus Torvalds
532249a8c11SDavid Chinner count++;
533249a8c11SDavid Chinner
534249a8c11SDavid Chinner /*
535249a8c11SDavid Chinner * Are there too many items we can't do anything with?
53643ff2122SChristoph Hellwig *
537b63da6c8SRandy Dunlap * If we are skipping too many items because we can't flush
538249a8c11SDavid Chinner * them or they are already being flushed, we back off and
539249a8c11SDavid Chinner * given them time to complete whatever operation is being
540249a8c11SDavid Chinner * done. i.e. remove pressure from the AIL while we can't make
541249a8c11SDavid Chinner * progress so traversals don't slow down further inserts and
542249a8c11SDavid Chinner * removals to/from the AIL.
543249a8c11SDavid Chinner *
544249a8c11SDavid Chinner * The value of 100 is an arbitrary magic number based on
545249a8c11SDavid Chinner * observation.
546249a8c11SDavid Chinner */
547249a8c11SDavid Chinner if (stuck > 100)
548249a8c11SDavid Chinner break;
549249a8c11SDavid Chinner
550af3e4022SDave Chinner lip = xfs_trans_ail_cursor_next(ailp, &cur);
551249a8c11SDavid Chinner if (lip == NULL)
552249a8c11SDavid Chinner break;
553249a8c11SDavid Chinner lsn = lip->li_lsn;
5541da177e4SLinus Torvalds }
555f376b45eSBrian Foster
556f376b45eSBrian Foster out_done:
557e4a1e29cSEric Sandeen xfs_trans_ail_cursor_done(&cur);
55857e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock);
5591da177e4SLinus Torvalds
56057e80956SMatthew Wilcox if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list))
56157e80956SMatthew Wilcox ailp->ail_log_flush++;
562d808f617SDave Chinner
56343ff2122SChristoph Hellwig if (!count || XFS_LSN_CMP(lsn, target) >= 0) {
564249a8c11SDavid Chinner /*
56543ff2122SChristoph Hellwig * We reached the target or the AIL is empty, so wait a bit
56643ff2122SChristoph Hellwig * longer for I/O to complete and remove pushed items from the
56743ff2122SChristoph Hellwig * AIL before we start the next scan from the start of the AIL.
568249a8c11SDavid Chinner */
569453eac8aSDave Chinner tout = 50;
57057e80956SMatthew Wilcox ailp->ail_last_pushed_lsn = 0;
57143ff2122SChristoph Hellwig } else if (((stuck + flushing) * 100) / count > 90) {
572249a8c11SDavid Chinner /*
57343ff2122SChristoph Hellwig * Either there is a lot of contention on the AIL or we are
57443ff2122SChristoph Hellwig * stuck due to operations in progress. "Stuck" in this case
57543ff2122SChristoph Hellwig * is defined as >90% of the items we tried to push were stuck.
576249a8c11SDavid Chinner *
577249a8c11SDavid Chinner * Backoff a bit more to allow some I/O to complete before
57843ff2122SChristoph Hellwig * restarting from the start of the AIL. This prevents us from
57943ff2122SChristoph Hellwig * spinning on the same items, and if they are pinned will all
58043ff2122SChristoph Hellwig * the restart to issue a log force to unpin the stuck items.
581249a8c11SDavid Chinner */
582453eac8aSDave Chinner tout = 20;
58357e80956SMatthew Wilcox ailp->ail_last_pushed_lsn = 0;
58443ff2122SChristoph Hellwig } else {
58543ff2122SChristoph Hellwig /*
58643ff2122SChristoph Hellwig * Assume we have more work to do in a short while.
58743ff2122SChristoph Hellwig */
58843ff2122SChristoph Hellwig tout = 10;
589453eac8aSDave Chinner }
5901da177e4SLinus Torvalds
5910030807cSChristoph Hellwig return tout;
5920030807cSChristoph Hellwig }
5930030807cSChristoph Hellwig
5940030807cSChristoph Hellwig static int
xfsaild(void * data)5950030807cSChristoph Hellwig xfsaild(
5960030807cSChristoph Hellwig void *data)
5970030807cSChristoph Hellwig {
5980030807cSChristoph Hellwig struct xfs_ail *ailp = data;
5990030807cSChristoph Hellwig long tout = 0; /* milliseconds */
60010a98cb1SEric Biggers unsigned int noreclaim_flag;
6010030807cSChristoph Hellwig
60210a98cb1SEric Biggers noreclaim_flag = memalloc_noreclaim_save();
60318f1df4eSMichal Hocko set_freezable();
60443ff2122SChristoph Hellwig
6050bd89676SHou Tao while (1) {
6060030807cSChristoph Hellwig if (tout && tout <= 20)
607f5d39b02SPeter Zijlstra set_current_state(TASK_KILLABLE|TASK_FREEZABLE);
6080030807cSChristoph Hellwig else
609f5d39b02SPeter Zijlstra set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
6100bd89676SHou Tao
6110bd89676SHou Tao /*
612efc3289cSBrian Foster * Check kthread_should_stop() after we set the task state to
613efc3289cSBrian Foster * guarantee that we either see the stop bit and exit or the
614efc3289cSBrian Foster * task state is reset to runnable such that it's not scheduled
615efc3289cSBrian Foster * out indefinitely and detects the stop bit at next iteration.
6160bd89676SHou Tao * A memory barrier is included in above task state set to
6170bd89676SHou Tao * serialize again kthread_stop().
6180bd89676SHou Tao */
6190bd89676SHou Tao if (kthread_should_stop()) {
6200bd89676SHou Tao __set_current_state(TASK_RUNNING);
621efc3289cSBrian Foster
622efc3289cSBrian Foster /*
623efc3289cSBrian Foster * The caller forces out the AIL before stopping the
624efc3289cSBrian Foster * thread in the common case, which means the delwri
625efc3289cSBrian Foster * queue is drained. In the shutdown case, the queue may
626efc3289cSBrian Foster * still hold relogged buffers that haven't been
627efc3289cSBrian Foster * submitted because they were pinned since added to the
628efc3289cSBrian Foster * queue.
629efc3289cSBrian Foster *
630efc3289cSBrian Foster * Log I/O error processing stales the underlying buffer
631efc3289cSBrian Foster * and clears the delwri state, expecting the buf to be
632efc3289cSBrian Foster * removed on the next submission attempt. That won't
633efc3289cSBrian Foster * happen if we're shutting down, so this is the last
634efc3289cSBrian Foster * opportunity to release such buffers from the queue.
635efc3289cSBrian Foster */
636efc3289cSBrian Foster ASSERT(list_empty(&ailp->ail_buf_list) ||
6378eda8721SDave Chinner xlog_is_shutdown(ailp->ail_log));
638efc3289cSBrian Foster xfs_buf_delwri_cancel(&ailp->ail_buf_list);
6390bd89676SHou Tao break;
6400bd89676SHou Tao }
6418375f922SBrian Foster
64257e80956SMatthew Wilcox spin_lock(&ailp->ail_lock);
6438375f922SBrian Foster
6448375f922SBrian Foster /*
6458375f922SBrian Foster * Idle if the AIL is empty and we are not racing with a target
6468375f922SBrian Foster * update. We check the AIL after we set the task to a sleep
64757e80956SMatthew Wilcox * state to guarantee that we either catch an ail_target update
6488375f922SBrian Foster * or that a wake_up resets the state to TASK_RUNNING.
6498375f922SBrian Foster * Otherwise, we run the risk of sleeping indefinitely.
6508375f922SBrian Foster *
65157e80956SMatthew Wilcox * The barrier matches the ail_target update in xfs_ail_push().
6528375f922SBrian Foster */
6538375f922SBrian Foster smp_rmb();
6548375f922SBrian Foster if (!xfs_ail_min(ailp) &&
655f376b45eSBrian Foster ailp->ail_target == ailp->ail_target_prev &&
656f376b45eSBrian Foster list_empty(&ailp->ail_buf_list)) {
65757e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock);
658f5d39b02SPeter Zijlstra schedule();
6598375f922SBrian Foster tout = 0;
6608375f922SBrian Foster continue;
6618375f922SBrian Foster }
66257e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock);
6638375f922SBrian Foster
6648375f922SBrian Foster if (tout)
665f5d39b02SPeter Zijlstra schedule_timeout(msecs_to_jiffies(tout));
6668375f922SBrian Foster
6678375f922SBrian Foster __set_current_state(TASK_RUNNING);
6680030807cSChristoph Hellwig
6690030807cSChristoph Hellwig try_to_freeze();
6700030807cSChristoph Hellwig
6710030807cSChristoph Hellwig tout = xfsaild_push(ailp);
6720030807cSChristoph Hellwig }
6730030807cSChristoph Hellwig
67410a98cb1SEric Biggers memalloc_noreclaim_restore(noreclaim_flag);
6750030807cSChristoph Hellwig return 0;
6760bf6a5bdSDave Chinner }
6770bf6a5bdSDave Chinner
6780bf6a5bdSDave Chinner /*
6790bf6a5bdSDave Chinner * This routine is called to move the tail of the AIL forward. It does this by
6800bf6a5bdSDave Chinner * trying to flush items in the AIL whose lsns are below the given
6810bf6a5bdSDave Chinner * threshold_lsn.
6820bf6a5bdSDave Chinner *
6830bf6a5bdSDave Chinner * The push is run asynchronously in a workqueue, which means the caller needs
6840bf6a5bdSDave Chinner * to handle waiting on the async flush for space to become available.
6850bf6a5bdSDave Chinner * We don't want to interrupt any push that is in progress, hence we only queue
686cf085a1bSJoe Perches * work if we set the pushing bit appropriately.
6870bf6a5bdSDave Chinner *
6880bf6a5bdSDave Chinner * We do this unlocked - we only need to know whether there is anything in the
6890bf6a5bdSDave Chinner * AIL at the time we are called. We don't need to access the contents of
6900bf6a5bdSDave Chinner * any of the objects, so the lock is not needed.
6910bf6a5bdSDave Chinner */
6920bf6a5bdSDave Chinner void
xfs_ail_push(struct xfs_ail * ailp,xfs_lsn_t threshold_lsn)693fd074841SDave Chinner xfs_ail_push(
6940bf6a5bdSDave Chinner struct xfs_ail *ailp,
6950bf6a5bdSDave Chinner xfs_lsn_t threshold_lsn)
6960bf6a5bdSDave Chinner {
697efe2330fSChristoph Hellwig struct xfs_log_item *lip;
6980bf6a5bdSDave Chinner
6990bf6a5bdSDave Chinner lip = xfs_ail_min(ailp);
7008eda8721SDave Chinner if (!lip || xlog_is_shutdown(ailp->ail_log) ||
70157e80956SMatthew Wilcox XFS_LSN_CMP(threshold_lsn, ailp->ail_target) <= 0)
7020bf6a5bdSDave Chinner return;
7030bf6a5bdSDave Chinner
7040bf6a5bdSDave Chinner /*
7050bf6a5bdSDave Chinner * Ensure that the new target is noticed in push code before it clears
7060bf6a5bdSDave Chinner * the XFS_AIL_PUSHING_BIT.
7070bf6a5bdSDave Chinner */
7080bf6a5bdSDave Chinner smp_wmb();
70957e80956SMatthew Wilcox xfs_trans_ail_copy_lsn(ailp, &ailp->ail_target, &threshold_lsn);
7100030807cSChristoph Hellwig smp_wmb();
7110030807cSChristoph Hellwig
71257e80956SMatthew Wilcox wake_up_process(ailp->ail_task);
7130bf6a5bdSDave Chinner }
7141da177e4SLinus Torvalds
7151da177e4SLinus Torvalds /*
716fd074841SDave Chinner * Push out all items in the AIL immediately
717fd074841SDave Chinner */
718fd074841SDave Chinner void
xfs_ail_push_all(struct xfs_ail * ailp)719fd074841SDave Chinner xfs_ail_push_all(
720fd074841SDave Chinner struct xfs_ail *ailp)
721fd074841SDave Chinner {
722fd074841SDave Chinner xfs_lsn_t threshold_lsn = xfs_ail_max_lsn(ailp);
723fd074841SDave Chinner
724fd074841SDave Chinner if (threshold_lsn)
725fd074841SDave Chinner xfs_ail_push(ailp, threshold_lsn);
726fd074841SDave Chinner }
727fd074841SDave Chinner
728fd074841SDave Chinner /*
729211e4d43SChristoph Hellwig * Push out all items in the AIL immediately and wait until the AIL is empty.
730211e4d43SChristoph Hellwig */
731211e4d43SChristoph Hellwig void
xfs_ail_push_all_sync(struct xfs_ail * ailp)732211e4d43SChristoph Hellwig xfs_ail_push_all_sync(
733211e4d43SChristoph Hellwig struct xfs_ail *ailp)
734211e4d43SChristoph Hellwig {
735211e4d43SChristoph Hellwig DEFINE_WAIT(wait);
736211e4d43SChristoph Hellwig
73757e80956SMatthew Wilcox spin_lock(&ailp->ail_lock);
738fc93812cSColin Ian King while (xfs_ail_max(ailp) != NULL) {
73957e80956SMatthew Wilcox prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE);
74057e80956SMatthew Wilcox wake_up_process(ailp->ail_task);
74157e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock);
742211e4d43SChristoph Hellwig schedule();
74357e80956SMatthew Wilcox spin_lock(&ailp->ail_lock);
744211e4d43SChristoph Hellwig }
74557e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock);
746211e4d43SChristoph Hellwig
74757e80956SMatthew Wilcox finish_wait(&ailp->ail_empty, &wait);
748211e4d43SChristoph Hellwig }
749211e4d43SChristoph Hellwig
7504165994aSDave Chinner void
xfs_ail_update_finish(struct xfs_ail * ailp,xfs_lsn_t old_lsn)7514165994aSDave Chinner xfs_ail_update_finish(
7524165994aSDave Chinner struct xfs_ail *ailp,
7538eb807bdSDave Chinner xfs_lsn_t old_lsn) __releases(ailp->ail_lock)
7544165994aSDave Chinner {
7558eda8721SDave Chinner struct xlog *log = ailp->ail_log;
7564165994aSDave Chinner
7578eb807bdSDave Chinner /* if the tail lsn hasn't changed, don't do updates or wakeups. */
7588eb807bdSDave Chinner if (!old_lsn || old_lsn == __xfs_ail_min_lsn(ailp)) {
7594165994aSDave Chinner spin_unlock(&ailp->ail_lock);
7604165994aSDave Chinner return;
7614165994aSDave Chinner }
7624165994aSDave Chinner
7638eda8721SDave Chinner if (!xlog_is_shutdown(log))
7648eda8721SDave Chinner xlog_assign_tail_lsn_locked(log->l_mp);
7654165994aSDave Chinner
7664165994aSDave Chinner if (list_empty(&ailp->ail_head))
7674165994aSDave Chinner wake_up_all(&ailp->ail_empty);
7684165994aSDave Chinner spin_unlock(&ailp->ail_lock);
7698eda8721SDave Chinner xfs_log_space_wake(log->l_mp);
7704165994aSDave Chinner }
7714165994aSDave Chinner
772211e4d43SChristoph Hellwig /*
7730e57f6a3SDave Chinner * xfs_trans_ail_update - bulk AIL insertion operation.
7740e57f6a3SDave Chinner *
7750e57f6a3SDave Chinner * @xfs_trans_ail_update takes an array of log items that all need to be
7760e57f6a3SDave Chinner * positioned at the same LSN in the AIL. If an item is not in the AIL, it will
7770e57f6a3SDave Chinner * be added. Otherwise, it will be repositioned by removing it and re-adding
7780e57f6a3SDave Chinner * it to the AIL. If we move the first item in the AIL, update the log tail to
7790e57f6a3SDave Chinner * match the new minimum LSN in the AIL.
7800e57f6a3SDave Chinner *
7810e57f6a3SDave Chinner * This function takes the AIL lock once to execute the update operations on
7820e57f6a3SDave Chinner * all the items in the array, and as such should not be called with the AIL
7830e57f6a3SDave Chinner * lock held. As a result, once we have the AIL lock, we need to check each log
7840e57f6a3SDave Chinner * item LSN to confirm it needs to be moved forward in the AIL.
7850e57f6a3SDave Chinner *
7860e57f6a3SDave Chinner * To optimise the insert operation, we delete all the items from the AIL in
7870e57f6a3SDave Chinner * the first pass, moving them into a temporary list, then splice the temporary
7880e57f6a3SDave Chinner * list into the correct position in the AIL. This avoids needing to do an
7890e57f6a3SDave Chinner * insert operation on every item.
7900e57f6a3SDave Chinner *
7910e57f6a3SDave Chinner * This function must be called with the AIL lock held. The lock is dropped
7920e57f6a3SDave Chinner * before returning.
7930e57f6a3SDave Chinner */
7940e57f6a3SDave Chinner void
xfs_trans_ail_update_bulk(struct xfs_ail * ailp,struct xfs_ail_cursor * cur,struct xfs_log_item ** log_items,int nr_items,xfs_lsn_t lsn)7950e57f6a3SDave Chinner xfs_trans_ail_update_bulk(
7960e57f6a3SDave Chinner struct xfs_ail *ailp,
7971d8c95a3SDave Chinner struct xfs_ail_cursor *cur,
7980e57f6a3SDave Chinner struct xfs_log_item **log_items,
7990e57f6a3SDave Chinner int nr_items,
80057e80956SMatthew Wilcox xfs_lsn_t lsn) __releases(ailp->ail_lock)
8010e57f6a3SDave Chinner {
802efe2330fSChristoph Hellwig struct xfs_log_item *mlip;
8038eb807bdSDave Chinner xfs_lsn_t tail_lsn = 0;
8040e57f6a3SDave Chinner int i;
8050e57f6a3SDave Chinner LIST_HEAD(tmp);
8060e57f6a3SDave Chinner
807e44f4112SAlex Elder ASSERT(nr_items > 0); /* Not required, but true. */
8080e57f6a3SDave Chinner mlip = xfs_ail_min(ailp);
8090e57f6a3SDave Chinner
8100e57f6a3SDave Chinner for (i = 0; i < nr_items; i++) {
8110e57f6a3SDave Chinner struct xfs_log_item *lip = log_items[i];
81222525c17SDave Chinner if (test_and_set_bit(XFS_LI_IN_AIL, &lip->li_flags)) {
8130e57f6a3SDave Chinner /* check if we really need to move the item */
8140e57f6a3SDave Chinner if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0)
8150e57f6a3SDave Chinner continue;
8160e57f6a3SDave Chinner
817750b9c90SDave Chinner trace_xfs_ail_move(lip, lip->li_lsn, lsn);
8188eb807bdSDave Chinner if (mlip == lip && !tail_lsn)
8198eb807bdSDave Chinner tail_lsn = lip->li_lsn;
8208eb807bdSDave Chinner
8210e57f6a3SDave Chinner xfs_ail_delete(ailp, lip);
8220e57f6a3SDave Chinner } else {
823750b9c90SDave Chinner trace_xfs_ail_insert(lip, 0, lsn);
8240e57f6a3SDave Chinner }
8250e57f6a3SDave Chinner lip->li_lsn = lsn;
826*939bd50dSDave Chinner list_add_tail(&lip->li_ail, &tmp);
8270e57f6a3SDave Chinner }
8280e57f6a3SDave Chinner
829e44f4112SAlex Elder if (!list_empty(&tmp))
8301d8c95a3SDave Chinner xfs_ail_splice(ailp, cur, &tmp, lsn);
8311c304625SChristoph Hellwig
8328eb807bdSDave Chinner xfs_ail_update_finish(ailp, tail_lsn);
8330e57f6a3SDave Chinner }
8340e57f6a3SDave Chinner
83586a37174SDarrick J. Wong /* Insert a log item into the AIL. */
83686a37174SDarrick J. Wong void
xfs_trans_ail_insert(struct xfs_ail * ailp,struct xfs_log_item * lip,xfs_lsn_t lsn)83786a37174SDarrick J. Wong xfs_trans_ail_insert(
83886a37174SDarrick J. Wong struct xfs_ail *ailp,
83986a37174SDarrick J. Wong struct xfs_log_item *lip,
84086a37174SDarrick J. Wong xfs_lsn_t lsn)
84186a37174SDarrick J. Wong {
84286a37174SDarrick J. Wong spin_lock(&ailp->ail_lock);
84386a37174SDarrick J. Wong xfs_trans_ail_update_bulk(ailp, NULL, &lip, 1, lsn);
84486a37174SDarrick J. Wong }
84586a37174SDarrick J. Wong
8468eb807bdSDave Chinner /*
8478eb807bdSDave Chinner * Delete one log item from the AIL.
8488eb807bdSDave Chinner *
8498eb807bdSDave Chinner * If this item was at the tail of the AIL, return the LSN of the log item so
8508eb807bdSDave Chinner * that we can use it to check if the LSN of the tail of the log has moved
8518eb807bdSDave Chinner * when finishing up the AIL delete process in xfs_ail_update_finish().
8528eb807bdSDave Chinner */
8538eb807bdSDave Chinner xfs_lsn_t
xfs_ail_delete_one(struct xfs_ail * ailp,struct xfs_log_item * lip)85427af1bbfSChristoph Hellwig xfs_ail_delete_one(
85527af1bbfSChristoph Hellwig struct xfs_ail *ailp,
85627af1bbfSChristoph Hellwig struct xfs_log_item *lip)
85727af1bbfSChristoph Hellwig {
85827af1bbfSChristoph Hellwig struct xfs_log_item *mlip = xfs_ail_min(ailp);
8598eb807bdSDave Chinner xfs_lsn_t lsn = lip->li_lsn;
86027af1bbfSChristoph Hellwig
86127af1bbfSChristoph Hellwig trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);
86227af1bbfSChristoph Hellwig xfs_ail_delete(ailp, lip);
86322525c17SDave Chinner clear_bit(XFS_LI_IN_AIL, &lip->li_flags);
86427af1bbfSChristoph Hellwig lip->li_lsn = 0;
86527af1bbfSChristoph Hellwig
8668eb807bdSDave Chinner if (mlip == lip)
8678eb807bdSDave Chinner return lsn;
8688eb807bdSDave Chinner return 0;
86927af1bbfSChristoph Hellwig }
87027af1bbfSChristoph Hellwig
87130136832SDave Chinner void
xfs_trans_ail_delete(struct xfs_log_item * lip,int shutdown_type)87227af1bbfSChristoph Hellwig xfs_trans_ail_delete(
87327af1bbfSChristoph Hellwig struct xfs_log_item *lip,
8744165994aSDave Chinner int shutdown_type)
87530136832SDave Chinner {
876849274c1SBrian Foster struct xfs_ail *ailp = lip->li_ailp;
877b5f17becSDave Chinner struct xlog *log = ailp->ail_log;
8788eb807bdSDave Chinner xfs_lsn_t tail_lsn;
87930136832SDave Chinner
880849274c1SBrian Foster spin_lock(&ailp->ail_lock);
88122525c17SDave Chinner if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) {
88257e80956SMatthew Wilcox spin_unlock(&ailp->ail_lock);
883b5f17becSDave Chinner if (shutdown_type && !xlog_is_shutdown(log)) {
884b5f17becSDave Chinner xfs_alert_tag(log->l_mp, XFS_PTAG_AILDELETE,
88530136832SDave Chinner "%s: attempting to delete a log item that is not in the AIL",
88630136832SDave Chinner __func__);
887b5f17becSDave Chinner xlog_force_shutdown(log, shutdown_type);
88830136832SDave Chinner }
88930136832SDave Chinner return;
89030136832SDave Chinner }
89130136832SDave Chinner
8922b3cf093SBrian Foster /* xfs_ail_update_finish() drops the AIL lock */
893e98084b8SDave Chinner xfs_clear_li_failed(lip);
8948eb807bdSDave Chinner tail_lsn = xfs_ail_delete_one(ailp, lip);
8958eb807bdSDave Chinner xfs_ail_update_finish(ailp, tail_lsn);
89630136832SDave Chinner }
8971da177e4SLinus Torvalds
898249a8c11SDavid Chinner int
xfs_trans_ail_init(xfs_mount_t * mp)8991da177e4SLinus Torvalds xfs_trans_ail_init(
9001da177e4SLinus Torvalds xfs_mount_t *mp)
9011da177e4SLinus Torvalds {
90282fa9012SDavid Chinner struct xfs_ail *ailp;
90382fa9012SDavid Chinner
90482fa9012SDavid Chinner ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL);
90582fa9012SDavid Chinner if (!ailp)
9062451337dSDave Chinner return -ENOMEM;
90782fa9012SDavid Chinner
9088eda8721SDave Chinner ailp->ail_log = mp->m_log;
90957e80956SMatthew Wilcox INIT_LIST_HEAD(&ailp->ail_head);
91057e80956SMatthew Wilcox INIT_LIST_HEAD(&ailp->ail_cursors);
91157e80956SMatthew Wilcox spin_lock_init(&ailp->ail_lock);
91257e80956SMatthew Wilcox INIT_LIST_HEAD(&ailp->ail_buf_list);
91357e80956SMatthew Wilcox init_waitqueue_head(&ailp->ail_empty);
9140030807cSChristoph Hellwig
91557e80956SMatthew Wilcox ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s",
9168eda8721SDave Chinner mp->m_super->s_id);
91757e80956SMatthew Wilcox if (IS_ERR(ailp->ail_task))
9180030807cSChristoph Hellwig goto out_free_ailp;
9190030807cSChristoph Hellwig
92027d8d5feSDavid Chinner mp->m_ail = ailp;
92127d8d5feSDavid Chinner return 0;
9220030807cSChristoph Hellwig
9230030807cSChristoph Hellwig out_free_ailp:
9240030807cSChristoph Hellwig kmem_free(ailp);
9252451337dSDave Chinner return -ENOMEM;
926249a8c11SDavid Chinner }
927249a8c11SDavid Chinner
928249a8c11SDavid Chinner void
xfs_trans_ail_destroy(xfs_mount_t * mp)929249a8c11SDavid Chinner xfs_trans_ail_destroy(
930249a8c11SDavid Chinner xfs_mount_t *mp)
931249a8c11SDavid Chinner {
93282fa9012SDavid Chinner struct xfs_ail *ailp = mp->m_ail;
93382fa9012SDavid Chinner
93457e80956SMatthew Wilcox kthread_stop(ailp->ail_task);
93582fa9012SDavid Chinner kmem_free(ailp);
9361da177e4SLinus Torvalds }
937