10b61f8a4SDave Chinner // SPDX-License-Identifier: GPL-2.0
271e330b5SDave Chinner /*
371e330b5SDave Chinner * Copyright (c) 2010 Red Hat, Inc. All Rights Reserved.
471e330b5SDave Chinner */
571e330b5SDave Chinner
671e330b5SDave Chinner #include "xfs.h"
771e330b5SDave Chinner #include "xfs_fs.h"
84fb6e8adSChristoph Hellwig #include "xfs_format.h"
9239880efSDave Chinner #include "xfs_log_format.h"
1070a9883cSDave Chinner #include "xfs_shared.h"
11239880efSDave Chinner #include "xfs_trans_resv.h"
1271e330b5SDave Chinner #include "xfs_mount.h"
13efc27b52SDave Chinner #include "xfs_extent_busy.h"
14239880efSDave Chinner #include "xfs_trans.h"
15239880efSDave Chinner #include "xfs_trans_priv.h"
16239880efSDave Chinner #include "xfs_log.h"
17239880efSDave Chinner #include "xfs_log_priv.h"
184560e78fSChristoph Hellwig #include "xfs_trace.h"
19*428c4435SDave Chinner #include "xfs_discard.h"
2071e330b5SDave Chinner
2171e330b5SDave Chinner /*
2271e330b5SDave Chinner * Allocate a new ticket. Failing to get a new ticket makes it really hard to
2371e330b5SDave Chinner * recover, so we don't allow failure here. Also, we allocate in a context that
2471e330b5SDave Chinner * we don't want to be issuing transactions from, so we need to tell the
2571e330b5SDave Chinner * allocation code this as well.
2671e330b5SDave Chinner *
2771e330b5SDave Chinner * We don't reserve any space for the ticket - we are going to steal whatever
2871e330b5SDave Chinner * space we require from transactions as they commit. To ensure we reserve all
2971e330b5SDave Chinner * the space required, we need to set the current reservation of the ticket to
3071e330b5SDave Chinner * zero so that we know to steal the initial transaction overhead from the
3171e330b5SDave Chinner * first transaction commit.
3271e330b5SDave Chinner */
3371e330b5SDave Chinner static struct xlog_ticket *
xlog_cil_ticket_alloc(struct xlog * log)3471e330b5SDave Chinner xlog_cil_ticket_alloc(
35f7bdf03aSMark Tinguely struct xlog *log)
3671e330b5SDave Chinner {
3771e330b5SDave Chinner struct xlog_ticket *tic;
3871e330b5SDave Chinner
39c7610dceSDave Chinner tic = xlog_ticket_alloc(log, 0, 1, 0);
4071e330b5SDave Chinner
4171e330b5SDave Chinner /*
4271e330b5SDave Chinner * set the current reservation to zero so we know to steal the basic
4371e330b5SDave Chinner * transaction overhead reservation from the first transaction commit.
4471e330b5SDave Chinner */
4571e330b5SDave Chinner tic->t_curr_res = 0;
4631151cc3SDave Chinner tic->t_iclog_hdrs = 0;
4771e330b5SDave Chinner return tic;
4871e330b5SDave Chinner }
4971e330b5SDave Chinner
5031151cc3SDave Chinner static inline void
xlog_cil_set_iclog_hdr_count(struct xfs_cil * cil)5131151cc3SDave Chinner xlog_cil_set_iclog_hdr_count(struct xfs_cil *cil)
5231151cc3SDave Chinner {
5331151cc3SDave Chinner struct xlog *log = cil->xc_log;
5431151cc3SDave Chinner
5531151cc3SDave Chinner atomic_set(&cil->xc_iclog_hdrs,
5631151cc3SDave Chinner (XLOG_CIL_BLOCKING_SPACE_LIMIT(log) /
5731151cc3SDave Chinner (log->l_iclog_size - log->l_iclog_hsize)));
5831151cc3SDave Chinner }
5931151cc3SDave Chinner
6071e330b5SDave Chinner /*
6122b1afc5SDave Chinner * Check if the current log item was first committed in this sequence.
6222b1afc5SDave Chinner * We can't rely on just the log item being in the CIL, we have to check
6322b1afc5SDave Chinner * the recorded commit sequence number.
6422b1afc5SDave Chinner *
6522b1afc5SDave Chinner * Note: for this to be used in a non-racy manner, it has to be called with
6622b1afc5SDave Chinner * CIL flushing locked out. As a result, it should only be used during the
6722b1afc5SDave Chinner * transaction commit process when deciding what to format into the item.
6822b1afc5SDave Chinner */
6922b1afc5SDave Chinner static bool
xlog_item_in_current_chkpt(struct xfs_cil * cil,struct xfs_log_item * lip)7022b1afc5SDave Chinner xlog_item_in_current_chkpt(
7122b1afc5SDave Chinner struct xfs_cil *cil,
7222b1afc5SDave Chinner struct xfs_log_item *lip)
7322b1afc5SDave Chinner {
7488591e7fSDave Chinner if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags))
7522b1afc5SDave Chinner return false;
7622b1afc5SDave Chinner
7722b1afc5SDave Chinner /*
7822b1afc5SDave Chinner * li_seq is written on the first commit of a log item to record the
7922b1afc5SDave Chinner * first checkpoint it is written to. Hence if it is different to the
8022b1afc5SDave Chinner * current sequence, we're in a new checkpoint.
8122b1afc5SDave Chinner */
8222b1afc5SDave Chinner return lip->li_seq == READ_ONCE(cil->xc_current_sequence);
8322b1afc5SDave Chinner }
8422b1afc5SDave Chinner
8522b1afc5SDave Chinner bool
xfs_log_item_in_current_chkpt(struct xfs_log_item * lip)8622b1afc5SDave Chinner xfs_log_item_in_current_chkpt(
8722b1afc5SDave Chinner struct xfs_log_item *lip)
8822b1afc5SDave Chinner {
8922b1afc5SDave Chinner return xlog_item_in_current_chkpt(lip->li_log->l_cilp, lip);
9022b1afc5SDave Chinner }
9122b1afc5SDave Chinner
9222b1afc5SDave Chinner /*
9339823d0fSDave Chinner * Unavoidable forward declaration - xlog_cil_push_work() calls
9439823d0fSDave Chinner * xlog_cil_ctx_alloc() itself.
9539823d0fSDave Chinner */
9639823d0fSDave Chinner static void xlog_cil_push_work(struct work_struct *work);
9739823d0fSDave Chinner
9839823d0fSDave Chinner static struct xfs_cil_ctx *
xlog_cil_ctx_alloc(void)9939823d0fSDave Chinner xlog_cil_ctx_alloc(void)
10039823d0fSDave Chinner {
10139823d0fSDave Chinner struct xfs_cil_ctx *ctx;
10239823d0fSDave Chinner
10339823d0fSDave Chinner ctx = kmem_zalloc(sizeof(*ctx), KM_NOFS);
10439823d0fSDave Chinner INIT_LIST_HEAD(&ctx->committing);
105*428c4435SDave Chinner INIT_LIST_HEAD(&ctx->busy_extents.extent_list);
106c0fb4765SDave Chinner INIT_LIST_HEAD(&ctx->log_items);
10716924853SDave Chinner INIT_LIST_HEAD(&ctx->lv_chain);
10839823d0fSDave Chinner INIT_WORK(&ctx->push_work, xlog_cil_push_work);
10939823d0fSDave Chinner return ctx;
11039823d0fSDave Chinner }
11139823d0fSDave Chinner
1127c8ade21SDave Chinner /*
1137c8ade21SDave Chinner * Aggregate the CIL per cpu structures into global counts, lists, etc and
1147c8ade21SDave Chinner * clear the percpu state ready for the next context to use. This is called
1157c8ade21SDave Chinner * from the push code with the context lock held exclusively, hence nothing else
1167c8ade21SDave Chinner * will be accessing or modifying the per-cpu counters.
1177c8ade21SDave Chinner */
1187c8ade21SDave Chinner static void
xlog_cil_push_pcp_aggregate(struct xfs_cil * cil,struct xfs_cil_ctx * ctx)1197c8ade21SDave Chinner xlog_cil_push_pcp_aggregate(
1207c8ade21SDave Chinner struct xfs_cil *cil,
1217c8ade21SDave Chinner struct xfs_cil_ctx *ctx)
1227c8ade21SDave Chinner {
1237c8ade21SDave Chinner struct xlog_cil_pcp *cilpcp;
1247c8ade21SDave Chinner int cpu;
1257c8ade21SDave Chinner
126ecd49f7aSDarrick J. Wong for_each_cpu(cpu, &ctx->cil_pcpmask) {
1277c8ade21SDave Chinner cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
1287c8ade21SDave Chinner
1291dd2a2c1SDave Chinner ctx->ticket->t_curr_res += cilpcp->space_reserved;
1301dd2a2c1SDave Chinner cilpcp->space_reserved = 0;
1311dd2a2c1SDave Chinner
132df7a4a21SDave Chinner if (!list_empty(&cilpcp->busy_extents)) {
133df7a4a21SDave Chinner list_splice_init(&cilpcp->busy_extents,
134*428c4435SDave Chinner &ctx->busy_extents.extent_list);
135df7a4a21SDave Chinner }
136c0fb4765SDave Chinner if (!list_empty(&cilpcp->log_items))
137c0fb4765SDave Chinner list_splice_init(&cilpcp->log_items, &ctx->log_items);
138df7a4a21SDave Chinner
1397c8ade21SDave Chinner /*
1407c8ade21SDave Chinner * We're in the middle of switching cil contexts. Reset the
1417c8ade21SDave Chinner * counter we use to detect when the current context is nearing
1427c8ade21SDave Chinner * full.
1437c8ade21SDave Chinner */
1447c8ade21SDave Chinner cilpcp->space_used = 0;
1457c8ade21SDave Chinner }
1467c8ade21SDave Chinner }
1477c8ade21SDave Chinner
1487c8ade21SDave Chinner /*
1497c8ade21SDave Chinner * Aggregate the CIL per-cpu space used counters into the global atomic value.
1507c8ade21SDave Chinner * This is called when the per-cpu counter aggregation will first pass the soft
1517c8ade21SDave Chinner * limit threshold so we can switch to atomic counter aggregation for accurate
1527c8ade21SDave Chinner * detection of hard limit traversal.
1537c8ade21SDave Chinner */
1547c8ade21SDave Chinner static void
xlog_cil_insert_pcp_aggregate(struct xfs_cil * cil,struct xfs_cil_ctx * ctx)1557c8ade21SDave Chinner xlog_cil_insert_pcp_aggregate(
1567c8ade21SDave Chinner struct xfs_cil *cil,
1577c8ade21SDave Chinner struct xfs_cil_ctx *ctx)
1587c8ade21SDave Chinner {
1597c8ade21SDave Chinner struct xlog_cil_pcp *cilpcp;
1607c8ade21SDave Chinner int cpu;
1617c8ade21SDave Chinner int count = 0;
1627c8ade21SDave Chinner
1637c8ade21SDave Chinner /* Trigger atomic updates then aggregate only for the first caller */
1647c8ade21SDave Chinner if (!test_and_clear_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags))
1657c8ade21SDave Chinner return;
1667c8ade21SDave Chinner
167ecd49f7aSDarrick J. Wong /*
168ecd49f7aSDarrick J. Wong * We can race with other cpus setting cil_pcpmask. However, we've
169ecd49f7aSDarrick J. Wong * atomically cleared PCP_SPACE which forces other threads to add to
170ecd49f7aSDarrick J. Wong * the global space used count. cil_pcpmask is a superset of cilpcp
171ecd49f7aSDarrick J. Wong * structures that could have a nonzero space_used.
172ecd49f7aSDarrick J. Wong */
173ecd49f7aSDarrick J. Wong for_each_cpu(cpu, &ctx->cil_pcpmask) {
1747c8ade21SDave Chinner int old, prev;
1757c8ade21SDave Chinner
1767c8ade21SDave Chinner cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
1777c8ade21SDave Chinner do {
1787c8ade21SDave Chinner old = cilpcp->space_used;
1797c8ade21SDave Chinner prev = cmpxchg(&cilpcp->space_used, old, 0);
1807c8ade21SDave Chinner } while (old != prev);
1817c8ade21SDave Chinner count += old;
1827c8ade21SDave Chinner }
1837c8ade21SDave Chinner atomic_add(count, &ctx->space_used);
1847c8ade21SDave Chinner }
1857c8ade21SDave Chinner
18639823d0fSDave Chinner static void
xlog_cil_ctx_switch(struct xfs_cil * cil,struct xfs_cil_ctx * ctx)18739823d0fSDave Chinner xlog_cil_ctx_switch(
18839823d0fSDave Chinner struct xfs_cil *cil,
18939823d0fSDave Chinner struct xfs_cil_ctx *ctx)
19039823d0fSDave Chinner {
19131151cc3SDave Chinner xlog_cil_set_iclog_hdr_count(cil);
19288591e7fSDave Chinner set_bit(XLOG_CIL_EMPTY, &cil->xc_flags);
1937c8ade21SDave Chinner set_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags);
19439823d0fSDave Chinner ctx->sequence = ++cil->xc_current_sequence;
19539823d0fSDave Chinner ctx->cil = cil;
19639823d0fSDave Chinner cil->xc_ctx = ctx;
19739823d0fSDave Chinner }
19839823d0fSDave Chinner
19939823d0fSDave Chinner /*
20071e330b5SDave Chinner * After the first stage of log recovery is done, we know where the head and
20171e330b5SDave Chinner * tail of the log are. We need this log initialisation done before we can
20271e330b5SDave Chinner * initialise the first CIL checkpoint context.
20371e330b5SDave Chinner *
20471e330b5SDave Chinner * Here we allocate a log ticket to track space usage during a CIL push. This
20571e330b5SDave Chinner * ticket is passed to xlog_write() directly so that we don't slowly leak log
20671e330b5SDave Chinner * space by failing to account for space used by log headers and additional
20771e330b5SDave Chinner * region headers for split regions.
20871e330b5SDave Chinner */
20971e330b5SDave Chinner void
xlog_cil_init_post_recovery(struct xlog * log)21071e330b5SDave Chinner xlog_cil_init_post_recovery(
211f7bdf03aSMark Tinguely struct xlog *log)
21271e330b5SDave Chinner {
21371e330b5SDave Chinner log->l_cilp->xc_ctx->ticket = xlog_cil_ticket_alloc(log);
21471e330b5SDave Chinner log->l_cilp->xc_ctx->sequence = 1;
21531151cc3SDave Chinner xlog_cil_set_iclog_hdr_count(log->l_cilp);
21671e330b5SDave Chinner }
21771e330b5SDave Chinner
218b1c5ebb2SDave Chinner static inline int
xlog_cil_iovec_space(uint niovecs)219b1c5ebb2SDave Chinner xlog_cil_iovec_space(
220b1c5ebb2SDave Chinner uint niovecs)
221b1c5ebb2SDave Chinner {
222b1c5ebb2SDave Chinner return round_up((sizeof(struct xfs_log_vec) +
223b1c5ebb2SDave Chinner niovecs * sizeof(struct xfs_log_iovec)),
224b1c5ebb2SDave Chinner sizeof(uint64_t));
225b1c5ebb2SDave Chinner }
226b1c5ebb2SDave Chinner
227b1c5ebb2SDave Chinner /*
228b1c5ebb2SDave Chinner * Allocate or pin log vector buffers for CIL insertion.
229b1c5ebb2SDave Chinner *
230b1c5ebb2SDave Chinner * The CIL currently uses disposable buffers for copying a snapshot of the
231b1c5ebb2SDave Chinner * modified items into the log during a push. The biggest problem with this is
232b1c5ebb2SDave Chinner * the requirement to allocate the disposable buffer during the commit if:
233b1c5ebb2SDave Chinner * a) does not exist; or
234b1c5ebb2SDave Chinner * b) it is too small
235b1c5ebb2SDave Chinner *
236b1c5ebb2SDave Chinner * If we do this allocation within xlog_cil_insert_format_items(), it is done
237b1c5ebb2SDave Chinner * under the xc_ctx_lock, which means that a CIL push cannot occur during
238b1c5ebb2SDave Chinner * the memory allocation. This means that we have a potential deadlock situation
239b1c5ebb2SDave Chinner * under low memory conditions when we have lots of dirty metadata pinned in
240b1c5ebb2SDave Chinner * the CIL and we need a CIL commit to occur to free memory.
241b1c5ebb2SDave Chinner *
242b1c5ebb2SDave Chinner * To avoid this, we need to move the memory allocation outside the
243b1c5ebb2SDave Chinner * xc_ctx_lock, but because the log vector buffers are disposable, that opens
244b1c5ebb2SDave Chinner * up a TOCTOU race condition w.r.t. the CIL committing and removing the log
245b1c5ebb2SDave Chinner * vector buffers between the check and the formatting of the item into the
246b1c5ebb2SDave Chinner * log vector buffer within the xc_ctx_lock.
247b1c5ebb2SDave Chinner *
248b1c5ebb2SDave Chinner * Because the log vector buffer needs to be unchanged during the CIL push
249b1c5ebb2SDave Chinner * process, we cannot share the buffer between the transaction commit (which
250b1c5ebb2SDave Chinner * modifies the buffer) and the CIL push context that is writing the changes
251b1c5ebb2SDave Chinner * into the log. This means skipping preallocation of buffer space is
252b1c5ebb2SDave Chinner * unreliable, but we most definitely do not want to be allocating and freeing
253b1c5ebb2SDave Chinner * buffers unnecessarily during commits when overwrites can be done safely.
254b1c5ebb2SDave Chinner *
255b1c5ebb2SDave Chinner * The simplest solution to this problem is to allocate a shadow buffer when a
256b1c5ebb2SDave Chinner * log item is committed for the second time, and then to only use this buffer
257b1c5ebb2SDave Chinner * if necessary. The buffer can remain attached to the log item until such time
258b1c5ebb2SDave Chinner * it is needed, and this is the buffer that is reallocated to match the size of
259b1c5ebb2SDave Chinner * the incoming modification. Then during the formatting of the item we can swap
260b1c5ebb2SDave Chinner * the active buffer with the new one if we can't reuse the existing buffer. We
261b1c5ebb2SDave Chinner * don't free the old buffer as it may be reused on the next modification if
262b1c5ebb2SDave Chinner * it's size is right, otherwise we'll free and reallocate it at that point.
263b1c5ebb2SDave Chinner *
264b1c5ebb2SDave Chinner * This function builds a vector for the changes in each log item in the
265b1c5ebb2SDave Chinner * transaction. It then works out the length of the buffer needed for each log
266b1c5ebb2SDave Chinner * item, allocates them and attaches the vector to the log item in preparation
267b1c5ebb2SDave Chinner * for the formatting step which occurs under the xc_ctx_lock.
268b1c5ebb2SDave Chinner *
269b1c5ebb2SDave Chinner * While this means the memory footprint goes up, it avoids the repeated
270b1c5ebb2SDave Chinner * alloc/free pattern that repeated modifications of an item would otherwise
271b1c5ebb2SDave Chinner * cause, and hence minimises the CPU overhead of such behaviour.
272b1c5ebb2SDave Chinner */
273b1c5ebb2SDave Chinner static void
xlog_cil_alloc_shadow_bufs(struct xlog * log,struct xfs_trans * tp)274b1c5ebb2SDave Chinner xlog_cil_alloc_shadow_bufs(
275b1c5ebb2SDave Chinner struct xlog *log,
276b1c5ebb2SDave Chinner struct xfs_trans *tp)
277b1c5ebb2SDave Chinner {
278e6631f85SDave Chinner struct xfs_log_item *lip;
279b1c5ebb2SDave Chinner
280e6631f85SDave Chinner list_for_each_entry(lip, &tp->t_items, li_trans) {
281b1c5ebb2SDave Chinner struct xfs_log_vec *lv;
282b1c5ebb2SDave Chinner int niovecs = 0;
283b1c5ebb2SDave Chinner int nbytes = 0;
284b1c5ebb2SDave Chinner int buf_size;
285b1c5ebb2SDave Chinner bool ordered = false;
286b1c5ebb2SDave Chinner
287b1c5ebb2SDave Chinner /* Skip items which aren't dirty in this transaction. */
288e6631f85SDave Chinner if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
289b1c5ebb2SDave Chinner continue;
290b1c5ebb2SDave Chinner
291b1c5ebb2SDave Chinner /* get number of vecs and size of data to be stored */
292b1c5ebb2SDave Chinner lip->li_ops->iop_size(lip, &niovecs, &nbytes);
293b1c5ebb2SDave Chinner
294b1c5ebb2SDave Chinner /*
295b1c5ebb2SDave Chinner * Ordered items need to be tracked but we do not wish to write
296b1c5ebb2SDave Chinner * them. We need a logvec to track the object, but we do not
297b1c5ebb2SDave Chinner * need an iovec or buffer to be allocated for copying data.
298b1c5ebb2SDave Chinner */
299b1c5ebb2SDave Chinner if (niovecs == XFS_LOG_VEC_ORDERED) {
300b1c5ebb2SDave Chinner ordered = true;
301b1c5ebb2SDave Chinner niovecs = 0;
302b1c5ebb2SDave Chinner nbytes = 0;
303b1c5ebb2SDave Chinner }
304b1c5ebb2SDave Chinner
305b1c5ebb2SDave Chinner /*
3068d547cf9SDave Chinner * We 64-bit align the length of each iovec so that the start of
3078d547cf9SDave Chinner * the next one is naturally aligned. We'll need to account for
3088d547cf9SDave Chinner * that slack space here.
3098d547cf9SDave Chinner *
3108d547cf9SDave Chinner * We also add the xlog_op_header to each region when
3118d547cf9SDave Chinner * formatting, but that's not accounted to the size of the item
3128d547cf9SDave Chinner * at this point. Hence we'll need an addition number of bytes
3138d547cf9SDave Chinner * for each vector to hold an opheader.
3148d547cf9SDave Chinner *
3158d547cf9SDave Chinner * Then round nbytes up to 64-bit alignment so that the initial
3168d547cf9SDave Chinner * buffer alignment is easy to calculate and verify.
317b1c5ebb2SDave Chinner */
3188d547cf9SDave Chinner nbytes += niovecs *
3198d547cf9SDave Chinner (sizeof(uint64_t) + sizeof(struct xlog_op_header));
320b1c5ebb2SDave Chinner nbytes = round_up(nbytes, sizeof(uint64_t));
321b1c5ebb2SDave Chinner
322b1c5ebb2SDave Chinner /*
323b1c5ebb2SDave Chinner * The data buffer needs to start 64-bit aligned, so round up
324b1c5ebb2SDave Chinner * that space to ensure we can align it appropriately and not
325b1c5ebb2SDave Chinner * overrun the buffer.
326b1c5ebb2SDave Chinner */
327b1c5ebb2SDave Chinner buf_size = nbytes + xlog_cil_iovec_space(niovecs);
328b1c5ebb2SDave Chinner
329b1c5ebb2SDave Chinner /*
330b1c5ebb2SDave Chinner * if we have no shadow buffer, or it is too small, we need to
331b1c5ebb2SDave Chinner * reallocate it.
332b1c5ebb2SDave Chinner */
333b1c5ebb2SDave Chinner if (!lip->li_lv_shadow ||
334b1c5ebb2SDave Chinner buf_size > lip->li_lv_shadow->lv_size) {
335b1c5ebb2SDave Chinner /*
336b1c5ebb2SDave Chinner * We free and allocate here as a realloc would copy
3378dc9384bSDave Chinner * unnecessary data. We don't use kvzalloc() for the
338b1c5ebb2SDave Chinner * same reason - we don't need to zero the data area in
339b1c5ebb2SDave Chinner * the buffer, only the log vector header and the iovec
340b1c5ebb2SDave Chinner * storage.
341b1c5ebb2SDave Chinner */
342b1c5ebb2SDave Chinner kmem_free(lip->li_lv_shadow);
34345ff8b47SDave Chinner lv = xlog_kvmalloc(buf_size);
344b1c5ebb2SDave Chinner
345b1c5ebb2SDave Chinner memset(lv, 0, xlog_cil_iovec_space(niovecs));
346b1c5ebb2SDave Chinner
34716924853SDave Chinner INIT_LIST_HEAD(&lv->lv_list);
348b1c5ebb2SDave Chinner lv->lv_item = lip;
349b1c5ebb2SDave Chinner lv->lv_size = buf_size;
350b1c5ebb2SDave Chinner if (ordered)
351b1c5ebb2SDave Chinner lv->lv_buf_len = XFS_LOG_VEC_ORDERED;
352b1c5ebb2SDave Chinner else
353b1c5ebb2SDave Chinner lv->lv_iovecp = (struct xfs_log_iovec *)&lv[1];
354b1c5ebb2SDave Chinner lip->li_lv_shadow = lv;
355b1c5ebb2SDave Chinner } else {
356b1c5ebb2SDave Chinner /* same or smaller, optimise common overwrite case */
357b1c5ebb2SDave Chinner lv = lip->li_lv_shadow;
358b1c5ebb2SDave Chinner if (ordered)
359b1c5ebb2SDave Chinner lv->lv_buf_len = XFS_LOG_VEC_ORDERED;
360b1c5ebb2SDave Chinner else
361b1c5ebb2SDave Chinner lv->lv_buf_len = 0;
362b1c5ebb2SDave Chinner lv->lv_bytes = 0;
363b1c5ebb2SDave Chinner }
364b1c5ebb2SDave Chinner
365b1c5ebb2SDave Chinner /* Ensure the lv is set up according to ->iop_size */
366b1c5ebb2SDave Chinner lv->lv_niovecs = niovecs;
367b1c5ebb2SDave Chinner
368b1c5ebb2SDave Chinner /* The allocated data region lies beyond the iovec region */
369b1c5ebb2SDave Chinner lv->lv_buf = (char *)lv + xlog_cil_iovec_space(niovecs);
370b1c5ebb2SDave Chinner }
371b1c5ebb2SDave Chinner
372b1c5ebb2SDave Chinner }
373b1c5ebb2SDave Chinner
37471e330b5SDave Chinner /*
375991aaf65SDave Chinner * Prepare the log item for insertion into the CIL. Calculate the difference in
376593e3439SDave Chinner * log space it will consume, and if it is a new item pin it as well.
377991aaf65SDave Chinner */
378991aaf65SDave Chinner STATIC void
xfs_cil_prepare_item(struct xlog * log,struct xfs_log_vec * lv,struct xfs_log_vec * old_lv,int * diff_len)379991aaf65SDave Chinner xfs_cil_prepare_item(
380991aaf65SDave Chinner struct xlog *log,
381991aaf65SDave Chinner struct xfs_log_vec *lv,
382991aaf65SDave Chinner struct xfs_log_vec *old_lv,
383593e3439SDave Chinner int *diff_len)
384991aaf65SDave Chinner {
385991aaf65SDave Chinner /* Account for the new LV being passed in */
386593e3439SDave Chinner if (lv->lv_buf_len != XFS_LOG_VEC_ORDERED)
387110dc24aSDave Chinner *diff_len += lv->lv_bytes;
388991aaf65SDave Chinner
389991aaf65SDave Chinner /*
390991aaf65SDave Chinner * If there is no old LV, this is the first time we've seen the item in
391991aaf65SDave Chinner * this CIL context and so we need to pin it. If we are replacing the
392b1c5ebb2SDave Chinner * old_lv, then remove the space it accounts for and make it the shadow
393b1c5ebb2SDave Chinner * buffer for later freeing. In both cases we are now switching to the
394b63da6c8SRandy Dunlap * shadow buffer, so update the pointer to it appropriately.
395991aaf65SDave Chinner */
396b1c5ebb2SDave Chinner if (!old_lv) {
397e8b78db7SChristoph Hellwig if (lv->lv_item->li_ops->iop_pin)
398991aaf65SDave Chinner lv->lv_item->li_ops->iop_pin(lv->lv_item);
399b1c5ebb2SDave Chinner lv->lv_item->li_lv_shadow = NULL;
400b1c5ebb2SDave Chinner } else if (old_lv != lv) {
401991aaf65SDave Chinner ASSERT(lv->lv_buf_len != XFS_LOG_VEC_ORDERED);
402991aaf65SDave Chinner
403110dc24aSDave Chinner *diff_len -= old_lv->lv_bytes;
404b1c5ebb2SDave Chinner lv->lv_item->li_lv_shadow = old_lv;
405991aaf65SDave Chinner }
406991aaf65SDave Chinner
407991aaf65SDave Chinner /* attach new log vector to log item */
408991aaf65SDave Chinner lv->lv_item->li_lv = lv;
409991aaf65SDave Chinner
410991aaf65SDave Chinner /*
411991aaf65SDave Chinner * If this is the first time the item is being committed to the
412991aaf65SDave Chinner * CIL, store the sequence number on the log item so we can
413991aaf65SDave Chinner * tell in future commits whether this is the first checkpoint
414991aaf65SDave Chinner * the item is being committed into.
415991aaf65SDave Chinner */
416991aaf65SDave Chinner if (!lv->lv_item->li_seq)
417991aaf65SDave Chinner lv->lv_item->li_seq = log->l_cilp->xc_ctx->sequence;
418991aaf65SDave Chinner }
419991aaf65SDave Chinner
420991aaf65SDave Chinner /*
42171e330b5SDave Chinner * Format log item into a flat buffers
42271e330b5SDave Chinner *
42371e330b5SDave Chinner * For delayed logging, we need to hold a formatted buffer containing all the
42471e330b5SDave Chinner * changes on the log item. This enables us to relog the item in memory and
42571e330b5SDave Chinner * write it out asynchronously without needing to relock the object that was
42671e330b5SDave Chinner * modified at the time it gets written into the iclog.
42771e330b5SDave Chinner *
428b1c5ebb2SDave Chinner * This function takes the prepared log vectors attached to each log item, and
429b1c5ebb2SDave Chinner * formats the changes into the log vector buffer. The buffer it uses is
430b1c5ebb2SDave Chinner * dependent on the current state of the vector in the CIL - the shadow lv is
431b1c5ebb2SDave Chinner * guaranteed to be large enough for the current modification, but we will only
432b1c5ebb2SDave Chinner * use that if we can't reuse the existing lv. If we can't reuse the existing
433b1c5ebb2SDave Chinner * lv, then simple swap it out for the shadow lv. We don't free it - that is
434b1c5ebb2SDave Chinner * done lazily either by th enext modification or the freeing of the log item.
43571e330b5SDave Chinner *
43671e330b5SDave Chinner * We don't set up region headers during this process; we simply copy the
43771e330b5SDave Chinner * regions into the flat buffer. We can do this because we still have to do a
43871e330b5SDave Chinner * formatting step to write the regions into the iclog buffer. Writing the
43971e330b5SDave Chinner * ophdrs during the iclog write means that we can support splitting large
44071e330b5SDave Chinner * regions across iclog boundares without needing a change in the format of the
44171e330b5SDave Chinner * item/region encapsulation.
44271e330b5SDave Chinner *
44371e330b5SDave Chinner * Hence what we need to do now is change the rewrite the vector array to point
44471e330b5SDave Chinner * to the copied region inside the buffer we just allocated. This allows us to
44571e330b5SDave Chinner * format the regions into the iclog as though they are being formatted
44671e330b5SDave Chinner * directly out of the objects themselves.
44771e330b5SDave Chinner */
448991aaf65SDave Chinner static void
xlog_cil_insert_format_items(struct xlog * log,struct xfs_trans * tp,int * diff_len)449991aaf65SDave Chinner xlog_cil_insert_format_items(
450991aaf65SDave Chinner struct xlog *log,
451991aaf65SDave Chinner struct xfs_trans *tp,
452593e3439SDave Chinner int *diff_len)
45371e330b5SDave Chinner {
454e6631f85SDave Chinner struct xfs_log_item *lip;
45571e330b5SDave Chinner
4560244b960SChristoph Hellwig /* Bail out if we didn't find a log item. */
4570244b960SChristoph Hellwig if (list_empty(&tp->t_items)) {
4580244b960SChristoph Hellwig ASSERT(0);
459991aaf65SDave Chinner return;
4600244b960SChristoph Hellwig }
4610244b960SChristoph Hellwig
462e6631f85SDave Chinner list_for_each_entry(lip, &tp->t_items, li_trans) {
4637492c5b4SDave Chinner struct xfs_log_vec *lv;
464b1c5ebb2SDave Chinner struct xfs_log_vec *old_lv = NULL;
465b1c5ebb2SDave Chinner struct xfs_log_vec *shadow;
466fd63875cSDave Chinner bool ordered = false;
46771e330b5SDave Chinner
4680244b960SChristoph Hellwig /* Skip items which aren't dirty in this transaction. */
469e6631f85SDave Chinner if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
4700244b960SChristoph Hellwig continue;
4710244b960SChristoph Hellwig
472b1c5ebb2SDave Chinner /*
473b1c5ebb2SDave Chinner * The formatting size information is already attached to
474b1c5ebb2SDave Chinner * the shadow lv on the log item.
475b1c5ebb2SDave Chinner */
476b1c5ebb2SDave Chinner shadow = lip->li_lv_shadow;
477b1c5ebb2SDave Chinner if (shadow->lv_buf_len == XFS_LOG_VEC_ORDERED)
478b1c5ebb2SDave Chinner ordered = true;
479166d1368SDave Chinner
4800244b960SChristoph Hellwig /* Skip items that do not have any vectors for writing */
481b1c5ebb2SDave Chinner if (!shadow->lv_niovecs && !ordered)
4820244b960SChristoph Hellwig continue;
4830244b960SChristoph Hellwig
484f5baac35SDave Chinner /* compare to existing item size */
485b1c5ebb2SDave Chinner old_lv = lip->li_lv;
486b1c5ebb2SDave Chinner if (lip->li_lv && shadow->lv_size <= lip->li_lv->lv_size) {
487f5baac35SDave Chinner /* same or smaller, optimise common overwrite case */
488f5baac35SDave Chinner lv = lip->li_lv;
489f5baac35SDave Chinner
490f5baac35SDave Chinner if (ordered)
491f5baac35SDave Chinner goto insert;
492f5baac35SDave Chinner
493991aaf65SDave Chinner /*
494991aaf65SDave Chinner * set the item up as though it is a new insertion so
495991aaf65SDave Chinner * that the space reservation accounting is correct.
496991aaf65SDave Chinner */
497110dc24aSDave Chinner *diff_len -= lv->lv_bytes;
498b1c5ebb2SDave Chinner
499b1c5ebb2SDave Chinner /* Ensure the lv is set up according to ->iop_size */
500b1c5ebb2SDave Chinner lv->lv_niovecs = shadow->lv_niovecs;
501b1c5ebb2SDave Chinner
502b1c5ebb2SDave Chinner /* reset the lv buffer information for new formatting */
503b1c5ebb2SDave Chinner lv->lv_buf_len = 0;
504b1c5ebb2SDave Chinner lv->lv_bytes = 0;
505b1c5ebb2SDave Chinner lv->lv_buf = (char *)lv +
506b1c5ebb2SDave Chinner xlog_cil_iovec_space(lv->lv_niovecs);
5079597df6bSChristoph Hellwig } else {
508b1c5ebb2SDave Chinner /* switch to shadow buffer! */
509b1c5ebb2SDave Chinner lv = shadow;
5107492c5b4SDave Chinner lv->lv_item = lip;
511fd63875cSDave Chinner if (ordered) {
512fd63875cSDave Chinner /* track as an ordered logvec */
5137492c5b4SDave Chinner ASSERT(lip->li_lv == NULL);
5147492c5b4SDave Chinner goto insert;
515fd63875cSDave Chinner }
5169597df6bSChristoph Hellwig }
5179597df6bSChristoph Hellwig
5183895e51fSDave Chinner ASSERT(IS_ALIGNED((unsigned long)lv->lv_buf, sizeof(uint64_t)));
519bde7cff6SChristoph Hellwig lip->li_ops->iop_format(lip, lv);
5207492c5b4SDave Chinner insert:
521593e3439SDave Chinner xfs_cil_prepare_item(log, lv, old_lv, diff_len);
52271e330b5SDave Chinner }
523d1583a38SDave Chinner }
524d1583a38SDave Chinner
525d1583a38SDave Chinner /*
5267c8ade21SDave Chinner * The use of lockless waitqueue_active() requires that the caller has
5277c8ade21SDave Chinner * serialised itself against the wakeup call in xlog_cil_push_work(). That
5287c8ade21SDave Chinner * can be done by either holding the push lock or the context lock.
5297c8ade21SDave Chinner */
5307c8ade21SDave Chinner static inline bool
xlog_cil_over_hard_limit(struct xlog * log,int32_t space_used)5317c8ade21SDave Chinner xlog_cil_over_hard_limit(
5327c8ade21SDave Chinner struct xlog *log,
5337c8ade21SDave Chinner int32_t space_used)
5347c8ade21SDave Chinner {
5357c8ade21SDave Chinner if (waitqueue_active(&log->l_cilp->xc_push_wait))
5367c8ade21SDave Chinner return true;
5377c8ade21SDave Chinner if (space_used >= XLOG_CIL_BLOCKING_SPACE_LIMIT(log))
5387c8ade21SDave Chinner return true;
5397c8ade21SDave Chinner return false;
5407c8ade21SDave Chinner }
5417c8ade21SDave Chinner
5427c8ade21SDave Chinner /*
543d1583a38SDave Chinner * Insert the log items into the CIL and calculate the difference in space
544d1583a38SDave Chinner * consumed by the item. Add the space to the checkpoint ticket and calculate
545d1583a38SDave Chinner * if the change requires additional log metadata. If it does, take that space
54642b2aa86SJustin P. Mattock * as well. Remove the amount of space we added to the checkpoint ticket from
547d1583a38SDave Chinner * the current transaction ticket so that the accounting works out correctly.
548d1583a38SDave Chinner */
54971e330b5SDave Chinner static void
xlog_cil_insert_items(struct xlog * log,struct xfs_trans * tp,uint32_t released_space)5503b93c7aaSDave Chinner xlog_cil_insert_items(
551f7bdf03aSMark Tinguely struct xlog *log,
5520d227466SDave Chinner struct xfs_trans *tp,
5530d227466SDave Chinner uint32_t released_space)
5543b93c7aaSDave Chinner {
555d1583a38SDave Chinner struct xfs_cil *cil = log->l_cilp;
556d1583a38SDave Chinner struct xfs_cil_ctx *ctx = cil->xc_ctx;
557e6631f85SDave Chinner struct xfs_log_item *lip;
558d1583a38SDave Chinner int len = 0;
559e2f23426SBrian Foster int iovhdr_res = 0, split_res = 0, ctx_res = 0;
5607c8ade21SDave Chinner int space_used;
561016a2338SDave Chinner int order;
562ecd49f7aSDarrick J. Wong unsigned int cpu_nr;
5637c8ade21SDave Chinner struct xlog_cil_pcp *cilpcp;
5643b93c7aaSDave Chinner
565991aaf65SDave Chinner ASSERT(tp);
566d1583a38SDave Chinner
567d1583a38SDave Chinner /*
568d1583a38SDave Chinner * We can do this safely because the context can't checkpoint until we
569d1583a38SDave Chinner * are done so it doesn't matter exactly how we update the CIL.
570d1583a38SDave Chinner */
571593e3439SDave Chinner xlog_cil_insert_format_items(log, tp, &len);
572fd63875cSDave Chinner
573e2f23426SBrian Foster /*
5747c8ade21SDave Chinner * Subtract the space released by intent cancelation from the space we
5757c8ade21SDave Chinner * consumed so that we remove it from the CIL space and add it back to
5767c8ade21SDave Chinner * the current transaction reservation context.
5777c8ade21SDave Chinner */
5787c8ade21SDave Chinner len -= released_space;
5797c8ade21SDave Chinner
5807c8ade21SDave Chinner /*
5817c8ade21SDave Chinner * Grab the per-cpu pointer for the CIL before we start any accounting.
5827c8ade21SDave Chinner * That ensures that we are running with pre-emption disabled and so we
5837c8ade21SDave Chinner * can't be scheduled away between split sample/update operations that
5847c8ade21SDave Chinner * are done without outside locking to serialise them.
5857c8ade21SDave Chinner */
586ecd49f7aSDarrick J. Wong cpu_nr = get_cpu();
587ecd49f7aSDarrick J. Wong cilpcp = this_cpu_ptr(cil->xc_pcp);
588ecd49f7aSDarrick J. Wong
589ecd49f7aSDarrick J. Wong /* Tell the future push that there was work added by this CPU. */
590ecd49f7aSDarrick J. Wong if (!cpumask_test_cpu(cpu_nr, &ctx->cil_pcpmask))
591ecd49f7aSDarrick J. Wong cpumask_test_and_set_cpu(cpu_nr, &ctx->cil_pcpmask);
5927c8ade21SDave Chinner
5937c8ade21SDave Chinner /*
59488591e7fSDave Chinner * We need to take the CIL checkpoint unit reservation on the first
59588591e7fSDave Chinner * commit into the CIL. Test the XLOG_CIL_EMPTY bit first so we don't
596c0fb4765SDave Chinner * unnecessarily do an atomic op in the fast path here. We can clear the
597c0fb4765SDave Chinner * XLOG_CIL_EMPTY bit as we are under the xc_ctx_lock here and that
598c0fb4765SDave Chinner * needs to be held exclusively to reset the XLOG_CIL_EMPTY bit.
599e2f23426SBrian Foster */
60088591e7fSDave Chinner if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) &&
60112380d23SDave Chinner test_and_clear_bit(XLOG_CIL_EMPTY, &cil->xc_flags))
602e2f23426SBrian Foster ctx_res = ctx->ticket->t_unit_res;
60312380d23SDave Chinner
60431151cc3SDave Chinner /*
60531151cc3SDave Chinner * Check if we need to steal iclog headers. atomic_read() is not a
60631151cc3SDave Chinner * locked atomic operation, so we can check the value before we do any
60731151cc3SDave Chinner * real atomic ops in the fast path. If we've already taken the CIL unit
60831151cc3SDave Chinner * reservation from this commit, we've already got one iclog header
60931151cc3SDave Chinner * space reserved so we have to account for that otherwise we risk
61031151cc3SDave Chinner * overrunning the reservation on this ticket.
61131151cc3SDave Chinner *
61231151cc3SDave Chinner * If the CIL is already at the hard limit, we might need more header
61331151cc3SDave Chinner * space that originally reserved. So steal more header space from every
61431151cc3SDave Chinner * commit that occurs once we are over the hard limit to ensure the CIL
61531151cc3SDave Chinner * push won't run out of reservation space.
61631151cc3SDave Chinner *
61731151cc3SDave Chinner * This can steal more than we need, but that's OK.
6187c8ade21SDave Chinner *
6197c8ade21SDave Chinner * The cil->xc_ctx_lock provides the serialisation necessary for safely
6207c8ade21SDave Chinner * calling xlog_cil_over_hard_limit() in this context.
62131151cc3SDave Chinner */
6227c8ade21SDave Chinner space_used = atomic_read(&ctx->space_used) + cilpcp->space_used + len;
62331151cc3SDave Chinner if (atomic_read(&cil->xc_iclog_hdrs) > 0 ||
6247c8ade21SDave Chinner xlog_cil_over_hard_limit(log, space_used)) {
6257c8ade21SDave Chinner split_res = log->l_iclog_hsize +
62631151cc3SDave Chinner sizeof(struct xlog_op_header);
62731151cc3SDave Chinner if (ctx_res)
62831151cc3SDave Chinner ctx_res += split_res * (tp->t_ticket->t_iclog_hdrs - 1);
62931151cc3SDave Chinner else
63031151cc3SDave Chinner ctx_res = split_res * tp->t_ticket->t_iclog_hdrs;
63131151cc3SDave Chinner atomic_sub(tp->t_ticket->t_iclog_hdrs, &cil->xc_iclog_hdrs);
632e2f23426SBrian Foster }
6331dd2a2c1SDave Chinner cilpcp->space_reserved += ctx_res;
63431151cc3SDave Chinner
6357c8ade21SDave Chinner /*
6367c8ade21SDave Chinner * Accurately account when over the soft limit, otherwise fold the
6377c8ade21SDave Chinner * percpu count into the global count if over the per-cpu threshold.
6387c8ade21SDave Chinner */
6397c8ade21SDave Chinner if (!test_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags)) {
6407c8ade21SDave Chinner atomic_add(len, &ctx->space_used);
6417c8ade21SDave Chinner } else if (cilpcp->space_used + len >
6427c8ade21SDave Chinner (XLOG_CIL_SPACE_LIMIT(log) / num_online_cpus())) {
6437c8ade21SDave Chinner space_used = atomic_add_return(cilpcp->space_used + len,
6447c8ade21SDave Chinner &ctx->space_used);
6457c8ade21SDave Chinner cilpcp->space_used = 0;
646e2f23426SBrian Foster
647fd63875cSDave Chinner /*
6487c8ade21SDave Chinner * If we just transitioned over the soft limit, we need to
6497c8ade21SDave Chinner * transition to the global atomic counter.
650d4ca1d55SBrian Foster */
6517c8ade21SDave Chinner if (space_used >= XLOG_CIL_SPACE_LIMIT(log))
6527c8ade21SDave Chinner xlog_cil_insert_pcp_aggregate(cil, ctx);
6537c8ade21SDave Chinner } else {
6547c8ade21SDave Chinner cilpcp->space_used += len;
655d4ca1d55SBrian Foster }
656df7a4a21SDave Chinner /* attach the transaction to the CIL if it has any busy extents */
657df7a4a21SDave Chinner if (!list_empty(&tp->t_busy))
658df7a4a21SDave Chinner list_splice_init(&tp->t_busy, &cilpcp->busy_extents);
6597c8ade21SDave Chinner
660d4ca1d55SBrian Foster /*
661016a2338SDave Chinner * Now update the order of everything modified in the transaction
662016a2338SDave Chinner * and insert items into the CIL if they aren't already there.
663991aaf65SDave Chinner * We do this here so we only need to take the CIL lock once during
664991aaf65SDave Chinner * the transaction commit.
665fd63875cSDave Chinner */
666016a2338SDave Chinner order = atomic_inc_return(&ctx->order_id);
667e6631f85SDave Chinner list_for_each_entry(lip, &tp->t_items, li_trans) {
668991aaf65SDave Chinner /* Skip items which aren't dirty in this transaction. */
669e6631f85SDave Chinner if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
670991aaf65SDave Chinner continue;
671991aaf65SDave Chinner
672016a2338SDave Chinner lip->li_order_id = order;
673016a2338SDave Chinner if (!list_empty(&lip->li_cil))
674016a2338SDave Chinner continue;
675c0fb4765SDave Chinner list_add_tail(&lip->li_cil, &cilpcp->log_items);
676fd63875cSDave Chinner }
677ecd49f7aSDarrick J. Wong put_cpu();
678d4ca1d55SBrian Foster
6797c8ade21SDave Chinner /*
6807c8ade21SDave Chinner * If we've overrun the reservation, dump the tx details before we move
6817c8ade21SDave Chinner * the log items. Shutdown is imminent...
6827c8ade21SDave Chinner */
6837c8ade21SDave Chinner tp->t_ticket->t_curr_res -= ctx_res + len;
6847c8ade21SDave Chinner if (WARN_ON(tp->t_ticket->t_curr_res < 0)) {
6857c8ade21SDave Chinner xfs_warn(log->l_mp, "Transaction log reservation overrun:");
6867c8ade21SDave Chinner xfs_warn(log->l_mp,
6877c8ade21SDave Chinner " log items: %d bytes (iov hdrs: %d bytes)",
6887c8ade21SDave Chinner len, iovhdr_res);
6897c8ade21SDave Chinner xfs_warn(log->l_mp, " split region headers: %d bytes",
6907c8ade21SDave Chinner split_res);
6917c8ade21SDave Chinner xfs_warn(log->l_mp, " ctx ticket: %d bytes", ctx_res);
6927c8ade21SDave Chinner xlog_print_trans(tp);
693b5f17becSDave Chinner xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR);
6943b93c7aaSDave Chinner }
6957c8ade21SDave Chinner }
6963b93c7aaSDave Chinner
6973b93c7aaSDave Chinner static void
xlog_cil_free_logvec(struct list_head * lv_chain)69871e330b5SDave Chinner xlog_cil_free_logvec(
69916924853SDave Chinner struct list_head *lv_chain)
70071e330b5SDave Chinner {
70171e330b5SDave Chinner struct xfs_log_vec *lv;
70271e330b5SDave Chinner
70316924853SDave Chinner while (!list_empty(lv_chain)) {
70416924853SDave Chinner lv = list_first_entry(lv_chain, struct xfs_log_vec, lv_list);
70516924853SDave Chinner list_del_init(&lv->lv_list);
70671e330b5SDave Chinner kmem_free(lv);
70771e330b5SDave Chinner }
70871e330b5SDave Chinner }
70971e330b5SDave Chinner
71071e330b5SDave Chinner /*
71171e330b5SDave Chinner * Mark all items committed and clear busy extents. We free the log vector
71271e330b5SDave Chinner * chains in a separate pass so that we unpin the log items as quickly as
71371e330b5SDave Chinner * possible.
71471e330b5SDave Chinner */
71571e330b5SDave Chinner static void
xlog_cil_committed(struct xfs_cil_ctx * ctx)71671e330b5SDave Chinner xlog_cil_committed(
71712e6a0f4SChristoph Hellwig struct xfs_cil_ctx *ctx)
71871e330b5SDave Chinner {
719e84661aaSChristoph Hellwig struct xfs_mount *mp = ctx->cil->xc_log->l_mp;
7202039a272SDave Chinner bool abort = xlog_is_shutdown(ctx->cil->xc_log);
72171e330b5SDave Chinner
722545aa41fSBrian Foster /*
723545aa41fSBrian Foster * If the I/O failed, we're aborting the commit and already shutdown.
724545aa41fSBrian Foster * Wake any commit waiters before aborting the log items so we don't
725545aa41fSBrian Foster * block async log pushers on callbacks. Async log pushers explicitly do
726545aa41fSBrian Foster * not wait on log force completion because they may be holding locks
727545aa41fSBrian Foster * required to unpin items.
728545aa41fSBrian Foster */
729545aa41fSBrian Foster if (abort) {
730545aa41fSBrian Foster spin_lock(&ctx->cil->xc_push_lock);
73168a74dcaSDave Chinner wake_up_all(&ctx->cil->xc_start_wait);
732545aa41fSBrian Foster wake_up_all(&ctx->cil->xc_commit_wait);
733545aa41fSBrian Foster spin_unlock(&ctx->cil->xc_push_lock);
734545aa41fSBrian Foster }
735545aa41fSBrian Foster
73616924853SDave Chinner xfs_trans_committed_bulk(ctx->cil->xc_log->l_ailp, &ctx->lv_chain,
7370e57f6a3SDave Chinner ctx->start_lsn, abort);
73871e330b5SDave Chinner
739*428c4435SDave Chinner xfs_extent_busy_sort(&ctx->busy_extents.extent_list);
740*428c4435SDave Chinner xfs_extent_busy_clear(mp, &ctx->busy_extents.extent_list,
7410560f31aSDave Chinner xfs_has_discard(mp) && !abort);
74271e330b5SDave Chinner
7434bb928cdSDave Chinner spin_lock(&ctx->cil->xc_push_lock);
74471e330b5SDave Chinner list_del(&ctx->committing);
7454bb928cdSDave Chinner spin_unlock(&ctx->cil->xc_push_lock);
74671e330b5SDave Chinner
74716924853SDave Chinner xlog_cil_free_logvec(&ctx->lv_chain);
748e84661aaSChristoph Hellwig
749*428c4435SDave Chinner if (!list_empty(&ctx->busy_extents.extent_list)) {
750*428c4435SDave Chinner ctx->busy_extents.mount = mp;
751*428c4435SDave Chinner ctx->busy_extents.owner = ctx;
752*428c4435SDave Chinner xfs_discard_extents(mp, &ctx->busy_extents);
753*428c4435SDave Chinner return;
754*428c4435SDave Chinner }
755*428c4435SDave Chinner
75671e330b5SDave Chinner kmem_free(ctx);
75771e330b5SDave Chinner }
75871e330b5SDave Chinner
75989ae379dSChristoph Hellwig void
xlog_cil_process_committed(struct list_head * list)76089ae379dSChristoph Hellwig xlog_cil_process_committed(
76112e6a0f4SChristoph Hellwig struct list_head *list)
76289ae379dSChristoph Hellwig {
76389ae379dSChristoph Hellwig struct xfs_cil_ctx *ctx;
76489ae379dSChristoph Hellwig
76589ae379dSChristoph Hellwig while ((ctx = list_first_entry_or_null(list,
76689ae379dSChristoph Hellwig struct xfs_cil_ctx, iclog_entry))) {
76789ae379dSChristoph Hellwig list_del(&ctx->iclog_entry);
76812e6a0f4SChristoph Hellwig xlog_cil_committed(ctx);
76989ae379dSChristoph Hellwig }
77089ae379dSChristoph Hellwig }
77189ae379dSChristoph Hellwig
77271e330b5SDave Chinner /*
773c45aba40SDave Chinner * Record the LSN of the iclog we were just granted space to start writing into.
774c45aba40SDave Chinner * If the context doesn't have a start_lsn recorded, then this iclog will
775c45aba40SDave Chinner * contain the start record for the checkpoint. Otherwise this write contains
776c45aba40SDave Chinner * the commit record for the checkpoint.
777c45aba40SDave Chinner */
778c45aba40SDave Chinner void
xlog_cil_set_ctx_write_state(struct xfs_cil_ctx * ctx,struct xlog_in_core * iclog)779c45aba40SDave Chinner xlog_cil_set_ctx_write_state(
780c45aba40SDave Chinner struct xfs_cil_ctx *ctx,
781c45aba40SDave Chinner struct xlog_in_core *iclog)
782c45aba40SDave Chinner {
783c45aba40SDave Chinner struct xfs_cil *cil = ctx->cil;
784c45aba40SDave Chinner xfs_lsn_t lsn = be64_to_cpu(iclog->ic_header.h_lsn);
785c45aba40SDave Chinner
786c45aba40SDave Chinner ASSERT(!ctx->commit_lsn);
787caa80090SDave Chinner if (!ctx->start_lsn) {
788c45aba40SDave Chinner spin_lock(&cil->xc_push_lock);
78968a74dcaSDave Chinner /*
79068a74dcaSDave Chinner * The LSN we need to pass to the log items on transaction
79168a74dcaSDave Chinner * commit is the LSN reported by the first log vector write, not
79268a74dcaSDave Chinner * the commit lsn. If we use the commit record lsn then we can
793919edbadSDave Chinner * move the grant write head beyond the tail LSN and overwrite
794919edbadSDave Chinner * it.
79568a74dcaSDave Chinner */
796c45aba40SDave Chinner ctx->start_lsn = lsn;
79768a74dcaSDave Chinner wake_up_all(&cil->xc_start_wait);
798caa80090SDave Chinner spin_unlock(&cil->xc_push_lock);
799919edbadSDave Chinner
800919edbadSDave Chinner /*
801919edbadSDave Chinner * Make sure the metadata we are about to overwrite in the log
802919edbadSDave Chinner * has been flushed to stable storage before this iclog is
803919edbadSDave Chinner * issued.
804919edbadSDave Chinner */
805919edbadSDave Chinner spin_lock(&cil->xc_log->l_icloglock);
806919edbadSDave Chinner iclog->ic_flags |= XLOG_ICL_NEED_FLUSH;
807919edbadSDave Chinner spin_unlock(&cil->xc_log->l_icloglock);
808caa80090SDave Chinner return;
809caa80090SDave Chinner }
810caa80090SDave Chinner
811caa80090SDave Chinner /*
812caa80090SDave Chinner * Take a reference to the iclog for the context so that we still hold
813caa80090SDave Chinner * it when xlog_write is done and has released it. This means the
814caa80090SDave Chinner * context controls when the iclog is released for IO.
815caa80090SDave Chinner */
816caa80090SDave Chinner atomic_inc(&iclog->ic_refcnt);
817caa80090SDave Chinner
818caa80090SDave Chinner /*
819caa80090SDave Chinner * xlog_state_get_iclog_space() guarantees there is enough space in the
820caa80090SDave Chinner * iclog for an entire commit record, so we can attach the context
821caa80090SDave Chinner * callbacks now. This needs to be done before we make the commit_lsn
822caa80090SDave Chinner * visible to waiters so that checkpoints with commit records in the
823caa80090SDave Chinner * same iclog order their IO completion callbacks in the same order that
824caa80090SDave Chinner * the commit records appear in the iclog.
825caa80090SDave Chinner */
826caa80090SDave Chinner spin_lock(&cil->xc_log->l_icloglock);
827caa80090SDave Chinner list_add_tail(&ctx->iclog_entry, &iclog->ic_callbacks);
828caa80090SDave Chinner spin_unlock(&cil->xc_log->l_icloglock);
829caa80090SDave Chinner
830caa80090SDave Chinner /*
831caa80090SDave Chinner * Now we can record the commit LSN and wake anyone waiting for this
832caa80090SDave Chinner * sequence to have the ordered commit record assigned to a physical
833caa80090SDave Chinner * location in the log.
834caa80090SDave Chinner */
835caa80090SDave Chinner spin_lock(&cil->xc_push_lock);
836caa80090SDave Chinner ctx->commit_iclog = iclog;
837c45aba40SDave Chinner ctx->commit_lsn = lsn;
838caa80090SDave Chinner wake_up_all(&cil->xc_commit_wait);
839c45aba40SDave Chinner spin_unlock(&cil->xc_push_lock);
840c45aba40SDave Chinner }
841c45aba40SDave Chinner
842c45aba40SDave Chinner
843c45aba40SDave Chinner /*
844bf034bc8SDave Chinner * Ensure that the order of log writes follows checkpoint sequence order. This
845bf034bc8SDave Chinner * relies on the context LSN being zero until the log write has guaranteed the
846bf034bc8SDave Chinner * LSN that the log write will start at via xlog_state_get_iclog_space().
847bf034bc8SDave Chinner */
84868a74dcaSDave Chinner enum _record_type {
84968a74dcaSDave Chinner _START_RECORD,
85068a74dcaSDave Chinner _COMMIT_RECORD,
85168a74dcaSDave Chinner };
85268a74dcaSDave Chinner
853bf034bc8SDave Chinner static int
xlog_cil_order_write(struct xfs_cil * cil,xfs_csn_t sequence,enum _record_type record)854bf034bc8SDave Chinner xlog_cil_order_write(
855bf034bc8SDave Chinner struct xfs_cil *cil,
85668a74dcaSDave Chinner xfs_csn_t sequence,
85768a74dcaSDave Chinner enum _record_type record)
858bf034bc8SDave Chinner {
859bf034bc8SDave Chinner struct xfs_cil_ctx *ctx;
860bf034bc8SDave Chinner
861bf034bc8SDave Chinner restart:
862bf034bc8SDave Chinner spin_lock(&cil->xc_push_lock);
863bf034bc8SDave Chinner list_for_each_entry(ctx, &cil->xc_committing, committing) {
864bf034bc8SDave Chinner /*
865bf034bc8SDave Chinner * Avoid getting stuck in this loop because we were woken by the
866bf034bc8SDave Chinner * shutdown, but then went back to sleep once already in the
867bf034bc8SDave Chinner * shutdown state.
868bf034bc8SDave Chinner */
869bf034bc8SDave Chinner if (xlog_is_shutdown(cil->xc_log)) {
870bf034bc8SDave Chinner spin_unlock(&cil->xc_push_lock);
871bf034bc8SDave Chinner return -EIO;
872bf034bc8SDave Chinner }
873bf034bc8SDave Chinner
874bf034bc8SDave Chinner /*
875bf034bc8SDave Chinner * Higher sequences will wait for this one so skip them.
876bf034bc8SDave Chinner * Don't wait for our own sequence, either.
877bf034bc8SDave Chinner */
878bf034bc8SDave Chinner if (ctx->sequence >= sequence)
879bf034bc8SDave Chinner continue;
88068a74dcaSDave Chinner
88168a74dcaSDave Chinner /* Wait until the LSN for the record has been recorded. */
88268a74dcaSDave Chinner switch (record) {
88368a74dcaSDave Chinner case _START_RECORD:
88468a74dcaSDave Chinner if (!ctx->start_lsn) {
88568a74dcaSDave Chinner xlog_wait(&cil->xc_start_wait, &cil->xc_push_lock);
88668a74dcaSDave Chinner goto restart;
88768a74dcaSDave Chinner }
88868a74dcaSDave Chinner break;
88968a74dcaSDave Chinner case _COMMIT_RECORD:
890bf034bc8SDave Chinner if (!ctx->commit_lsn) {
891bf034bc8SDave Chinner xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock);
892bf034bc8SDave Chinner goto restart;
893bf034bc8SDave Chinner }
89468a74dcaSDave Chinner break;
89568a74dcaSDave Chinner }
896bf034bc8SDave Chinner }
897bf034bc8SDave Chinner spin_unlock(&cil->xc_push_lock);
898bf034bc8SDave Chinner return 0;
899bf034bc8SDave Chinner }
900bf034bc8SDave Chinner
901bf034bc8SDave Chinner /*
90268a74dcaSDave Chinner * Write out the log vector change now attached to the CIL context. This will
90368a74dcaSDave Chinner * write a start record that needs to be strictly ordered in ascending CIL
90468a74dcaSDave Chinner * sequence order so that log recovery will always use in-order start LSNs when
90568a74dcaSDave Chinner * replaying checkpoints.
90668a74dcaSDave Chinner */
90768a74dcaSDave Chinner static int
xlog_cil_write_chain(struct xfs_cil_ctx * ctx,uint32_t chain_len)90868a74dcaSDave Chinner xlog_cil_write_chain(
90968a74dcaSDave Chinner struct xfs_cil_ctx *ctx,
910d80fc291SDave Chinner uint32_t chain_len)
91168a74dcaSDave Chinner {
91268a74dcaSDave Chinner struct xlog *log = ctx->cil->xc_log;
91368a74dcaSDave Chinner int error;
91468a74dcaSDave Chinner
91568a74dcaSDave Chinner error = xlog_cil_order_write(ctx->cil, ctx->sequence, _START_RECORD);
91668a74dcaSDave Chinner if (error)
91768a74dcaSDave Chinner return error;
91816924853SDave Chinner return xlog_write(log, ctx, &ctx->lv_chain, ctx->ticket, chain_len);
91968a74dcaSDave Chinner }
92068a74dcaSDave Chinner
92168a74dcaSDave Chinner /*
922bf034bc8SDave Chinner * Write out the commit record of a checkpoint transaction to close off a
923bf034bc8SDave Chinner * running log write. These commit records are strictly ordered in ascending CIL
924bf034bc8SDave Chinner * sequence order so that log recovery will always replay the checkpoints in the
925bf034bc8SDave Chinner * correct order.
9262ce82b72SDave Chinner */
9272ce82b72SDave Chinner static int
xlog_cil_write_commit_record(struct xfs_cil_ctx * ctx)9282ce82b72SDave Chinner xlog_cil_write_commit_record(
929caa80090SDave Chinner struct xfs_cil_ctx *ctx)
9302ce82b72SDave Chinner {
931c45aba40SDave Chinner struct xlog *log = ctx->cil->xc_log;
93254021b62SDave Chinner struct xlog_op_header ophdr = {
93354021b62SDave Chinner .oh_clientid = XFS_TRANSACTION,
93454021b62SDave Chinner .oh_tid = cpu_to_be32(ctx->ticket->t_tid),
93554021b62SDave Chinner .oh_flags = XLOG_COMMIT_TRANS,
93654021b62SDave Chinner };
9372ce82b72SDave Chinner struct xfs_log_iovec reg = {
93854021b62SDave Chinner .i_addr = &ophdr,
93954021b62SDave Chinner .i_len = sizeof(struct xlog_op_header),
9402ce82b72SDave Chinner .i_type = XLOG_REG_TYPE_COMMIT,
9412ce82b72SDave Chinner };
9422ce82b72SDave Chinner struct xfs_log_vec vec = {
9432ce82b72SDave Chinner .lv_niovecs = 1,
9442ce82b72SDave Chinner .lv_iovecp = ®,
9452ce82b72SDave Chinner };
9462ce82b72SDave Chinner int error;
94716924853SDave Chinner LIST_HEAD(lv_chain);
94816924853SDave Chinner list_add(&vec.lv_list, &lv_chain);
9492ce82b72SDave Chinner
9502ce82b72SDave Chinner if (xlog_is_shutdown(log))
9512ce82b72SDave Chinner return -EIO;
9522ce82b72SDave Chinner
95368a74dcaSDave Chinner error = xlog_cil_order_write(ctx->cil, ctx->sequence, _COMMIT_RECORD);
95468a74dcaSDave Chinner if (error)
95568a74dcaSDave Chinner return error;
95668a74dcaSDave Chinner
95754021b62SDave Chinner /* account for space used by record data */
95854021b62SDave Chinner ctx->ticket->t_curr_res -= reg.i_len;
95916924853SDave Chinner error = xlog_write(log, ctx, &lv_chain, ctx->ticket, reg.i_len);
9602ce82b72SDave Chinner if (error)
961b5f17becSDave Chinner xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR);
9622ce82b72SDave Chinner return error;
9632ce82b72SDave Chinner }
9642ce82b72SDave Chinner
965735fbf67SDave Chinner struct xlog_cil_trans_hdr {
9666eaed95eSDave Chinner struct xlog_op_header oph[2];
967735fbf67SDave Chinner struct xfs_trans_header thdr;
9686eaed95eSDave Chinner struct xfs_log_iovec lhdr[2];
969735fbf67SDave Chinner };
970735fbf67SDave Chinner
971735fbf67SDave Chinner /*
972735fbf67SDave Chinner * Build a checkpoint transaction header to begin the journal transaction. We
973735fbf67SDave Chinner * need to account for the space used by the transaction header here as it is
974735fbf67SDave Chinner * not accounted for in xlog_write().
9756eaed95eSDave Chinner *
9766eaed95eSDave Chinner * This is the only place we write a transaction header, so we also build the
9776eaed95eSDave Chinner * log opheaders that indicate the start of a log transaction and wrap the
9786eaed95eSDave Chinner * transaction header. We keep the start record in it's own log vector rather
9796eaed95eSDave Chinner * than compacting them into a single region as this ends up making the logic
9806eaed95eSDave Chinner * in xlog_write() for handling empty opheaders for start, commit and unmount
9816eaed95eSDave Chinner * records much simpler.
982735fbf67SDave Chinner */
983735fbf67SDave Chinner static void
xlog_cil_build_trans_hdr(struct xfs_cil_ctx * ctx,struct xlog_cil_trans_hdr * hdr,struct xfs_log_vec * lvhdr,int num_iovecs)984735fbf67SDave Chinner xlog_cil_build_trans_hdr(
985735fbf67SDave Chinner struct xfs_cil_ctx *ctx,
986735fbf67SDave Chinner struct xlog_cil_trans_hdr *hdr,
987735fbf67SDave Chinner struct xfs_log_vec *lvhdr,
988735fbf67SDave Chinner int num_iovecs)
989735fbf67SDave Chinner {
990735fbf67SDave Chinner struct xlog_ticket *tic = ctx->ticket;
9916eaed95eSDave Chinner __be32 tid = cpu_to_be32(tic->t_tid);
992735fbf67SDave Chinner
993735fbf67SDave Chinner memset(hdr, 0, sizeof(*hdr));
994735fbf67SDave Chinner
9956eaed95eSDave Chinner /* Log start record */
9966eaed95eSDave Chinner hdr->oph[0].oh_tid = tid;
9976eaed95eSDave Chinner hdr->oph[0].oh_clientid = XFS_TRANSACTION;
9986eaed95eSDave Chinner hdr->oph[0].oh_flags = XLOG_START_TRANS;
9996eaed95eSDave Chinner
10006eaed95eSDave Chinner /* log iovec region pointer */
10016eaed95eSDave Chinner hdr->lhdr[0].i_addr = &hdr->oph[0];
10026eaed95eSDave Chinner hdr->lhdr[0].i_len = sizeof(struct xlog_op_header);
10036eaed95eSDave Chinner hdr->lhdr[0].i_type = XLOG_REG_TYPE_LRHEADER;
10046eaed95eSDave Chinner
10056eaed95eSDave Chinner /* log opheader */
10066eaed95eSDave Chinner hdr->oph[1].oh_tid = tid;
10076eaed95eSDave Chinner hdr->oph[1].oh_clientid = XFS_TRANSACTION;
10086eaed95eSDave Chinner hdr->oph[1].oh_len = cpu_to_be32(sizeof(struct xfs_trans_header));
10096eaed95eSDave Chinner
10106eaed95eSDave Chinner /* transaction header in host byte order format */
1011735fbf67SDave Chinner hdr->thdr.th_magic = XFS_TRANS_HEADER_MAGIC;
1012735fbf67SDave Chinner hdr->thdr.th_type = XFS_TRANS_CHECKPOINT;
1013735fbf67SDave Chinner hdr->thdr.th_tid = tic->t_tid;
1014735fbf67SDave Chinner hdr->thdr.th_num_items = num_iovecs;
1015735fbf67SDave Chinner
10166eaed95eSDave Chinner /* log iovec region pointer */
10176eaed95eSDave Chinner hdr->lhdr[1].i_addr = &hdr->oph[1];
10186eaed95eSDave Chinner hdr->lhdr[1].i_len = sizeof(struct xlog_op_header) +
10196eaed95eSDave Chinner sizeof(struct xfs_trans_header);
10206eaed95eSDave Chinner hdr->lhdr[1].i_type = XLOG_REG_TYPE_TRANSHDR;
10216eaed95eSDave Chinner
10226eaed95eSDave Chinner lvhdr->lv_niovecs = 2;
10236eaed95eSDave Chinner lvhdr->lv_iovecp = &hdr->lhdr[0];
1024d80fc291SDave Chinner lvhdr->lv_bytes = hdr->lhdr[0].i_len + hdr->lhdr[1].i_len;
1025d80fc291SDave Chinner
1026d80fc291SDave Chinner tic->t_curr_res -= lvhdr->lv_bytes;
1027735fbf67SDave Chinner }
1028735fbf67SDave Chinner
10292ce82b72SDave Chinner /*
1030016a2338SDave Chinner * CIL item reordering compare function. We want to order in ascending ID order,
1031016a2338SDave Chinner * but we want to leave items with the same ID in the order they were added to
1032016a2338SDave Chinner * the list. This is important for operations like reflink where we log 4 order
1033016a2338SDave Chinner * dependent intents in a single transaction when we overwrite an existing
1034016a2338SDave Chinner * shared extent with a new shared extent. i.e. BUI(unmap), CUI(drop),
1035016a2338SDave Chinner * CUI (inc), BUI(remap)...
1036016a2338SDave Chinner */
1037016a2338SDave Chinner static int
xlog_cil_order_cmp(void * priv,const struct list_head * a,const struct list_head * b)1038016a2338SDave Chinner xlog_cil_order_cmp(
1039016a2338SDave Chinner void *priv,
1040016a2338SDave Chinner const struct list_head *a,
1041016a2338SDave Chinner const struct list_head *b)
1042016a2338SDave Chinner {
10434eb56069SDave Chinner struct xfs_log_vec *l1 = container_of(a, struct xfs_log_vec, lv_list);
10444eb56069SDave Chinner struct xfs_log_vec *l2 = container_of(b, struct xfs_log_vec, lv_list);
1045016a2338SDave Chinner
10464eb56069SDave Chinner return l1->lv_order_id > l2->lv_order_id;
1047016a2338SDave Chinner }
1048016a2338SDave Chinner
1049016a2338SDave Chinner /*
105022b1afc5SDave Chinner * Pull all the log vectors off the items in the CIL, and remove the items from
105122b1afc5SDave Chinner * the CIL. We don't need the CIL lock here because it's only needed on the
105222b1afc5SDave Chinner * transaction commit side which is currently locked out by the flush lock.
10530d227466SDave Chinner *
10540d227466SDave Chinner * If a log item is marked with a whiteout, we do not need to write it to the
10550d227466SDave Chinner * journal and so we just move them to the whiteout list for the caller to
10560d227466SDave Chinner * dispose of appropriately.
105722b1afc5SDave Chinner */
105822b1afc5SDave Chinner static void
xlog_cil_build_lv_chain(struct xfs_cil_ctx * ctx,struct list_head * whiteouts,uint32_t * num_iovecs,uint32_t * num_bytes)105922b1afc5SDave Chinner xlog_cil_build_lv_chain(
106022b1afc5SDave Chinner struct xfs_cil_ctx *ctx,
10610d227466SDave Chinner struct list_head *whiteouts,
106222b1afc5SDave Chinner uint32_t *num_iovecs,
106322b1afc5SDave Chinner uint32_t *num_bytes)
106422b1afc5SDave Chinner {
1065c0fb4765SDave Chinner while (!list_empty(&ctx->log_items)) {
106622b1afc5SDave Chinner struct xfs_log_item *item;
106716924853SDave Chinner struct xfs_log_vec *lv;
106822b1afc5SDave Chinner
1069c0fb4765SDave Chinner item = list_first_entry(&ctx->log_items,
107022b1afc5SDave Chinner struct xfs_log_item, li_cil);
10710d227466SDave Chinner
10720d227466SDave Chinner if (test_bit(XFS_LI_WHITEOUT, &item->li_flags)) {
10730d227466SDave Chinner list_move(&item->li_cil, whiteouts);
10740d227466SDave Chinner trace_xfs_cil_whiteout_skip(item);
10750d227466SDave Chinner continue;
10760d227466SDave Chinner }
10770d227466SDave Chinner
107822b1afc5SDave Chinner lv = item->li_lv;
10794eb56069SDave Chinner lv->lv_order_id = item->li_order_id;
108022b1afc5SDave Chinner
108122b1afc5SDave Chinner /* we don't write ordered log vectors */
108222b1afc5SDave Chinner if (lv->lv_buf_len != XFS_LOG_VEC_ORDERED)
108322b1afc5SDave Chinner *num_bytes += lv->lv_bytes;
108416924853SDave Chinner *num_iovecs += lv->lv_niovecs;
108516924853SDave Chinner list_add_tail(&lv->lv_list, &ctx->lv_chain);
108616924853SDave Chinner
108716924853SDave Chinner list_del_init(&item->li_cil);
108816924853SDave Chinner item->li_order_id = 0;
108916924853SDave Chinner item->li_lv = NULL;
109022b1afc5SDave Chinner }
109122b1afc5SDave Chinner }
109222b1afc5SDave Chinner
10930d227466SDave Chinner static void
xlog_cil_cleanup_whiteouts(struct list_head * whiteouts)10940d227466SDave Chinner xlog_cil_cleanup_whiteouts(
10950d227466SDave Chinner struct list_head *whiteouts)
10960d227466SDave Chinner {
10970d227466SDave Chinner while (!list_empty(whiteouts)) {
10980d227466SDave Chinner struct xfs_log_item *item = list_first_entry(whiteouts,
10990d227466SDave Chinner struct xfs_log_item, li_cil);
11000d227466SDave Chinner list_del_init(&item->li_cil);
11010d227466SDave Chinner trace_xfs_cil_whiteout_unpin(item);
11020d227466SDave Chinner item->li_ops->iop_unpin(item, 1);
11030d227466SDave Chinner }
11040d227466SDave Chinner }
11050d227466SDave Chinner
110671e330b5SDave Chinner /*
1107c7cc296dSChristoph Hellwig * Push the Committed Item List to the log.
1108c7cc296dSChristoph Hellwig *
1109c7cc296dSChristoph Hellwig * If the current sequence is the same as xc_push_seq we need to do a flush. If
1110c7cc296dSChristoph Hellwig * xc_push_seq is less than the current sequence, then it has already been
1111a44f13edSDave Chinner * flushed and we don't need to do anything - the caller will wait for it to
1112a44f13edSDave Chinner * complete if necessary.
1113a44f13edSDave Chinner *
1114c7cc296dSChristoph Hellwig * xc_push_seq is checked unlocked against the sequence number for a match.
1115c7cc296dSChristoph Hellwig * Hence we can allow log forces to run racily and not issue pushes for the
1116c7cc296dSChristoph Hellwig * same sequence twice. If we get a race between multiple pushes for the same
1117c7cc296dSChristoph Hellwig * sequence they will block on the first one and then abort, hence avoiding
1118c7cc296dSChristoph Hellwig * needless pushes.
111971e330b5SDave Chinner */
1120c7cc296dSChristoph Hellwig static void
xlog_cil_push_work(struct work_struct * work)1121c7cc296dSChristoph Hellwig xlog_cil_push_work(
1122c7cc296dSChristoph Hellwig struct work_struct *work)
112371e330b5SDave Chinner {
112439823d0fSDave Chinner struct xfs_cil_ctx *ctx =
112539823d0fSDave Chinner container_of(work, struct xfs_cil_ctx, push_work);
112639823d0fSDave Chinner struct xfs_cil *cil = ctx->cil;
1127c7cc296dSChristoph Hellwig struct xlog *log = cil->xc_log;
112871e330b5SDave Chinner struct xfs_cil_ctx *new_ctx;
1129d80fc291SDave Chinner int num_iovecs = 0;
1130d80fc291SDave Chinner int num_bytes = 0;
113171e330b5SDave Chinner int error = 0;
1132735fbf67SDave Chinner struct xlog_cil_trans_hdr thdr;
113316924853SDave Chinner struct xfs_log_vec lvhdr = {};
11340dc8f7f1SDave Chinner xfs_csn_t push_seq;
11350020a190SDave Chinner bool push_commit_stable;
11360d227466SDave Chinner LIST_HEAD (whiteouts);
1137d9f68777SDave Chinner struct xlog_ticket *ticket;
113871e330b5SDave Chinner
113939823d0fSDave Chinner new_ctx = xlog_cil_ctx_alloc();
114071e330b5SDave Chinner new_ctx->ticket = xlog_cil_ticket_alloc(log);
114171e330b5SDave Chinner
114271e330b5SDave Chinner down_write(&cil->xc_ctx_lock);
114371e330b5SDave Chinner
11444bb928cdSDave Chinner spin_lock(&cil->xc_push_lock);
11454c2d542fSDave Chinner push_seq = cil->xc_push_seq;
11464c2d542fSDave Chinner ASSERT(push_seq <= ctx->sequence);
11470020a190SDave Chinner push_commit_stable = cil->xc_push_commit_stable;
11480020a190SDave Chinner cil->xc_push_commit_stable = false;
114971e330b5SDave Chinner
11504c2d542fSDave Chinner /*
115119f4e7ccSDave Chinner * As we are about to switch to a new, empty CIL context, we no longer
115219f4e7ccSDave Chinner * need to throttle tasks on CIL space overruns. Wake any waiters that
115319f4e7ccSDave Chinner * the hard push throttle may have caught so they can start committing
115419f4e7ccSDave Chinner * to the new context. The ctx->xc_push_lock provides the serialisation
115519f4e7ccSDave Chinner * necessary for safely using the lockless waitqueue_active() check in
115619f4e7ccSDave Chinner * this context.
11570e7ab7efSDave Chinner */
115819f4e7ccSDave Chinner if (waitqueue_active(&cil->xc_push_wait))
1159c7f87f39SDave Chinner wake_up_all(&cil->xc_push_wait);
11600e7ab7efSDave Chinner
11617c8ade21SDave Chinner xlog_cil_push_pcp_aggregate(cil, ctx);
11627c8ade21SDave Chinner
11630e7ab7efSDave Chinner /*
11644c2d542fSDave Chinner * Check if we've anything to push. If there is nothing, then we don't
11654c2d542fSDave Chinner * move on to a new sequence number and so we have to be able to push
11664c2d542fSDave Chinner * this sequence again later.
11674c2d542fSDave Chinner */
116888591e7fSDave Chinner if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) {
11694c2d542fSDave Chinner cil->xc_push_seq = 0;
11704bb928cdSDave Chinner spin_unlock(&cil->xc_push_lock);
1171a44f13edSDave Chinner goto out_skip;
11724c2d542fSDave Chinner }
11734c2d542fSDave Chinner
1174a44f13edSDave Chinner
1175cf085a1bSJoe Perches /* check for a previously pushed sequence */
117639823d0fSDave Chinner if (push_seq < ctx->sequence) {
11778af3dcd3SDave Chinner spin_unlock(&cil->xc_push_lock);
1178df806158SDave Chinner goto out_skip;
11798af3dcd3SDave Chinner }
11808af3dcd3SDave Chinner
11818af3dcd3SDave Chinner /*
11828af3dcd3SDave Chinner * We are now going to push this context, so add it to the committing
11838af3dcd3SDave Chinner * list before we do anything else. This ensures that anyone waiting on
11848af3dcd3SDave Chinner * this push can easily detect the difference between a "push in
11858af3dcd3SDave Chinner * progress" and "CIL is empty, nothing to do".
11868af3dcd3SDave Chinner *
11878af3dcd3SDave Chinner * IOWs, a wait loop can now check for:
11888af3dcd3SDave Chinner * the current sequence not being found on the committing list;
11898af3dcd3SDave Chinner * an empty CIL; and
11908af3dcd3SDave Chinner * an unchanged sequence number
11918af3dcd3SDave Chinner * to detect a push that had nothing to do and therefore does not need
11928af3dcd3SDave Chinner * waiting on. If the CIL is not empty, we get put on the committing
11938af3dcd3SDave Chinner * list before emptying the CIL and bumping the sequence number. Hence
11948af3dcd3SDave Chinner * an empty CIL and an unchanged sequence number means we jumped out
11958af3dcd3SDave Chinner * above after doing nothing.
11968af3dcd3SDave Chinner *
11978af3dcd3SDave Chinner * Hence the waiter will either find the commit sequence on the
11988af3dcd3SDave Chinner * committing list or the sequence number will be unchanged and the CIL
11998af3dcd3SDave Chinner * still dirty. In that latter case, the push has not yet started, and
12008af3dcd3SDave Chinner * so the waiter will have to continue trying to check the CIL
12018af3dcd3SDave Chinner * committing list until it is found. In extreme cases of delay, the
12028af3dcd3SDave Chinner * sequence may fully commit between the attempts the wait makes to wait
12038af3dcd3SDave Chinner * on the commit sequence.
12048af3dcd3SDave Chinner */
12058af3dcd3SDave Chinner list_add(&ctx->committing, &cil->xc_committing);
12068af3dcd3SDave Chinner spin_unlock(&cil->xc_push_lock);
1207df806158SDave Chinner
1208c0fb4765SDave Chinner xlog_cil_build_lv_chain(ctx, &whiteouts, &num_iovecs, &num_bytes);
120971e330b5SDave Chinner
121071e330b5SDave Chinner /*
121139823d0fSDave Chinner * Switch the contexts so we can drop the context lock and move out
121271e330b5SDave Chinner * of a shared context. We can't just go straight to the commit record,
121371e330b5SDave Chinner * though - we need to synchronise with previous and future commits so
121471e330b5SDave Chinner * that the commit records are correctly ordered in the log to ensure
121571e330b5SDave Chinner * that we process items during log IO completion in the correct order.
121671e330b5SDave Chinner *
121771e330b5SDave Chinner * For example, if we get an EFI in one checkpoint and the EFD in the
121871e330b5SDave Chinner * next (e.g. due to log forces), we do not want the checkpoint with
121971e330b5SDave Chinner * the EFD to be committed before the checkpoint with the EFI. Hence
122071e330b5SDave Chinner * we must strictly order the commit records of the checkpoints so
122171e330b5SDave Chinner * that: a) the checkpoint callbacks are attached to the iclogs in the
122271e330b5SDave Chinner * correct order; and b) the checkpoints are replayed in correct order
122371e330b5SDave Chinner * in log recovery.
122471e330b5SDave Chinner *
122571e330b5SDave Chinner * Hence we need to add this context to the committing context list so
122671e330b5SDave Chinner * that higher sequences will wait for us to write out a commit record
122771e330b5SDave Chinner * before they do.
1228f876e446SDave Chinner *
12295f9b4b0dSDave Chinner * xfs_log_force_seq requires us to mirror the new sequence into the cil
1230f876e446SDave Chinner * structure atomically with the addition of this sequence to the
1231f876e446SDave Chinner * committing list. This also ensures that we can do unlocked checks
1232f876e446SDave Chinner * against the current sequence in log forces without risking
1233f876e446SDave Chinner * deferencing a freed context pointer.
123471e330b5SDave Chinner */
12354bb928cdSDave Chinner spin_lock(&cil->xc_push_lock);
123639823d0fSDave Chinner xlog_cil_ctx_switch(cil, new_ctx);
12374bb928cdSDave Chinner spin_unlock(&cil->xc_push_lock);
123871e330b5SDave Chinner up_write(&cil->xc_ctx_lock);
123971e330b5SDave Chinner
124071e330b5SDave Chinner /*
12414eb56069SDave Chinner * Sort the log vector chain before we add the transaction headers.
12424eb56069SDave Chinner * This ensures we always have the transaction headers at the start
12434eb56069SDave Chinner * of the chain.
12444eb56069SDave Chinner */
12454eb56069SDave Chinner list_sort(NULL, &ctx->lv_chain, xlog_cil_order_cmp);
12464eb56069SDave Chinner
12474eb56069SDave Chinner /*
124871e330b5SDave Chinner * Build a checkpoint transaction header and write it to the log to
124971e330b5SDave Chinner * begin the transaction. We need to account for the space used by the
125071e330b5SDave Chinner * transaction header here as it is not accounted for in xlog_write().
125116924853SDave Chinner * Add the lvhdr to the head of the lv chain we pass to xlog_write() so
125216924853SDave Chinner * it gets written into the iclog first.
125371e330b5SDave Chinner */
1254735fbf67SDave Chinner xlog_cil_build_trans_hdr(ctx, &thdr, &lvhdr, num_iovecs);
1255d80fc291SDave Chinner num_bytes += lvhdr.lv_bytes;
125616924853SDave Chinner list_add(&lvhdr.lv_list, &ctx->lv_chain);
125771e330b5SDave Chinner
125816924853SDave Chinner /*
125916924853SDave Chinner * Take the lvhdr back off the lv_chain immediately after calling
126016924853SDave Chinner * xlog_cil_write_chain() as it should not be passed to log IO
126116924853SDave Chinner * completion.
126216924853SDave Chinner */
126316924853SDave Chinner error = xlog_cil_write_chain(ctx, num_bytes);
126416924853SDave Chinner list_del(&lvhdr.lv_list);
1265bf034bc8SDave Chinner if (error)
1266ac983517SDave Chinner goto out_abort_free_ticket;
126771e330b5SDave Chinner
1268caa80090SDave Chinner error = xlog_cil_write_commit_record(ctx);
1269dd401770SDave Chinner if (error)
1270dd401770SDave Chinner goto out_abort_free_ticket;
1271dd401770SDave Chinner
1272d9f68777SDave Chinner /*
1273d9f68777SDave Chinner * Grab the ticket from the ctx so we can ungrant it after releasing the
1274d9f68777SDave Chinner * commit_iclog. The ctx may be freed by the time we return from
1275d9f68777SDave Chinner * releasing the commit_iclog (i.e. checkpoint has been completed and
1276d9f68777SDave Chinner * callback run) so we can't reference the ctx after the call to
1277d9f68777SDave Chinner * xlog_state_release_iclog().
1278d9f68777SDave Chinner */
1279d9f68777SDave Chinner ticket = ctx->ticket;
128071e330b5SDave Chinner
1281a1bb8505SDave Chinner /*
12821effb72aSDave Chinner * If the checkpoint spans multiple iclogs, wait for all previous iclogs
12831effb72aSDave Chinner * to complete before we submit the commit_iclog. We can't use state
12841effb72aSDave Chinner * checks for this - ACTIVE can be either a past completed iclog or a
12851effb72aSDave Chinner * future iclog being filled, while WANT_SYNC through SYNC_DONE can be a
12861effb72aSDave Chinner * past or future iclog awaiting IO or ordered IO completion to be run.
12871effb72aSDave Chinner * In the latter case, if it's a future iclog and we wait on it, the we
12881effb72aSDave Chinner * will hang because it won't get processed through to ic_force_wait
12891effb72aSDave Chinner * wakeup until this commit_iclog is written to disk. Hence we use the
12901effb72aSDave Chinner * iclog header lsn and compare it to the commit lsn to determine if we
12911effb72aSDave Chinner * need to wait on iclogs or not.
1292a79b28c2SDave Chinner */
1293caa80090SDave Chinner spin_lock(&log->l_icloglock);
1294c45aba40SDave Chinner if (ctx->start_lsn != ctx->commit_lsn) {
12951effb72aSDave Chinner xfs_lsn_t plsn;
12961effb72aSDave Chinner
1297caa80090SDave Chinner plsn = be64_to_cpu(ctx->commit_iclog->ic_prev->ic_header.h_lsn);
1298c45aba40SDave Chinner if (plsn && XFS_LSN_CMP(plsn, ctx->commit_lsn) < 0) {
12991effb72aSDave Chinner /*
13001effb72aSDave Chinner * Waiting on ic_force_wait orders the completion of
13011effb72aSDave Chinner * iclogs older than ic_prev. Hence we only need to wait
13021effb72aSDave Chinner * on the most recent older iclog here.
13031effb72aSDave Chinner */
1304caa80090SDave Chinner xlog_wait_on_iclog(ctx->commit_iclog->ic_prev);
1305eef983ffSDave Chinner spin_lock(&log->l_icloglock);
13061effb72aSDave Chinner }
13071effb72aSDave Chinner
13081effb72aSDave Chinner /*
13091effb72aSDave Chinner * We need to issue a pre-flush so that the ordering for this
13101effb72aSDave Chinner * checkpoint is correctly preserved down to stable storage.
13111effb72aSDave Chinner */
1312caa80090SDave Chinner ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FLUSH;
1313a79b28c2SDave Chinner }
1314a79b28c2SDave Chinner
1315eef983ffSDave Chinner /*
1316eef983ffSDave Chinner * The commit iclog must be written to stable storage to guarantee
1317eef983ffSDave Chinner * journal IO vs metadata writeback IO is correctly ordered on stable
1318eef983ffSDave Chinner * storage.
13190020a190SDave Chinner *
13200020a190SDave Chinner * If the push caller needs the commit to be immediately stable and the
13210020a190SDave Chinner * commit_iclog is not yet marked as XLOG_STATE_WANT_SYNC to indicate it
13220020a190SDave Chinner * will be written when released, switch it's state to WANT_SYNC right
13230020a190SDave Chinner * now.
1324eef983ffSDave Chinner */
1325caa80090SDave Chinner ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FUA;
13260020a190SDave Chinner if (push_commit_stable &&
13270020a190SDave Chinner ctx->commit_iclog->ic_state == XLOG_STATE_ACTIVE)
13280020a190SDave Chinner xlog_state_switch_iclogs(log, ctx->commit_iclog, 0);
1329d9f68777SDave Chinner ticket = ctx->ticket;
1330d9f68777SDave Chinner xlog_state_release_iclog(log, ctx->commit_iclog, ticket);
1331502a01faSDave Chinner
1332502a01faSDave Chinner /* Not safe to reference ctx now! */
1333502a01faSDave Chinner
1334eef983ffSDave Chinner spin_unlock(&log->l_icloglock);
13350d227466SDave Chinner xlog_cil_cleanup_whiteouts(&whiteouts);
1336d9f68777SDave Chinner xfs_log_ticket_ungrant(log, ticket);
1337c7cc296dSChristoph Hellwig return;
133871e330b5SDave Chinner
133971e330b5SDave Chinner out_skip:
134071e330b5SDave Chinner up_write(&cil->xc_ctx_lock);
134171e330b5SDave Chinner xfs_log_ticket_put(new_ctx->ticket);
134271e330b5SDave Chinner kmem_free(new_ctx);
1343c7cc296dSChristoph Hellwig return;
134471e330b5SDave Chinner
13457db37c5eSDave Chinner out_abort_free_ticket:
13462039a272SDave Chinner ASSERT(xlog_is_shutdown(log));
13470d227466SDave Chinner xlog_cil_cleanup_whiteouts(&whiteouts);
1348caa80090SDave Chinner if (!ctx->commit_iclog) {
1349d9f68777SDave Chinner xfs_log_ticket_ungrant(log, ctx->ticket);
135012e6a0f4SChristoph Hellwig xlog_cil_committed(ctx);
1351caa80090SDave Chinner return;
1352caa80090SDave Chinner }
1353caa80090SDave Chinner spin_lock(&log->l_icloglock);
1354d9f68777SDave Chinner ticket = ctx->ticket;
1355d9f68777SDave Chinner xlog_state_release_iclog(log, ctx->commit_iclog, ticket);
1356caa80090SDave Chinner /* Not safe to reference ctx now! */
1357caa80090SDave Chinner spin_unlock(&log->l_icloglock);
1358d9f68777SDave Chinner xfs_log_ticket_ungrant(log, ticket);
13594c2d542fSDave Chinner }
13604c2d542fSDave Chinner
13614c2d542fSDave Chinner /*
13624c2d542fSDave Chinner * We need to push CIL every so often so we don't cache more than we can fit in
13634c2d542fSDave Chinner * the log. The limit really is that a checkpoint can't be more than half the
13644c2d542fSDave Chinner * log (the current checkpoint is not allowed to overwrite the previous
13654c2d542fSDave Chinner * checkpoint), but commit latency and memory usage limit this to a smaller
13664c2d542fSDave Chinner * size.
13674c2d542fSDave Chinner */
13684c2d542fSDave Chinner static void
xlog_cil_push_background(struct xlog * log)13694c2d542fSDave Chinner xlog_cil_push_background(
13700e7ab7efSDave Chinner struct xlog *log) __releases(cil->xc_ctx_lock)
13714c2d542fSDave Chinner {
13724c2d542fSDave Chinner struct xfs_cil *cil = log->l_cilp;
13737c8ade21SDave Chinner int space_used = atomic_read(&cil->xc_ctx->space_used);
13744c2d542fSDave Chinner
13754c2d542fSDave Chinner /*
13764c2d542fSDave Chinner * The cil won't be empty because we are called while holding the
137788591e7fSDave Chinner * context lock so whatever we added to the CIL will still be there.
13784c2d542fSDave Chinner */
137988591e7fSDave Chinner ASSERT(!test_bit(XLOG_CIL_EMPTY, &cil->xc_flags));
13804c2d542fSDave Chinner
13814c2d542fSDave Chinner /*
13821ccb0745SDave Chinner * We are done if:
13831ccb0745SDave Chinner * - we haven't used up all the space available yet; or
13841ccb0745SDave Chinner * - we've already queued up a push; and
13851ccb0745SDave Chinner * - we're not over the hard limit; and
13861ccb0745SDave Chinner * - nothing has been over the hard limit.
13871ccb0745SDave Chinner *
13881ccb0745SDave Chinner * If so, we don't need to take the push lock as there's nothing to do.
13894c2d542fSDave Chinner */
13901ccb0745SDave Chinner if (space_used < XLOG_CIL_SPACE_LIMIT(log) ||
13911ccb0745SDave Chinner (cil->xc_push_seq == cil->xc_current_sequence &&
13921ccb0745SDave Chinner space_used < XLOG_CIL_BLOCKING_SPACE_LIMIT(log) &&
13931ccb0745SDave Chinner !waitqueue_active(&cil->xc_push_wait))) {
13940e7ab7efSDave Chinner up_read(&cil->xc_ctx_lock);
13954c2d542fSDave Chinner return;
13960e7ab7efSDave Chinner }
13974c2d542fSDave Chinner
13984bb928cdSDave Chinner spin_lock(&cil->xc_push_lock);
13994c2d542fSDave Chinner if (cil->xc_push_seq < cil->xc_current_sequence) {
14004c2d542fSDave Chinner cil->xc_push_seq = cil->xc_current_sequence;
140133c0dd78SDave Chinner queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work);
14024c2d542fSDave Chinner }
14030e7ab7efSDave Chinner
14040e7ab7efSDave Chinner /*
14050e7ab7efSDave Chinner * Drop the context lock now, we can't hold that if we need to sleep
14060e7ab7efSDave Chinner * because we are over the blocking threshold. The push_lock is still
14070e7ab7efSDave Chinner * held, so blocking threshold sleep/wakeup is still correctly
14080e7ab7efSDave Chinner * serialised here.
14090e7ab7efSDave Chinner */
14100e7ab7efSDave Chinner up_read(&cil->xc_ctx_lock);
14110e7ab7efSDave Chinner
14120e7ab7efSDave Chinner /*
14130e7ab7efSDave Chinner * If we are well over the space limit, throttle the work that is being
141419f4e7ccSDave Chinner * done until the push work on this context has begun. Enforce the hard
141519f4e7ccSDave Chinner * throttle on all transaction commits once it has been activated, even
141619f4e7ccSDave Chinner * if the committing transactions have resulted in the space usage
141719f4e7ccSDave Chinner * dipping back down under the hard limit.
141819f4e7ccSDave Chinner *
141919f4e7ccSDave Chinner * The ctx->xc_push_lock provides the serialisation necessary for safely
14207c8ade21SDave Chinner * calling xlog_cil_over_hard_limit() in this context.
14210e7ab7efSDave Chinner */
14227c8ade21SDave Chinner if (xlog_cil_over_hard_limit(log, space_used)) {
14230e7ab7efSDave Chinner trace_xfs_log_cil_wait(log, cil->xc_ctx->ticket);
14247c8ade21SDave Chinner ASSERT(space_used < log->l_logsize);
1425c7f87f39SDave Chinner xlog_wait(&cil->xc_push_wait, &cil->xc_push_lock);
14260e7ab7efSDave Chinner return;
14270e7ab7efSDave Chinner }
14280e7ab7efSDave Chinner
14294bb928cdSDave Chinner spin_unlock(&cil->xc_push_lock);
14304c2d542fSDave Chinner
14314c2d542fSDave Chinner }
14324c2d542fSDave Chinner
1433f876e446SDave Chinner /*
1434f876e446SDave Chinner * xlog_cil_push_now() is used to trigger an immediate CIL push to the sequence
1435f876e446SDave Chinner * number that is passed. When it returns, the work will be queued for
14360020a190SDave Chinner * @push_seq, but it won't be completed.
14370020a190SDave Chinner *
14380020a190SDave Chinner * If the caller is performing a synchronous force, we will flush the workqueue
14390020a190SDave Chinner * to get previously queued work moving to minimise the wait time they will
14400020a190SDave Chinner * undergo waiting for all outstanding pushes to complete. The caller is
14410020a190SDave Chinner * expected to do the required waiting for push_seq to complete.
14420020a190SDave Chinner *
14430020a190SDave Chinner * If the caller is performing an async push, we need to ensure that the
14440020a190SDave Chinner * checkpoint is fully flushed out of the iclogs when we finish the push. If we
14450020a190SDave Chinner * don't do this, then the commit record may remain sitting in memory in an
14460020a190SDave Chinner * ACTIVE iclog. This then requires another full log force to push to disk,
14470020a190SDave Chinner * which defeats the purpose of having an async, non-blocking CIL force
14480020a190SDave Chinner * mechanism. Hence in this case we need to pass a flag to the push work to
14490020a190SDave Chinner * indicate it needs to flush the commit record itself.
1450f876e446SDave Chinner */
14514c2d542fSDave Chinner static void
xlog_cil_push_now(struct xlog * log,xfs_lsn_t push_seq,bool async)1452f876e446SDave Chinner xlog_cil_push_now(
1453f7bdf03aSMark Tinguely struct xlog *log,
14540020a190SDave Chinner xfs_lsn_t push_seq,
14550020a190SDave Chinner bool async)
14564c2d542fSDave Chinner {
14574c2d542fSDave Chinner struct xfs_cil *cil = log->l_cilp;
14584c2d542fSDave Chinner
14594c2d542fSDave Chinner if (!cil)
14604c2d542fSDave Chinner return;
14614c2d542fSDave Chinner
14624c2d542fSDave Chinner ASSERT(push_seq && push_seq <= cil->xc_current_sequence);
14634c2d542fSDave Chinner
14644c2d542fSDave Chinner /* start on any pending background push to minimise wait time on it */
14650020a190SDave Chinner if (!async)
146633c0dd78SDave Chinner flush_workqueue(cil->xc_push_wq);
14674c2d542fSDave Chinner
146870447e0aSDave Chinner spin_lock(&cil->xc_push_lock);
146970447e0aSDave Chinner
147070447e0aSDave Chinner /*
147170447e0aSDave Chinner * If this is an async flush request, we always need to set the
147270447e0aSDave Chinner * xc_push_commit_stable flag even if something else has already queued
147370447e0aSDave Chinner * a push. The flush caller is asking for the CIL to be on stable
147470447e0aSDave Chinner * storage when the next push completes, so regardless of who has queued
147570447e0aSDave Chinner * the push, the flush requires stable semantics from it.
147670447e0aSDave Chinner */
147770447e0aSDave Chinner cil->xc_push_commit_stable = async;
147870447e0aSDave Chinner
14794c2d542fSDave Chinner /*
14804c2d542fSDave Chinner * If the CIL is empty or we've already pushed the sequence then
148170447e0aSDave Chinner * there's no more work that we need to do.
14824c2d542fSDave Chinner */
148388591e7fSDave Chinner if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) ||
148488591e7fSDave Chinner push_seq <= cil->xc_push_seq) {
14854bb928cdSDave Chinner spin_unlock(&cil->xc_push_lock);
14864c2d542fSDave Chinner return;
14874c2d542fSDave Chinner }
14884c2d542fSDave Chinner
14894c2d542fSDave Chinner cil->xc_push_seq = push_seq;
149033c0dd78SDave Chinner queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work);
14914bb928cdSDave Chinner spin_unlock(&cil->xc_push_lock);
14924c2d542fSDave Chinner }
14934c2d542fSDave Chinner
14942c6e24ceSDave Chinner bool
xlog_cil_empty(struct xlog * log)14952c6e24ceSDave Chinner xlog_cil_empty(
14962c6e24ceSDave Chinner struct xlog *log)
14972c6e24ceSDave Chinner {
14982c6e24ceSDave Chinner struct xfs_cil *cil = log->l_cilp;
14992c6e24ceSDave Chinner bool empty = false;
15002c6e24ceSDave Chinner
15012c6e24ceSDave Chinner spin_lock(&cil->xc_push_lock);
150288591e7fSDave Chinner if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags))
15032c6e24ceSDave Chinner empty = true;
15042c6e24ceSDave Chinner spin_unlock(&cil->xc_push_lock);
15052c6e24ceSDave Chinner return empty;
15062c6e24ceSDave Chinner }
15072c6e24ceSDave Chinner
150871e330b5SDave Chinner /*
15090d227466SDave Chinner * If there are intent done items in this transaction and the related intent was
15100d227466SDave Chinner * committed in the current (same) CIL checkpoint, we don't need to write either
15110d227466SDave Chinner * the intent or intent done item to the journal as the change will be
15120d227466SDave Chinner * journalled atomically within this checkpoint. As we cannot remove items from
15130d227466SDave Chinner * the CIL here, mark the related intent with a whiteout so that the CIL push
15140d227466SDave Chinner * can remove it rather than writing it to the journal. Then remove the intent
15150d227466SDave Chinner * done item from the current transaction and release it so it doesn't get put
15160d227466SDave Chinner * into the CIL at all.
15170d227466SDave Chinner */
15180d227466SDave Chinner static uint32_t
xlog_cil_process_intents(struct xfs_cil * cil,struct xfs_trans * tp)15190d227466SDave Chinner xlog_cil_process_intents(
15200d227466SDave Chinner struct xfs_cil *cil,
15210d227466SDave Chinner struct xfs_trans *tp)
15220d227466SDave Chinner {
15230d227466SDave Chinner struct xfs_log_item *lip, *ilip, *next;
15240d227466SDave Chinner uint32_t len = 0;
15250d227466SDave Chinner
15260d227466SDave Chinner list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) {
15270d227466SDave Chinner if (!(lip->li_ops->flags & XFS_ITEM_INTENT_DONE))
15280d227466SDave Chinner continue;
15290d227466SDave Chinner
15300d227466SDave Chinner ilip = lip->li_ops->iop_intent(lip);
15310d227466SDave Chinner if (!ilip || !xlog_item_in_current_chkpt(cil, ilip))
15320d227466SDave Chinner continue;
15330d227466SDave Chinner set_bit(XFS_LI_WHITEOUT, &ilip->li_flags);
15340d227466SDave Chinner trace_xfs_cil_whiteout_mark(ilip);
15350d227466SDave Chinner len += ilip->li_lv->lv_bytes;
15360d227466SDave Chinner kmem_free(ilip->li_lv);
15370d227466SDave Chinner ilip->li_lv = NULL;
15380d227466SDave Chinner
15390d227466SDave Chinner xfs_trans_del_item(lip);
15400d227466SDave Chinner lip->li_ops->iop_release(lip);
15410d227466SDave Chinner }
15420d227466SDave Chinner return len;
15430d227466SDave Chinner }
15440d227466SDave Chinner
15450d227466SDave Chinner /*
1546a44f13edSDave Chinner * Commit a transaction with the given vector to the Committed Item List.
1547a44f13edSDave Chinner *
1548a44f13edSDave Chinner * To do this, we need to format the item, pin it in memory if required and
1549a44f13edSDave Chinner * account for the space used by the transaction. Once we have done that we
1550a44f13edSDave Chinner * need to release the unused reservation for the transaction, attach the
1551a44f13edSDave Chinner * transaction to the checkpoint context so we carry the busy extents through
1552a44f13edSDave Chinner * to checkpoint completion, and then unlock all the items in the transaction.
1553a44f13edSDave Chinner *
1554a44f13edSDave Chinner * Called with the context lock already held in read mode to lock out
1555a44f13edSDave Chinner * background commit, returns without it held once background commits are
1556a44f13edSDave Chinner * allowed again.
1557a44f13edSDave Chinner */
1558c6f97264SJie Liu void
xlog_cil_commit(struct xlog * log,struct xfs_trans * tp,xfs_csn_t * commit_seq,bool regrant)15595f9b4b0dSDave Chinner xlog_cil_commit(
15605f9b4b0dSDave Chinner struct xlog *log,
1561a44f13edSDave Chinner struct xfs_trans *tp,
15625f9b4b0dSDave Chinner xfs_csn_t *commit_seq,
156370393313SChristoph Hellwig bool regrant)
1564a44f13edSDave Chinner {
1565991aaf65SDave Chinner struct xfs_cil *cil = log->l_cilp;
1566195cd83dSChristoph Hellwig struct xfs_log_item *lip, *next;
15670d227466SDave Chinner uint32_t released_space = 0;
1568a44f13edSDave Chinner
1569b1c5ebb2SDave Chinner /*
1570b1c5ebb2SDave Chinner * Do all necessary memory allocation before we lock the CIL.
1571b1c5ebb2SDave Chinner * This ensures the allocation does not deadlock with a CIL
1572b1c5ebb2SDave Chinner * push in memory reclaim (e.g. from kswapd).
1573b1c5ebb2SDave Chinner */
1574b1c5ebb2SDave Chinner xlog_cil_alloc_shadow_bufs(log, tp);
1575b1c5ebb2SDave Chinner
1576f5baac35SDave Chinner /* lock out background commit */
1577991aaf65SDave Chinner down_read(&cil->xc_ctx_lock);
1578f5baac35SDave Chinner
15790d227466SDave Chinner if (tp->t_flags & XFS_TRANS_HAS_INTENT_DONE)
15800d227466SDave Chinner released_space = xlog_cil_process_intents(cil, tp);
15810d227466SDave Chinner
15820d227466SDave Chinner xlog_cil_insert_items(log, tp, released_space);
1583a44f13edSDave Chinner
15842039a272SDave Chinner if (regrant && !xlog_is_shutdown(log))
15858b41e3f9SChristoph Hellwig xfs_log_ticket_regrant(log, tp->t_ticket);
15868b41e3f9SChristoph Hellwig else
15878b41e3f9SChristoph Hellwig xfs_log_ticket_ungrant(log, tp->t_ticket);
1588ba18781bSDave Chinner tp->t_ticket = NULL;
1589a44f13edSDave Chinner xfs_trans_unreserve_and_mod_sb(tp);
1590a44f13edSDave Chinner
1591a44f13edSDave Chinner /*
1592a44f13edSDave Chinner * Once all the items of the transaction have been copied to the CIL,
1593195cd83dSChristoph Hellwig * the items can be unlocked and possibly freed.
1594a44f13edSDave Chinner *
1595a44f13edSDave Chinner * This needs to be done before we drop the CIL context lock because we
1596a44f13edSDave Chinner * have to update state in the log items and unlock them before they go
1597a44f13edSDave Chinner * to disk. If we don't, then the CIL checkpoint can race with us and
1598a44f13edSDave Chinner * we can run checkpoint completion before we've updated and unlocked
1599a44f13edSDave Chinner * the log items. This affects (at least) processing of stale buffers,
1600a44f13edSDave Chinner * inodes and EFIs.
1601a44f13edSDave Chinner */
1602195cd83dSChristoph Hellwig trace_xfs_trans_commit_items(tp, _RET_IP_);
1603195cd83dSChristoph Hellwig list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) {
1604195cd83dSChristoph Hellwig xfs_trans_del_item(lip);
1605195cd83dSChristoph Hellwig if (lip->li_ops->iop_committing)
16065f9b4b0dSDave Chinner lip->li_ops->iop_committing(lip, cil->xc_ctx->sequence);
1607195cd83dSChristoph Hellwig }
16085f9b4b0dSDave Chinner if (commit_seq)
16095f9b4b0dSDave Chinner *commit_seq = cil->xc_ctx->sequence;
1610a44f13edSDave Chinner
16110e7ab7efSDave Chinner /* xlog_cil_push_background() releases cil->xc_ctx_lock */
16120e7ab7efSDave Chinner xlog_cil_push_background(log);
1613a44f13edSDave Chinner }
1614a44f13edSDave Chinner
1615a44f13edSDave Chinner /*
16160020a190SDave Chinner * Flush the CIL to stable storage but don't wait for it to complete. This
16170020a190SDave Chinner * requires the CIL push to ensure the commit record for the push hits the disk,
16180020a190SDave Chinner * but otherwise is no different to a push done from a log force.
16190020a190SDave Chinner */
16200020a190SDave Chinner void
xlog_cil_flush(struct xlog * log)16210020a190SDave Chinner xlog_cil_flush(
16220020a190SDave Chinner struct xlog *log)
16230020a190SDave Chinner {
16240020a190SDave Chinner xfs_csn_t seq = log->l_cilp->xc_current_sequence;
16250020a190SDave Chinner
16260020a190SDave Chinner trace_xfs_log_force(log->l_mp, seq, _RET_IP_);
16270020a190SDave Chinner xlog_cil_push_now(log, seq, true);
162870447e0aSDave Chinner
162970447e0aSDave Chinner /*
163070447e0aSDave Chinner * If the CIL is empty, make sure that any previous checkpoint that may
163170447e0aSDave Chinner * still be in an active iclog is pushed to stable storage.
163270447e0aSDave Chinner */
1633c0fb4765SDave Chinner if (test_bit(XLOG_CIL_EMPTY, &log->l_cilp->xc_flags))
163470447e0aSDave Chinner xfs_log_force(log->l_mp, 0);
16350020a190SDave Chinner }
16360020a190SDave Chinner
16370020a190SDave Chinner /*
163871e330b5SDave Chinner * Conditionally push the CIL based on the sequence passed in.
163971e330b5SDave Chinner *
16400020a190SDave Chinner * We only need to push if we haven't already pushed the sequence number given.
16410020a190SDave Chinner * Hence the only time we will trigger a push here is if the push sequence is
16420020a190SDave Chinner * the same as the current context.
164371e330b5SDave Chinner *
164471e330b5SDave Chinner * We return the current commit lsn to allow the callers to determine if a
164571e330b5SDave Chinner * iclog flush is necessary following this call.
164671e330b5SDave Chinner */
164771e330b5SDave Chinner xfs_lsn_t
xlog_cil_force_seq(struct xlog * log,xfs_csn_t sequence)16485f9b4b0dSDave Chinner xlog_cil_force_seq(
1649f7bdf03aSMark Tinguely struct xlog *log,
16505f9b4b0dSDave Chinner xfs_csn_t sequence)
165171e330b5SDave Chinner {
165271e330b5SDave Chinner struct xfs_cil *cil = log->l_cilp;
165371e330b5SDave Chinner struct xfs_cil_ctx *ctx;
165471e330b5SDave Chinner xfs_lsn_t commit_lsn = NULLCOMMITLSN;
165571e330b5SDave Chinner
1656a44f13edSDave Chinner ASSERT(sequence <= cil->xc_current_sequence);
165771e330b5SDave Chinner
16580020a190SDave Chinner if (!sequence)
16590020a190SDave Chinner sequence = cil->xc_current_sequence;
16600020a190SDave Chinner trace_xfs_log_force(log->l_mp, sequence, _RET_IP_);
16610020a190SDave Chinner
1662a44f13edSDave Chinner /*
1663a44f13edSDave Chinner * check to see if we need to force out the current context.
1664a44f13edSDave Chinner * xlog_cil_push() handles racing pushes for the same sequence,
1665a44f13edSDave Chinner * so no need to deal with it here.
1666a44f13edSDave Chinner */
1667f876e446SDave Chinner restart:
16680020a190SDave Chinner xlog_cil_push_now(log, sequence, false);
166971e330b5SDave Chinner
167071e330b5SDave Chinner /*
167171e330b5SDave Chinner * See if we can find a previous sequence still committing.
167271e330b5SDave Chinner * We need to wait for all previous sequence commits to complete
167371e330b5SDave Chinner * before allowing the force of push_seq to go ahead. Hence block
167471e330b5SDave Chinner * on commits for those as well.
167571e330b5SDave Chinner */
16764bb928cdSDave Chinner spin_lock(&cil->xc_push_lock);
167771e330b5SDave Chinner list_for_each_entry(ctx, &cil->xc_committing, committing) {
1678ac983517SDave Chinner /*
1679ac983517SDave Chinner * Avoid getting stuck in this loop because we were woken by the
1680ac983517SDave Chinner * shutdown, but then went back to sleep once already in the
1681ac983517SDave Chinner * shutdown state.
1682ac983517SDave Chinner */
16832039a272SDave Chinner if (xlog_is_shutdown(log))
1684ac983517SDave Chinner goto out_shutdown;
1685a44f13edSDave Chinner if (ctx->sequence > sequence)
168671e330b5SDave Chinner continue;
168771e330b5SDave Chinner if (!ctx->commit_lsn) {
168871e330b5SDave Chinner /*
168971e330b5SDave Chinner * It is still being pushed! Wait for the push to
169071e330b5SDave Chinner * complete, then start again from the beginning.
169171e330b5SDave Chinner */
16920020a190SDave Chinner XFS_STATS_INC(log->l_mp, xs_log_force_sleep);
16934bb928cdSDave Chinner xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock);
169471e330b5SDave Chinner goto restart;
169571e330b5SDave Chinner }
1696a44f13edSDave Chinner if (ctx->sequence != sequence)
169771e330b5SDave Chinner continue;
169871e330b5SDave Chinner /* found it! */
169971e330b5SDave Chinner commit_lsn = ctx->commit_lsn;
170071e330b5SDave Chinner }
1701f876e446SDave Chinner
1702f876e446SDave Chinner /*
1703f876e446SDave Chinner * The call to xlog_cil_push_now() executes the push in the background.
1704f876e446SDave Chinner * Hence by the time we have got here it our sequence may not have been
1705f876e446SDave Chinner * pushed yet. This is true if the current sequence still matches the
1706f876e446SDave Chinner * push sequence after the above wait loop and the CIL still contains
17078af3dcd3SDave Chinner * dirty objects. This is guaranteed by the push code first adding the
17088af3dcd3SDave Chinner * context to the committing list before emptying the CIL.
1709f876e446SDave Chinner *
17108af3dcd3SDave Chinner * Hence if we don't find the context in the committing list and the
17118af3dcd3SDave Chinner * current sequence number is unchanged then the CIL contents are
17128af3dcd3SDave Chinner * significant. If the CIL is empty, if means there was nothing to push
17138af3dcd3SDave Chinner * and that means there is nothing to wait for. If the CIL is not empty,
17148af3dcd3SDave Chinner * it means we haven't yet started the push, because if it had started
17158af3dcd3SDave Chinner * we would have found the context on the committing list.
1716f876e446SDave Chinner */
1717f876e446SDave Chinner if (sequence == cil->xc_current_sequence &&
171888591e7fSDave Chinner !test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) {
1719f876e446SDave Chinner spin_unlock(&cil->xc_push_lock);
1720f876e446SDave Chinner goto restart;
1721f876e446SDave Chinner }
1722f876e446SDave Chinner
17234bb928cdSDave Chinner spin_unlock(&cil->xc_push_lock);
172471e330b5SDave Chinner return commit_lsn;
1725ac983517SDave Chinner
1726ac983517SDave Chinner /*
1727ac983517SDave Chinner * We detected a shutdown in progress. We need to trigger the log force
1728ac983517SDave Chinner * to pass through it's iclog state machine error handling, even though
1729ac983517SDave Chinner * we are already in a shutdown state. Hence we can't return
1730ac983517SDave Chinner * NULLCOMMITLSN here as that has special meaning to log forces (i.e.
1731ac983517SDave Chinner * LSN is already stable), so we return a zero LSN instead.
1732ac983517SDave Chinner */
1733ac983517SDave Chinner out_shutdown:
1734ac983517SDave Chinner spin_unlock(&cil->xc_push_lock);
1735ac983517SDave Chinner return 0;
173671e330b5SDave Chinner }
1737ccf7c23fSDave Chinner
1738ccf7c23fSDave Chinner /*
17394c2d542fSDave Chinner * Perform initial CIL structure initialisation.
17404c2d542fSDave Chinner */
17414c2d542fSDave Chinner int
xlog_cil_init(struct xlog * log)17424c2d542fSDave Chinner xlog_cil_init(
1743f7bdf03aSMark Tinguely struct xlog *log)
17444c2d542fSDave Chinner {
17454c2d542fSDave Chinner struct xfs_cil *cil;
17464c2d542fSDave Chinner struct xfs_cil_ctx *ctx;
1747df7a4a21SDave Chinner struct xlog_cil_pcp *cilpcp;
1748df7a4a21SDave Chinner int cpu;
17494c2d542fSDave Chinner
1750707e0ddaSTetsuo Handa cil = kmem_zalloc(sizeof(*cil), KM_MAYFAIL);
17514c2d542fSDave Chinner if (!cil)
17522451337dSDave Chinner return -ENOMEM;
175333c0dd78SDave Chinner /*
175433c0dd78SDave Chinner * Limit the CIL pipeline depth to 4 concurrent works to bound the
175533c0dd78SDave Chinner * concurrency the log spinlocks will be exposed to.
175633c0dd78SDave Chinner */
175733c0dd78SDave Chinner cil->xc_push_wq = alloc_workqueue("xfs-cil/%s",
175833c0dd78SDave Chinner XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_UNBOUND),
175933c0dd78SDave Chinner 4, log->l_mp->m_super->s_id);
176033c0dd78SDave Chinner if (!cil->xc_push_wq)
176133c0dd78SDave Chinner goto out_destroy_cil;
17624c2d542fSDave Chinner
1763af1c2146SDave Chinner cil->xc_log = log;
1764af1c2146SDave Chinner cil->xc_pcp = alloc_percpu(struct xlog_cil_pcp);
1765af1c2146SDave Chinner if (!cil->xc_pcp)
1766af1c2146SDave Chinner goto out_destroy_wq;
1767af1c2146SDave Chinner
1768df7a4a21SDave Chinner for_each_possible_cpu(cpu) {
1769df7a4a21SDave Chinner cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
1770df7a4a21SDave Chinner INIT_LIST_HEAD(&cilpcp->busy_extents);
1771c0fb4765SDave Chinner INIT_LIST_HEAD(&cilpcp->log_items);
1772df7a4a21SDave Chinner }
1773df7a4a21SDave Chinner
17744c2d542fSDave Chinner INIT_LIST_HEAD(&cil->xc_committing);
17754bb928cdSDave Chinner spin_lock_init(&cil->xc_push_lock);
1776c7f87f39SDave Chinner init_waitqueue_head(&cil->xc_push_wait);
17774c2d542fSDave Chinner init_rwsem(&cil->xc_ctx_lock);
177868a74dcaSDave Chinner init_waitqueue_head(&cil->xc_start_wait);
17794c2d542fSDave Chinner init_waitqueue_head(&cil->xc_commit_wait);
17804c2d542fSDave Chinner log->l_cilp = cil;
178139823d0fSDave Chinner
178239823d0fSDave Chinner ctx = xlog_cil_ctx_alloc();
178339823d0fSDave Chinner xlog_cil_ctx_switch(cil, ctx);
17844c2d542fSDave Chinner return 0;
178533c0dd78SDave Chinner
1786af1c2146SDave Chinner out_destroy_wq:
1787af1c2146SDave Chinner destroy_workqueue(cil->xc_push_wq);
178833c0dd78SDave Chinner out_destroy_cil:
178933c0dd78SDave Chinner kmem_free(cil);
179033c0dd78SDave Chinner return -ENOMEM;
17914c2d542fSDave Chinner }
17924c2d542fSDave Chinner
17934c2d542fSDave Chinner void
xlog_cil_destroy(struct xlog * log)17944c2d542fSDave Chinner xlog_cil_destroy(
1795f7bdf03aSMark Tinguely struct xlog *log)
17964c2d542fSDave Chinner {
179788591e7fSDave Chinner struct xfs_cil *cil = log->l_cilp;
179888591e7fSDave Chinner
179988591e7fSDave Chinner if (cil->xc_ctx) {
180088591e7fSDave Chinner if (cil->xc_ctx->ticket)
180188591e7fSDave Chinner xfs_log_ticket_put(cil->xc_ctx->ticket);
180288591e7fSDave Chinner kmem_free(cil->xc_ctx);
18034c2d542fSDave Chinner }
18044c2d542fSDave Chinner
180588591e7fSDave Chinner ASSERT(test_bit(XLOG_CIL_EMPTY, &cil->xc_flags));
1806af1c2146SDave Chinner free_percpu(cil->xc_pcp);
180788591e7fSDave Chinner destroy_workqueue(cil->xc_push_wq);
180888591e7fSDave Chinner kmem_free(cil);
18094c2d542fSDave Chinner }
18104c2d542fSDave Chinner
1811