xref: /openbmc/linux/fs/xfs/xfs_buf_item.c (revision 6774def6)
1 /*
2  * Copyright (c) 2000-2005 Silicon Graphics, Inc.
3  * All Rights Reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License as
7  * published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it would be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write the Free Software Foundation,
16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17  */
18 #include "xfs.h"
19 #include "xfs_fs.h"
20 #include "xfs_log_format.h"
21 #include "xfs_trans_resv.h"
22 #include "xfs_bit.h"
23 #include "xfs_sb.h"
24 #include "xfs_ag.h"
25 #include "xfs_mount.h"
26 #include "xfs_trans.h"
27 #include "xfs_buf_item.h"
28 #include "xfs_trans_priv.h"
29 #include "xfs_error.h"
30 #include "xfs_trace.h"
31 #include "xfs_log.h"
32 
33 
34 kmem_zone_t	*xfs_buf_item_zone;
35 
36 static inline struct xfs_buf_log_item *BUF_ITEM(struct xfs_log_item *lip)
37 {
38 	return container_of(lip, struct xfs_buf_log_item, bli_item);
39 }
40 
41 STATIC void	xfs_buf_do_callbacks(struct xfs_buf *bp);
42 
43 static inline int
44 xfs_buf_log_format_size(
45 	struct xfs_buf_log_format *blfp)
46 {
47 	return offsetof(struct xfs_buf_log_format, blf_data_map) +
48 			(blfp->blf_map_size * sizeof(blfp->blf_data_map[0]));
49 }
50 
51 /*
52  * This returns the number of log iovecs needed to log the
53  * given buf log item.
54  *
55  * It calculates this as 1 iovec for the buf log format structure
56  * and 1 for each stretch of non-contiguous chunks to be logged.
57  * Contiguous chunks are logged in a single iovec.
58  *
59  * If the XFS_BLI_STALE flag has been set, then log nothing.
60  */
61 STATIC void
62 xfs_buf_item_size_segment(
63 	struct xfs_buf_log_item	*bip,
64 	struct xfs_buf_log_format *blfp,
65 	int			*nvecs,
66 	int			*nbytes)
67 {
68 	struct xfs_buf		*bp = bip->bli_buf;
69 	int			next_bit;
70 	int			last_bit;
71 
72 	last_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
73 	if (last_bit == -1)
74 		return;
75 
76 	/*
77 	 * initial count for a dirty buffer is 2 vectors - the format structure
78 	 * and the first dirty region.
79 	 */
80 	*nvecs += 2;
81 	*nbytes += xfs_buf_log_format_size(blfp) + XFS_BLF_CHUNK;
82 
83 	while (last_bit != -1) {
84 		/*
85 		 * This takes the bit number to start looking from and
86 		 * returns the next set bit from there.  It returns -1
87 		 * if there are no more bits set or the start bit is
88 		 * beyond the end of the bitmap.
89 		 */
90 		next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
91 					last_bit + 1);
92 		/*
93 		 * If we run out of bits, leave the loop,
94 		 * else if we find a new set of bits bump the number of vecs,
95 		 * else keep scanning the current set of bits.
96 		 */
97 		if (next_bit == -1) {
98 			break;
99 		} else if (next_bit != last_bit + 1) {
100 			last_bit = next_bit;
101 			(*nvecs)++;
102 		} else if (xfs_buf_offset(bp, next_bit * XFS_BLF_CHUNK) !=
103 			   (xfs_buf_offset(bp, last_bit * XFS_BLF_CHUNK) +
104 			    XFS_BLF_CHUNK)) {
105 			last_bit = next_bit;
106 			(*nvecs)++;
107 		} else {
108 			last_bit++;
109 		}
110 		*nbytes += XFS_BLF_CHUNK;
111 	}
112 }
113 
114 /*
115  * This returns the number of log iovecs needed to log the given buf log item.
116  *
117  * It calculates this as 1 iovec for the buf log format structure and 1 for each
118  * stretch of non-contiguous chunks to be logged.  Contiguous chunks are logged
119  * in a single iovec.
120  *
121  * Discontiguous buffers need a format structure per region that that is being
122  * logged. This makes the changes in the buffer appear to log recovery as though
123  * they came from separate buffers, just like would occur if multiple buffers
124  * were used instead of a single discontiguous buffer. This enables
125  * discontiguous buffers to be in-memory constructs, completely transparent to
126  * what ends up on disk.
127  *
128  * If the XFS_BLI_STALE flag has been set, then log nothing but the buf log
129  * format structures.
130  */
131 STATIC void
132 xfs_buf_item_size(
133 	struct xfs_log_item	*lip,
134 	int			*nvecs,
135 	int			*nbytes)
136 {
137 	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
138 	int			i;
139 
140 	ASSERT(atomic_read(&bip->bli_refcount) > 0);
141 	if (bip->bli_flags & XFS_BLI_STALE) {
142 		/*
143 		 * The buffer is stale, so all we need to log
144 		 * is the buf log format structure with the
145 		 * cancel flag in it.
146 		 */
147 		trace_xfs_buf_item_size_stale(bip);
148 		ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
149 		*nvecs += bip->bli_format_count;
150 		for (i = 0; i < bip->bli_format_count; i++) {
151 			*nbytes += xfs_buf_log_format_size(&bip->bli_formats[i]);
152 		}
153 		return;
154 	}
155 
156 	ASSERT(bip->bli_flags & XFS_BLI_LOGGED);
157 
158 	if (bip->bli_flags & XFS_BLI_ORDERED) {
159 		/*
160 		 * The buffer has been logged just to order it.
161 		 * It is not being included in the transaction
162 		 * commit, so no vectors are used at all.
163 		 */
164 		trace_xfs_buf_item_size_ordered(bip);
165 		*nvecs = XFS_LOG_VEC_ORDERED;
166 		return;
167 	}
168 
169 	/*
170 	 * the vector count is based on the number of buffer vectors we have
171 	 * dirty bits in. This will only be greater than one when we have a
172 	 * compound buffer with more than one segment dirty. Hence for compound
173 	 * buffers we need to track which segment the dirty bits correspond to,
174 	 * and when we move from one segment to the next increment the vector
175 	 * count for the extra buf log format structure that will need to be
176 	 * written.
177 	 */
178 	for (i = 0; i < bip->bli_format_count; i++) {
179 		xfs_buf_item_size_segment(bip, &bip->bli_formats[i],
180 					  nvecs, nbytes);
181 	}
182 	trace_xfs_buf_item_size(bip);
183 }
184 
185 static inline void
186 xfs_buf_item_copy_iovec(
187 	struct xfs_log_vec	*lv,
188 	struct xfs_log_iovec	**vecp,
189 	struct xfs_buf		*bp,
190 	uint			offset,
191 	int			first_bit,
192 	uint			nbits)
193 {
194 	offset += first_bit * XFS_BLF_CHUNK;
195 	xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_BCHUNK,
196 			xfs_buf_offset(bp, offset),
197 			nbits * XFS_BLF_CHUNK);
198 }
199 
200 static inline bool
201 xfs_buf_item_straddle(
202 	struct xfs_buf		*bp,
203 	uint			offset,
204 	int			next_bit,
205 	int			last_bit)
206 {
207 	return xfs_buf_offset(bp, offset + (next_bit << XFS_BLF_SHIFT)) !=
208 		(xfs_buf_offset(bp, offset + (last_bit << XFS_BLF_SHIFT)) +
209 		 XFS_BLF_CHUNK);
210 }
211 
212 static void
213 xfs_buf_item_format_segment(
214 	struct xfs_buf_log_item	*bip,
215 	struct xfs_log_vec	*lv,
216 	struct xfs_log_iovec	**vecp,
217 	uint			offset,
218 	struct xfs_buf_log_format *blfp)
219 {
220 	struct xfs_buf	*bp = bip->bli_buf;
221 	uint		base_size;
222 	int		first_bit;
223 	int		last_bit;
224 	int		next_bit;
225 	uint		nbits;
226 
227 	/* copy the flags across from the base format item */
228 	blfp->blf_flags = bip->__bli_format.blf_flags;
229 
230 	/*
231 	 * Base size is the actual size of the ondisk structure - it reflects
232 	 * the actual size of the dirty bitmap rather than the size of the in
233 	 * memory structure.
234 	 */
235 	base_size = xfs_buf_log_format_size(blfp);
236 
237 	first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
238 	if (!(bip->bli_flags & XFS_BLI_STALE) && first_bit == -1) {
239 		/*
240 		 * If the map is not be dirty in the transaction, mark
241 		 * the size as zero and do not advance the vector pointer.
242 		 */
243 		return;
244 	}
245 
246 	blfp = xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_BFORMAT, blfp, base_size);
247 	blfp->blf_size = 1;
248 
249 	if (bip->bli_flags & XFS_BLI_STALE) {
250 		/*
251 		 * The buffer is stale, so all we need to log
252 		 * is the buf log format structure with the
253 		 * cancel flag in it.
254 		 */
255 		trace_xfs_buf_item_format_stale(bip);
256 		ASSERT(blfp->blf_flags & XFS_BLF_CANCEL);
257 		return;
258 	}
259 
260 
261 	/*
262 	 * Fill in an iovec for each set of contiguous chunks.
263 	 */
264 	last_bit = first_bit;
265 	nbits = 1;
266 	for (;;) {
267 		/*
268 		 * This takes the bit number to start looking from and
269 		 * returns the next set bit from there.  It returns -1
270 		 * if there are no more bits set or the start bit is
271 		 * beyond the end of the bitmap.
272 		 */
273 		next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
274 					(uint)last_bit + 1);
275 		/*
276 		 * If we run out of bits fill in the last iovec and get out of
277 		 * the loop.  Else if we start a new set of bits then fill in
278 		 * the iovec for the series we were looking at and start
279 		 * counting the bits in the new one.  Else we're still in the
280 		 * same set of bits so just keep counting and scanning.
281 		 */
282 		if (next_bit == -1) {
283 			xfs_buf_item_copy_iovec(lv, vecp, bp, offset,
284 						first_bit, nbits);
285 			blfp->blf_size++;
286 			break;
287 		} else if (next_bit != last_bit + 1 ||
288 		           xfs_buf_item_straddle(bp, offset, next_bit, last_bit)) {
289 			xfs_buf_item_copy_iovec(lv, vecp, bp, offset,
290 						first_bit, nbits);
291 			blfp->blf_size++;
292 			first_bit = next_bit;
293 			last_bit = next_bit;
294 			nbits = 1;
295 		} else {
296 			last_bit++;
297 			nbits++;
298 		}
299 	}
300 }
301 
302 /*
303  * This is called to fill in the vector of log iovecs for the
304  * given log buf item.  It fills the first entry with a buf log
305  * format structure, and the rest point to contiguous chunks
306  * within the buffer.
307  */
308 STATIC void
309 xfs_buf_item_format(
310 	struct xfs_log_item	*lip,
311 	struct xfs_log_vec	*lv)
312 {
313 	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
314 	struct xfs_buf		*bp = bip->bli_buf;
315 	struct xfs_log_iovec	*vecp = NULL;
316 	uint			offset = 0;
317 	int			i;
318 
319 	ASSERT(atomic_read(&bip->bli_refcount) > 0);
320 	ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
321 	       (bip->bli_flags & XFS_BLI_STALE));
322 
323 	/*
324 	 * If it is an inode buffer, transfer the in-memory state to the
325 	 * format flags and clear the in-memory state.
326 	 *
327 	 * For buffer based inode allocation, we do not transfer
328 	 * this state if the inode buffer allocation has not yet been committed
329 	 * to the log as setting the XFS_BLI_INODE_BUF flag will prevent
330 	 * correct replay of the inode allocation.
331 	 *
332 	 * For icreate item based inode allocation, the buffers aren't written
333 	 * to the journal during allocation, and hence we should always tag the
334 	 * buffer as an inode buffer so that the correct unlinked list replay
335 	 * occurs during recovery.
336 	 */
337 	if (bip->bli_flags & XFS_BLI_INODE_BUF) {
338 		if (xfs_sb_version_hascrc(&lip->li_mountp->m_sb) ||
339 		    !((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) &&
340 		      xfs_log_item_in_current_chkpt(lip)))
341 			bip->__bli_format.blf_flags |= XFS_BLF_INODE_BUF;
342 		bip->bli_flags &= ~XFS_BLI_INODE_BUF;
343 	}
344 
345 	if ((bip->bli_flags & (XFS_BLI_ORDERED|XFS_BLI_STALE)) ==
346 							XFS_BLI_ORDERED) {
347 		/*
348 		 * The buffer has been logged just to order it.  It is not being
349 		 * included in the transaction commit, so don't format it.
350 		 */
351 		trace_xfs_buf_item_format_ordered(bip);
352 		return;
353 	}
354 
355 	for (i = 0; i < bip->bli_format_count; i++) {
356 		xfs_buf_item_format_segment(bip, lv, &vecp, offset,
357 					    &bip->bli_formats[i]);
358 		offset += bp->b_maps[i].bm_len;
359 	}
360 
361 	/*
362 	 * Check to make sure everything is consistent.
363 	 */
364 	trace_xfs_buf_item_format(bip);
365 }
366 
367 /*
368  * This is called to pin the buffer associated with the buf log item in memory
369  * so it cannot be written out.
370  *
371  * We also always take a reference to the buffer log item here so that the bli
372  * is held while the item is pinned in memory. This means that we can
373  * unconditionally drop the reference count a transaction holds when the
374  * transaction is completed.
375  */
376 STATIC void
377 xfs_buf_item_pin(
378 	struct xfs_log_item	*lip)
379 {
380 	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
381 
382 	ASSERT(atomic_read(&bip->bli_refcount) > 0);
383 	ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
384 	       (bip->bli_flags & XFS_BLI_ORDERED) ||
385 	       (bip->bli_flags & XFS_BLI_STALE));
386 
387 	trace_xfs_buf_item_pin(bip);
388 
389 	atomic_inc(&bip->bli_refcount);
390 	atomic_inc(&bip->bli_buf->b_pin_count);
391 }
392 
393 /*
394  * This is called to unpin the buffer associated with the buf log
395  * item which was previously pinned with a call to xfs_buf_item_pin().
396  *
397  * Also drop the reference to the buf item for the current transaction.
398  * If the XFS_BLI_STALE flag is set and we are the last reference,
399  * then free up the buf log item and unlock the buffer.
400  *
401  * If the remove flag is set we are called from uncommit in the
402  * forced-shutdown path.  If that is true and the reference count on
403  * the log item is going to drop to zero we need to free the item's
404  * descriptor in the transaction.
405  */
406 STATIC void
407 xfs_buf_item_unpin(
408 	struct xfs_log_item	*lip,
409 	int			remove)
410 {
411 	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
412 	xfs_buf_t	*bp = bip->bli_buf;
413 	struct xfs_ail	*ailp = lip->li_ailp;
414 	int		stale = bip->bli_flags & XFS_BLI_STALE;
415 	int		freed;
416 
417 	ASSERT(bp->b_fspriv == bip);
418 	ASSERT(atomic_read(&bip->bli_refcount) > 0);
419 
420 	trace_xfs_buf_item_unpin(bip);
421 
422 	freed = atomic_dec_and_test(&bip->bli_refcount);
423 
424 	if (atomic_dec_and_test(&bp->b_pin_count))
425 		wake_up_all(&bp->b_waiters);
426 
427 	if (freed && stale) {
428 		ASSERT(bip->bli_flags & XFS_BLI_STALE);
429 		ASSERT(xfs_buf_islocked(bp));
430 		ASSERT(XFS_BUF_ISSTALE(bp));
431 		ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
432 
433 		trace_xfs_buf_item_unpin_stale(bip);
434 
435 		if (remove) {
436 			/*
437 			 * If we are in a transaction context, we have to
438 			 * remove the log item from the transaction as we are
439 			 * about to release our reference to the buffer.  If we
440 			 * don't, the unlock that occurs later in
441 			 * xfs_trans_uncommit() will try to reference the
442 			 * buffer which we no longer have a hold on.
443 			 */
444 			if (lip->li_desc)
445 				xfs_trans_del_item(lip);
446 
447 			/*
448 			 * Since the transaction no longer refers to the buffer,
449 			 * the buffer should no longer refer to the transaction.
450 			 */
451 			bp->b_transp = NULL;
452 		}
453 
454 		/*
455 		 * If we get called here because of an IO error, we may
456 		 * or may not have the item on the AIL. xfs_trans_ail_delete()
457 		 * will take care of that situation.
458 		 * xfs_trans_ail_delete() drops the AIL lock.
459 		 */
460 		if (bip->bli_flags & XFS_BLI_STALE_INODE) {
461 			xfs_buf_do_callbacks(bp);
462 			bp->b_fspriv = NULL;
463 			bp->b_iodone = NULL;
464 		} else {
465 			spin_lock(&ailp->xa_lock);
466 			xfs_trans_ail_delete(ailp, lip, SHUTDOWN_LOG_IO_ERROR);
467 			xfs_buf_item_relse(bp);
468 			ASSERT(bp->b_fspriv == NULL);
469 		}
470 		xfs_buf_relse(bp);
471 	} else if (freed && remove) {
472 		/*
473 		 * There are currently two references to the buffer - the active
474 		 * LRU reference and the buf log item. What we are about to do
475 		 * here - simulate a failed IO completion - requires 3
476 		 * references.
477 		 *
478 		 * The LRU reference is removed by the xfs_buf_stale() call. The
479 		 * buf item reference is removed by the xfs_buf_iodone()
480 		 * callback that is run by xfs_buf_do_callbacks() during ioend
481 		 * processing (via the bp->b_iodone callback), and then finally
482 		 * the ioend processing will drop the IO reference if the buffer
483 		 * is marked XBF_ASYNC.
484 		 *
485 		 * Hence we need to take an additional reference here so that IO
486 		 * completion processing doesn't free the buffer prematurely.
487 		 */
488 		xfs_buf_lock(bp);
489 		xfs_buf_hold(bp);
490 		bp->b_flags |= XBF_ASYNC;
491 		xfs_buf_ioerror(bp, -EIO);
492 		XFS_BUF_UNDONE(bp);
493 		xfs_buf_stale(bp);
494 		xfs_buf_ioend(bp);
495 	}
496 }
497 
498 /*
499  * Buffer IO error rate limiting. Limit it to no more than 10 messages per 30
500  * seconds so as to not spam logs too much on repeated detection of the same
501  * buffer being bad..
502  */
503 
504 static DEFINE_RATELIMIT_STATE(xfs_buf_write_fail_rl_state, 30 * HZ, 10);
505 
506 STATIC uint
507 xfs_buf_item_push(
508 	struct xfs_log_item	*lip,
509 	struct list_head	*buffer_list)
510 {
511 	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
512 	struct xfs_buf		*bp = bip->bli_buf;
513 	uint			rval = XFS_ITEM_SUCCESS;
514 
515 	if (xfs_buf_ispinned(bp))
516 		return XFS_ITEM_PINNED;
517 	if (!xfs_buf_trylock(bp)) {
518 		/*
519 		 * If we have just raced with a buffer being pinned and it has
520 		 * been marked stale, we could end up stalling until someone else
521 		 * issues a log force to unpin the stale buffer. Check for the
522 		 * race condition here so xfsaild recognizes the buffer is pinned
523 		 * and queues a log force to move it along.
524 		 */
525 		if (xfs_buf_ispinned(bp))
526 			return XFS_ITEM_PINNED;
527 		return XFS_ITEM_LOCKED;
528 	}
529 
530 	ASSERT(!(bip->bli_flags & XFS_BLI_STALE));
531 
532 	trace_xfs_buf_item_push(bip);
533 
534 	/* has a previous flush failed due to IO errors? */
535 	if ((bp->b_flags & XBF_WRITE_FAIL) &&
536 	    ___ratelimit(&xfs_buf_write_fail_rl_state, "XFS:")) {
537 		xfs_warn(bp->b_target->bt_mount,
538 "Detected failing async write on buffer block 0x%llx. Retrying async write.\n",
539 			 (long long)bp->b_bn);
540 	}
541 
542 	if (!xfs_buf_delwri_queue(bp, buffer_list))
543 		rval = XFS_ITEM_FLUSHING;
544 	xfs_buf_unlock(bp);
545 	return rval;
546 }
547 
548 /*
549  * Release the buffer associated with the buf log item.  If there is no dirty
550  * logged data associated with the buffer recorded in the buf log item, then
551  * free the buf log item and remove the reference to it in the buffer.
552  *
553  * This call ignores the recursion count.  It is only called when the buffer
554  * should REALLY be unlocked, regardless of the recursion count.
555  *
556  * We unconditionally drop the transaction's reference to the log item. If the
557  * item was logged, then another reference was taken when it was pinned, so we
558  * can safely drop the transaction reference now.  This also allows us to avoid
559  * potential races with the unpin code freeing the bli by not referencing the
560  * bli after we've dropped the reference count.
561  *
562  * If the XFS_BLI_HOLD flag is set in the buf log item, then free the log item
563  * if necessary but do not unlock the buffer.  This is for support of
564  * xfs_trans_bhold(). Make sure the XFS_BLI_HOLD field is cleared if we don't
565  * free the item.
566  */
567 STATIC void
568 xfs_buf_item_unlock(
569 	struct xfs_log_item	*lip)
570 {
571 	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
572 	struct xfs_buf		*bp = bip->bli_buf;
573 	bool			clean;
574 	bool			aborted;
575 	int			flags;
576 
577 	/* Clear the buffer's association with this transaction. */
578 	bp->b_transp = NULL;
579 
580 	/*
581 	 * If this is a transaction abort, don't return early.  Instead, allow
582 	 * the brelse to happen.  Normally it would be done for stale
583 	 * (cancelled) buffers at unpin time, but we'll never go through the
584 	 * pin/unpin cycle if we abort inside commit.
585 	 */
586 	aborted = (lip->li_flags & XFS_LI_ABORTED) ? true : false;
587 	/*
588 	 * Before possibly freeing the buf item, copy the per-transaction state
589 	 * so we can reference it safely later after clearing it from the
590 	 * buffer log item.
591 	 */
592 	flags = bip->bli_flags;
593 	bip->bli_flags &= ~(XFS_BLI_LOGGED | XFS_BLI_HOLD | XFS_BLI_ORDERED);
594 
595 	/*
596 	 * If the buf item is marked stale, then don't do anything.  We'll
597 	 * unlock the buffer and free the buf item when the buffer is unpinned
598 	 * for the last time.
599 	 */
600 	if (flags & XFS_BLI_STALE) {
601 		trace_xfs_buf_item_unlock_stale(bip);
602 		ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
603 		if (!aborted) {
604 			atomic_dec(&bip->bli_refcount);
605 			return;
606 		}
607 	}
608 
609 	trace_xfs_buf_item_unlock(bip);
610 
611 	/*
612 	 * If the buf item isn't tracking any data, free it, otherwise drop the
613 	 * reference we hold to it. If we are aborting the transaction, this may
614 	 * be the only reference to the buf item, so we free it anyway
615 	 * regardless of whether it is dirty or not. A dirty abort implies a
616 	 * shutdown, anyway.
617 	 *
618 	 * Ordered buffers are dirty but may have no recorded changes, so ensure
619 	 * we only release clean items here.
620 	 */
621 	clean = (flags & XFS_BLI_DIRTY) ? false : true;
622 	if (clean) {
623 		int i;
624 		for (i = 0; i < bip->bli_format_count; i++) {
625 			if (!xfs_bitmap_empty(bip->bli_formats[i].blf_data_map,
626 				     bip->bli_formats[i].blf_map_size)) {
627 				clean = false;
628 				break;
629 			}
630 		}
631 	}
632 
633 	/*
634 	 * Clean buffers, by definition, cannot be in the AIL. However, aborted
635 	 * buffers may be dirty and hence in the AIL. Therefore if we are
636 	 * aborting a buffer and we've just taken the last refernce away, we
637 	 * have to check if it is in the AIL before freeing it. We need to free
638 	 * it in this case, because an aborted transaction has already shut the
639 	 * filesystem down and this is the last chance we will have to do so.
640 	 */
641 	if (atomic_dec_and_test(&bip->bli_refcount)) {
642 		if (clean)
643 			xfs_buf_item_relse(bp);
644 		else if (aborted) {
645 			ASSERT(XFS_FORCED_SHUTDOWN(lip->li_mountp));
646 			if (lip->li_flags & XFS_LI_IN_AIL) {
647 				spin_lock(&lip->li_ailp->xa_lock);
648 				xfs_trans_ail_delete(lip->li_ailp, lip,
649 						     SHUTDOWN_LOG_IO_ERROR);
650 			}
651 			xfs_buf_item_relse(bp);
652 		}
653 	}
654 
655 	if (!(flags & XFS_BLI_HOLD))
656 		xfs_buf_relse(bp);
657 }
658 
659 /*
660  * This is called to find out where the oldest active copy of the
661  * buf log item in the on disk log resides now that the last log
662  * write of it completed at the given lsn.
663  * We always re-log all the dirty data in a buffer, so usually the
664  * latest copy in the on disk log is the only one that matters.  For
665  * those cases we simply return the given lsn.
666  *
667  * The one exception to this is for buffers full of newly allocated
668  * inodes.  These buffers are only relogged with the XFS_BLI_INODE_BUF
669  * flag set, indicating that only the di_next_unlinked fields from the
670  * inodes in the buffers will be replayed during recovery.  If the
671  * original newly allocated inode images have not yet been flushed
672  * when the buffer is so relogged, then we need to make sure that we
673  * keep the old images in the 'active' portion of the log.  We do this
674  * by returning the original lsn of that transaction here rather than
675  * the current one.
676  */
677 STATIC xfs_lsn_t
678 xfs_buf_item_committed(
679 	struct xfs_log_item	*lip,
680 	xfs_lsn_t		lsn)
681 {
682 	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
683 
684 	trace_xfs_buf_item_committed(bip);
685 
686 	if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && lip->li_lsn != 0)
687 		return lip->li_lsn;
688 	return lsn;
689 }
690 
691 STATIC void
692 xfs_buf_item_committing(
693 	struct xfs_log_item	*lip,
694 	xfs_lsn_t		commit_lsn)
695 {
696 }
697 
698 /*
699  * This is the ops vector shared by all buf log items.
700  */
701 static const struct xfs_item_ops xfs_buf_item_ops = {
702 	.iop_size	= xfs_buf_item_size,
703 	.iop_format	= xfs_buf_item_format,
704 	.iop_pin	= xfs_buf_item_pin,
705 	.iop_unpin	= xfs_buf_item_unpin,
706 	.iop_unlock	= xfs_buf_item_unlock,
707 	.iop_committed	= xfs_buf_item_committed,
708 	.iop_push	= xfs_buf_item_push,
709 	.iop_committing = xfs_buf_item_committing
710 };
711 
712 STATIC int
713 xfs_buf_item_get_format(
714 	struct xfs_buf_log_item	*bip,
715 	int			count)
716 {
717 	ASSERT(bip->bli_formats == NULL);
718 	bip->bli_format_count = count;
719 
720 	if (count == 1) {
721 		bip->bli_formats = &bip->__bli_format;
722 		return 0;
723 	}
724 
725 	bip->bli_formats = kmem_zalloc(count * sizeof(struct xfs_buf_log_format),
726 				KM_SLEEP);
727 	if (!bip->bli_formats)
728 		return -ENOMEM;
729 	return 0;
730 }
731 
732 STATIC void
733 xfs_buf_item_free_format(
734 	struct xfs_buf_log_item	*bip)
735 {
736 	if (bip->bli_formats != &bip->__bli_format) {
737 		kmem_free(bip->bli_formats);
738 		bip->bli_formats = NULL;
739 	}
740 }
741 
742 /*
743  * Allocate a new buf log item to go with the given buffer.
744  * Set the buffer's b_fsprivate field to point to the new
745  * buf log item.  If there are other item's attached to the
746  * buffer (see xfs_buf_attach_iodone() below), then put the
747  * buf log item at the front.
748  */
749 void
750 xfs_buf_item_init(
751 	xfs_buf_t	*bp,
752 	xfs_mount_t	*mp)
753 {
754 	xfs_log_item_t		*lip = bp->b_fspriv;
755 	xfs_buf_log_item_t	*bip;
756 	int			chunks;
757 	int			map_size;
758 	int			error;
759 	int			i;
760 
761 	/*
762 	 * Check to see if there is already a buf log item for
763 	 * this buffer.  If there is, it is guaranteed to be
764 	 * the first.  If we do already have one, there is
765 	 * nothing to do here so return.
766 	 */
767 	ASSERT(bp->b_target->bt_mount == mp);
768 	if (lip != NULL && lip->li_type == XFS_LI_BUF)
769 		return;
770 
771 	bip = kmem_zone_zalloc(xfs_buf_item_zone, KM_SLEEP);
772 	xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops);
773 	bip->bli_buf = bp;
774 	xfs_buf_hold(bp);
775 
776 	/*
777 	 * chunks is the number of XFS_BLF_CHUNK size pieces the buffer
778 	 * can be divided into. Make sure not to truncate any pieces.
779 	 * map_size is the size of the bitmap needed to describe the
780 	 * chunks of the buffer.
781 	 *
782 	 * Discontiguous buffer support follows the layout of the underlying
783 	 * buffer. This makes the implementation as simple as possible.
784 	 */
785 	error = xfs_buf_item_get_format(bip, bp->b_map_count);
786 	ASSERT(error == 0);
787 
788 	for (i = 0; i < bip->bli_format_count; i++) {
789 		chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len),
790 				      XFS_BLF_CHUNK);
791 		map_size = DIV_ROUND_UP(chunks, NBWORD);
792 
793 		bip->bli_formats[i].blf_type = XFS_LI_BUF;
794 		bip->bli_formats[i].blf_blkno = bp->b_maps[i].bm_bn;
795 		bip->bli_formats[i].blf_len = bp->b_maps[i].bm_len;
796 		bip->bli_formats[i].blf_map_size = map_size;
797 	}
798 
799 	/*
800 	 * Put the buf item into the list of items attached to the
801 	 * buffer at the front.
802 	 */
803 	if (bp->b_fspriv)
804 		bip->bli_item.li_bio_list = bp->b_fspriv;
805 	bp->b_fspriv = bip;
806 }
807 
808 
809 /*
810  * Mark bytes first through last inclusive as dirty in the buf
811  * item's bitmap.
812  */
813 static void
814 xfs_buf_item_log_segment(
815 	uint			first,
816 	uint			last,
817 	uint			*map)
818 {
819 	uint		first_bit;
820 	uint		last_bit;
821 	uint		bits_to_set;
822 	uint		bits_set;
823 	uint		word_num;
824 	uint		*wordp;
825 	uint		bit;
826 	uint		end_bit;
827 	uint		mask;
828 
829 	/*
830 	 * Convert byte offsets to bit numbers.
831 	 */
832 	first_bit = first >> XFS_BLF_SHIFT;
833 	last_bit = last >> XFS_BLF_SHIFT;
834 
835 	/*
836 	 * Calculate the total number of bits to be set.
837 	 */
838 	bits_to_set = last_bit - first_bit + 1;
839 
840 	/*
841 	 * Get a pointer to the first word in the bitmap
842 	 * to set a bit in.
843 	 */
844 	word_num = first_bit >> BIT_TO_WORD_SHIFT;
845 	wordp = &map[word_num];
846 
847 	/*
848 	 * Calculate the starting bit in the first word.
849 	 */
850 	bit = first_bit & (uint)(NBWORD - 1);
851 
852 	/*
853 	 * First set any bits in the first word of our range.
854 	 * If it starts at bit 0 of the word, it will be
855 	 * set below rather than here.  That is what the variable
856 	 * bit tells us. The variable bits_set tracks the number
857 	 * of bits that have been set so far.  End_bit is the number
858 	 * of the last bit to be set in this word plus one.
859 	 */
860 	if (bit) {
861 		end_bit = MIN(bit + bits_to_set, (uint)NBWORD);
862 		mask = ((1 << (end_bit - bit)) - 1) << bit;
863 		*wordp |= mask;
864 		wordp++;
865 		bits_set = end_bit - bit;
866 	} else {
867 		bits_set = 0;
868 	}
869 
870 	/*
871 	 * Now set bits a whole word at a time that are between
872 	 * first_bit and last_bit.
873 	 */
874 	while ((bits_to_set - bits_set) >= NBWORD) {
875 		*wordp |= 0xffffffff;
876 		bits_set += NBWORD;
877 		wordp++;
878 	}
879 
880 	/*
881 	 * Finally, set any bits left to be set in one last partial word.
882 	 */
883 	end_bit = bits_to_set - bits_set;
884 	if (end_bit) {
885 		mask = (1 << end_bit) - 1;
886 		*wordp |= mask;
887 	}
888 }
889 
890 /*
891  * Mark bytes first through last inclusive as dirty in the buf
892  * item's bitmap.
893  */
894 void
895 xfs_buf_item_log(
896 	xfs_buf_log_item_t	*bip,
897 	uint			first,
898 	uint			last)
899 {
900 	int			i;
901 	uint			start;
902 	uint			end;
903 	struct xfs_buf		*bp = bip->bli_buf;
904 
905 	/*
906 	 * walk each buffer segment and mark them dirty appropriately.
907 	 */
908 	start = 0;
909 	for (i = 0; i < bip->bli_format_count; i++) {
910 		if (start > last)
911 			break;
912 		end = start + BBTOB(bp->b_maps[i].bm_len);
913 		if (first > end) {
914 			start += BBTOB(bp->b_maps[i].bm_len);
915 			continue;
916 		}
917 		if (first < start)
918 			first = start;
919 		if (end > last)
920 			end = last;
921 
922 		xfs_buf_item_log_segment(first, end,
923 					 &bip->bli_formats[i].blf_data_map[0]);
924 
925 		start += bp->b_maps[i].bm_len;
926 	}
927 }
928 
929 
930 /*
931  * Return 1 if the buffer has been logged or ordered in a transaction (at any
932  * point, not just the current transaction) and 0 if not.
933  */
934 uint
935 xfs_buf_item_dirty(
936 	xfs_buf_log_item_t	*bip)
937 {
938 	return (bip->bli_flags & XFS_BLI_DIRTY);
939 }
940 
941 STATIC void
942 xfs_buf_item_free(
943 	xfs_buf_log_item_t	*bip)
944 {
945 	xfs_buf_item_free_format(bip);
946 	kmem_zone_free(xfs_buf_item_zone, bip);
947 }
948 
949 /*
950  * This is called when the buf log item is no longer needed.  It should
951  * free the buf log item associated with the given buffer and clear
952  * the buffer's pointer to the buf log item.  If there are no more
953  * items in the list, clear the b_iodone field of the buffer (see
954  * xfs_buf_attach_iodone() below).
955  */
956 void
957 xfs_buf_item_relse(
958 	xfs_buf_t	*bp)
959 {
960 	xfs_buf_log_item_t	*bip = bp->b_fspriv;
961 
962 	trace_xfs_buf_item_relse(bp, _RET_IP_);
963 	ASSERT(!(bip->bli_item.li_flags & XFS_LI_IN_AIL));
964 
965 	bp->b_fspriv = bip->bli_item.li_bio_list;
966 	if (bp->b_fspriv == NULL)
967 		bp->b_iodone = NULL;
968 
969 	xfs_buf_rele(bp);
970 	xfs_buf_item_free(bip);
971 }
972 
973 
974 /*
975  * Add the given log item with its callback to the list of callbacks
976  * to be called when the buffer's I/O completes.  If it is not set
977  * already, set the buffer's b_iodone() routine to be
978  * xfs_buf_iodone_callbacks() and link the log item into the list of
979  * items rooted at b_fsprivate.  Items are always added as the second
980  * entry in the list if there is a first, because the buf item code
981  * assumes that the buf log item is first.
982  */
983 void
984 xfs_buf_attach_iodone(
985 	xfs_buf_t	*bp,
986 	void		(*cb)(xfs_buf_t *, xfs_log_item_t *),
987 	xfs_log_item_t	*lip)
988 {
989 	xfs_log_item_t	*head_lip;
990 
991 	ASSERT(xfs_buf_islocked(bp));
992 
993 	lip->li_cb = cb;
994 	head_lip = bp->b_fspriv;
995 	if (head_lip) {
996 		lip->li_bio_list = head_lip->li_bio_list;
997 		head_lip->li_bio_list = lip;
998 	} else {
999 		bp->b_fspriv = lip;
1000 	}
1001 
1002 	ASSERT(bp->b_iodone == NULL ||
1003 	       bp->b_iodone == xfs_buf_iodone_callbacks);
1004 	bp->b_iodone = xfs_buf_iodone_callbacks;
1005 }
1006 
1007 /*
1008  * We can have many callbacks on a buffer. Running the callbacks individually
1009  * can cause a lot of contention on the AIL lock, so we allow for a single
1010  * callback to be able to scan the remaining lip->li_bio_list for other items
1011  * of the same type and callback to be processed in the first call.
1012  *
1013  * As a result, the loop walking the callback list below will also modify the
1014  * list. it removes the first item from the list and then runs the callback.
1015  * The loop then restarts from the new head of the list. This allows the
1016  * callback to scan and modify the list attached to the buffer and we don't
1017  * have to care about maintaining a next item pointer.
1018  */
1019 STATIC void
1020 xfs_buf_do_callbacks(
1021 	struct xfs_buf		*bp)
1022 {
1023 	struct xfs_log_item	*lip;
1024 
1025 	while ((lip = bp->b_fspriv) != NULL) {
1026 		bp->b_fspriv = lip->li_bio_list;
1027 		ASSERT(lip->li_cb != NULL);
1028 		/*
1029 		 * Clear the next pointer so we don't have any
1030 		 * confusion if the item is added to another buf.
1031 		 * Don't touch the log item after calling its
1032 		 * callback, because it could have freed itself.
1033 		 */
1034 		lip->li_bio_list = NULL;
1035 		lip->li_cb(bp, lip);
1036 	}
1037 }
1038 
1039 /*
1040  * This is the iodone() function for buffers which have had callbacks
1041  * attached to them by xfs_buf_attach_iodone().  It should remove each
1042  * log item from the buffer's list and call the callback of each in turn.
1043  * When done, the buffer's fsprivate field is set to NULL and the buffer
1044  * is unlocked with a call to iodone().
1045  */
1046 void
1047 xfs_buf_iodone_callbacks(
1048 	struct xfs_buf		*bp)
1049 {
1050 	struct xfs_log_item	*lip = bp->b_fspriv;
1051 	struct xfs_mount	*mp = lip->li_mountp;
1052 	static ulong		lasttime;
1053 	static xfs_buftarg_t	*lasttarg;
1054 
1055 	if (likely(!bp->b_error))
1056 		goto do_callbacks;
1057 
1058 	/*
1059 	 * If we've already decided to shutdown the filesystem because of
1060 	 * I/O errors, there's no point in giving this a retry.
1061 	 */
1062 	if (XFS_FORCED_SHUTDOWN(mp)) {
1063 		xfs_buf_stale(bp);
1064 		XFS_BUF_DONE(bp);
1065 		trace_xfs_buf_item_iodone(bp, _RET_IP_);
1066 		goto do_callbacks;
1067 	}
1068 
1069 	if (bp->b_target != lasttarg ||
1070 	    time_after(jiffies, (lasttime + 5*HZ))) {
1071 		lasttime = jiffies;
1072 		xfs_buf_ioerror_alert(bp, __func__);
1073 	}
1074 	lasttarg = bp->b_target;
1075 
1076 	/*
1077 	 * If the write was asynchronous then no one will be looking for the
1078 	 * error.  Clear the error state and write the buffer out again.
1079 	 *
1080 	 * XXX: This helps against transient write errors, but we need to find
1081 	 * a way to shut the filesystem down if the writes keep failing.
1082 	 *
1083 	 * In practice we'll shut the filesystem down soon as non-transient
1084 	 * errors tend to affect the whole device and a failing log write
1085 	 * will make us give up.  But we really ought to do better here.
1086 	 */
1087 	if (XFS_BUF_ISASYNC(bp)) {
1088 		ASSERT(bp->b_iodone != NULL);
1089 
1090 		trace_xfs_buf_item_iodone_async(bp, _RET_IP_);
1091 
1092 		xfs_buf_ioerror(bp, 0); /* errno of 0 unsets the flag */
1093 
1094 		if (!(bp->b_flags & (XBF_STALE|XBF_WRITE_FAIL))) {
1095 			bp->b_flags |= XBF_WRITE | XBF_ASYNC |
1096 				       XBF_DONE | XBF_WRITE_FAIL;
1097 			xfs_buf_submit(bp);
1098 		} else {
1099 			xfs_buf_relse(bp);
1100 		}
1101 
1102 		return;
1103 	}
1104 
1105 	/*
1106 	 * If the write of the buffer was synchronous, we want to make
1107 	 * sure to return the error to the caller of xfs_bwrite().
1108 	 */
1109 	xfs_buf_stale(bp);
1110 	XFS_BUF_DONE(bp);
1111 
1112 	trace_xfs_buf_error_relse(bp, _RET_IP_);
1113 
1114 do_callbacks:
1115 	xfs_buf_do_callbacks(bp);
1116 	bp->b_fspriv = NULL;
1117 	bp->b_iodone = NULL;
1118 	xfs_buf_ioend(bp);
1119 }
1120 
1121 /*
1122  * This is the iodone() function for buffers which have been
1123  * logged.  It is called when they are eventually flushed out.
1124  * It should remove the buf item from the AIL, and free the buf item.
1125  * It is called by xfs_buf_iodone_callbacks() above which will take
1126  * care of cleaning up the buffer itself.
1127  */
1128 void
1129 xfs_buf_iodone(
1130 	struct xfs_buf		*bp,
1131 	struct xfs_log_item	*lip)
1132 {
1133 	struct xfs_ail		*ailp = lip->li_ailp;
1134 
1135 	ASSERT(BUF_ITEM(lip)->bli_buf == bp);
1136 
1137 	xfs_buf_rele(bp);
1138 
1139 	/*
1140 	 * If we are forcibly shutting down, this may well be
1141 	 * off the AIL already. That's because we simulate the
1142 	 * log-committed callbacks to unpin these buffers. Or we may never
1143 	 * have put this item on AIL because of the transaction was
1144 	 * aborted forcibly. xfs_trans_ail_delete() takes care of these.
1145 	 *
1146 	 * Either way, AIL is useless if we're forcing a shutdown.
1147 	 */
1148 	spin_lock(&ailp->xa_lock);
1149 	xfs_trans_ail_delete(ailp, lip, SHUTDOWN_CORRUPT_INCORE);
1150 	xfs_buf_item_free(BUF_ITEM(lip));
1151 }
1152