32 	xfs_mount_t	*mp,		/* file system mount structure */  in xfs_rtget_summary()  argument
40 	return xfs_rtmodify_summary_int(mp, tp, log, bbno, 0, rbpp, rsb, sum);  in xfs_rtget_summary()
49 	xfs_mount_t	*mp,		/* file system mount structure */  in xfs_rtany_summary()  argument
63 	if (mp->m_rsum_cache && low < mp->m_rsum_cache[bbno])  in xfs_rtany_summary()
64 		low = mp->m_rsum_cache[bbno];  in xfs_rtany_summary()
73 		error = xfs_rtget_summary(mp, tp, log, bbno, rbpp, rsb, &sum);  in xfs_rtany_summary()
91 	if (mp->m_rsum_cache && log > mp->m_rsum_cache[bbno])  in xfs_rtany_summary()
92 		mp->m_rsum_cache[bbno] = log;  in xfs_rtany_summary()
144 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_range()  argument
162 	error = xfs_rtfind_back(mp, tp, start, 0, &preblock);  in xfs_rtallocate_range()
169 	error = xfs_rtfind_forw(mp, tp, end, mp->m_sb.sb_rextents - 1,  in xfs_rtallocate_range()
178 	error = xfs_rtmodify_summary(mp, tp,  in xfs_rtallocate_range()
180 		XFS_BITTOBLOCK(mp, preblock), -1, rbpp, rsb);  in xfs_rtallocate_range()
189 		error = xfs_rtmodify_summary(mp, tp,  in xfs_rtallocate_range()
191 			XFS_BITTOBLOCK(mp, preblock), 1, rbpp, rsb);  in xfs_rtallocate_range()
201 		error = xfs_rtmodify_summary(mp, tp,  in xfs_rtallocate_range()
203 			XFS_BITTOBLOCK(mp, end + 1), 1, rbpp, rsb);  in xfs_rtallocate_range()
211 	error = xfs_rtmodify_range(mp, tp, start, len, 0);  in xfs_rtallocate_range()
221 	struct xfs_mount	*mp,  in xfs_rtallocate_clamp_len()  argument
228 	ret = min(mp->m_sb.sb_rextents, startrtx + rtxlen) - startrtx;  in xfs_rtallocate_clamp_len()
240 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_extent_block()  argument
264 	for (i = XFS_BLOCKTOBIT(mp, bbno), besti = -1, bestlen = 0,  in xfs_rtallocate_extent_block()
265 		end = XFS_BLOCKTOBIT(mp, bbno + 1) - 1;  in xfs_rtallocate_extent_block()
269 		maxlen = xfs_rtallocate_clamp_len(mp, i, maxlen, prod);  in xfs_rtallocate_extent_block()
275 		error = xfs_rtcheck_range(mp, tp, i, maxlen, 1, &next, &stat);  in xfs_rtallocate_extent_block()
283 			error = xfs_rtallocate_range(mp, tp, i, maxlen, rbpp,  in xfs_rtallocate_extent_block()
311 			error = xfs_rtfind_forw(mp, tp, next, end, &i);  in xfs_rtallocate_extent_block()
336 		error = xfs_rtallocate_range(mp, tp, besti, bestlen, rbpp, rsb);  in xfs_rtallocate_extent_block()
360 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_extent_exact()  argument
381 	error = xfs_rtcheck_range(mp, tp, bno, maxlen, 1, &next, &isfree);  in xfs_rtallocate_extent_exact()
389 		error = xfs_rtallocate_range(mp, tp, bno, maxlen, rbpp, rsb);  in xfs_rtallocate_extent_exact()
424 	error = xfs_rtallocate_range(mp, tp, bno, maxlen, rbpp, rsb);  in xfs_rtallocate_extent_exact()
440 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_extent_near()  argument
467 	if (bno >= mp->m_sb.sb_rextents)  in xfs_rtallocate_extent_near()
468 		bno = mp->m_sb.sb_rextents - 1;  in xfs_rtallocate_extent_near()
471 	maxlen = xfs_rtallocate_clamp_len(mp, bno, maxlen, prod);  in xfs_rtallocate_extent_near()
480 	error = xfs_rtallocate_extent_exact(mp, tp, bno, minlen, maxlen, len,  in xfs_rtallocate_extent_near()
492 	bbno = XFS_BITTOBLOCK(mp, bno);  in xfs_rtallocate_extent_near()
504 		error = xfs_rtany_summary(mp, tp, log2len, mp->m_rsumlevels - 1,  in xfs_rtallocate_extent_near()
522 				error = xfs_rtallocate_extent_block(mp, tp,  in xfs_rtallocate_extent_near()
551 					error = xfs_rtany_summary(mp, tp,  in xfs_rtallocate_extent_near()
552 						log2len, mp->m_rsumlevels - 1,  in xfs_rtallocate_extent_near()
567 					error = xfs_rtallocate_extent_block(mp,  in xfs_rtallocate_extent_near()
589 				error = xfs_rtallocate_extent_block(mp, tp,  in xfs_rtallocate_extent_near()
614 		else if (i > 0 && (int)bbno + i < mp->m_sb.sb_rbmblocks - 1)  in xfs_rtallocate_extent_near()
620 		else if (i <= 0 && (int)bbno - i < mp->m_sb.sb_rbmblocks - 1)  in xfs_rtallocate_extent_near()
645 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_extent_size()  argument
673 	for (l = xfs_highbit32(maxlen); l < mp->m_rsumlevels; l++) {  in xfs_rtallocate_extent_size()
677 		for (i = 0; i < mp->m_sb.sb_rbmblocks; i++) {  in xfs_rtallocate_extent_size()
681 			error = xfs_rtget_summary(mp, tp, l, i, rbpp, rsb,  in xfs_rtallocate_extent_size()
694 			error = xfs_rtallocate_extent_block(mp, tp, i, maxlen,  in xfs_rtallocate_extent_size()
711 			if (XFS_BITTOBLOCK(mp, n) > i + 1)  in xfs_rtallocate_extent_size()
712 				i = XFS_BITTOBLOCK(mp, n) - 1;  in xfs_rtallocate_extent_size()
736 		for (i = 0; i < mp->m_sb.sb_rbmblocks; i++) {  in xfs_rtallocate_extent_size()
740 			error =	xfs_rtget_summary(mp, tp, l, i, rbpp, rsb,  in xfs_rtallocate_extent_size()
755 			error = xfs_rtallocate_extent_block(mp, tp, i,  in xfs_rtallocate_extent_size()
774 			if (XFS_BITTOBLOCK(mp, n) > i + 1)  in xfs_rtallocate_extent_size()
775 				i = XFS_BITTOBLOCK(mp, n) - 1;  in xfs_rtallocate_extent_size()
790 	struct xfs_mount	*mp,		/* file system mount point */  in xfs_growfs_rt_alloc()  argument
806 	if (ip == mp->m_rsumip)  in xfs_growfs_rt_alloc()
815 		resblks = XFS_GROWFSRT_SPACE_RES(mp, nblocks - oblocks);  in xfs_growfs_rt_alloc()
819 		error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtalloc, resblks,  in xfs_growfs_rt_alloc()
861 			error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtzero,  in xfs_growfs_rt_alloc()
873 			d = XFS_FSB_TO_DADDR(mp, fsbno);  in xfs_growfs_rt_alloc()
874 			error = xfs_trans_get_buf(tp, mp->m_ddev_targp, d,  in xfs_growfs_rt_alloc()
875 					mp->m_bsize, 0, &bp);  in xfs_growfs_rt_alloc()
881 			memset(bp->b_addr, 0, mp->m_sb.sb_blocksize);  in xfs_growfs_rt_alloc()
882 			xfs_trans_log_buf(tp, bp, 0, mp->m_sb.sb_blocksize - 1);  in xfs_growfs_rt_alloc()
905 	xfs_mount_t	*mp,		/* file system mount structure */  in xfs_alloc_rsum_cache()  argument
913 	mp->m_rsum_cache = kvzalloc(rbmblocks, GFP_KERNEL);  in xfs_alloc_rsum_cache()
914 	if (!mp->m_rsum_cache)  in xfs_alloc_rsum_cache()
915 		xfs_warn(mp, "could not allocate realtime summary cache");  in xfs_alloc_rsum_cache()
926 	struct xfs_mount	*mp)  in xfs_growfs_rt_fixup_extsize()  argument
928 	struct xfs_inode	*ip = mp->m_rootip;  in xfs_growfs_rt_fixup_extsize()
937 	error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_ichange, 0, 0, false,  in xfs_growfs_rt_fixup_extsize()
960 	xfs_mount_t	*mp,		/* mount point for filesystem */  in xfs_growfs_rt()  argument
980 	xfs_agblock_t	old_rextsize = mp->m_sb.sb_rextsize;  in xfs_growfs_rt()
982 	sbp = &mp->m_sb;  in xfs_growfs_rt()
988 	if (!XFS_IS_REALTIME_MOUNT(mp))  in xfs_growfs_rt()
991 	if (!mutex_trylock(&mp->m_growlock))  in xfs_growfs_rt()
998 	if (!mp->m_rbmip || !mp->m_rsumip)  in xfs_growfs_rt()
1010 	if (XFS_FSB_TO_B(mp, in->extsize) > XFS_MAX_RTEXTSIZE ||  in xfs_growfs_rt()
1011 	    XFS_FSB_TO_B(mp, in->extsize) < XFS_MIN_RTEXTSIZE)  in xfs_growfs_rt()
1016 	if (xfs_has_rmapbt(mp) || xfs_has_reflink(mp) || xfs_has_quota(mp))  in xfs_growfs_rt()
1026 	error = xfs_buf_read_uncached(mp->m_rtdev_targp,  in xfs_growfs_rt()
1027 				XFS_FSB_TO_BB(mp, nrblocks - 1),  in xfs_growfs_rt()
1028 				XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);  in xfs_growfs_rt()
1046 	nrsumblocks = XFS_B_TO_FSB(mp, nrsumsize);  in xfs_growfs_rt()
1047 	nrsumsize = XFS_FSB_TO_B(mp, nrsumblocks);  in xfs_growfs_rt()
1053 	if (nrsumblocks > (mp->m_sb.sb_logblocks >> 1)) {  in xfs_growfs_rt()
1062 	rbmblocks = XFS_B_TO_FSB(mp, mp->m_rbmip->i_disk_size);  in xfs_growfs_rt()
1063 	rsumblocks = XFS_B_TO_FSB(mp, mp->m_rsumip->i_disk_size);  in xfs_growfs_rt()
1067 	error = xfs_growfs_rt_alloc(mp, rbmblocks, nrbmblocks, mp->m_rbmip);  in xfs_growfs_rt()
1070 	error = xfs_growfs_rt_alloc(mp, rsumblocks, nrsumblocks, mp->m_rsumip);  in xfs_growfs_rt()
1074 	rsum_cache = mp->m_rsum_cache;  in xfs_growfs_rt()
1076 		xfs_alloc_rsum_cache(mp, nrbmblocks);  in xfs_growfs_rt()
1089 		     ((sbp->sb_rextents & ((1 << mp->m_blkbit_log) - 1)) != 0);  in xfs_growfs_rt()
1095 		*nmp = *mp;  in xfs_growfs_rt()
1113 		nrsumblocks = XFS_B_TO_FSB(mp, nrsumsize);  in xfs_growfs_rt()
1114 		nmp->m_rsumsize = nrsumsize = XFS_FSB_TO_B(mp, nrsumblocks);  in xfs_growfs_rt()
1121 		error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtfree, 0, 0, 0,  in xfs_growfs_rt()
1128 		xfs_ilock(mp->m_rbmip, XFS_ILOCK_EXCL | XFS_ILOCK_RTBITMAP);  in xfs_growfs_rt()
1129 		xfs_trans_ijoin(tp, mp->m_rbmip, XFS_ILOCK_EXCL);  in xfs_growfs_rt()
1135 		mp->m_rbmip->i_disk_size =  in xfs_growfs_rt()
1137 		i_size_write(VFS_I(mp->m_rbmip), mp->m_rbmip->i_disk_size);  in xfs_growfs_rt()
1138 		xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);  in xfs_growfs_rt()
1142 		xfs_ilock(mp->m_rsumip, XFS_ILOCK_EXCL | XFS_ILOCK_RTSUM);  in xfs_growfs_rt()
1143 		xfs_trans_ijoin(tp, mp->m_rsumip, XFS_ILOCK_EXCL);  in xfs_growfs_rt()
1149 		mp->m_rsumip->i_disk_size = nmp->m_rsumsize;  in xfs_growfs_rt()
1150 		i_size_write(VFS_I(mp->m_rsumip), mp->m_rsumip->i_disk_size);  in xfs_growfs_rt()
1151 		xfs_trans_log_inode(tp, mp->m_rsumip, XFS_ILOG_CORE);  in xfs_growfs_rt()
1157 		    mp->m_rsumlevels != nmp->m_rsumlevels) {  in xfs_growfs_rt()
1158 			error = xfs_rtcopy_summary(mp, nmp, tp);  in xfs_growfs_rt()
1199 		mp->m_rsumlevels = nrsumlevels;  in xfs_growfs_rt()
1200 		mp->m_rsumsize = nrsumsize;  in xfs_growfs_rt()
1202 		xfs_trans_resv_calc(mp, &mp->m_resv);  in xfs_growfs_rt()
1209 		mp->m_features |= XFS_FEAT_REALTIME;  in xfs_growfs_rt()
1215 		error = xfs_growfs_rt_fixup_extsize(mp);  in xfs_growfs_rt()
1221 	error = xfs_update_secondary_sbs(mp);  in xfs_growfs_rt()
1234 	if (rsum_cache != mp->m_rsum_cache) {  in xfs_growfs_rt()
1236 			kmem_free(mp->m_rsum_cache);  in xfs_growfs_rt()
1237 			mp->m_rsum_cache = rsum_cache;  in xfs_growfs_rt()
1244 	mutex_unlock(&mp->m_growlock);  in xfs_growfs_rt()
1264 	xfs_mount_t	*mp = tp->t_mountp;  in xfs_rtallocate_extent()  local
1270 	ASSERT(xfs_isilocked(mp->m_rbmip, XFS_ILOCK_EXCL));  in xfs_rtallocate_extent()
1292 		error = xfs_rtallocate_extent_size(mp, tp, minlen, maxlen, len,  in xfs_rtallocate_extent()
1295 		error = xfs_rtallocate_extent_near(mp, tp, bno, minlen, maxlen,  in xfs_rtallocate_extent()
1327 	struct xfs_mount	*mp)	/* file system mount structure */  in xfs_rtmount_init()  argument
1334 	sbp = &mp->m_sb;  in xfs_rtmount_init()
1337 	if (mp->m_rtdev_targp == NULL) {  in xfs_rtmount_init()
1338 		xfs_warn(mp,  in xfs_rtmount_init()
1342 	mp->m_rsumlevels = sbp->sb_rextslog + 1;  in xfs_rtmount_init()
1343 	mp->m_rsumsize =  in xfs_rtmount_init()
1344 		(uint)sizeof(xfs_suminfo_t) * mp->m_rsumlevels *  in xfs_rtmount_init()
1346 	mp->m_rsumsize = roundup(mp->m_rsumsize, sbp->sb_blocksize);  in xfs_rtmount_init()
1347 	mp->m_rbmip = mp->m_rsumip = NULL;  in xfs_rtmount_init()
1351 	d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);  in xfs_rtmount_init()
1352 	if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_rblocks) {  in xfs_rtmount_init()
1353 		xfs_warn(mp, "realtime mount -- %llu != %llu",  in xfs_rtmount_init()
1354 			(unsigned long long) XFS_BB_TO_FSB(mp, d),  in xfs_rtmount_init()
1355 			(unsigned long long) mp->m_sb.sb_rblocks);  in xfs_rtmount_init()
1358 	error = xfs_buf_read_uncached(mp->m_rtdev_targp,  in xfs_rtmount_init()
1359 					d - XFS_FSB_TO_BB(mp, 1),  in xfs_rtmount_init()
1360 					XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);  in xfs_rtmount_init()
1362 		xfs_warn(mp, "realtime device size check failed");  in xfs_rtmount_init()
1371 	struct xfs_mount		*mp,  in xfs_rtalloc_count_frextent()  argument
1388 	struct xfs_mount	*mp)  in xfs_rtalloc_reinit_frextents()  argument
1393 	xfs_ilock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP);  in xfs_rtalloc_reinit_frextents()
1394 	error = xfs_rtalloc_query_all(mp, NULL, xfs_rtalloc_count_frextent,  in xfs_rtalloc_reinit_frextents()
1396 	xfs_iunlock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP);  in xfs_rtalloc_reinit_frextents()
1400 	spin_lock(&mp->m_sb_lock);  in xfs_rtalloc_reinit_frextents()
1401 	mp->m_sb.sb_frextents = val;  in xfs_rtalloc_reinit_frextents()
1402 	spin_unlock(&mp->m_sb_lock);  in xfs_rtalloc_reinit_frextents()
1403 	percpu_counter_set(&mp->m_frextents, mp->m_sb.sb_frextents);  in xfs_rtalloc_reinit_frextents()
1448 	xfs_mount_t	*mp)		/* file system mount structure */  in xfs_rtmount_inodes()  argument
1453 	sbp = &mp->m_sb;  in xfs_rtmount_inodes()
1454 	error = xfs_iget(mp, NULL, sbp->sb_rbmino, 0, 0, &mp->m_rbmip);  in xfs_rtmount_inodes()
1457 	ASSERT(mp->m_rbmip != NULL);  in xfs_rtmount_inodes()
1459 	error = xfs_rtmount_iread_extents(mp->m_rbmip, XFS_ILOCK_RTBITMAP);  in xfs_rtmount_inodes()
1463 	error = xfs_iget(mp, NULL, sbp->sb_rsumino, 0, 0, &mp->m_rsumip);  in xfs_rtmount_inodes()
1466 	ASSERT(mp->m_rsumip != NULL);  in xfs_rtmount_inodes()
1468 	error = xfs_rtmount_iread_extents(mp->m_rsumip, XFS_ILOCK_RTSUM);  in xfs_rtmount_inodes()
1472 	xfs_alloc_rsum_cache(mp, sbp->sb_rbmblocks);  in xfs_rtmount_inodes()
1476 	xfs_irele(mp->m_rsumip);  in xfs_rtmount_inodes()
1478 	xfs_irele(mp->m_rbmip);  in xfs_rtmount_inodes()
1484 	struct xfs_mount	*mp)  in xfs_rtunmount_inodes()  argument
1486 	kmem_free(mp->m_rsum_cache);  in xfs_rtunmount_inodes()
1487 	if (mp->m_rbmip)  in xfs_rtunmount_inodes()
1488 		xfs_irele(mp->m_rbmip);  in xfs_rtunmount_inodes()
1489 	if (mp->m_rsumip)  in xfs_rtunmount_inodes()
1490 		xfs_irele(mp->m_rsumip);  in xfs_rtunmount_inodes()
1502 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtpick_extent()  argument
1513 	ASSERT(xfs_isilocked(mp->m_rbmip, XFS_ILOCK_EXCL));  in xfs_rtpick_extent()
1515 	seqp = (uint64_t *)&VFS_I(mp->m_rbmip)->i_atime;  in xfs_rtpick_extent()
1516 	if (!(mp->m_rbmip->i_diflags & XFS_DIFLAG_NEWRTBM)) {  in xfs_rtpick_extent()
1517 		mp->m_rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM;  in xfs_rtpick_extent()
1525 		b = (mp->m_sb.sb_rextents * ((resid << 1) + 1ULL)) >>  in xfs_rtpick_extent()
1527 		if (b >= mp->m_sb.sb_rextents)  in xfs_rtpick_extent()
1528 			div64_u64_rem(b, mp->m_sb.sb_rextents, &b);  in xfs_rtpick_extent()
1529 		if (b + len > mp->m_sb.sb_rextents)  in xfs_rtpick_extent()
1530 			b = mp->m_sb.sb_rextents - len;  in xfs_rtpick_extent()
1533 	xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);  in xfs_rtpick_extent()