32 	xfs_mount_t	*mp,		/* file system mount structure */  in xfs_rtget_summary()  argument
40 	return xfs_rtmodify_summary_int(mp, tp, log, bbno, 0, rbpp, rsb, sum);  in xfs_rtget_summary()
49 	xfs_mount_t	*mp,		/* file system mount structure */  in xfs_rtany_summary()  argument
63 	if (mp->m_rsum_cache && low < mp->m_rsum_cache[bbno])  in xfs_rtany_summary()
64 		low = mp->m_rsum_cache[bbno];  in xfs_rtany_summary()
73 		error = xfs_rtget_summary(mp, tp, log, bbno, rbpp, rsb, &sum);  in xfs_rtany_summary()
91 	if (mp->m_rsum_cache && log > mp->m_rsum_cache[bbno])  in xfs_rtany_summary()
92 		mp->m_rsum_cache[bbno] = log;  in xfs_rtany_summary()
144 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_range()  argument
162 	error = xfs_rtfind_back(mp, tp, start, 0, &preblock);  in xfs_rtallocate_range()
169 	error = xfs_rtfind_forw(mp, tp, end, mp->m_sb.sb_rextents - 1,  in xfs_rtallocate_range()
178 	error = xfs_rtmodify_summary(mp, tp,  in xfs_rtallocate_range()
180 		XFS_BITTOBLOCK(mp, preblock), -1, rbpp, rsb);  in xfs_rtallocate_range()
189 		error = xfs_rtmodify_summary(mp, tp,  in xfs_rtallocate_range()
191 			XFS_BITTOBLOCK(mp, preblock), 1, rbpp, rsb);  in xfs_rtallocate_range()
201 		error = xfs_rtmodify_summary(mp, tp,  in xfs_rtallocate_range()
203 			XFS_BITTOBLOCK(mp, end + 1), 1, rbpp, rsb);  in xfs_rtallocate_range()
211 	error = xfs_rtmodify_range(mp, tp, start, len, 0);  in xfs_rtallocate_range()
221 	struct xfs_mount	*mp,  in xfs_rtallocate_clamp_len()  argument
228 	ret = min(mp->m_sb.sb_rextents, startrtx + rtxlen) - startrtx;  in xfs_rtallocate_clamp_len()
240 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_extent_block()  argument
264 	for (i = XFS_BLOCKTOBIT(mp, bbno), besti = -1, bestlen = 0,  in xfs_rtallocate_extent_block()
265 		end = XFS_BLOCKTOBIT(mp, bbno + 1) - 1;  in xfs_rtallocate_extent_block()
269 		maxlen = xfs_rtallocate_clamp_len(mp, i, maxlen, prod);  in xfs_rtallocate_extent_block()
275 		error = xfs_rtcheck_range(mp, tp, i, maxlen, 1, &next, &stat);  in xfs_rtallocate_extent_block()
283 			error = xfs_rtallocate_range(mp, tp, i, maxlen, rbpp,  in xfs_rtallocate_extent_block()
311 			error = xfs_rtfind_forw(mp, tp, next, end, &i);  in xfs_rtallocate_extent_block()
336 		error = xfs_rtallocate_range(mp, tp, besti, bestlen, rbpp, rsb);  in xfs_rtallocate_extent_block()
360 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_extent_exact()  argument
381 	error = xfs_rtcheck_range(mp, tp, bno, maxlen, 1, &next, &isfree);  in xfs_rtallocate_extent_exact()
389 		error = xfs_rtallocate_range(mp, tp, bno, maxlen, rbpp, rsb);  in xfs_rtallocate_extent_exact()
424 	error = xfs_rtallocate_range(mp, tp, bno, maxlen, rbpp, rsb);  in xfs_rtallocate_extent_exact()
440 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_extent_near()  argument
467 	if (bno >= mp->m_sb.sb_rextents)  in xfs_rtallocate_extent_near()
468 		bno = mp->m_sb.sb_rextents - 1;  in xfs_rtallocate_extent_near()
471 	maxlen = xfs_rtallocate_clamp_len(mp, bno, maxlen, prod);  in xfs_rtallocate_extent_near()
480 	error = xfs_rtallocate_extent_exact(mp, tp, bno, minlen, maxlen, len,  in xfs_rtallocate_extent_near()
492 	bbno = XFS_BITTOBLOCK(mp, bno);  in xfs_rtallocate_extent_near()
504 		error = xfs_rtany_summary(mp, tp, log2len, mp->m_rsumlevels - 1,  in xfs_rtallocate_extent_near()
522 				error = xfs_rtallocate_extent_block(mp, tp,  in xfs_rtallocate_extent_near()
551 					error = xfs_rtany_summary(mp, tp,  in xfs_rtallocate_extent_near()
552 						log2len, mp->m_rsumlevels - 1,  in xfs_rtallocate_extent_near()
567 					error = xfs_rtallocate_extent_block(mp,  in xfs_rtallocate_extent_near()
589 				error = xfs_rtallocate_extent_block(mp, tp,  in xfs_rtallocate_extent_near()
614 		else if (i > 0 && (int)bbno + i < mp->m_sb.sb_rbmblocks - 1)  in xfs_rtallocate_extent_near()
620 		else if (i <= 0 && (int)bbno - i < mp->m_sb.sb_rbmblocks - 1)  in xfs_rtallocate_extent_near()
645 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtallocate_extent_size()  argument
673 	for (l = xfs_highbit32(maxlen); l < mp->m_rsumlevels; l++) {  in xfs_rtallocate_extent_size()
677 		for (i = 0; i < mp->m_sb.sb_rbmblocks; i++) {  in xfs_rtallocate_extent_size()
681 			error = xfs_rtget_summary(mp, tp, l, i, rbpp, rsb,  in xfs_rtallocate_extent_size()
694 			error = xfs_rtallocate_extent_block(mp, tp, i, maxlen,  in xfs_rtallocate_extent_size()
711 			if (XFS_BITTOBLOCK(mp, n) > i + 1)  in xfs_rtallocate_extent_size()
712 				i = XFS_BITTOBLOCK(mp, n) - 1;  in xfs_rtallocate_extent_size()
736 		for (i = 0; i < mp->m_sb.sb_rbmblocks; i++) {  in xfs_rtallocate_extent_size()
740 			error =	xfs_rtget_summary(mp, tp, l, i, rbpp, rsb,  in xfs_rtallocate_extent_size()
755 			error = xfs_rtallocate_extent_block(mp, tp, i,  in xfs_rtallocate_extent_size()
774 			if (XFS_BITTOBLOCK(mp, n) > i + 1)  in xfs_rtallocate_extent_size()
775 				i = XFS_BITTOBLOCK(mp, n) - 1;  in xfs_rtallocate_extent_size()
790 	struct xfs_mount	*mp,		/* file system mount point */  in xfs_growfs_rt_alloc()  argument
806 	if (ip == mp->m_rsumip)  in xfs_growfs_rt_alloc()
815 		resblks = XFS_GROWFSRT_SPACE_RES(mp, nblocks - oblocks);  in xfs_growfs_rt_alloc()
819 		error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtalloc, resblks,  in xfs_growfs_rt_alloc()
861 			error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtzero,  in xfs_growfs_rt_alloc()
873 			d = XFS_FSB_TO_DADDR(mp, fsbno);  in xfs_growfs_rt_alloc()
874 			error = xfs_trans_get_buf(tp, mp->m_ddev_targp, d,  in xfs_growfs_rt_alloc()
875 					mp->m_bsize, 0, &bp);  in xfs_growfs_rt_alloc()
881 			memset(bp->b_addr, 0, mp->m_sb.sb_blocksize);  in xfs_growfs_rt_alloc()
882 			xfs_trans_log_buf(tp, bp, 0, mp->m_sb.sb_blocksize - 1);  in xfs_growfs_rt_alloc()
905 	xfs_mount_t	*mp,		/* file system mount structure */  in xfs_alloc_rsum_cache()  argument
913 	mp->m_rsum_cache = kvzalloc(rbmblocks, GFP_KERNEL);  in xfs_alloc_rsum_cache()
914 	if (!mp->m_rsum_cache)  in xfs_alloc_rsum_cache()
915 		xfs_warn(mp, "could not allocate realtime summary cache");  in xfs_alloc_rsum_cache()
927 	xfs_mount_t	*mp,		/* mount point for filesystem */  in xfs_growfs_rt()  argument
948 	sbp = &mp->m_sb;  in xfs_growfs_rt()
954 	if (!XFS_IS_REALTIME_MOUNT(mp))  in xfs_growfs_rt()
960 	if (!mp->m_rbmip || !mp->m_rsumip)  in xfs_growfs_rt()
972 	if (XFS_FSB_TO_B(mp, in->extsize) > XFS_MAX_RTEXTSIZE ||  in xfs_growfs_rt()
973 	    XFS_FSB_TO_B(mp, in->extsize) < XFS_MIN_RTEXTSIZE)  in xfs_growfs_rt()
977 	if (xfs_has_rmapbt(mp) || xfs_has_reflink(mp) || xfs_has_quota(mp))  in xfs_growfs_rt()
987 	error = xfs_buf_read_uncached(mp->m_rtdev_targp,  in xfs_growfs_rt()
988 				XFS_FSB_TO_BB(mp, nrblocks - 1),  in xfs_growfs_rt()
989 				XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);  in xfs_growfs_rt()
1005 	nrsumblocks = XFS_B_TO_FSB(mp, nrsumsize);  in xfs_growfs_rt()
1006 	nrsumsize = XFS_FSB_TO_B(mp, nrsumblocks);  in xfs_growfs_rt()
1012 	if (nrsumblocks > (mp->m_sb.sb_logblocks >> 1))  in xfs_growfs_rt()
1018 	rbmblocks = XFS_B_TO_FSB(mp, mp->m_rbmip->i_disk_size);  in xfs_growfs_rt()
1019 	rsumblocks = XFS_B_TO_FSB(mp, mp->m_rsumip->i_disk_size);  in xfs_growfs_rt()
1023 	error = xfs_growfs_rt_alloc(mp, rbmblocks, nrbmblocks, mp->m_rbmip);  in xfs_growfs_rt()
1026 	error = xfs_growfs_rt_alloc(mp, rsumblocks, nrsumblocks, mp->m_rsumip);  in xfs_growfs_rt()
1030 	rsum_cache = mp->m_rsum_cache;  in xfs_growfs_rt()
1032 		xfs_alloc_rsum_cache(mp, nrbmblocks);  in xfs_growfs_rt()
1045 		     ((sbp->sb_rextents & ((1 << mp->m_blkbit_log) - 1)) != 0);  in xfs_growfs_rt()
1051 		*nmp = *mp;  in xfs_growfs_rt()
1069 		nrsumblocks = XFS_B_TO_FSB(mp, nrsumsize);  in xfs_growfs_rt()
1070 		nmp->m_rsumsize = nrsumsize = XFS_FSB_TO_B(mp, nrsumblocks);  in xfs_growfs_rt()
1077 		error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtfree, 0, 0, 0,  in xfs_growfs_rt()
1084 		xfs_ilock(mp->m_rbmip, XFS_ILOCK_EXCL | XFS_ILOCK_RTBITMAP);  in xfs_growfs_rt()
1085 		xfs_trans_ijoin(tp, mp->m_rbmip, XFS_ILOCK_EXCL);  in xfs_growfs_rt()
1091 		mp->m_rbmip->i_disk_size =  in xfs_growfs_rt()
1093 		i_size_write(VFS_I(mp->m_rbmip), mp->m_rbmip->i_disk_size);  in xfs_growfs_rt()
1094 		xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);  in xfs_growfs_rt()
1098 		xfs_ilock(mp->m_rsumip, XFS_ILOCK_EXCL | XFS_ILOCK_RTSUM);  in xfs_growfs_rt()
1099 		xfs_trans_ijoin(tp, mp->m_rsumip, XFS_ILOCK_EXCL);  in xfs_growfs_rt()
1105 		mp->m_rsumip->i_disk_size = nmp->m_rsumsize;  in xfs_growfs_rt()
1106 		i_size_write(VFS_I(mp->m_rsumip), mp->m_rsumip->i_disk_size);  in xfs_growfs_rt()
1107 		xfs_trans_log_inode(tp, mp->m_rsumip, XFS_ILOG_CORE);  in xfs_growfs_rt()
1113 		    mp->m_rsumlevels != nmp->m_rsumlevels) {  in xfs_growfs_rt()
1114 			error = xfs_rtcopy_summary(mp, nmp, tp);  in xfs_growfs_rt()
1155 		mp->m_rsumlevels = nrsumlevels;  in xfs_growfs_rt()
1156 		mp->m_rsumsize = nrsumsize;  in xfs_growfs_rt()
1158 		xfs_trans_resv_calc(mp, &mp->m_resv);  in xfs_growfs_rt()
1165 		mp->m_features |= XFS_FEAT_REALTIME;  in xfs_growfs_rt()
1171 	error = xfs_update_secondary_sbs(mp);  in xfs_growfs_rt()
1184 	if (rsum_cache != mp->m_rsum_cache) {  in xfs_growfs_rt()
1186 			kmem_free(mp->m_rsum_cache);  in xfs_growfs_rt()
1187 			mp->m_rsum_cache = rsum_cache;  in xfs_growfs_rt()
1212 	xfs_mount_t	*mp = tp->t_mountp;  in xfs_rtallocate_extent()  local
1218 	ASSERT(xfs_isilocked(mp->m_rbmip, XFS_ILOCK_EXCL));  in xfs_rtallocate_extent()
1240 		error = xfs_rtallocate_extent_size(mp, tp, minlen, maxlen, len,  in xfs_rtallocate_extent()
1243 		error = xfs_rtallocate_extent_near(mp, tp, bno, minlen, maxlen,  in xfs_rtallocate_extent()
1275 	struct xfs_mount	*mp)	/* file system mount structure */  in xfs_rtmount_init()  argument
1282 	sbp = &mp->m_sb;  in xfs_rtmount_init()
1285 	if (mp->m_rtdev_targp == NULL) {  in xfs_rtmount_init()
1286 		xfs_warn(mp,  in xfs_rtmount_init()
1290 	mp->m_rsumlevels = sbp->sb_rextslog + 1;  in xfs_rtmount_init()
1291 	mp->m_rsumsize =  in xfs_rtmount_init()
1292 		(uint)sizeof(xfs_suminfo_t) * mp->m_rsumlevels *  in xfs_rtmount_init()
1294 	mp->m_rsumsize = roundup(mp->m_rsumsize, sbp->sb_blocksize);  in xfs_rtmount_init()
1295 	mp->m_rbmip = mp->m_rsumip = NULL;  in xfs_rtmount_init()
1299 	d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);  in xfs_rtmount_init()
1300 	if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_rblocks) {  in xfs_rtmount_init()
1301 		xfs_warn(mp, "realtime mount -- %llu != %llu",  in xfs_rtmount_init()
1302 			(unsigned long long) XFS_BB_TO_FSB(mp, d),  in xfs_rtmount_init()
1303 			(unsigned long long) mp->m_sb.sb_rblocks);  in xfs_rtmount_init()
1306 	error = xfs_buf_read_uncached(mp->m_rtdev_targp,  in xfs_rtmount_init()
1307 					d - XFS_FSB_TO_BB(mp, 1),  in xfs_rtmount_init()
1308 					XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);  in xfs_rtmount_init()
1310 		xfs_warn(mp, "realtime device size check failed");  in xfs_rtmount_init()
1319 	struct xfs_mount		*mp,  in xfs_rtalloc_count_frextent()  argument
1336 	struct xfs_mount	*mp)  in xfs_rtalloc_reinit_frextents()  argument
1341 	xfs_ilock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP);  in xfs_rtalloc_reinit_frextents()
1342 	error = xfs_rtalloc_query_all(mp, NULL, xfs_rtalloc_count_frextent,  in xfs_rtalloc_reinit_frextents()
1344 	xfs_iunlock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP);  in xfs_rtalloc_reinit_frextents()
1348 	spin_lock(&mp->m_sb_lock);  in xfs_rtalloc_reinit_frextents()
1349 	mp->m_sb.sb_frextents = val;  in xfs_rtalloc_reinit_frextents()
1350 	spin_unlock(&mp->m_sb_lock);  in xfs_rtalloc_reinit_frextents()
1351 	percpu_counter_set(&mp->m_frextents, mp->m_sb.sb_frextents);  in xfs_rtalloc_reinit_frextents()
1396 	xfs_mount_t	*mp)		/* file system mount structure */  in xfs_rtmount_inodes()  argument
1401 	sbp = &mp->m_sb;  in xfs_rtmount_inodes()
1402 	error = xfs_iget(mp, NULL, sbp->sb_rbmino, 0, 0, &mp->m_rbmip);  in xfs_rtmount_inodes()
1405 	ASSERT(mp->m_rbmip != NULL);  in xfs_rtmount_inodes()
1407 	error = xfs_rtmount_iread_extents(mp->m_rbmip, XFS_ILOCK_RTBITMAP);  in xfs_rtmount_inodes()
1411 	error = xfs_iget(mp, NULL, sbp->sb_rsumino, 0, 0, &mp->m_rsumip);  in xfs_rtmount_inodes()
1414 	ASSERT(mp->m_rsumip != NULL);  in xfs_rtmount_inodes()
1416 	error = xfs_rtmount_iread_extents(mp->m_rsumip, XFS_ILOCK_RTSUM);  in xfs_rtmount_inodes()
1420 	xfs_alloc_rsum_cache(mp, sbp->sb_rbmblocks);  in xfs_rtmount_inodes()
1424 	xfs_irele(mp->m_rsumip);  in xfs_rtmount_inodes()
1426 	xfs_irele(mp->m_rbmip);  in xfs_rtmount_inodes()
1432 	struct xfs_mount	*mp)  in xfs_rtunmount_inodes()  argument
1434 	kmem_free(mp->m_rsum_cache);  in xfs_rtunmount_inodes()
1435 	if (mp->m_rbmip)  in xfs_rtunmount_inodes()
1436 		xfs_irele(mp->m_rbmip);  in xfs_rtunmount_inodes()
1437 	if (mp->m_rsumip)  in xfs_rtunmount_inodes()
1438 		xfs_irele(mp->m_rsumip);  in xfs_rtunmount_inodes()
1450 	xfs_mount_t	*mp,		/* file system mount point */  in xfs_rtpick_extent()  argument
1461 	ASSERT(xfs_isilocked(mp->m_rbmip, XFS_ILOCK_EXCL));  in xfs_rtpick_extent()
1463 	seqp = (uint64_t *)&VFS_I(mp->m_rbmip)->i_atime;  in xfs_rtpick_extent()
1464 	if (!(mp->m_rbmip->i_diflags & XFS_DIFLAG_NEWRTBM)) {  in xfs_rtpick_extent()
1465 		mp->m_rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM;  in xfs_rtpick_extent()
1473 		b = (mp->m_sb.sb_rextents * ((resid << 1) + 1ULL)) >>  in xfs_rtpick_extent()
1475 		if (b >= mp->m_sb.sb_rextents)  in xfs_rtpick_extent()
1476 			div64_u64_rem(b, mp->m_sb.sb_rextents, &b);  in xfs_rtpick_extent()
1477 		if (b + len > mp->m_sb.sb_rextents)  in xfs_rtpick_extent()
1478 			b = mp->m_sb.sb_rextents - len;  in xfs_rtpick_extent()
1481 	xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);  in xfs_rtpick_extent()