157 	ret = tg->bps[rw][td->limit_index];  in tg_bps_limit()
167 	if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_LOW] &&  in tg_bps_limit()
168 	    tg->bps[rw][LIMIT_LOW] != tg->bps[rw][LIMIT_MAX]) {  in tg_bps_limit()
171 		adjusted = throtl_adjusted_limit(tg->bps[rw][LIMIT_LOW], td);  in tg_bps_limit()
172 		ret = min(tg->bps[rw][LIMIT_MAX], adjusted);  in tg_bps_limit()
189 		/* intermediate node or bps isn't 0 */  in tg_iops_limit()
191 		    tg->bps[rw][td->limit_index])  in tg_iops_limit()
362 	tg->bps[READ][LIMIT_MAX] = U64_MAX;  in throtl_pd_alloc()
363 	tg->bps[WRITE][LIMIT_MAX] = U64_MAX;  in throtl_pd_alloc()
397 	 * read_bps limit is set on a parent group, summary bps of  in throtl_pd_init()
457 		if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] ||  in blk_throtl_update_limit_valid()
478 	tg->bps[READ][LIMIT_LOW] = 0;  in throtl_pd_offline()
479 	tg->bps[WRITE][LIMIT_LOW] = 0;  in throtl_pd_offline()
745 	 * If bps are unlimited (-1), then time slice don't get  in throtl_trim_slice()
918 	/* If tg->bps = -1, then BW is unlimited */  in tg_may_dispatch()
1447 		.private = offsetof(struct throtl_grp, bps[READ][LIMIT_MAX]),
1453 		.private = offsetof(struct throtl_grp, bps[WRITE][LIMIT_MAX]),
1635 		tg->bps[READ][index] = v[0];  in tg_set_limit()
1636 		tg->bps[WRITE][index] = v[1];  in tg_set_limit()
1640 	tg->bps[READ][LIMIT_LOW] = min(tg->bps_conf[READ][LIMIT_LOW],  in tg_set_limit()
1642 	tg->bps[WRITE][LIMIT_LOW] = min(tg->bps_conf[WRITE][LIMIT_LOW],  in tg_set_limit()
1652 	if (!(tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW] ||  in tg_set_limit()
1653 	      tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) ||  in tg_set_limit()
1656 		tg->bps[READ][LIMIT_LOW] = 0;  in tg_set_limit()
1657 		tg->bps[WRITE][LIMIT_LOW] = 0;  in tg_set_limit()
1769 	if (tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW])  in __tg_last_low_overflow_time()
1771 	if (tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW])  in __tg_last_low_overflow_time()
1792 		if (!parent->bps[READ][LIMIT_LOW] &&  in tg_last_low_overflow_time()
1794 		    !parent->bps[WRITE][LIMIT_LOW] &&  in tg_last_low_overflow_time()
1832 	bool limit = tg->bps[rw][LIMIT_LOW] || tg->iops[rw][LIMIT_LOW];  in throtl_low_limit_reached()
1996 	uint64_t bps;  in throtl_downgrade_check()  local
2016 	if (tg->bps[READ][LIMIT_LOW]) {  in throtl_downgrade_check()
2017 		bps = tg->last_bytes_disp[READ] * HZ;  in throtl_downgrade_check()
2018 		do_div(bps, elapsed_time);  in throtl_downgrade_check()
2019 		if (bps >= tg->bps[READ][LIMIT_LOW])  in throtl_downgrade_check()
2023 	if (tg->bps[WRITE][LIMIT_LOW]) {  in throtl_downgrade_check()
2024 		bps = tg->last_bytes_disp[WRITE] * HZ;  in throtl_downgrade_check()
2025 		do_div(bps, elapsed_time);  in throtl_downgrade_check()
2026 		if (bps >= tg->bps[WRITE][LIMIT_LOW])  in throtl_downgrade_check()
2248 	throtl_log(sq, "[%c] bio. bdisp=%llu sz=%u bps=%llu iodisp=%u iops=%u queued=%d/%d",  in __blk_throtl_bio()