561 	u64				now;  member
963 static void ioc_refresh_vrate(struct ioc *ioc, struct ioc_now *now)  in ioc_refresh_vrate()  argument
965 	s64 pleft = ioc->period_at + ioc->period_us - now->now;  in ioc_refresh_vrate()
1042 static void ioc_now(struct ioc *ioc, struct ioc_now *now)  in ioc_now()  argument
1047 	now->now_ns = ktime_get();  in ioc_now()
1048 	now->now = ktime_to_us(now->now_ns);  in ioc_now()
1061 		now->vnow = ioc->period_at_vtime +  in ioc_now()
1062 			(now->now - ioc->period_at) * vrate;  in ioc_now()
1066 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now)  in ioc_start_period()  argument
1071 	ioc->period_at = now->now;  in ioc_start_period()
1072 	ioc->period_at_vtime = now->vnow;  in ioc_start_period()
1085 				bool save, struct ioc_now *now)  in __propagate_weights()  argument
1113 		iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);  in __propagate_weights()
1167 			      bool save, struct ioc_now *now)  in propagate_weights()  argument
1169 	__propagate_weights(iocg, active, inuse, save, now);  in propagate_weights()
1253 static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now)  in weight_updated()  argument
1264 		propagate_weights(iocg, weight, iocg->inuse, true, now);  in weight_updated()
1268 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)  in iocg_activate()  argument
1280 		ioc_now(ioc, now);  in iocg_activate()
1293 	ioc_now(ioc, now);  in iocg_activate()
1314 	vtarget = now->vnow - ioc->margins.target;  in iocg_activate()
1330 			  iocg->last_inuse ?: iocg->weight, true, now);  in iocg_activate()
1332 	TRACE_IOCG_PATH(iocg_activate, iocg, now,  in iocg_activate()
1335 	iocg->activated_at = now->now;  in iocg_activate()
1339 		ioc->dfgv_period_at = now->now;  in iocg_activate()
1341 		ioc_start_period(ioc, now);  in iocg_activate()
1353 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)  in iocg_kick_delay()  argument
1367 	if (time_before64(now->now, iocg->delay_at))  in iocg_kick_delay()
1371 	tdelta = now->now - iocg->delay_at;  in iocg_kick_delay()
1381 		abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow;  in iocg_kick_delay()
1398 		iocg->delay_at = now->now;  in iocg_kick_delay()
1404 			iocg->indelay_since = now->now;  in iocg_kick_delay()
1409 			iocg->stat.indelay_us += now->now - iocg->indelay_since;  in iocg_kick_delay()
1419 			    struct ioc_now *now)  in iocg_incur_debt()  argument
1432 		iocg->indebt_since = now->now;  in iocg_incur_debt()
1433 		propagate_weights(iocg, iocg->active, 0, false, now);  in iocg_incur_debt()
1444 			  struct ioc_now *now)  in iocg_pay_debt()  argument
1460 		iocg->stat.indebt_us += now->now - iocg->indebt_since;  in iocg_pay_debt()
1464 				  false, now);  in iocg_pay_debt()
1501 			    struct ioc_now *now)  in iocg_kick_waitq()  argument
1512 	vbudget = now->vnow - atomic64_read(&iocg->vtime);  in iocg_kick_waitq()
1524 		iocg_pay_debt(iocg, abs_vpay, now);  in iocg_kick_waitq()
1529 		iocg_kick_delay(iocg, now);  in iocg_kick_waitq()
1554 			iocg->stat.wait_us += now->now - iocg->wait_since;  in iocg_kick_waitq()
1561 		iocg->wait_since = now->now;  in iocg_kick_waitq()
1568 	expires = now->now_ns +  in iocg_kick_waitq()
1587 	struct ioc_now now;  in iocg_waitq_timer_fn()  local
1590 	ioc_now(iocg->ioc, &now);  in iocg_waitq_timer_fn()
1593 	iocg_kick_waitq(iocg, pay_debt, &now);  in iocg_waitq_timer_fn()
1703 static void iocg_flush_stat_leaf(struct ioc_gq *iocg, struct ioc_now *now)  in iocg_flush_stat_leaf()  argument
1727 static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now)  in iocg_flush_stat()  argument
1734 		iocg_flush_stat_leaf(iocg, now);  in iocg_flush_stat()
1751 				  u32 usage, struct ioc_now *now)  in hweight_after_donation()  argument
1763 	    time_after64(vtime, now->vnow - ioc->margins.min))  in hweight_after_donation()
1767 	excess = now->vnow - vtime - ioc->margins.target;  in hweight_after_donation()
1791 	delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime),  in hweight_after_donation()
1792 			  now->vnow - ioc->period_at_vtime);  in hweight_after_donation()
1856 static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)  in transfer_surpluses()  argument
2058 		TRACE_IOCG_PATH(inuse_transfer, iocg, now,  in transfer_surpluses()
2063 		__propagate_weights(iocg, iocg->active, inuse, true, now);  in transfer_surpluses()
2083 			      struct ioc_now *now)  in ioc_forgive_debts()  argument
2090 		ioc->dfgv_period_at = now->now;  in ioc_forgive_debts()
2106 	if (time_before64(now->now, ioc->dfgv_period_at + DFGV_PERIOD))  in ioc_forgive_debts()
2113 	dur = now->now - ioc->dfgv_period_at;  in ioc_forgive_debts()
2116 	ioc->dfgv_period_at = now->now;  in ioc_forgive_debts()
2155 		iocg_kick_waitq(iocg, true, now);  in ioc_forgive_debts()
2157 		TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct,  in ioc_forgive_debts()
2175 static int ioc_check_iocgs(struct ioc *ioc, struct ioc_now *now)  in ioc_check_iocgs()  argument
2189 			iocg->stat.wait_us += now->now - iocg->wait_since;  in ioc_check_iocgs()
2190 			iocg->wait_since = now->now;  in ioc_check_iocgs()
2194 				now->now - iocg->indebt_since;  in ioc_check_iocgs()
2195 			iocg->indebt_since = now->now;  in ioc_check_iocgs()
2199 				now->now - iocg->indelay_since;  in ioc_check_iocgs()
2200 			iocg->indelay_since = now->now;  in ioc_check_iocgs()
2206 			iocg_kick_waitq(iocg, true, now);  in ioc_check_iocgs()
2220 			excess = now->vnow - vtime - ioc->margins.target;  in ioc_check_iocgs()
2229 			TRACE_IOCG_PATH(iocg_idle, iocg, now,  in ioc_check_iocgs()
2232 			__propagate_weights(iocg, 0, 0, false, now);  in ioc_check_iocgs()
2247 	struct ioc_now now;  in ioc_timer_fn()  local
2265 	ioc_now(ioc, &now);  in ioc_timer_fn()
2267 	period_vtime = now.vnow - ioc->period_at_vtime;  in ioc_timer_fn()
2273 	nr_debtors = ioc_check_iocgs(ioc, &now);  in ioc_timer_fn()
2279 	iocg_flush_stat(&ioc->active_iocgs, &now);  in ioc_timer_fn()
2303 		    time_after64(vtime, now.vnow -  in ioc_timer_fn()
2305 		    time_before64(vdone, now.vnow - period_vtime))  in ioc_timer_fn()
2319 		     time_before64(vtime, now.vnow - ioc->margins.low))) {  in ioc_timer_fn()
2333 				usage_dur = max_t(u64, now.now - iocg->activated_at, 1);  in ioc_timer_fn()
2335 				usage_dur = max_t(u64, now.now - ioc->period_at, 1);  in ioc_timer_fn()
2349 							 usage, &now);  in ioc_timer_fn()
2372 				TRACE_IOCG_PATH(inuse_shortage, iocg, &now,  in ioc_timer_fn()
2377 						    iocg->active, true, &now);  in ioc_timer_fn()
2387 		transfer_surpluses(&surpluses, &now);  in ioc_timer_fn()
2446 	ioc_forgive_debts(ioc, usage_us_sum, nr_debtors, &now);  in ioc_timer_fn()
2456 			ioc_start_period(ioc, &now);  in ioc_timer_fn()
2463 		ioc_refresh_vrate(ioc, &now);  in ioc_timer_fn()
2470 				      u64 abs_cost, struct ioc_now *now)  in adjust_inuse_and_calc_cost()  argument
2483 	margin = now->vnow - vtime - cost;  in adjust_inuse_and_calc_cost()
2516 		propagate_weights(iocg, iocg->active, new_inuse, true, now);  in adjust_inuse_and_calc_cost()
2519 	} while (time_after64(vtime + cost, now->vnow) &&  in adjust_inuse_and_calc_cost()
2524 	TRACE_IOCG_PATH(inuse_adjust, iocg, now,  in adjust_inuse_and_calc_cost()
2613 	struct ioc_now now;  in ioc_rqos_throttle()  local
2628 	if (!iocg_activate(iocg, &now))  in ioc_rqos_throttle()
2633 	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);  in ioc_rqos_throttle()
2641 	    time_before_eq64(vtime + cost, now.vnow)) {  in ioc_rqos_throttle()
2689 		iocg_incur_debt(iocg, abs_cost, &now);  in ioc_rqos_throttle()
2690 		if (iocg_kick_delay(iocg, &now))  in ioc_rqos_throttle()
2705 				  &now);  in ioc_rqos_throttle()
2728 	iocg_kick_waitq(iocg, ioc_locked, &now);  in ioc_rqos_throttle()
2749 	struct ioc_now now;  in ioc_rqos_merge()  local
2761 	ioc_now(ioc, &now);  in ioc_rqos_merge()
2764 	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);  in ioc_rqos_merge()
2776 	    time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {  in ioc_rqos_merge()
2790 		iocg_incur_debt(iocg, abs_cost, &now);  in ioc_rqos_merge()
2791 		if (iocg_kick_delay(iocg, &now))  in ioc_rqos_merge()
2990 	struct ioc_now now;  in ioc_pd_init()  local
2994 	ioc_now(ioc, &now);  in ioc_pd_init()
2997 	atomic64_set(&iocg->vtime, now.vnow);  in ioc_pd_init()
2998 	atomic64_set(&iocg->done_vtime, now.vnow);  in ioc_pd_init()
3018 	weight_updated(iocg, &now);  in ioc_pd_init()
3032 			struct ioc_now now;  in ioc_pd_free()  local
3034 			ioc_now(ioc, &now);  in ioc_pd_free()
3035 			propagate_weights(iocg, 0, 0, false, &now);  in ioc_pd_free()
3103 	struct ioc_now now;  in ioc_weight_write()  local
3124 				ioc_now(iocg->ioc, &now);  in ioc_weight_write()
3125 				weight_updated(iocg, &now);  in ioc_weight_write()
3153 	ioc_now(iocg->ioc, &now);  in ioc_weight_write()
3154 	weight_updated(iocg, &now);  in ioc_weight_write()