xref: /openbmc/linux/kernel/sched/pelt.h (revision e2f3e35f)
1 #ifdef CONFIG_SMP
2 #include "sched-pelt.h"
3 
4 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
5 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
6 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
9 
10 #ifdef CONFIG_SCHED_THERMAL_PRESSURE
11 int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
12 
thermal_load_avg(struct rq * rq)13 static inline u64 thermal_load_avg(struct rq *rq)
14 {
15 	return READ_ONCE(rq->avg_thermal.load_avg);
16 }
17 #else
18 static inline int
update_thermal_load_avg(u64 now,struct rq * rq,u64 capacity)19 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
20 {
21 	return 0;
22 }
23 
thermal_load_avg(struct rq * rq)24 static inline u64 thermal_load_avg(struct rq *rq)
25 {
26 	return 0;
27 }
28 #endif
29 
30 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
31 int update_irq_load_avg(struct rq *rq, u64 running);
32 #else
33 static inline int
update_irq_load_avg(struct rq * rq,u64 running)34 update_irq_load_avg(struct rq *rq, u64 running)
35 {
36 	return 0;
37 }
38 #endif
39 
40 #define PELT_MIN_DIVIDER	(LOAD_AVG_MAX - 1024)
41 
get_pelt_divider(struct sched_avg * avg)42 static inline u32 get_pelt_divider(struct sched_avg *avg)
43 {
44 	return PELT_MIN_DIVIDER + avg->period_contrib;
45 }
46 
cfs_se_util_change(struct sched_avg * avg)47 static inline void cfs_se_util_change(struct sched_avg *avg)
48 {
49 	unsigned int enqueued;
50 
51 	if (!sched_feat(UTIL_EST))
52 		return;
53 
54 	/* Avoid store if the flag has been already reset */
55 	enqueued = avg->util_est.enqueued;
56 	if (!(enqueued & UTIL_AVG_UNCHANGED))
57 		return;
58 
59 	/* Reset flag to report util_avg has been updated */
60 	enqueued &= ~UTIL_AVG_UNCHANGED;
61 	WRITE_ONCE(avg->util_est.enqueued, enqueued);
62 }
63 
rq_clock_pelt(struct rq * rq)64 static inline u64 rq_clock_pelt(struct rq *rq)
65 {
66 	lockdep_assert_rq_held(rq);
67 	assert_clock_updated(rq);
68 
69 	return rq->clock_pelt - rq->lost_idle_time;
70 }
71 
72 /* The rq is idle, we can sync to clock_task */
_update_idle_rq_clock_pelt(struct rq * rq)73 static inline void _update_idle_rq_clock_pelt(struct rq *rq)
74 {
75 	rq->clock_pelt  = rq_clock_task(rq);
76 
77 	u64_u32_store(rq->clock_idle, rq_clock(rq));
78 	/* Paired with smp_rmb in migrate_se_pelt_lag() */
79 	smp_wmb();
80 	u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));
81 }
82 
83 /*
84  * The clock_pelt scales the time to reflect the effective amount of
85  * computation done during the running delta time but then sync back to
86  * clock_task when rq is idle.
87  *
88  *
89  * absolute time   | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
90  * @ max capacity  ------******---------------******---------------
91  * @ half capacity ------************---------************---------
92  * clock pelt      | 1| 2|    3|    4| 7| 8| 9|   10|   11|14|15|16
93  *
94  */
update_rq_clock_pelt(struct rq * rq,s64 delta)95 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
96 {
97 	if (unlikely(is_idle_task(rq->curr))) {
98 		_update_idle_rq_clock_pelt(rq);
99 		return;
100 	}
101 
102 	/*
103 	 * When a rq runs at a lower compute capacity, it will need
104 	 * more time to do the same amount of work than at max
105 	 * capacity. In order to be invariant, we scale the delta to
106 	 * reflect how much work has been really done.
107 	 * Running longer results in stealing idle time that will
108 	 * disturb the load signal compared to max capacity. This
109 	 * stolen idle time will be automatically reflected when the
110 	 * rq will be idle and the clock will be synced with
111 	 * rq_clock_task.
112 	 */
113 
114 	/*
115 	 * Scale the elapsed time to reflect the real amount of
116 	 * computation
117 	 */
118 	delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
119 	delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
120 
121 	rq->clock_pelt += delta;
122 }
123 
124 /*
125  * When rq becomes idle, we have to check if it has lost idle time
126  * because it was fully busy. A rq is fully used when the /Sum util_sum
127  * is greater or equal to:
128  * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
129  * For optimization and computing rounding purpose, we don't take into account
130  * the position in the current window (period_contrib) and we use the higher
131  * bound of util_sum to decide.
132  */
update_idle_rq_clock_pelt(struct rq * rq)133 static inline void update_idle_rq_clock_pelt(struct rq *rq)
134 {
135 	u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
136 	u32 util_sum = rq->cfs.avg.util_sum;
137 	util_sum += rq->avg_rt.util_sum;
138 	util_sum += rq->avg_dl.util_sum;
139 
140 	/*
141 	 * Reflecting stolen time makes sense only if the idle
142 	 * phase would be present at max capacity. As soon as the
143 	 * utilization of a rq has reached the maximum value, it is
144 	 * considered as an always running rq without idle time to
145 	 * steal. This potential idle time is considered as lost in
146 	 * this case. We keep track of this lost idle time compare to
147 	 * rq's clock_task.
148 	 */
149 	if (util_sum >= divider)
150 		rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
151 
152 	_update_idle_rq_clock_pelt(rq);
153 }
154 
155 #ifdef CONFIG_CFS_BANDWIDTH
update_idle_cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)156 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
157 {
158 	u64 throttled;
159 
160 	if (unlikely(cfs_rq->throttle_count))
161 		throttled = U64_MAX;
162 	else
163 		throttled = cfs_rq->throttled_clock_pelt_time;
164 
165 	u64_u32_store(cfs_rq->throttled_pelt_idle, throttled);
166 }
167 
168 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)169 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
170 {
171 	if (unlikely(cfs_rq->throttle_count))
172 		return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
173 
174 	return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
175 }
176 #else
update_idle_cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)177 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)178 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
179 {
180 	return rq_clock_pelt(rq_of(cfs_rq));
181 }
182 #endif
183 
184 #else
185 
186 static inline int
update_cfs_rq_load_avg(u64 now,struct cfs_rq * cfs_rq)187 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
188 {
189 	return 0;
190 }
191 
192 static inline int
update_rt_rq_load_avg(u64 now,struct rq * rq,int running)193 update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
194 {
195 	return 0;
196 }
197 
198 static inline int
update_dl_rq_load_avg(u64 now,struct rq * rq,int running)199 update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
200 {
201 	return 0;
202 }
203 
204 static inline int
update_thermal_load_avg(u64 now,struct rq * rq,u64 capacity)205 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
206 {
207 	return 0;
208 }
209 
thermal_load_avg(struct rq * rq)210 static inline u64 thermal_load_avg(struct rq *rq)
211 {
212 	return 0;
213 }
214 
215 static inline int
update_irq_load_avg(struct rq * rq,u64 running)216 update_irq_load_avg(struct rq *rq, u64 running)
217 {
218 	return 0;
219 }
220 
rq_clock_pelt(struct rq * rq)221 static inline u64 rq_clock_pelt(struct rq *rq)
222 {
223 	return rq_clock_task(rq);
224 }
225 
226 static inline void
update_rq_clock_pelt(struct rq * rq,s64 delta)227 update_rq_clock_pelt(struct rq *rq, s64 delta) { }
228 
229 static inline void
update_idle_rq_clock_pelt(struct rq * rq)230 update_idle_rq_clock_pelt(struct rq *rq) { }
231 
update_idle_cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)232 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
233 #endif
234 
235 
236