1 #ifdef CONFIG_SMP 2 #include "sched-pelt.h" 3 4 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se); 5 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se); 6 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq); 7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running); 8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running); 9 10 #ifdef CONFIG_SCHED_THERMAL_PRESSURE 11 int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity); 12 13 static inline u64 thermal_load_avg(struct rq *rq) 14 { 15 return READ_ONCE(rq->avg_thermal.load_avg); 16 } 17 #else 18 static inline int 19 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) 20 { 21 return 0; 22 } 23 24 static inline u64 thermal_load_avg(struct rq *rq) 25 { 26 return 0; 27 } 28 #endif 29 30 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 31 int update_irq_load_avg(struct rq *rq, u64 running); 32 #else 33 static inline int 34 update_irq_load_avg(struct rq *rq, u64 running) 35 { 36 return 0; 37 } 38 #endif 39 40 static inline u32 get_pelt_divider(struct sched_avg *avg) 41 { 42 return LOAD_AVG_MAX - 1024 + avg->period_contrib; 43 } 44 45 /* 46 * When a task is dequeued, its estimated utilization should not be update if 47 * its util_avg has not been updated at least once. 48 * This flag is used to synchronize util_avg updates with util_est updates. 49 * We map this information into the LSB bit of the utilization saved at 50 * dequeue time (i.e. util_est.dequeued). 51 */ 52 #define UTIL_AVG_UNCHANGED 0x1 53 54 static inline void cfs_se_util_change(struct sched_avg *avg) 55 { 56 unsigned int enqueued; 57 58 if (!sched_feat(UTIL_EST)) 59 return; 60 61 /* Avoid store if the flag has been already set */ 62 enqueued = avg->util_est.enqueued; 63 if (!(enqueued & UTIL_AVG_UNCHANGED)) 64 return; 65 66 /* Reset flag to report util_avg has been updated */ 67 enqueued &= ~UTIL_AVG_UNCHANGED; 68 WRITE_ONCE(avg->util_est.enqueued, enqueued); 69 } 70 71 /* 72 * The clock_pelt scales the time to reflect the effective amount of 73 * computation done during the running delta time but then sync back to 74 * clock_task when rq is idle. 75 * 76 * 77 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16 78 * @ max capacity ------******---------------******--------------- 79 * @ half capacity ------************---------************--------- 80 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16 81 * 82 */ 83 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) 84 { 85 if (unlikely(is_idle_task(rq->curr))) { 86 /* The rq is idle, we can sync to clock_task */ 87 rq->clock_pelt = rq_clock_task(rq); 88 return; 89 } 90 91 /* 92 * When a rq runs at a lower compute capacity, it will need 93 * more time to do the same amount of work than at max 94 * capacity. In order to be invariant, we scale the delta to 95 * reflect how much work has been really done. 96 * Running longer results in stealing idle time that will 97 * disturb the load signal compared to max capacity. This 98 * stolen idle time will be automatically reflected when the 99 * rq will be idle and the clock will be synced with 100 * rq_clock_task. 101 */ 102 103 /* 104 * Scale the elapsed time to reflect the real amount of 105 * computation 106 */ 107 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); 108 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); 109 110 rq->clock_pelt += delta; 111 } 112 113 /* 114 * When rq becomes idle, we have to check if it has lost idle time 115 * because it was fully busy. A rq is fully used when the /Sum util_sum 116 * is greater or equal to: 117 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT; 118 * For optimization and computing rounding purpose, we don't take into account 119 * the position in the current window (period_contrib) and we use the higher 120 * bound of util_sum to decide. 121 */ 122 static inline void update_idle_rq_clock_pelt(struct rq *rq) 123 { 124 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX; 125 u32 util_sum = rq->cfs.avg.util_sum; 126 util_sum += rq->avg_rt.util_sum; 127 util_sum += rq->avg_dl.util_sum; 128 129 /* 130 * Reflecting stolen time makes sense only if the idle 131 * phase would be present at max capacity. As soon as the 132 * utilization of a rq has reached the maximum value, it is 133 * considered as an always runnig rq without idle time to 134 * steal. This potential idle time is considered as lost in 135 * this case. We keep track of this lost idle time compare to 136 * rq's clock_task. 137 */ 138 if (util_sum >= divider) 139 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; 140 } 141 142 static inline u64 rq_clock_pelt(struct rq *rq) 143 { 144 lockdep_assert_held(&rq->lock); 145 assert_clock_updated(rq); 146 147 return rq->clock_pelt - rq->lost_idle_time; 148 } 149 150 #ifdef CONFIG_CFS_BANDWIDTH 151 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ 152 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 153 { 154 if (unlikely(cfs_rq->throttle_count)) 155 return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time; 156 157 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; 158 } 159 #else 160 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 161 { 162 return rq_clock_pelt(rq_of(cfs_rq)); 163 } 164 #endif 165 166 #else 167 168 static inline int 169 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 170 { 171 return 0; 172 } 173 174 static inline int 175 update_rt_rq_load_avg(u64 now, struct rq *rq, int running) 176 { 177 return 0; 178 } 179 180 static inline int 181 update_dl_rq_load_avg(u64 now, struct rq *rq, int running) 182 { 183 return 0; 184 } 185 186 static inline int 187 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) 188 { 189 return 0; 190 } 191 192 static inline u64 thermal_load_avg(struct rq *rq) 193 { 194 return 0; 195 } 196 197 static inline int 198 update_irq_load_avg(struct rq *rq, u64 running) 199 { 200 return 0; 201 } 202 203 static inline u64 rq_clock_pelt(struct rq *rq) 204 { 205 return rq_clock_task(rq); 206 } 207 208 static inline void 209 update_rq_clock_pelt(struct rq *rq, s64 delta) { } 210 211 static inline void 212 update_idle_rq_clock_pelt(struct rq *rq) { } 213 214 #endif 215 216 217