1 #ifdef CONFIG_SMP 2 #include "sched-pelt.h" 3 4 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se); 5 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se); 6 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq); 7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running); 8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running); 9 10 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 11 int update_irq_load_avg(struct rq *rq, u64 running); 12 #else 13 static inline int 14 update_irq_load_avg(struct rq *rq, u64 running) 15 { 16 return 0; 17 } 18 #endif 19 20 /* 21 * When a task is dequeued, its estimated utilization should not be update if 22 * its util_avg has not been updated at least once. 23 * This flag is used to synchronize util_avg updates with util_est updates. 24 * We map this information into the LSB bit of the utilization saved at 25 * dequeue time (i.e. util_est.dequeued). 26 */ 27 #define UTIL_AVG_UNCHANGED 0x1 28 29 static inline void cfs_se_util_change(struct sched_avg *avg) 30 { 31 unsigned int enqueued; 32 33 if (!sched_feat(UTIL_EST)) 34 return; 35 36 /* Avoid store if the flag has been already set */ 37 enqueued = avg->util_est.enqueued; 38 if (!(enqueued & UTIL_AVG_UNCHANGED)) 39 return; 40 41 /* Reset flag to report util_avg has been updated */ 42 enqueued &= ~UTIL_AVG_UNCHANGED; 43 WRITE_ONCE(avg->util_est.enqueued, enqueued); 44 } 45 46 /* 47 * The clock_pelt scales the time to reflect the effective amount of 48 * computation done during the running delta time but then sync back to 49 * clock_task when rq is idle. 50 * 51 * 52 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16 53 * @ max capacity ------******---------------******--------------- 54 * @ half capacity ------************---------************--------- 55 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16 56 * 57 */ 58 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) 59 { 60 if (unlikely(is_idle_task(rq->curr))) { 61 /* The rq is idle, we can sync to clock_task */ 62 rq->clock_pelt = rq_clock_task(rq); 63 return; 64 } 65 66 /* 67 * When a rq runs at a lower compute capacity, it will need 68 * more time to do the same amount of work than at max 69 * capacity. In order to be invariant, we scale the delta to 70 * reflect how much work has been really done. 71 * Running longer results in stealing idle time that will 72 * disturb the load signal compared to max capacity. This 73 * stolen idle time will be automatically reflected when the 74 * rq will be idle and the clock will be synced with 75 * rq_clock_task. 76 */ 77 78 /* 79 * Scale the elapsed time to reflect the real amount of 80 * computation 81 */ 82 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); 83 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); 84 85 rq->clock_pelt += delta; 86 } 87 88 /* 89 * When rq becomes idle, we have to check if it has lost idle time 90 * because it was fully busy. A rq is fully used when the /Sum util_sum 91 * is greater or equal to: 92 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT; 93 * For optimization and computing rounding purpose, we don't take into account 94 * the position in the current window (period_contrib) and we use the higher 95 * bound of util_sum to decide. 96 */ 97 static inline void update_idle_rq_clock_pelt(struct rq *rq) 98 { 99 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX; 100 u32 util_sum = rq->cfs.avg.util_sum; 101 util_sum += rq->avg_rt.util_sum; 102 util_sum += rq->avg_dl.util_sum; 103 104 /* 105 * Reflecting stolen time makes sense only if the idle 106 * phase would be present at max capacity. As soon as the 107 * utilization of a rq has reached the maximum value, it is 108 * considered as an always runnig rq without idle time to 109 * steal. This potential idle time is considered as lost in 110 * this case. We keep track of this lost idle time compare to 111 * rq's clock_task. 112 */ 113 if (util_sum >= divider) 114 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; 115 } 116 117 static inline u64 rq_clock_pelt(struct rq *rq) 118 { 119 lockdep_assert_held(&rq->lock); 120 assert_clock_updated(rq); 121 122 return rq->clock_pelt - rq->lost_idle_time; 123 } 124 125 #ifdef CONFIG_CFS_BANDWIDTH 126 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ 127 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 128 { 129 if (unlikely(cfs_rq->throttle_count)) 130 return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time; 131 132 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; 133 } 134 #else 135 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 136 { 137 return rq_clock_pelt(rq_of(cfs_rq)); 138 } 139 #endif 140 141 #else 142 143 static inline int 144 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 145 { 146 return 0; 147 } 148 149 static inline int 150 update_rt_rq_load_avg(u64 now, struct rq *rq, int running) 151 { 152 return 0; 153 } 154 155 static inline int 156 update_dl_rq_load_avg(u64 now, struct rq *rq, int running) 157 { 158 return 0; 159 } 160 161 static inline int 162 update_irq_load_avg(struct rq *rq, u64 running) 163 { 164 return 0; 165 } 166 167 static inline u64 rq_clock_pelt(struct rq *rq) 168 { 169 return rq_clock_task(rq); 170 } 171 172 static inline void 173 update_rq_clock_pelt(struct rq *rq, s64 delta) { } 174 175 static inline void 176 update_idle_rq_clock_pelt(struct rq *rq) { } 177 178 #endif 179 180 181