xref: /openbmc/linux/kernel/sched/fair.c (revision 75020f2d)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
4  *
5  *  Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
6  *
7  *  Interactivity improvements by Mike Galbraith
8  *  (C) 2007 Mike Galbraith <efault@gmx.de>
9  *
10  *  Various enhancements by Dmitry Adamushko.
11  *  (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
12  *
13  *  Group scheduling enhancements by Srivatsa Vaddagiri
14  *  Copyright IBM Corporation, 2007
15  *  Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
16  *
17  *  Scaled math optimizations by Thomas Gleixner
18  *  Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
19  *
20  *  Adaptive scheduling granularity, math enhancements by Peter Zijlstra
21  *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
22  */
23 #include "sched.h"
24 
25 /*
26  * Targeted preemption latency for CPU-bound tasks:
27  *
28  * NOTE: this latency value is not the same as the concept of
29  * 'timeslice length' - timeslices in CFS are of variable length
30  * and have no persistent notion like in traditional, time-slice
31  * based scheduling concepts.
32  *
33  * (to see the precise effective timeslice length of your workload,
34  *  run vmstat and monitor the context-switches (cs) field)
35  *
36  * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
37  */
38 unsigned int sysctl_sched_latency			= 6000000ULL;
39 static unsigned int normalized_sysctl_sched_latency	= 6000000ULL;
40 
41 /*
42  * The initial- and re-scaling of tunables is configurable
43  *
44  * Options are:
45  *
46  *   SCHED_TUNABLESCALING_NONE - unscaled, always *1
47  *   SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
48  *   SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
49  *
50  * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
51  */
52 unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
53 
54 /*
55  * Minimal preemption granularity for CPU-bound tasks:
56  *
57  * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
58  */
59 unsigned int sysctl_sched_min_granularity			= 750000ULL;
60 static unsigned int normalized_sysctl_sched_min_granularity	= 750000ULL;
61 
62 /*
63  * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity
64  */
65 static unsigned int sched_nr_latency = 8;
66 
67 /*
68  * After fork, child runs first. If set to 0 (default) then
69  * parent will (try to) run first.
70  */
71 unsigned int sysctl_sched_child_runs_first __read_mostly;
72 
73 /*
74  * SCHED_OTHER wake-up granularity.
75  *
76  * This option delays the preemption effects of decoupled workloads
77  * and reduces their over-scheduling. Synchronous workloads will still
78  * have immediate wakeup/sleep latencies.
79  *
80  * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
81  */
82 unsigned int sysctl_sched_wakeup_granularity			= 1000000UL;
83 static unsigned int normalized_sysctl_sched_wakeup_granularity	= 1000000UL;
84 
85 const_debug unsigned int sysctl_sched_migration_cost	= 500000UL;
86 
87 int sched_thermal_decay_shift;
88 static int __init setup_sched_thermal_decay_shift(char *str)
89 {
90 	int _shift = 0;
91 
92 	if (kstrtoint(str, 0, &_shift))
93 		pr_warn("Unable to set scheduler thermal pressure decay shift parameter\n");
94 
95 	sched_thermal_decay_shift = clamp(_shift, 0, 10);
96 	return 1;
97 }
98 __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift);
99 
100 #ifdef CONFIG_SMP
101 /*
102  * For asym packing, by default the lower numbered CPU has higher priority.
103  */
104 int __weak arch_asym_cpu_priority(int cpu)
105 {
106 	return -cpu;
107 }
108 
109 /*
110  * The margin used when comparing utilization with CPU capacity.
111  *
112  * (default: ~20%)
113  */
114 #define fits_capacity(cap, max)	((cap) * 1280 < (max) * 1024)
115 
116 /*
117  * The margin used when comparing CPU capacities.
118  * is 'cap1' noticeably greater than 'cap2'
119  *
120  * (default: ~5%)
121  */
122 #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078)
123 #endif
124 
125 #ifdef CONFIG_CFS_BANDWIDTH
126 /*
127  * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
128  * each time a cfs_rq requests quota.
129  *
130  * Note: in the case that the slice exceeds the runtime remaining (either due
131  * to consumption or the quota being specified to be smaller than the slice)
132  * we will always only issue the remaining available time.
133  *
134  * (default: 5 msec, units: microseconds)
135  */
136 unsigned int sysctl_sched_cfs_bandwidth_slice		= 5000UL;
137 #endif
138 
139 static inline void update_load_add(struct load_weight *lw, unsigned long inc)
140 {
141 	lw->weight += inc;
142 	lw->inv_weight = 0;
143 }
144 
145 static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
146 {
147 	lw->weight -= dec;
148 	lw->inv_weight = 0;
149 }
150 
151 static inline void update_load_set(struct load_weight *lw, unsigned long w)
152 {
153 	lw->weight = w;
154 	lw->inv_weight = 0;
155 }
156 
157 /*
158  * Increase the granularity value when there are more CPUs,
159  * because with more CPUs the 'effective latency' as visible
160  * to users decreases. But the relationship is not linear,
161  * so pick a second-best guess by going with the log2 of the
162  * number of CPUs.
163  *
164  * This idea comes from the SD scheduler of Con Kolivas:
165  */
166 static unsigned int get_update_sysctl_factor(void)
167 {
168 	unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
169 	unsigned int factor;
170 
171 	switch (sysctl_sched_tunable_scaling) {
172 	case SCHED_TUNABLESCALING_NONE:
173 		factor = 1;
174 		break;
175 	case SCHED_TUNABLESCALING_LINEAR:
176 		factor = cpus;
177 		break;
178 	case SCHED_TUNABLESCALING_LOG:
179 	default:
180 		factor = 1 + ilog2(cpus);
181 		break;
182 	}
183 
184 	return factor;
185 }
186 
187 static void update_sysctl(void)
188 {
189 	unsigned int factor = get_update_sysctl_factor();
190 
191 #define SET_SYSCTL(name) \
192 	(sysctl_##name = (factor) * normalized_sysctl_##name)
193 	SET_SYSCTL(sched_min_granularity);
194 	SET_SYSCTL(sched_latency);
195 	SET_SYSCTL(sched_wakeup_granularity);
196 #undef SET_SYSCTL
197 }
198 
199 void __init sched_init_granularity(void)
200 {
201 	update_sysctl();
202 }
203 
204 #define WMULT_CONST	(~0U)
205 #define WMULT_SHIFT	32
206 
207 static void __update_inv_weight(struct load_weight *lw)
208 {
209 	unsigned long w;
210 
211 	if (likely(lw->inv_weight))
212 		return;
213 
214 	w = scale_load_down(lw->weight);
215 
216 	if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
217 		lw->inv_weight = 1;
218 	else if (unlikely(!w))
219 		lw->inv_weight = WMULT_CONST;
220 	else
221 		lw->inv_weight = WMULT_CONST / w;
222 }
223 
224 /*
225  * delta_exec * weight / lw.weight
226  *   OR
227  * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT
228  *
229  * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case
230  * we're guaranteed shift stays positive because inv_weight is guaranteed to
231  * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22.
232  *
233  * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
234  * weight/lw.weight <= 1, and therefore our shift will also be positive.
235  */
236 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
237 {
238 	u64 fact = scale_load_down(weight);
239 	u32 fact_hi = (u32)(fact >> 32);
240 	int shift = WMULT_SHIFT;
241 	int fs;
242 
243 	__update_inv_weight(lw);
244 
245 	if (unlikely(fact_hi)) {
246 		fs = fls(fact_hi);
247 		shift -= fs;
248 		fact >>= fs;
249 	}
250 
251 	fact = mul_u32_u32(fact, lw->inv_weight);
252 
253 	fact_hi = (u32)(fact >> 32);
254 	if (fact_hi) {
255 		fs = fls(fact_hi);
256 		shift -= fs;
257 		fact >>= fs;
258 	}
259 
260 	return mul_u64_u32_shr(delta_exec, fact, shift);
261 }
262 
263 
264 const struct sched_class fair_sched_class;
265 
266 /**************************************************************
267  * CFS operations on generic schedulable entities:
268  */
269 
270 #ifdef CONFIG_FAIR_GROUP_SCHED
271 static inline struct task_struct *task_of(struct sched_entity *se)
272 {
273 	SCHED_WARN_ON(!entity_is_task(se));
274 	return container_of(se, struct task_struct, se);
275 }
276 
277 /* Walk up scheduling entities hierarchy */
278 #define for_each_sched_entity(se) \
279 		for (; se; se = se->parent)
280 
281 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
282 {
283 	return p->se.cfs_rq;
284 }
285 
286 /* runqueue on which this entity is (to be) queued */
287 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
288 {
289 	return se->cfs_rq;
290 }
291 
292 /* runqueue "owned" by this group */
293 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
294 {
295 	return grp->my_q;
296 }
297 
298 static inline void cfs_rq_tg_path(struct cfs_rq *cfs_rq, char *path, int len)
299 {
300 	if (!path)
301 		return;
302 
303 	if (cfs_rq && task_group_is_autogroup(cfs_rq->tg))
304 		autogroup_path(cfs_rq->tg, path, len);
305 	else if (cfs_rq && cfs_rq->tg->css.cgroup)
306 		cgroup_path(cfs_rq->tg->css.cgroup, path, len);
307 	else
308 		strlcpy(path, "(null)", len);
309 }
310 
311 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
312 {
313 	struct rq *rq = rq_of(cfs_rq);
314 	int cpu = cpu_of(rq);
315 
316 	if (cfs_rq->on_list)
317 		return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list;
318 
319 	cfs_rq->on_list = 1;
320 
321 	/*
322 	 * Ensure we either appear before our parent (if already
323 	 * enqueued) or force our parent to appear after us when it is
324 	 * enqueued. The fact that we always enqueue bottom-up
325 	 * reduces this to two cases and a special case for the root
326 	 * cfs_rq. Furthermore, it also means that we will always reset
327 	 * tmp_alone_branch either when the branch is connected
328 	 * to a tree or when we reach the top of the tree
329 	 */
330 	if (cfs_rq->tg->parent &&
331 	    cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {
332 		/*
333 		 * If parent is already on the list, we add the child
334 		 * just before. Thanks to circular linked property of
335 		 * the list, this means to put the child at the tail
336 		 * of the list that starts by parent.
337 		 */
338 		list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
339 			&(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));
340 		/*
341 		 * The branch is now connected to its tree so we can
342 		 * reset tmp_alone_branch to the beginning of the
343 		 * list.
344 		 */
345 		rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
346 		return true;
347 	}
348 
349 	if (!cfs_rq->tg->parent) {
350 		/*
351 		 * cfs rq without parent should be put
352 		 * at the tail of the list.
353 		 */
354 		list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
355 			&rq->leaf_cfs_rq_list);
356 		/*
357 		 * We have reach the top of a tree so we can reset
358 		 * tmp_alone_branch to the beginning of the list.
359 		 */
360 		rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
361 		return true;
362 	}
363 
364 	/*
365 	 * The parent has not already been added so we want to
366 	 * make sure that it will be put after us.
367 	 * tmp_alone_branch points to the begin of the branch
368 	 * where we will add parent.
369 	 */
370 	list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch);
371 	/*
372 	 * update tmp_alone_branch to points to the new begin
373 	 * of the branch
374 	 */
375 	rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list;
376 	return false;
377 }
378 
379 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
380 {
381 	if (cfs_rq->on_list) {
382 		struct rq *rq = rq_of(cfs_rq);
383 
384 		/*
385 		 * With cfs_rq being unthrottled/throttled during an enqueue,
386 		 * it can happen the tmp_alone_branch points the a leaf that
387 		 * we finally want to del. In this case, tmp_alone_branch moves
388 		 * to the prev element but it will point to rq->leaf_cfs_rq_list
389 		 * at the end of the enqueue.
390 		 */
391 		if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list)
392 			rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev;
393 
394 		list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
395 		cfs_rq->on_list = 0;
396 	}
397 }
398 
399 static inline void assert_list_leaf_cfs_rq(struct rq *rq)
400 {
401 	SCHED_WARN_ON(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list);
402 }
403 
404 /* Iterate thr' all leaf cfs_rq's on a runqueue */
405 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos)			\
406 	list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list,	\
407 				 leaf_cfs_rq_list)
408 
409 /* Do the two (enqueued) entities belong to the same group ? */
410 static inline struct cfs_rq *
411 is_same_group(struct sched_entity *se, struct sched_entity *pse)
412 {
413 	if (se->cfs_rq == pse->cfs_rq)
414 		return se->cfs_rq;
415 
416 	return NULL;
417 }
418 
419 static inline struct sched_entity *parent_entity(struct sched_entity *se)
420 {
421 	return se->parent;
422 }
423 
424 static void
425 find_matching_se(struct sched_entity **se, struct sched_entity **pse)
426 {
427 	int se_depth, pse_depth;
428 
429 	/*
430 	 * preemption test can be made between sibling entities who are in the
431 	 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of
432 	 * both tasks until we find their ancestors who are siblings of common
433 	 * parent.
434 	 */
435 
436 	/* First walk up until both entities are at same depth */
437 	se_depth = (*se)->depth;
438 	pse_depth = (*pse)->depth;
439 
440 	while (se_depth > pse_depth) {
441 		se_depth--;
442 		*se = parent_entity(*se);
443 	}
444 
445 	while (pse_depth > se_depth) {
446 		pse_depth--;
447 		*pse = parent_entity(*pse);
448 	}
449 
450 	while (!is_same_group(*se, *pse)) {
451 		*se = parent_entity(*se);
452 		*pse = parent_entity(*pse);
453 	}
454 }
455 
456 #else	/* !CONFIG_FAIR_GROUP_SCHED */
457 
458 static inline struct task_struct *task_of(struct sched_entity *se)
459 {
460 	return container_of(se, struct task_struct, se);
461 }
462 
463 #define for_each_sched_entity(se) \
464 		for (; se; se = NULL)
465 
466 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
467 {
468 	return &task_rq(p)->cfs;
469 }
470 
471 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
472 {
473 	struct task_struct *p = task_of(se);
474 	struct rq *rq = task_rq(p);
475 
476 	return &rq->cfs;
477 }
478 
479 /* runqueue "owned" by this group */
480 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
481 {
482 	return NULL;
483 }
484 
485 static inline void cfs_rq_tg_path(struct cfs_rq *cfs_rq, char *path, int len)
486 {
487 	if (path)
488 		strlcpy(path, "(null)", len);
489 }
490 
491 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
492 {
493 	return true;
494 }
495 
496 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
497 {
498 }
499 
500 static inline void assert_list_leaf_cfs_rq(struct rq *rq)
501 {
502 }
503 
504 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos)	\
505 		for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos)
506 
507 static inline struct sched_entity *parent_entity(struct sched_entity *se)
508 {
509 	return NULL;
510 }
511 
512 static inline void
513 find_matching_se(struct sched_entity **se, struct sched_entity **pse)
514 {
515 }
516 
517 #endif	/* CONFIG_FAIR_GROUP_SCHED */
518 
519 static __always_inline
520 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
521 
522 /**************************************************************
523  * Scheduling class tree data structure manipulation methods:
524  */
525 
526 static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime)
527 {
528 	s64 delta = (s64)(vruntime - max_vruntime);
529 	if (delta > 0)
530 		max_vruntime = vruntime;
531 
532 	return max_vruntime;
533 }
534 
535 static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
536 {
537 	s64 delta = (s64)(vruntime - min_vruntime);
538 	if (delta < 0)
539 		min_vruntime = vruntime;
540 
541 	return min_vruntime;
542 }
543 
544 static inline bool entity_before(struct sched_entity *a,
545 				struct sched_entity *b)
546 {
547 	return (s64)(a->vruntime - b->vruntime) < 0;
548 }
549 
550 #define __node_2_se(node) \
551 	rb_entry((node), struct sched_entity, run_node)
552 
553 static void update_min_vruntime(struct cfs_rq *cfs_rq)
554 {
555 	struct sched_entity *curr = cfs_rq->curr;
556 	struct rb_node *leftmost = rb_first_cached(&cfs_rq->tasks_timeline);
557 
558 	u64 vruntime = cfs_rq->min_vruntime;
559 
560 	if (curr) {
561 		if (curr->on_rq)
562 			vruntime = curr->vruntime;
563 		else
564 			curr = NULL;
565 	}
566 
567 	if (leftmost) { /* non-empty tree */
568 		struct sched_entity *se = __node_2_se(leftmost);
569 
570 		if (!curr)
571 			vruntime = se->vruntime;
572 		else
573 			vruntime = min_vruntime(vruntime, se->vruntime);
574 	}
575 
576 	/* ensure we never gain time by being placed backwards. */
577 	cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime);
578 #ifndef CONFIG_64BIT
579 	smp_wmb();
580 	cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
581 #endif
582 }
583 
584 static inline bool __entity_less(struct rb_node *a, const struct rb_node *b)
585 {
586 	return entity_before(__node_2_se(a), __node_2_se(b));
587 }
588 
589 /*
590  * Enqueue an entity into the rb-tree:
591  */
592 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
593 {
594 	rb_add_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less);
595 }
596 
597 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
598 {
599 	rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline);
600 }
601 
602 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
603 {
604 	struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline);
605 
606 	if (!left)
607 		return NULL;
608 
609 	return __node_2_se(left);
610 }
611 
612 static struct sched_entity *__pick_next_entity(struct sched_entity *se)
613 {
614 	struct rb_node *next = rb_next(&se->run_node);
615 
616 	if (!next)
617 		return NULL;
618 
619 	return __node_2_se(next);
620 }
621 
622 #ifdef CONFIG_SCHED_DEBUG
623 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
624 {
625 	struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root);
626 
627 	if (!last)
628 		return NULL;
629 
630 	return __node_2_se(last);
631 }
632 
633 /**************************************************************
634  * Scheduling class statistics methods:
635  */
636 
637 int sched_update_scaling(void)
638 {
639 	unsigned int factor = get_update_sysctl_factor();
640 
641 	sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
642 					sysctl_sched_min_granularity);
643 
644 #define WRT_SYSCTL(name) \
645 	(normalized_sysctl_##name = sysctl_##name / (factor))
646 	WRT_SYSCTL(sched_min_granularity);
647 	WRT_SYSCTL(sched_latency);
648 	WRT_SYSCTL(sched_wakeup_granularity);
649 #undef WRT_SYSCTL
650 
651 	return 0;
652 }
653 #endif
654 
655 /*
656  * delta /= w
657  */
658 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
659 {
660 	if (unlikely(se->load.weight != NICE_0_LOAD))
661 		delta = __calc_delta(delta, NICE_0_LOAD, &se->load);
662 
663 	return delta;
664 }
665 
666 /*
667  * The idea is to set a period in which each task runs once.
668  *
669  * When there are too many tasks (sched_nr_latency) we have to stretch
670  * this period because otherwise the slices get too small.
671  *
672  * p = (nr <= nl) ? l : l*nr/nl
673  */
674 static u64 __sched_period(unsigned long nr_running)
675 {
676 	if (unlikely(nr_running > sched_nr_latency))
677 		return nr_running * sysctl_sched_min_granularity;
678 	else
679 		return sysctl_sched_latency;
680 }
681 
682 /*
683  * We calculate the wall-time slice from the period by taking a part
684  * proportional to the weight.
685  *
686  * s = p*P[w/rw]
687  */
688 static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
689 {
690 	unsigned int nr_running = cfs_rq->nr_running;
691 	u64 slice;
692 
693 	if (sched_feat(ALT_PERIOD))
694 		nr_running = rq_of(cfs_rq)->cfs.h_nr_running;
695 
696 	slice = __sched_period(nr_running + !se->on_rq);
697 
698 	for_each_sched_entity(se) {
699 		struct load_weight *load;
700 		struct load_weight lw;
701 
702 		cfs_rq = cfs_rq_of(se);
703 		load = &cfs_rq->load;
704 
705 		if (unlikely(!se->on_rq)) {
706 			lw = cfs_rq->load;
707 
708 			update_load_add(&lw, se->load.weight);
709 			load = &lw;
710 		}
711 		slice = __calc_delta(slice, se->load.weight, load);
712 	}
713 
714 	if (sched_feat(BASE_SLICE))
715 		slice = max(slice, (u64)sysctl_sched_min_granularity);
716 
717 	return slice;
718 }
719 
720 /*
721  * We calculate the vruntime slice of a to-be-inserted task.
722  *
723  * vs = s/w
724  */
725 static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
726 {
727 	return calc_delta_fair(sched_slice(cfs_rq, se), se);
728 }
729 
730 #include "pelt.h"
731 #ifdef CONFIG_SMP
732 
733 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);
734 static unsigned long task_h_load(struct task_struct *p);
735 static unsigned long capacity_of(int cpu);
736 
737 /* Give new sched_entity start runnable values to heavy its load in infant time */
738 void init_entity_runnable_average(struct sched_entity *se)
739 {
740 	struct sched_avg *sa = &se->avg;
741 
742 	memset(sa, 0, sizeof(*sa));
743 
744 	/*
745 	 * Tasks are initialized with full load to be seen as heavy tasks until
746 	 * they get a chance to stabilize to their real load level.
747 	 * Group entities are initialized with zero load to reflect the fact that
748 	 * nothing has been attached to the task group yet.
749 	 */
750 	if (entity_is_task(se))
751 		sa->load_avg = scale_load_down(se->load.weight);
752 
753 	/* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */
754 }
755 
756 static void attach_entity_cfs_rq(struct sched_entity *se);
757 
758 /*
759  * With new tasks being created, their initial util_avgs are extrapolated
760  * based on the cfs_rq's current util_avg:
761  *
762  *   util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight
763  *
764  * However, in many cases, the above util_avg does not give a desired
765  * value. Moreover, the sum of the util_avgs may be divergent, such
766  * as when the series is a harmonic series.
767  *
768  * To solve this problem, we also cap the util_avg of successive tasks to
769  * only 1/2 of the left utilization budget:
770  *
771  *   util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n
772  *
773  * where n denotes the nth task and cpu_scale the CPU capacity.
774  *
775  * For example, for a CPU with 1024 of capacity, a simplest series from
776  * the beginning would be like:
777  *
778  *  task  util_avg: 512, 256, 128,  64,  32,   16,    8, ...
779  * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ...
780  *
781  * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap)
782  * if util_avg > util_avg_cap.
783  */
784 void post_init_entity_util_avg(struct task_struct *p)
785 {
786 	struct sched_entity *se = &p->se;
787 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
788 	struct sched_avg *sa = &se->avg;
789 	long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq)));
790 	long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2;
791 
792 	if (cap > 0) {
793 		if (cfs_rq->avg.util_avg != 0) {
794 			sa->util_avg  = cfs_rq->avg.util_avg * se->load.weight;
795 			sa->util_avg /= (cfs_rq->avg.load_avg + 1);
796 
797 			if (sa->util_avg > cap)
798 				sa->util_avg = cap;
799 		} else {
800 			sa->util_avg = cap;
801 		}
802 	}
803 
804 	sa->runnable_avg = sa->util_avg;
805 
806 	if (p->sched_class != &fair_sched_class) {
807 		/*
808 		 * For !fair tasks do:
809 		 *
810 		update_cfs_rq_load_avg(now, cfs_rq);
811 		attach_entity_load_avg(cfs_rq, se);
812 		switched_from_fair(rq, p);
813 		 *
814 		 * such that the next switched_to_fair() has the
815 		 * expected state.
816 		 */
817 		se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq);
818 		return;
819 	}
820 
821 	attach_entity_cfs_rq(se);
822 }
823 
824 #else /* !CONFIG_SMP */
825 void init_entity_runnable_average(struct sched_entity *se)
826 {
827 }
828 void post_init_entity_util_avg(struct task_struct *p)
829 {
830 }
831 static void update_tg_load_avg(struct cfs_rq *cfs_rq)
832 {
833 }
834 #endif /* CONFIG_SMP */
835 
836 /*
837  * Update the current task's runtime statistics.
838  */
839 static void update_curr(struct cfs_rq *cfs_rq)
840 {
841 	struct sched_entity *curr = cfs_rq->curr;
842 	u64 now = rq_clock_task(rq_of(cfs_rq));
843 	u64 delta_exec;
844 
845 	if (unlikely(!curr))
846 		return;
847 
848 	delta_exec = now - curr->exec_start;
849 	if (unlikely((s64)delta_exec <= 0))
850 		return;
851 
852 	curr->exec_start = now;
853 
854 	schedstat_set(curr->statistics.exec_max,
855 		      max(delta_exec, curr->statistics.exec_max));
856 
857 	curr->sum_exec_runtime += delta_exec;
858 	schedstat_add(cfs_rq->exec_clock, delta_exec);
859 
860 	curr->vruntime += calc_delta_fair(delta_exec, curr);
861 	update_min_vruntime(cfs_rq);
862 
863 	if (entity_is_task(curr)) {
864 		struct task_struct *curtask = task_of(curr);
865 
866 		trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
867 		cgroup_account_cputime(curtask, delta_exec);
868 		account_group_exec_runtime(curtask, delta_exec);
869 	}
870 
871 	account_cfs_rq_runtime(cfs_rq, delta_exec);
872 }
873 
874 static void update_curr_fair(struct rq *rq)
875 {
876 	update_curr(cfs_rq_of(&rq->curr->se));
877 }
878 
879 static inline void
880 update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
881 {
882 	u64 wait_start, prev_wait_start;
883 
884 	if (!schedstat_enabled())
885 		return;
886 
887 	wait_start = rq_clock(rq_of(cfs_rq));
888 	prev_wait_start = schedstat_val(se->statistics.wait_start);
889 
890 	if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) &&
891 	    likely(wait_start > prev_wait_start))
892 		wait_start -= prev_wait_start;
893 
894 	__schedstat_set(se->statistics.wait_start, wait_start);
895 }
896 
897 static inline void
898 update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
899 {
900 	struct task_struct *p;
901 	u64 delta;
902 
903 	if (!schedstat_enabled())
904 		return;
905 
906 	/*
907 	 * When the sched_schedstat changes from 0 to 1, some sched se
908 	 * maybe already in the runqueue, the se->statistics.wait_start
909 	 * will be 0.So it will let the delta wrong. We need to avoid this
910 	 * scenario.
911 	 */
912 	if (unlikely(!schedstat_val(se->statistics.wait_start)))
913 		return;
914 
915 	delta = rq_clock(rq_of(cfs_rq)) - schedstat_val(se->statistics.wait_start);
916 
917 	if (entity_is_task(se)) {
918 		p = task_of(se);
919 		if (task_on_rq_migrating(p)) {
920 			/*
921 			 * Preserve migrating task's wait time so wait_start
922 			 * time stamp can be adjusted to accumulate wait time
923 			 * prior to migration.
924 			 */
925 			__schedstat_set(se->statistics.wait_start, delta);
926 			return;
927 		}
928 		trace_sched_stat_wait(p, delta);
929 	}
930 
931 	__schedstat_set(se->statistics.wait_max,
932 		      max(schedstat_val(se->statistics.wait_max), delta));
933 	__schedstat_inc(se->statistics.wait_count);
934 	__schedstat_add(se->statistics.wait_sum, delta);
935 	__schedstat_set(se->statistics.wait_start, 0);
936 }
937 
938 static inline void
939 update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
940 {
941 	struct task_struct *tsk = NULL;
942 	u64 sleep_start, block_start;
943 
944 	if (!schedstat_enabled())
945 		return;
946 
947 	sleep_start = schedstat_val(se->statistics.sleep_start);
948 	block_start = schedstat_val(se->statistics.block_start);
949 
950 	if (entity_is_task(se))
951 		tsk = task_of(se);
952 
953 	if (sleep_start) {
954 		u64 delta = rq_clock(rq_of(cfs_rq)) - sleep_start;
955 
956 		if ((s64)delta < 0)
957 			delta = 0;
958 
959 		if (unlikely(delta > schedstat_val(se->statistics.sleep_max)))
960 			__schedstat_set(se->statistics.sleep_max, delta);
961 
962 		__schedstat_set(se->statistics.sleep_start, 0);
963 		__schedstat_add(se->statistics.sum_sleep_runtime, delta);
964 
965 		if (tsk) {
966 			account_scheduler_latency(tsk, delta >> 10, 1);
967 			trace_sched_stat_sleep(tsk, delta);
968 		}
969 	}
970 	if (block_start) {
971 		u64 delta = rq_clock(rq_of(cfs_rq)) - block_start;
972 
973 		if ((s64)delta < 0)
974 			delta = 0;
975 
976 		if (unlikely(delta > schedstat_val(se->statistics.block_max)))
977 			__schedstat_set(se->statistics.block_max, delta);
978 
979 		__schedstat_set(se->statistics.block_start, 0);
980 		__schedstat_add(se->statistics.sum_sleep_runtime, delta);
981 
982 		if (tsk) {
983 			if (tsk->in_iowait) {
984 				__schedstat_add(se->statistics.iowait_sum, delta);
985 				__schedstat_inc(se->statistics.iowait_count);
986 				trace_sched_stat_iowait(tsk, delta);
987 			}
988 
989 			trace_sched_stat_blocked(tsk, delta);
990 
991 			/*
992 			 * Blocking time is in units of nanosecs, so shift by
993 			 * 20 to get a milliseconds-range estimation of the
994 			 * amount of time that the task spent sleeping:
995 			 */
996 			if (unlikely(prof_on == SLEEP_PROFILING)) {
997 				profile_hits(SLEEP_PROFILING,
998 						(void *)get_wchan(tsk),
999 						delta >> 20);
1000 			}
1001 			account_scheduler_latency(tsk, delta >> 10, 0);
1002 		}
1003 	}
1004 }
1005 
1006 /*
1007  * Task is being enqueued - update stats:
1008  */
1009 static inline void
1010 update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
1011 {
1012 	if (!schedstat_enabled())
1013 		return;
1014 
1015 	/*
1016 	 * Are we enqueueing a waiting task? (for current tasks
1017 	 * a dequeue/enqueue event is a NOP)
1018 	 */
1019 	if (se != cfs_rq->curr)
1020 		update_stats_wait_start(cfs_rq, se);
1021 
1022 	if (flags & ENQUEUE_WAKEUP)
1023 		update_stats_enqueue_sleeper(cfs_rq, se);
1024 }
1025 
1026 static inline void
1027 update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
1028 {
1029 
1030 	if (!schedstat_enabled())
1031 		return;
1032 
1033 	/*
1034 	 * Mark the end of the wait period if dequeueing a
1035 	 * waiting task:
1036 	 */
1037 	if (se != cfs_rq->curr)
1038 		update_stats_wait_end(cfs_rq, se);
1039 
1040 	if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) {
1041 		struct task_struct *tsk = task_of(se);
1042 
1043 		if (tsk->state & TASK_INTERRUPTIBLE)
1044 			__schedstat_set(se->statistics.sleep_start,
1045 				      rq_clock(rq_of(cfs_rq)));
1046 		if (tsk->state & TASK_UNINTERRUPTIBLE)
1047 			__schedstat_set(se->statistics.block_start,
1048 				      rq_clock(rq_of(cfs_rq)));
1049 	}
1050 }
1051 
1052 /*
1053  * We are picking a new current task - update its stats:
1054  */
1055 static inline void
1056 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
1057 {
1058 	/*
1059 	 * We are starting a new run period:
1060 	 */
1061 	se->exec_start = rq_clock_task(rq_of(cfs_rq));
1062 }
1063 
1064 /**************************************************
1065  * Scheduling class queueing methods:
1066  */
1067 
1068 #ifdef CONFIG_NUMA_BALANCING
1069 /*
1070  * Approximate time to scan a full NUMA task in ms. The task scan period is
1071  * calculated based on the tasks virtual memory size and
1072  * numa_balancing_scan_size.
1073  */
1074 unsigned int sysctl_numa_balancing_scan_period_min = 1000;
1075 unsigned int sysctl_numa_balancing_scan_period_max = 60000;
1076 
1077 /* Portion of address space to scan in MB */
1078 unsigned int sysctl_numa_balancing_scan_size = 256;
1079 
1080 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
1081 unsigned int sysctl_numa_balancing_scan_delay = 1000;
1082 
1083 struct numa_group {
1084 	refcount_t refcount;
1085 
1086 	spinlock_t lock; /* nr_tasks, tasks */
1087 	int nr_tasks;
1088 	pid_t gid;
1089 	int active_nodes;
1090 
1091 	struct rcu_head rcu;
1092 	unsigned long total_faults;
1093 	unsigned long max_faults_cpu;
1094 	/*
1095 	 * Faults_cpu is used to decide whether memory should move
1096 	 * towards the CPU. As a consequence, these stats are weighted
1097 	 * more by CPU use than by memory faults.
1098 	 */
1099 	unsigned long *faults_cpu;
1100 	unsigned long faults[];
1101 };
1102 
1103 /*
1104  * For functions that can be called in multiple contexts that permit reading
1105  * ->numa_group (see struct task_struct for locking rules).
1106  */
1107 static struct numa_group *deref_task_numa_group(struct task_struct *p)
1108 {
1109 	return rcu_dereference_check(p->numa_group, p == current ||
1110 		(lockdep_is_held(&task_rq(p)->lock) && !READ_ONCE(p->on_cpu)));
1111 }
1112 
1113 static struct numa_group *deref_curr_numa_group(struct task_struct *p)
1114 {
1115 	return rcu_dereference_protected(p->numa_group, p == current);
1116 }
1117 
1118 static inline unsigned long group_faults_priv(struct numa_group *ng);
1119 static inline unsigned long group_faults_shared(struct numa_group *ng);
1120 
1121 static unsigned int task_nr_scan_windows(struct task_struct *p)
1122 {
1123 	unsigned long rss = 0;
1124 	unsigned long nr_scan_pages;
1125 
1126 	/*
1127 	 * Calculations based on RSS as non-present and empty pages are skipped
1128 	 * by the PTE scanner and NUMA hinting faults should be trapped based
1129 	 * on resident pages
1130 	 */
1131 	nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT);
1132 	rss = get_mm_rss(p->mm);
1133 	if (!rss)
1134 		rss = nr_scan_pages;
1135 
1136 	rss = round_up(rss, nr_scan_pages);
1137 	return rss / nr_scan_pages;
1138 }
1139 
1140 /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */
1141 #define MAX_SCAN_WINDOW 2560
1142 
1143 static unsigned int task_scan_min(struct task_struct *p)
1144 {
1145 	unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size);
1146 	unsigned int scan, floor;
1147 	unsigned int windows = 1;
1148 
1149 	if (scan_size < MAX_SCAN_WINDOW)
1150 		windows = MAX_SCAN_WINDOW / scan_size;
1151 	floor = 1000 / windows;
1152 
1153 	scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p);
1154 	return max_t(unsigned int, floor, scan);
1155 }
1156 
1157 static unsigned int task_scan_start(struct task_struct *p)
1158 {
1159 	unsigned long smin = task_scan_min(p);
1160 	unsigned long period = smin;
1161 	struct numa_group *ng;
1162 
1163 	/* Scale the maximum scan period with the amount of shared memory. */
1164 	rcu_read_lock();
1165 	ng = rcu_dereference(p->numa_group);
1166 	if (ng) {
1167 		unsigned long shared = group_faults_shared(ng);
1168 		unsigned long private = group_faults_priv(ng);
1169 
1170 		period *= refcount_read(&ng->refcount);
1171 		period *= shared + 1;
1172 		period /= private + shared + 1;
1173 	}
1174 	rcu_read_unlock();
1175 
1176 	return max(smin, period);
1177 }
1178 
1179 static unsigned int task_scan_max(struct task_struct *p)
1180 {
1181 	unsigned long smin = task_scan_min(p);
1182 	unsigned long smax;
1183 	struct numa_group *ng;
1184 
1185 	/* Watch for min being lower than max due to floor calculations */
1186 	smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);
1187 
1188 	/* Scale the maximum scan period with the amount of shared memory. */
1189 	ng = deref_curr_numa_group(p);
1190 	if (ng) {
1191 		unsigned long shared = group_faults_shared(ng);
1192 		unsigned long private = group_faults_priv(ng);
1193 		unsigned long period = smax;
1194 
1195 		period *= refcount_read(&ng->refcount);
1196 		period *= shared + 1;
1197 		period /= private + shared + 1;
1198 
1199 		smax = max(smax, period);
1200 	}
1201 
1202 	return max(smin, smax);
1203 }
1204 
1205 static void account_numa_enqueue(struct rq *rq, struct task_struct *p)
1206 {
1207 	rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE);
1208 	rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p));
1209 }
1210 
1211 static void account_numa_dequeue(struct rq *rq, struct task_struct *p)
1212 {
1213 	rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE);
1214 	rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));
1215 }
1216 
1217 /* Shared or private faults. */
1218 #define NR_NUMA_HINT_FAULT_TYPES 2
1219 
1220 /* Memory and CPU locality */
1221 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2)
1222 
1223 /* Averaged statistics, and temporary buffers. */
1224 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2)
1225 
1226 pid_t task_numa_group_id(struct task_struct *p)
1227 {
1228 	struct numa_group *ng;
1229 	pid_t gid = 0;
1230 
1231 	rcu_read_lock();
1232 	ng = rcu_dereference(p->numa_group);
1233 	if (ng)
1234 		gid = ng->gid;
1235 	rcu_read_unlock();
1236 
1237 	return gid;
1238 }
1239 
1240 /*
1241  * The averaged statistics, shared & private, memory & CPU,
1242  * occupy the first half of the array. The second half of the
1243  * array is for current counters, which are averaged into the
1244  * first set by task_numa_placement.
1245  */
1246 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv)
1247 {
1248 	return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv;
1249 }
1250 
1251 static inline unsigned long task_faults(struct task_struct *p, int nid)
1252 {
1253 	if (!p->numa_faults)
1254 		return 0;
1255 
1256 	return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1257 		p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)];
1258 }
1259 
1260 static inline unsigned long group_faults(struct task_struct *p, int nid)
1261 {
1262 	struct numa_group *ng = deref_task_numa_group(p);
1263 
1264 	if (!ng)
1265 		return 0;
1266 
1267 	return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1268 		ng->faults[task_faults_idx(NUMA_MEM, nid, 1)];
1269 }
1270 
1271 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)
1272 {
1273 	return group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 0)] +
1274 		group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)];
1275 }
1276 
1277 static inline unsigned long group_faults_priv(struct numa_group *ng)
1278 {
1279 	unsigned long faults = 0;
1280 	int node;
1281 
1282 	for_each_online_node(node) {
1283 		faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
1284 	}
1285 
1286 	return faults;
1287 }
1288 
1289 static inline unsigned long group_faults_shared(struct numa_group *ng)
1290 {
1291 	unsigned long faults = 0;
1292 	int node;
1293 
1294 	for_each_online_node(node) {
1295 		faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)];
1296 	}
1297 
1298 	return faults;
1299 }
1300 
1301 /*
1302  * A node triggering more than 1/3 as many NUMA faults as the maximum is
1303  * considered part of a numa group's pseudo-interleaving set. Migrations
1304  * between these nodes are slowed down, to allow things to settle down.
1305  */
1306 #define ACTIVE_NODE_FRACTION 3
1307 
1308 static bool numa_is_active_node(int nid, struct numa_group *ng)
1309 {
1310 	return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu;
1311 }
1312 
1313 /* Handle placement on systems where not all nodes are directly connected. */
1314 static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
1315 					int maxdist, bool task)
1316 {
1317 	unsigned long score = 0;
1318 	int node;
1319 
1320 	/*
1321 	 * All nodes are directly connected, and the same distance
1322 	 * from each other. No need for fancy placement algorithms.
1323 	 */
1324 	if (sched_numa_topology_type == NUMA_DIRECT)
1325 		return 0;
1326 
1327 	/*
1328 	 * This code is called for each node, introducing N^2 complexity,
1329 	 * which should be ok given the number of nodes rarely exceeds 8.
1330 	 */
1331 	for_each_online_node(node) {
1332 		unsigned long faults;
1333 		int dist = node_distance(nid, node);
1334 
1335 		/*
1336 		 * The furthest away nodes in the system are not interesting
1337 		 * for placement; nid was already counted.
1338 		 */
1339 		if (dist == sched_max_numa_distance || node == nid)
1340 			continue;
1341 
1342 		/*
1343 		 * On systems with a backplane NUMA topology, compare groups
1344 		 * of nodes, and move tasks towards the group with the most
1345 		 * memory accesses. When comparing two nodes at distance
1346 		 * "hoplimit", only nodes closer by than "hoplimit" are part
1347 		 * of each group. Skip other nodes.
1348 		 */
1349 		if (sched_numa_topology_type == NUMA_BACKPLANE &&
1350 					dist >= maxdist)
1351 			continue;
1352 
1353 		/* Add up the faults from nearby nodes. */
1354 		if (task)
1355 			faults = task_faults(p, node);
1356 		else
1357 			faults = group_faults(p, node);
1358 
1359 		/*
1360 		 * On systems with a glueless mesh NUMA topology, there are
1361 		 * no fixed "groups of nodes". Instead, nodes that are not
1362 		 * directly connected bounce traffic through intermediate
1363 		 * nodes; a numa_group can occupy any set of nodes.
1364 		 * The further away a node is, the less the faults count.
1365 		 * This seems to result in good task placement.
1366 		 */
1367 		if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
1368 			faults *= (sched_max_numa_distance - dist);
1369 			faults /= (sched_max_numa_distance - LOCAL_DISTANCE);
1370 		}
1371 
1372 		score += faults;
1373 	}
1374 
1375 	return score;
1376 }
1377 
1378 /*
1379  * These return the fraction of accesses done by a particular task, or
1380  * task group, on a particular numa node.  The group weight is given a
1381  * larger multiplier, in order to group tasks together that are almost
1382  * evenly spread out between numa nodes.
1383  */
1384 static inline unsigned long task_weight(struct task_struct *p, int nid,
1385 					int dist)
1386 {
1387 	unsigned long faults, total_faults;
1388 
1389 	if (!p->numa_faults)
1390 		return 0;
1391 
1392 	total_faults = p->total_numa_faults;
1393 
1394 	if (!total_faults)
1395 		return 0;
1396 
1397 	faults = task_faults(p, nid);
1398 	faults += score_nearby_nodes(p, nid, dist, true);
1399 
1400 	return 1000 * faults / total_faults;
1401 }
1402 
1403 static inline unsigned long group_weight(struct task_struct *p, int nid,
1404 					 int dist)
1405 {
1406 	struct numa_group *ng = deref_task_numa_group(p);
1407 	unsigned long faults, total_faults;
1408 
1409 	if (!ng)
1410 		return 0;
1411 
1412 	total_faults = ng->total_faults;
1413 
1414 	if (!total_faults)
1415 		return 0;
1416 
1417 	faults = group_faults(p, nid);
1418 	faults += score_nearby_nodes(p, nid, dist, false);
1419 
1420 	return 1000 * faults / total_faults;
1421 }
1422 
1423 bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
1424 				int src_nid, int dst_cpu)
1425 {
1426 	struct numa_group *ng = deref_curr_numa_group(p);
1427 	int dst_nid = cpu_to_node(dst_cpu);
1428 	int last_cpupid, this_cpupid;
1429 
1430 	this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
1431 	last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
1432 
1433 	/*
1434 	 * Allow first faults or private faults to migrate immediately early in
1435 	 * the lifetime of a task. The magic number 4 is based on waiting for
1436 	 * two full passes of the "multi-stage node selection" test that is
1437 	 * executed below.
1438 	 */
1439 	if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) &&
1440 	    (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid)))
1441 		return true;
1442 
1443 	/*
1444 	 * Multi-stage node selection is used in conjunction with a periodic
1445 	 * migration fault to build a temporal task<->page relation. By using
1446 	 * a two-stage filter we remove short/unlikely relations.
1447 	 *
1448 	 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate
1449 	 * a task's usage of a particular page (n_p) per total usage of this
1450 	 * page (n_t) (in a given time-span) to a probability.
1451 	 *
1452 	 * Our periodic faults will sample this probability and getting the
1453 	 * same result twice in a row, given these samples are fully
1454 	 * independent, is then given by P(n)^2, provided our sample period
1455 	 * is sufficiently short compared to the usage pattern.
1456 	 *
1457 	 * This quadric squishes small probabilities, making it less likely we
1458 	 * act on an unlikely task<->page relation.
1459 	 */
1460 	if (!cpupid_pid_unset(last_cpupid) &&
1461 				cpupid_to_nid(last_cpupid) != dst_nid)
1462 		return false;
1463 
1464 	/* Always allow migrate on private faults */
1465 	if (cpupid_match_pid(p, last_cpupid))
1466 		return true;
1467 
1468 	/* A shared fault, but p->numa_group has not been set up yet. */
1469 	if (!ng)
1470 		return true;
1471 
1472 	/*
1473 	 * Destination node is much more heavily used than the source
1474 	 * node? Allow migration.
1475 	 */
1476 	if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) *
1477 					ACTIVE_NODE_FRACTION)
1478 		return true;
1479 
1480 	/*
1481 	 * Distribute memory according to CPU & memory use on each node,
1482 	 * with 3/4 hysteresis to avoid unnecessary memory migrations:
1483 	 *
1484 	 * faults_cpu(dst)   3   faults_cpu(src)
1485 	 * --------------- * - > ---------------
1486 	 * faults_mem(dst)   4   faults_mem(src)
1487 	 */
1488 	return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 >
1489 	       group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4;
1490 }
1491 
1492 /*
1493  * 'numa_type' describes the node at the moment of load balancing.
1494  */
1495 enum numa_type {
1496 	/* The node has spare capacity that can be used to run more tasks.  */
1497 	node_has_spare = 0,
1498 	/*
1499 	 * The node is fully used and the tasks don't compete for more CPU
1500 	 * cycles. Nevertheless, some tasks might wait before running.
1501 	 */
1502 	node_fully_busy,
1503 	/*
1504 	 * The node is overloaded and can't provide expected CPU cycles to all
1505 	 * tasks.
1506 	 */
1507 	node_overloaded
1508 };
1509 
1510 /* Cached statistics for all CPUs within a node */
1511 struct numa_stats {
1512 	unsigned long load;
1513 	unsigned long runnable;
1514 	unsigned long util;
1515 	/* Total compute capacity of CPUs on a node */
1516 	unsigned long compute_capacity;
1517 	unsigned int nr_running;
1518 	unsigned int weight;
1519 	enum numa_type node_type;
1520 	int idle_cpu;
1521 };
1522 
1523 static inline bool is_core_idle(int cpu)
1524 {
1525 #ifdef CONFIG_SCHED_SMT
1526 	int sibling;
1527 
1528 	for_each_cpu(sibling, cpu_smt_mask(cpu)) {
1529 		if (cpu == sibling)
1530 			continue;
1531 
1532 		if (!idle_cpu(cpu))
1533 			return false;
1534 	}
1535 #endif
1536 
1537 	return true;
1538 }
1539 
1540 struct task_numa_env {
1541 	struct task_struct *p;
1542 
1543 	int src_cpu, src_nid;
1544 	int dst_cpu, dst_nid;
1545 
1546 	struct numa_stats src_stats, dst_stats;
1547 
1548 	int imbalance_pct;
1549 	int dist;
1550 
1551 	struct task_struct *best_task;
1552 	long best_imp;
1553 	int best_cpu;
1554 };
1555 
1556 static unsigned long cpu_load(struct rq *rq);
1557 static unsigned long cpu_runnable(struct rq *rq);
1558 static unsigned long cpu_util(int cpu);
1559 static inline long adjust_numa_imbalance(int imbalance,
1560 					int dst_running, int dst_weight);
1561 
1562 static inline enum
1563 numa_type numa_classify(unsigned int imbalance_pct,
1564 			 struct numa_stats *ns)
1565 {
1566 	if ((ns->nr_running > ns->weight) &&
1567 	    (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) ||
1568 	     ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100))))
1569 		return node_overloaded;
1570 
1571 	if ((ns->nr_running < ns->weight) ||
1572 	    (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) &&
1573 	     ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100))))
1574 		return node_has_spare;
1575 
1576 	return node_fully_busy;
1577 }
1578 
1579 #ifdef CONFIG_SCHED_SMT
1580 /* Forward declarations of select_idle_sibling helpers */
1581 static inline bool test_idle_cores(int cpu, bool def);
1582 static inline int numa_idle_core(int idle_core, int cpu)
1583 {
1584 	if (!static_branch_likely(&sched_smt_present) ||
1585 	    idle_core >= 0 || !test_idle_cores(cpu, false))
1586 		return idle_core;
1587 
1588 	/*
1589 	 * Prefer cores instead of packing HT siblings
1590 	 * and triggering future load balancing.
1591 	 */
1592 	if (is_core_idle(cpu))
1593 		idle_core = cpu;
1594 
1595 	return idle_core;
1596 }
1597 #else
1598 static inline int numa_idle_core(int idle_core, int cpu)
1599 {
1600 	return idle_core;
1601 }
1602 #endif
1603 
1604 /*
1605  * Gather all necessary information to make NUMA balancing placement
1606  * decisions that are compatible with standard load balancer. This
1607  * borrows code and logic from update_sg_lb_stats but sharing a
1608  * common implementation is impractical.
1609  */
1610 static void update_numa_stats(struct task_numa_env *env,
1611 			      struct numa_stats *ns, int nid,
1612 			      bool find_idle)
1613 {
1614 	int cpu, idle_core = -1;
1615 
1616 	memset(ns, 0, sizeof(*ns));
1617 	ns->idle_cpu = -1;
1618 
1619 	rcu_read_lock();
1620 	for_each_cpu(cpu, cpumask_of_node(nid)) {
1621 		struct rq *rq = cpu_rq(cpu);
1622 
1623 		ns->load += cpu_load(rq);
1624 		ns->runnable += cpu_runnable(rq);
1625 		ns->util += cpu_util(cpu);
1626 		ns->nr_running += rq->cfs.h_nr_running;
1627 		ns->compute_capacity += capacity_of(cpu);
1628 
1629 		if (find_idle && !rq->nr_running && idle_cpu(cpu)) {
1630 			if (READ_ONCE(rq->numa_migrate_on) ||
1631 			    !cpumask_test_cpu(cpu, env->p->cpus_ptr))
1632 				continue;
1633 
1634 			if (ns->idle_cpu == -1)
1635 				ns->idle_cpu = cpu;
1636 
1637 			idle_core = numa_idle_core(idle_core, cpu);
1638 		}
1639 	}
1640 	rcu_read_unlock();
1641 
1642 	ns->weight = cpumask_weight(cpumask_of_node(nid));
1643 
1644 	ns->node_type = numa_classify(env->imbalance_pct, ns);
1645 
1646 	if (idle_core >= 0)
1647 		ns->idle_cpu = idle_core;
1648 }
1649 
1650 static void task_numa_assign(struct task_numa_env *env,
1651 			     struct task_struct *p, long imp)
1652 {
1653 	struct rq *rq = cpu_rq(env->dst_cpu);
1654 
1655 	/* Check if run-queue part of active NUMA balance. */
1656 	if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) {
1657 		int cpu;
1658 		int start = env->dst_cpu;
1659 
1660 		/* Find alternative idle CPU. */
1661 		for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start) {
1662 			if (cpu == env->best_cpu || !idle_cpu(cpu) ||
1663 			    !cpumask_test_cpu(cpu, env->p->cpus_ptr)) {
1664 				continue;
1665 			}
1666 
1667 			env->dst_cpu = cpu;
1668 			rq = cpu_rq(env->dst_cpu);
1669 			if (!xchg(&rq->numa_migrate_on, 1))
1670 				goto assign;
1671 		}
1672 
1673 		/* Failed to find an alternative idle CPU */
1674 		return;
1675 	}
1676 
1677 assign:
1678 	/*
1679 	 * Clear previous best_cpu/rq numa-migrate flag, since task now
1680 	 * found a better CPU to move/swap.
1681 	 */
1682 	if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) {
1683 		rq = cpu_rq(env->best_cpu);
1684 		WRITE_ONCE(rq->numa_migrate_on, 0);
1685 	}
1686 
1687 	if (env->best_task)
1688 		put_task_struct(env->best_task);
1689 	if (p)
1690 		get_task_struct(p);
1691 
1692 	env->best_task = p;
1693 	env->best_imp = imp;
1694 	env->best_cpu = env->dst_cpu;
1695 }
1696 
1697 static bool load_too_imbalanced(long src_load, long dst_load,
1698 				struct task_numa_env *env)
1699 {
1700 	long imb, old_imb;
1701 	long orig_src_load, orig_dst_load;
1702 	long src_capacity, dst_capacity;
1703 
1704 	/*
1705 	 * The load is corrected for the CPU capacity available on each node.
1706 	 *
1707 	 * src_load        dst_load
1708 	 * ------------ vs ---------
1709 	 * src_capacity    dst_capacity
1710 	 */
1711 	src_capacity = env->src_stats.compute_capacity;
1712 	dst_capacity = env->dst_stats.compute_capacity;
1713 
1714 	imb = abs(dst_load * src_capacity - src_load * dst_capacity);
1715 
1716 	orig_src_load = env->src_stats.load;
1717 	orig_dst_load = env->dst_stats.load;
1718 
1719 	old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity);
1720 
1721 	/* Would this change make things worse? */
1722 	return (imb > old_imb);
1723 }
1724 
1725 /*
1726  * Maximum NUMA importance can be 1998 (2*999);
1727  * SMALLIMP @ 30 would be close to 1998/64.
1728  * Used to deter task migration.
1729  */
1730 #define SMALLIMP	30
1731 
1732 /*
1733  * This checks if the overall compute and NUMA accesses of the system would
1734  * be improved if the source tasks was migrated to the target dst_cpu taking
1735  * into account that it might be best if task running on the dst_cpu should
1736  * be exchanged with the source task
1737  */
1738 static bool task_numa_compare(struct task_numa_env *env,
1739 			      long taskimp, long groupimp, bool maymove)
1740 {
1741 	struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p);
1742 	struct rq *dst_rq = cpu_rq(env->dst_cpu);
1743 	long imp = p_ng ? groupimp : taskimp;
1744 	struct task_struct *cur;
1745 	long src_load, dst_load;
1746 	int dist = env->dist;
1747 	long moveimp = imp;
1748 	long load;
1749 	bool stopsearch = false;
1750 
1751 	if (READ_ONCE(dst_rq->numa_migrate_on))
1752 		return false;
1753 
1754 	rcu_read_lock();
1755 	cur = rcu_dereference(dst_rq->curr);
1756 	if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur)))
1757 		cur = NULL;
1758 
1759 	/*
1760 	 * Because we have preemption enabled we can get migrated around and
1761 	 * end try selecting ourselves (current == env->p) as a swap candidate.
1762 	 */
1763 	if (cur == env->p) {
1764 		stopsearch = true;
1765 		goto unlock;
1766 	}
1767 
1768 	if (!cur) {
1769 		if (maymove && moveimp >= env->best_imp)
1770 			goto assign;
1771 		else
1772 			goto unlock;
1773 	}
1774 
1775 	/* Skip this swap candidate if cannot move to the source cpu. */
1776 	if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr))
1777 		goto unlock;
1778 
1779 	/*
1780 	 * Skip this swap candidate if it is not moving to its preferred
1781 	 * node and the best task is.
1782 	 */
1783 	if (env->best_task &&
1784 	    env->best_task->numa_preferred_nid == env->src_nid &&
1785 	    cur->numa_preferred_nid != env->src_nid) {
1786 		goto unlock;
1787 	}
1788 
1789 	/*
1790 	 * "imp" is the fault differential for the source task between the
1791 	 * source and destination node. Calculate the total differential for
1792 	 * the source task and potential destination task. The more negative
1793 	 * the value is, the more remote accesses that would be expected to
1794 	 * be incurred if the tasks were swapped.
1795 	 *
1796 	 * If dst and source tasks are in the same NUMA group, or not
1797 	 * in any group then look only at task weights.
1798 	 */
1799 	cur_ng = rcu_dereference(cur->numa_group);
1800 	if (cur_ng == p_ng) {
1801 		imp = taskimp + task_weight(cur, env->src_nid, dist) -
1802 		      task_weight(cur, env->dst_nid, dist);
1803 		/*
1804 		 * Add some hysteresis to prevent swapping the
1805 		 * tasks within a group over tiny differences.
1806 		 */
1807 		if (cur_ng)
1808 			imp -= imp / 16;
1809 	} else {
1810 		/*
1811 		 * Compare the group weights. If a task is all by itself
1812 		 * (not part of a group), use the task weight instead.
1813 		 */
1814 		if (cur_ng && p_ng)
1815 			imp += group_weight(cur, env->src_nid, dist) -
1816 			       group_weight(cur, env->dst_nid, dist);
1817 		else
1818 			imp += task_weight(cur, env->src_nid, dist) -
1819 			       task_weight(cur, env->dst_nid, dist);
1820 	}
1821 
1822 	/* Discourage picking a task already on its preferred node */
1823 	if (cur->numa_preferred_nid == env->dst_nid)
1824 		imp -= imp / 16;
1825 
1826 	/*
1827 	 * Encourage picking a task that moves to its preferred node.
1828 	 * This potentially makes imp larger than it's maximum of
1829 	 * 1998 (see SMALLIMP and task_weight for why) but in this
1830 	 * case, it does not matter.
1831 	 */
1832 	if (cur->numa_preferred_nid == env->src_nid)
1833 		imp += imp / 8;
1834 
1835 	if (maymove && moveimp > imp && moveimp > env->best_imp) {
1836 		imp = moveimp;
1837 		cur = NULL;
1838 		goto assign;
1839 	}
1840 
1841 	/*
1842 	 * Prefer swapping with a task moving to its preferred node over a
1843 	 * task that is not.
1844 	 */
1845 	if (env->best_task && cur->numa_preferred_nid == env->src_nid &&
1846 	    env->best_task->numa_preferred_nid != env->src_nid) {
1847 		goto assign;
1848 	}
1849 
1850 	/*
1851 	 * If the NUMA importance is less than SMALLIMP,
1852 	 * task migration might only result in ping pong
1853 	 * of tasks and also hurt performance due to cache
1854 	 * misses.
1855 	 */
1856 	if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2)
1857 		goto unlock;
1858 
1859 	/*
1860 	 * In the overloaded case, try and keep the load balanced.
1861 	 */
1862 	load = task_h_load(env->p) - task_h_load(cur);
1863 	if (!load)
1864 		goto assign;
1865 
1866 	dst_load = env->dst_stats.load + load;
1867 	src_load = env->src_stats.load - load;
1868 
1869 	if (load_too_imbalanced(src_load, dst_load, env))
1870 		goto unlock;
1871 
1872 assign:
1873 	/* Evaluate an idle CPU for a task numa move. */
1874 	if (!cur) {
1875 		int cpu = env->dst_stats.idle_cpu;
1876 
1877 		/* Nothing cached so current CPU went idle since the search. */
1878 		if (cpu < 0)
1879 			cpu = env->dst_cpu;
1880 
1881 		/*
1882 		 * If the CPU is no longer truly idle and the previous best CPU
1883 		 * is, keep using it.
1884 		 */
1885 		if (!idle_cpu(cpu) && env->best_cpu >= 0 &&
1886 		    idle_cpu(env->best_cpu)) {
1887 			cpu = env->best_cpu;
1888 		}
1889 
1890 		env->dst_cpu = cpu;
1891 	}
1892 
1893 	task_numa_assign(env, cur, imp);
1894 
1895 	/*
1896 	 * If a move to idle is allowed because there is capacity or load
1897 	 * balance improves then stop the search. While a better swap
1898 	 * candidate may exist, a search is not free.
1899 	 */
1900 	if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu))
1901 		stopsearch = true;
1902 
1903 	/*
1904 	 * If a swap candidate must be identified and the current best task
1905 	 * moves its preferred node then stop the search.
1906 	 */
1907 	if (!maymove && env->best_task &&
1908 	    env->best_task->numa_preferred_nid == env->src_nid) {
1909 		stopsearch = true;
1910 	}
1911 unlock:
1912 	rcu_read_unlock();
1913 
1914 	return stopsearch;
1915 }
1916 
1917 static void task_numa_find_cpu(struct task_numa_env *env,
1918 				long taskimp, long groupimp)
1919 {
1920 	bool maymove = false;
1921 	int cpu;
1922 
1923 	/*
1924 	 * If dst node has spare capacity, then check if there is an
1925 	 * imbalance that would be overruled by the load balancer.
1926 	 */
1927 	if (env->dst_stats.node_type == node_has_spare) {
1928 		unsigned int imbalance;
1929 		int src_running, dst_running;
1930 
1931 		/*
1932 		 * Would movement cause an imbalance? Note that if src has
1933 		 * more running tasks that the imbalance is ignored as the
1934 		 * move improves the imbalance from the perspective of the
1935 		 * CPU load balancer.
1936 		 * */
1937 		src_running = env->src_stats.nr_running - 1;
1938 		dst_running = env->dst_stats.nr_running + 1;
1939 		imbalance = max(0, dst_running - src_running);
1940 		imbalance = adjust_numa_imbalance(imbalance, dst_running,
1941 							env->dst_stats.weight);
1942 
1943 		/* Use idle CPU if there is no imbalance */
1944 		if (!imbalance) {
1945 			maymove = true;
1946 			if (env->dst_stats.idle_cpu >= 0) {
1947 				env->dst_cpu = env->dst_stats.idle_cpu;
1948 				task_numa_assign(env, NULL, 0);
1949 				return;
1950 			}
1951 		}
1952 	} else {
1953 		long src_load, dst_load, load;
1954 		/*
1955 		 * If the improvement from just moving env->p direction is better
1956 		 * than swapping tasks around, check if a move is possible.
1957 		 */
1958 		load = task_h_load(env->p);
1959 		dst_load = env->dst_stats.load + load;
1960 		src_load = env->src_stats.load - load;
1961 		maymove = !load_too_imbalanced(src_load, dst_load, env);
1962 	}
1963 
1964 	for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
1965 		/* Skip this CPU if the source task cannot migrate */
1966 		if (!cpumask_test_cpu(cpu, env->p->cpus_ptr))
1967 			continue;
1968 
1969 		env->dst_cpu = cpu;
1970 		if (task_numa_compare(env, taskimp, groupimp, maymove))
1971 			break;
1972 	}
1973 }
1974 
1975 static int task_numa_migrate(struct task_struct *p)
1976 {
1977 	struct task_numa_env env = {
1978 		.p = p,
1979 
1980 		.src_cpu = task_cpu(p),
1981 		.src_nid = task_node(p),
1982 
1983 		.imbalance_pct = 112,
1984 
1985 		.best_task = NULL,
1986 		.best_imp = 0,
1987 		.best_cpu = -1,
1988 	};
1989 	unsigned long taskweight, groupweight;
1990 	struct sched_domain *sd;
1991 	long taskimp, groupimp;
1992 	struct numa_group *ng;
1993 	struct rq *best_rq;
1994 	int nid, ret, dist;
1995 
1996 	/*
1997 	 * Pick the lowest SD_NUMA domain, as that would have the smallest
1998 	 * imbalance and would be the first to start moving tasks about.
1999 	 *
2000 	 * And we want to avoid any moving of tasks about, as that would create
2001 	 * random movement of tasks -- counter the numa conditions we're trying
2002 	 * to satisfy here.
2003 	 */
2004 	rcu_read_lock();
2005 	sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
2006 	if (sd)
2007 		env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
2008 	rcu_read_unlock();
2009 
2010 	/*
2011 	 * Cpusets can break the scheduler domain tree into smaller
2012 	 * balance domains, some of which do not cross NUMA boundaries.
2013 	 * Tasks that are "trapped" in such domains cannot be migrated
2014 	 * elsewhere, so there is no point in (re)trying.
2015 	 */
2016 	if (unlikely(!sd)) {
2017 		sched_setnuma(p, task_node(p));
2018 		return -EINVAL;
2019 	}
2020 
2021 	env.dst_nid = p->numa_preferred_nid;
2022 	dist = env.dist = node_distance(env.src_nid, env.dst_nid);
2023 	taskweight = task_weight(p, env.src_nid, dist);
2024 	groupweight = group_weight(p, env.src_nid, dist);
2025 	update_numa_stats(&env, &env.src_stats, env.src_nid, false);
2026 	taskimp = task_weight(p, env.dst_nid, dist) - taskweight;
2027 	groupimp = group_weight(p, env.dst_nid, dist) - groupweight;
2028 	update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
2029 
2030 	/* Try to find a spot on the preferred nid. */
2031 	task_numa_find_cpu(&env, taskimp, groupimp);
2032 
2033 	/*
2034 	 * Look at other nodes in these cases:
2035 	 * - there is no space available on the preferred_nid
2036 	 * - the task is part of a numa_group that is interleaved across
2037 	 *   multiple NUMA nodes; in order to better consolidate the group,
2038 	 *   we need to check other locations.
2039 	 */
2040 	ng = deref_curr_numa_group(p);
2041 	if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) {
2042 		for_each_online_node(nid) {
2043 			if (nid == env.src_nid || nid == p->numa_preferred_nid)
2044 				continue;
2045 
2046 			dist = node_distance(env.src_nid, env.dst_nid);
2047 			if (sched_numa_topology_type == NUMA_BACKPLANE &&
2048 						dist != env.dist) {
2049 				taskweight = task_weight(p, env.src_nid, dist);
2050 				groupweight = group_weight(p, env.src_nid, dist);
2051 			}
2052 
2053 			/* Only consider nodes where both task and groups benefit */
2054 			taskimp = task_weight(p, nid, dist) - taskweight;
2055 			groupimp = group_weight(p, nid, dist) - groupweight;
2056 			if (taskimp < 0 && groupimp < 0)
2057 				continue;
2058 
2059 			env.dist = dist;
2060 			env.dst_nid = nid;
2061 			update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
2062 			task_numa_find_cpu(&env, taskimp, groupimp);
2063 		}
2064 	}
2065 
2066 	/*
2067 	 * If the task is part of a workload that spans multiple NUMA nodes,
2068 	 * and is migrating into one of the workload's active nodes, remember
2069 	 * this node as the task's preferred numa node, so the workload can
2070 	 * settle down.
2071 	 * A task that migrated to a second choice node will be better off
2072 	 * trying for a better one later. Do not set the preferred node here.
2073 	 */
2074 	if (ng) {
2075 		if (env.best_cpu == -1)
2076 			nid = env.src_nid;
2077 		else
2078 			nid = cpu_to_node(env.best_cpu);
2079 
2080 		if (nid != p->numa_preferred_nid)
2081 			sched_setnuma(p, nid);
2082 	}
2083 
2084 	/* No better CPU than the current one was found. */
2085 	if (env.best_cpu == -1) {
2086 		trace_sched_stick_numa(p, env.src_cpu, NULL, -1);
2087 		return -EAGAIN;
2088 	}
2089 
2090 	best_rq = cpu_rq(env.best_cpu);
2091 	if (env.best_task == NULL) {
2092 		ret = migrate_task_to(p, env.best_cpu);
2093 		WRITE_ONCE(best_rq->numa_migrate_on, 0);
2094 		if (ret != 0)
2095 			trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu);
2096 		return ret;
2097 	}
2098 
2099 	ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu);
2100 	WRITE_ONCE(best_rq->numa_migrate_on, 0);
2101 
2102 	if (ret != 0)
2103 		trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu);
2104 	put_task_struct(env.best_task);
2105 	return ret;
2106 }
2107 
2108 /* Attempt to migrate a task to a CPU on the preferred node. */
2109 static void numa_migrate_preferred(struct task_struct *p)
2110 {
2111 	unsigned long interval = HZ;
2112 
2113 	/* This task has no NUMA fault statistics yet */
2114 	if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults))
2115 		return;
2116 
2117 	/* Periodically retry migrating the task to the preferred node */
2118 	interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16);
2119 	p->numa_migrate_retry = jiffies + interval;
2120 
2121 	/* Success if task is already running on preferred CPU */
2122 	if (task_node(p) == p->numa_preferred_nid)
2123 		return;
2124 
2125 	/* Otherwise, try migrate to a CPU on the preferred node */
2126 	task_numa_migrate(p);
2127 }
2128 
2129 /*
2130  * Find out how many nodes on the workload is actively running on. Do this by
2131  * tracking the nodes from which NUMA hinting faults are triggered. This can
2132  * be different from the set of nodes where the workload's memory is currently
2133  * located.
2134  */
2135 static void numa_group_count_active_nodes(struct numa_group *numa_group)
2136 {
2137 	unsigned long faults, max_faults = 0;
2138 	int nid, active_nodes = 0;
2139 
2140 	for_each_online_node(nid) {
2141 		faults = group_faults_cpu(numa_group, nid);
2142 		if (faults > max_faults)
2143 			max_faults = faults;
2144 	}
2145 
2146 	for_each_online_node(nid) {
2147 		faults = group_faults_cpu(numa_group, nid);
2148 		if (faults * ACTIVE_NODE_FRACTION > max_faults)
2149 			active_nodes++;
2150 	}
2151 
2152 	numa_group->max_faults_cpu = max_faults;
2153 	numa_group->active_nodes = active_nodes;
2154 }
2155 
2156 /*
2157  * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS
2158  * increments. The more local the fault statistics are, the higher the scan
2159  * period will be for the next scan window. If local/(local+remote) ratio is
2160  * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS)
2161  * the scan period will decrease. Aim for 70% local accesses.
2162  */
2163 #define NUMA_PERIOD_SLOTS 10
2164 #define NUMA_PERIOD_THRESHOLD 7
2165 
2166 /*
2167  * Increase the scan period (slow down scanning) if the majority of
2168  * our memory is already on our local node, or if the majority of
2169  * the page accesses are shared with other processes.
2170  * Otherwise, decrease the scan period.
2171  */
2172 static void update_task_scan_period(struct task_struct *p,
2173 			unsigned long shared, unsigned long private)
2174 {
2175 	unsigned int period_slot;
2176 	int lr_ratio, ps_ratio;
2177 	int diff;
2178 
2179 	unsigned long remote = p->numa_faults_locality[0];
2180 	unsigned long local = p->numa_faults_locality[1];
2181 
2182 	/*
2183 	 * If there were no record hinting faults then either the task is
2184 	 * completely idle or all activity is areas that are not of interest
2185 	 * to automatic numa balancing. Related to that, if there were failed
2186 	 * migration then it implies we are migrating too quickly or the local
2187 	 * node is overloaded. In either case, scan slower
2188 	 */
2189 	if (local + shared == 0 || p->numa_faults_locality[2]) {
2190 		p->numa_scan_period = min(p->numa_scan_period_max,
2191 			p->numa_scan_period << 1);
2192 
2193 		p->mm->numa_next_scan = jiffies +
2194 			msecs_to_jiffies(p->numa_scan_period);
2195 
2196 		return;
2197 	}
2198 
2199 	/*
2200 	 * Prepare to scale scan period relative to the current period.
2201 	 *	 == NUMA_PERIOD_THRESHOLD scan period stays the same
2202 	 *       <  NUMA_PERIOD_THRESHOLD scan period decreases (scan faster)
2203 	 *	 >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower)
2204 	 */
2205 	period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS);
2206 	lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote);
2207 	ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared);
2208 
2209 	if (ps_ratio >= NUMA_PERIOD_THRESHOLD) {
2210 		/*
2211 		 * Most memory accesses are local. There is no need to
2212 		 * do fast NUMA scanning, since memory is already local.
2213 		 */
2214 		int slot = ps_ratio - NUMA_PERIOD_THRESHOLD;
2215 		if (!slot)
2216 			slot = 1;
2217 		diff = slot * period_slot;
2218 	} else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) {
2219 		/*
2220 		 * Most memory accesses are shared with other tasks.
2221 		 * There is no point in continuing fast NUMA scanning,
2222 		 * since other tasks may just move the memory elsewhere.
2223 		 */
2224 		int slot = lr_ratio - NUMA_PERIOD_THRESHOLD;
2225 		if (!slot)
2226 			slot = 1;
2227 		diff = slot * period_slot;
2228 	} else {
2229 		/*
2230 		 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS,
2231 		 * yet they are not on the local NUMA node. Speed up
2232 		 * NUMA scanning to get the memory moved over.
2233 		 */
2234 		int ratio = max(lr_ratio, ps_ratio);
2235 		diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot;
2236 	}
2237 
2238 	p->numa_scan_period = clamp(p->numa_scan_period + diff,
2239 			task_scan_min(p), task_scan_max(p));
2240 	memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2241 }
2242 
2243 /*
2244  * Get the fraction of time the task has been running since the last
2245  * NUMA placement cycle. The scheduler keeps similar statistics, but
2246  * decays those on a 32ms period, which is orders of magnitude off
2247  * from the dozens-of-seconds NUMA balancing period. Use the scheduler
2248  * stats only if the task is so new there are no NUMA statistics yet.
2249  */
2250 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period)
2251 {
2252 	u64 runtime, delta, now;
2253 	/* Use the start of this time slice to avoid calculations. */
2254 	now = p->se.exec_start;
2255 	runtime = p->se.sum_exec_runtime;
2256 
2257 	if (p->last_task_numa_placement) {
2258 		delta = runtime - p->last_sum_exec_runtime;
2259 		*period = now - p->last_task_numa_placement;
2260 
2261 		/* Avoid time going backwards, prevent potential divide error: */
2262 		if (unlikely((s64)*period < 0))
2263 			*period = 0;
2264 	} else {
2265 		delta = p->se.avg.load_sum;
2266 		*period = LOAD_AVG_MAX;
2267 	}
2268 
2269 	p->last_sum_exec_runtime = runtime;
2270 	p->last_task_numa_placement = now;
2271 
2272 	return delta;
2273 }
2274 
2275 /*
2276  * Determine the preferred nid for a task in a numa_group. This needs to
2277  * be done in a way that produces consistent results with group_weight,
2278  * otherwise workloads might not converge.
2279  */
2280 static int preferred_group_nid(struct task_struct *p, int nid)
2281 {
2282 	nodemask_t nodes;
2283 	int dist;
2284 
2285 	/* Direct connections between all NUMA nodes. */
2286 	if (sched_numa_topology_type == NUMA_DIRECT)
2287 		return nid;
2288 
2289 	/*
2290 	 * On a system with glueless mesh NUMA topology, group_weight
2291 	 * scores nodes according to the number of NUMA hinting faults on
2292 	 * both the node itself, and on nearby nodes.
2293 	 */
2294 	if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
2295 		unsigned long score, max_score = 0;
2296 		int node, max_node = nid;
2297 
2298 		dist = sched_max_numa_distance;
2299 
2300 		for_each_online_node(node) {
2301 			score = group_weight(p, node, dist);
2302 			if (score > max_score) {
2303 				max_score = score;
2304 				max_node = node;
2305 			}
2306 		}
2307 		return max_node;
2308 	}
2309 
2310 	/*
2311 	 * Finding the preferred nid in a system with NUMA backplane
2312 	 * interconnect topology is more involved. The goal is to locate
2313 	 * tasks from numa_groups near each other in the system, and
2314 	 * untangle workloads from different sides of the system. This requires
2315 	 * searching down the hierarchy of node groups, recursively searching
2316 	 * inside the highest scoring group of nodes. The nodemask tricks
2317 	 * keep the complexity of the search down.
2318 	 */
2319 	nodes = node_online_map;
2320 	for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) {
2321 		unsigned long max_faults = 0;
2322 		nodemask_t max_group = NODE_MASK_NONE;
2323 		int a, b;
2324 
2325 		/* Are there nodes at this distance from each other? */
2326 		if (!find_numa_distance(dist))
2327 			continue;
2328 
2329 		for_each_node_mask(a, nodes) {
2330 			unsigned long faults = 0;
2331 			nodemask_t this_group;
2332 			nodes_clear(this_group);
2333 
2334 			/* Sum group's NUMA faults; includes a==b case. */
2335 			for_each_node_mask(b, nodes) {
2336 				if (node_distance(a, b) < dist) {
2337 					faults += group_faults(p, b);
2338 					node_set(b, this_group);
2339 					node_clear(b, nodes);
2340 				}
2341 			}
2342 
2343 			/* Remember the top group. */
2344 			if (faults > max_faults) {
2345 				max_faults = faults;
2346 				max_group = this_group;
2347 				/*
2348 				 * subtle: at the smallest distance there is
2349 				 * just one node left in each "group", the
2350 				 * winner is the preferred nid.
2351 				 */
2352 				nid = a;
2353 			}
2354 		}
2355 		/* Next round, evaluate the nodes within max_group. */
2356 		if (!max_faults)
2357 			break;
2358 		nodes = max_group;
2359 	}
2360 	return nid;
2361 }
2362 
2363 static void task_numa_placement(struct task_struct *p)
2364 {
2365 	int seq, nid, max_nid = NUMA_NO_NODE;
2366 	unsigned long max_faults = 0;
2367 	unsigned long fault_types[2] = { 0, 0 };
2368 	unsigned long total_faults;
2369 	u64 runtime, period;
2370 	spinlock_t *group_lock = NULL;
2371 	struct numa_group *ng;
2372 
2373 	/*
2374 	 * The p->mm->numa_scan_seq field gets updated without
2375 	 * exclusive access. Use READ_ONCE() here to ensure
2376 	 * that the field is read in a single access:
2377 	 */
2378 	seq = READ_ONCE(p->mm->numa_scan_seq);
2379 	if (p->numa_scan_seq == seq)
2380 		return;
2381 	p->numa_scan_seq = seq;
2382 	p->numa_scan_period_max = task_scan_max(p);
2383 
2384 	total_faults = p->numa_faults_locality[0] +
2385 		       p->numa_faults_locality[1];
2386 	runtime = numa_get_avg_runtime(p, &period);
2387 
2388 	/* If the task is part of a group prevent parallel updates to group stats */
2389 	ng = deref_curr_numa_group(p);
2390 	if (ng) {
2391 		group_lock = &ng->lock;
2392 		spin_lock_irq(group_lock);
2393 	}
2394 
2395 	/* Find the node with the highest number of faults */
2396 	for_each_online_node(nid) {
2397 		/* Keep track of the offsets in numa_faults array */
2398 		int mem_idx, membuf_idx, cpu_idx, cpubuf_idx;
2399 		unsigned long faults = 0, group_faults = 0;
2400 		int priv;
2401 
2402 		for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) {
2403 			long diff, f_diff, f_weight;
2404 
2405 			mem_idx = task_faults_idx(NUMA_MEM, nid, priv);
2406 			membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv);
2407 			cpu_idx = task_faults_idx(NUMA_CPU, nid, priv);
2408 			cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv);
2409 
2410 			/* Decay existing window, copy faults since last scan */
2411 			diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2;
2412 			fault_types[priv] += p->numa_faults[membuf_idx];
2413 			p->numa_faults[membuf_idx] = 0;
2414 
2415 			/*
2416 			 * Normalize the faults_from, so all tasks in a group
2417 			 * count according to CPU use, instead of by the raw
2418 			 * number of faults. Tasks with little runtime have
2419 			 * little over-all impact on throughput, and thus their
2420 			 * faults are less important.
2421 			 */
2422 			f_weight = div64_u64(runtime << 16, period + 1);
2423 			f_weight = (f_weight * p->numa_faults[cpubuf_idx]) /
2424 				   (total_faults + 1);
2425 			f_diff = f_weight - p->numa_faults[cpu_idx] / 2;
2426 			p->numa_faults[cpubuf_idx] = 0;
2427 
2428 			p->numa_faults[mem_idx] += diff;
2429 			p->numa_faults[cpu_idx] += f_diff;
2430 			faults += p->numa_faults[mem_idx];
2431 			p->total_numa_faults += diff;
2432 			if (ng) {
2433 				/*
2434 				 * safe because we can only change our own group
2435 				 *
2436 				 * mem_idx represents the offset for a given
2437 				 * nid and priv in a specific region because it
2438 				 * is at the beginning of the numa_faults array.
2439 				 */
2440 				ng->faults[mem_idx] += diff;
2441 				ng->faults_cpu[mem_idx] += f_diff;
2442 				ng->total_faults += diff;
2443 				group_faults += ng->faults[mem_idx];
2444 			}
2445 		}
2446 
2447 		if (!ng) {
2448 			if (faults > max_faults) {
2449 				max_faults = faults;
2450 				max_nid = nid;
2451 			}
2452 		} else if (group_faults > max_faults) {
2453 			max_faults = group_faults;
2454 			max_nid = nid;
2455 		}
2456 	}
2457 
2458 	if (ng) {
2459 		numa_group_count_active_nodes(ng);
2460 		spin_unlock_irq(group_lock);
2461 		max_nid = preferred_group_nid(p, max_nid);
2462 	}
2463 
2464 	if (max_faults) {
2465 		/* Set the new preferred node */
2466 		if (max_nid != p->numa_preferred_nid)
2467 			sched_setnuma(p, max_nid);
2468 	}
2469 
2470 	update_task_scan_period(p, fault_types[0], fault_types[1]);
2471 }
2472 
2473 static inline int get_numa_group(struct numa_group *grp)
2474 {
2475 	return refcount_inc_not_zero(&grp->refcount);
2476 }
2477 
2478 static inline void put_numa_group(struct numa_group *grp)
2479 {
2480 	if (refcount_dec_and_test(&grp->refcount))
2481 		kfree_rcu(grp, rcu);
2482 }
2483 
2484 static void task_numa_group(struct task_struct *p, int cpupid, int flags,
2485 			int *priv)
2486 {
2487 	struct numa_group *grp, *my_grp;
2488 	struct task_struct *tsk;
2489 	bool join = false;
2490 	int cpu = cpupid_to_cpu(cpupid);
2491 	int i;
2492 
2493 	if (unlikely(!deref_curr_numa_group(p))) {
2494 		unsigned int size = sizeof(struct numa_group) +
2495 				    4*nr_node_ids*sizeof(unsigned long);
2496 
2497 		grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
2498 		if (!grp)
2499 			return;
2500 
2501 		refcount_set(&grp->refcount, 1);
2502 		grp->active_nodes = 1;
2503 		grp->max_faults_cpu = 0;
2504 		spin_lock_init(&grp->lock);
2505 		grp->gid = p->pid;
2506 		/* Second half of the array tracks nids where faults happen */
2507 		grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES *
2508 						nr_node_ids;
2509 
2510 		for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2511 			grp->faults[i] = p->numa_faults[i];
2512 
2513 		grp->total_faults = p->total_numa_faults;
2514 
2515 		grp->nr_tasks++;
2516 		rcu_assign_pointer(p->numa_group, grp);
2517 	}
2518 
2519 	rcu_read_lock();
2520 	tsk = READ_ONCE(cpu_rq(cpu)->curr);
2521 
2522 	if (!cpupid_match_pid(tsk, cpupid))
2523 		goto no_join;
2524 
2525 	grp = rcu_dereference(tsk->numa_group);
2526 	if (!grp)
2527 		goto no_join;
2528 
2529 	my_grp = deref_curr_numa_group(p);
2530 	if (grp == my_grp)
2531 		goto no_join;
2532 
2533 	/*
2534 	 * Only join the other group if its bigger; if we're the bigger group,
2535 	 * the other task will join us.
2536 	 */
2537 	if (my_grp->nr_tasks > grp->nr_tasks)
2538 		goto no_join;
2539 
2540 	/*
2541 	 * Tie-break on the grp address.
2542 	 */
2543 	if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp)
2544 		goto no_join;
2545 
2546 	/* Always join threads in the same process. */
2547 	if (tsk->mm == current->mm)
2548 		join = true;
2549 
2550 	/* Simple filter to avoid false positives due to PID collisions */
2551 	if (flags & TNF_SHARED)
2552 		join = true;
2553 
2554 	/* Update priv based on whether false sharing was detected */
2555 	*priv = !join;
2556 
2557 	if (join && !get_numa_group(grp))
2558 		goto no_join;
2559 
2560 	rcu_read_unlock();
2561 
2562 	if (!join)
2563 		return;
2564 
2565 	BUG_ON(irqs_disabled());
2566 	double_lock_irq(&my_grp->lock, &grp->lock);
2567 
2568 	for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) {
2569 		my_grp->faults[i] -= p->numa_faults[i];
2570 		grp->faults[i] += p->numa_faults[i];
2571 	}
2572 	my_grp->total_faults -= p->total_numa_faults;
2573 	grp->total_faults += p->total_numa_faults;
2574 
2575 	my_grp->nr_tasks--;
2576 	grp->nr_tasks++;
2577 
2578 	spin_unlock(&my_grp->lock);
2579 	spin_unlock_irq(&grp->lock);
2580 
2581 	rcu_assign_pointer(p->numa_group, grp);
2582 
2583 	put_numa_group(my_grp);
2584 	return;
2585 
2586 no_join:
2587 	rcu_read_unlock();
2588 	return;
2589 }
2590 
2591 /*
2592  * Get rid of NUMA statistics associated with a task (either current or dead).
2593  * If @final is set, the task is dead and has reached refcount zero, so we can
2594  * safely free all relevant data structures. Otherwise, there might be
2595  * concurrent reads from places like load balancing and procfs, and we should
2596  * reset the data back to default state without freeing ->numa_faults.
2597  */
2598 void task_numa_free(struct task_struct *p, bool final)
2599 {
2600 	/* safe: p either is current or is being freed by current */
2601 	struct numa_group *grp = rcu_dereference_raw(p->numa_group);
2602 	unsigned long *numa_faults = p->numa_faults;
2603 	unsigned long flags;
2604 	int i;
2605 
2606 	if (!numa_faults)
2607 		return;
2608 
2609 	if (grp) {
2610 		spin_lock_irqsave(&grp->lock, flags);
2611 		for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2612 			grp->faults[i] -= p->numa_faults[i];
2613 		grp->total_faults -= p->total_numa_faults;
2614 
2615 		grp->nr_tasks--;
2616 		spin_unlock_irqrestore(&grp->lock, flags);
2617 		RCU_INIT_POINTER(p->numa_group, NULL);
2618 		put_numa_group(grp);
2619 	}
2620 
2621 	if (final) {
2622 		p->numa_faults = NULL;
2623 		kfree(numa_faults);
2624 	} else {
2625 		p->total_numa_faults = 0;
2626 		for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2627 			numa_faults[i] = 0;
2628 	}
2629 }
2630 
2631 /*
2632  * Got a PROT_NONE fault for a page on @node.
2633  */
2634 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
2635 {
2636 	struct task_struct *p = current;
2637 	bool migrated = flags & TNF_MIGRATED;
2638 	int cpu_node = task_node(current);
2639 	int local = !!(flags & TNF_FAULT_LOCAL);
2640 	struct numa_group *ng;
2641 	int priv;
2642 
2643 	if (!static_branch_likely(&sched_numa_balancing))
2644 		return;
2645 
2646 	/* for example, ksmd faulting in a user's mm */
2647 	if (!p->mm)
2648 		return;
2649 
2650 	/* Allocate buffer to track faults on a per-node basis */
2651 	if (unlikely(!p->numa_faults)) {
2652 		int size = sizeof(*p->numa_faults) *
2653 			   NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids;
2654 
2655 		p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN);
2656 		if (!p->numa_faults)
2657 			return;
2658 
2659 		p->total_numa_faults = 0;
2660 		memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2661 	}
2662 
2663 	/*
2664 	 * First accesses are treated as private, otherwise consider accesses
2665 	 * to be private if the accessing pid has not changed
2666 	 */
2667 	if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) {
2668 		priv = 1;
2669 	} else {
2670 		priv = cpupid_match_pid(p, last_cpupid);
2671 		if (!priv && !(flags & TNF_NO_GROUP))
2672 			task_numa_group(p, last_cpupid, flags, &priv);
2673 	}
2674 
2675 	/*
2676 	 * If a workload spans multiple NUMA nodes, a shared fault that
2677 	 * occurs wholly within the set of nodes that the workload is
2678 	 * actively using should be counted as local. This allows the
2679 	 * scan rate to slow down when a workload has settled down.
2680 	 */
2681 	ng = deref_curr_numa_group(p);
2682 	if (!priv && !local && ng && ng->active_nodes > 1 &&
2683 				numa_is_active_node(cpu_node, ng) &&
2684 				numa_is_active_node(mem_node, ng))
2685 		local = 1;
2686 
2687 	/*
2688 	 * Retry to migrate task to preferred node periodically, in case it
2689 	 * previously failed, or the scheduler moved us.
2690 	 */
2691 	if (time_after(jiffies, p->numa_migrate_retry)) {
2692 		task_numa_placement(p);
2693 		numa_migrate_preferred(p);
2694 	}
2695 
2696 	if (migrated)
2697 		p->numa_pages_migrated += pages;
2698 	if (flags & TNF_MIGRATE_FAIL)
2699 		p->numa_faults_locality[2] += pages;
2700 
2701 	p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages;
2702 	p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages;
2703 	p->numa_faults_locality[local] += pages;
2704 }
2705 
2706 static void reset_ptenuma_scan(struct task_struct *p)
2707 {
2708 	/*
2709 	 * We only did a read acquisition of the mmap sem, so
2710 	 * p->mm->numa_scan_seq is written to without exclusive access
2711 	 * and the update is not guaranteed to be atomic. That's not
2712 	 * much of an issue though, since this is just used for
2713 	 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not
2714 	 * expensive, to avoid any form of compiler optimizations:
2715 	 */
2716 	WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
2717 	p->mm->numa_scan_offset = 0;
2718 }
2719 
2720 /*
2721  * The expensive part of numa migration is done from task_work context.
2722  * Triggered from task_tick_numa().
2723  */
2724 static void task_numa_work(struct callback_head *work)
2725 {
2726 	unsigned long migrate, next_scan, now = jiffies;
2727 	struct task_struct *p = current;
2728 	struct mm_struct *mm = p->mm;
2729 	u64 runtime = p->se.sum_exec_runtime;
2730 	struct vm_area_struct *vma;
2731 	unsigned long start, end;
2732 	unsigned long nr_pte_updates = 0;
2733 	long pages, virtpages;
2734 
2735 	SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work));
2736 
2737 	work->next = work;
2738 	/*
2739 	 * Who cares about NUMA placement when they're dying.
2740 	 *
2741 	 * NOTE: make sure not to dereference p->mm before this check,
2742 	 * exit_task_work() happens _after_ exit_mm() so we could be called
2743 	 * without p->mm even though we still had it when we enqueued this
2744 	 * work.
2745 	 */
2746 	if (p->flags & PF_EXITING)
2747 		return;
2748 
2749 	if (!mm->numa_next_scan) {
2750 		mm->numa_next_scan = now +
2751 			msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
2752 	}
2753 
2754 	/*
2755 	 * Enforce maximal scan/migration frequency..
2756 	 */
2757 	migrate = mm->numa_next_scan;
2758 	if (time_before(now, migrate))
2759 		return;
2760 
2761 	if (p->numa_scan_period == 0) {
2762 		p->numa_scan_period_max = task_scan_max(p);
2763 		p->numa_scan_period = task_scan_start(p);
2764 	}
2765 
2766 	next_scan = now + msecs_to_jiffies(p->numa_scan_period);
2767 	if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
2768 		return;
2769 
2770 	/*
2771 	 * Delay this task enough that another task of this mm will likely win
2772 	 * the next time around.
2773 	 */
2774 	p->node_stamp += 2 * TICK_NSEC;
2775 
2776 	start = mm->numa_scan_offset;
2777 	pages = sysctl_numa_balancing_scan_size;
2778 	pages <<= 20 - PAGE_SHIFT; /* MB in pages */
2779 	virtpages = pages * 8;	   /* Scan up to this much virtual space */
2780 	if (!pages)
2781 		return;
2782 
2783 
2784 	if (!mmap_read_trylock(mm))
2785 		return;
2786 	vma = find_vma(mm, start);
2787 	if (!vma) {
2788 		reset_ptenuma_scan(p);
2789 		start = 0;
2790 		vma = mm->mmap;
2791 	}
2792 	for (; vma; vma = vma->vm_next) {
2793 		if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
2794 			is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
2795 			continue;
2796 		}
2797 
2798 		/*
2799 		 * Shared library pages mapped by multiple processes are not
2800 		 * migrated as it is expected they are cache replicated. Avoid
2801 		 * hinting faults in read-only file-backed mappings or the vdso
2802 		 * as migrating the pages will be of marginal benefit.
2803 		 */
2804 		if (!vma->vm_mm ||
2805 		    (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ)))
2806 			continue;
2807 
2808 		/*
2809 		 * Skip inaccessible VMAs to avoid any confusion between
2810 		 * PROT_NONE and NUMA hinting ptes
2811 		 */
2812 		if (!vma_is_accessible(vma))
2813 			continue;
2814 
2815 		do {
2816 			start = max(start, vma->vm_start);
2817 			end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
2818 			end = min(end, vma->vm_end);
2819 			nr_pte_updates = change_prot_numa(vma, start, end);
2820 
2821 			/*
2822 			 * Try to scan sysctl_numa_balancing_size worth of
2823 			 * hpages that have at least one present PTE that
2824 			 * is not already pte-numa. If the VMA contains
2825 			 * areas that are unused or already full of prot_numa
2826 			 * PTEs, scan up to virtpages, to skip through those
2827 			 * areas faster.
2828 			 */
2829 			if (nr_pte_updates)
2830 				pages -= (end - start) >> PAGE_SHIFT;
2831 			virtpages -= (end - start) >> PAGE_SHIFT;
2832 
2833 			start = end;
2834 			if (pages <= 0 || virtpages <= 0)
2835 				goto out;
2836 
2837 			cond_resched();
2838 		} while (end != vma->vm_end);
2839 	}
2840 
2841 out:
2842 	/*
2843 	 * It is possible to reach the end of the VMA list but the last few
2844 	 * VMAs are not guaranteed to the vma_migratable. If they are not, we
2845 	 * would find the !migratable VMA on the next scan but not reset the
2846 	 * scanner to the start so check it now.
2847 	 */
2848 	if (vma)
2849 		mm->numa_scan_offset = start;
2850 	else
2851 		reset_ptenuma_scan(p);
2852 	mmap_read_unlock(mm);
2853 
2854 	/*
2855 	 * Make sure tasks use at least 32x as much time to run other code
2856 	 * than they used here, to limit NUMA PTE scanning overhead to 3% max.
2857 	 * Usually update_task_scan_period slows down scanning enough; on an
2858 	 * overloaded system we need to limit overhead on a per task basis.
2859 	 */
2860 	if (unlikely(p->se.sum_exec_runtime != runtime)) {
2861 		u64 diff = p->se.sum_exec_runtime - runtime;
2862 		p->node_stamp += 32 * diff;
2863 	}
2864 }
2865 
2866 void init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
2867 {
2868 	int mm_users = 0;
2869 	struct mm_struct *mm = p->mm;
2870 
2871 	if (mm) {
2872 		mm_users = atomic_read(&mm->mm_users);
2873 		if (mm_users == 1) {
2874 			mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
2875 			mm->numa_scan_seq = 0;
2876 		}
2877 	}
2878 	p->node_stamp			= 0;
2879 	p->numa_scan_seq		= mm ? mm->numa_scan_seq : 0;
2880 	p->numa_scan_period		= sysctl_numa_balancing_scan_delay;
2881 	/* Protect against double add, see task_tick_numa and task_numa_work */
2882 	p->numa_work.next		= &p->numa_work;
2883 	p->numa_faults			= NULL;
2884 	RCU_INIT_POINTER(p->numa_group, NULL);
2885 	p->last_task_numa_placement	= 0;
2886 	p->last_sum_exec_runtime	= 0;
2887 
2888 	init_task_work(&p->numa_work, task_numa_work);
2889 
2890 	/* New address space, reset the preferred nid */
2891 	if (!(clone_flags & CLONE_VM)) {
2892 		p->numa_preferred_nid = NUMA_NO_NODE;
2893 		return;
2894 	}
2895 
2896 	/*
2897 	 * New thread, keep existing numa_preferred_nid which should be copied
2898 	 * already by arch_dup_task_struct but stagger when scans start.
2899 	 */
2900 	if (mm) {
2901 		unsigned int delay;
2902 
2903 		delay = min_t(unsigned int, task_scan_max(current),
2904 			current->numa_scan_period * mm_users * NSEC_PER_MSEC);
2905 		delay += 2 * TICK_NSEC;
2906 		p->node_stamp = delay;
2907 	}
2908 }
2909 
2910 /*
2911  * Drive the periodic memory faults..
2912  */
2913 static void task_tick_numa(struct rq *rq, struct task_struct *curr)
2914 {
2915 	struct callback_head *work = &curr->numa_work;
2916 	u64 period, now;
2917 
2918 	/*
2919 	 * We don't care about NUMA placement if we don't have memory.
2920 	 */
2921 	if ((curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work)
2922 		return;
2923 
2924 	/*
2925 	 * Using runtime rather than walltime has the dual advantage that
2926 	 * we (mostly) drive the selection from busy threads and that the
2927 	 * task needs to have done some actual work before we bother with
2928 	 * NUMA placement.
2929 	 */
2930 	now = curr->se.sum_exec_runtime;
2931 	period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
2932 
2933 	if (now > curr->node_stamp + period) {
2934 		if (!curr->node_stamp)
2935 			curr->numa_scan_period = task_scan_start(curr);
2936 		curr->node_stamp += period;
2937 
2938 		if (!time_before(jiffies, curr->mm->numa_next_scan))
2939 			task_work_add(curr, work, TWA_RESUME);
2940 	}
2941 }
2942 
2943 static void update_scan_period(struct task_struct *p, int new_cpu)
2944 {
2945 	int src_nid = cpu_to_node(task_cpu(p));
2946 	int dst_nid = cpu_to_node(new_cpu);
2947 
2948 	if (!static_branch_likely(&sched_numa_balancing))
2949 		return;
2950 
2951 	if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING))
2952 		return;
2953 
2954 	if (src_nid == dst_nid)
2955 		return;
2956 
2957 	/*
2958 	 * Allow resets if faults have been trapped before one scan
2959 	 * has completed. This is most likely due to a new task that
2960 	 * is pulled cross-node due to wakeups or load balancing.
2961 	 */
2962 	if (p->numa_scan_seq) {
2963 		/*
2964 		 * Avoid scan adjustments if moving to the preferred
2965 		 * node or if the task was not previously running on
2966 		 * the preferred node.
2967 		 */
2968 		if (dst_nid == p->numa_preferred_nid ||
2969 		    (p->numa_preferred_nid != NUMA_NO_NODE &&
2970 			src_nid != p->numa_preferred_nid))
2971 			return;
2972 	}
2973 
2974 	p->numa_scan_period = task_scan_start(p);
2975 }
2976 
2977 #else
2978 static void task_tick_numa(struct rq *rq, struct task_struct *curr)
2979 {
2980 }
2981 
2982 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p)
2983 {
2984 }
2985 
2986 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p)
2987 {
2988 }
2989 
2990 static inline void update_scan_period(struct task_struct *p, int new_cpu)
2991 {
2992 }
2993 
2994 #endif /* CONFIG_NUMA_BALANCING */
2995 
2996 static void
2997 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
2998 {
2999 	update_load_add(&cfs_rq->load, se->load.weight);
3000 #ifdef CONFIG_SMP
3001 	if (entity_is_task(se)) {
3002 		struct rq *rq = rq_of(cfs_rq);
3003 
3004 		account_numa_enqueue(rq, task_of(se));
3005 		list_add(&se->group_node, &rq->cfs_tasks);
3006 	}
3007 #endif
3008 	cfs_rq->nr_running++;
3009 }
3010 
3011 static void
3012 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
3013 {
3014 	update_load_sub(&cfs_rq->load, se->load.weight);
3015 #ifdef CONFIG_SMP
3016 	if (entity_is_task(se)) {
3017 		account_numa_dequeue(rq_of(cfs_rq), task_of(se));
3018 		list_del_init(&se->group_node);
3019 	}
3020 #endif
3021 	cfs_rq->nr_running--;
3022 }
3023 
3024 /*
3025  * Signed add and clamp on underflow.
3026  *
3027  * Explicitly do a load-store to ensure the intermediate value never hits
3028  * memory. This allows lockless observations without ever seeing the negative
3029  * values.
3030  */
3031 #define add_positive(_ptr, _val) do {                           \
3032 	typeof(_ptr) ptr = (_ptr);                              \
3033 	typeof(_val) val = (_val);                              \
3034 	typeof(*ptr) res, var = READ_ONCE(*ptr);                \
3035 								\
3036 	res = var + val;                                        \
3037 								\
3038 	if (val < 0 && res > var)                               \
3039 		res = 0;                                        \
3040 								\
3041 	WRITE_ONCE(*ptr, res);                                  \
3042 } while (0)
3043 
3044 /*
3045  * Unsigned subtract and clamp on underflow.
3046  *
3047  * Explicitly do a load-store to ensure the intermediate value never hits
3048  * memory. This allows lockless observations without ever seeing the negative
3049  * values.
3050  */
3051 #define sub_positive(_ptr, _val) do {				\
3052 	typeof(_ptr) ptr = (_ptr);				\
3053 	typeof(*ptr) val = (_val);				\
3054 	typeof(*ptr) res, var = READ_ONCE(*ptr);		\
3055 	res = var - val;					\
3056 	if (res > var)						\
3057 		res = 0;					\
3058 	WRITE_ONCE(*ptr, res);					\
3059 } while (0)
3060 
3061 /*
3062  * Remove and clamp on negative, from a local variable.
3063  *
3064  * A variant of sub_positive(), which does not use explicit load-store
3065  * and is thus optimized for local variable updates.
3066  */
3067 #define lsub_positive(_ptr, _val) do {				\
3068 	typeof(_ptr) ptr = (_ptr);				\
3069 	*ptr -= min_t(typeof(*ptr), *ptr, _val);		\
3070 } while (0)
3071 
3072 #ifdef CONFIG_SMP
3073 static inline void
3074 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3075 {
3076 	cfs_rq->avg.load_avg += se->avg.load_avg;
3077 	cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum;
3078 }
3079 
3080 static inline void
3081 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3082 {
3083 	sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);
3084 	sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum);
3085 }
3086 #else
3087 static inline void
3088 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
3089 static inline void
3090 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
3091 #endif
3092 
3093 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
3094 			    unsigned long weight)
3095 {
3096 	if (se->on_rq) {
3097 		/* commit outstanding execution time */
3098 		if (cfs_rq->curr == se)
3099 			update_curr(cfs_rq);
3100 		update_load_sub(&cfs_rq->load, se->load.weight);
3101 	}
3102 	dequeue_load_avg(cfs_rq, se);
3103 
3104 	update_load_set(&se->load, weight);
3105 
3106 #ifdef CONFIG_SMP
3107 	do {
3108 		u32 divider = get_pelt_divider(&se->avg);
3109 
3110 		se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);
3111 	} while (0);
3112 #endif
3113 
3114 	enqueue_load_avg(cfs_rq, se);
3115 	if (se->on_rq)
3116 		update_load_add(&cfs_rq->load, se->load.weight);
3117 
3118 }
3119 
3120 void reweight_task(struct task_struct *p, int prio)
3121 {
3122 	struct sched_entity *se = &p->se;
3123 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
3124 	struct load_weight *load = &se->load;
3125 	unsigned long weight = scale_load(sched_prio_to_weight[prio]);
3126 
3127 	reweight_entity(cfs_rq, se, weight);
3128 	load->inv_weight = sched_prio_to_wmult[prio];
3129 }
3130 
3131 #ifdef CONFIG_FAIR_GROUP_SCHED
3132 #ifdef CONFIG_SMP
3133 /*
3134  * All this does is approximate the hierarchical proportion which includes that
3135  * global sum we all love to hate.
3136  *
3137  * That is, the weight of a group entity, is the proportional share of the
3138  * group weight based on the group runqueue weights. That is:
3139  *
3140  *                     tg->weight * grq->load.weight
3141  *   ge->load.weight = -----------------------------               (1)
3142  *			  \Sum grq->load.weight
3143  *
3144  * Now, because computing that sum is prohibitively expensive to compute (been
3145  * there, done that) we approximate it with this average stuff. The average
3146  * moves slower and therefore the approximation is cheaper and more stable.
3147  *
3148  * So instead of the above, we substitute:
3149  *
3150  *   grq->load.weight -> grq->avg.load_avg                         (2)
3151  *
3152  * which yields the following:
3153  *
3154  *                     tg->weight * grq->avg.load_avg
3155  *   ge->load.weight = ------------------------------              (3)
3156  *				tg->load_avg
3157  *
3158  * Where: tg->load_avg ~= \Sum grq->avg.load_avg
3159  *
3160  * That is shares_avg, and it is right (given the approximation (2)).
3161  *
3162  * The problem with it is that because the average is slow -- it was designed
3163  * to be exactly that of course -- this leads to transients in boundary
3164  * conditions. In specific, the case where the group was idle and we start the
3165  * one task. It takes time for our CPU's grq->avg.load_avg to build up,
3166  * yielding bad latency etc..
3167  *
3168  * Now, in that special case (1) reduces to:
3169  *
3170  *                     tg->weight * grq->load.weight
3171  *   ge->load.weight = ----------------------------- = tg->weight   (4)
3172  *			    grp->load.weight
3173  *
3174  * That is, the sum collapses because all other CPUs are idle; the UP scenario.
3175  *
3176  * So what we do is modify our approximation (3) to approach (4) in the (near)
3177  * UP case, like:
3178  *
3179  *   ge->load.weight =
3180  *
3181  *              tg->weight * grq->load.weight
3182  *     ---------------------------------------------------         (5)
3183  *     tg->load_avg - grq->avg.load_avg + grq->load.weight
3184  *
3185  * But because grq->load.weight can drop to 0, resulting in a divide by zero,
3186  * we need to use grq->avg.load_avg as its lower bound, which then gives:
3187  *
3188  *
3189  *                     tg->weight * grq->load.weight
3190  *   ge->load.weight = -----------------------------		   (6)
3191  *				tg_load_avg'
3192  *
3193  * Where:
3194  *
3195  *   tg_load_avg' = tg->load_avg - grq->avg.load_avg +
3196  *                  max(grq->load.weight, grq->avg.load_avg)
3197  *
3198  * And that is shares_weight and is icky. In the (near) UP case it approaches
3199  * (4) while in the normal case it approaches (3). It consistently
3200  * overestimates the ge->load.weight and therefore:
3201  *
3202  *   \Sum ge->load.weight >= tg->weight
3203  *
3204  * hence icky!
3205  */
3206 static long calc_group_shares(struct cfs_rq *cfs_rq)
3207 {
3208 	long tg_weight, tg_shares, load, shares;
3209 	struct task_group *tg = cfs_rq->tg;
3210 
3211 	tg_shares = READ_ONCE(tg->shares);
3212 
3213 	load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg);
3214 
3215 	tg_weight = atomic_long_read(&tg->load_avg);
3216 
3217 	/* Ensure tg_weight >= load */
3218 	tg_weight -= cfs_rq->tg_load_avg_contrib;
3219 	tg_weight += load;
3220 
3221 	shares = (tg_shares * load);
3222 	if (tg_weight)
3223 		shares /= tg_weight;
3224 
3225 	/*
3226 	 * MIN_SHARES has to be unscaled here to support per-CPU partitioning
3227 	 * of a group with small tg->shares value. It is a floor value which is
3228 	 * assigned as a minimum load.weight to the sched_entity representing
3229 	 * the group on a CPU.
3230 	 *
3231 	 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024
3232 	 * on an 8-core system with 8 tasks each runnable on one CPU shares has
3233 	 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In
3234 	 * case no task is runnable on a CPU MIN_SHARES=2 should be returned
3235 	 * instead of 0.
3236 	 */
3237 	return clamp_t(long, shares, MIN_SHARES, tg_shares);
3238 }
3239 #endif /* CONFIG_SMP */
3240 
3241 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
3242 
3243 /*
3244  * Recomputes the group entity based on the current state of its group
3245  * runqueue.
3246  */
3247 static void update_cfs_group(struct sched_entity *se)
3248 {
3249 	struct cfs_rq *gcfs_rq = group_cfs_rq(se);
3250 	long shares;
3251 
3252 	if (!gcfs_rq)
3253 		return;
3254 
3255 	if (throttled_hierarchy(gcfs_rq))
3256 		return;
3257 
3258 #ifndef CONFIG_SMP
3259 	shares = READ_ONCE(gcfs_rq->tg->shares);
3260 
3261 	if (likely(se->load.weight == shares))
3262 		return;
3263 #else
3264 	shares   = calc_group_shares(gcfs_rq);
3265 #endif
3266 
3267 	reweight_entity(cfs_rq_of(se), se, shares);
3268 }
3269 
3270 #else /* CONFIG_FAIR_GROUP_SCHED */
3271 static inline void update_cfs_group(struct sched_entity *se)
3272 {
3273 }
3274 #endif /* CONFIG_FAIR_GROUP_SCHED */
3275 
3276 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags)
3277 {
3278 	struct rq *rq = rq_of(cfs_rq);
3279 
3280 	if (&rq->cfs == cfs_rq) {
3281 		/*
3282 		 * There are a few boundary cases this might miss but it should
3283 		 * get called often enough that that should (hopefully) not be
3284 		 * a real problem.
3285 		 *
3286 		 * It will not get called when we go idle, because the idle
3287 		 * thread is a different class (!fair), nor will the utilization
3288 		 * number include things like RT tasks.
3289 		 *
3290 		 * As is, the util number is not freq-invariant (we'd have to
3291 		 * implement arch_scale_freq_capacity() for that).
3292 		 *
3293 		 * See cpu_util().
3294 		 */
3295 		cpufreq_update_util(rq, flags);
3296 	}
3297 }
3298 
3299 #ifdef CONFIG_SMP
3300 #ifdef CONFIG_FAIR_GROUP_SCHED
3301 /**
3302  * update_tg_load_avg - update the tg's load avg
3303  * @cfs_rq: the cfs_rq whose avg changed
3304  *
3305  * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load.
3306  * However, because tg->load_avg is a global value there are performance
3307  * considerations.
3308  *
3309  * In order to avoid having to look at the other cfs_rq's, we use a
3310  * differential update where we store the last value we propagated. This in
3311  * turn allows skipping updates if the differential is 'small'.
3312  *
3313  * Updating tg's load_avg is necessary before update_cfs_share().
3314  */
3315 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
3316 {
3317 	long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
3318 
3319 	/*
3320 	 * No need to update load_avg for root_task_group as it is not used.
3321 	 */
3322 	if (cfs_rq->tg == &root_task_group)
3323 		return;
3324 
3325 	if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
3326 		atomic_long_add(delta, &cfs_rq->tg->load_avg);
3327 		cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
3328 	}
3329 }
3330 
3331 /*
3332  * Called within set_task_rq() right before setting a task's CPU. The
3333  * caller only guarantees p->pi_lock is held; no other assumptions,
3334  * including the state of rq->lock, should be made.
3335  */
3336 void set_task_rq_fair(struct sched_entity *se,
3337 		      struct cfs_rq *prev, struct cfs_rq *next)
3338 {
3339 	u64 p_last_update_time;
3340 	u64 n_last_update_time;
3341 
3342 	if (!sched_feat(ATTACH_AGE_LOAD))
3343 		return;
3344 
3345 	/*
3346 	 * We are supposed to update the task to "current" time, then its up to
3347 	 * date and ready to go to new CPU/cfs_rq. But we have difficulty in
3348 	 * getting what current time is, so simply throw away the out-of-date
3349 	 * time. This will result in the wakee task is less decayed, but giving
3350 	 * the wakee more load sounds not bad.
3351 	 */
3352 	if (!(se->avg.last_update_time && prev))
3353 		return;
3354 
3355 #ifndef CONFIG_64BIT
3356 	{
3357 		u64 p_last_update_time_copy;
3358 		u64 n_last_update_time_copy;
3359 
3360 		do {
3361 			p_last_update_time_copy = prev->load_last_update_time_copy;
3362 			n_last_update_time_copy = next->load_last_update_time_copy;
3363 
3364 			smp_rmb();
3365 
3366 			p_last_update_time = prev->avg.last_update_time;
3367 			n_last_update_time = next->avg.last_update_time;
3368 
3369 		} while (p_last_update_time != p_last_update_time_copy ||
3370 			 n_last_update_time != n_last_update_time_copy);
3371 	}
3372 #else
3373 	p_last_update_time = prev->avg.last_update_time;
3374 	n_last_update_time = next->avg.last_update_time;
3375 #endif
3376 	__update_load_avg_blocked_se(p_last_update_time, se);
3377 	se->avg.last_update_time = n_last_update_time;
3378 }
3379 
3380 
3381 /*
3382  * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to
3383  * propagate its contribution. The key to this propagation is the invariant
3384  * that for each group:
3385  *
3386  *   ge->avg == grq->avg						(1)
3387  *
3388  * _IFF_ we look at the pure running and runnable sums. Because they
3389  * represent the very same entity, just at different points in the hierarchy.
3390  *
3391  * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial
3392  * and simply copies the running/runnable sum over (but still wrong, because
3393  * the group entity and group rq do not have their PELT windows aligned).
3394  *
3395  * However, update_tg_cfs_load() is more complex. So we have:
3396  *
3397  *   ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg		(2)
3398  *
3399  * And since, like util, the runnable part should be directly transferable,
3400  * the following would _appear_ to be the straight forward approach:
3401  *
3402  *   grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg	(3)
3403  *
3404  * And per (1) we have:
3405  *
3406  *   ge->avg.runnable_avg == grq->avg.runnable_avg
3407  *
3408  * Which gives:
3409  *
3410  *                      ge->load.weight * grq->avg.load_avg
3411  *   ge->avg.load_avg = -----------------------------------		(4)
3412  *                               grq->load.weight
3413  *
3414  * Except that is wrong!
3415  *
3416  * Because while for entities historical weight is not important and we
3417  * really only care about our future and therefore can consider a pure
3418  * runnable sum, runqueues can NOT do this.
3419  *
3420  * We specifically want runqueues to have a load_avg that includes
3421  * historical weights. Those represent the blocked load, the load we expect
3422  * to (shortly) return to us. This only works by keeping the weights as
3423  * integral part of the sum. We therefore cannot decompose as per (3).
3424  *
3425  * Another reason this doesn't work is that runnable isn't a 0-sum entity.
3426  * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the
3427  * rq itself is runnable anywhere between 2/3 and 1 depending on how the
3428  * runnable section of these tasks overlap (or not). If they were to perfectly
3429  * align the rq as a whole would be runnable 2/3 of the time. If however we
3430  * always have at least 1 runnable task, the rq as a whole is always runnable.
3431  *
3432  * So we'll have to approximate.. :/
3433  *
3434  * Given the constraint:
3435  *
3436  *   ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX
3437  *
3438  * We can construct a rule that adds runnable to a rq by assuming minimal
3439  * overlap.
3440  *
3441  * On removal, we'll assume each task is equally runnable; which yields:
3442  *
3443  *   grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight
3444  *
3445  * XXX: only do this for the part of runnable > running ?
3446  *
3447  */
3448 
3449 static inline void
3450 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3451 {
3452 	long delta = gcfs_rq->avg.util_avg - se->avg.util_avg;
3453 	u32 divider;
3454 
3455 	/* Nothing to update */
3456 	if (!delta)
3457 		return;
3458 
3459 	/*
3460 	 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3461 	 * See ___update_load_avg() for details.
3462 	 */
3463 	divider = get_pelt_divider(&cfs_rq->avg);
3464 
3465 	/* Set new sched_entity's utilization */
3466 	se->avg.util_avg = gcfs_rq->avg.util_avg;
3467 	se->avg.util_sum = se->avg.util_avg * divider;
3468 
3469 	/* Update parent cfs_rq utilization */
3470 	add_positive(&cfs_rq->avg.util_avg, delta);
3471 	cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * divider;
3472 }
3473 
3474 static inline void
3475 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3476 {
3477 	long delta = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg;
3478 	u32 divider;
3479 
3480 	/* Nothing to update */
3481 	if (!delta)
3482 		return;
3483 
3484 	/*
3485 	 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3486 	 * See ___update_load_avg() for details.
3487 	 */
3488 	divider = get_pelt_divider(&cfs_rq->avg);
3489 
3490 	/* Set new sched_entity's runnable */
3491 	se->avg.runnable_avg = gcfs_rq->avg.runnable_avg;
3492 	se->avg.runnable_sum = se->avg.runnable_avg * divider;
3493 
3494 	/* Update parent cfs_rq runnable */
3495 	add_positive(&cfs_rq->avg.runnable_avg, delta);
3496 	cfs_rq->avg.runnable_sum = cfs_rq->avg.runnable_avg * divider;
3497 }
3498 
3499 static inline void
3500 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3501 {
3502 	long delta, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;
3503 	unsigned long load_avg;
3504 	u64 load_sum = 0;
3505 	u32 divider;
3506 
3507 	if (!runnable_sum)
3508 		return;
3509 
3510 	gcfs_rq->prop_runnable_sum = 0;
3511 
3512 	/*
3513 	 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3514 	 * See ___update_load_avg() for details.
3515 	 */
3516 	divider = get_pelt_divider(&cfs_rq->avg);
3517 
3518 	if (runnable_sum >= 0) {
3519 		/*
3520 		 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until
3521 		 * the CPU is saturated running == runnable.
3522 		 */
3523 		runnable_sum += se->avg.load_sum;
3524 		runnable_sum = min_t(long, runnable_sum, divider);
3525 	} else {
3526 		/*
3527 		 * Estimate the new unweighted runnable_sum of the gcfs_rq by
3528 		 * assuming all tasks are equally runnable.
3529 		 */
3530 		if (scale_load_down(gcfs_rq->load.weight)) {
3531 			load_sum = div_s64(gcfs_rq->avg.load_sum,
3532 				scale_load_down(gcfs_rq->load.weight));
3533 		}
3534 
3535 		/* But make sure to not inflate se's runnable */
3536 		runnable_sum = min(se->avg.load_sum, load_sum);
3537 	}
3538 
3539 	/*
3540 	 * runnable_sum can't be lower than running_sum
3541 	 * Rescale running sum to be in the same range as runnable sum
3542 	 * running_sum is in [0 : LOAD_AVG_MAX <<  SCHED_CAPACITY_SHIFT]
3543 	 * runnable_sum is in [0 : LOAD_AVG_MAX]
3544 	 */
3545 	running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT;
3546 	runnable_sum = max(runnable_sum, running_sum);
3547 
3548 	load_sum = (s64)se_weight(se) * runnable_sum;
3549 	load_avg = div_s64(load_sum, divider);
3550 
3551 	delta = load_avg - se->avg.load_avg;
3552 
3553 	se->avg.load_sum = runnable_sum;
3554 	se->avg.load_avg = load_avg;
3555 
3556 	add_positive(&cfs_rq->avg.load_avg, delta);
3557 	cfs_rq->avg.load_sum = cfs_rq->avg.load_avg * divider;
3558 }
3559 
3560 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)
3561 {
3562 	cfs_rq->propagate = 1;
3563 	cfs_rq->prop_runnable_sum += runnable_sum;
3564 }
3565 
3566 /* Update task and its cfs_rq load average */
3567 static inline int propagate_entity_load_avg(struct sched_entity *se)
3568 {
3569 	struct cfs_rq *cfs_rq, *gcfs_rq;
3570 
3571 	if (entity_is_task(se))
3572 		return 0;
3573 
3574 	gcfs_rq = group_cfs_rq(se);
3575 	if (!gcfs_rq->propagate)
3576 		return 0;
3577 
3578 	gcfs_rq->propagate = 0;
3579 
3580 	cfs_rq = cfs_rq_of(se);
3581 
3582 	add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum);
3583 
3584 	update_tg_cfs_util(cfs_rq, se, gcfs_rq);
3585 	update_tg_cfs_runnable(cfs_rq, se, gcfs_rq);
3586 	update_tg_cfs_load(cfs_rq, se, gcfs_rq);
3587 
3588 	trace_pelt_cfs_tp(cfs_rq);
3589 	trace_pelt_se_tp(se);
3590 
3591 	return 1;
3592 }
3593 
3594 /*
3595  * Check if we need to update the load and the utilization of a blocked
3596  * group_entity:
3597  */
3598 static inline bool skip_blocked_update(struct sched_entity *se)
3599 {
3600 	struct cfs_rq *gcfs_rq = group_cfs_rq(se);
3601 
3602 	/*
3603 	 * If sched_entity still have not zero load or utilization, we have to
3604 	 * decay it:
3605 	 */
3606 	if (se->avg.load_avg || se->avg.util_avg)
3607 		return false;
3608 
3609 	/*
3610 	 * If there is a pending propagation, we have to update the load and
3611 	 * the utilization of the sched_entity:
3612 	 */
3613 	if (gcfs_rq->propagate)
3614 		return false;
3615 
3616 	/*
3617 	 * Otherwise, the load and the utilization of the sched_entity is
3618 	 * already zero and there is no pending propagation, so it will be a
3619 	 * waste of time to try to decay it:
3620 	 */
3621 	return true;
3622 }
3623 
3624 #else /* CONFIG_FAIR_GROUP_SCHED */
3625 
3626 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {}
3627 
3628 static inline int propagate_entity_load_avg(struct sched_entity *se)
3629 {
3630 	return 0;
3631 }
3632 
3633 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {}
3634 
3635 #endif /* CONFIG_FAIR_GROUP_SCHED */
3636 
3637 /**
3638  * update_cfs_rq_load_avg - update the cfs_rq's load/util averages
3639  * @now: current time, as per cfs_rq_clock_pelt()
3640  * @cfs_rq: cfs_rq to update
3641  *
3642  * The cfs_rq avg is the direct sum of all its entities (blocked and runnable)
3643  * avg. The immediate corollary is that all (fair) tasks must be attached, see
3644  * post_init_entity_util_avg().
3645  *
3646  * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example.
3647  *
3648  * Returns true if the load decayed or we removed load.
3649  *
3650  * Since both these conditions indicate a changed cfs_rq->avg.load we should
3651  * call update_tg_load_avg() when this function returns true.
3652  */
3653 static inline int
3654 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
3655 {
3656 	unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0;
3657 	struct sched_avg *sa = &cfs_rq->avg;
3658 	int decayed = 0;
3659 
3660 	if (cfs_rq->removed.nr) {
3661 		unsigned long r;
3662 		u32 divider = get_pelt_divider(&cfs_rq->avg);
3663 
3664 		raw_spin_lock(&cfs_rq->removed.lock);
3665 		swap(cfs_rq->removed.util_avg, removed_util);
3666 		swap(cfs_rq->removed.load_avg, removed_load);
3667 		swap(cfs_rq->removed.runnable_avg, removed_runnable);
3668 		cfs_rq->removed.nr = 0;
3669 		raw_spin_unlock(&cfs_rq->removed.lock);
3670 
3671 		r = removed_load;
3672 		sub_positive(&sa->load_avg, r);
3673 		sub_positive(&sa->load_sum, r * divider);
3674 
3675 		r = removed_util;
3676 		sub_positive(&sa->util_avg, r);
3677 		sub_positive(&sa->util_sum, r * divider);
3678 
3679 		r = removed_runnable;
3680 		sub_positive(&sa->runnable_avg, r);
3681 		sub_positive(&sa->runnable_sum, r * divider);
3682 
3683 		/*
3684 		 * removed_runnable is the unweighted version of removed_load so we
3685 		 * can use it to estimate removed_load_sum.
3686 		 */
3687 		add_tg_cfs_propagate(cfs_rq,
3688 			-(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT);
3689 
3690 		decayed = 1;
3691 	}
3692 
3693 	decayed |= __update_load_avg_cfs_rq(now, cfs_rq);
3694 
3695 #ifndef CONFIG_64BIT
3696 	smp_wmb();
3697 	cfs_rq->load_last_update_time_copy = sa->last_update_time;
3698 #endif
3699 
3700 	return decayed;
3701 }
3702 
3703 /**
3704  * attach_entity_load_avg - attach this entity to its cfs_rq load avg
3705  * @cfs_rq: cfs_rq to attach to
3706  * @se: sched_entity to attach
3707  *
3708  * Must call update_cfs_rq_load_avg() before this, since we rely on
3709  * cfs_rq->avg.last_update_time being current.
3710  */
3711 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3712 {
3713 	/*
3714 	 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3715 	 * See ___update_load_avg() for details.
3716 	 */
3717 	u32 divider = get_pelt_divider(&cfs_rq->avg);
3718 
3719 	/*
3720 	 * When we attach the @se to the @cfs_rq, we must align the decay
3721 	 * window because without that, really weird and wonderful things can
3722 	 * happen.
3723 	 *
3724 	 * XXX illustrate
3725 	 */
3726 	se->avg.last_update_time = cfs_rq->avg.last_update_time;
3727 	se->avg.period_contrib = cfs_rq->avg.period_contrib;
3728 
3729 	/*
3730 	 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new
3731 	 * period_contrib. This isn't strictly correct, but since we're
3732 	 * entirely outside of the PELT hierarchy, nobody cares if we truncate
3733 	 * _sum a little.
3734 	 */
3735 	se->avg.util_sum = se->avg.util_avg * divider;
3736 
3737 	se->avg.runnable_sum = se->avg.runnable_avg * divider;
3738 
3739 	se->avg.load_sum = divider;
3740 	if (se_weight(se)) {
3741 		se->avg.load_sum =
3742 			div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se));
3743 	}
3744 
3745 	enqueue_load_avg(cfs_rq, se);
3746 	cfs_rq->avg.util_avg += se->avg.util_avg;
3747 	cfs_rq->avg.util_sum += se->avg.util_sum;
3748 	cfs_rq->avg.runnable_avg += se->avg.runnable_avg;
3749 	cfs_rq->avg.runnable_sum += se->avg.runnable_sum;
3750 
3751 	add_tg_cfs_propagate(cfs_rq, se->avg.load_sum);
3752 
3753 	cfs_rq_util_change(cfs_rq, 0);
3754 
3755 	trace_pelt_cfs_tp(cfs_rq);
3756 }
3757 
3758 /**
3759  * detach_entity_load_avg - detach this entity from its cfs_rq load avg
3760  * @cfs_rq: cfs_rq to detach from
3761  * @se: sched_entity to detach
3762  *
3763  * Must call update_cfs_rq_load_avg() before this, since we rely on
3764  * cfs_rq->avg.last_update_time being current.
3765  */
3766 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3767 {
3768 	/*
3769 	 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3770 	 * See ___update_load_avg() for details.
3771 	 */
3772 	u32 divider = get_pelt_divider(&cfs_rq->avg);
3773 
3774 	dequeue_load_avg(cfs_rq, se);
3775 	sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);
3776 	cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * divider;
3777 	sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg);
3778 	cfs_rq->avg.runnable_sum = cfs_rq->avg.runnable_avg * divider;
3779 
3780 	add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum);
3781 
3782 	cfs_rq_util_change(cfs_rq, 0);
3783 
3784 	trace_pelt_cfs_tp(cfs_rq);
3785 }
3786 
3787 /*
3788  * Optional action to be done while updating the load average
3789  */
3790 #define UPDATE_TG	0x1
3791 #define SKIP_AGE_LOAD	0x2
3792 #define DO_ATTACH	0x4
3793 
3794 /* Update task and its cfs_rq load average */
3795 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
3796 {
3797 	u64 now = cfs_rq_clock_pelt(cfs_rq);
3798 	int decayed;
3799 
3800 	/*
3801 	 * Track task load average for carrying it to new CPU after migrated, and
3802 	 * track group sched_entity load average for task_h_load calc in migration
3803 	 */
3804 	if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD))
3805 		__update_load_avg_se(now, cfs_rq, se);
3806 
3807 	decayed  = update_cfs_rq_load_avg(now, cfs_rq);
3808 	decayed |= propagate_entity_load_avg(se);
3809 
3810 	if (!se->avg.last_update_time && (flags & DO_ATTACH)) {
3811 
3812 		/*
3813 		 * DO_ATTACH means we're here from enqueue_entity().
3814 		 * !last_update_time means we've passed through
3815 		 * migrate_task_rq_fair() indicating we migrated.
3816 		 *
3817 		 * IOW we're enqueueing a task on a new CPU.
3818 		 */
3819 		attach_entity_load_avg(cfs_rq, se);
3820 		update_tg_load_avg(cfs_rq);
3821 
3822 	} else if (decayed) {
3823 		cfs_rq_util_change(cfs_rq, 0);
3824 
3825 		if (flags & UPDATE_TG)
3826 			update_tg_load_avg(cfs_rq);
3827 	}
3828 }
3829 
3830 #ifndef CONFIG_64BIT
3831 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
3832 {
3833 	u64 last_update_time_copy;
3834 	u64 last_update_time;
3835 
3836 	do {
3837 		last_update_time_copy = cfs_rq->load_last_update_time_copy;
3838 		smp_rmb();
3839 		last_update_time = cfs_rq->avg.last_update_time;
3840 	} while (last_update_time != last_update_time_copy);
3841 
3842 	return last_update_time;
3843 }
3844 #else
3845 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
3846 {
3847 	return cfs_rq->avg.last_update_time;
3848 }
3849 #endif
3850 
3851 /*
3852  * Synchronize entity load avg of dequeued entity without locking
3853  * the previous rq.
3854  */
3855 static void sync_entity_load_avg(struct sched_entity *se)
3856 {
3857 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
3858 	u64 last_update_time;
3859 
3860 	last_update_time = cfs_rq_last_update_time(cfs_rq);
3861 	__update_load_avg_blocked_se(last_update_time, se);
3862 }
3863 
3864 /*
3865  * Task first catches up with cfs_rq, and then subtract
3866  * itself from the cfs_rq (task must be off the queue now).
3867  */
3868 static void remove_entity_load_avg(struct sched_entity *se)
3869 {
3870 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
3871 	unsigned long flags;
3872 
3873 	/*
3874 	 * tasks cannot exit without having gone through wake_up_new_task() ->
3875 	 * post_init_entity_util_avg() which will have added things to the
3876 	 * cfs_rq, so we can remove unconditionally.
3877 	 */
3878 
3879 	sync_entity_load_avg(se);
3880 
3881 	raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags);
3882 	++cfs_rq->removed.nr;
3883 	cfs_rq->removed.util_avg	+= se->avg.util_avg;
3884 	cfs_rq->removed.load_avg	+= se->avg.load_avg;
3885 	cfs_rq->removed.runnable_avg	+= se->avg.runnable_avg;
3886 	raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags);
3887 }
3888 
3889 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq)
3890 {
3891 	return cfs_rq->avg.runnable_avg;
3892 }
3893 
3894 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
3895 {
3896 	return cfs_rq->avg.load_avg;
3897 }
3898 
3899 static int newidle_balance(struct rq *this_rq, struct rq_flags *rf);
3900 
3901 static inline unsigned long task_util(struct task_struct *p)
3902 {
3903 	return READ_ONCE(p->se.avg.util_avg);
3904 }
3905 
3906 static inline unsigned long _task_util_est(struct task_struct *p)
3907 {
3908 	struct util_est ue = READ_ONCE(p->se.avg.util_est);
3909 
3910 	return max(ue.ewma, (ue.enqueued & ~UTIL_AVG_UNCHANGED));
3911 }
3912 
3913 static inline unsigned long task_util_est(struct task_struct *p)
3914 {
3915 	return max(task_util(p), _task_util_est(p));
3916 }
3917 
3918 #ifdef CONFIG_UCLAMP_TASK
3919 static inline unsigned long uclamp_task_util(struct task_struct *p)
3920 {
3921 	return clamp(task_util_est(p),
3922 		     uclamp_eff_value(p, UCLAMP_MIN),
3923 		     uclamp_eff_value(p, UCLAMP_MAX));
3924 }
3925 #else
3926 static inline unsigned long uclamp_task_util(struct task_struct *p)
3927 {
3928 	return task_util_est(p);
3929 }
3930 #endif
3931 
3932 static inline void util_est_enqueue(struct cfs_rq *cfs_rq,
3933 				    struct task_struct *p)
3934 {
3935 	unsigned int enqueued;
3936 
3937 	if (!sched_feat(UTIL_EST))
3938 		return;
3939 
3940 	/* Update root cfs_rq's estimated utilization */
3941 	enqueued  = cfs_rq->avg.util_est.enqueued;
3942 	enqueued += _task_util_est(p);
3943 	WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);
3944 
3945 	trace_sched_util_est_cfs_tp(cfs_rq);
3946 }
3947 
3948 static inline void util_est_dequeue(struct cfs_rq *cfs_rq,
3949 				    struct task_struct *p)
3950 {
3951 	unsigned int enqueued;
3952 
3953 	if (!sched_feat(UTIL_EST))
3954 		return;
3955 
3956 	/* Update root cfs_rq's estimated utilization */
3957 	enqueued  = cfs_rq->avg.util_est.enqueued;
3958 	enqueued -= min_t(unsigned int, enqueued, _task_util_est(p));
3959 	WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);
3960 
3961 	trace_sched_util_est_cfs_tp(cfs_rq);
3962 }
3963 
3964 #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100)
3965 
3966 /*
3967  * Check if a (signed) value is within a specified (unsigned) margin,
3968  * based on the observation that:
3969  *
3970  *     abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1)
3971  *
3972  * NOTE: this only works when value + margin < INT_MAX.
3973  */
3974 static inline bool within_margin(int value, int margin)
3975 {
3976 	return ((unsigned int)(value + margin - 1) < (2 * margin - 1));
3977 }
3978 
3979 static inline void util_est_update(struct cfs_rq *cfs_rq,
3980 				   struct task_struct *p,
3981 				   bool task_sleep)
3982 {
3983 	long last_ewma_diff, last_enqueued_diff;
3984 	struct util_est ue;
3985 
3986 	if (!sched_feat(UTIL_EST))
3987 		return;
3988 
3989 	/*
3990 	 * Skip update of task's estimated utilization when the task has not
3991 	 * yet completed an activation, e.g. being migrated.
3992 	 */
3993 	if (!task_sleep)
3994 		return;
3995 
3996 	/*
3997 	 * If the PELT values haven't changed since enqueue time,
3998 	 * skip the util_est update.
3999 	 */
4000 	ue = p->se.avg.util_est;
4001 	if (ue.enqueued & UTIL_AVG_UNCHANGED)
4002 		return;
4003 
4004 	last_enqueued_diff = ue.enqueued;
4005 
4006 	/*
4007 	 * Reset EWMA on utilization increases, the moving average is used only
4008 	 * to smooth utilization decreases.
4009 	 */
4010 	ue.enqueued = task_util(p);
4011 	if (sched_feat(UTIL_EST_FASTUP)) {
4012 		if (ue.ewma < ue.enqueued) {
4013 			ue.ewma = ue.enqueued;
4014 			goto done;
4015 		}
4016 	}
4017 
4018 	/*
4019 	 * Skip update of task's estimated utilization when its members are
4020 	 * already ~1% close to its last activation value.
4021 	 */
4022 	last_ewma_diff = ue.enqueued - ue.ewma;
4023 	last_enqueued_diff -= ue.enqueued;
4024 	if (within_margin(last_ewma_diff, UTIL_EST_MARGIN)) {
4025 		if (!within_margin(last_enqueued_diff, UTIL_EST_MARGIN))
4026 			goto done;
4027 
4028 		return;
4029 	}
4030 
4031 	/*
4032 	 * To avoid overestimation of actual task utilization, skip updates if
4033 	 * we cannot grant there is idle time in this CPU.
4034 	 */
4035 	if (task_util(p) > capacity_orig_of(cpu_of(rq_of(cfs_rq))))
4036 		return;
4037 
4038 	/*
4039 	 * Update Task's estimated utilization
4040 	 *
4041 	 * When *p completes an activation we can consolidate another sample
4042 	 * of the task size. This is done by storing the current PELT value
4043 	 * as ue.enqueued and by using this value to update the Exponential
4044 	 * Weighted Moving Average (EWMA):
4045 	 *
4046 	 *  ewma(t) = w *  task_util(p) + (1-w) * ewma(t-1)
4047 	 *          = w *  task_util(p) +         ewma(t-1)  - w * ewma(t-1)
4048 	 *          = w * (task_util(p) -         ewma(t-1)) +     ewma(t-1)
4049 	 *          = w * (      last_ewma_diff            ) +     ewma(t-1)
4050 	 *          = w * (last_ewma_diff  +  ewma(t-1) / w)
4051 	 *
4052 	 * Where 'w' is the weight of new samples, which is configured to be
4053 	 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
4054 	 */
4055 	ue.ewma <<= UTIL_EST_WEIGHT_SHIFT;
4056 	ue.ewma  += last_ewma_diff;
4057 	ue.ewma >>= UTIL_EST_WEIGHT_SHIFT;
4058 done:
4059 	ue.enqueued |= UTIL_AVG_UNCHANGED;
4060 	WRITE_ONCE(p->se.avg.util_est, ue);
4061 
4062 	trace_sched_util_est_se_tp(&p->se);
4063 }
4064 
4065 static inline int task_fits_capacity(struct task_struct *p, long capacity)
4066 {
4067 	return fits_capacity(uclamp_task_util(p), capacity);
4068 }
4069 
4070 static inline void update_misfit_status(struct task_struct *p, struct rq *rq)
4071 {
4072 	if (!static_branch_unlikely(&sched_asym_cpucapacity))
4073 		return;
4074 
4075 	if (!p || p->nr_cpus_allowed == 1) {
4076 		rq->misfit_task_load = 0;
4077 		return;
4078 	}
4079 
4080 	if (task_fits_capacity(p, capacity_of(cpu_of(rq)))) {
4081 		rq->misfit_task_load = 0;
4082 		return;
4083 	}
4084 
4085 	/*
4086 	 * Make sure that misfit_task_load will not be null even if
4087 	 * task_h_load() returns 0.
4088 	 */
4089 	rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1);
4090 }
4091 
4092 #else /* CONFIG_SMP */
4093 
4094 #define UPDATE_TG	0x0
4095 #define SKIP_AGE_LOAD	0x0
4096 #define DO_ATTACH	0x0
4097 
4098 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1)
4099 {
4100 	cfs_rq_util_change(cfs_rq, 0);
4101 }
4102 
4103 static inline void remove_entity_load_avg(struct sched_entity *se) {}
4104 
4105 static inline void
4106 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
4107 static inline void
4108 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
4109 
4110 static inline int newidle_balance(struct rq *rq, struct rq_flags *rf)
4111 {
4112 	return 0;
4113 }
4114 
4115 static inline void
4116 util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {}
4117 
4118 static inline void
4119 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p) {}
4120 
4121 static inline void
4122 util_est_update(struct cfs_rq *cfs_rq, struct task_struct *p,
4123 		bool task_sleep) {}
4124 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {}
4125 
4126 #endif /* CONFIG_SMP */
4127 
4128 static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
4129 {
4130 #ifdef CONFIG_SCHED_DEBUG
4131 	s64 d = se->vruntime - cfs_rq->min_vruntime;
4132 
4133 	if (d < 0)
4134 		d = -d;
4135 
4136 	if (d > 3*sysctl_sched_latency)
4137 		schedstat_inc(cfs_rq->nr_spread_over);
4138 #endif
4139 }
4140 
4141 static void
4142 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
4143 {
4144 	u64 vruntime = cfs_rq->min_vruntime;
4145 
4146 	/*
4147 	 * The 'current' period is already promised to the current tasks,
4148 	 * however the extra weight of the new task will slow them down a
4149 	 * little, place the new task so that it fits in the slot that
4150 	 * stays open at the end.
4151 	 */
4152 	if (initial && sched_feat(START_DEBIT))
4153 		vruntime += sched_vslice(cfs_rq, se);
4154 
4155 	/* sleeps up to a single latency don't count. */
4156 	if (!initial) {
4157 		unsigned long thresh = sysctl_sched_latency;
4158 
4159 		/*
4160 		 * Halve their sleep time's effect, to allow
4161 		 * for a gentler effect of sleepers:
4162 		 */
4163 		if (sched_feat(GENTLE_FAIR_SLEEPERS))
4164 			thresh >>= 1;
4165 
4166 		vruntime -= thresh;
4167 	}
4168 
4169 	/* ensure we never gain time by being placed backwards. */
4170 	se->vruntime = max_vruntime(se->vruntime, vruntime);
4171 }
4172 
4173 static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
4174 
4175 static inline void check_schedstat_required(void)
4176 {
4177 #ifdef CONFIG_SCHEDSTATS
4178 	if (schedstat_enabled())
4179 		return;
4180 
4181 	/* Force schedstat enabled if a dependent tracepoint is active */
4182 	if (trace_sched_stat_wait_enabled()    ||
4183 			trace_sched_stat_sleep_enabled()   ||
4184 			trace_sched_stat_iowait_enabled()  ||
4185 			trace_sched_stat_blocked_enabled() ||
4186 			trace_sched_stat_runtime_enabled())  {
4187 		printk_deferred_once("Scheduler tracepoints stat_sleep, stat_iowait, "
4188 			     "stat_blocked and stat_runtime require the "
4189 			     "kernel parameter schedstats=enable or "
4190 			     "kernel.sched_schedstats=1\n");
4191 	}
4192 #endif
4193 }
4194 
4195 static inline bool cfs_bandwidth_used(void);
4196 
4197 /*
4198  * MIGRATION
4199  *
4200  *	dequeue
4201  *	  update_curr()
4202  *	    update_min_vruntime()
4203  *	  vruntime -= min_vruntime
4204  *
4205  *	enqueue
4206  *	  update_curr()
4207  *	    update_min_vruntime()
4208  *	  vruntime += min_vruntime
4209  *
4210  * this way the vruntime transition between RQs is done when both
4211  * min_vruntime are up-to-date.
4212  *
4213  * WAKEUP (remote)
4214  *
4215  *	->migrate_task_rq_fair() (p->state == TASK_WAKING)
4216  *	  vruntime -= min_vruntime
4217  *
4218  *	enqueue
4219  *	  update_curr()
4220  *	    update_min_vruntime()
4221  *	  vruntime += min_vruntime
4222  *
4223  * this way we don't have the most up-to-date min_vruntime on the originating
4224  * CPU and an up-to-date min_vruntime on the destination CPU.
4225  */
4226 
4227 static void
4228 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4229 {
4230 	bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED);
4231 	bool curr = cfs_rq->curr == se;
4232 
4233 	/*
4234 	 * If we're the current task, we must renormalise before calling
4235 	 * update_curr().
4236 	 */
4237 	if (renorm && curr)
4238 		se->vruntime += cfs_rq->min_vruntime;
4239 
4240 	update_curr(cfs_rq);
4241 
4242 	/*
4243 	 * Otherwise, renormalise after, such that we're placed at the current
4244 	 * moment in time, instead of some random moment in the past. Being
4245 	 * placed in the past could significantly boost this task to the
4246 	 * fairness detriment of existing tasks.
4247 	 */
4248 	if (renorm && !curr)
4249 		se->vruntime += cfs_rq->min_vruntime;
4250 
4251 	/*
4252 	 * When enqueuing a sched_entity, we must:
4253 	 *   - Update loads to have both entity and cfs_rq synced with now.
4254 	 *   - Add its load to cfs_rq->runnable_avg
4255 	 *   - For group_entity, update its weight to reflect the new share of
4256 	 *     its group cfs_rq
4257 	 *   - Add its new weight to cfs_rq->load.weight
4258 	 */
4259 	update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH);
4260 	se_update_runnable(se);
4261 	update_cfs_group(se);
4262 	account_entity_enqueue(cfs_rq, se);
4263 
4264 	if (flags & ENQUEUE_WAKEUP)
4265 		place_entity(cfs_rq, se, 0);
4266 
4267 	check_schedstat_required();
4268 	update_stats_enqueue(cfs_rq, se, flags);
4269 	check_spread(cfs_rq, se);
4270 	if (!curr)
4271 		__enqueue_entity(cfs_rq, se);
4272 	se->on_rq = 1;
4273 
4274 	/*
4275 	 * When bandwidth control is enabled, cfs might have been removed
4276 	 * because of a parent been throttled but cfs->nr_running > 1. Try to
4277 	 * add it unconditionally.
4278 	 */
4279 	if (cfs_rq->nr_running == 1 || cfs_bandwidth_used())
4280 		list_add_leaf_cfs_rq(cfs_rq);
4281 
4282 	if (cfs_rq->nr_running == 1)
4283 		check_enqueue_throttle(cfs_rq);
4284 }
4285 
4286 static void __clear_buddies_last(struct sched_entity *se)
4287 {
4288 	for_each_sched_entity(se) {
4289 		struct cfs_rq *cfs_rq = cfs_rq_of(se);
4290 		if (cfs_rq->last != se)
4291 			break;
4292 
4293 		cfs_rq->last = NULL;
4294 	}
4295 }
4296 
4297 static void __clear_buddies_next(struct sched_entity *se)
4298 {
4299 	for_each_sched_entity(se) {
4300 		struct cfs_rq *cfs_rq = cfs_rq_of(se);
4301 		if (cfs_rq->next != se)
4302 			break;
4303 
4304 		cfs_rq->next = NULL;
4305 	}
4306 }
4307 
4308 static void __clear_buddies_skip(struct sched_entity *se)
4309 {
4310 	for_each_sched_entity(se) {
4311 		struct cfs_rq *cfs_rq = cfs_rq_of(se);
4312 		if (cfs_rq->skip != se)
4313 			break;
4314 
4315 		cfs_rq->skip = NULL;
4316 	}
4317 }
4318 
4319 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
4320 {
4321 	if (cfs_rq->last == se)
4322 		__clear_buddies_last(se);
4323 
4324 	if (cfs_rq->next == se)
4325 		__clear_buddies_next(se);
4326 
4327 	if (cfs_rq->skip == se)
4328 		__clear_buddies_skip(se);
4329 }
4330 
4331 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
4332 
4333 static void
4334 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4335 {
4336 	/*
4337 	 * Update run-time statistics of the 'current'.
4338 	 */
4339 	update_curr(cfs_rq);
4340 
4341 	/*
4342 	 * When dequeuing a sched_entity, we must:
4343 	 *   - Update loads to have both entity and cfs_rq synced with now.
4344 	 *   - Subtract its load from the cfs_rq->runnable_avg.
4345 	 *   - Subtract its previous weight from cfs_rq->load.weight.
4346 	 *   - For group entity, update its weight to reflect the new share
4347 	 *     of its group cfs_rq.
4348 	 */
4349 	update_load_avg(cfs_rq, se, UPDATE_TG);
4350 	se_update_runnable(se);
4351 
4352 	update_stats_dequeue(cfs_rq, se, flags);
4353 
4354 	clear_buddies(cfs_rq, se);
4355 
4356 	if (se != cfs_rq->curr)
4357 		__dequeue_entity(cfs_rq, se);
4358 	se->on_rq = 0;
4359 	account_entity_dequeue(cfs_rq, se);
4360 
4361 	/*
4362 	 * Normalize after update_curr(); which will also have moved
4363 	 * min_vruntime if @se is the one holding it back. But before doing
4364 	 * update_min_vruntime() again, which will discount @se's position and
4365 	 * can move min_vruntime forward still more.
4366 	 */
4367 	if (!(flags & DEQUEUE_SLEEP))
4368 		se->vruntime -= cfs_rq->min_vruntime;
4369 
4370 	/* return excess runtime on last dequeue */
4371 	return_cfs_rq_runtime(cfs_rq);
4372 
4373 	update_cfs_group(se);
4374 
4375 	/*
4376 	 * Now advance min_vruntime if @se was the entity holding it back,
4377 	 * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be
4378 	 * put back on, and if we advance min_vruntime, we'll be placed back
4379 	 * further than we started -- ie. we'll be penalized.
4380 	 */
4381 	if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE)
4382 		update_min_vruntime(cfs_rq);
4383 }
4384 
4385 /*
4386  * Preempt the current task with a newly woken task if needed:
4387  */
4388 static void
4389 check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
4390 {
4391 	unsigned long ideal_runtime, delta_exec;
4392 	struct sched_entity *se;
4393 	s64 delta;
4394 
4395 	ideal_runtime = sched_slice(cfs_rq, curr);
4396 	delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
4397 	if (delta_exec > ideal_runtime) {
4398 		resched_curr(rq_of(cfs_rq));
4399 		/*
4400 		 * The current task ran long enough, ensure it doesn't get
4401 		 * re-elected due to buddy favours.
4402 		 */
4403 		clear_buddies(cfs_rq, curr);
4404 		return;
4405 	}
4406 
4407 	/*
4408 	 * Ensure that a task that missed wakeup preemption by a
4409 	 * narrow margin doesn't have to wait for a full slice.
4410 	 * This also mitigates buddy induced latencies under load.
4411 	 */
4412 	if (delta_exec < sysctl_sched_min_granularity)
4413 		return;
4414 
4415 	se = __pick_first_entity(cfs_rq);
4416 	delta = curr->vruntime - se->vruntime;
4417 
4418 	if (delta < 0)
4419 		return;
4420 
4421 	if (delta > ideal_runtime)
4422 		resched_curr(rq_of(cfs_rq));
4423 }
4424 
4425 static void
4426 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
4427 {
4428 	/* 'current' is not kept within the tree. */
4429 	if (se->on_rq) {
4430 		/*
4431 		 * Any task has to be enqueued before it get to execute on
4432 		 * a CPU. So account for the time it spent waiting on the
4433 		 * runqueue.
4434 		 */
4435 		update_stats_wait_end(cfs_rq, se);
4436 		__dequeue_entity(cfs_rq, se);
4437 		update_load_avg(cfs_rq, se, UPDATE_TG);
4438 	}
4439 
4440 	update_stats_curr_start(cfs_rq, se);
4441 	cfs_rq->curr = se;
4442 
4443 	/*
4444 	 * Track our maximum slice length, if the CPU's load is at
4445 	 * least twice that of our own weight (i.e. dont track it
4446 	 * when there are only lesser-weight tasks around):
4447 	 */
4448 	if (schedstat_enabled() &&
4449 	    rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) {
4450 		schedstat_set(se->statistics.slice_max,
4451 			max((u64)schedstat_val(se->statistics.slice_max),
4452 			    se->sum_exec_runtime - se->prev_sum_exec_runtime));
4453 	}
4454 
4455 	se->prev_sum_exec_runtime = se->sum_exec_runtime;
4456 }
4457 
4458 static int
4459 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
4460 
4461 /*
4462  * Pick the next process, keeping these things in mind, in this order:
4463  * 1) keep things fair between processes/task groups
4464  * 2) pick the "next" process, since someone really wants that to run
4465  * 3) pick the "last" process, for cache locality
4466  * 4) do not run the "skip" process, if something else is available
4467  */
4468 static struct sched_entity *
4469 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr)
4470 {
4471 	struct sched_entity *left = __pick_first_entity(cfs_rq);
4472 	struct sched_entity *se;
4473 
4474 	/*
4475 	 * If curr is set we have to see if its left of the leftmost entity
4476 	 * still in the tree, provided there was anything in the tree at all.
4477 	 */
4478 	if (!left || (curr && entity_before(curr, left)))
4479 		left = curr;
4480 
4481 	se = left; /* ideally we run the leftmost entity */
4482 
4483 	/*
4484 	 * Avoid running the skip buddy, if running something else can
4485 	 * be done without getting too unfair.
4486 	 */
4487 	if (cfs_rq->skip == se) {
4488 		struct sched_entity *second;
4489 
4490 		if (se == curr) {
4491 			second = __pick_first_entity(cfs_rq);
4492 		} else {
4493 			second = __pick_next_entity(se);
4494 			if (!second || (curr && entity_before(curr, second)))
4495 				second = curr;
4496 		}
4497 
4498 		if (second && wakeup_preempt_entity(second, left) < 1)
4499 			se = second;
4500 	}
4501 
4502 	if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) {
4503 		/*
4504 		 * Someone really wants this to run. If it's not unfair, run it.
4505 		 */
4506 		se = cfs_rq->next;
4507 	} else if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) {
4508 		/*
4509 		 * Prefer last buddy, try to return the CPU to a preempted task.
4510 		 */
4511 		se = cfs_rq->last;
4512 	}
4513 
4514 	clear_buddies(cfs_rq, se);
4515 
4516 	return se;
4517 }
4518 
4519 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
4520 
4521 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
4522 {
4523 	/*
4524 	 * If still on the runqueue then deactivate_task()
4525 	 * was not called and update_curr() has to be done:
4526 	 */
4527 	if (prev->on_rq)
4528 		update_curr(cfs_rq);
4529 
4530 	/* throttle cfs_rqs exceeding runtime */
4531 	check_cfs_rq_runtime(cfs_rq);
4532 
4533 	check_spread(cfs_rq, prev);
4534 
4535 	if (prev->on_rq) {
4536 		update_stats_wait_start(cfs_rq, prev);
4537 		/* Put 'current' back into the tree. */
4538 		__enqueue_entity(cfs_rq, prev);
4539 		/* in !on_rq case, update occurred at dequeue */
4540 		update_load_avg(cfs_rq, prev, 0);
4541 	}
4542 	cfs_rq->curr = NULL;
4543 }
4544 
4545 static void
4546 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
4547 {
4548 	/*
4549 	 * Update run-time statistics of the 'current'.
4550 	 */
4551 	update_curr(cfs_rq);
4552 
4553 	/*
4554 	 * Ensure that runnable average is periodically updated.
4555 	 */
4556 	update_load_avg(cfs_rq, curr, UPDATE_TG);
4557 	update_cfs_group(curr);
4558 
4559 #ifdef CONFIG_SCHED_HRTICK
4560 	/*
4561 	 * queued ticks are scheduled to match the slice, so don't bother
4562 	 * validating it and just reschedule.
4563 	 */
4564 	if (queued) {
4565 		resched_curr(rq_of(cfs_rq));
4566 		return;
4567 	}
4568 	/*
4569 	 * don't let the period tick interfere with the hrtick preemption
4570 	 */
4571 	if (!sched_feat(DOUBLE_TICK) &&
4572 			hrtimer_active(&rq_of(cfs_rq)->hrtick_timer))
4573 		return;
4574 #endif
4575 
4576 	if (cfs_rq->nr_running > 1)
4577 		check_preempt_tick(cfs_rq, curr);
4578 }
4579 
4580 
4581 /**************************************************
4582  * CFS bandwidth control machinery
4583  */
4584 
4585 #ifdef CONFIG_CFS_BANDWIDTH
4586 
4587 #ifdef CONFIG_JUMP_LABEL
4588 static struct static_key __cfs_bandwidth_used;
4589 
4590 static inline bool cfs_bandwidth_used(void)
4591 {
4592 	return static_key_false(&__cfs_bandwidth_used);
4593 }
4594 
4595 void cfs_bandwidth_usage_inc(void)
4596 {
4597 	static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used);
4598 }
4599 
4600 void cfs_bandwidth_usage_dec(void)
4601 {
4602 	static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used);
4603 }
4604 #else /* CONFIG_JUMP_LABEL */
4605 static bool cfs_bandwidth_used(void)
4606 {
4607 	return true;
4608 }
4609 
4610 void cfs_bandwidth_usage_inc(void) {}
4611 void cfs_bandwidth_usage_dec(void) {}
4612 #endif /* CONFIG_JUMP_LABEL */
4613 
4614 /*
4615  * default period for cfs group bandwidth.
4616  * default: 0.1s, units: nanoseconds
4617  */
4618 static inline u64 default_cfs_period(void)
4619 {
4620 	return 100000000ULL;
4621 }
4622 
4623 static inline u64 sched_cfs_bandwidth_slice(void)
4624 {
4625 	return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
4626 }
4627 
4628 /*
4629  * Replenish runtime according to assigned quota. We use sched_clock_cpu
4630  * directly instead of rq->clock to avoid adding additional synchronization
4631  * around rq->lock.
4632  *
4633  * requires cfs_b->lock
4634  */
4635 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
4636 {
4637 	if (cfs_b->quota != RUNTIME_INF)
4638 		cfs_b->runtime = cfs_b->quota;
4639 }
4640 
4641 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
4642 {
4643 	return &tg->cfs_bandwidth;
4644 }
4645 
4646 /* returns 0 on failure to allocate runtime */
4647 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,
4648 				   struct cfs_rq *cfs_rq, u64 target_runtime)
4649 {
4650 	u64 min_amount, amount = 0;
4651 
4652 	lockdep_assert_held(&cfs_b->lock);
4653 
4654 	/* note: this is a positive sum as runtime_remaining <= 0 */
4655 	min_amount = target_runtime - cfs_rq->runtime_remaining;
4656 
4657 	if (cfs_b->quota == RUNTIME_INF)
4658 		amount = min_amount;
4659 	else {
4660 		start_cfs_bandwidth(cfs_b);
4661 
4662 		if (cfs_b->runtime > 0) {
4663 			amount = min(cfs_b->runtime, min_amount);
4664 			cfs_b->runtime -= amount;
4665 			cfs_b->idle = 0;
4666 		}
4667 	}
4668 
4669 	cfs_rq->runtime_remaining += amount;
4670 
4671 	return cfs_rq->runtime_remaining > 0;
4672 }
4673 
4674 /* returns 0 on failure to allocate runtime */
4675 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4676 {
4677 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4678 	int ret;
4679 
4680 	raw_spin_lock(&cfs_b->lock);
4681 	ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());
4682 	raw_spin_unlock(&cfs_b->lock);
4683 
4684 	return ret;
4685 }
4686 
4687 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
4688 {
4689 	/* dock delta_exec before expiring quota (as it could span periods) */
4690 	cfs_rq->runtime_remaining -= delta_exec;
4691 
4692 	if (likely(cfs_rq->runtime_remaining > 0))
4693 		return;
4694 
4695 	if (cfs_rq->throttled)
4696 		return;
4697 	/*
4698 	 * if we're unable to extend our runtime we resched so that the active
4699 	 * hierarchy can be throttled
4700 	 */
4701 	if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
4702 		resched_curr(rq_of(cfs_rq));
4703 }
4704 
4705 static __always_inline
4706 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
4707 {
4708 	if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
4709 		return;
4710 
4711 	__account_cfs_rq_runtime(cfs_rq, delta_exec);
4712 }
4713 
4714 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
4715 {
4716 	return cfs_bandwidth_used() && cfs_rq->throttled;
4717 }
4718 
4719 /* check whether cfs_rq, or any parent, is throttled */
4720 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
4721 {
4722 	return cfs_bandwidth_used() && cfs_rq->throttle_count;
4723 }
4724 
4725 /*
4726  * Ensure that neither of the group entities corresponding to src_cpu or
4727  * dest_cpu are members of a throttled hierarchy when performing group
4728  * load-balance operations.
4729  */
4730 static inline int throttled_lb_pair(struct task_group *tg,
4731 				    int src_cpu, int dest_cpu)
4732 {
4733 	struct cfs_rq *src_cfs_rq, *dest_cfs_rq;
4734 
4735 	src_cfs_rq = tg->cfs_rq[src_cpu];
4736 	dest_cfs_rq = tg->cfs_rq[dest_cpu];
4737 
4738 	return throttled_hierarchy(src_cfs_rq) ||
4739 	       throttled_hierarchy(dest_cfs_rq);
4740 }
4741 
4742 static int tg_unthrottle_up(struct task_group *tg, void *data)
4743 {
4744 	struct rq *rq = data;
4745 	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
4746 
4747 	cfs_rq->throttle_count--;
4748 	if (!cfs_rq->throttle_count) {
4749 		cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
4750 					     cfs_rq->throttled_clock_task;
4751 
4752 		/* Add cfs_rq with already running entity in the list */
4753 		if (cfs_rq->nr_running >= 1)
4754 			list_add_leaf_cfs_rq(cfs_rq);
4755 	}
4756 
4757 	return 0;
4758 }
4759 
4760 static int tg_throttle_down(struct task_group *tg, void *data)
4761 {
4762 	struct rq *rq = data;
4763 	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
4764 
4765 	/* group is entering throttled state, stop time */
4766 	if (!cfs_rq->throttle_count) {
4767 		cfs_rq->throttled_clock_task = rq_clock_task(rq);
4768 		list_del_leaf_cfs_rq(cfs_rq);
4769 	}
4770 	cfs_rq->throttle_count++;
4771 
4772 	return 0;
4773 }
4774 
4775 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
4776 {
4777 	struct rq *rq = rq_of(cfs_rq);
4778 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4779 	struct sched_entity *se;
4780 	long task_delta, idle_task_delta, dequeue = 1;
4781 
4782 	raw_spin_lock(&cfs_b->lock);
4783 	/* This will start the period timer if necessary */
4784 	if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {
4785 		/*
4786 		 * We have raced with bandwidth becoming available, and if we
4787 		 * actually throttled the timer might not unthrottle us for an
4788 		 * entire period. We additionally needed to make sure that any
4789 		 * subsequent check_cfs_rq_runtime calls agree not to throttle
4790 		 * us, as we may commit to do cfs put_prev+pick_next, so we ask
4791 		 * for 1ns of runtime rather than just check cfs_b.
4792 		 */
4793 		dequeue = 0;
4794 	} else {
4795 		list_add_tail_rcu(&cfs_rq->throttled_list,
4796 				  &cfs_b->throttled_cfs_rq);
4797 	}
4798 	raw_spin_unlock(&cfs_b->lock);
4799 
4800 	if (!dequeue)
4801 		return false;  /* Throttle no longer required. */
4802 
4803 	se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
4804 
4805 	/* freeze hierarchy runnable averages while throttled */
4806 	rcu_read_lock();
4807 	walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
4808 	rcu_read_unlock();
4809 
4810 	task_delta = cfs_rq->h_nr_running;
4811 	idle_task_delta = cfs_rq->idle_h_nr_running;
4812 	for_each_sched_entity(se) {
4813 		struct cfs_rq *qcfs_rq = cfs_rq_of(se);
4814 		/* throttled entity or throttle-on-deactivate */
4815 		if (!se->on_rq)
4816 			goto done;
4817 
4818 		dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
4819 
4820 		qcfs_rq->h_nr_running -= task_delta;
4821 		qcfs_rq->idle_h_nr_running -= idle_task_delta;
4822 
4823 		if (qcfs_rq->load.weight) {
4824 			/* Avoid re-evaluating load for this entity: */
4825 			se = parent_entity(se);
4826 			break;
4827 		}
4828 	}
4829 
4830 	for_each_sched_entity(se) {
4831 		struct cfs_rq *qcfs_rq = cfs_rq_of(se);
4832 		/* throttled entity or throttle-on-deactivate */
4833 		if (!se->on_rq)
4834 			goto done;
4835 
4836 		update_load_avg(qcfs_rq, se, 0);
4837 		se_update_runnable(se);
4838 
4839 		qcfs_rq->h_nr_running -= task_delta;
4840 		qcfs_rq->idle_h_nr_running -= idle_task_delta;
4841 	}
4842 
4843 	/* At this point se is NULL and we are at root level*/
4844 	sub_nr_running(rq, task_delta);
4845 
4846 done:
4847 	/*
4848 	 * Note: distribution will already see us throttled via the
4849 	 * throttled-list.  rq->lock protects completion.
4850 	 */
4851 	cfs_rq->throttled = 1;
4852 	cfs_rq->throttled_clock = rq_clock(rq);
4853 	return true;
4854 }
4855 
4856 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
4857 {
4858 	struct rq *rq = rq_of(cfs_rq);
4859 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4860 	struct sched_entity *se;
4861 	long task_delta, idle_task_delta;
4862 
4863 	se = cfs_rq->tg->se[cpu_of(rq)];
4864 
4865 	cfs_rq->throttled = 0;
4866 
4867 	update_rq_clock(rq);
4868 
4869 	raw_spin_lock(&cfs_b->lock);
4870 	cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
4871 	list_del_rcu(&cfs_rq->throttled_list);
4872 	raw_spin_unlock(&cfs_b->lock);
4873 
4874 	/* update hierarchical throttle state */
4875 	walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
4876 
4877 	if (!cfs_rq->load.weight)
4878 		return;
4879 
4880 	task_delta = cfs_rq->h_nr_running;
4881 	idle_task_delta = cfs_rq->idle_h_nr_running;
4882 	for_each_sched_entity(se) {
4883 		if (se->on_rq)
4884 			break;
4885 		cfs_rq = cfs_rq_of(se);
4886 		enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP);
4887 
4888 		cfs_rq->h_nr_running += task_delta;
4889 		cfs_rq->idle_h_nr_running += idle_task_delta;
4890 
4891 		/* end evaluation on encountering a throttled cfs_rq */
4892 		if (cfs_rq_throttled(cfs_rq))
4893 			goto unthrottle_throttle;
4894 	}
4895 
4896 	for_each_sched_entity(se) {
4897 		cfs_rq = cfs_rq_of(se);
4898 
4899 		update_load_avg(cfs_rq, se, UPDATE_TG);
4900 		se_update_runnable(se);
4901 
4902 		cfs_rq->h_nr_running += task_delta;
4903 		cfs_rq->idle_h_nr_running += idle_task_delta;
4904 
4905 
4906 		/* end evaluation on encountering a throttled cfs_rq */
4907 		if (cfs_rq_throttled(cfs_rq))
4908 			goto unthrottle_throttle;
4909 
4910 		/*
4911 		 * One parent has been throttled and cfs_rq removed from the
4912 		 * list. Add it back to not break the leaf list.
4913 		 */
4914 		if (throttled_hierarchy(cfs_rq))
4915 			list_add_leaf_cfs_rq(cfs_rq);
4916 	}
4917 
4918 	/* At this point se is NULL and we are at root level*/
4919 	add_nr_running(rq, task_delta);
4920 
4921 unthrottle_throttle:
4922 	/*
4923 	 * The cfs_rq_throttled() breaks in the above iteration can result in
4924 	 * incomplete leaf list maintenance, resulting in triggering the
4925 	 * assertion below.
4926 	 */
4927 	for_each_sched_entity(se) {
4928 		cfs_rq = cfs_rq_of(se);
4929 
4930 		if (list_add_leaf_cfs_rq(cfs_rq))
4931 			break;
4932 	}
4933 
4934 	assert_list_leaf_cfs_rq(rq);
4935 
4936 	/* Determine whether we need to wake up potentially idle CPU: */
4937 	if (rq->curr == rq->idle && rq->cfs.nr_running)
4938 		resched_curr(rq);
4939 }
4940 
4941 static void distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
4942 {
4943 	struct cfs_rq *cfs_rq;
4944 	u64 runtime, remaining = 1;
4945 
4946 	rcu_read_lock();
4947 	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
4948 				throttled_list) {
4949 		struct rq *rq = rq_of(cfs_rq);
4950 		struct rq_flags rf;
4951 
4952 		rq_lock_irqsave(rq, &rf);
4953 		if (!cfs_rq_throttled(cfs_rq))
4954 			goto next;
4955 
4956 		/* By the above check, this should never be true */
4957 		SCHED_WARN_ON(cfs_rq->runtime_remaining > 0);
4958 
4959 		raw_spin_lock(&cfs_b->lock);
4960 		runtime = -cfs_rq->runtime_remaining + 1;
4961 		if (runtime > cfs_b->runtime)
4962 			runtime = cfs_b->runtime;
4963 		cfs_b->runtime -= runtime;
4964 		remaining = cfs_b->runtime;
4965 		raw_spin_unlock(&cfs_b->lock);
4966 
4967 		cfs_rq->runtime_remaining += runtime;
4968 
4969 		/* we check whether we're throttled above */
4970 		if (cfs_rq->runtime_remaining > 0)
4971 			unthrottle_cfs_rq(cfs_rq);
4972 
4973 next:
4974 		rq_unlock_irqrestore(rq, &rf);
4975 
4976 		if (!remaining)
4977 			break;
4978 	}
4979 	rcu_read_unlock();
4980 }
4981 
4982 /*
4983  * Responsible for refilling a task_group's bandwidth and unthrottling its
4984  * cfs_rqs as appropriate. If there has been no activity within the last
4985  * period the timer is deactivated until scheduling resumes; cfs_b->idle is
4986  * used to track this state.
4987  */
4988 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
4989 {
4990 	int throttled;
4991 
4992 	/* no need to continue the timer with no bandwidth constraint */
4993 	if (cfs_b->quota == RUNTIME_INF)
4994 		goto out_deactivate;
4995 
4996 	throttled = !list_empty(&cfs_b->throttled_cfs_rq);
4997 	cfs_b->nr_periods += overrun;
4998 
4999 	/*
5000 	 * idle depends on !throttled (for the case of a large deficit), and if
5001 	 * we're going inactive then everything else can be deferred
5002 	 */
5003 	if (cfs_b->idle && !throttled)
5004 		goto out_deactivate;
5005 
5006 	__refill_cfs_bandwidth_runtime(cfs_b);
5007 
5008 	if (!throttled) {
5009 		/* mark as potentially idle for the upcoming period */
5010 		cfs_b->idle = 1;
5011 		return 0;
5012 	}
5013 
5014 	/* account preceding periods in which throttling occurred */
5015 	cfs_b->nr_throttled += overrun;
5016 
5017 	/*
5018 	 * This check is repeated as we release cfs_b->lock while we unthrottle.
5019 	 */
5020 	while (throttled && cfs_b->runtime > 0) {
5021 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5022 		/* we can't nest cfs_b->lock while distributing bandwidth */
5023 		distribute_cfs_runtime(cfs_b);
5024 		raw_spin_lock_irqsave(&cfs_b->lock, flags);
5025 
5026 		throttled = !list_empty(&cfs_b->throttled_cfs_rq);
5027 	}
5028 
5029 	/*
5030 	 * While we are ensured activity in the period following an
5031 	 * unthrottle, this also covers the case in which the new bandwidth is
5032 	 * insufficient to cover the existing bandwidth deficit.  (Forcing the
5033 	 * timer to remain active while there are any throttled entities.)
5034 	 */
5035 	cfs_b->idle = 0;
5036 
5037 	return 0;
5038 
5039 out_deactivate:
5040 	return 1;
5041 }
5042 
5043 /* a cfs_rq won't donate quota below this amount */
5044 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
5045 /* minimum remaining period time to redistribute slack quota */
5046 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
5047 /* how long we wait to gather additional slack before distributing */
5048 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
5049 
5050 /*
5051  * Are we near the end of the current quota period?
5052  *
5053  * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the
5054  * hrtimer base being cleared by hrtimer_start. In the case of
5055  * migrate_hrtimers, base is never cleared, so we are fine.
5056  */
5057 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
5058 {
5059 	struct hrtimer *refresh_timer = &cfs_b->period_timer;
5060 	u64 remaining;
5061 
5062 	/* if the call-back is running a quota refresh is already occurring */
5063 	if (hrtimer_callback_running(refresh_timer))
5064 		return 1;
5065 
5066 	/* is a quota refresh about to occur? */
5067 	remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer));
5068 	if (remaining < min_expire)
5069 		return 1;
5070 
5071 	return 0;
5072 }
5073 
5074 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
5075 {
5076 	u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
5077 
5078 	/* if there's a quota refresh soon don't bother with slack */
5079 	if (runtime_refresh_within(cfs_b, min_left))
5080 		return;
5081 
5082 	/* don't push forwards an existing deferred unthrottle */
5083 	if (cfs_b->slack_started)
5084 		return;
5085 	cfs_b->slack_started = true;
5086 
5087 	hrtimer_start(&cfs_b->slack_timer,
5088 			ns_to_ktime(cfs_bandwidth_slack_period),
5089 			HRTIMER_MODE_REL);
5090 }
5091 
5092 /* we know any runtime found here is valid as update_curr() precedes return */
5093 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5094 {
5095 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5096 	s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
5097 
5098 	if (slack_runtime <= 0)
5099 		return;
5100 
5101 	raw_spin_lock(&cfs_b->lock);
5102 	if (cfs_b->quota != RUNTIME_INF) {
5103 		cfs_b->runtime += slack_runtime;
5104 
5105 		/* we are under rq->lock, defer unthrottling using a timer */
5106 		if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
5107 		    !list_empty(&cfs_b->throttled_cfs_rq))
5108 			start_cfs_slack_bandwidth(cfs_b);
5109 	}
5110 	raw_spin_unlock(&cfs_b->lock);
5111 
5112 	/* even if it's not valid for return we don't want to try again */
5113 	cfs_rq->runtime_remaining -= slack_runtime;
5114 }
5115 
5116 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5117 {
5118 	if (!cfs_bandwidth_used())
5119 		return;
5120 
5121 	if (!cfs_rq->runtime_enabled || cfs_rq->nr_running)
5122 		return;
5123 
5124 	__return_cfs_rq_runtime(cfs_rq);
5125 }
5126 
5127 /*
5128  * This is done with a timer (instead of inline with bandwidth return) since
5129  * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
5130  */
5131 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
5132 {
5133 	u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
5134 	unsigned long flags;
5135 
5136 	/* confirm we're still not at a refresh boundary */
5137 	raw_spin_lock_irqsave(&cfs_b->lock, flags);
5138 	cfs_b->slack_started = false;
5139 
5140 	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {
5141 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5142 		return;
5143 	}
5144 
5145 	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)
5146 		runtime = cfs_b->runtime;
5147 
5148 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5149 
5150 	if (!runtime)
5151 		return;
5152 
5153 	distribute_cfs_runtime(cfs_b);
5154 }
5155 
5156 /*
5157  * When a group wakes up we want to make sure that its quota is not already
5158  * expired/exceeded, otherwise it may be allowed to steal additional ticks of
5159  * runtime as update_curr() throttling can not trigger until it's on-rq.
5160  */
5161 static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
5162 {
5163 	if (!cfs_bandwidth_used())
5164 		return;
5165 
5166 	/* an active group must be handled by the update_curr()->put() path */
5167 	if (!cfs_rq->runtime_enabled || cfs_rq->curr)
5168 		return;
5169 
5170 	/* ensure the group is not already throttled */
5171 	if (cfs_rq_throttled(cfs_rq))
5172 		return;
5173 
5174 	/* update runtime allocation */
5175 	account_cfs_rq_runtime(cfs_rq, 0);
5176 	if (cfs_rq->runtime_remaining <= 0)
5177 		throttle_cfs_rq(cfs_rq);
5178 }
5179 
5180 static void sync_throttle(struct task_group *tg, int cpu)
5181 {
5182 	struct cfs_rq *pcfs_rq, *cfs_rq;
5183 
5184 	if (!cfs_bandwidth_used())
5185 		return;
5186 
5187 	if (!tg->parent)
5188 		return;
5189 
5190 	cfs_rq = tg->cfs_rq[cpu];
5191 	pcfs_rq = tg->parent->cfs_rq[cpu];
5192 
5193 	cfs_rq->throttle_count = pcfs_rq->throttle_count;
5194 	cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu));
5195 }
5196 
5197 /* conditionally throttle active cfs_rq's from put_prev_entity() */
5198 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5199 {
5200 	if (!cfs_bandwidth_used())
5201 		return false;
5202 
5203 	if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
5204 		return false;
5205 
5206 	/*
5207 	 * it's possible for a throttled entity to be forced into a running
5208 	 * state (e.g. set_curr_task), in this case we're finished.
5209 	 */
5210 	if (cfs_rq_throttled(cfs_rq))
5211 		return true;
5212 
5213 	return throttle_cfs_rq(cfs_rq);
5214 }
5215 
5216 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
5217 {
5218 	struct cfs_bandwidth *cfs_b =
5219 		container_of(timer, struct cfs_bandwidth, slack_timer);
5220 
5221 	do_sched_cfs_slack_timer(cfs_b);
5222 
5223 	return HRTIMER_NORESTART;
5224 }
5225 
5226 extern const u64 max_cfs_quota_period;
5227 
5228 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
5229 {
5230 	struct cfs_bandwidth *cfs_b =
5231 		container_of(timer, struct cfs_bandwidth, period_timer);
5232 	unsigned long flags;
5233 	int overrun;
5234 	int idle = 0;
5235 	int count = 0;
5236 
5237 	raw_spin_lock_irqsave(&cfs_b->lock, flags);
5238 	for (;;) {
5239 		overrun = hrtimer_forward_now(timer, cfs_b->period);
5240 		if (!overrun)
5241 			break;
5242 
5243 		idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);
5244 
5245 		if (++count > 3) {
5246 			u64 new, old = ktime_to_ns(cfs_b->period);
5247 
5248 			/*
5249 			 * Grow period by a factor of 2 to avoid losing precision.
5250 			 * Precision loss in the quota/period ratio can cause __cfs_schedulable
5251 			 * to fail.
5252 			 */
5253 			new = old * 2;
5254 			if (new < max_cfs_quota_period) {
5255 				cfs_b->period = ns_to_ktime(new);
5256 				cfs_b->quota *= 2;
5257 
5258 				pr_warn_ratelimited(
5259 	"cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n",
5260 					smp_processor_id(),
5261 					div_u64(new, NSEC_PER_USEC),
5262 					div_u64(cfs_b->quota, NSEC_PER_USEC));
5263 			} else {
5264 				pr_warn_ratelimited(
5265 	"cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n",
5266 					smp_processor_id(),
5267 					div_u64(old, NSEC_PER_USEC),
5268 					div_u64(cfs_b->quota, NSEC_PER_USEC));
5269 			}
5270 
5271 			/* reset count so we don't come right back in here */
5272 			count = 0;
5273 		}
5274 	}
5275 	if (idle)
5276 		cfs_b->period_active = 0;
5277 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5278 
5279 	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
5280 }
5281 
5282 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5283 {
5284 	raw_spin_lock_init(&cfs_b->lock);
5285 	cfs_b->runtime = 0;
5286 	cfs_b->quota = RUNTIME_INF;
5287 	cfs_b->period = ns_to_ktime(default_cfs_period());
5288 
5289 	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
5290 	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
5291 	cfs_b->period_timer.function = sched_cfs_period_timer;
5292 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5293 	cfs_b->slack_timer.function = sched_cfs_slack_timer;
5294 	cfs_b->slack_started = false;
5295 }
5296 
5297 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5298 {
5299 	cfs_rq->runtime_enabled = 0;
5300 	INIT_LIST_HEAD(&cfs_rq->throttled_list);
5301 }
5302 
5303 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5304 {
5305 	lockdep_assert_held(&cfs_b->lock);
5306 
5307 	if (cfs_b->period_active)
5308 		return;
5309 
5310 	cfs_b->period_active = 1;
5311 	hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);
5312 	hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);
5313 }
5314 
5315 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5316 {
5317 	/* init_cfs_bandwidth() was not called */
5318 	if (!cfs_b->throttled_cfs_rq.next)
5319 		return;
5320 
5321 	hrtimer_cancel(&cfs_b->period_timer);
5322 	hrtimer_cancel(&cfs_b->slack_timer);
5323 }
5324 
5325 /*
5326  * Both these CPU hotplug callbacks race against unregister_fair_sched_group()
5327  *
5328  * The race is harmless, since modifying bandwidth settings of unhooked group
5329  * bits doesn't do much.
5330  */
5331 
5332 /* cpu online callback */
5333 static void __maybe_unused update_runtime_enabled(struct rq *rq)
5334 {
5335 	struct task_group *tg;
5336 
5337 	lockdep_assert_held(&rq->lock);
5338 
5339 	rcu_read_lock();
5340 	list_for_each_entry_rcu(tg, &task_groups, list) {
5341 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
5342 		struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5343 
5344 		raw_spin_lock(&cfs_b->lock);
5345 		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
5346 		raw_spin_unlock(&cfs_b->lock);
5347 	}
5348 	rcu_read_unlock();
5349 }
5350 
5351 /* cpu offline callback */
5352 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
5353 {
5354 	struct task_group *tg;
5355 
5356 	lockdep_assert_held(&rq->lock);
5357 
5358 	rcu_read_lock();
5359 	list_for_each_entry_rcu(tg, &task_groups, list) {
5360 		struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5361 
5362 		if (!cfs_rq->runtime_enabled)
5363 			continue;
5364 
5365 		/*
5366 		 * clock_task is not advancing so we just need to make sure
5367 		 * there's some valid quota amount
5368 		 */
5369 		cfs_rq->runtime_remaining = 1;
5370 		/*
5371 		 * Offline rq is schedulable till CPU is completely disabled
5372 		 * in take_cpu_down(), so we prevent new cfs throttling here.
5373 		 */
5374 		cfs_rq->runtime_enabled = 0;
5375 
5376 		if (cfs_rq_throttled(cfs_rq))
5377 			unthrottle_cfs_rq(cfs_rq);
5378 	}
5379 	rcu_read_unlock();
5380 }
5381 
5382 #else /* CONFIG_CFS_BANDWIDTH */
5383 
5384 static inline bool cfs_bandwidth_used(void)
5385 {
5386 	return false;
5387 }
5388 
5389 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {}
5390 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; }
5391 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
5392 static inline void sync_throttle(struct task_group *tg, int cpu) {}
5393 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
5394 
5395 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
5396 {
5397 	return 0;
5398 }
5399 
5400 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
5401 {
5402 	return 0;
5403 }
5404 
5405 static inline int throttled_lb_pair(struct task_group *tg,
5406 				    int src_cpu, int dest_cpu)
5407 {
5408 	return 0;
5409 }
5410 
5411 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
5412 
5413 #ifdef CONFIG_FAIR_GROUP_SCHED
5414 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
5415 #endif
5416 
5417 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
5418 {
5419 	return NULL;
5420 }
5421 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
5422 static inline void update_runtime_enabled(struct rq *rq) {}
5423 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
5424 
5425 #endif /* CONFIG_CFS_BANDWIDTH */
5426 
5427 /**************************************************
5428  * CFS operations on tasks:
5429  */
5430 
5431 #ifdef CONFIG_SCHED_HRTICK
5432 static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
5433 {
5434 	struct sched_entity *se = &p->se;
5435 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
5436 
5437 	SCHED_WARN_ON(task_rq(p) != rq);
5438 
5439 	if (rq->cfs.h_nr_running > 1) {
5440 		u64 slice = sched_slice(cfs_rq, se);
5441 		u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
5442 		s64 delta = slice - ran;
5443 
5444 		if (delta < 0) {
5445 			if (task_current(rq, p))
5446 				resched_curr(rq);
5447 			return;
5448 		}
5449 		hrtick_start(rq, delta);
5450 	}
5451 }
5452 
5453 /*
5454  * called from enqueue/dequeue and updates the hrtick when the
5455  * current task is from our class and nr_running is low enough
5456  * to matter.
5457  */
5458 static void hrtick_update(struct rq *rq)
5459 {
5460 	struct task_struct *curr = rq->curr;
5461 
5462 	if (!hrtick_enabled_fair(rq) || curr->sched_class != &fair_sched_class)
5463 		return;
5464 
5465 	if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency)
5466 		hrtick_start_fair(rq, curr);
5467 }
5468 #else /* !CONFIG_SCHED_HRTICK */
5469 static inline void
5470 hrtick_start_fair(struct rq *rq, struct task_struct *p)
5471 {
5472 }
5473 
5474 static inline void hrtick_update(struct rq *rq)
5475 {
5476 }
5477 #endif
5478 
5479 #ifdef CONFIG_SMP
5480 static inline unsigned long cpu_util(int cpu);
5481 
5482 static inline bool cpu_overutilized(int cpu)
5483 {
5484 	return !fits_capacity(cpu_util(cpu), capacity_of(cpu));
5485 }
5486 
5487 static inline void update_overutilized_status(struct rq *rq)
5488 {
5489 	if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) {
5490 		WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED);
5491 		trace_sched_overutilized_tp(rq->rd, SG_OVERUTILIZED);
5492 	}
5493 }
5494 #else
5495 static inline void update_overutilized_status(struct rq *rq) { }
5496 #endif
5497 
5498 /* Runqueue only has SCHED_IDLE tasks enqueued */
5499 static int sched_idle_rq(struct rq *rq)
5500 {
5501 	return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running &&
5502 			rq->nr_running);
5503 }
5504 
5505 #ifdef CONFIG_SMP
5506 static int sched_idle_cpu(int cpu)
5507 {
5508 	return sched_idle_rq(cpu_rq(cpu));
5509 }
5510 #endif
5511 
5512 /*
5513  * The enqueue_task method is called before nr_running is
5514  * increased. Here we update the fair scheduling stats and
5515  * then put the task into the rbtree:
5516  */
5517 static void
5518 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
5519 {
5520 	struct cfs_rq *cfs_rq;
5521 	struct sched_entity *se = &p->se;
5522 	int idle_h_nr_running = task_has_idle_policy(p);
5523 	int task_new = !(flags & ENQUEUE_WAKEUP);
5524 
5525 	/*
5526 	 * The code below (indirectly) updates schedutil which looks at
5527 	 * the cfs_rq utilization to select a frequency.
5528 	 * Let's add the task's estimated utilization to the cfs_rq's
5529 	 * estimated utilization, before we update schedutil.
5530 	 */
5531 	util_est_enqueue(&rq->cfs, p);
5532 
5533 	/*
5534 	 * If in_iowait is set, the code below may not trigger any cpufreq
5535 	 * utilization updates, so do it here explicitly with the IOWAIT flag
5536 	 * passed.
5537 	 */
5538 	if (p->in_iowait)
5539 		cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
5540 
5541 	for_each_sched_entity(se) {
5542 		if (se->on_rq)
5543 			break;
5544 		cfs_rq = cfs_rq_of(se);
5545 		enqueue_entity(cfs_rq, se, flags);
5546 
5547 		cfs_rq->h_nr_running++;
5548 		cfs_rq->idle_h_nr_running += idle_h_nr_running;
5549 
5550 		/* end evaluation on encountering a throttled cfs_rq */
5551 		if (cfs_rq_throttled(cfs_rq))
5552 			goto enqueue_throttle;
5553 
5554 		flags = ENQUEUE_WAKEUP;
5555 	}
5556 
5557 	for_each_sched_entity(se) {
5558 		cfs_rq = cfs_rq_of(se);
5559 
5560 		update_load_avg(cfs_rq, se, UPDATE_TG);
5561 		se_update_runnable(se);
5562 		update_cfs_group(se);
5563 
5564 		cfs_rq->h_nr_running++;
5565 		cfs_rq->idle_h_nr_running += idle_h_nr_running;
5566 
5567 		/* end evaluation on encountering a throttled cfs_rq */
5568 		if (cfs_rq_throttled(cfs_rq))
5569 			goto enqueue_throttle;
5570 
5571                /*
5572                 * One parent has been throttled and cfs_rq removed from the
5573                 * list. Add it back to not break the leaf list.
5574                 */
5575                if (throttled_hierarchy(cfs_rq))
5576                        list_add_leaf_cfs_rq(cfs_rq);
5577 	}
5578 
5579 	/* At this point se is NULL and we are at root level*/
5580 	add_nr_running(rq, 1);
5581 
5582 	/*
5583 	 * Since new tasks are assigned an initial util_avg equal to
5584 	 * half of the spare capacity of their CPU, tiny tasks have the
5585 	 * ability to cross the overutilized threshold, which will
5586 	 * result in the load balancer ruining all the task placement
5587 	 * done by EAS. As a way to mitigate that effect, do not account
5588 	 * for the first enqueue operation of new tasks during the
5589 	 * overutilized flag detection.
5590 	 *
5591 	 * A better way of solving this problem would be to wait for
5592 	 * the PELT signals of tasks to converge before taking them
5593 	 * into account, but that is not straightforward to implement,
5594 	 * and the following generally works well enough in practice.
5595 	 */
5596 	if (!task_new)
5597 		update_overutilized_status(rq);
5598 
5599 enqueue_throttle:
5600 	if (cfs_bandwidth_used()) {
5601 		/*
5602 		 * When bandwidth control is enabled; the cfs_rq_throttled()
5603 		 * breaks in the above iteration can result in incomplete
5604 		 * leaf list maintenance, resulting in triggering the assertion
5605 		 * below.
5606 		 */
5607 		for_each_sched_entity(se) {
5608 			cfs_rq = cfs_rq_of(se);
5609 
5610 			if (list_add_leaf_cfs_rq(cfs_rq))
5611 				break;
5612 		}
5613 	}
5614 
5615 	assert_list_leaf_cfs_rq(rq);
5616 
5617 	hrtick_update(rq);
5618 }
5619 
5620 static void set_next_buddy(struct sched_entity *se);
5621 
5622 /*
5623  * The dequeue_task method is called before nr_running is
5624  * decreased. We remove the task from the rbtree and
5625  * update the fair scheduling stats:
5626  */
5627 static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
5628 {
5629 	struct cfs_rq *cfs_rq;
5630 	struct sched_entity *se = &p->se;
5631 	int task_sleep = flags & DEQUEUE_SLEEP;
5632 	int idle_h_nr_running = task_has_idle_policy(p);
5633 	bool was_sched_idle = sched_idle_rq(rq);
5634 
5635 	util_est_dequeue(&rq->cfs, p);
5636 
5637 	for_each_sched_entity(se) {
5638 		cfs_rq = cfs_rq_of(se);
5639 		dequeue_entity(cfs_rq, se, flags);
5640 
5641 		cfs_rq->h_nr_running--;
5642 		cfs_rq->idle_h_nr_running -= idle_h_nr_running;
5643 
5644 		/* end evaluation on encountering a throttled cfs_rq */
5645 		if (cfs_rq_throttled(cfs_rq))
5646 			goto dequeue_throttle;
5647 
5648 		/* Don't dequeue parent if it has other entities besides us */
5649 		if (cfs_rq->load.weight) {
5650 			/* Avoid re-evaluating load for this entity: */
5651 			se = parent_entity(se);
5652 			/*
5653 			 * Bias pick_next to pick a task from this cfs_rq, as
5654 			 * p is sleeping when it is within its sched_slice.
5655 			 */
5656 			if (task_sleep && se && !throttled_hierarchy(cfs_rq))
5657 				set_next_buddy(se);
5658 			break;
5659 		}
5660 		flags |= DEQUEUE_SLEEP;
5661 	}
5662 
5663 	for_each_sched_entity(se) {
5664 		cfs_rq = cfs_rq_of(se);
5665 
5666 		update_load_avg(cfs_rq, se, UPDATE_TG);
5667 		se_update_runnable(se);
5668 		update_cfs_group(se);
5669 
5670 		cfs_rq->h_nr_running--;
5671 		cfs_rq->idle_h_nr_running -= idle_h_nr_running;
5672 
5673 		/* end evaluation on encountering a throttled cfs_rq */
5674 		if (cfs_rq_throttled(cfs_rq))
5675 			goto dequeue_throttle;
5676 
5677 	}
5678 
5679 	/* At this point se is NULL and we are at root level*/
5680 	sub_nr_running(rq, 1);
5681 
5682 	/* balance early to pull high priority tasks */
5683 	if (unlikely(!was_sched_idle && sched_idle_rq(rq)))
5684 		rq->next_balance = jiffies;
5685 
5686 dequeue_throttle:
5687 	util_est_update(&rq->cfs, p, task_sleep);
5688 	hrtick_update(rq);
5689 }
5690 
5691 #ifdef CONFIG_SMP
5692 
5693 /* Working cpumask for: load_balance, load_balance_newidle. */
5694 DEFINE_PER_CPU(cpumask_var_t, load_balance_mask);
5695 DEFINE_PER_CPU(cpumask_var_t, select_idle_mask);
5696 
5697 #ifdef CONFIG_NO_HZ_COMMON
5698 
5699 static struct {
5700 	cpumask_var_t idle_cpus_mask;
5701 	atomic_t nr_cpus;
5702 	int has_blocked;		/* Idle CPUS has blocked load */
5703 	unsigned long next_balance;     /* in jiffy units */
5704 	unsigned long next_blocked;	/* Next update of blocked load in jiffies */
5705 } nohz ____cacheline_aligned;
5706 
5707 #endif /* CONFIG_NO_HZ_COMMON */
5708 
5709 static unsigned long cpu_load(struct rq *rq)
5710 {
5711 	return cfs_rq_load_avg(&rq->cfs);
5712 }
5713 
5714 /*
5715  * cpu_load_without - compute CPU load without any contributions from *p
5716  * @cpu: the CPU which load is requested
5717  * @p: the task which load should be discounted
5718  *
5719  * The load of a CPU is defined by the load of tasks currently enqueued on that
5720  * CPU as well as tasks which are currently sleeping after an execution on that
5721  * CPU.
5722  *
5723  * This method returns the load of the specified CPU by discounting the load of
5724  * the specified task, whenever the task is currently contributing to the CPU
5725  * load.
5726  */
5727 static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p)
5728 {
5729 	struct cfs_rq *cfs_rq;
5730 	unsigned int load;
5731 
5732 	/* Task has no contribution or is new */
5733 	if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
5734 		return cpu_load(rq);
5735 
5736 	cfs_rq = &rq->cfs;
5737 	load = READ_ONCE(cfs_rq->avg.load_avg);
5738 
5739 	/* Discount task's util from CPU's util */
5740 	lsub_positive(&load, task_h_load(p));
5741 
5742 	return load;
5743 }
5744 
5745 static unsigned long cpu_runnable(struct rq *rq)
5746 {
5747 	return cfs_rq_runnable_avg(&rq->cfs);
5748 }
5749 
5750 static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p)
5751 {
5752 	struct cfs_rq *cfs_rq;
5753 	unsigned int runnable;
5754 
5755 	/* Task has no contribution or is new */
5756 	if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
5757 		return cpu_runnable(rq);
5758 
5759 	cfs_rq = &rq->cfs;
5760 	runnable = READ_ONCE(cfs_rq->avg.runnable_avg);
5761 
5762 	/* Discount task's runnable from CPU's runnable */
5763 	lsub_positive(&runnable, p->se.avg.runnable_avg);
5764 
5765 	return runnable;
5766 }
5767 
5768 static unsigned long capacity_of(int cpu)
5769 {
5770 	return cpu_rq(cpu)->cpu_capacity;
5771 }
5772 
5773 static void record_wakee(struct task_struct *p)
5774 {
5775 	/*
5776 	 * Only decay a single time; tasks that have less then 1 wakeup per
5777 	 * jiffy will not have built up many flips.
5778 	 */
5779 	if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) {
5780 		current->wakee_flips >>= 1;
5781 		current->wakee_flip_decay_ts = jiffies;
5782 	}
5783 
5784 	if (current->last_wakee != p) {
5785 		current->last_wakee = p;
5786 		current->wakee_flips++;
5787 	}
5788 }
5789 
5790 /*
5791  * Detect M:N waker/wakee relationships via a switching-frequency heuristic.
5792  *
5793  * A waker of many should wake a different task than the one last awakened
5794  * at a frequency roughly N times higher than one of its wakees.
5795  *
5796  * In order to determine whether we should let the load spread vs consolidating
5797  * to shared cache, we look for a minimum 'flip' frequency of llc_size in one
5798  * partner, and a factor of lls_size higher frequency in the other.
5799  *
5800  * With both conditions met, we can be relatively sure that the relationship is
5801  * non-monogamous, with partner count exceeding socket size.
5802  *
5803  * Waker/wakee being client/server, worker/dispatcher, interrupt source or
5804  * whatever is irrelevant, spread criteria is apparent partner count exceeds
5805  * socket size.
5806  */
5807 static int wake_wide(struct task_struct *p)
5808 {
5809 	unsigned int master = current->wakee_flips;
5810 	unsigned int slave = p->wakee_flips;
5811 	int factor = __this_cpu_read(sd_llc_size);
5812 
5813 	if (master < slave)
5814 		swap(master, slave);
5815 	if (slave < factor || master < slave * factor)
5816 		return 0;
5817 	return 1;
5818 }
5819 
5820 /*
5821  * The purpose of wake_affine() is to quickly determine on which CPU we can run
5822  * soonest. For the purpose of speed we only consider the waking and previous
5823  * CPU.
5824  *
5825  * wake_affine_idle() - only considers 'now', it check if the waking CPU is
5826  *			cache-affine and is (or	will be) idle.
5827  *
5828  * wake_affine_weight() - considers the weight to reflect the average
5829  *			  scheduling latency of the CPUs. This seems to work
5830  *			  for the overloaded case.
5831  */
5832 static int
5833 wake_affine_idle(int this_cpu, int prev_cpu, int sync)
5834 {
5835 	/*
5836 	 * If this_cpu is idle, it implies the wakeup is from interrupt
5837 	 * context. Only allow the move if cache is shared. Otherwise an
5838 	 * interrupt intensive workload could force all tasks onto one
5839 	 * node depending on the IO topology or IRQ affinity settings.
5840 	 *
5841 	 * If the prev_cpu is idle and cache affine then avoid a migration.
5842 	 * There is no guarantee that the cache hot data from an interrupt
5843 	 * is more important than cache hot data on the prev_cpu and from
5844 	 * a cpufreq perspective, it's better to have higher utilisation
5845 	 * on one CPU.
5846 	 */
5847 	if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu))
5848 		return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu;
5849 
5850 	if (sync && cpu_rq(this_cpu)->nr_running == 1)
5851 		return this_cpu;
5852 
5853 	if (available_idle_cpu(prev_cpu))
5854 		return prev_cpu;
5855 
5856 	return nr_cpumask_bits;
5857 }
5858 
5859 static int
5860 wake_affine_weight(struct sched_domain *sd, struct task_struct *p,
5861 		   int this_cpu, int prev_cpu, int sync)
5862 {
5863 	s64 this_eff_load, prev_eff_load;
5864 	unsigned long task_load;
5865 
5866 	this_eff_load = cpu_load(cpu_rq(this_cpu));
5867 
5868 	if (sync) {
5869 		unsigned long current_load = task_h_load(current);
5870 
5871 		if (current_load > this_eff_load)
5872 			return this_cpu;
5873 
5874 		this_eff_load -= current_load;
5875 	}
5876 
5877 	task_load = task_h_load(p);
5878 
5879 	this_eff_load += task_load;
5880 	if (sched_feat(WA_BIAS))
5881 		this_eff_load *= 100;
5882 	this_eff_load *= capacity_of(prev_cpu);
5883 
5884 	prev_eff_load = cpu_load(cpu_rq(prev_cpu));
5885 	prev_eff_load -= task_load;
5886 	if (sched_feat(WA_BIAS))
5887 		prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2;
5888 	prev_eff_load *= capacity_of(this_cpu);
5889 
5890 	/*
5891 	 * If sync, adjust the weight of prev_eff_load such that if
5892 	 * prev_eff == this_eff that select_idle_sibling() will consider
5893 	 * stacking the wakee on top of the waker if no other CPU is
5894 	 * idle.
5895 	 */
5896 	if (sync)
5897 		prev_eff_load += 1;
5898 
5899 	return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits;
5900 }
5901 
5902 static int wake_affine(struct sched_domain *sd, struct task_struct *p,
5903 		       int this_cpu, int prev_cpu, int sync)
5904 {
5905 	int target = nr_cpumask_bits;
5906 
5907 	if (sched_feat(WA_IDLE))
5908 		target = wake_affine_idle(this_cpu, prev_cpu, sync);
5909 
5910 	if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits)
5911 		target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync);
5912 
5913 	schedstat_inc(p->se.statistics.nr_wakeups_affine_attempts);
5914 	if (target == nr_cpumask_bits)
5915 		return prev_cpu;
5916 
5917 	schedstat_inc(sd->ttwu_move_affine);
5918 	schedstat_inc(p->se.statistics.nr_wakeups_affine);
5919 	return target;
5920 }
5921 
5922 static struct sched_group *
5923 find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu);
5924 
5925 /*
5926  * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group.
5927  */
5928 static int
5929 find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
5930 {
5931 	unsigned long load, min_load = ULONG_MAX;
5932 	unsigned int min_exit_latency = UINT_MAX;
5933 	u64 latest_idle_timestamp = 0;
5934 	int least_loaded_cpu = this_cpu;
5935 	int shallowest_idle_cpu = -1;
5936 	int i;
5937 
5938 	/* Check if we have any choice: */
5939 	if (group->group_weight == 1)
5940 		return cpumask_first(sched_group_span(group));
5941 
5942 	/* Traverse only the allowed CPUs */
5943 	for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
5944 		if (sched_idle_cpu(i))
5945 			return i;
5946 
5947 		if (available_idle_cpu(i)) {
5948 			struct rq *rq = cpu_rq(i);
5949 			struct cpuidle_state *idle = idle_get_state(rq);
5950 			if (idle && idle->exit_latency < min_exit_latency) {
5951 				/*
5952 				 * We give priority to a CPU whose idle state
5953 				 * has the smallest exit latency irrespective
5954 				 * of any idle timestamp.
5955 				 */
5956 				min_exit_latency = idle->exit_latency;
5957 				latest_idle_timestamp = rq->idle_stamp;
5958 				shallowest_idle_cpu = i;
5959 			} else if ((!idle || idle->exit_latency == min_exit_latency) &&
5960 				   rq->idle_stamp > latest_idle_timestamp) {
5961 				/*
5962 				 * If equal or no active idle state, then
5963 				 * the most recently idled CPU might have
5964 				 * a warmer cache.
5965 				 */
5966 				latest_idle_timestamp = rq->idle_stamp;
5967 				shallowest_idle_cpu = i;
5968 			}
5969 		} else if (shallowest_idle_cpu == -1) {
5970 			load = cpu_load(cpu_rq(i));
5971 			if (load < min_load) {
5972 				min_load = load;
5973 				least_loaded_cpu = i;
5974 			}
5975 		}
5976 	}
5977 
5978 	return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu;
5979 }
5980 
5981 static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p,
5982 				  int cpu, int prev_cpu, int sd_flag)
5983 {
5984 	int new_cpu = cpu;
5985 
5986 	if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr))
5987 		return prev_cpu;
5988 
5989 	/*
5990 	 * We need task's util for cpu_util_without, sync it up to
5991 	 * prev_cpu's last_update_time.
5992 	 */
5993 	if (!(sd_flag & SD_BALANCE_FORK))
5994 		sync_entity_load_avg(&p->se);
5995 
5996 	while (sd) {
5997 		struct sched_group *group;
5998 		struct sched_domain *tmp;
5999 		int weight;
6000 
6001 		if (!(sd->flags & sd_flag)) {
6002 			sd = sd->child;
6003 			continue;
6004 		}
6005 
6006 		group = find_idlest_group(sd, p, cpu);
6007 		if (!group) {
6008 			sd = sd->child;
6009 			continue;
6010 		}
6011 
6012 		new_cpu = find_idlest_group_cpu(group, p, cpu);
6013 		if (new_cpu == cpu) {
6014 			/* Now try balancing at a lower domain level of 'cpu': */
6015 			sd = sd->child;
6016 			continue;
6017 		}
6018 
6019 		/* Now try balancing at a lower domain level of 'new_cpu': */
6020 		cpu = new_cpu;
6021 		weight = sd->span_weight;
6022 		sd = NULL;
6023 		for_each_domain(cpu, tmp) {
6024 			if (weight <= tmp->span_weight)
6025 				break;
6026 			if (tmp->flags & sd_flag)
6027 				sd = tmp;
6028 		}
6029 	}
6030 
6031 	return new_cpu;
6032 }
6033 
6034 static inline int __select_idle_cpu(int cpu)
6035 {
6036 	if (available_idle_cpu(cpu) || sched_idle_cpu(cpu))
6037 		return cpu;
6038 
6039 	return -1;
6040 }
6041 
6042 #ifdef CONFIG_SCHED_SMT
6043 DEFINE_STATIC_KEY_FALSE(sched_smt_present);
6044 EXPORT_SYMBOL_GPL(sched_smt_present);
6045 
6046 static inline void set_idle_cores(int cpu, int val)
6047 {
6048 	struct sched_domain_shared *sds;
6049 
6050 	sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
6051 	if (sds)
6052 		WRITE_ONCE(sds->has_idle_cores, val);
6053 }
6054 
6055 static inline bool test_idle_cores(int cpu, bool def)
6056 {
6057 	struct sched_domain_shared *sds;
6058 
6059 	sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
6060 	if (sds)
6061 		return READ_ONCE(sds->has_idle_cores);
6062 
6063 	return def;
6064 }
6065 
6066 /*
6067  * Scans the local SMT mask to see if the entire core is idle, and records this
6068  * information in sd_llc_shared->has_idle_cores.
6069  *
6070  * Since SMT siblings share all cache levels, inspecting this limited remote
6071  * state should be fairly cheap.
6072  */
6073 void __update_idle_core(struct rq *rq)
6074 {
6075 	int core = cpu_of(rq);
6076 	int cpu;
6077 
6078 	rcu_read_lock();
6079 	if (test_idle_cores(core, true))
6080 		goto unlock;
6081 
6082 	for_each_cpu(cpu, cpu_smt_mask(core)) {
6083 		if (cpu == core)
6084 			continue;
6085 
6086 		if (!available_idle_cpu(cpu))
6087 			goto unlock;
6088 	}
6089 
6090 	set_idle_cores(core, 1);
6091 unlock:
6092 	rcu_read_unlock();
6093 }
6094 
6095 /*
6096  * Scan the entire LLC domain for idle cores; this dynamically switches off if
6097  * there are no idle cores left in the system; tracked through
6098  * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above.
6099  */
6100 static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
6101 {
6102 	bool idle = true;
6103 	int cpu;
6104 
6105 	if (!static_branch_likely(&sched_smt_present))
6106 		return __select_idle_cpu(core);
6107 
6108 	for_each_cpu(cpu, cpu_smt_mask(core)) {
6109 		if (!available_idle_cpu(cpu)) {
6110 			idle = false;
6111 			if (*idle_cpu == -1) {
6112 				if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, p->cpus_ptr)) {
6113 					*idle_cpu = cpu;
6114 					break;
6115 				}
6116 				continue;
6117 			}
6118 			break;
6119 		}
6120 		if (*idle_cpu == -1 && cpumask_test_cpu(cpu, p->cpus_ptr))
6121 			*idle_cpu = cpu;
6122 	}
6123 
6124 	if (idle)
6125 		return core;
6126 
6127 	cpumask_andnot(cpus, cpus, cpu_smt_mask(core));
6128 	return -1;
6129 }
6130 
6131 /*
6132  * Scan the local SMT mask for idle CPUs.
6133  */
6134 static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
6135 {
6136 	int cpu;
6137 
6138 	for_each_cpu(cpu, cpu_smt_mask(target)) {
6139 		if (!cpumask_test_cpu(cpu, p->cpus_ptr) ||
6140 		    !cpumask_test_cpu(cpu, sched_domain_span(sd)))
6141 			continue;
6142 		if (available_idle_cpu(cpu) || sched_idle_cpu(cpu))
6143 			return cpu;
6144 	}
6145 
6146 	return -1;
6147 }
6148 
6149 #else /* CONFIG_SCHED_SMT */
6150 
6151 static inline void set_idle_cores(int cpu, int val)
6152 {
6153 }
6154 
6155 static inline bool test_idle_cores(int cpu, bool def)
6156 {
6157 	return def;
6158 }
6159 
6160 static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
6161 {
6162 	return __select_idle_cpu(core);
6163 }
6164 
6165 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
6166 {
6167 	return -1;
6168 }
6169 
6170 #endif /* CONFIG_SCHED_SMT */
6171 
6172 /*
6173  * Scan the LLC domain for idle CPUs; this is dynamically regulated by
6174  * comparing the average scan cost (tracked in sd->avg_scan_cost) against the
6175  * average idle time for this rq (as found in rq->avg_idle).
6176  */
6177 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target)
6178 {
6179 	struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask);
6180 	int i, cpu, idle_cpu = -1, nr = INT_MAX;
6181 	int this = smp_processor_id();
6182 	struct sched_domain *this_sd;
6183 	u64 time;
6184 
6185 	this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc));
6186 	if (!this_sd)
6187 		return -1;
6188 
6189 	cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
6190 
6191 	if (sched_feat(SIS_PROP) && !has_idle_core) {
6192 		u64 avg_cost, avg_idle, span_avg;
6193 
6194 		/*
6195 		 * Due to large variance we need a large fuzz factor;
6196 		 * hackbench in particularly is sensitive here.
6197 		 */
6198 		avg_idle = this_rq()->avg_idle / 512;
6199 		avg_cost = this_sd->avg_scan_cost + 1;
6200 
6201 		span_avg = sd->span_weight * avg_idle;
6202 		if (span_avg > 4*avg_cost)
6203 			nr = div_u64(span_avg, avg_cost);
6204 		else
6205 			nr = 4;
6206 
6207 		time = cpu_clock(this);
6208 	}
6209 
6210 	for_each_cpu_wrap(cpu, cpus, target) {
6211 		if (has_idle_core) {
6212 			i = select_idle_core(p, cpu, cpus, &idle_cpu);
6213 			if ((unsigned int)i < nr_cpumask_bits)
6214 				return i;
6215 
6216 		} else {
6217 			if (!--nr)
6218 				return -1;
6219 			idle_cpu = __select_idle_cpu(cpu);
6220 			if ((unsigned int)idle_cpu < nr_cpumask_bits)
6221 				break;
6222 		}
6223 	}
6224 
6225 	if (has_idle_core)
6226 		set_idle_cores(target, false);
6227 
6228 	if (sched_feat(SIS_PROP) && !has_idle_core) {
6229 		time = cpu_clock(this) - time;
6230 		update_avg(&this_sd->avg_scan_cost, time);
6231 	}
6232 
6233 	return idle_cpu;
6234 }
6235 
6236 /*
6237  * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which
6238  * the task fits. If no CPU is big enough, but there are idle ones, try to
6239  * maximize capacity.
6240  */
6241 static int
6242 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
6243 {
6244 	unsigned long task_util, best_cap = 0;
6245 	int cpu, best_cpu = -1;
6246 	struct cpumask *cpus;
6247 
6248 	cpus = this_cpu_cpumask_var_ptr(select_idle_mask);
6249 	cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
6250 
6251 	task_util = uclamp_task_util(p);
6252 
6253 	for_each_cpu_wrap(cpu, cpus, target) {
6254 		unsigned long cpu_cap = capacity_of(cpu);
6255 
6256 		if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu))
6257 			continue;
6258 		if (fits_capacity(task_util, cpu_cap))
6259 			return cpu;
6260 
6261 		if (cpu_cap > best_cap) {
6262 			best_cap = cpu_cap;
6263 			best_cpu = cpu;
6264 		}
6265 	}
6266 
6267 	return best_cpu;
6268 }
6269 
6270 static inline bool asym_fits_capacity(int task_util, int cpu)
6271 {
6272 	if (static_branch_unlikely(&sched_asym_cpucapacity))
6273 		return fits_capacity(task_util, capacity_of(cpu));
6274 
6275 	return true;
6276 }
6277 
6278 /*
6279  * Try and locate an idle core/thread in the LLC cache domain.
6280  */
6281 static int select_idle_sibling(struct task_struct *p, int prev, int target)
6282 {
6283 	bool has_idle_core = false;
6284 	struct sched_domain *sd;
6285 	unsigned long task_util;
6286 	int i, recent_used_cpu;
6287 
6288 	/*
6289 	 * On asymmetric system, update task utilization because we will check
6290 	 * that the task fits with cpu's capacity.
6291 	 */
6292 	if (static_branch_unlikely(&sched_asym_cpucapacity)) {
6293 		sync_entity_load_avg(&p->se);
6294 		task_util = uclamp_task_util(p);
6295 	}
6296 
6297 	if ((available_idle_cpu(target) || sched_idle_cpu(target)) &&
6298 	    asym_fits_capacity(task_util, target))
6299 		return target;
6300 
6301 	/*
6302 	 * If the previous CPU is cache affine and idle, don't be stupid:
6303 	 */
6304 	if (prev != target && cpus_share_cache(prev, target) &&
6305 	    (available_idle_cpu(prev) || sched_idle_cpu(prev)) &&
6306 	    asym_fits_capacity(task_util, prev))
6307 		return prev;
6308 
6309 	/*
6310 	 * Allow a per-cpu kthread to stack with the wakee if the
6311 	 * kworker thread and the tasks previous CPUs are the same.
6312 	 * The assumption is that the wakee queued work for the
6313 	 * per-cpu kthread that is now complete and the wakeup is
6314 	 * essentially a sync wakeup. An obvious example of this
6315 	 * pattern is IO completions.
6316 	 */
6317 	if (is_per_cpu_kthread(current) &&
6318 	    prev == smp_processor_id() &&
6319 	    this_rq()->nr_running <= 1) {
6320 		return prev;
6321 	}
6322 
6323 	/* Check a recently used CPU as a potential idle candidate: */
6324 	recent_used_cpu = p->recent_used_cpu;
6325 	if (recent_used_cpu != prev &&
6326 	    recent_used_cpu != target &&
6327 	    cpus_share_cache(recent_used_cpu, target) &&
6328 	    (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) &&
6329 	    cpumask_test_cpu(p->recent_used_cpu, p->cpus_ptr) &&
6330 	    asym_fits_capacity(task_util, recent_used_cpu)) {
6331 		/*
6332 		 * Replace recent_used_cpu with prev as it is a potential
6333 		 * candidate for the next wake:
6334 		 */
6335 		p->recent_used_cpu = prev;
6336 		return recent_used_cpu;
6337 	}
6338 
6339 	/*
6340 	 * For asymmetric CPU capacity systems, our domain of interest is
6341 	 * sd_asym_cpucapacity rather than sd_llc.
6342 	 */
6343 	if (static_branch_unlikely(&sched_asym_cpucapacity)) {
6344 		sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target));
6345 		/*
6346 		 * On an asymmetric CPU capacity system where an exclusive
6347 		 * cpuset defines a symmetric island (i.e. one unique
6348 		 * capacity_orig value through the cpuset), the key will be set
6349 		 * but the CPUs within that cpuset will not have a domain with
6350 		 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric
6351 		 * capacity path.
6352 		 */
6353 		if (sd) {
6354 			i = select_idle_capacity(p, sd, target);
6355 			return ((unsigned)i < nr_cpumask_bits) ? i : target;
6356 		}
6357 	}
6358 
6359 	sd = rcu_dereference(per_cpu(sd_llc, target));
6360 	if (!sd)
6361 		return target;
6362 
6363 	if (sched_smt_active()) {
6364 		has_idle_core = test_idle_cores(target, false);
6365 
6366 		if (!has_idle_core && cpus_share_cache(prev, target)) {
6367 			i = select_idle_smt(p, sd, prev);
6368 			if ((unsigned int)i < nr_cpumask_bits)
6369 				return i;
6370 		}
6371 	}
6372 
6373 	i = select_idle_cpu(p, sd, has_idle_core, target);
6374 	if ((unsigned)i < nr_cpumask_bits)
6375 		return i;
6376 
6377 	return target;
6378 }
6379 
6380 /**
6381  * cpu_util - Estimates the amount of capacity of a CPU used by CFS tasks.
6382  * @cpu: the CPU to get the utilization of
6383  *
6384  * The unit of the return value must be the one of capacity so we can compare
6385  * the utilization with the capacity of the CPU that is available for CFS task
6386  * (ie cpu_capacity).
6387  *
6388  * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the
6389  * recent utilization of currently non-runnable tasks on a CPU. It represents
6390  * the amount of utilization of a CPU in the range [0..capacity_orig] where
6391  * capacity_orig is the cpu_capacity available at the highest frequency
6392  * (arch_scale_freq_capacity()).
6393  * The utilization of a CPU converges towards a sum equal to or less than the
6394  * current capacity (capacity_curr <= capacity_orig) of the CPU because it is
6395  * the running time on this CPU scaled by capacity_curr.
6396  *
6397  * The estimated utilization of a CPU is defined to be the maximum between its
6398  * cfs_rq.avg.util_avg and the sum of the estimated utilization of the tasks
6399  * currently RUNNABLE on that CPU.
6400  * This allows to properly represent the expected utilization of a CPU which
6401  * has just got a big task running since a long sleep period. At the same time
6402  * however it preserves the benefits of the "blocked utilization" in
6403  * describing the potential for other tasks waking up on the same CPU.
6404  *
6405  * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even
6406  * higher than capacity_orig because of unfortunate rounding in
6407  * cfs.avg.util_avg or just after migrating tasks and new task wakeups until
6408  * the average stabilizes with the new running time. We need to check that the
6409  * utilization stays within the range of [0..capacity_orig] and cap it if
6410  * necessary. Without utilization capping, a group could be seen as overloaded
6411  * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of
6412  * available capacity. We allow utilization to overshoot capacity_curr (but not
6413  * capacity_orig) as it useful for predicting the capacity required after task
6414  * migrations (scheduler-driven DVFS).
6415  *
6416  * Return: the (estimated) utilization for the specified CPU
6417  */
6418 static inline unsigned long cpu_util(int cpu)
6419 {
6420 	struct cfs_rq *cfs_rq;
6421 	unsigned int util;
6422 
6423 	cfs_rq = &cpu_rq(cpu)->cfs;
6424 	util = READ_ONCE(cfs_rq->avg.util_avg);
6425 
6426 	if (sched_feat(UTIL_EST))
6427 		util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued));
6428 
6429 	return min_t(unsigned long, util, capacity_orig_of(cpu));
6430 }
6431 
6432 /*
6433  * cpu_util_without: compute cpu utilization without any contributions from *p
6434  * @cpu: the CPU which utilization is requested
6435  * @p: the task which utilization should be discounted
6436  *
6437  * The utilization of a CPU is defined by the utilization of tasks currently
6438  * enqueued on that CPU as well as tasks which are currently sleeping after an
6439  * execution on that CPU.
6440  *
6441  * This method returns the utilization of the specified CPU by discounting the
6442  * utilization of the specified task, whenever the task is currently
6443  * contributing to the CPU utilization.
6444  */
6445 static unsigned long cpu_util_without(int cpu, struct task_struct *p)
6446 {
6447 	struct cfs_rq *cfs_rq;
6448 	unsigned int util;
6449 
6450 	/* Task has no contribution or is new */
6451 	if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
6452 		return cpu_util(cpu);
6453 
6454 	cfs_rq = &cpu_rq(cpu)->cfs;
6455 	util = READ_ONCE(cfs_rq->avg.util_avg);
6456 
6457 	/* Discount task's util from CPU's util */
6458 	lsub_positive(&util, task_util(p));
6459 
6460 	/*
6461 	 * Covered cases:
6462 	 *
6463 	 * a) if *p is the only task sleeping on this CPU, then:
6464 	 *      cpu_util (== task_util) > util_est (== 0)
6465 	 *    and thus we return:
6466 	 *      cpu_util_without = (cpu_util - task_util) = 0
6467 	 *
6468 	 * b) if other tasks are SLEEPING on this CPU, which is now exiting
6469 	 *    IDLE, then:
6470 	 *      cpu_util >= task_util
6471 	 *      cpu_util > util_est (== 0)
6472 	 *    and thus we discount *p's blocked utilization to return:
6473 	 *      cpu_util_without = (cpu_util - task_util) >= 0
6474 	 *
6475 	 * c) if other tasks are RUNNABLE on that CPU and
6476 	 *      util_est > cpu_util
6477 	 *    then we use util_est since it returns a more restrictive
6478 	 *    estimation of the spare capacity on that CPU, by just
6479 	 *    considering the expected utilization of tasks already
6480 	 *    runnable on that CPU.
6481 	 *
6482 	 * Cases a) and b) are covered by the above code, while case c) is
6483 	 * covered by the following code when estimated utilization is
6484 	 * enabled.
6485 	 */
6486 	if (sched_feat(UTIL_EST)) {
6487 		unsigned int estimated =
6488 			READ_ONCE(cfs_rq->avg.util_est.enqueued);
6489 
6490 		/*
6491 		 * Despite the following checks we still have a small window
6492 		 * for a possible race, when an execl's select_task_rq_fair()
6493 		 * races with LB's detach_task():
6494 		 *
6495 		 *   detach_task()
6496 		 *     p->on_rq = TASK_ON_RQ_MIGRATING;
6497 		 *     ---------------------------------- A
6498 		 *     deactivate_task()                   \
6499 		 *       dequeue_task()                     + RaceTime
6500 		 *         util_est_dequeue()              /
6501 		 *     ---------------------------------- B
6502 		 *
6503 		 * The additional check on "current == p" it's required to
6504 		 * properly fix the execl regression and it helps in further
6505 		 * reducing the chances for the above race.
6506 		 */
6507 		if (unlikely(task_on_rq_queued(p) || current == p))
6508 			lsub_positive(&estimated, _task_util_est(p));
6509 
6510 		util = max(util, estimated);
6511 	}
6512 
6513 	/*
6514 	 * Utilization (estimated) can exceed the CPU capacity, thus let's
6515 	 * clamp to the maximum CPU capacity to ensure consistency with
6516 	 * the cpu_util call.
6517 	 */
6518 	return min_t(unsigned long, util, capacity_orig_of(cpu));
6519 }
6520 
6521 /*
6522  * Predicts what cpu_util(@cpu) would return if @p was migrated (and enqueued)
6523  * to @dst_cpu.
6524  */
6525 static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu)
6526 {
6527 	struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;
6528 	unsigned long util_est, util = READ_ONCE(cfs_rq->avg.util_avg);
6529 
6530 	/*
6531 	 * If @p migrates from @cpu to another, remove its contribution. Or,
6532 	 * if @p migrates from another CPU to @cpu, add its contribution. In
6533 	 * the other cases, @cpu is not impacted by the migration, so the
6534 	 * util_avg should already be correct.
6535 	 */
6536 	if (task_cpu(p) == cpu && dst_cpu != cpu)
6537 		lsub_positive(&util, task_util(p));
6538 	else if (task_cpu(p) != cpu && dst_cpu == cpu)
6539 		util += task_util(p);
6540 
6541 	if (sched_feat(UTIL_EST)) {
6542 		util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued);
6543 
6544 		/*
6545 		 * During wake-up, the task isn't enqueued yet and doesn't
6546 		 * appear in the cfs_rq->avg.util_est.enqueued of any rq,
6547 		 * so just add it (if needed) to "simulate" what will be
6548 		 * cpu_util() after the task has been enqueued.
6549 		 */
6550 		if (dst_cpu == cpu)
6551 			util_est += _task_util_est(p);
6552 
6553 		util = max(util, util_est);
6554 	}
6555 
6556 	return min(util, capacity_orig_of(cpu));
6557 }
6558 
6559 /*
6560  * compute_energy(): Estimates the energy that @pd would consume if @p was
6561  * migrated to @dst_cpu. compute_energy() predicts what will be the utilization
6562  * landscape of @pd's CPUs after the task migration, and uses the Energy Model
6563  * to compute what would be the energy if we decided to actually migrate that
6564  * task.
6565  */
6566 static long
6567 compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd)
6568 {
6569 	struct cpumask *pd_mask = perf_domain_span(pd);
6570 	unsigned long cpu_cap = arch_scale_cpu_capacity(cpumask_first(pd_mask));
6571 	unsigned long max_util = 0, sum_util = 0;
6572 	int cpu;
6573 
6574 	/*
6575 	 * The capacity state of CPUs of the current rd can be driven by CPUs
6576 	 * of another rd if they belong to the same pd. So, account for the
6577 	 * utilization of these CPUs too by masking pd with cpu_online_mask
6578 	 * instead of the rd span.
6579 	 *
6580 	 * If an entire pd is outside of the current rd, it will not appear in
6581 	 * its pd list and will not be accounted by compute_energy().
6582 	 */
6583 	for_each_cpu_and(cpu, pd_mask, cpu_online_mask) {
6584 		unsigned long util_freq = cpu_util_next(cpu, p, dst_cpu);
6585 		unsigned long cpu_util, util_running = util_freq;
6586 		struct task_struct *tsk = NULL;
6587 
6588 		/*
6589 		 * When @p is placed on @cpu:
6590 		 *
6591 		 * util_running = max(cpu_util, cpu_util_est) +
6592 		 *		  max(task_util, _task_util_est)
6593 		 *
6594 		 * while cpu_util_next is: max(cpu_util + task_util,
6595 		 *			       cpu_util_est + _task_util_est)
6596 		 */
6597 		if (cpu == dst_cpu) {
6598 			tsk = p;
6599 			util_running =
6600 				cpu_util_next(cpu, p, -1) + task_util_est(p);
6601 		}
6602 
6603 		/*
6604 		 * Busy time computation: utilization clamping is not
6605 		 * required since the ratio (sum_util / cpu_capacity)
6606 		 * is already enough to scale the EM reported power
6607 		 * consumption at the (eventually clamped) cpu_capacity.
6608 		 */
6609 		sum_util += effective_cpu_util(cpu, util_running, cpu_cap,
6610 					       ENERGY_UTIL, NULL);
6611 
6612 		/*
6613 		 * Performance domain frequency: utilization clamping
6614 		 * must be considered since it affects the selection
6615 		 * of the performance domain frequency.
6616 		 * NOTE: in case RT tasks are running, by default the
6617 		 * FREQUENCY_UTIL's utilization can be max OPP.
6618 		 */
6619 		cpu_util = effective_cpu_util(cpu, util_freq, cpu_cap,
6620 					      FREQUENCY_UTIL, tsk);
6621 		max_util = max(max_util, cpu_util);
6622 	}
6623 
6624 	return em_cpu_energy(pd->em_pd, max_util, sum_util);
6625 }
6626 
6627 /*
6628  * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the
6629  * waking task. find_energy_efficient_cpu() looks for the CPU with maximum
6630  * spare capacity in each performance domain and uses it as a potential
6631  * candidate to execute the task. Then, it uses the Energy Model to figure
6632  * out which of the CPU candidates is the most energy-efficient.
6633  *
6634  * The rationale for this heuristic is as follows. In a performance domain,
6635  * all the most energy efficient CPU candidates (according to the Energy
6636  * Model) are those for which we'll request a low frequency. When there are
6637  * several CPUs for which the frequency request will be the same, we don't
6638  * have enough data to break the tie between them, because the Energy Model
6639  * only includes active power costs. With this model, if we assume that
6640  * frequency requests follow utilization (e.g. using schedutil), the CPU with
6641  * the maximum spare capacity in a performance domain is guaranteed to be among
6642  * the best candidates of the performance domain.
6643  *
6644  * In practice, it could be preferable from an energy standpoint to pack
6645  * small tasks on a CPU in order to let other CPUs go in deeper idle states,
6646  * but that could also hurt our chances to go cluster idle, and we have no
6647  * ways to tell with the current Energy Model if this is actually a good
6648  * idea or not. So, find_energy_efficient_cpu() basically favors
6649  * cluster-packing, and spreading inside a cluster. That should at least be
6650  * a good thing for latency, and this is consistent with the idea that most
6651  * of the energy savings of EAS come from the asymmetry of the system, and
6652  * not so much from breaking the tie between identical CPUs. That's also the
6653  * reason why EAS is enabled in the topology code only for systems where
6654  * SD_ASYM_CPUCAPACITY is set.
6655  *
6656  * NOTE: Forkees are not accepted in the energy-aware wake-up path because
6657  * they don't have any useful utilization data yet and it's not possible to
6658  * forecast their impact on energy consumption. Consequently, they will be
6659  * placed by find_idlest_cpu() on the least loaded CPU, which might turn out
6660  * to be energy-inefficient in some use-cases. The alternative would be to
6661  * bias new tasks towards specific types of CPUs first, or to try to infer
6662  * their util_avg from the parent task, but those heuristics could hurt
6663  * other use-cases too. So, until someone finds a better way to solve this,
6664  * let's keep things simple by re-using the existing slow path.
6665  */
6666 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
6667 {
6668 	unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX;
6669 	struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
6670 	unsigned long cpu_cap, util, base_energy = 0;
6671 	int cpu, best_energy_cpu = prev_cpu;
6672 	struct sched_domain *sd;
6673 	struct perf_domain *pd;
6674 
6675 	rcu_read_lock();
6676 	pd = rcu_dereference(rd->pd);
6677 	if (!pd || READ_ONCE(rd->overutilized))
6678 		goto fail;
6679 
6680 	/*
6681 	 * Energy-aware wake-up happens on the lowest sched_domain starting
6682 	 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu.
6683 	 */
6684 	sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity));
6685 	while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
6686 		sd = sd->parent;
6687 	if (!sd)
6688 		goto fail;
6689 
6690 	sync_entity_load_avg(&p->se);
6691 	if (!task_util_est(p))
6692 		goto unlock;
6693 
6694 	for (; pd; pd = pd->next) {
6695 		unsigned long cur_delta, spare_cap, max_spare_cap = 0;
6696 		unsigned long base_energy_pd;
6697 		int max_spare_cap_cpu = -1;
6698 
6699 		/* Compute the 'base' energy of the pd, without @p */
6700 		base_energy_pd = compute_energy(p, -1, pd);
6701 		base_energy += base_energy_pd;
6702 
6703 		for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) {
6704 			if (!cpumask_test_cpu(cpu, p->cpus_ptr))
6705 				continue;
6706 
6707 			util = cpu_util_next(cpu, p, cpu);
6708 			cpu_cap = capacity_of(cpu);
6709 			spare_cap = cpu_cap;
6710 			lsub_positive(&spare_cap, util);
6711 
6712 			/*
6713 			 * Skip CPUs that cannot satisfy the capacity request.
6714 			 * IOW, placing the task there would make the CPU
6715 			 * overutilized. Take uclamp into account to see how
6716 			 * much capacity we can get out of the CPU; this is
6717 			 * aligned with sched_cpu_util().
6718 			 */
6719 			util = uclamp_rq_util_with(cpu_rq(cpu), util, p);
6720 			if (!fits_capacity(util, cpu_cap))
6721 				continue;
6722 
6723 			/* Always use prev_cpu as a candidate. */
6724 			if (cpu == prev_cpu) {
6725 				prev_delta = compute_energy(p, prev_cpu, pd);
6726 				prev_delta -= base_energy_pd;
6727 				best_delta = min(best_delta, prev_delta);
6728 			}
6729 
6730 			/*
6731 			 * Find the CPU with the maximum spare capacity in
6732 			 * the performance domain
6733 			 */
6734 			if (spare_cap > max_spare_cap) {
6735 				max_spare_cap = spare_cap;
6736 				max_spare_cap_cpu = cpu;
6737 			}
6738 		}
6739 
6740 		/* Evaluate the energy impact of using this CPU. */
6741 		if (max_spare_cap_cpu >= 0 && max_spare_cap_cpu != prev_cpu) {
6742 			cur_delta = compute_energy(p, max_spare_cap_cpu, pd);
6743 			cur_delta -= base_energy_pd;
6744 			if (cur_delta < best_delta) {
6745 				best_delta = cur_delta;
6746 				best_energy_cpu = max_spare_cap_cpu;
6747 			}
6748 		}
6749 	}
6750 unlock:
6751 	rcu_read_unlock();
6752 
6753 	/*
6754 	 * Pick the best CPU if prev_cpu cannot be used, or if it saves at
6755 	 * least 6% of the energy used by prev_cpu.
6756 	 */
6757 	if (prev_delta == ULONG_MAX)
6758 		return best_energy_cpu;
6759 
6760 	if ((prev_delta - best_delta) > ((prev_delta + base_energy) >> 4))
6761 		return best_energy_cpu;
6762 
6763 	return prev_cpu;
6764 
6765 fail:
6766 	rcu_read_unlock();
6767 
6768 	return -1;
6769 }
6770 
6771 /*
6772  * select_task_rq_fair: Select target runqueue for the waking task in domains
6773  * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE,
6774  * SD_BALANCE_FORK, or SD_BALANCE_EXEC.
6775  *
6776  * Balances load by selecting the idlest CPU in the idlest group, or under
6777  * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set.
6778  *
6779  * Returns the target CPU number.
6780  *
6781  * preempt must be disabled.
6782  */
6783 static int
6784 select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
6785 {
6786 	int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING);
6787 	struct sched_domain *tmp, *sd = NULL;
6788 	int cpu = smp_processor_id();
6789 	int new_cpu = prev_cpu;
6790 	int want_affine = 0;
6791 	/* SD_flags and WF_flags share the first nibble */
6792 	int sd_flag = wake_flags & 0xF;
6793 
6794 	if (wake_flags & WF_TTWU) {
6795 		record_wakee(p);
6796 
6797 		if (sched_energy_enabled()) {
6798 			new_cpu = find_energy_efficient_cpu(p, prev_cpu);
6799 			if (new_cpu >= 0)
6800 				return new_cpu;
6801 			new_cpu = prev_cpu;
6802 		}
6803 
6804 		want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr);
6805 	}
6806 
6807 	rcu_read_lock();
6808 	for_each_domain(cpu, tmp) {
6809 		/*
6810 		 * If both 'cpu' and 'prev_cpu' are part of this domain,
6811 		 * cpu is a valid SD_WAKE_AFFINE target.
6812 		 */
6813 		if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
6814 		    cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
6815 			if (cpu != prev_cpu)
6816 				new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync);
6817 
6818 			sd = NULL; /* Prefer wake_affine over balance flags */
6819 			break;
6820 		}
6821 
6822 		if (tmp->flags & sd_flag)
6823 			sd = tmp;
6824 		else if (!want_affine)
6825 			break;
6826 	}
6827 
6828 	if (unlikely(sd)) {
6829 		/* Slow path */
6830 		new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag);
6831 	} else if (wake_flags & WF_TTWU) { /* XXX always ? */
6832 		/* Fast path */
6833 		new_cpu = select_idle_sibling(p, prev_cpu, new_cpu);
6834 
6835 		if (want_affine)
6836 			current->recent_used_cpu = cpu;
6837 	}
6838 	rcu_read_unlock();
6839 
6840 	return new_cpu;
6841 }
6842 
6843 static void detach_entity_cfs_rq(struct sched_entity *se);
6844 
6845 /*
6846  * Called immediately before a task is migrated to a new CPU; task_cpu(p) and
6847  * cfs_rq_of(p) references at time of call are still valid and identify the
6848  * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held.
6849  */
6850 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu)
6851 {
6852 	/*
6853 	 * As blocked tasks retain absolute vruntime the migration needs to
6854 	 * deal with this by subtracting the old and adding the new
6855 	 * min_vruntime -- the latter is done by enqueue_entity() when placing
6856 	 * the task on the new runqueue.
6857 	 */
6858 	if (p->state == TASK_WAKING) {
6859 		struct sched_entity *se = &p->se;
6860 		struct cfs_rq *cfs_rq = cfs_rq_of(se);
6861 		u64 min_vruntime;
6862 
6863 #ifndef CONFIG_64BIT
6864 		u64 min_vruntime_copy;
6865 
6866 		do {
6867 			min_vruntime_copy = cfs_rq->min_vruntime_copy;
6868 			smp_rmb();
6869 			min_vruntime = cfs_rq->min_vruntime;
6870 		} while (min_vruntime != min_vruntime_copy);
6871 #else
6872 		min_vruntime = cfs_rq->min_vruntime;
6873 #endif
6874 
6875 		se->vruntime -= min_vruntime;
6876 	}
6877 
6878 	if (p->on_rq == TASK_ON_RQ_MIGRATING) {
6879 		/*
6880 		 * In case of TASK_ON_RQ_MIGRATING we in fact hold the 'old'
6881 		 * rq->lock and can modify state directly.
6882 		 */
6883 		lockdep_assert_held(&task_rq(p)->lock);
6884 		detach_entity_cfs_rq(&p->se);
6885 
6886 	} else {
6887 		/*
6888 		 * We are supposed to update the task to "current" time, then
6889 		 * its up to date and ready to go to new CPU/cfs_rq. But we
6890 		 * have difficulty in getting what current time is, so simply
6891 		 * throw away the out-of-date time. This will result in the
6892 		 * wakee task is less decayed, but giving the wakee more load
6893 		 * sounds not bad.
6894 		 */
6895 		remove_entity_load_avg(&p->se);
6896 	}
6897 
6898 	/* Tell new CPU we are migrated */
6899 	p->se.avg.last_update_time = 0;
6900 
6901 	/* We have migrated, no longer consider this task hot */
6902 	p->se.exec_start = 0;
6903 
6904 	update_scan_period(p, new_cpu);
6905 }
6906 
6907 static void task_dead_fair(struct task_struct *p)
6908 {
6909 	remove_entity_load_avg(&p->se);
6910 }
6911 
6912 static int
6913 balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
6914 {
6915 	if (rq->nr_running)
6916 		return 1;
6917 
6918 	return newidle_balance(rq, rf) != 0;
6919 }
6920 #endif /* CONFIG_SMP */
6921 
6922 static unsigned long wakeup_gran(struct sched_entity *se)
6923 {
6924 	unsigned long gran = sysctl_sched_wakeup_granularity;
6925 
6926 	/*
6927 	 * Since its curr running now, convert the gran from real-time
6928 	 * to virtual-time in his units.
6929 	 *
6930 	 * By using 'se' instead of 'curr' we penalize light tasks, so
6931 	 * they get preempted easier. That is, if 'se' < 'curr' then
6932 	 * the resulting gran will be larger, therefore penalizing the
6933 	 * lighter, if otoh 'se' > 'curr' then the resulting gran will
6934 	 * be smaller, again penalizing the lighter task.
6935 	 *
6936 	 * This is especially important for buddies when the leftmost
6937 	 * task is higher priority than the buddy.
6938 	 */
6939 	return calc_delta_fair(gran, se);
6940 }
6941 
6942 /*
6943  * Should 'se' preempt 'curr'.
6944  *
6945  *             |s1
6946  *        |s2
6947  *   |s3
6948  *         g
6949  *      |<--->|c
6950  *
6951  *  w(c, s1) = -1
6952  *  w(c, s2) =  0
6953  *  w(c, s3) =  1
6954  *
6955  */
6956 static int
6957 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se)
6958 {
6959 	s64 gran, vdiff = curr->vruntime - se->vruntime;
6960 
6961 	if (vdiff <= 0)
6962 		return -1;
6963 
6964 	gran = wakeup_gran(se);
6965 	if (vdiff > gran)
6966 		return 1;
6967 
6968 	return 0;
6969 }
6970 
6971 static void set_last_buddy(struct sched_entity *se)
6972 {
6973 	if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se))))
6974 		return;
6975 
6976 	for_each_sched_entity(se) {
6977 		if (SCHED_WARN_ON(!se->on_rq))
6978 			return;
6979 		cfs_rq_of(se)->last = se;
6980 	}
6981 }
6982 
6983 static void set_next_buddy(struct sched_entity *se)
6984 {
6985 	if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se))))
6986 		return;
6987 
6988 	for_each_sched_entity(se) {
6989 		if (SCHED_WARN_ON(!se->on_rq))
6990 			return;
6991 		cfs_rq_of(se)->next = se;
6992 	}
6993 }
6994 
6995 static void set_skip_buddy(struct sched_entity *se)
6996 {
6997 	for_each_sched_entity(se)
6998 		cfs_rq_of(se)->skip = se;
6999 }
7000 
7001 /*
7002  * Preempt the current task with a newly woken task if needed:
7003  */
7004 static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
7005 {
7006 	struct task_struct *curr = rq->curr;
7007 	struct sched_entity *se = &curr->se, *pse = &p->se;
7008 	struct cfs_rq *cfs_rq = task_cfs_rq(curr);
7009 	int scale = cfs_rq->nr_running >= sched_nr_latency;
7010 	int next_buddy_marked = 0;
7011 
7012 	if (unlikely(se == pse))
7013 		return;
7014 
7015 	/*
7016 	 * This is possible from callers such as attach_tasks(), in which we
7017 	 * unconditionally check_preempt_curr() after an enqueue (which may have
7018 	 * lead to a throttle).  This both saves work and prevents false
7019 	 * next-buddy nomination below.
7020 	 */
7021 	if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
7022 		return;
7023 
7024 	if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) {
7025 		set_next_buddy(pse);
7026 		next_buddy_marked = 1;
7027 	}
7028 
7029 	/*
7030 	 * We can come here with TIF_NEED_RESCHED already set from new task
7031 	 * wake up path.
7032 	 *
7033 	 * Note: this also catches the edge-case of curr being in a throttled
7034 	 * group (e.g. via set_curr_task), since update_curr() (in the
7035 	 * enqueue of curr) will have resulted in resched being set.  This
7036 	 * prevents us from potentially nominating it as a false LAST_BUDDY
7037 	 * below.
7038 	 */
7039 	if (test_tsk_need_resched(curr))
7040 		return;
7041 
7042 	/* Idle tasks are by definition preempted by non-idle tasks. */
7043 	if (unlikely(task_has_idle_policy(curr)) &&
7044 	    likely(!task_has_idle_policy(p)))
7045 		goto preempt;
7046 
7047 	/*
7048 	 * Batch and idle tasks do not preempt non-idle tasks (their preemption
7049 	 * is driven by the tick):
7050 	 */
7051 	if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION))
7052 		return;
7053 
7054 	find_matching_se(&se, &pse);
7055 	update_curr(cfs_rq_of(se));
7056 	BUG_ON(!pse);
7057 	if (wakeup_preempt_entity(se, pse) == 1) {
7058 		/*
7059 		 * Bias pick_next to pick the sched entity that is
7060 		 * triggering this preemption.
7061 		 */
7062 		if (!next_buddy_marked)
7063 			set_next_buddy(pse);
7064 		goto preempt;
7065 	}
7066 
7067 	return;
7068 
7069 preempt:
7070 	resched_curr(rq);
7071 	/*
7072 	 * Only set the backward buddy when the current task is still
7073 	 * on the rq. This can happen when a wakeup gets interleaved
7074 	 * with schedule on the ->pre_schedule() or idle_balance()
7075 	 * point, either of which can * drop the rq lock.
7076 	 *
7077 	 * Also, during early boot the idle thread is in the fair class,
7078 	 * for obvious reasons its a bad idea to schedule back to it.
7079 	 */
7080 	if (unlikely(!se->on_rq || curr == rq->idle))
7081 		return;
7082 
7083 	if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se))
7084 		set_last_buddy(se);
7085 }
7086 
7087 struct task_struct *
7088 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
7089 {
7090 	struct cfs_rq *cfs_rq = &rq->cfs;
7091 	struct sched_entity *se;
7092 	struct task_struct *p;
7093 	int new_tasks;
7094 
7095 again:
7096 	if (!sched_fair_runnable(rq))
7097 		goto idle;
7098 
7099 #ifdef CONFIG_FAIR_GROUP_SCHED
7100 	if (!prev || prev->sched_class != &fair_sched_class)
7101 		goto simple;
7102 
7103 	/*
7104 	 * Because of the set_next_buddy() in dequeue_task_fair() it is rather
7105 	 * likely that a next task is from the same cgroup as the current.
7106 	 *
7107 	 * Therefore attempt to avoid putting and setting the entire cgroup
7108 	 * hierarchy, only change the part that actually changes.
7109 	 */
7110 
7111 	do {
7112 		struct sched_entity *curr = cfs_rq->curr;
7113 
7114 		/*
7115 		 * Since we got here without doing put_prev_entity() we also
7116 		 * have to consider cfs_rq->curr. If it is still a runnable
7117 		 * entity, update_curr() will update its vruntime, otherwise
7118 		 * forget we've ever seen it.
7119 		 */
7120 		if (curr) {
7121 			if (curr->on_rq)
7122 				update_curr(cfs_rq);
7123 			else
7124 				curr = NULL;
7125 
7126 			/*
7127 			 * This call to check_cfs_rq_runtime() will do the
7128 			 * throttle and dequeue its entity in the parent(s).
7129 			 * Therefore the nr_running test will indeed
7130 			 * be correct.
7131 			 */
7132 			if (unlikely(check_cfs_rq_runtime(cfs_rq))) {
7133 				cfs_rq = &rq->cfs;
7134 
7135 				if (!cfs_rq->nr_running)
7136 					goto idle;
7137 
7138 				goto simple;
7139 			}
7140 		}
7141 
7142 		se = pick_next_entity(cfs_rq, curr);
7143 		cfs_rq = group_cfs_rq(se);
7144 	} while (cfs_rq);
7145 
7146 	p = task_of(se);
7147 
7148 	/*
7149 	 * Since we haven't yet done put_prev_entity and if the selected task
7150 	 * is a different task than we started out with, try and touch the
7151 	 * least amount of cfs_rqs.
7152 	 */
7153 	if (prev != p) {
7154 		struct sched_entity *pse = &prev->se;
7155 
7156 		while (!(cfs_rq = is_same_group(se, pse))) {
7157 			int se_depth = se->depth;
7158 			int pse_depth = pse->depth;
7159 
7160 			if (se_depth <= pse_depth) {
7161 				put_prev_entity(cfs_rq_of(pse), pse);
7162 				pse = parent_entity(pse);
7163 			}
7164 			if (se_depth >= pse_depth) {
7165 				set_next_entity(cfs_rq_of(se), se);
7166 				se = parent_entity(se);
7167 			}
7168 		}
7169 
7170 		put_prev_entity(cfs_rq, pse);
7171 		set_next_entity(cfs_rq, se);
7172 	}
7173 
7174 	goto done;
7175 simple:
7176 #endif
7177 	if (prev)
7178 		put_prev_task(rq, prev);
7179 
7180 	do {
7181 		se = pick_next_entity(cfs_rq, NULL);
7182 		set_next_entity(cfs_rq, se);
7183 		cfs_rq = group_cfs_rq(se);
7184 	} while (cfs_rq);
7185 
7186 	p = task_of(se);
7187 
7188 done: __maybe_unused;
7189 #ifdef CONFIG_SMP
7190 	/*
7191 	 * Move the next running task to the front of
7192 	 * the list, so our cfs_tasks list becomes MRU
7193 	 * one.
7194 	 */
7195 	list_move(&p->se.group_node, &rq->cfs_tasks);
7196 #endif
7197 
7198 	if (hrtick_enabled_fair(rq))
7199 		hrtick_start_fair(rq, p);
7200 
7201 	update_misfit_status(p, rq);
7202 
7203 	return p;
7204 
7205 idle:
7206 	if (!rf)
7207 		return NULL;
7208 
7209 	new_tasks = newidle_balance(rq, rf);
7210 
7211 	/*
7212 	 * Because newidle_balance() releases (and re-acquires) rq->lock, it is
7213 	 * possible for any higher priority task to appear. In that case we
7214 	 * must re-start the pick_next_entity() loop.
7215 	 */
7216 	if (new_tasks < 0)
7217 		return RETRY_TASK;
7218 
7219 	if (new_tasks > 0)
7220 		goto again;
7221 
7222 	/*
7223 	 * rq is about to be idle, check if we need to update the
7224 	 * lost_idle_time of clock_pelt
7225 	 */
7226 	update_idle_rq_clock_pelt(rq);
7227 
7228 	return NULL;
7229 }
7230 
7231 static struct task_struct *__pick_next_task_fair(struct rq *rq)
7232 {
7233 	return pick_next_task_fair(rq, NULL, NULL);
7234 }
7235 
7236 /*
7237  * Account for a descheduled task:
7238  */
7239 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
7240 {
7241 	struct sched_entity *se = &prev->se;
7242 	struct cfs_rq *cfs_rq;
7243 
7244 	for_each_sched_entity(se) {
7245 		cfs_rq = cfs_rq_of(se);
7246 		put_prev_entity(cfs_rq, se);
7247 	}
7248 }
7249 
7250 /*
7251  * sched_yield() is very simple
7252  *
7253  * The magic of dealing with the ->skip buddy is in pick_next_entity.
7254  */
7255 static void yield_task_fair(struct rq *rq)
7256 {
7257 	struct task_struct *curr = rq->curr;
7258 	struct cfs_rq *cfs_rq = task_cfs_rq(curr);
7259 	struct sched_entity *se = &curr->se;
7260 
7261 	/*
7262 	 * Are we the only task in the tree?
7263 	 */
7264 	if (unlikely(rq->nr_running == 1))
7265 		return;
7266 
7267 	clear_buddies(cfs_rq, se);
7268 
7269 	if (curr->policy != SCHED_BATCH) {
7270 		update_rq_clock(rq);
7271 		/*
7272 		 * Update run-time statistics of the 'current'.
7273 		 */
7274 		update_curr(cfs_rq);
7275 		/*
7276 		 * Tell update_rq_clock() that we've just updated,
7277 		 * so we don't do microscopic update in schedule()
7278 		 * and double the fastpath cost.
7279 		 */
7280 		rq_clock_skip_update(rq);
7281 	}
7282 
7283 	set_skip_buddy(se);
7284 }
7285 
7286 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p)
7287 {
7288 	struct sched_entity *se = &p->se;
7289 
7290 	/* throttled hierarchies are not runnable */
7291 	if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se)))
7292 		return false;
7293 
7294 	/* Tell the scheduler that we'd really like pse to run next. */
7295 	set_next_buddy(se);
7296 
7297 	yield_task_fair(rq);
7298 
7299 	return true;
7300 }
7301 
7302 #ifdef CONFIG_SMP
7303 /**************************************************
7304  * Fair scheduling class load-balancing methods.
7305  *
7306  * BASICS
7307  *
7308  * The purpose of load-balancing is to achieve the same basic fairness the
7309  * per-CPU scheduler provides, namely provide a proportional amount of compute
7310  * time to each task. This is expressed in the following equation:
7311  *
7312  *   W_i,n/P_i == W_j,n/P_j for all i,j                               (1)
7313  *
7314  * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight
7315  * W_i,0 is defined as:
7316  *
7317  *   W_i,0 = \Sum_j w_i,j                                             (2)
7318  *
7319  * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight
7320  * is derived from the nice value as per sched_prio_to_weight[].
7321  *
7322  * The weight average is an exponential decay average of the instantaneous
7323  * weight:
7324  *
7325  *   W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0               (3)
7326  *
7327  * C_i is the compute capacity of CPU i, typically it is the
7328  * fraction of 'recent' time available for SCHED_OTHER task execution. But it
7329  * can also include other factors [XXX].
7330  *
7331  * To achieve this balance we define a measure of imbalance which follows
7332  * directly from (1):
7333  *
7334  *   imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j }    (4)
7335  *
7336  * We them move tasks around to minimize the imbalance. In the continuous
7337  * function space it is obvious this converges, in the discrete case we get
7338  * a few fun cases generally called infeasible weight scenarios.
7339  *
7340  * [XXX expand on:
7341  *     - infeasible weights;
7342  *     - local vs global optima in the discrete case. ]
7343  *
7344  *
7345  * SCHED DOMAINS
7346  *
7347  * In order to solve the imbalance equation (4), and avoid the obvious O(n^2)
7348  * for all i,j solution, we create a tree of CPUs that follows the hardware
7349  * topology where each level pairs two lower groups (or better). This results
7350  * in O(log n) layers. Furthermore we reduce the number of CPUs going up the
7351  * tree to only the first of the previous level and we decrease the frequency
7352  * of load-balance at each level inv. proportional to the number of CPUs in
7353  * the groups.
7354  *
7355  * This yields:
7356  *
7357  *     log_2 n     1     n
7358  *   \Sum       { --- * --- * 2^i } = O(n)                            (5)
7359  *     i = 0      2^i   2^i
7360  *                               `- size of each group
7361  *         |         |     `- number of CPUs doing load-balance
7362  *         |         `- freq
7363  *         `- sum over all levels
7364  *
7365  * Coupled with a limit on how many tasks we can migrate every balance pass,
7366  * this makes (5) the runtime complexity of the balancer.
7367  *
7368  * An important property here is that each CPU is still (indirectly) connected
7369  * to every other CPU in at most O(log n) steps:
7370  *
7371  * The adjacency matrix of the resulting graph is given by:
7372  *
7373  *             log_2 n
7374  *   A_i,j = \Union     (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1)  (6)
7375  *             k = 0
7376  *
7377  * And you'll find that:
7378  *
7379  *   A^(log_2 n)_i,j != 0  for all i,j                                (7)
7380  *
7381  * Showing there's indeed a path between every CPU in at most O(log n) steps.
7382  * The task movement gives a factor of O(m), giving a convergence complexity
7383  * of:
7384  *
7385  *   O(nm log n),  n := nr_cpus, m := nr_tasks                        (8)
7386  *
7387  *
7388  * WORK CONSERVING
7389  *
7390  * In order to avoid CPUs going idle while there's still work to do, new idle
7391  * balancing is more aggressive and has the newly idle CPU iterate up the domain
7392  * tree itself instead of relying on other CPUs to bring it work.
7393  *
7394  * This adds some complexity to both (5) and (8) but it reduces the total idle
7395  * time.
7396  *
7397  * [XXX more?]
7398  *
7399  *
7400  * CGROUPS
7401  *
7402  * Cgroups make a horror show out of (2), instead of a simple sum we get:
7403  *
7404  *                                s_k,i
7405  *   W_i,0 = \Sum_j \Prod_k w_k * -----                               (9)
7406  *                                 S_k
7407  *
7408  * Where
7409  *
7410  *   s_k,i = \Sum_j w_i,j,k  and  S_k = \Sum_i s_k,i                 (10)
7411  *
7412  * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i.
7413  *
7414  * The big problem is S_k, its a global sum needed to compute a local (W_i)
7415  * property.
7416  *
7417  * [XXX write more on how we solve this.. _after_ merging pjt's patches that
7418  *      rewrite all of this once again.]
7419  */
7420 
7421 static unsigned long __read_mostly max_load_balance_interval = HZ/10;
7422 
7423 enum fbq_type { regular, remote, all };
7424 
7425 /*
7426  * 'group_type' describes the group of CPUs at the moment of load balancing.
7427  *
7428  * The enum is ordered by pulling priority, with the group with lowest priority
7429  * first so the group_type can simply be compared when selecting the busiest
7430  * group. See update_sd_pick_busiest().
7431  */
7432 enum group_type {
7433 	/* The group has spare capacity that can be used to run more tasks.  */
7434 	group_has_spare = 0,
7435 	/*
7436 	 * The group is fully used and the tasks don't compete for more CPU
7437 	 * cycles. Nevertheless, some tasks might wait before running.
7438 	 */
7439 	group_fully_busy,
7440 	/*
7441 	 * SD_ASYM_CPUCAPACITY only: One task doesn't fit with CPU's capacity
7442 	 * and must be migrated to a more powerful CPU.
7443 	 */
7444 	group_misfit_task,
7445 	/*
7446 	 * SD_ASYM_PACKING only: One local CPU with higher capacity is available,
7447 	 * and the task should be migrated to it instead of running on the
7448 	 * current CPU.
7449 	 */
7450 	group_asym_packing,
7451 	/*
7452 	 * The tasks' affinity constraints previously prevented the scheduler
7453 	 * from balancing the load across the system.
7454 	 */
7455 	group_imbalanced,
7456 	/*
7457 	 * The CPU is overloaded and can't provide expected CPU cycles to all
7458 	 * tasks.
7459 	 */
7460 	group_overloaded
7461 };
7462 
7463 enum migration_type {
7464 	migrate_load = 0,
7465 	migrate_util,
7466 	migrate_task,
7467 	migrate_misfit
7468 };
7469 
7470 #define LBF_ALL_PINNED	0x01
7471 #define LBF_NEED_BREAK	0x02
7472 #define LBF_DST_PINNED  0x04
7473 #define LBF_SOME_PINNED	0x08
7474 #define LBF_ACTIVE_LB	0x10
7475 
7476 struct lb_env {
7477 	struct sched_domain	*sd;
7478 
7479 	struct rq		*src_rq;
7480 	int			src_cpu;
7481 
7482 	int			dst_cpu;
7483 	struct rq		*dst_rq;
7484 
7485 	struct cpumask		*dst_grpmask;
7486 	int			new_dst_cpu;
7487 	enum cpu_idle_type	idle;
7488 	long			imbalance;
7489 	/* The set of CPUs under consideration for load-balancing */
7490 	struct cpumask		*cpus;
7491 
7492 	unsigned int		flags;
7493 
7494 	unsigned int		loop;
7495 	unsigned int		loop_break;
7496 	unsigned int		loop_max;
7497 
7498 	enum fbq_type		fbq_type;
7499 	enum migration_type	migration_type;
7500 	struct list_head	tasks;
7501 };
7502 
7503 /*
7504  * Is this task likely cache-hot:
7505  */
7506 static int task_hot(struct task_struct *p, struct lb_env *env)
7507 {
7508 	s64 delta;
7509 
7510 	lockdep_assert_held(&env->src_rq->lock);
7511 
7512 	if (p->sched_class != &fair_sched_class)
7513 		return 0;
7514 
7515 	if (unlikely(task_has_idle_policy(p)))
7516 		return 0;
7517 
7518 	/* SMT siblings share cache */
7519 	if (env->sd->flags & SD_SHARE_CPUCAPACITY)
7520 		return 0;
7521 
7522 	/*
7523 	 * Buddy candidates are cache hot:
7524 	 */
7525 	if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&
7526 			(&p->se == cfs_rq_of(&p->se)->next ||
7527 			 &p->se == cfs_rq_of(&p->se)->last))
7528 		return 1;
7529 
7530 	if (sysctl_sched_migration_cost == -1)
7531 		return 1;
7532 	if (sysctl_sched_migration_cost == 0)
7533 		return 0;
7534 
7535 	delta = rq_clock_task(env->src_rq) - p->se.exec_start;
7536 
7537 	return delta < (s64)sysctl_sched_migration_cost;
7538 }
7539 
7540 #ifdef CONFIG_NUMA_BALANCING
7541 /*
7542  * Returns 1, if task migration degrades locality
7543  * Returns 0, if task migration improves locality i.e migration preferred.
7544  * Returns -1, if task migration is not affected by locality.
7545  */
7546 static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
7547 {
7548 	struct numa_group *numa_group = rcu_dereference(p->numa_group);
7549 	unsigned long src_weight, dst_weight;
7550 	int src_nid, dst_nid, dist;
7551 
7552 	if (!static_branch_likely(&sched_numa_balancing))
7553 		return -1;
7554 
7555 	if (!p->numa_faults || !(env->sd->flags & SD_NUMA))
7556 		return -1;
7557 
7558 	src_nid = cpu_to_node(env->src_cpu);
7559 	dst_nid = cpu_to_node(env->dst_cpu);
7560 
7561 	if (src_nid == dst_nid)
7562 		return -1;
7563 
7564 	/* Migrating away from the preferred node is always bad. */
7565 	if (src_nid == p->numa_preferred_nid) {
7566 		if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)
7567 			return 1;
7568 		else
7569 			return -1;
7570 	}
7571 
7572 	/* Encourage migration to the preferred node. */
7573 	if (dst_nid == p->numa_preferred_nid)
7574 		return 0;
7575 
7576 	/* Leaving a core idle is often worse than degrading locality. */
7577 	if (env->idle == CPU_IDLE)
7578 		return -1;
7579 
7580 	dist = node_distance(src_nid, dst_nid);
7581 	if (numa_group) {
7582 		src_weight = group_weight(p, src_nid, dist);
7583 		dst_weight = group_weight(p, dst_nid, dist);
7584 	} else {
7585 		src_weight = task_weight(p, src_nid, dist);
7586 		dst_weight = task_weight(p, dst_nid, dist);
7587 	}
7588 
7589 	return dst_weight < src_weight;
7590 }
7591 
7592 #else
7593 static inline int migrate_degrades_locality(struct task_struct *p,
7594 					     struct lb_env *env)
7595 {
7596 	return -1;
7597 }
7598 #endif
7599 
7600 /*
7601  * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
7602  */
7603 static
7604 int can_migrate_task(struct task_struct *p, struct lb_env *env)
7605 {
7606 	int tsk_cache_hot;
7607 
7608 	lockdep_assert_held(&env->src_rq->lock);
7609 
7610 	/*
7611 	 * We do not migrate tasks that are:
7612 	 * 1) throttled_lb_pair, or
7613 	 * 2) cannot be migrated to this CPU due to cpus_ptr, or
7614 	 * 3) running (obviously), or
7615 	 * 4) are cache-hot on their current CPU.
7616 	 */
7617 	if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))
7618 		return 0;
7619 
7620 	/* Disregard pcpu kthreads; they are where they need to be. */
7621 	if (kthread_is_per_cpu(p))
7622 		return 0;
7623 
7624 	if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) {
7625 		int cpu;
7626 
7627 		schedstat_inc(p->se.statistics.nr_failed_migrations_affine);
7628 
7629 		env->flags |= LBF_SOME_PINNED;
7630 
7631 		/*
7632 		 * Remember if this task can be migrated to any other CPU in
7633 		 * our sched_group. We may want to revisit it if we couldn't
7634 		 * meet load balance goals by pulling other tasks on src_cpu.
7635 		 *
7636 		 * Avoid computing new_dst_cpu
7637 		 * - for NEWLY_IDLE
7638 		 * - if we have already computed one in current iteration
7639 		 * - if it's an active balance
7640 		 */
7641 		if (env->idle == CPU_NEWLY_IDLE ||
7642 		    env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB))
7643 			return 0;
7644 
7645 		/* Prevent to re-select dst_cpu via env's CPUs: */
7646 		for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
7647 			if (cpumask_test_cpu(cpu, p->cpus_ptr)) {
7648 				env->flags |= LBF_DST_PINNED;
7649 				env->new_dst_cpu = cpu;
7650 				break;
7651 			}
7652 		}
7653 
7654 		return 0;
7655 	}
7656 
7657 	/* Record that we found at least one task that could run on dst_cpu */
7658 	env->flags &= ~LBF_ALL_PINNED;
7659 
7660 	if (task_running(env->src_rq, p)) {
7661 		schedstat_inc(p->se.statistics.nr_failed_migrations_running);
7662 		return 0;
7663 	}
7664 
7665 	/*
7666 	 * Aggressive migration if:
7667 	 * 1) active balance
7668 	 * 2) destination numa is preferred
7669 	 * 3) task is cache cold, or
7670 	 * 4) too many balance attempts have failed.
7671 	 */
7672 	if (env->flags & LBF_ACTIVE_LB)
7673 		return 1;
7674 
7675 	tsk_cache_hot = migrate_degrades_locality(p, env);
7676 	if (tsk_cache_hot == -1)
7677 		tsk_cache_hot = task_hot(p, env);
7678 
7679 	if (tsk_cache_hot <= 0 ||
7680 	    env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
7681 		if (tsk_cache_hot == 1) {
7682 			schedstat_inc(env->sd->lb_hot_gained[env->idle]);
7683 			schedstat_inc(p->se.statistics.nr_forced_migrations);
7684 		}
7685 		return 1;
7686 	}
7687 
7688 	schedstat_inc(p->se.statistics.nr_failed_migrations_hot);
7689 	return 0;
7690 }
7691 
7692 /*
7693  * detach_task() -- detach the task for the migration specified in env
7694  */
7695 static void detach_task(struct task_struct *p, struct lb_env *env)
7696 {
7697 	lockdep_assert_held(&env->src_rq->lock);
7698 
7699 	deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);
7700 	set_task_cpu(p, env->dst_cpu);
7701 }
7702 
7703 /*
7704  * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as
7705  * part of active balancing operations within "domain".
7706  *
7707  * Returns a task if successful and NULL otherwise.
7708  */
7709 static struct task_struct *detach_one_task(struct lb_env *env)
7710 {
7711 	struct task_struct *p;
7712 
7713 	lockdep_assert_held(&env->src_rq->lock);
7714 
7715 	list_for_each_entry_reverse(p,
7716 			&env->src_rq->cfs_tasks, se.group_node) {
7717 		if (!can_migrate_task(p, env))
7718 			continue;
7719 
7720 		detach_task(p, env);
7721 
7722 		/*
7723 		 * Right now, this is only the second place where
7724 		 * lb_gained[env->idle] is updated (other is detach_tasks)
7725 		 * so we can safely collect stats here rather than
7726 		 * inside detach_tasks().
7727 		 */
7728 		schedstat_inc(env->sd->lb_gained[env->idle]);
7729 		return p;
7730 	}
7731 	return NULL;
7732 }
7733 
7734 static const unsigned int sched_nr_migrate_break = 32;
7735 
7736 /*
7737  * detach_tasks() -- tries to detach up to imbalance load/util/tasks from
7738  * busiest_rq, as part of a balancing operation within domain "sd".
7739  *
7740  * Returns number of detached tasks if successful and 0 otherwise.
7741  */
7742 static int detach_tasks(struct lb_env *env)
7743 {
7744 	struct list_head *tasks = &env->src_rq->cfs_tasks;
7745 	unsigned long util, load;
7746 	struct task_struct *p;
7747 	int detached = 0;
7748 
7749 	lockdep_assert_held(&env->src_rq->lock);
7750 
7751 	/*
7752 	 * Source run queue has been emptied by another CPU, clear
7753 	 * LBF_ALL_PINNED flag as we will not test any task.
7754 	 */
7755 	if (env->src_rq->nr_running <= 1) {
7756 		env->flags &= ~LBF_ALL_PINNED;
7757 		return 0;
7758 	}
7759 
7760 	if (env->imbalance <= 0)
7761 		return 0;
7762 
7763 	while (!list_empty(tasks)) {
7764 		/*
7765 		 * We don't want to steal all, otherwise we may be treated likewise,
7766 		 * which could at worst lead to a livelock crash.
7767 		 */
7768 		if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)
7769 			break;
7770 
7771 		p = list_last_entry(tasks, struct task_struct, se.group_node);
7772 
7773 		env->loop++;
7774 		/* We've more or less seen every task there is, call it quits */
7775 		if (env->loop > env->loop_max)
7776 			break;
7777 
7778 		/* take a breather every nr_migrate tasks */
7779 		if (env->loop > env->loop_break) {
7780 			env->loop_break += sched_nr_migrate_break;
7781 			env->flags |= LBF_NEED_BREAK;
7782 			break;
7783 		}
7784 
7785 		if (!can_migrate_task(p, env))
7786 			goto next;
7787 
7788 		switch (env->migration_type) {
7789 		case migrate_load:
7790 			/*
7791 			 * Depending of the number of CPUs and tasks and the
7792 			 * cgroup hierarchy, task_h_load() can return a null
7793 			 * value. Make sure that env->imbalance decreases
7794 			 * otherwise detach_tasks() will stop only after
7795 			 * detaching up to loop_max tasks.
7796 			 */
7797 			load = max_t(unsigned long, task_h_load(p), 1);
7798 
7799 			if (sched_feat(LB_MIN) &&
7800 			    load < 16 && !env->sd->nr_balance_failed)
7801 				goto next;
7802 
7803 			/*
7804 			 * Make sure that we don't migrate too much load.
7805 			 * Nevertheless, let relax the constraint if
7806 			 * scheduler fails to find a good waiting task to
7807 			 * migrate.
7808 			 */
7809 			if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance)
7810 				goto next;
7811 
7812 			env->imbalance -= load;
7813 			break;
7814 
7815 		case migrate_util:
7816 			util = task_util_est(p);
7817 
7818 			if (util > env->imbalance)
7819 				goto next;
7820 
7821 			env->imbalance -= util;
7822 			break;
7823 
7824 		case migrate_task:
7825 			env->imbalance--;
7826 			break;
7827 
7828 		case migrate_misfit:
7829 			/* This is not a misfit task */
7830 			if (task_fits_capacity(p, capacity_of(env->src_cpu)))
7831 				goto next;
7832 
7833 			env->imbalance = 0;
7834 			break;
7835 		}
7836 
7837 		detach_task(p, env);
7838 		list_add(&p->se.group_node, &env->tasks);
7839 
7840 		detached++;
7841 
7842 #ifdef CONFIG_PREEMPTION
7843 		/*
7844 		 * NEWIDLE balancing is a source of latency, so preemptible
7845 		 * kernels will stop after the first task is detached to minimize
7846 		 * the critical section.
7847 		 */
7848 		if (env->idle == CPU_NEWLY_IDLE)
7849 			break;
7850 #endif
7851 
7852 		/*
7853 		 * We only want to steal up to the prescribed amount of
7854 		 * load/util/tasks.
7855 		 */
7856 		if (env->imbalance <= 0)
7857 			break;
7858 
7859 		continue;
7860 next:
7861 		list_move(&p->se.group_node, tasks);
7862 	}
7863 
7864 	/*
7865 	 * Right now, this is one of only two places we collect this stat
7866 	 * so we can safely collect detach_one_task() stats here rather
7867 	 * than inside detach_one_task().
7868 	 */
7869 	schedstat_add(env->sd->lb_gained[env->idle], detached);
7870 
7871 	return detached;
7872 }
7873 
7874 /*
7875  * attach_task() -- attach the task detached by detach_task() to its new rq.
7876  */
7877 static void attach_task(struct rq *rq, struct task_struct *p)
7878 {
7879 	lockdep_assert_held(&rq->lock);
7880 
7881 	BUG_ON(task_rq(p) != rq);
7882 	activate_task(rq, p, ENQUEUE_NOCLOCK);
7883 	check_preempt_curr(rq, p, 0);
7884 }
7885 
7886 /*
7887  * attach_one_task() -- attaches the task returned from detach_one_task() to
7888  * its new rq.
7889  */
7890 static void attach_one_task(struct rq *rq, struct task_struct *p)
7891 {
7892 	struct rq_flags rf;
7893 
7894 	rq_lock(rq, &rf);
7895 	update_rq_clock(rq);
7896 	attach_task(rq, p);
7897 	rq_unlock(rq, &rf);
7898 }
7899 
7900 /*
7901  * attach_tasks() -- attaches all tasks detached by detach_tasks() to their
7902  * new rq.
7903  */
7904 static void attach_tasks(struct lb_env *env)
7905 {
7906 	struct list_head *tasks = &env->tasks;
7907 	struct task_struct *p;
7908 	struct rq_flags rf;
7909 
7910 	rq_lock(env->dst_rq, &rf);
7911 	update_rq_clock(env->dst_rq);
7912 
7913 	while (!list_empty(tasks)) {
7914 		p = list_first_entry(tasks, struct task_struct, se.group_node);
7915 		list_del_init(&p->se.group_node);
7916 
7917 		attach_task(env->dst_rq, p);
7918 	}
7919 
7920 	rq_unlock(env->dst_rq, &rf);
7921 }
7922 
7923 #ifdef CONFIG_NO_HZ_COMMON
7924 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq)
7925 {
7926 	if (cfs_rq->avg.load_avg)
7927 		return true;
7928 
7929 	if (cfs_rq->avg.util_avg)
7930 		return true;
7931 
7932 	return false;
7933 }
7934 
7935 static inline bool others_have_blocked(struct rq *rq)
7936 {
7937 	if (READ_ONCE(rq->avg_rt.util_avg))
7938 		return true;
7939 
7940 	if (READ_ONCE(rq->avg_dl.util_avg))
7941 		return true;
7942 
7943 	if (thermal_load_avg(rq))
7944 		return true;
7945 
7946 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
7947 	if (READ_ONCE(rq->avg_irq.util_avg))
7948 		return true;
7949 #endif
7950 
7951 	return false;
7952 }
7953 
7954 static inline void update_blocked_load_tick(struct rq *rq)
7955 {
7956 	WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies);
7957 }
7958 
7959 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked)
7960 {
7961 	if (!has_blocked)
7962 		rq->has_blocked_load = 0;
7963 }
7964 #else
7965 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; }
7966 static inline bool others_have_blocked(struct rq *rq) { return false; }
7967 static inline void update_blocked_load_tick(struct rq *rq) {}
7968 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {}
7969 #endif
7970 
7971 static bool __update_blocked_others(struct rq *rq, bool *done)
7972 {
7973 	const struct sched_class *curr_class;
7974 	u64 now = rq_clock_pelt(rq);
7975 	unsigned long thermal_pressure;
7976 	bool decayed;
7977 
7978 	/*
7979 	 * update_load_avg() can call cpufreq_update_util(). Make sure that RT,
7980 	 * DL and IRQ signals have been updated before updating CFS.
7981 	 */
7982 	curr_class = rq->curr->sched_class;
7983 
7984 	thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq));
7985 
7986 	decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) |
7987 		  update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) |
7988 		  update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure) |
7989 		  update_irq_load_avg(rq, 0);
7990 
7991 	if (others_have_blocked(rq))
7992 		*done = false;
7993 
7994 	return decayed;
7995 }
7996 
7997 #ifdef CONFIG_FAIR_GROUP_SCHED
7998 
7999 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
8000 {
8001 	if (cfs_rq->load.weight)
8002 		return false;
8003 
8004 	if (cfs_rq->avg.load_sum)
8005 		return false;
8006 
8007 	if (cfs_rq->avg.util_sum)
8008 		return false;
8009 
8010 	if (cfs_rq->avg.runnable_sum)
8011 		return false;
8012 
8013 	return true;
8014 }
8015 
8016 static bool __update_blocked_fair(struct rq *rq, bool *done)
8017 {
8018 	struct cfs_rq *cfs_rq, *pos;
8019 	bool decayed = false;
8020 	int cpu = cpu_of(rq);
8021 
8022 	/*
8023 	 * Iterates the task_group tree in a bottom up fashion, see
8024 	 * list_add_leaf_cfs_rq() for details.
8025 	 */
8026 	for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) {
8027 		struct sched_entity *se;
8028 
8029 		if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) {
8030 			update_tg_load_avg(cfs_rq);
8031 
8032 			if (cfs_rq == &rq->cfs)
8033 				decayed = true;
8034 		}
8035 
8036 		/* Propagate pending load changes to the parent, if any: */
8037 		se = cfs_rq->tg->se[cpu];
8038 		if (se && !skip_blocked_update(se))
8039 			update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
8040 
8041 		/*
8042 		 * There can be a lot of idle CPU cgroups.  Don't let fully
8043 		 * decayed cfs_rqs linger on the list.
8044 		 */
8045 		if (cfs_rq_is_decayed(cfs_rq))
8046 			list_del_leaf_cfs_rq(cfs_rq);
8047 
8048 		/* Don't need periodic decay once load/util_avg are null */
8049 		if (cfs_rq_has_blocked(cfs_rq))
8050 			*done = false;
8051 	}
8052 
8053 	return decayed;
8054 }
8055 
8056 /*
8057  * Compute the hierarchical load factor for cfs_rq and all its ascendants.
8058  * This needs to be done in a top-down fashion because the load of a child
8059  * group is a fraction of its parents load.
8060  */
8061 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
8062 {
8063 	struct rq *rq = rq_of(cfs_rq);
8064 	struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
8065 	unsigned long now = jiffies;
8066 	unsigned long load;
8067 
8068 	if (cfs_rq->last_h_load_update == now)
8069 		return;
8070 
8071 	WRITE_ONCE(cfs_rq->h_load_next, NULL);
8072 	for_each_sched_entity(se) {
8073 		cfs_rq = cfs_rq_of(se);
8074 		WRITE_ONCE(cfs_rq->h_load_next, se);
8075 		if (cfs_rq->last_h_load_update == now)
8076 			break;
8077 	}
8078 
8079 	if (!se) {
8080 		cfs_rq->h_load = cfs_rq_load_avg(cfs_rq);
8081 		cfs_rq->last_h_load_update = now;
8082 	}
8083 
8084 	while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) {
8085 		load = cfs_rq->h_load;
8086 		load = div64_ul(load * se->avg.load_avg,
8087 			cfs_rq_load_avg(cfs_rq) + 1);
8088 		cfs_rq = group_cfs_rq(se);
8089 		cfs_rq->h_load = load;
8090 		cfs_rq->last_h_load_update = now;
8091 	}
8092 }
8093 
8094 static unsigned long task_h_load(struct task_struct *p)
8095 {
8096 	struct cfs_rq *cfs_rq = task_cfs_rq(p);
8097 
8098 	update_cfs_rq_h_load(cfs_rq);
8099 	return div64_ul(p->se.avg.load_avg * cfs_rq->h_load,
8100 			cfs_rq_load_avg(cfs_rq) + 1);
8101 }
8102 #else
8103 static bool __update_blocked_fair(struct rq *rq, bool *done)
8104 {
8105 	struct cfs_rq *cfs_rq = &rq->cfs;
8106 	bool decayed;
8107 
8108 	decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq);
8109 	if (cfs_rq_has_blocked(cfs_rq))
8110 		*done = false;
8111 
8112 	return decayed;
8113 }
8114 
8115 static unsigned long task_h_load(struct task_struct *p)
8116 {
8117 	return p->se.avg.load_avg;
8118 }
8119 #endif
8120 
8121 static void update_blocked_averages(int cpu)
8122 {
8123 	bool decayed = false, done = true;
8124 	struct rq *rq = cpu_rq(cpu);
8125 	struct rq_flags rf;
8126 
8127 	rq_lock_irqsave(rq, &rf);
8128 	update_blocked_load_tick(rq);
8129 	update_rq_clock(rq);
8130 
8131 	decayed |= __update_blocked_others(rq, &done);
8132 	decayed |= __update_blocked_fair(rq, &done);
8133 
8134 	update_blocked_load_status(rq, !done);
8135 	if (decayed)
8136 		cpufreq_update_util(rq, 0);
8137 	rq_unlock_irqrestore(rq, &rf);
8138 }
8139 
8140 /********** Helpers for find_busiest_group ************************/
8141 
8142 /*
8143  * sg_lb_stats - stats of a sched_group required for load_balancing
8144  */
8145 struct sg_lb_stats {
8146 	unsigned long avg_load; /*Avg load across the CPUs of the group */
8147 	unsigned long group_load; /* Total load over the CPUs of the group */
8148 	unsigned long group_capacity;
8149 	unsigned long group_util; /* Total utilization over the CPUs of the group */
8150 	unsigned long group_runnable; /* Total runnable time over the CPUs of the group */
8151 	unsigned int sum_nr_running; /* Nr of tasks running in the group */
8152 	unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */
8153 	unsigned int idle_cpus;
8154 	unsigned int group_weight;
8155 	enum group_type group_type;
8156 	unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */
8157 	unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */
8158 #ifdef CONFIG_NUMA_BALANCING
8159 	unsigned int nr_numa_running;
8160 	unsigned int nr_preferred_running;
8161 #endif
8162 };
8163 
8164 /*
8165  * sd_lb_stats - Structure to store the statistics of a sched_domain
8166  *		 during load balancing.
8167  */
8168 struct sd_lb_stats {
8169 	struct sched_group *busiest;	/* Busiest group in this sd */
8170 	struct sched_group *local;	/* Local group in this sd */
8171 	unsigned long total_load;	/* Total load of all groups in sd */
8172 	unsigned long total_capacity;	/* Total capacity of all groups in sd */
8173 	unsigned long avg_load;	/* Average load across all groups in sd */
8174 	unsigned int prefer_sibling; /* tasks should go to sibling first */
8175 
8176 	struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */
8177 	struct sg_lb_stats local_stat;	/* Statistics of the local group */
8178 };
8179 
8180 static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
8181 {
8182 	/*
8183 	 * Skimp on the clearing to avoid duplicate work. We can avoid clearing
8184 	 * local_stat because update_sg_lb_stats() does a full clear/assignment.
8185 	 * We must however set busiest_stat::group_type and
8186 	 * busiest_stat::idle_cpus to the worst busiest group because
8187 	 * update_sd_pick_busiest() reads these before assignment.
8188 	 */
8189 	*sds = (struct sd_lb_stats){
8190 		.busiest = NULL,
8191 		.local = NULL,
8192 		.total_load = 0UL,
8193 		.total_capacity = 0UL,
8194 		.busiest_stat = {
8195 			.idle_cpus = UINT_MAX,
8196 			.group_type = group_has_spare,
8197 		},
8198 	};
8199 }
8200 
8201 static unsigned long scale_rt_capacity(int cpu)
8202 {
8203 	struct rq *rq = cpu_rq(cpu);
8204 	unsigned long max = arch_scale_cpu_capacity(cpu);
8205 	unsigned long used, free;
8206 	unsigned long irq;
8207 
8208 	irq = cpu_util_irq(rq);
8209 
8210 	if (unlikely(irq >= max))
8211 		return 1;
8212 
8213 	/*
8214 	 * avg_rt.util_avg and avg_dl.util_avg track binary signals
8215 	 * (running and not running) with weights 0 and 1024 respectively.
8216 	 * avg_thermal.load_avg tracks thermal pressure and the weighted
8217 	 * average uses the actual delta max capacity(load).
8218 	 */
8219 	used = READ_ONCE(rq->avg_rt.util_avg);
8220 	used += READ_ONCE(rq->avg_dl.util_avg);
8221 	used += thermal_load_avg(rq);
8222 
8223 	if (unlikely(used >= max))
8224 		return 1;
8225 
8226 	free = max - used;
8227 
8228 	return scale_irq_capacity(free, irq, max);
8229 }
8230 
8231 static void update_cpu_capacity(struct sched_domain *sd, int cpu)
8232 {
8233 	unsigned long capacity = scale_rt_capacity(cpu);
8234 	struct sched_group *sdg = sd->groups;
8235 
8236 	cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu);
8237 
8238 	if (!capacity)
8239 		capacity = 1;
8240 
8241 	cpu_rq(cpu)->cpu_capacity = capacity;
8242 	trace_sched_cpu_capacity_tp(cpu_rq(cpu));
8243 
8244 	sdg->sgc->capacity = capacity;
8245 	sdg->sgc->min_capacity = capacity;
8246 	sdg->sgc->max_capacity = capacity;
8247 }
8248 
8249 void update_group_capacity(struct sched_domain *sd, int cpu)
8250 {
8251 	struct sched_domain *child = sd->child;
8252 	struct sched_group *group, *sdg = sd->groups;
8253 	unsigned long capacity, min_capacity, max_capacity;
8254 	unsigned long interval;
8255 
8256 	interval = msecs_to_jiffies(sd->balance_interval);
8257 	interval = clamp(interval, 1UL, max_load_balance_interval);
8258 	sdg->sgc->next_update = jiffies + interval;
8259 
8260 	if (!child) {
8261 		update_cpu_capacity(sd, cpu);
8262 		return;
8263 	}
8264 
8265 	capacity = 0;
8266 	min_capacity = ULONG_MAX;
8267 	max_capacity = 0;
8268 
8269 	if (child->flags & SD_OVERLAP) {
8270 		/*
8271 		 * SD_OVERLAP domains cannot assume that child groups
8272 		 * span the current group.
8273 		 */
8274 
8275 		for_each_cpu(cpu, sched_group_span(sdg)) {
8276 			unsigned long cpu_cap = capacity_of(cpu);
8277 
8278 			capacity += cpu_cap;
8279 			min_capacity = min(cpu_cap, min_capacity);
8280 			max_capacity = max(cpu_cap, max_capacity);
8281 		}
8282 	} else  {
8283 		/*
8284 		 * !SD_OVERLAP domains can assume that child groups
8285 		 * span the current group.
8286 		 */
8287 
8288 		group = child->groups;
8289 		do {
8290 			struct sched_group_capacity *sgc = group->sgc;
8291 
8292 			capacity += sgc->capacity;
8293 			min_capacity = min(sgc->min_capacity, min_capacity);
8294 			max_capacity = max(sgc->max_capacity, max_capacity);
8295 			group = group->next;
8296 		} while (group != child->groups);
8297 	}
8298 
8299 	sdg->sgc->capacity = capacity;
8300 	sdg->sgc->min_capacity = min_capacity;
8301 	sdg->sgc->max_capacity = max_capacity;
8302 }
8303 
8304 /*
8305  * Check whether the capacity of the rq has been noticeably reduced by side
8306  * activity. The imbalance_pct is used for the threshold.
8307  * Return true is the capacity is reduced
8308  */
8309 static inline int
8310 check_cpu_capacity(struct rq *rq, struct sched_domain *sd)
8311 {
8312 	return ((rq->cpu_capacity * sd->imbalance_pct) <
8313 				(rq->cpu_capacity_orig * 100));
8314 }
8315 
8316 /*
8317  * Check whether a rq has a misfit task and if it looks like we can actually
8318  * help that task: we can migrate the task to a CPU of higher capacity, or
8319  * the task's current CPU is heavily pressured.
8320  */
8321 static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd)
8322 {
8323 	return rq->misfit_task_load &&
8324 		(rq->cpu_capacity_orig < rq->rd->max_cpu_capacity ||
8325 		 check_cpu_capacity(rq, sd));
8326 }
8327 
8328 /*
8329  * Group imbalance indicates (and tries to solve) the problem where balancing
8330  * groups is inadequate due to ->cpus_ptr constraints.
8331  *
8332  * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a
8333  * cpumask covering 1 CPU of the first group and 3 CPUs of the second group.
8334  * Something like:
8335  *
8336  *	{ 0 1 2 3 } { 4 5 6 7 }
8337  *	        *     * * *
8338  *
8339  * If we were to balance group-wise we'd place two tasks in the first group and
8340  * two tasks in the second group. Clearly this is undesired as it will overload
8341  * cpu 3 and leave one of the CPUs in the second group unused.
8342  *
8343  * The current solution to this issue is detecting the skew in the first group
8344  * by noticing the lower domain failed to reach balance and had difficulty
8345  * moving tasks due to affinity constraints.
8346  *
8347  * When this is so detected; this group becomes a candidate for busiest; see
8348  * update_sd_pick_busiest(). And calculate_imbalance() and
8349  * find_busiest_group() avoid some of the usual balance conditions to allow it
8350  * to create an effective group imbalance.
8351  *
8352  * This is a somewhat tricky proposition since the next run might not find the
8353  * group imbalance and decide the groups need to be balanced again. A most
8354  * subtle and fragile situation.
8355  */
8356 
8357 static inline int sg_imbalanced(struct sched_group *group)
8358 {
8359 	return group->sgc->imbalance;
8360 }
8361 
8362 /*
8363  * group_has_capacity returns true if the group has spare capacity that could
8364  * be used by some tasks.
8365  * We consider that a group has spare capacity if the  * number of task is
8366  * smaller than the number of CPUs or if the utilization is lower than the
8367  * available capacity for CFS tasks.
8368  * For the latter, we use a threshold to stabilize the state, to take into
8369  * account the variance of the tasks' load and to return true if the available
8370  * capacity in meaningful for the load balancer.
8371  * As an example, an available capacity of 1% can appear but it doesn't make
8372  * any benefit for the load balance.
8373  */
8374 static inline bool
8375 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
8376 {
8377 	if (sgs->sum_nr_running < sgs->group_weight)
8378 		return true;
8379 
8380 	if ((sgs->group_capacity * imbalance_pct) <
8381 			(sgs->group_runnable * 100))
8382 		return false;
8383 
8384 	if ((sgs->group_capacity * 100) >
8385 			(sgs->group_util * imbalance_pct))
8386 		return true;
8387 
8388 	return false;
8389 }
8390 
8391 /*
8392  *  group_is_overloaded returns true if the group has more tasks than it can
8393  *  handle.
8394  *  group_is_overloaded is not equals to !group_has_capacity because a group
8395  *  with the exact right number of tasks, has no more spare capacity but is not
8396  *  overloaded so both group_has_capacity and group_is_overloaded return
8397  *  false.
8398  */
8399 static inline bool
8400 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
8401 {
8402 	if (sgs->sum_nr_running <= sgs->group_weight)
8403 		return false;
8404 
8405 	if ((sgs->group_capacity * 100) <
8406 			(sgs->group_util * imbalance_pct))
8407 		return true;
8408 
8409 	if ((sgs->group_capacity * imbalance_pct) <
8410 			(sgs->group_runnable * 100))
8411 		return true;
8412 
8413 	return false;
8414 }
8415 
8416 static inline enum
8417 group_type group_classify(unsigned int imbalance_pct,
8418 			  struct sched_group *group,
8419 			  struct sg_lb_stats *sgs)
8420 {
8421 	if (group_is_overloaded(imbalance_pct, sgs))
8422 		return group_overloaded;
8423 
8424 	if (sg_imbalanced(group))
8425 		return group_imbalanced;
8426 
8427 	if (sgs->group_asym_packing)
8428 		return group_asym_packing;
8429 
8430 	if (sgs->group_misfit_task_load)
8431 		return group_misfit_task;
8432 
8433 	if (!group_has_capacity(imbalance_pct, sgs))
8434 		return group_fully_busy;
8435 
8436 	return group_has_spare;
8437 }
8438 
8439 /**
8440  * update_sg_lb_stats - Update sched_group's statistics for load balancing.
8441  * @env: The load balancing environment.
8442  * @group: sched_group whose statistics are to be updated.
8443  * @sgs: variable to hold the statistics for this group.
8444  * @sg_status: Holds flag indicating the status of the sched_group
8445  */
8446 static inline void update_sg_lb_stats(struct lb_env *env,
8447 				      struct sched_group *group,
8448 				      struct sg_lb_stats *sgs,
8449 				      int *sg_status)
8450 {
8451 	int i, nr_running, local_group;
8452 
8453 	memset(sgs, 0, sizeof(*sgs));
8454 
8455 	local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(group));
8456 
8457 	for_each_cpu_and(i, sched_group_span(group), env->cpus) {
8458 		struct rq *rq = cpu_rq(i);
8459 
8460 		sgs->group_load += cpu_load(rq);
8461 		sgs->group_util += cpu_util(i);
8462 		sgs->group_runnable += cpu_runnable(rq);
8463 		sgs->sum_h_nr_running += rq->cfs.h_nr_running;
8464 
8465 		nr_running = rq->nr_running;
8466 		sgs->sum_nr_running += nr_running;
8467 
8468 		if (nr_running > 1)
8469 			*sg_status |= SG_OVERLOAD;
8470 
8471 		if (cpu_overutilized(i))
8472 			*sg_status |= SG_OVERUTILIZED;
8473 
8474 #ifdef CONFIG_NUMA_BALANCING
8475 		sgs->nr_numa_running += rq->nr_numa_running;
8476 		sgs->nr_preferred_running += rq->nr_preferred_running;
8477 #endif
8478 		/*
8479 		 * No need to call idle_cpu() if nr_running is not 0
8480 		 */
8481 		if (!nr_running && idle_cpu(i)) {
8482 			sgs->idle_cpus++;
8483 			/* Idle cpu can't have misfit task */
8484 			continue;
8485 		}
8486 
8487 		if (local_group)
8488 			continue;
8489 
8490 		/* Check for a misfit task on the cpu */
8491 		if (env->sd->flags & SD_ASYM_CPUCAPACITY &&
8492 		    sgs->group_misfit_task_load < rq->misfit_task_load) {
8493 			sgs->group_misfit_task_load = rq->misfit_task_load;
8494 			*sg_status |= SG_OVERLOAD;
8495 		}
8496 	}
8497 
8498 	/* Check if dst CPU is idle and preferred to this group */
8499 	if (env->sd->flags & SD_ASYM_PACKING &&
8500 	    env->idle != CPU_NOT_IDLE &&
8501 	    sgs->sum_h_nr_running &&
8502 	    sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu)) {
8503 		sgs->group_asym_packing = 1;
8504 	}
8505 
8506 	sgs->group_capacity = group->sgc->capacity;
8507 
8508 	sgs->group_weight = group->group_weight;
8509 
8510 	sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);
8511 
8512 	/* Computing avg_load makes sense only when group is overloaded */
8513 	if (sgs->group_type == group_overloaded)
8514 		sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
8515 				sgs->group_capacity;
8516 }
8517 
8518 /**
8519  * update_sd_pick_busiest - return 1 on busiest group
8520  * @env: The load balancing environment.
8521  * @sds: sched_domain statistics
8522  * @sg: sched_group candidate to be checked for being the busiest
8523  * @sgs: sched_group statistics
8524  *
8525  * Determine if @sg is a busier group than the previously selected
8526  * busiest group.
8527  *
8528  * Return: %true if @sg is a busier group than the previously selected
8529  * busiest group. %false otherwise.
8530  */
8531 static bool update_sd_pick_busiest(struct lb_env *env,
8532 				   struct sd_lb_stats *sds,
8533 				   struct sched_group *sg,
8534 				   struct sg_lb_stats *sgs)
8535 {
8536 	struct sg_lb_stats *busiest = &sds->busiest_stat;
8537 
8538 	/* Make sure that there is at least one task to pull */
8539 	if (!sgs->sum_h_nr_running)
8540 		return false;
8541 
8542 	/*
8543 	 * Don't try to pull misfit tasks we can't help.
8544 	 * We can use max_capacity here as reduction in capacity on some
8545 	 * CPUs in the group should either be possible to resolve
8546 	 * internally or be covered by avg_load imbalance (eventually).
8547 	 */
8548 	if (sgs->group_type == group_misfit_task &&
8549 	    (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||
8550 	     sds->local_stat.group_type != group_has_spare))
8551 		return false;
8552 
8553 	if (sgs->group_type > busiest->group_type)
8554 		return true;
8555 
8556 	if (sgs->group_type < busiest->group_type)
8557 		return false;
8558 
8559 	/*
8560 	 * The candidate and the current busiest group are the same type of
8561 	 * group. Let check which one is the busiest according to the type.
8562 	 */
8563 
8564 	switch (sgs->group_type) {
8565 	case group_overloaded:
8566 		/* Select the overloaded group with highest avg_load. */
8567 		if (sgs->avg_load <= busiest->avg_load)
8568 			return false;
8569 		break;
8570 
8571 	case group_imbalanced:
8572 		/*
8573 		 * Select the 1st imbalanced group as we don't have any way to
8574 		 * choose one more than another.
8575 		 */
8576 		return false;
8577 
8578 	case group_asym_packing:
8579 		/* Prefer to move from lowest priority CPU's work */
8580 		if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu))
8581 			return false;
8582 		break;
8583 
8584 	case group_misfit_task:
8585 		/*
8586 		 * If we have more than one misfit sg go with the biggest
8587 		 * misfit.
8588 		 */
8589 		if (sgs->group_misfit_task_load < busiest->group_misfit_task_load)
8590 			return false;
8591 		break;
8592 
8593 	case group_fully_busy:
8594 		/*
8595 		 * Select the fully busy group with highest avg_load. In
8596 		 * theory, there is no need to pull task from such kind of
8597 		 * group because tasks have all compute capacity that they need
8598 		 * but we can still improve the overall throughput by reducing
8599 		 * contention when accessing shared HW resources.
8600 		 *
8601 		 * XXX for now avg_load is not computed and always 0 so we
8602 		 * select the 1st one.
8603 		 */
8604 		if (sgs->avg_load <= busiest->avg_load)
8605 			return false;
8606 		break;
8607 
8608 	case group_has_spare:
8609 		/*
8610 		 * Select not overloaded group with lowest number of idle cpus
8611 		 * and highest number of running tasks. We could also compare
8612 		 * the spare capacity which is more stable but it can end up
8613 		 * that the group has less spare capacity but finally more idle
8614 		 * CPUs which means less opportunity to pull tasks.
8615 		 */
8616 		if (sgs->idle_cpus > busiest->idle_cpus)
8617 			return false;
8618 		else if ((sgs->idle_cpus == busiest->idle_cpus) &&
8619 			 (sgs->sum_nr_running <= busiest->sum_nr_running))
8620 			return false;
8621 
8622 		break;
8623 	}
8624 
8625 	/*
8626 	 * Candidate sg has no more than one task per CPU and has higher
8627 	 * per-CPU capacity. Migrating tasks to less capable CPUs may harm
8628 	 * throughput. Maximize throughput, power/energy consequences are not
8629 	 * considered.
8630 	 */
8631 	if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
8632 	    (sgs->group_type <= group_fully_busy) &&
8633 	    (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu))))
8634 		return false;
8635 
8636 	return true;
8637 }
8638 
8639 #ifdef CONFIG_NUMA_BALANCING
8640 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
8641 {
8642 	if (sgs->sum_h_nr_running > sgs->nr_numa_running)
8643 		return regular;
8644 	if (sgs->sum_h_nr_running > sgs->nr_preferred_running)
8645 		return remote;
8646 	return all;
8647 }
8648 
8649 static inline enum fbq_type fbq_classify_rq(struct rq *rq)
8650 {
8651 	if (rq->nr_running > rq->nr_numa_running)
8652 		return regular;
8653 	if (rq->nr_running > rq->nr_preferred_running)
8654 		return remote;
8655 	return all;
8656 }
8657 #else
8658 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
8659 {
8660 	return all;
8661 }
8662 
8663 static inline enum fbq_type fbq_classify_rq(struct rq *rq)
8664 {
8665 	return regular;
8666 }
8667 #endif /* CONFIG_NUMA_BALANCING */
8668 
8669 
8670 struct sg_lb_stats;
8671 
8672 /*
8673  * task_running_on_cpu - return 1 if @p is running on @cpu.
8674  */
8675 
8676 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p)
8677 {
8678 	/* Task has no contribution or is new */
8679 	if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
8680 		return 0;
8681 
8682 	if (task_on_rq_queued(p))
8683 		return 1;
8684 
8685 	return 0;
8686 }
8687 
8688 /**
8689  * idle_cpu_without - would a given CPU be idle without p ?
8690  * @cpu: the processor on which idleness is tested.
8691  * @p: task which should be ignored.
8692  *
8693  * Return: 1 if the CPU would be idle. 0 otherwise.
8694  */
8695 static int idle_cpu_without(int cpu, struct task_struct *p)
8696 {
8697 	struct rq *rq = cpu_rq(cpu);
8698 
8699 	if (rq->curr != rq->idle && rq->curr != p)
8700 		return 0;
8701 
8702 	/*
8703 	 * rq->nr_running can't be used but an updated version without the
8704 	 * impact of p on cpu must be used instead. The updated nr_running
8705 	 * be computed and tested before calling idle_cpu_without().
8706 	 */
8707 
8708 #ifdef CONFIG_SMP
8709 	if (rq->ttwu_pending)
8710 		return 0;
8711 #endif
8712 
8713 	return 1;
8714 }
8715 
8716 /*
8717  * update_sg_wakeup_stats - Update sched_group's statistics for wakeup.
8718  * @sd: The sched_domain level to look for idlest group.
8719  * @group: sched_group whose statistics are to be updated.
8720  * @sgs: variable to hold the statistics for this group.
8721  * @p: The task for which we look for the idlest group/CPU.
8722  */
8723 static inline void update_sg_wakeup_stats(struct sched_domain *sd,
8724 					  struct sched_group *group,
8725 					  struct sg_lb_stats *sgs,
8726 					  struct task_struct *p)
8727 {
8728 	int i, nr_running;
8729 
8730 	memset(sgs, 0, sizeof(*sgs));
8731 
8732 	for_each_cpu(i, sched_group_span(group)) {
8733 		struct rq *rq = cpu_rq(i);
8734 		unsigned int local;
8735 
8736 		sgs->group_load += cpu_load_without(rq, p);
8737 		sgs->group_util += cpu_util_without(i, p);
8738 		sgs->group_runnable += cpu_runnable_without(rq, p);
8739 		local = task_running_on_cpu(i, p);
8740 		sgs->sum_h_nr_running += rq->cfs.h_nr_running - local;
8741 
8742 		nr_running = rq->nr_running - local;
8743 		sgs->sum_nr_running += nr_running;
8744 
8745 		/*
8746 		 * No need to call idle_cpu_without() if nr_running is not 0
8747 		 */
8748 		if (!nr_running && idle_cpu_without(i, p))
8749 			sgs->idle_cpus++;
8750 
8751 	}
8752 
8753 	/* Check if task fits in the group */
8754 	if (sd->flags & SD_ASYM_CPUCAPACITY &&
8755 	    !task_fits_capacity(p, group->sgc->max_capacity)) {
8756 		sgs->group_misfit_task_load = 1;
8757 	}
8758 
8759 	sgs->group_capacity = group->sgc->capacity;
8760 
8761 	sgs->group_weight = group->group_weight;
8762 
8763 	sgs->group_type = group_classify(sd->imbalance_pct, group, sgs);
8764 
8765 	/*
8766 	 * Computing avg_load makes sense only when group is fully busy or
8767 	 * overloaded
8768 	 */
8769 	if (sgs->group_type == group_fully_busy ||
8770 		sgs->group_type == group_overloaded)
8771 		sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
8772 				sgs->group_capacity;
8773 }
8774 
8775 static bool update_pick_idlest(struct sched_group *idlest,
8776 			       struct sg_lb_stats *idlest_sgs,
8777 			       struct sched_group *group,
8778 			       struct sg_lb_stats *sgs)
8779 {
8780 	if (sgs->group_type < idlest_sgs->group_type)
8781 		return true;
8782 
8783 	if (sgs->group_type > idlest_sgs->group_type)
8784 		return false;
8785 
8786 	/*
8787 	 * The candidate and the current idlest group are the same type of
8788 	 * group. Let check which one is the idlest according to the type.
8789 	 */
8790 
8791 	switch (sgs->group_type) {
8792 	case group_overloaded:
8793 	case group_fully_busy:
8794 		/* Select the group with lowest avg_load. */
8795 		if (idlest_sgs->avg_load <= sgs->avg_load)
8796 			return false;
8797 		break;
8798 
8799 	case group_imbalanced:
8800 	case group_asym_packing:
8801 		/* Those types are not used in the slow wakeup path */
8802 		return false;
8803 
8804 	case group_misfit_task:
8805 		/* Select group with the highest max capacity */
8806 		if (idlest->sgc->max_capacity >= group->sgc->max_capacity)
8807 			return false;
8808 		break;
8809 
8810 	case group_has_spare:
8811 		/* Select group with most idle CPUs */
8812 		if (idlest_sgs->idle_cpus > sgs->idle_cpus)
8813 			return false;
8814 
8815 		/* Select group with lowest group_util */
8816 		if (idlest_sgs->idle_cpus == sgs->idle_cpus &&
8817 			idlest_sgs->group_util <= sgs->group_util)
8818 			return false;
8819 
8820 		break;
8821 	}
8822 
8823 	return true;
8824 }
8825 
8826 /*
8827  * Allow a NUMA imbalance if busy CPUs is less than 25% of the domain.
8828  * This is an approximation as the number of running tasks may not be
8829  * related to the number of busy CPUs due to sched_setaffinity.
8830  */
8831 static inline bool allow_numa_imbalance(int dst_running, int dst_weight)
8832 {
8833 	return (dst_running < (dst_weight >> 2));
8834 }
8835 
8836 /*
8837  * find_idlest_group() finds and returns the least busy CPU group within the
8838  * domain.
8839  *
8840  * Assumes p is allowed on at least one CPU in sd.
8841  */
8842 static struct sched_group *
8843 find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
8844 {
8845 	struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups;
8846 	struct sg_lb_stats local_sgs, tmp_sgs;
8847 	struct sg_lb_stats *sgs;
8848 	unsigned long imbalance;
8849 	struct sg_lb_stats idlest_sgs = {
8850 			.avg_load = UINT_MAX,
8851 			.group_type = group_overloaded,
8852 	};
8853 
8854 	do {
8855 		int local_group;
8856 
8857 		/* Skip over this group if it has no CPUs allowed */
8858 		if (!cpumask_intersects(sched_group_span(group),
8859 					p->cpus_ptr))
8860 			continue;
8861 
8862 		local_group = cpumask_test_cpu(this_cpu,
8863 					       sched_group_span(group));
8864 
8865 		if (local_group) {
8866 			sgs = &local_sgs;
8867 			local = group;
8868 		} else {
8869 			sgs = &tmp_sgs;
8870 		}
8871 
8872 		update_sg_wakeup_stats(sd, group, sgs, p);
8873 
8874 		if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) {
8875 			idlest = group;
8876 			idlest_sgs = *sgs;
8877 		}
8878 
8879 	} while (group = group->next, group != sd->groups);
8880 
8881 
8882 	/* There is no idlest group to push tasks to */
8883 	if (!idlest)
8884 		return NULL;
8885 
8886 	/* The local group has been skipped because of CPU affinity */
8887 	if (!local)
8888 		return idlest;
8889 
8890 	/*
8891 	 * If the local group is idler than the selected idlest group
8892 	 * don't try and push the task.
8893 	 */
8894 	if (local_sgs.group_type < idlest_sgs.group_type)
8895 		return NULL;
8896 
8897 	/*
8898 	 * If the local group is busier than the selected idlest group
8899 	 * try and push the task.
8900 	 */
8901 	if (local_sgs.group_type > idlest_sgs.group_type)
8902 		return idlest;
8903 
8904 	switch (local_sgs.group_type) {
8905 	case group_overloaded:
8906 	case group_fully_busy:
8907 
8908 		/* Calculate allowed imbalance based on load */
8909 		imbalance = scale_load_down(NICE_0_LOAD) *
8910 				(sd->imbalance_pct-100) / 100;
8911 
8912 		/*
8913 		 * When comparing groups across NUMA domains, it's possible for
8914 		 * the local domain to be very lightly loaded relative to the
8915 		 * remote domains but "imbalance" skews the comparison making
8916 		 * remote CPUs look much more favourable. When considering
8917 		 * cross-domain, add imbalance to the load on the remote node
8918 		 * and consider staying local.
8919 		 */
8920 
8921 		if ((sd->flags & SD_NUMA) &&
8922 		    ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load))
8923 			return NULL;
8924 
8925 		/*
8926 		 * If the local group is less loaded than the selected
8927 		 * idlest group don't try and push any tasks.
8928 		 */
8929 		if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance))
8930 			return NULL;
8931 
8932 		if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load)
8933 			return NULL;
8934 		break;
8935 
8936 	case group_imbalanced:
8937 	case group_asym_packing:
8938 		/* Those type are not used in the slow wakeup path */
8939 		return NULL;
8940 
8941 	case group_misfit_task:
8942 		/* Select group with the highest max capacity */
8943 		if (local->sgc->max_capacity >= idlest->sgc->max_capacity)
8944 			return NULL;
8945 		break;
8946 
8947 	case group_has_spare:
8948 		if (sd->flags & SD_NUMA) {
8949 #ifdef CONFIG_NUMA_BALANCING
8950 			int idlest_cpu;
8951 			/*
8952 			 * If there is spare capacity at NUMA, try to select
8953 			 * the preferred node
8954 			 */
8955 			if (cpu_to_node(this_cpu) == p->numa_preferred_nid)
8956 				return NULL;
8957 
8958 			idlest_cpu = cpumask_first(sched_group_span(idlest));
8959 			if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid)
8960 				return idlest;
8961 #endif
8962 			/*
8963 			 * Otherwise, keep the task on this node to stay close
8964 			 * its wakeup source and improve locality. If there is
8965 			 * a real need of migration, periodic load balance will
8966 			 * take care of it.
8967 			 */
8968 			if (allow_numa_imbalance(local_sgs.sum_nr_running, sd->span_weight))
8969 				return NULL;
8970 		}
8971 
8972 		/*
8973 		 * Select group with highest number of idle CPUs. We could also
8974 		 * compare the utilization which is more stable but it can end
8975 		 * up that the group has less spare capacity but finally more
8976 		 * idle CPUs which means more opportunity to run task.
8977 		 */
8978 		if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus)
8979 			return NULL;
8980 		break;
8981 	}
8982 
8983 	return idlest;
8984 }
8985 
8986 /**
8987  * update_sd_lb_stats - Update sched_domain's statistics for load balancing.
8988  * @env: The load balancing environment.
8989  * @sds: variable to hold the statistics for this sched_domain.
8990  */
8991 
8992 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)
8993 {
8994 	struct sched_domain *child = env->sd->child;
8995 	struct sched_group *sg = env->sd->groups;
8996 	struct sg_lb_stats *local = &sds->local_stat;
8997 	struct sg_lb_stats tmp_sgs;
8998 	int sg_status = 0;
8999 
9000 	do {
9001 		struct sg_lb_stats *sgs = &tmp_sgs;
9002 		int local_group;
9003 
9004 		local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg));
9005 		if (local_group) {
9006 			sds->local = sg;
9007 			sgs = local;
9008 
9009 			if (env->idle != CPU_NEWLY_IDLE ||
9010 			    time_after_eq(jiffies, sg->sgc->next_update))
9011 				update_group_capacity(env->sd, env->dst_cpu);
9012 		}
9013 
9014 		update_sg_lb_stats(env, sg, sgs, &sg_status);
9015 
9016 		if (local_group)
9017 			goto next_group;
9018 
9019 
9020 		if (update_sd_pick_busiest(env, sds, sg, sgs)) {
9021 			sds->busiest = sg;
9022 			sds->busiest_stat = *sgs;
9023 		}
9024 
9025 next_group:
9026 		/* Now, start updating sd_lb_stats */
9027 		sds->total_load += sgs->group_load;
9028 		sds->total_capacity += sgs->group_capacity;
9029 
9030 		sg = sg->next;
9031 	} while (sg != env->sd->groups);
9032 
9033 	/* Tag domain that child domain prefers tasks go to siblings first */
9034 	sds->prefer_sibling = child && child->flags & SD_PREFER_SIBLING;
9035 
9036 
9037 	if (env->sd->flags & SD_NUMA)
9038 		env->fbq_type = fbq_classify_group(&sds->busiest_stat);
9039 
9040 	if (!env->sd->parent) {
9041 		struct root_domain *rd = env->dst_rq->rd;
9042 
9043 		/* update overload indicator if we are at root domain */
9044 		WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD);
9045 
9046 		/* Update over-utilization (tipping point, U >= 0) indicator */
9047 		WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED);
9048 		trace_sched_overutilized_tp(rd, sg_status & SG_OVERUTILIZED);
9049 	} else if (sg_status & SG_OVERUTILIZED) {
9050 		struct root_domain *rd = env->dst_rq->rd;
9051 
9052 		WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED);
9053 		trace_sched_overutilized_tp(rd, SG_OVERUTILIZED);
9054 	}
9055 }
9056 
9057 #define NUMA_IMBALANCE_MIN 2
9058 
9059 static inline long adjust_numa_imbalance(int imbalance,
9060 				int dst_running, int dst_weight)
9061 {
9062 	if (!allow_numa_imbalance(dst_running, dst_weight))
9063 		return imbalance;
9064 
9065 	/*
9066 	 * Allow a small imbalance based on a simple pair of communicating
9067 	 * tasks that remain local when the destination is lightly loaded.
9068 	 */
9069 	if (imbalance <= NUMA_IMBALANCE_MIN)
9070 		return 0;
9071 
9072 	return imbalance;
9073 }
9074 
9075 /**
9076  * calculate_imbalance - Calculate the amount of imbalance present within the
9077  *			 groups of a given sched_domain during load balance.
9078  * @env: load balance environment
9079  * @sds: statistics of the sched_domain whose imbalance is to be calculated.
9080  */
9081 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
9082 {
9083 	struct sg_lb_stats *local, *busiest;
9084 
9085 	local = &sds->local_stat;
9086 	busiest = &sds->busiest_stat;
9087 
9088 	if (busiest->group_type == group_misfit_task) {
9089 		/* Set imbalance to allow misfit tasks to be balanced. */
9090 		env->migration_type = migrate_misfit;
9091 		env->imbalance = 1;
9092 		return;
9093 	}
9094 
9095 	if (busiest->group_type == group_asym_packing) {
9096 		/*
9097 		 * In case of asym capacity, we will try to migrate all load to
9098 		 * the preferred CPU.
9099 		 */
9100 		env->migration_type = migrate_task;
9101 		env->imbalance = busiest->sum_h_nr_running;
9102 		return;
9103 	}
9104 
9105 	if (busiest->group_type == group_imbalanced) {
9106 		/*
9107 		 * In the group_imb case we cannot rely on group-wide averages
9108 		 * to ensure CPU-load equilibrium, try to move any task to fix
9109 		 * the imbalance. The next load balance will take care of
9110 		 * balancing back the system.
9111 		 */
9112 		env->migration_type = migrate_task;
9113 		env->imbalance = 1;
9114 		return;
9115 	}
9116 
9117 	/*
9118 	 * Try to use spare capacity of local group without overloading it or
9119 	 * emptying busiest.
9120 	 */
9121 	if (local->group_type == group_has_spare) {
9122 		if ((busiest->group_type > group_fully_busy) &&
9123 		    !(env->sd->flags & SD_SHARE_PKG_RESOURCES)) {
9124 			/*
9125 			 * If busiest is overloaded, try to fill spare
9126 			 * capacity. This might end up creating spare capacity
9127 			 * in busiest or busiest still being overloaded but
9128 			 * there is no simple way to directly compute the
9129 			 * amount of load to migrate in order to balance the
9130 			 * system.
9131 			 */
9132 			env->migration_type = migrate_util;
9133 			env->imbalance = max(local->group_capacity, local->group_util) -
9134 					 local->group_util;
9135 
9136 			/*
9137 			 * In some cases, the group's utilization is max or even
9138 			 * higher than capacity because of migrations but the
9139 			 * local CPU is (newly) idle. There is at least one
9140 			 * waiting task in this overloaded busiest group. Let's
9141 			 * try to pull it.
9142 			 */
9143 			if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) {
9144 				env->migration_type = migrate_task;
9145 				env->imbalance = 1;
9146 			}
9147 
9148 			return;
9149 		}
9150 
9151 		if (busiest->group_weight == 1 || sds->prefer_sibling) {
9152 			unsigned int nr_diff = busiest->sum_nr_running;
9153 			/*
9154 			 * When prefer sibling, evenly spread running tasks on
9155 			 * groups.
9156 			 */
9157 			env->migration_type = migrate_task;
9158 			lsub_positive(&nr_diff, local->sum_nr_running);
9159 			env->imbalance = nr_diff >> 1;
9160 		} else {
9161 
9162 			/*
9163 			 * If there is no overload, we just want to even the number of
9164 			 * idle cpus.
9165 			 */
9166 			env->migration_type = migrate_task;
9167 			env->imbalance = max_t(long, 0, (local->idle_cpus -
9168 						 busiest->idle_cpus) >> 1);
9169 		}
9170 
9171 		/* Consider allowing a small imbalance between NUMA groups */
9172 		if (env->sd->flags & SD_NUMA) {
9173 			env->imbalance = adjust_numa_imbalance(env->imbalance,
9174 				busiest->sum_nr_running, busiest->group_weight);
9175 		}
9176 
9177 		return;
9178 	}
9179 
9180 	/*
9181 	 * Local is fully busy but has to take more load to relieve the
9182 	 * busiest group
9183 	 */
9184 	if (local->group_type < group_overloaded) {
9185 		/*
9186 		 * Local will become overloaded so the avg_load metrics are
9187 		 * finally needed.
9188 		 */
9189 
9190 		local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /
9191 				  local->group_capacity;
9192 
9193 		sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
9194 				sds->total_capacity;
9195 		/*
9196 		 * If the local group is more loaded than the selected
9197 		 * busiest group don't try to pull any tasks.
9198 		 */
9199 		if (local->avg_load >= busiest->avg_load) {
9200 			env->imbalance = 0;
9201 			return;
9202 		}
9203 	}
9204 
9205 	/*
9206 	 * Both group are or will become overloaded and we're trying to get all
9207 	 * the CPUs to the average_load, so we don't want to push ourselves
9208 	 * above the average load, nor do we wish to reduce the max loaded CPU
9209 	 * below the average load. At the same time, we also don't want to
9210 	 * reduce the group load below the group capacity. Thus we look for
9211 	 * the minimum possible imbalance.
9212 	 */
9213 	env->migration_type = migrate_load;
9214 	env->imbalance = min(
9215 		(busiest->avg_load - sds->avg_load) * busiest->group_capacity,
9216 		(sds->avg_load - local->avg_load) * local->group_capacity
9217 	) / SCHED_CAPACITY_SCALE;
9218 }
9219 
9220 /******* find_busiest_group() helpers end here *********************/
9221 
9222 /*
9223  * Decision matrix according to the local and busiest group type:
9224  *
9225  * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded
9226  * has_spare        nr_idle   balanced   N/A    N/A  balanced   balanced
9227  * fully_busy       nr_idle   nr_idle    N/A    N/A  balanced   balanced
9228  * misfit_task      force     N/A        N/A    N/A  force      force
9229  * asym_packing     force     force      N/A    N/A  force      force
9230  * imbalanced       force     force      N/A    N/A  force      force
9231  * overloaded       force     force      N/A    N/A  force      avg_load
9232  *
9233  * N/A :      Not Applicable because already filtered while updating
9234  *            statistics.
9235  * balanced : The system is balanced for these 2 groups.
9236  * force :    Calculate the imbalance as load migration is probably needed.
9237  * avg_load : Only if imbalance is significant enough.
9238  * nr_idle :  dst_cpu is not busy and the number of idle CPUs is quite
9239  *            different in groups.
9240  */
9241 
9242 /**
9243  * find_busiest_group - Returns the busiest group within the sched_domain
9244  * if there is an imbalance.
9245  *
9246  * Also calculates the amount of runnable load which should be moved
9247  * to restore balance.
9248  *
9249  * @env: The load balancing environment.
9250  *
9251  * Return:	- The busiest group if imbalance exists.
9252  */
9253 static struct sched_group *find_busiest_group(struct lb_env *env)
9254 {
9255 	struct sg_lb_stats *local, *busiest;
9256 	struct sd_lb_stats sds;
9257 
9258 	init_sd_lb_stats(&sds);
9259 
9260 	/*
9261 	 * Compute the various statistics relevant for load balancing at
9262 	 * this level.
9263 	 */
9264 	update_sd_lb_stats(env, &sds);
9265 
9266 	if (sched_energy_enabled()) {
9267 		struct root_domain *rd = env->dst_rq->rd;
9268 
9269 		if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized))
9270 			goto out_balanced;
9271 	}
9272 
9273 	local = &sds.local_stat;
9274 	busiest = &sds.busiest_stat;
9275 
9276 	/* There is no busy sibling group to pull tasks from */
9277 	if (!sds.busiest)
9278 		goto out_balanced;
9279 
9280 	/* Misfit tasks should be dealt with regardless of the avg load */
9281 	if (busiest->group_type == group_misfit_task)
9282 		goto force_balance;
9283 
9284 	/* ASYM feature bypasses nice load balance check */
9285 	if (busiest->group_type == group_asym_packing)
9286 		goto force_balance;
9287 
9288 	/*
9289 	 * If the busiest group is imbalanced the below checks don't
9290 	 * work because they assume all things are equal, which typically
9291 	 * isn't true due to cpus_ptr constraints and the like.
9292 	 */
9293 	if (busiest->group_type == group_imbalanced)
9294 		goto force_balance;
9295 
9296 	/*
9297 	 * If the local group is busier than the selected busiest group
9298 	 * don't try and pull any tasks.
9299 	 */
9300 	if (local->group_type > busiest->group_type)
9301 		goto out_balanced;
9302 
9303 	/*
9304 	 * When groups are overloaded, use the avg_load to ensure fairness
9305 	 * between tasks.
9306 	 */
9307 	if (local->group_type == group_overloaded) {
9308 		/*
9309 		 * If the local group is more loaded than the selected
9310 		 * busiest group don't try to pull any tasks.
9311 		 */
9312 		if (local->avg_load >= busiest->avg_load)
9313 			goto out_balanced;
9314 
9315 		/* XXX broken for overlapping NUMA groups */
9316 		sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) /
9317 				sds.total_capacity;
9318 
9319 		/*
9320 		 * Don't pull any tasks if this group is already above the
9321 		 * domain average load.
9322 		 */
9323 		if (local->avg_load >= sds.avg_load)
9324 			goto out_balanced;
9325 
9326 		/*
9327 		 * If the busiest group is more loaded, use imbalance_pct to be
9328 		 * conservative.
9329 		 */
9330 		if (100 * busiest->avg_load <=
9331 				env->sd->imbalance_pct * local->avg_load)
9332 			goto out_balanced;
9333 	}
9334 
9335 	/* Try to move all excess tasks to child's sibling domain */
9336 	if (sds.prefer_sibling && local->group_type == group_has_spare &&
9337 	    busiest->sum_nr_running > local->sum_nr_running + 1)
9338 		goto force_balance;
9339 
9340 	if (busiest->group_type != group_overloaded) {
9341 		if (env->idle == CPU_NOT_IDLE)
9342 			/*
9343 			 * If the busiest group is not overloaded (and as a
9344 			 * result the local one too) but this CPU is already
9345 			 * busy, let another idle CPU try to pull task.
9346 			 */
9347 			goto out_balanced;
9348 
9349 		if (busiest->group_weight > 1 &&
9350 		    local->idle_cpus <= (busiest->idle_cpus + 1))
9351 			/*
9352 			 * If the busiest group is not overloaded
9353 			 * and there is no imbalance between this and busiest
9354 			 * group wrt idle CPUs, it is balanced. The imbalance
9355 			 * becomes significant if the diff is greater than 1
9356 			 * otherwise we might end up to just move the imbalance
9357 			 * on another group. Of course this applies only if
9358 			 * there is more than 1 CPU per group.
9359 			 */
9360 			goto out_balanced;
9361 
9362 		if (busiest->sum_h_nr_running == 1)
9363 			/*
9364 			 * busiest doesn't have any tasks waiting to run
9365 			 */
9366 			goto out_balanced;
9367 	}
9368 
9369 force_balance:
9370 	/* Looks like there is an imbalance. Compute it */
9371 	calculate_imbalance(env, &sds);
9372 	return env->imbalance ? sds.busiest : NULL;
9373 
9374 out_balanced:
9375 	env->imbalance = 0;
9376 	return NULL;
9377 }
9378 
9379 /*
9380  * find_busiest_queue - find the busiest runqueue among the CPUs in the group.
9381  */
9382 static struct rq *find_busiest_queue(struct lb_env *env,
9383 				     struct sched_group *group)
9384 {
9385 	struct rq *busiest = NULL, *rq;
9386 	unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1;
9387 	unsigned int busiest_nr = 0;
9388 	int i;
9389 
9390 	for_each_cpu_and(i, sched_group_span(group), env->cpus) {
9391 		unsigned long capacity, load, util;
9392 		unsigned int nr_running;
9393 		enum fbq_type rt;
9394 
9395 		rq = cpu_rq(i);
9396 		rt = fbq_classify_rq(rq);
9397 
9398 		/*
9399 		 * We classify groups/runqueues into three groups:
9400 		 *  - regular: there are !numa tasks
9401 		 *  - remote:  there are numa tasks that run on the 'wrong' node
9402 		 *  - all:     there is no distinction
9403 		 *
9404 		 * In order to avoid migrating ideally placed numa tasks,
9405 		 * ignore those when there's better options.
9406 		 *
9407 		 * If we ignore the actual busiest queue to migrate another
9408 		 * task, the next balance pass can still reduce the busiest
9409 		 * queue by moving tasks around inside the node.
9410 		 *
9411 		 * If we cannot move enough load due to this classification
9412 		 * the next pass will adjust the group classification and
9413 		 * allow migration of more tasks.
9414 		 *
9415 		 * Both cases only affect the total convergence complexity.
9416 		 */
9417 		if (rt > env->fbq_type)
9418 			continue;
9419 
9420 		nr_running = rq->cfs.h_nr_running;
9421 		if (!nr_running)
9422 			continue;
9423 
9424 		capacity = capacity_of(i);
9425 
9426 		/*
9427 		 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could
9428 		 * eventually lead to active_balancing high->low capacity.
9429 		 * Higher per-CPU capacity is considered better than balancing
9430 		 * average load.
9431 		 */
9432 		if (env->sd->flags & SD_ASYM_CPUCAPACITY &&
9433 		    !capacity_greater(capacity_of(env->dst_cpu), capacity) &&
9434 		    nr_running == 1)
9435 			continue;
9436 
9437 		switch (env->migration_type) {
9438 		case migrate_load:
9439 			/*
9440 			 * When comparing with load imbalance, use cpu_load()
9441 			 * which is not scaled with the CPU capacity.
9442 			 */
9443 			load = cpu_load(rq);
9444 
9445 			if (nr_running == 1 && load > env->imbalance &&
9446 			    !check_cpu_capacity(rq, env->sd))
9447 				break;
9448 
9449 			/*
9450 			 * For the load comparisons with the other CPUs,
9451 			 * consider the cpu_load() scaled with the CPU
9452 			 * capacity, so that the load can be moved away
9453 			 * from the CPU that is potentially running at a
9454 			 * lower capacity.
9455 			 *
9456 			 * Thus we're looking for max(load_i / capacity_i),
9457 			 * crosswise multiplication to rid ourselves of the
9458 			 * division works out to:
9459 			 * load_i * capacity_j > load_j * capacity_i;
9460 			 * where j is our previous maximum.
9461 			 */
9462 			if (load * busiest_capacity > busiest_load * capacity) {
9463 				busiest_load = load;
9464 				busiest_capacity = capacity;
9465 				busiest = rq;
9466 			}
9467 			break;
9468 
9469 		case migrate_util:
9470 			util = cpu_util(cpu_of(rq));
9471 
9472 			/*
9473 			 * Don't try to pull utilization from a CPU with one
9474 			 * running task. Whatever its utilization, we will fail
9475 			 * detach the task.
9476 			 */
9477 			if (nr_running <= 1)
9478 				continue;
9479 
9480 			if (busiest_util < util) {
9481 				busiest_util = util;
9482 				busiest = rq;
9483 			}
9484 			break;
9485 
9486 		case migrate_task:
9487 			if (busiest_nr < nr_running) {
9488 				busiest_nr = nr_running;
9489 				busiest = rq;
9490 			}
9491 			break;
9492 
9493 		case migrate_misfit:
9494 			/*
9495 			 * For ASYM_CPUCAPACITY domains with misfit tasks we
9496 			 * simply seek the "biggest" misfit task.
9497 			 */
9498 			if (rq->misfit_task_load > busiest_load) {
9499 				busiest_load = rq->misfit_task_load;
9500 				busiest = rq;
9501 			}
9502 
9503 			break;
9504 
9505 		}
9506 	}
9507 
9508 	return busiest;
9509 }
9510 
9511 /*
9512  * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
9513  * so long as it is large enough.
9514  */
9515 #define MAX_PINNED_INTERVAL	512
9516 
9517 static inline bool
9518 asym_active_balance(struct lb_env *env)
9519 {
9520 	/*
9521 	 * ASYM_PACKING needs to force migrate tasks from busy but
9522 	 * lower priority CPUs in order to pack all tasks in the
9523 	 * highest priority CPUs.
9524 	 */
9525 	return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) &&
9526 	       sched_asym_prefer(env->dst_cpu, env->src_cpu);
9527 }
9528 
9529 static inline bool
9530 imbalanced_active_balance(struct lb_env *env)
9531 {
9532 	struct sched_domain *sd = env->sd;
9533 
9534 	/*
9535 	 * The imbalanced case includes the case of pinned tasks preventing a fair
9536 	 * distribution of the load on the system but also the even distribution of the
9537 	 * threads on a system with spare capacity
9538 	 */
9539 	if ((env->migration_type == migrate_task) &&
9540 	    (sd->nr_balance_failed > sd->cache_nice_tries+2))
9541 		return 1;
9542 
9543 	return 0;
9544 }
9545 
9546 static int need_active_balance(struct lb_env *env)
9547 {
9548 	struct sched_domain *sd = env->sd;
9549 
9550 	if (asym_active_balance(env))
9551 		return 1;
9552 
9553 	if (imbalanced_active_balance(env))
9554 		return 1;
9555 
9556 	/*
9557 	 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task.
9558 	 * It's worth migrating the task if the src_cpu's capacity is reduced
9559 	 * because of other sched_class or IRQs if more capacity stays
9560 	 * available on dst_cpu.
9561 	 */
9562 	if ((env->idle != CPU_NOT_IDLE) &&
9563 	    (env->src_rq->cfs.h_nr_running == 1)) {
9564 		if ((check_cpu_capacity(env->src_rq, sd)) &&
9565 		    (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100))
9566 			return 1;
9567 	}
9568 
9569 	if (env->migration_type == migrate_misfit)
9570 		return 1;
9571 
9572 	return 0;
9573 }
9574 
9575 static int active_load_balance_cpu_stop(void *data);
9576 
9577 static int should_we_balance(struct lb_env *env)
9578 {
9579 	struct sched_group *sg = env->sd->groups;
9580 	int cpu;
9581 
9582 	/*
9583 	 * Ensure the balancing environment is consistent; can happen
9584 	 * when the softirq triggers 'during' hotplug.
9585 	 */
9586 	if (!cpumask_test_cpu(env->dst_cpu, env->cpus))
9587 		return 0;
9588 
9589 	/*
9590 	 * In the newly idle case, we will allow all the CPUs
9591 	 * to do the newly idle load balance.
9592 	 */
9593 	if (env->idle == CPU_NEWLY_IDLE)
9594 		return 1;
9595 
9596 	/* Try to find first idle CPU */
9597 	for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) {
9598 		if (!idle_cpu(cpu))
9599 			continue;
9600 
9601 		/* Are we the first idle CPU? */
9602 		return cpu == env->dst_cpu;
9603 	}
9604 
9605 	/* Are we the first CPU of this group ? */
9606 	return group_balance_cpu(sg) == env->dst_cpu;
9607 }
9608 
9609 /*
9610  * Check this_cpu to ensure it is balanced within domain. Attempt to move
9611  * tasks if there is an imbalance.
9612  */
9613 static int load_balance(int this_cpu, struct rq *this_rq,
9614 			struct sched_domain *sd, enum cpu_idle_type idle,
9615 			int *continue_balancing)
9616 {
9617 	int ld_moved, cur_ld_moved, active_balance = 0;
9618 	struct sched_domain *sd_parent = sd->parent;
9619 	struct sched_group *group;
9620 	struct rq *busiest;
9621 	struct rq_flags rf;
9622 	struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask);
9623 
9624 	struct lb_env env = {
9625 		.sd		= sd,
9626 		.dst_cpu	= this_cpu,
9627 		.dst_rq		= this_rq,
9628 		.dst_grpmask    = sched_group_span(sd->groups),
9629 		.idle		= idle,
9630 		.loop_break	= sched_nr_migrate_break,
9631 		.cpus		= cpus,
9632 		.fbq_type	= all,
9633 		.tasks		= LIST_HEAD_INIT(env.tasks),
9634 	};
9635 
9636 	cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask);
9637 
9638 	schedstat_inc(sd->lb_count[idle]);
9639 
9640 redo:
9641 	if (!should_we_balance(&env)) {
9642 		*continue_balancing = 0;
9643 		goto out_balanced;
9644 	}
9645 
9646 	group = find_busiest_group(&env);
9647 	if (!group) {
9648 		schedstat_inc(sd->lb_nobusyg[idle]);
9649 		goto out_balanced;
9650 	}
9651 
9652 	busiest = find_busiest_queue(&env, group);
9653 	if (!busiest) {
9654 		schedstat_inc(sd->lb_nobusyq[idle]);
9655 		goto out_balanced;
9656 	}
9657 
9658 	BUG_ON(busiest == env.dst_rq);
9659 
9660 	schedstat_add(sd->lb_imbalance[idle], env.imbalance);
9661 
9662 	env.src_cpu = busiest->cpu;
9663 	env.src_rq = busiest;
9664 
9665 	ld_moved = 0;
9666 	/* Clear this flag as soon as we find a pullable task */
9667 	env.flags |= LBF_ALL_PINNED;
9668 	if (busiest->nr_running > 1) {
9669 		/*
9670 		 * Attempt to move tasks. If find_busiest_group has found
9671 		 * an imbalance but busiest->nr_running <= 1, the group is
9672 		 * still unbalanced. ld_moved simply stays zero, so it is
9673 		 * correctly treated as an imbalance.
9674 		 */
9675 		env.loop_max  = min(sysctl_sched_nr_migrate, busiest->nr_running);
9676 
9677 more_balance:
9678 		rq_lock_irqsave(busiest, &rf);
9679 		update_rq_clock(busiest);
9680 
9681 		/*
9682 		 * cur_ld_moved - load moved in current iteration
9683 		 * ld_moved     - cumulative load moved across iterations
9684 		 */
9685 		cur_ld_moved = detach_tasks(&env);
9686 
9687 		/*
9688 		 * We've detached some tasks from busiest_rq. Every
9689 		 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely
9690 		 * unlock busiest->lock, and we are able to be sure
9691 		 * that nobody can manipulate the tasks in parallel.
9692 		 * See task_rq_lock() family for the details.
9693 		 */
9694 
9695 		rq_unlock(busiest, &rf);
9696 
9697 		if (cur_ld_moved) {
9698 			attach_tasks(&env);
9699 			ld_moved += cur_ld_moved;
9700 		}
9701 
9702 		local_irq_restore(rf.flags);
9703 
9704 		if (env.flags & LBF_NEED_BREAK) {
9705 			env.flags &= ~LBF_NEED_BREAK;
9706 			goto more_balance;
9707 		}
9708 
9709 		/*
9710 		 * Revisit (affine) tasks on src_cpu that couldn't be moved to
9711 		 * us and move them to an alternate dst_cpu in our sched_group
9712 		 * where they can run. The upper limit on how many times we
9713 		 * iterate on same src_cpu is dependent on number of CPUs in our
9714 		 * sched_group.
9715 		 *
9716 		 * This changes load balance semantics a bit on who can move
9717 		 * load to a given_cpu. In addition to the given_cpu itself
9718 		 * (or a ilb_cpu acting on its behalf where given_cpu is
9719 		 * nohz-idle), we now have balance_cpu in a position to move
9720 		 * load to given_cpu. In rare situations, this may cause
9721 		 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding
9722 		 * _independently_ and at _same_ time to move some load to
9723 		 * given_cpu) causing excess load to be moved to given_cpu.
9724 		 * This however should not happen so much in practice and
9725 		 * moreover subsequent load balance cycles should correct the
9726 		 * excess load moved.
9727 		 */
9728 		if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {
9729 
9730 			/* Prevent to re-select dst_cpu via env's CPUs */
9731 			__cpumask_clear_cpu(env.dst_cpu, env.cpus);
9732 
9733 			env.dst_rq	 = cpu_rq(env.new_dst_cpu);
9734 			env.dst_cpu	 = env.new_dst_cpu;
9735 			env.flags	&= ~LBF_DST_PINNED;
9736 			env.loop	 = 0;
9737 			env.loop_break	 = sched_nr_migrate_break;
9738 
9739 			/*
9740 			 * Go back to "more_balance" rather than "redo" since we
9741 			 * need to continue with same src_cpu.
9742 			 */
9743 			goto more_balance;
9744 		}
9745 
9746 		/*
9747 		 * We failed to reach balance because of affinity.
9748 		 */
9749 		if (sd_parent) {
9750 			int *group_imbalance = &sd_parent->groups->sgc->imbalance;
9751 
9752 			if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)
9753 				*group_imbalance = 1;
9754 		}
9755 
9756 		/* All tasks on this runqueue were pinned by CPU affinity */
9757 		if (unlikely(env.flags & LBF_ALL_PINNED)) {
9758 			__cpumask_clear_cpu(cpu_of(busiest), cpus);
9759 			/*
9760 			 * Attempting to continue load balancing at the current
9761 			 * sched_domain level only makes sense if there are
9762 			 * active CPUs remaining as possible busiest CPUs to
9763 			 * pull load from which are not contained within the
9764 			 * destination group that is receiving any migrated
9765 			 * load.
9766 			 */
9767 			if (!cpumask_subset(cpus, env.dst_grpmask)) {
9768 				env.loop = 0;
9769 				env.loop_break = sched_nr_migrate_break;
9770 				goto redo;
9771 			}
9772 			goto out_all_pinned;
9773 		}
9774 	}
9775 
9776 	if (!ld_moved) {
9777 		schedstat_inc(sd->lb_failed[idle]);
9778 		/*
9779 		 * Increment the failure counter only on periodic balance.
9780 		 * We do not want newidle balance, which can be very
9781 		 * frequent, pollute the failure counter causing
9782 		 * excessive cache_hot migrations and active balances.
9783 		 */
9784 		if (idle != CPU_NEWLY_IDLE)
9785 			sd->nr_balance_failed++;
9786 
9787 		if (need_active_balance(&env)) {
9788 			unsigned long flags;
9789 
9790 			raw_spin_lock_irqsave(&busiest->lock, flags);
9791 
9792 			/*
9793 			 * Don't kick the active_load_balance_cpu_stop,
9794 			 * if the curr task on busiest CPU can't be
9795 			 * moved to this_cpu:
9796 			 */
9797 			if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) {
9798 				raw_spin_unlock_irqrestore(&busiest->lock,
9799 							    flags);
9800 				goto out_one_pinned;
9801 			}
9802 
9803 			/* Record that we found at least one task that could run on this_cpu */
9804 			env.flags &= ~LBF_ALL_PINNED;
9805 
9806 			/*
9807 			 * ->active_balance synchronizes accesses to
9808 			 * ->active_balance_work.  Once set, it's cleared
9809 			 * only after active load balance is finished.
9810 			 */
9811 			if (!busiest->active_balance) {
9812 				busiest->active_balance = 1;
9813 				busiest->push_cpu = this_cpu;
9814 				active_balance = 1;
9815 			}
9816 			raw_spin_unlock_irqrestore(&busiest->lock, flags);
9817 
9818 			if (active_balance) {
9819 				stop_one_cpu_nowait(cpu_of(busiest),
9820 					active_load_balance_cpu_stop, busiest,
9821 					&busiest->active_balance_work);
9822 			}
9823 		}
9824 	} else {
9825 		sd->nr_balance_failed = 0;
9826 	}
9827 
9828 	if (likely(!active_balance) || need_active_balance(&env)) {
9829 		/* We were unbalanced, so reset the balancing interval */
9830 		sd->balance_interval = sd->min_interval;
9831 	}
9832 
9833 	goto out;
9834 
9835 out_balanced:
9836 	/*
9837 	 * We reach balance although we may have faced some affinity
9838 	 * constraints. Clear the imbalance flag only if other tasks got
9839 	 * a chance to move and fix the imbalance.
9840 	 */
9841 	if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {
9842 		int *group_imbalance = &sd_parent->groups->sgc->imbalance;
9843 
9844 		if (*group_imbalance)
9845 			*group_imbalance = 0;
9846 	}
9847 
9848 out_all_pinned:
9849 	/*
9850 	 * We reach balance because all tasks are pinned at this level so
9851 	 * we can't migrate them. Let the imbalance flag set so parent level
9852 	 * can try to migrate them.
9853 	 */
9854 	schedstat_inc(sd->lb_balanced[idle]);
9855 
9856 	sd->nr_balance_failed = 0;
9857 
9858 out_one_pinned:
9859 	ld_moved = 0;
9860 
9861 	/*
9862 	 * newidle_balance() disregards balance intervals, so we could
9863 	 * repeatedly reach this code, which would lead to balance_interval
9864 	 * skyrocketing in a short amount of time. Skip the balance_interval
9865 	 * increase logic to avoid that.
9866 	 */
9867 	if (env.idle == CPU_NEWLY_IDLE)
9868 		goto out;
9869 
9870 	/* tune up the balancing interval */
9871 	if ((env.flags & LBF_ALL_PINNED &&
9872 	     sd->balance_interval < MAX_PINNED_INTERVAL) ||
9873 	    sd->balance_interval < sd->max_interval)
9874 		sd->balance_interval *= 2;
9875 out:
9876 	return ld_moved;
9877 }
9878 
9879 static inline unsigned long
9880 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy)
9881 {
9882 	unsigned long interval = sd->balance_interval;
9883 
9884 	if (cpu_busy)
9885 		interval *= sd->busy_factor;
9886 
9887 	/* scale ms to jiffies */
9888 	interval = msecs_to_jiffies(interval);
9889 
9890 	/*
9891 	 * Reduce likelihood of busy balancing at higher domains racing with
9892 	 * balancing at lower domains by preventing their balancing periods
9893 	 * from being multiples of each other.
9894 	 */
9895 	if (cpu_busy)
9896 		interval -= 1;
9897 
9898 	interval = clamp(interval, 1UL, max_load_balance_interval);
9899 
9900 	return interval;
9901 }
9902 
9903 static inline void
9904 update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
9905 {
9906 	unsigned long interval, next;
9907 
9908 	/* used by idle balance, so cpu_busy = 0 */
9909 	interval = get_sd_balance_interval(sd, 0);
9910 	next = sd->last_balance + interval;
9911 
9912 	if (time_after(*next_balance, next))
9913 		*next_balance = next;
9914 }
9915 
9916 /*
9917  * active_load_balance_cpu_stop is run by the CPU stopper. It pushes
9918  * running tasks off the busiest CPU onto idle CPUs. It requires at
9919  * least 1 task to be running on each physical CPU where possible, and
9920  * avoids physical / logical imbalances.
9921  */
9922 static int active_load_balance_cpu_stop(void *data)
9923 {
9924 	struct rq *busiest_rq = data;
9925 	int busiest_cpu = cpu_of(busiest_rq);
9926 	int target_cpu = busiest_rq->push_cpu;
9927 	struct rq *target_rq = cpu_rq(target_cpu);
9928 	struct sched_domain *sd;
9929 	struct task_struct *p = NULL;
9930 	struct rq_flags rf;
9931 
9932 	rq_lock_irq(busiest_rq, &rf);
9933 	/*
9934 	 * Between queueing the stop-work and running it is a hole in which
9935 	 * CPUs can become inactive. We should not move tasks from or to
9936 	 * inactive CPUs.
9937 	 */
9938 	if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu))
9939 		goto out_unlock;
9940 
9941 	/* Make sure the requested CPU hasn't gone down in the meantime: */
9942 	if (unlikely(busiest_cpu != smp_processor_id() ||
9943 		     !busiest_rq->active_balance))
9944 		goto out_unlock;
9945 
9946 	/* Is there any task to move? */
9947 	if (busiest_rq->nr_running <= 1)
9948 		goto out_unlock;
9949 
9950 	/*
9951 	 * This condition is "impossible", if it occurs
9952 	 * we need to fix it. Originally reported by
9953 	 * Bjorn Helgaas on a 128-CPU setup.
9954 	 */
9955 	BUG_ON(busiest_rq == target_rq);
9956 
9957 	/* Search for an sd spanning us and the target CPU. */
9958 	rcu_read_lock();
9959 	for_each_domain(target_cpu, sd) {
9960 		if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
9961 			break;
9962 	}
9963 
9964 	if (likely(sd)) {
9965 		struct lb_env env = {
9966 			.sd		= sd,
9967 			.dst_cpu	= target_cpu,
9968 			.dst_rq		= target_rq,
9969 			.src_cpu	= busiest_rq->cpu,
9970 			.src_rq		= busiest_rq,
9971 			.idle		= CPU_IDLE,
9972 			.flags		= LBF_ACTIVE_LB,
9973 		};
9974 
9975 		schedstat_inc(sd->alb_count);
9976 		update_rq_clock(busiest_rq);
9977 
9978 		p = detach_one_task(&env);
9979 		if (p) {
9980 			schedstat_inc(sd->alb_pushed);
9981 			/* Active balancing done, reset the failure counter. */
9982 			sd->nr_balance_failed = 0;
9983 		} else {
9984 			schedstat_inc(sd->alb_failed);
9985 		}
9986 	}
9987 	rcu_read_unlock();
9988 out_unlock:
9989 	busiest_rq->active_balance = 0;
9990 	rq_unlock(busiest_rq, &rf);
9991 
9992 	if (p)
9993 		attach_one_task(target_rq, p);
9994 
9995 	local_irq_enable();
9996 
9997 	return 0;
9998 }
9999 
10000 static DEFINE_SPINLOCK(balancing);
10001 
10002 /*
10003  * Scale the max load_balance interval with the number of CPUs in the system.
10004  * This trades load-balance latency on larger machines for less cross talk.
10005  */
10006 void update_max_interval(void)
10007 {
10008 	max_load_balance_interval = HZ*num_online_cpus()/10;
10009 }
10010 
10011 /*
10012  * It checks each scheduling domain to see if it is due to be balanced,
10013  * and initiates a balancing operation if so.
10014  *
10015  * Balancing parameters are set up in init_sched_domains.
10016  */
10017 static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
10018 {
10019 	int continue_balancing = 1;
10020 	int cpu = rq->cpu;
10021 	int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);
10022 	unsigned long interval;
10023 	struct sched_domain *sd;
10024 	/* Earliest time when we have to do rebalance again */
10025 	unsigned long next_balance = jiffies + 60*HZ;
10026 	int update_next_balance = 0;
10027 	int need_serialize, need_decay = 0;
10028 	u64 max_cost = 0;
10029 
10030 	rcu_read_lock();
10031 	for_each_domain(cpu, sd) {
10032 		/*
10033 		 * Decay the newidle max times here because this is a regular
10034 		 * visit to all the domains. Decay ~1% per second.
10035 		 */
10036 		if (time_after(jiffies, sd->next_decay_max_lb_cost)) {
10037 			sd->max_newidle_lb_cost =
10038 				(sd->max_newidle_lb_cost * 253) / 256;
10039 			sd->next_decay_max_lb_cost = jiffies + HZ;
10040 			need_decay = 1;
10041 		}
10042 		max_cost += sd->max_newidle_lb_cost;
10043 
10044 		/*
10045 		 * Stop the load balance at this level. There is another
10046 		 * CPU in our sched group which is doing load balancing more
10047 		 * actively.
10048 		 */
10049 		if (!continue_balancing) {
10050 			if (need_decay)
10051 				continue;
10052 			break;
10053 		}
10054 
10055 		interval = get_sd_balance_interval(sd, busy);
10056 
10057 		need_serialize = sd->flags & SD_SERIALIZE;
10058 		if (need_serialize) {
10059 			if (!spin_trylock(&balancing))
10060 				goto out;
10061 		}
10062 
10063 		if (time_after_eq(jiffies, sd->last_balance + interval)) {
10064 			if (load_balance(cpu, rq, sd, idle, &continue_balancing)) {
10065 				/*
10066 				 * The LBF_DST_PINNED logic could have changed
10067 				 * env->dst_cpu, so we can't know our idle
10068 				 * state even if we migrated tasks. Update it.
10069 				 */
10070 				idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE;
10071 				busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);
10072 			}
10073 			sd->last_balance = jiffies;
10074 			interval = get_sd_balance_interval(sd, busy);
10075 		}
10076 		if (need_serialize)
10077 			spin_unlock(&balancing);
10078 out:
10079 		if (time_after(next_balance, sd->last_balance + interval)) {
10080 			next_balance = sd->last_balance + interval;
10081 			update_next_balance = 1;
10082 		}
10083 	}
10084 	if (need_decay) {
10085 		/*
10086 		 * Ensure the rq-wide value also decays but keep it at a
10087 		 * reasonable floor to avoid funnies with rq->avg_idle.
10088 		 */
10089 		rq->max_idle_balance_cost =
10090 			max((u64)sysctl_sched_migration_cost, max_cost);
10091 	}
10092 	rcu_read_unlock();
10093 
10094 	/*
10095 	 * next_balance will be updated only when there is a need.
10096 	 * When the cpu is attached to null domain for ex, it will not be
10097 	 * updated.
10098 	 */
10099 	if (likely(update_next_balance))
10100 		rq->next_balance = next_balance;
10101 
10102 }
10103 
10104 static inline int on_null_domain(struct rq *rq)
10105 {
10106 	return unlikely(!rcu_dereference_sched(rq->sd));
10107 }
10108 
10109 #ifdef CONFIG_NO_HZ_COMMON
10110 /*
10111  * idle load balancing details
10112  * - When one of the busy CPUs notice that there may be an idle rebalancing
10113  *   needed, they will kick the idle load balancer, which then does idle
10114  *   load balancing for all the idle CPUs.
10115  * - HK_FLAG_MISC CPUs are used for this task, because HK_FLAG_SCHED not set
10116  *   anywhere yet.
10117  */
10118 
10119 static inline int find_new_ilb(void)
10120 {
10121 	int ilb;
10122 
10123 	for_each_cpu_and(ilb, nohz.idle_cpus_mask,
10124 			      housekeeping_cpumask(HK_FLAG_MISC)) {
10125 
10126 		if (ilb == smp_processor_id())
10127 			continue;
10128 
10129 		if (idle_cpu(ilb))
10130 			return ilb;
10131 	}
10132 
10133 	return nr_cpu_ids;
10134 }
10135 
10136 /*
10137  * Kick a CPU to do the nohz balancing, if it is time for it. We pick any
10138  * idle CPU in the HK_FLAG_MISC housekeeping set (if there is one).
10139  */
10140 static void kick_ilb(unsigned int flags)
10141 {
10142 	int ilb_cpu;
10143 
10144 	/*
10145 	 * Increase nohz.next_balance only when if full ilb is triggered but
10146 	 * not if we only update stats.
10147 	 */
10148 	if (flags & NOHZ_BALANCE_KICK)
10149 		nohz.next_balance = jiffies+1;
10150 
10151 	ilb_cpu = find_new_ilb();
10152 
10153 	if (ilb_cpu >= nr_cpu_ids)
10154 		return;
10155 
10156 	/*
10157 	 * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets
10158 	 * the first flag owns it; cleared by nohz_csd_func().
10159 	 */
10160 	flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu));
10161 	if (flags & NOHZ_KICK_MASK)
10162 		return;
10163 
10164 	/*
10165 	 * This way we generate an IPI on the target CPU which
10166 	 * is idle. And the softirq performing nohz idle load balance
10167 	 * will be run before returning from the IPI.
10168 	 */
10169 	smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);
10170 }
10171 
10172 /*
10173  * Current decision point for kicking the idle load balancer in the presence
10174  * of idle CPUs in the system.
10175  */
10176 static void nohz_balancer_kick(struct rq *rq)
10177 {
10178 	unsigned long now = jiffies;
10179 	struct sched_domain_shared *sds;
10180 	struct sched_domain *sd;
10181 	int nr_busy, i, cpu = rq->cpu;
10182 	unsigned int flags = 0;
10183 
10184 	if (unlikely(rq->idle_balance))
10185 		return;
10186 
10187 	/*
10188 	 * We may be recently in ticked or tickless idle mode. At the first
10189 	 * busy tick after returning from idle, we will update the busy stats.
10190 	 */
10191 	nohz_balance_exit_idle(rq);
10192 
10193 	/*
10194 	 * None are in tickless mode and hence no need for NOHZ idle load
10195 	 * balancing.
10196 	 */
10197 	if (likely(!atomic_read(&nohz.nr_cpus)))
10198 		return;
10199 
10200 	if (READ_ONCE(nohz.has_blocked) &&
10201 	    time_after(now, READ_ONCE(nohz.next_blocked)))
10202 		flags = NOHZ_STATS_KICK;
10203 
10204 	if (time_before(now, nohz.next_balance))
10205 		goto out;
10206 
10207 	if (rq->nr_running >= 2) {
10208 		flags = NOHZ_KICK_MASK;
10209 		goto out;
10210 	}
10211 
10212 	rcu_read_lock();
10213 
10214 	sd = rcu_dereference(rq->sd);
10215 	if (sd) {
10216 		/*
10217 		 * If there's a CFS task and the current CPU has reduced
10218 		 * capacity; kick the ILB to see if there's a better CPU to run
10219 		 * on.
10220 		 */
10221 		if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) {
10222 			flags = NOHZ_KICK_MASK;
10223 			goto unlock;
10224 		}
10225 	}
10226 
10227 	sd = rcu_dereference(per_cpu(sd_asym_packing, cpu));
10228 	if (sd) {
10229 		/*
10230 		 * When ASYM_PACKING; see if there's a more preferred CPU
10231 		 * currently idle; in which case, kick the ILB to move tasks
10232 		 * around.
10233 		 */
10234 		for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
10235 			if (sched_asym_prefer(i, cpu)) {
10236 				flags = NOHZ_KICK_MASK;
10237 				goto unlock;
10238 			}
10239 		}
10240 	}
10241 
10242 	sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu));
10243 	if (sd) {
10244 		/*
10245 		 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
10246 		 * to run the misfit task on.
10247 		 */
10248 		if (check_misfit_status(rq, sd)) {
10249 			flags = NOHZ_KICK_MASK;
10250 			goto unlock;
10251 		}
10252 
10253 		/*
10254 		 * For asymmetric systems, we do not want to nicely balance
10255 		 * cache use, instead we want to embrace asymmetry and only
10256 		 * ensure tasks have enough CPU capacity.
10257 		 *
10258 		 * Skip the LLC logic because it's not relevant in that case.
10259 		 */
10260 		goto unlock;
10261 	}
10262 
10263 	sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
10264 	if (sds) {
10265 		/*
10266 		 * If there is an imbalance between LLC domains (IOW we could
10267 		 * increase the overall cache use), we need some less-loaded LLC
10268 		 * domain to pull some load. Likewise, we may need to spread
10269 		 * load within the current LLC domain (e.g. packed SMT cores but
10270 		 * other CPUs are idle). We can't really know from here how busy
10271 		 * the others are - so just get a nohz balance going if it looks
10272 		 * like this LLC domain has tasks we could move.
10273 		 */
10274 		nr_busy = atomic_read(&sds->nr_busy_cpus);
10275 		if (nr_busy > 1) {
10276 			flags = NOHZ_KICK_MASK;
10277 			goto unlock;
10278 		}
10279 	}
10280 unlock:
10281 	rcu_read_unlock();
10282 out:
10283 	if (flags)
10284 		kick_ilb(flags);
10285 }
10286 
10287 static void set_cpu_sd_state_busy(int cpu)
10288 {
10289 	struct sched_domain *sd;
10290 
10291 	rcu_read_lock();
10292 	sd = rcu_dereference(per_cpu(sd_llc, cpu));
10293 
10294 	if (!sd || !sd->nohz_idle)
10295 		goto unlock;
10296 	sd->nohz_idle = 0;
10297 
10298 	atomic_inc(&sd->shared->nr_busy_cpus);
10299 unlock:
10300 	rcu_read_unlock();
10301 }
10302 
10303 void nohz_balance_exit_idle(struct rq *rq)
10304 {
10305 	SCHED_WARN_ON(rq != this_rq());
10306 
10307 	if (likely(!rq->nohz_tick_stopped))
10308 		return;
10309 
10310 	rq->nohz_tick_stopped = 0;
10311 	cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask);
10312 	atomic_dec(&nohz.nr_cpus);
10313 
10314 	set_cpu_sd_state_busy(rq->cpu);
10315 }
10316 
10317 static void set_cpu_sd_state_idle(int cpu)
10318 {
10319 	struct sched_domain *sd;
10320 
10321 	rcu_read_lock();
10322 	sd = rcu_dereference(per_cpu(sd_llc, cpu));
10323 
10324 	if (!sd || sd->nohz_idle)
10325 		goto unlock;
10326 	sd->nohz_idle = 1;
10327 
10328 	atomic_dec(&sd->shared->nr_busy_cpus);
10329 unlock:
10330 	rcu_read_unlock();
10331 }
10332 
10333 /*
10334  * This routine will record that the CPU is going idle with tick stopped.
10335  * This info will be used in performing idle load balancing in the future.
10336  */
10337 void nohz_balance_enter_idle(int cpu)
10338 {
10339 	struct rq *rq = cpu_rq(cpu);
10340 
10341 	SCHED_WARN_ON(cpu != smp_processor_id());
10342 
10343 	/* If this CPU is going down, then nothing needs to be done: */
10344 	if (!cpu_active(cpu))
10345 		return;
10346 
10347 	/* Spare idle load balancing on CPUs that don't want to be disturbed: */
10348 	if (!housekeeping_cpu(cpu, HK_FLAG_SCHED))
10349 		return;
10350 
10351 	/*
10352 	 * Can be set safely without rq->lock held
10353 	 * If a clear happens, it will have evaluated last additions because
10354 	 * rq->lock is held during the check and the clear
10355 	 */
10356 	rq->has_blocked_load = 1;
10357 
10358 	/*
10359 	 * The tick is still stopped but load could have been added in the
10360 	 * meantime. We set the nohz.has_blocked flag to trig a check of the
10361 	 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear
10362 	 * of nohz.has_blocked can only happen after checking the new load
10363 	 */
10364 	if (rq->nohz_tick_stopped)
10365 		goto out;
10366 
10367 	/* If we're a completely isolated CPU, we don't play: */
10368 	if (on_null_domain(rq))
10369 		return;
10370 
10371 	rq->nohz_tick_stopped = 1;
10372 
10373 	cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
10374 	atomic_inc(&nohz.nr_cpus);
10375 
10376 	/*
10377 	 * Ensures that if nohz_idle_balance() fails to observe our
10378 	 * @idle_cpus_mask store, it must observe the @has_blocked
10379 	 * store.
10380 	 */
10381 	smp_mb__after_atomic();
10382 
10383 	set_cpu_sd_state_idle(cpu);
10384 
10385 out:
10386 	/*
10387 	 * Each time a cpu enter idle, we assume that it has blocked load and
10388 	 * enable the periodic update of the load of idle cpus
10389 	 */
10390 	WRITE_ONCE(nohz.has_blocked, 1);
10391 }
10392 
10393 static bool update_nohz_stats(struct rq *rq)
10394 {
10395 	unsigned int cpu = rq->cpu;
10396 
10397 	if (!rq->has_blocked_load)
10398 		return false;
10399 
10400 	if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask))
10401 		return false;
10402 
10403 	if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick)))
10404 		return true;
10405 
10406 	update_blocked_averages(cpu);
10407 
10408 	return rq->has_blocked_load;
10409 }
10410 
10411 /*
10412  * Internal function that runs load balance for all idle cpus. The load balance
10413  * can be a simple update of blocked load or a complete load balance with
10414  * tasks movement depending of flags.
10415  */
10416 static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
10417 			       enum cpu_idle_type idle)
10418 {
10419 	/* Earliest time when we have to do rebalance again */
10420 	unsigned long now = jiffies;
10421 	unsigned long next_balance = now + 60*HZ;
10422 	bool has_blocked_load = false;
10423 	int update_next_balance = 0;
10424 	int this_cpu = this_rq->cpu;
10425 	int balance_cpu;
10426 	struct rq *rq;
10427 
10428 	SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);
10429 
10430 	/*
10431 	 * We assume there will be no idle load after this update and clear
10432 	 * the has_blocked flag. If a cpu enters idle in the mean time, it will
10433 	 * set the has_blocked flag and trig another update of idle load.
10434 	 * Because a cpu that becomes idle, is added to idle_cpus_mask before
10435 	 * setting the flag, we are sure to not clear the state and not
10436 	 * check the load of an idle cpu.
10437 	 */
10438 	WRITE_ONCE(nohz.has_blocked, 0);
10439 
10440 	/*
10441 	 * Ensures that if we miss the CPU, we must see the has_blocked
10442 	 * store from nohz_balance_enter_idle().
10443 	 */
10444 	smp_mb();
10445 
10446 	/*
10447 	 * Start with the next CPU after this_cpu so we will end with this_cpu and let a
10448 	 * chance for other idle cpu to pull load.
10449 	 */
10450 	for_each_cpu_wrap(balance_cpu,  nohz.idle_cpus_mask, this_cpu+1) {
10451 		if (!idle_cpu(balance_cpu))
10452 			continue;
10453 
10454 		/*
10455 		 * If this CPU gets work to do, stop the load balancing
10456 		 * work being done for other CPUs. Next load
10457 		 * balancing owner will pick it up.
10458 		 */
10459 		if (need_resched()) {
10460 			has_blocked_load = true;
10461 			goto abort;
10462 		}
10463 
10464 		rq = cpu_rq(balance_cpu);
10465 
10466 		has_blocked_load |= update_nohz_stats(rq);
10467 
10468 		/*
10469 		 * If time for next balance is due,
10470 		 * do the balance.
10471 		 */
10472 		if (time_after_eq(jiffies, rq->next_balance)) {
10473 			struct rq_flags rf;
10474 
10475 			rq_lock_irqsave(rq, &rf);
10476 			update_rq_clock(rq);
10477 			rq_unlock_irqrestore(rq, &rf);
10478 
10479 			if (flags & NOHZ_BALANCE_KICK)
10480 				rebalance_domains(rq, CPU_IDLE);
10481 		}
10482 
10483 		if (time_after(next_balance, rq->next_balance)) {
10484 			next_balance = rq->next_balance;
10485 			update_next_balance = 1;
10486 		}
10487 	}
10488 
10489 	/*
10490 	 * next_balance will be updated only when there is a need.
10491 	 * When the CPU is attached to null domain for ex, it will not be
10492 	 * updated.
10493 	 */
10494 	if (likely(update_next_balance))
10495 		nohz.next_balance = next_balance;
10496 
10497 	WRITE_ONCE(nohz.next_blocked,
10498 		now + msecs_to_jiffies(LOAD_AVG_PERIOD));
10499 
10500 abort:
10501 	/* There is still blocked load, enable periodic update */
10502 	if (has_blocked_load)
10503 		WRITE_ONCE(nohz.has_blocked, 1);
10504 }
10505 
10506 /*
10507  * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
10508  * rebalancing for all the cpus for whom scheduler ticks are stopped.
10509  */
10510 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
10511 {
10512 	unsigned int flags = this_rq->nohz_idle_balance;
10513 
10514 	if (!flags)
10515 		return false;
10516 
10517 	this_rq->nohz_idle_balance = 0;
10518 
10519 	if (idle != CPU_IDLE)
10520 		return false;
10521 
10522 	_nohz_idle_balance(this_rq, flags, idle);
10523 
10524 	return true;
10525 }
10526 
10527 /*
10528  * Check if we need to run the ILB for updating blocked load before entering
10529  * idle state.
10530  */
10531 void nohz_run_idle_balance(int cpu)
10532 {
10533 	unsigned int flags;
10534 
10535 	flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu));
10536 
10537 	/*
10538 	 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen
10539 	 * (ie NOHZ_STATS_KICK set) and will do the same.
10540 	 */
10541 	if ((flags == NOHZ_NEWILB_KICK) && !need_resched())
10542 		_nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK, CPU_IDLE);
10543 }
10544 
10545 static void nohz_newidle_balance(struct rq *this_rq)
10546 {
10547 	int this_cpu = this_rq->cpu;
10548 
10549 	/*
10550 	 * This CPU doesn't want to be disturbed by scheduler
10551 	 * housekeeping
10552 	 */
10553 	if (!housekeeping_cpu(this_cpu, HK_FLAG_SCHED))
10554 		return;
10555 
10556 	/* Will wake up very soon. No time for doing anything else*/
10557 	if (this_rq->avg_idle < sysctl_sched_migration_cost)
10558 		return;
10559 
10560 	/* Don't need to update blocked load of idle CPUs*/
10561 	if (!READ_ONCE(nohz.has_blocked) ||
10562 	    time_before(jiffies, READ_ONCE(nohz.next_blocked)))
10563 		return;
10564 
10565 	/*
10566 	 * Set the need to trigger ILB in order to update blocked load
10567 	 * before entering idle state.
10568 	 */
10569 	atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu));
10570 }
10571 
10572 #else /* !CONFIG_NO_HZ_COMMON */
10573 static inline void nohz_balancer_kick(struct rq *rq) { }
10574 
10575 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
10576 {
10577 	return false;
10578 }
10579 
10580 static inline void nohz_newidle_balance(struct rq *this_rq) { }
10581 #endif /* CONFIG_NO_HZ_COMMON */
10582 
10583 /*
10584  * newidle_balance is called by schedule() if this_cpu is about to become
10585  * idle. Attempts to pull tasks from other CPUs.
10586  *
10587  * Returns:
10588  *   < 0 - we released the lock and there are !fair tasks present
10589  *     0 - failed, no new tasks
10590  *   > 0 - success, new (fair) tasks present
10591  */
10592 static int newidle_balance(struct rq *this_rq, struct rq_flags *rf)
10593 {
10594 	unsigned long next_balance = jiffies + HZ;
10595 	int this_cpu = this_rq->cpu;
10596 	struct sched_domain *sd;
10597 	int pulled_task = 0;
10598 	u64 curr_cost = 0;
10599 
10600 	update_misfit_status(NULL, this_rq);
10601 	/*
10602 	 * We must set idle_stamp _before_ calling idle_balance(), such that we
10603 	 * measure the duration of idle_balance() as idle time.
10604 	 */
10605 	this_rq->idle_stamp = rq_clock(this_rq);
10606 
10607 	/*
10608 	 * Do not pull tasks towards !active CPUs...
10609 	 */
10610 	if (!cpu_active(this_cpu))
10611 		return 0;
10612 
10613 	/*
10614 	 * This is OK, because current is on_cpu, which avoids it being picked
10615 	 * for load-balance and preemption/IRQs are still disabled avoiding
10616 	 * further scheduler activity on it and we're being very careful to
10617 	 * re-start the picking loop.
10618 	 */
10619 	rq_unpin_lock(this_rq, rf);
10620 
10621 	if (this_rq->avg_idle < sysctl_sched_migration_cost ||
10622 	    !READ_ONCE(this_rq->rd->overload)) {
10623 
10624 		rcu_read_lock();
10625 		sd = rcu_dereference_check_sched_domain(this_rq->sd);
10626 		if (sd)
10627 			update_next_balance(sd, &next_balance);
10628 		rcu_read_unlock();
10629 
10630 		goto out;
10631 	}
10632 
10633 	raw_spin_unlock(&this_rq->lock);
10634 
10635 	update_blocked_averages(this_cpu);
10636 	rcu_read_lock();
10637 	for_each_domain(this_cpu, sd) {
10638 		int continue_balancing = 1;
10639 		u64 t0, domain_cost;
10640 
10641 		if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) {
10642 			update_next_balance(sd, &next_balance);
10643 			break;
10644 		}
10645 
10646 		if (sd->flags & SD_BALANCE_NEWIDLE) {
10647 			t0 = sched_clock_cpu(this_cpu);
10648 
10649 			pulled_task = load_balance(this_cpu, this_rq,
10650 						   sd, CPU_NEWLY_IDLE,
10651 						   &continue_balancing);
10652 
10653 			domain_cost = sched_clock_cpu(this_cpu) - t0;
10654 			if (domain_cost > sd->max_newidle_lb_cost)
10655 				sd->max_newidle_lb_cost = domain_cost;
10656 
10657 			curr_cost += domain_cost;
10658 		}
10659 
10660 		update_next_balance(sd, &next_balance);
10661 
10662 		/*
10663 		 * Stop searching for tasks to pull if there are
10664 		 * now runnable tasks on this rq.
10665 		 */
10666 		if (pulled_task || this_rq->nr_running > 0)
10667 			break;
10668 	}
10669 	rcu_read_unlock();
10670 
10671 	raw_spin_lock(&this_rq->lock);
10672 
10673 	if (curr_cost > this_rq->max_idle_balance_cost)
10674 		this_rq->max_idle_balance_cost = curr_cost;
10675 
10676 	/*
10677 	 * While browsing the domains, we released the rq lock, a task could
10678 	 * have been enqueued in the meantime. Since we're not going idle,
10679 	 * pretend we pulled a task.
10680 	 */
10681 	if (this_rq->cfs.h_nr_running && !pulled_task)
10682 		pulled_task = 1;
10683 
10684 	/* Is there a task of a high priority class? */
10685 	if (this_rq->nr_running != this_rq->cfs.h_nr_running)
10686 		pulled_task = -1;
10687 
10688 out:
10689 	/* Move the next balance forward */
10690 	if (time_after(this_rq->next_balance, next_balance))
10691 		this_rq->next_balance = next_balance;
10692 
10693 	if (pulled_task)
10694 		this_rq->idle_stamp = 0;
10695 	else
10696 		nohz_newidle_balance(this_rq);
10697 
10698 	rq_repin_lock(this_rq, rf);
10699 
10700 	return pulled_task;
10701 }
10702 
10703 /*
10704  * run_rebalance_domains is triggered when needed from the scheduler tick.
10705  * Also triggered for nohz idle balancing (with nohz_balancing_kick set).
10706  */
10707 static __latent_entropy void run_rebalance_domains(struct softirq_action *h)
10708 {
10709 	struct rq *this_rq = this_rq();
10710 	enum cpu_idle_type idle = this_rq->idle_balance ?
10711 						CPU_IDLE : CPU_NOT_IDLE;
10712 
10713 	/*
10714 	 * If this CPU has a pending nohz_balance_kick, then do the
10715 	 * balancing on behalf of the other idle CPUs whose ticks are
10716 	 * stopped. Do nohz_idle_balance *before* rebalance_domains to
10717 	 * give the idle CPUs a chance to load balance. Else we may
10718 	 * load balance only within the local sched_domain hierarchy
10719 	 * and abort nohz_idle_balance altogether if we pull some load.
10720 	 */
10721 	if (nohz_idle_balance(this_rq, idle))
10722 		return;
10723 
10724 	/* normal load balance */
10725 	update_blocked_averages(this_rq->cpu);
10726 	rebalance_domains(this_rq, idle);
10727 }
10728 
10729 /*
10730  * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
10731  */
10732 void trigger_load_balance(struct rq *rq)
10733 {
10734 	/*
10735 	 * Don't need to rebalance while attached to NULL domain or
10736 	 * runqueue CPU is not active
10737 	 */
10738 	if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq))))
10739 		return;
10740 
10741 	if (time_after_eq(jiffies, rq->next_balance))
10742 		raise_softirq(SCHED_SOFTIRQ);
10743 
10744 	nohz_balancer_kick(rq);
10745 }
10746 
10747 static void rq_online_fair(struct rq *rq)
10748 {
10749 	update_sysctl();
10750 
10751 	update_runtime_enabled(rq);
10752 }
10753 
10754 static void rq_offline_fair(struct rq *rq)
10755 {
10756 	update_sysctl();
10757 
10758 	/* Ensure any throttled groups are reachable by pick_next_task */
10759 	unthrottle_offline_cfs_rqs(rq);
10760 }
10761 
10762 #endif /* CONFIG_SMP */
10763 
10764 /*
10765  * scheduler tick hitting a task of our scheduling class.
10766  *
10767  * NOTE: This function can be called remotely by the tick offload that
10768  * goes along full dynticks. Therefore no local assumption can be made
10769  * and everything must be accessed through the @rq and @curr passed in
10770  * parameters.
10771  */
10772 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
10773 {
10774 	struct cfs_rq *cfs_rq;
10775 	struct sched_entity *se = &curr->se;
10776 
10777 	for_each_sched_entity(se) {
10778 		cfs_rq = cfs_rq_of(se);
10779 		entity_tick(cfs_rq, se, queued);
10780 	}
10781 
10782 	if (static_branch_unlikely(&sched_numa_balancing))
10783 		task_tick_numa(rq, curr);
10784 
10785 	update_misfit_status(curr, rq);
10786 	update_overutilized_status(task_rq(curr));
10787 }
10788 
10789 /*
10790  * called on fork with the child task as argument from the parent's context
10791  *  - child not yet on the tasklist
10792  *  - preemption disabled
10793  */
10794 static void task_fork_fair(struct task_struct *p)
10795 {
10796 	struct cfs_rq *cfs_rq;
10797 	struct sched_entity *se = &p->se, *curr;
10798 	struct rq *rq = this_rq();
10799 	struct rq_flags rf;
10800 
10801 	rq_lock(rq, &rf);
10802 	update_rq_clock(rq);
10803 
10804 	cfs_rq = task_cfs_rq(current);
10805 	curr = cfs_rq->curr;
10806 	if (curr) {
10807 		update_curr(cfs_rq);
10808 		se->vruntime = curr->vruntime;
10809 	}
10810 	place_entity(cfs_rq, se, 1);
10811 
10812 	if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) {
10813 		/*
10814 		 * Upon rescheduling, sched_class::put_prev_task() will place
10815 		 * 'current' within the tree based on its new key value.
10816 		 */
10817 		swap(curr->vruntime, se->vruntime);
10818 		resched_curr(rq);
10819 	}
10820 
10821 	se->vruntime -= cfs_rq->min_vruntime;
10822 	rq_unlock(rq, &rf);
10823 }
10824 
10825 /*
10826  * Priority of the task has changed. Check to see if we preempt
10827  * the current task.
10828  */
10829 static void
10830 prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio)
10831 {
10832 	if (!task_on_rq_queued(p))
10833 		return;
10834 
10835 	if (rq->cfs.nr_running == 1)
10836 		return;
10837 
10838 	/*
10839 	 * Reschedule if we are currently running on this runqueue and
10840 	 * our priority decreased, or if we are not currently running on
10841 	 * this runqueue and our priority is higher than the current's
10842 	 */
10843 	if (task_current(rq, p)) {
10844 		if (p->prio > oldprio)
10845 			resched_curr(rq);
10846 	} else
10847 		check_preempt_curr(rq, p, 0);
10848 }
10849 
10850 static inline bool vruntime_normalized(struct task_struct *p)
10851 {
10852 	struct sched_entity *se = &p->se;
10853 
10854 	/*
10855 	 * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases,
10856 	 * the dequeue_entity(.flags=0) will already have normalized the
10857 	 * vruntime.
10858 	 */
10859 	if (p->on_rq)
10860 		return true;
10861 
10862 	/*
10863 	 * When !on_rq, vruntime of the task has usually NOT been normalized.
10864 	 * But there are some cases where it has already been normalized:
10865 	 *
10866 	 * - A forked child which is waiting for being woken up by
10867 	 *   wake_up_new_task().
10868 	 * - A task which has been woken up by try_to_wake_up() and
10869 	 *   waiting for actually being woken up by sched_ttwu_pending().
10870 	 */
10871 	if (!se->sum_exec_runtime ||
10872 	    (p->state == TASK_WAKING && p->sched_remote_wakeup))
10873 		return true;
10874 
10875 	return false;
10876 }
10877 
10878 #ifdef CONFIG_FAIR_GROUP_SCHED
10879 /*
10880  * Propagate the changes of the sched_entity across the tg tree to make it
10881  * visible to the root
10882  */
10883 static void propagate_entity_cfs_rq(struct sched_entity *se)
10884 {
10885 	struct cfs_rq *cfs_rq;
10886 
10887 	list_add_leaf_cfs_rq(cfs_rq_of(se));
10888 
10889 	/* Start to propagate at parent */
10890 	se = se->parent;
10891 
10892 	for_each_sched_entity(se) {
10893 		cfs_rq = cfs_rq_of(se);
10894 
10895 		if (!cfs_rq_throttled(cfs_rq)){
10896 			update_load_avg(cfs_rq, se, UPDATE_TG);
10897 			list_add_leaf_cfs_rq(cfs_rq);
10898 			continue;
10899 		}
10900 
10901 		if (list_add_leaf_cfs_rq(cfs_rq))
10902 			break;
10903 	}
10904 }
10905 #else
10906 static void propagate_entity_cfs_rq(struct sched_entity *se) { }
10907 #endif
10908 
10909 static void detach_entity_cfs_rq(struct sched_entity *se)
10910 {
10911 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
10912 
10913 	/* Catch up with the cfs_rq and remove our load when we leave */
10914 	update_load_avg(cfs_rq, se, 0);
10915 	detach_entity_load_avg(cfs_rq, se);
10916 	update_tg_load_avg(cfs_rq);
10917 	propagate_entity_cfs_rq(se);
10918 }
10919 
10920 static void attach_entity_cfs_rq(struct sched_entity *se)
10921 {
10922 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
10923 
10924 #ifdef CONFIG_FAIR_GROUP_SCHED
10925 	/*
10926 	 * Since the real-depth could have been changed (only FAIR
10927 	 * class maintain depth value), reset depth properly.
10928 	 */
10929 	se->depth = se->parent ? se->parent->depth + 1 : 0;
10930 #endif
10931 
10932 	/* Synchronize entity with its cfs_rq */
10933 	update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD);
10934 	attach_entity_load_avg(cfs_rq, se);
10935 	update_tg_load_avg(cfs_rq);
10936 	propagate_entity_cfs_rq(se);
10937 }
10938 
10939 static void detach_task_cfs_rq(struct task_struct *p)
10940 {
10941 	struct sched_entity *se = &p->se;
10942 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
10943 
10944 	if (!vruntime_normalized(p)) {
10945 		/*
10946 		 * Fix up our vruntime so that the current sleep doesn't
10947 		 * cause 'unlimited' sleep bonus.
10948 		 */
10949 		place_entity(cfs_rq, se, 0);
10950 		se->vruntime -= cfs_rq->min_vruntime;
10951 	}
10952 
10953 	detach_entity_cfs_rq(se);
10954 }
10955 
10956 static void attach_task_cfs_rq(struct task_struct *p)
10957 {
10958 	struct sched_entity *se = &p->se;
10959 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
10960 
10961 	attach_entity_cfs_rq(se);
10962 
10963 	if (!vruntime_normalized(p))
10964 		se->vruntime += cfs_rq->min_vruntime;
10965 }
10966 
10967 static void switched_from_fair(struct rq *rq, struct task_struct *p)
10968 {
10969 	detach_task_cfs_rq(p);
10970 }
10971 
10972 static void switched_to_fair(struct rq *rq, struct task_struct *p)
10973 {
10974 	attach_task_cfs_rq(p);
10975 
10976 	if (task_on_rq_queued(p)) {
10977 		/*
10978 		 * We were most likely switched from sched_rt, so
10979 		 * kick off the schedule if running, otherwise just see
10980 		 * if we can still preempt the current task.
10981 		 */
10982 		if (task_current(rq, p))
10983 			resched_curr(rq);
10984 		else
10985 			check_preempt_curr(rq, p, 0);
10986 	}
10987 }
10988 
10989 /* Account for a task changing its policy or group.
10990  *
10991  * This routine is mostly called to set cfs_rq->curr field when a task
10992  * migrates between groups/classes.
10993  */
10994 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
10995 {
10996 	struct sched_entity *se = &p->se;
10997 
10998 #ifdef CONFIG_SMP
10999 	if (task_on_rq_queued(p)) {
11000 		/*
11001 		 * Move the next running task to the front of the list, so our
11002 		 * cfs_tasks list becomes MRU one.
11003 		 */
11004 		list_move(&se->group_node, &rq->cfs_tasks);
11005 	}
11006 #endif
11007 
11008 	for_each_sched_entity(se) {
11009 		struct cfs_rq *cfs_rq = cfs_rq_of(se);
11010 
11011 		set_next_entity(cfs_rq, se);
11012 		/* ensure bandwidth has been allocated on our new cfs_rq */
11013 		account_cfs_rq_runtime(cfs_rq, 0);
11014 	}
11015 }
11016 
11017 void init_cfs_rq(struct cfs_rq *cfs_rq)
11018 {
11019 	cfs_rq->tasks_timeline = RB_ROOT_CACHED;
11020 	cfs_rq->min_vruntime = (u64)(-(1LL << 20));
11021 #ifndef CONFIG_64BIT
11022 	cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
11023 #endif
11024 #ifdef CONFIG_SMP
11025 	raw_spin_lock_init(&cfs_rq->removed.lock);
11026 #endif
11027 }
11028 
11029 #ifdef CONFIG_FAIR_GROUP_SCHED
11030 static void task_set_group_fair(struct task_struct *p)
11031 {
11032 	struct sched_entity *se = &p->se;
11033 
11034 	set_task_rq(p, task_cpu(p));
11035 	se->depth = se->parent ? se->parent->depth + 1 : 0;
11036 }
11037 
11038 static void task_move_group_fair(struct task_struct *p)
11039 {
11040 	detach_task_cfs_rq(p);
11041 	set_task_rq(p, task_cpu(p));
11042 
11043 #ifdef CONFIG_SMP
11044 	/* Tell se's cfs_rq has been changed -- migrated */
11045 	p->se.avg.last_update_time = 0;
11046 #endif
11047 	attach_task_cfs_rq(p);
11048 }
11049 
11050 static void task_change_group_fair(struct task_struct *p, int type)
11051 {
11052 	switch (type) {
11053 	case TASK_SET_GROUP:
11054 		task_set_group_fair(p);
11055 		break;
11056 
11057 	case TASK_MOVE_GROUP:
11058 		task_move_group_fair(p);
11059 		break;
11060 	}
11061 }
11062 
11063 void free_fair_sched_group(struct task_group *tg)
11064 {
11065 	int i;
11066 
11067 	destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
11068 
11069 	for_each_possible_cpu(i) {
11070 		if (tg->cfs_rq)
11071 			kfree(tg->cfs_rq[i]);
11072 		if (tg->se)
11073 			kfree(tg->se[i]);
11074 	}
11075 
11076 	kfree(tg->cfs_rq);
11077 	kfree(tg->se);
11078 }
11079 
11080 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
11081 {
11082 	struct sched_entity *se;
11083 	struct cfs_rq *cfs_rq;
11084 	int i;
11085 
11086 	tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);
11087 	if (!tg->cfs_rq)
11088 		goto err;
11089 	tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL);
11090 	if (!tg->se)
11091 		goto err;
11092 
11093 	tg->shares = NICE_0_LOAD;
11094 
11095 	init_cfs_bandwidth(tg_cfs_bandwidth(tg));
11096 
11097 	for_each_possible_cpu(i) {
11098 		cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
11099 				      GFP_KERNEL, cpu_to_node(i));
11100 		if (!cfs_rq)
11101 			goto err;
11102 
11103 		se = kzalloc_node(sizeof(struct sched_entity),
11104 				  GFP_KERNEL, cpu_to_node(i));
11105 		if (!se)
11106 			goto err_free_rq;
11107 
11108 		init_cfs_rq(cfs_rq);
11109 		init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
11110 		init_entity_runnable_average(se);
11111 	}
11112 
11113 	return 1;
11114 
11115 err_free_rq:
11116 	kfree(cfs_rq);
11117 err:
11118 	return 0;
11119 }
11120 
11121 void online_fair_sched_group(struct task_group *tg)
11122 {
11123 	struct sched_entity *se;
11124 	struct rq_flags rf;
11125 	struct rq *rq;
11126 	int i;
11127 
11128 	for_each_possible_cpu(i) {
11129 		rq = cpu_rq(i);
11130 		se = tg->se[i];
11131 		rq_lock_irq(rq, &rf);
11132 		update_rq_clock(rq);
11133 		attach_entity_cfs_rq(se);
11134 		sync_throttle(tg, i);
11135 		rq_unlock_irq(rq, &rf);
11136 	}
11137 }
11138 
11139 void unregister_fair_sched_group(struct task_group *tg)
11140 {
11141 	unsigned long flags;
11142 	struct rq *rq;
11143 	int cpu;
11144 
11145 	for_each_possible_cpu(cpu) {
11146 		if (tg->se[cpu])
11147 			remove_entity_load_avg(tg->se[cpu]);
11148 
11149 		/*
11150 		 * Only empty task groups can be destroyed; so we can speculatively
11151 		 * check on_list without danger of it being re-added.
11152 		 */
11153 		if (!tg->cfs_rq[cpu]->on_list)
11154 			continue;
11155 
11156 		rq = cpu_rq(cpu);
11157 
11158 		raw_spin_lock_irqsave(&rq->lock, flags);
11159 		list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
11160 		raw_spin_unlock_irqrestore(&rq->lock, flags);
11161 	}
11162 }
11163 
11164 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
11165 			struct sched_entity *se, int cpu,
11166 			struct sched_entity *parent)
11167 {
11168 	struct rq *rq = cpu_rq(cpu);
11169 
11170 	cfs_rq->tg = tg;
11171 	cfs_rq->rq = rq;
11172 	init_cfs_rq_runtime(cfs_rq);
11173 
11174 	tg->cfs_rq[cpu] = cfs_rq;
11175 	tg->se[cpu] = se;
11176 
11177 	/* se could be NULL for root_task_group */
11178 	if (!se)
11179 		return;
11180 
11181 	if (!parent) {
11182 		se->cfs_rq = &rq->cfs;
11183 		se->depth = 0;
11184 	} else {
11185 		se->cfs_rq = parent->my_q;
11186 		se->depth = parent->depth + 1;
11187 	}
11188 
11189 	se->my_q = cfs_rq;
11190 	/* guarantee group entities always have weight */
11191 	update_load_set(&se->load, NICE_0_LOAD);
11192 	se->parent = parent;
11193 }
11194 
11195 static DEFINE_MUTEX(shares_mutex);
11196 
11197 int sched_group_set_shares(struct task_group *tg, unsigned long shares)
11198 {
11199 	int i;
11200 
11201 	/*
11202 	 * We can't change the weight of the root cgroup.
11203 	 */
11204 	if (!tg->se[0])
11205 		return -EINVAL;
11206 
11207 	shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
11208 
11209 	mutex_lock(&shares_mutex);
11210 	if (tg->shares == shares)
11211 		goto done;
11212 
11213 	tg->shares = shares;
11214 	for_each_possible_cpu(i) {
11215 		struct rq *rq = cpu_rq(i);
11216 		struct sched_entity *se = tg->se[i];
11217 		struct rq_flags rf;
11218 
11219 		/* Propagate contribution to hierarchy */
11220 		rq_lock_irqsave(rq, &rf);
11221 		update_rq_clock(rq);
11222 		for_each_sched_entity(se) {
11223 			update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
11224 			update_cfs_group(se);
11225 		}
11226 		rq_unlock_irqrestore(rq, &rf);
11227 	}
11228 
11229 done:
11230 	mutex_unlock(&shares_mutex);
11231 	return 0;
11232 }
11233 #else /* CONFIG_FAIR_GROUP_SCHED */
11234 
11235 void free_fair_sched_group(struct task_group *tg) { }
11236 
11237 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
11238 {
11239 	return 1;
11240 }
11241 
11242 void online_fair_sched_group(struct task_group *tg) { }
11243 
11244 void unregister_fair_sched_group(struct task_group *tg) { }
11245 
11246 #endif /* CONFIG_FAIR_GROUP_SCHED */
11247 
11248 
11249 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
11250 {
11251 	struct sched_entity *se = &task->se;
11252 	unsigned int rr_interval = 0;
11253 
11254 	/*
11255 	 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise
11256 	 * idle runqueue:
11257 	 */
11258 	if (rq->cfs.load.weight)
11259 		rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se));
11260 
11261 	return rr_interval;
11262 }
11263 
11264 /*
11265  * All the scheduling class methods:
11266  */
11267 DEFINE_SCHED_CLASS(fair) = {
11268 
11269 	.enqueue_task		= enqueue_task_fair,
11270 	.dequeue_task		= dequeue_task_fair,
11271 	.yield_task		= yield_task_fair,
11272 	.yield_to_task		= yield_to_task_fair,
11273 
11274 	.check_preempt_curr	= check_preempt_wakeup,
11275 
11276 	.pick_next_task		= __pick_next_task_fair,
11277 	.put_prev_task		= put_prev_task_fair,
11278 	.set_next_task          = set_next_task_fair,
11279 
11280 #ifdef CONFIG_SMP
11281 	.balance		= balance_fair,
11282 	.select_task_rq		= select_task_rq_fair,
11283 	.migrate_task_rq	= migrate_task_rq_fair,
11284 
11285 	.rq_online		= rq_online_fair,
11286 	.rq_offline		= rq_offline_fair,
11287 
11288 	.task_dead		= task_dead_fair,
11289 	.set_cpus_allowed	= set_cpus_allowed_common,
11290 #endif
11291 
11292 	.task_tick		= task_tick_fair,
11293 	.task_fork		= task_fork_fair,
11294 
11295 	.prio_changed		= prio_changed_fair,
11296 	.switched_from		= switched_from_fair,
11297 	.switched_to		= switched_to_fair,
11298 
11299 	.get_rr_interval	= get_rr_interval_fair,
11300 
11301 	.update_curr		= update_curr_fair,
11302 
11303 #ifdef CONFIG_FAIR_GROUP_SCHED
11304 	.task_change_group	= task_change_group_fair,
11305 #endif
11306 
11307 #ifdef CONFIG_UCLAMP_TASK
11308 	.uclamp_enabled		= 1,
11309 #endif
11310 };
11311 
11312 #ifdef CONFIG_SCHED_DEBUG
11313 void print_cfs_stats(struct seq_file *m, int cpu)
11314 {
11315 	struct cfs_rq *cfs_rq, *pos;
11316 
11317 	rcu_read_lock();
11318 	for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos)
11319 		print_cfs_rq(m, cpu, cfs_rq);
11320 	rcu_read_unlock();
11321 }
11322 
11323 #ifdef CONFIG_NUMA_BALANCING
11324 void show_numa_stats(struct task_struct *p, struct seq_file *m)
11325 {
11326 	int node;
11327 	unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0;
11328 	struct numa_group *ng;
11329 
11330 	rcu_read_lock();
11331 	ng = rcu_dereference(p->numa_group);
11332 	for_each_online_node(node) {
11333 		if (p->numa_faults) {
11334 			tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)];
11335 			tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)];
11336 		}
11337 		if (ng) {
11338 			gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)],
11339 			gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
11340 		}
11341 		print_numa_stats(m, node, tsf, tpf, gsf, gpf);
11342 	}
11343 	rcu_read_unlock();
11344 }
11345 #endif /* CONFIG_NUMA_BALANCING */
11346 #endif /* CONFIG_SCHED_DEBUG */
11347 
11348 __init void init_sched_fair_class(void)
11349 {
11350 #ifdef CONFIG_SMP
11351 	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
11352 
11353 #ifdef CONFIG_NO_HZ_COMMON
11354 	nohz.next_balance = jiffies;
11355 	nohz.next_blocked = jiffies;
11356 	zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
11357 #endif
11358 #endif /* SMP */
11359 
11360 }
11361 
11362 /*
11363  * Helper functions to facilitate extracting info from tracepoints.
11364  */
11365 
11366 const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq)
11367 {
11368 #ifdef CONFIG_SMP
11369 	return cfs_rq ? &cfs_rq->avg : NULL;
11370 #else
11371 	return NULL;
11372 #endif
11373 }
11374 EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_avg);
11375 
11376 char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len)
11377 {
11378 	if (!cfs_rq) {
11379 		if (str)
11380 			strlcpy(str, "(null)", len);
11381 		else
11382 			return NULL;
11383 	}
11384 
11385 	cfs_rq_tg_path(cfs_rq, str, len);
11386 	return str;
11387 }
11388 EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_path);
11389 
11390 int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq)
11391 {
11392 	return cfs_rq ? cpu_of(rq_of(cfs_rq)) : -1;
11393 }
11394 EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_cpu);
11395 
11396 const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq)
11397 {
11398 #ifdef CONFIG_SMP
11399 	return rq ? &rq->avg_rt : NULL;
11400 #else
11401 	return NULL;
11402 #endif
11403 }
11404 EXPORT_SYMBOL_GPL(sched_trace_rq_avg_rt);
11405 
11406 const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq)
11407 {
11408 #ifdef CONFIG_SMP
11409 	return rq ? &rq->avg_dl : NULL;
11410 #else
11411 	return NULL;
11412 #endif
11413 }
11414 EXPORT_SYMBOL_GPL(sched_trace_rq_avg_dl);
11415 
11416 const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq)
11417 {
11418 #if defined(CONFIG_SMP) && defined(CONFIG_HAVE_SCHED_AVG_IRQ)
11419 	return rq ? &rq->avg_irq : NULL;
11420 #else
11421 	return NULL;
11422 #endif
11423 }
11424 EXPORT_SYMBOL_GPL(sched_trace_rq_avg_irq);
11425 
11426 int sched_trace_rq_cpu(struct rq *rq)
11427 {
11428 	return rq ? cpu_of(rq) : -1;
11429 }
11430 EXPORT_SYMBOL_GPL(sched_trace_rq_cpu);
11431 
11432 int sched_trace_rq_cpu_capacity(struct rq *rq)
11433 {
11434 	return rq ?
11435 #ifdef CONFIG_SMP
11436 		rq->cpu_capacity
11437 #else
11438 		SCHED_CAPACITY_SCALE
11439 #endif
11440 		: -1;
11441 }
11442 EXPORT_SYMBOL_GPL(sched_trace_rq_cpu_capacity);
11443 
11444 const struct cpumask *sched_trace_rd_span(struct root_domain *rd)
11445 {
11446 #ifdef CONFIG_SMP
11447 	return rd ? rd->span : NULL;
11448 #else
11449 	return NULL;
11450 #endif
11451 }
11452 EXPORT_SYMBOL_GPL(sched_trace_rd_span);
11453 
11454 int sched_trace_rq_nr_running(struct rq *rq)
11455 {
11456         return rq ? rq->nr_running : -1;
11457 }
11458 EXPORT_SYMBOL_GPL(sched_trace_rq_nr_running);
11459