xref: /openbmc/linux/kernel/sched/topology.c (revision a8c5cb99)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Scheduler topology setup/handling methods
4  */
5 #include "sched.h"
6 
7 DEFINE_MUTEX(sched_domains_mutex);
8 
9 /* Protected by sched_domains_mutex: */
10 cpumask_var_t sched_domains_tmpmask;
11 cpumask_var_t sched_domains_tmpmask2;
12 
13 #ifdef CONFIG_SCHED_DEBUG
14 
15 static int __init sched_debug_setup(char *str)
16 {
17 	sched_debug_enabled = true;
18 
19 	return 0;
20 }
21 early_param("sched_debug", sched_debug_setup);
22 
23 static inline bool sched_debug(void)
24 {
25 	return sched_debug_enabled;
26 }
27 
28 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
29 				  struct cpumask *groupmask)
30 {
31 	struct sched_group *group = sd->groups;
32 
33 	cpumask_clear(groupmask);
34 
35 	printk(KERN_DEBUG "%*s domain-%d: ", level, "", level);
36 
37 	if (!(sd->flags & SD_LOAD_BALANCE)) {
38 		printk("does not load-balance\n");
39 		if (sd->parent)
40 			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain has parent");
41 		return -1;
42 	}
43 
44 	printk(KERN_CONT "span=%*pbl level=%s\n",
45 	       cpumask_pr_args(sched_domain_span(sd)), sd->name);
46 
47 	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
48 		printk(KERN_ERR "ERROR: domain->span does not contain CPU%d\n", cpu);
49 	}
50 	if (!cpumask_test_cpu(cpu, sched_group_span(group))) {
51 		printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n", cpu);
52 	}
53 
54 	printk(KERN_DEBUG "%*s groups:", level + 1, "");
55 	do {
56 		if (!group) {
57 			printk("\n");
58 			printk(KERN_ERR "ERROR: group is NULL\n");
59 			break;
60 		}
61 
62 		if (!cpumask_weight(sched_group_span(group))) {
63 			printk(KERN_CONT "\n");
64 			printk(KERN_ERR "ERROR: empty group\n");
65 			break;
66 		}
67 
68 		if (!(sd->flags & SD_OVERLAP) &&
69 		    cpumask_intersects(groupmask, sched_group_span(group))) {
70 			printk(KERN_CONT "\n");
71 			printk(KERN_ERR "ERROR: repeated CPUs\n");
72 			break;
73 		}
74 
75 		cpumask_or(groupmask, groupmask, sched_group_span(group));
76 
77 		printk(KERN_CONT " %d:{ span=%*pbl",
78 				group->sgc->id,
79 				cpumask_pr_args(sched_group_span(group)));
80 
81 		if ((sd->flags & SD_OVERLAP) &&
82 		    !cpumask_equal(group_balance_mask(group), sched_group_span(group))) {
83 			printk(KERN_CONT " mask=%*pbl",
84 				cpumask_pr_args(group_balance_mask(group)));
85 		}
86 
87 		if (group->sgc->capacity != SCHED_CAPACITY_SCALE)
88 			printk(KERN_CONT " cap=%lu", group->sgc->capacity);
89 
90 		if (group == sd->groups && sd->child &&
91 		    !cpumask_equal(sched_domain_span(sd->child),
92 				   sched_group_span(group))) {
93 			printk(KERN_ERR "ERROR: domain->groups does not match domain->child\n");
94 		}
95 
96 		printk(KERN_CONT " }");
97 
98 		group = group->next;
99 
100 		if (group != sd->groups)
101 			printk(KERN_CONT ",");
102 
103 	} while (group != sd->groups);
104 	printk(KERN_CONT "\n");
105 
106 	if (!cpumask_equal(sched_domain_span(sd), groupmask))
107 		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
108 
109 	if (sd->parent &&
110 	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
111 		printk(KERN_ERR "ERROR: parent span is not a superset of domain->span\n");
112 	return 0;
113 }
114 
115 static void sched_domain_debug(struct sched_domain *sd, int cpu)
116 {
117 	int level = 0;
118 
119 	if (!sched_debug_enabled)
120 		return;
121 
122 	if (!sd) {
123 		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
124 		return;
125 	}
126 
127 	printk(KERN_DEBUG "CPU%d attaching sched-domain(s):\n", cpu);
128 
129 	for (;;) {
130 		if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
131 			break;
132 		level++;
133 		sd = sd->parent;
134 		if (!sd)
135 			break;
136 	}
137 }
138 #else /* !CONFIG_SCHED_DEBUG */
139 
140 # define sched_debug_enabled 0
141 # define sched_domain_debug(sd, cpu) do { } while (0)
142 static inline bool sched_debug(void)
143 {
144 	return false;
145 }
146 #endif /* CONFIG_SCHED_DEBUG */
147 
148 static int sd_degenerate(struct sched_domain *sd)
149 {
150 	if (cpumask_weight(sched_domain_span(sd)) == 1)
151 		return 1;
152 
153 	/* Following flags need at least 2 groups */
154 	if (sd->flags & (SD_LOAD_BALANCE |
155 			 SD_BALANCE_NEWIDLE |
156 			 SD_BALANCE_FORK |
157 			 SD_BALANCE_EXEC |
158 			 SD_SHARE_CPUCAPACITY |
159 			 SD_ASYM_CPUCAPACITY |
160 			 SD_SHARE_PKG_RESOURCES |
161 			 SD_SHARE_POWERDOMAIN)) {
162 		if (sd->groups != sd->groups->next)
163 			return 0;
164 	}
165 
166 	/* Following flags don't use groups */
167 	if (sd->flags & (SD_WAKE_AFFINE))
168 		return 0;
169 
170 	return 1;
171 }
172 
173 static int
174 sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
175 {
176 	unsigned long cflags = sd->flags, pflags = parent->flags;
177 
178 	if (sd_degenerate(parent))
179 		return 1;
180 
181 	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
182 		return 0;
183 
184 	/* Flags needing groups don't count if only 1 group in parent */
185 	if (parent->groups == parent->groups->next) {
186 		pflags &= ~(SD_LOAD_BALANCE |
187 				SD_BALANCE_NEWIDLE |
188 				SD_BALANCE_FORK |
189 				SD_BALANCE_EXEC |
190 				SD_ASYM_CPUCAPACITY |
191 				SD_SHARE_CPUCAPACITY |
192 				SD_SHARE_PKG_RESOURCES |
193 				SD_PREFER_SIBLING |
194 				SD_SHARE_POWERDOMAIN);
195 		if (nr_node_ids == 1)
196 			pflags &= ~SD_SERIALIZE;
197 	}
198 	if (~cflags & pflags)
199 		return 0;
200 
201 	return 1;
202 }
203 
204 static void free_rootdomain(struct rcu_head *rcu)
205 {
206 	struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
207 
208 	cpupri_cleanup(&rd->cpupri);
209 	cpudl_cleanup(&rd->cpudl);
210 	free_cpumask_var(rd->dlo_mask);
211 	free_cpumask_var(rd->rto_mask);
212 	free_cpumask_var(rd->online);
213 	free_cpumask_var(rd->span);
214 	kfree(rd);
215 }
216 
217 void rq_attach_root(struct rq *rq, struct root_domain *rd)
218 {
219 	struct root_domain *old_rd = NULL;
220 	unsigned long flags;
221 
222 	raw_spin_lock_irqsave(&rq->lock, flags);
223 
224 	if (rq->rd) {
225 		old_rd = rq->rd;
226 
227 		if (cpumask_test_cpu(rq->cpu, old_rd->online))
228 			set_rq_offline(rq);
229 
230 		cpumask_clear_cpu(rq->cpu, old_rd->span);
231 
232 		/*
233 		 * If we dont want to free the old_rd yet then
234 		 * set old_rd to NULL to skip the freeing later
235 		 * in this function:
236 		 */
237 		if (!atomic_dec_and_test(&old_rd->refcount))
238 			old_rd = NULL;
239 	}
240 
241 	atomic_inc(&rd->refcount);
242 	rq->rd = rd;
243 
244 	cpumask_set_cpu(rq->cpu, rd->span);
245 	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
246 		set_rq_online(rq);
247 
248 	raw_spin_unlock_irqrestore(&rq->lock, flags);
249 
250 	if (old_rd)
251 		call_rcu_sched(&old_rd->rcu, free_rootdomain);
252 }
253 
254 void sched_get_rd(struct root_domain *rd)
255 {
256 	atomic_inc(&rd->refcount);
257 }
258 
259 void sched_put_rd(struct root_domain *rd)
260 {
261 	if (!atomic_dec_and_test(&rd->refcount))
262 		return;
263 
264 	call_rcu_sched(&rd->rcu, free_rootdomain);
265 }
266 
267 static int init_rootdomain(struct root_domain *rd)
268 {
269 	if (!zalloc_cpumask_var(&rd->span, GFP_KERNEL))
270 		goto out;
271 	if (!zalloc_cpumask_var(&rd->online, GFP_KERNEL))
272 		goto free_span;
273 	if (!zalloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
274 		goto free_online;
275 	if (!zalloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
276 		goto free_dlo_mask;
277 
278 #ifdef HAVE_RT_PUSH_IPI
279 	rd->rto_cpu = -1;
280 	raw_spin_lock_init(&rd->rto_lock);
281 	init_irq_work(&rd->rto_push_work, rto_push_irq_work_func);
282 #endif
283 
284 	init_dl_bw(&rd->dl_bw);
285 	if (cpudl_init(&rd->cpudl) != 0)
286 		goto free_rto_mask;
287 
288 	if (cpupri_init(&rd->cpupri) != 0)
289 		goto free_cpudl;
290 	return 0;
291 
292 free_cpudl:
293 	cpudl_cleanup(&rd->cpudl);
294 free_rto_mask:
295 	free_cpumask_var(rd->rto_mask);
296 free_dlo_mask:
297 	free_cpumask_var(rd->dlo_mask);
298 free_online:
299 	free_cpumask_var(rd->online);
300 free_span:
301 	free_cpumask_var(rd->span);
302 out:
303 	return -ENOMEM;
304 }
305 
306 /*
307  * By default the system creates a single root-domain with all CPUs as
308  * members (mimicking the global state we have today).
309  */
310 struct root_domain def_root_domain;
311 
312 void init_defrootdomain(void)
313 {
314 	init_rootdomain(&def_root_domain);
315 
316 	atomic_set(&def_root_domain.refcount, 1);
317 }
318 
319 static struct root_domain *alloc_rootdomain(void)
320 {
321 	struct root_domain *rd;
322 
323 	rd = kzalloc(sizeof(*rd), GFP_KERNEL);
324 	if (!rd)
325 		return NULL;
326 
327 	if (init_rootdomain(rd) != 0) {
328 		kfree(rd);
329 		return NULL;
330 	}
331 
332 	return rd;
333 }
334 
335 static void free_sched_groups(struct sched_group *sg, int free_sgc)
336 {
337 	struct sched_group *tmp, *first;
338 
339 	if (!sg)
340 		return;
341 
342 	first = sg;
343 	do {
344 		tmp = sg->next;
345 
346 		if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
347 			kfree(sg->sgc);
348 
349 		if (atomic_dec_and_test(&sg->ref))
350 			kfree(sg);
351 		sg = tmp;
352 	} while (sg != first);
353 }
354 
355 static void destroy_sched_domain(struct sched_domain *sd)
356 {
357 	/*
358 	 * A normal sched domain may have multiple group references, an
359 	 * overlapping domain, having private groups, only one.  Iterate,
360 	 * dropping group/capacity references, freeing where none remain.
361 	 */
362 	free_sched_groups(sd->groups, 1);
363 
364 	if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
365 		kfree(sd->shared);
366 	kfree(sd);
367 }
368 
369 static void destroy_sched_domains_rcu(struct rcu_head *rcu)
370 {
371 	struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
372 
373 	while (sd) {
374 		struct sched_domain *parent = sd->parent;
375 		destroy_sched_domain(sd);
376 		sd = parent;
377 	}
378 }
379 
380 static void destroy_sched_domains(struct sched_domain *sd)
381 {
382 	if (sd)
383 		call_rcu(&sd->rcu, destroy_sched_domains_rcu);
384 }
385 
386 /*
387  * Keep a special pointer to the highest sched_domain that has
388  * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
389  * allows us to avoid some pointer chasing select_idle_sibling().
390  *
391  * Also keep a unique ID per domain (we use the first CPU number in
392  * the cpumask of the domain), this allows us to quickly tell if
393  * two CPUs are in the same cache domain, see cpus_share_cache().
394  */
395 DEFINE_PER_CPU(struct sched_domain *, sd_llc);
396 DEFINE_PER_CPU(int, sd_llc_size);
397 DEFINE_PER_CPU(int, sd_llc_id);
398 DEFINE_PER_CPU(struct sched_domain_shared *, sd_llc_shared);
399 DEFINE_PER_CPU(struct sched_domain *, sd_numa);
400 DEFINE_PER_CPU(struct sched_domain *, sd_asym);
401 
402 static void update_top_cache_domain(int cpu)
403 {
404 	struct sched_domain_shared *sds = NULL;
405 	struct sched_domain *sd;
406 	int id = cpu;
407 	int size = 1;
408 
409 	sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
410 	if (sd) {
411 		id = cpumask_first(sched_domain_span(sd));
412 		size = cpumask_weight(sched_domain_span(sd));
413 		sds = sd->shared;
414 	}
415 
416 	rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
417 	per_cpu(sd_llc_size, cpu) = size;
418 	per_cpu(sd_llc_id, cpu) = id;
419 	rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
420 
421 	sd = lowest_flag_domain(cpu, SD_NUMA);
422 	rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
423 
424 	sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
425 	rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
426 }
427 
428 /*
429  * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
430  * hold the hotplug lock.
431  */
432 static void
433 cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
434 {
435 	struct rq *rq = cpu_rq(cpu);
436 	struct sched_domain *tmp;
437 
438 	/* Remove the sched domains which do not contribute to scheduling. */
439 	for (tmp = sd; tmp; ) {
440 		struct sched_domain *parent = tmp->parent;
441 		if (!parent)
442 			break;
443 
444 		if (sd_parent_degenerate(tmp, parent)) {
445 			tmp->parent = parent->parent;
446 			if (parent->parent)
447 				parent->parent->child = tmp;
448 			/*
449 			 * Transfer SD_PREFER_SIBLING down in case of a
450 			 * degenerate parent; the spans match for this
451 			 * so the property transfers.
452 			 */
453 			if (parent->flags & SD_PREFER_SIBLING)
454 				tmp->flags |= SD_PREFER_SIBLING;
455 			destroy_sched_domain(parent);
456 		} else
457 			tmp = tmp->parent;
458 	}
459 
460 	if (sd && sd_degenerate(sd)) {
461 		tmp = sd;
462 		sd = sd->parent;
463 		destroy_sched_domain(tmp);
464 		if (sd)
465 			sd->child = NULL;
466 	}
467 
468 	sched_domain_debug(sd, cpu);
469 
470 	rq_attach_root(rq, rd);
471 	tmp = rq->sd;
472 	rcu_assign_pointer(rq->sd, sd);
473 	dirty_sched_domain_sysctl(cpu);
474 	destroy_sched_domains(tmp);
475 
476 	update_top_cache_domain(cpu);
477 }
478 
479 struct s_data {
480 	struct sched_domain ** __percpu sd;
481 	struct root_domain	*rd;
482 };
483 
484 enum s_alloc {
485 	sa_rootdomain,
486 	sa_sd,
487 	sa_sd_storage,
488 	sa_none,
489 };
490 
491 /*
492  * Return the canonical balance CPU for this group, this is the first CPU
493  * of this group that's also in the balance mask.
494  *
495  * The balance mask are all those CPUs that could actually end up at this
496  * group. See build_balance_mask().
497  *
498  * Also see should_we_balance().
499  */
500 int group_balance_cpu(struct sched_group *sg)
501 {
502 	return cpumask_first(group_balance_mask(sg));
503 }
504 
505 
506 /*
507  * NUMA topology (first read the regular topology blurb below)
508  *
509  * Given a node-distance table, for example:
510  *
511  *   node   0   1   2   3
512  *     0:  10  20  30  20
513  *     1:  20  10  20  30
514  *     2:  30  20  10  20
515  *     3:  20  30  20  10
516  *
517  * which represents a 4 node ring topology like:
518  *
519  *   0 ----- 1
520  *   |       |
521  *   |       |
522  *   |       |
523  *   3 ----- 2
524  *
525  * We want to construct domains and groups to represent this. The way we go
526  * about doing this is to build the domains on 'hops'. For each NUMA level we
527  * construct the mask of all nodes reachable in @level hops.
528  *
529  * For the above NUMA topology that gives 3 levels:
530  *
531  * NUMA-2	0-3		0-3		0-3		0-3
532  *  groups:	{0-1,3},{1-3}	{0-2},{0,2-3}	{1-3},{0-1,3}	{0,2-3},{0-2}
533  *
534  * NUMA-1	0-1,3		0-2		1-3		0,2-3
535  *  groups:	{0},{1},{3}	{0},{1},{2}	{1},{2},{3}	{0},{2},{3}
536  *
537  * NUMA-0	0		1		2		3
538  *
539  *
540  * As can be seen; things don't nicely line up as with the regular topology.
541  * When we iterate a domain in child domain chunks some nodes can be
542  * represented multiple times -- hence the "overlap" naming for this part of
543  * the topology.
544  *
545  * In order to minimize this overlap, we only build enough groups to cover the
546  * domain. For instance Node-0 NUMA-2 would only get groups: 0-1,3 and 1-3.
547  *
548  * Because:
549  *
550  *  - the first group of each domain is its child domain; this
551  *    gets us the first 0-1,3
552  *  - the only uncovered node is 2, who's child domain is 1-3.
553  *
554  * However, because of the overlap, computing a unique CPU for each group is
555  * more complicated. Consider for instance the groups of NODE-1 NUMA-2, both
556  * groups include the CPUs of Node-0, while those CPUs would not in fact ever
557  * end up at those groups (they would end up in group: 0-1,3).
558  *
559  * To correct this we have to introduce the group balance mask. This mask
560  * will contain those CPUs in the group that can reach this group given the
561  * (child) domain tree.
562  *
563  * With this we can once again compute balance_cpu and sched_group_capacity
564  * relations.
565  *
566  * XXX include words on how balance_cpu is unique and therefore can be
567  * used for sched_group_capacity links.
568  *
569  *
570  * Another 'interesting' topology is:
571  *
572  *   node   0   1   2   3
573  *     0:  10  20  20  30
574  *     1:  20  10  20  20
575  *     2:  20  20  10  20
576  *     3:  30  20  20  10
577  *
578  * Which looks a little like:
579  *
580  *   0 ----- 1
581  *   |     / |
582  *   |   /   |
583  *   | /     |
584  *   2 ----- 3
585  *
586  * This topology is asymmetric, nodes 1,2 are fully connected, but nodes 0,3
587  * are not.
588  *
589  * This leads to a few particularly weird cases where the sched_domain's are
590  * not of the same number for each CPU. Consider:
591  *
592  * NUMA-2	0-3						0-3
593  *  groups:	{0-2},{1-3}					{1-3},{0-2}
594  *
595  * NUMA-1	0-2		0-3		0-3		1-3
596  *
597  * NUMA-0	0		1		2		3
598  *
599  */
600 
601 
602 /*
603  * Build the balance mask; it contains only those CPUs that can arrive at this
604  * group and should be considered to continue balancing.
605  *
606  * We do this during the group creation pass, therefore the group information
607  * isn't complete yet, however since each group represents a (child) domain we
608  * can fully construct this using the sched_domain bits (which are already
609  * complete).
610  */
611 static void
612 build_balance_mask(struct sched_domain *sd, struct sched_group *sg, struct cpumask *mask)
613 {
614 	const struct cpumask *sg_span = sched_group_span(sg);
615 	struct sd_data *sdd = sd->private;
616 	struct sched_domain *sibling;
617 	int i;
618 
619 	cpumask_clear(mask);
620 
621 	for_each_cpu(i, sg_span) {
622 		sibling = *per_cpu_ptr(sdd->sd, i);
623 
624 		/*
625 		 * Can happen in the asymmetric case, where these siblings are
626 		 * unused. The mask will not be empty because those CPUs that
627 		 * do have the top domain _should_ span the domain.
628 		 */
629 		if (!sibling->child)
630 			continue;
631 
632 		/* If we would not end up here, we can't continue from here */
633 		if (!cpumask_equal(sg_span, sched_domain_span(sibling->child)))
634 			continue;
635 
636 		cpumask_set_cpu(i, mask);
637 	}
638 
639 	/* We must not have empty masks here */
640 	WARN_ON_ONCE(cpumask_empty(mask));
641 }
642 
643 /*
644  * XXX: This creates per-node group entries; since the load-balancer will
645  * immediately access remote memory to construct this group's load-balance
646  * statistics having the groups node local is of dubious benefit.
647  */
648 static struct sched_group *
649 build_group_from_child_sched_domain(struct sched_domain *sd, int cpu)
650 {
651 	struct sched_group *sg;
652 	struct cpumask *sg_span;
653 
654 	sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
655 			GFP_KERNEL, cpu_to_node(cpu));
656 
657 	if (!sg)
658 		return NULL;
659 
660 	sg_span = sched_group_span(sg);
661 	if (sd->child)
662 		cpumask_copy(sg_span, sched_domain_span(sd->child));
663 	else
664 		cpumask_copy(sg_span, sched_domain_span(sd));
665 
666 	atomic_inc(&sg->ref);
667 	return sg;
668 }
669 
670 static void init_overlap_sched_group(struct sched_domain *sd,
671 				     struct sched_group *sg)
672 {
673 	struct cpumask *mask = sched_domains_tmpmask2;
674 	struct sd_data *sdd = sd->private;
675 	struct cpumask *sg_span;
676 	int cpu;
677 
678 	build_balance_mask(sd, sg, mask);
679 	cpu = cpumask_first_and(sched_group_span(sg), mask);
680 
681 	sg->sgc = *per_cpu_ptr(sdd->sgc, cpu);
682 	if (atomic_inc_return(&sg->sgc->ref) == 1)
683 		cpumask_copy(group_balance_mask(sg), mask);
684 	else
685 		WARN_ON_ONCE(!cpumask_equal(group_balance_mask(sg), mask));
686 
687 	/*
688 	 * Initialize sgc->capacity such that even if we mess up the
689 	 * domains and no possible iteration will get us here, we won't
690 	 * die on a /0 trap.
691 	 */
692 	sg_span = sched_group_span(sg);
693 	sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
694 	sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;
695 }
696 
697 static int
698 build_overlap_sched_groups(struct sched_domain *sd, int cpu)
699 {
700 	struct sched_group *first = NULL, *last = NULL, *sg;
701 	const struct cpumask *span = sched_domain_span(sd);
702 	struct cpumask *covered = sched_domains_tmpmask;
703 	struct sd_data *sdd = sd->private;
704 	struct sched_domain *sibling;
705 	int i;
706 
707 	cpumask_clear(covered);
708 
709 	for_each_cpu_wrap(i, span, cpu) {
710 		struct cpumask *sg_span;
711 
712 		if (cpumask_test_cpu(i, covered))
713 			continue;
714 
715 		sibling = *per_cpu_ptr(sdd->sd, i);
716 
717 		/*
718 		 * Asymmetric node setups can result in situations where the
719 		 * domain tree is of unequal depth, make sure to skip domains
720 		 * that already cover the entire range.
721 		 *
722 		 * In that case build_sched_domains() will have terminated the
723 		 * iteration early and our sibling sd spans will be empty.
724 		 * Domains should always include the CPU they're built on, so
725 		 * check that.
726 		 */
727 		if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
728 			continue;
729 
730 		sg = build_group_from_child_sched_domain(sibling, cpu);
731 		if (!sg)
732 			goto fail;
733 
734 		sg_span = sched_group_span(sg);
735 		cpumask_or(covered, covered, sg_span);
736 
737 		init_overlap_sched_group(sd, sg);
738 
739 		if (!first)
740 			first = sg;
741 		if (last)
742 			last->next = sg;
743 		last = sg;
744 		last->next = first;
745 	}
746 	sd->groups = first;
747 
748 	return 0;
749 
750 fail:
751 	free_sched_groups(first, 0);
752 
753 	return -ENOMEM;
754 }
755 
756 
757 /*
758  * Package topology (also see the load-balance blurb in fair.c)
759  *
760  * The scheduler builds a tree structure to represent a number of important
761  * topology features. By default (default_topology[]) these include:
762  *
763  *  - Simultaneous multithreading (SMT)
764  *  - Multi-Core Cache (MC)
765  *  - Package (DIE)
766  *
767  * Where the last one more or less denotes everything up to a NUMA node.
768  *
769  * The tree consists of 3 primary data structures:
770  *
771  *	sched_domain -> sched_group -> sched_group_capacity
772  *	    ^ ^             ^ ^
773  *          `-'             `-'
774  *
775  * The sched_domains are per-CPU and have a two way link (parent & child) and
776  * denote the ever growing mask of CPUs belonging to that level of topology.
777  *
778  * Each sched_domain has a circular (double) linked list of sched_group's, each
779  * denoting the domains of the level below (or individual CPUs in case of the
780  * first domain level). The sched_group linked by a sched_domain includes the
781  * CPU of that sched_domain [*].
782  *
783  * Take for instance a 2 threaded, 2 core, 2 cache cluster part:
784  *
785  * CPU   0   1   2   3   4   5   6   7
786  *
787  * DIE  [                             ]
788  * MC   [             ] [             ]
789  * SMT  [     ] [     ] [     ] [     ]
790  *
791  *  - or -
792  *
793  * DIE  0-7 0-7 0-7 0-7 0-7 0-7 0-7 0-7
794  * MC	0-3 0-3 0-3 0-3 4-7 4-7 4-7 4-7
795  * SMT  0-1 0-1 2-3 2-3 4-5 4-5 6-7 6-7
796  *
797  * CPU   0   1   2   3   4   5   6   7
798  *
799  * One way to think about it is: sched_domain moves you up and down among these
800  * topology levels, while sched_group moves you sideways through it, at child
801  * domain granularity.
802  *
803  * sched_group_capacity ensures each unique sched_group has shared storage.
804  *
805  * There are two related construction problems, both require a CPU that
806  * uniquely identify each group (for a given domain):
807  *
808  *  - The first is the balance_cpu (see should_we_balance() and the
809  *    load-balance blub in fair.c); for each group we only want 1 CPU to
810  *    continue balancing at a higher domain.
811  *
812  *  - The second is the sched_group_capacity; we want all identical groups
813  *    to share a single sched_group_capacity.
814  *
815  * Since these topologies are exclusive by construction. That is, its
816  * impossible for an SMT thread to belong to multiple cores, and cores to
817  * be part of multiple caches. There is a very clear and unique location
818  * for each CPU in the hierarchy.
819  *
820  * Therefore computing a unique CPU for each group is trivial (the iteration
821  * mask is redundant and set all 1s; all CPUs in a group will end up at _that_
822  * group), we can simply pick the first CPU in each group.
823  *
824  *
825  * [*] in other words, the first group of each domain is its child domain.
826  */
827 
828 static struct sched_group *get_group(int cpu, struct sd_data *sdd)
829 {
830 	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
831 	struct sched_domain *child = sd->child;
832 	struct sched_group *sg;
833 
834 	if (child)
835 		cpu = cpumask_first(sched_domain_span(child));
836 
837 	sg = *per_cpu_ptr(sdd->sg, cpu);
838 	sg->sgc = *per_cpu_ptr(sdd->sgc, cpu);
839 
840 	/* For claim_allocations: */
841 	atomic_inc(&sg->ref);
842 	atomic_inc(&sg->sgc->ref);
843 
844 	if (child) {
845 		cpumask_copy(sched_group_span(sg), sched_domain_span(child));
846 		cpumask_copy(group_balance_mask(sg), sched_group_span(sg));
847 	} else {
848 		cpumask_set_cpu(cpu, sched_group_span(sg));
849 		cpumask_set_cpu(cpu, group_balance_mask(sg));
850 	}
851 
852 	sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sched_group_span(sg));
853 	sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;
854 
855 	return sg;
856 }
857 
858 /*
859  * build_sched_groups will build a circular linked list of the groups
860  * covered by the given span, and will set each group's ->cpumask correctly,
861  * and ->cpu_capacity to 0.
862  *
863  * Assumes the sched_domain tree is fully constructed
864  */
865 static int
866 build_sched_groups(struct sched_domain *sd, int cpu)
867 {
868 	struct sched_group *first = NULL, *last = NULL;
869 	struct sd_data *sdd = sd->private;
870 	const struct cpumask *span = sched_domain_span(sd);
871 	struct cpumask *covered;
872 	int i;
873 
874 	lockdep_assert_held(&sched_domains_mutex);
875 	covered = sched_domains_tmpmask;
876 
877 	cpumask_clear(covered);
878 
879 	for_each_cpu_wrap(i, span, cpu) {
880 		struct sched_group *sg;
881 
882 		if (cpumask_test_cpu(i, covered))
883 			continue;
884 
885 		sg = get_group(i, sdd);
886 
887 		cpumask_or(covered, covered, sched_group_span(sg));
888 
889 		if (!first)
890 			first = sg;
891 		if (last)
892 			last->next = sg;
893 		last = sg;
894 	}
895 	last->next = first;
896 	sd->groups = first;
897 
898 	return 0;
899 }
900 
901 /*
902  * Initialize sched groups cpu_capacity.
903  *
904  * cpu_capacity indicates the capacity of sched group, which is used while
905  * distributing the load between different sched groups in a sched domain.
906  * Typically cpu_capacity for all the groups in a sched domain will be same
907  * unless there are asymmetries in the topology. If there are asymmetries,
908  * group having more cpu_capacity will pickup more load compared to the
909  * group having less cpu_capacity.
910  */
911 static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
912 {
913 	struct sched_group *sg = sd->groups;
914 
915 	WARN_ON(!sg);
916 
917 	do {
918 		int cpu, max_cpu = -1;
919 
920 		sg->group_weight = cpumask_weight(sched_group_span(sg));
921 
922 		if (!(sd->flags & SD_ASYM_PACKING))
923 			goto next;
924 
925 		for_each_cpu(cpu, sched_group_span(sg)) {
926 			if (max_cpu < 0)
927 				max_cpu = cpu;
928 			else if (sched_asym_prefer(cpu, max_cpu))
929 				max_cpu = cpu;
930 		}
931 		sg->asym_prefer_cpu = max_cpu;
932 
933 next:
934 		sg = sg->next;
935 	} while (sg != sd->groups);
936 
937 	if (cpu != group_balance_cpu(sg))
938 		return;
939 
940 	update_group_capacity(sd, cpu);
941 }
942 
943 /*
944  * Initializers for schedule domains
945  * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
946  */
947 
948 static int default_relax_domain_level = -1;
949 int sched_domain_level_max;
950 
951 static int __init setup_relax_domain_level(char *str)
952 {
953 	if (kstrtoint(str, 0, &default_relax_domain_level))
954 		pr_warn("Unable to set relax_domain_level\n");
955 
956 	return 1;
957 }
958 __setup("relax_domain_level=", setup_relax_domain_level);
959 
960 static void set_domain_attribute(struct sched_domain *sd,
961 				 struct sched_domain_attr *attr)
962 {
963 	int request;
964 
965 	if (!attr || attr->relax_domain_level < 0) {
966 		if (default_relax_domain_level < 0)
967 			return;
968 		else
969 			request = default_relax_domain_level;
970 	} else
971 		request = attr->relax_domain_level;
972 	if (request < sd->level) {
973 		/* Turn off idle balance on this domain: */
974 		sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
975 	} else {
976 		/* Turn on idle balance on this domain: */
977 		sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
978 	}
979 }
980 
981 static void __sdt_free(const struct cpumask *cpu_map);
982 static int __sdt_alloc(const struct cpumask *cpu_map);
983 
984 static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
985 				 const struct cpumask *cpu_map)
986 {
987 	switch (what) {
988 	case sa_rootdomain:
989 		if (!atomic_read(&d->rd->refcount))
990 			free_rootdomain(&d->rd->rcu);
991 		/* Fall through */
992 	case sa_sd:
993 		free_percpu(d->sd);
994 		/* Fall through */
995 	case sa_sd_storage:
996 		__sdt_free(cpu_map);
997 		/* Fall through */
998 	case sa_none:
999 		break;
1000 	}
1001 }
1002 
1003 static enum s_alloc
1004 __visit_domain_allocation_hell(struct s_data *d, const struct cpumask *cpu_map)
1005 {
1006 	memset(d, 0, sizeof(*d));
1007 
1008 	if (__sdt_alloc(cpu_map))
1009 		return sa_sd_storage;
1010 	d->sd = alloc_percpu(struct sched_domain *);
1011 	if (!d->sd)
1012 		return sa_sd_storage;
1013 	d->rd = alloc_rootdomain();
1014 	if (!d->rd)
1015 		return sa_sd;
1016 
1017 	return sa_rootdomain;
1018 }
1019 
1020 /*
1021  * NULL the sd_data elements we've used to build the sched_domain and
1022  * sched_group structure so that the subsequent __free_domain_allocs()
1023  * will not free the data we're using.
1024  */
1025 static void claim_allocations(int cpu, struct sched_domain *sd)
1026 {
1027 	struct sd_data *sdd = sd->private;
1028 
1029 	WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
1030 	*per_cpu_ptr(sdd->sd, cpu) = NULL;
1031 
1032 	if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref))
1033 		*per_cpu_ptr(sdd->sds, cpu) = NULL;
1034 
1035 	if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
1036 		*per_cpu_ptr(sdd->sg, cpu) = NULL;
1037 
1038 	if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
1039 		*per_cpu_ptr(sdd->sgc, cpu) = NULL;
1040 }
1041 
1042 #ifdef CONFIG_NUMA
1043 enum numa_topology_type sched_numa_topology_type;
1044 
1045 static int			sched_domains_numa_levels;
1046 static int			sched_domains_curr_level;
1047 
1048 int				sched_max_numa_distance;
1049 static int			*sched_domains_numa_distance;
1050 static struct cpumask		***sched_domains_numa_masks;
1051 #endif
1052 
1053 /*
1054  * SD_flags allowed in topology descriptions.
1055  *
1056  * These flags are purely descriptive of the topology and do not prescribe
1057  * behaviour. Behaviour is artificial and mapped in the below sd_init()
1058  * function:
1059  *
1060  *   SD_SHARE_CPUCAPACITY   - describes SMT topologies
1061  *   SD_SHARE_PKG_RESOURCES - describes shared caches
1062  *   SD_NUMA                - describes NUMA topologies
1063  *   SD_SHARE_POWERDOMAIN   - describes shared power domain
1064  *   SD_ASYM_CPUCAPACITY    - describes mixed capacity topologies
1065  *
1066  * Odd one out, which beside describing the topology has a quirk also
1067  * prescribes the desired behaviour that goes along with it:
1068  *
1069  *   SD_ASYM_PACKING        - describes SMT quirks
1070  */
1071 #define TOPOLOGY_SD_FLAGS		\
1072 	(SD_SHARE_CPUCAPACITY	|	\
1073 	 SD_SHARE_PKG_RESOURCES |	\
1074 	 SD_NUMA		|	\
1075 	 SD_ASYM_PACKING	|	\
1076 	 SD_ASYM_CPUCAPACITY	|	\
1077 	 SD_SHARE_POWERDOMAIN)
1078 
1079 static struct sched_domain *
1080 sd_init(struct sched_domain_topology_level *tl,
1081 	const struct cpumask *cpu_map,
1082 	struct sched_domain *child, int cpu)
1083 {
1084 	struct sd_data *sdd = &tl->data;
1085 	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
1086 	int sd_id, sd_weight, sd_flags = 0;
1087 
1088 #ifdef CONFIG_NUMA
1089 	/*
1090 	 * Ugly hack to pass state to sd_numa_mask()...
1091 	 */
1092 	sched_domains_curr_level = tl->numa_level;
1093 #endif
1094 
1095 	sd_weight = cpumask_weight(tl->mask(cpu));
1096 
1097 	if (tl->sd_flags)
1098 		sd_flags = (*tl->sd_flags)();
1099 	if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
1100 			"wrong sd_flags in topology description\n"))
1101 		sd_flags &= ~TOPOLOGY_SD_FLAGS;
1102 
1103 	*sd = (struct sched_domain){
1104 		.min_interval		= sd_weight,
1105 		.max_interval		= 2*sd_weight,
1106 		.busy_factor		= 32,
1107 		.imbalance_pct		= 125,
1108 
1109 		.cache_nice_tries	= 0,
1110 		.busy_idx		= 0,
1111 		.idle_idx		= 0,
1112 		.newidle_idx		= 0,
1113 		.wake_idx		= 0,
1114 		.forkexec_idx		= 0,
1115 
1116 		.flags			= 1*SD_LOAD_BALANCE
1117 					| 1*SD_BALANCE_NEWIDLE
1118 					| 1*SD_BALANCE_EXEC
1119 					| 1*SD_BALANCE_FORK
1120 					| 0*SD_BALANCE_WAKE
1121 					| 1*SD_WAKE_AFFINE
1122 					| 0*SD_SHARE_CPUCAPACITY
1123 					| 0*SD_SHARE_PKG_RESOURCES
1124 					| 0*SD_SERIALIZE
1125 					| 0*SD_PREFER_SIBLING
1126 					| 0*SD_NUMA
1127 					| sd_flags
1128 					,
1129 
1130 		.last_balance		= jiffies,
1131 		.balance_interval	= sd_weight,
1132 		.smt_gain		= 0,
1133 		.max_newidle_lb_cost	= 0,
1134 		.next_decay_max_lb_cost	= jiffies,
1135 		.child			= child,
1136 #ifdef CONFIG_SCHED_DEBUG
1137 		.name			= tl->name,
1138 #endif
1139 	};
1140 
1141 	cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
1142 	sd_id = cpumask_first(sched_domain_span(sd));
1143 
1144 	/*
1145 	 * Convert topological properties into behaviour.
1146 	 */
1147 
1148 	if (sd->flags & SD_ASYM_CPUCAPACITY) {
1149 		struct sched_domain *t = sd;
1150 
1151 		for_each_lower_domain(t)
1152 			t->flags |= SD_BALANCE_WAKE;
1153 	}
1154 
1155 	if (sd->flags & SD_SHARE_CPUCAPACITY) {
1156 		sd->flags |= SD_PREFER_SIBLING;
1157 		sd->imbalance_pct = 110;
1158 		sd->smt_gain = 1178; /* ~15% */
1159 
1160 	} else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
1161 		sd->flags |= SD_PREFER_SIBLING;
1162 		sd->imbalance_pct = 117;
1163 		sd->cache_nice_tries = 1;
1164 		sd->busy_idx = 2;
1165 
1166 #ifdef CONFIG_NUMA
1167 	} else if (sd->flags & SD_NUMA) {
1168 		sd->cache_nice_tries = 2;
1169 		sd->busy_idx = 3;
1170 		sd->idle_idx = 2;
1171 
1172 		sd->flags |= SD_SERIALIZE;
1173 		if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
1174 			sd->flags &= ~(SD_BALANCE_EXEC |
1175 				       SD_BALANCE_FORK |
1176 				       SD_WAKE_AFFINE);
1177 		}
1178 
1179 #endif
1180 	} else {
1181 		sd->flags |= SD_PREFER_SIBLING;
1182 		sd->cache_nice_tries = 1;
1183 		sd->busy_idx = 2;
1184 		sd->idle_idx = 1;
1185 	}
1186 
1187 	/*
1188 	 * For all levels sharing cache; connect a sched_domain_shared
1189 	 * instance.
1190 	 */
1191 	if (sd->flags & SD_SHARE_PKG_RESOURCES) {
1192 		sd->shared = *per_cpu_ptr(sdd->sds, sd_id);
1193 		atomic_inc(&sd->shared->ref);
1194 		atomic_set(&sd->shared->nr_busy_cpus, sd_weight);
1195 	}
1196 
1197 	sd->private = sdd;
1198 
1199 	return sd;
1200 }
1201 
1202 /*
1203  * Topology list, bottom-up.
1204  */
1205 static struct sched_domain_topology_level default_topology[] = {
1206 #ifdef CONFIG_SCHED_SMT
1207 	{ cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
1208 #endif
1209 #ifdef CONFIG_SCHED_MC
1210 	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
1211 #endif
1212 	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
1213 	{ NULL, },
1214 };
1215 
1216 static struct sched_domain_topology_level *sched_domain_topology =
1217 	default_topology;
1218 
1219 #define for_each_sd_topology(tl)			\
1220 	for (tl = sched_domain_topology; tl->mask; tl++)
1221 
1222 void set_sched_topology(struct sched_domain_topology_level *tl)
1223 {
1224 	if (WARN_ON_ONCE(sched_smp_initialized))
1225 		return;
1226 
1227 	sched_domain_topology = tl;
1228 }
1229 
1230 #ifdef CONFIG_NUMA
1231 
1232 static const struct cpumask *sd_numa_mask(int cpu)
1233 {
1234 	return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
1235 }
1236 
1237 static void sched_numa_warn(const char *str)
1238 {
1239 	static int done = false;
1240 	int i,j;
1241 
1242 	if (done)
1243 		return;
1244 
1245 	done = true;
1246 
1247 	printk(KERN_WARNING "ERROR: %s\n\n", str);
1248 
1249 	for (i = 0; i < nr_node_ids; i++) {
1250 		printk(KERN_WARNING "  ");
1251 		for (j = 0; j < nr_node_ids; j++)
1252 			printk(KERN_CONT "%02d ", node_distance(i,j));
1253 		printk(KERN_CONT "\n");
1254 	}
1255 	printk(KERN_WARNING "\n");
1256 }
1257 
1258 bool find_numa_distance(int distance)
1259 {
1260 	int i;
1261 
1262 	if (distance == node_distance(0, 0))
1263 		return true;
1264 
1265 	for (i = 0; i < sched_domains_numa_levels; i++) {
1266 		if (sched_domains_numa_distance[i] == distance)
1267 			return true;
1268 	}
1269 
1270 	return false;
1271 }
1272 
1273 /*
1274  * A system can have three types of NUMA topology:
1275  * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system
1276  * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes
1277  * NUMA_BACKPLANE: nodes can reach other nodes through a backplane
1278  *
1279  * The difference between a glueless mesh topology and a backplane
1280  * topology lies in whether communication between not directly
1281  * connected nodes goes through intermediary nodes (where programs
1282  * could run), or through backplane controllers. This affects
1283  * placement of programs.
1284  *
1285  * The type of topology can be discerned with the following tests:
1286  * - If the maximum distance between any nodes is 1 hop, the system
1287  *   is directly connected.
1288  * - If for two nodes A and B, located N > 1 hops away from each other,
1289  *   there is an intermediary node C, which is < N hops away from both
1290  *   nodes A and B, the system is a glueless mesh.
1291  */
1292 static void init_numa_topology_type(void)
1293 {
1294 	int a, b, c, n;
1295 
1296 	n = sched_max_numa_distance;
1297 
1298 	if (sched_domains_numa_levels <= 1) {
1299 		sched_numa_topology_type = NUMA_DIRECT;
1300 		return;
1301 	}
1302 
1303 	for_each_online_node(a) {
1304 		for_each_online_node(b) {
1305 			/* Find two nodes furthest removed from each other. */
1306 			if (node_distance(a, b) < n)
1307 				continue;
1308 
1309 			/* Is there an intermediary node between a and b? */
1310 			for_each_online_node(c) {
1311 				if (node_distance(a, c) < n &&
1312 				    node_distance(b, c) < n) {
1313 					sched_numa_topology_type =
1314 							NUMA_GLUELESS_MESH;
1315 					return;
1316 				}
1317 			}
1318 
1319 			sched_numa_topology_type = NUMA_BACKPLANE;
1320 			return;
1321 		}
1322 	}
1323 }
1324 
1325 void sched_init_numa(void)
1326 {
1327 	int next_distance, curr_distance = node_distance(0, 0);
1328 	struct sched_domain_topology_level *tl;
1329 	int level = 0;
1330 	int i, j, k;
1331 
1332 	sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
1333 	if (!sched_domains_numa_distance)
1334 		return;
1335 
1336 	/* Includes NUMA identity node at level 0. */
1337 	sched_domains_numa_distance[level++] = curr_distance;
1338 	sched_domains_numa_levels = level;
1339 
1340 	/*
1341 	 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
1342 	 * unique distances in the node_distance() table.
1343 	 *
1344 	 * Assumes node_distance(0,j) includes all distances in
1345 	 * node_distance(i,j) in order to avoid cubic time.
1346 	 */
1347 	next_distance = curr_distance;
1348 	for (i = 0; i < nr_node_ids; i++) {
1349 		for (j = 0; j < nr_node_ids; j++) {
1350 			for (k = 0; k < nr_node_ids; k++) {
1351 				int distance = node_distance(i, k);
1352 
1353 				if (distance > curr_distance &&
1354 				    (distance < next_distance ||
1355 				     next_distance == curr_distance))
1356 					next_distance = distance;
1357 
1358 				/*
1359 				 * While not a strong assumption it would be nice to know
1360 				 * about cases where if node A is connected to B, B is not
1361 				 * equally connected to A.
1362 				 */
1363 				if (sched_debug() && node_distance(k, i) != distance)
1364 					sched_numa_warn("Node-distance not symmetric");
1365 
1366 				if (sched_debug() && i && !find_numa_distance(distance))
1367 					sched_numa_warn("Node-0 not representative");
1368 			}
1369 			if (next_distance != curr_distance) {
1370 				sched_domains_numa_distance[level++] = next_distance;
1371 				sched_domains_numa_levels = level;
1372 				curr_distance = next_distance;
1373 			} else break;
1374 		}
1375 
1376 		/*
1377 		 * In case of sched_debug() we verify the above assumption.
1378 		 */
1379 		if (!sched_debug())
1380 			break;
1381 	}
1382 
1383 	if (!level)
1384 		return;
1385 
1386 	/*
1387 	 * 'level' contains the number of unique distances
1388 	 *
1389 	 * The sched_domains_numa_distance[] array includes the actual distance
1390 	 * numbers.
1391 	 */
1392 
1393 	/*
1394 	 * Here, we should temporarily reset sched_domains_numa_levels to 0.
1395 	 * If it fails to allocate memory for array sched_domains_numa_masks[][],
1396 	 * the array will contain less then 'level' members. This could be
1397 	 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
1398 	 * in other functions.
1399 	 *
1400 	 * We reset it to 'level' at the end of this function.
1401 	 */
1402 	sched_domains_numa_levels = 0;
1403 
1404 	sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
1405 	if (!sched_domains_numa_masks)
1406 		return;
1407 
1408 	/*
1409 	 * Now for each level, construct a mask per node which contains all
1410 	 * CPUs of nodes that are that many hops away from us.
1411 	 */
1412 	for (i = 0; i < level; i++) {
1413 		sched_domains_numa_masks[i] =
1414 			kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
1415 		if (!sched_domains_numa_masks[i])
1416 			return;
1417 
1418 		for (j = 0; j < nr_node_ids; j++) {
1419 			struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
1420 			if (!mask)
1421 				return;
1422 
1423 			sched_domains_numa_masks[i][j] = mask;
1424 
1425 			for_each_node(k) {
1426 				if (node_distance(j, k) > sched_domains_numa_distance[i])
1427 					continue;
1428 
1429 				cpumask_or(mask, mask, cpumask_of_node(k));
1430 			}
1431 		}
1432 	}
1433 
1434 	/* Compute default topology size */
1435 	for (i = 0; sched_domain_topology[i].mask; i++);
1436 
1437 	tl = kzalloc((i + level + 1) *
1438 			sizeof(struct sched_domain_topology_level), GFP_KERNEL);
1439 	if (!tl)
1440 		return;
1441 
1442 	/*
1443 	 * Copy the default topology bits..
1444 	 */
1445 	for (i = 0; sched_domain_topology[i].mask; i++)
1446 		tl[i] = sched_domain_topology[i];
1447 
1448 	/*
1449 	 * Add the NUMA identity distance, aka single NODE.
1450 	 */
1451 	tl[i++] = (struct sched_domain_topology_level){
1452 		.mask = sd_numa_mask,
1453 		.numa_level = 0,
1454 		SD_INIT_NAME(NODE)
1455 	};
1456 
1457 	/*
1458 	 * .. and append 'j' levels of NUMA goodness.
1459 	 */
1460 	for (j = 1; j < level; i++, j++) {
1461 		tl[i] = (struct sched_domain_topology_level){
1462 			.mask = sd_numa_mask,
1463 			.sd_flags = cpu_numa_flags,
1464 			.flags = SDTL_OVERLAP,
1465 			.numa_level = j,
1466 			SD_INIT_NAME(NUMA)
1467 		};
1468 	}
1469 
1470 	sched_domain_topology = tl;
1471 
1472 	sched_domains_numa_levels = level;
1473 	sched_max_numa_distance = sched_domains_numa_distance[level - 1];
1474 
1475 	init_numa_topology_type();
1476 }
1477 
1478 void sched_domains_numa_masks_set(unsigned int cpu)
1479 {
1480 	int node = cpu_to_node(cpu);
1481 	int i, j;
1482 
1483 	for (i = 0; i < sched_domains_numa_levels; i++) {
1484 		for (j = 0; j < nr_node_ids; j++) {
1485 			if (node_distance(j, node) <= sched_domains_numa_distance[i])
1486 				cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
1487 		}
1488 	}
1489 }
1490 
1491 void sched_domains_numa_masks_clear(unsigned int cpu)
1492 {
1493 	int i, j;
1494 
1495 	for (i = 0; i < sched_domains_numa_levels; i++) {
1496 		for (j = 0; j < nr_node_ids; j++)
1497 			cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
1498 	}
1499 }
1500 
1501 #endif /* CONFIG_NUMA */
1502 
1503 static int __sdt_alloc(const struct cpumask *cpu_map)
1504 {
1505 	struct sched_domain_topology_level *tl;
1506 	int j;
1507 
1508 	for_each_sd_topology(tl) {
1509 		struct sd_data *sdd = &tl->data;
1510 
1511 		sdd->sd = alloc_percpu(struct sched_domain *);
1512 		if (!sdd->sd)
1513 			return -ENOMEM;
1514 
1515 		sdd->sds = alloc_percpu(struct sched_domain_shared *);
1516 		if (!sdd->sds)
1517 			return -ENOMEM;
1518 
1519 		sdd->sg = alloc_percpu(struct sched_group *);
1520 		if (!sdd->sg)
1521 			return -ENOMEM;
1522 
1523 		sdd->sgc = alloc_percpu(struct sched_group_capacity *);
1524 		if (!sdd->sgc)
1525 			return -ENOMEM;
1526 
1527 		for_each_cpu(j, cpu_map) {
1528 			struct sched_domain *sd;
1529 			struct sched_domain_shared *sds;
1530 			struct sched_group *sg;
1531 			struct sched_group_capacity *sgc;
1532 
1533 			sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
1534 					GFP_KERNEL, cpu_to_node(j));
1535 			if (!sd)
1536 				return -ENOMEM;
1537 
1538 			*per_cpu_ptr(sdd->sd, j) = sd;
1539 
1540 			sds = kzalloc_node(sizeof(struct sched_domain_shared),
1541 					GFP_KERNEL, cpu_to_node(j));
1542 			if (!sds)
1543 				return -ENOMEM;
1544 
1545 			*per_cpu_ptr(sdd->sds, j) = sds;
1546 
1547 			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
1548 					GFP_KERNEL, cpu_to_node(j));
1549 			if (!sg)
1550 				return -ENOMEM;
1551 
1552 			sg->next = sg;
1553 
1554 			*per_cpu_ptr(sdd->sg, j) = sg;
1555 
1556 			sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
1557 					GFP_KERNEL, cpu_to_node(j));
1558 			if (!sgc)
1559 				return -ENOMEM;
1560 
1561 #ifdef CONFIG_SCHED_DEBUG
1562 			sgc->id = j;
1563 #endif
1564 
1565 			*per_cpu_ptr(sdd->sgc, j) = sgc;
1566 		}
1567 	}
1568 
1569 	return 0;
1570 }
1571 
1572 static void __sdt_free(const struct cpumask *cpu_map)
1573 {
1574 	struct sched_domain_topology_level *tl;
1575 	int j;
1576 
1577 	for_each_sd_topology(tl) {
1578 		struct sd_data *sdd = &tl->data;
1579 
1580 		for_each_cpu(j, cpu_map) {
1581 			struct sched_domain *sd;
1582 
1583 			if (sdd->sd) {
1584 				sd = *per_cpu_ptr(sdd->sd, j);
1585 				if (sd && (sd->flags & SD_OVERLAP))
1586 					free_sched_groups(sd->groups, 0);
1587 				kfree(*per_cpu_ptr(sdd->sd, j));
1588 			}
1589 
1590 			if (sdd->sds)
1591 				kfree(*per_cpu_ptr(sdd->sds, j));
1592 			if (sdd->sg)
1593 				kfree(*per_cpu_ptr(sdd->sg, j));
1594 			if (sdd->sgc)
1595 				kfree(*per_cpu_ptr(sdd->sgc, j));
1596 		}
1597 		free_percpu(sdd->sd);
1598 		sdd->sd = NULL;
1599 		free_percpu(sdd->sds);
1600 		sdd->sds = NULL;
1601 		free_percpu(sdd->sg);
1602 		sdd->sg = NULL;
1603 		free_percpu(sdd->sgc);
1604 		sdd->sgc = NULL;
1605 	}
1606 }
1607 
1608 static struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
1609 		const struct cpumask *cpu_map, struct sched_domain_attr *attr,
1610 		struct sched_domain *child, int cpu)
1611 {
1612 	struct sched_domain *sd = sd_init(tl, cpu_map, child, cpu);
1613 
1614 	if (child) {
1615 		sd->level = child->level + 1;
1616 		sched_domain_level_max = max(sched_domain_level_max, sd->level);
1617 		child->parent = sd;
1618 
1619 		if (!cpumask_subset(sched_domain_span(child),
1620 				    sched_domain_span(sd))) {
1621 			pr_err("BUG: arch topology borken\n");
1622 #ifdef CONFIG_SCHED_DEBUG
1623 			pr_err("     the %s domain not a subset of the %s domain\n",
1624 					child->name, sd->name);
1625 #endif
1626 			/* Fixup, ensure @sd has at least @child CPUs. */
1627 			cpumask_or(sched_domain_span(sd),
1628 				   sched_domain_span(sd),
1629 				   sched_domain_span(child));
1630 		}
1631 
1632 	}
1633 	set_domain_attribute(sd, attr);
1634 
1635 	return sd;
1636 }
1637 
1638 /*
1639  * Build sched domains for a given set of CPUs and attach the sched domains
1640  * to the individual CPUs
1641  */
1642 static int
1643 build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr)
1644 {
1645 	enum s_alloc alloc_state;
1646 	struct sched_domain *sd;
1647 	struct s_data d;
1648 	struct rq *rq = NULL;
1649 	int i, ret = -ENOMEM;
1650 
1651 	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
1652 	if (alloc_state != sa_rootdomain)
1653 		goto error;
1654 
1655 	/* Set up domains for CPUs specified by the cpu_map: */
1656 	for_each_cpu(i, cpu_map) {
1657 		struct sched_domain_topology_level *tl;
1658 
1659 		sd = NULL;
1660 		for_each_sd_topology(tl) {
1661 			sd = build_sched_domain(tl, cpu_map, attr, sd, i);
1662 			if (tl == sched_domain_topology)
1663 				*per_cpu_ptr(d.sd, i) = sd;
1664 			if (tl->flags & SDTL_OVERLAP)
1665 				sd->flags |= SD_OVERLAP;
1666 			if (cpumask_equal(cpu_map, sched_domain_span(sd)))
1667 				break;
1668 		}
1669 	}
1670 
1671 	/* Build the groups for the domains */
1672 	for_each_cpu(i, cpu_map) {
1673 		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
1674 			sd->span_weight = cpumask_weight(sched_domain_span(sd));
1675 			if (sd->flags & SD_OVERLAP) {
1676 				if (build_overlap_sched_groups(sd, i))
1677 					goto error;
1678 			} else {
1679 				if (build_sched_groups(sd, i))
1680 					goto error;
1681 			}
1682 		}
1683 	}
1684 
1685 	/* Calculate CPU capacity for physical packages and nodes */
1686 	for (i = nr_cpumask_bits-1; i >= 0; i--) {
1687 		if (!cpumask_test_cpu(i, cpu_map))
1688 			continue;
1689 
1690 		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
1691 			claim_allocations(i, sd);
1692 			init_sched_groups_capacity(i, sd);
1693 		}
1694 	}
1695 
1696 	/* Attach the domains */
1697 	rcu_read_lock();
1698 	for_each_cpu(i, cpu_map) {
1699 		rq = cpu_rq(i);
1700 		sd = *per_cpu_ptr(d.sd, i);
1701 
1702 		/* Use READ_ONCE()/WRITE_ONCE() to avoid load/store tearing: */
1703 		if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity))
1704 			WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig);
1705 
1706 		cpu_attach_domain(sd, d.rd, i);
1707 	}
1708 	rcu_read_unlock();
1709 
1710 	if (rq && sched_debug_enabled) {
1711 		pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n",
1712 			cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity);
1713 	}
1714 
1715 	ret = 0;
1716 error:
1717 	__free_domain_allocs(&d, alloc_state, cpu_map);
1718 
1719 	return ret;
1720 }
1721 
1722 /* Current sched domains: */
1723 static cpumask_var_t			*doms_cur;
1724 
1725 /* Number of sched domains in 'doms_cur': */
1726 static int				ndoms_cur;
1727 
1728 /* Attribues of custom domains in 'doms_cur' */
1729 static struct sched_domain_attr		*dattr_cur;
1730 
1731 /*
1732  * Special case: If a kmalloc() of a doms_cur partition (array of
1733  * cpumask) fails, then fallback to a single sched domain,
1734  * as determined by the single cpumask fallback_doms.
1735  */
1736 static cpumask_var_t			fallback_doms;
1737 
1738 /*
1739  * arch_update_cpu_topology lets virtualized architectures update the
1740  * CPU core maps. It is supposed to return 1 if the topology changed
1741  * or 0 if it stayed the same.
1742  */
1743 int __weak arch_update_cpu_topology(void)
1744 {
1745 	return 0;
1746 }
1747 
1748 cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
1749 {
1750 	int i;
1751 	cpumask_var_t *doms;
1752 
1753 	doms = kmalloc_array(ndoms, sizeof(*doms), GFP_KERNEL);
1754 	if (!doms)
1755 		return NULL;
1756 	for (i = 0; i < ndoms; i++) {
1757 		if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
1758 			free_sched_domains(doms, i);
1759 			return NULL;
1760 		}
1761 	}
1762 	return doms;
1763 }
1764 
1765 void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
1766 {
1767 	unsigned int i;
1768 	for (i = 0; i < ndoms; i++)
1769 		free_cpumask_var(doms[i]);
1770 	kfree(doms);
1771 }
1772 
1773 /*
1774  * Set up scheduler domains and groups. Callers must hold the hotplug lock.
1775  * For now this just excludes isolated CPUs, but could be used to
1776  * exclude other special cases in the future.
1777  */
1778 int sched_init_domains(const struct cpumask *cpu_map)
1779 {
1780 	int err;
1781 
1782 	zalloc_cpumask_var(&sched_domains_tmpmask, GFP_KERNEL);
1783 	zalloc_cpumask_var(&sched_domains_tmpmask2, GFP_KERNEL);
1784 	zalloc_cpumask_var(&fallback_doms, GFP_KERNEL);
1785 
1786 	arch_update_cpu_topology();
1787 	ndoms_cur = 1;
1788 	doms_cur = alloc_sched_domains(ndoms_cur);
1789 	if (!doms_cur)
1790 		doms_cur = &fallback_doms;
1791 	cpumask_and(doms_cur[0], cpu_map, housekeeping_cpumask(HK_FLAG_DOMAIN));
1792 	err = build_sched_domains(doms_cur[0], NULL);
1793 	register_sched_domain_sysctl();
1794 
1795 	return err;
1796 }
1797 
1798 /*
1799  * Detach sched domains from a group of CPUs specified in cpu_map
1800  * These CPUs will now be attached to the NULL domain
1801  */
1802 static void detach_destroy_domains(const struct cpumask *cpu_map)
1803 {
1804 	int i;
1805 
1806 	rcu_read_lock();
1807 	for_each_cpu(i, cpu_map)
1808 		cpu_attach_domain(NULL, &def_root_domain, i);
1809 	rcu_read_unlock();
1810 }
1811 
1812 /* handle null as "default" */
1813 static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
1814 			struct sched_domain_attr *new, int idx_new)
1815 {
1816 	struct sched_domain_attr tmp;
1817 
1818 	/* Fast path: */
1819 	if (!new && !cur)
1820 		return 1;
1821 
1822 	tmp = SD_ATTR_INIT;
1823 
1824 	return !memcmp(cur ? (cur + idx_cur) : &tmp,
1825 			new ? (new + idx_new) : &tmp,
1826 			sizeof(struct sched_domain_attr));
1827 }
1828 
1829 /*
1830  * Partition sched domains as specified by the 'ndoms_new'
1831  * cpumasks in the array doms_new[] of cpumasks. This compares
1832  * doms_new[] to the current sched domain partitioning, doms_cur[].
1833  * It destroys each deleted domain and builds each new domain.
1834  *
1835  * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
1836  * The masks don't intersect (don't overlap.) We should setup one
1837  * sched domain for each mask. CPUs not in any of the cpumasks will
1838  * not be load balanced. If the same cpumask appears both in the
1839  * current 'doms_cur' domains and in the new 'doms_new', we can leave
1840  * it as it is.
1841  *
1842  * The passed in 'doms_new' should be allocated using
1843  * alloc_sched_domains.  This routine takes ownership of it and will
1844  * free_sched_domains it when done with it. If the caller failed the
1845  * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
1846  * and partition_sched_domains() will fallback to the single partition
1847  * 'fallback_doms', it also forces the domains to be rebuilt.
1848  *
1849  * If doms_new == NULL it will be replaced with cpu_online_mask.
1850  * ndoms_new == 0 is a special case for destroying existing domains,
1851  * and it will not create the default domain.
1852  *
1853  * Call with hotplug lock held
1854  */
1855 void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
1856 			     struct sched_domain_attr *dattr_new)
1857 {
1858 	int i, j, n;
1859 	int new_topology;
1860 
1861 	mutex_lock(&sched_domains_mutex);
1862 
1863 	/* Always unregister in case we don't destroy any domains: */
1864 	unregister_sched_domain_sysctl();
1865 
1866 	/* Let the architecture update CPU core mappings: */
1867 	new_topology = arch_update_cpu_topology();
1868 
1869 	if (!doms_new) {
1870 		WARN_ON_ONCE(dattr_new);
1871 		n = 0;
1872 		doms_new = alloc_sched_domains(1);
1873 		if (doms_new) {
1874 			n = 1;
1875 			cpumask_and(doms_new[0], cpu_active_mask,
1876 				    housekeeping_cpumask(HK_FLAG_DOMAIN));
1877 		}
1878 	} else {
1879 		n = ndoms_new;
1880 	}
1881 
1882 	/* Destroy deleted domains: */
1883 	for (i = 0; i < ndoms_cur; i++) {
1884 		for (j = 0; j < n && !new_topology; j++) {
1885 			if (cpumask_equal(doms_cur[i], doms_new[j])
1886 			    && dattrs_equal(dattr_cur, i, dattr_new, j))
1887 				goto match1;
1888 		}
1889 		/* No match - a current sched domain not in new doms_new[] */
1890 		detach_destroy_domains(doms_cur[i]);
1891 match1:
1892 		;
1893 	}
1894 
1895 	n = ndoms_cur;
1896 	if (!doms_new) {
1897 		n = 0;
1898 		doms_new = &fallback_doms;
1899 		cpumask_and(doms_new[0], cpu_active_mask,
1900 			    housekeeping_cpumask(HK_FLAG_DOMAIN));
1901 	}
1902 
1903 	/* Build new domains: */
1904 	for (i = 0; i < ndoms_new; i++) {
1905 		for (j = 0; j < n && !new_topology; j++) {
1906 			if (cpumask_equal(doms_new[i], doms_cur[j])
1907 			    && dattrs_equal(dattr_new, i, dattr_cur, j))
1908 				goto match2;
1909 		}
1910 		/* No match - add a new doms_new */
1911 		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
1912 match2:
1913 		;
1914 	}
1915 
1916 	/* Remember the new sched domains: */
1917 	if (doms_cur != &fallback_doms)
1918 		free_sched_domains(doms_cur, ndoms_cur);
1919 
1920 	kfree(dattr_cur);
1921 	doms_cur = doms_new;
1922 	dattr_cur = dattr_new;
1923 	ndoms_cur = ndoms_new;
1924 
1925 	register_sched_domain_sysctl();
1926 
1927 	mutex_unlock(&sched_domains_mutex);
1928 }
1929