1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Scheduler topology setup/handling methods 4 */ 5 6 #include <linux/bsearch.h> 7 8 DEFINE_MUTEX(sched_domains_mutex); 9 10 /* Protected by sched_domains_mutex: */ 11 static cpumask_var_t sched_domains_tmpmask; 12 static cpumask_var_t sched_domains_tmpmask2; 13 14 #ifdef CONFIG_SCHED_DEBUG 15 16 static int __init sched_debug_setup(char *str) 17 { 18 sched_debug_verbose = true; 19 20 return 0; 21 } 22 early_param("sched_verbose", sched_debug_setup); 23 24 static inline bool sched_debug(void) 25 { 26 return sched_debug_verbose; 27 } 28 29 #define SD_FLAG(_name, mflags) [__##_name] = { .meta_flags = mflags, .name = #_name }, 30 const struct sd_flag_debug sd_flag_debug[] = { 31 #include <linux/sched/sd_flags.h> 32 }; 33 #undef SD_FLAG 34 35 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 36 struct cpumask *groupmask) 37 { 38 struct sched_group *group = sd->groups; 39 unsigned long flags = sd->flags; 40 unsigned int idx; 41 42 cpumask_clear(groupmask); 43 44 printk(KERN_DEBUG "%*s domain-%d: ", level, "", level); 45 printk(KERN_CONT "span=%*pbl level=%s\n", 46 cpumask_pr_args(sched_domain_span(sd)), sd->name); 47 48 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) { 49 printk(KERN_ERR "ERROR: domain->span does not contain CPU%d\n", cpu); 50 } 51 if (group && !cpumask_test_cpu(cpu, sched_group_span(group))) { 52 printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n", cpu); 53 } 54 55 for_each_set_bit(idx, &flags, __SD_FLAG_CNT) { 56 unsigned int flag = BIT(idx); 57 unsigned int meta_flags = sd_flag_debug[idx].meta_flags; 58 59 if ((meta_flags & SDF_SHARED_CHILD) && sd->child && 60 !(sd->child->flags & flag)) 61 printk(KERN_ERR "ERROR: flag %s set here but not in child\n", 62 sd_flag_debug[idx].name); 63 64 if ((meta_flags & SDF_SHARED_PARENT) && sd->parent && 65 !(sd->parent->flags & flag)) 66 printk(KERN_ERR "ERROR: flag %s set here but not in parent\n", 67 sd_flag_debug[idx].name); 68 } 69 70 printk(KERN_DEBUG "%*s groups:", level + 1, ""); 71 do { 72 if (!group) { 73 printk("\n"); 74 printk(KERN_ERR "ERROR: group is NULL\n"); 75 break; 76 } 77 78 if (cpumask_empty(sched_group_span(group))) { 79 printk(KERN_CONT "\n"); 80 printk(KERN_ERR "ERROR: empty group\n"); 81 break; 82 } 83 84 if (!(sd->flags & SD_OVERLAP) && 85 cpumask_intersects(groupmask, sched_group_span(group))) { 86 printk(KERN_CONT "\n"); 87 printk(KERN_ERR "ERROR: repeated CPUs\n"); 88 break; 89 } 90 91 cpumask_or(groupmask, groupmask, sched_group_span(group)); 92 93 printk(KERN_CONT " %d:{ span=%*pbl", 94 group->sgc->id, 95 cpumask_pr_args(sched_group_span(group))); 96 97 if ((sd->flags & SD_OVERLAP) && 98 !cpumask_equal(group_balance_mask(group), sched_group_span(group))) { 99 printk(KERN_CONT " mask=%*pbl", 100 cpumask_pr_args(group_balance_mask(group))); 101 } 102 103 if (group->sgc->capacity != SCHED_CAPACITY_SCALE) 104 printk(KERN_CONT " cap=%lu", group->sgc->capacity); 105 106 if (group == sd->groups && sd->child && 107 !cpumask_equal(sched_domain_span(sd->child), 108 sched_group_span(group))) { 109 printk(KERN_ERR "ERROR: domain->groups does not match domain->child\n"); 110 } 111 112 printk(KERN_CONT " }"); 113 114 group = group->next; 115 116 if (group != sd->groups) 117 printk(KERN_CONT ","); 118 119 } while (group != sd->groups); 120 printk(KERN_CONT "\n"); 121 122 if (!cpumask_equal(sched_domain_span(sd), groupmask)) 123 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); 124 125 if (sd->parent && 126 !cpumask_subset(groupmask, sched_domain_span(sd->parent))) 127 printk(KERN_ERR "ERROR: parent span is not a superset of domain->span\n"); 128 return 0; 129 } 130 131 static void sched_domain_debug(struct sched_domain *sd, int cpu) 132 { 133 int level = 0; 134 135 if (!sched_debug_verbose) 136 return; 137 138 if (!sd) { 139 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); 140 return; 141 } 142 143 printk(KERN_DEBUG "CPU%d attaching sched-domain(s):\n", cpu); 144 145 for (;;) { 146 if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask)) 147 break; 148 level++; 149 sd = sd->parent; 150 if (!sd) 151 break; 152 } 153 } 154 #else /* !CONFIG_SCHED_DEBUG */ 155 156 # define sched_debug_verbose 0 157 # define sched_domain_debug(sd, cpu) do { } while (0) 158 static inline bool sched_debug(void) 159 { 160 return false; 161 } 162 #endif /* CONFIG_SCHED_DEBUG */ 163 164 /* Generate a mask of SD flags with the SDF_NEEDS_GROUPS metaflag */ 165 #define SD_FLAG(name, mflags) (name * !!((mflags) & SDF_NEEDS_GROUPS)) | 166 static const unsigned int SD_DEGENERATE_GROUPS_MASK = 167 #include <linux/sched/sd_flags.h> 168 0; 169 #undef SD_FLAG 170 171 static int sd_degenerate(struct sched_domain *sd) 172 { 173 if (cpumask_weight(sched_domain_span(sd)) == 1) 174 return 1; 175 176 /* Following flags need at least 2 groups */ 177 if ((sd->flags & SD_DEGENERATE_GROUPS_MASK) && 178 (sd->groups != sd->groups->next)) 179 return 0; 180 181 /* Following flags don't use groups */ 182 if (sd->flags & (SD_WAKE_AFFINE)) 183 return 0; 184 185 return 1; 186 } 187 188 static int 189 sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) 190 { 191 unsigned long cflags = sd->flags, pflags = parent->flags; 192 193 if (sd_degenerate(parent)) 194 return 1; 195 196 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent))) 197 return 0; 198 199 /* Flags needing groups don't count if only 1 group in parent */ 200 if (parent->groups == parent->groups->next) 201 pflags &= ~SD_DEGENERATE_GROUPS_MASK; 202 203 if (~cflags & pflags) 204 return 0; 205 206 return 1; 207 } 208 209 #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) 210 DEFINE_STATIC_KEY_FALSE(sched_energy_present); 211 static unsigned int sysctl_sched_energy_aware = 1; 212 static DEFINE_MUTEX(sched_energy_mutex); 213 static bool sched_energy_update; 214 215 void rebuild_sched_domains_energy(void) 216 { 217 mutex_lock(&sched_energy_mutex); 218 sched_energy_update = true; 219 rebuild_sched_domains(); 220 sched_energy_update = false; 221 mutex_unlock(&sched_energy_mutex); 222 } 223 224 #ifdef CONFIG_PROC_SYSCTL 225 static int sched_energy_aware_handler(struct ctl_table *table, int write, 226 void *buffer, size_t *lenp, loff_t *ppos) 227 { 228 int ret, state; 229 230 if (write && !capable(CAP_SYS_ADMIN)) 231 return -EPERM; 232 233 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 234 if (!ret && write) { 235 state = static_branch_unlikely(&sched_energy_present); 236 if (state != sysctl_sched_energy_aware) 237 rebuild_sched_domains_energy(); 238 } 239 240 return ret; 241 } 242 243 static struct ctl_table sched_energy_aware_sysctls[] = { 244 { 245 .procname = "sched_energy_aware", 246 .data = &sysctl_sched_energy_aware, 247 .maxlen = sizeof(unsigned int), 248 .mode = 0644, 249 .proc_handler = sched_energy_aware_handler, 250 .extra1 = SYSCTL_ZERO, 251 .extra2 = SYSCTL_ONE, 252 }, 253 {} 254 }; 255 256 static int __init sched_energy_aware_sysctl_init(void) 257 { 258 register_sysctl_init("kernel", sched_energy_aware_sysctls); 259 return 0; 260 } 261 262 late_initcall(sched_energy_aware_sysctl_init); 263 #endif 264 265 static void free_pd(struct perf_domain *pd) 266 { 267 struct perf_domain *tmp; 268 269 while (pd) { 270 tmp = pd->next; 271 kfree(pd); 272 pd = tmp; 273 } 274 } 275 276 static struct perf_domain *find_pd(struct perf_domain *pd, int cpu) 277 { 278 while (pd) { 279 if (cpumask_test_cpu(cpu, perf_domain_span(pd))) 280 return pd; 281 pd = pd->next; 282 } 283 284 return NULL; 285 } 286 287 static struct perf_domain *pd_init(int cpu) 288 { 289 struct em_perf_domain *obj = em_cpu_get(cpu); 290 struct perf_domain *pd; 291 292 if (!obj) { 293 if (sched_debug()) 294 pr_info("%s: no EM found for CPU%d\n", __func__, cpu); 295 return NULL; 296 } 297 298 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 299 if (!pd) 300 return NULL; 301 pd->em_pd = obj; 302 303 return pd; 304 } 305 306 static void perf_domain_debug(const struct cpumask *cpu_map, 307 struct perf_domain *pd) 308 { 309 if (!sched_debug() || !pd) 310 return; 311 312 printk(KERN_DEBUG "root_domain %*pbl:", cpumask_pr_args(cpu_map)); 313 314 while (pd) { 315 printk(KERN_CONT " pd%d:{ cpus=%*pbl nr_pstate=%d }", 316 cpumask_first(perf_domain_span(pd)), 317 cpumask_pr_args(perf_domain_span(pd)), 318 em_pd_nr_perf_states(pd->em_pd)); 319 pd = pd->next; 320 } 321 322 printk(KERN_CONT "\n"); 323 } 324 325 static void destroy_perf_domain_rcu(struct rcu_head *rp) 326 { 327 struct perf_domain *pd; 328 329 pd = container_of(rp, struct perf_domain, rcu); 330 free_pd(pd); 331 } 332 333 static void sched_energy_set(bool has_eas) 334 { 335 if (!has_eas && static_branch_unlikely(&sched_energy_present)) { 336 if (sched_debug()) 337 pr_info("%s: stopping EAS\n", __func__); 338 static_branch_disable_cpuslocked(&sched_energy_present); 339 } else if (has_eas && !static_branch_unlikely(&sched_energy_present)) { 340 if (sched_debug()) 341 pr_info("%s: starting EAS\n", __func__); 342 static_branch_enable_cpuslocked(&sched_energy_present); 343 } 344 } 345 346 /* 347 * EAS can be used on a root domain if it meets all the following conditions: 348 * 1. an Energy Model (EM) is available; 349 * 2. the SD_ASYM_CPUCAPACITY flag is set in the sched_domain hierarchy. 350 * 3. no SMT is detected. 351 * 4. the EM complexity is low enough to keep scheduling overheads low; 352 * 5. schedutil is driving the frequency of all CPUs of the rd; 353 * 6. frequency invariance support is present; 354 * 355 * The complexity of the Energy Model is defined as: 356 * 357 * C = nr_pd * (nr_cpus + nr_ps) 358 * 359 * with parameters defined as: 360 * - nr_pd: the number of performance domains 361 * - nr_cpus: the number of CPUs 362 * - nr_ps: the sum of the number of performance states of all performance 363 * domains (for example, on a system with 2 performance domains, 364 * with 10 performance states each, nr_ps = 2 * 10 = 20). 365 * 366 * It is generally not a good idea to use such a model in the wake-up path on 367 * very complex platforms because of the associated scheduling overheads. The 368 * arbitrary constraint below prevents that. It makes EAS usable up to 16 CPUs 369 * with per-CPU DVFS and less than 8 performance states each, for example. 370 */ 371 #define EM_MAX_COMPLEXITY 2048 372 373 extern struct cpufreq_governor schedutil_gov; 374 static bool build_perf_domains(const struct cpumask *cpu_map) 375 { 376 int i, nr_pd = 0, nr_ps = 0, nr_cpus = cpumask_weight(cpu_map); 377 struct perf_domain *pd = NULL, *tmp; 378 int cpu = cpumask_first(cpu_map); 379 struct root_domain *rd = cpu_rq(cpu)->rd; 380 struct cpufreq_policy *policy; 381 struct cpufreq_governor *gov; 382 383 if (!sysctl_sched_energy_aware) 384 goto free; 385 386 /* EAS is enabled for asymmetric CPU capacity topologies. */ 387 if (!per_cpu(sd_asym_cpucapacity, cpu)) { 388 if (sched_debug()) { 389 pr_info("rd %*pbl: CPUs do not have asymmetric capacities\n", 390 cpumask_pr_args(cpu_map)); 391 } 392 goto free; 393 } 394 395 /* EAS definitely does *not* handle SMT */ 396 if (sched_smt_active()) { 397 pr_warn("rd %*pbl: Disabling EAS, SMT is not supported\n", 398 cpumask_pr_args(cpu_map)); 399 goto free; 400 } 401 402 if (!arch_scale_freq_invariant()) { 403 if (sched_debug()) { 404 pr_warn("rd %*pbl: Disabling EAS: frequency-invariant load tracking not yet supported", 405 cpumask_pr_args(cpu_map)); 406 } 407 goto free; 408 } 409 410 for_each_cpu(i, cpu_map) { 411 /* Skip already covered CPUs. */ 412 if (find_pd(pd, i)) 413 continue; 414 415 /* Do not attempt EAS if schedutil is not being used. */ 416 policy = cpufreq_cpu_get(i); 417 if (!policy) 418 goto free; 419 gov = policy->governor; 420 cpufreq_cpu_put(policy); 421 if (gov != &schedutil_gov) { 422 if (rd->pd) 423 pr_warn("rd %*pbl: Disabling EAS, schedutil is mandatory\n", 424 cpumask_pr_args(cpu_map)); 425 goto free; 426 } 427 428 /* Create the new pd and add it to the local list. */ 429 tmp = pd_init(i); 430 if (!tmp) 431 goto free; 432 tmp->next = pd; 433 pd = tmp; 434 435 /* 436 * Count performance domains and performance states for the 437 * complexity check. 438 */ 439 nr_pd++; 440 nr_ps += em_pd_nr_perf_states(pd->em_pd); 441 } 442 443 /* Bail out if the Energy Model complexity is too high. */ 444 if (nr_pd * (nr_ps + nr_cpus) > EM_MAX_COMPLEXITY) { 445 WARN(1, "rd %*pbl: Failed to start EAS, EM complexity is too high\n", 446 cpumask_pr_args(cpu_map)); 447 goto free; 448 } 449 450 perf_domain_debug(cpu_map, pd); 451 452 /* Attach the new list of performance domains to the root domain. */ 453 tmp = rd->pd; 454 rcu_assign_pointer(rd->pd, pd); 455 if (tmp) 456 call_rcu(&tmp->rcu, destroy_perf_domain_rcu); 457 458 return !!pd; 459 460 free: 461 free_pd(pd); 462 tmp = rd->pd; 463 rcu_assign_pointer(rd->pd, NULL); 464 if (tmp) 465 call_rcu(&tmp->rcu, destroy_perf_domain_rcu); 466 467 return false; 468 } 469 #else 470 static void free_pd(struct perf_domain *pd) { } 471 #endif /* CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL*/ 472 473 static void free_rootdomain(struct rcu_head *rcu) 474 { 475 struct root_domain *rd = container_of(rcu, struct root_domain, rcu); 476 477 cpupri_cleanup(&rd->cpupri); 478 cpudl_cleanup(&rd->cpudl); 479 free_cpumask_var(rd->dlo_mask); 480 free_cpumask_var(rd->rto_mask); 481 free_cpumask_var(rd->online); 482 free_cpumask_var(rd->span); 483 free_pd(rd->pd); 484 kfree(rd); 485 } 486 487 void rq_attach_root(struct rq *rq, struct root_domain *rd) 488 { 489 struct root_domain *old_rd = NULL; 490 struct rq_flags rf; 491 492 rq_lock_irqsave(rq, &rf); 493 494 if (rq->rd) { 495 old_rd = rq->rd; 496 497 if (cpumask_test_cpu(rq->cpu, old_rd->online)) 498 set_rq_offline(rq); 499 500 cpumask_clear_cpu(rq->cpu, old_rd->span); 501 502 /* 503 * If we dont want to free the old_rd yet then 504 * set old_rd to NULL to skip the freeing later 505 * in this function: 506 */ 507 if (!atomic_dec_and_test(&old_rd->refcount)) 508 old_rd = NULL; 509 } 510 511 atomic_inc(&rd->refcount); 512 rq->rd = rd; 513 514 cpumask_set_cpu(rq->cpu, rd->span); 515 if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) 516 set_rq_online(rq); 517 518 rq_unlock_irqrestore(rq, &rf); 519 520 if (old_rd) 521 call_rcu(&old_rd->rcu, free_rootdomain); 522 } 523 524 void sched_get_rd(struct root_domain *rd) 525 { 526 atomic_inc(&rd->refcount); 527 } 528 529 void sched_put_rd(struct root_domain *rd) 530 { 531 if (!atomic_dec_and_test(&rd->refcount)) 532 return; 533 534 call_rcu(&rd->rcu, free_rootdomain); 535 } 536 537 static int init_rootdomain(struct root_domain *rd) 538 { 539 if (!zalloc_cpumask_var(&rd->span, GFP_KERNEL)) 540 goto out; 541 if (!zalloc_cpumask_var(&rd->online, GFP_KERNEL)) 542 goto free_span; 543 if (!zalloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL)) 544 goto free_online; 545 if (!zalloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) 546 goto free_dlo_mask; 547 548 #ifdef HAVE_RT_PUSH_IPI 549 rd->rto_cpu = -1; 550 raw_spin_lock_init(&rd->rto_lock); 551 rd->rto_push_work = IRQ_WORK_INIT_HARD(rto_push_irq_work_func); 552 #endif 553 554 rd->visit_gen = 0; 555 init_dl_bw(&rd->dl_bw); 556 if (cpudl_init(&rd->cpudl) != 0) 557 goto free_rto_mask; 558 559 if (cpupri_init(&rd->cpupri) != 0) 560 goto free_cpudl; 561 return 0; 562 563 free_cpudl: 564 cpudl_cleanup(&rd->cpudl); 565 free_rto_mask: 566 free_cpumask_var(rd->rto_mask); 567 free_dlo_mask: 568 free_cpumask_var(rd->dlo_mask); 569 free_online: 570 free_cpumask_var(rd->online); 571 free_span: 572 free_cpumask_var(rd->span); 573 out: 574 return -ENOMEM; 575 } 576 577 /* 578 * By default the system creates a single root-domain with all CPUs as 579 * members (mimicking the global state we have today). 580 */ 581 struct root_domain def_root_domain; 582 583 void __init init_defrootdomain(void) 584 { 585 init_rootdomain(&def_root_domain); 586 587 atomic_set(&def_root_domain.refcount, 1); 588 } 589 590 static struct root_domain *alloc_rootdomain(void) 591 { 592 struct root_domain *rd; 593 594 rd = kzalloc(sizeof(*rd), GFP_KERNEL); 595 if (!rd) 596 return NULL; 597 598 if (init_rootdomain(rd) != 0) { 599 kfree(rd); 600 return NULL; 601 } 602 603 return rd; 604 } 605 606 static void free_sched_groups(struct sched_group *sg, int free_sgc) 607 { 608 struct sched_group *tmp, *first; 609 610 if (!sg) 611 return; 612 613 first = sg; 614 do { 615 tmp = sg->next; 616 617 if (free_sgc && atomic_dec_and_test(&sg->sgc->ref)) 618 kfree(sg->sgc); 619 620 if (atomic_dec_and_test(&sg->ref)) 621 kfree(sg); 622 sg = tmp; 623 } while (sg != first); 624 } 625 626 static void destroy_sched_domain(struct sched_domain *sd) 627 { 628 /* 629 * A normal sched domain may have multiple group references, an 630 * overlapping domain, having private groups, only one. Iterate, 631 * dropping group/capacity references, freeing where none remain. 632 */ 633 free_sched_groups(sd->groups, 1); 634 635 if (sd->shared && atomic_dec_and_test(&sd->shared->ref)) 636 kfree(sd->shared); 637 kfree(sd); 638 } 639 640 static void destroy_sched_domains_rcu(struct rcu_head *rcu) 641 { 642 struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu); 643 644 while (sd) { 645 struct sched_domain *parent = sd->parent; 646 destroy_sched_domain(sd); 647 sd = parent; 648 } 649 } 650 651 static void destroy_sched_domains(struct sched_domain *sd) 652 { 653 if (sd) 654 call_rcu(&sd->rcu, destroy_sched_domains_rcu); 655 } 656 657 /* 658 * Keep a special pointer to the highest sched_domain that has 659 * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this 660 * allows us to avoid some pointer chasing select_idle_sibling(). 661 * 662 * Also keep a unique ID per domain (we use the first CPU number in 663 * the cpumask of the domain), this allows us to quickly tell if 664 * two CPUs are in the same cache domain, see cpus_share_cache(). 665 */ 666 DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc); 667 DEFINE_PER_CPU(int, sd_llc_size); 668 DEFINE_PER_CPU(int, sd_llc_id); 669 DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); 670 DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); 671 DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); 672 DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); 673 DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity); 674 675 static void update_top_cache_domain(int cpu) 676 { 677 struct sched_domain_shared *sds = NULL; 678 struct sched_domain *sd; 679 int id = cpu; 680 int size = 1; 681 682 sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES); 683 if (sd) { 684 id = cpumask_first(sched_domain_span(sd)); 685 size = cpumask_weight(sched_domain_span(sd)); 686 sds = sd->shared; 687 } 688 689 rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); 690 per_cpu(sd_llc_size, cpu) = size; 691 per_cpu(sd_llc_id, cpu) = id; 692 rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); 693 694 sd = lowest_flag_domain(cpu, SD_NUMA); 695 rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); 696 697 sd = highest_flag_domain(cpu, SD_ASYM_PACKING); 698 rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd); 699 700 sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL); 701 rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd); 702 } 703 704 /* 705 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must 706 * hold the hotplug lock. 707 */ 708 static void 709 cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) 710 { 711 struct rq *rq = cpu_rq(cpu); 712 struct sched_domain *tmp; 713 714 /* Remove the sched domains which do not contribute to scheduling. */ 715 for (tmp = sd; tmp; ) { 716 struct sched_domain *parent = tmp->parent; 717 if (!parent) 718 break; 719 720 if (sd_parent_degenerate(tmp, parent)) { 721 tmp->parent = parent->parent; 722 723 if (parent->parent) { 724 parent->parent->child = tmp; 725 if (tmp->flags & SD_SHARE_CPUCAPACITY) 726 parent->parent->groups->flags |= SD_SHARE_CPUCAPACITY; 727 } 728 729 /* 730 * Transfer SD_PREFER_SIBLING down in case of a 731 * degenerate parent; the spans match for this 732 * so the property transfers. 733 */ 734 if (parent->flags & SD_PREFER_SIBLING) 735 tmp->flags |= SD_PREFER_SIBLING; 736 destroy_sched_domain(parent); 737 } else 738 tmp = tmp->parent; 739 } 740 741 if (sd && sd_degenerate(sd)) { 742 tmp = sd; 743 sd = sd->parent; 744 destroy_sched_domain(tmp); 745 if (sd) { 746 struct sched_group *sg = sd->groups; 747 748 /* 749 * sched groups hold the flags of the child sched 750 * domain for convenience. Clear such flags since 751 * the child is being destroyed. 752 */ 753 do { 754 sg->flags = 0; 755 } while (sg != sd->groups); 756 757 sd->child = NULL; 758 } 759 } 760 761 sched_domain_debug(sd, cpu); 762 763 rq_attach_root(rq, rd); 764 tmp = rq->sd; 765 rcu_assign_pointer(rq->sd, sd); 766 dirty_sched_domain_sysctl(cpu); 767 destroy_sched_domains(tmp); 768 769 update_top_cache_domain(cpu); 770 } 771 772 struct s_data { 773 struct sched_domain * __percpu *sd; 774 struct root_domain *rd; 775 }; 776 777 enum s_alloc { 778 sa_rootdomain, 779 sa_sd, 780 sa_sd_storage, 781 sa_none, 782 }; 783 784 /* 785 * Return the canonical balance CPU for this group, this is the first CPU 786 * of this group that's also in the balance mask. 787 * 788 * The balance mask are all those CPUs that could actually end up at this 789 * group. See build_balance_mask(). 790 * 791 * Also see should_we_balance(). 792 */ 793 int group_balance_cpu(struct sched_group *sg) 794 { 795 return cpumask_first(group_balance_mask(sg)); 796 } 797 798 799 /* 800 * NUMA topology (first read the regular topology blurb below) 801 * 802 * Given a node-distance table, for example: 803 * 804 * node 0 1 2 3 805 * 0: 10 20 30 20 806 * 1: 20 10 20 30 807 * 2: 30 20 10 20 808 * 3: 20 30 20 10 809 * 810 * which represents a 4 node ring topology like: 811 * 812 * 0 ----- 1 813 * | | 814 * | | 815 * | | 816 * 3 ----- 2 817 * 818 * We want to construct domains and groups to represent this. The way we go 819 * about doing this is to build the domains on 'hops'. For each NUMA level we 820 * construct the mask of all nodes reachable in @level hops. 821 * 822 * For the above NUMA topology that gives 3 levels: 823 * 824 * NUMA-2 0-3 0-3 0-3 0-3 825 * groups: {0-1,3},{1-3} {0-2},{0,2-3} {1-3},{0-1,3} {0,2-3},{0-2} 826 * 827 * NUMA-1 0-1,3 0-2 1-3 0,2-3 828 * groups: {0},{1},{3} {0},{1},{2} {1},{2},{3} {0},{2},{3} 829 * 830 * NUMA-0 0 1 2 3 831 * 832 * 833 * As can be seen; things don't nicely line up as with the regular topology. 834 * When we iterate a domain in child domain chunks some nodes can be 835 * represented multiple times -- hence the "overlap" naming for this part of 836 * the topology. 837 * 838 * In order to minimize this overlap, we only build enough groups to cover the 839 * domain. For instance Node-0 NUMA-2 would only get groups: 0-1,3 and 1-3. 840 * 841 * Because: 842 * 843 * - the first group of each domain is its child domain; this 844 * gets us the first 0-1,3 845 * - the only uncovered node is 2, who's child domain is 1-3. 846 * 847 * However, because of the overlap, computing a unique CPU for each group is 848 * more complicated. Consider for instance the groups of NODE-1 NUMA-2, both 849 * groups include the CPUs of Node-0, while those CPUs would not in fact ever 850 * end up at those groups (they would end up in group: 0-1,3). 851 * 852 * To correct this we have to introduce the group balance mask. This mask 853 * will contain those CPUs in the group that can reach this group given the 854 * (child) domain tree. 855 * 856 * With this we can once again compute balance_cpu and sched_group_capacity 857 * relations. 858 * 859 * XXX include words on how balance_cpu is unique and therefore can be 860 * used for sched_group_capacity links. 861 * 862 * 863 * Another 'interesting' topology is: 864 * 865 * node 0 1 2 3 866 * 0: 10 20 20 30 867 * 1: 20 10 20 20 868 * 2: 20 20 10 20 869 * 3: 30 20 20 10 870 * 871 * Which looks a little like: 872 * 873 * 0 ----- 1 874 * | / | 875 * | / | 876 * | / | 877 * 2 ----- 3 878 * 879 * This topology is asymmetric, nodes 1,2 are fully connected, but nodes 0,3 880 * are not. 881 * 882 * This leads to a few particularly weird cases where the sched_domain's are 883 * not of the same number for each CPU. Consider: 884 * 885 * NUMA-2 0-3 0-3 886 * groups: {0-2},{1-3} {1-3},{0-2} 887 * 888 * NUMA-1 0-2 0-3 0-3 1-3 889 * 890 * NUMA-0 0 1 2 3 891 * 892 */ 893 894 895 /* 896 * Build the balance mask; it contains only those CPUs that can arrive at this 897 * group and should be considered to continue balancing. 898 * 899 * We do this during the group creation pass, therefore the group information 900 * isn't complete yet, however since each group represents a (child) domain we 901 * can fully construct this using the sched_domain bits (which are already 902 * complete). 903 */ 904 static void 905 build_balance_mask(struct sched_domain *sd, struct sched_group *sg, struct cpumask *mask) 906 { 907 const struct cpumask *sg_span = sched_group_span(sg); 908 struct sd_data *sdd = sd->private; 909 struct sched_domain *sibling; 910 int i; 911 912 cpumask_clear(mask); 913 914 for_each_cpu(i, sg_span) { 915 sibling = *per_cpu_ptr(sdd->sd, i); 916 917 /* 918 * Can happen in the asymmetric case, where these siblings are 919 * unused. The mask will not be empty because those CPUs that 920 * do have the top domain _should_ span the domain. 921 */ 922 if (!sibling->child) 923 continue; 924 925 /* If we would not end up here, we can't continue from here */ 926 if (!cpumask_equal(sg_span, sched_domain_span(sibling->child))) 927 continue; 928 929 cpumask_set_cpu(i, mask); 930 } 931 932 /* We must not have empty masks here */ 933 WARN_ON_ONCE(cpumask_empty(mask)); 934 } 935 936 /* 937 * XXX: This creates per-node group entries; since the load-balancer will 938 * immediately access remote memory to construct this group's load-balance 939 * statistics having the groups node local is of dubious benefit. 940 */ 941 static struct sched_group * 942 build_group_from_child_sched_domain(struct sched_domain *sd, int cpu) 943 { 944 struct sched_group *sg; 945 struct cpumask *sg_span; 946 947 sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), 948 GFP_KERNEL, cpu_to_node(cpu)); 949 950 if (!sg) 951 return NULL; 952 953 sg_span = sched_group_span(sg); 954 if (sd->child) { 955 cpumask_copy(sg_span, sched_domain_span(sd->child)); 956 sg->flags = sd->child->flags; 957 } else { 958 cpumask_copy(sg_span, sched_domain_span(sd)); 959 } 960 961 atomic_inc(&sg->ref); 962 return sg; 963 } 964 965 static void init_overlap_sched_group(struct sched_domain *sd, 966 struct sched_group *sg) 967 { 968 struct cpumask *mask = sched_domains_tmpmask2; 969 struct sd_data *sdd = sd->private; 970 struct cpumask *sg_span; 971 int cpu; 972 973 build_balance_mask(sd, sg, mask); 974 cpu = cpumask_first(mask); 975 976 sg->sgc = *per_cpu_ptr(sdd->sgc, cpu); 977 if (atomic_inc_return(&sg->sgc->ref) == 1) 978 cpumask_copy(group_balance_mask(sg), mask); 979 else 980 WARN_ON_ONCE(!cpumask_equal(group_balance_mask(sg), mask)); 981 982 /* 983 * Initialize sgc->capacity such that even if we mess up the 984 * domains and no possible iteration will get us here, we won't 985 * die on a /0 trap. 986 */ 987 sg_span = sched_group_span(sg); 988 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); 989 sg->sgc->min_capacity = SCHED_CAPACITY_SCALE; 990 sg->sgc->max_capacity = SCHED_CAPACITY_SCALE; 991 } 992 993 static struct sched_domain * 994 find_descended_sibling(struct sched_domain *sd, struct sched_domain *sibling) 995 { 996 /* 997 * The proper descendant would be the one whose child won't span out 998 * of sd 999 */ 1000 while (sibling->child && 1001 !cpumask_subset(sched_domain_span(sibling->child), 1002 sched_domain_span(sd))) 1003 sibling = sibling->child; 1004 1005 /* 1006 * As we are referencing sgc across different topology level, we need 1007 * to go down to skip those sched_domains which don't contribute to 1008 * scheduling because they will be degenerated in cpu_attach_domain 1009 */ 1010 while (sibling->child && 1011 cpumask_equal(sched_domain_span(sibling->child), 1012 sched_domain_span(sibling))) 1013 sibling = sibling->child; 1014 1015 return sibling; 1016 } 1017 1018 static int 1019 build_overlap_sched_groups(struct sched_domain *sd, int cpu) 1020 { 1021 struct sched_group *first = NULL, *last = NULL, *sg; 1022 const struct cpumask *span = sched_domain_span(sd); 1023 struct cpumask *covered = sched_domains_tmpmask; 1024 struct sd_data *sdd = sd->private; 1025 struct sched_domain *sibling; 1026 int i; 1027 1028 cpumask_clear(covered); 1029 1030 for_each_cpu_wrap(i, span, cpu) { 1031 struct cpumask *sg_span; 1032 1033 if (cpumask_test_cpu(i, covered)) 1034 continue; 1035 1036 sibling = *per_cpu_ptr(sdd->sd, i); 1037 1038 /* 1039 * Asymmetric node setups can result in situations where the 1040 * domain tree is of unequal depth, make sure to skip domains 1041 * that already cover the entire range. 1042 * 1043 * In that case build_sched_domains() will have terminated the 1044 * iteration early and our sibling sd spans will be empty. 1045 * Domains should always include the CPU they're built on, so 1046 * check that. 1047 */ 1048 if (!cpumask_test_cpu(i, sched_domain_span(sibling))) 1049 continue; 1050 1051 /* 1052 * Usually we build sched_group by sibling's child sched_domain 1053 * But for machines whose NUMA diameter are 3 or above, we move 1054 * to build sched_group by sibling's proper descendant's child 1055 * domain because sibling's child sched_domain will span out of 1056 * the sched_domain being built as below. 1057 * 1058 * Smallest diameter=3 topology is: 1059 * 1060 * node 0 1 2 3 1061 * 0: 10 20 30 40 1062 * 1: 20 10 20 30 1063 * 2: 30 20 10 20 1064 * 3: 40 30 20 10 1065 * 1066 * 0 --- 1 --- 2 --- 3 1067 * 1068 * NUMA-3 0-3 N/A N/A 0-3 1069 * groups: {0-2},{1-3} {1-3},{0-2} 1070 * 1071 * NUMA-2 0-2 0-3 0-3 1-3 1072 * groups: {0-1},{1-3} {0-2},{2-3} {1-3},{0-1} {2-3},{0-2} 1073 * 1074 * NUMA-1 0-1 0-2 1-3 2-3 1075 * groups: {0},{1} {1},{2},{0} {2},{3},{1} {3},{2} 1076 * 1077 * NUMA-0 0 1 2 3 1078 * 1079 * The NUMA-2 groups for nodes 0 and 3 are obviously buggered, as the 1080 * group span isn't a subset of the domain span. 1081 */ 1082 if (sibling->child && 1083 !cpumask_subset(sched_domain_span(sibling->child), span)) 1084 sibling = find_descended_sibling(sd, sibling); 1085 1086 sg = build_group_from_child_sched_domain(sibling, cpu); 1087 if (!sg) 1088 goto fail; 1089 1090 sg_span = sched_group_span(sg); 1091 cpumask_or(covered, covered, sg_span); 1092 1093 init_overlap_sched_group(sibling, sg); 1094 1095 if (!first) 1096 first = sg; 1097 if (last) 1098 last->next = sg; 1099 last = sg; 1100 last->next = first; 1101 } 1102 sd->groups = first; 1103 1104 return 0; 1105 1106 fail: 1107 free_sched_groups(first, 0); 1108 1109 return -ENOMEM; 1110 } 1111 1112 1113 /* 1114 * Package topology (also see the load-balance blurb in fair.c) 1115 * 1116 * The scheduler builds a tree structure to represent a number of important 1117 * topology features. By default (default_topology[]) these include: 1118 * 1119 * - Simultaneous multithreading (SMT) 1120 * - Multi-Core Cache (MC) 1121 * - Package (DIE) 1122 * 1123 * Where the last one more or less denotes everything up to a NUMA node. 1124 * 1125 * The tree consists of 3 primary data structures: 1126 * 1127 * sched_domain -> sched_group -> sched_group_capacity 1128 * ^ ^ ^ ^ 1129 * `-' `-' 1130 * 1131 * The sched_domains are per-CPU and have a two way link (parent & child) and 1132 * denote the ever growing mask of CPUs belonging to that level of topology. 1133 * 1134 * Each sched_domain has a circular (double) linked list of sched_group's, each 1135 * denoting the domains of the level below (or individual CPUs in case of the 1136 * first domain level). The sched_group linked by a sched_domain includes the 1137 * CPU of that sched_domain [*]. 1138 * 1139 * Take for instance a 2 threaded, 2 core, 2 cache cluster part: 1140 * 1141 * CPU 0 1 2 3 4 5 6 7 1142 * 1143 * DIE [ ] 1144 * MC [ ] [ ] 1145 * SMT [ ] [ ] [ ] [ ] 1146 * 1147 * - or - 1148 * 1149 * DIE 0-7 0-7 0-7 0-7 0-7 0-7 0-7 0-7 1150 * MC 0-3 0-3 0-3 0-3 4-7 4-7 4-7 4-7 1151 * SMT 0-1 0-1 2-3 2-3 4-5 4-5 6-7 6-7 1152 * 1153 * CPU 0 1 2 3 4 5 6 7 1154 * 1155 * One way to think about it is: sched_domain moves you up and down among these 1156 * topology levels, while sched_group moves you sideways through it, at child 1157 * domain granularity. 1158 * 1159 * sched_group_capacity ensures each unique sched_group has shared storage. 1160 * 1161 * There are two related construction problems, both require a CPU that 1162 * uniquely identify each group (for a given domain): 1163 * 1164 * - The first is the balance_cpu (see should_we_balance() and the 1165 * load-balance blub in fair.c); for each group we only want 1 CPU to 1166 * continue balancing at a higher domain. 1167 * 1168 * - The second is the sched_group_capacity; we want all identical groups 1169 * to share a single sched_group_capacity. 1170 * 1171 * Since these topologies are exclusive by construction. That is, its 1172 * impossible for an SMT thread to belong to multiple cores, and cores to 1173 * be part of multiple caches. There is a very clear and unique location 1174 * for each CPU in the hierarchy. 1175 * 1176 * Therefore computing a unique CPU for each group is trivial (the iteration 1177 * mask is redundant and set all 1s; all CPUs in a group will end up at _that_ 1178 * group), we can simply pick the first CPU in each group. 1179 * 1180 * 1181 * [*] in other words, the first group of each domain is its child domain. 1182 */ 1183 1184 static struct sched_group *get_group(int cpu, struct sd_data *sdd) 1185 { 1186 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); 1187 struct sched_domain *child = sd->child; 1188 struct sched_group *sg; 1189 bool already_visited; 1190 1191 if (child) 1192 cpu = cpumask_first(sched_domain_span(child)); 1193 1194 sg = *per_cpu_ptr(sdd->sg, cpu); 1195 sg->sgc = *per_cpu_ptr(sdd->sgc, cpu); 1196 1197 /* Increase refcounts for claim_allocations: */ 1198 already_visited = atomic_inc_return(&sg->ref) > 1; 1199 /* sgc visits should follow a similar trend as sg */ 1200 WARN_ON(already_visited != (atomic_inc_return(&sg->sgc->ref) > 1)); 1201 1202 /* If we have already visited that group, it's already initialized. */ 1203 if (already_visited) 1204 return sg; 1205 1206 if (child) { 1207 cpumask_copy(sched_group_span(sg), sched_domain_span(child)); 1208 cpumask_copy(group_balance_mask(sg), sched_group_span(sg)); 1209 sg->flags = child->flags; 1210 } else { 1211 cpumask_set_cpu(cpu, sched_group_span(sg)); 1212 cpumask_set_cpu(cpu, group_balance_mask(sg)); 1213 } 1214 1215 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sched_group_span(sg)); 1216 sg->sgc->min_capacity = SCHED_CAPACITY_SCALE; 1217 sg->sgc->max_capacity = SCHED_CAPACITY_SCALE; 1218 1219 return sg; 1220 } 1221 1222 /* 1223 * build_sched_groups will build a circular linked list of the groups 1224 * covered by the given span, will set each group's ->cpumask correctly, 1225 * and will initialize their ->sgc. 1226 * 1227 * Assumes the sched_domain tree is fully constructed 1228 */ 1229 static int 1230 build_sched_groups(struct sched_domain *sd, int cpu) 1231 { 1232 struct sched_group *first = NULL, *last = NULL; 1233 struct sd_data *sdd = sd->private; 1234 const struct cpumask *span = sched_domain_span(sd); 1235 struct cpumask *covered; 1236 int i; 1237 1238 lockdep_assert_held(&sched_domains_mutex); 1239 covered = sched_domains_tmpmask; 1240 1241 cpumask_clear(covered); 1242 1243 for_each_cpu_wrap(i, span, cpu) { 1244 struct sched_group *sg; 1245 1246 if (cpumask_test_cpu(i, covered)) 1247 continue; 1248 1249 sg = get_group(i, sdd); 1250 1251 cpumask_or(covered, covered, sched_group_span(sg)); 1252 1253 if (!first) 1254 first = sg; 1255 if (last) 1256 last->next = sg; 1257 last = sg; 1258 } 1259 last->next = first; 1260 sd->groups = first; 1261 1262 return 0; 1263 } 1264 1265 /* 1266 * Initialize sched groups cpu_capacity. 1267 * 1268 * cpu_capacity indicates the capacity of sched group, which is used while 1269 * distributing the load between different sched groups in a sched domain. 1270 * Typically cpu_capacity for all the groups in a sched domain will be same 1271 * unless there are asymmetries in the topology. If there are asymmetries, 1272 * group having more cpu_capacity will pickup more load compared to the 1273 * group having less cpu_capacity. 1274 */ 1275 static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) 1276 { 1277 struct sched_group *sg = sd->groups; 1278 struct cpumask *mask = sched_domains_tmpmask2; 1279 1280 WARN_ON(!sg); 1281 1282 do { 1283 int cpu, cores = 0, max_cpu = -1; 1284 1285 sg->group_weight = cpumask_weight(sched_group_span(sg)); 1286 1287 cpumask_copy(mask, sched_group_span(sg)); 1288 for_each_cpu(cpu, mask) { 1289 cores++; 1290 #ifdef CONFIG_SCHED_SMT 1291 cpumask_andnot(mask, mask, cpu_smt_mask(cpu)); 1292 #endif 1293 } 1294 sg->cores = cores; 1295 1296 if (!(sd->flags & SD_ASYM_PACKING)) 1297 goto next; 1298 1299 for_each_cpu(cpu, sched_group_span(sg)) { 1300 if (max_cpu < 0) 1301 max_cpu = cpu; 1302 else if (sched_asym_prefer(cpu, max_cpu)) 1303 max_cpu = cpu; 1304 } 1305 sg->asym_prefer_cpu = max_cpu; 1306 1307 next: 1308 sg = sg->next; 1309 } while (sg != sd->groups); 1310 1311 if (cpu != group_balance_cpu(sg)) 1312 return; 1313 1314 update_group_capacity(sd, cpu); 1315 } 1316 1317 /* 1318 * Asymmetric CPU capacity bits 1319 */ 1320 struct asym_cap_data { 1321 struct list_head link; 1322 unsigned long capacity; 1323 unsigned long cpus[]; 1324 }; 1325 1326 /* 1327 * Set of available CPUs grouped by their corresponding capacities 1328 * Each list entry contains a CPU mask reflecting CPUs that share the same 1329 * capacity. 1330 * The lifespan of data is unlimited. 1331 */ 1332 static LIST_HEAD(asym_cap_list); 1333 1334 #define cpu_capacity_span(asym_data) to_cpumask((asym_data)->cpus) 1335 1336 /* 1337 * Verify whether there is any CPU capacity asymmetry in a given sched domain. 1338 * Provides sd_flags reflecting the asymmetry scope. 1339 */ 1340 static inline int 1341 asym_cpu_capacity_classify(const struct cpumask *sd_span, 1342 const struct cpumask *cpu_map) 1343 { 1344 struct asym_cap_data *entry; 1345 int count = 0, miss = 0; 1346 1347 /* 1348 * Count how many unique CPU capacities this domain spans across 1349 * (compare sched_domain CPUs mask with ones representing available 1350 * CPUs capacities). Take into account CPUs that might be offline: 1351 * skip those. 1352 */ 1353 list_for_each_entry(entry, &asym_cap_list, link) { 1354 if (cpumask_intersects(sd_span, cpu_capacity_span(entry))) 1355 ++count; 1356 else if (cpumask_intersects(cpu_map, cpu_capacity_span(entry))) 1357 ++miss; 1358 } 1359 1360 WARN_ON_ONCE(!count && !list_empty(&asym_cap_list)); 1361 1362 /* No asymmetry detected */ 1363 if (count < 2) 1364 return 0; 1365 /* Some of the available CPU capacity values have not been detected */ 1366 if (miss) 1367 return SD_ASYM_CPUCAPACITY; 1368 1369 /* Full asymmetry */ 1370 return SD_ASYM_CPUCAPACITY | SD_ASYM_CPUCAPACITY_FULL; 1371 1372 } 1373 1374 static inline void asym_cpu_capacity_update_data(int cpu) 1375 { 1376 unsigned long capacity = arch_scale_cpu_capacity(cpu); 1377 struct asym_cap_data *entry = NULL; 1378 1379 list_for_each_entry(entry, &asym_cap_list, link) { 1380 if (capacity == entry->capacity) 1381 goto done; 1382 } 1383 1384 entry = kzalloc(sizeof(*entry) + cpumask_size(), GFP_KERNEL); 1385 if (WARN_ONCE(!entry, "Failed to allocate memory for asymmetry data\n")) 1386 return; 1387 entry->capacity = capacity; 1388 list_add(&entry->link, &asym_cap_list); 1389 done: 1390 __cpumask_set_cpu(cpu, cpu_capacity_span(entry)); 1391 } 1392 1393 /* 1394 * Build-up/update list of CPUs grouped by their capacities 1395 * An update requires explicit request to rebuild sched domains 1396 * with state indicating CPU topology changes. 1397 */ 1398 static void asym_cpu_capacity_scan(void) 1399 { 1400 struct asym_cap_data *entry, *next; 1401 int cpu; 1402 1403 list_for_each_entry(entry, &asym_cap_list, link) 1404 cpumask_clear(cpu_capacity_span(entry)); 1405 1406 for_each_cpu_and(cpu, cpu_possible_mask, housekeeping_cpumask(HK_TYPE_DOMAIN)) 1407 asym_cpu_capacity_update_data(cpu); 1408 1409 list_for_each_entry_safe(entry, next, &asym_cap_list, link) { 1410 if (cpumask_empty(cpu_capacity_span(entry))) { 1411 list_del(&entry->link); 1412 kfree(entry); 1413 } 1414 } 1415 1416 /* 1417 * Only one capacity value has been detected i.e. this system is symmetric. 1418 * No need to keep this data around. 1419 */ 1420 if (list_is_singular(&asym_cap_list)) { 1421 entry = list_first_entry(&asym_cap_list, typeof(*entry), link); 1422 list_del(&entry->link); 1423 kfree(entry); 1424 } 1425 } 1426 1427 /* 1428 * Initializers for schedule domains 1429 * Non-inlined to reduce accumulated stack pressure in build_sched_domains() 1430 */ 1431 1432 static int default_relax_domain_level = -1; 1433 int sched_domain_level_max; 1434 1435 static int __init setup_relax_domain_level(char *str) 1436 { 1437 if (kstrtoint(str, 0, &default_relax_domain_level)) 1438 pr_warn("Unable to set relax_domain_level\n"); 1439 1440 return 1; 1441 } 1442 __setup("relax_domain_level=", setup_relax_domain_level); 1443 1444 static void set_domain_attribute(struct sched_domain *sd, 1445 struct sched_domain_attr *attr) 1446 { 1447 int request; 1448 1449 if (!attr || attr->relax_domain_level < 0) { 1450 if (default_relax_domain_level < 0) 1451 return; 1452 request = default_relax_domain_level; 1453 } else 1454 request = attr->relax_domain_level; 1455 1456 if (sd->level > request) { 1457 /* Turn off idle balance on this domain: */ 1458 sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); 1459 } 1460 } 1461 1462 static void __sdt_free(const struct cpumask *cpu_map); 1463 static int __sdt_alloc(const struct cpumask *cpu_map); 1464 1465 static void __free_domain_allocs(struct s_data *d, enum s_alloc what, 1466 const struct cpumask *cpu_map) 1467 { 1468 switch (what) { 1469 case sa_rootdomain: 1470 if (!atomic_read(&d->rd->refcount)) 1471 free_rootdomain(&d->rd->rcu); 1472 fallthrough; 1473 case sa_sd: 1474 free_percpu(d->sd); 1475 fallthrough; 1476 case sa_sd_storage: 1477 __sdt_free(cpu_map); 1478 fallthrough; 1479 case sa_none: 1480 break; 1481 } 1482 } 1483 1484 static enum s_alloc 1485 __visit_domain_allocation_hell(struct s_data *d, const struct cpumask *cpu_map) 1486 { 1487 memset(d, 0, sizeof(*d)); 1488 1489 if (__sdt_alloc(cpu_map)) 1490 return sa_sd_storage; 1491 d->sd = alloc_percpu(struct sched_domain *); 1492 if (!d->sd) 1493 return sa_sd_storage; 1494 d->rd = alloc_rootdomain(); 1495 if (!d->rd) 1496 return sa_sd; 1497 1498 return sa_rootdomain; 1499 } 1500 1501 /* 1502 * NULL the sd_data elements we've used to build the sched_domain and 1503 * sched_group structure so that the subsequent __free_domain_allocs() 1504 * will not free the data we're using. 1505 */ 1506 static void claim_allocations(int cpu, struct sched_domain *sd) 1507 { 1508 struct sd_data *sdd = sd->private; 1509 1510 WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd); 1511 *per_cpu_ptr(sdd->sd, cpu) = NULL; 1512 1513 if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref)) 1514 *per_cpu_ptr(sdd->sds, cpu) = NULL; 1515 1516 if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) 1517 *per_cpu_ptr(sdd->sg, cpu) = NULL; 1518 1519 if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref)) 1520 *per_cpu_ptr(sdd->sgc, cpu) = NULL; 1521 } 1522 1523 #ifdef CONFIG_NUMA 1524 enum numa_topology_type sched_numa_topology_type; 1525 1526 static int sched_domains_numa_levels; 1527 static int sched_domains_curr_level; 1528 1529 int sched_max_numa_distance; 1530 static int *sched_domains_numa_distance; 1531 static struct cpumask ***sched_domains_numa_masks; 1532 #endif 1533 1534 /* 1535 * SD_flags allowed in topology descriptions. 1536 * 1537 * These flags are purely descriptive of the topology and do not prescribe 1538 * behaviour. Behaviour is artificial and mapped in the below sd_init() 1539 * function: 1540 * 1541 * SD_SHARE_CPUCAPACITY - describes SMT topologies 1542 * SD_SHARE_PKG_RESOURCES - describes shared caches 1543 * SD_NUMA - describes NUMA topologies 1544 * 1545 * Odd one out, which beside describing the topology has a quirk also 1546 * prescribes the desired behaviour that goes along with it: 1547 * 1548 * SD_ASYM_PACKING - describes SMT quirks 1549 */ 1550 #define TOPOLOGY_SD_FLAGS \ 1551 (SD_SHARE_CPUCAPACITY | \ 1552 SD_SHARE_PKG_RESOURCES | \ 1553 SD_NUMA | \ 1554 SD_ASYM_PACKING) 1555 1556 static struct sched_domain * 1557 sd_init(struct sched_domain_topology_level *tl, 1558 const struct cpumask *cpu_map, 1559 struct sched_domain *child, int cpu) 1560 { 1561 struct sd_data *sdd = &tl->data; 1562 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); 1563 int sd_id, sd_weight, sd_flags = 0; 1564 struct cpumask *sd_span; 1565 1566 #ifdef CONFIG_NUMA 1567 /* 1568 * Ugly hack to pass state to sd_numa_mask()... 1569 */ 1570 sched_domains_curr_level = tl->numa_level; 1571 #endif 1572 1573 sd_weight = cpumask_weight(tl->mask(cpu)); 1574 1575 if (tl->sd_flags) 1576 sd_flags = (*tl->sd_flags)(); 1577 if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS, 1578 "wrong sd_flags in topology description\n")) 1579 sd_flags &= TOPOLOGY_SD_FLAGS; 1580 1581 *sd = (struct sched_domain){ 1582 .min_interval = sd_weight, 1583 .max_interval = 2*sd_weight, 1584 .busy_factor = 16, 1585 .imbalance_pct = 117, 1586 1587 .cache_nice_tries = 0, 1588 1589 .flags = 1*SD_BALANCE_NEWIDLE 1590 | 1*SD_BALANCE_EXEC 1591 | 1*SD_BALANCE_FORK 1592 | 0*SD_BALANCE_WAKE 1593 | 1*SD_WAKE_AFFINE 1594 | 0*SD_SHARE_CPUCAPACITY 1595 | 0*SD_SHARE_PKG_RESOURCES 1596 | 0*SD_SERIALIZE 1597 | 1*SD_PREFER_SIBLING 1598 | 0*SD_NUMA 1599 | sd_flags 1600 , 1601 1602 .last_balance = jiffies, 1603 .balance_interval = sd_weight, 1604 .max_newidle_lb_cost = 0, 1605 .last_decay_max_lb_cost = jiffies, 1606 .child = child, 1607 #ifdef CONFIG_SCHED_DEBUG 1608 .name = tl->name, 1609 #endif 1610 }; 1611 1612 sd_span = sched_domain_span(sd); 1613 cpumask_and(sd_span, cpu_map, tl->mask(cpu)); 1614 sd_id = cpumask_first(sd_span); 1615 1616 sd->flags |= asym_cpu_capacity_classify(sd_span, cpu_map); 1617 1618 WARN_ONCE((sd->flags & (SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY)) == 1619 (SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY), 1620 "CPU capacity asymmetry not supported on SMT\n"); 1621 1622 /* 1623 * Convert topological properties into behaviour. 1624 */ 1625 /* Don't attempt to spread across CPUs of different capacities. */ 1626 if ((sd->flags & SD_ASYM_CPUCAPACITY) && sd->child) 1627 sd->child->flags &= ~SD_PREFER_SIBLING; 1628 1629 if (sd->flags & SD_SHARE_CPUCAPACITY) { 1630 sd->imbalance_pct = 110; 1631 1632 } else if (sd->flags & SD_SHARE_PKG_RESOURCES) { 1633 sd->imbalance_pct = 117; 1634 sd->cache_nice_tries = 1; 1635 1636 #ifdef CONFIG_NUMA 1637 } else if (sd->flags & SD_NUMA) { 1638 sd->cache_nice_tries = 2; 1639 1640 sd->flags &= ~SD_PREFER_SIBLING; 1641 sd->flags |= SD_SERIALIZE; 1642 if (sched_domains_numa_distance[tl->numa_level] > node_reclaim_distance) { 1643 sd->flags &= ~(SD_BALANCE_EXEC | 1644 SD_BALANCE_FORK | 1645 SD_WAKE_AFFINE); 1646 } 1647 1648 #endif 1649 } else { 1650 sd->cache_nice_tries = 1; 1651 } 1652 1653 /* 1654 * For all levels sharing cache; connect a sched_domain_shared 1655 * instance. 1656 */ 1657 if (sd->flags & SD_SHARE_PKG_RESOURCES) { 1658 sd->shared = *per_cpu_ptr(sdd->sds, sd_id); 1659 atomic_inc(&sd->shared->ref); 1660 atomic_set(&sd->shared->nr_busy_cpus, sd_weight); 1661 } 1662 1663 sd->private = sdd; 1664 1665 return sd; 1666 } 1667 1668 /* 1669 * Topology list, bottom-up. 1670 */ 1671 static struct sched_domain_topology_level default_topology[] = { 1672 #ifdef CONFIG_SCHED_SMT 1673 { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, 1674 #endif 1675 1676 #ifdef CONFIG_SCHED_CLUSTER 1677 { cpu_clustergroup_mask, cpu_cluster_flags, SD_INIT_NAME(CLS) }, 1678 #endif 1679 1680 #ifdef CONFIG_SCHED_MC 1681 { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, 1682 #endif 1683 { cpu_cpu_mask, SD_INIT_NAME(DIE) }, 1684 { NULL, }, 1685 }; 1686 1687 static struct sched_domain_topology_level *sched_domain_topology = 1688 default_topology; 1689 static struct sched_domain_topology_level *sched_domain_topology_saved; 1690 1691 #define for_each_sd_topology(tl) \ 1692 for (tl = sched_domain_topology; tl->mask; tl++) 1693 1694 void __init set_sched_topology(struct sched_domain_topology_level *tl) 1695 { 1696 if (WARN_ON_ONCE(sched_smp_initialized)) 1697 return; 1698 1699 sched_domain_topology = tl; 1700 sched_domain_topology_saved = NULL; 1701 } 1702 1703 #ifdef CONFIG_NUMA 1704 1705 static const struct cpumask *sd_numa_mask(int cpu) 1706 { 1707 return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)]; 1708 } 1709 1710 static void sched_numa_warn(const char *str) 1711 { 1712 static int done = false; 1713 int i,j; 1714 1715 if (done) 1716 return; 1717 1718 done = true; 1719 1720 printk(KERN_WARNING "ERROR: %s\n\n", str); 1721 1722 for (i = 0; i < nr_node_ids; i++) { 1723 printk(KERN_WARNING " "); 1724 for (j = 0; j < nr_node_ids; j++) { 1725 if (!node_state(i, N_CPU) || !node_state(j, N_CPU)) 1726 printk(KERN_CONT "(%02d) ", node_distance(i,j)); 1727 else 1728 printk(KERN_CONT " %02d ", node_distance(i,j)); 1729 } 1730 printk(KERN_CONT "\n"); 1731 } 1732 printk(KERN_WARNING "\n"); 1733 } 1734 1735 bool find_numa_distance(int distance) 1736 { 1737 bool found = false; 1738 int i, *distances; 1739 1740 if (distance == node_distance(0, 0)) 1741 return true; 1742 1743 rcu_read_lock(); 1744 distances = rcu_dereference(sched_domains_numa_distance); 1745 if (!distances) 1746 goto unlock; 1747 for (i = 0; i < sched_domains_numa_levels; i++) { 1748 if (distances[i] == distance) { 1749 found = true; 1750 break; 1751 } 1752 } 1753 unlock: 1754 rcu_read_unlock(); 1755 1756 return found; 1757 } 1758 1759 #define for_each_cpu_node_but(n, nbut) \ 1760 for_each_node_state(n, N_CPU) \ 1761 if (n == nbut) \ 1762 continue; \ 1763 else 1764 1765 /* 1766 * A system can have three types of NUMA topology: 1767 * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system 1768 * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes 1769 * NUMA_BACKPLANE: nodes can reach other nodes through a backplane 1770 * 1771 * The difference between a glueless mesh topology and a backplane 1772 * topology lies in whether communication between not directly 1773 * connected nodes goes through intermediary nodes (where programs 1774 * could run), or through backplane controllers. This affects 1775 * placement of programs. 1776 * 1777 * The type of topology can be discerned with the following tests: 1778 * - If the maximum distance between any nodes is 1 hop, the system 1779 * is directly connected. 1780 * - If for two nodes A and B, located N > 1 hops away from each other, 1781 * there is an intermediary node C, which is < N hops away from both 1782 * nodes A and B, the system is a glueless mesh. 1783 */ 1784 static void init_numa_topology_type(int offline_node) 1785 { 1786 int a, b, c, n; 1787 1788 n = sched_max_numa_distance; 1789 1790 if (sched_domains_numa_levels <= 2) { 1791 sched_numa_topology_type = NUMA_DIRECT; 1792 return; 1793 } 1794 1795 for_each_cpu_node_but(a, offline_node) { 1796 for_each_cpu_node_but(b, offline_node) { 1797 /* Find two nodes furthest removed from each other. */ 1798 if (node_distance(a, b) < n) 1799 continue; 1800 1801 /* Is there an intermediary node between a and b? */ 1802 for_each_cpu_node_but(c, offline_node) { 1803 if (node_distance(a, c) < n && 1804 node_distance(b, c) < n) { 1805 sched_numa_topology_type = 1806 NUMA_GLUELESS_MESH; 1807 return; 1808 } 1809 } 1810 1811 sched_numa_topology_type = NUMA_BACKPLANE; 1812 return; 1813 } 1814 } 1815 1816 pr_err("Failed to find a NUMA topology type, defaulting to DIRECT\n"); 1817 sched_numa_topology_type = NUMA_DIRECT; 1818 } 1819 1820 1821 #define NR_DISTANCE_VALUES (1 << DISTANCE_BITS) 1822 1823 void sched_init_numa(int offline_node) 1824 { 1825 struct sched_domain_topology_level *tl; 1826 unsigned long *distance_map; 1827 int nr_levels = 0; 1828 int i, j; 1829 int *distances; 1830 struct cpumask ***masks; 1831 1832 /* 1833 * O(nr_nodes^2) deduplicating selection sort -- in order to find the 1834 * unique distances in the node_distance() table. 1835 */ 1836 distance_map = bitmap_alloc(NR_DISTANCE_VALUES, GFP_KERNEL); 1837 if (!distance_map) 1838 return; 1839 1840 bitmap_zero(distance_map, NR_DISTANCE_VALUES); 1841 for_each_cpu_node_but(i, offline_node) { 1842 for_each_cpu_node_but(j, offline_node) { 1843 int distance = node_distance(i, j); 1844 1845 if (distance < LOCAL_DISTANCE || distance >= NR_DISTANCE_VALUES) { 1846 sched_numa_warn("Invalid distance value range"); 1847 bitmap_free(distance_map); 1848 return; 1849 } 1850 1851 bitmap_set(distance_map, distance, 1); 1852 } 1853 } 1854 /* 1855 * We can now figure out how many unique distance values there are and 1856 * allocate memory accordingly. 1857 */ 1858 nr_levels = bitmap_weight(distance_map, NR_DISTANCE_VALUES); 1859 1860 distances = kcalloc(nr_levels, sizeof(int), GFP_KERNEL); 1861 if (!distances) { 1862 bitmap_free(distance_map); 1863 return; 1864 } 1865 1866 for (i = 0, j = 0; i < nr_levels; i++, j++) { 1867 j = find_next_bit(distance_map, NR_DISTANCE_VALUES, j); 1868 distances[i] = j; 1869 } 1870 rcu_assign_pointer(sched_domains_numa_distance, distances); 1871 1872 bitmap_free(distance_map); 1873 1874 /* 1875 * 'nr_levels' contains the number of unique distances 1876 * 1877 * The sched_domains_numa_distance[] array includes the actual distance 1878 * numbers. 1879 */ 1880 1881 /* 1882 * Here, we should temporarily reset sched_domains_numa_levels to 0. 1883 * If it fails to allocate memory for array sched_domains_numa_masks[][], 1884 * the array will contain less then 'nr_levels' members. This could be 1885 * dangerous when we use it to iterate array sched_domains_numa_masks[][] 1886 * in other functions. 1887 * 1888 * We reset it to 'nr_levels' at the end of this function. 1889 */ 1890 sched_domains_numa_levels = 0; 1891 1892 masks = kzalloc(sizeof(void *) * nr_levels, GFP_KERNEL); 1893 if (!masks) 1894 return; 1895 1896 /* 1897 * Now for each level, construct a mask per node which contains all 1898 * CPUs of nodes that are that many hops away from us. 1899 */ 1900 for (i = 0; i < nr_levels; i++) { 1901 masks[i] = kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL); 1902 if (!masks[i]) 1903 return; 1904 1905 for_each_cpu_node_but(j, offline_node) { 1906 struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL); 1907 int k; 1908 1909 if (!mask) 1910 return; 1911 1912 masks[i][j] = mask; 1913 1914 for_each_cpu_node_but(k, offline_node) { 1915 if (sched_debug() && (node_distance(j, k) != node_distance(k, j))) 1916 sched_numa_warn("Node-distance not symmetric"); 1917 1918 if (node_distance(j, k) > sched_domains_numa_distance[i]) 1919 continue; 1920 1921 cpumask_or(mask, mask, cpumask_of_node(k)); 1922 } 1923 } 1924 } 1925 rcu_assign_pointer(sched_domains_numa_masks, masks); 1926 1927 /* Compute default topology size */ 1928 for (i = 0; sched_domain_topology[i].mask; i++); 1929 1930 tl = kzalloc((i + nr_levels + 1) * 1931 sizeof(struct sched_domain_topology_level), GFP_KERNEL); 1932 if (!tl) 1933 return; 1934 1935 /* 1936 * Copy the default topology bits.. 1937 */ 1938 for (i = 0; sched_domain_topology[i].mask; i++) 1939 tl[i] = sched_domain_topology[i]; 1940 1941 /* 1942 * Add the NUMA identity distance, aka single NODE. 1943 */ 1944 tl[i++] = (struct sched_domain_topology_level){ 1945 .mask = sd_numa_mask, 1946 .numa_level = 0, 1947 SD_INIT_NAME(NODE) 1948 }; 1949 1950 /* 1951 * .. and append 'j' levels of NUMA goodness. 1952 */ 1953 for (j = 1; j < nr_levels; i++, j++) { 1954 tl[i] = (struct sched_domain_topology_level){ 1955 .mask = sd_numa_mask, 1956 .sd_flags = cpu_numa_flags, 1957 .flags = SDTL_OVERLAP, 1958 .numa_level = j, 1959 SD_INIT_NAME(NUMA) 1960 }; 1961 } 1962 1963 sched_domain_topology_saved = sched_domain_topology; 1964 sched_domain_topology = tl; 1965 1966 sched_domains_numa_levels = nr_levels; 1967 WRITE_ONCE(sched_max_numa_distance, sched_domains_numa_distance[nr_levels - 1]); 1968 1969 init_numa_topology_type(offline_node); 1970 } 1971 1972 1973 static void sched_reset_numa(void) 1974 { 1975 int nr_levels, *distances; 1976 struct cpumask ***masks; 1977 1978 nr_levels = sched_domains_numa_levels; 1979 sched_domains_numa_levels = 0; 1980 sched_max_numa_distance = 0; 1981 sched_numa_topology_type = NUMA_DIRECT; 1982 distances = sched_domains_numa_distance; 1983 rcu_assign_pointer(sched_domains_numa_distance, NULL); 1984 masks = sched_domains_numa_masks; 1985 rcu_assign_pointer(sched_domains_numa_masks, NULL); 1986 if (distances || masks) { 1987 int i, j; 1988 1989 synchronize_rcu(); 1990 kfree(distances); 1991 for (i = 0; i < nr_levels && masks; i++) { 1992 if (!masks[i]) 1993 continue; 1994 for_each_node(j) 1995 kfree(masks[i][j]); 1996 kfree(masks[i]); 1997 } 1998 kfree(masks); 1999 } 2000 if (sched_domain_topology_saved) { 2001 kfree(sched_domain_topology); 2002 sched_domain_topology = sched_domain_topology_saved; 2003 sched_domain_topology_saved = NULL; 2004 } 2005 } 2006 2007 /* 2008 * Call with hotplug lock held 2009 */ 2010 void sched_update_numa(int cpu, bool online) 2011 { 2012 int node; 2013 2014 node = cpu_to_node(cpu); 2015 /* 2016 * Scheduler NUMA topology is updated when the first CPU of a 2017 * node is onlined or the last CPU of a node is offlined. 2018 */ 2019 if (cpumask_weight(cpumask_of_node(node)) != 1) 2020 return; 2021 2022 sched_reset_numa(); 2023 sched_init_numa(online ? NUMA_NO_NODE : node); 2024 } 2025 2026 void sched_domains_numa_masks_set(unsigned int cpu) 2027 { 2028 int node = cpu_to_node(cpu); 2029 int i, j; 2030 2031 for (i = 0; i < sched_domains_numa_levels; i++) { 2032 for (j = 0; j < nr_node_ids; j++) { 2033 if (!node_state(j, N_CPU)) 2034 continue; 2035 2036 /* Set ourselves in the remote node's masks */ 2037 if (node_distance(j, node) <= sched_domains_numa_distance[i]) 2038 cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]); 2039 } 2040 } 2041 } 2042 2043 void sched_domains_numa_masks_clear(unsigned int cpu) 2044 { 2045 int i, j; 2046 2047 for (i = 0; i < sched_domains_numa_levels; i++) { 2048 for (j = 0; j < nr_node_ids; j++) { 2049 if (sched_domains_numa_masks[i][j]) 2050 cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]); 2051 } 2052 } 2053 } 2054 2055 /* 2056 * sched_numa_find_closest() - given the NUMA topology, find the cpu 2057 * closest to @cpu from @cpumask. 2058 * cpumask: cpumask to find a cpu from 2059 * cpu: cpu to be close to 2060 * 2061 * returns: cpu, or nr_cpu_ids when nothing found. 2062 */ 2063 int sched_numa_find_closest(const struct cpumask *cpus, int cpu) 2064 { 2065 int i, j = cpu_to_node(cpu), found = nr_cpu_ids; 2066 struct cpumask ***masks; 2067 2068 rcu_read_lock(); 2069 masks = rcu_dereference(sched_domains_numa_masks); 2070 if (!masks) 2071 goto unlock; 2072 for (i = 0; i < sched_domains_numa_levels; i++) { 2073 if (!masks[i][j]) 2074 break; 2075 cpu = cpumask_any_and(cpus, masks[i][j]); 2076 if (cpu < nr_cpu_ids) { 2077 found = cpu; 2078 break; 2079 } 2080 } 2081 unlock: 2082 rcu_read_unlock(); 2083 2084 return found; 2085 } 2086 2087 struct __cmp_key { 2088 const struct cpumask *cpus; 2089 struct cpumask ***masks; 2090 int node; 2091 int cpu; 2092 int w; 2093 }; 2094 2095 static int hop_cmp(const void *a, const void *b) 2096 { 2097 struct cpumask **prev_hop, **cur_hop = *(struct cpumask ***)b; 2098 struct __cmp_key *k = (struct __cmp_key *)a; 2099 2100 if (cpumask_weight_and(k->cpus, cur_hop[k->node]) <= k->cpu) 2101 return 1; 2102 2103 if (b == k->masks) { 2104 k->w = 0; 2105 return 0; 2106 } 2107 2108 prev_hop = *((struct cpumask ***)b - 1); 2109 k->w = cpumask_weight_and(k->cpus, prev_hop[k->node]); 2110 if (k->w <= k->cpu) 2111 return 0; 2112 2113 return -1; 2114 } 2115 2116 /* 2117 * sched_numa_find_nth_cpu() - given the NUMA topology, find the Nth next cpu 2118 * closest to @cpu from @cpumask. 2119 * cpumask: cpumask to find a cpu from 2120 * cpu: Nth cpu to find 2121 * 2122 * returns: cpu, or nr_cpu_ids when nothing found. 2123 */ 2124 int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node) 2125 { 2126 struct __cmp_key k = { .cpus = cpus, .node = node, .cpu = cpu }; 2127 struct cpumask ***hop_masks; 2128 int hop, ret = nr_cpu_ids; 2129 2130 rcu_read_lock(); 2131 2132 k.masks = rcu_dereference(sched_domains_numa_masks); 2133 if (!k.masks) 2134 goto unlock; 2135 2136 hop_masks = bsearch(&k, k.masks, sched_domains_numa_levels, sizeof(k.masks[0]), hop_cmp); 2137 hop = hop_masks - k.masks; 2138 2139 ret = hop ? 2140 cpumask_nth_and_andnot(cpu - k.w, cpus, k.masks[hop][node], k.masks[hop-1][node]) : 2141 cpumask_nth_and(cpu, cpus, k.masks[0][node]); 2142 unlock: 2143 rcu_read_unlock(); 2144 return ret; 2145 } 2146 EXPORT_SYMBOL_GPL(sched_numa_find_nth_cpu); 2147 2148 /** 2149 * sched_numa_hop_mask() - Get the cpumask of CPUs at most @hops hops away from 2150 * @node 2151 * @node: The node to count hops from. 2152 * @hops: Include CPUs up to that many hops away. 0 means local node. 2153 * 2154 * Return: On success, a pointer to a cpumask of CPUs at most @hops away from 2155 * @node, an error value otherwise. 2156 * 2157 * Requires rcu_lock to be held. Returned cpumask is only valid within that 2158 * read-side section, copy it if required beyond that. 2159 * 2160 * Note that not all hops are equal in distance; see sched_init_numa() for how 2161 * distances and masks are handled. 2162 * Also note that this is a reflection of sched_domains_numa_masks, which may change 2163 * during the lifetime of the system (offline nodes are taken out of the masks). 2164 */ 2165 const struct cpumask *sched_numa_hop_mask(unsigned int node, unsigned int hops) 2166 { 2167 struct cpumask ***masks; 2168 2169 if (node >= nr_node_ids || hops >= sched_domains_numa_levels) 2170 return ERR_PTR(-EINVAL); 2171 2172 masks = rcu_dereference(sched_domains_numa_masks); 2173 if (!masks) 2174 return ERR_PTR(-EBUSY); 2175 2176 return masks[hops][node]; 2177 } 2178 EXPORT_SYMBOL_GPL(sched_numa_hop_mask); 2179 2180 #endif /* CONFIG_NUMA */ 2181 2182 static int __sdt_alloc(const struct cpumask *cpu_map) 2183 { 2184 struct sched_domain_topology_level *tl; 2185 int j; 2186 2187 for_each_sd_topology(tl) { 2188 struct sd_data *sdd = &tl->data; 2189 2190 sdd->sd = alloc_percpu(struct sched_domain *); 2191 if (!sdd->sd) 2192 return -ENOMEM; 2193 2194 sdd->sds = alloc_percpu(struct sched_domain_shared *); 2195 if (!sdd->sds) 2196 return -ENOMEM; 2197 2198 sdd->sg = alloc_percpu(struct sched_group *); 2199 if (!sdd->sg) 2200 return -ENOMEM; 2201 2202 sdd->sgc = alloc_percpu(struct sched_group_capacity *); 2203 if (!sdd->sgc) 2204 return -ENOMEM; 2205 2206 for_each_cpu(j, cpu_map) { 2207 struct sched_domain *sd; 2208 struct sched_domain_shared *sds; 2209 struct sched_group *sg; 2210 struct sched_group_capacity *sgc; 2211 2212 sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), 2213 GFP_KERNEL, cpu_to_node(j)); 2214 if (!sd) 2215 return -ENOMEM; 2216 2217 *per_cpu_ptr(sdd->sd, j) = sd; 2218 2219 sds = kzalloc_node(sizeof(struct sched_domain_shared), 2220 GFP_KERNEL, cpu_to_node(j)); 2221 if (!sds) 2222 return -ENOMEM; 2223 2224 *per_cpu_ptr(sdd->sds, j) = sds; 2225 2226 sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), 2227 GFP_KERNEL, cpu_to_node(j)); 2228 if (!sg) 2229 return -ENOMEM; 2230 2231 sg->next = sg; 2232 2233 *per_cpu_ptr(sdd->sg, j) = sg; 2234 2235 sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(), 2236 GFP_KERNEL, cpu_to_node(j)); 2237 if (!sgc) 2238 return -ENOMEM; 2239 2240 #ifdef CONFIG_SCHED_DEBUG 2241 sgc->id = j; 2242 #endif 2243 2244 *per_cpu_ptr(sdd->sgc, j) = sgc; 2245 } 2246 } 2247 2248 return 0; 2249 } 2250 2251 static void __sdt_free(const struct cpumask *cpu_map) 2252 { 2253 struct sched_domain_topology_level *tl; 2254 int j; 2255 2256 for_each_sd_topology(tl) { 2257 struct sd_data *sdd = &tl->data; 2258 2259 for_each_cpu(j, cpu_map) { 2260 struct sched_domain *sd; 2261 2262 if (sdd->sd) { 2263 sd = *per_cpu_ptr(sdd->sd, j); 2264 if (sd && (sd->flags & SD_OVERLAP)) 2265 free_sched_groups(sd->groups, 0); 2266 kfree(*per_cpu_ptr(sdd->sd, j)); 2267 } 2268 2269 if (sdd->sds) 2270 kfree(*per_cpu_ptr(sdd->sds, j)); 2271 if (sdd->sg) 2272 kfree(*per_cpu_ptr(sdd->sg, j)); 2273 if (sdd->sgc) 2274 kfree(*per_cpu_ptr(sdd->sgc, j)); 2275 } 2276 free_percpu(sdd->sd); 2277 sdd->sd = NULL; 2278 free_percpu(sdd->sds); 2279 sdd->sds = NULL; 2280 free_percpu(sdd->sg); 2281 sdd->sg = NULL; 2282 free_percpu(sdd->sgc); 2283 sdd->sgc = NULL; 2284 } 2285 } 2286 2287 static struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, 2288 const struct cpumask *cpu_map, struct sched_domain_attr *attr, 2289 struct sched_domain *child, int cpu) 2290 { 2291 struct sched_domain *sd = sd_init(tl, cpu_map, child, cpu); 2292 2293 if (child) { 2294 sd->level = child->level + 1; 2295 sched_domain_level_max = max(sched_domain_level_max, sd->level); 2296 child->parent = sd; 2297 2298 if (!cpumask_subset(sched_domain_span(child), 2299 sched_domain_span(sd))) { 2300 pr_err("BUG: arch topology borken\n"); 2301 #ifdef CONFIG_SCHED_DEBUG 2302 pr_err(" the %s domain not a subset of the %s domain\n", 2303 child->name, sd->name); 2304 #endif 2305 /* Fixup, ensure @sd has at least @child CPUs. */ 2306 cpumask_or(sched_domain_span(sd), 2307 sched_domain_span(sd), 2308 sched_domain_span(child)); 2309 } 2310 2311 } 2312 set_domain_attribute(sd, attr); 2313 2314 return sd; 2315 } 2316 2317 /* 2318 * Ensure topology masks are sane, i.e. there are no conflicts (overlaps) for 2319 * any two given CPUs at this (non-NUMA) topology level. 2320 */ 2321 static bool topology_span_sane(struct sched_domain_topology_level *tl, 2322 const struct cpumask *cpu_map, int cpu) 2323 { 2324 int i; 2325 2326 /* NUMA levels are allowed to overlap */ 2327 if (tl->flags & SDTL_OVERLAP) 2328 return true; 2329 2330 /* 2331 * Non-NUMA levels cannot partially overlap - they must be either 2332 * completely equal or completely disjoint. Otherwise we can end up 2333 * breaking the sched_group lists - i.e. a later get_group() pass 2334 * breaks the linking done for an earlier span. 2335 */ 2336 for_each_cpu(i, cpu_map) { 2337 if (i == cpu) 2338 continue; 2339 /* 2340 * We should 'and' all those masks with 'cpu_map' to exactly 2341 * match the topology we're about to build, but that can only 2342 * remove CPUs, which only lessens our ability to detect 2343 * overlaps 2344 */ 2345 if (!cpumask_equal(tl->mask(cpu), tl->mask(i)) && 2346 cpumask_intersects(tl->mask(cpu), tl->mask(i))) 2347 return false; 2348 } 2349 2350 return true; 2351 } 2352 2353 /* 2354 * Build sched domains for a given set of CPUs and attach the sched domains 2355 * to the individual CPUs 2356 */ 2357 static int 2358 build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr) 2359 { 2360 enum s_alloc alloc_state = sa_none; 2361 struct sched_domain *sd; 2362 struct s_data d; 2363 struct rq *rq = NULL; 2364 int i, ret = -ENOMEM; 2365 bool has_asym = false; 2366 2367 if (WARN_ON(cpumask_empty(cpu_map))) 2368 goto error; 2369 2370 alloc_state = __visit_domain_allocation_hell(&d, cpu_map); 2371 if (alloc_state != sa_rootdomain) 2372 goto error; 2373 2374 /* Set up domains for CPUs specified by the cpu_map: */ 2375 for_each_cpu(i, cpu_map) { 2376 struct sched_domain_topology_level *tl; 2377 2378 sd = NULL; 2379 for_each_sd_topology(tl) { 2380 2381 if (WARN_ON(!topology_span_sane(tl, cpu_map, i))) 2382 goto error; 2383 2384 sd = build_sched_domain(tl, cpu_map, attr, sd, i); 2385 2386 has_asym |= sd->flags & SD_ASYM_CPUCAPACITY; 2387 2388 if (tl == sched_domain_topology) 2389 *per_cpu_ptr(d.sd, i) = sd; 2390 if (tl->flags & SDTL_OVERLAP) 2391 sd->flags |= SD_OVERLAP; 2392 if (cpumask_equal(cpu_map, sched_domain_span(sd))) 2393 break; 2394 } 2395 } 2396 2397 /* Build the groups for the domains */ 2398 for_each_cpu(i, cpu_map) { 2399 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { 2400 sd->span_weight = cpumask_weight(sched_domain_span(sd)); 2401 if (sd->flags & SD_OVERLAP) { 2402 if (build_overlap_sched_groups(sd, i)) 2403 goto error; 2404 } else { 2405 if (build_sched_groups(sd, i)) 2406 goto error; 2407 } 2408 } 2409 } 2410 2411 /* 2412 * Calculate an allowed NUMA imbalance such that LLCs do not get 2413 * imbalanced. 2414 */ 2415 for_each_cpu(i, cpu_map) { 2416 unsigned int imb = 0; 2417 unsigned int imb_span = 1; 2418 2419 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { 2420 struct sched_domain *child = sd->child; 2421 2422 if (!(sd->flags & SD_SHARE_PKG_RESOURCES) && child && 2423 (child->flags & SD_SHARE_PKG_RESOURCES)) { 2424 struct sched_domain __rcu *top_p; 2425 unsigned int nr_llcs; 2426 2427 /* 2428 * For a single LLC per node, allow an 2429 * imbalance up to 12.5% of the node. This is 2430 * arbitrary cutoff based two factors -- SMT and 2431 * memory channels. For SMT-2, the intent is to 2432 * avoid premature sharing of HT resources but 2433 * SMT-4 or SMT-8 *may* benefit from a different 2434 * cutoff. For memory channels, this is a very 2435 * rough estimate of how many channels may be 2436 * active and is based on recent CPUs with 2437 * many cores. 2438 * 2439 * For multiple LLCs, allow an imbalance 2440 * until multiple tasks would share an LLC 2441 * on one node while LLCs on another node 2442 * remain idle. This assumes that there are 2443 * enough logical CPUs per LLC to avoid SMT 2444 * factors and that there is a correlation 2445 * between LLCs and memory channels. 2446 */ 2447 nr_llcs = sd->span_weight / child->span_weight; 2448 if (nr_llcs == 1) 2449 imb = sd->span_weight >> 3; 2450 else 2451 imb = nr_llcs; 2452 imb = max(1U, imb); 2453 sd->imb_numa_nr = imb; 2454 2455 /* Set span based on the first NUMA domain. */ 2456 top_p = sd->parent; 2457 while (top_p && !(top_p->flags & SD_NUMA)) { 2458 top_p = top_p->parent; 2459 } 2460 imb_span = top_p ? top_p->span_weight : sd->span_weight; 2461 } else { 2462 int factor = max(1U, (sd->span_weight / imb_span)); 2463 2464 sd->imb_numa_nr = imb * factor; 2465 } 2466 } 2467 } 2468 2469 /* Calculate CPU capacity for physical packages and nodes */ 2470 for (i = nr_cpumask_bits-1; i >= 0; i--) { 2471 if (!cpumask_test_cpu(i, cpu_map)) 2472 continue; 2473 2474 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { 2475 claim_allocations(i, sd); 2476 init_sched_groups_capacity(i, sd); 2477 } 2478 } 2479 2480 /* Attach the domains */ 2481 rcu_read_lock(); 2482 for_each_cpu(i, cpu_map) { 2483 rq = cpu_rq(i); 2484 sd = *per_cpu_ptr(d.sd, i); 2485 2486 /* Use READ_ONCE()/WRITE_ONCE() to avoid load/store tearing: */ 2487 if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity)) 2488 WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig); 2489 2490 cpu_attach_domain(sd, d.rd, i); 2491 } 2492 rcu_read_unlock(); 2493 2494 if (has_asym) 2495 static_branch_inc_cpuslocked(&sched_asym_cpucapacity); 2496 2497 if (rq && sched_debug_verbose) { 2498 pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n", 2499 cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity); 2500 } 2501 2502 ret = 0; 2503 error: 2504 __free_domain_allocs(&d, alloc_state, cpu_map); 2505 2506 return ret; 2507 } 2508 2509 /* Current sched domains: */ 2510 static cpumask_var_t *doms_cur; 2511 2512 /* Number of sched domains in 'doms_cur': */ 2513 static int ndoms_cur; 2514 2515 /* Attributes of custom domains in 'doms_cur' */ 2516 static struct sched_domain_attr *dattr_cur; 2517 2518 /* 2519 * Special case: If a kmalloc() of a doms_cur partition (array of 2520 * cpumask) fails, then fallback to a single sched domain, 2521 * as determined by the single cpumask fallback_doms. 2522 */ 2523 static cpumask_var_t fallback_doms; 2524 2525 /* 2526 * arch_update_cpu_topology lets virtualized architectures update the 2527 * CPU core maps. It is supposed to return 1 if the topology changed 2528 * or 0 if it stayed the same. 2529 */ 2530 int __weak arch_update_cpu_topology(void) 2531 { 2532 return 0; 2533 } 2534 2535 cpumask_var_t *alloc_sched_domains(unsigned int ndoms) 2536 { 2537 int i; 2538 cpumask_var_t *doms; 2539 2540 doms = kmalloc_array(ndoms, sizeof(*doms), GFP_KERNEL); 2541 if (!doms) 2542 return NULL; 2543 for (i = 0; i < ndoms; i++) { 2544 if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) { 2545 free_sched_domains(doms, i); 2546 return NULL; 2547 } 2548 } 2549 return doms; 2550 } 2551 2552 void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms) 2553 { 2554 unsigned int i; 2555 for (i = 0; i < ndoms; i++) 2556 free_cpumask_var(doms[i]); 2557 kfree(doms); 2558 } 2559 2560 /* 2561 * Set up scheduler domains and groups. For now this just excludes isolated 2562 * CPUs, but could be used to exclude other special cases in the future. 2563 */ 2564 int __init sched_init_domains(const struct cpumask *cpu_map) 2565 { 2566 int err; 2567 2568 zalloc_cpumask_var(&sched_domains_tmpmask, GFP_KERNEL); 2569 zalloc_cpumask_var(&sched_domains_tmpmask2, GFP_KERNEL); 2570 zalloc_cpumask_var(&fallback_doms, GFP_KERNEL); 2571 2572 arch_update_cpu_topology(); 2573 asym_cpu_capacity_scan(); 2574 ndoms_cur = 1; 2575 doms_cur = alloc_sched_domains(ndoms_cur); 2576 if (!doms_cur) 2577 doms_cur = &fallback_doms; 2578 cpumask_and(doms_cur[0], cpu_map, housekeeping_cpumask(HK_TYPE_DOMAIN)); 2579 err = build_sched_domains(doms_cur[0], NULL); 2580 2581 return err; 2582 } 2583 2584 /* 2585 * Detach sched domains from a group of CPUs specified in cpu_map 2586 * These CPUs will now be attached to the NULL domain 2587 */ 2588 static void detach_destroy_domains(const struct cpumask *cpu_map) 2589 { 2590 unsigned int cpu = cpumask_any(cpu_map); 2591 int i; 2592 2593 if (rcu_access_pointer(per_cpu(sd_asym_cpucapacity, cpu))) 2594 static_branch_dec_cpuslocked(&sched_asym_cpucapacity); 2595 2596 rcu_read_lock(); 2597 for_each_cpu(i, cpu_map) 2598 cpu_attach_domain(NULL, &def_root_domain, i); 2599 rcu_read_unlock(); 2600 } 2601 2602 /* handle null as "default" */ 2603 static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur, 2604 struct sched_domain_attr *new, int idx_new) 2605 { 2606 struct sched_domain_attr tmp; 2607 2608 /* Fast path: */ 2609 if (!new && !cur) 2610 return 1; 2611 2612 tmp = SD_ATTR_INIT; 2613 2614 return !memcmp(cur ? (cur + idx_cur) : &tmp, 2615 new ? (new + idx_new) : &tmp, 2616 sizeof(struct sched_domain_attr)); 2617 } 2618 2619 /* 2620 * Partition sched domains as specified by the 'ndoms_new' 2621 * cpumasks in the array doms_new[] of cpumasks. This compares 2622 * doms_new[] to the current sched domain partitioning, doms_cur[]. 2623 * It destroys each deleted domain and builds each new domain. 2624 * 2625 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'. 2626 * The masks don't intersect (don't overlap.) We should setup one 2627 * sched domain for each mask. CPUs not in any of the cpumasks will 2628 * not be load balanced. If the same cpumask appears both in the 2629 * current 'doms_cur' domains and in the new 'doms_new', we can leave 2630 * it as it is. 2631 * 2632 * The passed in 'doms_new' should be allocated using 2633 * alloc_sched_domains. This routine takes ownership of it and will 2634 * free_sched_domains it when done with it. If the caller failed the 2635 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1, 2636 * and partition_sched_domains() will fallback to the single partition 2637 * 'fallback_doms', it also forces the domains to be rebuilt. 2638 * 2639 * If doms_new == NULL it will be replaced with cpu_online_mask. 2640 * ndoms_new == 0 is a special case for destroying existing domains, 2641 * and it will not create the default domain. 2642 * 2643 * Call with hotplug lock and sched_domains_mutex held 2644 */ 2645 void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new[], 2646 struct sched_domain_attr *dattr_new) 2647 { 2648 bool __maybe_unused has_eas = false; 2649 int i, j, n; 2650 int new_topology; 2651 2652 lockdep_assert_held(&sched_domains_mutex); 2653 2654 /* Let the architecture update CPU core mappings: */ 2655 new_topology = arch_update_cpu_topology(); 2656 /* Trigger rebuilding CPU capacity asymmetry data */ 2657 if (new_topology) 2658 asym_cpu_capacity_scan(); 2659 2660 if (!doms_new) { 2661 WARN_ON_ONCE(dattr_new); 2662 n = 0; 2663 doms_new = alloc_sched_domains(1); 2664 if (doms_new) { 2665 n = 1; 2666 cpumask_and(doms_new[0], cpu_active_mask, 2667 housekeeping_cpumask(HK_TYPE_DOMAIN)); 2668 } 2669 } else { 2670 n = ndoms_new; 2671 } 2672 2673 /* Destroy deleted domains: */ 2674 for (i = 0; i < ndoms_cur; i++) { 2675 for (j = 0; j < n && !new_topology; j++) { 2676 if (cpumask_equal(doms_cur[i], doms_new[j]) && 2677 dattrs_equal(dattr_cur, i, dattr_new, j)) { 2678 struct root_domain *rd; 2679 2680 /* 2681 * This domain won't be destroyed and as such 2682 * its dl_bw->total_bw needs to be cleared. It 2683 * will be recomputed in function 2684 * update_tasks_root_domain(). 2685 */ 2686 rd = cpu_rq(cpumask_any(doms_cur[i]))->rd; 2687 dl_clear_root_domain(rd); 2688 goto match1; 2689 } 2690 } 2691 /* No match - a current sched domain not in new doms_new[] */ 2692 detach_destroy_domains(doms_cur[i]); 2693 match1: 2694 ; 2695 } 2696 2697 n = ndoms_cur; 2698 if (!doms_new) { 2699 n = 0; 2700 doms_new = &fallback_doms; 2701 cpumask_and(doms_new[0], cpu_active_mask, 2702 housekeeping_cpumask(HK_TYPE_DOMAIN)); 2703 } 2704 2705 /* Build new domains: */ 2706 for (i = 0; i < ndoms_new; i++) { 2707 for (j = 0; j < n && !new_topology; j++) { 2708 if (cpumask_equal(doms_new[i], doms_cur[j]) && 2709 dattrs_equal(dattr_new, i, dattr_cur, j)) 2710 goto match2; 2711 } 2712 /* No match - add a new doms_new */ 2713 build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL); 2714 match2: 2715 ; 2716 } 2717 2718 #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) 2719 /* Build perf. domains: */ 2720 for (i = 0; i < ndoms_new; i++) { 2721 for (j = 0; j < n && !sched_energy_update; j++) { 2722 if (cpumask_equal(doms_new[i], doms_cur[j]) && 2723 cpu_rq(cpumask_first(doms_cur[j]))->rd->pd) { 2724 has_eas = true; 2725 goto match3; 2726 } 2727 } 2728 /* No match - add perf. domains for a new rd */ 2729 has_eas |= build_perf_domains(doms_new[i]); 2730 match3: 2731 ; 2732 } 2733 sched_energy_set(has_eas); 2734 #endif 2735 2736 /* Remember the new sched domains: */ 2737 if (doms_cur != &fallback_doms) 2738 free_sched_domains(doms_cur, ndoms_cur); 2739 2740 kfree(dattr_cur); 2741 doms_cur = doms_new; 2742 dattr_cur = dattr_new; 2743 ndoms_cur = ndoms_new; 2744 2745 update_sched_domain_debugfs(); 2746 } 2747 2748 /* 2749 * Call with hotplug lock held 2750 */ 2751 void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], 2752 struct sched_domain_attr *dattr_new) 2753 { 2754 mutex_lock(&sched_domains_mutex); 2755 partition_sched_domains_locked(ndoms_new, doms_new, dattr_new); 2756 mutex_unlock(&sched_domains_mutex); 2757 } 2758