1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2017, Nicholas Piggin, IBM Corporation 4 */ 5 6 #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt 7 8 #include <linux/export.h> 9 #include <linux/init.h> 10 #include <linux/jump_label.h> 11 #include <linux/libfdt.h> 12 #include <linux/memblock.h> 13 #include <linux/printk.h> 14 #include <linux/sched.h> 15 #include <linux/string.h> 16 #include <linux/threads.h> 17 18 #include <asm/cputable.h> 19 #include <asm/dt_cpu_ftrs.h> 20 #include <asm/mce.h> 21 #include <asm/mmu.h> 22 #include <asm/prom.h> 23 #include <asm/setup.h> 24 25 26 /* Device-tree visible constants follow */ 27 #define ISA_V3_0B 3000 28 #define ISA_V3_1 3100 29 30 #define USABLE_PR (1U << 0) 31 #define USABLE_OS (1U << 1) 32 #define USABLE_HV (1U << 2) 33 34 #define HV_SUPPORT_HFSCR (1U << 0) 35 #define OS_SUPPORT_FSCR (1U << 0) 36 37 /* For parsing, we define all bits set as "NONE" case */ 38 #define HV_SUPPORT_NONE 0xffffffffU 39 #define OS_SUPPORT_NONE 0xffffffffU 40 41 struct dt_cpu_feature { 42 const char *name; 43 uint32_t isa; 44 uint32_t usable_privilege; 45 uint32_t hv_support; 46 uint32_t os_support; 47 uint32_t hfscr_bit_nr; 48 uint32_t fscr_bit_nr; 49 uint32_t hwcap_bit_nr; 50 /* fdt parsing */ 51 unsigned long node; 52 int enabled; 53 int disabled; 54 }; 55 56 #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) 57 58 #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ 59 PPC_FEATURE_ARCH_2_06 |\ 60 PPC_FEATURE_ICACHE_SNOOP) 61 #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ 62 PPC_FEATURE2_ISEL) 63 /* 64 * Set up the base CPU 65 */ 66 67 static int hv_mode; 68 69 static struct { 70 u64 lpcr; 71 u64 hfscr; 72 u64 fscr; 73 u64 pcr; 74 } system_registers; 75 76 static void (*init_pmu_registers)(void); 77 78 static void __restore_cpu_cpufeatures(void) 79 { 80 mtspr(SPRN_LPCR, system_registers.lpcr); 81 if (hv_mode) { 82 mtspr(SPRN_LPID, 0); 83 mtspr(SPRN_HFSCR, system_registers.hfscr); 84 mtspr(SPRN_PCR, system_registers.pcr); 85 } 86 mtspr(SPRN_FSCR, system_registers.fscr); 87 88 if (init_pmu_registers) 89 init_pmu_registers(); 90 } 91 92 static char dt_cpu_name[64]; 93 94 static struct cpu_spec __initdata base_cpu_spec = { 95 .cpu_name = NULL, 96 .cpu_features = CPU_FTRS_DT_CPU_BASE, 97 .cpu_user_features = COMMON_USER_BASE, 98 .cpu_user_features2 = COMMON_USER2_BASE, 99 .mmu_features = 0, 100 .icache_bsize = 32, /* minimum block size, fixed by */ 101 .dcache_bsize = 32, /* cache info init. */ 102 .num_pmcs = 0, 103 .pmc_type = PPC_PMC_DEFAULT, 104 .oprofile_cpu_type = NULL, 105 .cpu_setup = NULL, 106 .cpu_restore = __restore_cpu_cpufeatures, 107 .machine_check_early = NULL, 108 .platform = NULL, 109 }; 110 111 static void __init cpufeatures_setup_cpu(void) 112 { 113 set_cur_cpu_spec(&base_cpu_spec); 114 115 cur_cpu_spec->pvr_mask = -1; 116 cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); 117 118 /* Initialize the base environment -- clear FSCR/HFSCR. */ 119 hv_mode = !!(mfmsr() & MSR_HV); 120 if (hv_mode) { 121 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 122 mtspr(SPRN_HFSCR, 0); 123 } 124 mtspr(SPRN_FSCR, 0); 125 mtspr(SPRN_PCR, PCR_MASK); 126 127 /* 128 * LPCR does not get cleared, to match behaviour with secondaries 129 * in __restore_cpu_cpufeatures. Once the idle code is fixed, this 130 * could clear LPCR too. 131 */ 132 } 133 134 static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) 135 { 136 if (f->hv_support == HV_SUPPORT_NONE) { 137 } else if (f->hv_support & HV_SUPPORT_HFSCR) { 138 u64 hfscr = mfspr(SPRN_HFSCR); 139 hfscr |= 1UL << f->hfscr_bit_nr; 140 mtspr(SPRN_HFSCR, hfscr); 141 } else { 142 /* Does not have a known recipe */ 143 return 0; 144 } 145 146 if (f->os_support == OS_SUPPORT_NONE) { 147 } else if (f->os_support & OS_SUPPORT_FSCR) { 148 u64 fscr = mfspr(SPRN_FSCR); 149 fscr |= 1UL << f->fscr_bit_nr; 150 mtspr(SPRN_FSCR, fscr); 151 } else { 152 /* Does not have a known recipe */ 153 return 0; 154 } 155 156 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 157 uint32_t word = f->hwcap_bit_nr / 32; 158 uint32_t bit = f->hwcap_bit_nr % 32; 159 160 if (word == 0) 161 cur_cpu_spec->cpu_user_features |= 1U << bit; 162 else if (word == 1) 163 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 164 else 165 pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); 166 } 167 168 return 1; 169 } 170 171 static int __init feat_enable(struct dt_cpu_feature *f) 172 { 173 if (f->hv_support != HV_SUPPORT_NONE) { 174 if (f->hfscr_bit_nr != -1) { 175 u64 hfscr = mfspr(SPRN_HFSCR); 176 hfscr |= 1UL << f->hfscr_bit_nr; 177 mtspr(SPRN_HFSCR, hfscr); 178 } 179 } 180 181 if (f->os_support != OS_SUPPORT_NONE) { 182 if (f->fscr_bit_nr != -1) { 183 u64 fscr = mfspr(SPRN_FSCR); 184 fscr |= 1UL << f->fscr_bit_nr; 185 mtspr(SPRN_FSCR, fscr); 186 } 187 } 188 189 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 190 uint32_t word = f->hwcap_bit_nr / 32; 191 uint32_t bit = f->hwcap_bit_nr % 32; 192 193 if (word == 0) 194 cur_cpu_spec->cpu_user_features |= 1U << bit; 195 else if (word == 1) 196 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 197 else 198 pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); 199 } 200 201 return 1; 202 } 203 204 static int __init feat_disable(struct dt_cpu_feature *f) 205 { 206 return 0; 207 } 208 209 static int __init feat_enable_hv(struct dt_cpu_feature *f) 210 { 211 u64 lpcr; 212 213 if (!hv_mode) { 214 pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); 215 return 0; 216 } 217 218 mtspr(SPRN_LPID, 0); 219 220 lpcr = mfspr(SPRN_LPCR); 221 lpcr &= ~LPCR_LPES0; /* HV external interrupts */ 222 mtspr(SPRN_LPCR, lpcr); 223 224 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 225 226 return 1; 227 } 228 229 static int __init feat_enable_le(struct dt_cpu_feature *f) 230 { 231 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; 232 return 1; 233 } 234 235 static int __init feat_enable_smt(struct dt_cpu_feature *f) 236 { 237 cur_cpu_spec->cpu_features |= CPU_FTR_SMT; 238 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; 239 return 1; 240 } 241 242 static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) 243 { 244 u64 lpcr; 245 246 /* Set PECE wakeup modes for ISA 207 */ 247 lpcr = mfspr(SPRN_LPCR); 248 lpcr |= LPCR_PECE0; 249 lpcr |= LPCR_PECE1; 250 lpcr |= LPCR_PECE2; 251 mtspr(SPRN_LPCR, lpcr); 252 253 return 1; 254 } 255 256 static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) 257 { 258 u64 lpcr; 259 260 /* Set PECE wakeup modes for ISAv3.0B */ 261 lpcr = mfspr(SPRN_LPCR); 262 lpcr |= LPCR_PECE0; 263 lpcr |= LPCR_PECE1; 264 lpcr |= LPCR_PECE2; 265 mtspr(SPRN_LPCR, lpcr); 266 267 return 1; 268 } 269 270 static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) 271 { 272 u64 lpcr; 273 274 lpcr = mfspr(SPRN_LPCR); 275 lpcr &= ~LPCR_ISL; 276 277 /* VRMASD */ 278 lpcr |= LPCR_VPM0; 279 lpcr &= ~LPCR_VPM1; 280 lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ 281 mtspr(SPRN_LPCR, lpcr); 282 283 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 284 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 285 286 return 1; 287 } 288 289 static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) 290 { 291 u64 lpcr; 292 293 lpcr = mfspr(SPRN_LPCR); 294 lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); 295 mtspr(SPRN_LPCR, lpcr); 296 297 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 298 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 299 300 return 1; 301 } 302 303 304 static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) 305 { 306 #ifdef CONFIG_PPC_RADIX_MMU 307 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; 308 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 309 cur_cpu_spec->mmu_features |= MMU_FTR_GTSE; 310 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 311 312 return 1; 313 #endif 314 return 0; 315 } 316 317 static int __init feat_enable_dscr(struct dt_cpu_feature *f) 318 { 319 u64 lpcr; 320 321 /* 322 * Linux relies on FSCR[DSCR] being clear, so that we can take the 323 * facility unavailable interrupt and track the task's usage of DSCR. 324 * See facility_unavailable_exception(). 325 * Clear the bit here so that feat_enable() doesn't set it. 326 */ 327 f->fscr_bit_nr = -1; 328 329 feat_enable(f); 330 331 lpcr = mfspr(SPRN_LPCR); 332 lpcr &= ~LPCR_DPFD; 333 lpcr |= (4UL << LPCR_DPFD_SH); 334 mtspr(SPRN_LPCR, lpcr); 335 336 return 1; 337 } 338 339 static void hfscr_pmu_enable(void) 340 { 341 u64 hfscr = mfspr(SPRN_HFSCR); 342 hfscr |= PPC_BIT(60); 343 mtspr(SPRN_HFSCR, hfscr); 344 } 345 346 static void init_pmu_power8(void) 347 { 348 if (hv_mode) { 349 mtspr(SPRN_MMCRC, 0); 350 mtspr(SPRN_MMCRH, 0); 351 } 352 353 mtspr(SPRN_MMCRA, 0); 354 mtspr(SPRN_MMCR0, 0); 355 mtspr(SPRN_MMCR1, 0); 356 mtspr(SPRN_MMCR2, 0); 357 mtspr(SPRN_MMCRS, 0); 358 } 359 360 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) 361 { 362 cur_cpu_spec->platform = "power8"; 363 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; 364 365 return 1; 366 } 367 368 static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) 369 { 370 hfscr_pmu_enable(); 371 372 init_pmu_power8(); 373 init_pmu_registers = init_pmu_power8; 374 375 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 376 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 377 if (pvr_version_is(PVR_POWER8E)) 378 cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; 379 380 cur_cpu_spec->num_pmcs = 6; 381 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 382 cur_cpu_spec->oprofile_cpu_type = "ppc64/power8"; 383 384 return 1; 385 } 386 387 static void init_pmu_power9(void) 388 { 389 if (hv_mode) 390 mtspr(SPRN_MMCRC, 0); 391 392 mtspr(SPRN_MMCRA, 0); 393 mtspr(SPRN_MMCR0, 0); 394 mtspr(SPRN_MMCR1, 0); 395 mtspr(SPRN_MMCR2, 0); 396 } 397 398 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) 399 { 400 cur_cpu_spec->platform = "power9"; 401 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; 402 403 return 1; 404 } 405 406 static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) 407 { 408 hfscr_pmu_enable(); 409 410 init_pmu_power9(); 411 init_pmu_registers = init_pmu_power9; 412 413 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 414 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 415 416 cur_cpu_spec->num_pmcs = 6; 417 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 418 cur_cpu_spec->oprofile_cpu_type = "ppc64/power9"; 419 420 return 1; 421 } 422 423 static void init_pmu_power10(void) 424 { 425 init_pmu_power9(); 426 427 mtspr(SPRN_MMCR3, 0); 428 mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); 429 mtspr(SPRN_MMCR0, MMCR0_PMCCEXT); 430 } 431 432 static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f) 433 { 434 hfscr_pmu_enable(); 435 436 init_pmu_power10(); 437 init_pmu_registers = init_pmu_power10; 438 439 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 440 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 441 442 cur_cpu_spec->num_pmcs = 6; 443 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 444 cur_cpu_spec->oprofile_cpu_type = "ppc64/power10"; 445 446 return 1; 447 } 448 449 static int __init feat_enable_mce_power10(struct dt_cpu_feature *f) 450 { 451 cur_cpu_spec->platform = "power10"; 452 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 453 454 return 1; 455 } 456 457 static int __init feat_enable_tm(struct dt_cpu_feature *f) 458 { 459 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 460 feat_enable(f); 461 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; 462 return 1; 463 #endif 464 return 0; 465 } 466 467 static int __init feat_enable_fp(struct dt_cpu_feature *f) 468 { 469 feat_enable(f); 470 cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; 471 472 return 1; 473 } 474 475 static int __init feat_enable_vector(struct dt_cpu_feature *f) 476 { 477 #ifdef CONFIG_ALTIVEC 478 feat_enable(f); 479 cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; 480 cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; 481 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; 482 483 return 1; 484 #endif 485 return 0; 486 } 487 488 static int __init feat_enable_vsx(struct dt_cpu_feature *f) 489 { 490 #ifdef CONFIG_VSX 491 feat_enable(f); 492 cur_cpu_spec->cpu_features |= CPU_FTR_VSX; 493 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; 494 495 return 1; 496 #endif 497 return 0; 498 } 499 500 static int __init feat_enable_purr(struct dt_cpu_feature *f) 501 { 502 cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; 503 504 return 1; 505 } 506 507 static int __init feat_enable_ebb(struct dt_cpu_feature *f) 508 { 509 /* 510 * PPC_FEATURE2_EBB is enabled in PMU init code because it has 511 * historically been related to the PMU facility. This may have 512 * to be decoupled if EBB becomes more generic. For now, follow 513 * existing convention. 514 */ 515 f->hwcap_bit_nr = -1; 516 feat_enable(f); 517 518 return 1; 519 } 520 521 static int __init feat_enable_dbell(struct dt_cpu_feature *f) 522 { 523 u64 lpcr; 524 525 /* P9 has an HFSCR for privileged state */ 526 feat_enable(f); 527 528 cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; 529 530 lpcr = mfspr(SPRN_LPCR); 531 lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ 532 mtspr(SPRN_LPCR, lpcr); 533 534 return 1; 535 } 536 537 static int __init feat_enable_hvi(struct dt_cpu_feature *f) 538 { 539 u64 lpcr; 540 541 /* 542 * POWER9 XIVE interrupts including in OPAL XICS compatibility 543 * are always delivered as hypervisor virtualization interrupts (HVI) 544 * rather than EE. 545 * 546 * However LPES0 is not set here, in the chance that an EE does get 547 * delivered to the host somehow, the EE handler would not expect it 548 * to be delivered in LPES0 mode (e.g., using SRR[01]). This could 549 * happen if there is a bug in interrupt controller code, or IC is 550 * misconfigured in systemsim. 551 */ 552 553 lpcr = mfspr(SPRN_LPCR); 554 lpcr |= LPCR_HVICE; /* enable hvi interrupts */ 555 lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ 556 lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ 557 mtspr(SPRN_LPCR, lpcr); 558 559 return 1; 560 } 561 562 static int __init feat_enable_large_ci(struct dt_cpu_feature *f) 563 { 564 cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; 565 566 return 1; 567 } 568 569 static int __init feat_enable_mma(struct dt_cpu_feature *f) 570 { 571 u64 pcr; 572 573 feat_enable(f); 574 pcr = mfspr(SPRN_PCR); 575 pcr &= ~PCR_MMA_DIS; 576 mtspr(SPRN_PCR, pcr); 577 578 return 1; 579 } 580 581 struct dt_cpu_feature_match { 582 const char *name; 583 int (*enable)(struct dt_cpu_feature *f); 584 u64 cpu_ftr_bit_mask; 585 }; 586 587 static struct dt_cpu_feature_match __initdata 588 dt_cpu_feature_match_table[] = { 589 {"hypervisor", feat_enable_hv, 0}, 590 {"big-endian", feat_enable, 0}, 591 {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, 592 {"smt", feat_enable_smt, 0}, 593 {"interrupt-facilities", feat_enable, 0}, 594 {"system-call-vectored", feat_enable, 0}, 595 {"timer-facilities", feat_enable, 0}, 596 {"timer-facilities-v3", feat_enable, 0}, 597 {"debug-facilities", feat_enable, 0}, 598 {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, 599 {"branch-tracing", feat_enable, 0}, 600 {"floating-point", feat_enable_fp, 0}, 601 {"vector", feat_enable_vector, 0}, 602 {"vector-scalar", feat_enable_vsx, 0}, 603 {"vector-scalar-v3", feat_enable, 0}, 604 {"decimal-floating-point", feat_enable, 0}, 605 {"decimal-integer", feat_enable, 0}, 606 {"quadword-load-store", feat_enable, 0}, 607 {"vector-crypto", feat_enable, 0}, 608 {"mmu-hash", feat_enable_mmu_hash, 0}, 609 {"mmu-radix", feat_enable_mmu_radix, 0}, 610 {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, 611 {"virtual-page-class-key-protection", feat_enable, 0}, 612 {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, 613 {"transactional-memory-v3", feat_enable_tm, 0}, 614 {"tm-suspend-hypervisor-assist", feat_enable, CPU_FTR_P9_TM_HV_ASSIST}, 615 {"tm-suspend-xer-so-bug", feat_enable, CPU_FTR_P9_TM_XER_SO_BUG}, 616 {"idle-nap", feat_enable_idle_nap, 0}, 617 /* alignment-interrupt-dsisr ignored */ 618 {"idle-stop", feat_enable_idle_stop, 0}, 619 {"machine-check-power8", feat_enable_mce_power8, 0}, 620 {"performance-monitor-power8", feat_enable_pmu_power8, 0}, 621 {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, 622 {"event-based-branch", feat_enable_ebb, 0}, 623 {"target-address-register", feat_enable, 0}, 624 {"branch-history-rolling-buffer", feat_enable, 0}, 625 {"control-register", feat_enable, CPU_FTR_CTRL}, 626 {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, 627 {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, 628 {"processor-utilization-of-resources-register", feat_enable_purr, 0}, 629 {"no-execute", feat_enable, 0}, 630 {"strong-access-ordering", feat_enable, CPU_FTR_SAO}, 631 {"cache-inhibited-large-page", feat_enable_large_ci, 0}, 632 {"coprocessor-icswx", feat_enable, 0}, 633 {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, 634 {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, 635 {"wait", feat_enable, 0}, 636 {"atomic-memory-operations", feat_enable, 0}, 637 {"branch-v3", feat_enable, 0}, 638 {"copy-paste", feat_enable, 0}, 639 {"decimal-floating-point-v3", feat_enable, 0}, 640 {"decimal-integer-v3", feat_enable, 0}, 641 {"fixed-point-v3", feat_enable, 0}, 642 {"floating-point-v3", feat_enable, 0}, 643 {"group-start-register", feat_enable, 0}, 644 {"pc-relative-addressing", feat_enable, 0}, 645 {"machine-check-power9", feat_enable_mce_power9, 0}, 646 {"machine-check-power10", feat_enable_mce_power10, 0}, 647 {"performance-monitor-power9", feat_enable_pmu_power9, 0}, 648 {"performance-monitor-power10", feat_enable_pmu_power10, 0}, 649 {"event-based-branch-v3", feat_enable, 0}, 650 {"random-number-generator", feat_enable, 0}, 651 {"system-call-vectored", feat_disable, 0}, 652 {"trace-interrupt-v3", feat_enable, 0}, 653 {"vector-v3", feat_enable, 0}, 654 {"vector-binary128", feat_enable, 0}, 655 {"vector-binary16", feat_enable, 0}, 656 {"wait-v3", feat_enable, 0}, 657 {"prefix-instructions", feat_enable, 0}, 658 {"matrix-multiply-assist", feat_enable_mma, 0}, 659 {"debug-facilities-v31", feat_enable, CPU_FTR_DAWR1}, 660 }; 661 662 static bool __initdata using_dt_cpu_ftrs; 663 static bool __initdata enable_unknown = true; 664 665 static int __init dt_cpu_ftrs_parse(char *str) 666 { 667 if (!str) 668 return 0; 669 670 if (!strcmp(str, "off")) 671 using_dt_cpu_ftrs = false; 672 else if (!strcmp(str, "known")) 673 enable_unknown = false; 674 else 675 return 1; 676 677 return 0; 678 } 679 early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); 680 681 static void __init cpufeatures_setup_start(u32 isa) 682 { 683 pr_info("setup for ISA %d\n", isa); 684 685 if (isa >= ISA_V3_0B) { 686 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; 687 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; 688 } 689 690 if (isa >= ISA_V3_1) { 691 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; 692 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; 693 } 694 } 695 696 static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) 697 { 698 const struct dt_cpu_feature_match *m; 699 bool known = false; 700 int i; 701 702 for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { 703 m = &dt_cpu_feature_match_table[i]; 704 if (!strcmp(f->name, m->name)) { 705 known = true; 706 if (m->enable(f)) { 707 cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; 708 break; 709 } 710 711 pr_info("not enabling: %s (disabled or unsupported by kernel)\n", 712 f->name); 713 return false; 714 } 715 } 716 717 if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) { 718 pr_info("not enabling: %s (unknown and unsupported by kernel)\n", 719 f->name); 720 return false; 721 } 722 723 if (known) 724 pr_debug("enabling: %s\n", f->name); 725 else 726 pr_debug("enabling: %s (unknown)\n", f->name); 727 728 return true; 729 } 730 731 /* 732 * Handle POWER9 broadcast tlbie invalidation issue using 733 * cpu feature flag. 734 */ 735 static __init void update_tlbie_feature_flag(unsigned long pvr) 736 { 737 if (PVR_VER(pvr) == PVR_POWER9) { 738 /* 739 * Set the tlbie feature flag for anything below 740 * Nimbus DD 2.3 and Cumulus DD 1.3 741 */ 742 if ((pvr & 0xe000) == 0) { 743 /* Nimbus */ 744 if ((pvr & 0xfff) < 0x203) 745 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 746 } else if ((pvr & 0xc000) == 0) { 747 /* Cumulus */ 748 if ((pvr & 0xfff) < 0x103) 749 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 750 } else { 751 WARN_ONCE(1, "Unknown PVR"); 752 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 753 } 754 755 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; 756 } 757 } 758 759 static __init void cpufeatures_cpu_quirks(void) 760 { 761 unsigned long version = mfspr(SPRN_PVR); 762 763 /* 764 * Not all quirks can be derived from the cpufeatures device tree. 765 */ 766 if ((version & 0xffffefff) == 0x004e0200) { 767 /* DD2.0 has no feature flag */ 768 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 769 } else if ((version & 0xffffefff) == 0x004e0201) { 770 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 771 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 772 } else if ((version & 0xffffefff) == 0x004e0202) { 773 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 774 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 775 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 776 } else if ((version & 0xffff0000) == 0x004e0000) { 777 /* DD2.1 and up have DD2_1 */ 778 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 779 } 780 781 if ((version & 0xffff0000) == 0x004e0000) { 782 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 783 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; 784 } 785 786 update_tlbie_feature_flag(version); 787 } 788 789 static void __init cpufeatures_setup_finished(void) 790 { 791 cpufeatures_cpu_quirks(); 792 793 if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { 794 pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); 795 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 796 } 797 798 /* Make sure powerpc_base_platform is non-NULL */ 799 powerpc_base_platform = cur_cpu_spec->platform; 800 801 system_registers.lpcr = mfspr(SPRN_LPCR); 802 system_registers.hfscr = mfspr(SPRN_HFSCR); 803 system_registers.fscr = mfspr(SPRN_FSCR); 804 system_registers.pcr = mfspr(SPRN_PCR); 805 806 pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", 807 cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); 808 } 809 810 static int __init disabled_on_cmdline(void) 811 { 812 unsigned long root, chosen; 813 const char *p; 814 815 root = of_get_flat_dt_root(); 816 chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); 817 if (chosen == -FDT_ERR_NOTFOUND) 818 return false; 819 820 p = of_get_flat_dt_prop(chosen, "bootargs", NULL); 821 if (!p) 822 return false; 823 824 if (strstr(p, "dt_cpu_ftrs=off")) 825 return true; 826 827 return false; 828 } 829 830 static int __init fdt_find_cpu_features(unsigned long node, const char *uname, 831 int depth, void *data) 832 { 833 if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") 834 && of_get_flat_dt_prop(node, "isa", NULL)) 835 return 1; 836 837 return 0; 838 } 839 840 bool __init dt_cpu_ftrs_in_use(void) 841 { 842 return using_dt_cpu_ftrs; 843 } 844 845 bool __init dt_cpu_ftrs_init(void *fdt) 846 { 847 using_dt_cpu_ftrs = false; 848 849 /* Setup and verify the FDT, if it fails we just bail */ 850 if (!early_init_dt_verify(fdt)) 851 return false; 852 853 if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) 854 return false; 855 856 if (disabled_on_cmdline()) 857 return false; 858 859 cpufeatures_setup_cpu(); 860 861 using_dt_cpu_ftrs = true; 862 return true; 863 } 864 865 static int nr_dt_cpu_features; 866 static struct dt_cpu_feature *dt_cpu_features; 867 868 static int __init process_cpufeatures_node(unsigned long node, 869 const char *uname, int i) 870 { 871 const __be32 *prop; 872 struct dt_cpu_feature *f; 873 int len; 874 875 f = &dt_cpu_features[i]; 876 877 f->node = node; 878 879 f->name = uname; 880 881 prop = of_get_flat_dt_prop(node, "isa", &len); 882 if (!prop) { 883 pr_warn("%s: missing isa property\n", uname); 884 return 0; 885 } 886 f->isa = be32_to_cpup(prop); 887 888 prop = of_get_flat_dt_prop(node, "usable-privilege", &len); 889 if (!prop) { 890 pr_warn("%s: missing usable-privilege property", uname); 891 return 0; 892 } 893 f->usable_privilege = be32_to_cpup(prop); 894 895 prop = of_get_flat_dt_prop(node, "hv-support", &len); 896 if (prop) 897 f->hv_support = be32_to_cpup(prop); 898 else 899 f->hv_support = HV_SUPPORT_NONE; 900 901 prop = of_get_flat_dt_prop(node, "os-support", &len); 902 if (prop) 903 f->os_support = be32_to_cpup(prop); 904 else 905 f->os_support = OS_SUPPORT_NONE; 906 907 prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); 908 if (prop) 909 f->hfscr_bit_nr = be32_to_cpup(prop); 910 else 911 f->hfscr_bit_nr = -1; 912 prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); 913 if (prop) 914 f->fscr_bit_nr = be32_to_cpup(prop); 915 else 916 f->fscr_bit_nr = -1; 917 prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); 918 if (prop) 919 f->hwcap_bit_nr = be32_to_cpup(prop); 920 else 921 f->hwcap_bit_nr = -1; 922 923 if (f->usable_privilege & USABLE_HV) { 924 if (!(mfmsr() & MSR_HV)) { 925 pr_warn("%s: HV feature passed to guest\n", uname); 926 return 0; 927 } 928 929 if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { 930 pr_warn("%s: unwanted hfscr_bit_nr\n", uname); 931 return 0; 932 } 933 934 if (f->hv_support == HV_SUPPORT_HFSCR) { 935 if (f->hfscr_bit_nr == -1) { 936 pr_warn("%s: missing hfscr_bit_nr\n", uname); 937 return 0; 938 } 939 } 940 } else { 941 if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { 942 pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); 943 return 0; 944 } 945 } 946 947 if (f->usable_privilege & USABLE_OS) { 948 if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { 949 pr_warn("%s: unwanted fscr_bit_nr\n", uname); 950 return 0; 951 } 952 953 if (f->os_support == OS_SUPPORT_FSCR) { 954 if (f->fscr_bit_nr == -1) { 955 pr_warn("%s: missing fscr_bit_nr\n", uname); 956 return 0; 957 } 958 } 959 } else { 960 if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { 961 pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); 962 return 0; 963 } 964 } 965 966 if (!(f->usable_privilege & USABLE_PR)) { 967 if (f->hwcap_bit_nr != -1) { 968 pr_warn("%s: unwanted hwcap_bit_nr\n", uname); 969 return 0; 970 } 971 } 972 973 /* Do all the independent features in the first pass */ 974 if (!of_get_flat_dt_prop(node, "dependencies", &len)) { 975 if (cpufeatures_process_feature(f)) 976 f->enabled = 1; 977 else 978 f->disabled = 1; 979 } 980 981 return 0; 982 } 983 984 static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) 985 { 986 const __be32 *prop; 987 int len; 988 int nr_deps; 989 int i; 990 991 if (f->enabled || f->disabled) 992 return; 993 994 prop = of_get_flat_dt_prop(f->node, "dependencies", &len); 995 if (!prop) { 996 pr_warn("%s: missing dependencies property", f->name); 997 return; 998 } 999 1000 nr_deps = len / sizeof(int); 1001 1002 for (i = 0; i < nr_deps; i++) { 1003 unsigned long phandle = be32_to_cpu(prop[i]); 1004 int j; 1005 1006 for (j = 0; j < nr_dt_cpu_features; j++) { 1007 struct dt_cpu_feature *d = &dt_cpu_features[j]; 1008 1009 if (of_get_flat_dt_phandle(d->node) == phandle) { 1010 cpufeatures_deps_enable(d); 1011 if (d->disabled) { 1012 f->disabled = 1; 1013 return; 1014 } 1015 } 1016 } 1017 } 1018 1019 if (cpufeatures_process_feature(f)) 1020 f->enabled = 1; 1021 else 1022 f->disabled = 1; 1023 } 1024 1025 static int __init scan_cpufeatures_subnodes(unsigned long node, 1026 const char *uname, 1027 void *data) 1028 { 1029 int *count = data; 1030 1031 process_cpufeatures_node(node, uname, *count); 1032 1033 (*count)++; 1034 1035 return 0; 1036 } 1037 1038 static int __init count_cpufeatures_subnodes(unsigned long node, 1039 const char *uname, 1040 void *data) 1041 { 1042 int *count = data; 1043 1044 (*count)++; 1045 1046 return 0; 1047 } 1048 1049 static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char 1050 *uname, int depth, void *data) 1051 { 1052 const __be32 *prop; 1053 int count, i; 1054 u32 isa; 1055 1056 /* We are scanning "ibm,powerpc-cpu-features" nodes only */ 1057 if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) 1058 return 0; 1059 1060 prop = of_get_flat_dt_prop(node, "isa", NULL); 1061 if (!prop) 1062 /* We checked before, "can't happen" */ 1063 return 0; 1064 1065 isa = be32_to_cpup(prop); 1066 1067 /* Count and allocate space for cpu features */ 1068 of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, 1069 &nr_dt_cpu_features); 1070 dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); 1071 if (!dt_cpu_features) 1072 panic("%s: Failed to allocate %zu bytes align=0x%lx\n", 1073 __func__, 1074 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, 1075 PAGE_SIZE); 1076 1077 cpufeatures_setup_start(isa); 1078 1079 /* Scan nodes into dt_cpu_features and enable those without deps */ 1080 count = 0; 1081 of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); 1082 1083 /* Recursive enable remaining features with dependencies */ 1084 for (i = 0; i < nr_dt_cpu_features; i++) { 1085 struct dt_cpu_feature *f = &dt_cpu_features[i]; 1086 1087 cpufeatures_deps_enable(f); 1088 } 1089 1090 prop = of_get_flat_dt_prop(node, "display-name", NULL); 1091 if (prop && strlen((char *)prop) != 0) { 1092 strlcpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); 1093 cur_cpu_spec->cpu_name = dt_cpu_name; 1094 } 1095 1096 cpufeatures_setup_finished(); 1097 1098 memblock_free(__pa(dt_cpu_features), 1099 sizeof(struct dt_cpu_feature)*nr_dt_cpu_features); 1100 1101 return 0; 1102 } 1103 1104 void __init dt_cpu_ftrs_scan(void) 1105 { 1106 if (!using_dt_cpu_ftrs) 1107 return; 1108 1109 of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); 1110 } 1111