1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2017, Nicholas Piggin, IBM Corporation 4 */ 5 6 #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt 7 8 #include <linux/export.h> 9 #include <linux/init.h> 10 #include <linux/jump_label.h> 11 #include <linux/libfdt.h> 12 #include <linux/memblock.h> 13 #include <linux/printk.h> 14 #include <linux/sched.h> 15 #include <linux/string.h> 16 #include <linux/threads.h> 17 18 #include <asm/cputable.h> 19 #include <asm/dt_cpu_ftrs.h> 20 #include <asm/mmu.h> 21 #include <asm/oprofile_impl.h> 22 #include <asm/prom.h> 23 #include <asm/setup.h> 24 25 26 /* Device-tree visible constants follow */ 27 #define ISA_V3_0B 3000 28 #define ISA_V3_1 3100 29 30 #define USABLE_PR (1U << 0) 31 #define USABLE_OS (1U << 1) 32 #define USABLE_HV (1U << 2) 33 34 #define HV_SUPPORT_HFSCR (1U << 0) 35 #define OS_SUPPORT_FSCR (1U << 0) 36 37 /* For parsing, we define all bits set as "NONE" case */ 38 #define HV_SUPPORT_NONE 0xffffffffU 39 #define OS_SUPPORT_NONE 0xffffffffU 40 41 struct dt_cpu_feature { 42 const char *name; 43 uint32_t isa; 44 uint32_t usable_privilege; 45 uint32_t hv_support; 46 uint32_t os_support; 47 uint32_t hfscr_bit_nr; 48 uint32_t fscr_bit_nr; 49 uint32_t hwcap_bit_nr; 50 /* fdt parsing */ 51 unsigned long node; 52 int enabled; 53 int disabled; 54 }; 55 56 #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) 57 58 #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ 59 PPC_FEATURE_ARCH_2_06 |\ 60 PPC_FEATURE_ICACHE_SNOOP) 61 #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ 62 PPC_FEATURE2_ISEL) 63 /* 64 * Set up the base CPU 65 */ 66 67 extern long __machine_check_early_realmode_p8(struct pt_regs *regs); 68 extern long __machine_check_early_realmode_p9(struct pt_regs *regs); 69 extern long __machine_check_early_realmode_p10(struct pt_regs *regs); 70 71 static int hv_mode; 72 73 static struct { 74 u64 lpcr; 75 u64 lpcr_clear; 76 u64 hfscr; 77 u64 fscr; 78 u64 pcr; 79 } system_registers; 80 81 static void (*init_pmu_registers)(void); 82 83 static void __restore_cpu_cpufeatures(void) 84 { 85 u64 lpcr; 86 87 /* 88 * LPCR is restored by the power on engine already. It can be changed 89 * after early init e.g., by radix enable, and we have no unified API 90 * for saving and restoring such SPRs. 91 * 92 * This ->restore hook should really be removed from idle and register 93 * restore moved directly into the idle restore code, because this code 94 * doesn't know how idle is implemented or what it needs restored here. 95 * 96 * The best we can do to accommodate secondary boot and idle restore 97 * for now is "or" LPCR with existing. 98 */ 99 lpcr = mfspr(SPRN_LPCR); 100 lpcr |= system_registers.lpcr; 101 lpcr &= ~system_registers.lpcr_clear; 102 mtspr(SPRN_LPCR, lpcr); 103 if (hv_mode) { 104 mtspr(SPRN_LPID, 0); 105 mtspr(SPRN_HFSCR, system_registers.hfscr); 106 mtspr(SPRN_PCR, system_registers.pcr); 107 } 108 mtspr(SPRN_FSCR, system_registers.fscr); 109 110 if (init_pmu_registers) 111 init_pmu_registers(); 112 } 113 114 static char dt_cpu_name[64]; 115 116 static struct cpu_spec __initdata base_cpu_spec = { 117 .cpu_name = NULL, 118 .cpu_features = CPU_FTRS_DT_CPU_BASE, 119 .cpu_user_features = COMMON_USER_BASE, 120 .cpu_user_features2 = COMMON_USER2_BASE, 121 .mmu_features = 0, 122 .icache_bsize = 32, /* minimum block size, fixed by */ 123 .dcache_bsize = 32, /* cache info init. */ 124 .num_pmcs = 0, 125 .pmc_type = PPC_PMC_DEFAULT, 126 .oprofile_cpu_type = NULL, 127 .oprofile_type = PPC_OPROFILE_INVALID, 128 .cpu_setup = NULL, 129 .cpu_restore = __restore_cpu_cpufeatures, 130 .machine_check_early = NULL, 131 .platform = NULL, 132 }; 133 134 static void __init cpufeatures_setup_cpu(void) 135 { 136 set_cur_cpu_spec(&base_cpu_spec); 137 138 cur_cpu_spec->pvr_mask = -1; 139 cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); 140 141 /* Initialize the base environment -- clear FSCR/HFSCR. */ 142 hv_mode = !!(mfmsr() & MSR_HV); 143 if (hv_mode) { 144 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 145 mtspr(SPRN_HFSCR, 0); 146 } 147 mtspr(SPRN_FSCR, 0); 148 mtspr(SPRN_PCR, PCR_MASK); 149 150 /* 151 * LPCR does not get cleared, to match behaviour with secondaries 152 * in __restore_cpu_cpufeatures. Once the idle code is fixed, this 153 * could clear LPCR too. 154 */ 155 } 156 157 static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) 158 { 159 if (f->hv_support == HV_SUPPORT_NONE) { 160 } else if (f->hv_support & HV_SUPPORT_HFSCR) { 161 u64 hfscr = mfspr(SPRN_HFSCR); 162 hfscr |= 1UL << f->hfscr_bit_nr; 163 mtspr(SPRN_HFSCR, hfscr); 164 } else { 165 /* Does not have a known recipe */ 166 return 0; 167 } 168 169 if (f->os_support == OS_SUPPORT_NONE) { 170 } else if (f->os_support & OS_SUPPORT_FSCR) { 171 u64 fscr = mfspr(SPRN_FSCR); 172 fscr |= 1UL << f->fscr_bit_nr; 173 mtspr(SPRN_FSCR, fscr); 174 } else { 175 /* Does not have a known recipe */ 176 return 0; 177 } 178 179 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 180 uint32_t word = f->hwcap_bit_nr / 32; 181 uint32_t bit = f->hwcap_bit_nr % 32; 182 183 if (word == 0) 184 cur_cpu_spec->cpu_user_features |= 1U << bit; 185 else if (word == 1) 186 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 187 else 188 pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); 189 } 190 191 return 1; 192 } 193 194 static int __init feat_enable(struct dt_cpu_feature *f) 195 { 196 if (f->hv_support != HV_SUPPORT_NONE) { 197 if (f->hfscr_bit_nr != -1) { 198 u64 hfscr = mfspr(SPRN_HFSCR); 199 hfscr |= 1UL << f->hfscr_bit_nr; 200 mtspr(SPRN_HFSCR, hfscr); 201 } 202 } 203 204 if (f->os_support != OS_SUPPORT_NONE) { 205 if (f->fscr_bit_nr != -1) { 206 u64 fscr = mfspr(SPRN_FSCR); 207 fscr |= 1UL << f->fscr_bit_nr; 208 mtspr(SPRN_FSCR, fscr); 209 } 210 } 211 212 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 213 uint32_t word = f->hwcap_bit_nr / 32; 214 uint32_t bit = f->hwcap_bit_nr % 32; 215 216 if (word == 0) 217 cur_cpu_spec->cpu_user_features |= 1U << bit; 218 else if (word == 1) 219 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 220 else 221 pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); 222 } 223 224 return 1; 225 } 226 227 static int __init feat_disable(struct dt_cpu_feature *f) 228 { 229 return 0; 230 } 231 232 static int __init feat_enable_hv(struct dt_cpu_feature *f) 233 { 234 u64 lpcr; 235 236 if (!hv_mode) { 237 pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); 238 return 0; 239 } 240 241 mtspr(SPRN_LPID, 0); 242 243 lpcr = mfspr(SPRN_LPCR); 244 lpcr &= ~LPCR_LPES0; /* HV external interrupts */ 245 mtspr(SPRN_LPCR, lpcr); 246 247 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 248 249 return 1; 250 } 251 252 static int __init feat_enable_le(struct dt_cpu_feature *f) 253 { 254 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; 255 return 1; 256 } 257 258 static int __init feat_enable_smt(struct dt_cpu_feature *f) 259 { 260 cur_cpu_spec->cpu_features |= CPU_FTR_SMT; 261 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; 262 return 1; 263 } 264 265 static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) 266 { 267 u64 lpcr; 268 269 /* Set PECE wakeup modes for ISA 207 */ 270 lpcr = mfspr(SPRN_LPCR); 271 lpcr |= LPCR_PECE0; 272 lpcr |= LPCR_PECE1; 273 lpcr |= LPCR_PECE2; 274 mtspr(SPRN_LPCR, lpcr); 275 276 return 1; 277 } 278 279 static int __init feat_enable_align_dsisr(struct dt_cpu_feature *f) 280 { 281 cur_cpu_spec->cpu_features &= ~CPU_FTR_NODSISRALIGN; 282 283 return 1; 284 } 285 286 static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) 287 { 288 u64 lpcr; 289 290 /* Set PECE wakeup modes for ISAv3.0B */ 291 lpcr = mfspr(SPRN_LPCR); 292 lpcr |= LPCR_PECE0; 293 lpcr |= LPCR_PECE1; 294 lpcr |= LPCR_PECE2; 295 mtspr(SPRN_LPCR, lpcr); 296 297 return 1; 298 } 299 300 static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) 301 { 302 u64 lpcr; 303 304 lpcr = mfspr(SPRN_LPCR); 305 lpcr &= ~LPCR_ISL; 306 307 /* VRMASD */ 308 lpcr |= LPCR_VPM0; 309 lpcr &= ~LPCR_VPM1; 310 lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ 311 mtspr(SPRN_LPCR, lpcr); 312 313 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 314 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 315 316 return 1; 317 } 318 319 static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) 320 { 321 u64 lpcr; 322 323 system_registers.lpcr_clear |= (LPCR_ISL | LPCR_UPRT | LPCR_HR); 324 lpcr = mfspr(SPRN_LPCR); 325 lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); 326 mtspr(SPRN_LPCR, lpcr); 327 328 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 329 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 330 331 return 1; 332 } 333 334 335 static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) 336 { 337 #ifdef CONFIG_PPC_RADIX_MMU 338 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; 339 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 340 cur_cpu_spec->mmu_features |= MMU_FTR_GTSE; 341 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 342 343 return 1; 344 #endif 345 return 0; 346 } 347 348 static int __init feat_enable_dscr(struct dt_cpu_feature *f) 349 { 350 u64 lpcr; 351 352 /* 353 * Linux relies on FSCR[DSCR] being clear, so that we can take the 354 * facility unavailable interrupt and track the task's usage of DSCR. 355 * See facility_unavailable_exception(). 356 * Clear the bit here so that feat_enable() doesn't set it. 357 */ 358 f->fscr_bit_nr = -1; 359 360 feat_enable(f); 361 362 lpcr = mfspr(SPRN_LPCR); 363 lpcr &= ~LPCR_DPFD; 364 lpcr |= (4UL << LPCR_DPFD_SH); 365 mtspr(SPRN_LPCR, lpcr); 366 367 return 1; 368 } 369 370 static void hfscr_pmu_enable(void) 371 { 372 u64 hfscr = mfspr(SPRN_HFSCR); 373 hfscr |= PPC_BIT(60); 374 mtspr(SPRN_HFSCR, hfscr); 375 } 376 377 static void init_pmu_power8(void) 378 { 379 if (hv_mode) { 380 mtspr(SPRN_MMCRC, 0); 381 mtspr(SPRN_MMCRH, 0); 382 } 383 384 mtspr(SPRN_MMCRA, 0); 385 mtspr(SPRN_MMCR0, 0); 386 mtspr(SPRN_MMCR1, 0); 387 mtspr(SPRN_MMCR2, 0); 388 mtspr(SPRN_MMCRS, 0); 389 } 390 391 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) 392 { 393 cur_cpu_spec->platform = "power8"; 394 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; 395 396 return 1; 397 } 398 399 static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) 400 { 401 hfscr_pmu_enable(); 402 403 init_pmu_power8(); 404 init_pmu_registers = init_pmu_power8; 405 406 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 407 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 408 if (pvr_version_is(PVR_POWER8E)) 409 cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; 410 411 cur_cpu_spec->num_pmcs = 6; 412 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 413 cur_cpu_spec->oprofile_cpu_type = "ppc64/power8"; 414 415 return 1; 416 } 417 418 static void init_pmu_power9(void) 419 { 420 if (hv_mode) 421 mtspr(SPRN_MMCRC, 0); 422 423 mtspr(SPRN_MMCRA, 0); 424 mtspr(SPRN_MMCR0, 0); 425 mtspr(SPRN_MMCR1, 0); 426 mtspr(SPRN_MMCR2, 0); 427 } 428 429 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) 430 { 431 cur_cpu_spec->platform = "power9"; 432 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; 433 434 return 1; 435 } 436 437 static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) 438 { 439 hfscr_pmu_enable(); 440 441 init_pmu_power9(); 442 init_pmu_registers = init_pmu_power9; 443 444 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 445 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 446 447 cur_cpu_spec->num_pmcs = 6; 448 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 449 cur_cpu_spec->oprofile_cpu_type = "ppc64/power9"; 450 451 return 1; 452 } 453 454 static void init_pmu_power10(void) 455 { 456 init_pmu_power9(); 457 458 mtspr(SPRN_MMCR3, 0); 459 mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); 460 } 461 462 static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f) 463 { 464 hfscr_pmu_enable(); 465 466 init_pmu_power10(); 467 init_pmu_registers = init_pmu_power10; 468 469 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 470 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 471 472 cur_cpu_spec->num_pmcs = 6; 473 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 474 cur_cpu_spec->oprofile_cpu_type = "ppc64/power10"; 475 476 return 1; 477 } 478 479 static int __init feat_enable_mce_power10(struct dt_cpu_feature *f) 480 { 481 cur_cpu_spec->platform = "power10"; 482 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 483 484 return 1; 485 } 486 487 static int __init feat_enable_tm(struct dt_cpu_feature *f) 488 { 489 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 490 feat_enable(f); 491 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; 492 return 1; 493 #endif 494 return 0; 495 } 496 497 static int __init feat_enable_fp(struct dt_cpu_feature *f) 498 { 499 feat_enable(f); 500 cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; 501 502 return 1; 503 } 504 505 static int __init feat_enable_vector(struct dt_cpu_feature *f) 506 { 507 #ifdef CONFIG_ALTIVEC 508 feat_enable(f); 509 cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; 510 cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; 511 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; 512 513 return 1; 514 #endif 515 return 0; 516 } 517 518 static int __init feat_enable_vsx(struct dt_cpu_feature *f) 519 { 520 #ifdef CONFIG_VSX 521 feat_enable(f); 522 cur_cpu_spec->cpu_features |= CPU_FTR_VSX; 523 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; 524 525 return 1; 526 #endif 527 return 0; 528 } 529 530 static int __init feat_enable_purr(struct dt_cpu_feature *f) 531 { 532 cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; 533 534 return 1; 535 } 536 537 static int __init feat_enable_ebb(struct dt_cpu_feature *f) 538 { 539 /* 540 * PPC_FEATURE2_EBB is enabled in PMU init code because it has 541 * historically been related to the PMU facility. This may have 542 * to be decoupled if EBB becomes more generic. For now, follow 543 * existing convention. 544 */ 545 f->hwcap_bit_nr = -1; 546 feat_enable(f); 547 548 return 1; 549 } 550 551 static int __init feat_enable_dbell(struct dt_cpu_feature *f) 552 { 553 u64 lpcr; 554 555 /* P9 has an HFSCR for privileged state */ 556 feat_enable(f); 557 558 cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; 559 560 lpcr = mfspr(SPRN_LPCR); 561 lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ 562 mtspr(SPRN_LPCR, lpcr); 563 564 return 1; 565 } 566 567 static int __init feat_enable_hvi(struct dt_cpu_feature *f) 568 { 569 u64 lpcr; 570 571 /* 572 * POWER9 XIVE interrupts including in OPAL XICS compatibility 573 * are always delivered as hypervisor virtualization interrupts (HVI) 574 * rather than EE. 575 * 576 * However LPES0 is not set here, in the chance that an EE does get 577 * delivered to the host somehow, the EE handler would not expect it 578 * to be delivered in LPES0 mode (e.g., using SRR[01]). This could 579 * happen if there is a bug in interrupt controller code, or IC is 580 * misconfigured in systemsim. 581 */ 582 583 lpcr = mfspr(SPRN_LPCR); 584 lpcr |= LPCR_HVICE; /* enable hvi interrupts */ 585 lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ 586 lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ 587 mtspr(SPRN_LPCR, lpcr); 588 589 return 1; 590 } 591 592 static int __init feat_enable_large_ci(struct dt_cpu_feature *f) 593 { 594 cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; 595 596 return 1; 597 } 598 599 static int __init feat_enable_mma(struct dt_cpu_feature *f) 600 { 601 u64 pcr; 602 603 feat_enable(f); 604 pcr = mfspr(SPRN_PCR); 605 pcr &= ~PCR_MMA_DIS; 606 mtspr(SPRN_PCR, pcr); 607 608 return 1; 609 } 610 611 struct dt_cpu_feature_match { 612 const char *name; 613 int (*enable)(struct dt_cpu_feature *f); 614 u64 cpu_ftr_bit_mask; 615 }; 616 617 static struct dt_cpu_feature_match __initdata 618 dt_cpu_feature_match_table[] = { 619 {"hypervisor", feat_enable_hv, 0}, 620 {"big-endian", feat_enable, 0}, 621 {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, 622 {"smt", feat_enable_smt, 0}, 623 {"interrupt-facilities", feat_enable, 0}, 624 {"system-call-vectored", feat_enable, 0}, 625 {"timer-facilities", feat_enable, 0}, 626 {"timer-facilities-v3", feat_enable, 0}, 627 {"debug-facilities", feat_enable, 0}, 628 {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, 629 {"branch-tracing", feat_enable, 0}, 630 {"floating-point", feat_enable_fp, 0}, 631 {"vector", feat_enable_vector, 0}, 632 {"vector-scalar", feat_enable_vsx, 0}, 633 {"vector-scalar-v3", feat_enable, 0}, 634 {"decimal-floating-point", feat_enable, 0}, 635 {"decimal-integer", feat_enable, 0}, 636 {"quadword-load-store", feat_enable, 0}, 637 {"vector-crypto", feat_enable, 0}, 638 {"mmu-hash", feat_enable_mmu_hash, 0}, 639 {"mmu-radix", feat_enable_mmu_radix, 0}, 640 {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, 641 {"virtual-page-class-key-protection", feat_enable, 0}, 642 {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, 643 {"transactional-memory-v3", feat_enable_tm, 0}, 644 {"tm-suspend-hypervisor-assist", feat_enable, CPU_FTR_P9_TM_HV_ASSIST}, 645 {"tm-suspend-xer-so-bug", feat_enable, CPU_FTR_P9_TM_XER_SO_BUG}, 646 {"idle-nap", feat_enable_idle_nap, 0}, 647 {"alignment-interrupt-dsisr", feat_enable_align_dsisr, 0}, 648 {"idle-stop", feat_enable_idle_stop, 0}, 649 {"machine-check-power8", feat_enable_mce_power8, 0}, 650 {"performance-monitor-power8", feat_enable_pmu_power8, 0}, 651 {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, 652 {"event-based-branch", feat_enable_ebb, 0}, 653 {"target-address-register", feat_enable, 0}, 654 {"branch-history-rolling-buffer", feat_enable, 0}, 655 {"control-register", feat_enable, CPU_FTR_CTRL}, 656 {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, 657 {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, 658 {"processor-utilization-of-resources-register", feat_enable_purr, 0}, 659 {"no-execute", feat_enable, 0}, 660 /* strong-access-ordering is unused */ 661 {"cache-inhibited-large-page", feat_enable_large_ci, 0}, 662 {"coprocessor-icswx", feat_enable, 0}, 663 {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, 664 {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, 665 {"wait", feat_enable, 0}, 666 {"atomic-memory-operations", feat_enable, 0}, 667 {"branch-v3", feat_enable, 0}, 668 {"copy-paste", feat_enable, 0}, 669 {"decimal-floating-point-v3", feat_enable, 0}, 670 {"decimal-integer-v3", feat_enable, 0}, 671 {"fixed-point-v3", feat_enable, 0}, 672 {"floating-point-v3", feat_enable, 0}, 673 {"group-start-register", feat_enable, 0}, 674 {"pc-relative-addressing", feat_enable, 0}, 675 {"machine-check-power9", feat_enable_mce_power9, 0}, 676 {"machine-check-power10", feat_enable_mce_power10, 0}, 677 {"performance-monitor-power9", feat_enable_pmu_power9, 0}, 678 {"performance-monitor-power10", feat_enable_pmu_power10, 0}, 679 {"event-based-branch-v3", feat_enable, 0}, 680 {"random-number-generator", feat_enable, 0}, 681 {"system-call-vectored", feat_disable, 0}, 682 {"trace-interrupt-v3", feat_enable, 0}, 683 {"vector-v3", feat_enable, 0}, 684 {"vector-binary128", feat_enable, 0}, 685 {"vector-binary16", feat_enable, 0}, 686 {"wait-v3", feat_enable, 0}, 687 {"prefix-instructions", feat_enable, 0}, 688 {"matrix-multiply-assist", feat_enable_mma, 0}, 689 {"debug-facilities-v31", feat_enable, CPU_FTR_DAWR1}, 690 }; 691 692 static bool __initdata using_dt_cpu_ftrs; 693 static bool __initdata enable_unknown = true; 694 695 static int __init dt_cpu_ftrs_parse(char *str) 696 { 697 if (!str) 698 return 0; 699 700 if (!strcmp(str, "off")) 701 using_dt_cpu_ftrs = false; 702 else if (!strcmp(str, "known")) 703 enable_unknown = false; 704 else 705 return 1; 706 707 return 0; 708 } 709 early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); 710 711 static void __init cpufeatures_setup_start(u32 isa) 712 { 713 pr_info("setup for ISA %d\n", isa); 714 715 if (isa >= ISA_V3_0B) { 716 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; 717 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; 718 } 719 720 if (isa >= ISA_V3_1) { 721 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; 722 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; 723 } 724 } 725 726 static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) 727 { 728 const struct dt_cpu_feature_match *m; 729 bool known = false; 730 int i; 731 732 for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { 733 m = &dt_cpu_feature_match_table[i]; 734 if (!strcmp(f->name, m->name)) { 735 known = true; 736 if (m->enable(f)) { 737 cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; 738 break; 739 } 740 741 pr_info("not enabling: %s (disabled or unsupported by kernel)\n", 742 f->name); 743 return false; 744 } 745 } 746 747 if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) { 748 pr_info("not enabling: %s (unknown and unsupported by kernel)\n", 749 f->name); 750 return false; 751 } 752 753 if (known) 754 pr_debug("enabling: %s\n", f->name); 755 else 756 pr_debug("enabling: %s (unknown)\n", f->name); 757 758 return true; 759 } 760 761 /* 762 * Handle POWER9 broadcast tlbie invalidation issue using 763 * cpu feature flag. 764 */ 765 static __init void update_tlbie_feature_flag(unsigned long pvr) 766 { 767 if (PVR_VER(pvr) == PVR_POWER9) { 768 /* 769 * Set the tlbie feature flag for anything below 770 * Nimbus DD 2.3 and Cumulus DD 1.3 771 */ 772 if ((pvr & 0xe000) == 0) { 773 /* Nimbus */ 774 if ((pvr & 0xfff) < 0x203) 775 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 776 } else if ((pvr & 0xc000) == 0) { 777 /* Cumulus */ 778 if ((pvr & 0xfff) < 0x103) 779 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 780 } else { 781 WARN_ONCE(1, "Unknown PVR"); 782 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 783 } 784 785 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; 786 } 787 } 788 789 static __init void cpufeatures_cpu_quirks(void) 790 { 791 unsigned long version = mfspr(SPRN_PVR); 792 793 /* 794 * Not all quirks can be derived from the cpufeatures device tree. 795 */ 796 if ((version & 0xffffefff) == 0x004e0200) { 797 /* DD2.0 has no feature flag */ 798 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 799 } else if ((version & 0xffffefff) == 0x004e0201) { 800 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 801 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 802 } else if ((version & 0xffffefff) == 0x004e0202) { 803 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 804 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 805 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 806 } else if ((version & 0xffff0000) == 0x004e0000) { 807 /* DD2.1 and up have DD2_1 */ 808 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 809 } 810 811 if ((version & 0xffff0000) == 0x004e0000) { 812 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 813 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; 814 } 815 816 update_tlbie_feature_flag(version); 817 } 818 819 static void __init cpufeatures_setup_finished(void) 820 { 821 cpufeatures_cpu_quirks(); 822 823 if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { 824 pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); 825 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 826 } 827 828 /* Make sure powerpc_base_platform is non-NULL */ 829 powerpc_base_platform = cur_cpu_spec->platform; 830 831 system_registers.lpcr = mfspr(SPRN_LPCR); 832 system_registers.hfscr = mfspr(SPRN_HFSCR); 833 system_registers.fscr = mfspr(SPRN_FSCR); 834 system_registers.pcr = mfspr(SPRN_PCR); 835 836 pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", 837 cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); 838 } 839 840 static int __init disabled_on_cmdline(void) 841 { 842 unsigned long root, chosen; 843 const char *p; 844 845 root = of_get_flat_dt_root(); 846 chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); 847 if (chosen == -FDT_ERR_NOTFOUND) 848 return false; 849 850 p = of_get_flat_dt_prop(chosen, "bootargs", NULL); 851 if (!p) 852 return false; 853 854 if (strstr(p, "dt_cpu_ftrs=off")) 855 return true; 856 857 return false; 858 } 859 860 static int __init fdt_find_cpu_features(unsigned long node, const char *uname, 861 int depth, void *data) 862 { 863 if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") 864 && of_get_flat_dt_prop(node, "isa", NULL)) 865 return 1; 866 867 return 0; 868 } 869 870 bool __init dt_cpu_ftrs_in_use(void) 871 { 872 return using_dt_cpu_ftrs; 873 } 874 875 bool __init dt_cpu_ftrs_init(void *fdt) 876 { 877 using_dt_cpu_ftrs = false; 878 879 /* Setup and verify the FDT, if it fails we just bail */ 880 if (!early_init_dt_verify(fdt)) 881 return false; 882 883 if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) 884 return false; 885 886 if (disabled_on_cmdline()) 887 return false; 888 889 cpufeatures_setup_cpu(); 890 891 using_dt_cpu_ftrs = true; 892 return true; 893 } 894 895 static int nr_dt_cpu_features; 896 static struct dt_cpu_feature *dt_cpu_features; 897 898 static int __init process_cpufeatures_node(unsigned long node, 899 const char *uname, int i) 900 { 901 const __be32 *prop; 902 struct dt_cpu_feature *f; 903 int len; 904 905 f = &dt_cpu_features[i]; 906 907 f->node = node; 908 909 f->name = uname; 910 911 prop = of_get_flat_dt_prop(node, "isa", &len); 912 if (!prop) { 913 pr_warn("%s: missing isa property\n", uname); 914 return 0; 915 } 916 f->isa = be32_to_cpup(prop); 917 918 prop = of_get_flat_dt_prop(node, "usable-privilege", &len); 919 if (!prop) { 920 pr_warn("%s: missing usable-privilege property", uname); 921 return 0; 922 } 923 f->usable_privilege = be32_to_cpup(prop); 924 925 prop = of_get_flat_dt_prop(node, "hv-support", &len); 926 if (prop) 927 f->hv_support = be32_to_cpup(prop); 928 else 929 f->hv_support = HV_SUPPORT_NONE; 930 931 prop = of_get_flat_dt_prop(node, "os-support", &len); 932 if (prop) 933 f->os_support = be32_to_cpup(prop); 934 else 935 f->os_support = OS_SUPPORT_NONE; 936 937 prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); 938 if (prop) 939 f->hfscr_bit_nr = be32_to_cpup(prop); 940 else 941 f->hfscr_bit_nr = -1; 942 prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); 943 if (prop) 944 f->fscr_bit_nr = be32_to_cpup(prop); 945 else 946 f->fscr_bit_nr = -1; 947 prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); 948 if (prop) 949 f->hwcap_bit_nr = be32_to_cpup(prop); 950 else 951 f->hwcap_bit_nr = -1; 952 953 if (f->usable_privilege & USABLE_HV) { 954 if (!(mfmsr() & MSR_HV)) { 955 pr_warn("%s: HV feature passed to guest\n", uname); 956 return 0; 957 } 958 959 if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { 960 pr_warn("%s: unwanted hfscr_bit_nr\n", uname); 961 return 0; 962 } 963 964 if (f->hv_support == HV_SUPPORT_HFSCR) { 965 if (f->hfscr_bit_nr == -1) { 966 pr_warn("%s: missing hfscr_bit_nr\n", uname); 967 return 0; 968 } 969 } 970 } else { 971 if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { 972 pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); 973 return 0; 974 } 975 } 976 977 if (f->usable_privilege & USABLE_OS) { 978 if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { 979 pr_warn("%s: unwanted fscr_bit_nr\n", uname); 980 return 0; 981 } 982 983 if (f->os_support == OS_SUPPORT_FSCR) { 984 if (f->fscr_bit_nr == -1) { 985 pr_warn("%s: missing fscr_bit_nr\n", uname); 986 return 0; 987 } 988 } 989 } else { 990 if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { 991 pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); 992 return 0; 993 } 994 } 995 996 if (!(f->usable_privilege & USABLE_PR)) { 997 if (f->hwcap_bit_nr != -1) { 998 pr_warn("%s: unwanted hwcap_bit_nr\n", uname); 999 return 0; 1000 } 1001 } 1002 1003 /* Do all the independent features in the first pass */ 1004 if (!of_get_flat_dt_prop(node, "dependencies", &len)) { 1005 if (cpufeatures_process_feature(f)) 1006 f->enabled = 1; 1007 else 1008 f->disabled = 1; 1009 } 1010 1011 return 0; 1012 } 1013 1014 static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) 1015 { 1016 const __be32 *prop; 1017 int len; 1018 int nr_deps; 1019 int i; 1020 1021 if (f->enabled || f->disabled) 1022 return; 1023 1024 prop = of_get_flat_dt_prop(f->node, "dependencies", &len); 1025 if (!prop) { 1026 pr_warn("%s: missing dependencies property", f->name); 1027 return; 1028 } 1029 1030 nr_deps = len / sizeof(int); 1031 1032 for (i = 0; i < nr_deps; i++) { 1033 unsigned long phandle = be32_to_cpu(prop[i]); 1034 int j; 1035 1036 for (j = 0; j < nr_dt_cpu_features; j++) { 1037 struct dt_cpu_feature *d = &dt_cpu_features[j]; 1038 1039 if (of_get_flat_dt_phandle(d->node) == phandle) { 1040 cpufeatures_deps_enable(d); 1041 if (d->disabled) { 1042 f->disabled = 1; 1043 return; 1044 } 1045 } 1046 } 1047 } 1048 1049 if (cpufeatures_process_feature(f)) 1050 f->enabled = 1; 1051 else 1052 f->disabled = 1; 1053 } 1054 1055 static int __init scan_cpufeatures_subnodes(unsigned long node, 1056 const char *uname, 1057 void *data) 1058 { 1059 int *count = data; 1060 1061 process_cpufeatures_node(node, uname, *count); 1062 1063 (*count)++; 1064 1065 return 0; 1066 } 1067 1068 static int __init count_cpufeatures_subnodes(unsigned long node, 1069 const char *uname, 1070 void *data) 1071 { 1072 int *count = data; 1073 1074 (*count)++; 1075 1076 return 0; 1077 } 1078 1079 static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char 1080 *uname, int depth, void *data) 1081 { 1082 const __be32 *prop; 1083 int count, i; 1084 u32 isa; 1085 1086 /* We are scanning "ibm,powerpc-cpu-features" nodes only */ 1087 if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) 1088 return 0; 1089 1090 prop = of_get_flat_dt_prop(node, "isa", NULL); 1091 if (!prop) 1092 /* We checked before, "can't happen" */ 1093 return 0; 1094 1095 isa = be32_to_cpup(prop); 1096 1097 /* Count and allocate space for cpu features */ 1098 of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, 1099 &nr_dt_cpu_features); 1100 dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); 1101 if (!dt_cpu_features) 1102 panic("%s: Failed to allocate %zu bytes align=0x%lx\n", 1103 __func__, 1104 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, 1105 PAGE_SIZE); 1106 1107 cpufeatures_setup_start(isa); 1108 1109 /* Scan nodes into dt_cpu_features and enable those without deps */ 1110 count = 0; 1111 of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); 1112 1113 /* Recursive enable remaining features with dependencies */ 1114 for (i = 0; i < nr_dt_cpu_features; i++) { 1115 struct dt_cpu_feature *f = &dt_cpu_features[i]; 1116 1117 cpufeatures_deps_enable(f); 1118 } 1119 1120 prop = of_get_flat_dt_prop(node, "display-name", NULL); 1121 if (prop && strlen((char *)prop) != 0) { 1122 strlcpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); 1123 cur_cpu_spec->cpu_name = dt_cpu_name; 1124 } 1125 1126 cpufeatures_setup_finished(); 1127 1128 memblock_free(__pa(dt_cpu_features), 1129 sizeof(struct dt_cpu_feature)*nr_dt_cpu_features); 1130 1131 return 0; 1132 } 1133 1134 void __init dt_cpu_ftrs_scan(void) 1135 { 1136 if (!using_dt_cpu_ftrs) 1137 return; 1138 1139 of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); 1140 } 1141