1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2017, Nicholas Piggin, IBM Corporation 4 */ 5 6 #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt 7 8 #include <linux/export.h> 9 #include <linux/init.h> 10 #include <linux/jump_label.h> 11 #include <linux/libfdt.h> 12 #include <linux/memblock.h> 13 #include <linux/printk.h> 14 #include <linux/sched.h> 15 #include <linux/string.h> 16 #include <linux/threads.h> 17 18 #include <asm/cputable.h> 19 #include <asm/dt_cpu_ftrs.h> 20 #include <asm/mce.h> 21 #include <asm/mmu.h> 22 #include <asm/oprofile_impl.h> 23 #include <asm/prom.h> 24 #include <asm/setup.h> 25 26 27 /* Device-tree visible constants follow */ 28 #define ISA_V3_0B 3000 29 #define ISA_V3_1 3100 30 31 #define USABLE_PR (1U << 0) 32 #define USABLE_OS (1U << 1) 33 #define USABLE_HV (1U << 2) 34 35 #define HV_SUPPORT_HFSCR (1U << 0) 36 #define OS_SUPPORT_FSCR (1U << 0) 37 38 /* For parsing, we define all bits set as "NONE" case */ 39 #define HV_SUPPORT_NONE 0xffffffffU 40 #define OS_SUPPORT_NONE 0xffffffffU 41 42 struct dt_cpu_feature { 43 const char *name; 44 uint32_t isa; 45 uint32_t usable_privilege; 46 uint32_t hv_support; 47 uint32_t os_support; 48 uint32_t hfscr_bit_nr; 49 uint32_t fscr_bit_nr; 50 uint32_t hwcap_bit_nr; 51 /* fdt parsing */ 52 unsigned long node; 53 int enabled; 54 int disabled; 55 }; 56 57 #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) 58 59 #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ 60 PPC_FEATURE_ARCH_2_06 |\ 61 PPC_FEATURE_ICACHE_SNOOP) 62 #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ 63 PPC_FEATURE2_ISEL) 64 /* 65 * Set up the base CPU 66 */ 67 68 static int hv_mode; 69 70 static struct { 71 u64 lpcr; 72 u64 lpcr_clear; 73 u64 hfscr; 74 u64 fscr; 75 u64 pcr; 76 } system_registers; 77 78 static void (*init_pmu_registers)(void); 79 80 static void __restore_cpu_cpufeatures(void) 81 { 82 u64 lpcr; 83 84 /* 85 * LPCR is restored by the power on engine already. It can be changed 86 * after early init e.g., by radix enable, and we have no unified API 87 * for saving and restoring such SPRs. 88 * 89 * This ->restore hook should really be removed from idle and register 90 * restore moved directly into the idle restore code, because this code 91 * doesn't know how idle is implemented or what it needs restored here. 92 * 93 * The best we can do to accommodate secondary boot and idle restore 94 * for now is "or" LPCR with existing. 95 */ 96 lpcr = mfspr(SPRN_LPCR); 97 lpcr |= system_registers.lpcr; 98 lpcr &= ~system_registers.lpcr_clear; 99 mtspr(SPRN_LPCR, lpcr); 100 if (hv_mode) { 101 mtspr(SPRN_LPID, 0); 102 mtspr(SPRN_HFSCR, system_registers.hfscr); 103 mtspr(SPRN_PCR, system_registers.pcr); 104 } 105 mtspr(SPRN_FSCR, system_registers.fscr); 106 107 if (init_pmu_registers) 108 init_pmu_registers(); 109 } 110 111 static char dt_cpu_name[64]; 112 113 static struct cpu_spec __initdata base_cpu_spec = { 114 .cpu_name = NULL, 115 .cpu_features = CPU_FTRS_DT_CPU_BASE, 116 .cpu_user_features = COMMON_USER_BASE, 117 .cpu_user_features2 = COMMON_USER2_BASE, 118 .mmu_features = 0, 119 .icache_bsize = 32, /* minimum block size, fixed by */ 120 .dcache_bsize = 32, /* cache info init. */ 121 .num_pmcs = 0, 122 .pmc_type = PPC_PMC_DEFAULT, 123 .oprofile_cpu_type = NULL, 124 .oprofile_type = PPC_OPROFILE_INVALID, 125 .cpu_setup = NULL, 126 .cpu_restore = __restore_cpu_cpufeatures, 127 .machine_check_early = NULL, 128 .platform = NULL, 129 }; 130 131 static void __init cpufeatures_setup_cpu(void) 132 { 133 set_cur_cpu_spec(&base_cpu_spec); 134 135 cur_cpu_spec->pvr_mask = -1; 136 cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); 137 138 /* Initialize the base environment -- clear FSCR/HFSCR. */ 139 hv_mode = !!(mfmsr() & MSR_HV); 140 if (hv_mode) { 141 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 142 mtspr(SPRN_HFSCR, 0); 143 } 144 mtspr(SPRN_FSCR, 0); 145 mtspr(SPRN_PCR, PCR_MASK); 146 147 /* 148 * LPCR does not get cleared, to match behaviour with secondaries 149 * in __restore_cpu_cpufeatures. Once the idle code is fixed, this 150 * could clear LPCR too. 151 */ 152 } 153 154 static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) 155 { 156 if (f->hv_support == HV_SUPPORT_NONE) { 157 } else if (f->hv_support & HV_SUPPORT_HFSCR) { 158 u64 hfscr = mfspr(SPRN_HFSCR); 159 hfscr |= 1UL << f->hfscr_bit_nr; 160 mtspr(SPRN_HFSCR, hfscr); 161 } else { 162 /* Does not have a known recipe */ 163 return 0; 164 } 165 166 if (f->os_support == OS_SUPPORT_NONE) { 167 } else if (f->os_support & OS_SUPPORT_FSCR) { 168 u64 fscr = mfspr(SPRN_FSCR); 169 fscr |= 1UL << f->fscr_bit_nr; 170 mtspr(SPRN_FSCR, fscr); 171 } else { 172 /* Does not have a known recipe */ 173 return 0; 174 } 175 176 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 177 uint32_t word = f->hwcap_bit_nr / 32; 178 uint32_t bit = f->hwcap_bit_nr % 32; 179 180 if (word == 0) 181 cur_cpu_spec->cpu_user_features |= 1U << bit; 182 else if (word == 1) 183 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 184 else 185 pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); 186 } 187 188 return 1; 189 } 190 191 static int __init feat_enable(struct dt_cpu_feature *f) 192 { 193 if (f->hv_support != HV_SUPPORT_NONE) { 194 if (f->hfscr_bit_nr != -1) { 195 u64 hfscr = mfspr(SPRN_HFSCR); 196 hfscr |= 1UL << f->hfscr_bit_nr; 197 mtspr(SPRN_HFSCR, hfscr); 198 } 199 } 200 201 if (f->os_support != OS_SUPPORT_NONE) { 202 if (f->fscr_bit_nr != -1) { 203 u64 fscr = mfspr(SPRN_FSCR); 204 fscr |= 1UL << f->fscr_bit_nr; 205 mtspr(SPRN_FSCR, fscr); 206 } 207 } 208 209 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 210 uint32_t word = f->hwcap_bit_nr / 32; 211 uint32_t bit = f->hwcap_bit_nr % 32; 212 213 if (word == 0) 214 cur_cpu_spec->cpu_user_features |= 1U << bit; 215 else if (word == 1) 216 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 217 else 218 pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); 219 } 220 221 return 1; 222 } 223 224 static int __init feat_disable(struct dt_cpu_feature *f) 225 { 226 return 0; 227 } 228 229 static int __init feat_enable_hv(struct dt_cpu_feature *f) 230 { 231 u64 lpcr; 232 233 if (!hv_mode) { 234 pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); 235 return 0; 236 } 237 238 mtspr(SPRN_LPID, 0); 239 240 lpcr = mfspr(SPRN_LPCR); 241 lpcr &= ~LPCR_LPES0; /* HV external interrupts */ 242 mtspr(SPRN_LPCR, lpcr); 243 244 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 245 246 return 1; 247 } 248 249 static int __init feat_enable_le(struct dt_cpu_feature *f) 250 { 251 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; 252 return 1; 253 } 254 255 static int __init feat_enable_smt(struct dt_cpu_feature *f) 256 { 257 cur_cpu_spec->cpu_features |= CPU_FTR_SMT; 258 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; 259 return 1; 260 } 261 262 static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) 263 { 264 u64 lpcr; 265 266 /* Set PECE wakeup modes for ISA 207 */ 267 lpcr = mfspr(SPRN_LPCR); 268 lpcr |= LPCR_PECE0; 269 lpcr |= LPCR_PECE1; 270 lpcr |= LPCR_PECE2; 271 mtspr(SPRN_LPCR, lpcr); 272 273 return 1; 274 } 275 276 static int __init feat_enable_align_dsisr(struct dt_cpu_feature *f) 277 { 278 cur_cpu_spec->cpu_features &= ~CPU_FTR_NODSISRALIGN; 279 280 return 1; 281 } 282 283 static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) 284 { 285 u64 lpcr; 286 287 /* Set PECE wakeup modes for ISAv3.0B */ 288 lpcr = mfspr(SPRN_LPCR); 289 lpcr |= LPCR_PECE0; 290 lpcr |= LPCR_PECE1; 291 lpcr |= LPCR_PECE2; 292 mtspr(SPRN_LPCR, lpcr); 293 294 return 1; 295 } 296 297 static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) 298 { 299 u64 lpcr; 300 301 lpcr = mfspr(SPRN_LPCR); 302 lpcr &= ~LPCR_ISL; 303 304 /* VRMASD */ 305 lpcr |= LPCR_VPM0; 306 lpcr &= ~LPCR_VPM1; 307 lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ 308 mtspr(SPRN_LPCR, lpcr); 309 310 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 311 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 312 313 return 1; 314 } 315 316 static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) 317 { 318 u64 lpcr; 319 320 system_registers.lpcr_clear |= (LPCR_ISL | LPCR_UPRT | LPCR_HR); 321 lpcr = mfspr(SPRN_LPCR); 322 lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); 323 mtspr(SPRN_LPCR, lpcr); 324 325 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 326 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 327 328 return 1; 329 } 330 331 332 static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) 333 { 334 #ifdef CONFIG_PPC_RADIX_MMU 335 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; 336 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 337 cur_cpu_spec->mmu_features |= MMU_FTR_GTSE; 338 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 339 340 return 1; 341 #endif 342 return 0; 343 } 344 345 static int __init feat_enable_dscr(struct dt_cpu_feature *f) 346 { 347 u64 lpcr; 348 349 /* 350 * Linux relies on FSCR[DSCR] being clear, so that we can take the 351 * facility unavailable interrupt and track the task's usage of DSCR. 352 * See facility_unavailable_exception(). 353 * Clear the bit here so that feat_enable() doesn't set it. 354 */ 355 f->fscr_bit_nr = -1; 356 357 feat_enable(f); 358 359 lpcr = mfspr(SPRN_LPCR); 360 lpcr &= ~LPCR_DPFD; 361 lpcr |= (4UL << LPCR_DPFD_SH); 362 mtspr(SPRN_LPCR, lpcr); 363 364 return 1; 365 } 366 367 static void hfscr_pmu_enable(void) 368 { 369 u64 hfscr = mfspr(SPRN_HFSCR); 370 hfscr |= PPC_BIT(60); 371 mtspr(SPRN_HFSCR, hfscr); 372 } 373 374 static void init_pmu_power8(void) 375 { 376 if (hv_mode) { 377 mtspr(SPRN_MMCRC, 0); 378 mtspr(SPRN_MMCRH, 0); 379 } 380 381 mtspr(SPRN_MMCRA, 0); 382 mtspr(SPRN_MMCR0, 0); 383 mtspr(SPRN_MMCR1, 0); 384 mtspr(SPRN_MMCR2, 0); 385 mtspr(SPRN_MMCRS, 0); 386 } 387 388 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) 389 { 390 cur_cpu_spec->platform = "power8"; 391 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; 392 393 return 1; 394 } 395 396 static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) 397 { 398 hfscr_pmu_enable(); 399 400 init_pmu_power8(); 401 init_pmu_registers = init_pmu_power8; 402 403 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 404 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 405 if (pvr_version_is(PVR_POWER8E)) 406 cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; 407 408 cur_cpu_spec->num_pmcs = 6; 409 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 410 cur_cpu_spec->oprofile_cpu_type = "ppc64/power8"; 411 412 return 1; 413 } 414 415 static void init_pmu_power9(void) 416 { 417 if (hv_mode) 418 mtspr(SPRN_MMCRC, 0); 419 420 mtspr(SPRN_MMCRA, 0); 421 mtspr(SPRN_MMCR0, 0); 422 mtspr(SPRN_MMCR1, 0); 423 mtspr(SPRN_MMCR2, 0); 424 } 425 426 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) 427 { 428 cur_cpu_spec->platform = "power9"; 429 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; 430 431 return 1; 432 } 433 434 static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) 435 { 436 hfscr_pmu_enable(); 437 438 init_pmu_power9(); 439 init_pmu_registers = init_pmu_power9; 440 441 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 442 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 443 444 cur_cpu_spec->num_pmcs = 6; 445 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 446 cur_cpu_spec->oprofile_cpu_type = "ppc64/power9"; 447 448 return 1; 449 } 450 451 static void init_pmu_power10(void) 452 { 453 init_pmu_power9(); 454 455 mtspr(SPRN_MMCR3, 0); 456 mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); 457 } 458 459 static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f) 460 { 461 hfscr_pmu_enable(); 462 463 init_pmu_power10(); 464 init_pmu_registers = init_pmu_power10; 465 466 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 467 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 468 469 cur_cpu_spec->num_pmcs = 6; 470 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 471 cur_cpu_spec->oprofile_cpu_type = "ppc64/power10"; 472 473 return 1; 474 } 475 476 static int __init feat_enable_mce_power10(struct dt_cpu_feature *f) 477 { 478 cur_cpu_spec->platform = "power10"; 479 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 480 481 return 1; 482 } 483 484 static int __init feat_enable_tm(struct dt_cpu_feature *f) 485 { 486 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 487 feat_enable(f); 488 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; 489 return 1; 490 #endif 491 return 0; 492 } 493 494 static int __init feat_enable_fp(struct dt_cpu_feature *f) 495 { 496 feat_enable(f); 497 cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; 498 499 return 1; 500 } 501 502 static int __init feat_enable_vector(struct dt_cpu_feature *f) 503 { 504 #ifdef CONFIG_ALTIVEC 505 feat_enable(f); 506 cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; 507 cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; 508 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; 509 510 return 1; 511 #endif 512 return 0; 513 } 514 515 static int __init feat_enable_vsx(struct dt_cpu_feature *f) 516 { 517 #ifdef CONFIG_VSX 518 feat_enable(f); 519 cur_cpu_spec->cpu_features |= CPU_FTR_VSX; 520 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; 521 522 return 1; 523 #endif 524 return 0; 525 } 526 527 static int __init feat_enable_purr(struct dt_cpu_feature *f) 528 { 529 cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; 530 531 return 1; 532 } 533 534 static int __init feat_enable_ebb(struct dt_cpu_feature *f) 535 { 536 /* 537 * PPC_FEATURE2_EBB is enabled in PMU init code because it has 538 * historically been related to the PMU facility. This may have 539 * to be decoupled if EBB becomes more generic. For now, follow 540 * existing convention. 541 */ 542 f->hwcap_bit_nr = -1; 543 feat_enable(f); 544 545 return 1; 546 } 547 548 static int __init feat_enable_dbell(struct dt_cpu_feature *f) 549 { 550 u64 lpcr; 551 552 /* P9 has an HFSCR for privileged state */ 553 feat_enable(f); 554 555 cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; 556 557 lpcr = mfspr(SPRN_LPCR); 558 lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ 559 mtspr(SPRN_LPCR, lpcr); 560 561 return 1; 562 } 563 564 static int __init feat_enable_hvi(struct dt_cpu_feature *f) 565 { 566 u64 lpcr; 567 568 /* 569 * POWER9 XIVE interrupts including in OPAL XICS compatibility 570 * are always delivered as hypervisor virtualization interrupts (HVI) 571 * rather than EE. 572 * 573 * However LPES0 is not set here, in the chance that an EE does get 574 * delivered to the host somehow, the EE handler would not expect it 575 * to be delivered in LPES0 mode (e.g., using SRR[01]). This could 576 * happen if there is a bug in interrupt controller code, or IC is 577 * misconfigured in systemsim. 578 */ 579 580 lpcr = mfspr(SPRN_LPCR); 581 lpcr |= LPCR_HVICE; /* enable hvi interrupts */ 582 lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ 583 lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ 584 mtspr(SPRN_LPCR, lpcr); 585 586 return 1; 587 } 588 589 static int __init feat_enable_large_ci(struct dt_cpu_feature *f) 590 { 591 cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; 592 593 return 1; 594 } 595 596 static int __init feat_enable_mma(struct dt_cpu_feature *f) 597 { 598 u64 pcr; 599 600 feat_enable(f); 601 pcr = mfspr(SPRN_PCR); 602 pcr &= ~PCR_MMA_DIS; 603 mtspr(SPRN_PCR, pcr); 604 605 return 1; 606 } 607 608 struct dt_cpu_feature_match { 609 const char *name; 610 int (*enable)(struct dt_cpu_feature *f); 611 u64 cpu_ftr_bit_mask; 612 }; 613 614 static struct dt_cpu_feature_match __initdata 615 dt_cpu_feature_match_table[] = { 616 {"hypervisor", feat_enable_hv, 0}, 617 {"big-endian", feat_enable, 0}, 618 {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, 619 {"smt", feat_enable_smt, 0}, 620 {"interrupt-facilities", feat_enable, 0}, 621 {"system-call-vectored", feat_enable, 0}, 622 {"timer-facilities", feat_enable, 0}, 623 {"timer-facilities-v3", feat_enable, 0}, 624 {"debug-facilities", feat_enable, 0}, 625 {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, 626 {"branch-tracing", feat_enable, 0}, 627 {"floating-point", feat_enable_fp, 0}, 628 {"vector", feat_enable_vector, 0}, 629 {"vector-scalar", feat_enable_vsx, 0}, 630 {"vector-scalar-v3", feat_enable, 0}, 631 {"decimal-floating-point", feat_enable, 0}, 632 {"decimal-integer", feat_enable, 0}, 633 {"quadword-load-store", feat_enable, 0}, 634 {"vector-crypto", feat_enable, 0}, 635 {"mmu-hash", feat_enable_mmu_hash, 0}, 636 {"mmu-radix", feat_enable_mmu_radix, 0}, 637 {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, 638 {"virtual-page-class-key-protection", feat_enable, 0}, 639 {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, 640 {"transactional-memory-v3", feat_enable_tm, 0}, 641 {"tm-suspend-hypervisor-assist", feat_enable, CPU_FTR_P9_TM_HV_ASSIST}, 642 {"tm-suspend-xer-so-bug", feat_enable, CPU_FTR_P9_TM_XER_SO_BUG}, 643 {"idle-nap", feat_enable_idle_nap, 0}, 644 {"alignment-interrupt-dsisr", feat_enable_align_dsisr, 0}, 645 {"idle-stop", feat_enable_idle_stop, 0}, 646 {"machine-check-power8", feat_enable_mce_power8, 0}, 647 {"performance-monitor-power8", feat_enable_pmu_power8, 0}, 648 {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, 649 {"event-based-branch", feat_enable_ebb, 0}, 650 {"target-address-register", feat_enable, 0}, 651 {"branch-history-rolling-buffer", feat_enable, 0}, 652 {"control-register", feat_enable, CPU_FTR_CTRL}, 653 {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, 654 {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, 655 {"processor-utilization-of-resources-register", feat_enable_purr, 0}, 656 {"no-execute", feat_enable, 0}, 657 {"strong-access-ordering", feat_enable, CPU_FTR_SAO}, 658 {"cache-inhibited-large-page", feat_enable_large_ci, 0}, 659 {"coprocessor-icswx", feat_enable, 0}, 660 {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, 661 {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, 662 {"wait", feat_enable, 0}, 663 {"atomic-memory-operations", feat_enable, 0}, 664 {"branch-v3", feat_enable, 0}, 665 {"copy-paste", feat_enable, 0}, 666 {"decimal-floating-point-v3", feat_enable, 0}, 667 {"decimal-integer-v3", feat_enable, 0}, 668 {"fixed-point-v3", feat_enable, 0}, 669 {"floating-point-v3", feat_enable, 0}, 670 {"group-start-register", feat_enable, 0}, 671 {"pc-relative-addressing", feat_enable, 0}, 672 {"machine-check-power9", feat_enable_mce_power9, 0}, 673 {"machine-check-power10", feat_enable_mce_power10, 0}, 674 {"performance-monitor-power9", feat_enable_pmu_power9, 0}, 675 {"performance-monitor-power10", feat_enable_pmu_power10, 0}, 676 {"event-based-branch-v3", feat_enable, 0}, 677 {"random-number-generator", feat_enable, 0}, 678 {"system-call-vectored", feat_disable, 0}, 679 {"trace-interrupt-v3", feat_enable, 0}, 680 {"vector-v3", feat_enable, 0}, 681 {"vector-binary128", feat_enable, 0}, 682 {"vector-binary16", feat_enable, 0}, 683 {"wait-v3", feat_enable, 0}, 684 {"prefix-instructions", feat_enable, 0}, 685 {"matrix-multiply-assist", feat_enable_mma, 0}, 686 {"debug-facilities-v31", feat_enable, CPU_FTR_DAWR1}, 687 }; 688 689 static bool __initdata using_dt_cpu_ftrs; 690 static bool __initdata enable_unknown = true; 691 692 static int __init dt_cpu_ftrs_parse(char *str) 693 { 694 if (!str) 695 return 0; 696 697 if (!strcmp(str, "off")) 698 using_dt_cpu_ftrs = false; 699 else if (!strcmp(str, "known")) 700 enable_unknown = false; 701 else 702 return 1; 703 704 return 0; 705 } 706 early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); 707 708 static void __init cpufeatures_setup_start(u32 isa) 709 { 710 pr_info("setup for ISA %d\n", isa); 711 712 if (isa >= ISA_V3_0B) { 713 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; 714 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; 715 } 716 717 if (isa >= ISA_V3_1) { 718 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; 719 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; 720 } 721 } 722 723 static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) 724 { 725 const struct dt_cpu_feature_match *m; 726 bool known = false; 727 int i; 728 729 for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { 730 m = &dt_cpu_feature_match_table[i]; 731 if (!strcmp(f->name, m->name)) { 732 known = true; 733 if (m->enable(f)) { 734 cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; 735 break; 736 } 737 738 pr_info("not enabling: %s (disabled or unsupported by kernel)\n", 739 f->name); 740 return false; 741 } 742 } 743 744 if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) { 745 pr_info("not enabling: %s (unknown and unsupported by kernel)\n", 746 f->name); 747 return false; 748 } 749 750 if (known) 751 pr_debug("enabling: %s\n", f->name); 752 else 753 pr_debug("enabling: %s (unknown)\n", f->name); 754 755 return true; 756 } 757 758 /* 759 * Handle POWER9 broadcast tlbie invalidation issue using 760 * cpu feature flag. 761 */ 762 static __init void update_tlbie_feature_flag(unsigned long pvr) 763 { 764 if (PVR_VER(pvr) == PVR_POWER9) { 765 /* 766 * Set the tlbie feature flag for anything below 767 * Nimbus DD 2.3 and Cumulus DD 1.3 768 */ 769 if ((pvr & 0xe000) == 0) { 770 /* Nimbus */ 771 if ((pvr & 0xfff) < 0x203) 772 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 773 } else if ((pvr & 0xc000) == 0) { 774 /* Cumulus */ 775 if ((pvr & 0xfff) < 0x103) 776 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 777 } else { 778 WARN_ONCE(1, "Unknown PVR"); 779 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 780 } 781 782 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; 783 } 784 } 785 786 static __init void cpufeatures_cpu_quirks(void) 787 { 788 unsigned long version = mfspr(SPRN_PVR); 789 790 /* 791 * Not all quirks can be derived from the cpufeatures device tree. 792 */ 793 if ((version & 0xffffefff) == 0x004e0200) { 794 /* DD2.0 has no feature flag */ 795 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 796 } else if ((version & 0xffffefff) == 0x004e0201) { 797 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 798 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 799 } else if ((version & 0xffffefff) == 0x004e0202) { 800 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 801 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 802 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 803 } else if ((version & 0xffff0000) == 0x004e0000) { 804 /* DD2.1 and up have DD2_1 */ 805 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 806 } 807 808 if ((version & 0xffff0000) == 0x004e0000) { 809 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 810 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; 811 } 812 813 update_tlbie_feature_flag(version); 814 } 815 816 static void __init cpufeatures_setup_finished(void) 817 { 818 cpufeatures_cpu_quirks(); 819 820 if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { 821 pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); 822 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 823 } 824 825 /* Make sure powerpc_base_platform is non-NULL */ 826 powerpc_base_platform = cur_cpu_spec->platform; 827 828 system_registers.lpcr = mfspr(SPRN_LPCR); 829 system_registers.hfscr = mfspr(SPRN_HFSCR); 830 system_registers.fscr = mfspr(SPRN_FSCR); 831 system_registers.pcr = mfspr(SPRN_PCR); 832 833 pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", 834 cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); 835 } 836 837 static int __init disabled_on_cmdline(void) 838 { 839 unsigned long root, chosen; 840 const char *p; 841 842 root = of_get_flat_dt_root(); 843 chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); 844 if (chosen == -FDT_ERR_NOTFOUND) 845 return false; 846 847 p = of_get_flat_dt_prop(chosen, "bootargs", NULL); 848 if (!p) 849 return false; 850 851 if (strstr(p, "dt_cpu_ftrs=off")) 852 return true; 853 854 return false; 855 } 856 857 static int __init fdt_find_cpu_features(unsigned long node, const char *uname, 858 int depth, void *data) 859 { 860 if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") 861 && of_get_flat_dt_prop(node, "isa", NULL)) 862 return 1; 863 864 return 0; 865 } 866 867 bool __init dt_cpu_ftrs_in_use(void) 868 { 869 return using_dt_cpu_ftrs; 870 } 871 872 bool __init dt_cpu_ftrs_init(void *fdt) 873 { 874 using_dt_cpu_ftrs = false; 875 876 /* Setup and verify the FDT, if it fails we just bail */ 877 if (!early_init_dt_verify(fdt)) 878 return false; 879 880 if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) 881 return false; 882 883 if (disabled_on_cmdline()) 884 return false; 885 886 cpufeatures_setup_cpu(); 887 888 using_dt_cpu_ftrs = true; 889 return true; 890 } 891 892 static int nr_dt_cpu_features; 893 static struct dt_cpu_feature *dt_cpu_features; 894 895 static int __init process_cpufeatures_node(unsigned long node, 896 const char *uname, int i) 897 { 898 const __be32 *prop; 899 struct dt_cpu_feature *f; 900 int len; 901 902 f = &dt_cpu_features[i]; 903 904 f->node = node; 905 906 f->name = uname; 907 908 prop = of_get_flat_dt_prop(node, "isa", &len); 909 if (!prop) { 910 pr_warn("%s: missing isa property\n", uname); 911 return 0; 912 } 913 f->isa = be32_to_cpup(prop); 914 915 prop = of_get_flat_dt_prop(node, "usable-privilege", &len); 916 if (!prop) { 917 pr_warn("%s: missing usable-privilege property", uname); 918 return 0; 919 } 920 f->usable_privilege = be32_to_cpup(prop); 921 922 prop = of_get_flat_dt_prop(node, "hv-support", &len); 923 if (prop) 924 f->hv_support = be32_to_cpup(prop); 925 else 926 f->hv_support = HV_SUPPORT_NONE; 927 928 prop = of_get_flat_dt_prop(node, "os-support", &len); 929 if (prop) 930 f->os_support = be32_to_cpup(prop); 931 else 932 f->os_support = OS_SUPPORT_NONE; 933 934 prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); 935 if (prop) 936 f->hfscr_bit_nr = be32_to_cpup(prop); 937 else 938 f->hfscr_bit_nr = -1; 939 prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); 940 if (prop) 941 f->fscr_bit_nr = be32_to_cpup(prop); 942 else 943 f->fscr_bit_nr = -1; 944 prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); 945 if (prop) 946 f->hwcap_bit_nr = be32_to_cpup(prop); 947 else 948 f->hwcap_bit_nr = -1; 949 950 if (f->usable_privilege & USABLE_HV) { 951 if (!(mfmsr() & MSR_HV)) { 952 pr_warn("%s: HV feature passed to guest\n", uname); 953 return 0; 954 } 955 956 if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { 957 pr_warn("%s: unwanted hfscr_bit_nr\n", uname); 958 return 0; 959 } 960 961 if (f->hv_support == HV_SUPPORT_HFSCR) { 962 if (f->hfscr_bit_nr == -1) { 963 pr_warn("%s: missing hfscr_bit_nr\n", uname); 964 return 0; 965 } 966 } 967 } else { 968 if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { 969 pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); 970 return 0; 971 } 972 } 973 974 if (f->usable_privilege & USABLE_OS) { 975 if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { 976 pr_warn("%s: unwanted fscr_bit_nr\n", uname); 977 return 0; 978 } 979 980 if (f->os_support == OS_SUPPORT_FSCR) { 981 if (f->fscr_bit_nr == -1) { 982 pr_warn("%s: missing fscr_bit_nr\n", uname); 983 return 0; 984 } 985 } 986 } else { 987 if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { 988 pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); 989 return 0; 990 } 991 } 992 993 if (!(f->usable_privilege & USABLE_PR)) { 994 if (f->hwcap_bit_nr != -1) { 995 pr_warn("%s: unwanted hwcap_bit_nr\n", uname); 996 return 0; 997 } 998 } 999 1000 /* Do all the independent features in the first pass */ 1001 if (!of_get_flat_dt_prop(node, "dependencies", &len)) { 1002 if (cpufeatures_process_feature(f)) 1003 f->enabled = 1; 1004 else 1005 f->disabled = 1; 1006 } 1007 1008 return 0; 1009 } 1010 1011 static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) 1012 { 1013 const __be32 *prop; 1014 int len; 1015 int nr_deps; 1016 int i; 1017 1018 if (f->enabled || f->disabled) 1019 return; 1020 1021 prop = of_get_flat_dt_prop(f->node, "dependencies", &len); 1022 if (!prop) { 1023 pr_warn("%s: missing dependencies property", f->name); 1024 return; 1025 } 1026 1027 nr_deps = len / sizeof(int); 1028 1029 for (i = 0; i < nr_deps; i++) { 1030 unsigned long phandle = be32_to_cpu(prop[i]); 1031 int j; 1032 1033 for (j = 0; j < nr_dt_cpu_features; j++) { 1034 struct dt_cpu_feature *d = &dt_cpu_features[j]; 1035 1036 if (of_get_flat_dt_phandle(d->node) == phandle) { 1037 cpufeatures_deps_enable(d); 1038 if (d->disabled) { 1039 f->disabled = 1; 1040 return; 1041 } 1042 } 1043 } 1044 } 1045 1046 if (cpufeatures_process_feature(f)) 1047 f->enabled = 1; 1048 else 1049 f->disabled = 1; 1050 } 1051 1052 static int __init scan_cpufeatures_subnodes(unsigned long node, 1053 const char *uname, 1054 void *data) 1055 { 1056 int *count = data; 1057 1058 process_cpufeatures_node(node, uname, *count); 1059 1060 (*count)++; 1061 1062 return 0; 1063 } 1064 1065 static int __init count_cpufeatures_subnodes(unsigned long node, 1066 const char *uname, 1067 void *data) 1068 { 1069 int *count = data; 1070 1071 (*count)++; 1072 1073 return 0; 1074 } 1075 1076 static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char 1077 *uname, int depth, void *data) 1078 { 1079 const __be32 *prop; 1080 int count, i; 1081 u32 isa; 1082 1083 /* We are scanning "ibm,powerpc-cpu-features" nodes only */ 1084 if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) 1085 return 0; 1086 1087 prop = of_get_flat_dt_prop(node, "isa", NULL); 1088 if (!prop) 1089 /* We checked before, "can't happen" */ 1090 return 0; 1091 1092 isa = be32_to_cpup(prop); 1093 1094 /* Count and allocate space for cpu features */ 1095 of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, 1096 &nr_dt_cpu_features); 1097 dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); 1098 if (!dt_cpu_features) 1099 panic("%s: Failed to allocate %zu bytes align=0x%lx\n", 1100 __func__, 1101 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, 1102 PAGE_SIZE); 1103 1104 cpufeatures_setup_start(isa); 1105 1106 /* Scan nodes into dt_cpu_features and enable those without deps */ 1107 count = 0; 1108 of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); 1109 1110 /* Recursive enable remaining features with dependencies */ 1111 for (i = 0; i < nr_dt_cpu_features; i++) { 1112 struct dt_cpu_feature *f = &dt_cpu_features[i]; 1113 1114 cpufeatures_deps_enable(f); 1115 } 1116 1117 prop = of_get_flat_dt_prop(node, "display-name", NULL); 1118 if (prop && strlen((char *)prop) != 0) { 1119 strlcpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); 1120 cur_cpu_spec->cpu_name = dt_cpu_name; 1121 } 1122 1123 cpufeatures_setup_finished(); 1124 1125 memblock_free(__pa(dt_cpu_features), 1126 sizeof(struct dt_cpu_feature)*nr_dt_cpu_features); 1127 1128 return 0; 1129 } 1130 1131 void __init dt_cpu_ftrs_scan(void) 1132 { 1133 if (!using_dt_cpu_ftrs) 1134 return; 1135 1136 of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); 1137 } 1138