1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2017, Nicholas Piggin, IBM Corporation 4 */ 5 6 #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt 7 8 #include <linux/export.h> 9 #include <linux/init.h> 10 #include <linux/jump_label.h> 11 #include <linux/libfdt.h> 12 #include <linux/memblock.h> 13 #include <linux/printk.h> 14 #include <linux/sched.h> 15 #include <linux/string.h> 16 #include <linux/threads.h> 17 18 #include <asm/cputable.h> 19 #include <asm/dt_cpu_ftrs.h> 20 #include <asm/mmu.h> 21 #include <asm/oprofile_impl.h> 22 #include <asm/prom.h> 23 #include <asm/setup.h> 24 25 26 /* Device-tree visible constants follow */ 27 #define ISA_V3_0B 3000 28 #define ISA_V3_1 3100 29 30 #define USABLE_PR (1U << 0) 31 #define USABLE_OS (1U << 1) 32 #define USABLE_HV (1U << 2) 33 34 #define HV_SUPPORT_HFSCR (1U << 0) 35 #define OS_SUPPORT_FSCR (1U << 0) 36 37 /* For parsing, we define all bits set as "NONE" case */ 38 #define HV_SUPPORT_NONE 0xffffffffU 39 #define OS_SUPPORT_NONE 0xffffffffU 40 41 struct dt_cpu_feature { 42 const char *name; 43 uint32_t isa; 44 uint32_t usable_privilege; 45 uint32_t hv_support; 46 uint32_t os_support; 47 uint32_t hfscr_bit_nr; 48 uint32_t fscr_bit_nr; 49 uint32_t hwcap_bit_nr; 50 /* fdt parsing */ 51 unsigned long node; 52 int enabled; 53 int disabled; 54 }; 55 56 #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) 57 58 #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ 59 PPC_FEATURE_ARCH_2_06 |\ 60 PPC_FEATURE_ICACHE_SNOOP) 61 #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ 62 PPC_FEATURE2_ISEL) 63 /* 64 * Set up the base CPU 65 */ 66 67 static int hv_mode; 68 69 static struct { 70 u64 lpcr; 71 u64 lpcr_clear; 72 u64 hfscr; 73 u64 fscr; 74 u64 pcr; 75 } system_registers; 76 77 static void (*init_pmu_registers)(void); 78 79 static void __restore_cpu_cpufeatures(void) 80 { 81 u64 lpcr; 82 83 /* 84 * LPCR is restored by the power on engine already. It can be changed 85 * after early init e.g., by radix enable, and we have no unified API 86 * for saving and restoring such SPRs. 87 * 88 * This ->restore hook should really be removed from idle and register 89 * restore moved directly into the idle restore code, because this code 90 * doesn't know how idle is implemented or what it needs restored here. 91 * 92 * The best we can do to accommodate secondary boot and idle restore 93 * for now is "or" LPCR with existing. 94 */ 95 lpcr = mfspr(SPRN_LPCR); 96 lpcr |= system_registers.lpcr; 97 lpcr &= ~system_registers.lpcr_clear; 98 mtspr(SPRN_LPCR, lpcr); 99 if (hv_mode) { 100 mtspr(SPRN_LPID, 0); 101 mtspr(SPRN_HFSCR, system_registers.hfscr); 102 mtspr(SPRN_PCR, system_registers.pcr); 103 } 104 mtspr(SPRN_FSCR, system_registers.fscr); 105 106 if (init_pmu_registers) 107 init_pmu_registers(); 108 } 109 110 static char dt_cpu_name[64]; 111 112 static struct cpu_spec __initdata base_cpu_spec = { 113 .cpu_name = NULL, 114 .cpu_features = CPU_FTRS_DT_CPU_BASE, 115 .cpu_user_features = COMMON_USER_BASE, 116 .cpu_user_features2 = COMMON_USER2_BASE, 117 .mmu_features = 0, 118 .icache_bsize = 32, /* minimum block size, fixed by */ 119 .dcache_bsize = 32, /* cache info init. */ 120 .num_pmcs = 0, 121 .pmc_type = PPC_PMC_DEFAULT, 122 .oprofile_cpu_type = NULL, 123 .oprofile_type = PPC_OPROFILE_INVALID, 124 .cpu_setup = NULL, 125 .cpu_restore = __restore_cpu_cpufeatures, 126 .machine_check_early = NULL, 127 .platform = NULL, 128 }; 129 130 static void __init cpufeatures_setup_cpu(void) 131 { 132 set_cur_cpu_spec(&base_cpu_spec); 133 134 cur_cpu_spec->pvr_mask = -1; 135 cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); 136 137 /* Initialize the base environment -- clear FSCR/HFSCR. */ 138 hv_mode = !!(mfmsr() & MSR_HV); 139 if (hv_mode) { 140 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 141 mtspr(SPRN_HFSCR, 0); 142 } 143 mtspr(SPRN_FSCR, 0); 144 mtspr(SPRN_PCR, PCR_MASK); 145 146 /* 147 * LPCR does not get cleared, to match behaviour with secondaries 148 * in __restore_cpu_cpufeatures. Once the idle code is fixed, this 149 * could clear LPCR too. 150 */ 151 } 152 153 static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) 154 { 155 if (f->hv_support == HV_SUPPORT_NONE) { 156 } else if (f->hv_support & HV_SUPPORT_HFSCR) { 157 u64 hfscr = mfspr(SPRN_HFSCR); 158 hfscr |= 1UL << f->hfscr_bit_nr; 159 mtspr(SPRN_HFSCR, hfscr); 160 } else { 161 /* Does not have a known recipe */ 162 return 0; 163 } 164 165 if (f->os_support == OS_SUPPORT_NONE) { 166 } else if (f->os_support & OS_SUPPORT_FSCR) { 167 u64 fscr = mfspr(SPRN_FSCR); 168 fscr |= 1UL << f->fscr_bit_nr; 169 mtspr(SPRN_FSCR, fscr); 170 } else { 171 /* Does not have a known recipe */ 172 return 0; 173 } 174 175 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 176 uint32_t word = f->hwcap_bit_nr / 32; 177 uint32_t bit = f->hwcap_bit_nr % 32; 178 179 if (word == 0) 180 cur_cpu_spec->cpu_user_features |= 1U << bit; 181 else if (word == 1) 182 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 183 else 184 pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); 185 } 186 187 return 1; 188 } 189 190 static int __init feat_enable(struct dt_cpu_feature *f) 191 { 192 if (f->hv_support != HV_SUPPORT_NONE) { 193 if (f->hfscr_bit_nr != -1) { 194 u64 hfscr = mfspr(SPRN_HFSCR); 195 hfscr |= 1UL << f->hfscr_bit_nr; 196 mtspr(SPRN_HFSCR, hfscr); 197 } 198 } 199 200 if (f->os_support != OS_SUPPORT_NONE) { 201 if (f->fscr_bit_nr != -1) { 202 u64 fscr = mfspr(SPRN_FSCR); 203 fscr |= 1UL << f->fscr_bit_nr; 204 mtspr(SPRN_FSCR, fscr); 205 } 206 } 207 208 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 209 uint32_t word = f->hwcap_bit_nr / 32; 210 uint32_t bit = f->hwcap_bit_nr % 32; 211 212 if (word == 0) 213 cur_cpu_spec->cpu_user_features |= 1U << bit; 214 else if (word == 1) 215 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 216 else 217 pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); 218 } 219 220 return 1; 221 } 222 223 static int __init feat_disable(struct dt_cpu_feature *f) 224 { 225 return 0; 226 } 227 228 static int __init feat_enable_hv(struct dt_cpu_feature *f) 229 { 230 u64 lpcr; 231 232 if (!hv_mode) { 233 pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); 234 return 0; 235 } 236 237 mtspr(SPRN_LPID, 0); 238 239 lpcr = mfspr(SPRN_LPCR); 240 lpcr &= ~LPCR_LPES0; /* HV external interrupts */ 241 mtspr(SPRN_LPCR, lpcr); 242 243 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 244 245 return 1; 246 } 247 248 static int __init feat_enable_le(struct dt_cpu_feature *f) 249 { 250 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; 251 return 1; 252 } 253 254 static int __init feat_enable_smt(struct dt_cpu_feature *f) 255 { 256 cur_cpu_spec->cpu_features |= CPU_FTR_SMT; 257 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; 258 return 1; 259 } 260 261 static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) 262 { 263 u64 lpcr; 264 265 /* Set PECE wakeup modes for ISA 207 */ 266 lpcr = mfspr(SPRN_LPCR); 267 lpcr |= LPCR_PECE0; 268 lpcr |= LPCR_PECE1; 269 lpcr |= LPCR_PECE2; 270 mtspr(SPRN_LPCR, lpcr); 271 272 return 1; 273 } 274 275 static int __init feat_enable_align_dsisr(struct dt_cpu_feature *f) 276 { 277 cur_cpu_spec->cpu_features &= ~CPU_FTR_NODSISRALIGN; 278 279 return 1; 280 } 281 282 static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) 283 { 284 u64 lpcr; 285 286 /* Set PECE wakeup modes for ISAv3.0B */ 287 lpcr = mfspr(SPRN_LPCR); 288 lpcr |= LPCR_PECE0; 289 lpcr |= LPCR_PECE1; 290 lpcr |= LPCR_PECE2; 291 mtspr(SPRN_LPCR, lpcr); 292 293 return 1; 294 } 295 296 static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) 297 { 298 u64 lpcr; 299 300 lpcr = mfspr(SPRN_LPCR); 301 lpcr &= ~LPCR_ISL; 302 303 /* VRMASD */ 304 lpcr |= LPCR_VPM0; 305 lpcr &= ~LPCR_VPM1; 306 lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ 307 mtspr(SPRN_LPCR, lpcr); 308 309 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 310 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 311 312 return 1; 313 } 314 315 static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) 316 { 317 u64 lpcr; 318 319 system_registers.lpcr_clear |= (LPCR_ISL | LPCR_UPRT | LPCR_HR); 320 lpcr = mfspr(SPRN_LPCR); 321 lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); 322 mtspr(SPRN_LPCR, lpcr); 323 324 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 325 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 326 327 return 1; 328 } 329 330 331 static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) 332 { 333 #ifdef CONFIG_PPC_RADIX_MMU 334 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; 335 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 336 cur_cpu_spec->mmu_features |= MMU_FTR_GTSE; 337 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 338 339 return 1; 340 #endif 341 return 0; 342 } 343 344 static int __init feat_enable_dscr(struct dt_cpu_feature *f) 345 { 346 u64 lpcr; 347 348 /* 349 * Linux relies on FSCR[DSCR] being clear, so that we can take the 350 * facility unavailable interrupt and track the task's usage of DSCR. 351 * See facility_unavailable_exception(). 352 * Clear the bit here so that feat_enable() doesn't set it. 353 */ 354 f->fscr_bit_nr = -1; 355 356 feat_enable(f); 357 358 lpcr = mfspr(SPRN_LPCR); 359 lpcr &= ~LPCR_DPFD; 360 lpcr |= (4UL << LPCR_DPFD_SH); 361 mtspr(SPRN_LPCR, lpcr); 362 363 return 1; 364 } 365 366 static void hfscr_pmu_enable(void) 367 { 368 u64 hfscr = mfspr(SPRN_HFSCR); 369 hfscr |= PPC_BIT(60); 370 mtspr(SPRN_HFSCR, hfscr); 371 } 372 373 static void init_pmu_power8(void) 374 { 375 if (hv_mode) { 376 mtspr(SPRN_MMCRC, 0); 377 mtspr(SPRN_MMCRH, 0); 378 } 379 380 mtspr(SPRN_MMCRA, 0); 381 mtspr(SPRN_MMCR0, 0); 382 mtspr(SPRN_MMCR1, 0); 383 mtspr(SPRN_MMCR2, 0); 384 mtspr(SPRN_MMCRS, 0); 385 } 386 387 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) 388 { 389 cur_cpu_spec->platform = "power8"; 390 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; 391 392 return 1; 393 } 394 395 static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) 396 { 397 hfscr_pmu_enable(); 398 399 init_pmu_power8(); 400 init_pmu_registers = init_pmu_power8; 401 402 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 403 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 404 if (pvr_version_is(PVR_POWER8E)) 405 cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; 406 407 cur_cpu_spec->num_pmcs = 6; 408 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 409 cur_cpu_spec->oprofile_cpu_type = "ppc64/power8"; 410 411 return 1; 412 } 413 414 static void init_pmu_power9(void) 415 { 416 if (hv_mode) 417 mtspr(SPRN_MMCRC, 0); 418 419 mtspr(SPRN_MMCRA, 0); 420 mtspr(SPRN_MMCR0, 0); 421 mtspr(SPRN_MMCR1, 0); 422 mtspr(SPRN_MMCR2, 0); 423 } 424 425 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) 426 { 427 cur_cpu_spec->platform = "power9"; 428 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; 429 430 return 1; 431 } 432 433 static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) 434 { 435 hfscr_pmu_enable(); 436 437 init_pmu_power9(); 438 init_pmu_registers = init_pmu_power9; 439 440 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 441 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 442 443 cur_cpu_spec->num_pmcs = 6; 444 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 445 cur_cpu_spec->oprofile_cpu_type = "ppc64/power9"; 446 447 return 1; 448 } 449 450 static void init_pmu_power10(void) 451 { 452 init_pmu_power9(); 453 454 mtspr(SPRN_MMCR3, 0); 455 mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); 456 } 457 458 static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f) 459 { 460 hfscr_pmu_enable(); 461 462 init_pmu_power10(); 463 init_pmu_registers = init_pmu_power10; 464 465 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 466 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 467 468 cur_cpu_spec->num_pmcs = 6; 469 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 470 cur_cpu_spec->oprofile_cpu_type = "ppc64/power10"; 471 472 return 1; 473 } 474 475 static int __init feat_enable_mce_power10(struct dt_cpu_feature *f) 476 { 477 cur_cpu_spec->platform = "power10"; 478 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; 479 480 return 1; 481 } 482 483 static int __init feat_enable_tm(struct dt_cpu_feature *f) 484 { 485 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 486 feat_enable(f); 487 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; 488 return 1; 489 #endif 490 return 0; 491 } 492 493 static int __init feat_enable_fp(struct dt_cpu_feature *f) 494 { 495 feat_enable(f); 496 cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; 497 498 return 1; 499 } 500 501 static int __init feat_enable_vector(struct dt_cpu_feature *f) 502 { 503 #ifdef CONFIG_ALTIVEC 504 feat_enable(f); 505 cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; 506 cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; 507 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; 508 509 return 1; 510 #endif 511 return 0; 512 } 513 514 static int __init feat_enable_vsx(struct dt_cpu_feature *f) 515 { 516 #ifdef CONFIG_VSX 517 feat_enable(f); 518 cur_cpu_spec->cpu_features |= CPU_FTR_VSX; 519 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; 520 521 return 1; 522 #endif 523 return 0; 524 } 525 526 static int __init feat_enable_purr(struct dt_cpu_feature *f) 527 { 528 cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; 529 530 return 1; 531 } 532 533 static int __init feat_enable_ebb(struct dt_cpu_feature *f) 534 { 535 /* 536 * PPC_FEATURE2_EBB is enabled in PMU init code because it has 537 * historically been related to the PMU facility. This may have 538 * to be decoupled if EBB becomes more generic. For now, follow 539 * existing convention. 540 */ 541 f->hwcap_bit_nr = -1; 542 feat_enable(f); 543 544 return 1; 545 } 546 547 static int __init feat_enable_dbell(struct dt_cpu_feature *f) 548 { 549 u64 lpcr; 550 551 /* P9 has an HFSCR for privileged state */ 552 feat_enable(f); 553 554 cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; 555 556 lpcr = mfspr(SPRN_LPCR); 557 lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ 558 mtspr(SPRN_LPCR, lpcr); 559 560 return 1; 561 } 562 563 static int __init feat_enable_hvi(struct dt_cpu_feature *f) 564 { 565 u64 lpcr; 566 567 /* 568 * POWER9 XIVE interrupts including in OPAL XICS compatibility 569 * are always delivered as hypervisor virtualization interrupts (HVI) 570 * rather than EE. 571 * 572 * However LPES0 is not set here, in the chance that an EE does get 573 * delivered to the host somehow, the EE handler would not expect it 574 * to be delivered in LPES0 mode (e.g., using SRR[01]). This could 575 * happen if there is a bug in interrupt controller code, or IC is 576 * misconfigured in systemsim. 577 */ 578 579 lpcr = mfspr(SPRN_LPCR); 580 lpcr |= LPCR_HVICE; /* enable hvi interrupts */ 581 lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ 582 lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ 583 mtspr(SPRN_LPCR, lpcr); 584 585 return 1; 586 } 587 588 static int __init feat_enable_large_ci(struct dt_cpu_feature *f) 589 { 590 cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; 591 592 return 1; 593 } 594 595 static int __init feat_enable_mma(struct dt_cpu_feature *f) 596 { 597 u64 pcr; 598 599 feat_enable(f); 600 pcr = mfspr(SPRN_PCR); 601 pcr &= ~PCR_MMA_DIS; 602 mtspr(SPRN_PCR, pcr); 603 604 return 1; 605 } 606 607 struct dt_cpu_feature_match { 608 const char *name; 609 int (*enable)(struct dt_cpu_feature *f); 610 u64 cpu_ftr_bit_mask; 611 }; 612 613 static struct dt_cpu_feature_match __initdata 614 dt_cpu_feature_match_table[] = { 615 {"hypervisor", feat_enable_hv, 0}, 616 {"big-endian", feat_enable, 0}, 617 {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, 618 {"smt", feat_enable_smt, 0}, 619 {"interrupt-facilities", feat_enable, 0}, 620 {"system-call-vectored", feat_enable, 0}, 621 {"timer-facilities", feat_enable, 0}, 622 {"timer-facilities-v3", feat_enable, 0}, 623 {"debug-facilities", feat_enable, 0}, 624 {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, 625 {"branch-tracing", feat_enable, 0}, 626 {"floating-point", feat_enable_fp, 0}, 627 {"vector", feat_enable_vector, 0}, 628 {"vector-scalar", feat_enable_vsx, 0}, 629 {"vector-scalar-v3", feat_enable, 0}, 630 {"decimal-floating-point", feat_enable, 0}, 631 {"decimal-integer", feat_enable, 0}, 632 {"quadword-load-store", feat_enable, 0}, 633 {"vector-crypto", feat_enable, 0}, 634 {"mmu-hash", feat_enable_mmu_hash, 0}, 635 {"mmu-radix", feat_enable_mmu_radix, 0}, 636 {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, 637 {"virtual-page-class-key-protection", feat_enable, 0}, 638 {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, 639 {"transactional-memory-v3", feat_enable_tm, 0}, 640 {"tm-suspend-hypervisor-assist", feat_enable, CPU_FTR_P9_TM_HV_ASSIST}, 641 {"tm-suspend-xer-so-bug", feat_enable, CPU_FTR_P9_TM_XER_SO_BUG}, 642 {"idle-nap", feat_enable_idle_nap, 0}, 643 {"alignment-interrupt-dsisr", feat_enable_align_dsisr, 0}, 644 {"idle-stop", feat_enable_idle_stop, 0}, 645 {"machine-check-power8", feat_enable_mce_power8, 0}, 646 {"performance-monitor-power8", feat_enable_pmu_power8, 0}, 647 {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, 648 {"event-based-branch", feat_enable_ebb, 0}, 649 {"target-address-register", feat_enable, 0}, 650 {"branch-history-rolling-buffer", feat_enable, 0}, 651 {"control-register", feat_enable, CPU_FTR_CTRL}, 652 {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, 653 {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, 654 {"processor-utilization-of-resources-register", feat_enable_purr, 0}, 655 {"no-execute", feat_enable, 0}, 656 /* strong-access-ordering is unused */ 657 {"cache-inhibited-large-page", feat_enable_large_ci, 0}, 658 {"coprocessor-icswx", feat_enable, 0}, 659 {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, 660 {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, 661 {"wait", feat_enable, 0}, 662 {"atomic-memory-operations", feat_enable, 0}, 663 {"branch-v3", feat_enable, 0}, 664 {"copy-paste", feat_enable, 0}, 665 {"decimal-floating-point-v3", feat_enable, 0}, 666 {"decimal-integer-v3", feat_enable, 0}, 667 {"fixed-point-v3", feat_enable, 0}, 668 {"floating-point-v3", feat_enable, 0}, 669 {"group-start-register", feat_enable, 0}, 670 {"pc-relative-addressing", feat_enable, 0}, 671 {"machine-check-power9", feat_enable_mce_power9, 0}, 672 {"machine-check-power10", feat_enable_mce_power10, 0}, 673 {"performance-monitor-power9", feat_enable_pmu_power9, 0}, 674 {"performance-monitor-power10", feat_enable_pmu_power10, 0}, 675 {"event-based-branch-v3", feat_enable, 0}, 676 {"random-number-generator", feat_enable, 0}, 677 {"system-call-vectored", feat_disable, 0}, 678 {"trace-interrupt-v3", feat_enable, 0}, 679 {"vector-v3", feat_enable, 0}, 680 {"vector-binary128", feat_enable, 0}, 681 {"vector-binary16", feat_enable, 0}, 682 {"wait-v3", feat_enable, 0}, 683 {"prefix-instructions", feat_enable, 0}, 684 {"matrix-multiply-assist", feat_enable_mma, 0}, 685 {"debug-facilities-v31", feat_enable, CPU_FTR_DAWR1}, 686 }; 687 688 static bool __initdata using_dt_cpu_ftrs; 689 static bool __initdata enable_unknown = true; 690 691 static int __init dt_cpu_ftrs_parse(char *str) 692 { 693 if (!str) 694 return 0; 695 696 if (!strcmp(str, "off")) 697 using_dt_cpu_ftrs = false; 698 else if (!strcmp(str, "known")) 699 enable_unknown = false; 700 else 701 return 1; 702 703 return 0; 704 } 705 early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); 706 707 static void __init cpufeatures_setup_start(u32 isa) 708 { 709 pr_info("setup for ISA %d\n", isa); 710 711 if (isa >= ISA_V3_0B) { 712 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; 713 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; 714 } 715 716 if (isa >= ISA_V3_1) { 717 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; 718 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; 719 } 720 } 721 722 static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) 723 { 724 const struct dt_cpu_feature_match *m; 725 bool known = false; 726 int i; 727 728 for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { 729 m = &dt_cpu_feature_match_table[i]; 730 if (!strcmp(f->name, m->name)) { 731 known = true; 732 if (m->enable(f)) { 733 cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; 734 break; 735 } 736 737 pr_info("not enabling: %s (disabled or unsupported by kernel)\n", 738 f->name); 739 return false; 740 } 741 } 742 743 if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) { 744 pr_info("not enabling: %s (unknown and unsupported by kernel)\n", 745 f->name); 746 return false; 747 } 748 749 if (known) 750 pr_debug("enabling: %s\n", f->name); 751 else 752 pr_debug("enabling: %s (unknown)\n", f->name); 753 754 return true; 755 } 756 757 /* 758 * Handle POWER9 broadcast tlbie invalidation issue using 759 * cpu feature flag. 760 */ 761 static __init void update_tlbie_feature_flag(unsigned long pvr) 762 { 763 if (PVR_VER(pvr) == PVR_POWER9) { 764 /* 765 * Set the tlbie feature flag for anything below 766 * Nimbus DD 2.3 and Cumulus DD 1.3 767 */ 768 if ((pvr & 0xe000) == 0) { 769 /* Nimbus */ 770 if ((pvr & 0xfff) < 0x203) 771 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 772 } else if ((pvr & 0xc000) == 0) { 773 /* Cumulus */ 774 if ((pvr & 0xfff) < 0x103) 775 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 776 } else { 777 WARN_ONCE(1, "Unknown PVR"); 778 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 779 } 780 781 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; 782 } 783 } 784 785 static __init void cpufeatures_cpu_quirks(void) 786 { 787 unsigned long version = mfspr(SPRN_PVR); 788 789 /* 790 * Not all quirks can be derived from the cpufeatures device tree. 791 */ 792 if ((version & 0xffffefff) == 0x004e0200) { 793 /* DD2.0 has no feature flag */ 794 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 795 } else if ((version & 0xffffefff) == 0x004e0201) { 796 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 797 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 798 } else if ((version & 0xffffefff) == 0x004e0202) { 799 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 800 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 801 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 802 } else if ((version & 0xffff0000) == 0x004e0000) { 803 /* DD2.1 and up have DD2_1 */ 804 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 805 } 806 807 if ((version & 0xffff0000) == 0x004e0000) { 808 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 809 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; 810 } 811 812 update_tlbie_feature_flag(version); 813 } 814 815 static void __init cpufeatures_setup_finished(void) 816 { 817 cpufeatures_cpu_quirks(); 818 819 if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { 820 pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); 821 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 822 } 823 824 /* Make sure powerpc_base_platform is non-NULL */ 825 powerpc_base_platform = cur_cpu_spec->platform; 826 827 system_registers.lpcr = mfspr(SPRN_LPCR); 828 system_registers.hfscr = mfspr(SPRN_HFSCR); 829 system_registers.fscr = mfspr(SPRN_FSCR); 830 system_registers.pcr = mfspr(SPRN_PCR); 831 832 pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", 833 cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); 834 } 835 836 static int __init disabled_on_cmdline(void) 837 { 838 unsigned long root, chosen; 839 const char *p; 840 841 root = of_get_flat_dt_root(); 842 chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); 843 if (chosen == -FDT_ERR_NOTFOUND) 844 return false; 845 846 p = of_get_flat_dt_prop(chosen, "bootargs", NULL); 847 if (!p) 848 return false; 849 850 if (strstr(p, "dt_cpu_ftrs=off")) 851 return true; 852 853 return false; 854 } 855 856 static int __init fdt_find_cpu_features(unsigned long node, const char *uname, 857 int depth, void *data) 858 { 859 if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") 860 && of_get_flat_dt_prop(node, "isa", NULL)) 861 return 1; 862 863 return 0; 864 } 865 866 bool __init dt_cpu_ftrs_in_use(void) 867 { 868 return using_dt_cpu_ftrs; 869 } 870 871 bool __init dt_cpu_ftrs_init(void *fdt) 872 { 873 using_dt_cpu_ftrs = false; 874 875 /* Setup and verify the FDT, if it fails we just bail */ 876 if (!early_init_dt_verify(fdt)) 877 return false; 878 879 if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) 880 return false; 881 882 if (disabled_on_cmdline()) 883 return false; 884 885 cpufeatures_setup_cpu(); 886 887 using_dt_cpu_ftrs = true; 888 return true; 889 } 890 891 static int nr_dt_cpu_features; 892 static struct dt_cpu_feature *dt_cpu_features; 893 894 static int __init process_cpufeatures_node(unsigned long node, 895 const char *uname, int i) 896 { 897 const __be32 *prop; 898 struct dt_cpu_feature *f; 899 int len; 900 901 f = &dt_cpu_features[i]; 902 903 f->node = node; 904 905 f->name = uname; 906 907 prop = of_get_flat_dt_prop(node, "isa", &len); 908 if (!prop) { 909 pr_warn("%s: missing isa property\n", uname); 910 return 0; 911 } 912 f->isa = be32_to_cpup(prop); 913 914 prop = of_get_flat_dt_prop(node, "usable-privilege", &len); 915 if (!prop) { 916 pr_warn("%s: missing usable-privilege property", uname); 917 return 0; 918 } 919 f->usable_privilege = be32_to_cpup(prop); 920 921 prop = of_get_flat_dt_prop(node, "hv-support", &len); 922 if (prop) 923 f->hv_support = be32_to_cpup(prop); 924 else 925 f->hv_support = HV_SUPPORT_NONE; 926 927 prop = of_get_flat_dt_prop(node, "os-support", &len); 928 if (prop) 929 f->os_support = be32_to_cpup(prop); 930 else 931 f->os_support = OS_SUPPORT_NONE; 932 933 prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); 934 if (prop) 935 f->hfscr_bit_nr = be32_to_cpup(prop); 936 else 937 f->hfscr_bit_nr = -1; 938 prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); 939 if (prop) 940 f->fscr_bit_nr = be32_to_cpup(prop); 941 else 942 f->fscr_bit_nr = -1; 943 prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); 944 if (prop) 945 f->hwcap_bit_nr = be32_to_cpup(prop); 946 else 947 f->hwcap_bit_nr = -1; 948 949 if (f->usable_privilege & USABLE_HV) { 950 if (!(mfmsr() & MSR_HV)) { 951 pr_warn("%s: HV feature passed to guest\n", uname); 952 return 0; 953 } 954 955 if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { 956 pr_warn("%s: unwanted hfscr_bit_nr\n", uname); 957 return 0; 958 } 959 960 if (f->hv_support == HV_SUPPORT_HFSCR) { 961 if (f->hfscr_bit_nr == -1) { 962 pr_warn("%s: missing hfscr_bit_nr\n", uname); 963 return 0; 964 } 965 } 966 } else { 967 if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { 968 pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); 969 return 0; 970 } 971 } 972 973 if (f->usable_privilege & USABLE_OS) { 974 if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { 975 pr_warn("%s: unwanted fscr_bit_nr\n", uname); 976 return 0; 977 } 978 979 if (f->os_support == OS_SUPPORT_FSCR) { 980 if (f->fscr_bit_nr == -1) { 981 pr_warn("%s: missing fscr_bit_nr\n", uname); 982 return 0; 983 } 984 } 985 } else { 986 if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { 987 pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); 988 return 0; 989 } 990 } 991 992 if (!(f->usable_privilege & USABLE_PR)) { 993 if (f->hwcap_bit_nr != -1) { 994 pr_warn("%s: unwanted hwcap_bit_nr\n", uname); 995 return 0; 996 } 997 } 998 999 /* Do all the independent features in the first pass */ 1000 if (!of_get_flat_dt_prop(node, "dependencies", &len)) { 1001 if (cpufeatures_process_feature(f)) 1002 f->enabled = 1; 1003 else 1004 f->disabled = 1; 1005 } 1006 1007 return 0; 1008 } 1009 1010 static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) 1011 { 1012 const __be32 *prop; 1013 int len; 1014 int nr_deps; 1015 int i; 1016 1017 if (f->enabled || f->disabled) 1018 return; 1019 1020 prop = of_get_flat_dt_prop(f->node, "dependencies", &len); 1021 if (!prop) { 1022 pr_warn("%s: missing dependencies property", f->name); 1023 return; 1024 } 1025 1026 nr_deps = len / sizeof(int); 1027 1028 for (i = 0; i < nr_deps; i++) { 1029 unsigned long phandle = be32_to_cpu(prop[i]); 1030 int j; 1031 1032 for (j = 0; j < nr_dt_cpu_features; j++) { 1033 struct dt_cpu_feature *d = &dt_cpu_features[j]; 1034 1035 if (of_get_flat_dt_phandle(d->node) == phandle) { 1036 cpufeatures_deps_enable(d); 1037 if (d->disabled) { 1038 f->disabled = 1; 1039 return; 1040 } 1041 } 1042 } 1043 } 1044 1045 if (cpufeatures_process_feature(f)) 1046 f->enabled = 1; 1047 else 1048 f->disabled = 1; 1049 } 1050 1051 static int __init scan_cpufeatures_subnodes(unsigned long node, 1052 const char *uname, 1053 void *data) 1054 { 1055 int *count = data; 1056 1057 process_cpufeatures_node(node, uname, *count); 1058 1059 (*count)++; 1060 1061 return 0; 1062 } 1063 1064 static int __init count_cpufeatures_subnodes(unsigned long node, 1065 const char *uname, 1066 void *data) 1067 { 1068 int *count = data; 1069 1070 (*count)++; 1071 1072 return 0; 1073 } 1074 1075 static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char 1076 *uname, int depth, void *data) 1077 { 1078 const __be32 *prop; 1079 int count, i; 1080 u32 isa; 1081 1082 /* We are scanning "ibm,powerpc-cpu-features" nodes only */ 1083 if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) 1084 return 0; 1085 1086 prop = of_get_flat_dt_prop(node, "isa", NULL); 1087 if (!prop) 1088 /* We checked before, "can't happen" */ 1089 return 0; 1090 1091 isa = be32_to_cpup(prop); 1092 1093 /* Count and allocate space for cpu features */ 1094 of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, 1095 &nr_dt_cpu_features); 1096 dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); 1097 if (!dt_cpu_features) 1098 panic("%s: Failed to allocate %zu bytes align=0x%lx\n", 1099 __func__, 1100 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, 1101 PAGE_SIZE); 1102 1103 cpufeatures_setup_start(isa); 1104 1105 /* Scan nodes into dt_cpu_features and enable those without deps */ 1106 count = 0; 1107 of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); 1108 1109 /* Recursive enable remaining features with dependencies */ 1110 for (i = 0; i < nr_dt_cpu_features; i++) { 1111 struct dt_cpu_feature *f = &dt_cpu_features[i]; 1112 1113 cpufeatures_deps_enable(f); 1114 } 1115 1116 prop = of_get_flat_dt_prop(node, "display-name", NULL); 1117 if (prop && strlen((char *)prop) != 0) { 1118 strlcpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); 1119 cur_cpu_spec->cpu_name = dt_cpu_name; 1120 } 1121 1122 cpufeatures_setup_finished(); 1123 1124 memblock_free(__pa(dt_cpu_features), 1125 sizeof(struct dt_cpu_feature)*nr_dt_cpu_features); 1126 1127 return 0; 1128 } 1129 1130 void __init dt_cpu_ftrs_scan(void) 1131 { 1132 if (!using_dt_cpu_ftrs) 1133 return; 1134 1135 of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); 1136 } 1137