1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2017, Nicholas Piggin, IBM Corporation 4 */ 5 6 #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt 7 8 #include <linux/export.h> 9 #include <linux/init.h> 10 #include <linux/jump_label.h> 11 #include <linux/libfdt.h> 12 #include <linux/memblock.h> 13 #include <linux/printk.h> 14 #include <linux/sched.h> 15 #include <linux/string.h> 16 #include <linux/threads.h> 17 18 #include <asm/cputable.h> 19 #include <asm/dt_cpu_ftrs.h> 20 #include <asm/mmu.h> 21 #include <asm/oprofile_impl.h> 22 #include <asm/prom.h> 23 #include <asm/setup.h> 24 25 26 /* Device-tree visible constants follow */ 27 #define ISA_V2_07B 2070 28 #define ISA_V3_0B 3000 29 30 #define USABLE_PR (1U << 0) 31 #define USABLE_OS (1U << 1) 32 #define USABLE_HV (1U << 2) 33 34 #define HV_SUPPORT_HFSCR (1U << 0) 35 #define OS_SUPPORT_FSCR (1U << 0) 36 37 /* For parsing, we define all bits set as "NONE" case */ 38 #define HV_SUPPORT_NONE 0xffffffffU 39 #define OS_SUPPORT_NONE 0xffffffffU 40 41 struct dt_cpu_feature { 42 const char *name; 43 uint32_t isa; 44 uint32_t usable_privilege; 45 uint32_t hv_support; 46 uint32_t os_support; 47 uint32_t hfscr_bit_nr; 48 uint32_t fscr_bit_nr; 49 uint32_t hwcap_bit_nr; 50 /* fdt parsing */ 51 unsigned long node; 52 int enabled; 53 int disabled; 54 }; 55 56 #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) 57 58 #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ 59 PPC_FEATURE_ARCH_2_06 |\ 60 PPC_FEATURE_ICACHE_SNOOP) 61 #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ 62 PPC_FEATURE2_ISEL) 63 /* 64 * Set up the base CPU 65 */ 66 67 extern long __machine_check_early_realmode_p8(struct pt_regs *regs); 68 extern long __machine_check_early_realmode_p9(struct pt_regs *regs); 69 70 static int hv_mode; 71 72 static struct { 73 u64 lpcr; 74 u64 lpcr_clear; 75 u64 hfscr; 76 u64 fscr; 77 } system_registers; 78 79 static void (*init_pmu_registers)(void); 80 81 static void __restore_cpu_cpufeatures(void) 82 { 83 u64 lpcr; 84 85 /* 86 * LPCR is restored by the power on engine already. It can be changed 87 * after early init e.g., by radix enable, and we have no unified API 88 * for saving and restoring such SPRs. 89 * 90 * This ->restore hook should really be removed from idle and register 91 * restore moved directly into the idle restore code, because this code 92 * doesn't know how idle is implemented or what it needs restored here. 93 * 94 * The best we can do to accommodate secondary boot and idle restore 95 * for now is "or" LPCR with existing. 96 */ 97 lpcr = mfspr(SPRN_LPCR); 98 lpcr |= system_registers.lpcr; 99 lpcr &= ~system_registers.lpcr_clear; 100 mtspr(SPRN_LPCR, lpcr); 101 if (hv_mode) { 102 mtspr(SPRN_LPID, 0); 103 mtspr(SPRN_HFSCR, system_registers.hfscr); 104 mtspr(SPRN_PCR, PCR_MASK); 105 } 106 mtspr(SPRN_FSCR, system_registers.fscr); 107 108 if (init_pmu_registers) 109 init_pmu_registers(); 110 } 111 112 static char dt_cpu_name[64]; 113 114 static struct cpu_spec __initdata base_cpu_spec = { 115 .cpu_name = NULL, 116 .cpu_features = CPU_FTRS_DT_CPU_BASE, 117 .cpu_user_features = COMMON_USER_BASE, 118 .cpu_user_features2 = COMMON_USER2_BASE, 119 .mmu_features = 0, 120 .icache_bsize = 32, /* minimum block size, fixed by */ 121 .dcache_bsize = 32, /* cache info init. */ 122 .num_pmcs = 0, 123 .pmc_type = PPC_PMC_DEFAULT, 124 .oprofile_cpu_type = NULL, 125 .oprofile_type = PPC_OPROFILE_INVALID, 126 .cpu_setup = NULL, 127 .cpu_restore = __restore_cpu_cpufeatures, 128 .machine_check_early = NULL, 129 .platform = NULL, 130 }; 131 132 static void __init cpufeatures_setup_cpu(void) 133 { 134 set_cur_cpu_spec(&base_cpu_spec); 135 136 cur_cpu_spec->pvr_mask = -1; 137 cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); 138 139 /* Initialize the base environment -- clear FSCR/HFSCR. */ 140 hv_mode = !!(mfmsr() & MSR_HV); 141 if (hv_mode) { 142 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 143 mtspr(SPRN_HFSCR, 0); 144 } 145 mtspr(SPRN_FSCR, 0); 146 mtspr(SPRN_PCR, PCR_MASK); 147 148 /* 149 * LPCR does not get cleared, to match behaviour with secondaries 150 * in __restore_cpu_cpufeatures. Once the idle code is fixed, this 151 * could clear LPCR too. 152 */ 153 } 154 155 static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) 156 { 157 if (f->hv_support == HV_SUPPORT_NONE) { 158 } else if (f->hv_support & HV_SUPPORT_HFSCR) { 159 u64 hfscr = mfspr(SPRN_HFSCR); 160 hfscr |= 1UL << f->hfscr_bit_nr; 161 mtspr(SPRN_HFSCR, hfscr); 162 } else { 163 /* Does not have a known recipe */ 164 return 0; 165 } 166 167 if (f->os_support == OS_SUPPORT_NONE) { 168 } else if (f->os_support & OS_SUPPORT_FSCR) { 169 u64 fscr = mfspr(SPRN_FSCR); 170 fscr |= 1UL << f->fscr_bit_nr; 171 mtspr(SPRN_FSCR, fscr); 172 } else { 173 /* Does not have a known recipe */ 174 return 0; 175 } 176 177 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 178 uint32_t word = f->hwcap_bit_nr / 32; 179 uint32_t bit = f->hwcap_bit_nr % 32; 180 181 if (word == 0) 182 cur_cpu_spec->cpu_user_features |= 1U << bit; 183 else if (word == 1) 184 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 185 else 186 pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); 187 } 188 189 return 1; 190 } 191 192 static int __init feat_enable(struct dt_cpu_feature *f) 193 { 194 if (f->hv_support != HV_SUPPORT_NONE) { 195 if (f->hfscr_bit_nr != -1) { 196 u64 hfscr = mfspr(SPRN_HFSCR); 197 hfscr |= 1UL << f->hfscr_bit_nr; 198 mtspr(SPRN_HFSCR, hfscr); 199 } 200 } 201 202 if (f->os_support != OS_SUPPORT_NONE) { 203 if (f->fscr_bit_nr != -1) { 204 u64 fscr = mfspr(SPRN_FSCR); 205 fscr |= 1UL << f->fscr_bit_nr; 206 mtspr(SPRN_FSCR, fscr); 207 } 208 } 209 210 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 211 uint32_t word = f->hwcap_bit_nr / 32; 212 uint32_t bit = f->hwcap_bit_nr % 32; 213 214 if (word == 0) 215 cur_cpu_spec->cpu_user_features |= 1U << bit; 216 else if (word == 1) 217 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 218 else 219 pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); 220 } 221 222 return 1; 223 } 224 225 static int __init feat_disable(struct dt_cpu_feature *f) 226 { 227 return 0; 228 } 229 230 static int __init feat_enable_hv(struct dt_cpu_feature *f) 231 { 232 u64 lpcr; 233 234 if (!hv_mode) { 235 pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); 236 return 0; 237 } 238 239 mtspr(SPRN_LPID, 0); 240 241 lpcr = mfspr(SPRN_LPCR); 242 lpcr &= ~LPCR_LPES0; /* HV external interrupts */ 243 mtspr(SPRN_LPCR, lpcr); 244 245 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 246 247 return 1; 248 } 249 250 static int __init feat_enable_le(struct dt_cpu_feature *f) 251 { 252 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; 253 return 1; 254 } 255 256 static int __init feat_enable_smt(struct dt_cpu_feature *f) 257 { 258 cur_cpu_spec->cpu_features |= CPU_FTR_SMT; 259 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; 260 return 1; 261 } 262 263 static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) 264 { 265 u64 lpcr; 266 267 /* Set PECE wakeup modes for ISA 207 */ 268 lpcr = mfspr(SPRN_LPCR); 269 lpcr |= LPCR_PECE0; 270 lpcr |= LPCR_PECE1; 271 lpcr |= LPCR_PECE2; 272 mtspr(SPRN_LPCR, lpcr); 273 274 return 1; 275 } 276 277 static int __init feat_enable_align_dsisr(struct dt_cpu_feature *f) 278 { 279 cur_cpu_spec->cpu_features &= ~CPU_FTR_NODSISRALIGN; 280 281 return 1; 282 } 283 284 static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) 285 { 286 u64 lpcr; 287 288 /* Set PECE wakeup modes for ISAv3.0B */ 289 lpcr = mfspr(SPRN_LPCR); 290 lpcr |= LPCR_PECE0; 291 lpcr |= LPCR_PECE1; 292 lpcr |= LPCR_PECE2; 293 mtspr(SPRN_LPCR, lpcr); 294 295 return 1; 296 } 297 298 static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) 299 { 300 u64 lpcr; 301 302 lpcr = mfspr(SPRN_LPCR); 303 lpcr &= ~LPCR_ISL; 304 305 /* VRMASD */ 306 lpcr |= LPCR_VPM0; 307 lpcr &= ~LPCR_VPM1; 308 lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ 309 mtspr(SPRN_LPCR, lpcr); 310 311 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 312 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 313 314 return 1; 315 } 316 317 static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) 318 { 319 u64 lpcr; 320 321 system_registers.lpcr_clear |= (LPCR_ISL | LPCR_UPRT | LPCR_HR); 322 lpcr = mfspr(SPRN_LPCR); 323 lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); 324 mtspr(SPRN_LPCR, lpcr); 325 326 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 327 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 328 329 return 1; 330 } 331 332 333 static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) 334 { 335 #ifdef CONFIG_PPC_RADIX_MMU 336 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; 337 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 338 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 339 340 return 1; 341 #endif 342 return 0; 343 } 344 345 static int __init feat_enable_dscr(struct dt_cpu_feature *f) 346 { 347 u64 lpcr; 348 349 feat_enable(f); 350 351 lpcr = mfspr(SPRN_LPCR); 352 lpcr &= ~LPCR_DPFD; 353 lpcr |= (4UL << LPCR_DPFD_SH); 354 mtspr(SPRN_LPCR, lpcr); 355 356 return 1; 357 } 358 359 static void hfscr_pmu_enable(void) 360 { 361 u64 hfscr = mfspr(SPRN_HFSCR); 362 hfscr |= PPC_BIT(60); 363 mtspr(SPRN_HFSCR, hfscr); 364 } 365 366 static void init_pmu_power8(void) 367 { 368 if (hv_mode) { 369 mtspr(SPRN_MMCRC, 0); 370 mtspr(SPRN_MMCRH, 0); 371 } 372 373 mtspr(SPRN_MMCRA, 0); 374 mtspr(SPRN_MMCR0, 0); 375 mtspr(SPRN_MMCR1, 0); 376 mtspr(SPRN_MMCR2, 0); 377 mtspr(SPRN_MMCRS, 0); 378 } 379 380 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) 381 { 382 cur_cpu_spec->platform = "power8"; 383 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; 384 385 return 1; 386 } 387 388 static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) 389 { 390 hfscr_pmu_enable(); 391 392 init_pmu_power8(); 393 init_pmu_registers = init_pmu_power8; 394 395 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 396 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 397 if (pvr_version_is(PVR_POWER8E)) 398 cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; 399 400 cur_cpu_spec->num_pmcs = 6; 401 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 402 cur_cpu_spec->oprofile_cpu_type = "ppc64/power8"; 403 404 return 1; 405 } 406 407 static void init_pmu_power9(void) 408 { 409 if (hv_mode) 410 mtspr(SPRN_MMCRC, 0); 411 412 mtspr(SPRN_MMCRA, 0); 413 mtspr(SPRN_MMCR0, 0); 414 mtspr(SPRN_MMCR1, 0); 415 mtspr(SPRN_MMCR2, 0); 416 } 417 418 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) 419 { 420 cur_cpu_spec->platform = "power9"; 421 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; 422 423 return 1; 424 } 425 426 static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) 427 { 428 hfscr_pmu_enable(); 429 430 init_pmu_power9(); 431 init_pmu_registers = init_pmu_power9; 432 433 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 434 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 435 436 cur_cpu_spec->num_pmcs = 6; 437 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 438 cur_cpu_spec->oprofile_cpu_type = "ppc64/power9"; 439 440 return 1; 441 } 442 443 static int __init feat_enable_tm(struct dt_cpu_feature *f) 444 { 445 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 446 feat_enable(f); 447 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; 448 return 1; 449 #endif 450 return 0; 451 } 452 453 static int __init feat_enable_fp(struct dt_cpu_feature *f) 454 { 455 feat_enable(f); 456 cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; 457 458 return 1; 459 } 460 461 static int __init feat_enable_vector(struct dt_cpu_feature *f) 462 { 463 #ifdef CONFIG_ALTIVEC 464 feat_enable(f); 465 cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; 466 cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; 467 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; 468 469 return 1; 470 #endif 471 return 0; 472 } 473 474 static int __init feat_enable_vsx(struct dt_cpu_feature *f) 475 { 476 #ifdef CONFIG_VSX 477 feat_enable(f); 478 cur_cpu_spec->cpu_features |= CPU_FTR_VSX; 479 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; 480 481 return 1; 482 #endif 483 return 0; 484 } 485 486 static int __init feat_enable_purr(struct dt_cpu_feature *f) 487 { 488 cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; 489 490 return 1; 491 } 492 493 static int __init feat_enable_ebb(struct dt_cpu_feature *f) 494 { 495 /* 496 * PPC_FEATURE2_EBB is enabled in PMU init code because it has 497 * historically been related to the PMU facility. This may have 498 * to be decoupled if EBB becomes more generic. For now, follow 499 * existing convention. 500 */ 501 f->hwcap_bit_nr = -1; 502 feat_enable(f); 503 504 return 1; 505 } 506 507 static int __init feat_enable_dbell(struct dt_cpu_feature *f) 508 { 509 u64 lpcr; 510 511 /* P9 has an HFSCR for privileged state */ 512 feat_enable(f); 513 514 cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; 515 516 lpcr = mfspr(SPRN_LPCR); 517 lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ 518 mtspr(SPRN_LPCR, lpcr); 519 520 return 1; 521 } 522 523 static int __init feat_enable_hvi(struct dt_cpu_feature *f) 524 { 525 u64 lpcr; 526 527 /* 528 * POWER9 XIVE interrupts including in OPAL XICS compatibility 529 * are always delivered as hypervisor virtualization interrupts (HVI) 530 * rather than EE. 531 * 532 * However LPES0 is not set here, in the chance that an EE does get 533 * delivered to the host somehow, the EE handler would not expect it 534 * to be delivered in LPES0 mode (e.g., using SRR[01]). This could 535 * happen if there is a bug in interrupt controller code, or IC is 536 * misconfigured in systemsim. 537 */ 538 539 lpcr = mfspr(SPRN_LPCR); 540 lpcr |= LPCR_HVICE; /* enable hvi interrupts */ 541 lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ 542 lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ 543 mtspr(SPRN_LPCR, lpcr); 544 545 return 1; 546 } 547 548 static int __init feat_enable_large_ci(struct dt_cpu_feature *f) 549 { 550 cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; 551 552 return 1; 553 } 554 555 struct dt_cpu_feature_match { 556 const char *name; 557 int (*enable)(struct dt_cpu_feature *f); 558 u64 cpu_ftr_bit_mask; 559 }; 560 561 static struct dt_cpu_feature_match __initdata 562 dt_cpu_feature_match_table[] = { 563 {"hypervisor", feat_enable_hv, 0}, 564 {"big-endian", feat_enable, 0}, 565 {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, 566 {"smt", feat_enable_smt, 0}, 567 {"interrupt-facilities", feat_enable, 0}, 568 {"timer-facilities", feat_enable, 0}, 569 {"timer-facilities-v3", feat_enable, 0}, 570 {"debug-facilities", feat_enable, 0}, 571 {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, 572 {"branch-tracing", feat_enable, 0}, 573 {"floating-point", feat_enable_fp, 0}, 574 {"vector", feat_enable_vector, 0}, 575 {"vector-scalar", feat_enable_vsx, 0}, 576 {"vector-scalar-v3", feat_enable, 0}, 577 {"decimal-floating-point", feat_enable, 0}, 578 {"decimal-integer", feat_enable, 0}, 579 {"quadword-load-store", feat_enable, 0}, 580 {"vector-crypto", feat_enable, 0}, 581 {"mmu-hash", feat_enable_mmu_hash, 0}, 582 {"mmu-radix", feat_enable_mmu_radix, 0}, 583 {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, 584 {"virtual-page-class-key-protection", feat_enable, 0}, 585 {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, 586 {"transactional-memory-v3", feat_enable_tm, 0}, 587 {"tm-suspend-hypervisor-assist", feat_enable, CPU_FTR_P9_TM_HV_ASSIST}, 588 {"tm-suspend-xer-so-bug", feat_enable, CPU_FTR_P9_TM_XER_SO_BUG}, 589 {"idle-nap", feat_enable_idle_nap, 0}, 590 {"alignment-interrupt-dsisr", feat_enable_align_dsisr, 0}, 591 {"idle-stop", feat_enable_idle_stop, 0}, 592 {"machine-check-power8", feat_enable_mce_power8, 0}, 593 {"performance-monitor-power8", feat_enable_pmu_power8, 0}, 594 {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, 595 {"event-based-branch", feat_enable_ebb, 0}, 596 {"target-address-register", feat_enable, 0}, 597 {"branch-history-rolling-buffer", feat_enable, 0}, 598 {"control-register", feat_enable, CPU_FTR_CTRL}, 599 {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, 600 {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, 601 {"processor-utilization-of-resources-register", feat_enable_purr, 0}, 602 {"no-execute", feat_enable, 0}, 603 {"strong-access-ordering", feat_enable, CPU_FTR_SAO}, 604 {"cache-inhibited-large-page", feat_enable_large_ci, 0}, 605 {"coprocessor-icswx", feat_enable, 0}, 606 {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, 607 {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, 608 {"wait", feat_enable, 0}, 609 {"atomic-memory-operations", feat_enable, 0}, 610 {"branch-v3", feat_enable, 0}, 611 {"copy-paste", feat_enable, 0}, 612 {"decimal-floating-point-v3", feat_enable, 0}, 613 {"decimal-integer-v3", feat_enable, 0}, 614 {"fixed-point-v3", feat_enable, 0}, 615 {"floating-point-v3", feat_enable, 0}, 616 {"group-start-register", feat_enable, 0}, 617 {"pc-relative-addressing", feat_enable, 0}, 618 {"machine-check-power9", feat_enable_mce_power9, 0}, 619 {"performance-monitor-power9", feat_enable_pmu_power9, 0}, 620 {"event-based-branch-v3", feat_enable, 0}, 621 {"random-number-generator", feat_enable, 0}, 622 {"system-call-vectored", feat_disable, 0}, 623 {"trace-interrupt-v3", feat_enable, 0}, 624 {"vector-v3", feat_enable, 0}, 625 {"vector-binary128", feat_enable, 0}, 626 {"vector-binary16", feat_enable, 0}, 627 {"wait-v3", feat_enable, 0}, 628 }; 629 630 static bool __initdata using_dt_cpu_ftrs; 631 static bool __initdata enable_unknown = true; 632 633 static int __init dt_cpu_ftrs_parse(char *str) 634 { 635 if (!str) 636 return 0; 637 638 if (!strcmp(str, "off")) 639 using_dt_cpu_ftrs = false; 640 else if (!strcmp(str, "known")) 641 enable_unknown = false; 642 else 643 return 1; 644 645 return 0; 646 } 647 early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); 648 649 static void __init cpufeatures_setup_start(u32 isa) 650 { 651 pr_info("setup for ISA %d\n", isa); 652 653 if (isa >= 3000) { 654 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; 655 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; 656 } 657 } 658 659 static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) 660 { 661 const struct dt_cpu_feature_match *m; 662 bool known = false; 663 int i; 664 665 for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { 666 m = &dt_cpu_feature_match_table[i]; 667 if (!strcmp(f->name, m->name)) { 668 known = true; 669 if (m->enable(f)) { 670 cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; 671 break; 672 } 673 674 pr_info("not enabling: %s (disabled or unsupported by kernel)\n", 675 f->name); 676 return false; 677 } 678 } 679 680 if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) { 681 pr_info("not enabling: %s (unknown and unsupported by kernel)\n", 682 f->name); 683 return false; 684 } 685 686 if (known) 687 pr_debug("enabling: %s\n", f->name); 688 else 689 pr_debug("enabling: %s (unknown)\n", f->name); 690 691 return true; 692 } 693 694 /* 695 * Handle POWER9 broadcast tlbie invalidation issue using 696 * cpu feature flag. 697 */ 698 static __init void update_tlbie_feature_flag(unsigned long pvr) 699 { 700 if (PVR_VER(pvr) == PVR_POWER9) { 701 /* 702 * Set the tlbie feature flag for anything below 703 * Nimbus DD 2.3 and Cumulus DD 1.3 704 */ 705 if ((pvr & 0xe000) == 0) { 706 /* Nimbus */ 707 if ((pvr & 0xfff) < 0x203) 708 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 709 } else if ((pvr & 0xc000) == 0) { 710 /* Cumulus */ 711 if ((pvr & 0xfff) < 0x103) 712 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 713 } else { 714 WARN_ONCE(1, "Unknown PVR"); 715 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 716 } 717 718 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; 719 } 720 } 721 722 static __init void cpufeatures_cpu_quirks(void) 723 { 724 unsigned long version = mfspr(SPRN_PVR); 725 726 /* 727 * Not all quirks can be derived from the cpufeatures device tree. 728 */ 729 if ((version & 0xffffefff) == 0x004e0200) { 730 /* DD2.0 has no feature flag */ 731 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 732 } else if ((version & 0xffffefff) == 0x004e0201) { 733 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 734 cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; 735 } else if ((version & 0xffffefff) == 0x004e0202) { 736 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; 737 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; 738 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 739 } else if ((version & 0xffff0000) == 0x004e0000) { 740 /* DD2.1 and up have DD2_1 */ 741 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; 742 } 743 744 if ((version & 0xffff0000) == 0x004e0000) { 745 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 746 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; 747 } 748 749 update_tlbie_feature_flag(version); 750 /* 751 * PKEY was not in the initial base or feature node 752 * specification, but it should become optional in the next 753 * cpu feature version sequence. 754 */ 755 cur_cpu_spec->cpu_features |= CPU_FTR_PKEY; 756 } 757 758 static void __init cpufeatures_setup_finished(void) 759 { 760 cpufeatures_cpu_quirks(); 761 762 if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { 763 pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); 764 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 765 } 766 767 /* Make sure powerpc_base_platform is non-NULL */ 768 powerpc_base_platform = cur_cpu_spec->platform; 769 770 system_registers.lpcr = mfspr(SPRN_LPCR); 771 system_registers.hfscr = mfspr(SPRN_HFSCR); 772 system_registers.fscr = mfspr(SPRN_FSCR); 773 774 pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", 775 cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); 776 } 777 778 static int __init disabled_on_cmdline(void) 779 { 780 unsigned long root, chosen; 781 const char *p; 782 783 root = of_get_flat_dt_root(); 784 chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); 785 if (chosen == -FDT_ERR_NOTFOUND) 786 return false; 787 788 p = of_get_flat_dt_prop(chosen, "bootargs", NULL); 789 if (!p) 790 return false; 791 792 if (strstr(p, "dt_cpu_ftrs=off")) 793 return true; 794 795 return false; 796 } 797 798 static int __init fdt_find_cpu_features(unsigned long node, const char *uname, 799 int depth, void *data) 800 { 801 if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") 802 && of_get_flat_dt_prop(node, "isa", NULL)) 803 return 1; 804 805 return 0; 806 } 807 808 bool __init dt_cpu_ftrs_in_use(void) 809 { 810 return using_dt_cpu_ftrs; 811 } 812 813 bool __init dt_cpu_ftrs_init(void *fdt) 814 { 815 using_dt_cpu_ftrs = false; 816 817 /* Setup and verify the FDT, if it fails we just bail */ 818 if (!early_init_dt_verify(fdt)) 819 return false; 820 821 if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) 822 return false; 823 824 if (disabled_on_cmdline()) 825 return false; 826 827 cpufeatures_setup_cpu(); 828 829 using_dt_cpu_ftrs = true; 830 return true; 831 } 832 833 static int nr_dt_cpu_features; 834 static struct dt_cpu_feature *dt_cpu_features; 835 836 static int __init process_cpufeatures_node(unsigned long node, 837 const char *uname, int i) 838 { 839 const __be32 *prop; 840 struct dt_cpu_feature *f; 841 int len; 842 843 f = &dt_cpu_features[i]; 844 845 f->node = node; 846 847 f->name = uname; 848 849 prop = of_get_flat_dt_prop(node, "isa", &len); 850 if (!prop) { 851 pr_warn("%s: missing isa property\n", uname); 852 return 0; 853 } 854 f->isa = be32_to_cpup(prop); 855 856 prop = of_get_flat_dt_prop(node, "usable-privilege", &len); 857 if (!prop) { 858 pr_warn("%s: missing usable-privilege property", uname); 859 return 0; 860 } 861 f->usable_privilege = be32_to_cpup(prop); 862 863 prop = of_get_flat_dt_prop(node, "hv-support", &len); 864 if (prop) 865 f->hv_support = be32_to_cpup(prop); 866 else 867 f->hv_support = HV_SUPPORT_NONE; 868 869 prop = of_get_flat_dt_prop(node, "os-support", &len); 870 if (prop) 871 f->os_support = be32_to_cpup(prop); 872 else 873 f->os_support = OS_SUPPORT_NONE; 874 875 prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); 876 if (prop) 877 f->hfscr_bit_nr = be32_to_cpup(prop); 878 else 879 f->hfscr_bit_nr = -1; 880 prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); 881 if (prop) 882 f->fscr_bit_nr = be32_to_cpup(prop); 883 else 884 f->fscr_bit_nr = -1; 885 prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); 886 if (prop) 887 f->hwcap_bit_nr = be32_to_cpup(prop); 888 else 889 f->hwcap_bit_nr = -1; 890 891 if (f->usable_privilege & USABLE_HV) { 892 if (!(mfmsr() & MSR_HV)) { 893 pr_warn("%s: HV feature passed to guest\n", uname); 894 return 0; 895 } 896 897 if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { 898 pr_warn("%s: unwanted hfscr_bit_nr\n", uname); 899 return 0; 900 } 901 902 if (f->hv_support == HV_SUPPORT_HFSCR) { 903 if (f->hfscr_bit_nr == -1) { 904 pr_warn("%s: missing hfscr_bit_nr\n", uname); 905 return 0; 906 } 907 } 908 } else { 909 if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { 910 pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); 911 return 0; 912 } 913 } 914 915 if (f->usable_privilege & USABLE_OS) { 916 if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { 917 pr_warn("%s: unwanted fscr_bit_nr\n", uname); 918 return 0; 919 } 920 921 if (f->os_support == OS_SUPPORT_FSCR) { 922 if (f->fscr_bit_nr == -1) { 923 pr_warn("%s: missing fscr_bit_nr\n", uname); 924 return 0; 925 } 926 } 927 } else { 928 if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { 929 pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); 930 return 0; 931 } 932 } 933 934 if (!(f->usable_privilege & USABLE_PR)) { 935 if (f->hwcap_bit_nr != -1) { 936 pr_warn("%s: unwanted hwcap_bit_nr\n", uname); 937 return 0; 938 } 939 } 940 941 /* Do all the independent features in the first pass */ 942 if (!of_get_flat_dt_prop(node, "dependencies", &len)) { 943 if (cpufeatures_process_feature(f)) 944 f->enabled = 1; 945 else 946 f->disabled = 1; 947 } 948 949 return 0; 950 } 951 952 static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) 953 { 954 const __be32 *prop; 955 int len; 956 int nr_deps; 957 int i; 958 959 if (f->enabled || f->disabled) 960 return; 961 962 prop = of_get_flat_dt_prop(f->node, "dependencies", &len); 963 if (!prop) { 964 pr_warn("%s: missing dependencies property", f->name); 965 return; 966 } 967 968 nr_deps = len / sizeof(int); 969 970 for (i = 0; i < nr_deps; i++) { 971 unsigned long phandle = be32_to_cpu(prop[i]); 972 int j; 973 974 for (j = 0; j < nr_dt_cpu_features; j++) { 975 struct dt_cpu_feature *d = &dt_cpu_features[j]; 976 977 if (of_get_flat_dt_phandle(d->node) == phandle) { 978 cpufeatures_deps_enable(d); 979 if (d->disabled) { 980 f->disabled = 1; 981 return; 982 } 983 } 984 } 985 } 986 987 if (cpufeatures_process_feature(f)) 988 f->enabled = 1; 989 else 990 f->disabled = 1; 991 } 992 993 static int __init scan_cpufeatures_subnodes(unsigned long node, 994 const char *uname, 995 void *data) 996 { 997 int *count = data; 998 999 process_cpufeatures_node(node, uname, *count); 1000 1001 (*count)++; 1002 1003 return 0; 1004 } 1005 1006 static int __init count_cpufeatures_subnodes(unsigned long node, 1007 const char *uname, 1008 void *data) 1009 { 1010 int *count = data; 1011 1012 (*count)++; 1013 1014 return 0; 1015 } 1016 1017 static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char 1018 *uname, int depth, void *data) 1019 { 1020 const __be32 *prop; 1021 int count, i; 1022 u32 isa; 1023 1024 /* We are scanning "ibm,powerpc-cpu-features" nodes only */ 1025 if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) 1026 return 0; 1027 1028 prop = of_get_flat_dt_prop(node, "isa", NULL); 1029 if (!prop) 1030 /* We checked before, "can't happen" */ 1031 return 0; 1032 1033 isa = be32_to_cpup(prop); 1034 1035 /* Count and allocate space for cpu features */ 1036 of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, 1037 &nr_dt_cpu_features); 1038 dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); 1039 if (!dt_cpu_features) 1040 panic("%s: Failed to allocate %zu bytes align=0x%lx\n", 1041 __func__, 1042 sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, 1043 PAGE_SIZE); 1044 1045 cpufeatures_setup_start(isa); 1046 1047 /* Scan nodes into dt_cpu_features and enable those without deps */ 1048 count = 0; 1049 of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); 1050 1051 /* Recursive enable remaining features with dependencies */ 1052 for (i = 0; i < nr_dt_cpu_features; i++) { 1053 struct dt_cpu_feature *f = &dt_cpu_features[i]; 1054 1055 cpufeatures_deps_enable(f); 1056 } 1057 1058 prop = of_get_flat_dt_prop(node, "display-name", NULL); 1059 if (prop && strlen((char *)prop) != 0) { 1060 strlcpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); 1061 cur_cpu_spec->cpu_name = dt_cpu_name; 1062 } 1063 1064 cpufeatures_setup_finished(); 1065 1066 memblock_free(__pa(dt_cpu_features), 1067 sizeof(struct dt_cpu_feature)*nr_dt_cpu_features); 1068 1069 return 0; 1070 } 1071 1072 void __init dt_cpu_ftrs_scan(void) 1073 { 1074 if (!using_dt_cpu_ftrs) 1075 return; 1076 1077 of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); 1078 } 1079