1 /* 2 * Copyright 2017, Nicholas Piggin, IBM Corporation 3 * Licensed under GPLv2. 4 */ 5 6 #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt 7 8 #include <linux/export.h> 9 #include <linux/init.h> 10 #include <linux/jump_label.h> 11 #include <linux/libfdt.h> 12 #include <linux/memblock.h> 13 #include <linux/printk.h> 14 #include <linux/sched.h> 15 #include <linux/string.h> 16 #include <linux/threads.h> 17 18 #include <asm/cputable.h> 19 #include <asm/dt_cpu_ftrs.h> 20 #include <asm/mmu.h> 21 #include <asm/oprofile_impl.h> 22 #include <asm/prom.h> 23 #include <asm/setup.h> 24 25 26 /* Device-tree visible constants follow */ 27 #define ISA_V2_07B 2070 28 #define ISA_V3_0B 3000 29 30 #define USABLE_PR (1U << 0) 31 #define USABLE_OS (1U << 1) 32 #define USABLE_HV (1U << 2) 33 34 #define HV_SUPPORT_HFSCR (1U << 0) 35 #define OS_SUPPORT_FSCR (1U << 0) 36 37 /* For parsing, we define all bits set as "NONE" case */ 38 #define HV_SUPPORT_NONE 0xffffffffU 39 #define OS_SUPPORT_NONE 0xffffffffU 40 41 struct dt_cpu_feature { 42 const char *name; 43 uint32_t isa; 44 uint32_t usable_privilege; 45 uint32_t hv_support; 46 uint32_t os_support; 47 uint32_t hfscr_bit_nr; 48 uint32_t fscr_bit_nr; 49 uint32_t hwcap_bit_nr; 50 /* fdt parsing */ 51 unsigned long node; 52 int enabled; 53 int disabled; 54 }; 55 56 #define CPU_FTRS_BASE \ 57 (CPU_FTR_USE_TB | \ 58 CPU_FTR_LWSYNC | \ 59 CPU_FTR_FPU_UNAVAILABLE |\ 60 CPU_FTR_NODSISRALIGN |\ 61 CPU_FTR_NOEXECUTE |\ 62 CPU_FTR_COHERENT_ICACHE | \ 63 CPU_FTR_STCX_CHECKS_ADDRESS |\ 64 CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ 65 CPU_FTR_DAWR | \ 66 CPU_FTR_ARCH_206 |\ 67 CPU_FTR_ARCH_207S) 68 69 #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) 70 71 #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ 72 PPC_FEATURE_ARCH_2_06 |\ 73 PPC_FEATURE_ICACHE_SNOOP) 74 #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ 75 PPC_FEATURE2_ISEL) 76 /* 77 * Set up the base CPU 78 */ 79 80 extern void __flush_tlb_power8(unsigned int action); 81 extern void __flush_tlb_power9(unsigned int action); 82 extern long __machine_check_early_realmode_p8(struct pt_regs *regs); 83 extern long __machine_check_early_realmode_p9(struct pt_regs *regs); 84 85 static int hv_mode; 86 87 static struct { 88 u64 lpcr; 89 u64 hfscr; 90 u64 fscr; 91 } system_registers; 92 93 static void (*init_pmu_registers)(void); 94 95 static void cpufeatures_flush_tlb(void) 96 { 97 unsigned long rb; 98 unsigned int i, num_sets; 99 100 /* 101 * This is a temporary measure to keep equivalent TLB flush as the 102 * cputable based setup code. 103 */ 104 switch (PVR_VER(mfspr(SPRN_PVR))) { 105 case PVR_POWER8: 106 case PVR_POWER8E: 107 case PVR_POWER8NVL: 108 num_sets = POWER8_TLB_SETS; 109 break; 110 case PVR_POWER9: 111 num_sets = POWER9_TLB_SETS_HASH; 112 break; 113 default: 114 num_sets = 1; 115 pr_err("unknown CPU version for boot TLB flush\n"); 116 break; 117 } 118 119 asm volatile("ptesync" : : : "memory"); 120 rb = TLBIEL_INVAL_SET; 121 for (i = 0; i < num_sets; i++) { 122 asm volatile("tlbiel %0" : : "r" (rb)); 123 rb += 1 << TLBIEL_INVAL_SET_SHIFT; 124 } 125 asm volatile("ptesync" : : : "memory"); 126 } 127 128 static void __restore_cpu_cpufeatures(void) 129 { 130 /* 131 * LPCR is restored by the power on engine already. It can be changed 132 * after early init e.g., by radix enable, and we have no unified API 133 * for saving and restoring such SPRs. 134 * 135 * This ->restore hook should really be removed from idle and register 136 * restore moved directly into the idle restore code, because this code 137 * doesn't know how idle is implemented or what it needs restored here. 138 * 139 * The best we can do to accommodate secondary boot and idle restore 140 * for now is "or" LPCR with existing. 141 */ 142 143 mtspr(SPRN_LPCR, system_registers.lpcr | mfspr(SPRN_LPCR)); 144 if (hv_mode) { 145 mtspr(SPRN_LPID, 0); 146 mtspr(SPRN_HFSCR, system_registers.hfscr); 147 } 148 mtspr(SPRN_FSCR, system_registers.fscr); 149 150 if (init_pmu_registers) 151 init_pmu_registers(); 152 153 cpufeatures_flush_tlb(); 154 } 155 156 static char dt_cpu_name[64]; 157 158 static struct cpu_spec __initdata base_cpu_spec = { 159 .cpu_name = NULL, 160 .cpu_features = CPU_FTRS_BASE, 161 .cpu_user_features = COMMON_USER_BASE, 162 .cpu_user_features2 = COMMON_USER2_BASE, 163 .mmu_features = 0, 164 .icache_bsize = 32, /* minimum block size, fixed by */ 165 .dcache_bsize = 32, /* cache info init. */ 166 .num_pmcs = 0, 167 .pmc_type = PPC_PMC_DEFAULT, 168 .oprofile_cpu_type = NULL, 169 .oprofile_type = PPC_OPROFILE_INVALID, 170 .cpu_setup = NULL, 171 .cpu_restore = __restore_cpu_cpufeatures, 172 .flush_tlb = NULL, 173 .machine_check_early = NULL, 174 .platform = NULL, 175 }; 176 177 static void __init cpufeatures_setup_cpu(void) 178 { 179 set_cur_cpu_spec(&base_cpu_spec); 180 181 cur_cpu_spec->pvr_mask = -1; 182 cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); 183 184 /* Initialize the base environment -- clear FSCR/HFSCR. */ 185 hv_mode = !!(mfmsr() & MSR_HV); 186 if (hv_mode) { 187 /* CPU_FTR_HVMODE is used early in PACA setup */ 188 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 189 mtspr(SPRN_HFSCR, 0); 190 } 191 mtspr(SPRN_FSCR, 0); 192 193 /* 194 * LPCR does not get cleared, to match behaviour with secondaries 195 * in __restore_cpu_cpufeatures. Once the idle code is fixed, this 196 * could clear LPCR too. 197 */ 198 } 199 200 static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) 201 { 202 if (f->hv_support == HV_SUPPORT_NONE) { 203 } else if (f->hv_support & HV_SUPPORT_HFSCR) { 204 u64 hfscr = mfspr(SPRN_HFSCR); 205 hfscr |= 1UL << f->hfscr_bit_nr; 206 mtspr(SPRN_HFSCR, hfscr); 207 } else { 208 /* Does not have a known recipe */ 209 return 0; 210 } 211 212 if (f->os_support == OS_SUPPORT_NONE) { 213 } else if (f->os_support & OS_SUPPORT_FSCR) { 214 u64 fscr = mfspr(SPRN_FSCR); 215 fscr |= 1UL << f->fscr_bit_nr; 216 mtspr(SPRN_FSCR, fscr); 217 } else { 218 /* Does not have a known recipe */ 219 return 0; 220 } 221 222 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 223 uint32_t word = f->hwcap_bit_nr / 32; 224 uint32_t bit = f->hwcap_bit_nr % 32; 225 226 if (word == 0) 227 cur_cpu_spec->cpu_user_features |= 1U << bit; 228 else if (word == 1) 229 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 230 else 231 pr_err("%s could not advertise to user (no hwcap bits)\n", f->name); 232 } 233 234 return 1; 235 } 236 237 static int __init feat_enable(struct dt_cpu_feature *f) 238 { 239 if (f->hv_support != HV_SUPPORT_NONE) { 240 if (f->hfscr_bit_nr != -1) { 241 u64 hfscr = mfspr(SPRN_HFSCR); 242 hfscr |= 1UL << f->hfscr_bit_nr; 243 mtspr(SPRN_HFSCR, hfscr); 244 } 245 } 246 247 if (f->os_support != OS_SUPPORT_NONE) { 248 if (f->fscr_bit_nr != -1) { 249 u64 fscr = mfspr(SPRN_FSCR); 250 fscr |= 1UL << f->fscr_bit_nr; 251 mtspr(SPRN_FSCR, fscr); 252 } 253 } 254 255 if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { 256 uint32_t word = f->hwcap_bit_nr / 32; 257 uint32_t bit = f->hwcap_bit_nr % 32; 258 259 if (word == 0) 260 cur_cpu_spec->cpu_user_features |= 1U << bit; 261 else if (word == 1) 262 cur_cpu_spec->cpu_user_features2 |= 1U << bit; 263 else 264 pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n", f->name); 265 } 266 267 return 1; 268 } 269 270 static int __init feat_disable(struct dt_cpu_feature *f) 271 { 272 return 0; 273 } 274 275 static int __init feat_enable_hv(struct dt_cpu_feature *f) 276 { 277 u64 lpcr; 278 279 if (!hv_mode) { 280 pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n"); 281 return 0; 282 } 283 284 mtspr(SPRN_LPID, 0); 285 286 lpcr = mfspr(SPRN_LPCR); 287 lpcr &= ~LPCR_LPES0; /* HV external interrupts */ 288 mtspr(SPRN_LPCR, lpcr); 289 290 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 291 292 return 1; 293 } 294 295 static int __init feat_enable_le(struct dt_cpu_feature *f) 296 { 297 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; 298 return 1; 299 } 300 301 static int __init feat_enable_smt(struct dt_cpu_feature *f) 302 { 303 cur_cpu_spec->cpu_features |= CPU_FTR_SMT; 304 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; 305 return 1; 306 } 307 308 static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) 309 { 310 u64 lpcr; 311 312 /* Set PECE wakeup modes for ISA 207 */ 313 lpcr = mfspr(SPRN_LPCR); 314 lpcr |= LPCR_PECE0; 315 lpcr |= LPCR_PECE1; 316 lpcr |= LPCR_PECE2; 317 mtspr(SPRN_LPCR, lpcr); 318 319 return 1; 320 } 321 322 static int __init feat_enable_align_dsisr(struct dt_cpu_feature *f) 323 { 324 cur_cpu_spec->cpu_features &= ~CPU_FTR_NODSISRALIGN; 325 326 return 1; 327 } 328 329 static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) 330 { 331 u64 lpcr; 332 333 /* Set PECE wakeup modes for ISAv3.0B */ 334 lpcr = mfspr(SPRN_LPCR); 335 lpcr |= LPCR_PECE0; 336 lpcr |= LPCR_PECE1; 337 lpcr |= LPCR_PECE2; 338 mtspr(SPRN_LPCR, lpcr); 339 340 return 1; 341 } 342 343 static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) 344 { 345 u64 lpcr; 346 347 lpcr = mfspr(SPRN_LPCR); 348 lpcr &= ~LPCR_ISL; 349 350 /* VRMASD */ 351 lpcr |= LPCR_VPM0; 352 lpcr &= ~LPCR_VPM1; 353 lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ 354 mtspr(SPRN_LPCR, lpcr); 355 356 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 357 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 358 359 return 1; 360 } 361 362 static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) 363 { 364 u64 lpcr; 365 366 lpcr = mfspr(SPRN_LPCR); 367 lpcr &= ~LPCR_ISL; 368 mtspr(SPRN_LPCR, lpcr); 369 370 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 371 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 372 373 return 1; 374 } 375 376 377 static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) 378 { 379 #ifdef CONFIG_PPC_RADIX_MMU 380 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; 381 cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; 382 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; 383 384 return 1; 385 #endif 386 return 0; 387 } 388 389 static int __init feat_enable_dscr(struct dt_cpu_feature *f) 390 { 391 u64 lpcr; 392 393 feat_enable(f); 394 395 lpcr = mfspr(SPRN_LPCR); 396 lpcr &= ~LPCR_DPFD; 397 lpcr |= (4UL << LPCR_DPFD_SH); 398 mtspr(SPRN_LPCR, lpcr); 399 400 return 1; 401 } 402 403 static void hfscr_pmu_enable(void) 404 { 405 u64 hfscr = mfspr(SPRN_HFSCR); 406 hfscr |= PPC_BIT(60); 407 mtspr(SPRN_HFSCR, hfscr); 408 } 409 410 static void init_pmu_power8(void) 411 { 412 if (hv_mode) { 413 mtspr(SPRN_MMCRC, 0); 414 mtspr(SPRN_MMCRH, 0); 415 } 416 417 mtspr(SPRN_MMCRA, 0); 418 mtspr(SPRN_MMCR0, 0); 419 mtspr(SPRN_MMCR1, 0); 420 mtspr(SPRN_MMCR2, 0); 421 mtspr(SPRN_MMCRS, 0); 422 } 423 424 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) 425 { 426 cur_cpu_spec->platform = "power8"; 427 cur_cpu_spec->flush_tlb = __flush_tlb_power8; 428 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; 429 430 return 1; 431 } 432 433 static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) 434 { 435 hfscr_pmu_enable(); 436 437 init_pmu_power8(); 438 init_pmu_registers = init_pmu_power8; 439 440 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 441 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 442 if (pvr_version_is(PVR_POWER8E)) 443 cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; 444 445 cur_cpu_spec->num_pmcs = 6; 446 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 447 cur_cpu_spec->oprofile_cpu_type = "ppc64/power8"; 448 449 return 1; 450 } 451 452 static void init_pmu_power9(void) 453 { 454 if (hv_mode) 455 mtspr(SPRN_MMCRC, 0); 456 457 mtspr(SPRN_MMCRA, 0); 458 mtspr(SPRN_MMCR0, 0); 459 mtspr(SPRN_MMCR1, 0); 460 mtspr(SPRN_MMCR2, 0); 461 } 462 463 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) 464 { 465 cur_cpu_spec->platform = "power9"; 466 cur_cpu_spec->flush_tlb = __flush_tlb_power9; 467 cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; 468 469 return 1; 470 } 471 472 static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) 473 { 474 hfscr_pmu_enable(); 475 476 init_pmu_power9(); 477 init_pmu_registers = init_pmu_power9; 478 479 cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; 480 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; 481 482 cur_cpu_spec->num_pmcs = 6; 483 cur_cpu_spec->pmc_type = PPC_PMC_IBM; 484 cur_cpu_spec->oprofile_cpu_type = "ppc64/power9"; 485 486 return 1; 487 } 488 489 static int __init feat_enable_tm(struct dt_cpu_feature *f) 490 { 491 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 492 feat_enable(f); 493 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; 494 return 1; 495 #endif 496 return 0; 497 } 498 499 static int __init feat_enable_fp(struct dt_cpu_feature *f) 500 { 501 feat_enable(f); 502 cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; 503 504 return 1; 505 } 506 507 static int __init feat_enable_vector(struct dt_cpu_feature *f) 508 { 509 #ifdef CONFIG_ALTIVEC 510 feat_enable(f); 511 cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; 512 cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; 513 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; 514 515 return 1; 516 #endif 517 return 0; 518 } 519 520 static int __init feat_enable_vsx(struct dt_cpu_feature *f) 521 { 522 #ifdef CONFIG_VSX 523 feat_enable(f); 524 cur_cpu_spec->cpu_features |= CPU_FTR_VSX; 525 cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; 526 527 return 1; 528 #endif 529 return 0; 530 } 531 532 static int __init feat_enable_purr(struct dt_cpu_feature *f) 533 { 534 cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; 535 536 return 1; 537 } 538 539 static int __init feat_enable_ebb(struct dt_cpu_feature *f) 540 { 541 /* 542 * PPC_FEATURE2_EBB is enabled in PMU init code because it has 543 * historically been related to the PMU facility. This may have 544 * to be decoupled if EBB becomes more generic. For now, follow 545 * existing convention. 546 */ 547 f->hwcap_bit_nr = -1; 548 feat_enable(f); 549 550 return 1; 551 } 552 553 static int __init feat_enable_dbell(struct dt_cpu_feature *f) 554 { 555 u64 lpcr; 556 557 /* P9 has an HFSCR for privileged state */ 558 feat_enable(f); 559 560 cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; 561 562 lpcr = mfspr(SPRN_LPCR); 563 lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ 564 mtspr(SPRN_LPCR, lpcr); 565 566 return 1; 567 } 568 569 static int __init feat_enable_hvi(struct dt_cpu_feature *f) 570 { 571 u64 lpcr; 572 573 /* 574 * POWER9 XIVE interrupts including in OPAL XICS compatibility 575 * are always delivered as hypervisor virtualization interrupts (HVI) 576 * rather than EE. 577 * 578 * However LPES0 is not set here, in the chance that an EE does get 579 * delivered to the host somehow, the EE handler would not expect it 580 * to be delivered in LPES0 mode (e.g., using SRR[01]). This could 581 * happen if there is a bug in interrupt controller code, or IC is 582 * misconfigured in systemsim. 583 */ 584 585 lpcr = mfspr(SPRN_LPCR); 586 lpcr |= LPCR_HVICE; /* enable hvi interrupts */ 587 lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ 588 lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ 589 mtspr(SPRN_LPCR, lpcr); 590 591 return 1; 592 } 593 594 static int __init feat_enable_large_ci(struct dt_cpu_feature *f) 595 { 596 cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; 597 598 return 1; 599 } 600 601 struct dt_cpu_feature_match { 602 const char *name; 603 int (*enable)(struct dt_cpu_feature *f); 604 u64 cpu_ftr_bit_mask; 605 }; 606 607 static struct dt_cpu_feature_match __initdata 608 dt_cpu_feature_match_table[] = { 609 {"hypervisor", feat_enable_hv, 0}, 610 {"big-endian", feat_enable, 0}, 611 {"little-endian", feat_enable_le, CPU_FTR_REAL_LE}, 612 {"smt", feat_enable_smt, 0}, 613 {"interrupt-facilities", feat_enable, 0}, 614 {"timer-facilities", feat_enable, 0}, 615 {"timer-facilities-v3", feat_enable, 0}, 616 {"debug-facilities", feat_enable, 0}, 617 {"come-from-address-register", feat_enable, CPU_FTR_CFAR}, 618 {"branch-tracing", feat_enable, 0}, 619 {"floating-point", feat_enable_fp, 0}, 620 {"vector", feat_enable_vector, 0}, 621 {"vector-scalar", feat_enable_vsx, 0}, 622 {"vector-scalar-v3", feat_enable, 0}, 623 {"decimal-floating-point", feat_enable, 0}, 624 {"decimal-integer", feat_enable, 0}, 625 {"quadword-load-store", feat_enable, 0}, 626 {"vector-crypto", feat_enable, 0}, 627 {"mmu-hash", feat_enable_mmu_hash, 0}, 628 {"mmu-radix", feat_enable_mmu_radix, 0}, 629 {"mmu-hash-v3", feat_enable_mmu_hash_v3, 0}, 630 {"virtual-page-class-key-protection", feat_enable, 0}, 631 {"transactional-memory", feat_enable_tm, CPU_FTR_TM}, 632 {"transactional-memory-v3", feat_enable_tm, 0}, 633 {"idle-nap", feat_enable_idle_nap, 0}, 634 {"alignment-interrupt-dsisr", feat_enable_align_dsisr, 0}, 635 {"idle-stop", feat_enable_idle_stop, 0}, 636 {"machine-check-power8", feat_enable_mce_power8, 0}, 637 {"performance-monitor-power8", feat_enable_pmu_power8, 0}, 638 {"data-stream-control-register", feat_enable_dscr, CPU_FTR_DSCR}, 639 {"event-based-branch", feat_enable_ebb, 0}, 640 {"target-address-register", feat_enable, 0}, 641 {"branch-history-rolling-buffer", feat_enable, 0}, 642 {"control-register", feat_enable, CPU_FTR_CTRL}, 643 {"processor-control-facility", feat_enable_dbell, CPU_FTR_DBELL}, 644 {"processor-control-facility-v3", feat_enable_dbell, CPU_FTR_DBELL}, 645 {"processor-utilization-of-resources-register", feat_enable_purr, 0}, 646 {"no-execute", feat_enable, 0}, 647 {"strong-access-ordering", feat_enable, CPU_FTR_SAO}, 648 {"cache-inhibited-large-page", feat_enable_large_ci, 0}, 649 {"coprocessor-icswx", feat_enable, CPU_FTR_ICSWX}, 650 {"hypervisor-virtualization-interrupt", feat_enable_hvi, 0}, 651 {"program-priority-register", feat_enable, CPU_FTR_HAS_PPR}, 652 {"wait", feat_enable, 0}, 653 {"atomic-memory-operations", feat_enable, 0}, 654 {"branch-v3", feat_enable, 0}, 655 {"copy-paste", feat_enable, 0}, 656 {"decimal-floating-point-v3", feat_enable, 0}, 657 {"decimal-integer-v3", feat_enable, 0}, 658 {"fixed-point-v3", feat_enable, 0}, 659 {"floating-point-v3", feat_enable, 0}, 660 {"group-start-register", feat_enable, 0}, 661 {"pc-relative-addressing", feat_enable, 0}, 662 {"machine-check-power9", feat_enable_mce_power9, 0}, 663 {"performance-monitor-power9", feat_enable_pmu_power9, 0}, 664 {"event-based-branch-v3", feat_enable, 0}, 665 {"random-number-generator", feat_enable, 0}, 666 {"system-call-vectored", feat_disable, 0}, 667 {"trace-interrupt-v3", feat_enable, 0}, 668 {"vector-v3", feat_enable, 0}, 669 {"vector-binary128", feat_enable, 0}, 670 {"vector-binary16", feat_enable, 0}, 671 {"wait-v3", feat_enable, 0}, 672 }; 673 674 static bool __initdata using_dt_cpu_ftrs; 675 static bool __initdata enable_unknown = true; 676 677 static int __init dt_cpu_ftrs_parse(char *str) 678 { 679 if (!str) 680 return 0; 681 682 if (!strcmp(str, "off")) 683 using_dt_cpu_ftrs = false; 684 else if (!strcmp(str, "known")) 685 enable_unknown = false; 686 else 687 return 1; 688 689 return 0; 690 } 691 early_param("dt_cpu_ftrs", dt_cpu_ftrs_parse); 692 693 static void __init cpufeatures_setup_start(u32 isa) 694 { 695 pr_info("setup for ISA %d\n", isa); 696 697 if (isa >= 3000) { 698 cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; 699 cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; 700 } 701 } 702 703 static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) 704 { 705 const struct dt_cpu_feature_match *m; 706 bool known = false; 707 int i; 708 709 for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { 710 m = &dt_cpu_feature_match_table[i]; 711 if (!strcmp(f->name, m->name)) { 712 known = true; 713 if (m->enable(f)) 714 break; 715 716 pr_info("not enabling: %s (disabled or unsupported by kernel)\n", 717 f->name); 718 return false; 719 } 720 } 721 722 if (!known && enable_unknown) { 723 if (!feat_try_enable_unknown(f)) { 724 pr_info("not enabling: %s (unknown and unsupported by kernel)\n", 725 f->name); 726 return false; 727 } 728 } 729 730 if (m->cpu_ftr_bit_mask) 731 cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; 732 733 if (known) 734 pr_debug("enabling: %s\n", f->name); 735 else 736 pr_debug("enabling: %s (unknown)\n", f->name); 737 738 return true; 739 } 740 741 static __init void cpufeatures_cpu_quirks(void) 742 { 743 int version = mfspr(SPRN_PVR); 744 745 /* 746 * Not all quirks can be derived from the cpufeatures device tree. 747 */ 748 if ((version & 0xffffff00) == 0x004e0100) 749 cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1; 750 } 751 752 static void __init cpufeatures_setup_finished(void) 753 { 754 cpufeatures_cpu_quirks(); 755 756 if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { 757 pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n"); 758 cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; 759 } 760 761 system_registers.lpcr = mfspr(SPRN_LPCR); 762 system_registers.hfscr = mfspr(SPRN_HFSCR); 763 system_registers.fscr = mfspr(SPRN_FSCR); 764 765 cpufeatures_flush_tlb(); 766 767 pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", 768 cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); 769 } 770 771 static int __init disabled_on_cmdline(void) 772 { 773 unsigned long root, chosen; 774 const char *p; 775 776 root = of_get_flat_dt_root(); 777 chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); 778 if (chosen == -FDT_ERR_NOTFOUND) 779 return false; 780 781 p = of_get_flat_dt_prop(chosen, "bootargs", NULL); 782 if (!p) 783 return false; 784 785 if (strstr(p, "dt_cpu_ftrs=off")) 786 return true; 787 788 return false; 789 } 790 791 static int __init fdt_find_cpu_features(unsigned long node, const char *uname, 792 int depth, void *data) 793 { 794 if (of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features") 795 && of_get_flat_dt_prop(node, "isa", NULL)) 796 return 1; 797 798 return 0; 799 } 800 801 bool __init dt_cpu_ftrs_in_use(void) 802 { 803 return using_dt_cpu_ftrs; 804 } 805 806 bool __init dt_cpu_ftrs_init(void *fdt) 807 { 808 using_dt_cpu_ftrs = false; 809 810 /* Setup and verify the FDT, if it fails we just bail */ 811 if (!early_init_dt_verify(fdt)) 812 return false; 813 814 if (!of_scan_flat_dt(fdt_find_cpu_features, NULL)) 815 return false; 816 817 if (disabled_on_cmdline()) 818 return false; 819 820 cpufeatures_setup_cpu(); 821 822 using_dt_cpu_ftrs = true; 823 return true; 824 } 825 826 static int nr_dt_cpu_features; 827 static struct dt_cpu_feature *dt_cpu_features; 828 829 static int __init process_cpufeatures_node(unsigned long node, 830 const char *uname, int i) 831 { 832 const __be32 *prop; 833 struct dt_cpu_feature *f; 834 int len; 835 836 f = &dt_cpu_features[i]; 837 memset(f, 0, sizeof(struct dt_cpu_feature)); 838 839 f->node = node; 840 841 f->name = uname; 842 843 prop = of_get_flat_dt_prop(node, "isa", &len); 844 if (!prop) { 845 pr_warn("%s: missing isa property\n", uname); 846 return 0; 847 } 848 f->isa = be32_to_cpup(prop); 849 850 prop = of_get_flat_dt_prop(node, "usable-privilege", &len); 851 if (!prop) { 852 pr_warn("%s: missing usable-privilege property", uname); 853 return 0; 854 } 855 f->usable_privilege = be32_to_cpup(prop); 856 857 prop = of_get_flat_dt_prop(node, "hv-support", &len); 858 if (prop) 859 f->hv_support = be32_to_cpup(prop); 860 else 861 f->hv_support = HV_SUPPORT_NONE; 862 863 prop = of_get_flat_dt_prop(node, "os-support", &len); 864 if (prop) 865 f->os_support = be32_to_cpup(prop); 866 else 867 f->os_support = OS_SUPPORT_NONE; 868 869 prop = of_get_flat_dt_prop(node, "hfscr-bit-nr", &len); 870 if (prop) 871 f->hfscr_bit_nr = be32_to_cpup(prop); 872 else 873 f->hfscr_bit_nr = -1; 874 prop = of_get_flat_dt_prop(node, "fscr-bit-nr", &len); 875 if (prop) 876 f->fscr_bit_nr = be32_to_cpup(prop); 877 else 878 f->fscr_bit_nr = -1; 879 prop = of_get_flat_dt_prop(node, "hwcap-bit-nr", &len); 880 if (prop) 881 f->hwcap_bit_nr = be32_to_cpup(prop); 882 else 883 f->hwcap_bit_nr = -1; 884 885 if (f->usable_privilege & USABLE_HV) { 886 if (!(mfmsr() & MSR_HV)) { 887 pr_warn("%s: HV feature passed to guest\n", uname); 888 return 0; 889 } 890 891 if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { 892 pr_warn("%s: unwanted hfscr_bit_nr\n", uname); 893 return 0; 894 } 895 896 if (f->hv_support == HV_SUPPORT_HFSCR) { 897 if (f->hfscr_bit_nr == -1) { 898 pr_warn("%s: missing hfscr_bit_nr\n", uname); 899 return 0; 900 } 901 } 902 } else { 903 if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { 904 pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n", uname); 905 return 0; 906 } 907 } 908 909 if (f->usable_privilege & USABLE_OS) { 910 if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { 911 pr_warn("%s: unwanted fscr_bit_nr\n", uname); 912 return 0; 913 } 914 915 if (f->os_support == OS_SUPPORT_FSCR) { 916 if (f->fscr_bit_nr == -1) { 917 pr_warn("%s: missing fscr_bit_nr\n", uname); 918 return 0; 919 } 920 } 921 } else { 922 if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { 923 pr_warn("%s: unwanted os_support/fscr_bit_nr\n", uname); 924 return 0; 925 } 926 } 927 928 if (!(f->usable_privilege & USABLE_PR)) { 929 if (f->hwcap_bit_nr != -1) { 930 pr_warn("%s: unwanted hwcap_bit_nr\n", uname); 931 return 0; 932 } 933 } 934 935 /* Do all the independent features in the first pass */ 936 if (!of_get_flat_dt_prop(node, "dependencies", &len)) { 937 if (cpufeatures_process_feature(f)) 938 f->enabled = 1; 939 else 940 f->disabled = 1; 941 } 942 943 return 0; 944 } 945 946 static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) 947 { 948 const __be32 *prop; 949 int len; 950 int nr_deps; 951 int i; 952 953 if (f->enabled || f->disabled) 954 return; 955 956 prop = of_get_flat_dt_prop(f->node, "dependencies", &len); 957 if (!prop) { 958 pr_warn("%s: missing dependencies property", f->name); 959 return; 960 } 961 962 nr_deps = len / sizeof(int); 963 964 for (i = 0; i < nr_deps; i++) { 965 unsigned long phandle = be32_to_cpu(prop[i]); 966 int j; 967 968 for (j = 0; j < nr_dt_cpu_features; j++) { 969 struct dt_cpu_feature *d = &dt_cpu_features[j]; 970 971 if (of_get_flat_dt_phandle(d->node) == phandle) { 972 cpufeatures_deps_enable(d); 973 if (d->disabled) { 974 f->disabled = 1; 975 return; 976 } 977 } 978 } 979 } 980 981 if (cpufeatures_process_feature(f)) 982 f->enabled = 1; 983 else 984 f->disabled = 1; 985 } 986 987 static int __init scan_cpufeatures_subnodes(unsigned long node, 988 const char *uname, 989 void *data) 990 { 991 int *count = data; 992 993 process_cpufeatures_node(node, uname, *count); 994 995 (*count)++; 996 997 return 0; 998 } 999 1000 static int __init count_cpufeatures_subnodes(unsigned long node, 1001 const char *uname, 1002 void *data) 1003 { 1004 int *count = data; 1005 1006 (*count)++; 1007 1008 return 0; 1009 } 1010 1011 static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char 1012 *uname, int depth, void *data) 1013 { 1014 const __be32 *prop; 1015 int count, i; 1016 u32 isa; 1017 1018 /* We are scanning "ibm,powerpc-cpu-features" nodes only */ 1019 if (!of_flat_dt_is_compatible(node, "ibm,powerpc-cpu-features")) 1020 return 0; 1021 1022 prop = of_get_flat_dt_prop(node, "isa", NULL); 1023 if (!prop) 1024 /* We checked before, "can't happen" */ 1025 return 0; 1026 1027 isa = be32_to_cpup(prop); 1028 1029 /* Count and allocate space for cpu features */ 1030 of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, 1031 &nr_dt_cpu_features); 1032 dt_cpu_features = __va( 1033 memblock_alloc(sizeof(struct dt_cpu_feature)* 1034 nr_dt_cpu_features, PAGE_SIZE)); 1035 1036 cpufeatures_setup_start(isa); 1037 1038 /* Scan nodes into dt_cpu_features and enable those without deps */ 1039 count = 0; 1040 of_scan_flat_dt_subnodes(node, scan_cpufeatures_subnodes, &count); 1041 1042 /* Recursive enable remaining features with dependencies */ 1043 for (i = 0; i < nr_dt_cpu_features; i++) { 1044 struct dt_cpu_feature *f = &dt_cpu_features[i]; 1045 1046 cpufeatures_deps_enable(f); 1047 } 1048 1049 prop = of_get_flat_dt_prop(node, "display-name", NULL); 1050 if (prop && strlen((char *)prop) != 0) { 1051 strlcpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); 1052 cur_cpu_spec->cpu_name = dt_cpu_name; 1053 } 1054 1055 cpufeatures_setup_finished(); 1056 1057 memblock_free(__pa(dt_cpu_features), 1058 sizeof(struct dt_cpu_feature)*nr_dt_cpu_features); 1059 1060 return 0; 1061 } 1062 1063 void __init dt_cpu_ftrs_scan(void) 1064 { 1065 if (!using_dt_cpu_ftrs) 1066 return; 1067 1068 of_scan_flat_dt(dt_cpu_ftrs_scan_callback, NULL); 1069 } 1070