1e388b802SRussell King // SPDX-License-Identifier: GPL-2.0 210115105SRussell King #include <linux/arm-smccc.h> 3e388b802SRussell King #include <linux/kernel.h> 4e388b802SRussell King #include <linux/smp.h> 5e388b802SRussell King 6f5fe12b1SRussell King #include <asm/cp15.h> 7f5fe12b1SRussell King #include <asm/cputype.h> 810115105SRussell King #include <asm/proc-fns.h> 9f02cab2bSRussell King (Oracle) #include <asm/spectre.h> 10f5fe12b1SRussell King #include <asm/system_misc.h> 11f5fe12b1SRussell King 12f02cab2bSRussell King (Oracle) #ifdef CONFIG_ARM_PSCI 13f02cab2bSRussell King (Oracle) static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) 14f02cab2bSRussell King (Oracle) { 15f02cab2bSRussell King (Oracle) struct arm_smccc_res res; 16f02cab2bSRussell King (Oracle) 17f02cab2bSRussell King (Oracle) arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, 18f02cab2bSRussell King (Oracle) ARM_SMCCC_ARCH_WORKAROUND_1, &res); 19f02cab2bSRussell King (Oracle) 20f02cab2bSRussell King (Oracle) switch ((int)res.a0) { 21f02cab2bSRussell King (Oracle) case SMCCC_RET_SUCCESS: 22f02cab2bSRussell King (Oracle) return SPECTRE_MITIGATED; 23f02cab2bSRussell King (Oracle) 24f02cab2bSRussell King (Oracle) case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: 25f02cab2bSRussell King (Oracle) return SPECTRE_UNAFFECTED; 26f02cab2bSRussell King (Oracle) 27f02cab2bSRussell King (Oracle) default: 28f02cab2bSRussell King (Oracle) return SPECTRE_VULNERABLE; 29f02cab2bSRussell King (Oracle) } 30f02cab2bSRussell King (Oracle) } 31f02cab2bSRussell King (Oracle) #else 32f02cab2bSRussell King (Oracle) static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) 33f02cab2bSRussell King (Oracle) { 34f02cab2bSRussell King (Oracle) return SPECTRE_VULNERABLE; 35f02cab2bSRussell King (Oracle) } 36f02cab2bSRussell King (Oracle) #endif 37f02cab2bSRussell King (Oracle) 38f5fe12b1SRussell King #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR 39f5fe12b1SRussell King DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); 40f5fe12b1SRussell King 41c44f366eSRussell King extern void cpu_v7_iciallu_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); 42c44f366eSRussell King extern void cpu_v7_bpiall_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); 4310115105SRussell King extern void cpu_v7_smc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); 4410115105SRussell King extern void cpu_v7_hvc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); 4510115105SRussell King 46f5fe12b1SRussell King static void harden_branch_predictor_bpiall(void) 47f5fe12b1SRussell King { 48f5fe12b1SRussell King write_sysreg(0, BPIALL); 49f5fe12b1SRussell King } 50f5fe12b1SRussell King 51f5fe12b1SRussell King static void harden_branch_predictor_iciallu(void) 52f5fe12b1SRussell King { 53f5fe12b1SRussell King write_sysreg(0, ICIALLU); 54f5fe12b1SRussell King } 55f5fe12b1SRussell King 5610115105SRussell King static void __maybe_unused call_smc_arch_workaround_1(void) 5710115105SRussell King { 5810115105SRussell King arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); 5910115105SRussell King } 6010115105SRussell King 6110115105SRussell King static void __maybe_unused call_hvc_arch_workaround_1(void) 6210115105SRussell King { 6310115105SRussell King arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); 6410115105SRussell King } 6510115105SRussell King 66f02cab2bSRussell King (Oracle) static unsigned int spectre_v2_install_workaround(unsigned int method) 67f5fe12b1SRussell King { 68f5fe12b1SRussell King const char *spectre_v2_method = NULL; 69f5fe12b1SRussell King int cpu = smp_processor_id(); 70f5fe12b1SRussell King 71f5fe12b1SRussell King if (per_cpu(harden_branch_predictor_fn, cpu)) 72f02cab2bSRussell King (Oracle) return SPECTRE_MITIGATED; 73f02cab2bSRussell King (Oracle) 74f02cab2bSRussell King (Oracle) switch (method) { 75f02cab2bSRussell King (Oracle) case SPECTRE_V2_METHOD_BPIALL: 76f02cab2bSRussell King (Oracle) per_cpu(harden_branch_predictor_fn, cpu) = 77f02cab2bSRussell King (Oracle) harden_branch_predictor_bpiall; 78f02cab2bSRussell King (Oracle) spectre_v2_method = "BPIALL"; 79f02cab2bSRussell King (Oracle) break; 80f02cab2bSRussell King (Oracle) 81f02cab2bSRussell King (Oracle) case SPECTRE_V2_METHOD_ICIALLU: 82f02cab2bSRussell King (Oracle) per_cpu(harden_branch_predictor_fn, cpu) = 83f02cab2bSRussell King (Oracle) harden_branch_predictor_iciallu; 84f02cab2bSRussell King (Oracle) spectre_v2_method = "ICIALLU"; 85f02cab2bSRussell King (Oracle) break; 86f02cab2bSRussell King (Oracle) 87f02cab2bSRussell King (Oracle) case SPECTRE_V2_METHOD_HVC: 88f02cab2bSRussell King (Oracle) per_cpu(harden_branch_predictor_fn, cpu) = 89f02cab2bSRussell King (Oracle) call_hvc_arch_workaround_1; 90f02cab2bSRussell King (Oracle) cpu_do_switch_mm = cpu_v7_hvc_switch_mm; 91f02cab2bSRussell King (Oracle) spectre_v2_method = "hypervisor"; 92f02cab2bSRussell King (Oracle) break; 93f02cab2bSRussell King (Oracle) 94f02cab2bSRussell King (Oracle) case SPECTRE_V2_METHOD_SMC: 95f02cab2bSRussell King (Oracle) per_cpu(harden_branch_predictor_fn, cpu) = 96f02cab2bSRussell King (Oracle) call_smc_arch_workaround_1; 97f02cab2bSRussell King (Oracle) cpu_do_switch_mm = cpu_v7_smc_switch_mm; 98f02cab2bSRussell King (Oracle) spectre_v2_method = "firmware"; 99f02cab2bSRussell King (Oracle) break; 100f02cab2bSRussell King (Oracle) } 101f02cab2bSRussell King (Oracle) 102f02cab2bSRussell King (Oracle) if (spectre_v2_method) 103f02cab2bSRussell King (Oracle) pr_info("CPU%u: Spectre v2: using %s workaround\n", 104f02cab2bSRussell King (Oracle) smp_processor_id(), spectre_v2_method); 105f02cab2bSRussell King (Oracle) 106f02cab2bSRussell King (Oracle) return SPECTRE_MITIGATED; 107f02cab2bSRussell King (Oracle) } 108f02cab2bSRussell King (Oracle) #else 109f02cab2bSRussell King (Oracle) static unsigned int spectre_v2_install_workaround(unsigned int method) 110f02cab2bSRussell King (Oracle) { 111*2b2d2a8bSRussell King (Oracle) pr_info("CPU%u: Spectre V2: workarounds disabled by configuration\n", 112*2b2d2a8bSRussell King (Oracle) smp_processor_id()); 113f02cab2bSRussell King (Oracle) 114f02cab2bSRussell King (Oracle) return SPECTRE_VULNERABLE; 115f02cab2bSRussell King (Oracle) } 116f02cab2bSRussell King (Oracle) #endif 117f02cab2bSRussell King (Oracle) 118f02cab2bSRussell King (Oracle) static void cpu_v7_spectre_v2_init(void) 119f02cab2bSRussell King (Oracle) { 120f02cab2bSRussell King (Oracle) unsigned int state, method = 0; 121f5fe12b1SRussell King 122f5fe12b1SRussell King switch (read_cpuid_part()) { 123f5fe12b1SRussell King case ARM_CPU_PART_CORTEX_A8: 124f5fe12b1SRussell King case ARM_CPU_PART_CORTEX_A9: 125f5fe12b1SRussell King case ARM_CPU_PART_CORTEX_A12: 126f5fe12b1SRussell King case ARM_CPU_PART_CORTEX_A17: 127f5fe12b1SRussell King case ARM_CPU_PART_CORTEX_A73: 128f5fe12b1SRussell King case ARM_CPU_PART_CORTEX_A75: 129f02cab2bSRussell King (Oracle) state = SPECTRE_MITIGATED; 130f02cab2bSRussell King (Oracle) method = SPECTRE_V2_METHOD_BPIALL; 131f5fe12b1SRussell King break; 132f5fe12b1SRussell King 133f5fe12b1SRussell King case ARM_CPU_PART_CORTEX_A15: 134f5fe12b1SRussell King case ARM_CPU_PART_BRAHMA_B15: 135f02cab2bSRussell King (Oracle) state = SPECTRE_MITIGATED; 136f02cab2bSRussell King (Oracle) method = SPECTRE_V2_METHOD_ICIALLU; 137f5fe12b1SRussell King break; 13810115105SRussell King 1394ae5061aSDoug Berger case ARM_CPU_PART_BRAHMA_B53: 1404ae5061aSDoug Berger /* Requires no workaround */ 141f02cab2bSRussell King (Oracle) state = SPECTRE_UNAFFECTED; 1424ae5061aSDoug Berger break; 143f02cab2bSRussell King (Oracle) 14410115105SRussell King default: 14510115105SRussell King /* Other ARM CPUs require no workaround */ 146f02cab2bSRussell King (Oracle) if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) { 147f02cab2bSRussell King (Oracle) state = SPECTRE_UNAFFECTED; 14810115105SRussell King break; 149f02cab2bSRussell King (Oracle) } 150f02cab2bSRussell King (Oracle) 151df561f66SGustavo A. R. Silva fallthrough; 152f02cab2bSRussell King (Oracle) 15310115105SRussell King /* Cortex A57/A72 require firmware workaround */ 15410115105SRussell King case ARM_CPU_PART_CORTEX_A57: 155f02cab2bSRussell King (Oracle) case ARM_CPU_PART_CORTEX_A72: 156f02cab2bSRussell King (Oracle) state = spectre_v2_get_cpu_fw_mitigation_state(); 157f02cab2bSRussell King (Oracle) if (state != SPECTRE_MITIGATED) 158f02cab2bSRussell King (Oracle) break; 159ce4d5ca2SSteven Price 160ce4d5ca2SSteven Price switch (arm_smccc_1_1_get_conduit()) { 161ce4d5ca2SSteven Price case SMCCC_CONDUIT_HVC: 162f02cab2bSRussell King (Oracle) method = SPECTRE_V2_METHOD_HVC; 16310115105SRussell King break; 16410115105SRussell King 1656848253dSMark Rutland case SMCCC_CONDUIT_SMC: 166f02cab2bSRussell King (Oracle) method = SPECTRE_V2_METHOD_SMC; 16710115105SRussell King break; 16810115105SRussell King 16910115105SRussell King default: 170f02cab2bSRussell King (Oracle) state = SPECTRE_VULNERABLE; 17110115105SRussell King break; 172f5fe12b1SRussell King } 17310115105SRussell King } 17410115105SRussell King 175f02cab2bSRussell King (Oracle) if (state == SPECTRE_MITIGATED) 176f02cab2bSRussell King (Oracle) state = spectre_v2_install_workaround(method); 177f02cab2bSRussell King (Oracle) 178f02cab2bSRussell King (Oracle) spectre_v2_update_state(state, method); 179f5fe12b1SRussell King } 180f5fe12b1SRussell King 18157654884SRussell King (Oracle) #ifdef CONFIG_HARDEN_BRANCH_HISTORY 18257654884SRussell King (Oracle) static int spectre_bhb_method; 18357654884SRussell King (Oracle) 18457654884SRussell King (Oracle) static const char *spectre_bhb_method_name(int method) 18557654884SRussell King (Oracle) { 18657654884SRussell King (Oracle) switch (method) { 18757654884SRussell King (Oracle) case SPECTRE_V2_METHOD_LOOP8: 18857654884SRussell King (Oracle) return "loop"; 18957654884SRussell King (Oracle) 19057654884SRussell King (Oracle) case SPECTRE_V2_METHOD_BPIALL: 19157654884SRussell King (Oracle) return "BPIALL"; 19257654884SRussell King (Oracle) 19357654884SRussell King (Oracle) default: 19457654884SRussell King (Oracle) return "unknown"; 19557654884SRussell King (Oracle) } 19657654884SRussell King (Oracle) } 19757654884SRussell King (Oracle) 19857654884SRussell King (Oracle) static int spectre_bhb_install_workaround(int method) 19957654884SRussell King (Oracle) { 20057654884SRussell King (Oracle) if (spectre_bhb_method != method) { 20157654884SRussell King (Oracle) if (spectre_bhb_method) { 20257654884SRussell King (Oracle) pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n", 20357654884SRussell King (Oracle) smp_processor_id()); 20457654884SRussell King (Oracle) 20557654884SRussell King (Oracle) return SPECTRE_VULNERABLE; 20657654884SRussell King (Oracle) } 20757654884SRussell King (Oracle) 20857654884SRussell King (Oracle) if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE) 20957654884SRussell King (Oracle) return SPECTRE_VULNERABLE; 21057654884SRussell King (Oracle) 21157654884SRussell King (Oracle) spectre_bhb_method = method; 21257654884SRussell King (Oracle) } 21357654884SRussell King (Oracle) 21457654884SRussell King (Oracle) pr_info("CPU%u: Spectre BHB: using %s workaround\n", 21557654884SRussell King (Oracle) smp_processor_id(), spectre_bhb_method_name(method)); 21657654884SRussell King (Oracle) 21757654884SRussell King (Oracle) return SPECTRE_MITIGATED; 21857654884SRussell King (Oracle) } 21957654884SRussell King (Oracle) #else 22057654884SRussell King (Oracle) static int spectre_bhb_install_workaround(int method) 22157654884SRussell King (Oracle) { 22257654884SRussell King (Oracle) return SPECTRE_VULNERABLE; 22357654884SRussell King (Oracle) } 22457654884SRussell King (Oracle) #endif 22557654884SRussell King (Oracle) 22657654884SRussell King (Oracle) static void cpu_v7_spectre_bhb_init(void) 22757654884SRussell King (Oracle) { 22857654884SRussell King (Oracle) unsigned int state, method = 0; 22957654884SRussell King (Oracle) 23057654884SRussell King (Oracle) switch (read_cpuid_part()) { 23157654884SRussell King (Oracle) case ARM_CPU_PART_CORTEX_A15: 23257654884SRussell King (Oracle) case ARM_CPU_PART_BRAHMA_B15: 23357654884SRussell King (Oracle) case ARM_CPU_PART_CORTEX_A57: 23457654884SRussell King (Oracle) case ARM_CPU_PART_CORTEX_A72: 23557654884SRussell King (Oracle) state = SPECTRE_MITIGATED; 23657654884SRussell King (Oracle) method = SPECTRE_V2_METHOD_LOOP8; 23757654884SRussell King (Oracle) break; 23857654884SRussell King (Oracle) 23957654884SRussell King (Oracle) case ARM_CPU_PART_CORTEX_A73: 24057654884SRussell King (Oracle) case ARM_CPU_PART_CORTEX_A75: 24157654884SRussell King (Oracle) state = SPECTRE_MITIGATED; 24257654884SRussell King (Oracle) method = SPECTRE_V2_METHOD_BPIALL; 24357654884SRussell King (Oracle) break; 24457654884SRussell King (Oracle) 24557654884SRussell King (Oracle) default: 24657654884SRussell King (Oracle) state = SPECTRE_UNAFFECTED; 24757654884SRussell King (Oracle) break; 24857654884SRussell King (Oracle) } 24957654884SRussell King (Oracle) 25057654884SRussell King (Oracle) if (state == SPECTRE_MITIGATED) 25157654884SRussell King (Oracle) state = spectre_bhb_install_workaround(method); 25257654884SRussell King (Oracle) 25357654884SRussell King (Oracle) spectre_v2_update_state(state, method); 25457654884SRussell King (Oracle) } 25557654884SRussell King (Oracle) 256f5fe12b1SRussell King static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned, 257e388b802SRussell King u32 mask, const char *msg) 258e388b802SRussell King { 259e388b802SRussell King u32 aux_cr; 260e388b802SRussell King 261e388b802SRussell King asm("mrc p15, 0, %0, c1, c0, 1" : "=r" (aux_cr)); 262e388b802SRussell King 263e388b802SRussell King if ((aux_cr & mask) != mask) { 264e388b802SRussell King if (!*warned) 265e388b802SRussell King pr_err("CPU%u: %s", smp_processor_id(), msg); 266e388b802SRussell King *warned = true; 267f5fe12b1SRussell King return false; 268e388b802SRussell King } 269f5fe12b1SRussell King return true; 270e388b802SRussell King } 271e388b802SRussell King 272e388b802SRussell King static DEFINE_PER_CPU(bool, spectre_warned); 273e388b802SRussell King 274f5fe12b1SRussell King static bool check_spectre_auxcr(bool *warned, u32 bit) 275e388b802SRussell King { 276f5fe12b1SRussell King return IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR) && 277e388b802SRussell King cpu_v7_check_auxcr_set(warned, bit, 278e388b802SRussell King "Spectre v2: firmware did not set auxiliary control register IBE bit, system vulnerable\n"); 279e388b802SRussell King } 280e388b802SRussell King 281e388b802SRussell King void cpu_v7_ca8_ibe(void) 282e388b802SRussell King { 283f5fe12b1SRussell King if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6))) 284f02cab2bSRussell King (Oracle) cpu_v7_spectre_v2_init(); 285e388b802SRussell King } 286e388b802SRussell King 287e388b802SRussell King void cpu_v7_ca15_ibe(void) 288e388b802SRussell King { 289f5fe12b1SRussell King if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0))) 290f02cab2bSRussell King (Oracle) cpu_v7_spectre_v2_init(); 291f5fe12b1SRussell King } 292f5fe12b1SRussell King 293f5fe12b1SRussell King void cpu_v7_bugs_init(void) 294f5fe12b1SRussell King { 295f02cab2bSRussell King (Oracle) cpu_v7_spectre_v2_init(); 29657654884SRussell King (Oracle) cpu_v7_spectre_bhb_init(); 297e388b802SRussell King } 298