1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2f4672752SMarc Zyngier /* 3f4672752SMarc Zyngier * Copyright (C) 2012,2013 - ARM Ltd 4f4672752SMarc Zyngier * Author: Marc Zyngier <marc.zyngier@arm.com> 5f4672752SMarc Zyngier * 6f4672752SMarc Zyngier * Derived from arch/arm/kvm/reset.c 7f4672752SMarc Zyngier * Copyright (C) 2012 - Virtual Open Systems and Columbia University 8f4672752SMarc Zyngier * Author: Christoffer Dall <c.dall@virtualopensystems.com> 9f4672752SMarc Zyngier */ 10f4672752SMarc Zyngier 11f4672752SMarc Zyngier #include <linux/errno.h> 129a3cdf26SDave Martin #include <linux/kernel.h> 13f4672752SMarc Zyngier #include <linux/kvm_host.h> 14f4672752SMarc Zyngier #include <linux/kvm.h> 15834bf887SAlex Bennée #include <linux/hw_breakpoint.h> 169033bba4SDave Martin #include <linux/slab.h> 179a3cdf26SDave Martin #include <linux/string.h> 189033bba4SDave Martin #include <linux/types.h> 19f4672752SMarc Zyngier 20003300deSMarc Zyngier #include <kvm/arm_arch_timer.h> 21003300deSMarc Zyngier 227665f3a8SSuzuki K Poulose #include <asm/cpufeature.h> 23f4672752SMarc Zyngier #include <asm/cputype.h> 249033bba4SDave Martin #include <asm/fpsimd.h> 25f4672752SMarc Zyngier #include <asm/ptrace.h> 26f4672752SMarc Zyngier #include <asm/kvm_arm.h> 2767f69197SAKASHI Takahiro #include <asm/kvm_asm.h> 28f4672752SMarc Zyngier #include <asm/kvm_coproc.h> 29358b28f0SMarc Zyngier #include <asm/kvm_emulate.h> 3067f69197SAKASHI Takahiro #include <asm/kvm_mmu.h> 319a3cdf26SDave Martin #include <asm/virt.h> 32f4672752SMarc Zyngier 330f62f0e9SSuzuki K Poulose /* Maximum phys_shift supported for any VM on this host */ 340f62f0e9SSuzuki K Poulose static u32 kvm_ipa_limit; 350f62f0e9SSuzuki K Poulose 36f4672752SMarc Zyngier /* 37f4672752SMarc Zyngier * ARMv8 Reset Values 38f4672752SMarc Zyngier */ 39349c330cSMarc Zyngier #define VCPU_RESET_PSTATE_EL1 (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | \ 40349c330cSMarc Zyngier PSR_F_BIT | PSR_D_BIT) 41f4672752SMarc Zyngier 42349c330cSMarc Zyngier #define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \ 43349c330cSMarc Zyngier PSR_AA32_I_BIT | PSR_AA32_F_BIT) 440d854a60SMarc Zyngier 45aff7cce0SMarc Zyngier static bool system_has_full_ptr_auth(void) 46aff7cce0SMarc Zyngier { 47aff7cce0SMarc Zyngier return system_supports_address_auth() && system_supports_generic_auth(); 48aff7cce0SMarc Zyngier } 49aff7cce0SMarc Zyngier 50834bf887SAlex Bennée /** 51375bdd3bSDongjiu Geng * kvm_arch_vm_ioctl_check_extension 52834bf887SAlex Bennée * 53834bf887SAlex Bennée * We currently assume that the number of HW registers is uniform 54834bf887SAlex Bennée * across all CPUs (see cpuinfo_sanity_check). 55834bf887SAlex Bennée */ 56375bdd3bSDongjiu Geng int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext) 57f4672752SMarc Zyngier { 58f4672752SMarc Zyngier int r; 59f4672752SMarc Zyngier 60f4672752SMarc Zyngier switch (ext) { 610d854a60SMarc Zyngier case KVM_CAP_ARM_EL1_32BIT: 62540f76d1SWill Deacon r = cpus_have_const_cap(ARM64_HAS_32BIT_EL1); 630d854a60SMarc Zyngier break; 64834bf887SAlex Bennée case KVM_CAP_GUEST_DEBUG_HW_BPS: 65834bf887SAlex Bennée r = get_num_brps(); 66834bf887SAlex Bennée break; 67834bf887SAlex Bennée case KVM_CAP_GUEST_DEBUG_HW_WPS: 68834bf887SAlex Bennée r = get_num_wrps(); 69834bf887SAlex Bennée break; 70808e7381SShannon Zhao case KVM_CAP_ARM_PMU_V3: 71808e7381SShannon Zhao r = kvm_arm_support_pmu_v3(); 72808e7381SShannon Zhao break; 73be26b3a7SDongjiu Geng case KVM_CAP_ARM_INJECT_SERROR_ESR: 74be26b3a7SDongjiu Geng r = cpus_have_const_cap(ARM64_HAS_RAS_EXTN); 75be26b3a7SDongjiu Geng break; 76834bf887SAlex Bennée case KVM_CAP_SET_GUEST_DEBUG: 77f577f6c2SShannon Zhao case KVM_CAP_VCPU_ATTRIBUTES: 78834bf887SAlex Bennée r = 1; 79834bf887SAlex Bennée break; 80233a7cb2SSuzuki K Poulose case KVM_CAP_ARM_VM_IPA_SIZE: 81233a7cb2SSuzuki K Poulose r = kvm_ipa_limit; 82233a7cb2SSuzuki K Poulose break; 83555f3d03SDave Martin case KVM_CAP_ARM_SVE: 84555f3d03SDave Martin r = system_supports_sve(); 85555f3d03SDave Martin break; 86a243c16dSAmit Daniel Kachhap case KVM_CAP_ARM_PTRAUTH_ADDRESS: 87a243c16dSAmit Daniel Kachhap case KVM_CAP_ARM_PTRAUTH_GENERIC: 88aff7cce0SMarc Zyngier r = system_has_full_ptr_auth(); 89a243c16dSAmit Daniel Kachhap break; 90f4672752SMarc Zyngier default: 91f4672752SMarc Zyngier r = 0; 92f4672752SMarc Zyngier } 93f4672752SMarc Zyngier 94f4672752SMarc Zyngier return r; 95f4672752SMarc Zyngier } 96f4672752SMarc Zyngier 979033bba4SDave Martin unsigned int kvm_sve_max_vl; 989033bba4SDave Martin 99a3be836dSDave Martin int kvm_arm_init_sve(void) 1009033bba4SDave Martin { 1019033bba4SDave Martin if (system_supports_sve()) { 1029033bba4SDave Martin kvm_sve_max_vl = sve_max_virtualisable_vl; 1039033bba4SDave Martin 1049033bba4SDave Martin /* 1059033bba4SDave Martin * The get_sve_reg()/set_sve_reg() ioctl interface will need 1069033bba4SDave Martin * to be extended with multiple register slice support in 1079033bba4SDave Martin * order to support vector lengths greater than 1089033bba4SDave Martin * SVE_VL_ARCH_MAX: 1099033bba4SDave Martin */ 1109033bba4SDave Martin if (WARN_ON(kvm_sve_max_vl > SVE_VL_ARCH_MAX)) 1119033bba4SDave Martin kvm_sve_max_vl = SVE_VL_ARCH_MAX; 1129033bba4SDave Martin 1139033bba4SDave Martin /* 1149033bba4SDave Martin * Don't even try to make use of vector lengths that 1159033bba4SDave Martin * aren't available on all CPUs, for now: 1169033bba4SDave Martin */ 1179033bba4SDave Martin if (kvm_sve_max_vl < sve_max_vl) 1189033bba4SDave Martin pr_warn("KVM: SVE vector length for guests limited to %u bytes\n", 1199033bba4SDave Martin kvm_sve_max_vl); 1209033bba4SDave Martin } 1219033bba4SDave Martin 1229033bba4SDave Martin return 0; 1239033bba4SDave Martin } 1249033bba4SDave Martin 1259a3cdf26SDave Martin static int kvm_vcpu_enable_sve(struct kvm_vcpu *vcpu) 1269a3cdf26SDave Martin { 1279a3cdf26SDave Martin if (!system_supports_sve()) 1289a3cdf26SDave Martin return -EINVAL; 1299a3cdf26SDave Martin 1309a3cdf26SDave Martin /* Verify that KVM startup enforced this when SVE was detected: */ 1319a3cdf26SDave Martin if (WARN_ON(!has_vhe())) 1329a3cdf26SDave Martin return -EINVAL; 1339a3cdf26SDave Martin 1349a3cdf26SDave Martin vcpu->arch.sve_max_vl = kvm_sve_max_vl; 1359a3cdf26SDave Martin 1369a3cdf26SDave Martin /* 1379a3cdf26SDave Martin * Userspace can still customize the vector lengths by writing 1389a3cdf26SDave Martin * KVM_REG_ARM64_SVE_VLS. Allocation is deferred until 1399a3cdf26SDave Martin * kvm_arm_vcpu_finalize(), which freezes the configuration. 1409a3cdf26SDave Martin */ 1419a3cdf26SDave Martin vcpu->arch.flags |= KVM_ARM64_GUEST_HAS_SVE; 1429a3cdf26SDave Martin 1439a3cdf26SDave Martin return 0; 1449a3cdf26SDave Martin } 1459a3cdf26SDave Martin 1469033bba4SDave Martin /* 1479033bba4SDave Martin * Finalize vcpu's maximum SVE vector length, allocating 1489033bba4SDave Martin * vcpu->arch.sve_state as necessary. 1499033bba4SDave Martin */ 1509033bba4SDave Martin static int kvm_vcpu_finalize_sve(struct kvm_vcpu *vcpu) 1519033bba4SDave Martin { 1529033bba4SDave Martin void *buf; 1539033bba4SDave Martin unsigned int vl; 1549033bba4SDave Martin 1559033bba4SDave Martin vl = vcpu->arch.sve_max_vl; 1569033bba4SDave Martin 1579033bba4SDave Martin /* 158656012c7SFuad Tabba * Responsibility for these properties is shared between 1599033bba4SDave Martin * kvm_arm_init_arch_resources(), kvm_vcpu_enable_sve() and 1609033bba4SDave Martin * set_sve_vls(). Double-check here just to be sure: 1619033bba4SDave Martin */ 1629033bba4SDave Martin if (WARN_ON(!sve_vl_valid(vl) || vl > sve_max_virtualisable_vl || 1639033bba4SDave Martin vl > SVE_VL_ARCH_MAX)) 1649033bba4SDave Martin return -EIO; 1659033bba4SDave Martin 1669033bba4SDave Martin buf = kzalloc(SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl)), GFP_KERNEL); 1679033bba4SDave Martin if (!buf) 1689033bba4SDave Martin return -ENOMEM; 1699033bba4SDave Martin 1709033bba4SDave Martin vcpu->arch.sve_state = buf; 1719033bba4SDave Martin vcpu->arch.flags |= KVM_ARM64_VCPU_SVE_FINALIZED; 1729033bba4SDave Martin return 0; 1739033bba4SDave Martin } 1749033bba4SDave Martin 17592e68b2bSDave Martin int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature) 1769033bba4SDave Martin { 17792e68b2bSDave Martin switch (feature) { 1789033bba4SDave Martin case KVM_ARM_VCPU_SVE: 1799033bba4SDave Martin if (!vcpu_has_sve(vcpu)) 1809033bba4SDave Martin return -EINVAL; 1819033bba4SDave Martin 1829033bba4SDave Martin if (kvm_arm_vcpu_sve_finalized(vcpu)) 1839033bba4SDave Martin return -EPERM; 1849033bba4SDave Martin 1859033bba4SDave Martin return kvm_vcpu_finalize_sve(vcpu); 1869033bba4SDave Martin } 1879033bba4SDave Martin 1889033bba4SDave Martin return -EINVAL; 1899033bba4SDave Martin } 1909033bba4SDave Martin 1919033bba4SDave Martin bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu) 1929033bba4SDave Martin { 1939033bba4SDave Martin if (vcpu_has_sve(vcpu) && !kvm_arm_vcpu_sve_finalized(vcpu)) 1949033bba4SDave Martin return false; 1959033bba4SDave Martin 1969033bba4SDave Martin return true; 1979033bba4SDave Martin } 1989033bba4SDave Martin 19919bcc89eSSean Christopherson void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu) 20019bcc89eSSean Christopherson { 2019033bba4SDave Martin kfree(vcpu->arch.sve_state); 2029033bba4SDave Martin } 2039033bba4SDave Martin 2049a3cdf26SDave Martin static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu) 2059a3cdf26SDave Martin { 2069a3cdf26SDave Martin if (vcpu_has_sve(vcpu)) 2079a3cdf26SDave Martin memset(vcpu->arch.sve_state, 0, vcpu_sve_state_size(vcpu)); 2089a3cdf26SDave Martin } 2099a3cdf26SDave Martin 210a22fa321SAmit Daniel Kachhap static int kvm_vcpu_enable_ptrauth(struct kvm_vcpu *vcpu) 211a22fa321SAmit Daniel Kachhap { 212a22fa321SAmit Daniel Kachhap /* 213a22fa321SAmit Daniel Kachhap * For now make sure that both address/generic pointer authentication 214aff7cce0SMarc Zyngier * features are requested by the userspace together and the system 215aff7cce0SMarc Zyngier * supports these capabilities. 216a22fa321SAmit Daniel Kachhap */ 217a22fa321SAmit Daniel Kachhap if (!test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, vcpu->arch.features) || 218aff7cce0SMarc Zyngier !test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, vcpu->arch.features) || 219aff7cce0SMarc Zyngier !system_has_full_ptr_auth()) 220a22fa321SAmit Daniel Kachhap return -EINVAL; 221a22fa321SAmit Daniel Kachhap 222a22fa321SAmit Daniel Kachhap vcpu->arch.flags |= KVM_ARM64_GUEST_HAS_PTRAUTH; 223a22fa321SAmit Daniel Kachhap return 0; 224a22fa321SAmit Daniel Kachhap } 225a22fa321SAmit Daniel Kachhap 226f4672752SMarc Zyngier /** 227f4672752SMarc Zyngier * kvm_reset_vcpu - sets core registers and sys_regs to reset value 228f4672752SMarc Zyngier * @vcpu: The VCPU pointer 229f4672752SMarc Zyngier * 230f4672752SMarc Zyngier * This function finds the right table above and sets the registers on 231edce2292SAndrea Gelmini * the virtual CPU struct to their architecturally defined reset 2329a3cdf26SDave Martin * values, except for registers whose reset is deferred until 2339a3cdf26SDave Martin * kvm_arm_vcpu_finalize(). 234e761a927SChristoffer Dall * 235e761a927SChristoffer Dall * Note: This function can be called from two paths: The KVM_ARM_VCPU_INIT 236e761a927SChristoffer Dall * ioctl or as part of handling a request issued by another VCPU in the PSCI 237e761a927SChristoffer Dall * handling code. In the first case, the VCPU will not be loaded, and in the 238e761a927SChristoffer Dall * second case the VCPU will be loaded. Because this function operates purely 239656012c7SFuad Tabba * on the memory-backed values of system registers, we want to do a full put if 240e761a927SChristoffer Dall * we were loaded (handling a request) and load the values back at the end of 241e761a927SChristoffer Dall * the function. Otherwise we leave the state alone. In both cases, we 242e761a927SChristoffer Dall * disable preemption around the vcpu reset as we would otherwise race with 243e761a927SChristoffer Dall * preempt notifiers which also call put/load. 244f4672752SMarc Zyngier */ 245f4672752SMarc Zyngier int kvm_reset_vcpu(struct kvm_vcpu *vcpu) 246f4672752SMarc Zyngier { 24766b7e05dSSteven Price int ret; 248e761a927SChristoffer Dall bool loaded; 249349c330cSMarc Zyngier u32 pstate; 250e761a927SChristoffer Dall 251ebff0b0eSMarc Zyngier /* Reset PMU outside of the non-preemptible section */ 252ebff0b0eSMarc Zyngier kvm_pmu_vcpu_reset(vcpu); 253ebff0b0eSMarc Zyngier 254e761a927SChristoffer Dall preempt_disable(); 255e761a927SChristoffer Dall loaded = (vcpu->cpu != -1); 256e761a927SChristoffer Dall if (loaded) 257e761a927SChristoffer Dall kvm_arch_vcpu_put(vcpu); 258f4672752SMarc Zyngier 2599a3cdf26SDave Martin if (!kvm_arm_vcpu_sve_finalized(vcpu)) { 2609a3cdf26SDave Martin if (test_bit(KVM_ARM_VCPU_SVE, vcpu->arch.features)) { 2619a3cdf26SDave Martin ret = kvm_vcpu_enable_sve(vcpu); 2629a3cdf26SDave Martin if (ret) 2639a3cdf26SDave Martin goto out; 2649a3cdf26SDave Martin } 2659a3cdf26SDave Martin } else { 2669a3cdf26SDave Martin kvm_vcpu_reset_sve(vcpu); 2679a3cdf26SDave Martin } 2689a3cdf26SDave Martin 269a22fa321SAmit Daniel Kachhap if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, vcpu->arch.features) || 270a22fa321SAmit Daniel Kachhap test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, vcpu->arch.features)) { 27166b7e05dSSteven Price if (kvm_vcpu_enable_ptrauth(vcpu)) { 27266b7e05dSSteven Price ret = -EINVAL; 273a22fa321SAmit Daniel Kachhap goto out; 274a22fa321SAmit Daniel Kachhap } 27566b7e05dSSteven Price } 276a22fa321SAmit Daniel Kachhap 277f4672752SMarc Zyngier switch (vcpu->arch.target) { 278f4672752SMarc Zyngier default: 2790d854a60SMarc Zyngier if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) { 28066b7e05dSSteven Price if (!cpus_have_const_cap(ARM64_HAS_32BIT_EL1)) { 28166b7e05dSSteven Price ret = -EINVAL; 282e761a927SChristoffer Dall goto out; 28366b7e05dSSteven Price } 284349c330cSMarc Zyngier pstate = VCPU_RESET_PSTATE_SVC; 2850d854a60SMarc Zyngier } else { 286349c330cSMarc Zyngier pstate = VCPU_RESET_PSTATE_EL1; 2870d854a60SMarc Zyngier } 2880d854a60SMarc Zyngier 289f4672752SMarc Zyngier break; 290f4672752SMarc Zyngier } 291f4672752SMarc Zyngier 292f4672752SMarc Zyngier /* Reset core registers */ 293349c330cSMarc Zyngier memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); 294e47c2055SMarc Zyngier vcpu_gp_regs(vcpu)->pstate = pstate; 295f4672752SMarc Zyngier 296f4672752SMarc Zyngier /* Reset system registers */ 297f4672752SMarc Zyngier kvm_reset_sys_regs(vcpu); 298f4672752SMarc Zyngier 299358b28f0SMarc Zyngier /* 300358b28f0SMarc Zyngier * Additional reset state handling that PSCI may have imposed on us. 301358b28f0SMarc Zyngier * Must be done after all the sys_reg reset. 302358b28f0SMarc Zyngier */ 303358b28f0SMarc Zyngier if (vcpu->arch.reset_state.reset) { 304358b28f0SMarc Zyngier unsigned long target_pc = vcpu->arch.reset_state.pc; 305358b28f0SMarc Zyngier 306358b28f0SMarc Zyngier /* Gracefully handle Thumb2 entry point */ 307358b28f0SMarc Zyngier if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) { 308358b28f0SMarc Zyngier target_pc &= ~1UL; 309358b28f0SMarc Zyngier vcpu_set_thumb(vcpu); 310358b28f0SMarc Zyngier } 311358b28f0SMarc Zyngier 312358b28f0SMarc Zyngier /* Propagate caller endianness */ 313358b28f0SMarc Zyngier if (vcpu->arch.reset_state.be) 314358b28f0SMarc Zyngier kvm_vcpu_set_be(vcpu); 315358b28f0SMarc Zyngier 316358b28f0SMarc Zyngier *vcpu_pc(vcpu) = target_pc; 317358b28f0SMarc Zyngier vcpu_set_reg(vcpu, 0, vcpu->arch.reset_state.r0); 318358b28f0SMarc Zyngier 319358b28f0SMarc Zyngier vcpu->arch.reset_state.reset = false; 320358b28f0SMarc Zyngier } 321358b28f0SMarc Zyngier 3225d81f7dcSMarc Zyngier /* Default workaround setup is enabled (if supported) */ 3235d81f7dcSMarc Zyngier if (kvm_arm_have_ssbd() == KVM_SSBD_KERNEL) 3245d81f7dcSMarc Zyngier vcpu->arch.workaround_flags |= VCPU_WORKAROUND_2_FLAG; 3255d81f7dcSMarc Zyngier 326003300deSMarc Zyngier /* Reset timer */ 327e761a927SChristoffer Dall ret = kvm_timer_vcpu_reset(vcpu); 328e761a927SChristoffer Dall out: 329e761a927SChristoffer Dall if (loaded) 330e761a927SChristoffer Dall kvm_arch_vcpu_load(vcpu, smp_processor_id()); 331e761a927SChristoffer Dall preempt_enable(); 332e761a927SChristoffer Dall return ret; 333f4672752SMarc Zyngier } 3345b6c6742SSuzuki K Poulose 335c73433fcSAnshuman Khandual u32 get_kvm_ipa_limit(void) 336c73433fcSAnshuman Khandual { 337c73433fcSAnshuman Khandual return kvm_ipa_limit; 338c73433fcSAnshuman Khandual } 339c73433fcSAnshuman Khandual 340b130a8f7SMarc Zyngier int kvm_set_ipa_limit(void) 3410f62f0e9SSuzuki K Poulose { 342c9b69a0cSWill Deacon unsigned int parange, tgran_2; 343f73531f0SAnshuman Khandual u64 mmfr0; 3440f62f0e9SSuzuki K Poulose 345f73531f0SAnshuman Khandual mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 346f73531f0SAnshuman Khandual parange = cpuid_feature_extract_unsigned_field(mmfr0, 347f73531f0SAnshuman Khandual ID_AA64MMFR0_PARANGE_SHIFT); 348b130a8f7SMarc Zyngier 349b130a8f7SMarc Zyngier /* 350b130a8f7SMarc Zyngier * Check with ARMv8.5-GTG that our PAGE_SIZE is supported at 351b130a8f7SMarc Zyngier * Stage-2. If not, things will stop very quickly. 352b130a8f7SMarc Zyngier */ 353b130a8f7SMarc Zyngier switch (PAGE_SIZE) { 354b130a8f7SMarc Zyngier default: 355b130a8f7SMarc Zyngier case SZ_4K: 356b130a8f7SMarc Zyngier tgran_2 = ID_AA64MMFR0_TGRAN4_2_SHIFT; 357b130a8f7SMarc Zyngier break; 358b130a8f7SMarc Zyngier case SZ_16K: 359b130a8f7SMarc Zyngier tgran_2 = ID_AA64MMFR0_TGRAN16_2_SHIFT; 360b130a8f7SMarc Zyngier break; 361b130a8f7SMarc Zyngier case SZ_64K: 362b130a8f7SMarc Zyngier tgran_2 = ID_AA64MMFR0_TGRAN64_2_SHIFT; 363b130a8f7SMarc Zyngier break; 364b130a8f7SMarc Zyngier } 365b130a8f7SMarc Zyngier 366b130a8f7SMarc Zyngier switch (cpuid_feature_extract_unsigned_field(mmfr0, tgran_2)) { 367b130a8f7SMarc Zyngier default: 368b130a8f7SMarc Zyngier case 1: 369b130a8f7SMarc Zyngier kvm_err("PAGE_SIZE not supported at Stage-2, giving up\n"); 370b130a8f7SMarc Zyngier return -EINVAL; 371b130a8f7SMarc Zyngier case 0: 372b130a8f7SMarc Zyngier kvm_debug("PAGE_SIZE supported at Stage-2 (default)\n"); 373b130a8f7SMarc Zyngier break; 374b130a8f7SMarc Zyngier case 2: 375b130a8f7SMarc Zyngier kvm_debug("PAGE_SIZE supported at Stage-2 (advertised)\n"); 376b130a8f7SMarc Zyngier break; 377b130a8f7SMarc Zyngier } 378b130a8f7SMarc Zyngier 379c9b69a0cSWill Deacon kvm_ipa_limit = id_aa64mmfr0_parange_to_phys_shift(parange); 380c9b69a0cSWill Deacon WARN(kvm_ipa_limit < KVM_PHYS_SHIFT, 381c9b69a0cSWill Deacon "KVM IPA Size Limit (%d bits) is smaller than default size\n", 382c9b69a0cSWill Deacon kvm_ipa_limit); 3830f62f0e9SSuzuki K Poulose kvm_info("IPA Size Limit: %d bits\n", kvm_ipa_limit); 384b130a8f7SMarc Zyngier 385b130a8f7SMarc Zyngier return 0; 3860f62f0e9SSuzuki K Poulose } 3870f62f0e9SSuzuki K Poulose 3887665f3a8SSuzuki K Poulose /* 3897665f3a8SSuzuki K Poulose * Configure the VTCR_EL2 for this VM. The VTCR value is common 3907665f3a8SSuzuki K Poulose * across all the physical CPUs on the system. We use system wide 3917665f3a8SSuzuki K Poulose * sanitised values to fill in different fields, except for Hardware 3927665f3a8SSuzuki K Poulose * Management of Access Flags. HA Flag is set unconditionally on 3937665f3a8SSuzuki K Poulose * all CPUs, as it is safe to run with or without the feature and 3947665f3a8SSuzuki K Poulose * the bit is RES0 on CPUs that don't support it. 3957665f3a8SSuzuki K Poulose */ 396bca607ebSMarc Zyngier int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type) 3975b6c6742SSuzuki K Poulose { 398f73531f0SAnshuman Khandual u64 vtcr = VTCR_EL2_FLAGS, mmfr0; 3997665f3a8SSuzuki K Poulose u32 parange, phys_shift; 40058b3efc8SSuzuki K Poulose u8 lvls; 4017665f3a8SSuzuki K Poulose 402233a7cb2SSuzuki K Poulose if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) 4035b6c6742SSuzuki K Poulose return -EINVAL; 4047665f3a8SSuzuki K Poulose 405233a7cb2SSuzuki K Poulose phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type); 406233a7cb2SSuzuki K Poulose if (phys_shift) { 407233a7cb2SSuzuki K Poulose if (phys_shift > kvm_ipa_limit || 408233a7cb2SSuzuki K Poulose phys_shift < 32) 409233a7cb2SSuzuki K Poulose return -EINVAL; 410233a7cb2SSuzuki K Poulose } else { 411233a7cb2SSuzuki K Poulose phys_shift = KVM_PHYS_SHIFT; 412233a7cb2SSuzuki K Poulose } 413233a7cb2SSuzuki K Poulose 414f73531f0SAnshuman Khandual mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 415f73531f0SAnshuman Khandual parange = cpuid_feature_extract_unsigned_field(mmfr0, 416f73531f0SAnshuman Khandual ID_AA64MMFR0_PARANGE_SHIFT); 4177665f3a8SSuzuki K Poulose if (parange > ID_AA64MMFR0_PARANGE_MAX) 4187665f3a8SSuzuki K Poulose parange = ID_AA64MMFR0_PARANGE_MAX; 4197665f3a8SSuzuki K Poulose vtcr |= parange << VTCR_EL2_PS_SHIFT; 4207665f3a8SSuzuki K Poulose 4217665f3a8SSuzuki K Poulose vtcr |= VTCR_EL2_T0SZ(phys_shift); 42258b3efc8SSuzuki K Poulose /* 42358b3efc8SSuzuki K Poulose * Use a minimum 2 level page table to prevent splitting 42458b3efc8SSuzuki K Poulose * host PMD huge pages at stage2. 42558b3efc8SSuzuki K Poulose */ 42658b3efc8SSuzuki K Poulose lvls = stage2_pgtable_levels(phys_shift); 42758b3efc8SSuzuki K Poulose if (lvls < 2) 42858b3efc8SSuzuki K Poulose lvls = 2; 42958b3efc8SSuzuki K Poulose vtcr |= VTCR_EL2_LVLS_TO_SL0(lvls); 4307665f3a8SSuzuki K Poulose 4317665f3a8SSuzuki K Poulose /* 4327665f3a8SSuzuki K Poulose * Enable the Hardware Access Flag management, unconditionally 4337665f3a8SSuzuki K Poulose * on all CPUs. The features is RES0 on CPUs without the support 4347665f3a8SSuzuki K Poulose * and must be ignored by the CPUs. 4357665f3a8SSuzuki K Poulose */ 4367665f3a8SSuzuki K Poulose vtcr |= VTCR_EL2_HA; 4377665f3a8SSuzuki K Poulose 4387665f3a8SSuzuki K Poulose /* Set the vmid bits */ 4397665f3a8SSuzuki K Poulose vtcr |= (kvm_get_vmid_bits() == 16) ? 4407665f3a8SSuzuki K Poulose VTCR_EL2_VS_16BIT : 4417665f3a8SSuzuki K Poulose VTCR_EL2_VS_8BIT; 4427665f3a8SSuzuki K Poulose kvm->arch.vtcr = vtcr; 4435b6c6742SSuzuki K Poulose return 0; 4445b6c6742SSuzuki K Poulose } 445