1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <asm/kvm_hyp.h> 8 #include <asm/kvm_mmu.h> 9 #include <asm/tlbflush.h> 10 11 struct tlb_inv_context { 12 u64 tcr; 13 }; 14 15 static void __tlb_switch_to_guest(struct kvm_s2_mmu *mmu, 16 struct tlb_inv_context *cxt) 17 { 18 if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { 19 u64 val; 20 21 /* 22 * For CPUs that are affected by ARM 1319367, we need to 23 * avoid a host Stage-1 walk while we have the guest's 24 * VMID set in the VTTBR in order to invalidate TLBs. 25 * We're guaranteed that the S1 MMU is enabled, so we can 26 * simply set the EPD bits to avoid any further TLB fill. 27 */ 28 val = cxt->tcr = read_sysreg_el1(SYS_TCR); 29 val |= TCR_EPD1_MASK | TCR_EPD0_MASK; 30 write_sysreg_el1(val, SYS_TCR); 31 isb(); 32 } 33 34 __load_guest_stage2(mmu); 35 } 36 37 static void __tlb_switch_to_host(struct tlb_inv_context *cxt) 38 { 39 write_sysreg(0, vttbr_el2); 40 41 if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { 42 /* Ensure write of the host VMID */ 43 isb(); 44 /* Restore the host's TCR_EL1 */ 45 write_sysreg_el1(cxt->tcr, SYS_TCR); 46 } 47 } 48 49 void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, 50 phys_addr_t ipa, int level) 51 { 52 struct tlb_inv_context cxt; 53 54 dsb(ishst); 55 56 /* Switch to requested VMID */ 57 mmu = kern_hyp_va(mmu); 58 __tlb_switch_to_guest(mmu, &cxt); 59 60 /* 61 * We could do so much better if we had the VA as well. 62 * Instead, we invalidate Stage-2 for this IPA, and the 63 * whole of Stage-1. Weep... 64 */ 65 ipa >>= 12; 66 __tlbi_level(ipas2e1is, ipa, level); 67 68 /* 69 * We have to ensure completion of the invalidation at Stage-2, 70 * since a table walk on another CPU could refill a TLB with a 71 * complete (S1 + S2) walk based on the old Stage-2 mapping if 72 * the Stage-1 invalidation happened first. 73 */ 74 dsb(ish); 75 __tlbi(vmalle1is); 76 dsb(ish); 77 isb(); 78 79 /* 80 * If the host is running at EL1 and we have a VPIPT I-cache, 81 * then we must perform I-cache maintenance at EL2 in order for 82 * it to have an effect on the guest. Since the guest cannot hit 83 * I-cache lines allocated with a different VMID, we don't need 84 * to worry about junk out of guest reset (we nuke the I-cache on 85 * VMID rollover), but we do need to be careful when remapping 86 * executable pages for the same guest. This can happen when KSM 87 * takes a CoW fault on an executable page, copies the page into 88 * a page that was previously mapped in the guest and then needs 89 * to invalidate the guest view of the I-cache for that page 90 * from EL1. To solve this, we invalidate the entire I-cache when 91 * unmapping a page from a guest if we have a VPIPT I-cache but 92 * the host is running at EL1. As above, we could do better if 93 * we had the VA. 94 * 95 * The moral of this story is: if you have a VPIPT I-cache, then 96 * you should be running with VHE enabled. 97 */ 98 if (icache_is_vpipt()) 99 __flush_icache_all(); 100 101 __tlb_switch_to_host(&cxt); 102 } 103 104 void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu) 105 { 106 struct tlb_inv_context cxt; 107 108 dsb(ishst); 109 110 /* Switch to requested VMID */ 111 mmu = kern_hyp_va(mmu); 112 __tlb_switch_to_guest(mmu, &cxt); 113 114 __tlbi(vmalls12e1is); 115 dsb(ish); 116 isb(); 117 118 __tlb_switch_to_host(&cxt); 119 } 120 121 void __kvm_tlb_flush_local_vmid(struct kvm_s2_mmu *mmu) 122 { 123 struct tlb_inv_context cxt; 124 125 /* Switch to requested VMID */ 126 mmu = kern_hyp_va(mmu); 127 __tlb_switch_to_guest(mmu, &cxt); 128 129 __tlbi(vmalle1); 130 dsb(nsh); 131 isb(); 132 133 __tlb_switch_to_host(&cxt); 134 } 135 136 void __kvm_flush_vm_context(void) 137 { 138 dsb(ishst); 139 __tlbi(alle1is); 140 141 /* 142 * VIPT and PIPT caches are not affected by VMID, so no maintenance 143 * is necessary across a VMID rollover. 144 * 145 * VPIPT caches constrain lookup and maintenance to the active VMID, 146 * so we need to invalidate lines with a stale VMID to avoid an ABA 147 * race after multiple rollovers. 148 * 149 */ 150 if (icache_is_vpipt()) 151 asm volatile("ic ialluis"); 152 153 dsb(ish); 154 } 155