1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Google LLC 4 * Author: Quentin Perret <qperret@google.com> 5 */ 6 7 #include <linux/kvm_host.h> 8 #include <asm/kvm_emulate.h> 9 #include <asm/kvm_hyp.h> 10 #include <asm/kvm_mmu.h> 11 #include <asm/kvm_pgtable.h> 12 #include <asm/stage2_pgtable.h> 13 14 #include <hyp/switch.h> 15 16 #include <nvhe/gfp.h> 17 #include <nvhe/memory.h> 18 #include <nvhe/mem_protect.h> 19 #include <nvhe/mm.h> 20 21 #define KVM_HOST_S2_FLAGS (KVM_PGTABLE_S2_NOFWB | KVM_PGTABLE_S2_IDMAP) 22 23 extern unsigned long hyp_nr_cpus; 24 struct host_kvm host_kvm; 25 26 static struct hyp_pool host_s2_pool; 27 28 /* 29 * Copies of the host's CPU features registers holding sanitized values. 30 */ 31 u64 id_aa64mmfr0_el1_sys_val; 32 u64 id_aa64mmfr1_el1_sys_val; 33 34 static const u8 pkvm_hyp_id = 1; 35 36 static void *host_s2_zalloc_pages_exact(size_t size) 37 { 38 return hyp_alloc_pages(&host_s2_pool, get_order(size)); 39 } 40 41 static void *host_s2_zalloc_page(void *pool) 42 { 43 return hyp_alloc_pages(pool, 0); 44 } 45 46 static void host_s2_get_page(void *addr) 47 { 48 hyp_get_page(&host_s2_pool, addr); 49 } 50 51 static void host_s2_put_page(void *addr) 52 { 53 hyp_put_page(&host_s2_pool, addr); 54 } 55 56 static int prepare_s2_pool(void *pgt_pool_base) 57 { 58 unsigned long nr_pages, pfn; 59 int ret; 60 61 pfn = hyp_virt_to_pfn(pgt_pool_base); 62 nr_pages = host_s2_pgtable_pages(); 63 ret = hyp_pool_init(&host_s2_pool, pfn, nr_pages, 0); 64 if (ret) 65 return ret; 66 67 host_kvm.mm_ops = (struct kvm_pgtable_mm_ops) { 68 .zalloc_pages_exact = host_s2_zalloc_pages_exact, 69 .zalloc_page = host_s2_zalloc_page, 70 .phys_to_virt = hyp_phys_to_virt, 71 .virt_to_phys = hyp_virt_to_phys, 72 .page_count = hyp_page_count, 73 .get_page = host_s2_get_page, 74 .put_page = host_s2_put_page, 75 }; 76 77 return 0; 78 } 79 80 static void prepare_host_vtcr(void) 81 { 82 u32 parange, phys_shift; 83 84 /* The host stage 2 is id-mapped, so use parange for T0SZ */ 85 parange = kvm_get_parange(id_aa64mmfr0_el1_sys_val); 86 phys_shift = id_aa64mmfr0_parange_to_phys_shift(parange); 87 88 host_kvm.arch.vtcr = kvm_get_vtcr(id_aa64mmfr0_el1_sys_val, 89 id_aa64mmfr1_el1_sys_val, phys_shift); 90 } 91 92 int kvm_host_prepare_stage2(void *pgt_pool_base) 93 { 94 struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu; 95 int ret; 96 97 prepare_host_vtcr(); 98 hyp_spin_lock_init(&host_kvm.lock); 99 100 ret = prepare_s2_pool(pgt_pool_base); 101 if (ret) 102 return ret; 103 104 ret = kvm_pgtable_stage2_init_flags(&host_kvm.pgt, &host_kvm.arch, 105 &host_kvm.mm_ops, KVM_HOST_S2_FLAGS); 106 if (ret) 107 return ret; 108 109 mmu->pgd_phys = __hyp_pa(host_kvm.pgt.pgd); 110 mmu->arch = &host_kvm.arch; 111 mmu->pgt = &host_kvm.pgt; 112 mmu->vmid.vmid_gen = 0; 113 mmu->vmid.vmid = 0; 114 115 return 0; 116 } 117 118 int __pkvm_prot_finalize(void) 119 { 120 struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu; 121 struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params); 122 123 params->vttbr = kvm_get_vttbr(mmu); 124 params->vtcr = host_kvm.arch.vtcr; 125 params->hcr_el2 |= HCR_VM; 126 kvm_flush_dcache_to_poc(params, sizeof(*params)); 127 128 write_sysreg(params->hcr_el2, hcr_el2); 129 __load_stage2(&host_kvm.arch.mmu, host_kvm.arch.vtcr); 130 131 /* 132 * Make sure to have an ISB before the TLB maintenance below but only 133 * when __load_stage2() doesn't include one already. 134 */ 135 asm(ALTERNATIVE("isb", "nop", ARM64_WORKAROUND_SPECULATIVE_AT)); 136 137 /* Invalidate stale HCR bits that may be cached in TLBs */ 138 __tlbi(vmalls12e1); 139 dsb(nsh); 140 isb(); 141 142 return 0; 143 } 144 145 static int host_stage2_unmap_dev_all(void) 146 { 147 struct kvm_pgtable *pgt = &host_kvm.pgt; 148 struct memblock_region *reg; 149 u64 addr = 0; 150 int i, ret; 151 152 /* Unmap all non-memory regions to recycle the pages */ 153 for (i = 0; i < hyp_memblock_nr; i++, addr = reg->base + reg->size) { 154 reg = &hyp_memory[i]; 155 ret = kvm_pgtable_stage2_unmap(pgt, addr, reg->base - addr); 156 if (ret) 157 return ret; 158 } 159 return kvm_pgtable_stage2_unmap(pgt, addr, BIT(pgt->ia_bits) - addr); 160 } 161 162 static bool find_mem_range(phys_addr_t addr, struct kvm_mem_range *range) 163 { 164 int cur, left = 0, right = hyp_memblock_nr; 165 struct memblock_region *reg; 166 phys_addr_t end; 167 168 range->start = 0; 169 range->end = ULONG_MAX; 170 171 /* The list of memblock regions is sorted, binary search it */ 172 while (left < right) { 173 cur = (left + right) >> 1; 174 reg = &hyp_memory[cur]; 175 end = reg->base + reg->size; 176 if (addr < reg->base) { 177 right = cur; 178 range->end = reg->base; 179 } else if (addr >= end) { 180 left = cur + 1; 181 range->start = end; 182 } else { 183 range->start = reg->base; 184 range->end = end; 185 return true; 186 } 187 } 188 189 return false; 190 } 191 192 static bool range_is_memory(u64 start, u64 end) 193 { 194 struct kvm_mem_range r1, r2; 195 196 if (!find_mem_range(start, &r1) || !find_mem_range(end, &r2)) 197 return false; 198 if (r1.start != r2.start) 199 return false; 200 201 return true; 202 } 203 204 static inline int __host_stage2_idmap(u64 start, u64 end, 205 enum kvm_pgtable_prot prot) 206 { 207 return kvm_pgtable_stage2_map(&host_kvm.pgt, start, end - start, start, 208 prot, &host_s2_pool); 209 } 210 211 static int host_stage2_idmap(u64 addr) 212 { 213 enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W; 214 struct kvm_mem_range range; 215 bool is_memory = find_mem_range(addr, &range); 216 int ret; 217 218 if (is_memory) 219 prot |= KVM_PGTABLE_PROT_X; 220 221 hyp_spin_lock(&host_kvm.lock); 222 ret = kvm_pgtable_stage2_find_range(&host_kvm.pgt, addr, prot, &range); 223 if (ret) 224 goto unlock; 225 226 ret = __host_stage2_idmap(range.start, range.end, prot); 227 if (ret != -ENOMEM) 228 goto unlock; 229 230 /* 231 * The pool has been provided with enough pages to cover all of memory 232 * with page granularity, but it is difficult to know how much of the 233 * MMIO range we will need to cover upfront, so we may need to 'recycle' 234 * the pages if we run out. 235 */ 236 ret = host_stage2_unmap_dev_all(); 237 if (ret) 238 goto unlock; 239 240 ret = __host_stage2_idmap(range.start, range.end, prot); 241 242 unlock: 243 hyp_spin_unlock(&host_kvm.lock); 244 245 return ret; 246 } 247 248 int __pkvm_mark_hyp(phys_addr_t start, phys_addr_t end) 249 { 250 int ret; 251 252 /* 253 * host_stage2_unmap_dev_all() currently relies on MMIO mappings being 254 * non-persistent, so don't allow changing page ownership in MMIO range. 255 */ 256 if (!range_is_memory(start, end)) 257 return -EINVAL; 258 259 hyp_spin_lock(&host_kvm.lock); 260 ret = kvm_pgtable_stage2_set_owner(&host_kvm.pgt, start, end - start, 261 &host_s2_pool, pkvm_hyp_id); 262 hyp_spin_unlock(&host_kvm.lock); 263 264 return ret != -EAGAIN ? ret : 0; 265 } 266 267 void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) 268 { 269 struct kvm_vcpu_fault_info fault; 270 u64 esr, addr; 271 int ret = 0; 272 273 esr = read_sysreg_el2(SYS_ESR); 274 BUG_ON(!__get_fault_info(esr, &fault)); 275 276 addr = (fault.hpfar_el2 & HPFAR_MASK) << 8; 277 ret = host_stage2_idmap(addr); 278 BUG_ON(ret && ret != -EAGAIN); 279 } 280