1 /* 2 * Copyright (C) 2012,2013 - ARM Ltd 3 * Author: Marc Zyngier <marc.zyngier@arm.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18 #ifndef __ARM64_KVM_MMU_H__ 19 #define __ARM64_KVM_MMU_H__ 20 21 #include <asm/page.h> 22 #include <asm/memory.h> 23 #include <asm/cpufeature.h> 24 25 /* 26 * As ARMv8.0 only has the TTBR0_EL2 register, we cannot express 27 * "negative" addresses. This makes it impossible to directly share 28 * mappings with the kernel. 29 * 30 * Instead, give the HYP mode its own VA region at a fixed offset from 31 * the kernel by just masking the top bits (which are all ones for a 32 * kernel address). We need to find out how many bits to mask. 33 * 34 * We want to build a set of page tables that cover both parts of the 35 * idmap (the trampoline page used to initialize EL2), and our normal 36 * runtime VA space, at the same time. 37 * 38 * Given that the kernel uses VA_BITS for its entire address space, 39 * and that half of that space (VA_BITS - 1) is used for the linear 40 * mapping, we can also limit the EL2 space to (VA_BITS - 1). 41 * 42 * The main question is "Within the VA_BITS space, does EL2 use the 43 * top or the bottom half of that space to shadow the kernel's linear 44 * mapping?". As we need to idmap the trampoline page, this is 45 * determined by the range in which this page lives. 46 * 47 * If the page is in the bottom half, we have to use the top half. If 48 * the page is in the top half, we have to use the bottom half: 49 * 50 * T = __pa_symbol(__hyp_idmap_text_start) 51 * if (T & BIT(VA_BITS - 1)) 52 * HYP_VA_MIN = 0 //idmap in upper half 53 * else 54 * HYP_VA_MIN = 1 << (VA_BITS - 1) 55 * HYP_VA_MAX = HYP_VA_MIN + (1 << (VA_BITS - 1)) - 1 56 * 57 * This of course assumes that the trampoline page exists within the 58 * VA_BITS range. If it doesn't, then it means we're in the odd case 59 * where the kernel idmap (as well as HYP) uses more levels than the 60 * kernel runtime page tables (as seen when the kernel is configured 61 * for 4k pages, 39bits VA, and yet memory lives just above that 62 * limit, forcing the idmap to use 4 levels of page tables while the 63 * kernel itself only uses 3). In this particular case, it doesn't 64 * matter which side of VA_BITS we use, as we're guaranteed not to 65 * conflict with anything. 66 * 67 * When using VHE, there are no separate hyp mappings and all KVM 68 * functionality is already mapped as part of the main kernel 69 * mappings, and none of this applies in that case. 70 */ 71 72 #ifdef __ASSEMBLY__ 73 74 #include <asm/alternative.h> 75 #include <asm/cpufeature.h> 76 77 /* 78 * Convert a kernel VA into a HYP VA. 79 * reg: VA to be converted. 80 * 81 * The actual code generation takes place in kvm_update_va_mask, and 82 * the instructions below are only there to reserve the space and 83 * perform the register allocation (kvm_update_va_mask uses the 84 * specific registers encoded in the instructions). 85 */ 86 .macro kern_hyp_va reg 87 alternative_cb kvm_update_va_mask 88 and \reg, \reg, #1 /* mask with va_mask */ 89 ror \reg, \reg, #1 /* rotate to the first tag bit */ 90 add \reg, \reg, #0 /* insert the low 12 bits of the tag */ 91 add \reg, \reg, #0, lsl 12 /* insert the top 12 bits of the tag */ 92 ror \reg, \reg, #63 /* rotate back */ 93 alternative_cb_end 94 .endm 95 96 #else 97 98 #include <asm/pgalloc.h> 99 #include <asm/cache.h> 100 #include <asm/cacheflush.h> 101 #include <asm/mmu_context.h> 102 #include <asm/pgtable.h> 103 104 void kvm_update_va_mask(struct alt_instr *alt, 105 __le32 *origptr, __le32 *updptr, int nr_inst); 106 107 static inline unsigned long __kern_hyp_va(unsigned long v) 108 { 109 asm volatile(ALTERNATIVE_CB("and %0, %0, #1\n" 110 "ror %0, %0, #1\n" 111 "add %0, %0, #0\n" 112 "add %0, %0, #0, lsl 12\n" 113 "ror %0, %0, #63\n", 114 kvm_update_va_mask) 115 : "+r" (v)); 116 return v; 117 } 118 119 #define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v)))) 120 121 /* 122 * Obtain the PC-relative address of a kernel symbol 123 * s: symbol 124 * 125 * The goal of this macro is to return a symbol's address based on a 126 * PC-relative computation, as opposed to a loading the VA from a 127 * constant pool or something similar. This works well for HYP, as an 128 * absolute VA is guaranteed to be wrong. Only use this if trying to 129 * obtain the address of a symbol (i.e. not something you obtained by 130 * following a pointer). 131 */ 132 #define hyp_symbol_addr(s) \ 133 ({ \ 134 typeof(s) *addr; \ 135 asm("adrp %0, %1\n" \ 136 "add %0, %0, :lo12:%1\n" \ 137 : "=r" (addr) : "S" (&s)); \ 138 addr; \ 139 }) 140 141 /* 142 * We currently only support a 40bit IPA. 143 */ 144 #define KVM_PHYS_SHIFT (40) 145 #define KVM_PHYS_SIZE (1UL << KVM_PHYS_SHIFT) 146 #define KVM_PHYS_MASK (KVM_PHYS_SIZE - 1UL) 147 148 #include <asm/stage2_pgtable.h> 149 150 int create_hyp_mappings(void *from, void *to, pgprot_t prot); 151 int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, 152 void __iomem **kaddr, 153 void __iomem **haddr); 154 int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size, 155 void **haddr); 156 void free_hyp_pgds(void); 157 158 void stage2_unmap_vm(struct kvm *kvm); 159 int kvm_alloc_stage2_pgd(struct kvm *kvm); 160 void kvm_free_stage2_pgd(struct kvm *kvm); 161 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, 162 phys_addr_t pa, unsigned long size, bool writable); 163 164 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run); 165 166 void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); 167 168 phys_addr_t kvm_mmu_get_httbr(void); 169 phys_addr_t kvm_get_idmap_vector(void); 170 int kvm_mmu_init(void); 171 void kvm_clear_hyp_idmap(void); 172 173 #define kvm_set_pte(ptep, pte) set_pte(ptep, pte) 174 #define kvm_set_pmd(pmdp, pmd) set_pmd(pmdp, pmd) 175 176 static inline pte_t kvm_s2pte_mkwrite(pte_t pte) 177 { 178 pte_val(pte) |= PTE_S2_RDWR; 179 return pte; 180 } 181 182 static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd) 183 { 184 pmd_val(pmd) |= PMD_S2_RDWR; 185 return pmd; 186 } 187 188 static inline pte_t kvm_s2pte_mkexec(pte_t pte) 189 { 190 pte_val(pte) &= ~PTE_S2_XN; 191 return pte; 192 } 193 194 static inline pmd_t kvm_s2pmd_mkexec(pmd_t pmd) 195 { 196 pmd_val(pmd) &= ~PMD_S2_XN; 197 return pmd; 198 } 199 200 static inline void kvm_set_s2pte_readonly(pte_t *ptep) 201 { 202 pteval_t old_pteval, pteval; 203 204 pteval = READ_ONCE(pte_val(*ptep)); 205 do { 206 old_pteval = pteval; 207 pteval &= ~PTE_S2_RDWR; 208 pteval |= PTE_S2_RDONLY; 209 pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); 210 } while (pteval != old_pteval); 211 } 212 213 static inline bool kvm_s2pte_readonly(pte_t *ptep) 214 { 215 return (READ_ONCE(pte_val(*ptep)) & PTE_S2_RDWR) == PTE_S2_RDONLY; 216 } 217 218 static inline bool kvm_s2pte_exec(pte_t *ptep) 219 { 220 return !(READ_ONCE(pte_val(*ptep)) & PTE_S2_XN); 221 } 222 223 static inline void kvm_set_s2pmd_readonly(pmd_t *pmdp) 224 { 225 kvm_set_s2pte_readonly((pte_t *)pmdp); 226 } 227 228 static inline bool kvm_s2pmd_readonly(pmd_t *pmdp) 229 { 230 return kvm_s2pte_readonly((pte_t *)pmdp); 231 } 232 233 static inline bool kvm_s2pmd_exec(pmd_t *pmdp) 234 { 235 return !(READ_ONCE(pmd_val(*pmdp)) & PMD_S2_XN); 236 } 237 238 static inline bool kvm_page_empty(void *ptr) 239 { 240 struct page *ptr_page = virt_to_page(ptr); 241 return page_count(ptr_page) == 1; 242 } 243 244 #define hyp_pte_table_empty(ptep) kvm_page_empty(ptep) 245 246 #ifdef __PAGETABLE_PMD_FOLDED 247 #define hyp_pmd_table_empty(pmdp) (0) 248 #else 249 #define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp) 250 #endif 251 252 #ifdef __PAGETABLE_PUD_FOLDED 253 #define hyp_pud_table_empty(pudp) (0) 254 #else 255 #define hyp_pud_table_empty(pudp) kvm_page_empty(pudp) 256 #endif 257 258 struct kvm; 259 260 #define kvm_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l)) 261 262 static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu) 263 { 264 return (vcpu_read_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101; 265 } 266 267 static inline void __clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size) 268 { 269 void *va = page_address(pfn_to_page(pfn)); 270 271 kvm_flush_dcache_to_poc(va, size); 272 } 273 274 static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn, 275 unsigned long size) 276 { 277 if (icache_is_aliasing()) { 278 /* any kind of VIPT cache */ 279 __flush_icache_all(); 280 } else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) { 281 /* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */ 282 void *va = page_address(pfn_to_page(pfn)); 283 284 invalidate_icache_range((unsigned long)va, 285 (unsigned long)va + size); 286 } 287 } 288 289 static inline void __kvm_flush_dcache_pte(pte_t pte) 290 { 291 struct page *page = pte_page(pte); 292 kvm_flush_dcache_to_poc(page_address(page), PAGE_SIZE); 293 } 294 295 static inline void __kvm_flush_dcache_pmd(pmd_t pmd) 296 { 297 struct page *page = pmd_page(pmd); 298 kvm_flush_dcache_to_poc(page_address(page), PMD_SIZE); 299 } 300 301 static inline void __kvm_flush_dcache_pud(pud_t pud) 302 { 303 struct page *page = pud_page(pud); 304 kvm_flush_dcache_to_poc(page_address(page), PUD_SIZE); 305 } 306 307 #define kvm_virt_to_phys(x) __pa_symbol(x) 308 309 void kvm_set_way_flush(struct kvm_vcpu *vcpu); 310 void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled); 311 312 static inline bool __kvm_cpu_uses_extended_idmap(void) 313 { 314 return __cpu_uses_extended_idmap_level(); 315 } 316 317 static inline unsigned long __kvm_idmap_ptrs_per_pgd(void) 318 { 319 return idmap_ptrs_per_pgd; 320 } 321 322 /* 323 * Can't use pgd_populate here, because the extended idmap adds an extra level 324 * above CONFIG_PGTABLE_LEVELS (which is 2 or 3 if we're using the extended 325 * idmap), and pgd_populate is only available if CONFIG_PGTABLE_LEVELS = 4. 326 */ 327 static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd, 328 pgd_t *hyp_pgd, 329 pgd_t *merged_hyp_pgd, 330 unsigned long hyp_idmap_start) 331 { 332 int idmap_idx; 333 u64 pgd_addr; 334 335 /* 336 * Use the first entry to access the HYP mappings. It is 337 * guaranteed to be free, otherwise we wouldn't use an 338 * extended idmap. 339 */ 340 VM_BUG_ON(pgd_val(merged_hyp_pgd[0])); 341 pgd_addr = __phys_to_pgd_val(__pa(hyp_pgd)); 342 merged_hyp_pgd[0] = __pgd(pgd_addr | PMD_TYPE_TABLE); 343 344 /* 345 * Create another extended level entry that points to the boot HYP map, 346 * which contains an ID mapping of the HYP init code. We essentially 347 * merge the boot and runtime HYP maps by doing so, but they don't 348 * overlap anyway, so this is fine. 349 */ 350 idmap_idx = hyp_idmap_start >> VA_BITS; 351 VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx])); 352 pgd_addr = __phys_to_pgd_val(__pa(boot_hyp_pgd)); 353 merged_hyp_pgd[idmap_idx] = __pgd(pgd_addr | PMD_TYPE_TABLE); 354 } 355 356 static inline unsigned int kvm_get_vmid_bits(void) 357 { 358 int reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); 359 360 return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8; 361 } 362 363 #ifdef CONFIG_KVM_INDIRECT_VECTORS 364 /* 365 * EL2 vectors can be mapped and rerouted in a number of ways, 366 * depending on the kernel configuration and CPU present: 367 * 368 * - If the CPU has the ARM64_HARDEN_BRANCH_PREDICTOR cap, the 369 * hardening sequence is placed in one of the vector slots, which is 370 * executed before jumping to the real vectors. 371 * 372 * - If the CPU has both the ARM64_HARDEN_EL2_VECTORS cap and the 373 * ARM64_HARDEN_BRANCH_PREDICTOR cap, the slot containing the 374 * hardening sequence is mapped next to the idmap page, and executed 375 * before jumping to the real vectors. 376 * 377 * - If the CPU only has the ARM64_HARDEN_EL2_VECTORS cap, then an 378 * empty slot is selected, mapped next to the idmap page, and 379 * executed before jumping to the real vectors. 380 * 381 * Note that ARM64_HARDEN_EL2_VECTORS is somewhat incompatible with 382 * VHE, as we don't have hypervisor-specific mappings. If the system 383 * is VHE and yet selects this capability, it will be ignored. 384 */ 385 #include <asm/mmu.h> 386 387 extern void *__kvm_bp_vect_base; 388 extern int __kvm_harden_el2_vector_slot; 389 390 static inline void *kvm_get_hyp_vector(void) 391 { 392 struct bp_hardening_data *data = arm64_get_bp_hardening_data(); 393 void *vect = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector)); 394 int slot = -1; 395 396 if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR) && data->fn) { 397 vect = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs_start)); 398 slot = data->hyp_vectors_slot; 399 } 400 401 if (this_cpu_has_cap(ARM64_HARDEN_EL2_VECTORS) && !has_vhe()) { 402 vect = __kvm_bp_vect_base; 403 if (slot == -1) 404 slot = __kvm_harden_el2_vector_slot; 405 } 406 407 if (slot != -1) 408 vect += slot * SZ_2K; 409 410 return vect; 411 } 412 413 /* This is only called on a !VHE system */ 414 static inline int kvm_map_vectors(void) 415 { 416 /* 417 * HBP = ARM64_HARDEN_BRANCH_PREDICTOR 418 * HEL2 = ARM64_HARDEN_EL2_VECTORS 419 * 420 * !HBP + !HEL2 -> use direct vectors 421 * HBP + !HEL2 -> use hardened vectors in place 422 * !HBP + HEL2 -> allocate one vector slot and use exec mapping 423 * HBP + HEL2 -> use hardened vertors and use exec mapping 424 */ 425 if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR)) { 426 __kvm_bp_vect_base = kvm_ksym_ref(__bp_harden_hyp_vecs_start); 427 __kvm_bp_vect_base = kern_hyp_va(__kvm_bp_vect_base); 428 } 429 430 if (cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) { 431 phys_addr_t vect_pa = __pa_symbol(__bp_harden_hyp_vecs_start); 432 unsigned long size = (__bp_harden_hyp_vecs_end - 433 __bp_harden_hyp_vecs_start); 434 435 /* 436 * Always allocate a spare vector slot, as we don't 437 * know yet which CPUs have a BP hardening slot that 438 * we can reuse. 439 */ 440 __kvm_harden_el2_vector_slot = atomic_inc_return(&arm64_el2_vector_last_slot); 441 BUG_ON(__kvm_harden_el2_vector_slot >= BP_HARDEN_EL2_SLOTS); 442 return create_hyp_exec_mappings(vect_pa, size, 443 &__kvm_bp_vect_base); 444 } 445 446 return 0; 447 } 448 #else 449 static inline void *kvm_get_hyp_vector(void) 450 { 451 return kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector)); 452 } 453 454 static inline int kvm_map_vectors(void) 455 { 456 return 0; 457 } 458 #endif 459 460 #define kvm_phys_to_vttbr(addr) phys_to_ttbr(addr) 461 462 #endif /* __ASSEMBLY__ */ 463 #endif /* __ARM64_KVM_MMU_H__ */ 464