1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 4 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 5 */ 6 7 #include <linux/mman.h> 8 #include <linux/kvm_host.h> 9 #include <linux/io.h> 10 #include <linux/hugetlb.h> 11 #include <linux/sched/signal.h> 12 #include <trace/events/kvm.h> 13 #include <asm/pgalloc.h> 14 #include <asm/cacheflush.h> 15 #include <asm/kvm_arm.h> 16 #include <asm/kvm_mmu.h> 17 #include <asm/kvm_pgtable.h> 18 #include <asm/kvm_ras.h> 19 #include <asm/kvm_asm.h> 20 #include <asm/kvm_emulate.h> 21 #include <asm/virt.h> 22 23 #include "trace.h" 24 25 static struct kvm_pgtable *hyp_pgtable; 26 static DEFINE_MUTEX(kvm_hyp_pgd_mutex); 27 28 static unsigned long hyp_idmap_start; 29 static unsigned long hyp_idmap_end; 30 static phys_addr_t hyp_idmap_vector; 31 32 static unsigned long io_map_base; 33 34 35 /* 36 * Release kvm_mmu_lock periodically if the memory region is large. Otherwise, 37 * we may see kernel panics with CONFIG_DETECT_HUNG_TASK, 38 * CONFIG_LOCKUP_DETECTOR, CONFIG_LOCKDEP. Additionally, holding the lock too 39 * long will also starve other vCPUs. We have to also make sure that the page 40 * tables are not freed while we released the lock. 41 */ 42 static int stage2_apply_range(struct kvm *kvm, phys_addr_t addr, 43 phys_addr_t end, 44 int (*fn)(struct kvm_pgtable *, u64, u64), 45 bool resched) 46 { 47 int ret; 48 u64 next; 49 50 do { 51 struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; 52 if (!pgt) 53 return -EINVAL; 54 55 next = stage2_pgd_addr_end(kvm, addr, end); 56 ret = fn(pgt, addr, next - addr); 57 if (ret) 58 break; 59 60 if (resched && next != end) 61 cond_resched_lock(&kvm->mmu_lock); 62 } while (addr = next, addr != end); 63 64 return ret; 65 } 66 67 #define stage2_apply_range_resched(kvm, addr, end, fn) \ 68 stage2_apply_range(kvm, addr, end, fn, true) 69 70 static bool memslot_is_logging(struct kvm_memory_slot *memslot) 71 { 72 return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); 73 } 74 75 /** 76 * kvm_flush_remote_tlbs() - flush all VM TLB entries for v7/8 77 * @kvm: pointer to kvm structure. 78 * 79 * Interface to HYP function to flush all VM TLB entries 80 */ 81 void kvm_flush_remote_tlbs(struct kvm *kvm) 82 { 83 kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); 84 } 85 86 static bool kvm_is_device_pfn(unsigned long pfn) 87 { 88 return !pfn_valid(pfn); 89 } 90 91 /* 92 * Unmapping vs dcache management: 93 * 94 * If a guest maps certain memory pages as uncached, all writes will 95 * bypass the data cache and go directly to RAM. However, the CPUs 96 * can still speculate reads (not writes) and fill cache lines with 97 * data. 98 * 99 * Those cache lines will be *clean* cache lines though, so a 100 * clean+invalidate operation is equivalent to an invalidate 101 * operation, because no cache lines are marked dirty. 102 * 103 * Those clean cache lines could be filled prior to an uncached write 104 * by the guest, and the cache coherent IO subsystem would therefore 105 * end up writing old data to disk. 106 * 107 * This is why right after unmapping a page/section and invalidating 108 * the corresponding TLBs, we flush to make sure the IO subsystem will 109 * never hit in the cache. 110 * 111 * This is all avoided on systems that have ARM64_HAS_STAGE2_FWB, as 112 * we then fully enforce cacheability of RAM, no matter what the guest 113 * does. 114 */ 115 /** 116 * unmap_stage2_range -- Clear stage2 page table entries to unmap a range 117 * @mmu: The KVM stage-2 MMU pointer 118 * @start: The intermediate physical base address of the range to unmap 119 * @size: The size of the area to unmap 120 * @may_block: Whether or not we are permitted to block 121 * 122 * Clear a range of stage-2 mappings, lowering the various ref-counts. Must 123 * be called while holding mmu_lock (unless for freeing the stage2 pgd before 124 * destroying the VM), otherwise another faulting VCPU may come in and mess 125 * with things behind our backs. 126 */ 127 static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, 128 bool may_block) 129 { 130 struct kvm *kvm = mmu->kvm; 131 phys_addr_t end = start + size; 132 133 assert_spin_locked(&kvm->mmu_lock); 134 WARN_ON(size & ~PAGE_MASK); 135 WARN_ON(stage2_apply_range(kvm, start, end, kvm_pgtable_stage2_unmap, 136 may_block)); 137 } 138 139 static void unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size) 140 { 141 __unmap_stage2_range(mmu, start, size, true); 142 } 143 144 static void stage2_flush_memslot(struct kvm *kvm, 145 struct kvm_memory_slot *memslot) 146 { 147 phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; 148 phys_addr_t end = addr + PAGE_SIZE * memslot->npages; 149 150 stage2_apply_range_resched(kvm, addr, end, kvm_pgtable_stage2_flush); 151 } 152 153 /** 154 * stage2_flush_vm - Invalidate cache for pages mapped in stage 2 155 * @kvm: The struct kvm pointer 156 * 157 * Go through the stage 2 page tables and invalidate any cache lines 158 * backing memory already mapped to the VM. 159 */ 160 static void stage2_flush_vm(struct kvm *kvm) 161 { 162 struct kvm_memslots *slots; 163 struct kvm_memory_slot *memslot; 164 int idx; 165 166 idx = srcu_read_lock(&kvm->srcu); 167 spin_lock(&kvm->mmu_lock); 168 169 slots = kvm_memslots(kvm); 170 kvm_for_each_memslot(memslot, slots) 171 stage2_flush_memslot(kvm, memslot); 172 173 spin_unlock(&kvm->mmu_lock); 174 srcu_read_unlock(&kvm->srcu, idx); 175 } 176 177 /** 178 * free_hyp_pgds - free Hyp-mode page tables 179 */ 180 void free_hyp_pgds(void) 181 { 182 mutex_lock(&kvm_hyp_pgd_mutex); 183 if (hyp_pgtable) { 184 kvm_pgtable_hyp_destroy(hyp_pgtable); 185 kfree(hyp_pgtable); 186 } 187 mutex_unlock(&kvm_hyp_pgd_mutex); 188 } 189 190 static int __create_hyp_mappings(unsigned long start, unsigned long size, 191 unsigned long phys, enum kvm_pgtable_prot prot) 192 { 193 int err; 194 195 mutex_lock(&kvm_hyp_pgd_mutex); 196 err = kvm_pgtable_hyp_map(hyp_pgtable, start, size, phys, prot); 197 mutex_unlock(&kvm_hyp_pgd_mutex); 198 199 return err; 200 } 201 202 static phys_addr_t kvm_kaddr_to_phys(void *kaddr) 203 { 204 if (!is_vmalloc_addr(kaddr)) { 205 BUG_ON(!virt_addr_valid(kaddr)); 206 return __pa(kaddr); 207 } else { 208 return page_to_phys(vmalloc_to_page(kaddr)) + 209 offset_in_page(kaddr); 210 } 211 } 212 213 /** 214 * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode 215 * @from: The virtual kernel start address of the range 216 * @to: The virtual kernel end address of the range (exclusive) 217 * @prot: The protection to be applied to this range 218 * 219 * The same virtual address as the kernel virtual address is also used 220 * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying 221 * physical pages. 222 */ 223 int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot) 224 { 225 phys_addr_t phys_addr; 226 unsigned long virt_addr; 227 unsigned long start = kern_hyp_va((unsigned long)from); 228 unsigned long end = kern_hyp_va((unsigned long)to); 229 230 if (is_kernel_in_hyp_mode()) 231 return 0; 232 233 start = start & PAGE_MASK; 234 end = PAGE_ALIGN(end); 235 236 for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) { 237 int err; 238 239 phys_addr = kvm_kaddr_to_phys(from + virt_addr - start); 240 err = __create_hyp_mappings(virt_addr, PAGE_SIZE, phys_addr, 241 prot); 242 if (err) 243 return err; 244 } 245 246 return 0; 247 } 248 249 static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size, 250 unsigned long *haddr, 251 enum kvm_pgtable_prot prot) 252 { 253 unsigned long base; 254 int ret = 0; 255 256 mutex_lock(&kvm_hyp_pgd_mutex); 257 258 /* 259 * This assumes that we have enough space below the idmap 260 * page to allocate our VAs. If not, the check below will 261 * kick. A potential alternative would be to detect that 262 * overflow and switch to an allocation above the idmap. 263 * 264 * The allocated size is always a multiple of PAGE_SIZE. 265 */ 266 size = PAGE_ALIGN(size + offset_in_page(phys_addr)); 267 base = io_map_base - size; 268 269 /* 270 * Verify that BIT(VA_BITS - 1) hasn't been flipped by 271 * allocating the new area, as it would indicate we've 272 * overflowed the idmap/IO address range. 273 */ 274 if ((base ^ io_map_base) & BIT(VA_BITS - 1)) 275 ret = -ENOMEM; 276 else 277 io_map_base = base; 278 279 mutex_unlock(&kvm_hyp_pgd_mutex); 280 281 if (ret) 282 goto out; 283 284 ret = __create_hyp_mappings(base, size, phys_addr, prot); 285 if (ret) 286 goto out; 287 288 *haddr = base + offset_in_page(phys_addr); 289 out: 290 return ret; 291 } 292 293 /** 294 * create_hyp_io_mappings - Map IO into both kernel and HYP 295 * @phys_addr: The physical start address which gets mapped 296 * @size: Size of the region being mapped 297 * @kaddr: Kernel VA for this mapping 298 * @haddr: HYP VA for this mapping 299 */ 300 int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, 301 void __iomem **kaddr, 302 void __iomem **haddr) 303 { 304 unsigned long addr; 305 int ret; 306 307 *kaddr = ioremap(phys_addr, size); 308 if (!*kaddr) 309 return -ENOMEM; 310 311 if (is_kernel_in_hyp_mode()) { 312 *haddr = *kaddr; 313 return 0; 314 } 315 316 ret = __create_hyp_private_mapping(phys_addr, size, 317 &addr, PAGE_HYP_DEVICE); 318 if (ret) { 319 iounmap(*kaddr); 320 *kaddr = NULL; 321 *haddr = NULL; 322 return ret; 323 } 324 325 *haddr = (void __iomem *)addr; 326 return 0; 327 } 328 329 /** 330 * create_hyp_exec_mappings - Map an executable range into HYP 331 * @phys_addr: The physical start address which gets mapped 332 * @size: Size of the region being mapped 333 * @haddr: HYP VA for this mapping 334 */ 335 int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size, 336 void **haddr) 337 { 338 unsigned long addr; 339 int ret; 340 341 BUG_ON(is_kernel_in_hyp_mode()); 342 343 ret = __create_hyp_private_mapping(phys_addr, size, 344 &addr, PAGE_HYP_EXEC); 345 if (ret) { 346 *haddr = NULL; 347 return ret; 348 } 349 350 *haddr = (void *)addr; 351 return 0; 352 } 353 354 /** 355 * kvm_init_stage2_mmu - Initialise a S2 MMU strucrure 356 * @kvm: The pointer to the KVM structure 357 * @mmu: The pointer to the s2 MMU structure 358 * 359 * Allocates only the stage-2 HW PGD level table(s). 360 * Note we don't need locking here as this is only called when the VM is 361 * created, which can only be done once. 362 */ 363 int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) 364 { 365 int cpu, err; 366 struct kvm_pgtable *pgt; 367 368 if (mmu->pgt != NULL) { 369 kvm_err("kvm_arch already initialized?\n"); 370 return -EINVAL; 371 } 372 373 pgt = kzalloc(sizeof(*pgt), GFP_KERNEL); 374 if (!pgt) 375 return -ENOMEM; 376 377 err = kvm_pgtable_stage2_init(pgt, kvm); 378 if (err) 379 goto out_free_pgtable; 380 381 mmu->last_vcpu_ran = alloc_percpu(typeof(*mmu->last_vcpu_ran)); 382 if (!mmu->last_vcpu_ran) { 383 err = -ENOMEM; 384 goto out_destroy_pgtable; 385 } 386 387 for_each_possible_cpu(cpu) 388 *per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1; 389 390 mmu->kvm = kvm; 391 mmu->pgt = pgt; 392 mmu->pgd_phys = __pa(pgt->pgd); 393 mmu->vmid.vmid_gen = 0; 394 return 0; 395 396 out_destroy_pgtable: 397 kvm_pgtable_stage2_destroy(pgt); 398 out_free_pgtable: 399 kfree(pgt); 400 return err; 401 } 402 403 static void stage2_unmap_memslot(struct kvm *kvm, 404 struct kvm_memory_slot *memslot) 405 { 406 hva_t hva = memslot->userspace_addr; 407 phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; 408 phys_addr_t size = PAGE_SIZE * memslot->npages; 409 hva_t reg_end = hva + size; 410 411 /* 412 * A memory region could potentially cover multiple VMAs, and any holes 413 * between them, so iterate over all of them to find out if we should 414 * unmap any of them. 415 * 416 * +--------------------------------------------+ 417 * +---------------+----------------+ +----------------+ 418 * | : VMA 1 | VMA 2 | | VMA 3 : | 419 * +---------------+----------------+ +----------------+ 420 * | memory region | 421 * +--------------------------------------------+ 422 */ 423 do { 424 struct vm_area_struct *vma = find_vma(current->mm, hva); 425 hva_t vm_start, vm_end; 426 427 if (!vma || vma->vm_start >= reg_end) 428 break; 429 430 /* 431 * Take the intersection of this VMA with the memory region 432 */ 433 vm_start = max(hva, vma->vm_start); 434 vm_end = min(reg_end, vma->vm_end); 435 436 if (!(vma->vm_flags & VM_PFNMAP)) { 437 gpa_t gpa = addr + (vm_start - memslot->userspace_addr); 438 unmap_stage2_range(&kvm->arch.mmu, gpa, vm_end - vm_start); 439 } 440 hva = vm_end; 441 } while (hva < reg_end); 442 } 443 444 /** 445 * stage2_unmap_vm - Unmap Stage-2 RAM mappings 446 * @kvm: The struct kvm pointer 447 * 448 * Go through the memregions and unmap any regular RAM 449 * backing memory already mapped to the VM. 450 */ 451 void stage2_unmap_vm(struct kvm *kvm) 452 { 453 struct kvm_memslots *slots; 454 struct kvm_memory_slot *memslot; 455 int idx; 456 457 idx = srcu_read_lock(&kvm->srcu); 458 mmap_read_lock(current->mm); 459 spin_lock(&kvm->mmu_lock); 460 461 slots = kvm_memslots(kvm); 462 kvm_for_each_memslot(memslot, slots) 463 stage2_unmap_memslot(kvm, memslot); 464 465 spin_unlock(&kvm->mmu_lock); 466 mmap_read_unlock(current->mm); 467 srcu_read_unlock(&kvm->srcu, idx); 468 } 469 470 void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) 471 { 472 struct kvm *kvm = mmu->kvm; 473 struct kvm_pgtable *pgt = NULL; 474 475 spin_lock(&kvm->mmu_lock); 476 pgt = mmu->pgt; 477 if (pgt) { 478 mmu->pgd_phys = 0; 479 mmu->pgt = NULL; 480 free_percpu(mmu->last_vcpu_ran); 481 } 482 spin_unlock(&kvm->mmu_lock); 483 484 if (pgt) { 485 kvm_pgtable_stage2_destroy(pgt); 486 kfree(pgt); 487 } 488 } 489 490 /** 491 * kvm_phys_addr_ioremap - map a device range to guest IPA 492 * 493 * @kvm: The KVM pointer 494 * @guest_ipa: The IPA at which to insert the mapping 495 * @pa: The physical address of the device 496 * @size: The size of the mapping 497 * @writable: Whether or not to create a writable mapping 498 */ 499 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, 500 phys_addr_t pa, unsigned long size, bool writable) 501 { 502 phys_addr_t addr; 503 int ret = 0; 504 struct kvm_mmu_memory_cache cache = { 0, __GFP_ZERO, NULL, }; 505 struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; 506 enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_DEVICE | 507 KVM_PGTABLE_PROT_R | 508 (writable ? KVM_PGTABLE_PROT_W : 0); 509 510 size += offset_in_page(guest_ipa); 511 guest_ipa &= PAGE_MASK; 512 513 for (addr = guest_ipa; addr < guest_ipa + size; addr += PAGE_SIZE) { 514 ret = kvm_mmu_topup_memory_cache(&cache, 515 kvm_mmu_cache_min_pages(kvm)); 516 if (ret) 517 break; 518 519 spin_lock(&kvm->mmu_lock); 520 ret = kvm_pgtable_stage2_map(pgt, addr, PAGE_SIZE, pa, prot, 521 &cache); 522 spin_unlock(&kvm->mmu_lock); 523 if (ret) 524 break; 525 526 pa += PAGE_SIZE; 527 } 528 529 kvm_mmu_free_memory_cache(&cache); 530 return ret; 531 } 532 533 /** 534 * stage2_wp_range() - write protect stage2 memory region range 535 * @mmu: The KVM stage-2 MMU pointer 536 * @addr: Start address of range 537 * @end: End address of range 538 */ 539 static void stage2_wp_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end) 540 { 541 struct kvm *kvm = mmu->kvm; 542 stage2_apply_range_resched(kvm, addr, end, kvm_pgtable_stage2_wrprotect); 543 } 544 545 /** 546 * kvm_mmu_wp_memory_region() - write protect stage 2 entries for memory slot 547 * @kvm: The KVM pointer 548 * @slot: The memory slot to write protect 549 * 550 * Called to start logging dirty pages after memory region 551 * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns 552 * all present PUD, PMD and PTEs are write protected in the memory region. 553 * Afterwards read of dirty page log can be called. 554 * 555 * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired, 556 * serializing operations for VM memory regions. 557 */ 558 void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot) 559 { 560 struct kvm_memslots *slots = kvm_memslots(kvm); 561 struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); 562 phys_addr_t start, end; 563 564 if (WARN_ON_ONCE(!memslot)) 565 return; 566 567 start = memslot->base_gfn << PAGE_SHIFT; 568 end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; 569 570 spin_lock(&kvm->mmu_lock); 571 stage2_wp_range(&kvm->arch.mmu, start, end); 572 spin_unlock(&kvm->mmu_lock); 573 kvm_flush_remote_tlbs(kvm); 574 } 575 576 /** 577 * kvm_mmu_write_protect_pt_masked() - write protect dirty pages 578 * @kvm: The KVM pointer 579 * @slot: The memory slot associated with mask 580 * @gfn_offset: The gfn offset in memory slot 581 * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory 582 * slot to be write protected 583 * 584 * Walks bits set in mask write protects the associated pte's. Caller must 585 * acquire kvm_mmu_lock. 586 */ 587 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, 588 struct kvm_memory_slot *slot, 589 gfn_t gfn_offset, unsigned long mask) 590 { 591 phys_addr_t base_gfn = slot->base_gfn + gfn_offset; 592 phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; 593 phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; 594 595 stage2_wp_range(&kvm->arch.mmu, start, end); 596 } 597 598 /* 599 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected 600 * dirty pages. 601 * 602 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to 603 * enable dirty logging for them. 604 */ 605 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 606 struct kvm_memory_slot *slot, 607 gfn_t gfn_offset, unsigned long mask) 608 { 609 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); 610 } 611 612 static void clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size) 613 { 614 __clean_dcache_guest_page(pfn, size); 615 } 616 617 static void invalidate_icache_guest_page(kvm_pfn_t pfn, unsigned long size) 618 { 619 __invalidate_icache_guest_page(pfn, size); 620 } 621 622 static void kvm_send_hwpoison_signal(unsigned long address, short lsb) 623 { 624 send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current); 625 } 626 627 static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, 628 unsigned long hva, 629 unsigned long map_size) 630 { 631 gpa_t gpa_start; 632 hva_t uaddr_start, uaddr_end; 633 size_t size; 634 635 /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ 636 if (map_size == PAGE_SIZE) 637 return true; 638 639 size = memslot->npages * PAGE_SIZE; 640 641 gpa_start = memslot->base_gfn << PAGE_SHIFT; 642 643 uaddr_start = memslot->userspace_addr; 644 uaddr_end = uaddr_start + size; 645 646 /* 647 * Pages belonging to memslots that don't have the same alignment 648 * within a PMD/PUD for userspace and IPA cannot be mapped with stage-2 649 * PMD/PUD entries, because we'll end up mapping the wrong pages. 650 * 651 * Consider a layout like the following: 652 * 653 * memslot->userspace_addr: 654 * +-----+--------------------+--------------------+---+ 655 * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| 656 * +-----+--------------------+--------------------+---+ 657 * 658 * memslot->base_gfn << PAGE_SHIFT: 659 * +---+--------------------+--------------------+-----+ 660 * |abc|def Stage-2 block | Stage-2 block |tvxyz| 661 * +---+--------------------+--------------------+-----+ 662 * 663 * If we create those stage-2 blocks, we'll end up with this incorrect 664 * mapping: 665 * d -> f 666 * e -> g 667 * f -> h 668 */ 669 if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1))) 670 return false; 671 672 /* 673 * Next, let's make sure we're not trying to map anything not covered 674 * by the memslot. This means we have to prohibit block size mappings 675 * for the beginning and end of a non-block aligned and non-block sized 676 * memory slot (illustrated by the head and tail parts of the 677 * userspace view above containing pages 'abcde' and 'xyz', 678 * respectively). 679 * 680 * Note that it doesn't matter if we do the check using the 681 * userspace_addr or the base_gfn, as both are equally aligned (per 682 * the check above) and equally sized. 683 */ 684 return (hva & ~(map_size - 1)) >= uaddr_start && 685 (hva & ~(map_size - 1)) + map_size <= uaddr_end; 686 } 687 688 /* 689 * Check if the given hva is backed by a transparent huge page (THP) and 690 * whether it can be mapped using block mapping in stage2. If so, adjust 691 * the stage2 PFN and IPA accordingly. Only PMD_SIZE THPs are currently 692 * supported. This will need to be updated to support other THP sizes. 693 * 694 * Returns the size of the mapping. 695 */ 696 static unsigned long 697 transparent_hugepage_adjust(struct kvm_memory_slot *memslot, 698 unsigned long hva, kvm_pfn_t *pfnp, 699 phys_addr_t *ipap) 700 { 701 kvm_pfn_t pfn = *pfnp; 702 703 /* 704 * Make sure the adjustment is done only for THP pages. Also make 705 * sure that the HVA and IPA are sufficiently aligned and that the 706 * block map is contained within the memslot. 707 */ 708 if (kvm_is_transparent_hugepage(pfn) && 709 fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) { 710 /* 711 * The address we faulted on is backed by a transparent huge 712 * page. However, because we map the compound huge page and 713 * not the individual tail page, we need to transfer the 714 * refcount to the head page. We have to be careful that the 715 * THP doesn't start to split while we are adjusting the 716 * refcounts. 717 * 718 * We are sure this doesn't happen, because mmu_notifier_retry 719 * was successful and we are holding the mmu_lock, so if this 720 * THP is trying to split, it will be blocked in the mmu 721 * notifier before touching any of the pages, specifically 722 * before being able to call __split_huge_page_refcount(). 723 * 724 * We can therefore safely transfer the refcount from PG_tail 725 * to PG_head and switch the pfn from a tail page to the head 726 * page accordingly. 727 */ 728 *ipap &= PMD_MASK; 729 kvm_release_pfn_clean(pfn); 730 pfn &= ~(PTRS_PER_PMD - 1); 731 kvm_get_pfn(pfn); 732 *pfnp = pfn; 733 734 return PMD_SIZE; 735 } 736 737 /* Use page mapping if we cannot use block mapping. */ 738 return PAGE_SIZE; 739 } 740 741 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 742 struct kvm_memory_slot *memslot, unsigned long hva, 743 unsigned long fault_status) 744 { 745 int ret = 0; 746 bool write_fault, writable, force_pte = false; 747 bool exec_fault; 748 bool device = false; 749 unsigned long mmu_seq; 750 struct kvm *kvm = vcpu->kvm; 751 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; 752 struct vm_area_struct *vma; 753 short vma_shift; 754 gfn_t gfn; 755 kvm_pfn_t pfn; 756 bool logging_active = memslot_is_logging(memslot); 757 unsigned long vma_pagesize; 758 enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; 759 struct kvm_pgtable *pgt; 760 761 write_fault = kvm_is_write_fault(vcpu); 762 exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu); 763 VM_BUG_ON(write_fault && exec_fault); 764 765 if (fault_status == FSC_PERM && !write_fault && !exec_fault) { 766 kvm_err("Unexpected L2 read permission error\n"); 767 return -EFAULT; 768 } 769 770 /* Let's check if we will get back a huge page backed by hugetlbfs */ 771 mmap_read_lock(current->mm); 772 vma = find_vma_intersection(current->mm, hva, hva + 1); 773 if (unlikely(!vma)) { 774 kvm_err("Failed to find VMA for hva 0x%lx\n", hva); 775 mmap_read_unlock(current->mm); 776 return -EFAULT; 777 } 778 779 if (is_vm_hugetlb_page(vma)) 780 vma_shift = huge_page_shift(hstate_vma(vma)); 781 else 782 vma_shift = PAGE_SHIFT; 783 784 if (logging_active || 785 (vma->vm_flags & VM_PFNMAP)) { 786 force_pte = true; 787 vma_shift = PAGE_SHIFT; 788 } 789 790 if (vma_shift == PUD_SHIFT && 791 !fault_supports_stage2_huge_mapping(memslot, hva, PUD_SIZE)) 792 vma_shift = PMD_SHIFT; 793 794 if (vma_shift == PMD_SHIFT && 795 !fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) { 796 force_pte = true; 797 vma_shift = PAGE_SHIFT; 798 } 799 800 vma_pagesize = 1UL << vma_shift; 801 if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE) 802 fault_ipa &= ~(vma_pagesize - 1); 803 804 gfn = fault_ipa >> PAGE_SHIFT; 805 mmap_read_unlock(current->mm); 806 807 /* 808 * Permission faults just need to update the existing leaf entry, 809 * and so normally don't require allocations from the memcache. The 810 * only exception to this is when dirty logging is enabled at runtime 811 * and a write fault needs to collapse a block entry into a table. 812 */ 813 if (fault_status != FSC_PERM || (logging_active && write_fault)) { 814 ret = kvm_mmu_topup_memory_cache(memcache, 815 kvm_mmu_cache_min_pages(kvm)); 816 if (ret) 817 return ret; 818 } 819 820 mmu_seq = vcpu->kvm->mmu_notifier_seq; 821 /* 822 * Ensure the read of mmu_notifier_seq happens before we call 823 * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk 824 * the page we just got a reference to gets unmapped before we have a 825 * chance to grab the mmu_lock, which ensure that if the page gets 826 * unmapped afterwards, the call to kvm_unmap_hva will take it away 827 * from us again properly. This smp_rmb() interacts with the smp_wmb() 828 * in kvm_mmu_notifier_invalidate_<page|range_end>. 829 */ 830 smp_rmb(); 831 832 pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable); 833 if (pfn == KVM_PFN_ERR_HWPOISON) { 834 kvm_send_hwpoison_signal(hva, vma_shift); 835 return 0; 836 } 837 if (is_error_noslot_pfn(pfn)) 838 return -EFAULT; 839 840 if (kvm_is_device_pfn(pfn)) { 841 device = true; 842 } else if (logging_active && !write_fault) { 843 /* 844 * Only actually map the page as writable if this was a write 845 * fault. 846 */ 847 writable = false; 848 } 849 850 if (exec_fault && device) 851 return -ENOEXEC; 852 853 spin_lock(&kvm->mmu_lock); 854 pgt = vcpu->arch.hw_mmu->pgt; 855 if (mmu_notifier_retry(kvm, mmu_seq)) 856 goto out_unlock; 857 858 /* 859 * If we are not forced to use page mapping, check if we are 860 * backed by a THP and thus use block mapping if possible. 861 */ 862 if (vma_pagesize == PAGE_SIZE && !force_pte) 863 vma_pagesize = transparent_hugepage_adjust(memslot, hva, 864 &pfn, &fault_ipa); 865 if (writable) { 866 prot |= KVM_PGTABLE_PROT_W; 867 kvm_set_pfn_dirty(pfn); 868 mark_page_dirty(kvm, gfn); 869 } 870 871 if (fault_status != FSC_PERM && !device) 872 clean_dcache_guest_page(pfn, vma_pagesize); 873 874 if (exec_fault) { 875 prot |= KVM_PGTABLE_PROT_X; 876 invalidate_icache_guest_page(pfn, vma_pagesize); 877 } 878 879 if (device) 880 prot |= KVM_PGTABLE_PROT_DEVICE; 881 else if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC)) 882 prot |= KVM_PGTABLE_PROT_X; 883 884 if (fault_status == FSC_PERM && !(logging_active && writable)) { 885 ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot); 886 } else { 887 ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize, 888 __pfn_to_phys(pfn), prot, 889 memcache); 890 } 891 892 out_unlock: 893 spin_unlock(&kvm->mmu_lock); 894 kvm_set_pfn_accessed(pfn); 895 kvm_release_pfn_clean(pfn); 896 return ret; 897 } 898 899 /* Resolve the access fault by making the page young again. */ 900 static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) 901 { 902 pte_t pte; 903 kvm_pte_t kpte; 904 struct kvm_s2_mmu *mmu; 905 906 trace_kvm_access_fault(fault_ipa); 907 908 spin_lock(&vcpu->kvm->mmu_lock); 909 mmu = vcpu->arch.hw_mmu; 910 kpte = kvm_pgtable_stage2_mkyoung(mmu->pgt, fault_ipa); 911 spin_unlock(&vcpu->kvm->mmu_lock); 912 913 pte = __pte(kpte); 914 if (pte_valid(pte)) 915 kvm_set_pfn_accessed(pte_pfn(pte)); 916 } 917 918 /** 919 * kvm_handle_guest_abort - handles all 2nd stage aborts 920 * @vcpu: the VCPU pointer 921 * 922 * Any abort that gets to the host is almost guaranteed to be caused by a 923 * missing second stage translation table entry, which can mean that either the 924 * guest simply needs more memory and we must allocate an appropriate page or it 925 * can mean that the guest tried to access I/O memory, which is emulated by user 926 * space. The distinction is based on the IPA causing the fault and whether this 927 * memory region has been registered as standard RAM by user space. 928 */ 929 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) 930 { 931 unsigned long fault_status; 932 phys_addr_t fault_ipa; 933 struct kvm_memory_slot *memslot; 934 unsigned long hva; 935 bool is_iabt, write_fault, writable; 936 gfn_t gfn; 937 int ret, idx; 938 939 fault_status = kvm_vcpu_trap_get_fault_type(vcpu); 940 941 fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); 942 is_iabt = kvm_vcpu_trap_is_iabt(vcpu); 943 944 /* Synchronous External Abort? */ 945 if (kvm_vcpu_abt_issea(vcpu)) { 946 /* 947 * For RAS the host kernel may handle this abort. 948 * There is no need to pass the error into the guest. 949 */ 950 if (kvm_handle_guest_sea(fault_ipa, kvm_vcpu_get_esr(vcpu))) 951 kvm_inject_vabt(vcpu); 952 953 return 1; 954 } 955 956 trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu), 957 kvm_vcpu_get_hfar(vcpu), fault_ipa); 958 959 /* Check the stage-2 fault is trans. fault or write fault */ 960 if (fault_status != FSC_FAULT && fault_status != FSC_PERM && 961 fault_status != FSC_ACCESS) { 962 kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", 963 kvm_vcpu_trap_get_class(vcpu), 964 (unsigned long)kvm_vcpu_trap_get_fault(vcpu), 965 (unsigned long)kvm_vcpu_get_esr(vcpu)); 966 return -EFAULT; 967 } 968 969 idx = srcu_read_lock(&vcpu->kvm->srcu); 970 971 gfn = fault_ipa >> PAGE_SHIFT; 972 memslot = gfn_to_memslot(vcpu->kvm, gfn); 973 hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); 974 write_fault = kvm_is_write_fault(vcpu); 975 if (kvm_is_error_hva(hva) || (write_fault && !writable)) { 976 /* 977 * The guest has put either its instructions or its page-tables 978 * somewhere it shouldn't have. Userspace won't be able to do 979 * anything about this (there's no syndrome for a start), so 980 * re-inject the abort back into the guest. 981 */ 982 if (is_iabt) { 983 ret = -ENOEXEC; 984 goto out; 985 } 986 987 if (kvm_vcpu_abt_iss1tw(vcpu)) { 988 kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); 989 ret = 1; 990 goto out_unlock; 991 } 992 993 /* 994 * Check for a cache maintenance operation. Since we 995 * ended-up here, we know it is outside of any memory 996 * slot. But we can't find out if that is for a device, 997 * or if the guest is just being stupid. The only thing 998 * we know for sure is that this range cannot be cached. 999 * 1000 * So let's assume that the guest is just being 1001 * cautious, and skip the instruction. 1002 */ 1003 if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) { 1004 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); 1005 ret = 1; 1006 goto out_unlock; 1007 } 1008 1009 /* 1010 * The IPA is reported as [MAX:12], so we need to 1011 * complement it with the bottom 12 bits from the 1012 * faulting VA. This is always 12 bits, irrespective 1013 * of the page size. 1014 */ 1015 fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1); 1016 ret = io_mem_abort(vcpu, fault_ipa); 1017 goto out_unlock; 1018 } 1019 1020 /* Userspace should not be able to register out-of-bounds IPAs */ 1021 VM_BUG_ON(fault_ipa >= kvm_phys_size(vcpu->kvm)); 1022 1023 if (fault_status == FSC_ACCESS) { 1024 handle_access_fault(vcpu, fault_ipa); 1025 ret = 1; 1026 goto out_unlock; 1027 } 1028 1029 ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status); 1030 if (ret == 0) 1031 ret = 1; 1032 out: 1033 if (ret == -ENOEXEC) { 1034 kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu)); 1035 ret = 1; 1036 } 1037 out_unlock: 1038 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1039 return ret; 1040 } 1041 1042 static int handle_hva_to_gpa(struct kvm *kvm, 1043 unsigned long start, 1044 unsigned long end, 1045 int (*handler)(struct kvm *kvm, 1046 gpa_t gpa, u64 size, 1047 void *data), 1048 void *data) 1049 { 1050 struct kvm_memslots *slots; 1051 struct kvm_memory_slot *memslot; 1052 int ret = 0; 1053 1054 slots = kvm_memslots(kvm); 1055 1056 /* we only care about the pages that the guest sees */ 1057 kvm_for_each_memslot(memslot, slots) { 1058 unsigned long hva_start, hva_end; 1059 gfn_t gpa; 1060 1061 hva_start = max(start, memslot->userspace_addr); 1062 hva_end = min(end, memslot->userspace_addr + 1063 (memslot->npages << PAGE_SHIFT)); 1064 if (hva_start >= hva_end) 1065 continue; 1066 1067 gpa = hva_to_gfn_memslot(hva_start, memslot) << PAGE_SHIFT; 1068 ret |= handler(kvm, gpa, (u64)(hva_end - hva_start), data); 1069 } 1070 1071 return ret; 1072 } 1073 1074 static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) 1075 { 1076 unsigned flags = *(unsigned *)data; 1077 bool may_block = flags & MMU_NOTIFIER_RANGE_BLOCKABLE; 1078 1079 __unmap_stage2_range(&kvm->arch.mmu, gpa, size, may_block); 1080 return 0; 1081 } 1082 1083 int kvm_unmap_hva_range(struct kvm *kvm, 1084 unsigned long start, unsigned long end, unsigned flags) 1085 { 1086 if (!kvm->arch.mmu.pgt) 1087 return 0; 1088 1089 trace_kvm_unmap_hva_range(start, end); 1090 handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, &flags); 1091 return 0; 1092 } 1093 1094 static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) 1095 { 1096 kvm_pfn_t *pfn = (kvm_pfn_t *)data; 1097 1098 WARN_ON(size != PAGE_SIZE); 1099 1100 /* 1101 * The MMU notifiers will have unmapped a huge PMD before calling 1102 * ->change_pte() (which in turn calls kvm_set_spte_hva()) and 1103 * therefore we never need to clear out a huge PMD through this 1104 * calling path and a memcache is not required. 1105 */ 1106 kvm_pgtable_stage2_map(kvm->arch.mmu.pgt, gpa, PAGE_SIZE, 1107 __pfn_to_phys(*pfn), KVM_PGTABLE_PROT_R, NULL); 1108 return 0; 1109 } 1110 1111 int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) 1112 { 1113 unsigned long end = hva + PAGE_SIZE; 1114 kvm_pfn_t pfn = pte_pfn(pte); 1115 1116 if (!kvm->arch.mmu.pgt) 1117 return 0; 1118 1119 trace_kvm_set_spte_hva(hva); 1120 1121 /* 1122 * We've moved a page around, probably through CoW, so let's treat it 1123 * just like a translation fault and clean the cache to the PoC. 1124 */ 1125 clean_dcache_guest_page(pfn, PAGE_SIZE); 1126 handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &pfn); 1127 return 0; 1128 } 1129 1130 static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) 1131 { 1132 pte_t pte; 1133 kvm_pte_t kpte; 1134 1135 WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); 1136 kpte = kvm_pgtable_stage2_mkold(kvm->arch.mmu.pgt, gpa); 1137 pte = __pte(kpte); 1138 return pte_valid(pte) && pte_young(pte); 1139 } 1140 1141 static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) 1142 { 1143 WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); 1144 return kvm_pgtable_stage2_is_young(kvm->arch.mmu.pgt, gpa); 1145 } 1146 1147 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) 1148 { 1149 if (!kvm->arch.mmu.pgt) 1150 return 0; 1151 trace_kvm_age_hva(start, end); 1152 return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); 1153 } 1154 1155 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) 1156 { 1157 if (!kvm->arch.mmu.pgt) 1158 return 0; 1159 trace_kvm_test_age_hva(hva); 1160 return handle_hva_to_gpa(kvm, hva, hva + PAGE_SIZE, 1161 kvm_test_age_hva_handler, NULL); 1162 } 1163 1164 phys_addr_t kvm_mmu_get_httbr(void) 1165 { 1166 return __pa(hyp_pgtable->pgd); 1167 } 1168 1169 phys_addr_t kvm_get_idmap_vector(void) 1170 { 1171 return hyp_idmap_vector; 1172 } 1173 1174 static int kvm_map_idmap_text(void) 1175 { 1176 unsigned long size = hyp_idmap_end - hyp_idmap_start; 1177 int err = __create_hyp_mappings(hyp_idmap_start, size, hyp_idmap_start, 1178 PAGE_HYP_EXEC); 1179 if (err) 1180 kvm_err("Failed to idmap %lx-%lx\n", 1181 hyp_idmap_start, hyp_idmap_end); 1182 1183 return err; 1184 } 1185 1186 int kvm_mmu_init(void) 1187 { 1188 int err; 1189 u32 hyp_va_bits; 1190 1191 hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start); 1192 hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE); 1193 hyp_idmap_end = __pa_symbol(__hyp_idmap_text_end); 1194 hyp_idmap_end = ALIGN(hyp_idmap_end, PAGE_SIZE); 1195 hyp_idmap_vector = __pa_symbol(__kvm_hyp_init); 1196 1197 /* 1198 * We rely on the linker script to ensure at build time that the HYP 1199 * init code does not cross a page boundary. 1200 */ 1201 BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK); 1202 1203 hyp_va_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET); 1204 kvm_debug("Using %u-bit virtual addresses at EL2\n", hyp_va_bits); 1205 kvm_debug("IDMAP page: %lx\n", hyp_idmap_start); 1206 kvm_debug("HYP VA range: %lx:%lx\n", 1207 kern_hyp_va(PAGE_OFFSET), 1208 kern_hyp_va((unsigned long)high_memory - 1)); 1209 1210 if (hyp_idmap_start >= kern_hyp_va(PAGE_OFFSET) && 1211 hyp_idmap_start < kern_hyp_va((unsigned long)high_memory - 1) && 1212 hyp_idmap_start != (unsigned long)__hyp_idmap_text_start) { 1213 /* 1214 * The idmap page is intersecting with the VA space, 1215 * it is not safe to continue further. 1216 */ 1217 kvm_err("IDMAP intersecting with HYP VA, unable to continue\n"); 1218 err = -EINVAL; 1219 goto out; 1220 } 1221 1222 hyp_pgtable = kzalloc(sizeof(*hyp_pgtable), GFP_KERNEL); 1223 if (!hyp_pgtable) { 1224 kvm_err("Hyp mode page-table not allocated\n"); 1225 err = -ENOMEM; 1226 goto out; 1227 } 1228 1229 err = kvm_pgtable_hyp_init(hyp_pgtable, hyp_va_bits); 1230 if (err) 1231 goto out_free_pgtable; 1232 1233 err = kvm_map_idmap_text(); 1234 if (err) 1235 goto out_destroy_pgtable; 1236 1237 io_map_base = hyp_idmap_start; 1238 return 0; 1239 1240 out_destroy_pgtable: 1241 kvm_pgtable_hyp_destroy(hyp_pgtable); 1242 out_free_pgtable: 1243 kfree(hyp_pgtable); 1244 hyp_pgtable = NULL; 1245 out: 1246 return err; 1247 } 1248 1249 void kvm_arch_commit_memory_region(struct kvm *kvm, 1250 const struct kvm_userspace_memory_region *mem, 1251 struct kvm_memory_slot *old, 1252 const struct kvm_memory_slot *new, 1253 enum kvm_mr_change change) 1254 { 1255 /* 1256 * At this point memslot has been committed and there is an 1257 * allocated dirty_bitmap[], dirty pages will be tracked while the 1258 * memory slot is write protected. 1259 */ 1260 if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { 1261 /* 1262 * If we're with initial-all-set, we don't need to write 1263 * protect any pages because they're all reported as dirty. 1264 * Huge pages and normal pages will be write protect gradually. 1265 */ 1266 if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) { 1267 kvm_mmu_wp_memory_region(kvm, mem->slot); 1268 } 1269 } 1270 } 1271 1272 int kvm_arch_prepare_memory_region(struct kvm *kvm, 1273 struct kvm_memory_slot *memslot, 1274 const struct kvm_userspace_memory_region *mem, 1275 enum kvm_mr_change change) 1276 { 1277 hva_t hva = mem->userspace_addr; 1278 hva_t reg_end = hva + mem->memory_size; 1279 bool writable = !(mem->flags & KVM_MEM_READONLY); 1280 int ret = 0; 1281 1282 if (change != KVM_MR_CREATE && change != KVM_MR_MOVE && 1283 change != KVM_MR_FLAGS_ONLY) 1284 return 0; 1285 1286 /* 1287 * Prevent userspace from creating a memory region outside of the IPA 1288 * space addressable by the KVM guest IPA space. 1289 */ 1290 if (memslot->base_gfn + memslot->npages >= 1291 (kvm_phys_size(kvm) >> PAGE_SHIFT)) 1292 return -EFAULT; 1293 1294 mmap_read_lock(current->mm); 1295 /* 1296 * A memory region could potentially cover multiple VMAs, and any holes 1297 * between them, so iterate over all of them to find out if we can map 1298 * any of them right now. 1299 * 1300 * +--------------------------------------------+ 1301 * +---------------+----------------+ +----------------+ 1302 * | : VMA 1 | VMA 2 | | VMA 3 : | 1303 * +---------------+----------------+ +----------------+ 1304 * | memory region | 1305 * +--------------------------------------------+ 1306 */ 1307 do { 1308 struct vm_area_struct *vma = find_vma(current->mm, hva); 1309 hva_t vm_start, vm_end; 1310 1311 if (!vma || vma->vm_start >= reg_end) 1312 break; 1313 1314 /* 1315 * Take the intersection of this VMA with the memory region 1316 */ 1317 vm_start = max(hva, vma->vm_start); 1318 vm_end = min(reg_end, vma->vm_end); 1319 1320 if (vma->vm_flags & VM_PFNMAP) { 1321 gpa_t gpa = mem->guest_phys_addr + 1322 (vm_start - mem->userspace_addr); 1323 phys_addr_t pa; 1324 1325 pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT; 1326 pa += vm_start - vma->vm_start; 1327 1328 /* IO region dirty page logging not allowed */ 1329 if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES) { 1330 ret = -EINVAL; 1331 goto out; 1332 } 1333 1334 ret = kvm_phys_addr_ioremap(kvm, gpa, pa, 1335 vm_end - vm_start, 1336 writable); 1337 if (ret) 1338 break; 1339 } 1340 hva = vm_end; 1341 } while (hva < reg_end); 1342 1343 if (change == KVM_MR_FLAGS_ONLY) 1344 goto out; 1345 1346 spin_lock(&kvm->mmu_lock); 1347 if (ret) 1348 unmap_stage2_range(&kvm->arch.mmu, mem->guest_phys_addr, mem->memory_size); 1349 else if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 1350 stage2_flush_memslot(kvm, memslot); 1351 spin_unlock(&kvm->mmu_lock); 1352 out: 1353 mmap_read_unlock(current->mm); 1354 return ret; 1355 } 1356 1357 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) 1358 { 1359 } 1360 1361 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) 1362 { 1363 } 1364 1365 void kvm_arch_flush_shadow_all(struct kvm *kvm) 1366 { 1367 kvm_free_stage2_pgd(&kvm->arch.mmu); 1368 } 1369 1370 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 1371 struct kvm_memory_slot *slot) 1372 { 1373 gpa_t gpa = slot->base_gfn << PAGE_SHIFT; 1374 phys_addr_t size = slot->npages << PAGE_SHIFT; 1375 1376 spin_lock(&kvm->mmu_lock); 1377 unmap_stage2_range(&kvm->arch.mmu, gpa, size); 1378 spin_unlock(&kvm->mmu_lock); 1379 } 1380 1381 /* 1382 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized). 1383 * 1384 * Main problems: 1385 * - S/W ops are local to a CPU (not broadcast) 1386 * - We have line migration behind our back (speculation) 1387 * - System caches don't support S/W at all (damn!) 1388 * 1389 * In the face of the above, the best we can do is to try and convert 1390 * S/W ops to VA ops. Because the guest is not allowed to infer the 1391 * S/W to PA mapping, it can only use S/W to nuke the whole cache, 1392 * which is a rather good thing for us. 1393 * 1394 * Also, it is only used when turning caches on/off ("The expected 1395 * usage of the cache maintenance instructions that operate by set/way 1396 * is associated with the cache maintenance instructions associated 1397 * with the powerdown and powerup of caches, if this is required by 1398 * the implementation."). 1399 * 1400 * We use the following policy: 1401 * 1402 * - If we trap a S/W operation, we enable VM trapping to detect 1403 * caches being turned on/off, and do a full clean. 1404 * 1405 * - We flush the caches on both caches being turned on and off. 1406 * 1407 * - Once the caches are enabled, we stop trapping VM ops. 1408 */ 1409 void kvm_set_way_flush(struct kvm_vcpu *vcpu) 1410 { 1411 unsigned long hcr = *vcpu_hcr(vcpu); 1412 1413 /* 1414 * If this is the first time we do a S/W operation 1415 * (i.e. HCR_TVM not set) flush the whole memory, and set the 1416 * VM trapping. 1417 * 1418 * Otherwise, rely on the VM trapping to wait for the MMU + 1419 * Caches to be turned off. At that point, we'll be able to 1420 * clean the caches again. 1421 */ 1422 if (!(hcr & HCR_TVM)) { 1423 trace_kvm_set_way_flush(*vcpu_pc(vcpu), 1424 vcpu_has_cache_enabled(vcpu)); 1425 stage2_flush_vm(vcpu->kvm); 1426 *vcpu_hcr(vcpu) = hcr | HCR_TVM; 1427 } 1428 } 1429 1430 void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) 1431 { 1432 bool now_enabled = vcpu_has_cache_enabled(vcpu); 1433 1434 /* 1435 * If switching the MMU+caches on, need to invalidate the caches. 1436 * If switching it off, need to clean the caches. 1437 * Clean + invalidate does the trick always. 1438 */ 1439 if (now_enabled != was_enabled) 1440 stage2_flush_vm(vcpu->kvm); 1441 1442 /* Caches are now on, stop trapping VM ops (until a S/W op) */ 1443 if (now_enabled) 1444 *vcpu_hcr(vcpu) &= ~HCR_TVM; 1445 1446 trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); 1447 } 1448