1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 4 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 5 */ 6 7 #include <linux/bug.h> 8 #include <linux/cpu_pm.h> 9 #include <linux/errno.h> 10 #include <linux/err.h> 11 #include <linux/kvm_host.h> 12 #include <linux/list.h> 13 #include <linux/module.h> 14 #include <linux/vmalloc.h> 15 #include <linux/fs.h> 16 #include <linux/mman.h> 17 #include <linux/sched.h> 18 #include <linux/kvm.h> 19 #include <linux/kvm_irqfd.h> 20 #include <linux/irqbypass.h> 21 #include <linux/sched/stat.h> 22 #include <linux/psci.h> 23 #include <trace/events/kvm.h> 24 25 #define CREATE_TRACE_POINTS 26 #include "trace_arm.h" 27 28 #include <linux/uaccess.h> 29 #include <asm/ptrace.h> 30 #include <asm/mman.h> 31 #include <asm/tlbflush.h> 32 #include <asm/cacheflush.h> 33 #include <asm/cpufeature.h> 34 #include <asm/virt.h> 35 #include <asm/kvm_arm.h> 36 #include <asm/kvm_asm.h> 37 #include <asm/kvm_mmu.h> 38 #include <asm/kvm_emulate.h> 39 #include <asm/sections.h> 40 41 #include <kvm/arm_hypercalls.h> 42 #include <kvm/arm_pmu.h> 43 #include <kvm/arm_psci.h> 44 45 #ifdef REQUIRES_VIRT 46 __asm__(".arch_extension virt"); 47 #endif 48 49 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT; 50 DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized); 51 52 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector); 53 54 static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page); 55 unsigned long kvm_arm_hyp_percpu_base[NR_CPUS]; 56 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); 57 58 /* The VMID used in the VTTBR */ 59 static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1); 60 static u32 kvm_next_vmid; 61 static DEFINE_SPINLOCK(kvm_vmid_lock); 62 63 static bool vgic_present; 64 65 static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled); 66 DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use); 67 68 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 69 { 70 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 71 } 72 73 int kvm_arch_hardware_setup(void *opaque) 74 { 75 return 0; 76 } 77 78 int kvm_arch_check_processor_compat(void *opaque) 79 { 80 return 0; 81 } 82 83 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 84 struct kvm_enable_cap *cap) 85 { 86 int r; 87 88 if (cap->flags) 89 return -EINVAL; 90 91 switch (cap->cap) { 92 case KVM_CAP_ARM_NISV_TO_USER: 93 r = 0; 94 kvm->arch.return_nisv_io_abort_to_user = true; 95 break; 96 default: 97 r = -EINVAL; 98 break; 99 } 100 101 return r; 102 } 103 104 static int kvm_arm_default_max_vcpus(void) 105 { 106 return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; 107 } 108 109 static void set_default_spectre(struct kvm *kvm) 110 { 111 /* 112 * The default is to expose CSV2 == 1 if the HW isn't affected. 113 * Although this is a per-CPU feature, we make it global because 114 * asymmetric systems are just a nuisance. 115 * 116 * Userspace can override this as long as it doesn't promise 117 * the impossible. 118 */ 119 if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED) 120 kvm->arch.pfr0_csv2 = 1; 121 if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED) 122 kvm->arch.pfr0_csv3 = 1; 123 } 124 125 /** 126 * kvm_arch_init_vm - initializes a VM data structure 127 * @kvm: pointer to the KVM struct 128 */ 129 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 130 { 131 int ret; 132 133 ret = kvm_arm_setup_stage2(kvm, type); 134 if (ret) 135 return ret; 136 137 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu); 138 if (ret) 139 return ret; 140 141 ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP); 142 if (ret) 143 goto out_free_stage2_pgd; 144 145 kvm_vgic_early_init(kvm); 146 147 /* The maximum number of VCPUs is limited by the host's GIC model */ 148 kvm->arch.max_vcpus = kvm_arm_default_max_vcpus(); 149 150 set_default_spectre(kvm); 151 152 return ret; 153 out_free_stage2_pgd: 154 kvm_free_stage2_pgd(&kvm->arch.mmu); 155 return ret; 156 } 157 158 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 159 { 160 return VM_FAULT_SIGBUS; 161 } 162 163 164 /** 165 * kvm_arch_destroy_vm - destroy the VM data structure 166 * @kvm: pointer to the KVM struct 167 */ 168 void kvm_arch_destroy_vm(struct kvm *kvm) 169 { 170 int i; 171 172 bitmap_free(kvm->arch.pmu_filter); 173 174 kvm_vgic_destroy(kvm); 175 176 for (i = 0; i < KVM_MAX_VCPUS; ++i) { 177 if (kvm->vcpus[i]) { 178 kvm_vcpu_destroy(kvm->vcpus[i]); 179 kvm->vcpus[i] = NULL; 180 } 181 } 182 atomic_set(&kvm->online_vcpus, 0); 183 } 184 185 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 186 { 187 int r; 188 switch (ext) { 189 case KVM_CAP_IRQCHIP: 190 r = vgic_present; 191 break; 192 case KVM_CAP_IOEVENTFD: 193 case KVM_CAP_DEVICE_CTRL: 194 case KVM_CAP_USER_MEMORY: 195 case KVM_CAP_SYNC_MMU: 196 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 197 case KVM_CAP_ONE_REG: 198 case KVM_CAP_ARM_PSCI: 199 case KVM_CAP_ARM_PSCI_0_2: 200 case KVM_CAP_READONLY_MEM: 201 case KVM_CAP_MP_STATE: 202 case KVM_CAP_IMMEDIATE_EXIT: 203 case KVM_CAP_VCPU_EVENTS: 204 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2: 205 case KVM_CAP_ARM_NISV_TO_USER: 206 case KVM_CAP_ARM_INJECT_EXT_DABT: 207 case KVM_CAP_SET_GUEST_DEBUG: 208 case KVM_CAP_VCPU_ATTRIBUTES: 209 r = 1; 210 break; 211 case KVM_CAP_ARM_SET_DEVICE_ADDR: 212 r = 1; 213 break; 214 case KVM_CAP_NR_VCPUS: 215 r = num_online_cpus(); 216 break; 217 case KVM_CAP_MAX_VCPUS: 218 case KVM_CAP_MAX_VCPU_ID: 219 if (kvm) 220 r = kvm->arch.max_vcpus; 221 else 222 r = kvm_arm_default_max_vcpus(); 223 break; 224 case KVM_CAP_MSI_DEVID: 225 if (!kvm) 226 r = -EINVAL; 227 else 228 r = kvm->arch.vgic.msis_require_devid; 229 break; 230 case KVM_CAP_ARM_USER_IRQ: 231 /* 232 * 1: EL1_VTIMER, EL1_PTIMER, and PMU. 233 * (bump this number if adding more devices) 234 */ 235 r = 1; 236 break; 237 case KVM_CAP_STEAL_TIME: 238 r = kvm_arm_pvtime_supported(); 239 break; 240 case KVM_CAP_ARM_EL1_32BIT: 241 r = cpus_have_const_cap(ARM64_HAS_32BIT_EL1); 242 break; 243 case KVM_CAP_GUEST_DEBUG_HW_BPS: 244 r = get_num_brps(); 245 break; 246 case KVM_CAP_GUEST_DEBUG_HW_WPS: 247 r = get_num_wrps(); 248 break; 249 case KVM_CAP_ARM_PMU_V3: 250 r = kvm_arm_support_pmu_v3(); 251 break; 252 case KVM_CAP_ARM_INJECT_SERROR_ESR: 253 r = cpus_have_const_cap(ARM64_HAS_RAS_EXTN); 254 break; 255 case KVM_CAP_ARM_VM_IPA_SIZE: 256 r = get_kvm_ipa_limit(); 257 break; 258 case KVM_CAP_ARM_SVE: 259 r = system_supports_sve(); 260 break; 261 case KVM_CAP_ARM_PTRAUTH_ADDRESS: 262 case KVM_CAP_ARM_PTRAUTH_GENERIC: 263 r = system_has_full_ptr_auth(); 264 break; 265 default: 266 r = 0; 267 } 268 269 return r; 270 } 271 272 long kvm_arch_dev_ioctl(struct file *filp, 273 unsigned int ioctl, unsigned long arg) 274 { 275 return -EINVAL; 276 } 277 278 struct kvm *kvm_arch_alloc_vm(void) 279 { 280 if (!has_vhe()) 281 return kzalloc(sizeof(struct kvm), GFP_KERNEL); 282 283 return vzalloc(sizeof(struct kvm)); 284 } 285 286 void kvm_arch_free_vm(struct kvm *kvm) 287 { 288 if (!has_vhe()) 289 kfree(kvm); 290 else 291 vfree(kvm); 292 } 293 294 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 295 { 296 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) 297 return -EBUSY; 298 299 if (id >= kvm->arch.max_vcpus) 300 return -EINVAL; 301 302 return 0; 303 } 304 305 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 306 { 307 int err; 308 309 /* Force users to call KVM_ARM_VCPU_INIT */ 310 vcpu->arch.target = -1; 311 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES); 312 313 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; 314 315 /* Set up the timer */ 316 kvm_timer_vcpu_init(vcpu); 317 318 kvm_pmu_vcpu_init(vcpu); 319 320 kvm_arm_reset_debug_ptr(vcpu); 321 322 kvm_arm_pvtime_vcpu_init(&vcpu->arch); 323 324 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 325 326 err = kvm_vgic_vcpu_init(vcpu); 327 if (err) 328 return err; 329 330 return create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP); 331 } 332 333 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 334 { 335 } 336 337 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 338 { 339 if (vcpu->arch.has_run_once && unlikely(!irqchip_in_kernel(vcpu->kvm))) 340 static_branch_dec(&userspace_irqchip_in_use); 341 342 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 343 kvm_timer_vcpu_terminate(vcpu); 344 kvm_pmu_vcpu_destroy(vcpu); 345 346 kvm_arm_vcpu_destroy(vcpu); 347 } 348 349 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 350 { 351 return kvm_timer_is_pending(vcpu); 352 } 353 354 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 355 { 356 /* 357 * If we're about to block (most likely because we've just hit a 358 * WFI), we need to sync back the state of the GIC CPU interface 359 * so that we have the latest PMR and group enables. This ensures 360 * that kvm_arch_vcpu_runnable has up-to-date data to decide 361 * whether we have pending interrupts. 362 * 363 * For the same reason, we want to tell GICv4 that we need 364 * doorbells to be signalled, should an interrupt become pending. 365 */ 366 preempt_disable(); 367 kvm_vgic_vmcr_sync(vcpu); 368 vgic_v4_put(vcpu, true); 369 preempt_enable(); 370 } 371 372 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 373 { 374 preempt_disable(); 375 vgic_v4_load(vcpu); 376 preempt_enable(); 377 } 378 379 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 380 { 381 struct kvm_s2_mmu *mmu; 382 int *last_ran; 383 384 mmu = vcpu->arch.hw_mmu; 385 last_ran = this_cpu_ptr(mmu->last_vcpu_ran); 386 387 /* 388 * We might get preempted before the vCPU actually runs, but 389 * over-invalidation doesn't affect correctness. 390 */ 391 if (*last_ran != vcpu->vcpu_id) { 392 kvm_call_hyp(__kvm_tlb_flush_local_vmid, mmu); 393 *last_ran = vcpu->vcpu_id; 394 } 395 396 vcpu->cpu = cpu; 397 398 kvm_vgic_load(vcpu); 399 kvm_timer_vcpu_load(vcpu); 400 if (has_vhe()) 401 kvm_vcpu_load_sysregs_vhe(vcpu); 402 kvm_arch_vcpu_load_fp(vcpu); 403 kvm_vcpu_pmu_restore_guest(vcpu); 404 if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) 405 kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); 406 407 if (single_task_running()) 408 vcpu_clear_wfx_traps(vcpu); 409 else 410 vcpu_set_wfx_traps(vcpu); 411 412 if (vcpu_has_ptrauth(vcpu)) 413 vcpu_ptrauth_disable(vcpu); 414 } 415 416 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 417 { 418 kvm_arch_vcpu_put_fp(vcpu); 419 if (has_vhe()) 420 kvm_vcpu_put_sysregs_vhe(vcpu); 421 kvm_timer_vcpu_put(vcpu); 422 kvm_vgic_put(vcpu); 423 kvm_vcpu_pmu_restore_host(vcpu); 424 425 vcpu->cpu = -1; 426 } 427 428 static void vcpu_power_off(struct kvm_vcpu *vcpu) 429 { 430 vcpu->arch.power_off = true; 431 kvm_make_request(KVM_REQ_SLEEP, vcpu); 432 kvm_vcpu_kick(vcpu); 433 } 434 435 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 436 struct kvm_mp_state *mp_state) 437 { 438 if (vcpu->arch.power_off) 439 mp_state->mp_state = KVM_MP_STATE_STOPPED; 440 else 441 mp_state->mp_state = KVM_MP_STATE_RUNNABLE; 442 443 return 0; 444 } 445 446 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 447 struct kvm_mp_state *mp_state) 448 { 449 int ret = 0; 450 451 switch (mp_state->mp_state) { 452 case KVM_MP_STATE_RUNNABLE: 453 vcpu->arch.power_off = false; 454 break; 455 case KVM_MP_STATE_STOPPED: 456 vcpu_power_off(vcpu); 457 break; 458 default: 459 ret = -EINVAL; 460 } 461 462 return ret; 463 } 464 465 /** 466 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled 467 * @v: The VCPU pointer 468 * 469 * If the guest CPU is not waiting for interrupts or an interrupt line is 470 * asserted, the CPU is by definition runnable. 471 */ 472 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) 473 { 474 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF); 475 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) 476 && !v->arch.power_off && !v->arch.pause); 477 } 478 479 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 480 { 481 return vcpu_mode_priv(vcpu); 482 } 483 484 /* Just ensure a guest exit from a particular CPU */ 485 static void exit_vm_noop(void *info) 486 { 487 } 488 489 void force_vm_exit(const cpumask_t *mask) 490 { 491 preempt_disable(); 492 smp_call_function_many(mask, exit_vm_noop, NULL, true); 493 preempt_enable(); 494 } 495 496 /** 497 * need_new_vmid_gen - check that the VMID is still valid 498 * @vmid: The VMID to check 499 * 500 * return true if there is a new generation of VMIDs being used 501 * 502 * The hardware supports a limited set of values with the value zero reserved 503 * for the host, so we check if an assigned value belongs to a previous 504 * generation, which requires us to assign a new value. If we're the first to 505 * use a VMID for the new generation, we must flush necessary caches and TLBs 506 * on all CPUs. 507 */ 508 static bool need_new_vmid_gen(struct kvm_vmid *vmid) 509 { 510 u64 current_vmid_gen = atomic64_read(&kvm_vmid_gen); 511 smp_rmb(); /* Orders read of kvm_vmid_gen and kvm->arch.vmid */ 512 return unlikely(READ_ONCE(vmid->vmid_gen) != current_vmid_gen); 513 } 514 515 /** 516 * update_vmid - Update the vmid with a valid VMID for the current generation 517 * @vmid: The stage-2 VMID information struct 518 */ 519 static void update_vmid(struct kvm_vmid *vmid) 520 { 521 if (!need_new_vmid_gen(vmid)) 522 return; 523 524 spin_lock(&kvm_vmid_lock); 525 526 /* 527 * We need to re-check the vmid_gen here to ensure that if another vcpu 528 * already allocated a valid vmid for this vm, then this vcpu should 529 * use the same vmid. 530 */ 531 if (!need_new_vmid_gen(vmid)) { 532 spin_unlock(&kvm_vmid_lock); 533 return; 534 } 535 536 /* First user of a new VMID generation? */ 537 if (unlikely(kvm_next_vmid == 0)) { 538 atomic64_inc(&kvm_vmid_gen); 539 kvm_next_vmid = 1; 540 541 /* 542 * On SMP we know no other CPUs can use this CPU's or each 543 * other's VMID after force_vm_exit returns since the 544 * kvm_vmid_lock blocks them from reentry to the guest. 545 */ 546 force_vm_exit(cpu_all_mask); 547 /* 548 * Now broadcast TLB + ICACHE invalidation over the inner 549 * shareable domain to make sure all data structures are 550 * clean. 551 */ 552 kvm_call_hyp(__kvm_flush_vm_context); 553 } 554 555 vmid->vmid = kvm_next_vmid; 556 kvm_next_vmid++; 557 kvm_next_vmid &= (1 << kvm_get_vmid_bits()) - 1; 558 559 smp_wmb(); 560 WRITE_ONCE(vmid->vmid_gen, atomic64_read(&kvm_vmid_gen)); 561 562 spin_unlock(&kvm_vmid_lock); 563 } 564 565 static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) 566 { 567 struct kvm *kvm = vcpu->kvm; 568 int ret = 0; 569 570 if (likely(vcpu->arch.has_run_once)) 571 return 0; 572 573 if (!kvm_arm_vcpu_is_finalized(vcpu)) 574 return -EPERM; 575 576 vcpu->arch.has_run_once = true; 577 578 if (likely(irqchip_in_kernel(kvm))) { 579 /* 580 * Map the VGIC hardware resources before running a vcpu the 581 * first time on this VM. 582 */ 583 ret = kvm_vgic_map_resources(kvm); 584 if (ret) 585 return ret; 586 } else { 587 /* 588 * Tell the rest of the code that there are userspace irqchip 589 * VMs in the wild. 590 */ 591 static_branch_inc(&userspace_irqchip_in_use); 592 } 593 594 ret = kvm_timer_enable(vcpu); 595 if (ret) 596 return ret; 597 598 ret = kvm_arm_pmu_v3_enable(vcpu); 599 600 return ret; 601 } 602 603 bool kvm_arch_intc_initialized(struct kvm *kvm) 604 { 605 return vgic_initialized(kvm); 606 } 607 608 void kvm_arm_halt_guest(struct kvm *kvm) 609 { 610 int i; 611 struct kvm_vcpu *vcpu; 612 613 kvm_for_each_vcpu(i, vcpu, kvm) 614 vcpu->arch.pause = true; 615 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP); 616 } 617 618 void kvm_arm_resume_guest(struct kvm *kvm) 619 { 620 int i; 621 struct kvm_vcpu *vcpu; 622 623 kvm_for_each_vcpu(i, vcpu, kvm) { 624 vcpu->arch.pause = false; 625 rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu)); 626 } 627 } 628 629 static void vcpu_req_sleep(struct kvm_vcpu *vcpu) 630 { 631 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); 632 633 rcuwait_wait_event(wait, 634 (!vcpu->arch.power_off) &&(!vcpu->arch.pause), 635 TASK_INTERRUPTIBLE); 636 637 if (vcpu->arch.power_off || vcpu->arch.pause) { 638 /* Awaken to handle a signal, request we sleep again later. */ 639 kvm_make_request(KVM_REQ_SLEEP, vcpu); 640 } 641 642 /* 643 * Make sure we will observe a potential reset request if we've 644 * observed a change to the power state. Pairs with the smp_wmb() in 645 * kvm_psci_vcpu_on(). 646 */ 647 smp_rmb(); 648 } 649 650 static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu) 651 { 652 return vcpu->arch.target >= 0; 653 } 654 655 static void check_vcpu_requests(struct kvm_vcpu *vcpu) 656 { 657 if (kvm_request_pending(vcpu)) { 658 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) 659 vcpu_req_sleep(vcpu); 660 661 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 662 kvm_reset_vcpu(vcpu); 663 664 /* 665 * Clear IRQ_PENDING requests that were made to guarantee 666 * that a VCPU sees new virtual interrupts. 667 */ 668 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu); 669 670 if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) 671 kvm_update_stolen_time(vcpu); 672 673 if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) { 674 /* The distributor enable bits were changed */ 675 preempt_disable(); 676 vgic_v4_put(vcpu, false); 677 vgic_v4_load(vcpu); 678 preempt_enable(); 679 } 680 } 681 } 682 683 /** 684 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code 685 * @vcpu: The VCPU pointer 686 * 687 * This function is called through the VCPU_RUN ioctl called from user space. It 688 * will execute VM code in a loop until the time slice for the process is used 689 * or some emulation is needed from user space in which case the function will 690 * return with return value 0 and with the kvm_run structure filled in with the 691 * required data for the requested emulation. 692 */ 693 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 694 { 695 struct kvm_run *run = vcpu->run; 696 int ret; 697 698 if (unlikely(!kvm_vcpu_initialized(vcpu))) 699 return -ENOEXEC; 700 701 ret = kvm_vcpu_first_run_init(vcpu); 702 if (ret) 703 return ret; 704 705 if (run->exit_reason == KVM_EXIT_MMIO) { 706 ret = kvm_handle_mmio_return(vcpu); 707 if (ret) 708 return ret; 709 } 710 711 if (run->immediate_exit) 712 return -EINTR; 713 714 vcpu_load(vcpu); 715 716 kvm_sigset_activate(vcpu); 717 718 ret = 1; 719 run->exit_reason = KVM_EXIT_UNKNOWN; 720 while (ret > 0) { 721 /* 722 * Check conditions before entering the guest 723 */ 724 cond_resched(); 725 726 update_vmid(&vcpu->arch.hw_mmu->vmid); 727 728 check_vcpu_requests(vcpu); 729 730 /* 731 * Preparing the interrupts to be injected also 732 * involves poking the GIC, which must be done in a 733 * non-preemptible context. 734 */ 735 preempt_disable(); 736 737 kvm_pmu_flush_hwstate(vcpu); 738 739 local_irq_disable(); 740 741 kvm_vgic_flush_hwstate(vcpu); 742 743 /* 744 * Exit if we have a signal pending so that we can deliver the 745 * signal to user space. 746 */ 747 if (signal_pending(current)) { 748 ret = -EINTR; 749 run->exit_reason = KVM_EXIT_INTR; 750 } 751 752 /* 753 * If we're using a userspace irqchip, then check if we need 754 * to tell a userspace irqchip about timer or PMU level 755 * changes and if so, exit to userspace (the actual level 756 * state gets updated in kvm_timer_update_run and 757 * kvm_pmu_update_run below). 758 */ 759 if (static_branch_unlikely(&userspace_irqchip_in_use)) { 760 if (kvm_timer_should_notify_user(vcpu) || 761 kvm_pmu_should_notify_user(vcpu)) { 762 ret = -EINTR; 763 run->exit_reason = KVM_EXIT_INTR; 764 } 765 } 766 767 /* 768 * Ensure we set mode to IN_GUEST_MODE after we disable 769 * interrupts and before the final VCPU requests check. 770 * See the comment in kvm_vcpu_exiting_guest_mode() and 771 * Documentation/virt/kvm/vcpu-requests.rst 772 */ 773 smp_store_mb(vcpu->mode, IN_GUEST_MODE); 774 775 if (ret <= 0 || need_new_vmid_gen(&vcpu->arch.hw_mmu->vmid) || 776 kvm_request_pending(vcpu)) { 777 vcpu->mode = OUTSIDE_GUEST_MODE; 778 isb(); /* Ensure work in x_flush_hwstate is committed */ 779 kvm_pmu_sync_hwstate(vcpu); 780 if (static_branch_unlikely(&userspace_irqchip_in_use)) 781 kvm_timer_sync_user(vcpu); 782 kvm_vgic_sync_hwstate(vcpu); 783 local_irq_enable(); 784 preempt_enable(); 785 continue; 786 } 787 788 kvm_arm_setup_debug(vcpu); 789 790 /************************************************************** 791 * Enter the guest 792 */ 793 trace_kvm_entry(*vcpu_pc(vcpu)); 794 guest_enter_irqoff(); 795 796 ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu); 797 798 vcpu->mode = OUTSIDE_GUEST_MODE; 799 vcpu->stat.exits++; 800 /* 801 * Back from guest 802 *************************************************************/ 803 804 kvm_arm_clear_debug(vcpu); 805 806 /* 807 * We must sync the PMU state before the vgic state so 808 * that the vgic can properly sample the updated state of the 809 * interrupt line. 810 */ 811 kvm_pmu_sync_hwstate(vcpu); 812 813 /* 814 * Sync the vgic state before syncing the timer state because 815 * the timer code needs to know if the virtual timer 816 * interrupts are active. 817 */ 818 kvm_vgic_sync_hwstate(vcpu); 819 820 /* 821 * Sync the timer hardware state before enabling interrupts as 822 * we don't want vtimer interrupts to race with syncing the 823 * timer virtual interrupt state. 824 */ 825 if (static_branch_unlikely(&userspace_irqchip_in_use)) 826 kvm_timer_sync_user(vcpu); 827 828 kvm_arch_vcpu_ctxsync_fp(vcpu); 829 830 /* 831 * We may have taken a host interrupt in HYP mode (ie 832 * while executing the guest). This interrupt is still 833 * pending, as we haven't serviced it yet! 834 * 835 * We're now back in SVC mode, with interrupts 836 * disabled. Enabling the interrupts now will have 837 * the effect of taking the interrupt again, in SVC 838 * mode this time. 839 */ 840 local_irq_enable(); 841 842 /* 843 * We do local_irq_enable() before calling guest_exit() so 844 * that if a timer interrupt hits while running the guest we 845 * account that tick as being spent in the guest. We enable 846 * preemption after calling guest_exit() so that if we get 847 * preempted we make sure ticks after that is not counted as 848 * guest time. 849 */ 850 guest_exit(); 851 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); 852 853 /* Exit types that need handling before we can be preempted */ 854 handle_exit_early(vcpu, ret); 855 856 preempt_enable(); 857 858 /* 859 * The ARMv8 architecture doesn't give the hypervisor 860 * a mechanism to prevent a guest from dropping to AArch32 EL0 861 * if implemented by the CPU. If we spot the guest in such 862 * state and that we decided it wasn't supposed to do so (like 863 * with the asymmetric AArch32 case), return to userspace with 864 * a fatal error. 865 */ 866 if (!system_supports_32bit_el0() && vcpu_mode_is_32bit(vcpu)) { 867 /* 868 * As we have caught the guest red-handed, decide that 869 * it isn't fit for purpose anymore by making the vcpu 870 * invalid. The VMM can try and fix it by issuing a 871 * KVM_ARM_VCPU_INIT if it really wants to. 872 */ 873 vcpu->arch.target = -1; 874 ret = ARM_EXCEPTION_IL; 875 } 876 877 ret = handle_exit(vcpu, ret); 878 } 879 880 /* Tell userspace about in-kernel device output levels */ 881 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) { 882 kvm_timer_update_run(vcpu); 883 kvm_pmu_update_run(vcpu); 884 } 885 886 kvm_sigset_deactivate(vcpu); 887 888 vcpu_put(vcpu); 889 return ret; 890 } 891 892 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) 893 { 894 int bit_index; 895 bool set; 896 unsigned long *hcr; 897 898 if (number == KVM_ARM_IRQ_CPU_IRQ) 899 bit_index = __ffs(HCR_VI); 900 else /* KVM_ARM_IRQ_CPU_FIQ */ 901 bit_index = __ffs(HCR_VF); 902 903 hcr = vcpu_hcr(vcpu); 904 if (level) 905 set = test_and_set_bit(bit_index, hcr); 906 else 907 set = test_and_clear_bit(bit_index, hcr); 908 909 /* 910 * If we didn't change anything, no need to wake up or kick other CPUs 911 */ 912 if (set == level) 913 return 0; 914 915 /* 916 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and 917 * trigger a world-switch round on the running physical CPU to set the 918 * virtual IRQ/FIQ fields in the HCR appropriately. 919 */ 920 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); 921 kvm_vcpu_kick(vcpu); 922 923 return 0; 924 } 925 926 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 927 bool line_status) 928 { 929 u32 irq = irq_level->irq; 930 unsigned int irq_type, vcpu_idx, irq_num; 931 int nrcpus = atomic_read(&kvm->online_vcpus); 932 struct kvm_vcpu *vcpu = NULL; 933 bool level = irq_level->level; 934 935 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK; 936 vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK; 937 vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1); 938 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK; 939 940 trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level); 941 942 switch (irq_type) { 943 case KVM_ARM_IRQ_TYPE_CPU: 944 if (irqchip_in_kernel(kvm)) 945 return -ENXIO; 946 947 if (vcpu_idx >= nrcpus) 948 return -EINVAL; 949 950 vcpu = kvm_get_vcpu(kvm, vcpu_idx); 951 if (!vcpu) 952 return -EINVAL; 953 954 if (irq_num > KVM_ARM_IRQ_CPU_FIQ) 955 return -EINVAL; 956 957 return vcpu_interrupt_line(vcpu, irq_num, level); 958 case KVM_ARM_IRQ_TYPE_PPI: 959 if (!irqchip_in_kernel(kvm)) 960 return -ENXIO; 961 962 if (vcpu_idx >= nrcpus) 963 return -EINVAL; 964 965 vcpu = kvm_get_vcpu(kvm, vcpu_idx); 966 if (!vcpu) 967 return -EINVAL; 968 969 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS) 970 return -EINVAL; 971 972 return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL); 973 case KVM_ARM_IRQ_TYPE_SPI: 974 if (!irqchip_in_kernel(kvm)) 975 return -ENXIO; 976 977 if (irq_num < VGIC_NR_PRIVATE_IRQS) 978 return -EINVAL; 979 980 return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL); 981 } 982 983 return -EINVAL; 984 } 985 986 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, 987 const struct kvm_vcpu_init *init) 988 { 989 unsigned int i, ret; 990 int phys_target = kvm_target_cpu(); 991 992 if (init->target != phys_target) 993 return -EINVAL; 994 995 /* 996 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must 997 * use the same target. 998 */ 999 if (vcpu->arch.target != -1 && vcpu->arch.target != init->target) 1000 return -EINVAL; 1001 1002 /* -ENOENT for unknown features, -EINVAL for invalid combinations. */ 1003 for (i = 0; i < sizeof(init->features) * 8; i++) { 1004 bool set = (init->features[i / 32] & (1 << (i % 32))); 1005 1006 if (set && i >= KVM_VCPU_MAX_FEATURES) 1007 return -ENOENT; 1008 1009 /* 1010 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must 1011 * use the same feature set. 1012 */ 1013 if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES && 1014 test_bit(i, vcpu->arch.features) != set) 1015 return -EINVAL; 1016 1017 if (set) 1018 set_bit(i, vcpu->arch.features); 1019 } 1020 1021 vcpu->arch.target = phys_target; 1022 1023 /* Now we know what it is, we can reset it. */ 1024 ret = kvm_reset_vcpu(vcpu); 1025 if (ret) { 1026 vcpu->arch.target = -1; 1027 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES); 1028 } 1029 1030 return ret; 1031 } 1032 1033 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, 1034 struct kvm_vcpu_init *init) 1035 { 1036 int ret; 1037 1038 ret = kvm_vcpu_set_target(vcpu, init); 1039 if (ret) 1040 return ret; 1041 1042 /* 1043 * Ensure a rebooted VM will fault in RAM pages and detect if the 1044 * guest MMU is turned off and flush the caches as needed. 1045 * 1046 * S2FWB enforces all memory accesses to RAM being cacheable, 1047 * ensuring that the data side is always coherent. We still 1048 * need to invalidate the I-cache though, as FWB does *not* 1049 * imply CTR_EL0.DIC. 1050 */ 1051 if (vcpu->arch.has_run_once) { 1052 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 1053 stage2_unmap_vm(vcpu->kvm); 1054 else 1055 __flush_icache_all(); 1056 } 1057 1058 vcpu_reset_hcr(vcpu); 1059 1060 /* 1061 * Handle the "start in power-off" case. 1062 */ 1063 if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features)) 1064 vcpu_power_off(vcpu); 1065 else 1066 vcpu->arch.power_off = false; 1067 1068 return 0; 1069 } 1070 1071 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu, 1072 struct kvm_device_attr *attr) 1073 { 1074 int ret = -ENXIO; 1075 1076 switch (attr->group) { 1077 default: 1078 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr); 1079 break; 1080 } 1081 1082 return ret; 1083 } 1084 1085 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu, 1086 struct kvm_device_attr *attr) 1087 { 1088 int ret = -ENXIO; 1089 1090 switch (attr->group) { 1091 default: 1092 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr); 1093 break; 1094 } 1095 1096 return ret; 1097 } 1098 1099 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu, 1100 struct kvm_device_attr *attr) 1101 { 1102 int ret = -ENXIO; 1103 1104 switch (attr->group) { 1105 default: 1106 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr); 1107 break; 1108 } 1109 1110 return ret; 1111 } 1112 1113 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, 1114 struct kvm_vcpu_events *events) 1115 { 1116 memset(events, 0, sizeof(*events)); 1117 1118 return __kvm_arm_vcpu_get_events(vcpu, events); 1119 } 1120 1121 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, 1122 struct kvm_vcpu_events *events) 1123 { 1124 int i; 1125 1126 /* check whether the reserved field is zero */ 1127 for (i = 0; i < ARRAY_SIZE(events->reserved); i++) 1128 if (events->reserved[i]) 1129 return -EINVAL; 1130 1131 /* check whether the pad field is zero */ 1132 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++) 1133 if (events->exception.pad[i]) 1134 return -EINVAL; 1135 1136 return __kvm_arm_vcpu_set_events(vcpu, events); 1137 } 1138 1139 long kvm_arch_vcpu_ioctl(struct file *filp, 1140 unsigned int ioctl, unsigned long arg) 1141 { 1142 struct kvm_vcpu *vcpu = filp->private_data; 1143 void __user *argp = (void __user *)arg; 1144 struct kvm_device_attr attr; 1145 long r; 1146 1147 switch (ioctl) { 1148 case KVM_ARM_VCPU_INIT: { 1149 struct kvm_vcpu_init init; 1150 1151 r = -EFAULT; 1152 if (copy_from_user(&init, argp, sizeof(init))) 1153 break; 1154 1155 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init); 1156 break; 1157 } 1158 case KVM_SET_ONE_REG: 1159 case KVM_GET_ONE_REG: { 1160 struct kvm_one_reg reg; 1161 1162 r = -ENOEXEC; 1163 if (unlikely(!kvm_vcpu_initialized(vcpu))) 1164 break; 1165 1166 r = -EFAULT; 1167 if (copy_from_user(®, argp, sizeof(reg))) 1168 break; 1169 1170 if (ioctl == KVM_SET_ONE_REG) 1171 r = kvm_arm_set_reg(vcpu, ®); 1172 else 1173 r = kvm_arm_get_reg(vcpu, ®); 1174 break; 1175 } 1176 case KVM_GET_REG_LIST: { 1177 struct kvm_reg_list __user *user_list = argp; 1178 struct kvm_reg_list reg_list; 1179 unsigned n; 1180 1181 r = -ENOEXEC; 1182 if (unlikely(!kvm_vcpu_initialized(vcpu))) 1183 break; 1184 1185 r = -EPERM; 1186 if (!kvm_arm_vcpu_is_finalized(vcpu)) 1187 break; 1188 1189 r = -EFAULT; 1190 if (copy_from_user(®_list, user_list, sizeof(reg_list))) 1191 break; 1192 n = reg_list.n; 1193 reg_list.n = kvm_arm_num_regs(vcpu); 1194 if (copy_to_user(user_list, ®_list, sizeof(reg_list))) 1195 break; 1196 r = -E2BIG; 1197 if (n < reg_list.n) 1198 break; 1199 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg); 1200 break; 1201 } 1202 case KVM_SET_DEVICE_ATTR: { 1203 r = -EFAULT; 1204 if (copy_from_user(&attr, argp, sizeof(attr))) 1205 break; 1206 r = kvm_arm_vcpu_set_attr(vcpu, &attr); 1207 break; 1208 } 1209 case KVM_GET_DEVICE_ATTR: { 1210 r = -EFAULT; 1211 if (copy_from_user(&attr, argp, sizeof(attr))) 1212 break; 1213 r = kvm_arm_vcpu_get_attr(vcpu, &attr); 1214 break; 1215 } 1216 case KVM_HAS_DEVICE_ATTR: { 1217 r = -EFAULT; 1218 if (copy_from_user(&attr, argp, sizeof(attr))) 1219 break; 1220 r = kvm_arm_vcpu_has_attr(vcpu, &attr); 1221 break; 1222 } 1223 case KVM_GET_VCPU_EVENTS: { 1224 struct kvm_vcpu_events events; 1225 1226 if (kvm_arm_vcpu_get_events(vcpu, &events)) 1227 return -EINVAL; 1228 1229 if (copy_to_user(argp, &events, sizeof(events))) 1230 return -EFAULT; 1231 1232 return 0; 1233 } 1234 case KVM_SET_VCPU_EVENTS: { 1235 struct kvm_vcpu_events events; 1236 1237 if (copy_from_user(&events, argp, sizeof(events))) 1238 return -EFAULT; 1239 1240 return kvm_arm_vcpu_set_events(vcpu, &events); 1241 } 1242 case KVM_ARM_VCPU_FINALIZE: { 1243 int what; 1244 1245 if (!kvm_vcpu_initialized(vcpu)) 1246 return -ENOEXEC; 1247 1248 if (get_user(what, (const int __user *)argp)) 1249 return -EFAULT; 1250 1251 return kvm_arm_vcpu_finalize(vcpu, what); 1252 } 1253 default: 1254 r = -EINVAL; 1255 } 1256 1257 return r; 1258 } 1259 1260 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 1261 { 1262 1263 } 1264 1265 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, 1266 struct kvm_memory_slot *memslot) 1267 { 1268 kvm_flush_remote_tlbs(kvm); 1269 } 1270 1271 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm, 1272 struct kvm_arm_device_addr *dev_addr) 1273 { 1274 unsigned long dev_id, type; 1275 1276 dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >> 1277 KVM_ARM_DEVICE_ID_SHIFT; 1278 type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >> 1279 KVM_ARM_DEVICE_TYPE_SHIFT; 1280 1281 switch (dev_id) { 1282 case KVM_ARM_DEVICE_VGIC_V2: 1283 if (!vgic_present) 1284 return -ENXIO; 1285 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true); 1286 default: 1287 return -ENODEV; 1288 } 1289 } 1290 1291 long kvm_arch_vm_ioctl(struct file *filp, 1292 unsigned int ioctl, unsigned long arg) 1293 { 1294 struct kvm *kvm = filp->private_data; 1295 void __user *argp = (void __user *)arg; 1296 1297 switch (ioctl) { 1298 case KVM_CREATE_IRQCHIP: { 1299 int ret; 1300 if (!vgic_present) 1301 return -ENXIO; 1302 mutex_lock(&kvm->lock); 1303 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2); 1304 mutex_unlock(&kvm->lock); 1305 return ret; 1306 } 1307 case KVM_ARM_SET_DEVICE_ADDR: { 1308 struct kvm_arm_device_addr dev_addr; 1309 1310 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr))) 1311 return -EFAULT; 1312 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr); 1313 } 1314 case KVM_ARM_PREFERRED_TARGET: { 1315 int err; 1316 struct kvm_vcpu_init init; 1317 1318 err = kvm_vcpu_preferred_target(&init); 1319 if (err) 1320 return err; 1321 1322 if (copy_to_user(argp, &init, sizeof(init))) 1323 return -EFAULT; 1324 1325 return 0; 1326 } 1327 default: 1328 return -EINVAL; 1329 } 1330 } 1331 1332 static unsigned long nvhe_percpu_size(void) 1333 { 1334 return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) - 1335 (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start); 1336 } 1337 1338 static unsigned long nvhe_percpu_order(void) 1339 { 1340 unsigned long size = nvhe_percpu_size(); 1341 1342 return size ? get_order(size) : 0; 1343 } 1344 1345 /* A lookup table holding the hypervisor VA for each vector slot */ 1346 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS]; 1347 1348 static int __kvm_vector_slot2idx(enum arm64_hyp_spectre_vector slot) 1349 { 1350 return slot - (slot != HYP_VECTOR_DIRECT); 1351 } 1352 1353 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot) 1354 { 1355 int idx = __kvm_vector_slot2idx(slot); 1356 1357 hyp_spectre_vector_selector[slot] = base + (idx * SZ_2K); 1358 } 1359 1360 static int kvm_init_vector_slots(void) 1361 { 1362 int err; 1363 void *base; 1364 1365 base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector)); 1366 kvm_init_vector_slot(base, HYP_VECTOR_DIRECT); 1367 1368 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs)); 1369 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT); 1370 1371 if (!cpus_have_const_cap(ARM64_SPECTRE_V3A)) 1372 return 0; 1373 1374 if (!has_vhe()) { 1375 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs), 1376 __BP_HARDEN_HYP_VECS_SZ, &base); 1377 if (err) 1378 return err; 1379 } 1380 1381 kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT); 1382 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT); 1383 return 0; 1384 } 1385 1386 static void cpu_init_hyp_mode(void) 1387 { 1388 struct kvm_nvhe_init_params *params = this_cpu_ptr_nvhe_sym(kvm_init_params); 1389 struct arm_smccc_res res; 1390 unsigned long tcr; 1391 1392 /* Switch from the HYP stub to our own HYP init vector */ 1393 __hyp_set_vectors(kvm_get_idmap_vector()); 1394 1395 /* 1396 * Calculate the raw per-cpu offset without a translation from the 1397 * kernel's mapping to the linear mapping, and store it in tpidr_el2 1398 * so that we can use adr_l to access per-cpu variables in EL2. 1399 */ 1400 params->tpidr_el2 = (unsigned long)this_cpu_ptr_nvhe_sym(__per_cpu_start) - 1401 (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start)); 1402 1403 params->mair_el2 = read_sysreg(mair_el1); 1404 1405 /* 1406 * The ID map may be configured to use an extended virtual address 1407 * range. This is only the case if system RAM is out of range for the 1408 * currently configured page size and VA_BITS, in which case we will 1409 * also need the extended virtual range for the HYP ID map, or we won't 1410 * be able to enable the EL2 MMU. 1411 * 1412 * However, at EL2, there is only one TTBR register, and we can't switch 1413 * between translation tables *and* update TCR_EL2.T0SZ at the same 1414 * time. Bottom line: we need to use the extended range with *both* our 1415 * translation tables. 1416 * 1417 * So use the same T0SZ value we use for the ID map. 1418 */ 1419 tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1; 1420 tcr &= ~TCR_T0SZ_MASK; 1421 tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET; 1422 params->tcr_el2 = tcr; 1423 1424 params->stack_hyp_va = kern_hyp_va(__this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE); 1425 params->pgd_pa = kvm_mmu_get_httbr(); 1426 1427 /* 1428 * Flush the init params from the data cache because the struct will 1429 * be read while the MMU is off. 1430 */ 1431 kvm_flush_dcache_to_poc(params, sizeof(*params)); 1432 1433 /* 1434 * Call initialization code, and switch to the full blown HYP code. 1435 * If the cpucaps haven't been finalized yet, something has gone very 1436 * wrong, and hyp will crash and burn when it uses any 1437 * cpus_have_const_cap() wrapper. 1438 */ 1439 BUG_ON(!system_capabilities_finalized()); 1440 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res); 1441 WARN_ON(res.a0 != SMCCC_RET_SUCCESS); 1442 1443 /* 1444 * Disabling SSBD on a non-VHE system requires us to enable SSBS 1445 * at EL2. 1446 */ 1447 if (this_cpu_has_cap(ARM64_SSBS) && 1448 arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) { 1449 kvm_call_hyp_nvhe(__kvm_enable_ssbs); 1450 } 1451 } 1452 1453 static void cpu_hyp_reset(void) 1454 { 1455 if (!is_kernel_in_hyp_mode()) 1456 __hyp_reset_vectors(); 1457 } 1458 1459 /* 1460 * EL2 vectors can be mapped and rerouted in a number of ways, 1461 * depending on the kernel configuration and CPU present: 1462 * 1463 * - If the CPU is affected by Spectre-v2, the hardening sequence is 1464 * placed in one of the vector slots, which is executed before jumping 1465 * to the real vectors. 1466 * 1467 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot 1468 * containing the hardening sequence is mapped next to the idmap page, 1469 * and executed before jumping to the real vectors. 1470 * 1471 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an 1472 * empty slot is selected, mapped next to the idmap page, and 1473 * executed before jumping to the real vectors. 1474 * 1475 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with 1476 * VHE, as we don't have hypervisor-specific mappings. If the system 1477 * is VHE and yet selects this capability, it will be ignored. 1478 */ 1479 static void cpu_set_hyp_vector(void) 1480 { 1481 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); 1482 void *vector = hyp_spectre_vector_selector[data->slot]; 1483 1484 *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector; 1485 } 1486 1487 static void cpu_hyp_reinit(void) 1488 { 1489 kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt); 1490 1491 cpu_hyp_reset(); 1492 cpu_set_hyp_vector(); 1493 1494 if (is_kernel_in_hyp_mode()) 1495 kvm_timer_init_vhe(); 1496 else 1497 cpu_init_hyp_mode(); 1498 1499 kvm_arm_init_debug(); 1500 1501 if (vgic_present) 1502 kvm_vgic_init_cpu_hardware(); 1503 } 1504 1505 static void _kvm_arch_hardware_enable(void *discard) 1506 { 1507 if (!__this_cpu_read(kvm_arm_hardware_enabled)) { 1508 cpu_hyp_reinit(); 1509 __this_cpu_write(kvm_arm_hardware_enabled, 1); 1510 } 1511 } 1512 1513 int kvm_arch_hardware_enable(void) 1514 { 1515 _kvm_arch_hardware_enable(NULL); 1516 return 0; 1517 } 1518 1519 static void _kvm_arch_hardware_disable(void *discard) 1520 { 1521 if (__this_cpu_read(kvm_arm_hardware_enabled)) { 1522 cpu_hyp_reset(); 1523 __this_cpu_write(kvm_arm_hardware_enabled, 0); 1524 } 1525 } 1526 1527 void kvm_arch_hardware_disable(void) 1528 { 1529 if (!is_protected_kvm_enabled()) 1530 _kvm_arch_hardware_disable(NULL); 1531 } 1532 1533 #ifdef CONFIG_CPU_PM 1534 static int hyp_init_cpu_pm_notifier(struct notifier_block *self, 1535 unsigned long cmd, 1536 void *v) 1537 { 1538 /* 1539 * kvm_arm_hardware_enabled is left with its old value over 1540 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should 1541 * re-enable hyp. 1542 */ 1543 switch (cmd) { 1544 case CPU_PM_ENTER: 1545 if (__this_cpu_read(kvm_arm_hardware_enabled)) 1546 /* 1547 * don't update kvm_arm_hardware_enabled here 1548 * so that the hardware will be re-enabled 1549 * when we resume. See below. 1550 */ 1551 cpu_hyp_reset(); 1552 1553 return NOTIFY_OK; 1554 case CPU_PM_ENTER_FAILED: 1555 case CPU_PM_EXIT: 1556 if (__this_cpu_read(kvm_arm_hardware_enabled)) 1557 /* The hardware was enabled before suspend. */ 1558 cpu_hyp_reinit(); 1559 1560 return NOTIFY_OK; 1561 1562 default: 1563 return NOTIFY_DONE; 1564 } 1565 } 1566 1567 static struct notifier_block hyp_init_cpu_pm_nb = { 1568 .notifier_call = hyp_init_cpu_pm_notifier, 1569 }; 1570 1571 static void hyp_cpu_pm_init(void) 1572 { 1573 if (!is_protected_kvm_enabled()) 1574 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb); 1575 } 1576 static void hyp_cpu_pm_exit(void) 1577 { 1578 if (!is_protected_kvm_enabled()) 1579 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb); 1580 } 1581 #else 1582 static inline void hyp_cpu_pm_init(void) 1583 { 1584 } 1585 static inline void hyp_cpu_pm_exit(void) 1586 { 1587 } 1588 #endif 1589 1590 static void init_cpu_logical_map(void) 1591 { 1592 unsigned int cpu; 1593 1594 /* 1595 * Copy the MPIDR <-> logical CPU ID mapping to hyp. 1596 * Only copy the set of online CPUs whose features have been chacked 1597 * against the finalized system capabilities. The hypervisor will not 1598 * allow any other CPUs from the `possible` set to boot. 1599 */ 1600 for_each_online_cpu(cpu) 1601 hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu); 1602 } 1603 1604 #define init_psci_0_1_impl_state(config, what) \ 1605 config.psci_0_1_ ## what ## _implemented = psci_ops.what 1606 1607 static bool init_psci_relay(void) 1608 { 1609 /* 1610 * If PSCI has not been initialized, protected KVM cannot install 1611 * itself on newly booted CPUs. 1612 */ 1613 if (!psci_ops.get_version) { 1614 kvm_err("Cannot initialize protected mode without PSCI\n"); 1615 return false; 1616 } 1617 1618 kvm_host_psci_config.version = psci_ops.get_version(); 1619 1620 if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) { 1621 kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids(); 1622 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend); 1623 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on); 1624 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off); 1625 init_psci_0_1_impl_state(kvm_host_psci_config, migrate); 1626 } 1627 return true; 1628 } 1629 1630 static int init_common_resources(void) 1631 { 1632 return kvm_set_ipa_limit(); 1633 } 1634 1635 static int init_subsystems(void) 1636 { 1637 int err = 0; 1638 1639 /* 1640 * Enable hardware so that subsystem initialisation can access EL2. 1641 */ 1642 on_each_cpu(_kvm_arch_hardware_enable, NULL, 1); 1643 1644 /* 1645 * Register CPU lower-power notifier 1646 */ 1647 hyp_cpu_pm_init(); 1648 1649 /* 1650 * Init HYP view of VGIC 1651 */ 1652 err = kvm_vgic_hyp_init(); 1653 switch (err) { 1654 case 0: 1655 vgic_present = true; 1656 break; 1657 case -ENODEV: 1658 case -ENXIO: 1659 vgic_present = false; 1660 err = 0; 1661 break; 1662 default: 1663 goto out; 1664 } 1665 1666 /* 1667 * Init HYP architected timer support 1668 */ 1669 err = kvm_timer_hyp_init(vgic_present); 1670 if (err) 1671 goto out; 1672 1673 kvm_perf_init(); 1674 kvm_sys_reg_table_init(); 1675 1676 out: 1677 if (err || !is_protected_kvm_enabled()) 1678 on_each_cpu(_kvm_arch_hardware_disable, NULL, 1); 1679 1680 return err; 1681 } 1682 1683 static void teardown_hyp_mode(void) 1684 { 1685 int cpu; 1686 1687 free_hyp_pgds(); 1688 for_each_possible_cpu(cpu) { 1689 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu)); 1690 free_pages(kvm_arm_hyp_percpu_base[cpu], nvhe_percpu_order()); 1691 } 1692 } 1693 1694 /** 1695 * Inits Hyp-mode on all online CPUs 1696 */ 1697 static int init_hyp_mode(void) 1698 { 1699 int cpu; 1700 int err = 0; 1701 1702 /* 1703 * Allocate Hyp PGD and setup Hyp identity mapping 1704 */ 1705 err = kvm_mmu_init(); 1706 if (err) 1707 goto out_err; 1708 1709 /* 1710 * Allocate stack pages for Hypervisor-mode 1711 */ 1712 for_each_possible_cpu(cpu) { 1713 unsigned long stack_page; 1714 1715 stack_page = __get_free_page(GFP_KERNEL); 1716 if (!stack_page) { 1717 err = -ENOMEM; 1718 goto out_err; 1719 } 1720 1721 per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page; 1722 } 1723 1724 /* 1725 * Allocate and initialize pages for Hypervisor-mode percpu regions. 1726 */ 1727 for_each_possible_cpu(cpu) { 1728 struct page *page; 1729 void *page_addr; 1730 1731 page = alloc_pages(GFP_KERNEL, nvhe_percpu_order()); 1732 if (!page) { 1733 err = -ENOMEM; 1734 goto out_err; 1735 } 1736 1737 page_addr = page_address(page); 1738 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size()); 1739 kvm_arm_hyp_percpu_base[cpu] = (unsigned long)page_addr; 1740 } 1741 1742 /* 1743 * Map the Hyp-code called directly from the host 1744 */ 1745 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start), 1746 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC); 1747 if (err) { 1748 kvm_err("Cannot map world-switch code\n"); 1749 goto out_err; 1750 } 1751 1752 err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_ro_after_init_start), 1753 kvm_ksym_ref(__hyp_data_ro_after_init_end), 1754 PAGE_HYP_RO); 1755 if (err) { 1756 kvm_err("Cannot map .hyp.data..ro_after_init section\n"); 1757 goto out_err; 1758 } 1759 1760 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata), 1761 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO); 1762 if (err) { 1763 kvm_err("Cannot map rodata section\n"); 1764 goto out_err; 1765 } 1766 1767 err = create_hyp_mappings(kvm_ksym_ref(__bss_start), 1768 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO); 1769 if (err) { 1770 kvm_err("Cannot map bss section\n"); 1771 goto out_err; 1772 } 1773 1774 /* 1775 * Map the Hyp stack pages 1776 */ 1777 for_each_possible_cpu(cpu) { 1778 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu); 1779 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE, 1780 PAGE_HYP); 1781 1782 if (err) { 1783 kvm_err("Cannot map hyp stack\n"); 1784 goto out_err; 1785 } 1786 } 1787 1788 /* 1789 * Map Hyp percpu pages 1790 */ 1791 for_each_possible_cpu(cpu) { 1792 char *percpu_begin = (char *)kvm_arm_hyp_percpu_base[cpu]; 1793 char *percpu_end = percpu_begin + nvhe_percpu_size(); 1794 1795 err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP); 1796 1797 if (err) { 1798 kvm_err("Cannot map hyp percpu region\n"); 1799 goto out_err; 1800 } 1801 } 1802 1803 if (is_protected_kvm_enabled()) { 1804 init_cpu_logical_map(); 1805 1806 if (!init_psci_relay()) 1807 goto out_err; 1808 } 1809 1810 return 0; 1811 1812 out_err: 1813 teardown_hyp_mode(); 1814 kvm_err("error initializing Hyp mode: %d\n", err); 1815 return err; 1816 } 1817 1818 static void check_kvm_target_cpu(void *ret) 1819 { 1820 *(int *)ret = kvm_target_cpu(); 1821 } 1822 1823 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr) 1824 { 1825 struct kvm_vcpu *vcpu; 1826 int i; 1827 1828 mpidr &= MPIDR_HWID_BITMASK; 1829 kvm_for_each_vcpu(i, vcpu, kvm) { 1830 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu)) 1831 return vcpu; 1832 } 1833 return NULL; 1834 } 1835 1836 bool kvm_arch_has_irq_bypass(void) 1837 { 1838 return true; 1839 } 1840 1841 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, 1842 struct irq_bypass_producer *prod) 1843 { 1844 struct kvm_kernel_irqfd *irqfd = 1845 container_of(cons, struct kvm_kernel_irqfd, consumer); 1846 1847 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq, 1848 &irqfd->irq_entry); 1849 } 1850 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, 1851 struct irq_bypass_producer *prod) 1852 { 1853 struct kvm_kernel_irqfd *irqfd = 1854 container_of(cons, struct kvm_kernel_irqfd, consumer); 1855 1856 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq, 1857 &irqfd->irq_entry); 1858 } 1859 1860 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons) 1861 { 1862 struct kvm_kernel_irqfd *irqfd = 1863 container_of(cons, struct kvm_kernel_irqfd, consumer); 1864 1865 kvm_arm_halt_guest(irqfd->kvm); 1866 } 1867 1868 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons) 1869 { 1870 struct kvm_kernel_irqfd *irqfd = 1871 container_of(cons, struct kvm_kernel_irqfd, consumer); 1872 1873 kvm_arm_resume_guest(irqfd->kvm); 1874 } 1875 1876 /** 1877 * Initialize Hyp-mode and memory mappings on all CPUs. 1878 */ 1879 int kvm_arch_init(void *opaque) 1880 { 1881 int err; 1882 int ret, cpu; 1883 bool in_hyp_mode; 1884 1885 if (!is_hyp_mode_available()) { 1886 kvm_info("HYP mode not available\n"); 1887 return -ENODEV; 1888 } 1889 1890 in_hyp_mode = is_kernel_in_hyp_mode(); 1891 1892 if (!in_hyp_mode && kvm_arch_requires_vhe()) { 1893 kvm_pr_unimpl("CPU unsupported in non-VHE mode, not initializing\n"); 1894 return -ENODEV; 1895 } 1896 1897 if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) || 1898 cpus_have_final_cap(ARM64_WORKAROUND_1508412)) 1899 kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \ 1900 "Only trusted guests should be used on this system.\n"); 1901 1902 for_each_online_cpu(cpu) { 1903 smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1); 1904 if (ret < 0) { 1905 kvm_err("Error, CPU %d not supported!\n", cpu); 1906 return -ENODEV; 1907 } 1908 } 1909 1910 err = init_common_resources(); 1911 if (err) 1912 return err; 1913 1914 err = kvm_arm_init_sve(); 1915 if (err) 1916 return err; 1917 1918 if (!in_hyp_mode) { 1919 err = init_hyp_mode(); 1920 if (err) 1921 goto out_err; 1922 } 1923 1924 err = kvm_init_vector_slots(); 1925 if (err) { 1926 kvm_err("Cannot initialise vector slots\n"); 1927 goto out_err; 1928 } 1929 1930 err = init_subsystems(); 1931 if (err) 1932 goto out_hyp; 1933 1934 if (is_protected_kvm_enabled()) { 1935 static_branch_enable(&kvm_protected_mode_initialized); 1936 kvm_info("Protected nVHE mode initialized successfully\n"); 1937 } else if (in_hyp_mode) { 1938 kvm_info("VHE mode initialized successfully\n"); 1939 } else { 1940 kvm_info("Hyp mode initialized successfully\n"); 1941 } 1942 1943 return 0; 1944 1945 out_hyp: 1946 hyp_cpu_pm_exit(); 1947 if (!in_hyp_mode) 1948 teardown_hyp_mode(); 1949 out_err: 1950 return err; 1951 } 1952 1953 /* NOP: Compiling as a module not supported */ 1954 void kvm_arch_exit(void) 1955 { 1956 kvm_perf_teardown(); 1957 } 1958 1959 static int __init early_kvm_mode_cfg(char *arg) 1960 { 1961 if (!arg) 1962 return -EINVAL; 1963 1964 if (strcmp(arg, "protected") == 0) { 1965 kvm_mode = KVM_MODE_PROTECTED; 1966 return 0; 1967 } 1968 1969 return -EINVAL; 1970 } 1971 early_param("kvm-arm.mode", early_kvm_mode_cfg); 1972 1973 enum kvm_mode kvm_get_mode(void) 1974 { 1975 return kvm_mode; 1976 } 1977 1978 static int arm_init(void) 1979 { 1980 int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 1981 return rc; 1982 } 1983 1984 module_init(arm_init); 1985