1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * arch/arm64/kvm/fpsimd.c: Guest/host FPSIMD context coordination helpers 4 * 5 * Copyright 2018 Arm Limited 6 * Author: Dave Martin <Dave.Martin@arm.com> 7 */ 8 #include <linux/bottom_half.h> 9 #include <linux/sched.h> 10 #include <linux/thread_info.h> 11 #include <linux/kvm_host.h> 12 #include <asm/kvm_asm.h> 13 #include <asm/kvm_host.h> 14 #include <asm/kvm_mmu.h> 15 16 /* 17 * Called on entry to KVM_RUN unless this vcpu previously ran at least 18 * once and the most recent prior KVM_RUN for this vcpu was called from 19 * the same task as current (highly likely). 20 * 21 * This is guaranteed to execute before kvm_arch_vcpu_load_fp(vcpu), 22 * such that on entering hyp the relevant parts of current are already 23 * mapped. 24 */ 25 int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) 26 { 27 int ret; 28 29 struct thread_info *ti = ¤t->thread_info; 30 struct user_fpsimd_state *fpsimd = ¤t->thread.uw.fpsimd_state; 31 32 /* 33 * Make sure the host task thread flags and fpsimd state are 34 * visible to hyp: 35 */ 36 ret = create_hyp_mappings(ti, ti + 1, PAGE_HYP); 37 if (ret) 38 goto error; 39 40 ret = create_hyp_mappings(fpsimd, fpsimd + 1, PAGE_HYP); 41 if (ret) 42 goto error; 43 44 vcpu->arch.host_thread_info = kern_hyp_va(ti); 45 vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd); 46 error: 47 return ret; 48 } 49 50 /* 51 * Prepare vcpu for saving the host's FPSIMD state and loading the guest's. 52 * The actual loading is done by the FPSIMD access trap taken to hyp. 53 * 54 * Here, we just set the correct metadata to indicate that the FPSIMD 55 * state in the cpu regs (if any) belongs to current on the host. 56 * 57 * TIF_SVE is backed up here, since it may get clobbered with guest state. 58 * This flag is restored by kvm_arch_vcpu_put_fp(vcpu). 59 */ 60 void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) 61 { 62 BUG_ON(!current->mm); 63 64 vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED | KVM_ARM64_HOST_SVE_IN_USE); 65 vcpu->arch.flags |= KVM_ARM64_FP_HOST; 66 if (test_thread_flag(TIF_SVE)) 67 vcpu->arch.flags |= KVM_ARM64_HOST_SVE_IN_USE; 68 } 69 70 /* 71 * If the guest FPSIMD state was loaded, update the host's context 72 * tracking data mark the CPU FPSIMD regs as dirty and belonging to vcpu 73 * so that they will be written back if the kernel clobbers them due to 74 * kernel-mode NEON before re-entry into the guest. 75 */ 76 void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) 77 { 78 WARN_ON_ONCE(!irqs_disabled()); 79 80 if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) { 81 fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.gp_regs.fp_regs); 82 clear_thread_flag(TIF_FOREIGN_FPSTATE); 83 clear_thread_flag(TIF_SVE); 84 } 85 } 86 87 /* 88 * Write back the vcpu FPSIMD regs if they are dirty, and invalidate the 89 * cpu FPSIMD regs so that they can't be spuriously reused if this vcpu 90 * disappears and another task or vcpu appears that recycles the same 91 * struct fpsimd_state. 92 */ 93 void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) 94 { 95 local_bh_disable(); 96 97 update_thread_flag(TIF_SVE, 98 vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE); 99 100 if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) { 101 /* Clean guest FP state to memory and invalidate cpu view */ 102 fpsimd_save(); 103 fpsimd_flush_cpu_state(); 104 } else if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { 105 /* Ensure user trap controls are correctly restored */ 106 fpsimd_bind_task_to_cpu(); 107 } 108 109 local_bh_enable(); 110 } 111