1caab277bSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-only */
24f8d6632SMarc Zyngier /*
34f8d6632SMarc Zyngier * Copyright (C) 2012,2013 - ARM Ltd
44f8d6632SMarc Zyngier * Author: Marc Zyngier <marc.zyngier@arm.com>
54f8d6632SMarc Zyngier *
64f8d6632SMarc Zyngier * Derived from arch/arm/include/asm/kvm_host.h:
74f8d6632SMarc Zyngier * Copyright (C) 2012 - Virtual Open Systems and Columbia University
84f8d6632SMarc Zyngier * Author: Christoffer Dall <c.dall@virtualopensystems.com>
94f8d6632SMarc Zyngier */
104f8d6632SMarc Zyngier
114f8d6632SMarc Zyngier #ifndef __ARM64_KVM_HOST_H__
124f8d6632SMarc Zyngier #define __ARM64_KVM_HOST_H__
134f8d6632SMarc Zyngier
1405469831SAndrew Scull #include <linux/arm-smccc.h>
153f61f409SDave Martin #include <linux/bitmap.h>
1665647300SPaolo Bonzini #include <linux/types.h>
173f61f409SDave Martin #include <linux/jump_label.h>
1865647300SPaolo Bonzini #include <linux/kvm_types.h>
19fb88707dSOliver Upton #include <linux/maple_tree.h>
203f61f409SDave Martin #include <linux/percpu.h>
21ff367fe4SDavid Brazdil #include <linux/psci.h>
2285738e05SJulien Thierry #include <asm/arch_gicv3.h>
233f61f409SDave Martin #include <asm/barrier.h>
2463a1e1c9SMark Rutland #include <asm/cpufeature.h>
251e0cf16cSMarc Zyngier #include <asm/cputype.h>
264f5abad9SJames Morse #include <asm/daifflags.h>
2717eed27bSDave Martin #include <asm/fpsimd.h>
284f8d6632SMarc Zyngier #include <asm/kvm.h>
293a3604bcSMarc Zyngier #include <asm/kvm_asm.h>
304f8d6632SMarc Zyngier
31c1426e4cSEric Auger #define __KVM_HAVE_ARCH_INTC_INITIALIZED
32c1426e4cSEric Auger
33920552b2SDavid Hildenbrand #define KVM_HALT_POLL_NS_DEFAULT 500000
344f8d6632SMarc Zyngier
354f8d6632SMarc Zyngier #include <kvm/arm_vgic.h>
364f8d6632SMarc Zyngier #include <kvm/arm_arch_timer.h>
3704fe4726SShannon Zhao #include <kvm/arm_pmu.h>
384f8d6632SMarc Zyngier
39ef748917SMing Lei #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
40ef748917SMing Lei
41a22fa321SAmit Daniel Kachhap #define KVM_VCPU_MAX_FEATURES 7
42a7a2c72aSOliver Upton #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1)
434f8d6632SMarc Zyngier
447b244e2bSAndrew Jones #define KVM_REQ_SLEEP \
452387149eSAndrew Jones KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
46325f9c64SAndrew Jones #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1)
47358b28f0SMarc Zyngier #define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2)
488564d637SSteven Price #define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(3)
49d9c3872cSMarc Zyngier #define KVM_REQ_RELOAD_GICv4 KVM_ARCH_REQ(4)
50d0c94c49SMarc Zyngier #define KVM_REQ_RELOAD_PMU KVM_ARCH_REQ(5)
517b33a09dSOliver Upton #define KVM_REQ_SUSPEND KVM_ARCH_REQ(6)
52*b1f778a2SMarc Zyngier #define KVM_REQ_RESYNC_PMU_EL0 KVM_ARCH_REQ(7)
53b13216cfSChristoffer Dall
54c862626eSKeqian Zhu #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
55c862626eSKeqian Zhu KVM_DIRTY_LOG_INITIALLY_SET)
56c862626eSKeqian Zhu
57fcc5bf89SJing Zhang #define KVM_HAVE_MMU_RWLOCK
58fcc5bf89SJing Zhang
59d8b369c4SDavid Brazdil /*
60d8b369c4SDavid Brazdil * Mode of operation configurable with kvm-arm.mode early param.
61d8b369c4SDavid Brazdil * See Documentation/admin-guide/kernel-parameters.txt for more information.
62d8b369c4SDavid Brazdil */
63d8b369c4SDavid Brazdil enum kvm_mode {
64d8b369c4SDavid Brazdil KVM_MODE_DEFAULT,
65d8b369c4SDavid Brazdil KVM_MODE_PROTECTED,
66675cabc8SJintack Lim KVM_MODE_NV,
67b6a68b97SMarc Zyngier KVM_MODE_NONE,
68d8b369c4SDavid Brazdil };
69675cabc8SJintack Lim #ifdef CONFIG_KVM
703eb681fbSDavid Brazdil enum kvm_mode kvm_get_mode(void);
71675cabc8SJintack Lim #else
kvm_get_mode(void)72675cabc8SJintack Lim static inline enum kvm_mode kvm_get_mode(void) { return KVM_MODE_NONE; };
73675cabc8SJintack Lim #endif
74d8b369c4SDavid Brazdil
7561bbe380SChristoffer Dall DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
7661bbe380SChristoffer Dall
778d20bd63SSean Christopherson extern unsigned int __ro_after_init kvm_sve_max_vl;
788d20bd63SSean Christopherson int __init kvm_arm_init_sve(void);
790f062bfeSDave Martin
806b7982feSAnshuman Khandual u32 __attribute_const__ kvm_target_cpu(void);
814f8d6632SMarc Zyngier int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
8219bcc89eSSean Christopherson void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu);
834f8d6632SMarc Zyngier
84717a7eebSQuentin Perret struct kvm_hyp_memcache {
85717a7eebSQuentin Perret phys_addr_t head;
86717a7eebSQuentin Perret unsigned long nr_pages;
87717a7eebSQuentin Perret };
88717a7eebSQuentin Perret
push_hyp_memcache(struct kvm_hyp_memcache * mc,phys_addr_t * p,phys_addr_t (* to_pa)(void * virt))89717a7eebSQuentin Perret static inline void push_hyp_memcache(struct kvm_hyp_memcache *mc,
90717a7eebSQuentin Perret phys_addr_t *p,
91717a7eebSQuentin Perret phys_addr_t (*to_pa)(void *virt))
92717a7eebSQuentin Perret {
93717a7eebSQuentin Perret *p = mc->head;
94717a7eebSQuentin Perret mc->head = to_pa(p);
95717a7eebSQuentin Perret mc->nr_pages++;
96717a7eebSQuentin Perret }
97717a7eebSQuentin Perret
pop_hyp_memcache(struct kvm_hyp_memcache * mc,void * (* to_va)(phys_addr_t phys))98717a7eebSQuentin Perret static inline void *pop_hyp_memcache(struct kvm_hyp_memcache *mc,
99717a7eebSQuentin Perret void *(*to_va)(phys_addr_t phys))
100717a7eebSQuentin Perret {
101717a7eebSQuentin Perret phys_addr_t *p = to_va(mc->head);
102717a7eebSQuentin Perret
103717a7eebSQuentin Perret if (!mc->nr_pages)
104717a7eebSQuentin Perret return NULL;
105717a7eebSQuentin Perret
106717a7eebSQuentin Perret mc->head = *p;
107717a7eebSQuentin Perret mc->nr_pages--;
108717a7eebSQuentin Perret
109717a7eebSQuentin Perret return p;
110717a7eebSQuentin Perret }
111717a7eebSQuentin Perret
__topup_hyp_memcache(struct kvm_hyp_memcache * mc,unsigned long min_pages,void * (* alloc_fn)(void * arg),phys_addr_t (* to_pa)(void * virt),void * arg)112717a7eebSQuentin Perret static inline int __topup_hyp_memcache(struct kvm_hyp_memcache *mc,
113717a7eebSQuentin Perret unsigned long min_pages,
114717a7eebSQuentin Perret void *(*alloc_fn)(void *arg),
115717a7eebSQuentin Perret phys_addr_t (*to_pa)(void *virt),
116717a7eebSQuentin Perret void *arg)
117717a7eebSQuentin Perret {
118717a7eebSQuentin Perret while (mc->nr_pages < min_pages) {
119717a7eebSQuentin Perret phys_addr_t *p = alloc_fn(arg);
120717a7eebSQuentin Perret
121717a7eebSQuentin Perret if (!p)
122717a7eebSQuentin Perret return -ENOMEM;
123717a7eebSQuentin Perret push_hyp_memcache(mc, p, to_pa);
124717a7eebSQuentin Perret }
125717a7eebSQuentin Perret
126717a7eebSQuentin Perret return 0;
127717a7eebSQuentin Perret }
128717a7eebSQuentin Perret
__free_hyp_memcache(struct kvm_hyp_memcache * mc,void (* free_fn)(void * virt,void * arg),void * (* to_va)(phys_addr_t phys),void * arg)129717a7eebSQuentin Perret static inline void __free_hyp_memcache(struct kvm_hyp_memcache *mc,
130717a7eebSQuentin Perret void (*free_fn)(void *virt, void *arg),
131717a7eebSQuentin Perret void *(*to_va)(phys_addr_t phys),
132717a7eebSQuentin Perret void *arg)
133717a7eebSQuentin Perret {
134717a7eebSQuentin Perret while (mc->nr_pages)
135717a7eebSQuentin Perret free_fn(pop_hyp_memcache(mc, to_va), arg);
136717a7eebSQuentin Perret }
137717a7eebSQuentin Perret
138717a7eebSQuentin Perret void free_hyp_memcache(struct kvm_hyp_memcache *mc);
139717a7eebSQuentin Perret int topup_hyp_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages);
140717a7eebSQuentin Perret
141e329fb75SChristoffer Dall struct kvm_vmid {
1423248136bSJulien Grall atomic64_t id;
143e329fb75SChristoffer Dall };
144e329fb75SChristoffer Dall
145a0e50aa3SChristoffer Dall struct kvm_s2_mmu {
146e329fb75SChristoffer Dall struct kvm_vmid vmid;
1474f8d6632SMarc Zyngier
148a0e50aa3SChristoffer Dall /*
149a0e50aa3SChristoffer Dall * stage2 entry level table
150a0e50aa3SChristoffer Dall *
151a0e50aa3SChristoffer Dall * Two kvm_s2_mmu structures in the same VM can point to the same
152a0e50aa3SChristoffer Dall * pgd here. This happens when running a guest using a
153a0e50aa3SChristoffer Dall * translation regime that isn't affected by its own stage-2
154a0e50aa3SChristoffer Dall * translation, such as a non-VHE hypervisor running at vEL2, or
155a0e50aa3SChristoffer Dall * for vEL1/EL0 with vHCR_EL2.VM == 0. In that case, we use the
156a0e50aa3SChristoffer Dall * canonical stage-2 page tables.
157a0e50aa3SChristoffer Dall */
158e329fb75SChristoffer Dall phys_addr_t pgd_phys;
15971233d05SWill Deacon struct kvm_pgtable *pgt;
1604f8d6632SMarc Zyngier
16194d0e598SMarc Zyngier /* The last vcpu id that ran on each physical CPU */
16294d0e598SMarc Zyngier int __percpu *last_vcpu_ran;
16394d0e598SMarc Zyngier
1642f440b72SRicardo Koller #define KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT 0
1652f440b72SRicardo Koller /*
1662f440b72SRicardo Koller * Memory cache used to split
1672f440b72SRicardo Koller * KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE worth of huge pages. It
1682f440b72SRicardo Koller * is used to allocate stage2 page tables while splitting huge
1692f440b72SRicardo Koller * pages. The choice of KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE
1702f440b72SRicardo Koller * influences both the capacity of the split page cache, and
1712f440b72SRicardo Koller * how often KVM reschedules. Be wary of raising CHUNK_SIZE
1722f440b72SRicardo Koller * too high.
1732f440b72SRicardo Koller *
1742f440b72SRicardo Koller * Protected by kvm->slots_lock.
1752f440b72SRicardo Koller */
1762f440b72SRicardo Koller struct kvm_mmu_memory_cache split_page_cache;
1772f440b72SRicardo Koller uint64_t split_page_chunk_size;
1782f440b72SRicardo Koller
179cfb1a98dSQuentin Perret struct kvm_arch *arch;
180a0e50aa3SChristoffer Dall };
181a0e50aa3SChristoffer Dall
1828d14797bSWill Deacon struct kvm_arch_memory_slot {
1838d14797bSWill Deacon };
1848d14797bSWill Deacon
18505714cabSRaghavendra Rao Ananta /**
18605714cabSRaghavendra Rao Ananta * struct kvm_smccc_features: Descriptor of the hypercall services exposed to the guests
18705714cabSRaghavendra Rao Ananta *
18805714cabSRaghavendra Rao Ananta * @std_bmap: Bitmap of standard secure service calls
189428fd678SRaghavendra Rao Ananta * @std_hyp_bmap: Bitmap of standard hypervisor service calls
190b22216e1SRaghavendra Rao Ananta * @vendor_hyp_bmap: Bitmap of vendor specific hypervisor service calls
19105714cabSRaghavendra Rao Ananta */
19205714cabSRaghavendra Rao Ananta struct kvm_smccc_features {
19305714cabSRaghavendra Rao Ananta unsigned long std_bmap;
194428fd678SRaghavendra Rao Ananta unsigned long std_hyp_bmap;
195b22216e1SRaghavendra Rao Ananta unsigned long vendor_hyp_bmap;
19605714cabSRaghavendra Rao Ananta };
19705714cabSRaghavendra Rao Ananta
198a1ec5c70SFuad Tabba typedef unsigned int pkvm_handle_t;
199a1ec5c70SFuad Tabba
2009d0c063aSFuad Tabba struct kvm_protected_vm {
2019d0c063aSFuad Tabba pkvm_handle_t handle;
202f41dff4eSQuentin Perret struct kvm_hyp_memcache teardown_mc;
2039d0c063aSFuad Tabba };
2049d0c063aSFuad Tabba
205a0e50aa3SChristoffer Dall struct kvm_arch {
206a0e50aa3SChristoffer Dall struct kvm_s2_mmu mmu;
207a0e50aa3SChristoffer Dall
208a0e50aa3SChristoffer Dall /* VTCR_EL2 value for this VM */
209a0e50aa3SChristoffer Dall u64 vtcr;
210a0e50aa3SChristoffer Dall
2114f8d6632SMarc Zyngier /* Interrupt controller */
2124f8d6632SMarc Zyngier struct vgic_dist vgic;
21385bd0ba1SMarc Zyngier
21447053904SMarc Zyngier /* Timers */
21547053904SMarc Zyngier struct arch_timer_vm_data timer_data;
21647053904SMarc Zyngier
21785bd0ba1SMarc Zyngier /* Mandated version of PSCI */
21885bd0ba1SMarc Zyngier u32 psci_version;
219c726200dSChristoffer Dall
220c43120afSOliver Upton /* Protects VM-scoped configuration data */
221c43120afSOliver Upton struct mutex config_lock;
222c43120afSOliver Upton
223c726200dSChristoffer Dall /*
224c726200dSChristoffer Dall * If we encounter a data abort without valid instruction syndrome
225c726200dSChristoffer Dall * information, report this to user space. User space can (and
226c726200dSChristoffer Dall * should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is
227c726200dSChristoffer Dall * supported.
228c726200dSChristoffer Dall */
22906394531SMarc Zyngier #define KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER 0
23006394531SMarc Zyngier /* Memory Tagging Extension enabled for the guest */
23106394531SMarc Zyngier #define KVM_ARCH_FLAG_MTE_ENABLED 1
23206394531SMarc Zyngier /* At least one vCPU has ran in the VM */
23306394531SMarc Zyngier #define KVM_ARCH_FLAG_HAS_RAN_ONCE 2
2342251e9ffSOliver Upton /* The vCPU feature set for the VM is configured */
2352251e9ffSOliver Upton #define KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED 3
236bfbab445SOliver Upton /* PSCI SYSTEM_SUSPEND enabled for the guest */
2372251e9ffSOliver Upton #define KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED 4
23830ec7997SMarc Zyngier /* VM counter offset */
2392251e9ffSOliver Upton #define KVM_ARCH_FLAG_VM_COUNTER_OFFSET 5
2408a5eb2d2SMarc Zyngier /* Timer PPIs made immutable */
2412251e9ffSOliver Upton #define KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE 6
242fb88707dSOliver Upton /* SMCCC filter initialized for the VM */
2432251e9ffSOliver Upton #define KVM_ARCH_FLAG_SMCCC_FILTER_CONFIGURED 7
24447334146SJing Zhang /* Initial ID reg values loaded */
24547334146SJing Zhang #define KVM_ARCH_FLAG_ID_REGS_INITIALIZED 8
24606394531SMarc Zyngier unsigned long flags;
247fd65a3b5SMarc Zyngier
2482251e9ffSOliver Upton /* VM-wide vCPU feature set */
2492251e9ffSOliver Upton DECLARE_BITMAP(vcpu_features, KVM_VCPU_MAX_FEATURES);
2502251e9ffSOliver Upton
251d7eec236SMarc Zyngier /*
252d7eec236SMarc Zyngier * VM-wide PMU filter, implemented as a bitmap and big enough for
253d7eec236SMarc Zyngier * up to 2^10 events (ARMv8.0) or 2^16 events (ARMv8.1+).
254d7eec236SMarc Zyngier */
255d7eec236SMarc Zyngier unsigned long *pmu_filter;
25646b18782SMarc Zyngier struct arm_pmu *arm_pmu;
25723711a5eSMarc Zyngier
258583cda1bSAlexandru Elisei cpumask_var_t supported_cpus;
25923711a5eSMarc Zyngier
26005714cabSRaghavendra Rao Ananta /* Hypercall features firmware registers' descriptor */
26105714cabSRaghavendra Rao Ananta struct kvm_smccc_features smccc_feat;
262fb88707dSOliver Upton struct maple_tree smccc_filter;
263a1ec5c70SFuad Tabba
264a1ec5c70SFuad Tabba /*
26547334146SJing Zhang * Emulated CPU ID registers per VM
26647334146SJing Zhang * (Op0, Op1, CRn, CRm, Op2) of the ID registers to be saved in it
26747334146SJing Zhang * is (3, 0, 0, crm, op2), where 1<=crm<8, 0<=op2<8.
26847334146SJing Zhang *
26947334146SJing Zhang * These emulated idregs are VM-wide, but accessed from the context of a vCPU.
27047334146SJing Zhang * Atomic access to multiple idregs are guarded by kvm_arch.config_lock.
27147334146SJing Zhang */
27247334146SJing Zhang #define IDREG_IDX(id) (((sys_reg_CRm(id) - 1) << 3) | sys_reg_Op2(id))
27347334146SJing Zhang #define IDREG(kvm, id) ((kvm)->arch.id_regs[IDREG_IDX(id)])
27447334146SJing Zhang #define KVM_ARM_ID_REG_NUM (IDREG_IDX(sys_reg(3, 0, 0, 7, 7)) + 1)
27547334146SJing Zhang u64 id_regs[KVM_ARM_ID_REG_NUM];
27647334146SJing Zhang
27747334146SJing Zhang /*
2789d0c063aSFuad Tabba * For an untrusted host VM, 'pkvm.handle' is used to lookup
279a1ec5c70SFuad Tabba * the associated pKVM instance in the hypervisor.
280a1ec5c70SFuad Tabba */
2819d0c063aSFuad Tabba struct kvm_protected_vm pkvm;
2824f8d6632SMarc Zyngier };
2834f8d6632SMarc Zyngier
2844f8d6632SMarc Zyngier struct kvm_vcpu_fault_info {
2850b12620fSAlexandru Elisei u64 esr_el2; /* Hyp Syndrom Register */
2864f8d6632SMarc Zyngier u64 far_el2; /* Hyp Fault Address Register */
2874f8d6632SMarc Zyngier u64 hpfar_el2; /* Hyp IPA Fault Address Register */
2880067df41SJames Morse u64 disr_el1; /* Deferred [SError] Status Register */
2894f8d6632SMarc Zyngier };
2904f8d6632SMarc Zyngier
2919d8415d6SMarc Zyngier enum vcpu_sysreg {
2928f7f4fe7SMarc Zyngier __INVALID_SYSREG__, /* 0 is reserved as an invalid value */
2939d8415d6SMarc Zyngier MPIDR_EL1, /* MultiProcessor Affinity Register */
2947af0c253SAkihiko Odaki CLIDR_EL1, /* Cache Level ID Register */
2959d8415d6SMarc Zyngier CSSELR_EL1, /* Cache Size Selection Register */
2969d8415d6SMarc Zyngier SCTLR_EL1, /* System Control Register */
2979d8415d6SMarc Zyngier ACTLR_EL1, /* Auxiliary Control Register */
2989d8415d6SMarc Zyngier CPACR_EL1, /* Coprocessor Access Control */
29973433762SDave Martin ZCR_EL1, /* SVE Control */
3009d8415d6SMarc Zyngier TTBR0_EL1, /* Translation Table Base Register 0 */
3019d8415d6SMarc Zyngier TTBR1_EL1, /* Translation Table Base Register 1 */
3029d8415d6SMarc Zyngier TCR_EL1, /* Translation Control Register */
303fbff5606SJoey Gouly TCR2_EL1, /* Extended Translation Control Register */
3049d8415d6SMarc Zyngier ESR_EL1, /* Exception Syndrome Register */
305ef769e32SAdam Buchbinder AFSR0_EL1, /* Auxiliary Fault Status Register 0 */
306ef769e32SAdam Buchbinder AFSR1_EL1, /* Auxiliary Fault Status Register 1 */
3079d8415d6SMarc Zyngier FAR_EL1, /* Fault Address Register */
3089d8415d6SMarc Zyngier MAIR_EL1, /* Memory Attribute Indirection Register */
3099d8415d6SMarc Zyngier VBAR_EL1, /* Vector Base Address Register */
3109d8415d6SMarc Zyngier CONTEXTIDR_EL1, /* Context ID Register */
3119d8415d6SMarc Zyngier TPIDR_EL0, /* Thread ID, User R/W */
3129d8415d6SMarc Zyngier TPIDRRO_EL0, /* Thread ID, User R/O */
3139d8415d6SMarc Zyngier TPIDR_EL1, /* Thread ID, Privileged */
3149d8415d6SMarc Zyngier AMAIR_EL1, /* Aux Memory Attribute Indirection Register */
3159d8415d6SMarc Zyngier CNTKCTL_EL1, /* Timer Control Register (EL1) */
3169d8415d6SMarc Zyngier PAR_EL1, /* Physical Address Register */
3179d8415d6SMarc Zyngier MDSCR_EL1, /* Monitor Debug System Control Register */
3189d8415d6SMarc Zyngier MDCCINT_EL1, /* Monitor Debug Comms Channel Interrupt Enable Reg */
319d42e2671SOliver Upton OSLSR_EL1, /* OS Lock Status Register */
320c773ae2bSJames Morse DISR_EL1, /* Deferred Interrupt Status Register */
3219d8415d6SMarc Zyngier
322ab946834SShannon Zhao /* Performance Monitors Registers */
323ab946834SShannon Zhao PMCR_EL0, /* Control Register */
3243965c3ceSShannon Zhao PMSELR_EL0, /* Event Counter Selection Register */
325051ff581SShannon Zhao PMEVCNTR0_EL0, /* Event Counter Register (0-30) */
326051ff581SShannon Zhao PMEVCNTR30_EL0 = PMEVCNTR0_EL0 + 30,
327051ff581SShannon Zhao PMCCNTR_EL0, /* Cycle Counter Register */
3289feb21acSShannon Zhao PMEVTYPER0_EL0, /* Event Type Register (0-30) */
3299feb21acSShannon Zhao PMEVTYPER30_EL0 = PMEVTYPER0_EL0 + 30,
3309feb21acSShannon Zhao PMCCFILTR_EL0, /* Cycle Count Filter Register */
33196b0eebcSShannon Zhao PMCNTENSET_EL0, /* Count Enable Set Register */
3329db52c78SShannon Zhao PMINTENSET_EL1, /* Interrupt Enable Set Register */
33376d883c4SShannon Zhao PMOVSSET_EL0, /* Overflow Flag Status Set Register */
334d692b8adSShannon Zhao PMUSERENR_EL0, /* User Enable Register */
335ab946834SShannon Zhao
336384b40caSMark Rutland /* Pointer Authentication Registers in a strict increasing order. */
337384b40caSMark Rutland APIAKEYLO_EL1,
338384b40caSMark Rutland APIAKEYHI_EL1,
339384b40caSMark Rutland APIBKEYLO_EL1,
340384b40caSMark Rutland APIBKEYHI_EL1,
341384b40caSMark Rutland APDAKEYLO_EL1,
342384b40caSMark Rutland APDAKEYHI_EL1,
343384b40caSMark Rutland APDBKEYLO_EL1,
344384b40caSMark Rutland APDBKEYHI_EL1,
345384b40caSMark Rutland APGAKEYLO_EL1,
346384b40caSMark Rutland APGAKEYHI_EL1,
347384b40caSMark Rutland
34898909e6dSMarc Zyngier ELR_EL1,
3491bded23eSMarc Zyngier SP_EL1,
350710f1982SMarc Zyngier SPSR_EL1,
35198909e6dSMarc Zyngier
35241ce82f6SMarc Zyngier CNTVOFF_EL2,
35341ce82f6SMarc Zyngier CNTV_CVAL_EL0,
35441ce82f6SMarc Zyngier CNTV_CTL_EL0,
35541ce82f6SMarc Zyngier CNTP_CVAL_EL0,
35641ce82f6SMarc Zyngier CNTP_CTL_EL0,
35741ce82f6SMarc Zyngier
358e1f358b5SSteven Price /* Memory Tagging Extension registers */
359e1f358b5SSteven Price RGSR_EL1, /* Random Allocation Tag Seed Register */
360e1f358b5SSteven Price GCR_EL1, /* Tag Control Register */
361e1f358b5SSteven Price TFSR_EL1, /* Tag Fault Status Register (EL1) */
362e1f358b5SSteven Price TFSRE0_EL1, /* Tag Fault Status Register (EL0) */
363e1f358b5SSteven Price
36486f9de9dSJoey Gouly /* Permission Indirection Extension registers */
36586f9de9dSJoey Gouly PIR_EL1, /* Permission Indirection Register 1 (EL1) */
36686f9de9dSJoey Gouly PIRE0_EL1, /* Permission Indirection Register 0 (EL1) */
36786f9de9dSJoey Gouly
3685305cc2cSMarc Zyngier /* 32bit specific registers. */
3699d8415d6SMarc Zyngier DACR32_EL2, /* Domain Access Control Register */
3709d8415d6SMarc Zyngier IFSR32_EL2, /* Instruction Fault Status Register */
3719d8415d6SMarc Zyngier FPEXC32_EL2, /* Floating-Point Exception Control Register */
3729d8415d6SMarc Zyngier DBGVCR32_EL2, /* Debug Vector Catch Register */
3739d8415d6SMarc Zyngier
3745305cc2cSMarc Zyngier /* EL2 registers */
3755305cc2cSMarc Zyngier VPIDR_EL2, /* Virtualization Processor ID Register */
3765305cc2cSMarc Zyngier VMPIDR_EL2, /* Virtualization Multiprocessor ID Register */
3775305cc2cSMarc Zyngier SCTLR_EL2, /* System Control Register (EL2) */
3785305cc2cSMarc Zyngier ACTLR_EL2, /* Auxiliary Control Register (EL2) */
3795305cc2cSMarc Zyngier HCR_EL2, /* Hypervisor Configuration Register */
3805305cc2cSMarc Zyngier MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */
3815305cc2cSMarc Zyngier CPTR_EL2, /* Architectural Feature Trap Register (EL2) */
3825305cc2cSMarc Zyngier HSTR_EL2, /* Hypervisor System Trap Register */
3835305cc2cSMarc Zyngier HACR_EL2, /* Hypervisor Auxiliary Control Register */
3845305cc2cSMarc Zyngier HCRX_EL2, /* Extended Hypervisor Configuration Register */
3855305cc2cSMarc Zyngier TTBR0_EL2, /* Translation Table Base Register 0 (EL2) */
3865305cc2cSMarc Zyngier TTBR1_EL2, /* Translation Table Base Register 1 (EL2) */
3875305cc2cSMarc Zyngier TCR_EL2, /* Translation Control Register (EL2) */
3885305cc2cSMarc Zyngier VTTBR_EL2, /* Virtualization Translation Table Base Register */
3895305cc2cSMarc Zyngier VTCR_EL2, /* Virtualization Translation Control Register */
3905305cc2cSMarc Zyngier SPSR_EL2, /* EL2 saved program status register */
3915305cc2cSMarc Zyngier ELR_EL2, /* EL2 exception link register */
3925305cc2cSMarc Zyngier AFSR0_EL2, /* Auxiliary Fault Status Register 0 (EL2) */
3935305cc2cSMarc Zyngier AFSR1_EL2, /* Auxiliary Fault Status Register 1 (EL2) */
3945305cc2cSMarc Zyngier ESR_EL2, /* Exception Syndrome Register (EL2) */
3955305cc2cSMarc Zyngier FAR_EL2, /* Fault Address Register (EL2) */
3965305cc2cSMarc Zyngier HPFAR_EL2, /* Hypervisor IPA Fault Address Register */
3975305cc2cSMarc Zyngier MAIR_EL2, /* Memory Attribute Indirection Register (EL2) */
3985305cc2cSMarc Zyngier AMAIR_EL2, /* Auxiliary Memory Attribute Indirection Register (EL2) */
3995305cc2cSMarc Zyngier VBAR_EL2, /* Vector Base Address Register (EL2) */
4005305cc2cSMarc Zyngier RVBAR_EL2, /* Reset Vector Base Address Register */
4015305cc2cSMarc Zyngier CONTEXTIDR_EL2, /* Context ID Register (EL2) */
4025305cc2cSMarc Zyngier TPIDR_EL2, /* EL2 Software Thread ID Register */
4035305cc2cSMarc Zyngier CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */
40481dc9504SMarc Zyngier SP_EL2, /* EL2 Stack Pointer */
40581dc9504SMarc Zyngier HFGRTR_EL2,
40681dc9504SMarc Zyngier HFGWTR_EL2,
40781dc9504SMarc Zyngier HFGITR_EL2,
4085305cc2cSMarc Zyngier HDFGRTR_EL2,
4099d8415d6SMarc Zyngier HDFGWTR_EL2,
4109d8415d6SMarc Zyngier CNTHP_CTL_EL2,
4119d8415d6SMarc Zyngier CNTHP_CVAL_EL2,
4124f8d6632SMarc Zyngier CNTHV_CTL_EL2,
413e47c2055SMarc Zyngier CNTHV_CVAL_EL2,
414e47c2055SMarc Zyngier
415fd85b667SMarc Zyngier NR_SYS_REGS /* Nothing after this line! */
416fd85b667SMarc Zyngier };
417fd85b667SMarc Zyngier
418fd85b667SMarc Zyngier struct kvm_cpu_context {
419e47c2055SMarc Zyngier struct user_pt_regs regs; /* sp = sp_el0 */
420e47c2055SMarc Zyngier
421e47c2055SMarc Zyngier u64 spsr_abt;
4224f8d6632SMarc Zyngier u64 spsr_und;
423c97e166eSJames Morse u64 spsr_irq;
424c97e166eSJames Morse u64 spsr_fiq;
4254f8d6632SMarc Zyngier
4264f8d6632SMarc Zyngier struct user_fpsimd_state fp_regs;
427630a1685SAndrew Murray
428630a1685SAndrew Murray u64 sys_regs[NR_SYS_REGS];
429630a1685SAndrew Murray
430630a1685SAndrew Murray struct kvm_vcpu *__hyp_running_vcpu;
431ff367fe4SDavid Brazdil };
432ff367fe4SDavid Brazdil
433ff367fe4SDavid Brazdil struct kvm_host_data {
43412bdce4fSWill Deacon struct kvm_cpu_context host_ctxt;
435ff367fe4SDavid Brazdil };
436ff367fe4SDavid Brazdil
437ff367fe4SDavid Brazdil struct kvm_host_psci_config {
438ff367fe4SDavid Brazdil /* PSCI version used by host. */
439767c973fSMarc Zyngier u32 version;
440767c973fSMarc Zyngier u32 smccc_version;
441767c973fSMarc Zyngier
442767c973fSMarc Zyngier /* Function IDs used by host if version is v0.1. */
443ff367fe4SDavid Brazdil struct psci_0_1_function_ids function_ids_0_1;
444ff367fe4SDavid Brazdil
445ff367fe4SDavid Brazdil bool psci_0_1_cpu_suspend_implemented;
446ff367fe4SDavid Brazdil bool psci_0_1_cpu_on_implemented;
447ff367fe4SDavid Brazdil bool psci_0_1_cpu_off_implemented;
44861fe0c37SDavid Brazdil bool psci_0_1_migrate_implemented;
44961fe0c37SDavid Brazdil };
45061fe0c37SDavid Brazdil
45161fe0c37SDavid Brazdil extern struct kvm_host_psci_config kvm_nvhe_sym(kvm_host_psci_config);
45261fe0c37SDavid Brazdil #define kvm_host_psci_config CHOOSE_NVHE_SYM(kvm_host_psci_config)
45361fe0c37SDavid Brazdil
454358b28f0SMarc Zyngier extern s64 kvm_nvhe_sym(hyp_physvirt_offset);
455358b28f0SMarc Zyngier #define hyp_physvirt_offset CHOOSE_NVHE_SYM(hyp_physvirt_offset)
456358b28f0SMarc Zyngier
457358b28f0SMarc Zyngier extern u64 kvm_nvhe_sym(hyp_cpu_logical_map)[NR_CPUS];
458358b28f0SMarc Zyngier #define hyp_cpu_logical_map CHOOSE_NVHE_SYM(hyp_cpu_logical_map)
459358b28f0SMarc Zyngier
460358b28f0SMarc Zyngier struct vcpu_reset_state {
4614f8d6632SMarc Zyngier unsigned long pc;
4624f8d6632SMarc Zyngier unsigned long r0;
4630033cd93SMark Brown bool be;
464baa85152SMark Brown bool reset;
465baa85152SMark Brown };
466baa85152SMark Brown
467baa85152SMark Brown struct kvm_vcpu_arch {
468baa85152SMark Brown struct kvm_cpu_context ctxt;
469baa85152SMark Brown
470baa85152SMark Brown /*
471baa85152SMark Brown * Guest floating point state
472baa85152SMark Brown *
473baa85152SMark Brown * The architecture has two main floating point extensions,
474b43b5dd9SDave Martin * the original FPSIMD and SVE. These have overlapping
475baa85152SMark Brown * register views, with the FPSIMD V registers occupying the
476b43b5dd9SDave Martin * low 128 bits of the SVE Z registers. When the core
4770033cd93SMark Brown * floating point code saves the register state of a task it
4784f8d6632SMarc Zyngier * records which view it saved in fp_type.
479a0e50aa3SChristoffer Dall */
480a0e50aa3SChristoffer Dall void *sve_state;
481a0e50aa3SChristoffer Dall enum fp_type fp_type;
4821460b4b2SFuad Tabba unsigned int sve_max_vl;
4834f8d6632SMarc Zyngier u64 svcr;
484d6c850ddSFuad Tabba
485cd496228SFuad Tabba /* Stage 2 paging state used by the hardware on next switch */
4864f8d6632SMarc Zyngier struct kvm_s2_mmu *hw_mmu;
4871460b4b2SFuad Tabba
4881460b4b2SFuad Tabba /* Values of trap registers for the guest. */
4894f8d6632SMarc Zyngier u64 hcr_el2;
4904f8d6632SMarc Zyngier u64 mdcr_el2;
4914f8d6632SMarc Zyngier u64 cptr_el2;
4924f8d6632SMarc Zyngier
493f8077b0dSMarc Zyngier /* Values of trap registers for the host before guest entry. */
494f8077b0dSMarc Zyngier u64 mdcr_el2_host;
495f8077b0dSMarc Zyngier
496f8077b0dSMarc Zyngier /* Exception Information */
497f8077b0dSMarc Zyngier struct kvm_vcpu_fault_info fault;
498f8077b0dSMarc Zyngier
499f8077b0dSMarc Zyngier /* Ownership of the FP regs */
500690bacb8SMarc Zyngier enum {
50154ddda91SMarc Zyngier FP_STATE_FREE,
502690bacb8SMarc Zyngier FP_STATE_HOST_OWNED,
503690bacb8SMarc Zyngier FP_STATE_GUEST_OWNED,
50454ddda91SMarc Zyngier } fp_state;
505690bacb8SMarc Zyngier
506690bacb8SMarc Zyngier /* Configuration flags, set once and for all before the vcpu can run */
50754ddda91SMarc Zyngier u8 cflags;
508690bacb8SMarc Zyngier
50984e690bfSAlex Bennée /* Input flags to the hypervisor code, potentially cleared after use */
5100fa4a313SMarc Zyngier u8 iflags;
5110fa4a313SMarc Zyngier
5120fa4a313SMarc Zyngier /* State flags for kernel bookkeeping, unused by the hypervisor code */
5130fa4a313SMarc Zyngier u8 sflags;
5140fa4a313SMarc Zyngier
5150fa4a313SMarc Zyngier /*
5160fa4a313SMarc Zyngier * Don't run the guest (internal implementation need).
5170c557ed4SMarc Zyngier *
51884e690bfSAlex Bennée * Contrary to the flags above, this is set/cleared outside of
51984e690bfSAlex Bennée * a vcpu context, and thus cannot be mixed with the flags
52084e690bfSAlex Bennée * themselves (or the flag accesses need to be made atomic).
52184e690bfSAlex Bennée */
52284e690bfSAlex Bennée bool pause;
523834bf887SAlex Bennée
524834bf887SAlex Bennée /*
525834bf887SAlex Bennée * We maintain more than a single set of debug registers to support
526834bf887SAlex Bennée * debugging the guest from the host and to maintain separate host and
52784e690bfSAlex Bennée * guest state during world switches. vcpu_debug_state are the debug
52884e690bfSAlex Bennée * registers of the vcpu as the guest sees them. host_debug_state are
52984e690bfSAlex Bennée * the host registers which are saved and restored during
53084e690bfSAlex Bennée * world switches. external_debug_state contains the debug
53184e690bfSAlex Bennée * values we want to debug the guest. This is set via the
53284e690bfSAlex Bennée * KVM_SET_GUEST_DEBUG ioctl.
533834bf887SAlex Bennée *
53484e690bfSAlex Bennée * debug_ptr points to the set of debug registers that should be loaded
535e6b673b7SDave Martin * onto the hardware when running the guest.
53652b28657SQuentin Perret */
537e6b673b7SDave Martin struct kvm_guest_debug_arch *debug_ptr;
538f85279b4SWill Deacon struct kvm_guest_debug_arch vcpu_debug_state;
539f85279b4SWill Deacon struct kvm_guest_debug_arch external_debug_state;
540f85279b4SWill Deacon
541f85279b4SWill Deacon struct user_fpsimd_state *host_fpsimd_state; /* hyp VA */
542f85279b4SWill Deacon struct task_struct *parent_task;
543a1319260SSuzuki K Poulose
544a1319260SSuzuki K Poulose struct {
545f85279b4SWill Deacon /* {Break,watch}point registers */
5464f8d6632SMarc Zyngier struct kvm_guest_debug_arch regs;
5474f8d6632SMarc Zyngier /* Statistical profiling extension */
5484f8d6632SMarc Zyngier u64 pmscr_el1;
5494f8d6632SMarc Zyngier /* Self-hosted trace */
55004fe4726SShannon Zhao u64 trfcr_el1;
5514f8d6632SMarc Zyngier } host_debug_state;
5524f8d6632SMarc Zyngier
553337b99bfSAlex Bennée /* VGIC state */
554337b99bfSAlex Bennée struct vgic_cpu vgic_cpu;
555337b99bfSAlex Bennée struct arch_timer_cpu timer_cpu;
556337b99bfSAlex Bennée struct kvm_pmu pmu;
557337b99bfSAlex Bennée
558337b99bfSAlex Bennée /*
559337b99bfSAlex Bennée * Guest registers we preserve during guest debugging.
560337b99bfSAlex Bennée *
56134fbdee0SReiji Watanabe * These shadow registers are updated by the kvm_handle_sys_reg
562337b99bfSAlex Bennée * trap handler if the guest accesses or updates them while we
563337b99bfSAlex Bennée * are using guest debug.
564b171f9bbSOliver Upton */
565b171f9bbSOliver Upton struct {
5660acc7239SOliver Upton u32 mdscr_el1;
5674f8d6632SMarc Zyngier bool pstate_ss;
5684f8d6632SMarc Zyngier } guest_debug_preserved;
5694f8d6632SMarc Zyngier
5704f8d6632SMarc Zyngier /* vcpu power state */
5714f8d6632SMarc Zyngier struct kvm_mp_state mp_state;
5726c8c0c4dSChen Gang spinlock_t mp_state_lock;
5734f8d6632SMarc Zyngier
5744f8d6632SMarc Zyngier /* Cache some mmu pages needed inside spinlock regions */
5754715c14bSJames Morse struct kvm_mmu_memory_cache mmu_page_cache;
5764715c14bSJames Morse
577d47533daSChristoffer Dall /* feature flags */
578358b28f0SMarc Zyngier DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
579358b28f0SMarc Zyngier
580358b28f0SMarc Zyngier /* Virtual SError ESR to restore when HCR_EL2.VSE is set */
5818564d637SSteven Price u64 vsesr_el2;
5828564d637SSteven Price
5838564d637SSteven Price /* Additional reset state */
5848564d637SSteven Price struct vcpu_reset_state reset_state;
5858564d637SSteven Price
5867af0c253SAkihiko Odaki /* Guest PV state */
5877af0c253SAkihiko Odaki struct {
5887af0c253SAkihiko Odaki u64 last_steal;
5894f8d6632SMarc Zyngier gpa_t base;
5904f8d6632SMarc Zyngier } steal;
591e87abb73SMarc Zyngier
592e87abb73SMarc Zyngier /* Per-vcpu CCSIDR override or NULL */
593e87abb73SMarc Zyngier u32 *ccsidr;
594e87abb73SMarc Zyngier };
595e87abb73SMarc Zyngier
596e87abb73SMarc Zyngier /*
597e87abb73SMarc Zyngier * Each 'flag' is composed of a comma-separated triplet:
598e87abb73SMarc Zyngier *
599e87abb73SMarc Zyngier * - the flag-set it belongs to in the vcpu->arch structure
600e87abb73SMarc Zyngier * - the value for that flag
601e87abb73SMarc Zyngier * - the mask for that flag
602e87abb73SMarc Zyngier *
603e87abb73SMarc Zyngier * __vcpu_single_flag() builds such a triplet for a single-bit flag.
604e87abb73SMarc Zyngier * unpack_vcpu_flag() extract the flag value from the triplet for
605e87abb73SMarc Zyngier * direct use outside of the flag accessors.
606e87abb73SMarc Zyngier */
6075a3984f4SMarc Zyngier #define __vcpu_single_flag(_set, _f) _set, (_f), (_f)
6085a3984f4SMarc Zyngier
6095a3984f4SMarc Zyngier #define __unpack_flag(_set, _f, _m) _f
6105a3984f4SMarc Zyngier #define unpack_vcpu_flag(...) __unpack_flag(__VA_ARGS__)
6115a3984f4SMarc Zyngier
6125a3984f4SMarc Zyngier #define __build_check_flag(v, flagset, f, m) \
6135a3984f4SMarc Zyngier do { \
6145a3984f4SMarc Zyngier typeof(v->arch.flagset) *_fset; \
6155a3984f4SMarc Zyngier \
6165a3984f4SMarc Zyngier /* Check that the flags fit in the mask */ \
617e87abb73SMarc Zyngier BUILD_BUG_ON(HWEIGHT(m) != HWEIGHT((f) | (m))); \
618e87abb73SMarc Zyngier /* Check that the flags fit in the type */ \
6195a3984f4SMarc Zyngier BUILD_BUG_ON((sizeof(*_fset) * 8) <= __fls(m)); \
6205a3984f4SMarc Zyngier } while (0)
62135dcb3acSMarc Zyngier
622e87abb73SMarc Zyngier #define __vcpu_get_flag(v, flagset, f, m) \
623e87abb73SMarc Zyngier ({ \
62435dcb3acSMarc Zyngier __build_check_flag(v, flagset, f, m); \
62535dcb3acSMarc Zyngier \
62635dcb3acSMarc Zyngier READ_ONCE(v->arch.flagset) & (m); \
62735dcb3acSMarc Zyngier })
62835dcb3acSMarc Zyngier
62935dcb3acSMarc Zyngier /*
63035dcb3acSMarc Zyngier * Note that the set/clear accessors must be preempt-safe in order to
63135dcb3acSMarc Zyngier * avoid nesting them with load/put which also manipulate flags...
63235dcb3acSMarc Zyngier */
63335dcb3acSMarc Zyngier #ifdef __KVM_NVHE_HYPERVISOR__
63435dcb3acSMarc Zyngier /* the nVHE hypervisor is always non-preemptible */
63535dcb3acSMarc Zyngier #define __vcpu_flags_preempt_disable()
63635dcb3acSMarc Zyngier #define __vcpu_flags_preempt_enable()
637e87abb73SMarc Zyngier #else
638e87abb73SMarc Zyngier #define __vcpu_flags_preempt_disable() preempt_disable()
639e87abb73SMarc Zyngier #define __vcpu_flags_preempt_enable() preempt_enable()
640e87abb73SMarc Zyngier #endif
6415a3984f4SMarc Zyngier
6425a3984f4SMarc Zyngier #define __vcpu_set_flag(v, flagset, f, m) \
643e87abb73SMarc Zyngier do { \
64435dcb3acSMarc Zyngier typeof(v->arch.flagset) *fset; \
645e87abb73SMarc Zyngier \
646e87abb73SMarc Zyngier __build_check_flag(v, flagset, f, m); \
647e87abb73SMarc Zyngier \
64835dcb3acSMarc Zyngier fset = &v->arch.flagset; \
649e87abb73SMarc Zyngier __vcpu_flags_preempt_disable(); \
650e87abb73SMarc Zyngier if (HWEIGHT(m) > 1) \
651e87abb73SMarc Zyngier *fset &= ~(m); \
652e87abb73SMarc Zyngier *fset |= (f); \
653e87abb73SMarc Zyngier __vcpu_flags_preempt_enable(); \
654e87abb73SMarc Zyngier } while (0)
6555a3984f4SMarc Zyngier
6565a3984f4SMarc Zyngier #define __vcpu_clear_flag(v, flagset, f, m) \
657e87abb73SMarc Zyngier do { \
65835dcb3acSMarc Zyngier typeof(v->arch.flagset) *fset; \
659e87abb73SMarc Zyngier \
66035dcb3acSMarc Zyngier __build_check_flag(v, flagset, f, m); \
661e87abb73SMarc Zyngier \
662e87abb73SMarc Zyngier fset = &v->arch.flagset; \
663e87abb73SMarc Zyngier __vcpu_flags_preempt_disable(); \
664e87abb73SMarc Zyngier *fset &= ~(m); \
665e87abb73SMarc Zyngier __vcpu_flags_preempt_enable(); \
666e87abb73SMarc Zyngier } while (0)
6674c0680d3SMarc Zyngier
6684c0680d3SMarc Zyngier #define vcpu_get_flag(v, ...) __vcpu_get_flag((v), __VA_ARGS__)
6694c0680d3SMarc Zyngier #define vcpu_set_flag(v, ...) __vcpu_set_flag((v), __VA_ARGS__)
6704c0680d3SMarc Zyngier #define vcpu_clear_flag(v, ...) __vcpu_clear_flag((v), __VA_ARGS__)
6714c0680d3SMarc Zyngier
6724c0680d3SMarc Zyngier /* SVE exposed to guest */
6734c0680d3SMarc Zyngier #define GUEST_HAS_SVE __vcpu_single_flag(cflags, BIT(0))
674699bb2e0SMarc Zyngier /* SVE config completed */
675699bb2e0SMarc Zyngier #define VCPU_SVE_FINALIZED __vcpu_single_flag(cflags, BIT(1))
676699bb2e0SMarc Zyngier /* PTRAUTH exposed to guest */
677699bb2e0SMarc Zyngier #define GUEST_HAS_PTRAUTH __vcpu_single_flag(cflags, BIT(2))
678699bb2e0SMarc Zyngier /* KVM_ARM_VCPU_INIT completed */
679699bb2e0SMarc Zyngier #define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(3))
680699bb2e0SMarc Zyngier
681699bb2e0SMarc Zyngier /* Exception pending */
682699bb2e0SMarc Zyngier #define PENDING_EXCEPTION __vcpu_single_flag(iflags, BIT(0))
683699bb2e0SMarc Zyngier /*
684699bb2e0SMarc Zyngier * PC increment. Overlaps with EXCEPT_MASK on purpose so that it can't
685699bb2e0SMarc Zyngier * be set together with an exception...
686699bb2e0SMarc Zyngier */
687699bb2e0SMarc Zyngier #define INCREMENT_PC __vcpu_single_flag(iflags, BIT(1))
688699bb2e0SMarc Zyngier /* Target EL/MODE (not a single flag, but let's abuse the macro) */
689699bb2e0SMarc Zyngier #define EXCEPT_MASK __vcpu_single_flag(iflags, GENMASK(3, 1))
690699bb2e0SMarc Zyngier
691699bb2e0SMarc Zyngier /* Helpers to encode exceptions with minimum fuss */
692699bb2e0SMarc Zyngier #define __EXCEPT_MASK_VAL unpack_vcpu_flag(EXCEPT_MASK)
693699bb2e0SMarc Zyngier #define __EXCEPT_SHIFT __builtin_ctzl(__EXCEPT_MASK_VAL)
694699bb2e0SMarc Zyngier #define __vcpu_except_flags(_f) iflags, (_f << __EXCEPT_SHIFT), __EXCEPT_MASK_VAL
695699bb2e0SMarc Zyngier
696699bb2e0SMarc Zyngier /*
697699bb2e0SMarc Zyngier * When PENDING_EXCEPTION is set, EXCEPT_MASK can take the following
698699bb2e0SMarc Zyngier * values:
699699bb2e0SMarc Zyngier *
700699bb2e0SMarc Zyngier * For AArch32 EL1:
701699bb2e0SMarc Zyngier */
702699bb2e0SMarc Zyngier #define EXCEPT_AA32_UND __vcpu_except_flags(0)
70347f3a2fcSJintack Lim #define EXCEPT_AA32_IABT __vcpu_except_flags(1)
704699bb2e0SMarc Zyngier #define EXCEPT_AA32_DABT __vcpu_except_flags(2)
705699bb2e0SMarc Zyngier /* For AArch64: */
706699bb2e0SMarc Zyngier #define EXCEPT_AA64_EL1_SYNC __vcpu_except_flags(0)
707699bb2e0SMarc Zyngier #define EXCEPT_AA64_EL1_IRQ __vcpu_except_flags(1)
708b1da4908SMarc Zyngier #define EXCEPT_AA64_EL1_FIQ __vcpu_except_flags(2)
709b1da4908SMarc Zyngier #define EXCEPT_AA64_EL1_SERR __vcpu_except_flags(3)
710b1da4908SMarc Zyngier /* For AArch64 with NV: */
711b1da4908SMarc Zyngier #define EXCEPT_AA64_EL2_SYNC __vcpu_except_flags(4)
712b1da4908SMarc Zyngier #define EXCEPT_AA64_EL2_IRQ __vcpu_except_flags(5)
713b1da4908SMarc Zyngier #define EXCEPT_AA64_EL2_FIQ __vcpu_except_flags(6)
714d9552fe1SMarc Zyngier #define EXCEPT_AA64_EL2_SERR __vcpu_except_flags(7)
715d9552fe1SMarc Zyngier /* Guest debug is live */
716e87abb73SMarc Zyngier #define DEBUG_DIRTY __vcpu_single_flag(iflags, BIT(4))
7170affa37fSMarc Zyngier /* Save SPE context if active */
7180affa37fSMarc Zyngier #define DEBUG_STATE_SAVE_SPE __vcpu_single_flag(iflags, BIT(5))
7190affa37fSMarc Zyngier /* Save TRBE context if active */
7200affa37fSMarc Zyngier #define DEBUG_STATE_SAVE_TRBE __vcpu_single_flag(iflags, BIT(6))
721aff3ccd7SMarc Zyngier /* vcpu running in HYP context */
722aff3ccd7SMarc Zyngier #define VCPU_HYP_CONTEXT __vcpu_single_flag(iflags, BIT(7))
723eebc538dSMarc Zyngier
724eebc538dSMarc Zyngier /* SVE enabled for host EL0 */
72530b6ab45SMarc Zyngier #define HOST_SVE_ENABLED __vcpu_single_flag(sflags, BIT(0))
72630b6ab45SMarc Zyngier /* SME enabled for EL0 */
727370531d1SReiji Watanabe #define HOST_SME_ENABLED __vcpu_single_flag(sflags, BIT(1))
728370531d1SReiji Watanabe /* Physical CPU not in supported_cpus */
7290c2f9acfSReiji Watanabe #define ON_UNSUPPORTED_CPU __vcpu_single_flag(sflags, BIT(2))
7300c2f9acfSReiji Watanabe /* WFIT instruction trapped */
731b321c31cSMarc Zyngier #define IN_WFIT __vcpu_single_flag(sflags, BIT(3))
732b321c31cSMarc Zyngier /* vcpu system registers loaded on physical CPU */
733370531d1SReiji Watanabe #define SYSREGS_ON_CPU __vcpu_single_flag(sflags, BIT(4))
7340affa37fSMarc Zyngier /* Software step state is Active-pending */
735b43b5dd9SDave Martin #define DBG_SS_ACTIVE_PENDING __vcpu_single_flag(sflags, BIT(5))
736985d3a1bSMarc Zyngier /* PMUSERENR for the guest EL0 is on physical CPU */
737985d3a1bSMarc Zyngier #define PMUSERENR_ON_CPU __vcpu_single_flag(sflags, BIT(6))
738b43b5dd9SDave Martin /* WFI instruction trapped */
739468f3477SMarc Zyngier #define IN_WFI __vcpu_single_flag(sflags, BIT(7))
740b3eb56b6SDave Martin
741e1c9c983SDave Martin
742e1c9c983SDave Martin /* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
743e1c9c983SDave Martin #define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \
744e1c9c983SDave Martin sve_ffr_offset((vcpu)->arch.sve_max_vl))
745e1c9c983SDave Martin
746e1c9c983SDave Martin #define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl)
747e1c9c983SDave Martin
748468f3477SMarc Zyngier #define vcpu_sve_state_size(vcpu) ({ \
749e1c9c983SDave Martin size_t __size_ret; \
750e1c9c983SDave Martin unsigned int __vcpu_vq; \
751e1c9c983SDave Martin \
752e1c9c983SDave Martin if (WARN_ON(!sve_vl_valid((vcpu)->arch.sve_max_vl))) { \
753e1c9c983SDave Martin __size_ret = 0; \
754e1c9c983SDave Martin } else { \
755892fd259SMarc Zyngier __vcpu_vq = vcpu_sve_max_vq(vcpu); \
756892fd259SMarc Zyngier __size_ret = SVE_SIG_REGS_SIZE(__vcpu_vq); \
757892fd259SMarc Zyngier } \
758892fd259SMarc Zyngier \
7591765edbaSDave Martin __size_ret; \
7601765edbaSDave Martin })
7614c0680d3SMarc Zyngier
762fa89d31cSDave Martin #define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \
763bf4086b1SMarc Zyngier KVM_GUESTDBG_USE_SW_BP | \
764bf4086b1SMarc Zyngier KVM_GUESTDBG_USE_HW | \
765bf4086b1SMarc Zyngier KVM_GUESTDBG_SINGLESTEP)
766bf4086b1SMarc Zyngier
7674c0680d3SMarc Zyngier #define vcpu_has_sve(vcpu) (system_supports_sve() && \
768bf4086b1SMarc Zyngier vcpu_get_flag(vcpu, GUEST_HAS_SVE))
769bf4086b1SMarc Zyngier
770bf4086b1SMarc Zyngier #ifdef CONFIG_ARM64_PTR_AUTH
771b890d75cSAmit Daniel Kachhap #define vcpu_has_ptrauth(vcpu) \
772583cda1bSAlexandru Elisei ((cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH) || \
773aff3ccd7SMarc Zyngier cpus_have_final_cap(ARM64_HAS_GENERIC_AUTH)) && \
774583cda1bSAlexandru Elisei vcpu_get_flag(vcpu, GUEST_HAS_PTRAUTH))
775583cda1bSAlexandru Elisei #else
776aff3ccd7SMarc Zyngier #define vcpu_has_ptrauth(vcpu) false
777583cda1bSAlexandru Elisei #endif
778583cda1bSAlexandru Elisei
779aff3ccd7SMarc Zyngier #define vcpu_on_unsupported_cpu(vcpu) \
780583cda1bSAlexandru Elisei vcpu_get_flag(vcpu, ON_UNSUPPORTED_CPU)
781e47c2055SMarc Zyngier
7828d404c4cSChristoffer Dall #define vcpu_set_on_unsupported_cpu(vcpu) \
7838d404c4cSChristoffer Dall vcpu_set_flag(vcpu, ON_UNSUPPORTED_CPU)
7841b422dd7SMarc Zyngier
7851b422dd7SMarc Zyngier #define vcpu_clear_on_unsupported_cpu(vcpu) \
7861b422dd7SMarc Zyngier vcpu_clear_flag(vcpu, ON_UNSUPPORTED_CPU)
7871b422dd7SMarc Zyngier
7881b422dd7SMarc Zyngier #define vcpu_gp_regs(v) (&(v)->arch.ctxt.regs)
7898d404c4cSChristoffer Dall
7901b422dd7SMarc Zyngier /*
7911b422dd7SMarc Zyngier * Only use __vcpu_sys_reg/ctxt_sys_reg if you know you want the
7921b422dd7SMarc Zyngier * memory backed version of a register, and not the one most recently
7931b422dd7SMarc Zyngier * accessed by a running VCPU. For example, for userspace access or
7941b422dd7SMarc Zyngier * for system registers that are never context switched, but only
7958d404c4cSChristoffer Dall * emulated.
796da6f1666SChristoffer Dall */
797d47533daSChristoffer Dall #define __ctxt_sys_reg(c,r) (&(c)->sys_regs[(r)])
7988d404c4cSChristoffer Dall
79921c81001SMarc Zyngier #define ctxt_sys_reg(c,r) (*__ctxt_sys_reg(c,r))
80021c81001SMarc Zyngier
80121c81001SMarc Zyngier #define __vcpu_sys_reg(v,r) (ctxt_sys_reg(&(v)->arch.ctxt, (r)))
80221c81001SMarc Zyngier
80321c81001SMarc Zyngier u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg);
80421c81001SMarc Zyngier void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg);
80521c81001SMarc Zyngier
__vcpu_read_sys_reg_from_cpu(int reg,u64 * val)80621c81001SMarc Zyngier static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val)
80721c81001SMarc Zyngier {
80821c81001SMarc Zyngier /*
80921c81001SMarc Zyngier * *** VHE ONLY ***
81021c81001SMarc Zyngier *
81121c81001SMarc Zyngier * System registers listed in the switch are not saved on every
81221c81001SMarc Zyngier * exit from the guest but are only saved on vcpu_put.
81321c81001SMarc Zyngier *
81421c81001SMarc Zyngier * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
81521c81001SMarc Zyngier * should never be listed below, because the guest cannot modify its
81621c81001SMarc Zyngier * own MPIDR_EL1 and MPIDR_EL1 is accessed for VCPU A from VCPU B's
81721c81001SMarc Zyngier * thread when emulating cross-VCPU communication.
81821c81001SMarc Zyngier */
81921c81001SMarc Zyngier if (!has_vhe())
82021c81001SMarc Zyngier return false;
82121c81001SMarc Zyngier
82221c81001SMarc Zyngier switch (reg) {
82321c81001SMarc Zyngier case SCTLR_EL1: *val = read_sysreg_s(SYS_SCTLR_EL12); break;
82421c81001SMarc Zyngier case CPACR_EL1: *val = read_sysreg_s(SYS_CPACR_EL12); break;
82521c81001SMarc Zyngier case TTBR0_EL1: *val = read_sysreg_s(SYS_TTBR0_EL12); break;
82621c81001SMarc Zyngier case TTBR1_EL1: *val = read_sysreg_s(SYS_TTBR1_EL12); break;
82721c81001SMarc Zyngier case TCR_EL1: *val = read_sysreg_s(SYS_TCR_EL12); break;
82821c81001SMarc Zyngier case ESR_EL1: *val = read_sysreg_s(SYS_ESR_EL12); break;
82921c81001SMarc Zyngier case AFSR0_EL1: *val = read_sysreg_s(SYS_AFSR0_EL12); break;
83021c81001SMarc Zyngier case AFSR1_EL1: *val = read_sysreg_s(SYS_AFSR1_EL12); break;
83121c81001SMarc Zyngier case FAR_EL1: *val = read_sysreg_s(SYS_FAR_EL12); break;
83221c81001SMarc Zyngier case MAIR_EL1: *val = read_sysreg_s(SYS_MAIR_EL12); break;
83321c81001SMarc Zyngier case VBAR_EL1: *val = read_sysreg_s(SYS_VBAR_EL12); break;
83421c81001SMarc Zyngier case CONTEXTIDR_EL1: *val = read_sysreg_s(SYS_CONTEXTIDR_EL12);break;
83521c81001SMarc Zyngier case TPIDR_EL0: *val = read_sysreg_s(SYS_TPIDR_EL0); break;
83621c81001SMarc Zyngier case TPIDRRO_EL0: *val = read_sysreg_s(SYS_TPIDRRO_EL0); break;
83721c81001SMarc Zyngier case TPIDR_EL1: *val = read_sysreg_s(SYS_TPIDR_EL1); break;
83821c81001SMarc Zyngier case AMAIR_EL1: *val = read_sysreg_s(SYS_AMAIR_EL12); break;
83921c81001SMarc Zyngier case CNTKCTL_EL1: *val = read_sysreg_s(SYS_CNTKCTL_EL12); break;
84021c81001SMarc Zyngier case ELR_EL1: *val = read_sysreg_s(SYS_ELR_EL12); break;
84121c81001SMarc Zyngier case PAR_EL1: *val = read_sysreg_par(); break;
84221c81001SMarc Zyngier case DACR32_EL2: *val = read_sysreg_s(SYS_DACR32_EL2); break;
84321c81001SMarc Zyngier case IFSR32_EL2: *val = read_sysreg_s(SYS_IFSR32_EL2); break;
84421c81001SMarc Zyngier case DBGVCR32_EL2: *val = read_sysreg_s(SYS_DBGVCR32_EL2); break;
84521c81001SMarc Zyngier default: return false;
84621c81001SMarc Zyngier }
84721c81001SMarc Zyngier
84821c81001SMarc Zyngier return true;
84921c81001SMarc Zyngier }
85021c81001SMarc Zyngier
__vcpu_write_sys_reg_to_cpu(u64 val,int reg)85121c81001SMarc Zyngier static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg)
85221c81001SMarc Zyngier {
85321c81001SMarc Zyngier /*
85421c81001SMarc Zyngier * *** VHE ONLY ***
85521c81001SMarc Zyngier *
85621c81001SMarc Zyngier * System registers listed in the switch are not restored on every
85721c81001SMarc Zyngier * entry to the guest but are only restored on vcpu_load.
85821c81001SMarc Zyngier *
85921c81001SMarc Zyngier * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
86021c81001SMarc Zyngier * should never be listed below, because the MPIDR should only be set
86121c81001SMarc Zyngier * once, before running the VCPU, and never changed later.
86221c81001SMarc Zyngier */
86321c81001SMarc Zyngier if (!has_vhe())
86421c81001SMarc Zyngier return false;
86521c81001SMarc Zyngier
86621c81001SMarc Zyngier switch (reg) {
86721c81001SMarc Zyngier case SCTLR_EL1: write_sysreg_s(val, SYS_SCTLR_EL12); break;
86821c81001SMarc Zyngier case CPACR_EL1: write_sysreg_s(val, SYS_CPACR_EL12); break;
86921c81001SMarc Zyngier case TTBR0_EL1: write_sysreg_s(val, SYS_TTBR0_EL12); break;
87021c81001SMarc Zyngier case TTBR1_EL1: write_sysreg_s(val, SYS_TTBR1_EL12); break;
87121c81001SMarc Zyngier case TCR_EL1: write_sysreg_s(val, SYS_TCR_EL12); break;
87221c81001SMarc Zyngier case ESR_EL1: write_sysreg_s(val, SYS_ESR_EL12); break;
87321c81001SMarc Zyngier case AFSR0_EL1: write_sysreg_s(val, SYS_AFSR0_EL12); break;
87421c81001SMarc Zyngier case AFSR1_EL1: write_sysreg_s(val, SYS_AFSR1_EL12); break;
87521c81001SMarc Zyngier case FAR_EL1: write_sysreg_s(val, SYS_FAR_EL12); break;
87621c81001SMarc Zyngier case MAIR_EL1: write_sysreg_s(val, SYS_MAIR_EL12); break;
87721c81001SMarc Zyngier case VBAR_EL1: write_sysreg_s(val, SYS_VBAR_EL12); break;
87821c81001SMarc Zyngier case CONTEXTIDR_EL1: write_sysreg_s(val, SYS_CONTEXTIDR_EL12);break;
87921c81001SMarc Zyngier case TPIDR_EL0: write_sysreg_s(val, SYS_TPIDR_EL0); break;
88021c81001SMarc Zyngier case TPIDRRO_EL0: write_sysreg_s(val, SYS_TPIDRRO_EL0); break;
88121c81001SMarc Zyngier case TPIDR_EL1: write_sysreg_s(val, SYS_TPIDR_EL1); break;
88221c81001SMarc Zyngier case AMAIR_EL1: write_sysreg_s(val, SYS_AMAIR_EL12); break;
88321c81001SMarc Zyngier case CNTKCTL_EL1: write_sysreg_s(val, SYS_CNTKCTL_EL12); break;
88421c81001SMarc Zyngier case ELR_EL1: write_sysreg_s(val, SYS_ELR_EL12); break;
88521c81001SMarc Zyngier case PAR_EL1: write_sysreg_s(val, SYS_PAR_EL1); break;
88621c81001SMarc Zyngier case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); break;
88721c81001SMarc Zyngier case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break;
8884f8d6632SMarc Zyngier case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break;
8890193cc90SJing Zhang default: return false;
8904f8d6632SMarc Zyngier }
8914f8d6632SMarc Zyngier
8924f8d6632SMarc Zyngier return true;
8930193cc90SJing Zhang }
8948a7e75d4SSuraj Jitindar Singh
895b19e6892SAmit Tomar struct kvm_vm_stat {
896b19e6892SAmit Tomar struct kvm_vm_stat_generic generic;
897b19e6892SAmit Tomar };
898b19e6892SAmit Tomar
899fe5161d2SOliver Upton struct kvm_vcpu_stat {
900b19e6892SAmit Tomar struct kvm_vcpu_stat_generic generic;
9014f8d6632SMarc Zyngier u64 hvc_exit_stat;
9024f8d6632SMarc Zyngier u64 wfe_exit_stat;
90308e873cbSYueHaibing u64 wfi_exit_stat;
9044f8d6632SMarc Zyngier u64 mmio_exit_user;
9054f8d6632SMarc Zyngier u64 mmio_exit_kernel;
9064f8d6632SMarc Zyngier u64 signal_exits;
9074f8d6632SMarc Zyngier u64 exits;
9086ac4a5acSMarc Zyngier };
9096ac4a5acSMarc Zyngier
9106ac4a5acSMarc Zyngier unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
9116ac4a5acSMarc Zyngier int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
912539aee0eSJames Morse int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
913b7b27facSDongjiu Geng int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
914b7b27facSDongjiu Geng
915539aee0eSJames Morse unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu);
916b7b27facSDongjiu Geng int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices);
9174f8d6632SMarc Zyngier
9184f8d6632SMarc Zyngier int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
9194f8d6632SMarc Zyngier struct kvm_vcpu_events *events);
920b13216cfSChristoffer Dall
921b13216cfSChristoffer Dall int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
9224f8d6632SMarc Zyngier struct kvm_vcpu_events *events);
923cc5705fbSMarc Zyngier
924cc5705fbSMarc Zyngier #define KVM_ARCH_WANT_MMU_NOTIFIER
92540a50853SQuentin Perret
926f50b6f6aSAndrew Scull void kvm_arm_halt_guest(struct kvm *kvm);
927f50b6f6aSAndrew Scull void kvm_arm_resume_guest(struct kvm *kvm);
92805469831SAndrew Scull
92905469831SAndrew Scull #define vcpu_has_run_once(vcpu) !!rcu_access_pointer((vcpu)->pid)
93005469831SAndrew Scull
93105469831SAndrew Scull #ifndef __KVM_NVHE_HYPERVISOR__
93205469831SAndrew Scull #define kvm_call_hyp_nvhe(f, ...) \
93305469831SAndrew Scull ({ \
93405469831SAndrew Scull struct arm_smccc_res res; \
935f50b6f6aSAndrew Scull \
936f50b6f6aSAndrew Scull arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \
93718fc7bf8SMarc Zyngier ##__VA_ARGS__, &res); \
93818fc7bf8SMarc Zyngier WARN_ON(res.a0 != SMCCC_RET_SUCCESS); \
93918fc7bf8SMarc Zyngier \
94018fc7bf8SMarc Zyngier res.a1; \
94118fc7bf8SMarc Zyngier })
94218fc7bf8SMarc Zyngier
94318fc7bf8SMarc Zyngier /*
94418fc7bf8SMarc Zyngier * The couple of isb() below are there to guarantee the same behaviour
94518fc7bf8SMarc Zyngier * on VHE as on !VHE, where the eret to EL1 acts as a context
94618fc7bf8SMarc Zyngier * synchronization event.
94718fc7bf8SMarc Zyngier */
948f50b6f6aSAndrew Scull #define kvm_call_hyp(f, ...) \
94918fc7bf8SMarc Zyngier do { \
95018fc7bf8SMarc Zyngier if (has_vhe()) { \
95118fc7bf8SMarc Zyngier f(__VA_ARGS__); \
95218fc7bf8SMarc Zyngier isb(); \
95318fc7bf8SMarc Zyngier } else { \
95418fc7bf8SMarc Zyngier kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \
95518fc7bf8SMarc Zyngier } \
95618fc7bf8SMarc Zyngier } while(0)
95718fc7bf8SMarc Zyngier
95818fc7bf8SMarc Zyngier #define kvm_call_hyp_ret(f, ...) \
95918fc7bf8SMarc Zyngier ({ \
96005469831SAndrew Scull typeof(f(__VA_ARGS__)) ret; \
96118fc7bf8SMarc Zyngier \
96218fc7bf8SMarc Zyngier if (has_vhe()) { \
96318fc7bf8SMarc Zyngier ret = f(__VA_ARGS__); \
96418fc7bf8SMarc Zyngier isb(); \
96540a50853SQuentin Perret } else { \
96640a50853SQuentin Perret ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \
96740a50853SQuentin Perret } \
96840a50853SQuentin Perret \
96940a50853SQuentin Perret ret; \
97022b39ca3SMarc Zyngier })
971cf5d3188SChristoffer Dall #else /* __KVM_NVHE_HYPERVISOR__ */
9724f8d6632SMarc Zyngier #define kvm_call_hyp(f, ...) f(__VA_ARGS__)
97374cc7e0cSTianjia Zhang #define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__)
97474cc7e0cSTianjia Zhang #define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__)
9754f8d6632SMarc Zyngier #endif /* __KVM_NVHE_HYPERVISOR__ */
9766ac4a5acSMarc Zyngier
9776ac4a5acSMarc Zyngier int handle_exit(struct kvm_vcpu *vcpu, int exception_index);
9786ac4a5acSMarc Zyngier void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index);
9796ac4a5acSMarc Zyngier
9806ac4a5acSMarc Zyngier int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu);
9816ac4a5acSMarc Zyngier int kvm_handle_cp14_32(struct kvm_vcpu *vcpu);
9829369bc5cSOliver Upton int kvm_handle_cp14_64(struct kvm_vcpu *vcpu);
9836ac4a5acSMarc Zyngier int kvm_handle_cp15_32(struct kvm_vcpu *vcpu);
9846ac4a5acSMarc Zyngier int kvm_handle_cp15_64(struct kvm_vcpu *vcpu);
9856ac4a5acSMarc Zyngier int kvm_handle_sys_reg(struct kvm_vcpu *vcpu);
9868d20bd63SSean Christopherson int kvm_handle_cp10_id(struct kvm_vcpu *vcpu);
9876ac4a5acSMarc Zyngier
98896906a91SMarc Zyngier void kvm_reset_sys_regs(struct kvm_vcpu *vcpu);
98996906a91SMarc Zyngier
99096906a91SMarc Zyngier int __init kvm_sys_reg_table_init(void);
9910e20f5e2SMarc Zyngier int __init populate_nv_trap_config(void);
9920e20f5e2SMarc Zyngier
9930e20f5e2SMarc Zyngier bool lock_all_vcpus(struct kvm *kvm);
9940e20f5e2SMarc Zyngier void unlock_all_vcpus(struct kvm *kvm);
99574cc7e0cSTianjia Zhang
99674cc7e0cSTianjia Zhang /* MMIO helpers */
9970e20f5e2SMarc Zyngier void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data);
998e1bfc245SSean Christopherson unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len);
999e1bfc245SSean Christopherson
1000e1bfc245SSean Christopherson int kvm_handle_mmio_return(struct kvm_vcpu *vcpu);
1001e1bfc245SSean Christopherson int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa);
1002e1bfc245SSean Christopherson
1003e1bfc245SSean Christopherson /*
1004e1bfc245SSean Christopherson * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event,
1005e1bfc245SSean Christopherson * arrived in guest context. For arm64, any event that arrives while a vCPU is
1006e1bfc245SSean Christopherson * loaded is considered to be "in guest".
1007e1bfc245SSean Christopherson */
kvm_arch_pmi_in_guest(struct kvm_vcpu * vcpu)1008b48c1a45SSteven Price static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu)
10098564d637SSteven Price {
10108564d637SSteven Price return IS_ENABLED(CONFIG_GUEST_PERF_EVENTS) && !!vcpu;
10118564d637SSteven Price }
1012004a0124SAndrew Jones
101358772e9aSSteven Price long kvm_hypercall_pv_features(struct kvm_vcpu *vcpu);
101458772e9aSSteven Price gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu);
101558772e9aSSteven Price void kvm_update_stolen_time(struct kvm_vcpu *vcpu);
101658772e9aSSteven Price
101758772e9aSSteven Price bool kvm_arm_pvtime_supported(void);
101858772e9aSSteven Price int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu,
101958772e9aSSteven Price struct kvm_device_attr *attr);
10208d20bd63SSean Christopherson int kvm_arm_pvtime_get_attr(struct kvm_vcpu *vcpu,
10218d20bd63SSean Christopherson struct kvm_device_attr *attr);
10228d20bd63SSean Christopherson int kvm_arm_pvtime_has_attr(struct kvm_vcpu *vcpu,
102341783839SShameer Kolothum struct kvm_device_attr *attr);
1024100b4f09SShameer Kolothum
102541783839SShameer Kolothum extern unsigned int __ro_after_init kvm_arm_vmid_bits;
10268564d637SSteven Price int __init kvm_arm_vmid_alloc_init(void);
10278564d637SSteven Price void __init kvm_arm_vmid_alloc_free(void);
1028cecafc0aSYu Zhang void kvm_arm_vmid_update(struct kvm_vmid *kvm_vmid);
10298564d637SSteven Price void kvm_arm_vmid_clear_active(void);
10308564d637SSteven Price
kvm_arm_pvtime_vcpu_init(struct kvm_vcpu_arch * vcpu_arch)10318564d637SSteven Price static inline void kvm_arm_pvtime_vcpu_init(struct kvm_vcpu_arch *vcpu_arch)
10328564d637SSteven Price {
1033cecafc0aSYu Zhang vcpu_arch->steal.base = INVALID_GPA;
10348564d637SSteven Price }
1035b48c1a45SSteven Price
kvm_arm_is_pvtime_enabled(struct kvm_vcpu_arch * vcpu_arch)1036b7b27facSDongjiu Geng static inline bool kvm_arm_is_pvtime_enabled(struct kvm_vcpu_arch *vcpu_arch)
1037b7b27facSDongjiu Geng {
10384429fc64SAndre Przywara return (vcpu_arch->steal.base != INVALID_GPA);
10394429fc64SAndre Przywara }
104014ef9d04SMarc Zyngier
10414464e210SChristoffer Dall void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 syndrome);
10421e0cf16cSMarc Zyngier
104332f13955SMarc Zyngier struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
104432f13955SMarc Zyngier
104571071acfSMarc Zyngier DECLARE_KVM_HYP_PER_CPU(struct kvm_host_data, kvm_host_data);
104632f13955SMarc Zyngier
kvm_init_host_cpu_context(struct kvm_cpu_context * cpu_ctxt)104732f13955SMarc Zyngier static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt)
10485bdf3437SJames Morse {
10495bdf3437SJames Morse /* The host's MPIDR is immutable, so let's set it up at boot time */
10505bdf3437SJames Morse ctxt_sys_reg(cpu_ctxt, MPIDR_EL1) = read_cpuid_mpidr();
10515bdf3437SJames Morse }
10525bdf3437SJames Morse
kvm_system_needs_idmapped_vectors(void)1053384b40caSMark Rutland static inline bool kvm_system_needs_idmapped_vectors(void)
1054384b40caSMark Rutland {
10550865e636SRadim Krčmář return cpus_have_const_cap(ARM64_SPECTRE_V3A);
10560865e636SRadim Krčmář }
10570865e636SRadim Krčmář
kvm_arch_sync_events(struct kvm * kvm)105856c7f5e7SAlex Bennée static inline void kvm_arch_sync_events(struct kvm *kvm) {}
kvm_arch_sched_in(struct kvm_vcpu * vcpu,int cpu)1059263d6287SAlexandru Elisei static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
106056c7f5e7SAlex Bennée
106156c7f5e7SAlex Bennée void kvm_arm_init_debug(void);
106284e690bfSAlex Bennée void kvm_arm_vcpu_init_debug(struct kvm_vcpu *vcpu);
10637dabf02fSOliver Upton void kvm_arm_setup_debug(struct kvm_vcpu *vcpu);
10647dabf02fSOliver Upton void kvm_arm_clear_debug(struct kvm_vcpu *vcpu);
1065187de7c2SMark Brown void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu);
10667dabf02fSOliver Upton
1067bb0c70bcSShannon Zhao #define kvm_vcpu_os_lock_enabled(vcpu) \
1068bb0c70bcSShannon Zhao (!!(__vcpu_sys_reg(vcpu, OSLSR_EL1) & OSLSR_EL1_OSLK))
1069bb0c70bcSShannon Zhao
1070bb0c70bcSShannon Zhao int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
1071bb0c70bcSShannon Zhao struct kvm_device_attr *attr);
1072bb0c70bcSShannon Zhao int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
107356c7f5e7SAlex Bennée struct kvm_device_attr *attr);
10742def950cSThomas Huth int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
1075f0376edbSSteven Price struct kvm_device_attr *attr);
107630ec7997SMarc Zyngier
107730ec7997SMarc Zyngier int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm,
1078f0376edbSSteven Price struct kvm_arm_copy_mte_tags *copy_tags);
1079e6b673b7SDave Martin int kvm_vm_ioctl_set_counter_offset(struct kvm *kvm,
1080e6b673b7SDave Martin struct kvm_arm_counter_offset *offset);
1081e6b673b7SDave Martin
1082af9a0e21SMarc Zyngier /* Guest/host FPSIMD coordination helpers */
1083e6b673b7SDave Martin int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu);
1084e6b673b7SDave Martin void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu);
108552b28657SQuentin Perret void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu);
1086e6b673b7SDave Martin void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu);
1087eb41238cSAndrew Murray void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu);
1088eb41238cSAndrew Murray void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu);
1089435e53fbSAndrew Murray
kvm_pmu_counter_deferred(struct perf_event_attr * attr)1090eb41238cSAndrew Murray static inline bool kvm_pmu_counter_deferred(struct perf_event_attr *attr)
1091eb41238cSAndrew Murray {
1092d2602bb4SSuzuki K Poulose return (!has_vhe() && attr->exclude_host);
1093d2602bb4SSuzuki K Poulose }
1094d2602bb4SSuzuki K Poulose
1095d2602bb4SSuzuki K Poulose /* Flags for host debug state */
1096052f064dSMarc Zyngier void kvm_arch_vcpu_load_debug_state_flags(struct kvm_vcpu *vcpu);
1097eb41238cSAndrew Murray void kvm_arch_vcpu_put_debug_state_flags(struct kvm_vcpu *vcpu);
1098eb41238cSAndrew Murray
10990c2f9acfSReiji Watanabe #ifdef CONFIG_KVM
1100eb41238cSAndrew Murray void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr);
1101eb41238cSAndrew Murray void kvm_clr_pmu_events(u32 clr);
1102eb41238cSAndrew Murray bool kvm_set_pmuserenr(u64 val);
11030c2f9acfSReiji Watanabe #else
kvm_set_pmu_events(u32 set,struct perf_event_attr * attr)11040c2f9acfSReiji Watanabe static inline void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) {}
kvm_clr_pmu_events(u32 clr)11050c2f9acfSReiji Watanabe static inline void kvm_clr_pmu_events(u32 clr) {}
kvm_set_pmuserenr(u64 val)11060c2f9acfSReiji Watanabe static inline bool kvm_set_pmuserenr(u64 val)
1107e6b673b7SDave Martin {
110817eed27bSDave Martin return false;
110913aeb9b4SDavid Brazdil }
111013aeb9b4SDavid Brazdil #endif
1111bc192ceeSChristoffer Dall
11128d20bd63SSean Christopherson void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu);
11130f62f0e9SSuzuki K Poulose void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu);
1114d1e5b0e9SMarc Orr
1115d1e5b0e9SMarc Orr int __init kvm_set_ipa_limit(void);
1116d1e5b0e9SMarc Orr
11172ea7f655SFuad Tabba #define __KVM_HAVE_ARCH_VM_ALLOC
11182ea7f655SFuad Tabba struct kvm *kvm_arch_alloc_vm(void);
11192ea7f655SFuad Tabba
11202ea7f655SFuad Tabba #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS
11212ea7f655SFuad Tabba
11222a0c3433SFuad Tabba #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE
11232a0c3433SFuad Tabba
kvm_vm_is_protected(struct kvm * kvm)112492e68b2bSDave Martin static inline bool kvm_vm_is_protected(struct kvm *kvm)
11259033bba4SDave Martin {
11269033bba4SDave Martin return false;
11274c0680d3SMarc Zyngier }
11287dd32a0dSDave Martin
112906394531SMarc Zyngier int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature);
113006394531SMarc Zyngier bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu);
113106394531SMarc Zyngier
113214bda7a9SMarc Zyngier #define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED)
1133f3c6efc7SOliver Upton
1134f3c6efc7SOliver Upton #define kvm_has_mte(kvm) \
1135f3c6efc7SOliver Upton (system_supports_mte() && \
1136f3c6efc7SOliver Upton test_bit(KVM_ARCH_FLAG_MTE_ENABLED, &(kvm)->arch.flags))
1137de40bb8aSOliver Upton
1138de40bb8aSOliver Upton #define kvm_supports_32bit_el0() \
1139de40bb8aSOliver Upton (system_supports_32bit_el0() && \
1140a8e190cdSArd Biesheuvel !static_branch_unlikely(&arm64_mismatched_32bit_el0))
1141f320bc74SQuentin Perret
1142f320bc74SQuentin Perret #define kvm_vm_has_ran_once(kvm) \
1143f320bc74SQuentin Perret (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &(kvm)->arch.flags))
1144f320bc74SQuentin Perret
1145f320bc74SQuentin Perret int kvm_trng_call(struct kvm_vcpu *vcpu);
1146f320bc74SQuentin Perret #ifdef CONFIG_KVM
1147f320bc74SQuentin Perret extern phys_addr_t hyp_mem_base;
1148a8e190cdSArd Biesheuvel extern phys_addr_t hyp_mem_size;
11491e579429SOliver Upton void __init kvm_hyp_reserve(void);
1150b171f9bbSOliver Upton #else
kvm_hyp_reserve(void)11511e579429SOliver Upton static inline void kvm_hyp_reserve(void) { }
11524f8d6632SMarc Zyngier #endif
1153
1154 void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu);
1155 bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu);
1156
1157 #endif /* __ARM64_KVM_HOST_H__ */
1158