xref: /openbmc/linux/arch/arm64/include/asm/kvm_mmu.h (revision f17f06a0)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Copyright (C) 2012,2013 - ARM Ltd
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  */
6 
7 #ifndef __ARM64_KVM_MMU_H__
8 #define __ARM64_KVM_MMU_H__
9 
10 #include <asm/page.h>
11 #include <asm/memory.h>
12 #include <asm/cpufeature.h>
13 
14 /*
15  * As ARMv8.0 only has the TTBR0_EL2 register, we cannot express
16  * "negative" addresses. This makes it impossible to directly share
17  * mappings with the kernel.
18  *
19  * Instead, give the HYP mode its own VA region at a fixed offset from
20  * the kernel by just masking the top bits (which are all ones for a
21  * kernel address). We need to find out how many bits to mask.
22  *
23  * We want to build a set of page tables that cover both parts of the
24  * idmap (the trampoline page used to initialize EL2), and our normal
25  * runtime VA space, at the same time.
26  *
27  * Given that the kernel uses VA_BITS for its entire address space,
28  * and that half of that space (VA_BITS - 1) is used for the linear
29  * mapping, we can also limit the EL2 space to (VA_BITS - 1).
30  *
31  * The main question is "Within the VA_BITS space, does EL2 use the
32  * top or the bottom half of that space to shadow the kernel's linear
33  * mapping?". As we need to idmap the trampoline page, this is
34  * determined by the range in which this page lives.
35  *
36  * If the page is in the bottom half, we have to use the top half. If
37  * the page is in the top half, we have to use the bottom half:
38  *
39  * T = __pa_symbol(__hyp_idmap_text_start)
40  * if (T & BIT(VA_BITS - 1))
41  *	HYP_VA_MIN = 0  //idmap in upper half
42  * else
43  *	HYP_VA_MIN = 1 << (VA_BITS - 1)
44  * HYP_VA_MAX = HYP_VA_MIN + (1 << (VA_BITS - 1)) - 1
45  *
46  * This of course assumes that the trampoline page exists within the
47  * VA_BITS range. If it doesn't, then it means we're in the odd case
48  * where the kernel idmap (as well as HYP) uses more levels than the
49  * kernel runtime page tables (as seen when the kernel is configured
50  * for 4k pages, 39bits VA, and yet memory lives just above that
51  * limit, forcing the idmap to use 4 levels of page tables while the
52  * kernel itself only uses 3). In this particular case, it doesn't
53  * matter which side of VA_BITS we use, as we're guaranteed not to
54  * conflict with anything.
55  *
56  * When using VHE, there are no separate hyp mappings and all KVM
57  * functionality is already mapped as part of the main kernel
58  * mappings, and none of this applies in that case.
59  */
60 
61 #ifdef __ASSEMBLY__
62 
63 #include <asm/alternative.h>
64 
65 /*
66  * Convert a kernel VA into a HYP VA.
67  * reg: VA to be converted.
68  *
69  * The actual code generation takes place in kvm_update_va_mask, and
70  * the instructions below are only there to reserve the space and
71  * perform the register allocation (kvm_update_va_mask uses the
72  * specific registers encoded in the instructions).
73  */
74 .macro kern_hyp_va	reg
75 alternative_cb kvm_update_va_mask
76 	and     \reg, \reg, #1		/* mask with va_mask */
77 	ror	\reg, \reg, #1		/* rotate to the first tag bit */
78 	add	\reg, \reg, #0		/* insert the low 12 bits of the tag */
79 	add	\reg, \reg, #0, lsl 12	/* insert the top 12 bits of the tag */
80 	ror	\reg, \reg, #63		/* rotate back */
81 alternative_cb_end
82 .endm
83 
84 #else
85 
86 #include <asm/pgalloc.h>
87 #include <asm/cache.h>
88 #include <asm/cacheflush.h>
89 #include <asm/mmu_context.h>
90 #include <asm/pgtable.h>
91 
92 void kvm_update_va_mask(struct alt_instr *alt,
93 			__le32 *origptr, __le32 *updptr, int nr_inst);
94 void kvm_compute_layout(void);
95 
96 static __always_inline unsigned long __kern_hyp_va(unsigned long v)
97 {
98 	asm volatile(ALTERNATIVE_CB("and %0, %0, #1\n"
99 				    "ror %0, %0, #1\n"
100 				    "add %0, %0, #0\n"
101 				    "add %0, %0, #0, lsl 12\n"
102 				    "ror %0, %0, #63\n",
103 				    kvm_update_va_mask)
104 		     : "+r" (v));
105 	return v;
106 }
107 
108 #define kern_hyp_va(v) 	((typeof(v))(__kern_hyp_va((unsigned long)(v))))
109 
110 /*
111  * Obtain the PC-relative address of a kernel symbol
112  * s: symbol
113  *
114  * The goal of this macro is to return a symbol's address based on a
115  * PC-relative computation, as opposed to a loading the VA from a
116  * constant pool or something similar. This works well for HYP, as an
117  * absolute VA is guaranteed to be wrong. Only use this if trying to
118  * obtain the address of a symbol (i.e. not something you obtained by
119  * following a pointer).
120  */
121 #define hyp_symbol_addr(s)						\
122 	({								\
123 		typeof(s) *addr;					\
124 		asm("adrp	%0, %1\n"				\
125 		    "add	%0, %0, :lo12:%1\n"			\
126 		    : "=r" (addr) : "S" (&s));				\
127 		addr;							\
128 	})
129 
130 /*
131  * We currently support using a VM-specified IPA size. For backward
132  * compatibility, the default IPA size is fixed to 40bits.
133  */
134 #define KVM_PHYS_SHIFT	(40)
135 
136 #define kvm_phys_shift(kvm)		VTCR_EL2_IPA(kvm->arch.vtcr)
137 #define kvm_phys_size(kvm)		(_AC(1, ULL) << kvm_phys_shift(kvm))
138 #define kvm_phys_mask(kvm)		(kvm_phys_size(kvm) - _AC(1, ULL))
139 
140 static inline bool kvm_page_empty(void *ptr)
141 {
142 	struct page *ptr_page = virt_to_page(ptr);
143 	return page_count(ptr_page) == 1;
144 }
145 
146 #include <asm/stage2_pgtable.h>
147 
148 int create_hyp_mappings(void *from, void *to, pgprot_t prot);
149 int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
150 			   void __iomem **kaddr,
151 			   void __iomem **haddr);
152 int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size,
153 			     void **haddr);
154 void free_hyp_pgds(void);
155 
156 void stage2_unmap_vm(struct kvm *kvm);
157 int kvm_alloc_stage2_pgd(struct kvm *kvm);
158 void kvm_free_stage2_pgd(struct kvm *kvm);
159 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
160 			  phys_addr_t pa, unsigned long size, bool writable);
161 
162 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);
163 
164 void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
165 
166 phys_addr_t kvm_mmu_get_httbr(void);
167 phys_addr_t kvm_get_idmap_vector(void);
168 int kvm_mmu_init(void);
169 void kvm_clear_hyp_idmap(void);
170 
171 #define kvm_mk_pmd(ptep)					\
172 	__pmd(__phys_to_pmd_val(__pa(ptep)) | PMD_TYPE_TABLE)
173 #define kvm_mk_pud(pmdp)					\
174 	__pud(__phys_to_pud_val(__pa(pmdp)) | PMD_TYPE_TABLE)
175 #define kvm_mk_pgd(pudp)					\
176 	__pgd(__phys_to_pgd_val(__pa(pudp)) | PUD_TYPE_TABLE)
177 
178 #define kvm_set_pud(pudp, pud)		set_pud(pudp, pud)
179 
180 #define kvm_pfn_pte(pfn, prot)		pfn_pte(pfn, prot)
181 #define kvm_pfn_pmd(pfn, prot)		pfn_pmd(pfn, prot)
182 #define kvm_pfn_pud(pfn, prot)		pfn_pud(pfn, prot)
183 
184 #define kvm_pud_pfn(pud)		pud_pfn(pud)
185 
186 #define kvm_pmd_mkhuge(pmd)		pmd_mkhuge(pmd)
187 #define kvm_pud_mkhuge(pud)		pud_mkhuge(pud)
188 
189 static inline pte_t kvm_s2pte_mkwrite(pte_t pte)
190 {
191 	pte_val(pte) |= PTE_S2_RDWR;
192 	return pte;
193 }
194 
195 static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd)
196 {
197 	pmd_val(pmd) |= PMD_S2_RDWR;
198 	return pmd;
199 }
200 
201 static inline pud_t kvm_s2pud_mkwrite(pud_t pud)
202 {
203 	pud_val(pud) |= PUD_S2_RDWR;
204 	return pud;
205 }
206 
207 static inline pte_t kvm_s2pte_mkexec(pte_t pte)
208 {
209 	pte_val(pte) &= ~PTE_S2_XN;
210 	return pte;
211 }
212 
213 static inline pmd_t kvm_s2pmd_mkexec(pmd_t pmd)
214 {
215 	pmd_val(pmd) &= ~PMD_S2_XN;
216 	return pmd;
217 }
218 
219 static inline pud_t kvm_s2pud_mkexec(pud_t pud)
220 {
221 	pud_val(pud) &= ~PUD_S2_XN;
222 	return pud;
223 }
224 
225 static inline void kvm_set_s2pte_readonly(pte_t *ptep)
226 {
227 	pteval_t old_pteval, pteval;
228 
229 	pteval = READ_ONCE(pte_val(*ptep));
230 	do {
231 		old_pteval = pteval;
232 		pteval &= ~PTE_S2_RDWR;
233 		pteval |= PTE_S2_RDONLY;
234 		pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
235 	} while (pteval != old_pteval);
236 }
237 
238 static inline bool kvm_s2pte_readonly(pte_t *ptep)
239 {
240 	return (READ_ONCE(pte_val(*ptep)) & PTE_S2_RDWR) == PTE_S2_RDONLY;
241 }
242 
243 static inline bool kvm_s2pte_exec(pte_t *ptep)
244 {
245 	return !(READ_ONCE(pte_val(*ptep)) & PTE_S2_XN);
246 }
247 
248 static inline void kvm_set_s2pmd_readonly(pmd_t *pmdp)
249 {
250 	kvm_set_s2pte_readonly((pte_t *)pmdp);
251 }
252 
253 static inline bool kvm_s2pmd_readonly(pmd_t *pmdp)
254 {
255 	return kvm_s2pte_readonly((pte_t *)pmdp);
256 }
257 
258 static inline bool kvm_s2pmd_exec(pmd_t *pmdp)
259 {
260 	return !(READ_ONCE(pmd_val(*pmdp)) & PMD_S2_XN);
261 }
262 
263 static inline void kvm_set_s2pud_readonly(pud_t *pudp)
264 {
265 	kvm_set_s2pte_readonly((pte_t *)pudp);
266 }
267 
268 static inline bool kvm_s2pud_readonly(pud_t *pudp)
269 {
270 	return kvm_s2pte_readonly((pte_t *)pudp);
271 }
272 
273 static inline bool kvm_s2pud_exec(pud_t *pudp)
274 {
275 	return !(READ_ONCE(pud_val(*pudp)) & PUD_S2_XN);
276 }
277 
278 static inline pud_t kvm_s2pud_mkyoung(pud_t pud)
279 {
280 	return pud_mkyoung(pud);
281 }
282 
283 static inline bool kvm_s2pud_young(pud_t pud)
284 {
285 	return pud_young(pud);
286 }
287 
288 #define hyp_pte_table_empty(ptep) kvm_page_empty(ptep)
289 
290 #ifdef __PAGETABLE_PMD_FOLDED
291 #define hyp_pmd_table_empty(pmdp) (0)
292 #else
293 #define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp)
294 #endif
295 
296 #ifdef __PAGETABLE_PUD_FOLDED
297 #define hyp_pud_table_empty(pudp) (0)
298 #else
299 #define hyp_pud_table_empty(pudp) kvm_page_empty(pudp)
300 #endif
301 
302 struct kvm;
303 
304 #define kvm_flush_dcache_to_poc(a,l)	__flush_dcache_area((a), (l))
305 
306 static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
307 {
308 	return (vcpu_read_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101;
309 }
310 
311 static inline void __clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size)
312 {
313 	void *va = page_address(pfn_to_page(pfn));
314 
315 	/*
316 	 * With FWB, we ensure that the guest always accesses memory using
317 	 * cacheable attributes, and we don't have to clean to PoC when
318 	 * faulting in pages. Furthermore, FWB implies IDC, so cleaning to
319 	 * PoU is not required either in this case.
320 	 */
321 	if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
322 		return;
323 
324 	kvm_flush_dcache_to_poc(va, size);
325 }
326 
327 static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn,
328 						  unsigned long size)
329 {
330 	if (icache_is_aliasing()) {
331 		/* any kind of VIPT cache */
332 		__flush_icache_all();
333 	} else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
334 		/* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
335 		void *va = page_address(pfn_to_page(pfn));
336 
337 		invalidate_icache_range((unsigned long)va,
338 					(unsigned long)va + size);
339 	}
340 }
341 
342 static inline void __kvm_flush_dcache_pte(pte_t pte)
343 {
344 	if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) {
345 		struct page *page = pte_page(pte);
346 		kvm_flush_dcache_to_poc(page_address(page), PAGE_SIZE);
347 	}
348 }
349 
350 static inline void __kvm_flush_dcache_pmd(pmd_t pmd)
351 {
352 	if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) {
353 		struct page *page = pmd_page(pmd);
354 		kvm_flush_dcache_to_poc(page_address(page), PMD_SIZE);
355 	}
356 }
357 
358 static inline void __kvm_flush_dcache_pud(pud_t pud)
359 {
360 	if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) {
361 		struct page *page = pud_page(pud);
362 		kvm_flush_dcache_to_poc(page_address(page), PUD_SIZE);
363 	}
364 }
365 
366 #define kvm_virt_to_phys(x)		__pa_symbol(x)
367 
368 void kvm_set_way_flush(struct kvm_vcpu *vcpu);
369 void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
370 
371 static inline bool __kvm_cpu_uses_extended_idmap(void)
372 {
373 	return __cpu_uses_extended_idmap_level();
374 }
375 
376 static inline unsigned long __kvm_idmap_ptrs_per_pgd(void)
377 {
378 	return idmap_ptrs_per_pgd;
379 }
380 
381 /*
382  * Can't use pgd_populate here, because the extended idmap adds an extra level
383  * above CONFIG_PGTABLE_LEVELS (which is 2 or 3 if we're using the extended
384  * idmap), and pgd_populate is only available if CONFIG_PGTABLE_LEVELS = 4.
385  */
386 static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
387 				       pgd_t *hyp_pgd,
388 				       pgd_t *merged_hyp_pgd,
389 				       unsigned long hyp_idmap_start)
390 {
391 	int idmap_idx;
392 	u64 pgd_addr;
393 
394 	/*
395 	 * Use the first entry to access the HYP mappings. It is
396 	 * guaranteed to be free, otherwise we wouldn't use an
397 	 * extended idmap.
398 	 */
399 	VM_BUG_ON(pgd_val(merged_hyp_pgd[0]));
400 	pgd_addr = __phys_to_pgd_val(__pa(hyp_pgd));
401 	merged_hyp_pgd[0] = __pgd(pgd_addr | PMD_TYPE_TABLE);
402 
403 	/*
404 	 * Create another extended level entry that points to the boot HYP map,
405 	 * which contains an ID mapping of the HYP init code. We essentially
406 	 * merge the boot and runtime HYP maps by doing so, but they don't
407 	 * overlap anyway, so this is fine.
408 	 */
409 	idmap_idx = hyp_idmap_start >> VA_BITS;
410 	VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx]));
411 	pgd_addr = __phys_to_pgd_val(__pa(boot_hyp_pgd));
412 	merged_hyp_pgd[idmap_idx] = __pgd(pgd_addr | PMD_TYPE_TABLE);
413 }
414 
415 static inline unsigned int kvm_get_vmid_bits(void)
416 {
417 	int reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
418 
419 	return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
420 }
421 
422 /*
423  * We are not in the kvm->srcu critical section most of the time, so we take
424  * the SRCU read lock here. Since we copy the data from the user page, we
425  * can immediately drop the lock again.
426  */
427 static inline int kvm_read_guest_lock(struct kvm *kvm,
428 				      gpa_t gpa, void *data, unsigned long len)
429 {
430 	int srcu_idx = srcu_read_lock(&kvm->srcu);
431 	int ret = kvm_read_guest(kvm, gpa, data, len);
432 
433 	srcu_read_unlock(&kvm->srcu, srcu_idx);
434 
435 	return ret;
436 }
437 
438 static inline int kvm_write_guest_lock(struct kvm *kvm, gpa_t gpa,
439 				       const void *data, unsigned long len)
440 {
441 	int srcu_idx = srcu_read_lock(&kvm->srcu);
442 	int ret = kvm_write_guest(kvm, gpa, data, len);
443 
444 	srcu_read_unlock(&kvm->srcu, srcu_idx);
445 
446 	return ret;
447 }
448 
449 #ifdef CONFIG_KVM_INDIRECT_VECTORS
450 /*
451  * EL2 vectors can be mapped and rerouted in a number of ways,
452  * depending on the kernel configuration and CPU present:
453  *
454  * - If the CPU has the ARM64_HARDEN_BRANCH_PREDICTOR cap, the
455  *   hardening sequence is placed in one of the vector slots, which is
456  *   executed before jumping to the real vectors.
457  *
458  * - If the CPU has both the ARM64_HARDEN_EL2_VECTORS cap and the
459  *   ARM64_HARDEN_BRANCH_PREDICTOR cap, the slot containing the
460  *   hardening sequence is mapped next to the idmap page, and executed
461  *   before jumping to the real vectors.
462  *
463  * - If the CPU only has the ARM64_HARDEN_EL2_VECTORS cap, then an
464  *   empty slot is selected, mapped next to the idmap page, and
465  *   executed before jumping to the real vectors.
466  *
467  * Note that ARM64_HARDEN_EL2_VECTORS is somewhat incompatible with
468  * VHE, as we don't have hypervisor-specific mappings. If the system
469  * is VHE and yet selects this capability, it will be ignored.
470  */
471 #include <asm/mmu.h>
472 
473 extern void *__kvm_bp_vect_base;
474 extern int __kvm_harden_el2_vector_slot;
475 
476 /*  This is only called on a VHE system */
477 static inline void *kvm_get_hyp_vector(void)
478 {
479 	struct bp_hardening_data *data = arm64_get_bp_hardening_data();
480 	void *vect = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
481 	int slot = -1;
482 
483 	if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR) && data->fn) {
484 		vect = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs_start));
485 		slot = data->hyp_vectors_slot;
486 	}
487 
488 	if (this_cpu_has_cap(ARM64_HARDEN_EL2_VECTORS) && !has_vhe()) {
489 		vect = __kvm_bp_vect_base;
490 		if (slot == -1)
491 			slot = __kvm_harden_el2_vector_slot;
492 	}
493 
494 	if (slot != -1)
495 		vect += slot * SZ_2K;
496 
497 	return vect;
498 }
499 
500 /*  This is only called on a !VHE system */
501 static inline int kvm_map_vectors(void)
502 {
503 	/*
504 	 * HBP  = ARM64_HARDEN_BRANCH_PREDICTOR
505 	 * HEL2 = ARM64_HARDEN_EL2_VECTORS
506 	 *
507 	 * !HBP + !HEL2 -> use direct vectors
508 	 *  HBP + !HEL2 -> use hardened vectors in place
509 	 * !HBP +  HEL2 -> allocate one vector slot and use exec mapping
510 	 *  HBP +  HEL2 -> use hardened vertors and use exec mapping
511 	 */
512 	if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR)) {
513 		__kvm_bp_vect_base = kvm_ksym_ref(__bp_harden_hyp_vecs_start);
514 		__kvm_bp_vect_base = kern_hyp_va(__kvm_bp_vect_base);
515 	}
516 
517 	if (cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) {
518 		phys_addr_t vect_pa = __pa_symbol(__bp_harden_hyp_vecs_start);
519 		unsigned long size = (__bp_harden_hyp_vecs_end -
520 				      __bp_harden_hyp_vecs_start);
521 
522 		/*
523 		 * Always allocate a spare vector slot, as we don't
524 		 * know yet which CPUs have a BP hardening slot that
525 		 * we can reuse.
526 		 */
527 		__kvm_harden_el2_vector_slot = atomic_inc_return(&arm64_el2_vector_last_slot);
528 		BUG_ON(__kvm_harden_el2_vector_slot >= BP_HARDEN_EL2_SLOTS);
529 		return create_hyp_exec_mappings(vect_pa, size,
530 						&__kvm_bp_vect_base);
531 	}
532 
533 	return 0;
534 }
535 #else
536 static inline void *kvm_get_hyp_vector(void)
537 {
538 	return kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
539 }
540 
541 static inline int kvm_map_vectors(void)
542 {
543 	return 0;
544 }
545 #endif
546 
547 #ifdef CONFIG_ARM64_SSBD
548 DECLARE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
549 
550 static inline int hyp_map_aux_data(void)
551 {
552 	int cpu, err;
553 
554 	for_each_possible_cpu(cpu) {
555 		u64 *ptr;
556 
557 		ptr = per_cpu_ptr(&arm64_ssbd_callback_required, cpu);
558 		err = create_hyp_mappings(ptr, ptr + 1, PAGE_HYP);
559 		if (err)
560 			return err;
561 	}
562 	return 0;
563 }
564 #else
565 static inline int hyp_map_aux_data(void)
566 {
567 	return 0;
568 }
569 #endif
570 
571 #define kvm_phys_to_vttbr(addr)		phys_to_ttbr(addr)
572 
573 /*
574  * Get the magic number 'x' for VTTBR:BADDR of this KVM instance.
575  * With v8.2 LVA extensions, 'x' should be a minimum of 6 with
576  * 52bit IPS.
577  */
578 static inline int arm64_vttbr_x(u32 ipa_shift, u32 levels)
579 {
580 	int x = ARM64_VTTBR_X(ipa_shift, levels);
581 
582 	return (IS_ENABLED(CONFIG_ARM64_PA_BITS_52) && x < 6) ? 6 : x;
583 }
584 
585 static inline u64 vttbr_baddr_mask(u32 ipa_shift, u32 levels)
586 {
587 	unsigned int x = arm64_vttbr_x(ipa_shift, levels);
588 
589 	return GENMASK_ULL(PHYS_MASK_SHIFT - 1, x);
590 }
591 
592 static inline u64 kvm_vttbr_baddr_mask(struct kvm *kvm)
593 {
594 	return vttbr_baddr_mask(kvm_phys_shift(kvm), kvm_stage2_levels(kvm));
595 }
596 
597 static __always_inline u64 kvm_get_vttbr(struct kvm *kvm)
598 {
599 	struct kvm_vmid *vmid = &kvm->arch.vmid;
600 	u64 vmid_field, baddr;
601 	u64 cnp = system_supports_cnp() ? VTTBR_CNP_BIT : 0;
602 
603 	baddr = kvm->arch.pgd_phys;
604 	vmid_field = (u64)vmid->vmid << VTTBR_VMID_SHIFT;
605 	return kvm_phys_to_vttbr(baddr) | vmid_field | cnp;
606 }
607 
608 #endif /* __ASSEMBLY__ */
609 #endif /* __ARM64_KVM_MMU_H__ */
610