1dbca5e1aSKuppuswamy Sathyanarayanan // SPDX-License-Identifier: GPL-2.0-only
2dbca5e1aSKuppuswamy Sathyanarayanan /*
3dbca5e1aSKuppuswamy Sathyanarayanan * AMD Memory Encryption Support
4dbca5e1aSKuppuswamy Sathyanarayanan *
5dbca5e1aSKuppuswamy Sathyanarayanan * Copyright (C) 2016 Advanced Micro Devices, Inc.
6dbca5e1aSKuppuswamy Sathyanarayanan *
7dbca5e1aSKuppuswamy Sathyanarayanan * Author: Tom Lendacky <thomas.lendacky@amd.com>
8dbca5e1aSKuppuswamy Sathyanarayanan */
9dbca5e1aSKuppuswamy Sathyanarayanan
10dbca5e1aSKuppuswamy Sathyanarayanan #define DISABLE_BRANCH_PROFILING
11dbca5e1aSKuppuswamy Sathyanarayanan
12dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/linkage.h>
13dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/init.h>
14dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/mm.h>
15dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/dma-direct.h>
16dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/swiotlb.h>
17dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/mem_encrypt.h>
18dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/device.h>
19dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/kernel.h>
20dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/bitops.h>
21dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/dma-mapping.h>
22dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/virtio_config.h>
23a603002eSJuergen Gross #include <linux/virtio_anchor.h>
24dbca5e1aSKuppuswamy Sathyanarayanan #include <linux/cc_platform.h>
25dbca5e1aSKuppuswamy Sathyanarayanan
26dbca5e1aSKuppuswamy Sathyanarayanan #include <asm/tlbflush.h>
27dbca5e1aSKuppuswamy Sathyanarayanan #include <asm/fixmap.h>
28dbca5e1aSKuppuswamy Sathyanarayanan #include <asm/setup.h>
294867fbbdSAnshuman Khandual #include <asm/mem_encrypt.h>
30dbca5e1aSKuppuswamy Sathyanarayanan #include <asm/bootparam.h>
31dbca5e1aSKuppuswamy Sathyanarayanan #include <asm/set_memory.h>
32dbca5e1aSKuppuswamy Sathyanarayanan #include <asm/cacheflush.h>
33dbca5e1aSKuppuswamy Sathyanarayanan #include <asm/processor-flags.h>
34dbca5e1aSKuppuswamy Sathyanarayanan #include <asm/msr.h>
35dbca5e1aSKuppuswamy Sathyanarayanan #include <asm/cmdline.h>
365e5ccff6SBrijesh Singh #include <asm/sev.h>
3734c686e5SKirill A. Shutemov #include <asm/ia32.h>
38dbca5e1aSKuppuswamy Sathyanarayanan
39dbca5e1aSKuppuswamy Sathyanarayanan #include "mm_internal.h"
40dbca5e1aSKuppuswamy Sathyanarayanan
41dbca5e1aSKuppuswamy Sathyanarayanan /*
42dbca5e1aSKuppuswamy Sathyanarayanan * Since SME related variables are set early in the boot process they must
43dbca5e1aSKuppuswamy Sathyanarayanan * reside in the .data section so as not to be zeroed out when the .bss
44dbca5e1aSKuppuswamy Sathyanarayanan * section is later cleared.
45dbca5e1aSKuppuswamy Sathyanarayanan */
46dbca5e1aSKuppuswamy Sathyanarayanan u64 sme_me_mask __section(".data") = 0;
47dbca5e1aSKuppuswamy Sathyanarayanan u64 sev_status __section(".data") = 0;
48dbca5e1aSKuppuswamy Sathyanarayanan u64 sev_check_data __section(".data") = 0;
49dbca5e1aSKuppuswamy Sathyanarayanan EXPORT_SYMBOL(sme_me_mask);
50dbca5e1aSKuppuswamy Sathyanarayanan
51dbca5e1aSKuppuswamy Sathyanarayanan /* Buffer used for early in-place encryption by BSP, no locking needed */
52dbca5e1aSKuppuswamy Sathyanarayanan static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);
53dbca5e1aSKuppuswamy Sathyanarayanan
54dbca5e1aSKuppuswamy Sathyanarayanan /*
555e5ccff6SBrijesh Singh * SNP-specific routine which needs to additionally change the page state from
565e5ccff6SBrijesh Singh * private to shared before copying the data from the source to destination and
575e5ccff6SBrijesh Singh * restore after the copy.
585e5ccff6SBrijesh Singh */
snp_memcpy(void * dst,void * src,size_t sz,unsigned long paddr,bool decrypt)595e5ccff6SBrijesh Singh static inline void __init snp_memcpy(void *dst, void *src, size_t sz,
605e5ccff6SBrijesh Singh unsigned long paddr, bool decrypt)
615e5ccff6SBrijesh Singh {
625e5ccff6SBrijesh Singh unsigned long npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
635e5ccff6SBrijesh Singh
645e5ccff6SBrijesh Singh if (decrypt) {
655e5ccff6SBrijesh Singh /*
665e5ccff6SBrijesh Singh * @paddr needs to be accessed decrypted, mark the page shared in
675e5ccff6SBrijesh Singh * the RMP table before copying it.
685e5ccff6SBrijesh Singh */
695e5ccff6SBrijesh Singh early_snp_set_memory_shared((unsigned long)__va(paddr), paddr, npages);
705e5ccff6SBrijesh Singh
715e5ccff6SBrijesh Singh memcpy(dst, src, sz);
725e5ccff6SBrijesh Singh
735e5ccff6SBrijesh Singh /* Restore the page state after the memcpy. */
745e5ccff6SBrijesh Singh early_snp_set_memory_private((unsigned long)__va(paddr), paddr, npages);
755e5ccff6SBrijesh Singh } else {
765e5ccff6SBrijesh Singh /*
775e5ccff6SBrijesh Singh * @paddr need to be accessed encrypted, no need for the page state
785e5ccff6SBrijesh Singh * change.
795e5ccff6SBrijesh Singh */
805e5ccff6SBrijesh Singh memcpy(dst, src, sz);
815e5ccff6SBrijesh Singh }
825e5ccff6SBrijesh Singh }
835e5ccff6SBrijesh Singh
845e5ccff6SBrijesh Singh /*
85dbca5e1aSKuppuswamy Sathyanarayanan * This routine does not change the underlying encryption setting of the
86dbca5e1aSKuppuswamy Sathyanarayanan * page(s) that map this memory. It assumes that eventually the memory is
87dbca5e1aSKuppuswamy Sathyanarayanan * meant to be accessed as either encrypted or decrypted but the contents
88dbca5e1aSKuppuswamy Sathyanarayanan * are currently not in the desired state.
89dbca5e1aSKuppuswamy Sathyanarayanan *
90dbca5e1aSKuppuswamy Sathyanarayanan * This routine follows the steps outlined in the AMD64 Architecture
91dbca5e1aSKuppuswamy Sathyanarayanan * Programmer's Manual Volume 2, Section 7.10.8 Encrypt-in-Place.
92dbca5e1aSKuppuswamy Sathyanarayanan */
__sme_early_enc_dec(resource_size_t paddr,unsigned long size,bool enc)93dbca5e1aSKuppuswamy Sathyanarayanan static void __init __sme_early_enc_dec(resource_size_t paddr,
94dbca5e1aSKuppuswamy Sathyanarayanan unsigned long size, bool enc)
95dbca5e1aSKuppuswamy Sathyanarayanan {
96dbca5e1aSKuppuswamy Sathyanarayanan void *src, *dst;
97dbca5e1aSKuppuswamy Sathyanarayanan size_t len;
98dbca5e1aSKuppuswamy Sathyanarayanan
99dbca5e1aSKuppuswamy Sathyanarayanan if (!sme_me_mask)
100dbca5e1aSKuppuswamy Sathyanarayanan return;
101dbca5e1aSKuppuswamy Sathyanarayanan
102dbca5e1aSKuppuswamy Sathyanarayanan wbinvd();
103dbca5e1aSKuppuswamy Sathyanarayanan
104dbca5e1aSKuppuswamy Sathyanarayanan /*
105dbca5e1aSKuppuswamy Sathyanarayanan * There are limited number of early mapping slots, so map (at most)
106dbca5e1aSKuppuswamy Sathyanarayanan * one page at time.
107dbca5e1aSKuppuswamy Sathyanarayanan */
108dbca5e1aSKuppuswamy Sathyanarayanan while (size) {
109dbca5e1aSKuppuswamy Sathyanarayanan len = min_t(size_t, sizeof(sme_early_buffer), size);
110dbca5e1aSKuppuswamy Sathyanarayanan
111dbca5e1aSKuppuswamy Sathyanarayanan /*
112dbca5e1aSKuppuswamy Sathyanarayanan * Create mappings for the current and desired format of
113dbca5e1aSKuppuswamy Sathyanarayanan * the memory. Use a write-protected mapping for the source.
114dbca5e1aSKuppuswamy Sathyanarayanan */
115dbca5e1aSKuppuswamy Sathyanarayanan src = enc ? early_memremap_decrypted_wp(paddr, len) :
116dbca5e1aSKuppuswamy Sathyanarayanan early_memremap_encrypted_wp(paddr, len);
117dbca5e1aSKuppuswamy Sathyanarayanan
118dbca5e1aSKuppuswamy Sathyanarayanan dst = enc ? early_memremap_encrypted(paddr, len) :
119dbca5e1aSKuppuswamy Sathyanarayanan early_memremap_decrypted(paddr, len);
120dbca5e1aSKuppuswamy Sathyanarayanan
121dbca5e1aSKuppuswamy Sathyanarayanan /*
122dbca5e1aSKuppuswamy Sathyanarayanan * If a mapping can't be obtained to perform the operation,
123dbca5e1aSKuppuswamy Sathyanarayanan * then eventual access of that area in the desired mode
124dbca5e1aSKuppuswamy Sathyanarayanan * will cause a crash.
125dbca5e1aSKuppuswamy Sathyanarayanan */
126dbca5e1aSKuppuswamy Sathyanarayanan BUG_ON(!src || !dst);
127dbca5e1aSKuppuswamy Sathyanarayanan
128dbca5e1aSKuppuswamy Sathyanarayanan /*
129dbca5e1aSKuppuswamy Sathyanarayanan * Use a temporary buffer, of cache-line multiple size, to
130dbca5e1aSKuppuswamy Sathyanarayanan * avoid data corruption as documented in the APM.
131dbca5e1aSKuppuswamy Sathyanarayanan */
1325e5ccff6SBrijesh Singh if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
1335e5ccff6SBrijesh Singh snp_memcpy(sme_early_buffer, src, len, paddr, enc);
1345e5ccff6SBrijesh Singh snp_memcpy(dst, sme_early_buffer, len, paddr, !enc);
1355e5ccff6SBrijesh Singh } else {
136dbca5e1aSKuppuswamy Sathyanarayanan memcpy(sme_early_buffer, src, len);
137dbca5e1aSKuppuswamy Sathyanarayanan memcpy(dst, sme_early_buffer, len);
1385e5ccff6SBrijesh Singh }
139dbca5e1aSKuppuswamy Sathyanarayanan
140dbca5e1aSKuppuswamy Sathyanarayanan early_memunmap(dst, len);
141dbca5e1aSKuppuswamy Sathyanarayanan early_memunmap(src, len);
142dbca5e1aSKuppuswamy Sathyanarayanan
143dbca5e1aSKuppuswamy Sathyanarayanan paddr += len;
144dbca5e1aSKuppuswamy Sathyanarayanan size -= len;
145dbca5e1aSKuppuswamy Sathyanarayanan }
146dbca5e1aSKuppuswamy Sathyanarayanan }
147dbca5e1aSKuppuswamy Sathyanarayanan
sme_early_encrypt(resource_size_t paddr,unsigned long size)148dbca5e1aSKuppuswamy Sathyanarayanan void __init sme_early_encrypt(resource_size_t paddr, unsigned long size)
149dbca5e1aSKuppuswamy Sathyanarayanan {
150dbca5e1aSKuppuswamy Sathyanarayanan __sme_early_enc_dec(paddr, size, true);
151dbca5e1aSKuppuswamy Sathyanarayanan }
152dbca5e1aSKuppuswamy Sathyanarayanan
sme_early_decrypt(resource_size_t paddr,unsigned long size)153dbca5e1aSKuppuswamy Sathyanarayanan void __init sme_early_decrypt(resource_size_t paddr, unsigned long size)
154dbca5e1aSKuppuswamy Sathyanarayanan {
155dbca5e1aSKuppuswamy Sathyanarayanan __sme_early_enc_dec(paddr, size, false);
156dbca5e1aSKuppuswamy Sathyanarayanan }
157dbca5e1aSKuppuswamy Sathyanarayanan
__sme_early_map_unmap_mem(void * vaddr,unsigned long size,bool map)158dbca5e1aSKuppuswamy Sathyanarayanan static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size,
159dbca5e1aSKuppuswamy Sathyanarayanan bool map)
160dbca5e1aSKuppuswamy Sathyanarayanan {
161dbca5e1aSKuppuswamy Sathyanarayanan unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET;
162dbca5e1aSKuppuswamy Sathyanarayanan pmdval_t pmd_flags, pmd;
163dbca5e1aSKuppuswamy Sathyanarayanan
164dbca5e1aSKuppuswamy Sathyanarayanan /* Use early_pmd_flags but remove the encryption mask */
165dbca5e1aSKuppuswamy Sathyanarayanan pmd_flags = __sme_clr(early_pmd_flags);
166dbca5e1aSKuppuswamy Sathyanarayanan
167dbca5e1aSKuppuswamy Sathyanarayanan do {
168dbca5e1aSKuppuswamy Sathyanarayanan pmd = map ? (paddr & PMD_MASK) + pmd_flags : 0;
169dbca5e1aSKuppuswamy Sathyanarayanan __early_make_pgtable((unsigned long)vaddr, pmd);
170dbca5e1aSKuppuswamy Sathyanarayanan
171dbca5e1aSKuppuswamy Sathyanarayanan vaddr += PMD_SIZE;
172dbca5e1aSKuppuswamy Sathyanarayanan paddr += PMD_SIZE;
173dbca5e1aSKuppuswamy Sathyanarayanan size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE;
174dbca5e1aSKuppuswamy Sathyanarayanan } while (size);
175dbca5e1aSKuppuswamy Sathyanarayanan
176dbca5e1aSKuppuswamy Sathyanarayanan flush_tlb_local();
177dbca5e1aSKuppuswamy Sathyanarayanan }
178dbca5e1aSKuppuswamy Sathyanarayanan
sme_unmap_bootdata(char * real_mode_data)179dbca5e1aSKuppuswamy Sathyanarayanan void __init sme_unmap_bootdata(char *real_mode_data)
180dbca5e1aSKuppuswamy Sathyanarayanan {
181dbca5e1aSKuppuswamy Sathyanarayanan struct boot_params *boot_data;
182dbca5e1aSKuppuswamy Sathyanarayanan unsigned long cmdline_paddr;
183dbca5e1aSKuppuswamy Sathyanarayanan
184dbca5e1aSKuppuswamy Sathyanarayanan if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
185dbca5e1aSKuppuswamy Sathyanarayanan return;
186dbca5e1aSKuppuswamy Sathyanarayanan
187dbca5e1aSKuppuswamy Sathyanarayanan /* Get the command line address before unmapping the real_mode_data */
188dbca5e1aSKuppuswamy Sathyanarayanan boot_data = (struct boot_params *)real_mode_data;
189dbca5e1aSKuppuswamy Sathyanarayanan cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
190dbca5e1aSKuppuswamy Sathyanarayanan
191dbca5e1aSKuppuswamy Sathyanarayanan __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), false);
192dbca5e1aSKuppuswamy Sathyanarayanan
193dbca5e1aSKuppuswamy Sathyanarayanan if (!cmdline_paddr)
194dbca5e1aSKuppuswamy Sathyanarayanan return;
195dbca5e1aSKuppuswamy Sathyanarayanan
196dbca5e1aSKuppuswamy Sathyanarayanan __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, false);
197dbca5e1aSKuppuswamy Sathyanarayanan }
198dbca5e1aSKuppuswamy Sathyanarayanan
sme_map_bootdata(char * real_mode_data)199dbca5e1aSKuppuswamy Sathyanarayanan void __init sme_map_bootdata(char *real_mode_data)
200dbca5e1aSKuppuswamy Sathyanarayanan {
201dbca5e1aSKuppuswamy Sathyanarayanan struct boot_params *boot_data;
202dbca5e1aSKuppuswamy Sathyanarayanan unsigned long cmdline_paddr;
203dbca5e1aSKuppuswamy Sathyanarayanan
204dbca5e1aSKuppuswamy Sathyanarayanan if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
205dbca5e1aSKuppuswamy Sathyanarayanan return;
206dbca5e1aSKuppuswamy Sathyanarayanan
207dbca5e1aSKuppuswamy Sathyanarayanan __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), true);
208dbca5e1aSKuppuswamy Sathyanarayanan
209dbca5e1aSKuppuswamy Sathyanarayanan /* Get the command line address after mapping the real_mode_data */
210dbca5e1aSKuppuswamy Sathyanarayanan boot_data = (struct boot_params *)real_mode_data;
211dbca5e1aSKuppuswamy Sathyanarayanan cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
212dbca5e1aSKuppuswamy Sathyanarayanan
213dbca5e1aSKuppuswamy Sathyanarayanan if (!cmdline_paddr)
214dbca5e1aSKuppuswamy Sathyanarayanan return;
215dbca5e1aSKuppuswamy Sathyanarayanan
216dbca5e1aSKuppuswamy Sathyanarayanan __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true);
217dbca5e1aSKuppuswamy Sathyanarayanan }
218dbca5e1aSKuppuswamy Sathyanarayanan
sev_setup_arch(void)219dbca5e1aSKuppuswamy Sathyanarayanan void __init sev_setup_arch(void)
220dbca5e1aSKuppuswamy Sathyanarayanan {
221dbca5e1aSKuppuswamy Sathyanarayanan phys_addr_t total_mem = memblock_phys_mem_size();
222dbca5e1aSKuppuswamy Sathyanarayanan unsigned long size;
223dbca5e1aSKuppuswamy Sathyanarayanan
224dbca5e1aSKuppuswamy Sathyanarayanan if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
225dbca5e1aSKuppuswamy Sathyanarayanan return;
226dbca5e1aSKuppuswamy Sathyanarayanan
227dbca5e1aSKuppuswamy Sathyanarayanan /*
228dbca5e1aSKuppuswamy Sathyanarayanan * For SEV, all DMA has to occur via shared/unencrypted pages.
229dbca5e1aSKuppuswamy Sathyanarayanan * SEV uses SWIOTLB to make this happen without changing device
230dbca5e1aSKuppuswamy Sathyanarayanan * drivers. However, depending on the workload being run, the
231dbca5e1aSKuppuswamy Sathyanarayanan * default 64MB of SWIOTLB may not be enough and SWIOTLB may
232dbca5e1aSKuppuswamy Sathyanarayanan * run out of buffers for DMA, resulting in I/O errors and/or
233dbca5e1aSKuppuswamy Sathyanarayanan * performance degradation especially with high I/O workloads.
234dbca5e1aSKuppuswamy Sathyanarayanan *
235dbca5e1aSKuppuswamy Sathyanarayanan * Adjust the default size of SWIOTLB for SEV guests using
236dbca5e1aSKuppuswamy Sathyanarayanan * a percentage of guest memory for SWIOTLB buffers.
237dbca5e1aSKuppuswamy Sathyanarayanan * Also, as the SWIOTLB bounce buffer memory is allocated
238dbca5e1aSKuppuswamy Sathyanarayanan * from low memory, ensure that the adjusted size is within
239dbca5e1aSKuppuswamy Sathyanarayanan * the limits of low available memory.
240dbca5e1aSKuppuswamy Sathyanarayanan *
241dbca5e1aSKuppuswamy Sathyanarayanan * The percentage of guest memory used here for SWIOTLB buffers
242dbca5e1aSKuppuswamy Sathyanarayanan * is more of an approximation of the static adjustment which
243dbca5e1aSKuppuswamy Sathyanarayanan * 64MB for <1G, and ~128M to 256M for 1G-to-4G, i.e., the 6%
244dbca5e1aSKuppuswamy Sathyanarayanan */
245dbca5e1aSKuppuswamy Sathyanarayanan size = total_mem * 6 / 100;
246dbca5e1aSKuppuswamy Sathyanarayanan size = clamp_val(size, IO_TLB_DEFAULT_SIZE, SZ_1G);
247dbca5e1aSKuppuswamy Sathyanarayanan swiotlb_adjust_size(size);
2483f9dfbebSJuergen Gross
2493f9dfbebSJuergen Gross /* Set restricted memory access for virtio. */
250a603002eSJuergen Gross virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc);
251dbca5e1aSKuppuswamy Sathyanarayanan }
252dbca5e1aSKuppuswamy Sathyanarayanan
pg_level_to_pfn(int level,pte_t * kpte,pgprot_t * ret_prot)253dbca5e1aSKuppuswamy Sathyanarayanan static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot)
254dbca5e1aSKuppuswamy Sathyanarayanan {
255dbca5e1aSKuppuswamy Sathyanarayanan unsigned long pfn = 0;
256dbca5e1aSKuppuswamy Sathyanarayanan pgprot_t prot;
257dbca5e1aSKuppuswamy Sathyanarayanan
258dbca5e1aSKuppuswamy Sathyanarayanan switch (level) {
259dbca5e1aSKuppuswamy Sathyanarayanan case PG_LEVEL_4K:
260dbca5e1aSKuppuswamy Sathyanarayanan pfn = pte_pfn(*kpte);
261dbca5e1aSKuppuswamy Sathyanarayanan prot = pte_pgprot(*kpte);
262dbca5e1aSKuppuswamy Sathyanarayanan break;
263dbca5e1aSKuppuswamy Sathyanarayanan case PG_LEVEL_2M:
264dbca5e1aSKuppuswamy Sathyanarayanan pfn = pmd_pfn(*(pmd_t *)kpte);
265dbca5e1aSKuppuswamy Sathyanarayanan prot = pmd_pgprot(*(pmd_t *)kpte);
266dbca5e1aSKuppuswamy Sathyanarayanan break;
267dbca5e1aSKuppuswamy Sathyanarayanan case PG_LEVEL_1G:
268dbca5e1aSKuppuswamy Sathyanarayanan pfn = pud_pfn(*(pud_t *)kpte);
269dbca5e1aSKuppuswamy Sathyanarayanan prot = pud_pgprot(*(pud_t *)kpte);
270dbca5e1aSKuppuswamy Sathyanarayanan break;
271dbca5e1aSKuppuswamy Sathyanarayanan default:
272dbca5e1aSKuppuswamy Sathyanarayanan WARN_ONCE(1, "Invalid level for kpte\n");
273dbca5e1aSKuppuswamy Sathyanarayanan return 0;
274dbca5e1aSKuppuswamy Sathyanarayanan }
275dbca5e1aSKuppuswamy Sathyanarayanan
276dbca5e1aSKuppuswamy Sathyanarayanan if (ret_prot)
277dbca5e1aSKuppuswamy Sathyanarayanan *ret_prot = prot;
278dbca5e1aSKuppuswamy Sathyanarayanan
279dbca5e1aSKuppuswamy Sathyanarayanan return pfn;
280dbca5e1aSKuppuswamy Sathyanarayanan }
281dbca5e1aSKuppuswamy Sathyanarayanan
amd_enc_tlb_flush_required(bool enc)2821e8c5971SBrijesh Singh static bool amd_enc_tlb_flush_required(bool enc)
2831e8c5971SBrijesh Singh {
2841e8c5971SBrijesh Singh return true;
2851e8c5971SBrijesh Singh }
2861e8c5971SBrijesh Singh
amd_enc_cache_flush_required(void)2871e8c5971SBrijesh Singh static bool amd_enc_cache_flush_required(void)
2881e8c5971SBrijesh Singh {
2891e8c5971SBrijesh Singh return !cpu_feature_enabled(X86_FEATURE_SME_COHERENT);
2901e8c5971SBrijesh Singh }
2911e8c5971SBrijesh Singh
enc_dec_hypercall(unsigned long vaddr,unsigned long size,bool enc)292ac3f9c9fSSteve Rutherford static void enc_dec_hypercall(unsigned long vaddr, unsigned long size, bool enc)
293dbca5e1aSKuppuswamy Sathyanarayanan {
294dbca5e1aSKuppuswamy Sathyanarayanan #ifdef CONFIG_PARAVIRT
295ac3f9c9fSSteve Rutherford unsigned long vaddr_end = vaddr + size;
296dbca5e1aSKuppuswamy Sathyanarayanan
297dbca5e1aSKuppuswamy Sathyanarayanan while (vaddr < vaddr_end) {
298dbca5e1aSKuppuswamy Sathyanarayanan int psize, pmask, level;
299dbca5e1aSKuppuswamy Sathyanarayanan unsigned long pfn;
300dbca5e1aSKuppuswamy Sathyanarayanan pte_t *kpte;
301dbca5e1aSKuppuswamy Sathyanarayanan
302dbca5e1aSKuppuswamy Sathyanarayanan kpte = lookup_address(vaddr, &level);
303dbca5e1aSKuppuswamy Sathyanarayanan if (!kpte || pte_none(*kpte)) {
304dbca5e1aSKuppuswamy Sathyanarayanan WARN_ONCE(1, "kpte lookup for vaddr\n");
305dbca5e1aSKuppuswamy Sathyanarayanan return;
306dbca5e1aSKuppuswamy Sathyanarayanan }
307dbca5e1aSKuppuswamy Sathyanarayanan
308dbca5e1aSKuppuswamy Sathyanarayanan pfn = pg_level_to_pfn(level, kpte, NULL);
309dbca5e1aSKuppuswamy Sathyanarayanan if (!pfn)
310dbca5e1aSKuppuswamy Sathyanarayanan continue;
311dbca5e1aSKuppuswamy Sathyanarayanan
312dbca5e1aSKuppuswamy Sathyanarayanan psize = page_level_size(level);
313dbca5e1aSKuppuswamy Sathyanarayanan pmask = page_level_mask(level);
314dbca5e1aSKuppuswamy Sathyanarayanan
315dbca5e1aSKuppuswamy Sathyanarayanan notify_page_enc_status_changed(pfn, psize >> PAGE_SHIFT, enc);
316dbca5e1aSKuppuswamy Sathyanarayanan
317dbca5e1aSKuppuswamy Sathyanarayanan vaddr = (vaddr & pmask) + psize;
318dbca5e1aSKuppuswamy Sathyanarayanan }
319dbca5e1aSKuppuswamy Sathyanarayanan #endif
320dbca5e1aSKuppuswamy Sathyanarayanan }
321dbca5e1aSKuppuswamy Sathyanarayanan
amd_enc_status_change_prepare(unsigned long vaddr,int npages,bool enc)3223f6819ddSKirill A. Shutemov static bool amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool enc)
3231e8c5971SBrijesh Singh {
324dc3f3d24SBrijesh Singh /*
325dc3f3d24SBrijesh Singh * To maintain the security guarantees of SEV-SNP guests, make sure
326dc3f3d24SBrijesh Singh * to invalidate the memory before encryption attribute is cleared.
327dc3f3d24SBrijesh Singh */
328dc3f3d24SBrijesh Singh if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && !enc)
329dc3f3d24SBrijesh Singh snp_set_memory_shared(vaddr, npages);
3303f6819ddSKirill A. Shutemov
3313f6819ddSKirill A. Shutemov return true;
3321e8c5971SBrijesh Singh }
3331e8c5971SBrijesh Singh
3341e8c5971SBrijesh Singh /* Return true unconditionally: return value doesn't matter for the SEV side */
amd_enc_status_change_finish(unsigned long vaddr,int npages,bool enc)3351e8c5971SBrijesh Singh static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool enc)
3361e8c5971SBrijesh Singh {
337dc3f3d24SBrijesh Singh /*
338dc3f3d24SBrijesh Singh * After memory is mapped encrypted in the page table, validate it
339dc3f3d24SBrijesh Singh * so that it is consistent with the page table updates.
340dc3f3d24SBrijesh Singh */
341dc3f3d24SBrijesh Singh if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && enc)
342dc3f3d24SBrijesh Singh snp_set_memory_private(vaddr, npages);
343dc3f3d24SBrijesh Singh
3441e8c5971SBrijesh Singh if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
345ac3f9c9fSSteve Rutherford enc_dec_hypercall(vaddr, npages << PAGE_SHIFT, enc);
3461e8c5971SBrijesh Singh
3471e8c5971SBrijesh Singh return true;
3481e8c5971SBrijesh Singh }
3491e8c5971SBrijesh Singh
__set_clr_pte_enc(pte_t * kpte,int level,bool enc)350dbca5e1aSKuppuswamy Sathyanarayanan static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
351dbca5e1aSKuppuswamy Sathyanarayanan {
352dbca5e1aSKuppuswamy Sathyanarayanan pgprot_t old_prot, new_prot;
353dbca5e1aSKuppuswamy Sathyanarayanan unsigned long pfn, pa, size;
354dbca5e1aSKuppuswamy Sathyanarayanan pte_t new_pte;
355dbca5e1aSKuppuswamy Sathyanarayanan
356dbca5e1aSKuppuswamy Sathyanarayanan pfn = pg_level_to_pfn(level, kpte, &old_prot);
357dbca5e1aSKuppuswamy Sathyanarayanan if (!pfn)
358dbca5e1aSKuppuswamy Sathyanarayanan return;
359dbca5e1aSKuppuswamy Sathyanarayanan
360dbca5e1aSKuppuswamy Sathyanarayanan new_prot = old_prot;
361dbca5e1aSKuppuswamy Sathyanarayanan if (enc)
362dbca5e1aSKuppuswamy Sathyanarayanan pgprot_val(new_prot) |= _PAGE_ENC;
363dbca5e1aSKuppuswamy Sathyanarayanan else
364dbca5e1aSKuppuswamy Sathyanarayanan pgprot_val(new_prot) &= ~_PAGE_ENC;
365dbca5e1aSKuppuswamy Sathyanarayanan
366dbca5e1aSKuppuswamy Sathyanarayanan /* If prot is same then do nothing. */
367dbca5e1aSKuppuswamy Sathyanarayanan if (pgprot_val(old_prot) == pgprot_val(new_prot))
368dbca5e1aSKuppuswamy Sathyanarayanan return;
369dbca5e1aSKuppuswamy Sathyanarayanan
370dbca5e1aSKuppuswamy Sathyanarayanan pa = pfn << PAGE_SHIFT;
371dbca5e1aSKuppuswamy Sathyanarayanan size = page_level_size(level);
372dbca5e1aSKuppuswamy Sathyanarayanan
373dbca5e1aSKuppuswamy Sathyanarayanan /*
374dbca5e1aSKuppuswamy Sathyanarayanan * We are going to perform in-place en-/decryption and change the
375dbca5e1aSKuppuswamy Sathyanarayanan * physical page attribute from C=1 to C=0 or vice versa. Flush the
376dbca5e1aSKuppuswamy Sathyanarayanan * caches to ensure that data gets accessed with the correct C-bit.
377dbca5e1aSKuppuswamy Sathyanarayanan */
378dbca5e1aSKuppuswamy Sathyanarayanan clflush_cache_range(__va(pa), size);
379dbca5e1aSKuppuswamy Sathyanarayanan
380dbca5e1aSKuppuswamy Sathyanarayanan /* Encrypt/decrypt the contents in-place */
3815e5ccff6SBrijesh Singh if (enc) {
382dbca5e1aSKuppuswamy Sathyanarayanan sme_early_encrypt(pa, size);
3835e5ccff6SBrijesh Singh } else {
384dbca5e1aSKuppuswamy Sathyanarayanan sme_early_decrypt(pa, size);
385dbca5e1aSKuppuswamy Sathyanarayanan
3865e5ccff6SBrijesh Singh /*
3875e5ccff6SBrijesh Singh * ON SNP, the page state in the RMP table must happen
3885e5ccff6SBrijesh Singh * before the page table updates.
3895e5ccff6SBrijesh Singh */
3905e5ccff6SBrijesh Singh early_snp_set_memory_shared((unsigned long)__va(pa), pa, 1);
3915e5ccff6SBrijesh Singh }
3925e5ccff6SBrijesh Singh
393dbca5e1aSKuppuswamy Sathyanarayanan /* Change the page encryption mask. */
394dbca5e1aSKuppuswamy Sathyanarayanan new_pte = pfn_pte(pfn, new_prot);
395dbca5e1aSKuppuswamy Sathyanarayanan set_pte_atomic(kpte, new_pte);
3965e5ccff6SBrijesh Singh
3975e5ccff6SBrijesh Singh /*
3985e5ccff6SBrijesh Singh * If page is set encrypted in the page table, then update the RMP table to
3995e5ccff6SBrijesh Singh * add this page as private.
4005e5ccff6SBrijesh Singh */
4015e5ccff6SBrijesh Singh if (enc)
4025e5ccff6SBrijesh Singh early_snp_set_memory_private((unsigned long)__va(pa), pa, 1);
403dbca5e1aSKuppuswamy Sathyanarayanan }
404dbca5e1aSKuppuswamy Sathyanarayanan
early_set_memory_enc_dec(unsigned long vaddr,unsigned long size,bool enc)405dbca5e1aSKuppuswamy Sathyanarayanan static int __init early_set_memory_enc_dec(unsigned long vaddr,
406dbca5e1aSKuppuswamy Sathyanarayanan unsigned long size, bool enc)
407dbca5e1aSKuppuswamy Sathyanarayanan {
408dbca5e1aSKuppuswamy Sathyanarayanan unsigned long vaddr_end, vaddr_next, start;
409dbca5e1aSKuppuswamy Sathyanarayanan unsigned long psize, pmask;
410dbca5e1aSKuppuswamy Sathyanarayanan int split_page_size_mask;
411dbca5e1aSKuppuswamy Sathyanarayanan int level, ret;
412dbca5e1aSKuppuswamy Sathyanarayanan pte_t *kpte;
413dbca5e1aSKuppuswamy Sathyanarayanan
414dbca5e1aSKuppuswamy Sathyanarayanan start = vaddr;
415dbca5e1aSKuppuswamy Sathyanarayanan vaddr_next = vaddr;
416dbca5e1aSKuppuswamy Sathyanarayanan vaddr_end = vaddr + size;
417dbca5e1aSKuppuswamy Sathyanarayanan
418dbca5e1aSKuppuswamy Sathyanarayanan for (; vaddr < vaddr_end; vaddr = vaddr_next) {
419dbca5e1aSKuppuswamy Sathyanarayanan kpte = lookup_address(vaddr, &level);
420dbca5e1aSKuppuswamy Sathyanarayanan if (!kpte || pte_none(*kpte)) {
421dbca5e1aSKuppuswamy Sathyanarayanan ret = 1;
422dbca5e1aSKuppuswamy Sathyanarayanan goto out;
423dbca5e1aSKuppuswamy Sathyanarayanan }
424dbca5e1aSKuppuswamy Sathyanarayanan
425dbca5e1aSKuppuswamy Sathyanarayanan if (level == PG_LEVEL_4K) {
426dbca5e1aSKuppuswamy Sathyanarayanan __set_clr_pte_enc(kpte, level, enc);
427dbca5e1aSKuppuswamy Sathyanarayanan vaddr_next = (vaddr & PAGE_MASK) + PAGE_SIZE;
428dbca5e1aSKuppuswamy Sathyanarayanan continue;
429dbca5e1aSKuppuswamy Sathyanarayanan }
430dbca5e1aSKuppuswamy Sathyanarayanan
431dbca5e1aSKuppuswamy Sathyanarayanan psize = page_level_size(level);
432dbca5e1aSKuppuswamy Sathyanarayanan pmask = page_level_mask(level);
433dbca5e1aSKuppuswamy Sathyanarayanan
434dbca5e1aSKuppuswamy Sathyanarayanan /*
435dbca5e1aSKuppuswamy Sathyanarayanan * Check whether we can change the large page in one go.
436dbca5e1aSKuppuswamy Sathyanarayanan * We request a split when the address is not aligned and
437dbca5e1aSKuppuswamy Sathyanarayanan * the number of pages to set/clear encryption bit is smaller
438dbca5e1aSKuppuswamy Sathyanarayanan * than the number of pages in the large page.
439dbca5e1aSKuppuswamy Sathyanarayanan */
440dbca5e1aSKuppuswamy Sathyanarayanan if (vaddr == (vaddr & pmask) &&
441dbca5e1aSKuppuswamy Sathyanarayanan ((vaddr_end - vaddr) >= psize)) {
442dbca5e1aSKuppuswamy Sathyanarayanan __set_clr_pte_enc(kpte, level, enc);
443dbca5e1aSKuppuswamy Sathyanarayanan vaddr_next = (vaddr & pmask) + psize;
444dbca5e1aSKuppuswamy Sathyanarayanan continue;
445dbca5e1aSKuppuswamy Sathyanarayanan }
446dbca5e1aSKuppuswamy Sathyanarayanan
447dbca5e1aSKuppuswamy Sathyanarayanan /*
448dbca5e1aSKuppuswamy Sathyanarayanan * The virtual address is part of a larger page, create the next
449dbca5e1aSKuppuswamy Sathyanarayanan * level page table mapping (4K or 2M). If it is part of a 2M
450dbca5e1aSKuppuswamy Sathyanarayanan * page then we request a split of the large page into 4K
451dbca5e1aSKuppuswamy Sathyanarayanan * chunks. A 1GB large page is split into 2M pages, resp.
452dbca5e1aSKuppuswamy Sathyanarayanan */
453dbca5e1aSKuppuswamy Sathyanarayanan if (level == PG_LEVEL_2M)
454dbca5e1aSKuppuswamy Sathyanarayanan split_page_size_mask = 0;
455dbca5e1aSKuppuswamy Sathyanarayanan else
456dbca5e1aSKuppuswamy Sathyanarayanan split_page_size_mask = 1 << PG_LEVEL_2M;
457dbca5e1aSKuppuswamy Sathyanarayanan
458dbca5e1aSKuppuswamy Sathyanarayanan /*
459dbca5e1aSKuppuswamy Sathyanarayanan * kernel_physical_mapping_change() does not flush the TLBs, so
460dbca5e1aSKuppuswamy Sathyanarayanan * a TLB flush is required after we exit from the for loop.
461dbca5e1aSKuppuswamy Sathyanarayanan */
462dbca5e1aSKuppuswamy Sathyanarayanan kernel_physical_mapping_change(__pa(vaddr & pmask),
463dbca5e1aSKuppuswamy Sathyanarayanan __pa((vaddr_end & pmask) + psize),
464dbca5e1aSKuppuswamy Sathyanarayanan split_page_size_mask);
465dbca5e1aSKuppuswamy Sathyanarayanan }
466dbca5e1aSKuppuswamy Sathyanarayanan
467dbca5e1aSKuppuswamy Sathyanarayanan ret = 0;
468dbca5e1aSKuppuswamy Sathyanarayanan
469ac3f9c9fSSteve Rutherford early_set_mem_enc_dec_hypercall(start, size, enc);
470dbca5e1aSKuppuswamy Sathyanarayanan out:
471dbca5e1aSKuppuswamy Sathyanarayanan __flush_tlb_all();
472dbca5e1aSKuppuswamy Sathyanarayanan return ret;
473dbca5e1aSKuppuswamy Sathyanarayanan }
474dbca5e1aSKuppuswamy Sathyanarayanan
early_set_memory_decrypted(unsigned long vaddr,unsigned long size)475dbca5e1aSKuppuswamy Sathyanarayanan int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size)
476dbca5e1aSKuppuswamy Sathyanarayanan {
477dbca5e1aSKuppuswamy Sathyanarayanan return early_set_memory_enc_dec(vaddr, size, false);
478dbca5e1aSKuppuswamy Sathyanarayanan }
479dbca5e1aSKuppuswamy Sathyanarayanan
early_set_memory_encrypted(unsigned long vaddr,unsigned long size)480dbca5e1aSKuppuswamy Sathyanarayanan int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)
481dbca5e1aSKuppuswamy Sathyanarayanan {
482dbca5e1aSKuppuswamy Sathyanarayanan return early_set_memory_enc_dec(vaddr, size, true);
483dbca5e1aSKuppuswamy Sathyanarayanan }
484dbca5e1aSKuppuswamy Sathyanarayanan
early_set_mem_enc_dec_hypercall(unsigned long vaddr,unsigned long size,bool enc)485ac3f9c9fSSteve Rutherford void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, unsigned long size, bool enc)
486dbca5e1aSKuppuswamy Sathyanarayanan {
487ac3f9c9fSSteve Rutherford enc_dec_hypercall(vaddr, size, enc);
4881e8c5971SBrijesh Singh }
4891e8c5971SBrijesh Singh
sme_early_init(void)4901e8c5971SBrijesh Singh void __init sme_early_init(void)
4911e8c5971SBrijesh Singh {
4921e8c5971SBrijesh Singh if (!sme_me_mask)
4931e8c5971SBrijesh Singh return;
4941e8c5971SBrijesh Singh
4951e8c5971SBrijesh Singh early_pmd_flags = __sme_set(early_pmd_flags);
4961e8c5971SBrijesh Singh
4971e8c5971SBrijesh Singh __supported_pte_mask = __sme_set(__supported_pte_mask);
4981e8c5971SBrijesh Singh
4991e8c5971SBrijesh Singh /* Update the protection map with memory encryption mask */
5004867fbbdSAnshuman Khandual add_encrypt_protection_map();
5011e8c5971SBrijesh Singh
5021e8c5971SBrijesh Singh x86_platform.guest.enc_status_change_prepare = amd_enc_status_change_prepare;
5031e8c5971SBrijesh Singh x86_platform.guest.enc_status_change_finish = amd_enc_status_change_finish;
5041e8c5971SBrijesh Singh x86_platform.guest.enc_tlb_flush_required = amd_enc_tlb_flush_required;
5051e8c5971SBrijesh Singh x86_platform.guest.enc_cache_flush_required = amd_enc_cache_flush_required;
506ff3cfcb0SThomas Gleixner
507ff3cfcb0SThomas Gleixner /*
508ff3cfcb0SThomas Gleixner * AMD-SEV-ES intercepts the RDMSR to read the X2APIC ID in the
509ff3cfcb0SThomas Gleixner * parallel bringup low level code. That raises #VC which cannot be
510ff3cfcb0SThomas Gleixner * handled there.
511ff3cfcb0SThomas Gleixner * It does not provide a RDMSR GHCB protocol so the early startup
512ff3cfcb0SThomas Gleixner * code cannot directly communicate with the secure firmware. The
513ff3cfcb0SThomas Gleixner * alternative solution to retrieve the APIC ID via CPUID(0xb),
514ff3cfcb0SThomas Gleixner * which is covered by the GHCB protocol, is not viable either
515ff3cfcb0SThomas Gleixner * because there is no enforcement of the CPUID(0xb) provided
516ff3cfcb0SThomas Gleixner * "initial" APIC ID to be the same as the real APIC ID.
517ff3cfcb0SThomas Gleixner * Disable parallel bootup.
518ff3cfcb0SThomas Gleixner */
519ff3cfcb0SThomas Gleixner if (sev_status & MSR_AMD64_SEV_ES_ENABLED)
520ff3cfcb0SThomas Gleixner x86_cpuinit.parallel_bringup = false;
52134c686e5SKirill A. Shutemov
52234c686e5SKirill A. Shutemov /*
52334c686e5SKirill A. Shutemov * The VMM is capable of injecting interrupt 0x80 and triggering the
52434c686e5SKirill A. Shutemov * compatibility syscall path.
52534c686e5SKirill A. Shutemov *
52634c686e5SKirill A. Shutemov * By default, the 32-bit emulation is disabled in order to ensure
52734c686e5SKirill A. Shutemov * the safety of the VM.
52834c686e5SKirill A. Shutemov */
52934c686e5SKirill A. Shutemov if (sev_status & MSR_AMD64_SEV_ENABLED)
53034c686e5SKirill A. Shutemov ia32_disable();
5314338e40dSKevin Loughlin
5324338e40dSKevin Loughlin /*
5334338e40dSKevin Loughlin * Override init functions that scan the ROM region in SEV-SNP guests,
5344338e40dSKevin Loughlin * as this memory is not pre-validated and would thus cause a crash.
5354338e40dSKevin Loughlin */
5364338e40dSKevin Loughlin if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
5374338e40dSKevin Loughlin x86_init.mpparse.find_smp_config = x86_init_noop;
5384338e40dSKevin Loughlin x86_init.pci.init_irq = x86_init_noop;
5394338e40dSKevin Loughlin x86_init.resources.probe_roms = x86_init_noop;
5404338e40dSKevin Loughlin
5414338e40dSKevin Loughlin /*
5424338e40dSKevin Loughlin * DMI setup behavior for SEV-SNP guests depends on
5434338e40dSKevin Loughlin * efi_enabled(EFI_CONFIG_TABLES), which hasn't been
5444338e40dSKevin Loughlin * parsed yet. snp_dmi_setup() will run after that
5454338e40dSKevin Loughlin * parsing has happened.
5464338e40dSKevin Loughlin */
5474338e40dSKevin Loughlin x86_init.resources.dmi_setup = snp_dmi_setup;
5484338e40dSKevin Loughlin }
549dbca5e1aSKuppuswamy Sathyanarayanan }
550dbca5e1aSKuppuswamy Sathyanarayanan
mem_encrypt_free_decrypted_mem(void)551dbca5e1aSKuppuswamy Sathyanarayanan void __init mem_encrypt_free_decrypted_mem(void)
552dbca5e1aSKuppuswamy Sathyanarayanan {
553dbca5e1aSKuppuswamy Sathyanarayanan unsigned long vaddr, vaddr_end, npages;
554dbca5e1aSKuppuswamy Sathyanarayanan int r;
555dbca5e1aSKuppuswamy Sathyanarayanan
556dbca5e1aSKuppuswamy Sathyanarayanan vaddr = (unsigned long)__start_bss_decrypted_unused;
557dbca5e1aSKuppuswamy Sathyanarayanan vaddr_end = (unsigned long)__end_bss_decrypted;
558dbca5e1aSKuppuswamy Sathyanarayanan npages = (vaddr_end - vaddr) >> PAGE_SHIFT;
559dbca5e1aSKuppuswamy Sathyanarayanan
560dbca5e1aSKuppuswamy Sathyanarayanan /*
561c7b5254bSMichael Kelley * If the unused memory range was mapped decrypted, change the encryption
562c7b5254bSMichael Kelley * attribute from decrypted to encrypted before freeing it. Base the
563c7b5254bSMichael Kelley * re-encryption on the same condition used for the decryption in
564c7b5254bSMichael Kelley * sme_postprocess_startup(). Higher level abstractions, such as
565c7b5254bSMichael Kelley * CC_ATTR_MEM_ENCRYPT, aren't necessarily equivalent in a Hyper-V VM
566c7b5254bSMichael Kelley * using vTOM, where sme_me_mask is always zero.
567dbca5e1aSKuppuswamy Sathyanarayanan */
568c7b5254bSMichael Kelley if (sme_me_mask) {
569dbca5e1aSKuppuswamy Sathyanarayanan r = set_memory_encrypted(vaddr, npages);
570dbca5e1aSKuppuswamy Sathyanarayanan if (r) {
571dbca5e1aSKuppuswamy Sathyanarayanan pr_warn("failed to free unused decrypted pages\n");
572dbca5e1aSKuppuswamy Sathyanarayanan return;
573dbca5e1aSKuppuswamy Sathyanarayanan }
574dbca5e1aSKuppuswamy Sathyanarayanan }
575dbca5e1aSKuppuswamy Sathyanarayanan
576dbca5e1aSKuppuswamy Sathyanarayanan free_init_pages("unused decrypted", vaddr, vaddr_end);
577dbca5e1aSKuppuswamy Sathyanarayanan }
578