1 #include <asm/processor.h>
2 #include "pgtable.h"
3 #include "../string.h"
4 
5 /*
6  * __force_order is used by special_insns.h asm code to force instruction
7  * serialization.
8  *
9  * It is not referenced from the code, but GCC < 5 with -fPIE would fail
10  * due to an undefined symbol. Define it to make these ancient GCCs work.
11  */
12 unsigned long __force_order;
13 
14 #define BIOS_START_MIN		0x20000U	/* 128K, less than this is insane */
15 #define BIOS_START_MAX		0x9f000U	/* 640K, absolute maximum */
16 
17 struct paging_config {
18 	unsigned long trampoline_start;
19 	unsigned long l5_required;
20 };
21 
22 /* Buffer to preserve trampoline memory */
23 static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
24 
25 /*
26  * Trampoline address will be printed by extract_kernel() for debugging
27  * purposes.
28  *
29  * Avoid putting the pointer into .bss as it will be cleared between
30  * paging_prepare() and extract_kernel().
31  */
32 unsigned long *trampoline_32bit __section(.data);
33 
34 struct paging_config paging_prepare(void)
35 {
36 	struct paging_config paging_config = {};
37 	unsigned long bios_start, ebda_start;
38 
39 	/*
40 	 * Check if LA57 is desired and supported.
41 	 *
42 	 * There are two parts to the check:
43 	 *   - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y
44 	 *   - if the machine supports 5-level paging:
45 	 *     + CPUID leaf 7 is supported
46 	 *     + the leaf has the feature bit set
47 	 *
48 	 * That's substitute for boot_cpu_has() in early boot code.
49 	 */
50 	if (IS_ENABLED(CONFIG_X86_5LEVEL) &&
51 			native_cpuid_eax(0) >= 7 &&
52 			(native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {
53 		paging_config.l5_required = 1;
54 	}
55 
56 	/*
57 	 * Find a suitable spot for the trampoline.
58 	 * This code is based on reserve_bios_regions().
59 	 */
60 
61 	ebda_start = *(unsigned short *)0x40e << 4;
62 	bios_start = *(unsigned short *)0x413 << 10;
63 
64 	if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX)
65 		bios_start = BIOS_START_MAX;
66 
67 	if (ebda_start > BIOS_START_MIN && ebda_start < bios_start)
68 		bios_start = ebda_start;
69 
70 	/* Place the trampoline just below the end of low memory, aligned to 4k */
71 	paging_config.trampoline_start = bios_start - TRAMPOLINE_32BIT_SIZE;
72 	paging_config.trampoline_start = round_down(paging_config.trampoline_start, PAGE_SIZE);
73 
74 	trampoline_32bit = (unsigned long *)paging_config.trampoline_start;
75 
76 	/* Preserve trampoline memory */
77 	memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
78 
79 	/* Clear trampoline memory first */
80 	memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
81 
82 	/* Copy trampoline code in place */
83 	memcpy(trampoline_32bit + TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
84 			&trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
85 
86 	/*
87 	 * The code below prepares page table in trampoline memory.
88 	 *
89 	 * The new page table will be used by trampoline code for switching
90 	 * from 4- to 5-level paging or vice versa.
91 	 *
92 	 * If switching is not required, the page table is unused: trampoline
93 	 * code wouldn't touch CR3.
94 	 */
95 
96 	/*
97 	 * We are not going to use the page table in trampoline memory if we
98 	 * are already in the desired paging mode.
99 	 */
100 	if (paging_config.l5_required == !!(native_read_cr4() & X86_CR4_LA57))
101 		goto out;
102 
103 	if (paging_config.l5_required) {
104 		/*
105 		 * For 4- to 5-level paging transition, set up current CR3 as
106 		 * the first and the only entry in a new top-level page table.
107 		 */
108 		trampoline_32bit[TRAMPOLINE_32BIT_PGTABLE_OFFSET] = __native_read_cr3() | _PAGE_TABLE_NOENC;
109 	} else {
110 		unsigned long src;
111 
112 		/*
113 		 * For 5- to 4-level paging transition, copy page table pointed
114 		 * by first entry in the current top-level page table as our
115 		 * new top-level page table.
116 		 *
117 		 * We cannot just point to the page table from trampoline as it
118 		 * may be above 4G.
119 		 */
120 		src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
121 		memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long),
122 		       (void *)src, PAGE_SIZE);
123 	}
124 
125 out:
126 	return paging_config;
127 }
128 
129 void cleanup_trampoline(void *pgtable)
130 {
131 	void *trampoline_pgtable;
132 
133 	trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET;
134 
135 	/*
136 	 * Move the top level page table out of trampoline memory,
137 	 * if it's there.
138 	 */
139 	if ((void *)__native_read_cr3() == trampoline_pgtable) {
140 		memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
141 		native_write_cr3((unsigned long)pgtable);
142 	}
143 
144 	/* Restore trampoline memory */
145 	memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
146 }
147