1 // SPDX-License-Identifier: GPL-2.0
2 #include "misc.h"
3 #include <asm/e820/types.h>
4 #include <asm/processor.h>
5 #include "pgtable.h"
6 #include "../string.h"
7 #include "efi.h"
8
9 #define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */
10 #define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */
11
12 #ifdef CONFIG_X86_5LEVEL
13 /* __pgtable_l5_enabled needs to be in .data to avoid being cleared along with .bss */
14 unsigned int __section(".data") __pgtable_l5_enabled;
15 unsigned int __section(".data") pgdir_shift = 39;
16 unsigned int __section(".data") ptrs_per_p4d = 1;
17 #endif
18
19 /* Buffer to preserve trampoline memory */
20 static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
21
22 /*
23 * Trampoline address will be printed by extract_kernel() for debugging
24 * purposes.
25 *
26 * Avoid putting the pointer into .bss as it will be cleared between
27 * configure_5level_paging() and extract_kernel().
28 */
29 unsigned long *trampoline_32bit __section(".data");
30
31 extern struct boot_params *boot_params;
32 int cmdline_find_option_bool(const char *option);
33
find_trampoline_placement(void)34 static unsigned long find_trampoline_placement(void)
35 {
36 unsigned long bios_start = 0, ebda_start = 0;
37 struct boot_e820_entry *entry;
38 char *signature;
39 int i;
40
41 /*
42 * Find a suitable spot for the trampoline.
43 * This code is based on reserve_bios_regions().
44 */
45
46 /*
47 * EFI systems may not provide legacy ROM. The memory may not be mapped
48 * at all.
49 *
50 * Only look for values in the legacy ROM for non-EFI system.
51 */
52 signature = (char *)&boot_params->efi_info.efi_loader_signature;
53 if (strncmp(signature, EFI32_LOADER_SIGNATURE, 4) &&
54 strncmp(signature, EFI64_LOADER_SIGNATURE, 4)) {
55 ebda_start = *(unsigned short *)0x40e << 4;
56 bios_start = *(unsigned short *)0x413 << 10;
57 }
58
59 if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX)
60 bios_start = BIOS_START_MAX;
61
62 if (ebda_start > BIOS_START_MIN && ebda_start < bios_start)
63 bios_start = ebda_start;
64
65 bios_start = round_down(bios_start, PAGE_SIZE);
66
67 /* Find the first usable memory region under bios_start. */
68 for (i = boot_params->e820_entries - 1; i >= 0; i--) {
69 unsigned long new = bios_start;
70
71 entry = &boot_params->e820_table[i];
72
73 /* Skip all entries above bios_start. */
74 if (bios_start <= entry->addr)
75 continue;
76
77 /* Skip non-RAM entries. */
78 if (entry->type != E820_TYPE_RAM)
79 continue;
80
81 /* Adjust bios_start to the end of the entry if needed. */
82 if (bios_start > entry->addr + entry->size)
83 new = entry->addr + entry->size;
84
85 /* Keep bios_start page-aligned. */
86 new = round_down(new, PAGE_SIZE);
87
88 /* Skip the entry if it's too small. */
89 if (new - TRAMPOLINE_32BIT_SIZE < entry->addr)
90 continue;
91
92 /* Protect against underflow. */
93 if (new - TRAMPOLINE_32BIT_SIZE > bios_start)
94 break;
95
96 bios_start = new;
97 break;
98 }
99
100 /* Place the trampoline just below the end of low memory */
101 return bios_start - TRAMPOLINE_32BIT_SIZE;
102 }
103
configure_5level_paging(struct boot_params * bp,void * pgtable)104 asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
105 {
106 void (*toggle_la57)(void *cr3);
107 bool l5_required = false;
108
109 /* Initialize boot_params. Required for cmdline_find_option_bool(). */
110 boot_params = bp;
111
112 /*
113 * Check if LA57 is desired and supported.
114 *
115 * There are several parts to the check:
116 * - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y
117 * - if user asked to disable 5-level paging: no5lvl in cmdline
118 * - if the machine supports 5-level paging:
119 * + CPUID leaf 7 is supported
120 * + the leaf has the feature bit set
121 *
122 * That's substitute for boot_cpu_has() in early boot code.
123 */
124 if (IS_ENABLED(CONFIG_X86_5LEVEL) &&
125 !cmdline_find_option_bool("no5lvl") &&
126 native_cpuid_eax(0) >= 7 &&
127 (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {
128 l5_required = true;
129
130 /* Initialize variables for 5-level paging */
131 __pgtable_l5_enabled = 1;
132 pgdir_shift = 48;
133 ptrs_per_p4d = 512;
134 }
135
136 /*
137 * The trampoline will not be used if the paging mode is already set to
138 * the desired one.
139 */
140 if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
141 return;
142
143 trampoline_32bit = (unsigned long *)find_trampoline_placement();
144
145 /* Preserve trampoline memory */
146 memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
147
148 /* Clear trampoline memory first */
149 memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
150
151 /* Copy trampoline code in place */
152 toggle_la57 = memcpy(trampoline_32bit +
153 TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
154 &trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
155
156 /*
157 * Avoid the need for a stack in the 32-bit trampoline code, by using
158 * LJMP rather than LRET to return back to long mode. LJMP takes an
159 * immediate absolute address, which needs to be adjusted based on the
160 * placement of the trampoline.
161 */
162 *(u32 *)((u8 *)toggle_la57 + trampoline_ljmp_imm_offset) +=
163 (unsigned long)toggle_la57;
164
165 /*
166 * The code below prepares page table in trampoline memory.
167 *
168 * The new page table will be used by trampoline code for switching
169 * from 4- to 5-level paging or vice versa.
170 */
171
172 if (l5_required) {
173 /*
174 * For 4- to 5-level paging transition, set up current CR3 as
175 * the first and the only entry in a new top-level page table.
176 */
177 *trampoline_32bit = __native_read_cr3() | _PAGE_TABLE_NOENC;
178 } else {
179 unsigned long src;
180
181 /*
182 * For 5- to 4-level paging transition, copy page table pointed
183 * by first entry in the current top-level page table as our
184 * new top-level page table.
185 *
186 * We cannot just point to the page table from trampoline as it
187 * may be above 4G.
188 */
189 src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
190 memcpy(trampoline_32bit, (void *)src, PAGE_SIZE);
191 }
192
193 toggle_la57(trampoline_32bit);
194
195 /*
196 * Move the top level page table out of trampoline memory.
197 */
198 memcpy(pgtable, trampoline_32bit, PAGE_SIZE);
199 native_write_cr3((unsigned long)pgtable);
200
201 /* Restore trampoline memory */
202 memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
203 }
204