1 #include <linux/efi.h> 2 #include <asm/e820/types.h> 3 #include <asm/processor.h> 4 #include <asm/efi.h> 5 #include "pgtable.h" 6 #include "../string.h" 7 8 #define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */ 9 #define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */ 10 11 struct paging_config { 12 unsigned long trampoline_start; 13 unsigned long l5_required; 14 }; 15 16 /* Buffer to preserve trampoline memory */ 17 static char trampoline_save[TRAMPOLINE_32BIT_SIZE]; 18 19 /* 20 * Trampoline address will be printed by extract_kernel() for debugging 21 * purposes. 22 * 23 * Avoid putting the pointer into .bss as it will be cleared between 24 * paging_prepare() and extract_kernel(). 25 */ 26 unsigned long *trampoline_32bit __section(.data); 27 28 extern struct boot_params *boot_params; 29 int cmdline_find_option_bool(const char *option); 30 31 static unsigned long find_trampoline_placement(void) 32 { 33 unsigned long bios_start = 0, ebda_start = 0; 34 struct boot_e820_entry *entry; 35 char *signature; 36 int i; 37 38 /* 39 * Find a suitable spot for the trampoline. 40 * This code is based on reserve_bios_regions(). 41 */ 42 43 /* 44 * EFI systems may not provide legacy ROM. The memory may not be mapped 45 * at all. 46 * 47 * Only look for values in the legacy ROM for non-EFI system. 48 */ 49 signature = (char *)&boot_params->efi_info.efi_loader_signature; 50 if (strncmp(signature, EFI32_LOADER_SIGNATURE, 4) && 51 strncmp(signature, EFI64_LOADER_SIGNATURE, 4)) { 52 ebda_start = *(unsigned short *)0x40e << 4; 53 bios_start = *(unsigned short *)0x413 << 10; 54 } 55 56 if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX) 57 bios_start = BIOS_START_MAX; 58 59 if (ebda_start > BIOS_START_MIN && ebda_start < bios_start) 60 bios_start = ebda_start; 61 62 bios_start = round_down(bios_start, PAGE_SIZE); 63 64 /* Find the first usable memory region under bios_start. */ 65 for (i = boot_params->e820_entries - 1; i >= 0; i--) { 66 unsigned long new = bios_start; 67 68 entry = &boot_params->e820_table[i]; 69 70 /* Skip all entries above bios_start. */ 71 if (bios_start <= entry->addr) 72 continue; 73 74 /* Skip non-RAM entries. */ 75 if (entry->type != E820_TYPE_RAM) 76 continue; 77 78 /* Adjust bios_start to the end of the entry if needed. */ 79 if (bios_start > entry->addr + entry->size) 80 new = entry->addr + entry->size; 81 82 /* Keep bios_start page-aligned. */ 83 new = round_down(new, PAGE_SIZE); 84 85 /* Skip the entry if it's too small. */ 86 if (new - TRAMPOLINE_32BIT_SIZE < entry->addr) 87 continue; 88 89 /* Protect against underflow. */ 90 if (new - TRAMPOLINE_32BIT_SIZE > bios_start) 91 break; 92 93 bios_start = new; 94 break; 95 } 96 97 /* Place the trampoline just below the end of low memory */ 98 return bios_start - TRAMPOLINE_32BIT_SIZE; 99 } 100 101 struct paging_config paging_prepare(void *rmode) 102 { 103 struct paging_config paging_config = {}; 104 105 /* Initialize boot_params. Required for cmdline_find_option_bool(). */ 106 boot_params = rmode; 107 108 /* 109 * Check if LA57 is desired and supported. 110 * 111 * There are several parts to the check: 112 * - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y 113 * - if user asked to disable 5-level paging: no5lvl in cmdline 114 * - if the machine supports 5-level paging: 115 * + CPUID leaf 7 is supported 116 * + the leaf has the feature bit set 117 * 118 * That's substitute for boot_cpu_has() in early boot code. 119 */ 120 if (IS_ENABLED(CONFIG_X86_5LEVEL) && 121 !cmdline_find_option_bool("no5lvl") && 122 native_cpuid_eax(0) >= 7 && 123 (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) { 124 paging_config.l5_required = 1; 125 } 126 127 paging_config.trampoline_start = find_trampoline_placement(); 128 129 trampoline_32bit = (unsigned long *)paging_config.trampoline_start; 130 131 /* Preserve trampoline memory */ 132 memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE); 133 134 /* Clear trampoline memory first */ 135 memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE); 136 137 /* Copy trampoline code in place */ 138 memcpy(trampoline_32bit + TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long), 139 &trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE); 140 141 /* 142 * The code below prepares page table in trampoline memory. 143 * 144 * The new page table will be used by trampoline code for switching 145 * from 4- to 5-level paging or vice versa. 146 * 147 * If switching is not required, the page table is unused: trampoline 148 * code wouldn't touch CR3. 149 */ 150 151 /* 152 * We are not going to use the page table in trampoline memory if we 153 * are already in the desired paging mode. 154 */ 155 if (paging_config.l5_required == !!(native_read_cr4() & X86_CR4_LA57)) 156 goto out; 157 158 if (paging_config.l5_required) { 159 /* 160 * For 4- to 5-level paging transition, set up current CR3 as 161 * the first and the only entry in a new top-level page table. 162 */ 163 trampoline_32bit[TRAMPOLINE_32BIT_PGTABLE_OFFSET] = __native_read_cr3() | _PAGE_TABLE_NOENC; 164 } else { 165 unsigned long src; 166 167 /* 168 * For 5- to 4-level paging transition, copy page table pointed 169 * by first entry in the current top-level page table as our 170 * new top-level page table. 171 * 172 * We cannot just point to the page table from trampoline as it 173 * may be above 4G. 174 */ 175 src = *(unsigned long *)__native_read_cr3() & PAGE_MASK; 176 memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long), 177 (void *)src, PAGE_SIZE); 178 } 179 180 out: 181 return paging_config; 182 } 183 184 void cleanup_trampoline(void *pgtable) 185 { 186 void *trampoline_pgtable; 187 188 trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long); 189 190 /* 191 * Move the top level page table out of trampoline memory, 192 * if it's there. 193 */ 194 if ((void *)__native_read_cr3() == trampoline_pgtable) { 195 memcpy(pgtable, trampoline_pgtable, PAGE_SIZE); 196 native_write_cr3((unsigned long)pgtable); 197 } 198 199 /* Restore trampoline memory */ 200 memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); 201 } 202