1 #include <asm/processor.h>
2 #include "pgtable.h"
3 #include "../string.h"
4 
5 /*
6  * __force_order is used by special_insns.h asm code to force instruction
7  * serialization.
8  *
9  * It is not referenced from the code, but GCC < 5 with -fPIE would fail
10  * due to an undefined symbol. Define it to make these ancient GCCs work.
11  */
12 unsigned long __force_order;
13 
14 #define BIOS_START_MIN		0x20000U	/* 128K, less than this is insane */
15 #define BIOS_START_MAX		0x9f000U	/* 640K, absolute maximum */
16 
17 struct paging_config {
18 	unsigned long trampoline_start;
19 	unsigned long l5_required;
20 };
21 
22 /* Buffer to preserve trampoline memory */
23 static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
24 
25 /*
26  * Trampoline address will be printed by extract_kernel() for debugging
27  * purposes.
28  *
29  * Avoid putting the pointer into .bss as it will be cleared between
30  * paging_prepare() and extract_kernel().
31  */
32 unsigned long *trampoline_32bit __section(.data);
33 
34 extern struct boot_params *boot_params;
35 int cmdline_find_option_bool(const char *option);
36 
37 struct paging_config paging_prepare(void *rmode)
38 {
39 	struct paging_config paging_config = {};
40 	unsigned long bios_start, ebda_start;
41 
42 	/* Initialize boot_params. Required for cmdline_find_option_bool(). */
43 	boot_params = rmode;
44 
45 	/*
46 	 * Check if LA57 is desired and supported.
47 	 *
48 	 * There are several parts to the check:
49 	 *   - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y
50 	 *   - if user asked to disable 5-level paging: no5lvl in cmdline
51 	 *   - if the machine supports 5-level paging:
52 	 *     + CPUID leaf 7 is supported
53 	 *     + the leaf has the feature bit set
54 	 *
55 	 * That's substitute for boot_cpu_has() in early boot code.
56 	 */
57 	if (IS_ENABLED(CONFIG_X86_5LEVEL) &&
58 			!cmdline_find_option_bool("no5lvl") &&
59 			native_cpuid_eax(0) >= 7 &&
60 			(native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {
61 		paging_config.l5_required = 1;
62 	}
63 
64 	/*
65 	 * Find a suitable spot for the trampoline.
66 	 * This code is based on reserve_bios_regions().
67 	 */
68 
69 	ebda_start = *(unsigned short *)0x40e << 4;
70 	bios_start = *(unsigned short *)0x413 << 10;
71 
72 	if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX)
73 		bios_start = BIOS_START_MAX;
74 
75 	if (ebda_start > BIOS_START_MIN && ebda_start < bios_start)
76 		bios_start = ebda_start;
77 
78 	/* Place the trampoline just below the end of low memory, aligned to 4k */
79 	paging_config.trampoline_start = bios_start - TRAMPOLINE_32BIT_SIZE;
80 	paging_config.trampoline_start = round_down(paging_config.trampoline_start, PAGE_SIZE);
81 
82 	trampoline_32bit = (unsigned long *)paging_config.trampoline_start;
83 
84 	/* Preserve trampoline memory */
85 	memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
86 
87 	/* Clear trampoline memory first */
88 	memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
89 
90 	/* Copy trampoline code in place */
91 	memcpy(trampoline_32bit + TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
92 			&trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
93 
94 	/*
95 	 * The code below prepares page table in trampoline memory.
96 	 *
97 	 * The new page table will be used by trampoline code for switching
98 	 * from 4- to 5-level paging or vice versa.
99 	 *
100 	 * If switching is not required, the page table is unused: trampoline
101 	 * code wouldn't touch CR3.
102 	 */
103 
104 	/*
105 	 * We are not going to use the page table in trampoline memory if we
106 	 * are already in the desired paging mode.
107 	 */
108 	if (paging_config.l5_required == !!(native_read_cr4() & X86_CR4_LA57))
109 		goto out;
110 
111 	if (paging_config.l5_required) {
112 		/*
113 		 * For 4- to 5-level paging transition, set up current CR3 as
114 		 * the first and the only entry in a new top-level page table.
115 		 */
116 		trampoline_32bit[TRAMPOLINE_32BIT_PGTABLE_OFFSET] = __native_read_cr3() | _PAGE_TABLE_NOENC;
117 	} else {
118 		unsigned long src;
119 
120 		/*
121 		 * For 5- to 4-level paging transition, copy page table pointed
122 		 * by first entry in the current top-level page table as our
123 		 * new top-level page table.
124 		 *
125 		 * We cannot just point to the page table from trampoline as it
126 		 * may be above 4G.
127 		 */
128 		src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
129 		memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long),
130 		       (void *)src, PAGE_SIZE);
131 	}
132 
133 out:
134 	return paging_config;
135 }
136 
137 void cleanup_trampoline(void *pgtable)
138 {
139 	void *trampoline_pgtable;
140 
141 	trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long);
142 
143 	/*
144 	 * Move the top level page table out of trampoline memory,
145 	 * if it's there.
146 	 */
147 	if ((void *)__native_read_cr3() == trampoline_pgtable) {
148 		memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
149 		native_write_cr3((unsigned long)pgtable);
150 	}
151 
152 	/* Restore trampoline memory */
153 	memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
154 }
155