1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2014, 2015 Intel Corporation; author Matt Fleming
4 *
5 * Early support for invoking 32-bit EFI services from a 64-bit kernel.
6 *
7 * Because this thunking occurs before ExitBootServices() we have to
8 * restore the firmware's 32-bit GDT and IDT before we make EFI service
9 * calls.
10 *
11 * On the plus side, we don't have to worry about mangling 64-bit
12 * addresses into 32-bits because we're executing with an identity
13 * mapped pagetable and haven't transitioned to 64-bit virtual addresses
14 * yet.
15 */
16
17#include <linux/linkage.h>
18#include <asm/asm-offsets.h>
19#include <asm/msr.h>
20#include <asm/page_types.h>
21#include <asm/processor-flags.h>
22#include <asm/segment.h>
23#include <asm/setup.h>
24
25	.code64
26	.text
27/*
28 * When booting in 64-bit mode on 32-bit EFI firmware, startup_64_mixed_mode()
29 * is the first thing that runs after switching to long mode. Depending on
30 * whether the EFI handover protocol or the compat entry point was used to
31 * enter the kernel, it will either branch to the common 64-bit EFI stub
32 * entrypoint efi_stub_entry() directly, or via the 64-bit EFI PE/COFF
33 * entrypoint efi_pe_entry(). In the former case, the bootloader must provide a
34 * struct bootparams pointer as the third argument, so the presence of such a
35 * pointer is used to disambiguate.
36 *
37 *                                                             +--------------+
38 *  +------------------+     +------------+            +------>| efi_pe_entry |
39 *  | efi32_pe_entry   |---->|            |            |       +-----------+--+
40 *  +------------------+     |            |     +------+----------------+  |
41 *                           | startup_32 |---->| startup_64_mixed_mode |  |
42 *  +------------------+     |            |     +------+----------------+  |
43 *  | efi32_stub_entry |---->|            |            |                   |
44 *  +------------------+     +------------+            |                   |
45 *                                                     V                   |
46 *                           +------------+     +----------------+         |
47 *                           | startup_64 |<----| efi_stub_entry |<--------+
48 *                           +------------+     +----------------+
49 */
50SYM_FUNC_START(startup_64_mixed_mode)
51	lea	efi32_boot_args(%rip), %rdx
52	mov	0(%rdx), %edi
53	mov	4(%rdx), %esi
54
55	/* Switch to the firmware's stack */
56	movl	efi32_boot_sp(%rip), %esp
57	andl	$~7, %esp
58
59#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
60	mov	8(%rdx), %edx		// saved bootparams pointer
61	test	%edx, %edx
62	jnz	efi_stub_entry
63#endif
64	/*
65	 * efi_pe_entry uses MS calling convention, which requires 32 bytes of
66	 * shadow space on the stack even if all arguments are passed in
67	 * registers. We also need an additional 8 bytes for the space that
68	 * would be occupied by the return address, and this also results in
69	 * the correct stack alignment for entry.
70	 */
71	sub	$40, %rsp
72	mov	%rdi, %rcx		// MS calling convention
73	mov	%rsi, %rdx
74	jmp	efi_pe_entry
75SYM_FUNC_END(startup_64_mixed_mode)
76
77SYM_FUNC_START(__efi64_thunk)
78	push	%rbp
79	push	%rbx
80
81	movl	%ds, %eax
82	push	%rax
83	movl	%es, %eax
84	push	%rax
85	movl	%ss, %eax
86	push	%rax
87
88	/* Copy args passed on stack */
89	movq	0x30(%rsp), %rbp
90	movq	0x38(%rsp), %rbx
91	movq	0x40(%rsp), %rax
92
93	/*
94	 * Convert x86-64 ABI params to i386 ABI
95	 */
96	subq	$64, %rsp
97	movl	%esi, 0x0(%rsp)
98	movl	%edx, 0x4(%rsp)
99	movl	%ecx, 0x8(%rsp)
100	movl	%r8d, 0xc(%rsp)
101	movl	%r9d, 0x10(%rsp)
102	movl	%ebp, 0x14(%rsp)
103	movl	%ebx, 0x18(%rsp)
104	movl	%eax, 0x1c(%rsp)
105
106	leaq	0x20(%rsp), %rbx
107	sgdt	(%rbx)
108	sidt	16(%rbx)
109
110	leaq	1f(%rip), %rbp
111
112	/*
113	 * Switch to IDT and GDT with 32-bit segments. These are the firmware
114	 * GDT and IDT that were installed when the kernel started executing.
115	 * The pointers were saved by the efi32_entry() routine below.
116	 *
117	 * Pass the saved DS selector to the 32-bit code, and use far return to
118	 * restore the saved CS selector.
119	 */
120	lidt	efi32_boot_idt(%rip)
121	lgdt	efi32_boot_gdt(%rip)
122
123	movzwl	efi32_boot_ds(%rip), %edx
124	movzwq	efi32_boot_cs(%rip), %rax
125	pushq	%rax
126	leaq	efi_enter32(%rip), %rax
127	pushq	%rax
128	lretq
129
1301:	addq	$64, %rsp
131	movq	%rdi, %rax
132
133	pop	%rbx
134	movl	%ebx, %ss
135	pop	%rbx
136	movl	%ebx, %es
137	pop	%rbx
138	movl	%ebx, %ds
139	/* Clear out 32-bit selector from FS and GS */
140	xorl	%ebx, %ebx
141	movl	%ebx, %fs
142	movl	%ebx, %gs
143
144	pop	%rbx
145	pop	%rbp
146	RET
147SYM_FUNC_END(__efi64_thunk)
148
149	.code32
150#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
151SYM_FUNC_START(efi32_stub_entry)
152	call	1f
1531:	popl	%ecx
154	leal	(efi32_boot_args - 1b)(%ecx), %ebx
155
156	/* Clear BSS */
157	xorl	%eax, %eax
158	leal	(_bss - 1b)(%ecx), %edi
159	leal	(_ebss - 1b)(%ecx), %ecx
160	subl	%edi, %ecx
161	shrl	$2, %ecx
162	cld
163	rep	stosl
164
165	add	$0x4, %esp		/* Discard return address */
166	popl	%ecx
167	popl	%edx
168	popl	%esi
169	movl	%esi, 8(%ebx)
170	jmp	efi32_entry
171SYM_FUNC_END(efi32_stub_entry)
172#endif
173
174/*
175 * EFI service pointer must be in %edi.
176 *
177 * The stack should represent the 32-bit calling convention.
178 */
179SYM_FUNC_START_LOCAL(efi_enter32)
180	/* Load firmware selector into data and stack segment registers */
181	movl	%edx, %ds
182	movl	%edx, %es
183	movl	%edx, %fs
184	movl	%edx, %gs
185	movl	%edx, %ss
186
187	/* Reload pgtables */
188	movl	%cr3, %eax
189	movl	%eax, %cr3
190
191	/* Disable paging */
192	movl	%cr0, %eax
193	btrl	$X86_CR0_PG_BIT, %eax
194	movl	%eax, %cr0
195
196	/* Disable long mode via EFER */
197	movl	$MSR_EFER, %ecx
198	rdmsr
199	btrl	$_EFER_LME, %eax
200	wrmsr
201
202	call	*%edi
203
204	/* We must preserve return value */
205	movl	%eax, %edi
206
207	/*
208	 * Some firmware will return with interrupts enabled. Be sure to
209	 * disable them before we switch GDTs and IDTs.
210	 */
211	cli
212
213	lidtl	16(%ebx)
214	lgdtl	(%ebx)
215
216	movl	%cr4, %eax
217	btsl	$(X86_CR4_PAE_BIT), %eax
218	movl	%eax, %cr4
219
220	movl	%cr3, %eax
221	movl	%eax, %cr3
222
223	movl	$MSR_EFER, %ecx
224	rdmsr
225	btsl	$_EFER_LME, %eax
226	wrmsr
227
228	xorl	%eax, %eax
229	lldt	%ax
230
231	pushl	$__KERNEL_CS
232	pushl	%ebp
233
234	/* Enable paging */
235	movl	%cr0, %eax
236	btsl	$X86_CR0_PG_BIT, %eax
237	movl	%eax, %cr0
238	lret
239SYM_FUNC_END(efi_enter32)
240
241/*
242 * This is the common EFI stub entry point for mixed mode.
243 *
244 * Arguments:	%ecx	image handle
245 * 		%edx	EFI system table pointer
246 *
247 * Since this is the point of no return for ordinary execution, no registers
248 * are considered live except for the function parameters. [Note that the EFI
249 * stub may still exit and return to the firmware using the Exit() EFI boot
250 * service.]
251 */
252SYM_FUNC_START_LOCAL(efi32_entry)
253	call	1f
2541:	pop	%ebx
255
256	/* Save firmware GDTR and code/data selectors */
257	sgdtl	(efi32_boot_gdt - 1b)(%ebx)
258	movw	%cs, (efi32_boot_cs - 1b)(%ebx)
259	movw	%ds, (efi32_boot_ds - 1b)(%ebx)
260
261	/* Store firmware IDT descriptor */
262	sidtl	(efi32_boot_idt - 1b)(%ebx)
263
264	/* Store firmware stack pointer */
265	movl	%esp, (efi32_boot_sp - 1b)(%ebx)
266
267	/* Store boot arguments */
268	leal	(efi32_boot_args - 1b)(%ebx), %ebx
269	movl	%ecx, 0(%ebx)
270	movl	%edx, 4(%ebx)
271	movb	$0x0, 12(%ebx)          // efi_is64
272
273	/*
274	 * Allocate some memory for a temporary struct boot_params, which only
275	 * needs the minimal pieces that startup_32() relies on.
276	 */
277	subl	$PARAM_SIZE, %esp
278	movl	%esp, %esi
279	movl	$PAGE_SIZE, BP_kernel_alignment(%esi)
280	movl	$_end - 1b, BP_init_size(%esi)
281	subl	$startup_32 - 1b, BP_init_size(%esi)
282
283	/* Disable paging */
284	movl	%cr0, %eax
285	btrl	$X86_CR0_PG_BIT, %eax
286	movl	%eax, %cr0
287
288	jmp	startup_32
289SYM_FUNC_END(efi32_entry)
290
291/*
292 * efi_status_t efi32_pe_entry(efi_handle_t image_handle,
293 *			       efi_system_table_32_t *sys_table)
294 */
295SYM_FUNC_START(efi32_pe_entry)
296	pushl	%ebp
297	movl	%esp, %ebp
298	pushl	%ebx				// save callee-save registers
299	pushl	%edi
300
301	call	verify_cpu			// check for long mode support
302	testl	%eax, %eax
303	movl	$0x80000003, %eax		// EFI_UNSUPPORTED
304	jnz	2f
305
306	movl	8(%ebp), %ecx			// image_handle
307	movl	12(%ebp), %edx			// sys_table
308	jmp	efi32_entry			// pass %ecx, %edx
309						// no other registers remain live
310
3112:	popl	%edi				// restore callee-save registers
312	popl	%ebx
313	leave
314	RET
315SYM_FUNC_END(efi32_pe_entry)
316
317#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
318	.org	efi32_stub_entry + 0x200
319	.code64
320SYM_FUNC_START_NOALIGN(efi64_stub_entry)
321	jmp	efi_handover_entry
322SYM_FUNC_END(efi64_stub_entry)
323#endif
324
325	.data
326	.balign	8
327SYM_DATA_START_LOCAL(efi32_boot_gdt)
328	.word	0
329	.quad	0
330SYM_DATA_END(efi32_boot_gdt)
331
332SYM_DATA_START_LOCAL(efi32_boot_idt)
333	.word	0
334	.quad	0
335SYM_DATA_END(efi32_boot_idt)
336
337SYM_DATA_LOCAL(efi32_boot_cs, .word 0)
338SYM_DATA_LOCAL(efi32_boot_ds, .word 0)
339SYM_DATA_LOCAL(efi32_boot_sp, .long 0)
340SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0)
341SYM_DATA(efi_is64, .byte 1)
342