xref: /openbmc/linux/arch/x86/boot/compressed/head_64.S (revision 4da722ca)
1/*
2 *  linux/boot/head.S
3 *
4 *  Copyright (C) 1991, 1992, 1993  Linus Torvalds
5 */
6
7/*
8 *  head.S contains the 32-bit startup code.
9 *
10 * NOTE!!! Startup happens at absolute address 0x00001000, which is also where
11 * the page directory will exist. The startup code will be overwritten by
12 * the page directory. [According to comments etc elsewhere on a compressed
13 * kernel it will end up at 0x1000 + 1Mb I hope so as I assume this. - AC]
14 *
15 * Page 0 is deliberately kept safe, since System Management Mode code in
16 * laptops may need to access the BIOS data stored there.  This is also
17 * useful for future device drivers that either access the BIOS via VM86
18 * mode.
19 */
20
21/*
22 * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996
23 */
24	.code32
25	.text
26
27#include <linux/init.h>
28#include <linux/linkage.h>
29#include <asm/segment.h>
30#include <asm/boot.h>
31#include <asm/msr.h>
32#include <asm/processor-flags.h>
33#include <asm/asm-offsets.h>
34#include <asm/bootparam.h>
35
36/*
37 * Locally defined symbols should be marked hidden:
38 */
39	.hidden _bss
40	.hidden _ebss
41	.hidden _got
42	.hidden _egot
43
44	__HEAD
45	.code32
46ENTRY(startup_32)
47	/*
48	 * 32bit entry is 0 and it is ABI so immutable!
49	 * If we come here directly from a bootloader,
50	 * kernel(text+data+bss+brk) ramdisk, zero_page, command line
51	 * all need to be under the 4G limit.
52	 */
53	cld
54	/*
55	 * Test KEEP_SEGMENTS flag to see if the bootloader is asking
56	 * us to not reload segments
57	 */
58	testb $KEEP_SEGMENTS, BP_loadflags(%esi)
59	jnz 1f
60
61	cli
62	movl	$(__BOOT_DS), %eax
63	movl	%eax, %ds
64	movl	%eax, %es
65	movl	%eax, %ss
661:
67
68/*
69 * Calculate the delta between where we were compiled to run
70 * at and where we were actually loaded at.  This can only be done
71 * with a short local call on x86.  Nothing  else will tell us what
72 * address we are running at.  The reserved chunk of the real-mode
73 * data at 0x1e4 (defined as a scratch field) are used as the stack
74 * for this calculation. Only 4 bytes are needed.
75 */
76	leal	(BP_scratch+4)(%esi), %esp
77	call	1f
781:	popl	%ebp
79	subl	$1b, %ebp
80
81/* setup a stack and make sure cpu supports long mode. */
82	movl	$boot_stack_end, %eax
83	addl	%ebp, %eax
84	movl	%eax, %esp
85
86	call	verify_cpu
87	testl	%eax, %eax
88	jnz	no_longmode
89
90/*
91 * Compute the delta between where we were compiled to run at
92 * and where the code will actually run at.
93 *
94 * %ebp contains the address we are loaded at by the boot loader and %ebx
95 * contains the address where we should move the kernel image temporarily
96 * for safe in-place decompression.
97 */
98
99#ifdef CONFIG_RELOCATABLE
100	movl	%ebp, %ebx
101	movl	BP_kernel_alignment(%esi), %eax
102	decl	%eax
103	addl	%eax, %ebx
104	notl	%eax
105	andl	%eax, %ebx
106	cmpl	$LOAD_PHYSICAL_ADDR, %ebx
107	jge	1f
108#endif
109	movl	$LOAD_PHYSICAL_ADDR, %ebx
1101:
111
112	/* Target address to relocate to for decompression */
113	movl	BP_init_size(%esi), %eax
114	subl	$_end, %eax
115	addl	%eax, %ebx
116
117/*
118 * Prepare for entering 64 bit mode
119 */
120
121	/* Load new GDT with the 64bit segments using 32bit descriptor */
122	addl	%ebp, gdt+2(%ebp)
123	lgdt	gdt(%ebp)
124
125	/* Enable PAE mode */
126	movl	%cr4, %eax
127	orl	$X86_CR4_PAE, %eax
128	movl	%eax, %cr4
129
130 /*
131  * Build early 4G boot pagetable
132  */
133	/* Initialize Page tables to 0 */
134	leal	pgtable(%ebx), %edi
135	xorl	%eax, %eax
136	movl	$(BOOT_INIT_PGT_SIZE/4), %ecx
137	rep	stosl
138
139	/* Build Level 4 */
140	leal	pgtable + 0(%ebx), %edi
141	leal	0x1007 (%edi), %eax
142	movl	%eax, 0(%edi)
143
144	/* Build Level 3 */
145	leal	pgtable + 0x1000(%ebx), %edi
146	leal	0x1007(%edi), %eax
147	movl	$4, %ecx
1481:	movl	%eax, 0x00(%edi)
149	addl	$0x00001000, %eax
150	addl	$8, %edi
151	decl	%ecx
152	jnz	1b
153
154	/* Build Level 2 */
155	leal	pgtable + 0x2000(%ebx), %edi
156	movl	$0x00000183, %eax
157	movl	$2048, %ecx
1581:	movl	%eax, 0(%edi)
159	addl	$0x00200000, %eax
160	addl	$8, %edi
161	decl	%ecx
162	jnz	1b
163
164	/* Enable the boot page tables */
165	leal	pgtable(%ebx), %eax
166	movl	%eax, %cr3
167
168	/* Enable Long mode in EFER (Extended Feature Enable Register) */
169	movl	$MSR_EFER, %ecx
170	rdmsr
171	btsl	$_EFER_LME, %eax
172	wrmsr
173
174	/* After gdt is loaded */
175	xorl	%eax, %eax
176	lldt	%ax
177	movl    $__BOOT_TSS, %eax
178	ltr	%ax
179
180	/*
181	 * Setup for the jump to 64bit mode
182	 *
183	 * When the jump is performend we will be in long mode but
184	 * in 32bit compatibility mode with EFER.LME = 1, CS.L = 0, CS.D = 1
185	 * (and in turn EFER.LMA = 1).	To jump into 64bit mode we use
186	 * the new gdt/idt that has __KERNEL_CS with CS.L = 1.
187	 * We place all of the values on our mini stack so lret can
188	 * used to perform that far jump.
189	 */
190	pushl	$__KERNEL_CS
191	leal	startup_64(%ebp), %eax
192#ifdef CONFIG_EFI_MIXED
193	movl	efi32_config(%ebp), %ebx
194	cmp	$0, %ebx
195	jz	1f
196	leal	handover_entry(%ebp), %eax
1971:
198#endif
199	pushl	%eax
200
201	/* Enter paged protected Mode, activating Long Mode */
202	movl	$(X86_CR0_PG | X86_CR0_PE), %eax /* Enable Paging and Protected mode */
203	movl	%eax, %cr0
204
205	/* Jump from 32bit compatibility mode into 64bit mode. */
206	lret
207ENDPROC(startup_32)
208
209#ifdef CONFIG_EFI_MIXED
210	.org 0x190
211ENTRY(efi32_stub_entry)
212	add	$0x4, %esp		/* Discard return address */
213	popl	%ecx
214	popl	%edx
215	popl	%esi
216
217	leal	(BP_scratch+4)(%esi), %esp
218	call	1f
2191:	pop	%ebp
220	subl	$1b, %ebp
221
222	movl	%ecx, efi32_config(%ebp)
223	movl	%edx, efi32_config+8(%ebp)
224	sgdtl	efi32_boot_gdt(%ebp)
225
226	leal	efi32_config(%ebp), %eax
227	movl	%eax, efi_config(%ebp)
228
229	jmp	startup_32
230ENDPROC(efi32_stub_entry)
231#endif
232
233	.code64
234	.org 0x200
235ENTRY(startup_64)
236	/*
237	 * 64bit entry is 0x200 and it is ABI so immutable!
238	 * We come here either from startup_32 or directly from a
239	 * 64bit bootloader.
240	 * If we come here from a bootloader, kernel(text+data+bss+brk),
241	 * ramdisk, zero_page, command line could be above 4G.
242	 * We depend on an identity mapped page table being provided
243	 * that maps our entire kernel(text+data+bss+brk), zero page
244	 * and command line.
245	 */
246#ifdef CONFIG_EFI_STUB
247	/*
248	 * The entry point for the PE/COFF executable is efi_pe_entry, so
249	 * only legacy boot loaders will execute this jmp.
250	 */
251	jmp	preferred_addr
252
253ENTRY(efi_pe_entry)
254	movq	%rcx, efi64_config(%rip)	/* Handle */
255	movq	%rdx, efi64_config+8(%rip) /* EFI System table pointer */
256
257	leaq	efi64_config(%rip), %rax
258	movq	%rax, efi_config(%rip)
259
260	call	1f
2611:	popq	%rbp
262	subq	$1b, %rbp
263
264	/*
265	 * Relocate efi_config->call().
266	 */
267	addq	%rbp, efi64_config+40(%rip)
268
269	movq	%rax, %rdi
270	call	make_boot_params
271	cmpq	$0,%rax
272	je	fail
273	mov	%rax, %rsi
274	leaq	startup_32(%rip), %rax
275	movl	%eax, BP_code32_start(%rsi)
276	jmp	2f		/* Skip the relocation */
277
278handover_entry:
279	call	1f
2801:	popq	%rbp
281	subq	$1b, %rbp
282
283	/*
284	 * Relocate efi_config->call().
285	 */
286	movq	efi_config(%rip), %rax
287	addq	%rbp, 40(%rax)
2882:
289	movq	efi_config(%rip), %rdi
290	call	efi_main
291	movq	%rax,%rsi
292	cmpq	$0,%rax
293	jne	2f
294fail:
295	/* EFI init failed, so hang. */
296	hlt
297	jmp	fail
2982:
299	movl	BP_code32_start(%esi), %eax
300	leaq	preferred_addr(%rax), %rax
301	jmp	*%rax
302
303preferred_addr:
304#endif
305
306	/* Setup data segments. */
307	xorl	%eax, %eax
308	movl	%eax, %ds
309	movl	%eax, %es
310	movl	%eax, %ss
311	movl	%eax, %fs
312	movl	%eax, %gs
313
314	/*
315	 * Compute the decompressed kernel start address.  It is where
316	 * we were loaded at aligned to a 2M boundary. %rbp contains the
317	 * decompressed kernel start address.
318	 *
319	 * If it is a relocatable kernel then decompress and run the kernel
320	 * from load address aligned to 2MB addr, otherwise decompress and
321	 * run the kernel from LOAD_PHYSICAL_ADDR
322	 *
323	 * We cannot rely on the calculation done in 32-bit mode, since we
324	 * may have been invoked via the 64-bit entry point.
325	 */
326
327	/* Start with the delta to where the kernel will run at. */
328#ifdef CONFIG_RELOCATABLE
329	leaq	startup_32(%rip) /* - $startup_32 */, %rbp
330	movl	BP_kernel_alignment(%rsi), %eax
331	decl	%eax
332	addq	%rax, %rbp
333	notq	%rax
334	andq	%rax, %rbp
335	cmpq	$LOAD_PHYSICAL_ADDR, %rbp
336	jge	1f
337#endif
338	movq	$LOAD_PHYSICAL_ADDR, %rbp
3391:
340
341	/* Target address to relocate to for decompression */
342	movl	BP_init_size(%rsi), %ebx
343	subl	$_end, %ebx
344	addq	%rbp, %rbx
345
346	/* Set up the stack */
347	leaq	boot_stack_end(%rbx), %rsp
348
349#ifdef CONFIG_X86_5LEVEL
350	/* Check if 5-level paging has already enabled */
351	movq	%cr4, %rax
352	testl	$X86_CR4_LA57, %eax
353	jnz	lvl5
354
355	/*
356	 * At this point we are in long mode with 4-level paging enabled,
357	 * but we want to enable 5-level paging.
358	 *
359	 * The problem is that we cannot do it directly. Setting LA57 in
360	 * long mode would trigger #GP. So we need to switch off long mode
361	 * first.
362	 *
363	 * NOTE: This is not going to work if bootloader put us above 4G
364	 * limit.
365	 *
366	 * The first step is go into compatibility mode.
367	 */
368
369	/* Clear additional page table */
370	leaq	lvl5_pgtable(%rbx), %rdi
371	xorq	%rax, %rax
372	movq	$(PAGE_SIZE/8), %rcx
373	rep	stosq
374
375	/*
376	 * Setup current CR3 as the first and only entry in a new top level
377	 * page table.
378	 */
379	movq	%cr3, %rdi
380	leaq	0x7 (%rdi), %rax
381	movq	%rax, lvl5_pgtable(%rbx)
382
383	/* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */
384	pushq	$__KERNEL32_CS
385	leaq	compatible_mode(%rip), %rax
386	pushq	%rax
387	lretq
388lvl5:
389#endif
390
391	/* Zero EFLAGS */
392	pushq	$0
393	popfq
394
395/*
396 * Copy the compressed kernel to the end of our buffer
397 * where decompression in place becomes safe.
398 */
399	pushq	%rsi
400	leaq	(_bss-8)(%rip), %rsi
401	leaq	(_bss-8)(%rbx), %rdi
402	movq	$_bss /* - $startup_32 */, %rcx
403	shrq	$3, %rcx
404	std
405	rep	movsq
406	cld
407	popq	%rsi
408
409/*
410 * Jump to the relocated address.
411 */
412	leaq	relocated(%rbx), %rax
413	jmp	*%rax
414
415#ifdef CONFIG_EFI_STUB
416	.org 0x390
417ENTRY(efi64_stub_entry)
418	movq	%rdi, efi64_config(%rip)	/* Handle */
419	movq	%rsi, efi64_config+8(%rip) /* EFI System table pointer */
420
421	leaq	efi64_config(%rip), %rax
422	movq	%rax, efi_config(%rip)
423
424	movq	%rdx, %rsi
425	jmp	handover_entry
426ENDPROC(efi64_stub_entry)
427#endif
428
429	.text
430relocated:
431
432/*
433 * Clear BSS (stack is currently empty)
434 */
435	xorl	%eax, %eax
436	leaq    _bss(%rip), %rdi
437	leaq    _ebss(%rip), %rcx
438	subq	%rdi, %rcx
439	shrq	$3, %rcx
440	rep	stosq
441
442/*
443 * Adjust our own GOT
444 */
445	leaq	_got(%rip), %rdx
446	leaq	_egot(%rip), %rcx
4471:
448	cmpq	%rcx, %rdx
449	jae	2f
450	addq	%rbx, (%rdx)
451	addq	$8, %rdx
452	jmp	1b
4532:
454
455/*
456 * Do the extraction, and jump to the new kernel..
457 */
458	pushq	%rsi			/* Save the real mode argument */
459	movq	%rsi, %rdi		/* real mode address */
460	leaq	boot_heap(%rip), %rsi	/* malloc area for uncompression */
461	leaq	input_data(%rip), %rdx  /* input_data */
462	movl	$z_input_len, %ecx	/* input_len */
463	movq	%rbp, %r8		/* output target address */
464	movq	$z_output_len, %r9	/* decompressed length, end of relocs */
465	call	extract_kernel		/* returns kernel location in %rax */
466	popq	%rsi
467
468/*
469 * Jump to the decompressed kernel.
470 */
471	jmp	*%rax
472
473	.code32
474#ifdef CONFIG_X86_5LEVEL
475compatible_mode:
476	/* Setup data and stack segments */
477	movl	$__KERNEL_DS, %eax
478	movl	%eax, %ds
479	movl	%eax, %ss
480
481	/* Disable paging */
482	movl	%cr0, %eax
483	btrl	$X86_CR0_PG_BIT, %eax
484	movl	%eax, %cr0
485
486	/* Point CR3 to 5-level paging */
487	leal	lvl5_pgtable(%ebx), %eax
488	movl	%eax, %cr3
489
490	/* Enable PAE and LA57 mode */
491	movl	%cr4, %eax
492	orl	$(X86_CR4_PAE | X86_CR4_LA57), %eax
493	movl	%eax, %cr4
494
495	/* Calculate address we are running at */
496	call	1f
4971:	popl	%edi
498	subl	$1b, %edi
499
500	/* Prepare stack for far return to Long Mode */
501	pushl	$__KERNEL_CS
502	leal	lvl5(%edi), %eax
503	push	%eax
504
505	/* Enable paging back */
506	movl	$(X86_CR0_PG | X86_CR0_PE), %eax
507	movl	%eax, %cr0
508
509	lret
510#endif
511
512no_longmode:
513	/* This isn't an x86-64 CPU so hang */
5141:
515	hlt
516	jmp     1b
517
518#include "../../kernel/verify_cpu.S"
519
520	.data
521gdt:
522	.word	gdt_end - gdt
523	.long	gdt
524	.word	0
525	.quad	0x00cf9a000000ffff	/* __KERNEL32_CS */
526	.quad	0x00af9a000000ffff	/* __KERNEL_CS */
527	.quad	0x00cf92000000ffff	/* __KERNEL_DS */
528	.quad	0x0080890000000000	/* TS descriptor */
529	.quad   0x0000000000000000	/* TS continued */
530gdt_end:
531
532#ifdef CONFIG_EFI_STUB
533efi_config:
534	.quad	0
535
536#ifdef CONFIG_EFI_MIXED
537	.global efi32_config
538efi32_config:
539	.fill	5,8,0
540	.quad	efi64_thunk
541	.byte	0
542#endif
543
544	.global efi64_config
545efi64_config:
546	.fill	5,8,0
547	.quad	efi_call
548	.byte	1
549#endif /* CONFIG_EFI_STUB */
550
551/*
552 * Stack and heap for uncompression
553 */
554	.bss
555	.balign 4
556boot_heap:
557	.fill BOOT_HEAP_SIZE, 1, 0
558boot_stack:
559	.fill BOOT_STACK_SIZE, 1, 0
560boot_stack_end:
561
562/*
563 * Space for page tables (not in .bss so not zeroed)
564 */
565	.section ".pgtable","a",@nobits
566	.balign 4096
567pgtable:
568	.fill BOOT_PGT_SIZE, 1, 0
569#ifdef CONFIG_X86_5LEVEL
570lvl5_pgtable:
571	.fill PAGE_SIZE, 1, 0
572#endif
573