1/* 2 * linux/arch/x86_64/kernel/head.S -- start in 32bit and switch to 64bit 3 * 4 * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE 5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz> 6 * Copyright (C) 2000 Karsten Keil <kkeil@suse.de> 7 * Copyright (C) 2001,2002 Andi Kleen <ak@suse.de> 8 * Copyright (C) 2005 Eric Biederman <ebiederm@xmission.com> 9 */ 10 11 12#include <linux/linkage.h> 13#include <linux/threads.h> 14#include <linux/init.h> 15#include <asm/segment.h> 16#include <asm/pgtable.h> 17#include <asm/page.h> 18#include <asm/msr.h> 19#include <asm/cache.h> 20#include <asm/processor-flags.h> 21#include <asm/percpu.h> 22 23#ifdef CONFIG_PARAVIRT 24#include <asm/asm-offsets.h> 25#include <asm/paravirt.h> 26#else 27#define GET_CR2_INTO_RCX movq %cr2, %rcx 28#endif 29 30/* we are not able to switch in one step to the final KERNEL ADDRESS SPACE 31 * because we need identity-mapped pages. 32 * 33 */ 34 35#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) 36 37L4_PAGE_OFFSET = pgd_index(__PAGE_OFFSET) 38L3_PAGE_OFFSET = pud_index(__PAGE_OFFSET) 39L4_START_KERNEL = pgd_index(__START_KERNEL_map) 40L3_START_KERNEL = pud_index(__START_KERNEL_map) 41 42 .text 43 __HEAD 44 .code64 45 .globl startup_64 46startup_64: 47 48 /* 49 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 1, 50 * and someone has loaded an identity mapped page table 51 * for us. These identity mapped page tables map all of the 52 * kernel pages and possibly all of memory. 53 * 54 * %esi holds a physical pointer to real_mode_data. 55 * 56 * We come here either directly from a 64bit bootloader, or from 57 * arch/x86_64/boot/compressed/head.S. 58 * 59 * We only come here initially at boot nothing else comes here. 60 * 61 * Since we may be loaded at an address different from what we were 62 * compiled to run at we first fixup the physical addresses in our page 63 * tables and then reload them. 64 */ 65 66 /* Compute the delta between the address I am compiled to run at and the 67 * address I am actually running at. 68 */ 69 leaq _text(%rip), %rbp 70 subq $_text - __START_KERNEL_map, %rbp 71 72 /* Is the address not 2M aligned? */ 73 movq %rbp, %rax 74 andl $~PMD_PAGE_MASK, %eax 75 testl %eax, %eax 76 jnz bad_address 77 78 /* Is the address too large? */ 79 leaq _text(%rip), %rdx 80 movq $PGDIR_SIZE, %rax 81 cmpq %rax, %rdx 82 jae bad_address 83 84 /* Fixup the physical addresses in the page table 85 */ 86 addq %rbp, init_level4_pgt + 0(%rip) 87 addq %rbp, init_level4_pgt + (L4_PAGE_OFFSET*8)(%rip) 88 addq %rbp, init_level4_pgt + (L4_START_KERNEL*8)(%rip) 89 90 addq %rbp, level3_ident_pgt + 0(%rip) 91 92 addq %rbp, level3_kernel_pgt + (510*8)(%rip) 93 addq %rbp, level3_kernel_pgt + (511*8)(%rip) 94 95 addq %rbp, level2_fixmap_pgt + (506*8)(%rip) 96 97 /* Add an Identity mapping if I am above 1G */ 98 leaq _text(%rip), %rdi 99 andq $PMD_PAGE_MASK, %rdi 100 101 movq %rdi, %rax 102 shrq $PUD_SHIFT, %rax 103 andq $(PTRS_PER_PUD - 1), %rax 104 jz ident_complete 105 106 leaq (level2_spare_pgt - __START_KERNEL_map + _KERNPG_TABLE)(%rbp), %rdx 107 leaq level3_ident_pgt(%rip), %rbx 108 movq %rdx, 0(%rbx, %rax, 8) 109 110 movq %rdi, %rax 111 shrq $PMD_SHIFT, %rax 112 andq $(PTRS_PER_PMD - 1), %rax 113 leaq __PAGE_KERNEL_IDENT_LARGE_EXEC(%rdi), %rdx 114 leaq level2_spare_pgt(%rip), %rbx 115 movq %rdx, 0(%rbx, %rax, 8) 116ident_complete: 117 118 /* 119 * Fixup the kernel text+data virtual addresses. Note that 120 * we might write invalid pmds, when the kernel is relocated 121 * cleanup_highmap() fixes this up along with the mappings 122 * beyond _end. 123 */ 124 125 leaq level2_kernel_pgt(%rip), %rdi 126 leaq 4096(%rdi), %r8 127 /* See if it is a valid page table entry */ 1281: testq $1, 0(%rdi) 129 jz 2f 130 addq %rbp, 0(%rdi) 131 /* Go to the next page */ 1322: addq $8, %rdi 133 cmp %r8, %rdi 134 jne 1b 135 136 /* Fixup phys_base */ 137 addq %rbp, phys_base(%rip) 138 139 /* Due to ENTRY(), sometimes the empty space gets filled with 140 * zeros. Better take a jmp than relying on empty space being 141 * filled with 0x90 (nop) 142 */ 143 jmp secondary_startup_64 144ENTRY(secondary_startup_64) 145 /* 146 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 1, 147 * and someone has loaded a mapped page table. 148 * 149 * %esi holds a physical pointer to real_mode_data. 150 * 151 * We come here either from startup_64 (using physical addresses) 152 * or from trampoline.S (using virtual addresses). 153 * 154 * Using virtual addresses from trampoline.S removes the need 155 * to have any identity mapped pages in the kernel page table 156 * after the boot processor executes this code. 157 */ 158 159 /* Enable PAE mode and PGE */ 160 movl $(X86_CR4_PAE | X86_CR4_PGE), %eax 161 movq %rax, %cr4 162 163 /* Setup early boot stage 4 level pagetables. */ 164 movq $(init_level4_pgt - __START_KERNEL_map), %rax 165 addq phys_base(%rip), %rax 166 movq %rax, %cr3 167 168 /* Ensure I am executing from virtual addresses */ 169 movq $1f, %rax 170 jmp *%rax 1711: 172 173 /* Check if nx is implemented */ 174 movl $0x80000001, %eax 175 cpuid 176 movl %edx,%edi 177 178 /* Setup EFER (Extended Feature Enable Register) */ 179 movl $MSR_EFER, %ecx 180 rdmsr 181 btsl $_EFER_SCE, %eax /* Enable System Call */ 182 btl $20,%edi /* No Execute supported? */ 183 jnc 1f 184 btsl $_EFER_NX, %eax 1851: wrmsr /* Make changes effective */ 186 187 /* Setup cr0 */ 188#define CR0_STATE (X86_CR0_PE | X86_CR0_MP | X86_CR0_ET | \ 189 X86_CR0_NE | X86_CR0_WP | X86_CR0_AM | \ 190 X86_CR0_PG) 191 movl $CR0_STATE, %eax 192 /* Make changes effective */ 193 movq %rax, %cr0 194 195 /* Setup a boot time stack */ 196 movq stack_start(%rip),%rsp 197 198 /* zero EFLAGS after setting rsp */ 199 pushq $0 200 popfq 201 202 /* 203 * We must switch to a new descriptor in kernel space for the GDT 204 * because soon the kernel won't have access anymore to the userspace 205 * addresses where we're currently running on. We have to do that here 206 * because in 32bit we couldn't load a 64bit linear address. 207 */ 208 lgdt early_gdt_descr(%rip) 209 210 /* set up data segments */ 211 xorl %eax,%eax 212 movl %eax,%ds 213 movl %eax,%ss 214 movl %eax,%es 215 216 /* 217 * We don't really need to load %fs or %gs, but load them anyway 218 * to kill any stale realmode selectors. This allows execution 219 * under VT hardware. 220 */ 221 movl %eax,%fs 222 movl %eax,%gs 223 224 /* Set up %gs. 225 * 226 * The base of %gs always points to the bottom of the irqstack 227 * union. If the stack protector canary is enabled, it is 228 * located at %gs:40. Note that, on SMP, the boot cpu uses 229 * init data section till per cpu areas are set up. 230 */ 231 movl $MSR_GS_BASE,%ecx 232 movl initial_gs(%rip),%eax 233 movl initial_gs+4(%rip),%edx 234 wrmsr 235 236 /* esi is pointer to real mode structure with interesting info. 237 pass it to C */ 238 movl %esi, %edi 239 240 /* Finally jump to run C code and to be on real kernel address 241 * Since we are running on identity-mapped space we have to jump 242 * to the full 64bit address, this is only possible as indirect 243 * jump. In addition we need to ensure %cs is set so we make this 244 * a far return. 245 */ 246 movq initial_code(%rip),%rax 247 pushq $0 # fake return address to stop unwinder 248 pushq $__KERNEL_CS # set correct cs 249 pushq %rax # target address in negative space 250 lretq 251 252 /* SMP bootup changes these two */ 253 __REFDATA 254 .align 8 255 ENTRY(initial_code) 256 .quad x86_64_start_kernel 257 ENTRY(initial_gs) 258 .quad INIT_PER_CPU_VAR(irq_stack_union) 259 260 ENTRY(stack_start) 261 .quad init_thread_union+THREAD_SIZE-8 262 .word 0 263 __FINITDATA 264 265bad_address: 266 jmp bad_address 267 268 .section ".init.text","ax" 269#ifdef CONFIG_EARLY_PRINTK 270 .globl early_idt_handlers 271early_idt_handlers: 272 i = 0 273 .rept NUM_EXCEPTION_VECTORS 274 movl $i, %esi 275 jmp early_idt_handler 276 i = i + 1 277 .endr 278#endif 279 280ENTRY(early_idt_handler) 281#ifdef CONFIG_EARLY_PRINTK 282 cmpl $2,early_recursion_flag(%rip) 283 jz 1f 284 incl early_recursion_flag(%rip) 285 GET_CR2_INTO_RCX 286 movq %rcx,%r9 287 xorl %r8d,%r8d # zero for error code 288 movl %esi,%ecx # get vector number 289 # Test %ecx against mask of vectors that push error code. 290 cmpl $31,%ecx 291 ja 0f 292 movl $1,%eax 293 salq %cl,%rax 294 testl $0x27d00,%eax 295 je 0f 296 popq %r8 # get error code 2970: movq 0(%rsp),%rcx # get ip 298 movq 8(%rsp),%rdx # get cs 299 xorl %eax,%eax 300 leaq early_idt_msg(%rip),%rdi 301 call early_printk 302 cmpl $2,early_recursion_flag(%rip) 303 jz 1f 304 call dump_stack 305#ifdef CONFIG_KALLSYMS 306 leaq early_idt_ripmsg(%rip),%rdi 307 movq 0(%rsp),%rsi # get rip again 308 call __print_symbol 309#endif 310#endif /* EARLY_PRINTK */ 3111: hlt 312 jmp 1b 313 314#ifdef CONFIG_EARLY_PRINTK 315early_recursion_flag: 316 .long 0 317 318early_idt_msg: 319 .asciz "PANIC: early exception %02lx rip %lx:%lx error %lx cr2 %lx\n" 320early_idt_ripmsg: 321 .asciz "RIP %s\n" 322#endif /* CONFIG_EARLY_PRINTK */ 323 .previous 324 325#define NEXT_PAGE(name) \ 326 .balign PAGE_SIZE; \ 327ENTRY(name) 328 329/* Automate the creation of 1 to 1 mapping pmd entries */ 330#define PMDS(START, PERM, COUNT) \ 331 i = 0 ; \ 332 .rept (COUNT) ; \ 333 .quad (START) + (i << PMD_SHIFT) + (PERM) ; \ 334 i = i + 1 ; \ 335 .endr 336 337 .data 338 /* 339 * This default setting generates an ident mapping at address 0x100000 340 * and a mapping for the kernel that precisely maps virtual address 341 * 0xffffffff80000000 to physical address 0x000000. (always using 342 * 2Mbyte large pages provided by PAE mode) 343 */ 344NEXT_PAGE(init_level4_pgt) 345 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE 346 .org init_level4_pgt + L4_PAGE_OFFSET*8, 0 347 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE 348 .org init_level4_pgt + L4_START_KERNEL*8, 0 349 /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */ 350 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE 351 352NEXT_PAGE(level3_ident_pgt) 353 .quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE 354 .fill 511,8,0 355 356NEXT_PAGE(level3_kernel_pgt) 357 .fill L3_START_KERNEL,8,0 358 /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */ 359 .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE 360 .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE 361 362NEXT_PAGE(level2_fixmap_pgt) 363 .fill 506,8,0 364 .quad level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE 365 /* 8MB reserved for vsyscalls + a 2MB hole = 4 + 1 entries */ 366 .fill 5,8,0 367 368NEXT_PAGE(level1_fixmap_pgt) 369 .fill 512,8,0 370 371NEXT_PAGE(level2_ident_pgt) 372 /* Since I easily can, map the first 1G. 373 * Don't set NX because code runs from these pages. 374 */ 375 PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD) 376 377NEXT_PAGE(level2_kernel_pgt) 378 /* 379 * 512 MB kernel mapping. We spend a full page on this pagetable 380 * anyway. 381 * 382 * The kernel code+data+bss must not be bigger than that. 383 * 384 * (NOTE: at +512MB starts the module area, see MODULES_VADDR. 385 * If you want to increase this then increase MODULES_VADDR 386 * too.) 387 */ 388 PMDS(0, __PAGE_KERNEL_LARGE_EXEC, 389 KERNEL_IMAGE_SIZE/PMD_SIZE) 390 391NEXT_PAGE(level2_spare_pgt) 392 .fill 512, 8, 0 393 394#undef PMDS 395#undef NEXT_PAGE 396 397 .data 398 .align 16 399 .globl early_gdt_descr 400early_gdt_descr: 401 .word GDT_ENTRIES*8-1 402early_gdt_descr_base: 403 .quad INIT_PER_CPU_VAR(gdt_page) 404 405ENTRY(phys_base) 406 /* This must match the first entry in level2_kernel_pgt */ 407 .quad 0x0000000000000000 408 409#include "../../x86/xen/xen-head.S" 410 411 .section .bss, "aw", @nobits 412 .align L1_CACHE_BYTES 413ENTRY(idt_table) 414 .skip IDT_ENTRIES * 16 415 416 .align L1_CACHE_BYTES 417ENTRY(nmi_idt_table) 418 .skip IDT_ENTRIES * 16 419 420 __PAGE_ALIGNED_BSS 421 .align PAGE_SIZE 422ENTRY(empty_zero_page) 423 .skip PAGE_SIZE 424