19a163ed8SThomas Gleixner/* 29a163ed8SThomas Gleixner * 39a163ed8SThomas Gleixner * Copyright (C) 1991, 1992 Linus Torvalds 49a163ed8SThomas Gleixner * 59a163ed8SThomas Gleixner * Enhanced CPU detection and feature setting code by Mike Jagdis 69a163ed8SThomas Gleixner * and Martin Mares, November 1997. 79a163ed8SThomas Gleixner */ 89a163ed8SThomas Gleixner 99a163ed8SThomas Gleixner.text 109a163ed8SThomas Gleixner#include <linux/threads.h> 118b2f7fffSSam Ravnborg#include <linux/init.h> 129a163ed8SThomas Gleixner#include <linux/linkage.h> 139a163ed8SThomas Gleixner#include <asm/segment.h> 140341c14dSJeremy Fitzhardinge#include <asm/page_types.h> 150341c14dSJeremy Fitzhardinge#include <asm/pgtable_types.h> 169a163ed8SThomas Gleixner#include <asm/cache.h> 179a163ed8SThomas Gleixner#include <asm/thread_info.h> 189a163ed8SThomas Gleixner#include <asm/asm-offsets.h> 199a163ed8SThomas Gleixner#include <asm/setup.h> 20551889a6SIan Campbell#include <asm/processor-flags.h> 218a50e513SH. Peter Anvin#include <asm/msr-index.h> 228a50e513SH. Peter Anvin#include <asm/cpufeature.h> 2360a5317fSTejun Heo#include <asm/percpu.h> 24551889a6SIan Campbell 25551889a6SIan Campbell/* Physical address */ 26551889a6SIan Campbell#define pa(X) ((X) - __PAGE_OFFSET) 279a163ed8SThomas Gleixner 289a163ed8SThomas Gleixner/* 299a163ed8SThomas Gleixner * References to members of the new_cpu_data structure. 309a163ed8SThomas Gleixner */ 319a163ed8SThomas Gleixner 329a163ed8SThomas Gleixner#define X86 new_cpu_data+CPUINFO_x86 339a163ed8SThomas Gleixner#define X86_VENDOR new_cpu_data+CPUINFO_x86_vendor 349a163ed8SThomas Gleixner#define X86_MODEL new_cpu_data+CPUINFO_x86_model 359a163ed8SThomas Gleixner#define X86_MASK new_cpu_data+CPUINFO_x86_mask 369a163ed8SThomas Gleixner#define X86_HARD_MATH new_cpu_data+CPUINFO_hard_math 379a163ed8SThomas Gleixner#define X86_CPUID new_cpu_data+CPUINFO_cpuid_level 389a163ed8SThomas Gleixner#define X86_CAPABILITY new_cpu_data+CPUINFO_x86_capability 399a163ed8SThomas Gleixner#define X86_VENDOR_ID new_cpu_data+CPUINFO_x86_vendor_id 409a163ed8SThomas Gleixner 419a163ed8SThomas Gleixner/* 42c090f532SJeremy Fitzhardinge * This is how much memory in addition to the memory covered up to 43c090f532SJeremy Fitzhardinge * and including _end we need mapped initially. 449a163ed8SThomas Gleixner * We need: 452bd2753fSYinghai Lu * (KERNEL_IMAGE_SIZE/4096) / 1024 pages (worst case, non PAE) 462bd2753fSYinghai Lu * (KERNEL_IMAGE_SIZE/4096) / 512 + 4 pages (worst case for PAE) 479a163ed8SThomas Gleixner * 489a163ed8SThomas Gleixner * Modulo rounding, each megabyte assigned here requires a kilobyte of 499a163ed8SThomas Gleixner * memory, which is currently unreclaimed. 509a163ed8SThomas Gleixner * 519a163ed8SThomas Gleixner * This should be a multiple of a page. 522bd2753fSYinghai Lu * 532bd2753fSYinghai Lu * KERNEL_IMAGE_SIZE should be greater than pa(_end) 542bd2753fSYinghai Lu * and small than max_low_pfn, otherwise will waste some page table entries 559a163ed8SThomas Gleixner */ 569a163ed8SThomas Gleixner 579a163ed8SThomas Gleixner#if PTRS_PER_PMD > 1 58c090f532SJeremy Fitzhardinge#define PAGE_TABLE_SIZE(pages) (((pages) / PTRS_PER_PMD) + PTRS_PER_PGD) 599a163ed8SThomas Gleixner#else 60c090f532SJeremy Fitzhardinge#define PAGE_TABLE_SIZE(pages) ((pages) / PTRS_PER_PGD) 619a163ed8SThomas Gleixner#endif 629a163ed8SThomas Gleixner 63147dd561SH. Peter Anvin/* Number of possible pages in the lowmem region */ 64147dd561SH. Peter AnvinLOWMEM_PAGES = (((1<<32) - __PAGE_OFFSET) >> PAGE_SHIFT) 65147dd561SH. Peter Anvin 66c090f532SJeremy Fitzhardinge/* Enough space to fit pagetables for the low memory linear map */ 67147dd561SH. Peter AnvinMAPPING_BEYOND_END = PAGE_TABLE_SIZE(LOWMEM_PAGES) << PAGE_SHIFT 68c090f532SJeremy Fitzhardinge 69c090f532SJeremy Fitzhardinge/* 70c090f532SJeremy Fitzhardinge * Worst-case size of the kernel mapping we need to make: 71147dd561SH. Peter Anvin * a relocatable kernel can live anywhere in lowmem, so we need to be able 72147dd561SH. Peter Anvin * to map all of lowmem. 73c090f532SJeremy Fitzhardinge */ 74147dd561SH. Peter AnvinKERNEL_PAGES = LOWMEM_PAGES 75c090f532SJeremy Fitzhardinge 767bf04be8SStratos PsomadakisINIT_MAP_SIZE = PAGE_TABLE_SIZE(KERNEL_PAGES) * PAGE_SIZE 772bd2753fSYinghai LuRESERVE_BRK(pagetables, INIT_MAP_SIZE) 78796216a5SJeremy Fitzhardinge 799a163ed8SThomas Gleixner/* 809a163ed8SThomas Gleixner * 32-bit kernel entrypoint; only used by the boot CPU. On entry, 819a163ed8SThomas Gleixner * %esi points to the real-mode code as a 32-bit pointer. 829a163ed8SThomas Gleixner * CS and DS must be 4 GB flat segments, but we don't depend on 839a163ed8SThomas Gleixner * any particular GDT layout, because we load our own as soon as we 849a163ed8SThomas Gleixner * can. 859a163ed8SThomas Gleixner */ 864ae59b91STim Abbott__HEAD 879a163ed8SThomas GleixnerENTRY(startup_32) 8811d4c3f9SH. Peter Anvin movl pa(stack_start),%ecx 8911d4c3f9SH. Peter Anvin 90a24e7851SRusty Russell /* test KEEP_SEGMENTS flag to see if the bootloader is asking 91a24e7851SRusty Russell us to not reload segments */ 92a24e7851SRusty Russell testb $(1<<6), BP_loadflags(%esi) 93a24e7851SRusty Russell jnz 2f 949a163ed8SThomas Gleixner 959a163ed8SThomas Gleixner/* 969a163ed8SThomas Gleixner * Set segments to known values. 979a163ed8SThomas Gleixner */ 98551889a6SIan Campbell lgdt pa(boot_gdt_descr) 999a163ed8SThomas Gleixner movl $(__BOOT_DS),%eax 1009a163ed8SThomas Gleixner movl %eax,%ds 1019a163ed8SThomas Gleixner movl %eax,%es 1029a163ed8SThomas Gleixner movl %eax,%fs 1039a163ed8SThomas Gleixner movl %eax,%gs 10411d4c3f9SH. Peter Anvin movl %eax,%ss 105a24e7851SRusty Russell2: 10611d4c3f9SH. Peter Anvin leal -__PAGE_OFFSET(%ecx),%esp 1079a163ed8SThomas Gleixner 1089a163ed8SThomas Gleixner/* 1099a163ed8SThomas Gleixner * Clear BSS first so that there are no surprises... 1109a163ed8SThomas Gleixner */ 111a24e7851SRusty Russell cld 1129a163ed8SThomas Gleixner xorl %eax,%eax 113551889a6SIan Campbell movl $pa(__bss_start),%edi 114551889a6SIan Campbell movl $pa(__bss_stop),%ecx 1159a163ed8SThomas Gleixner subl %edi,%ecx 1169a163ed8SThomas Gleixner shrl $2,%ecx 1179a163ed8SThomas Gleixner rep ; stosl 1189a163ed8SThomas Gleixner/* 1199a163ed8SThomas Gleixner * Copy bootup parameters out of the way. 1209a163ed8SThomas Gleixner * Note: %esi still has the pointer to the real-mode data. 1219a163ed8SThomas Gleixner * With the kexec as boot loader, parameter segment might be loaded beyond 1229a163ed8SThomas Gleixner * kernel image and might not even be addressable by early boot page tables. 1239a163ed8SThomas Gleixner * (kexec on panic case). Hence copy out the parameters before initializing 1249a163ed8SThomas Gleixner * page tables. 1259a163ed8SThomas Gleixner */ 126551889a6SIan Campbell movl $pa(boot_params),%edi 1279a163ed8SThomas Gleixner movl $(PARAM_SIZE/4),%ecx 1289a163ed8SThomas Gleixner cld 1299a163ed8SThomas Gleixner rep 1309a163ed8SThomas Gleixner movsl 131551889a6SIan Campbell movl pa(boot_params) + NEW_CL_POINTER,%esi 1329a163ed8SThomas Gleixner andl %esi,%esi 133b595076aSUwe Kleine-König jz 1f # No command line 134551889a6SIan Campbell movl $pa(boot_command_line),%edi 1359a163ed8SThomas Gleixner movl $(COMMAND_LINE_SIZE/4),%ecx 1369a163ed8SThomas Gleixner rep 1379a163ed8SThomas Gleixner movsl 1389a163ed8SThomas Gleixner1: 1399a163ed8SThomas Gleixner 140dc3119e7SThomas Gleixner#ifdef CONFIG_OLPC 141fd699c76SAndres Salomon /* save OFW's pgdir table for later use when calling into OFW */ 142fd699c76SAndres Salomon movl %cr3, %eax 143fd699c76SAndres Salomon movl %eax, pa(olpc_ofw_pgd) 144fd699c76SAndres Salomon#endif 145fd699c76SAndres Salomon 1469a163ed8SThomas Gleixner/* 1479a163ed8SThomas Gleixner * Initialize page tables. This creates a PDE and a set of page 1482bd2753fSYinghai Lu * tables, which are located immediately beyond __brk_base. The variable 149ccf3fe02SJeremy Fitzhardinge * _brk_end is set up to point to the first "safe" location. 1509a163ed8SThomas Gleixner * Mappings are created both at virtual address 0 (identity mapping) 1512bd2753fSYinghai Lu * and PAGE_OFFSET for up to _end. 1529a163ed8SThomas Gleixner */ 153551889a6SIan Campbell#ifdef CONFIG_X86_PAE 154551889a6SIan Campbell 155551889a6SIan Campbell /* 156b40827faSBorislav Petkov * In PAE mode initial_page_table is statically defined to contain 157b40827faSBorislav Petkov * enough entries to cover the VMSPLIT option (that is the top 1, 2 or 3 158b40827faSBorislav Petkov * entries). The identity mapping is handled by pointing two PGD entries 159b40827faSBorislav Petkov * to the first kernel PMD. 160551889a6SIan Campbell * 161b40827faSBorislav Petkov * Note the upper half of each PMD or PTE are always zero at this stage. 162551889a6SIan Campbell */ 163551889a6SIan Campbell 16486b2b70eSJoe Korty#define KPMDS (((-__PAGE_OFFSET) >> 30) & 3) /* Number of kernel PMDs */ 165551889a6SIan Campbell 166551889a6SIan Campbell xorl %ebx,%ebx /* %ebx is kept at zero */ 167551889a6SIan Campbell 168ccf3fe02SJeremy Fitzhardinge movl $pa(__brk_base), %edi 169b40827faSBorislav Petkov movl $pa(initial_pg_pmd), %edx 170b2bc2731SSuresh Siddha movl $PTE_IDENT_ATTR, %eax 1719a163ed8SThomas Gleixner10: 172b2bc2731SSuresh Siddha leal PDE_IDENT_ATTR(%edi),%ecx /* Create PMD entry */ 173551889a6SIan Campbell movl %ecx,(%edx) /* Store PMD entry */ 174551889a6SIan Campbell /* Upper half already zero */ 175551889a6SIan Campbell addl $8,%edx 176551889a6SIan Campbell movl $512,%ecx 177551889a6SIan Campbell11: 178551889a6SIan Campbell stosl 179551889a6SIan Campbell xchgl %eax,%ebx 180551889a6SIan Campbell stosl 181551889a6SIan Campbell xchgl %eax,%ebx 182551889a6SIan Campbell addl $0x1000,%eax 183551889a6SIan Campbell loop 11b 184551889a6SIan Campbell 185551889a6SIan Campbell /* 186c090f532SJeremy Fitzhardinge * End condition: we must map up to the end + MAPPING_BEYOND_END. 187551889a6SIan Campbell */ 188c090f532SJeremy Fitzhardinge movl $pa(_end) + MAPPING_BEYOND_END + PTE_IDENT_ATTR, %ebp 189551889a6SIan Campbell cmpl %ebp,%eax 190551889a6SIan Campbell jb 10b 191551889a6SIan Campbell1: 192ccf3fe02SJeremy Fitzhardinge addl $__PAGE_OFFSET, %edi 193ccf3fe02SJeremy Fitzhardinge movl %edi, pa(_brk_end) 1946af61a76SYinghai Lu shrl $12, %eax 1956af61a76SYinghai Lu movl %eax, pa(max_pfn_mapped) 196551889a6SIan Campbell 197551889a6SIan Campbell /* Do early initialization of the fixmap area */ 198b40827faSBorislav Petkov movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax 199b40827faSBorislav Petkov movl %eax,pa(initial_pg_pmd+0x1000*KPMDS-8) 200551889a6SIan Campbell#else /* Not PAE */ 201551889a6SIan Campbell 202551889a6SIan Campbellpage_pde_offset = (__PAGE_OFFSET >> 20); 203551889a6SIan Campbell 204ccf3fe02SJeremy Fitzhardinge movl $pa(__brk_base), %edi 205b40827faSBorislav Petkov movl $pa(initial_page_table), %edx 206b2bc2731SSuresh Siddha movl $PTE_IDENT_ATTR, %eax 207551889a6SIan Campbell10: 208b2bc2731SSuresh Siddha leal PDE_IDENT_ATTR(%edi),%ecx /* Create PDE entry */ 2099a163ed8SThomas Gleixner movl %ecx,(%edx) /* Store identity PDE entry */ 2109a163ed8SThomas Gleixner movl %ecx,page_pde_offset(%edx) /* Store kernel PDE entry */ 2119a163ed8SThomas Gleixner addl $4,%edx 2129a163ed8SThomas Gleixner movl $1024, %ecx 2139a163ed8SThomas Gleixner11: 2149a163ed8SThomas Gleixner stosl 2159a163ed8SThomas Gleixner addl $0x1000,%eax 2169a163ed8SThomas Gleixner loop 11b 217551889a6SIan Campbell /* 218c090f532SJeremy Fitzhardinge * End condition: we must map up to the end + MAPPING_BEYOND_END. 219551889a6SIan Campbell */ 220c090f532SJeremy Fitzhardinge movl $pa(_end) + MAPPING_BEYOND_END + PTE_IDENT_ATTR, %ebp 2219a163ed8SThomas Gleixner cmpl %ebp,%eax 2229a163ed8SThomas Gleixner jb 10b 223ccf3fe02SJeremy Fitzhardinge addl $__PAGE_OFFSET, %edi 224ccf3fe02SJeremy Fitzhardinge movl %edi, pa(_brk_end) 2256af61a76SYinghai Lu shrl $12, %eax 2266af61a76SYinghai Lu movl %eax, pa(max_pfn_mapped) 2279a163ed8SThomas Gleixner 228551889a6SIan Campbell /* Do early initialization of the fixmap area */ 229b40827faSBorislav Petkov movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax 230b40827faSBorislav Petkov movl %eax,pa(initial_page_table+0xffc) 231551889a6SIan Campbell#endif 232d50d8fe1SRusty Russell 233d50d8fe1SRusty Russell#ifdef CONFIG_PARAVIRT 234d50d8fe1SRusty Russell /* This is can only trip for a broken bootloader... */ 235d50d8fe1SRusty Russell cmpw $0x207, pa(boot_params + BP_version) 236d50d8fe1SRusty Russell jb default_entry 237d50d8fe1SRusty Russell 238d50d8fe1SRusty Russell /* Paravirt-compatible boot parameters. Look to see what architecture 239d50d8fe1SRusty Russell we're booting under. */ 240d50d8fe1SRusty Russell movl pa(boot_params + BP_hardware_subarch), %eax 241d50d8fe1SRusty Russell cmpl $num_subarch_entries, %eax 242d50d8fe1SRusty Russell jae bad_subarch 243d50d8fe1SRusty Russell 244d50d8fe1SRusty Russell movl pa(subarch_entries)(,%eax,4), %eax 245d50d8fe1SRusty Russell subl $__PAGE_OFFSET, %eax 246d50d8fe1SRusty Russell jmp *%eax 247d50d8fe1SRusty Russell 248d50d8fe1SRusty Russellbad_subarch: 249d50d8fe1SRusty RussellWEAK(lguest_entry) 250d50d8fe1SRusty RussellWEAK(xen_entry) 251d50d8fe1SRusty Russell /* Unknown implementation; there's really 252d50d8fe1SRusty Russell nothing we can do at this point. */ 253d50d8fe1SRusty Russell ud2a 254d50d8fe1SRusty Russell 255d50d8fe1SRusty Russell __INITDATA 256d50d8fe1SRusty Russell 257d50d8fe1SRusty Russellsubarch_entries: 258d50d8fe1SRusty Russell .long default_entry /* normal x86/PC */ 259d50d8fe1SRusty Russell .long lguest_entry /* lguest hypervisor */ 260d50d8fe1SRusty Russell .long xen_entry /* Xen hypervisor */ 261d50d8fe1SRusty Russell .long default_entry /* Moorestown MID */ 262d50d8fe1SRusty Russellnum_subarch_entries = (. - subarch_entries) / 4 263d50d8fe1SRusty Russell.previous 264d50d8fe1SRusty Russell#else 265d50d8fe1SRusty Russell jmp default_entry 266d50d8fe1SRusty Russell#endif /* CONFIG_PARAVIRT */ 267d50d8fe1SRusty Russell 2689a163ed8SThomas Gleixner/* 2699a163ed8SThomas Gleixner * Non-boot CPU entry point; entered from trampoline.S 2709a163ed8SThomas Gleixner * We can't lgdt here, because lgdt itself uses a data segment, but 2719a163ed8SThomas Gleixner * we know the trampoline has already loaded the boot_gdt for us. 2729a163ed8SThomas Gleixner * 2739a163ed8SThomas Gleixner * If cpu hotplug is not supported then this code can go in init section 2749a163ed8SThomas Gleixner * which will be freed later 2759a163ed8SThomas Gleixner */ 27678b89ecdSJan Beulich__CPUINIT 2779a163ed8SThomas GleixnerENTRY(startup_32_smp) 2789a163ed8SThomas Gleixner cld 2799a163ed8SThomas Gleixner movl $(__BOOT_DS),%eax 2809a163ed8SThomas Gleixner movl %eax,%ds 2819a163ed8SThomas Gleixner movl %eax,%es 2829a163ed8SThomas Gleixner movl %eax,%fs 2839a163ed8SThomas Gleixner movl %eax,%gs 28411d4c3f9SH. Peter Anvin movl pa(stack_start),%ecx 28511d4c3f9SH. Peter Anvin movl %eax,%ss 28611d4c3f9SH. Peter Anvin leal -__PAGE_OFFSET(%ecx),%esp 28748927bbbSJarkko Sakkinen 288d50d8fe1SRusty Russelldefault_entry: 2899a163ed8SThomas Gleixner 2909a163ed8SThomas Gleixner/* 2919a163ed8SThomas Gleixner * New page tables may be in 4Mbyte page mode and may 2929a163ed8SThomas Gleixner * be using the global pages. 2939a163ed8SThomas Gleixner * 2949a163ed8SThomas Gleixner * NOTE! If we are on a 486 we may have no cr4 at all! 2959a163ed8SThomas Gleixner * So we do not try to touch it unless we really have 2969a163ed8SThomas Gleixner * some bits in it to set. This won't work if the BSP 2979a163ed8SThomas Gleixner * implements cr4 but this AP does not -- very unlikely 2989a163ed8SThomas Gleixner * but be warned! The same applies to the pse feature 2999a163ed8SThomas Gleixner * if not equally supported. --macro 3009a163ed8SThomas Gleixner * 3019a163ed8SThomas Gleixner * NOTE! We have to correct for the fact that we're 3029a163ed8SThomas Gleixner * not yet offset PAGE_OFFSET.. 3039a163ed8SThomas Gleixner */ 304551889a6SIan Campbell#define cr4_bits pa(mmu_cr4_features) 3059a163ed8SThomas Gleixner movl cr4_bits,%edx 3069a163ed8SThomas Gleixner andl %edx,%edx 3079a163ed8SThomas Gleixner jz 6f 3089a163ed8SThomas Gleixner movl %cr4,%eax # Turn on paging options (PSE,PAE,..) 3099a163ed8SThomas Gleixner orl %edx,%eax 3109a163ed8SThomas Gleixner movl %eax,%cr4 3119a163ed8SThomas Gleixner 3128a50e513SH. Peter Anvin testb $X86_CR4_PAE, %al # check if PAE is enabled 3138a50e513SH. Peter Anvin jz 6f 3149a163ed8SThomas Gleixner 3159a163ed8SThomas Gleixner /* Check if extended functions are implemented */ 3169a163ed8SThomas Gleixner movl $0x80000000, %eax 3179a163ed8SThomas Gleixner cpuid 3188a50e513SH. Peter Anvin /* Value must be in the range 0x80000001 to 0x8000ffff */ 3198a50e513SH. Peter Anvin subl $0x80000001, %eax 3208a50e513SH. Peter Anvin cmpl $(0x8000ffff-0x80000001), %eax 3218a50e513SH. Peter Anvin ja 6f 322ebba638aSKees Cook 323ebba638aSKees Cook /* Clear bogus XD_DISABLE bits */ 324ebba638aSKees Cook call verify_cpu 325ebba638aSKees Cook 3269a163ed8SThomas Gleixner mov $0x80000001, %eax 3279a163ed8SThomas Gleixner cpuid 3289a163ed8SThomas Gleixner /* Execute Disable bit supported? */ 3298a50e513SH. Peter Anvin btl $(X86_FEATURE_NX & 31), %edx 3309a163ed8SThomas Gleixner jnc 6f 3319a163ed8SThomas Gleixner 3329a163ed8SThomas Gleixner /* Setup EFER (Extended Feature Enable Register) */ 3338a50e513SH. Peter Anvin movl $MSR_EFER, %ecx 3349a163ed8SThomas Gleixner rdmsr 3359a163ed8SThomas Gleixner 3368a50e513SH. Peter Anvin btsl $_EFER_NX, %eax 3379a163ed8SThomas Gleixner /* Make changes effective */ 3389a163ed8SThomas Gleixner wrmsr 3399a163ed8SThomas Gleixner 3409a163ed8SThomas Gleixner6: 3419a163ed8SThomas Gleixner 3429a163ed8SThomas Gleixner/* 3439a163ed8SThomas Gleixner * Enable paging 3449a163ed8SThomas Gleixner */ 345b40827faSBorislav Petkov movl $pa(initial_page_table), %eax 3469a163ed8SThomas Gleixner movl %eax,%cr3 /* set the page table pointer.. */ 3479a163ed8SThomas Gleixner movl %cr0,%eax 348551889a6SIan Campbell orl $X86_CR0_PG,%eax 3499a163ed8SThomas Gleixner movl %eax,%cr0 /* ..and set paging (PG) bit */ 3509a163ed8SThomas Gleixner ljmp $__BOOT_CS,$1f /* Clear prefetch and normalize %eip */ 3519a163ed8SThomas Gleixner1: 35211d4c3f9SH. Peter Anvin /* Shift the stack pointer to a virtual address */ 35311d4c3f9SH. Peter Anvin addl $__PAGE_OFFSET, %esp 3549a163ed8SThomas Gleixner 3559a163ed8SThomas Gleixner/* 3569a163ed8SThomas Gleixner * Initialize eflags. Some BIOS's leave bits like NT set. This would 3579a163ed8SThomas Gleixner * confuse the debugger if this code is traced. 3589a163ed8SThomas Gleixner * XXX - best to initialize before switching to protected mode. 3599a163ed8SThomas Gleixner */ 3609a163ed8SThomas Gleixner pushl $0 3619a163ed8SThomas Gleixner popfl 3629a163ed8SThomas Gleixner 3639a163ed8SThomas Gleixner#ifdef CONFIG_SMP 36450359501SIan Campbell cmpb $0, ready 36511d4c3f9SH. Peter Anvin jnz checkCPUtype 3669a163ed8SThomas Gleixner#endif /* CONFIG_SMP */ 3679a163ed8SThomas Gleixner 3689a163ed8SThomas Gleixner/* 3699a163ed8SThomas Gleixner * start system 32-bit setup. We need to re-do some of the things done 3709a163ed8SThomas Gleixner * in 16-bit mode for the "real" operations. 3719a163ed8SThomas Gleixner */ 3729a163ed8SThomas Gleixner call setup_idt 3739a163ed8SThomas Gleixner 3749a163ed8SThomas GleixnercheckCPUtype: 3759a163ed8SThomas Gleixner 3769a163ed8SThomas Gleixner movl $-1,X86_CPUID # -1 for no CPUID initially 3779a163ed8SThomas Gleixner 3789a163ed8SThomas Gleixner/* check if it is 486 or 386. */ 3799a163ed8SThomas Gleixner/* 3809a163ed8SThomas Gleixner * XXX - this does a lot of unnecessary setup. Alignment checks don't 3819a163ed8SThomas Gleixner * apply at our cpl of 0 and the stack ought to be aligned already, and 3829a163ed8SThomas Gleixner * we don't need to preserve eflags. 3839a163ed8SThomas Gleixner */ 3849a163ed8SThomas Gleixner 3859a163ed8SThomas Gleixner movb $3,X86 # at least 386 3869a163ed8SThomas Gleixner pushfl # push EFLAGS 3879a163ed8SThomas Gleixner popl %eax # get EFLAGS 3889a163ed8SThomas Gleixner movl %eax,%ecx # save original EFLAGS 3899a163ed8SThomas Gleixner xorl $0x240000,%eax # flip AC and ID bits in EFLAGS 3909a163ed8SThomas Gleixner pushl %eax # copy to EFLAGS 3919a163ed8SThomas Gleixner popfl # set EFLAGS 3929a163ed8SThomas Gleixner pushfl # get new EFLAGS 3939a163ed8SThomas Gleixner popl %eax # put it in eax 3949a163ed8SThomas Gleixner xorl %ecx,%eax # change in flags 3959a163ed8SThomas Gleixner pushl %ecx # restore original EFLAGS 3969a163ed8SThomas Gleixner popfl 3979a163ed8SThomas Gleixner testl $0x40000,%eax # check if AC bit changed 3989a163ed8SThomas Gleixner je is386 3999a163ed8SThomas Gleixner 4009a163ed8SThomas Gleixner movb $4,X86 # at least 486 4019a163ed8SThomas Gleixner testl $0x200000,%eax # check if ID bit changed 4029a163ed8SThomas Gleixner je is486 4039a163ed8SThomas Gleixner 4049a163ed8SThomas Gleixner /* get vendor info */ 4059a163ed8SThomas Gleixner xorl %eax,%eax # call CPUID with 0 -> return vendor ID 4069a163ed8SThomas Gleixner cpuid 4079a163ed8SThomas Gleixner movl %eax,X86_CPUID # save CPUID level 4089a163ed8SThomas Gleixner movl %ebx,X86_VENDOR_ID # lo 4 chars 4099a163ed8SThomas Gleixner movl %edx,X86_VENDOR_ID+4 # next 4 chars 4109a163ed8SThomas Gleixner movl %ecx,X86_VENDOR_ID+8 # last 4 chars 4119a163ed8SThomas Gleixner 4129a163ed8SThomas Gleixner orl %eax,%eax # do we have processor info as well? 4139a163ed8SThomas Gleixner je is486 4149a163ed8SThomas Gleixner 4159a163ed8SThomas Gleixner movl $1,%eax # Use the CPUID instruction to get CPU type 4169a163ed8SThomas Gleixner cpuid 4179a163ed8SThomas Gleixner movb %al,%cl # save reg for future use 4189a163ed8SThomas Gleixner andb $0x0f,%ah # mask processor family 4199a163ed8SThomas Gleixner movb %ah,X86 4209a163ed8SThomas Gleixner andb $0xf0,%al # mask model 4219a163ed8SThomas Gleixner shrb $4,%al 4229a163ed8SThomas Gleixner movb %al,X86_MODEL 4239a163ed8SThomas Gleixner andb $0x0f,%cl # mask mask revision 4249a163ed8SThomas Gleixner movb %cl,X86_MASK 4259a163ed8SThomas Gleixner movl %edx,X86_CAPABILITY 4269a163ed8SThomas Gleixner 4279a163ed8SThomas Gleixneris486: movl $0x50022,%ecx # set AM, WP, NE and MP 4289a163ed8SThomas Gleixner jmp 2f 4299a163ed8SThomas Gleixner 4309a163ed8SThomas Gleixneris386: movl $2,%ecx # set MP 4319a163ed8SThomas Gleixner2: movl %cr0,%eax 4329a163ed8SThomas Gleixner andl $0x80000011,%eax # Save PG,PE,ET 4339a163ed8SThomas Gleixner orl %ecx,%eax 4349a163ed8SThomas Gleixner movl %eax,%cr0 4359a163ed8SThomas Gleixner 4369a163ed8SThomas Gleixner call check_x87 4379a163ed8SThomas Gleixner lgdt early_gdt_descr 4389a163ed8SThomas Gleixner lidt idt_descr 4399a163ed8SThomas Gleixner ljmp $(__KERNEL_CS),$1f 4409a163ed8SThomas Gleixner1: movl $(__KERNEL_DS),%eax # reload all the segment registers 4419a163ed8SThomas Gleixner movl %eax,%ss # after changing gdt. 4429a163ed8SThomas Gleixner 4439a163ed8SThomas Gleixner movl $(__USER_DS),%eax # DS/ES contains default USER segment 4449a163ed8SThomas Gleixner movl %eax,%ds 4459a163ed8SThomas Gleixner movl %eax,%es 4469a163ed8SThomas Gleixner 4470dd76d73SBrian Gerst movl $(__KERNEL_PERCPU), %eax 4480dd76d73SBrian Gerst movl %eax,%fs # set this cpu's percpu 4490dd76d73SBrian Gerst 45060a5317fSTejun Heo#ifdef CONFIG_CC_STACKPROTECTOR 45160a5317fSTejun Heo /* 45260a5317fSTejun Heo * The linker can't handle this by relocation. Manually set 45360a5317fSTejun Heo * base address in stack canary segment descriptor. 45460a5317fSTejun Heo */ 45560a5317fSTejun Heo cmpb $0,ready 45660a5317fSTejun Heo jne 1f 457dd17c8f7SRusty Russell movl $gdt_page,%eax 458dd17c8f7SRusty Russell movl $stack_canary,%ecx 45960a5317fSTejun Heo movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax) 46060a5317fSTejun Heo shrl $16, %ecx 46160a5317fSTejun Heo movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax) 46260a5317fSTejun Heo movb %ch, 8 * GDT_ENTRY_STACK_CANARY + 7(%eax) 46360a5317fSTejun Heo1: 46460a5317fSTejun Heo#endif 46560a5317fSTejun Heo movl $(__KERNEL_STACK_CANARY),%eax 4669a163ed8SThomas Gleixner movl %eax,%gs 46760a5317fSTejun Heo 46860a5317fSTejun Heo xorl %eax,%eax # Clear LDT 4699a163ed8SThomas Gleixner lldt %ax 4709a163ed8SThomas Gleixner 4719a163ed8SThomas Gleixner cld # gcc2 wants the direction flag cleared at all times 4729a163ed8SThomas Gleixner pushl $0 # fake return address for unwinder 4739a163ed8SThomas Gleixner movb $1, ready 474e3f77edfSGlauber Costa jmp *(initial_code) 4759a163ed8SThomas Gleixner 4769a163ed8SThomas Gleixner/* 4779a163ed8SThomas Gleixner * We depend on ET to be correct. This checks for 287/387. 4789a163ed8SThomas Gleixner */ 4799a163ed8SThomas Gleixnercheck_x87: 4809a163ed8SThomas Gleixner movb $0,X86_HARD_MATH 4819a163ed8SThomas Gleixner clts 4829a163ed8SThomas Gleixner fninit 4839a163ed8SThomas Gleixner fstsw %ax 4849a163ed8SThomas Gleixner cmpb $0,%al 4859a163ed8SThomas Gleixner je 1f 4869a163ed8SThomas Gleixner movl %cr0,%eax /* no coprocessor: have to set bits */ 4879a163ed8SThomas Gleixner xorl $4,%eax /* set EM */ 4889a163ed8SThomas Gleixner movl %eax,%cr0 4899a163ed8SThomas Gleixner ret 4909a163ed8SThomas Gleixner ALIGN 4919a163ed8SThomas Gleixner1: movb $1,X86_HARD_MATH 4929a163ed8SThomas Gleixner .byte 0xDB,0xE4 /* fsetpm for 287, ignored by 387 */ 4939a163ed8SThomas Gleixner ret 4949a163ed8SThomas Gleixner 4959a163ed8SThomas Gleixner/* 4969a163ed8SThomas Gleixner * setup_idt 4979a163ed8SThomas Gleixner * 4989a163ed8SThomas Gleixner * sets up a idt with 256 entries pointing to 4999a163ed8SThomas Gleixner * ignore_int, interrupt gates. It doesn't actually load 5009a163ed8SThomas Gleixner * idt - that can be done only after paging has been enabled 5019a163ed8SThomas Gleixner * and the kernel moved to PAGE_OFFSET. Interrupts 5029a163ed8SThomas Gleixner * are enabled elsewhere, when we can be relatively 5039a163ed8SThomas Gleixner * sure everything is ok. 5049a163ed8SThomas Gleixner * 5059a163ed8SThomas Gleixner * Warning: %esi is live across this function. 5069a163ed8SThomas Gleixner */ 5079a163ed8SThomas Gleixnersetup_idt: 5089a163ed8SThomas Gleixner lea ignore_int,%edx 5099a163ed8SThomas Gleixner movl $(__KERNEL_CS << 16),%eax 5109a163ed8SThomas Gleixner movw %dx,%ax /* selector = 0x0010 = cs */ 5119a163ed8SThomas Gleixner movw $0x8E00,%dx /* interrupt gate - dpl=0, present */ 5129a163ed8SThomas Gleixner 5139a163ed8SThomas Gleixner lea idt_table,%edi 5149a163ed8SThomas Gleixner mov $256,%ecx 5159a163ed8SThomas Gleixnerrp_sidt: 5169a163ed8SThomas Gleixner movl %eax,(%edi) 5179a163ed8SThomas Gleixner movl %edx,4(%edi) 5189a163ed8SThomas Gleixner addl $8,%edi 5199a163ed8SThomas Gleixner dec %ecx 5209a163ed8SThomas Gleixner jne rp_sidt 5219a163ed8SThomas Gleixner 5229a163ed8SThomas Gleixner.macro set_early_handler handler,trapno 5239a163ed8SThomas Gleixner lea \handler,%edx 5249a163ed8SThomas Gleixner movl $(__KERNEL_CS << 16),%eax 5259a163ed8SThomas Gleixner movw %dx,%ax 5269a163ed8SThomas Gleixner movw $0x8E00,%dx /* interrupt gate - dpl=0, present */ 5279a163ed8SThomas Gleixner lea idt_table,%edi 5289a163ed8SThomas Gleixner movl %eax,8*\trapno(%edi) 5299a163ed8SThomas Gleixner movl %edx,8*\trapno+4(%edi) 5309a163ed8SThomas Gleixner.endm 5319a163ed8SThomas Gleixner 5329a163ed8SThomas Gleixner set_early_handler handler=early_divide_err,trapno=0 5339a163ed8SThomas Gleixner set_early_handler handler=early_illegal_opcode,trapno=6 5349a163ed8SThomas Gleixner set_early_handler handler=early_protection_fault,trapno=13 5359a163ed8SThomas Gleixner set_early_handler handler=early_page_fault,trapno=14 5369a163ed8SThomas Gleixner 5379a163ed8SThomas Gleixner ret 5389a163ed8SThomas Gleixner 5399a163ed8SThomas Gleixnerearly_divide_err: 5409a163ed8SThomas Gleixner xor %edx,%edx 5419a163ed8SThomas Gleixner pushl $0 /* fake errcode */ 5429a163ed8SThomas Gleixner jmp early_fault 5439a163ed8SThomas Gleixner 5449a163ed8SThomas Gleixnerearly_illegal_opcode: 5459a163ed8SThomas Gleixner movl $6,%edx 5469a163ed8SThomas Gleixner pushl $0 /* fake errcode */ 5479a163ed8SThomas Gleixner jmp early_fault 5489a163ed8SThomas Gleixner 5499a163ed8SThomas Gleixnerearly_protection_fault: 5509a163ed8SThomas Gleixner movl $13,%edx 5519a163ed8SThomas Gleixner jmp early_fault 5529a163ed8SThomas Gleixner 5539a163ed8SThomas Gleixnerearly_page_fault: 5549a163ed8SThomas Gleixner movl $14,%edx 5559a163ed8SThomas Gleixner jmp early_fault 5569a163ed8SThomas Gleixner 5579a163ed8SThomas Gleixnerearly_fault: 5589a163ed8SThomas Gleixner cld 5599a163ed8SThomas Gleixner#ifdef CONFIG_PRINTK 560382f64abSIngo Molnar pusha 5619a163ed8SThomas Gleixner movl $(__KERNEL_DS),%eax 5629a163ed8SThomas Gleixner movl %eax,%ds 5639a163ed8SThomas Gleixner movl %eax,%es 5649a163ed8SThomas Gleixner cmpl $2,early_recursion_flag 5659a163ed8SThomas Gleixner je hlt_loop 5669a163ed8SThomas Gleixner incl early_recursion_flag 5679a163ed8SThomas Gleixner movl %cr2,%eax 5689a163ed8SThomas Gleixner pushl %eax 5699a163ed8SThomas Gleixner pushl %edx /* trapno */ 5709a163ed8SThomas Gleixner pushl $fault_msg 5719a163ed8SThomas Gleixner call printk 5729a163ed8SThomas Gleixner#endif 57394878efdSIngo Molnar call dump_stack 5749a163ed8SThomas Gleixnerhlt_loop: 5759a163ed8SThomas Gleixner hlt 5769a163ed8SThomas Gleixner jmp hlt_loop 5779a163ed8SThomas Gleixner 5789a163ed8SThomas Gleixner/* This is the default interrupt "handler" :-) */ 5799a163ed8SThomas Gleixner ALIGN 5809a163ed8SThomas Gleixnerignore_int: 5819a163ed8SThomas Gleixner cld 5829a163ed8SThomas Gleixner#ifdef CONFIG_PRINTK 5839a163ed8SThomas Gleixner pushl %eax 5849a163ed8SThomas Gleixner pushl %ecx 5859a163ed8SThomas Gleixner pushl %edx 5869a163ed8SThomas Gleixner pushl %es 5879a163ed8SThomas Gleixner pushl %ds 5889a163ed8SThomas Gleixner movl $(__KERNEL_DS),%eax 5899a163ed8SThomas Gleixner movl %eax,%ds 5909a163ed8SThomas Gleixner movl %eax,%es 5919a163ed8SThomas Gleixner cmpl $2,early_recursion_flag 5929a163ed8SThomas Gleixner je hlt_loop 5939a163ed8SThomas Gleixner incl early_recursion_flag 5949a163ed8SThomas Gleixner pushl 16(%esp) 5959a163ed8SThomas Gleixner pushl 24(%esp) 5969a163ed8SThomas Gleixner pushl 32(%esp) 5979a163ed8SThomas Gleixner pushl 40(%esp) 5989a163ed8SThomas Gleixner pushl $int_msg 5999a163ed8SThomas Gleixner call printk 600d5e397cbSIngo Molnar 601d5e397cbSIngo Molnar call dump_stack 602d5e397cbSIngo Molnar 6039a163ed8SThomas Gleixner addl $(5*4),%esp 6049a163ed8SThomas Gleixner popl %ds 6059a163ed8SThomas Gleixner popl %es 6069a163ed8SThomas Gleixner popl %edx 6079a163ed8SThomas Gleixner popl %ecx 6089a163ed8SThomas Gleixner popl %eax 6099a163ed8SThomas Gleixner#endif 6109a163ed8SThomas Gleixner iret 6119a163ed8SThomas Gleixner 612ebba638aSKees Cook#include "verify_cpu.S" 613ebba638aSKees Cook 6140e83815bSRobert Richter __REFDATA 615583323b9SThomas Gleixner.align 4 616583323b9SThomas GleixnerENTRY(initial_code) 617583323b9SThomas Gleixner .long i386_start_kernel 618583323b9SThomas Gleixner 6199a163ed8SThomas Gleixner/* 6209a163ed8SThomas Gleixner * BSS section 6219a163ed8SThomas Gleixner */ 62202b7da37STim Abbott__PAGE_ALIGNED_BSS 6237bf04be8SStratos Psomadakis .align PAGE_SIZE 624551889a6SIan Campbell#ifdef CONFIG_X86_PAE 625d50d8fe1SRusty Russellinitial_pg_pmd: 626551889a6SIan Campbell .fill 1024*KPMDS,4,0 627551889a6SIan Campbell#else 628b40827faSBorislav PetkovENTRY(initial_page_table) 6299a163ed8SThomas Gleixner .fill 1024,4,0 630551889a6SIan Campbell#endif 631d50d8fe1SRusty Russellinitial_pg_fixmap: 6329a163ed8SThomas Gleixner .fill 1024,4,0 6339a163ed8SThomas GleixnerENTRY(empty_zero_page) 6349a163ed8SThomas Gleixner .fill 4096,1,0 635b40827faSBorislav PetkovENTRY(swapper_pg_dir) 636b40827faSBorislav Petkov .fill 1024,4,0 6372bd2753fSYinghai Lu 6389a163ed8SThomas Gleixner/* 6399a163ed8SThomas Gleixner * This starts the data section. 6409a163ed8SThomas Gleixner */ 641551889a6SIan Campbell#ifdef CONFIG_X86_PAE 642abe1ee3aSTim Abbott__PAGE_ALIGNED_DATA 643551889a6SIan Campbell /* Page-aligned for the benefit of paravirt? */ 6447bf04be8SStratos Psomadakis .align PAGE_SIZE 645b40827faSBorislav PetkovENTRY(initial_page_table) 646b40827faSBorislav Petkov .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 /* low identity map */ 647551889a6SIan Campbell# if KPMDS == 3 648b40827faSBorislav Petkov .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 649b40827faSBorislav Petkov .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0 650b40827faSBorislav Petkov .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x2000),0 651551889a6SIan Campbell# elif KPMDS == 2 652551889a6SIan Campbell .long 0,0 653b40827faSBorislav Petkov .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 654b40827faSBorislav Petkov .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0 655551889a6SIan Campbell# elif KPMDS == 1 656551889a6SIan Campbell .long 0,0 657551889a6SIan Campbell .long 0,0 658b40827faSBorislav Petkov .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 659551889a6SIan Campbell# else 660551889a6SIan Campbell# error "Kernel PMDs should be 1, 2 or 3" 661551889a6SIan Campbell# endif 6627bf04be8SStratos Psomadakis .align PAGE_SIZE /* needs to be page-sized too */ 663551889a6SIan Campbell#endif 664551889a6SIan Campbell 6659a163ed8SThomas Gleixner.data 66611d4c3f9SH. Peter Anvin.balign 4 6679a163ed8SThomas GleixnerENTRY(stack_start) 6689a163ed8SThomas Gleixner .long init_thread_union+THREAD_SIZE 6699a163ed8SThomas Gleixner 6709a163ed8SThomas Gleixnerearly_recursion_flag: 6719a163ed8SThomas Gleixner .long 0 6729a163ed8SThomas Gleixner 67311d4c3f9SH. Peter Anvinready: .byte 0 67411d4c3f9SH. Peter Anvin 6759a163ed8SThomas Gleixnerint_msg: 676d5e397cbSIngo Molnar .asciz "Unknown interrupt or fault at: %p %p %p\n" 6779a163ed8SThomas Gleixner 6789a163ed8SThomas Gleixnerfault_msg: 679575ca735SVegard Nossum/* fault info: */ 680575ca735SVegard Nossum .ascii "BUG: Int %d: CR2 %p\n" 681575ca735SVegard Nossum/* pusha regs: */ 682575ca735SVegard Nossum .ascii " EDI %p ESI %p EBP %p ESP %p\n" 683575ca735SVegard Nossum .ascii " EBX %p EDX %p ECX %p EAX %p\n" 684575ca735SVegard Nossum/* fault frame: */ 685575ca735SVegard Nossum .ascii " err %p EIP %p CS %p flg %p\n" 686575ca735SVegard Nossum .ascii "Stack: %p %p %p %p %p %p %p %p\n" 687575ca735SVegard Nossum .ascii " %p %p %p %p %p %p %p %p\n" 688575ca735SVegard Nossum .asciz " %p %p %p %p %p %p %p %p\n" 6899a163ed8SThomas Gleixner 6909a163ed8SThomas Gleixner#include "../../x86/xen/xen-head.S" 6919a163ed8SThomas Gleixner 6929a163ed8SThomas Gleixner/* 6939a163ed8SThomas Gleixner * The IDT and GDT 'descriptors' are a strange 48-bit object 6949a163ed8SThomas Gleixner * only used by the lidt and lgdt instructions. They are not 6959a163ed8SThomas Gleixner * like usual segment descriptors - they consist of a 16-bit 6969a163ed8SThomas Gleixner * segment size, and 32-bit linear address value: 6979a163ed8SThomas Gleixner */ 6989a163ed8SThomas Gleixner 6999a163ed8SThomas Gleixner.globl boot_gdt_descr 7009a163ed8SThomas Gleixner.globl idt_descr 7019a163ed8SThomas Gleixner 7029a163ed8SThomas Gleixner ALIGN 7039a163ed8SThomas Gleixner# early boot GDT descriptor (must use 1:1 address mapping) 7049a163ed8SThomas Gleixner .word 0 # 32 bit align gdt_desc.address 7059a163ed8SThomas Gleixnerboot_gdt_descr: 7069a163ed8SThomas Gleixner .word __BOOT_DS+7 7079a163ed8SThomas Gleixner .long boot_gdt - __PAGE_OFFSET 7089a163ed8SThomas Gleixner 7099a163ed8SThomas Gleixner .word 0 # 32-bit align idt_desc.address 7109a163ed8SThomas Gleixneridt_descr: 7119a163ed8SThomas Gleixner .word IDT_ENTRIES*8-1 # idt contains 256 entries 7129a163ed8SThomas Gleixner .long idt_table 7139a163ed8SThomas Gleixner 7149a163ed8SThomas Gleixner# boot GDT descriptor (later on used by CPU#0): 7159a163ed8SThomas Gleixner .word 0 # 32 bit align gdt_desc.address 7169a163ed8SThomas GleixnerENTRY(early_gdt_descr) 7179a163ed8SThomas Gleixner .word GDT_ENTRIES*8-1 718dd17c8f7SRusty Russell .long gdt_page /* Overwritten for secondary CPUs */ 7199a163ed8SThomas Gleixner 7209a163ed8SThomas Gleixner/* 7219a163ed8SThomas Gleixner * The boot_gdt must mirror the equivalent in setup.S and is 7229a163ed8SThomas Gleixner * used only for booting. 7239a163ed8SThomas Gleixner */ 7249a163ed8SThomas Gleixner .align L1_CACHE_BYTES 7259a163ed8SThomas GleixnerENTRY(boot_gdt) 7269a163ed8SThomas Gleixner .fill GDT_ENTRY_BOOT_CS,8,0 7279a163ed8SThomas Gleixner .quad 0x00cf9a000000ffff /* kernel 4GB code at 0x00000000 */ 7289a163ed8SThomas Gleixner .quad 0x00cf92000000ffff /* kernel 4GB data at 0x00000000 */ 729