xref: /openbmc/linux/arch/x86/include/asm/processor.h (revision 68198dca)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_PROCESSOR_H
3 #define _ASM_X86_PROCESSOR_H
4 
5 #include <asm/processor-flags.h>
6 
7 /* Forward declaration, a strange C thing */
8 struct task_struct;
9 struct mm_struct;
10 struct vm86;
11 
12 #include <asm/math_emu.h>
13 #include <asm/segment.h>
14 #include <asm/types.h>
15 #include <uapi/asm/sigcontext.h>
16 #include <asm/current.h>
17 #include <asm/cpufeatures.h>
18 #include <asm/page.h>
19 #include <asm/pgtable_types.h>
20 #include <asm/percpu.h>
21 #include <asm/msr.h>
22 #include <asm/desc_defs.h>
23 #include <asm/nops.h>
24 #include <asm/special_insns.h>
25 #include <asm/fpu/types.h>
26 #include <asm/unwind_hints.h>
27 
28 #include <linux/personality.h>
29 #include <linux/cache.h>
30 #include <linux/threads.h>
31 #include <linux/math64.h>
32 #include <linux/err.h>
33 #include <linux/irqflags.h>
34 #include <linux/mem_encrypt.h>
35 
36 /*
37  * We handle most unaligned accesses in hardware.  On the other hand
38  * unaligned DMA can be quite expensive on some Nehalem processors.
39  *
40  * Based on this we disable the IP header alignment in network drivers.
41  */
42 #define NET_IP_ALIGN	0
43 
44 #define HBP_NUM 4
45 /*
46  * Default implementation of macro that returns current
47  * instruction pointer ("program counter").
48  */
49 static inline void *current_text_addr(void)
50 {
51 	void *pc;
52 
53 	asm volatile("mov $1f, %0; 1:":"=r" (pc));
54 
55 	return pc;
56 }
57 
58 /*
59  * These alignment constraints are for performance in the vSMP case,
60  * but in the task_struct case we must also meet hardware imposed
61  * alignment requirements of the FPU state:
62  */
63 #ifdef CONFIG_X86_VSMP
64 # define ARCH_MIN_TASKALIGN		(1 << INTERNODE_CACHE_SHIFT)
65 # define ARCH_MIN_MMSTRUCT_ALIGN	(1 << INTERNODE_CACHE_SHIFT)
66 #else
67 # define ARCH_MIN_TASKALIGN		__alignof__(union fpregs_state)
68 # define ARCH_MIN_MMSTRUCT_ALIGN	0
69 #endif
70 
71 enum tlb_infos {
72 	ENTRIES,
73 	NR_INFO
74 };
75 
76 extern u16 __read_mostly tlb_lli_4k[NR_INFO];
77 extern u16 __read_mostly tlb_lli_2m[NR_INFO];
78 extern u16 __read_mostly tlb_lli_4m[NR_INFO];
79 extern u16 __read_mostly tlb_lld_4k[NR_INFO];
80 extern u16 __read_mostly tlb_lld_2m[NR_INFO];
81 extern u16 __read_mostly tlb_lld_4m[NR_INFO];
82 extern u16 __read_mostly tlb_lld_1g[NR_INFO];
83 
84 /*
85  *  CPU type and hardware bug flags. Kept separately for each CPU.
86  *  Members of this structure are referenced in head_32.S, so think twice
87  *  before touching them. [mj]
88  */
89 
90 struct cpuinfo_x86 {
91 	__u8			x86;		/* CPU family */
92 	__u8			x86_vendor;	/* CPU vendor */
93 	__u8			x86_model;
94 	__u8			x86_mask;
95 #ifdef CONFIG_X86_64
96 	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
97 	int			x86_tlbsize;
98 #endif
99 	__u8			x86_virt_bits;
100 	__u8			x86_phys_bits;
101 	/* CPUID returned core id bits: */
102 	__u8			x86_coreid_bits;
103 	__u8			cu_id;
104 	/* Max extended CPUID function supported: */
105 	__u32			extended_cpuid_level;
106 	/* Maximum supported CPUID level, -1=no CPUID: */
107 	int			cpuid_level;
108 	__u32			x86_capability[NCAPINTS + NBUGINTS];
109 	char			x86_vendor_id[16];
110 	char			x86_model_id[64];
111 	/* in KB - valid for CPUS which support this call: */
112 	int			x86_cache_size;
113 	int			x86_cache_alignment;	/* In bytes */
114 	/* Cache QoS architectural values: */
115 	int			x86_cache_max_rmid;	/* max index */
116 	int			x86_cache_occ_scale;	/* scale to bytes */
117 	int			x86_power;
118 	unsigned long		loops_per_jiffy;
119 	/* cpuid returned max cores value: */
120 	u16			 x86_max_cores;
121 	u16			apicid;
122 	u16			initial_apicid;
123 	u16			x86_clflush_size;
124 	/* number of cores as seen by the OS: */
125 	u16			booted_cores;
126 	/* Physical processor id: */
127 	u16			phys_proc_id;
128 	/* Logical processor id: */
129 	u16			logical_proc_id;
130 	/* Core id: */
131 	u16			cpu_core_id;
132 	/* Index into per_cpu list: */
133 	u16			cpu_index;
134 	u32			microcode;
135 	unsigned		initialized : 1;
136 } __randomize_layout;
137 
138 struct cpuid_regs {
139 	u32 eax, ebx, ecx, edx;
140 };
141 
142 enum cpuid_regs_idx {
143 	CPUID_EAX = 0,
144 	CPUID_EBX,
145 	CPUID_ECX,
146 	CPUID_EDX,
147 };
148 
149 #define X86_VENDOR_INTEL	0
150 #define X86_VENDOR_CYRIX	1
151 #define X86_VENDOR_AMD		2
152 #define X86_VENDOR_UMC		3
153 #define X86_VENDOR_CENTAUR	5
154 #define X86_VENDOR_TRANSMETA	7
155 #define X86_VENDOR_NSC		8
156 #define X86_VENDOR_NUM		9
157 
158 #define X86_VENDOR_UNKNOWN	0xff
159 
160 /*
161  * capabilities of CPUs
162  */
163 extern struct cpuinfo_x86	boot_cpu_data;
164 extern struct cpuinfo_x86	new_cpu_data;
165 
166 extern struct tss_struct	doublefault_tss;
167 extern __u32			cpu_caps_cleared[NCAPINTS];
168 extern __u32			cpu_caps_set[NCAPINTS];
169 
170 #ifdef CONFIG_SMP
171 DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
172 #define cpu_data(cpu)		per_cpu(cpu_info, cpu)
173 #else
174 #define cpu_info		boot_cpu_data
175 #define cpu_data(cpu)		boot_cpu_data
176 #endif
177 
178 extern const struct seq_operations cpuinfo_op;
179 
180 #define cache_line_size()	(boot_cpu_data.x86_cache_alignment)
181 
182 extern void cpu_detect(struct cpuinfo_x86 *c);
183 
184 extern void early_cpu_init(void);
185 extern void identify_boot_cpu(void);
186 extern void identify_secondary_cpu(struct cpuinfo_x86 *);
187 extern void print_cpu_info(struct cpuinfo_x86 *);
188 void print_cpu_msr(struct cpuinfo_x86 *);
189 extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
190 extern u32 get_scattered_cpuid_leaf(unsigned int level,
191 				    unsigned int sub_leaf,
192 				    enum cpuid_regs_idx reg);
193 extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
194 extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
195 
196 extern void detect_extended_topology(struct cpuinfo_x86 *c);
197 extern void detect_ht(struct cpuinfo_x86 *c);
198 
199 #ifdef CONFIG_X86_32
200 extern int have_cpuid_p(void);
201 #else
202 static inline int have_cpuid_p(void)
203 {
204 	return 1;
205 }
206 #endif
207 static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
208 				unsigned int *ecx, unsigned int *edx)
209 {
210 	/* ecx is often an input as well as an output. */
211 	asm volatile("cpuid"
212 	    : "=a" (*eax),
213 	      "=b" (*ebx),
214 	      "=c" (*ecx),
215 	      "=d" (*edx)
216 	    : "0" (*eax), "2" (*ecx)
217 	    : "memory");
218 }
219 
220 #define native_cpuid_reg(reg)					\
221 static inline unsigned int native_cpuid_##reg(unsigned int op)	\
222 {								\
223 	unsigned int eax = op, ebx, ecx = 0, edx;		\
224 								\
225 	native_cpuid(&eax, &ebx, &ecx, &edx);			\
226 								\
227 	return reg;						\
228 }
229 
230 /*
231  * Native CPUID functions returning a single datum.
232  */
233 native_cpuid_reg(eax)
234 native_cpuid_reg(ebx)
235 native_cpuid_reg(ecx)
236 native_cpuid_reg(edx)
237 
238 /*
239  * Friendlier CR3 helpers.
240  */
241 static inline unsigned long read_cr3_pa(void)
242 {
243 	return __read_cr3() & CR3_ADDR_MASK;
244 }
245 
246 static inline unsigned long native_read_cr3_pa(void)
247 {
248 	return __native_read_cr3() & CR3_ADDR_MASK;
249 }
250 
251 static inline void load_cr3(pgd_t *pgdir)
252 {
253 	write_cr3(__sme_pa(pgdir));
254 }
255 
256 #ifdef CONFIG_X86_32
257 /* This is the TSS defined by the hardware. */
258 struct x86_hw_tss {
259 	unsigned short		back_link, __blh;
260 	unsigned long		sp0;
261 	unsigned short		ss0, __ss0h;
262 	unsigned long		sp1;
263 
264 	/*
265 	 * We don't use ring 1, so ss1 is a convenient scratch space in
266 	 * the same cacheline as sp0.  We use ss1 to cache the value in
267 	 * MSR_IA32_SYSENTER_CS.  When we context switch
268 	 * MSR_IA32_SYSENTER_CS, we first check if the new value being
269 	 * written matches ss1, and, if it's not, then we wrmsr the new
270 	 * value and update ss1.
271 	 *
272 	 * The only reason we context switch MSR_IA32_SYSENTER_CS is
273 	 * that we set it to zero in vm86 tasks to avoid corrupting the
274 	 * stack if we were to go through the sysenter path from vm86
275 	 * mode.
276 	 */
277 	unsigned short		ss1;	/* MSR_IA32_SYSENTER_CS */
278 
279 	unsigned short		__ss1h;
280 	unsigned long		sp2;
281 	unsigned short		ss2, __ss2h;
282 	unsigned long		__cr3;
283 	unsigned long		ip;
284 	unsigned long		flags;
285 	unsigned long		ax;
286 	unsigned long		cx;
287 	unsigned long		dx;
288 	unsigned long		bx;
289 	unsigned long		sp;
290 	unsigned long		bp;
291 	unsigned long		si;
292 	unsigned long		di;
293 	unsigned short		es, __esh;
294 	unsigned short		cs, __csh;
295 	unsigned short		ss, __ssh;
296 	unsigned short		ds, __dsh;
297 	unsigned short		fs, __fsh;
298 	unsigned short		gs, __gsh;
299 	unsigned short		ldt, __ldth;
300 	unsigned short		trace;
301 	unsigned short		io_bitmap_base;
302 
303 } __attribute__((packed));
304 #else
305 struct x86_hw_tss {
306 	u32			reserved1;
307 	u64			sp0;
308 	u64			sp1;
309 	u64			sp2;
310 	u64			reserved2;
311 	u64			ist[7];
312 	u32			reserved3;
313 	u32			reserved4;
314 	u16			reserved5;
315 	u16			io_bitmap_base;
316 
317 } __attribute__((packed));
318 #endif
319 
320 /*
321  * IO-bitmap sizes:
322  */
323 #define IO_BITMAP_BITS			65536
324 #define IO_BITMAP_BYTES			(IO_BITMAP_BITS/8)
325 #define IO_BITMAP_LONGS			(IO_BITMAP_BYTES/sizeof(long))
326 #define IO_BITMAP_OFFSET		offsetof(struct tss_struct, io_bitmap)
327 #define INVALID_IO_BITMAP_OFFSET	0x8000
328 
329 struct tss_struct {
330 	/*
331 	 * The hardware state:
332 	 */
333 	struct x86_hw_tss	x86_tss;
334 
335 	/*
336 	 * The extra 1 is there because the CPU will access an
337 	 * additional byte beyond the end of the IO permission
338 	 * bitmap. The extra byte must be all 1 bits, and must
339 	 * be within the limit.
340 	 */
341 	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];
342 
343 #ifdef CONFIG_X86_32
344 	/*
345 	 * Space for the temporary SYSENTER stack.
346 	 */
347 	unsigned long		SYSENTER_stack_canary;
348 	unsigned long		SYSENTER_stack[64];
349 #endif
350 
351 } ____cacheline_aligned;
352 
353 DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss);
354 
355 /*
356  * sizeof(unsigned long) coming from an extra "long" at the end
357  * of the iobitmap.
358  *
359  * -1? seg base+limit should be pointing to the address of the
360  * last valid byte
361  */
362 #define __KERNEL_TSS_LIMIT	\
363 	(IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)
364 
365 #ifdef CONFIG_X86_32
366 DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
367 #endif
368 
369 /*
370  * Save the original ist values for checking stack pointers during debugging
371  */
372 struct orig_ist {
373 	unsigned long		ist[7];
374 };
375 
376 #ifdef CONFIG_X86_64
377 DECLARE_PER_CPU(struct orig_ist, orig_ist);
378 
379 union irq_stack_union {
380 	char irq_stack[IRQ_STACK_SIZE];
381 	/*
382 	 * GCC hardcodes the stack canary as %gs:40.  Since the
383 	 * irq_stack is the object at %gs:0, we reserve the bottom
384 	 * 48 bytes of the irq stack for the canary.
385 	 */
386 	struct {
387 		char gs_base[40];
388 		unsigned long stack_canary;
389 	};
390 };
391 
392 DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;
393 DECLARE_INIT_PER_CPU(irq_stack_union);
394 
395 DECLARE_PER_CPU(char *, irq_stack_ptr);
396 DECLARE_PER_CPU(unsigned int, irq_count);
397 extern asmlinkage void ignore_sysret(void);
398 #else	/* X86_64 */
399 #ifdef CONFIG_CC_STACKPROTECTOR
400 /*
401  * Make sure stack canary segment base is cached-aligned:
402  *   "For Intel Atom processors, avoid non zero segment base address
403  *    that is not aligned to cache line boundary at all cost."
404  * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
405  */
406 struct stack_canary {
407 	char __pad[20];		/* canary at %gs:20 */
408 	unsigned long canary;
409 };
410 DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
411 #endif
412 /*
413  * per-CPU IRQ handling stacks
414  */
415 struct irq_stack {
416 	u32                     stack[THREAD_SIZE/sizeof(u32)];
417 } __aligned(THREAD_SIZE);
418 
419 DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
420 DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
421 #endif	/* X86_64 */
422 
423 extern unsigned int fpu_kernel_xstate_size;
424 extern unsigned int fpu_user_xstate_size;
425 
426 struct perf_event;
427 
428 typedef struct {
429 	unsigned long		seg;
430 } mm_segment_t;
431 
432 struct thread_struct {
433 	/* Cached TLS descriptors: */
434 	struct desc_struct	tls_array[GDT_ENTRY_TLS_ENTRIES];
435 #ifdef CONFIG_X86_32
436 	unsigned long		sp0;
437 #endif
438 	unsigned long		sp;
439 #ifdef CONFIG_X86_32
440 	unsigned long		sysenter_cs;
441 #else
442 	unsigned short		es;
443 	unsigned short		ds;
444 	unsigned short		fsindex;
445 	unsigned short		gsindex;
446 #endif
447 
448 	u32			status;		/* thread synchronous flags */
449 
450 #ifdef CONFIG_X86_64
451 	unsigned long		fsbase;
452 	unsigned long		gsbase;
453 #else
454 	/*
455 	 * XXX: this could presumably be unsigned short.  Alternatively,
456 	 * 32-bit kernels could be taught to use fsindex instead.
457 	 */
458 	unsigned long fs;
459 	unsigned long gs;
460 #endif
461 
462 	/* Save middle states of ptrace breakpoints */
463 	struct perf_event	*ptrace_bps[HBP_NUM];
464 	/* Debug status used for traps, single steps, etc... */
465 	unsigned long           debugreg6;
466 	/* Keep track of the exact dr7 value set by the user */
467 	unsigned long           ptrace_dr7;
468 	/* Fault info: */
469 	unsigned long		cr2;
470 	unsigned long		trap_nr;
471 	unsigned long		error_code;
472 #ifdef CONFIG_VM86
473 	/* Virtual 86 mode info */
474 	struct vm86		*vm86;
475 #endif
476 	/* IO permissions: */
477 	unsigned long		*io_bitmap_ptr;
478 	unsigned long		iopl;
479 	/* Max allowed port in the bitmap, in bytes: */
480 	unsigned		io_bitmap_max;
481 
482 	mm_segment_t		addr_limit;
483 
484 	unsigned int		sig_on_uaccess_err:1;
485 	unsigned int		uaccess_err:1;	/* uaccess failed */
486 
487 	/* Floating point and extended processor state */
488 	struct fpu		fpu;
489 	/*
490 	 * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
491 	 * the end.
492 	 */
493 };
494 
495 /*
496  * Thread-synchronous status.
497  *
498  * This is different from the flags in that nobody else
499  * ever touches our thread-synchronous status, so we don't
500  * have to worry about atomic accesses.
501  */
502 #define TS_COMPAT		0x0002	/* 32bit syscall active (64BIT)*/
503 
504 /*
505  * Set IOPL bits in EFLAGS from given mask
506  */
507 static inline void native_set_iopl_mask(unsigned mask)
508 {
509 #ifdef CONFIG_X86_32
510 	unsigned int reg;
511 
512 	asm volatile ("pushfl;"
513 		      "popl %0;"
514 		      "andl %1, %0;"
515 		      "orl %2, %0;"
516 		      "pushl %0;"
517 		      "popfl"
518 		      : "=&r" (reg)
519 		      : "i" (~X86_EFLAGS_IOPL), "r" (mask));
520 #endif
521 }
522 
523 static inline void
524 native_load_sp0(unsigned long sp0)
525 {
526 	this_cpu_write(cpu_tss.x86_tss.sp0, sp0);
527 }
528 
529 static inline void native_swapgs(void)
530 {
531 #ifdef CONFIG_X86_64
532 	asm volatile("swapgs" ::: "memory");
533 #endif
534 }
535 
536 static inline unsigned long current_top_of_stack(void)
537 {
538 #ifdef CONFIG_X86_64
539 	return this_cpu_read_stable(cpu_tss.x86_tss.sp0);
540 #else
541 	/* sp0 on x86_32 is special in and around vm86 mode. */
542 	return this_cpu_read_stable(cpu_current_top_of_stack);
543 #endif
544 }
545 
546 static inline bool on_thread_stack(void)
547 {
548 	return (unsigned long)(current_top_of_stack() -
549 			       current_stack_pointer) < THREAD_SIZE;
550 }
551 
552 #ifdef CONFIG_PARAVIRT
553 #include <asm/paravirt.h>
554 #else
555 #define __cpuid			native_cpuid
556 
557 static inline void load_sp0(unsigned long sp0)
558 {
559 	native_load_sp0(sp0);
560 }
561 
562 #define set_iopl_mask native_set_iopl_mask
563 #endif /* CONFIG_PARAVIRT */
564 
565 /* Free all resources held by a thread. */
566 extern void release_thread(struct task_struct *);
567 
568 unsigned long get_wchan(struct task_struct *p);
569 
570 /*
571  * Generic CPUID function
572  * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
573  * resulting in stale register contents being returned.
574  */
575 static inline void cpuid(unsigned int op,
576 			 unsigned int *eax, unsigned int *ebx,
577 			 unsigned int *ecx, unsigned int *edx)
578 {
579 	*eax = op;
580 	*ecx = 0;
581 	__cpuid(eax, ebx, ecx, edx);
582 }
583 
584 /* Some CPUID calls want 'count' to be placed in ecx */
585 static inline void cpuid_count(unsigned int op, int count,
586 			       unsigned int *eax, unsigned int *ebx,
587 			       unsigned int *ecx, unsigned int *edx)
588 {
589 	*eax = op;
590 	*ecx = count;
591 	__cpuid(eax, ebx, ecx, edx);
592 }
593 
594 /*
595  * CPUID functions returning a single datum
596  */
597 static inline unsigned int cpuid_eax(unsigned int op)
598 {
599 	unsigned int eax, ebx, ecx, edx;
600 
601 	cpuid(op, &eax, &ebx, &ecx, &edx);
602 
603 	return eax;
604 }
605 
606 static inline unsigned int cpuid_ebx(unsigned int op)
607 {
608 	unsigned int eax, ebx, ecx, edx;
609 
610 	cpuid(op, &eax, &ebx, &ecx, &edx);
611 
612 	return ebx;
613 }
614 
615 static inline unsigned int cpuid_ecx(unsigned int op)
616 {
617 	unsigned int eax, ebx, ecx, edx;
618 
619 	cpuid(op, &eax, &ebx, &ecx, &edx);
620 
621 	return ecx;
622 }
623 
624 static inline unsigned int cpuid_edx(unsigned int op)
625 {
626 	unsigned int eax, ebx, ecx, edx;
627 
628 	cpuid(op, &eax, &ebx, &ecx, &edx);
629 
630 	return edx;
631 }
632 
633 /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
634 static __always_inline void rep_nop(void)
635 {
636 	asm volatile("rep; nop" ::: "memory");
637 }
638 
639 static __always_inline void cpu_relax(void)
640 {
641 	rep_nop();
642 }
643 
644 /*
645  * This function forces the icache and prefetched instruction stream to
646  * catch up with reality in two very specific cases:
647  *
648  *  a) Text was modified using one virtual address and is about to be executed
649  *     from the same physical page at a different virtual address.
650  *
651  *  b) Text was modified on a different CPU, may subsequently be
652  *     executed on this CPU, and you want to make sure the new version
653  *     gets executed.  This generally means you're calling this in a IPI.
654  *
655  * If you're calling this for a different reason, you're probably doing
656  * it wrong.
657  */
658 static inline void sync_core(void)
659 {
660 	/*
661 	 * There are quite a few ways to do this.  IRET-to-self is nice
662 	 * because it works on every CPU, at any CPL (so it's compatible
663 	 * with paravirtualization), and it never exits to a hypervisor.
664 	 * The only down sides are that it's a bit slow (it seems to be
665 	 * a bit more than 2x slower than the fastest options) and that
666 	 * it unmasks NMIs.  The "push %cs" is needed because, in
667 	 * paravirtual environments, __KERNEL_CS may not be a valid CS
668 	 * value when we do IRET directly.
669 	 *
670 	 * In case NMI unmasking or performance ever becomes a problem,
671 	 * the next best option appears to be MOV-to-CR2 and an
672 	 * unconditional jump.  That sequence also works on all CPUs,
673 	 * but it will fault at CPL3 (i.e. Xen PV).
674 	 *
675 	 * CPUID is the conventional way, but it's nasty: it doesn't
676 	 * exist on some 486-like CPUs, and it usually exits to a
677 	 * hypervisor.
678 	 *
679 	 * Like all of Linux's memory ordering operations, this is a
680 	 * compiler barrier as well.
681 	 */
682 #ifdef CONFIG_X86_32
683 	asm volatile (
684 		"pushfl\n\t"
685 		"pushl %%cs\n\t"
686 		"pushl $1f\n\t"
687 		"iret\n\t"
688 		"1:"
689 		: ASM_CALL_CONSTRAINT : : "memory");
690 #else
691 	unsigned int tmp;
692 
693 	asm volatile (
694 		UNWIND_HINT_SAVE
695 		"mov %%ss, %0\n\t"
696 		"pushq %q0\n\t"
697 		"pushq %%rsp\n\t"
698 		"addq $8, (%%rsp)\n\t"
699 		"pushfq\n\t"
700 		"mov %%cs, %0\n\t"
701 		"pushq %q0\n\t"
702 		"pushq $1f\n\t"
703 		"iretq\n\t"
704 		UNWIND_HINT_RESTORE
705 		"1:"
706 		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
707 #endif
708 }
709 
710 extern void select_idle_routine(const struct cpuinfo_x86 *c);
711 extern void amd_e400_c1e_apic_setup(void);
712 
713 extern unsigned long		boot_option_idle_override;
714 
715 enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
716 			 IDLE_POLL};
717 
718 extern void enable_sep_cpu(void);
719 extern int sysenter_setup(void);
720 
721 extern void early_trap_init(void);
722 void early_trap_pf_init(void);
723 
724 /* Defined in head.S */
725 extern struct desc_ptr		early_gdt_descr;
726 
727 extern void cpu_set_gdt(int);
728 extern void switch_to_new_gdt(int);
729 extern void load_direct_gdt(int);
730 extern void load_fixmap_gdt(int);
731 extern void load_percpu_segment(int);
732 extern void cpu_init(void);
733 
734 static inline unsigned long get_debugctlmsr(void)
735 {
736 	unsigned long debugctlmsr = 0;
737 
738 #ifndef CONFIG_X86_DEBUGCTLMSR
739 	if (boot_cpu_data.x86 < 6)
740 		return 0;
741 #endif
742 	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
743 
744 	return debugctlmsr;
745 }
746 
747 static inline void update_debugctlmsr(unsigned long debugctlmsr)
748 {
749 #ifndef CONFIG_X86_DEBUGCTLMSR
750 	if (boot_cpu_data.x86 < 6)
751 		return;
752 #endif
753 	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
754 }
755 
756 extern void set_task_blockstep(struct task_struct *task, bool on);
757 
758 /* Boot loader type from the setup header: */
759 extern int			bootloader_type;
760 extern int			bootloader_version;
761 
762 extern char			ignore_fpu_irq;
763 
764 #define HAVE_ARCH_PICK_MMAP_LAYOUT 1
765 #define ARCH_HAS_PREFETCHW
766 #define ARCH_HAS_SPINLOCK_PREFETCH
767 
768 #ifdef CONFIG_X86_32
769 # define BASE_PREFETCH		""
770 # define ARCH_HAS_PREFETCH
771 #else
772 # define BASE_PREFETCH		"prefetcht0 %P1"
773 #endif
774 
775 /*
776  * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
777  *
778  * It's not worth to care about 3dnow prefetches for the K6
779  * because they are microcoded there and very slow.
780  */
781 static inline void prefetch(const void *x)
782 {
783 	alternative_input(BASE_PREFETCH, "prefetchnta %P1",
784 			  X86_FEATURE_XMM,
785 			  "m" (*(const char *)x));
786 }
787 
788 /*
789  * 3dnow prefetch to get an exclusive cache line.
790  * Useful for spinlocks to avoid one state transition in the
791  * cache coherency protocol:
792  */
793 static inline void prefetchw(const void *x)
794 {
795 	alternative_input(BASE_PREFETCH, "prefetchw %P1",
796 			  X86_FEATURE_3DNOWPREFETCH,
797 			  "m" (*(const char *)x));
798 }
799 
800 static inline void spin_lock_prefetch(const void *x)
801 {
802 	prefetchw(x);
803 }
804 
805 #define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
806 			   TOP_OF_KERNEL_STACK_PADDING)
807 
808 #define task_top_of_stack(task) ((unsigned long)(task_pt_regs(task) + 1))
809 
810 #define task_pt_regs(task) \
811 ({									\
812 	unsigned long __ptr = (unsigned long)task_stack_page(task);	\
813 	__ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;		\
814 	((struct pt_regs *)__ptr) - 1;					\
815 })
816 
817 #ifdef CONFIG_X86_32
818 /*
819  * User space process size: 3GB (default).
820  */
821 #define IA32_PAGE_OFFSET	PAGE_OFFSET
822 #define TASK_SIZE		PAGE_OFFSET
823 #define TASK_SIZE_LOW		TASK_SIZE
824 #define TASK_SIZE_MAX		TASK_SIZE
825 #define DEFAULT_MAP_WINDOW	TASK_SIZE
826 #define STACK_TOP		TASK_SIZE
827 #define STACK_TOP_MAX		STACK_TOP
828 
829 #define INIT_THREAD  {							  \
830 	.sp0			= TOP_OF_INIT_STACK,			  \
831 	.sysenter_cs		= __KERNEL_CS,				  \
832 	.io_bitmap_ptr		= NULL,					  \
833 	.addr_limit		= KERNEL_DS,				  \
834 }
835 
836 #define KSTK_ESP(task)		(task_pt_regs(task)->sp)
837 
838 #else
839 /*
840  * User space process size. 47bits minus one guard page.  The guard
841  * page is necessary on Intel CPUs: if a SYSCALL instruction is at
842  * the highest possible canonical userspace address, then that
843  * syscall will enter the kernel with a non-canonical return
844  * address, and SYSRET will explode dangerously.  We avoid this
845  * particular problem by preventing anything from being mapped
846  * at the maximum canonical address.
847  */
848 #define TASK_SIZE_MAX	((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)
849 
850 #define DEFAULT_MAP_WINDOW	((1UL << 47) - PAGE_SIZE)
851 
852 /* This decides where the kernel will search for a free chunk of vm
853  * space during mmap's.
854  */
855 #define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \
856 					0xc0000000 : 0xFFFFe000)
857 
858 #define TASK_SIZE_LOW		(test_thread_flag(TIF_ADDR32) ? \
859 					IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW)
860 #define TASK_SIZE		(test_thread_flag(TIF_ADDR32) ? \
861 					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
862 #define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
863 					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
864 
865 #define STACK_TOP		TASK_SIZE_LOW
866 #define STACK_TOP_MAX		TASK_SIZE_MAX
867 
868 #define INIT_THREAD  {						\
869 	.addr_limit		= KERNEL_DS,			\
870 }
871 
872 extern unsigned long KSTK_ESP(struct task_struct *task);
873 
874 #endif /* CONFIG_X86_64 */
875 
876 extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
877 					       unsigned long new_sp);
878 
879 /*
880  * This decides where the kernel will search for a free chunk of vm
881  * space during mmap's.
882  */
883 #define __TASK_UNMAPPED_BASE(task_size)	(PAGE_ALIGN(task_size / 3))
884 #define TASK_UNMAPPED_BASE		__TASK_UNMAPPED_BASE(TASK_SIZE_LOW)
885 
886 #define KSTK_EIP(task)		(task_pt_regs(task)->ip)
887 
888 /* Get/set a process' ability to use the timestamp counter instruction */
889 #define GET_TSC_CTL(adr)	get_tsc_mode((adr))
890 #define SET_TSC_CTL(val)	set_tsc_mode((val))
891 
892 extern int get_tsc_mode(unsigned long adr);
893 extern int set_tsc_mode(unsigned int val);
894 
895 DECLARE_PER_CPU(u64, msr_misc_features_shadow);
896 
897 /* Register/unregister a process' MPX related resource */
898 #define MPX_ENABLE_MANAGEMENT()	mpx_enable_management()
899 #define MPX_DISABLE_MANAGEMENT()	mpx_disable_management()
900 
901 #ifdef CONFIG_X86_INTEL_MPX
902 extern int mpx_enable_management(void);
903 extern int mpx_disable_management(void);
904 #else
905 static inline int mpx_enable_management(void)
906 {
907 	return -EINVAL;
908 }
909 static inline int mpx_disable_management(void)
910 {
911 	return -EINVAL;
912 }
913 #endif /* CONFIG_X86_INTEL_MPX */
914 
915 #ifdef CONFIG_CPU_SUP_AMD
916 extern u16 amd_get_nb_id(int cpu);
917 extern u32 amd_get_nodes_per_socket(void);
918 #else
919 static inline u16 amd_get_nb_id(int cpu)		{ return 0; }
920 static inline u32 amd_get_nodes_per_socket(void)	{ return 0; }
921 #endif
922 
923 static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
924 {
925 	uint32_t base, eax, signature[3];
926 
927 	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
928 		cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);
929 
930 		if (!memcmp(sig, signature, 12) &&
931 		    (leaves == 0 || ((eax - base) >= leaves)))
932 			return base;
933 	}
934 
935 	return 0;
936 }
937 
938 extern unsigned long arch_align_stack(unsigned long sp);
939 extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
940 
941 void default_idle(void);
942 #ifdef	CONFIG_XEN
943 bool xen_set_default_idle(void);
944 #else
945 #define xen_set_default_idle 0
946 #endif
947 
948 void stop_this_cpu(void *dummy);
949 void df_debug(struct pt_regs *regs, long error_code);
950 #endif /* _ASM_X86_PROCESSOR_H */
951