xref: /openbmc/linux/arch/x86/include/asm/processor.h (revision c1d3fb8a)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_PROCESSOR_H
3 #define _ASM_X86_PROCESSOR_H
4 
5 #include <asm/processor-flags.h>
6 
7 /* Forward declaration, a strange C thing */
8 struct task_struct;
9 struct mm_struct;
10 struct io_bitmap;
11 struct vm86;
12 
13 #include <asm/math_emu.h>
14 #include <asm/segment.h>
15 #include <asm/types.h>
16 #include <uapi/asm/sigcontext.h>
17 #include <asm/current.h>
18 #include <asm/cpufeatures.h>
19 #include <asm/page.h>
20 #include <asm/pgtable_types.h>
21 #include <asm/percpu.h>
22 #include <asm/msr.h>
23 #include <asm/desc_defs.h>
24 #include <asm/nops.h>
25 #include <asm/special_insns.h>
26 #include <asm/fpu/types.h>
27 #include <asm/unwind_hints.h>
28 
29 #include <linux/personality.h>
30 #include <linux/cache.h>
31 #include <linux/threads.h>
32 #include <linux/math64.h>
33 #include <linux/err.h>
34 #include <linux/irqflags.h>
35 #include <linux/mem_encrypt.h>
36 
37 /*
38  * We handle most unaligned accesses in hardware.  On the other hand
39  * unaligned DMA can be quite expensive on some Nehalem processors.
40  *
41  * Based on this we disable the IP header alignment in network drivers.
42  */
43 #define NET_IP_ALIGN	0
44 
45 #define HBP_NUM 4
46 
47 /*
48  * These alignment constraints are for performance in the vSMP case,
49  * but in the task_struct case we must also meet hardware imposed
50  * alignment requirements of the FPU state:
51  */
52 #ifdef CONFIG_X86_VSMP
53 # define ARCH_MIN_TASKALIGN		(1 << INTERNODE_CACHE_SHIFT)
54 # define ARCH_MIN_MMSTRUCT_ALIGN	(1 << INTERNODE_CACHE_SHIFT)
55 #else
56 # define ARCH_MIN_TASKALIGN		__alignof__(union fpregs_state)
57 # define ARCH_MIN_MMSTRUCT_ALIGN	0
58 #endif
59 
60 enum tlb_infos {
61 	ENTRIES,
62 	NR_INFO
63 };
64 
65 extern u16 __read_mostly tlb_lli_4k[NR_INFO];
66 extern u16 __read_mostly tlb_lli_2m[NR_INFO];
67 extern u16 __read_mostly tlb_lli_4m[NR_INFO];
68 extern u16 __read_mostly tlb_lld_4k[NR_INFO];
69 extern u16 __read_mostly tlb_lld_2m[NR_INFO];
70 extern u16 __read_mostly tlb_lld_4m[NR_INFO];
71 extern u16 __read_mostly tlb_lld_1g[NR_INFO];
72 
73 /*
74  *  CPU type and hardware bug flags. Kept separately for each CPU.
75  *  Members of this structure are referenced in head_32.S, so think twice
76  *  before touching them. [mj]
77  */
78 
79 struct cpuinfo_x86 {
80 	__u8			x86;		/* CPU family */
81 	__u8			x86_vendor;	/* CPU vendor */
82 	__u8			x86_model;
83 	__u8			x86_stepping;
84 #ifdef CONFIG_X86_64
85 	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
86 	int			x86_tlbsize;
87 #endif
88 	__u8			x86_virt_bits;
89 	__u8			x86_phys_bits;
90 	/* CPUID returned core id bits: */
91 	__u8			x86_coreid_bits;
92 	__u8			cu_id;
93 	/* Max extended CPUID function supported: */
94 	__u32			extended_cpuid_level;
95 	/* Maximum supported CPUID level, -1=no CPUID: */
96 	int			cpuid_level;
97 	/*
98 	 * Align to size of unsigned long because the x86_capability array
99 	 * is passed to bitops which require the alignment. Use unnamed
100 	 * union to enforce the array is aligned to size of unsigned long.
101 	 */
102 	union {
103 		__u32		x86_capability[NCAPINTS + NBUGINTS];
104 		unsigned long	x86_capability_alignment;
105 	};
106 	char			x86_vendor_id[16];
107 	char			x86_model_id[64];
108 	/* in KB - valid for CPUS which support this call: */
109 	unsigned int		x86_cache_size;
110 	int			x86_cache_alignment;	/* In bytes */
111 	/* Cache QoS architectural values: */
112 	int			x86_cache_max_rmid;	/* max index */
113 	int			x86_cache_occ_scale;	/* scale to bytes */
114 	int			x86_power;
115 	unsigned long		loops_per_jiffy;
116 	/* cpuid returned max cores value: */
117 	u16			x86_max_cores;
118 	u16			apicid;
119 	u16			initial_apicid;
120 	u16			x86_clflush_size;
121 	/* number of cores as seen by the OS: */
122 	u16			booted_cores;
123 	/* Physical processor id: */
124 	u16			phys_proc_id;
125 	/* Logical processor id: */
126 	u16			logical_proc_id;
127 	/* Core id: */
128 	u16			cpu_core_id;
129 	u16			cpu_die_id;
130 	u16			logical_die_id;
131 	/* Index into per_cpu list: */
132 	u16			cpu_index;
133 	u32			microcode;
134 	/* Address space bits used by the cache internally */
135 	u8			x86_cache_bits;
136 	unsigned		initialized : 1;
137 } __randomize_layout;
138 
139 struct cpuid_regs {
140 	u32 eax, ebx, ecx, edx;
141 };
142 
143 enum cpuid_regs_idx {
144 	CPUID_EAX = 0,
145 	CPUID_EBX,
146 	CPUID_ECX,
147 	CPUID_EDX,
148 };
149 
150 #define X86_VENDOR_INTEL	0
151 #define X86_VENDOR_CYRIX	1
152 #define X86_VENDOR_AMD		2
153 #define X86_VENDOR_UMC		3
154 #define X86_VENDOR_CENTAUR	5
155 #define X86_VENDOR_TRANSMETA	7
156 #define X86_VENDOR_NSC		8
157 #define X86_VENDOR_HYGON	9
158 #define X86_VENDOR_ZHAOXIN	10
159 #define X86_VENDOR_NUM		11
160 
161 #define X86_VENDOR_UNKNOWN	0xff
162 
163 /*
164  * capabilities of CPUs
165  */
166 extern struct cpuinfo_x86	boot_cpu_data;
167 extern struct cpuinfo_x86	new_cpu_data;
168 
169 extern __u32			cpu_caps_cleared[NCAPINTS + NBUGINTS];
170 extern __u32			cpu_caps_set[NCAPINTS + NBUGINTS];
171 
172 #ifdef CONFIG_SMP
173 DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
174 #define cpu_data(cpu)		per_cpu(cpu_info, cpu)
175 #else
176 #define cpu_info		boot_cpu_data
177 #define cpu_data(cpu)		boot_cpu_data
178 #endif
179 
180 extern const struct seq_operations cpuinfo_op;
181 
182 #define cache_line_size()	(boot_cpu_data.x86_cache_alignment)
183 
184 extern void cpu_detect(struct cpuinfo_x86 *c);
185 
186 static inline unsigned long long l1tf_pfn_limit(void)
187 {
188 	return BIT_ULL(boot_cpu_data.x86_cache_bits - 1 - PAGE_SHIFT);
189 }
190 
191 extern void early_cpu_init(void);
192 extern void identify_boot_cpu(void);
193 extern void identify_secondary_cpu(struct cpuinfo_x86 *);
194 extern void print_cpu_info(struct cpuinfo_x86 *);
195 void print_cpu_msr(struct cpuinfo_x86 *);
196 
197 #ifdef CONFIG_X86_32
198 extern int have_cpuid_p(void);
199 #else
200 static inline int have_cpuid_p(void)
201 {
202 	return 1;
203 }
204 #endif
205 static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
206 				unsigned int *ecx, unsigned int *edx)
207 {
208 	/* ecx is often an input as well as an output. */
209 	asm volatile("cpuid"
210 	    : "=a" (*eax),
211 	      "=b" (*ebx),
212 	      "=c" (*ecx),
213 	      "=d" (*edx)
214 	    : "0" (*eax), "2" (*ecx)
215 	    : "memory");
216 }
217 
218 #define native_cpuid_reg(reg)					\
219 static inline unsigned int native_cpuid_##reg(unsigned int op)	\
220 {								\
221 	unsigned int eax = op, ebx, ecx = 0, edx;		\
222 								\
223 	native_cpuid(&eax, &ebx, &ecx, &edx);			\
224 								\
225 	return reg;						\
226 }
227 
228 /*
229  * Native CPUID functions returning a single datum.
230  */
231 native_cpuid_reg(eax)
232 native_cpuid_reg(ebx)
233 native_cpuid_reg(ecx)
234 native_cpuid_reg(edx)
235 
236 /*
237  * Friendlier CR3 helpers.
238  */
239 static inline unsigned long read_cr3_pa(void)
240 {
241 	return __read_cr3() & CR3_ADDR_MASK;
242 }
243 
244 static inline unsigned long native_read_cr3_pa(void)
245 {
246 	return __native_read_cr3() & CR3_ADDR_MASK;
247 }
248 
249 static inline void load_cr3(pgd_t *pgdir)
250 {
251 	write_cr3(__sme_pa(pgdir));
252 }
253 
254 /*
255  * Note that while the legacy 'TSS' name comes from 'Task State Segment',
256  * on modern x86 CPUs the TSS also holds information important to 64-bit mode,
257  * unrelated to the task-switch mechanism:
258  */
259 #ifdef CONFIG_X86_32
260 /* This is the TSS defined by the hardware. */
261 struct x86_hw_tss {
262 	unsigned short		back_link, __blh;
263 	unsigned long		sp0;
264 	unsigned short		ss0, __ss0h;
265 	unsigned long		sp1;
266 
267 	/*
268 	 * We don't use ring 1, so ss1 is a convenient scratch space in
269 	 * the same cacheline as sp0.  We use ss1 to cache the value in
270 	 * MSR_IA32_SYSENTER_CS.  When we context switch
271 	 * MSR_IA32_SYSENTER_CS, we first check if the new value being
272 	 * written matches ss1, and, if it's not, then we wrmsr the new
273 	 * value and update ss1.
274 	 *
275 	 * The only reason we context switch MSR_IA32_SYSENTER_CS is
276 	 * that we set it to zero in vm86 tasks to avoid corrupting the
277 	 * stack if we were to go through the sysenter path from vm86
278 	 * mode.
279 	 */
280 	unsigned short		ss1;	/* MSR_IA32_SYSENTER_CS */
281 
282 	unsigned short		__ss1h;
283 	unsigned long		sp2;
284 	unsigned short		ss2, __ss2h;
285 	unsigned long		__cr3;
286 	unsigned long		ip;
287 	unsigned long		flags;
288 	unsigned long		ax;
289 	unsigned long		cx;
290 	unsigned long		dx;
291 	unsigned long		bx;
292 	unsigned long		sp;
293 	unsigned long		bp;
294 	unsigned long		si;
295 	unsigned long		di;
296 	unsigned short		es, __esh;
297 	unsigned short		cs, __csh;
298 	unsigned short		ss, __ssh;
299 	unsigned short		ds, __dsh;
300 	unsigned short		fs, __fsh;
301 	unsigned short		gs, __gsh;
302 	unsigned short		ldt, __ldth;
303 	unsigned short		trace;
304 	unsigned short		io_bitmap_base;
305 
306 } __attribute__((packed));
307 #else
308 struct x86_hw_tss {
309 	u32			reserved1;
310 	u64			sp0;
311 
312 	/*
313 	 * We store cpu_current_top_of_stack in sp1 so it's always accessible.
314 	 * Linux does not use ring 1, so sp1 is not otherwise needed.
315 	 */
316 	u64			sp1;
317 
318 	/*
319 	 * Since Linux does not use ring 2, the 'sp2' slot is unused by
320 	 * hardware.  entry_SYSCALL_64 uses it as scratch space to stash
321 	 * the user RSP value.
322 	 */
323 	u64			sp2;
324 
325 	u64			reserved2;
326 	u64			ist[7];
327 	u32			reserved3;
328 	u32			reserved4;
329 	u16			reserved5;
330 	u16			io_bitmap_base;
331 
332 } __attribute__((packed));
333 #endif
334 
335 /*
336  * IO-bitmap sizes:
337  */
338 #define IO_BITMAP_BITS			65536
339 #define IO_BITMAP_BYTES			(IO_BITMAP_BITS / BITS_PER_BYTE)
340 #define IO_BITMAP_LONGS			(IO_BITMAP_BYTES / sizeof(long))
341 
342 #define IO_BITMAP_OFFSET_VALID_MAP				\
343 	(offsetof(struct tss_struct, io_bitmap.bitmap) -	\
344 	 offsetof(struct tss_struct, x86_tss))
345 
346 #define IO_BITMAP_OFFSET_VALID_ALL				\
347 	(offsetof(struct tss_struct, io_bitmap.mapall) -	\
348 	 offsetof(struct tss_struct, x86_tss))
349 
350 #ifdef CONFIG_X86_IOPL_IOPERM
351 /*
352  * sizeof(unsigned long) coming from an extra "long" at the end of the
353  * iobitmap. The limit is inclusive, i.e. the last valid byte.
354  */
355 # define __KERNEL_TSS_LIMIT	\
356 	(IO_BITMAP_OFFSET_VALID_ALL + IO_BITMAP_BYTES + \
357 	 sizeof(unsigned long) - 1)
358 #else
359 # define __KERNEL_TSS_LIMIT	\
360 	(offsetof(struct tss_struct, x86_tss) + sizeof(struct x86_hw_tss) - 1)
361 #endif
362 
363 /* Base offset outside of TSS_LIMIT so unpriviledged IO causes #GP */
364 #define IO_BITMAP_OFFSET_INVALID	(__KERNEL_TSS_LIMIT + 1)
365 
366 struct entry_stack {
367 	unsigned long		words[64];
368 };
369 
370 struct entry_stack_page {
371 	struct entry_stack stack;
372 } __aligned(PAGE_SIZE);
373 
374 /*
375  * All IO bitmap related data stored in the TSS:
376  */
377 struct x86_io_bitmap {
378 	/* The sequence number of the last active bitmap. */
379 	u64			prev_sequence;
380 
381 	/*
382 	 * Store the dirty size of the last io bitmap offender. The next
383 	 * one will have to do the cleanup as the switch out to a non io
384 	 * bitmap user will just set x86_tss.io_bitmap_base to a value
385 	 * outside of the TSS limit. So for sane tasks there is no need to
386 	 * actually touch the io_bitmap at all.
387 	 */
388 	unsigned int		prev_max;
389 
390 	/*
391 	 * The extra 1 is there because the CPU will access an
392 	 * additional byte beyond the end of the IO permission
393 	 * bitmap. The extra byte must be all 1 bits, and must
394 	 * be within the limit.
395 	 */
396 	unsigned long		bitmap[IO_BITMAP_LONGS + 1];
397 
398 	/*
399 	 * Special I/O bitmap to emulate IOPL(3). All bytes zero,
400 	 * except the additional byte at the end.
401 	 */
402 	unsigned long		mapall[IO_BITMAP_LONGS + 1];
403 };
404 
405 struct tss_struct {
406 	/*
407 	 * The fixed hardware portion.  This must not cross a page boundary
408 	 * at risk of violating the SDM's advice and potentially triggering
409 	 * errata.
410 	 */
411 	struct x86_hw_tss	x86_tss;
412 
413 	struct x86_io_bitmap	io_bitmap;
414 } __aligned(PAGE_SIZE);
415 
416 DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw);
417 
418 /* Per CPU interrupt stacks */
419 struct irq_stack {
420 	char		stack[IRQ_STACK_SIZE];
421 } __aligned(IRQ_STACK_SIZE);
422 
423 DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
424 
425 #ifdef CONFIG_X86_32
426 DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
427 #else
428 /* The RO copy can't be accessed with this_cpu_xyz(), so use the RW copy. */
429 #define cpu_current_top_of_stack cpu_tss_rw.x86_tss.sp1
430 #endif
431 
432 #ifdef CONFIG_X86_64
433 struct fixed_percpu_data {
434 	/*
435 	 * GCC hardcodes the stack canary as %gs:40.  Since the
436 	 * irq_stack is the object at %gs:0, we reserve the bottom
437 	 * 48 bytes of the irq stack for the canary.
438 	 */
439 	char		gs_base[40];
440 	unsigned long	stack_canary;
441 };
442 
443 DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;
444 DECLARE_INIT_PER_CPU(fixed_percpu_data);
445 
446 static inline unsigned long cpu_kernelmode_gs_base(int cpu)
447 {
448 	return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
449 }
450 
451 DECLARE_PER_CPU(unsigned int, irq_count);
452 extern asmlinkage void ignore_sysret(void);
453 
454 #if IS_ENABLED(CONFIG_KVM)
455 /* Save actual FS/GS selectors and bases to current->thread */
456 void save_fsgs_for_kvm(void);
457 #endif
458 #else	/* X86_64 */
459 #ifdef CONFIG_STACKPROTECTOR
460 /*
461  * Make sure stack canary segment base is cached-aligned:
462  *   "For Intel Atom processors, avoid non zero segment base address
463  *    that is not aligned to cache line boundary at all cost."
464  * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
465  */
466 struct stack_canary {
467 	char __pad[20];		/* canary at %gs:20 */
468 	unsigned long canary;
469 };
470 DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
471 #endif
472 /* Per CPU softirq stack pointer */
473 DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
474 #endif	/* X86_64 */
475 
476 extern unsigned int fpu_kernel_xstate_size;
477 extern unsigned int fpu_user_xstate_size;
478 
479 struct perf_event;
480 
481 typedef struct {
482 	unsigned long		seg;
483 } mm_segment_t;
484 
485 struct thread_struct {
486 	/* Cached TLS descriptors: */
487 	struct desc_struct	tls_array[GDT_ENTRY_TLS_ENTRIES];
488 #ifdef CONFIG_X86_32
489 	unsigned long		sp0;
490 #endif
491 	unsigned long		sp;
492 #ifdef CONFIG_X86_32
493 	unsigned long		sysenter_cs;
494 #else
495 	unsigned short		es;
496 	unsigned short		ds;
497 	unsigned short		fsindex;
498 	unsigned short		gsindex;
499 #endif
500 
501 #ifdef CONFIG_X86_64
502 	unsigned long		fsbase;
503 	unsigned long		gsbase;
504 #else
505 	/*
506 	 * XXX: this could presumably be unsigned short.  Alternatively,
507 	 * 32-bit kernels could be taught to use fsindex instead.
508 	 */
509 	unsigned long fs;
510 	unsigned long gs;
511 #endif
512 
513 	/* Save middle states of ptrace breakpoints */
514 	struct perf_event	*ptrace_bps[HBP_NUM];
515 	/* Debug status used for traps, single steps, etc... */
516 	unsigned long           debugreg6;
517 	/* Keep track of the exact dr7 value set by the user */
518 	unsigned long           ptrace_dr7;
519 	/* Fault info: */
520 	unsigned long		cr2;
521 	unsigned long		trap_nr;
522 	unsigned long		error_code;
523 #ifdef CONFIG_VM86
524 	/* Virtual 86 mode info */
525 	struct vm86		*vm86;
526 #endif
527 	/* IO permissions: */
528 	struct io_bitmap	*io_bitmap;
529 
530 	/*
531 	 * IOPL. Priviledge level dependent I/O permission which is
532 	 * emulated via the I/O bitmap to prevent user space from disabling
533 	 * interrupts.
534 	 */
535 	unsigned long		iopl_emul;
536 
537 	mm_segment_t		addr_limit;
538 
539 	unsigned int		sig_on_uaccess_err:1;
540 	unsigned int		uaccess_err:1;	/* uaccess failed */
541 
542 	/* Floating point and extended processor state */
543 	struct fpu		fpu;
544 	/*
545 	 * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
546 	 * the end.
547 	 */
548 };
549 
550 /* Whitelist the FPU state from the task_struct for hardened usercopy. */
551 static inline void arch_thread_struct_whitelist(unsigned long *offset,
552 						unsigned long *size)
553 {
554 	*offset = offsetof(struct thread_struct, fpu.state);
555 	*size = fpu_kernel_xstate_size;
556 }
557 
558 /*
559  * Thread-synchronous status.
560  *
561  * This is different from the flags in that nobody else
562  * ever touches our thread-synchronous status, so we don't
563  * have to worry about atomic accesses.
564  */
565 #define TS_COMPAT		0x0002	/* 32bit syscall active (64BIT)*/
566 
567 static inline void
568 native_load_sp0(unsigned long sp0)
569 {
570 	this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
571 }
572 
573 static inline void native_swapgs(void)
574 {
575 #ifdef CONFIG_X86_64
576 	asm volatile("swapgs" ::: "memory");
577 #endif
578 }
579 
580 static inline unsigned long current_top_of_stack(void)
581 {
582 	/*
583 	 *  We can't read directly from tss.sp0: sp0 on x86_32 is special in
584 	 *  and around vm86 mode and sp0 on x86_64 is special because of the
585 	 *  entry trampoline.
586 	 */
587 	return this_cpu_read_stable(cpu_current_top_of_stack);
588 }
589 
590 static inline bool on_thread_stack(void)
591 {
592 	return (unsigned long)(current_top_of_stack() -
593 			       current_stack_pointer) < THREAD_SIZE;
594 }
595 
596 #ifdef CONFIG_PARAVIRT_XXL
597 #include <asm/paravirt.h>
598 #else
599 #define __cpuid			native_cpuid
600 
601 static inline void load_sp0(unsigned long sp0)
602 {
603 	native_load_sp0(sp0);
604 }
605 
606 #endif /* CONFIG_PARAVIRT_XXL */
607 
608 /* Free all resources held by a thread. */
609 extern void release_thread(struct task_struct *);
610 
611 unsigned long get_wchan(struct task_struct *p);
612 
613 /*
614  * Generic CPUID function
615  * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
616  * resulting in stale register contents being returned.
617  */
618 static inline void cpuid(unsigned int op,
619 			 unsigned int *eax, unsigned int *ebx,
620 			 unsigned int *ecx, unsigned int *edx)
621 {
622 	*eax = op;
623 	*ecx = 0;
624 	__cpuid(eax, ebx, ecx, edx);
625 }
626 
627 /* Some CPUID calls want 'count' to be placed in ecx */
628 static inline void cpuid_count(unsigned int op, int count,
629 			       unsigned int *eax, unsigned int *ebx,
630 			       unsigned int *ecx, unsigned int *edx)
631 {
632 	*eax = op;
633 	*ecx = count;
634 	__cpuid(eax, ebx, ecx, edx);
635 }
636 
637 /*
638  * CPUID functions returning a single datum
639  */
640 static inline unsigned int cpuid_eax(unsigned int op)
641 {
642 	unsigned int eax, ebx, ecx, edx;
643 
644 	cpuid(op, &eax, &ebx, &ecx, &edx);
645 
646 	return eax;
647 }
648 
649 static inline unsigned int cpuid_ebx(unsigned int op)
650 {
651 	unsigned int eax, ebx, ecx, edx;
652 
653 	cpuid(op, &eax, &ebx, &ecx, &edx);
654 
655 	return ebx;
656 }
657 
658 static inline unsigned int cpuid_ecx(unsigned int op)
659 {
660 	unsigned int eax, ebx, ecx, edx;
661 
662 	cpuid(op, &eax, &ebx, &ecx, &edx);
663 
664 	return ecx;
665 }
666 
667 static inline unsigned int cpuid_edx(unsigned int op)
668 {
669 	unsigned int eax, ebx, ecx, edx;
670 
671 	cpuid(op, &eax, &ebx, &ecx, &edx);
672 
673 	return edx;
674 }
675 
676 /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
677 static __always_inline void rep_nop(void)
678 {
679 	asm volatile("rep; nop" ::: "memory");
680 }
681 
682 static __always_inline void cpu_relax(void)
683 {
684 	rep_nop();
685 }
686 
687 /*
688  * This function forces the icache and prefetched instruction stream to
689  * catch up with reality in two very specific cases:
690  *
691  *  a) Text was modified using one virtual address and is about to be executed
692  *     from the same physical page at a different virtual address.
693  *
694  *  b) Text was modified on a different CPU, may subsequently be
695  *     executed on this CPU, and you want to make sure the new version
696  *     gets executed.  This generally means you're calling this in a IPI.
697  *
698  * If you're calling this for a different reason, you're probably doing
699  * it wrong.
700  */
701 static inline void sync_core(void)
702 {
703 	/*
704 	 * There are quite a few ways to do this.  IRET-to-self is nice
705 	 * because it works on every CPU, at any CPL (so it's compatible
706 	 * with paravirtualization), and it never exits to a hypervisor.
707 	 * The only down sides are that it's a bit slow (it seems to be
708 	 * a bit more than 2x slower than the fastest options) and that
709 	 * it unmasks NMIs.  The "push %cs" is needed because, in
710 	 * paravirtual environments, __KERNEL_CS may not be a valid CS
711 	 * value when we do IRET directly.
712 	 *
713 	 * In case NMI unmasking or performance ever becomes a problem,
714 	 * the next best option appears to be MOV-to-CR2 and an
715 	 * unconditional jump.  That sequence also works on all CPUs,
716 	 * but it will fault at CPL3 (i.e. Xen PV).
717 	 *
718 	 * CPUID is the conventional way, but it's nasty: it doesn't
719 	 * exist on some 486-like CPUs, and it usually exits to a
720 	 * hypervisor.
721 	 *
722 	 * Like all of Linux's memory ordering operations, this is a
723 	 * compiler barrier as well.
724 	 */
725 #ifdef CONFIG_X86_32
726 	asm volatile (
727 		"pushfl\n\t"
728 		"pushl %%cs\n\t"
729 		"pushl $1f\n\t"
730 		"iret\n\t"
731 		"1:"
732 		: ASM_CALL_CONSTRAINT : : "memory");
733 #else
734 	unsigned int tmp;
735 
736 	asm volatile (
737 		UNWIND_HINT_SAVE
738 		"mov %%ss, %0\n\t"
739 		"pushq %q0\n\t"
740 		"pushq %%rsp\n\t"
741 		"addq $8, (%%rsp)\n\t"
742 		"pushfq\n\t"
743 		"mov %%cs, %0\n\t"
744 		"pushq %q0\n\t"
745 		"pushq $1f\n\t"
746 		"iretq\n\t"
747 		UNWIND_HINT_RESTORE
748 		"1:"
749 		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
750 #endif
751 }
752 
753 extern void select_idle_routine(const struct cpuinfo_x86 *c);
754 extern void amd_e400_c1e_apic_setup(void);
755 
756 extern unsigned long		boot_option_idle_override;
757 
758 enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
759 			 IDLE_POLL};
760 
761 extern void enable_sep_cpu(void);
762 extern int sysenter_setup(void);
763 
764 
765 /* Defined in head.S */
766 extern struct desc_ptr		early_gdt_descr;
767 
768 extern void switch_to_new_gdt(int);
769 extern void load_direct_gdt(int);
770 extern void load_fixmap_gdt(int);
771 extern void load_percpu_segment(int);
772 extern void cpu_init(void);
773 extern void cr4_init(void);
774 
775 static inline unsigned long get_debugctlmsr(void)
776 {
777 	unsigned long debugctlmsr = 0;
778 
779 #ifndef CONFIG_X86_DEBUGCTLMSR
780 	if (boot_cpu_data.x86 < 6)
781 		return 0;
782 #endif
783 	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
784 
785 	return debugctlmsr;
786 }
787 
788 static inline void update_debugctlmsr(unsigned long debugctlmsr)
789 {
790 #ifndef CONFIG_X86_DEBUGCTLMSR
791 	if (boot_cpu_data.x86 < 6)
792 		return;
793 #endif
794 	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
795 }
796 
797 extern void set_task_blockstep(struct task_struct *task, bool on);
798 
799 /* Boot loader type from the setup header: */
800 extern int			bootloader_type;
801 extern int			bootloader_version;
802 
803 extern char			ignore_fpu_irq;
804 
805 #define HAVE_ARCH_PICK_MMAP_LAYOUT 1
806 #define ARCH_HAS_PREFETCHW
807 #define ARCH_HAS_SPINLOCK_PREFETCH
808 
809 #ifdef CONFIG_X86_32
810 # define BASE_PREFETCH		""
811 # define ARCH_HAS_PREFETCH
812 #else
813 # define BASE_PREFETCH		"prefetcht0 %P1"
814 #endif
815 
816 /*
817  * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
818  *
819  * It's not worth to care about 3dnow prefetches for the K6
820  * because they are microcoded there and very slow.
821  */
822 static inline void prefetch(const void *x)
823 {
824 	alternative_input(BASE_PREFETCH, "prefetchnta %P1",
825 			  X86_FEATURE_XMM,
826 			  "m" (*(const char *)x));
827 }
828 
829 /*
830  * 3dnow prefetch to get an exclusive cache line.
831  * Useful for spinlocks to avoid one state transition in the
832  * cache coherency protocol:
833  */
834 static inline void prefetchw(const void *x)
835 {
836 	alternative_input(BASE_PREFETCH, "prefetchw %P1",
837 			  X86_FEATURE_3DNOWPREFETCH,
838 			  "m" (*(const char *)x));
839 }
840 
841 static inline void spin_lock_prefetch(const void *x)
842 {
843 	prefetchw(x);
844 }
845 
846 #define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
847 			   TOP_OF_KERNEL_STACK_PADDING)
848 
849 #define task_top_of_stack(task) ((unsigned long)(task_pt_regs(task) + 1))
850 
851 #define task_pt_regs(task) \
852 ({									\
853 	unsigned long __ptr = (unsigned long)task_stack_page(task);	\
854 	__ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;		\
855 	((struct pt_regs *)__ptr) - 1;					\
856 })
857 
858 #ifdef CONFIG_X86_32
859 /*
860  * User space process size: 3GB (default).
861  */
862 #define IA32_PAGE_OFFSET	PAGE_OFFSET
863 #define TASK_SIZE		PAGE_OFFSET
864 #define TASK_SIZE_LOW		TASK_SIZE
865 #define TASK_SIZE_MAX		TASK_SIZE
866 #define DEFAULT_MAP_WINDOW	TASK_SIZE
867 #define STACK_TOP		TASK_SIZE
868 #define STACK_TOP_MAX		STACK_TOP
869 
870 #define INIT_THREAD  {							  \
871 	.sp0			= TOP_OF_INIT_STACK,			  \
872 	.sysenter_cs		= __KERNEL_CS,				  \
873 	.addr_limit		= KERNEL_DS,				  \
874 }
875 
876 #define KSTK_ESP(task)		(task_pt_regs(task)->sp)
877 
878 #else
879 /*
880  * User space process size.  This is the first address outside the user range.
881  * There are a few constraints that determine this:
882  *
883  * On Intel CPUs, if a SYSCALL instruction is at the highest canonical
884  * address, then that syscall will enter the kernel with a
885  * non-canonical return address, and SYSRET will explode dangerously.
886  * We avoid this particular problem by preventing anything executable
887  * from being mapped at the maximum canonical address.
888  *
889  * On AMD CPUs in the Ryzen family, there's a nasty bug in which the
890  * CPUs malfunction if they execute code from the highest canonical page.
891  * They'll speculate right off the end of the canonical space, and
892  * bad things happen.  This is worked around in the same way as the
893  * Intel problem.
894  *
895  * With page table isolation enabled, we map the LDT in ... [stay tuned]
896  */
897 #define TASK_SIZE_MAX	((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)
898 
899 #define DEFAULT_MAP_WINDOW	((1UL << 47) - PAGE_SIZE)
900 
901 /* This decides where the kernel will search for a free chunk of vm
902  * space during mmap's.
903  */
904 #define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \
905 					0xc0000000 : 0xFFFFe000)
906 
907 #define TASK_SIZE_LOW		(test_thread_flag(TIF_ADDR32) ? \
908 					IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW)
909 #define TASK_SIZE		(test_thread_flag(TIF_ADDR32) ? \
910 					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
911 #define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
912 					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
913 
914 #define STACK_TOP		TASK_SIZE_LOW
915 #define STACK_TOP_MAX		TASK_SIZE_MAX
916 
917 #define INIT_THREAD  {						\
918 	.addr_limit		= KERNEL_DS,			\
919 }
920 
921 extern unsigned long KSTK_ESP(struct task_struct *task);
922 
923 #endif /* CONFIG_X86_64 */
924 
925 extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
926 					       unsigned long new_sp);
927 
928 /*
929  * This decides where the kernel will search for a free chunk of vm
930  * space during mmap's.
931  */
932 #define __TASK_UNMAPPED_BASE(task_size)	(PAGE_ALIGN(task_size / 3))
933 #define TASK_UNMAPPED_BASE		__TASK_UNMAPPED_BASE(TASK_SIZE_LOW)
934 
935 #define KSTK_EIP(task)		(task_pt_regs(task)->ip)
936 
937 /* Get/set a process' ability to use the timestamp counter instruction */
938 #define GET_TSC_CTL(adr)	get_tsc_mode((adr))
939 #define SET_TSC_CTL(val)	set_tsc_mode((val))
940 
941 extern int get_tsc_mode(unsigned long adr);
942 extern int set_tsc_mode(unsigned int val);
943 
944 DECLARE_PER_CPU(u64, msr_misc_features_shadow);
945 
946 /* Register/unregister a process' MPX related resource */
947 #define MPX_ENABLE_MANAGEMENT()	mpx_enable_management()
948 #define MPX_DISABLE_MANAGEMENT()	mpx_disable_management()
949 
950 #ifdef CONFIG_X86_INTEL_MPX
951 extern int mpx_enable_management(void);
952 extern int mpx_disable_management(void);
953 #else
954 static inline int mpx_enable_management(void)
955 {
956 	return -EINVAL;
957 }
958 static inline int mpx_disable_management(void)
959 {
960 	return -EINVAL;
961 }
962 #endif /* CONFIG_X86_INTEL_MPX */
963 
964 #ifdef CONFIG_CPU_SUP_AMD
965 extern u16 amd_get_nb_id(int cpu);
966 extern u32 amd_get_nodes_per_socket(void);
967 #else
968 static inline u16 amd_get_nb_id(int cpu)		{ return 0; }
969 static inline u32 amd_get_nodes_per_socket(void)	{ return 0; }
970 #endif
971 
972 static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
973 {
974 	uint32_t base, eax, signature[3];
975 
976 	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
977 		cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);
978 
979 		if (!memcmp(sig, signature, 12) &&
980 		    (leaves == 0 || ((eax - base) >= leaves)))
981 			return base;
982 	}
983 
984 	return 0;
985 }
986 
987 extern unsigned long arch_align_stack(unsigned long sp);
988 void free_init_pages(const char *what, unsigned long begin, unsigned long end);
989 extern void free_kernel_image_pages(const char *what, void *begin, void *end);
990 
991 void default_idle(void);
992 #ifdef	CONFIG_XEN
993 bool xen_set_default_idle(void);
994 #else
995 #define xen_set_default_idle 0
996 #endif
997 
998 void stop_this_cpu(void *dummy);
999 void microcode_check(void);
1000 
1001 enum l1tf_mitigations {
1002 	L1TF_MITIGATION_OFF,
1003 	L1TF_MITIGATION_FLUSH_NOWARN,
1004 	L1TF_MITIGATION_FLUSH,
1005 	L1TF_MITIGATION_FLUSH_NOSMT,
1006 	L1TF_MITIGATION_FULL,
1007 	L1TF_MITIGATION_FULL_FORCE
1008 };
1009 
1010 extern enum l1tf_mitigations l1tf_mitigation;
1011 
1012 enum mds_mitigations {
1013 	MDS_MITIGATION_OFF,
1014 	MDS_MITIGATION_FULL,
1015 	MDS_MITIGATION_VMWERV,
1016 };
1017 
1018 enum taa_mitigations {
1019 	TAA_MITIGATION_OFF,
1020 	TAA_MITIGATION_UCODE_NEEDED,
1021 	TAA_MITIGATION_VERW,
1022 	TAA_MITIGATION_TSX_DISABLED,
1023 };
1024 
1025 #endif /* _ASM_X86_PROCESSOR_H */
1026