1fd534e9bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
2b1df07bdSGlauber de Oliveira Costa /* Paravirtualization interfaces
3b1df07bdSGlauber de Oliveira Costa Copyright (C) 2006 Rusty Russell IBM Corporation
4b1df07bdSGlauber de Oliveira Costa
5b1df07bdSGlauber de Oliveira Costa
6b1df07bdSGlauber de Oliveira Costa 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc
7b1df07bdSGlauber de Oliveira Costa */
8b1df07bdSGlauber de Oliveira Costa
9b1df07bdSGlauber de Oliveira Costa #include <linux/errno.h>
10186f4360SPaul Gortmaker #include <linux/init.h>
11186f4360SPaul Gortmaker #include <linux/export.h>
12b1df07bdSGlauber de Oliveira Costa #include <linux/efi.h>
13b1df07bdSGlauber de Oliveira Costa #include <linux/bcd.h>
14b1df07bdSGlauber de Oliveira Costa #include <linux/highmem.h>
15376e2424SMasami Hiramatsu #include <linux/kprobes.h>
1665fddcfcSMike Rapoport #include <linux/pgtable.h>
17a0e2bf7cSJuergen Gross #include <linux/static_call.h>
18b1df07bdSGlauber de Oliveira Costa
19b1df07bdSGlauber de Oliveira Costa #include <asm/bug.h>
20b1df07bdSGlauber de Oliveira Costa #include <asm/paravirt.h>
2150af5eadSPaul Gortmaker #include <asm/debugreg.h>
22b1df07bdSGlauber de Oliveira Costa #include <asm/desc.h>
23b1df07bdSGlauber de Oliveira Costa #include <asm/setup.h>
24b1df07bdSGlauber de Oliveira Costa #include <asm/time.h>
25eba0045fSJeremy Fitzhardinge #include <asm/pgalloc.h>
26b1df07bdSGlauber de Oliveira Costa #include <asm/irq.h>
27b1df07bdSGlauber de Oliveira Costa #include <asm/delay.h>
28b1df07bdSGlauber de Oliveira Costa #include <asm/fixmap.h>
29b1df07bdSGlauber de Oliveira Costa #include <asm/apic.h>
30b1df07bdSGlauber de Oliveira Costa #include <asm/tlbflush.h>
31b1df07bdSGlauber de Oliveira Costa #include <asm/timer.h>
32f05e798aSDavid Howells #include <asm/special_insns.h>
3348a8b97cSPeter Zijlstra #include <asm/tlb.h>
3499bcd4a6SJuergen Gross #include <asm/io_bitmap.h>
35ae53fa18SH. Peter Anvin (Intel) #include <asm/gsseg.h>
36b1df07bdSGlauber de Oliveira Costa
37fc57a7c6SAndy Lutomirski /*
38fc57a7c6SAndy Lutomirski * nop stub, which must not clobber anything *including the stack* to
39fc57a7c6SAndy Lutomirski * avoid confusing the entry prologues.
40fc57a7c6SAndy Lutomirski */
41f1a033ccSJuergen Gross DEFINE_PARAVIRT_ASM(_paravirt_nop, "", .entry.text);
42b1df07bdSGlauber de Oliveira Costa
43e453f872SJuergen Gross /* stub always returning 0. */
44f1a033ccSJuergen Gross DEFINE_PARAVIRT_ASM(paravirt_ret0, "xor %eax,%eax", .entry.text);
45e453f872SJuergen Gross
default_banner(void)466f30c1acSThomas Gleixner void __init default_banner(void)
47b1df07bdSGlauber de Oliveira Costa {
48b1df07bdSGlauber de Oliveira Costa printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
49b1df07bdSGlauber de Oliveira Costa pv_info.name);
50b1df07bdSGlauber de Oliveira Costa }
51b1df07bdSGlauber de Oliveira Costa
52b1df07bdSGlauber de Oliveira Costa /* Undefined instruction for dealing with missing ops pointers. */
paravirt_BUG(void)53e453f872SJuergen Gross noinstr void paravirt_BUG(void)
54fafe5e74SJuergen Gross {
55fafe5e74SJuergen Gross BUG();
56fafe5e74SJuergen Gross }
57b1df07bdSGlauber de Oliveira Costa
paravirt_patch_call(void * insn_buff,const void * target,unsigned long addr,unsigned len)581fc654cfSIngo Molnar static unsigned paravirt_patch_call(void *insn_buff, const void *target,
59abc745f8SJuergen Gross unsigned long addr, unsigned len)
60b1df07bdSGlauber de Oliveira Costa {
61ba27d1a8SPeter Zijlstra __text_gen_insn(insn_buff, CALL_INSN_OPCODE,
62ba27d1a8SPeter Zijlstra (void *)addr, target, CALL_INSN_SIZE);
63ba27d1a8SPeter Zijlstra return CALL_INSN_SIZE;
64b1df07bdSGlauber de Oliveira Costa }
65b1df07bdSGlauber de Oliveira Costa
669bad5658SJuergen Gross #ifdef CONFIG_PARAVIRT_XXL
6711af36cbSJuergen Gross DEFINE_PARAVIRT_ASM(_paravirt_ident_64, "mov %rdi, %rax", .text);
6811af36cbSJuergen Gross DEFINE_PARAVIRT_ASM(pv_native_save_fl, "pushf; pop %rax", .noinstr.text);
6911af36cbSJuergen Gross DEFINE_PARAVIRT_ASM(pv_native_irq_disable, "cli", .noinstr.text);
7011af36cbSJuergen Gross DEFINE_PARAVIRT_ASM(pv_native_irq_enable, "sti", .noinstr.text);
7111af36cbSJuergen Gross DEFINE_PARAVIRT_ASM(pv_native_read_cr2, "mov %cr2, %rax", .noinstr.text);
729bad5658SJuergen Gross #endif
73b1df07bdSGlauber de Oliveira Costa
749043442bSJuergen Gross DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key);
759043442bSJuergen Gross
native_pv_lock_init(void)769043442bSJuergen Gross void __init native_pv_lock_init(void)
779043442bSJuergen Gross {
78ce0a1b60SArnd Bergmann if (IS_ENABLED(CONFIG_PARAVIRT_SPINLOCKS) &&
79ce0a1b60SArnd Bergmann !boot_cpu_has(X86_FEATURE_HYPERVISOR))
809043442bSJuergen Gross static_branch_disable(&virt_spin_lock_key);
819043442bSJuergen Gross }
829043442bSJuergen Gross
native_tlb_remove_table(struct mmu_gather * tlb,void * table)83*fcce1c6cSKees Cook static void native_tlb_remove_table(struct mmu_gather *tlb, void *table)
84*fcce1c6cSKees Cook {
85*fcce1c6cSKees Cook tlb_remove_page(tlb, table);
86*fcce1c6cSKees Cook }
87*fcce1c6cSKees Cook
paravirt_patch(u8 type,void * insn_buff,unsigned long addr,unsigned int len)88054ac8adSJuergen Gross unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr,
89054ac8adSJuergen Gross unsigned int len)
905c83511bSJuergen Gross {
919043442bSJuergen Gross /*
929043442bSJuergen Gross * Neat trick to map patch type back to the call within the
939043442bSJuergen Gross * corresponding structure.
949043442bSJuergen Gross */
955c83511bSJuergen Gross void *opfunc = *((void **)&pv_ops + type);
96b1df07bdSGlauber de Oliveira Costa unsigned ret;
97b1df07bdSGlauber de Oliveira Costa
98b1df07bdSGlauber de Oliveira Costa if (opfunc == NULL)
99fafe5e74SJuergen Gross /* If there's no function, patch it with paravirt_BUG() */
100fafe5e74SJuergen Gross ret = paravirt_patch_call(insn_buff, paravirt_BUG, addr, len);
10141edafdbSJeremy Fitzhardinge else if (opfunc == _paravirt_nop)
10279f1d836SBorislav Petkov ret = 0;
103b1df07bdSGlauber de Oliveira Costa else
104abc745f8SJuergen Gross /* Otherwise call the function. */
1051fc654cfSIngo Molnar ret = paravirt_patch_call(insn_buff, opfunc, addr, len);
106b1df07bdSGlauber de Oliveira Costa
107b1df07bdSGlauber de Oliveira Costa return ret;
108b1df07bdSGlauber de Oliveira Costa }
109b1df07bdSGlauber de Oliveira Costa
110c5905afbSIngo Molnar struct static_key paravirt_steal_enabled;
111c5905afbSIngo Molnar struct static_key paravirt_steal_rq_enabled;
1123c404b57SGlauber Costa
native_steal_clock(int cpu)1133c404b57SGlauber Costa static u64 native_steal_clock(int cpu)
1143c404b57SGlauber Costa {
1153c404b57SGlauber Costa return 0;
1163c404b57SGlauber Costa }
1173c404b57SGlauber Costa
118a0e2bf7cSJuergen Gross DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
119a0e2bf7cSJuergen Gross DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock);
120a0e2bf7cSJuergen Gross
paravirt_set_sched_clock(u64 (* func)(void))121a0e2bf7cSJuergen Gross void paravirt_set_sched_clock(u64 (*func)(void))
122a0e2bf7cSJuergen Gross {
123a0e2bf7cSJuergen Gross static_call_update(pv_sched_clock, func);
124a0e2bf7cSJuergen Gross }
125a0e2bf7cSJuergen Gross
126b1df07bdSGlauber de Oliveira Costa /* These are in entry.S */
127b1df07bdSGlauber de Oliveira Costa static struct resource reserve_ioports = {
128b1df07bdSGlauber de Oliveira Costa .start = 0,
129b1df07bdSGlauber de Oliveira Costa .end = IO_SPACE_LIMIT,
130b1df07bdSGlauber de Oliveira Costa .name = "paravirt-ioport",
131b1df07bdSGlauber de Oliveira Costa .flags = IORESOURCE_IO | IORESOURCE_BUSY,
132b1df07bdSGlauber de Oliveira Costa };
133b1df07bdSGlauber de Oliveira Costa
134b1df07bdSGlauber de Oliveira Costa /*
135b1df07bdSGlauber de Oliveira Costa * Reserve the whole legacy IO space to prevent any legacy drivers
136b1df07bdSGlauber de Oliveira Costa * from wasting time probing for their hardware. This is a fairly
137b1df07bdSGlauber de Oliveira Costa * brute-force approach to disabling all non-virtual drivers.
138b1df07bdSGlauber de Oliveira Costa *
139b1df07bdSGlauber de Oliveira Costa * Note that this must be called very early to have any effect.
140b1df07bdSGlauber de Oliveira Costa */
paravirt_disable_iospace(void)141b1df07bdSGlauber de Oliveira Costa int paravirt_disable_iospace(void)
142b1df07bdSGlauber de Oliveira Costa {
143f7743fe6SJeremy Fitzhardinge return request_resource(&ioport_resource, &reserve_ioports);
144b1df07bdSGlauber de Oliveira Costa }
145b1df07bdSGlauber de Oliveira Costa
1469bad5658SJuergen Gross #ifdef CONFIG_PARAVIRT_XXL
pv_native_write_cr2(unsigned long val)147209cfd0cSPeter Zijlstra static noinstr void pv_native_write_cr2(unsigned long val)
148209cfd0cSPeter Zijlstra {
149209cfd0cSPeter Zijlstra native_write_cr2(val);
150209cfd0cSPeter Zijlstra }
151f4afb713SPeter Zijlstra
pv_native_get_debugreg(int regno)152f4afb713SPeter Zijlstra static noinstr unsigned long pv_native_get_debugreg(int regno)
153f4afb713SPeter Zijlstra {
154f4afb713SPeter Zijlstra return native_get_debugreg(regno);
155f4afb713SPeter Zijlstra }
1567361fac0SPeter Zijlstra
pv_native_set_debugreg(int regno,unsigned long val)1577361fac0SPeter Zijlstra static noinstr void pv_native_set_debugreg(int regno, unsigned long val)
1587361fac0SPeter Zijlstra {
1597361fac0SPeter Zijlstra native_set_debugreg(regno, val);
1607361fac0SPeter Zijlstra }
161d7bfc7d5SPeter Zijlstra
pv_native_wbinvd(void)16210a09940SPeter Zijlstra noinstr void pv_native_wbinvd(void)
16310a09940SPeter Zijlstra {
16410a09940SPeter Zijlstra native_wbinvd();
16510a09940SPeter Zijlstra }
16610a09940SPeter Zijlstra
pv_native_safe_halt(void)16710a09940SPeter Zijlstra static noinstr void pv_native_safe_halt(void)
16810a09940SPeter Zijlstra {
16910a09940SPeter Zijlstra native_safe_halt();
17010a09940SPeter Zijlstra }
1719bad5658SJuergen Gross #endif
172b1df07bdSGlauber de Oliveira Costa
173b1df07bdSGlauber de Oliveira Costa struct pv_info pv_info = {
174b1df07bdSGlauber de Oliveira Costa .name = "bare hardware",
17540181646SJuergen Gross #ifdef CONFIG_PARAVIRT_XXL
176318f5a2aSAndy Lutomirski .extra_user_64bit_cs = __USER_CS,
177318f5a2aSAndy Lutomirski #endif
178b1df07bdSGlauber de Oliveira Costa };
179b1df07bdSGlauber de Oliveira Costa
18041edafdbSJeremy Fitzhardinge /* 64-bit pagetable entries */
181da5de7c2SJeremy Fitzhardinge #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
18241edafdbSJeremy Fitzhardinge
1835c83511bSJuergen Gross struct paravirt_patch_template pv_ops = {
1845c83511bSJuergen Gross /* Cpu ops. */
1859bad5658SJuergen Gross .cpu.io_delay = native_io_delay,
186b1df07bdSGlauber de Oliveira Costa
1879bad5658SJuergen Gross #ifdef CONFIG_PARAVIRT_XXL
1885c83511bSJuergen Gross .cpu.cpuid = native_cpuid,
189f4afb713SPeter Zijlstra .cpu.get_debugreg = pv_native_get_debugreg,
1907361fac0SPeter Zijlstra .cpu.set_debugreg = pv_native_set_debugreg,
1915c83511bSJuergen Gross .cpu.read_cr0 = native_read_cr0,
1925c83511bSJuergen Gross .cpu.write_cr0 = native_write_cr0,
1935c83511bSJuergen Gross .cpu.write_cr4 = native_write_cr4,
19410a09940SPeter Zijlstra .cpu.wbinvd = pv_native_wbinvd,
1955c83511bSJuergen Gross .cpu.read_msr = native_read_msr,
1965c83511bSJuergen Gross .cpu.write_msr = native_write_msr,
1975c83511bSJuergen Gross .cpu.read_msr_safe = native_read_msr_safe,
1985c83511bSJuergen Gross .cpu.write_msr_safe = native_write_msr_safe,
1995c83511bSJuergen Gross .cpu.read_pmc = native_read_pmc,
2005c83511bSJuergen Gross .cpu.load_tr_desc = native_load_tr_desc,
2015c83511bSJuergen Gross .cpu.set_ldt = native_set_ldt,
2025c83511bSJuergen Gross .cpu.load_gdt = native_load_gdt,
2035c83511bSJuergen Gross .cpu.load_idt = native_load_idt,
2045c83511bSJuergen Gross .cpu.store_tr = native_store_tr,
2055c83511bSJuergen Gross .cpu.load_tls = native_load_tls,
2065c83511bSJuergen Gross .cpu.load_gs_index = native_load_gs_index,
2075c83511bSJuergen Gross .cpu.write_ldt_entry = native_write_ldt_entry,
2085c83511bSJuergen Gross .cpu.write_gdt_entry = native_write_gdt_entry,
2095c83511bSJuergen Gross .cpu.write_idt_entry = native_write_idt_entry,
210eba0045fSJeremy Fitzhardinge
2115c83511bSJuergen Gross .cpu.alloc_ldt = paravirt_nop,
2125c83511bSJuergen Gross .cpu.free_ldt = paravirt_nop,
213b1df07bdSGlauber de Oliveira Costa
2145c83511bSJuergen Gross .cpu.load_sp0 = native_load_sp0,
215b1df07bdSGlauber de Oliveira Costa
21699bcd4a6SJuergen Gross #ifdef CONFIG_X86_IOPL_IOPERM
217cadfad87SAndy Lutomirski .cpu.invalidate_io_bitmap = native_tss_invalidate_io_bitmap,
21899bcd4a6SJuergen Gross .cpu.update_io_bitmap = native_tss_update_io_bitmap,
21999bcd4a6SJuergen Gross #endif
22099bcd4a6SJuergen Gross
2215c83511bSJuergen Gross .cpu.start_context_switch = paravirt_nop,
2225c83511bSJuergen Gross .cpu.end_context_switch = paravirt_nop,
2235c83511bSJuergen Gross
2245c83511bSJuergen Gross /* Irq ops. */
22511af36cbSJuergen Gross .irq.save_fl = __PV_IS_CALLEE_SAVE(pv_native_save_fl),
22609c41307SPeter Zijlstra .irq.irq_disable = __PV_IS_CALLEE_SAVE(pv_native_irq_disable),
227d7bfc7d5SPeter Zijlstra .irq.irq_enable = __PV_IS_CALLEE_SAVE(pv_native_irq_enable),
22810a09940SPeter Zijlstra .irq.safe_halt = pv_native_safe_halt,
2295c83511bSJuergen Gross .irq.halt = native_halt,
2306da63eb2SJuergen Gross #endif /* CONFIG_PARAVIRT_XXL */
2315c83511bSJuergen Gross
2325c83511bSJuergen Gross /* Mmu ops. */
2332faf153bSThomas Gleixner .mmu.flush_tlb_user = native_flush_tlb_local,
2345c83511bSJuergen Gross .mmu.flush_tlb_kernel = native_flush_tlb_global,
2355c83511bSJuergen Gross .mmu.flush_tlb_one_user = native_flush_tlb_one_user,
2364ce94eabSNadav Amit .mmu.flush_tlb_multi = native_flush_tlb_multi,
237*fcce1c6cSKees Cook .mmu.tlb_remove_table = native_tlb_remove_table,
2385c83511bSJuergen Gross
239fdc0269eSJuergen Gross .mmu.exit_mmap = paravirt_nop,
240064ce6c5SBrijesh Singh .mmu.notify_page_enc_status_changed = paravirt_nop,
241fdc0269eSJuergen Gross
242fdc0269eSJuergen Gross #ifdef CONFIG_PARAVIRT_XXL
2430a53c9acSPeter Zijlstra .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(pv_native_read_cr2),
244209cfd0cSPeter Zijlstra .mmu.write_cr2 = pv_native_write_cr2,
245fdc0269eSJuergen Gross .mmu.read_cr3 = __native_read_cr3,
246fdc0269eSJuergen Gross .mmu.write_cr3 = native_write_cr3,
247fdc0269eSJuergen Gross
2485c83511bSJuergen Gross .mmu.pgd_alloc = __paravirt_pgd_alloc,
2495c83511bSJuergen Gross .mmu.pgd_free = paravirt_nop,
2505c83511bSJuergen Gross
2515c83511bSJuergen Gross .mmu.alloc_pte = paravirt_nop,
2525c83511bSJuergen Gross .mmu.alloc_pmd = paravirt_nop,
2535c83511bSJuergen Gross .mmu.alloc_pud = paravirt_nop,
2545c83511bSJuergen Gross .mmu.alloc_p4d = paravirt_nop,
2555c83511bSJuergen Gross .mmu.release_pte = paravirt_nop,
2565c83511bSJuergen Gross .mmu.release_pmd = paravirt_nop,
2575c83511bSJuergen Gross .mmu.release_pud = paravirt_nop,
2585c83511bSJuergen Gross .mmu.release_p4d = paravirt_nop,
2595c83511bSJuergen Gross
2605c83511bSJuergen Gross .mmu.set_pte = native_set_pte,
2615c83511bSJuergen Gross .mmu.set_pmd = native_set_pmd,
2625c83511bSJuergen Gross
2635c83511bSJuergen Gross .mmu.ptep_modify_prot_start = __ptep_modify_prot_start,
2645c83511bSJuergen Gross .mmu.ptep_modify_prot_commit = __ptep_modify_prot_commit,
26508b882c6SJeremy Fitzhardinge
2665c83511bSJuergen Gross .mmu.set_pud = native_set_pud,
267da5de7c2SJeremy Fitzhardinge
2685c83511bSJuergen Gross .mmu.pmd_val = PTE_IDENT,
2695c83511bSJuergen Gross .mmu.make_pmd = PTE_IDENT,
270f95f2f7bSEduardo Habkost
2715c83511bSJuergen Gross .mmu.pud_val = PTE_IDENT,
2725c83511bSJuergen Gross .mmu.make_pud = PTE_IDENT,
273da5de7c2SJeremy Fitzhardinge
2745c83511bSJuergen Gross .mmu.set_p4d = native_set_p4d,
275f2a6a705SKirill A. Shutemov
276f2a6a705SKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS >= 5
2775c83511bSJuergen Gross .mmu.p4d_val = PTE_IDENT,
2785c83511bSJuergen Gross .mmu.make_p4d = PTE_IDENT,
279335437fbSKirill A. Shutemov
2805c83511bSJuergen Gross .mmu.set_pgd = native_set_pgd,
281335437fbSKirill A. Shutemov #endif /* CONFIG_PGTABLE_LEVELS >= 5 */
282b1df07bdSGlauber de Oliveira Costa
2835c83511bSJuergen Gross .mmu.pte_val = PTE_IDENT,
2845c83511bSJuergen Gross .mmu.pgd_val = PTE_IDENT,
285b1df07bdSGlauber de Oliveira Costa
2865c83511bSJuergen Gross .mmu.make_pte = PTE_IDENT,
2875c83511bSJuergen Gross .mmu.make_pgd = PTE_IDENT,
288b1df07bdSGlauber de Oliveira Costa
289c9ae1b10SJuergen Gross .mmu.enter_mmap = paravirt_nop,
290b1df07bdSGlauber de Oliveira Costa
2915c83511bSJuergen Gross .mmu.lazy_mode = {
292b1df07bdSGlauber de Oliveira Costa .enter = paravirt_nop,
293b1df07bdSGlauber de Oliveira Costa .leave = paravirt_nop,
294511ba86eSBoris Ostrovsky .flush = paravirt_nop,
295b1df07bdSGlauber de Oliveira Costa },
296aeaaa59cSJeremy Fitzhardinge
2975c83511bSJuergen Gross .mmu.set_fixmap = native_set_fixmap,
298fdc0269eSJuergen Gross #endif /* CONFIG_PARAVIRT_XXL */
2995c83511bSJuergen Gross
3005c83511bSJuergen Gross #if defined(CONFIG_PARAVIRT_SPINLOCKS)
3015c83511bSJuergen Gross /* Lock ops. */
3025c83511bSJuergen Gross #ifdef CONFIG_SMP
3035c83511bSJuergen Gross .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath,
3045c83511bSJuergen Gross .lock.queued_spin_unlock =
3055c83511bSJuergen Gross PV_CALLEE_SAVE(__native_queued_spin_unlock),
3065c83511bSJuergen Gross .lock.wait = paravirt_nop,
3075c83511bSJuergen Gross .lock.kick = paravirt_nop,
3085c83511bSJuergen Gross .lock.vcpu_is_preempted =
3095c83511bSJuergen Gross PV_CALLEE_SAVE(__native_vcpu_is_preempted),
3105c83511bSJuergen Gross #endif /* SMP */
3115c83511bSJuergen Gross #endif
312b1df07bdSGlauber de Oliveira Costa };
313b1df07bdSGlauber de Oliveira Costa
3149bad5658SJuergen Gross #ifdef CONFIG_PARAVIRT_XXL
3155c83511bSJuergen Gross NOKPROBE_SYMBOL(native_load_idt);
3169bad5658SJuergen Gross #endif
3175c83511bSJuergen Gross
3188af19095SJuergen Gross EXPORT_SYMBOL(pv_ops);
319b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL_GPL(pv_info);
320