1b1df07bdSGlauber de Oliveira Costa /* Paravirtualization interfaces 2b1df07bdSGlauber de Oliveira Costa Copyright (C) 2006 Rusty Russell IBM Corporation 3b1df07bdSGlauber de Oliveira Costa 4b1df07bdSGlauber de Oliveira Costa This program is free software; you can redistribute it and/or modify 5b1df07bdSGlauber de Oliveira Costa it under the terms of the GNU General Public License as published by 6b1df07bdSGlauber de Oliveira Costa the Free Software Foundation; either version 2 of the License, or 7b1df07bdSGlauber de Oliveira Costa (at your option) any later version. 8b1df07bdSGlauber de Oliveira Costa 9b1df07bdSGlauber de Oliveira Costa This program is distributed in the hope that it will be useful, 10b1df07bdSGlauber de Oliveira Costa but WITHOUT ANY WARRANTY; without even the implied warranty of 11b1df07bdSGlauber de Oliveira Costa MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12b1df07bdSGlauber de Oliveira Costa GNU General Public License for more details. 13b1df07bdSGlauber de Oliveira Costa 14b1df07bdSGlauber de Oliveira Costa You should have received a copy of the GNU General Public License 15b1df07bdSGlauber de Oliveira Costa along with this program; if not, write to the Free Software 16b1df07bdSGlauber de Oliveira Costa Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17b1df07bdSGlauber de Oliveira Costa 18b1df07bdSGlauber de Oliveira Costa 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc 19b1df07bdSGlauber de Oliveira Costa */ 20b1df07bdSGlauber de Oliveira Costa 21b1df07bdSGlauber de Oliveira Costa #include <linux/errno.h> 22186f4360SPaul Gortmaker #include <linux/init.h> 23186f4360SPaul Gortmaker #include <linux/export.h> 24b1df07bdSGlauber de Oliveira Costa #include <linux/efi.h> 25b1df07bdSGlauber de Oliveira Costa #include <linux/bcd.h> 26b1df07bdSGlauber de Oliveira Costa #include <linux/highmem.h> 27376e2424SMasami Hiramatsu #include <linux/kprobes.h> 28b1df07bdSGlauber de Oliveira Costa 29b1df07bdSGlauber de Oliveira Costa #include <asm/bug.h> 30b1df07bdSGlauber de Oliveira Costa #include <asm/paravirt.h> 3150af5eadSPaul Gortmaker #include <asm/debugreg.h> 32b1df07bdSGlauber de Oliveira Costa #include <asm/desc.h> 33b1df07bdSGlauber de Oliveira Costa #include <asm/setup.h> 34a312b37bSEduardo Habkost #include <asm/pgtable.h> 35b1df07bdSGlauber de Oliveira Costa #include <asm/time.h> 36eba0045fSJeremy Fitzhardinge #include <asm/pgalloc.h> 37b1df07bdSGlauber de Oliveira Costa #include <asm/irq.h> 38b1df07bdSGlauber de Oliveira Costa #include <asm/delay.h> 39b1df07bdSGlauber de Oliveira Costa #include <asm/fixmap.h> 40b1df07bdSGlauber de Oliveira Costa #include <asm/apic.h> 41b1df07bdSGlauber de Oliveira Costa #include <asm/tlbflush.h> 42b1df07bdSGlauber de Oliveira Costa #include <asm/timer.h> 43f05e798aSDavid Howells #include <asm/special_insns.h> 44b1df07bdSGlauber de Oliveira Costa 45fc57a7c6SAndy Lutomirski /* 46fc57a7c6SAndy Lutomirski * nop stub, which must not clobber anything *including the stack* to 47fc57a7c6SAndy Lutomirski * avoid confusing the entry prologues. 48fc57a7c6SAndy Lutomirski */ 49fc57a7c6SAndy Lutomirski extern void _paravirt_nop(void); 50fc57a7c6SAndy Lutomirski asm (".pushsection .entry.text, \"ax\"\n" 51fc57a7c6SAndy Lutomirski ".global _paravirt_nop\n" 52fc57a7c6SAndy Lutomirski "_paravirt_nop:\n\t" 53fc57a7c6SAndy Lutomirski "ret\n\t" 54fc57a7c6SAndy Lutomirski ".size _paravirt_nop, . - _paravirt_nop\n\t" 55fc57a7c6SAndy Lutomirski ".type _paravirt_nop, @function\n\t" 56fc57a7c6SAndy Lutomirski ".popsection"); 57b1df07bdSGlauber de Oliveira Costa 5841edafdbSJeremy Fitzhardinge /* identity function, which can be inlined */ 5915301a57SSteven Rostedt u32 notrace _paravirt_ident_32(u32 x) 6041edafdbSJeremy Fitzhardinge { 6141edafdbSJeremy Fitzhardinge return x; 6241edafdbSJeremy Fitzhardinge } 6341edafdbSJeremy Fitzhardinge 6415301a57SSteven Rostedt u64 notrace _paravirt_ident_64(u64 x) 6541edafdbSJeremy Fitzhardinge { 6641edafdbSJeremy Fitzhardinge return x; 6741edafdbSJeremy Fitzhardinge } 6841edafdbSJeremy Fitzhardinge 696f30c1acSThomas Gleixner void __init default_banner(void) 70b1df07bdSGlauber de Oliveira Costa { 71b1df07bdSGlauber de Oliveira Costa printk(KERN_INFO "Booting paravirtualized kernel on %s\n", 72b1df07bdSGlauber de Oliveira Costa pv_info.name); 73b1df07bdSGlauber de Oliveira Costa } 74b1df07bdSGlauber de Oliveira Costa 75b1df07bdSGlauber de Oliveira Costa /* Undefined instruction for dealing with missing ops pointers. */ 76b1df07bdSGlauber de Oliveira Costa static const unsigned char ud2a[] = { 0x0f, 0x0b }; 77b1df07bdSGlauber de Oliveira Costa 78b1df07bdSGlauber de Oliveira Costa struct branch { 79b1df07bdSGlauber de Oliveira Costa unsigned char opcode; 80b1df07bdSGlauber de Oliveira Costa u32 delta; 81b1df07bdSGlauber de Oliveira Costa } __attribute__((packed)); 82b1df07bdSGlauber de Oliveira Costa 83b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_call(void *insnbuf, 84b1df07bdSGlauber de Oliveira Costa const void *target, u16 tgt_clobbers, 85b1df07bdSGlauber de Oliveira Costa unsigned long addr, u16 site_clobbers, 86b1df07bdSGlauber de Oliveira Costa unsigned len) 87b1df07bdSGlauber de Oliveira Costa { 88b1df07bdSGlauber de Oliveira Costa struct branch *b = insnbuf; 89b1df07bdSGlauber de Oliveira Costa unsigned long delta = (unsigned long)target - (addr+5); 90b1df07bdSGlauber de Oliveira Costa 91b1df07bdSGlauber de Oliveira Costa if (tgt_clobbers & ~site_clobbers) 92b1df07bdSGlauber de Oliveira Costa return len; /* target would clobber too much for this site */ 93b1df07bdSGlauber de Oliveira Costa if (len < 5) 94b1df07bdSGlauber de Oliveira Costa return len; /* call too long for patch site */ 95b1df07bdSGlauber de Oliveira Costa 96b1df07bdSGlauber de Oliveira Costa b->opcode = 0xe8; /* call */ 97b1df07bdSGlauber de Oliveira Costa b->delta = delta; 98b1df07bdSGlauber de Oliveira Costa BUILD_BUG_ON(sizeof(*b) != 5); 99b1df07bdSGlauber de Oliveira Costa 100b1df07bdSGlauber de Oliveira Costa return 5; 101b1df07bdSGlauber de Oliveira Costa } 102b1df07bdSGlauber de Oliveira Costa 103b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_jmp(void *insnbuf, const void *target, 104b1df07bdSGlauber de Oliveira Costa unsigned long addr, unsigned len) 105b1df07bdSGlauber de Oliveira Costa { 106b1df07bdSGlauber de Oliveira Costa struct branch *b = insnbuf; 107b1df07bdSGlauber de Oliveira Costa unsigned long delta = (unsigned long)target - (addr+5); 108b1df07bdSGlauber de Oliveira Costa 109b1df07bdSGlauber de Oliveira Costa if (len < 5) 110b1df07bdSGlauber de Oliveira Costa return len; /* call too long for patch site */ 111b1df07bdSGlauber de Oliveira Costa 112b1df07bdSGlauber de Oliveira Costa b->opcode = 0xe9; /* jmp */ 113b1df07bdSGlauber de Oliveira Costa b->delta = delta; 114b1df07bdSGlauber de Oliveira Costa 115b1df07bdSGlauber de Oliveira Costa return 5; 116b1df07bdSGlauber de Oliveira Costa } 117b1df07bdSGlauber de Oliveira Costa 118b1df07bdSGlauber de Oliveira Costa /* Neat trick to map patch type back to the call within the 119b1df07bdSGlauber de Oliveira Costa * corresponding structure. */ 120b1df07bdSGlauber de Oliveira Costa static void *get_call_destination(u8 type) 121b1df07bdSGlauber de Oliveira Costa { 122b1df07bdSGlauber de Oliveira Costa struct paravirt_patch_template tmpl = { 123b1df07bdSGlauber de Oliveira Costa .pv_init_ops = pv_init_ops, 124b1df07bdSGlauber de Oliveira Costa .pv_time_ops = pv_time_ops, 125b1df07bdSGlauber de Oliveira Costa .pv_cpu_ops = pv_cpu_ops, 126b1df07bdSGlauber de Oliveira Costa .pv_irq_ops = pv_irq_ops, 127b1df07bdSGlauber de Oliveira Costa .pv_mmu_ops = pv_mmu_ops, 128b4ecc126SJeremy Fitzhardinge #ifdef CONFIG_PARAVIRT_SPINLOCKS 12974d4affdSJeremy Fitzhardinge .pv_lock_ops = pv_lock_ops, 130b4ecc126SJeremy Fitzhardinge #endif 131b1df07bdSGlauber de Oliveira Costa }; 132b1df07bdSGlauber de Oliveira Costa return *((void **)&tmpl + type); 133b1df07bdSGlauber de Oliveira Costa } 134b1df07bdSGlauber de Oliveira Costa 135b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, 136b1df07bdSGlauber de Oliveira Costa unsigned long addr, unsigned len) 137b1df07bdSGlauber de Oliveira Costa { 138b1df07bdSGlauber de Oliveira Costa void *opfunc = get_call_destination(type); 139b1df07bdSGlauber de Oliveira Costa unsigned ret; 140b1df07bdSGlauber de Oliveira Costa 141b1df07bdSGlauber de Oliveira Costa if (opfunc == NULL) 142b1df07bdSGlauber de Oliveira Costa /* If there's no function, patch it with a ud2a (BUG) */ 143b1df07bdSGlauber de Oliveira Costa ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a)); 14441edafdbSJeremy Fitzhardinge else if (opfunc == _paravirt_nop) 14579f1d836SBorislav Petkov ret = 0; 14641edafdbSJeremy Fitzhardinge 14741edafdbSJeremy Fitzhardinge /* identity functions just return their single argument */ 14841edafdbSJeremy Fitzhardinge else if (opfunc == _paravirt_ident_32) 14941edafdbSJeremy Fitzhardinge ret = paravirt_patch_ident_32(insnbuf, len); 15041edafdbSJeremy Fitzhardinge else if (opfunc == _paravirt_ident_64) 15141edafdbSJeremy Fitzhardinge ret = paravirt_patch_ident_64(insnbuf, len); 15241edafdbSJeremy Fitzhardinge 153b1df07bdSGlauber de Oliveira Costa else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) || 1542be29982SJeremy Fitzhardinge type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64)) 155b1df07bdSGlauber de Oliveira Costa /* If operation requires a jmp, then jmp */ 156b1df07bdSGlauber de Oliveira Costa ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len); 157b1df07bdSGlauber de Oliveira Costa else 158b1df07bdSGlauber de Oliveira Costa /* Otherwise call the function; assume target could 159b1df07bdSGlauber de Oliveira Costa clobber any caller-save reg */ 160b1df07bdSGlauber de Oliveira Costa ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY, 161b1df07bdSGlauber de Oliveira Costa addr, clobbers, len); 162b1df07bdSGlauber de Oliveira Costa 163b1df07bdSGlauber de Oliveira Costa return ret; 164b1df07bdSGlauber de Oliveira Costa } 165b1df07bdSGlauber de Oliveira Costa 166b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_insns(void *insnbuf, unsigned len, 167b1df07bdSGlauber de Oliveira Costa const char *start, const char *end) 168b1df07bdSGlauber de Oliveira Costa { 169b1df07bdSGlauber de Oliveira Costa unsigned insn_len = end - start; 170b1df07bdSGlauber de Oliveira Costa 171b1df07bdSGlauber de Oliveira Costa if (insn_len > len || start == NULL) 172b1df07bdSGlauber de Oliveira Costa insn_len = len; 173b1df07bdSGlauber de Oliveira Costa else 174b1df07bdSGlauber de Oliveira Costa memcpy(insnbuf, start, insn_len); 175b1df07bdSGlauber de Oliveira Costa 176b1df07bdSGlauber de Oliveira Costa return insn_len; 177b1df07bdSGlauber de Oliveira Costa } 178b1df07bdSGlauber de Oliveira Costa 179b1df07bdSGlauber de Oliveira Costa static void native_flush_tlb(void) 180b1df07bdSGlauber de Oliveira Costa { 181b1df07bdSGlauber de Oliveira Costa __native_flush_tlb(); 182b1df07bdSGlauber de Oliveira Costa } 183b1df07bdSGlauber de Oliveira Costa 184b1df07bdSGlauber de Oliveira Costa /* 185b1df07bdSGlauber de Oliveira Costa * Global pages have to be flushed a bit differently. Not a real 186b1df07bdSGlauber de Oliveira Costa * performance problem because this does not happen often. 187b1df07bdSGlauber de Oliveira Costa */ 188b1df07bdSGlauber de Oliveira Costa static void native_flush_tlb_global(void) 189b1df07bdSGlauber de Oliveira Costa { 190b1df07bdSGlauber de Oliveira Costa __native_flush_tlb_global(); 191b1df07bdSGlauber de Oliveira Costa } 192b1df07bdSGlauber de Oliveira Costa 193b1df07bdSGlauber de Oliveira Costa static void native_flush_tlb_single(unsigned long addr) 194b1df07bdSGlauber de Oliveira Costa { 195b1df07bdSGlauber de Oliveira Costa __native_flush_tlb_single(addr); 196b1df07bdSGlauber de Oliveira Costa } 197b1df07bdSGlauber de Oliveira Costa 198c5905afbSIngo Molnar struct static_key paravirt_steal_enabled; 199c5905afbSIngo Molnar struct static_key paravirt_steal_rq_enabled; 2003c404b57SGlauber Costa 2013c404b57SGlauber Costa static u64 native_steal_clock(int cpu) 2023c404b57SGlauber Costa { 2033c404b57SGlauber Costa return 0; 2043c404b57SGlauber Costa } 2053c404b57SGlauber Costa 206b1df07bdSGlauber de Oliveira Costa /* These are in entry.S */ 207b1df07bdSGlauber de Oliveira Costa extern void native_iret(void); 2082be29982SJeremy Fitzhardinge extern void native_usergs_sysret64(void); 209b1df07bdSGlauber de Oliveira Costa 210b1df07bdSGlauber de Oliveira Costa static struct resource reserve_ioports = { 211b1df07bdSGlauber de Oliveira Costa .start = 0, 212b1df07bdSGlauber de Oliveira Costa .end = IO_SPACE_LIMIT, 213b1df07bdSGlauber de Oliveira Costa .name = "paravirt-ioport", 214b1df07bdSGlauber de Oliveira Costa .flags = IORESOURCE_IO | IORESOURCE_BUSY, 215b1df07bdSGlauber de Oliveira Costa }; 216b1df07bdSGlauber de Oliveira Costa 217b1df07bdSGlauber de Oliveira Costa /* 218b1df07bdSGlauber de Oliveira Costa * Reserve the whole legacy IO space to prevent any legacy drivers 219b1df07bdSGlauber de Oliveira Costa * from wasting time probing for their hardware. This is a fairly 220b1df07bdSGlauber de Oliveira Costa * brute-force approach to disabling all non-virtual drivers. 221b1df07bdSGlauber de Oliveira Costa * 222b1df07bdSGlauber de Oliveira Costa * Note that this must be called very early to have any effect. 223b1df07bdSGlauber de Oliveira Costa */ 224b1df07bdSGlauber de Oliveira Costa int paravirt_disable_iospace(void) 225b1df07bdSGlauber de Oliveira Costa { 226f7743fe6SJeremy Fitzhardinge return request_resource(&ioport_resource, &reserve_ioports); 227b1df07bdSGlauber de Oliveira Costa } 228b1df07bdSGlauber de Oliveira Costa 229b1df07bdSGlauber de Oliveira Costa static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE; 230b1df07bdSGlauber de Oliveira Costa 231b1df07bdSGlauber de Oliveira Costa static inline void enter_lazy(enum paravirt_lazy_mode mode) 232b1df07bdSGlauber de Oliveira Costa { 233c6ae41e7SAlex Shi BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE); 234b1df07bdSGlauber de Oliveira Costa 235c6ae41e7SAlex Shi this_cpu_write(paravirt_lazy_mode, mode); 236b1df07bdSGlauber de Oliveira Costa } 237b1df07bdSGlauber de Oliveira Costa 238b407fc57SJeremy Fitzhardinge static void leave_lazy(enum paravirt_lazy_mode mode) 239b1df07bdSGlauber de Oliveira Costa { 240c6ae41e7SAlex Shi BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode); 241b1df07bdSGlauber de Oliveira Costa 242c6ae41e7SAlex Shi this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE); 243b1df07bdSGlauber de Oliveira Costa } 244b1df07bdSGlauber de Oliveira Costa 245b1df07bdSGlauber de Oliveira Costa void paravirt_enter_lazy_mmu(void) 246b1df07bdSGlauber de Oliveira Costa { 247b1df07bdSGlauber de Oliveira Costa enter_lazy(PARAVIRT_LAZY_MMU); 248b1df07bdSGlauber de Oliveira Costa } 249b1df07bdSGlauber de Oliveira Costa 250b1df07bdSGlauber de Oliveira Costa void paravirt_leave_lazy_mmu(void) 251b1df07bdSGlauber de Oliveira Costa { 252b407fc57SJeremy Fitzhardinge leave_lazy(PARAVIRT_LAZY_MMU); 253b1df07bdSGlauber de Oliveira Costa } 254b1df07bdSGlauber de Oliveira Costa 255511ba86eSBoris Ostrovsky void paravirt_flush_lazy_mmu(void) 256511ba86eSBoris Ostrovsky { 257511ba86eSBoris Ostrovsky preempt_disable(); 258511ba86eSBoris Ostrovsky 259511ba86eSBoris Ostrovsky if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) { 260511ba86eSBoris Ostrovsky arch_leave_lazy_mmu_mode(); 261511ba86eSBoris Ostrovsky arch_enter_lazy_mmu_mode(); 262511ba86eSBoris Ostrovsky } 263511ba86eSBoris Ostrovsky 264511ba86eSBoris Ostrovsky preempt_enable(); 265511ba86eSBoris Ostrovsky } 266511ba86eSBoris Ostrovsky 267224101edSJeremy Fitzhardinge void paravirt_start_context_switch(struct task_struct *prev) 268b1df07bdSGlauber de Oliveira Costa { 2692829b449SJeremy Fitzhardinge BUG_ON(preemptible()); 2702829b449SJeremy Fitzhardinge 271c6ae41e7SAlex Shi if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) { 272b407fc57SJeremy Fitzhardinge arch_leave_lazy_mmu_mode(); 273224101edSJeremy Fitzhardinge set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES); 274b407fc57SJeremy Fitzhardinge } 275b1df07bdSGlauber de Oliveira Costa enter_lazy(PARAVIRT_LAZY_CPU); 276b1df07bdSGlauber de Oliveira Costa } 277b1df07bdSGlauber de Oliveira Costa 278224101edSJeremy Fitzhardinge void paravirt_end_context_switch(struct task_struct *next) 279b1df07bdSGlauber de Oliveira Costa { 2802829b449SJeremy Fitzhardinge BUG_ON(preemptible()); 2812829b449SJeremy Fitzhardinge 282b407fc57SJeremy Fitzhardinge leave_lazy(PARAVIRT_LAZY_CPU); 283b407fc57SJeremy Fitzhardinge 284224101edSJeremy Fitzhardinge if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES)) 285b407fc57SJeremy Fitzhardinge arch_enter_lazy_mmu_mode(); 286b1df07bdSGlauber de Oliveira Costa } 287b1df07bdSGlauber de Oliveira Costa 288b1df07bdSGlauber de Oliveira Costa enum paravirt_lazy_mode paravirt_get_lazy_mode(void) 289b1df07bdSGlauber de Oliveira Costa { 290b8bcfe99SJeremy Fitzhardinge if (in_interrupt()) 291b8bcfe99SJeremy Fitzhardinge return PARAVIRT_LAZY_NONE; 292b8bcfe99SJeremy Fitzhardinge 293c6ae41e7SAlex Shi return this_cpu_read(paravirt_lazy_mode); 294b1df07bdSGlauber de Oliveira Costa } 295b1df07bdSGlauber de Oliveira Costa 296b1df07bdSGlauber de Oliveira Costa struct pv_info pv_info = { 297b1df07bdSGlauber de Oliveira Costa .name = "bare hardware", 298b1df07bdSGlauber de Oliveira Costa .kernel_rpl = 0, 299b1df07bdSGlauber de Oliveira Costa .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */ 300318f5a2aSAndy Lutomirski 301318f5a2aSAndy Lutomirski #ifdef CONFIG_X86_64 302318f5a2aSAndy Lutomirski .extra_user_64bit_cs = __USER_CS, 303318f5a2aSAndy Lutomirski #endif 304b1df07bdSGlauber de Oliveira Costa }; 305b1df07bdSGlauber de Oliveira Costa 306b1df07bdSGlauber de Oliveira Costa struct pv_init_ops pv_init_ops = { 307b1df07bdSGlauber de Oliveira Costa .patch = native_patch, 308b1df07bdSGlauber de Oliveira Costa }; 309b1df07bdSGlauber de Oliveira Costa 310b1df07bdSGlauber de Oliveira Costa struct pv_time_ops pv_time_ops = { 311b1df07bdSGlauber de Oliveira Costa .sched_clock = native_sched_clock, 3123c404b57SGlauber Costa .steal_clock = native_steal_clock, 313b1df07bdSGlauber de Oliveira Costa }; 314b1df07bdSGlauber de Oliveira Costa 3159a55fdbeSAndi Kleen __visible struct pv_irq_ops pv_irq_ops = { 316ecb93d1cSJeremy Fitzhardinge .save_fl = __PV_IS_CALLEE_SAVE(native_save_fl), 317ecb93d1cSJeremy Fitzhardinge .restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl), 318ecb93d1cSJeremy Fitzhardinge .irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable), 319ecb93d1cSJeremy Fitzhardinge .irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable), 320b1df07bdSGlauber de Oliveira Costa .safe_halt = native_safe_halt, 321b1df07bdSGlauber de Oliveira Costa .halt = native_halt, 322fab58420SJeremy Fitzhardinge #ifdef CONFIG_X86_64 323fab58420SJeremy Fitzhardinge .adjust_exception_frame = paravirt_nop, 324fab58420SJeremy Fitzhardinge #endif 325b1df07bdSGlauber de Oliveira Costa }; 326b1df07bdSGlauber de Oliveira Costa 3279a55fdbeSAndi Kleen __visible struct pv_cpu_ops pv_cpu_ops = { 328b1df07bdSGlauber de Oliveira Costa .cpuid = native_cpuid, 329b1df07bdSGlauber de Oliveira Costa .get_debugreg = native_get_debugreg, 330b1df07bdSGlauber de Oliveira Costa .set_debugreg = native_set_debugreg, 331b1df07bdSGlauber de Oliveira Costa .read_cr0 = native_read_cr0, 332b1df07bdSGlauber de Oliveira Costa .write_cr0 = native_write_cr0, 333b1df07bdSGlauber de Oliveira Costa .read_cr4 = native_read_cr4, 334b1df07bdSGlauber de Oliveira Costa .write_cr4 = native_write_cr4, 33588b4755fSGlauber de Oliveira Costa #ifdef CONFIG_X86_64 33688b4755fSGlauber de Oliveira Costa .read_cr8 = native_read_cr8, 33788b4755fSGlauber de Oliveira Costa .write_cr8 = native_write_cr8, 33888b4755fSGlauber de Oliveira Costa #endif 339b1df07bdSGlauber de Oliveira Costa .wbinvd = native_wbinvd, 340dd2f4a00SAndy Lutomirski .read_msr = native_read_msr, 341dd2f4a00SAndy Lutomirski .write_msr = native_write_msr, 342c2ee03b2SAndy Lutomirski .read_msr_safe = native_read_msr_safe, 343c2ee03b2SAndy Lutomirski .write_msr_safe = native_write_msr_safe, 344b1df07bdSGlauber de Oliveira Costa .read_pmc = native_read_pmc, 345b1df07bdSGlauber de Oliveira Costa .load_tr_desc = native_load_tr_desc, 346b1df07bdSGlauber de Oliveira Costa .set_ldt = native_set_ldt, 347b1df07bdSGlauber de Oliveira Costa .load_gdt = native_load_gdt, 348b1df07bdSGlauber de Oliveira Costa .load_idt = native_load_idt, 349b1df07bdSGlauber de Oliveira Costa .store_idt = native_store_idt, 350b1df07bdSGlauber de Oliveira Costa .store_tr = native_store_tr, 351b1df07bdSGlauber de Oliveira Costa .load_tls = native_load_tls, 3529f9d489aSJeremy Fitzhardinge #ifdef CONFIG_X86_64 3539f9d489aSJeremy Fitzhardinge .load_gs_index = native_load_gs_index, 3549f9d489aSJeremy Fitzhardinge #endif 355b1df07bdSGlauber de Oliveira Costa .write_ldt_entry = native_write_ldt_entry, 356b1df07bdSGlauber de Oliveira Costa .write_gdt_entry = native_write_gdt_entry, 357b1df07bdSGlauber de Oliveira Costa .write_idt_entry = native_write_idt_entry, 35838ffbe66SJeremy Fitzhardinge 35938ffbe66SJeremy Fitzhardinge .alloc_ldt = paravirt_nop, 36038ffbe66SJeremy Fitzhardinge .free_ldt = paravirt_nop, 36138ffbe66SJeremy Fitzhardinge 362b1df07bdSGlauber de Oliveira Costa .load_sp0 = native_load_sp0, 363b1df07bdSGlauber de Oliveira Costa 3642be29982SJeremy Fitzhardinge #ifdef CONFIG_X86_64 3652be29982SJeremy Fitzhardinge .usergs_sysret64 = native_usergs_sysret64, 366d75cd22fSJeremy Fitzhardinge #endif 367b1df07bdSGlauber de Oliveira Costa .iret = native_iret, 368e801f864SGlauber de Oliveira Costa .swapgs = native_swapgs, 369b1df07bdSGlauber de Oliveira Costa 370b1df07bdSGlauber de Oliveira Costa .set_iopl_mask = native_set_iopl_mask, 371b1df07bdSGlauber de Oliveira Costa .io_delay = native_io_delay, 372b1df07bdSGlauber de Oliveira Costa 373224101edSJeremy Fitzhardinge .start_context_switch = paravirt_nop, 374224101edSJeremy Fitzhardinge .end_context_switch = paravirt_nop, 375b1df07bdSGlauber de Oliveira Costa }; 376b1df07bdSGlauber de Oliveira Costa 37780271972SMasami Hiramatsu /* At this point, native_get/set_debugreg has real function entries */ 378376e2424SMasami Hiramatsu NOKPROBE_SYMBOL(native_get_debugreg); 37980271972SMasami Hiramatsu NOKPROBE_SYMBOL(native_set_debugreg); 38080271972SMasami Hiramatsu NOKPROBE_SYMBOL(native_load_idt); 381376e2424SMasami Hiramatsu 38241edafdbSJeremy Fitzhardinge #if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE) 38341edafdbSJeremy Fitzhardinge /* 32-bit pagetable entries */ 384da5de7c2SJeremy Fitzhardinge #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32) 38541edafdbSJeremy Fitzhardinge #else 38641edafdbSJeremy Fitzhardinge /* 64-bit pagetable entries */ 387da5de7c2SJeremy Fitzhardinge #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64) 38841edafdbSJeremy Fitzhardinge #endif 38941edafdbSJeremy Fitzhardinge 390404f6aacSKees Cook struct pv_mmu_ops pv_mmu_ops __ro_after_init = { 391b1df07bdSGlauber de Oliveira Costa 392b1df07bdSGlauber de Oliveira Costa .read_cr2 = native_read_cr2, 393b1df07bdSGlauber de Oliveira Costa .write_cr2 = native_write_cr2, 3946c690ee1SAndy Lutomirski .read_cr3 = __native_read_cr3, 395b1df07bdSGlauber de Oliveira Costa .write_cr3 = native_write_cr3, 396b1df07bdSGlauber de Oliveira Costa 397b1df07bdSGlauber de Oliveira Costa .flush_tlb_user = native_flush_tlb, 398b1df07bdSGlauber de Oliveira Costa .flush_tlb_kernel = native_flush_tlb_global, 399b1df07bdSGlauber de Oliveira Costa .flush_tlb_single = native_flush_tlb_single, 400b1df07bdSGlauber de Oliveira Costa .flush_tlb_others = native_flush_tlb_others, 401b1df07bdSGlauber de Oliveira Costa 402eba0045fSJeremy Fitzhardinge .pgd_alloc = __paravirt_pgd_alloc, 403eba0045fSJeremy Fitzhardinge .pgd_free = paravirt_nop, 404eba0045fSJeremy Fitzhardinge 4056944a9c8SJeremy Fitzhardinge .alloc_pte = paravirt_nop, 4066944a9c8SJeremy Fitzhardinge .alloc_pmd = paravirt_nop, 4072761fa09SJeremy Fitzhardinge .alloc_pud = paravirt_nop, 408335437fbSKirill A. Shutemov .alloc_p4d = paravirt_nop, 4096944a9c8SJeremy Fitzhardinge .release_pte = paravirt_nop, 4106944a9c8SJeremy Fitzhardinge .release_pmd = paravirt_nop, 4112761fa09SJeremy Fitzhardinge .release_pud = paravirt_nop, 412335437fbSKirill A. Shutemov .release_p4d = paravirt_nop, 413b1df07bdSGlauber de Oliveira Costa 414b1df07bdSGlauber de Oliveira Costa .set_pte = native_set_pte, 415b1df07bdSGlauber de Oliveira Costa .set_pte_at = native_set_pte_at, 416b1df07bdSGlauber de Oliveira Costa .set_pmd = native_set_pmd, 417331127f7SAndrea Arcangeli .set_pmd_at = native_set_pmd_at, 418b1df07bdSGlauber de Oliveira Costa .pte_update = paravirt_nop, 419b1df07bdSGlauber de Oliveira Costa 42008b882c6SJeremy Fitzhardinge .ptep_modify_prot_start = __ptep_modify_prot_start, 42108b882c6SJeremy Fitzhardinge .ptep_modify_prot_commit = __ptep_modify_prot_commit, 42208b882c6SJeremy Fitzhardinge 42398233368SKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS >= 3 424b1df07bdSGlauber de Oliveira Costa #ifdef CONFIG_X86_PAE 425b1df07bdSGlauber de Oliveira Costa .set_pte_atomic = native_set_pte_atomic, 426b1df07bdSGlauber de Oliveira Costa .pte_clear = native_pte_clear, 427b1df07bdSGlauber de Oliveira Costa .pmd_clear = native_pmd_clear, 428f95f2f7bSEduardo Habkost #endif 429f95f2f7bSEduardo Habkost .set_pud = native_set_pud, 430a00cc7d9SMatthew Wilcox .set_pud_at = native_set_pud_at, 431da5de7c2SJeremy Fitzhardinge 432da5de7c2SJeremy Fitzhardinge .pmd_val = PTE_IDENT, 433da5de7c2SJeremy Fitzhardinge .make_pmd = PTE_IDENT, 434f95f2f7bSEduardo Habkost 435f2a6a705SKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS >= 4 436da5de7c2SJeremy Fitzhardinge .pud_val = PTE_IDENT, 437da5de7c2SJeremy Fitzhardinge .make_pud = PTE_IDENT, 438da5de7c2SJeremy Fitzhardinge 439f2a6a705SKirill A. Shutemov .set_p4d = native_set_p4d, 440f2a6a705SKirill A. Shutemov 441f2a6a705SKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS >= 5 442335437fbSKirill A. Shutemov .p4d_val = PTE_IDENT, 443335437fbSKirill A. Shutemov .make_p4d = PTE_IDENT, 444335437fbSKirill A. Shutemov 445335437fbSKirill A. Shutemov .set_pgd = native_set_pgd, 446335437fbSKirill A. Shutemov #endif /* CONFIG_PGTABLE_LEVELS >= 5 */ 447f2a6a705SKirill A. Shutemov #endif /* CONFIG_PGTABLE_LEVELS >= 4 */ 44898233368SKirill A. Shutemov #endif /* CONFIG_PGTABLE_LEVELS >= 3 */ 449b1df07bdSGlauber de Oliveira Costa 450da5de7c2SJeremy Fitzhardinge .pte_val = PTE_IDENT, 451da5de7c2SJeremy Fitzhardinge .pgd_val = PTE_IDENT, 452b1df07bdSGlauber de Oliveira Costa 453da5de7c2SJeremy Fitzhardinge .make_pte = PTE_IDENT, 454da5de7c2SJeremy Fitzhardinge .make_pgd = PTE_IDENT, 455b1df07bdSGlauber de Oliveira Costa 456b1df07bdSGlauber de Oliveira Costa .dup_mmap = paravirt_nop, 457b1df07bdSGlauber de Oliveira Costa .exit_mmap = paravirt_nop, 458b1df07bdSGlauber de Oliveira Costa .activate_mm = paravirt_nop, 459b1df07bdSGlauber de Oliveira Costa 460b1df07bdSGlauber de Oliveira Costa .lazy_mode = { 461b1df07bdSGlauber de Oliveira Costa .enter = paravirt_nop, 462b1df07bdSGlauber de Oliveira Costa .leave = paravirt_nop, 463511ba86eSBoris Ostrovsky .flush = paravirt_nop, 464b1df07bdSGlauber de Oliveira Costa }, 465aeaaa59cSJeremy Fitzhardinge 466aeaaa59cSJeremy Fitzhardinge .set_fixmap = native_set_fixmap, 467b1df07bdSGlauber de Oliveira Costa }; 468b1df07bdSGlauber de Oliveira Costa 469b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL_GPL(pv_time_ops); 470b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL (pv_cpu_ops); 471b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL (pv_mmu_ops); 472b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL_GPL(pv_info); 473b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL (pv_irq_ops); 474