1b1df07bdSGlauber de Oliveira Costa /* Paravirtualization interfaces 2b1df07bdSGlauber de Oliveira Costa Copyright (C) 2006 Rusty Russell IBM Corporation 3b1df07bdSGlauber de Oliveira Costa 4b1df07bdSGlauber de Oliveira Costa This program is free software; you can redistribute it and/or modify 5b1df07bdSGlauber de Oliveira Costa it under the terms of the GNU General Public License as published by 6b1df07bdSGlauber de Oliveira Costa the Free Software Foundation; either version 2 of the License, or 7b1df07bdSGlauber de Oliveira Costa (at your option) any later version. 8b1df07bdSGlauber de Oliveira Costa 9b1df07bdSGlauber de Oliveira Costa This program is distributed in the hope that it will be useful, 10b1df07bdSGlauber de Oliveira Costa but WITHOUT ANY WARRANTY; without even the implied warranty of 11b1df07bdSGlauber de Oliveira Costa MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12b1df07bdSGlauber de Oliveira Costa GNU General Public License for more details. 13b1df07bdSGlauber de Oliveira Costa 14b1df07bdSGlauber de Oliveira Costa You should have received a copy of the GNU General Public License 15b1df07bdSGlauber de Oliveira Costa along with this program; if not, write to the Free Software 16b1df07bdSGlauber de Oliveira Costa Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17b1df07bdSGlauber de Oliveira Costa 18b1df07bdSGlauber de Oliveira Costa 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc 19b1df07bdSGlauber de Oliveira Costa */ 20b1df07bdSGlauber de Oliveira Costa 21b1df07bdSGlauber de Oliveira Costa #include <linux/errno.h> 22b1df07bdSGlauber de Oliveira Costa #include <linux/module.h> 23b1df07bdSGlauber de Oliveira Costa #include <linux/efi.h> 24b1df07bdSGlauber de Oliveira Costa #include <linux/bcd.h> 25b1df07bdSGlauber de Oliveira Costa #include <linux/highmem.h> 26376e2424SMasami Hiramatsu #include <linux/kprobes.h> 27b1df07bdSGlauber de Oliveira Costa 28b1df07bdSGlauber de Oliveira Costa #include <asm/bug.h> 29b1df07bdSGlauber de Oliveira Costa #include <asm/paravirt.h> 3050af5eadSPaul Gortmaker #include <asm/debugreg.h> 31b1df07bdSGlauber de Oliveira Costa #include <asm/desc.h> 32b1df07bdSGlauber de Oliveira Costa #include <asm/setup.h> 33a312b37bSEduardo Habkost #include <asm/pgtable.h> 34b1df07bdSGlauber de Oliveira Costa #include <asm/time.h> 35eba0045fSJeremy Fitzhardinge #include <asm/pgalloc.h> 36b1df07bdSGlauber de Oliveira Costa #include <asm/irq.h> 37b1df07bdSGlauber de Oliveira Costa #include <asm/delay.h> 38b1df07bdSGlauber de Oliveira Costa #include <asm/fixmap.h> 39b1df07bdSGlauber de Oliveira Costa #include <asm/apic.h> 40b1df07bdSGlauber de Oliveira Costa #include <asm/tlbflush.h> 41b1df07bdSGlauber de Oliveira Costa #include <asm/timer.h> 42f05e798aSDavid Howells #include <asm/special_insns.h> 43b1df07bdSGlauber de Oliveira Costa 44b1df07bdSGlauber de Oliveira Costa /* nop stub */ 45b1df07bdSGlauber de Oliveira Costa void _paravirt_nop(void) 46b1df07bdSGlauber de Oliveira Costa { 47b1df07bdSGlauber de Oliveira Costa } 48b1df07bdSGlauber de Oliveira Costa 4941edafdbSJeremy Fitzhardinge /* identity function, which can be inlined */ 5041edafdbSJeremy Fitzhardinge u32 _paravirt_ident_32(u32 x) 5141edafdbSJeremy Fitzhardinge { 5241edafdbSJeremy Fitzhardinge return x; 5341edafdbSJeremy Fitzhardinge } 5441edafdbSJeremy Fitzhardinge 5541edafdbSJeremy Fitzhardinge u64 _paravirt_ident_64(u64 x) 5641edafdbSJeremy Fitzhardinge { 5741edafdbSJeremy Fitzhardinge return x; 5841edafdbSJeremy Fitzhardinge } 5941edafdbSJeremy Fitzhardinge 606f30c1acSThomas Gleixner void __init default_banner(void) 61b1df07bdSGlauber de Oliveira Costa { 62b1df07bdSGlauber de Oliveira Costa printk(KERN_INFO "Booting paravirtualized kernel on %s\n", 63b1df07bdSGlauber de Oliveira Costa pv_info.name); 64b1df07bdSGlauber de Oliveira Costa } 65b1df07bdSGlauber de Oliveira Costa 66b1df07bdSGlauber de Oliveira Costa /* Undefined instruction for dealing with missing ops pointers. */ 67b1df07bdSGlauber de Oliveira Costa static const unsigned char ud2a[] = { 0x0f, 0x0b }; 68b1df07bdSGlauber de Oliveira Costa 69b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_nop(void) 70b1df07bdSGlauber de Oliveira Costa { 71b1df07bdSGlauber de Oliveira Costa return 0; 72b1df07bdSGlauber de Oliveira Costa } 73b1df07bdSGlauber de Oliveira Costa 74b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_ignore(unsigned len) 75b1df07bdSGlauber de Oliveira Costa { 76b1df07bdSGlauber de Oliveira Costa return len; 77b1df07bdSGlauber de Oliveira Costa } 78b1df07bdSGlauber de Oliveira Costa 79b1df07bdSGlauber de Oliveira Costa struct branch { 80b1df07bdSGlauber de Oliveira Costa unsigned char opcode; 81b1df07bdSGlauber de Oliveira Costa u32 delta; 82b1df07bdSGlauber de Oliveira Costa } __attribute__((packed)); 83b1df07bdSGlauber de Oliveira Costa 84b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_call(void *insnbuf, 85b1df07bdSGlauber de Oliveira Costa const void *target, u16 tgt_clobbers, 86b1df07bdSGlauber de Oliveira Costa unsigned long addr, u16 site_clobbers, 87b1df07bdSGlauber de Oliveira Costa unsigned len) 88b1df07bdSGlauber de Oliveira Costa { 89b1df07bdSGlauber de Oliveira Costa struct branch *b = insnbuf; 90b1df07bdSGlauber de Oliveira Costa unsigned long delta = (unsigned long)target - (addr+5); 91b1df07bdSGlauber de Oliveira Costa 92b1df07bdSGlauber de Oliveira Costa if (tgt_clobbers & ~site_clobbers) 93b1df07bdSGlauber de Oliveira Costa return len; /* target would clobber too much for this site */ 94b1df07bdSGlauber de Oliveira Costa if (len < 5) 95b1df07bdSGlauber de Oliveira Costa return len; /* call too long for patch site */ 96b1df07bdSGlauber de Oliveira Costa 97b1df07bdSGlauber de Oliveira Costa b->opcode = 0xe8; /* call */ 98b1df07bdSGlauber de Oliveira Costa b->delta = delta; 99b1df07bdSGlauber de Oliveira Costa BUILD_BUG_ON(sizeof(*b) != 5); 100b1df07bdSGlauber de Oliveira Costa 101b1df07bdSGlauber de Oliveira Costa return 5; 102b1df07bdSGlauber de Oliveira Costa } 103b1df07bdSGlauber de Oliveira Costa 104b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_jmp(void *insnbuf, const void *target, 105b1df07bdSGlauber de Oliveira Costa unsigned long addr, unsigned len) 106b1df07bdSGlauber de Oliveira Costa { 107b1df07bdSGlauber de Oliveira Costa struct branch *b = insnbuf; 108b1df07bdSGlauber de Oliveira Costa unsigned long delta = (unsigned long)target - (addr+5); 109b1df07bdSGlauber de Oliveira Costa 110b1df07bdSGlauber de Oliveira Costa if (len < 5) 111b1df07bdSGlauber de Oliveira Costa return len; /* call too long for patch site */ 112b1df07bdSGlauber de Oliveira Costa 113b1df07bdSGlauber de Oliveira Costa b->opcode = 0xe9; /* jmp */ 114b1df07bdSGlauber de Oliveira Costa b->delta = delta; 115b1df07bdSGlauber de Oliveira Costa 116b1df07bdSGlauber de Oliveira Costa return 5; 117b1df07bdSGlauber de Oliveira Costa } 118b1df07bdSGlauber de Oliveira Costa 119b1df07bdSGlauber de Oliveira Costa /* Neat trick to map patch type back to the call within the 120b1df07bdSGlauber de Oliveira Costa * corresponding structure. */ 121b1df07bdSGlauber de Oliveira Costa static void *get_call_destination(u8 type) 122b1df07bdSGlauber de Oliveira Costa { 123b1df07bdSGlauber de Oliveira Costa struct paravirt_patch_template tmpl = { 124b1df07bdSGlauber de Oliveira Costa .pv_init_ops = pv_init_ops, 125b1df07bdSGlauber de Oliveira Costa .pv_time_ops = pv_time_ops, 126b1df07bdSGlauber de Oliveira Costa .pv_cpu_ops = pv_cpu_ops, 127b1df07bdSGlauber de Oliveira Costa .pv_irq_ops = pv_irq_ops, 128b1df07bdSGlauber de Oliveira Costa .pv_apic_ops = pv_apic_ops, 129b1df07bdSGlauber de Oliveira Costa .pv_mmu_ops = pv_mmu_ops, 130b4ecc126SJeremy Fitzhardinge #ifdef CONFIG_PARAVIRT_SPINLOCKS 13174d4affdSJeremy Fitzhardinge .pv_lock_ops = pv_lock_ops, 132b4ecc126SJeremy Fitzhardinge #endif 133b1df07bdSGlauber de Oliveira Costa }; 134b1df07bdSGlauber de Oliveira Costa return *((void **)&tmpl + type); 135b1df07bdSGlauber de Oliveira Costa } 136b1df07bdSGlauber de Oliveira Costa 137b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, 138b1df07bdSGlauber de Oliveira Costa unsigned long addr, unsigned len) 139b1df07bdSGlauber de Oliveira Costa { 140b1df07bdSGlauber de Oliveira Costa void *opfunc = get_call_destination(type); 141b1df07bdSGlauber de Oliveira Costa unsigned ret; 142b1df07bdSGlauber de Oliveira Costa 143b1df07bdSGlauber de Oliveira Costa if (opfunc == NULL) 144b1df07bdSGlauber de Oliveira Costa /* If there's no function, patch it with a ud2a (BUG) */ 145b1df07bdSGlauber de Oliveira Costa ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a)); 14641edafdbSJeremy Fitzhardinge else if (opfunc == _paravirt_nop) 147b1df07bdSGlauber de Oliveira Costa /* If the operation is a nop, then nop the callsite */ 148b1df07bdSGlauber de Oliveira Costa ret = paravirt_patch_nop(); 14941edafdbSJeremy Fitzhardinge 15041edafdbSJeremy Fitzhardinge /* identity functions just return their single argument */ 15141edafdbSJeremy Fitzhardinge else if (opfunc == _paravirt_ident_32) 15241edafdbSJeremy Fitzhardinge ret = paravirt_patch_ident_32(insnbuf, len); 15341edafdbSJeremy Fitzhardinge else if (opfunc == _paravirt_ident_64) 15441edafdbSJeremy Fitzhardinge ret = paravirt_patch_ident_64(insnbuf, len); 15541edafdbSJeremy Fitzhardinge 156b1df07bdSGlauber de Oliveira Costa else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) || 157d75cd22fSJeremy Fitzhardinge type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) || 1582be29982SJeremy Fitzhardinge type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret32) || 1592be29982SJeremy Fitzhardinge type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64)) 160b1df07bdSGlauber de Oliveira Costa /* If operation requires a jmp, then jmp */ 161b1df07bdSGlauber de Oliveira Costa ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len); 162b1df07bdSGlauber de Oliveira Costa else 163b1df07bdSGlauber de Oliveira Costa /* Otherwise call the function; assume target could 164b1df07bdSGlauber de Oliveira Costa clobber any caller-save reg */ 165b1df07bdSGlauber de Oliveira Costa ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY, 166b1df07bdSGlauber de Oliveira Costa addr, clobbers, len); 167b1df07bdSGlauber de Oliveira Costa 168b1df07bdSGlauber de Oliveira Costa return ret; 169b1df07bdSGlauber de Oliveira Costa } 170b1df07bdSGlauber de Oliveira Costa 171b1df07bdSGlauber de Oliveira Costa unsigned paravirt_patch_insns(void *insnbuf, unsigned len, 172b1df07bdSGlauber de Oliveira Costa const char *start, const char *end) 173b1df07bdSGlauber de Oliveira Costa { 174b1df07bdSGlauber de Oliveira Costa unsigned insn_len = end - start; 175b1df07bdSGlauber de Oliveira Costa 176b1df07bdSGlauber de Oliveira Costa if (insn_len > len || start == NULL) 177b1df07bdSGlauber de Oliveira Costa insn_len = len; 178b1df07bdSGlauber de Oliveira Costa else 179b1df07bdSGlauber de Oliveira Costa memcpy(insnbuf, start, insn_len); 180b1df07bdSGlauber de Oliveira Costa 181b1df07bdSGlauber de Oliveira Costa return insn_len; 182b1df07bdSGlauber de Oliveira Costa } 183b1df07bdSGlauber de Oliveira Costa 184b1df07bdSGlauber de Oliveira Costa static void native_flush_tlb(void) 185b1df07bdSGlauber de Oliveira Costa { 186b1df07bdSGlauber de Oliveira Costa __native_flush_tlb(); 187b1df07bdSGlauber de Oliveira Costa } 188b1df07bdSGlauber de Oliveira Costa 189b1df07bdSGlauber de Oliveira Costa /* 190b1df07bdSGlauber de Oliveira Costa * Global pages have to be flushed a bit differently. Not a real 191b1df07bdSGlauber de Oliveira Costa * performance problem because this does not happen often. 192b1df07bdSGlauber de Oliveira Costa */ 193b1df07bdSGlauber de Oliveira Costa static void native_flush_tlb_global(void) 194b1df07bdSGlauber de Oliveira Costa { 195b1df07bdSGlauber de Oliveira Costa __native_flush_tlb_global(); 196b1df07bdSGlauber de Oliveira Costa } 197b1df07bdSGlauber de Oliveira Costa 198b1df07bdSGlauber de Oliveira Costa static void native_flush_tlb_single(unsigned long addr) 199b1df07bdSGlauber de Oliveira Costa { 200b1df07bdSGlauber de Oliveira Costa __native_flush_tlb_single(addr); 201b1df07bdSGlauber de Oliveira Costa } 202b1df07bdSGlauber de Oliveira Costa 203c5905afbSIngo Molnar struct static_key paravirt_steal_enabled; 204c5905afbSIngo Molnar struct static_key paravirt_steal_rq_enabled; 2053c404b57SGlauber Costa 2063c404b57SGlauber Costa static u64 native_steal_clock(int cpu) 2073c404b57SGlauber Costa { 2083c404b57SGlauber Costa return 0; 2093c404b57SGlauber Costa } 2103c404b57SGlauber Costa 211b1df07bdSGlauber de Oliveira Costa /* These are in entry.S */ 212b1df07bdSGlauber de Oliveira Costa extern void native_iret(void); 213d75cd22fSJeremy Fitzhardinge extern void native_irq_enable_sysexit(void); 2142be29982SJeremy Fitzhardinge extern void native_usergs_sysret32(void); 2152be29982SJeremy Fitzhardinge extern void native_usergs_sysret64(void); 216b1df07bdSGlauber de Oliveira Costa 217b1df07bdSGlauber de Oliveira Costa static struct resource reserve_ioports = { 218b1df07bdSGlauber de Oliveira Costa .start = 0, 219b1df07bdSGlauber de Oliveira Costa .end = IO_SPACE_LIMIT, 220b1df07bdSGlauber de Oliveira Costa .name = "paravirt-ioport", 221b1df07bdSGlauber de Oliveira Costa .flags = IORESOURCE_IO | IORESOURCE_BUSY, 222b1df07bdSGlauber de Oliveira Costa }; 223b1df07bdSGlauber de Oliveira Costa 224b1df07bdSGlauber de Oliveira Costa /* 225b1df07bdSGlauber de Oliveira Costa * Reserve the whole legacy IO space to prevent any legacy drivers 226b1df07bdSGlauber de Oliveira Costa * from wasting time probing for their hardware. This is a fairly 227b1df07bdSGlauber de Oliveira Costa * brute-force approach to disabling all non-virtual drivers. 228b1df07bdSGlauber de Oliveira Costa * 229b1df07bdSGlauber de Oliveira Costa * Note that this must be called very early to have any effect. 230b1df07bdSGlauber de Oliveira Costa */ 231b1df07bdSGlauber de Oliveira Costa int paravirt_disable_iospace(void) 232b1df07bdSGlauber de Oliveira Costa { 233f7743fe6SJeremy Fitzhardinge return request_resource(&ioport_resource, &reserve_ioports); 234b1df07bdSGlauber de Oliveira Costa } 235b1df07bdSGlauber de Oliveira Costa 236b1df07bdSGlauber de Oliveira Costa static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE; 237b1df07bdSGlauber de Oliveira Costa 238b1df07bdSGlauber de Oliveira Costa static inline void enter_lazy(enum paravirt_lazy_mode mode) 239b1df07bdSGlauber de Oliveira Costa { 240c6ae41e7SAlex Shi BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE); 241b1df07bdSGlauber de Oliveira Costa 242c6ae41e7SAlex Shi this_cpu_write(paravirt_lazy_mode, mode); 243b1df07bdSGlauber de Oliveira Costa } 244b1df07bdSGlauber de Oliveira Costa 245b407fc57SJeremy Fitzhardinge static void leave_lazy(enum paravirt_lazy_mode mode) 246b1df07bdSGlauber de Oliveira Costa { 247c6ae41e7SAlex Shi BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode); 248b1df07bdSGlauber de Oliveira Costa 249c6ae41e7SAlex Shi this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE); 250b1df07bdSGlauber de Oliveira Costa } 251b1df07bdSGlauber de Oliveira Costa 252b1df07bdSGlauber de Oliveira Costa void paravirt_enter_lazy_mmu(void) 253b1df07bdSGlauber de Oliveira Costa { 254b1df07bdSGlauber de Oliveira Costa enter_lazy(PARAVIRT_LAZY_MMU); 255b1df07bdSGlauber de Oliveira Costa } 256b1df07bdSGlauber de Oliveira Costa 257b1df07bdSGlauber de Oliveira Costa void paravirt_leave_lazy_mmu(void) 258b1df07bdSGlauber de Oliveira Costa { 259b407fc57SJeremy Fitzhardinge leave_lazy(PARAVIRT_LAZY_MMU); 260b1df07bdSGlauber de Oliveira Costa } 261b1df07bdSGlauber de Oliveira Costa 262511ba86eSBoris Ostrovsky void paravirt_flush_lazy_mmu(void) 263511ba86eSBoris Ostrovsky { 264511ba86eSBoris Ostrovsky preempt_disable(); 265511ba86eSBoris Ostrovsky 266511ba86eSBoris Ostrovsky if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) { 267511ba86eSBoris Ostrovsky arch_leave_lazy_mmu_mode(); 268511ba86eSBoris Ostrovsky arch_enter_lazy_mmu_mode(); 269511ba86eSBoris Ostrovsky } 270511ba86eSBoris Ostrovsky 271511ba86eSBoris Ostrovsky preempt_enable(); 272511ba86eSBoris Ostrovsky } 273511ba86eSBoris Ostrovsky 274224101edSJeremy Fitzhardinge void paravirt_start_context_switch(struct task_struct *prev) 275b1df07bdSGlauber de Oliveira Costa { 2762829b449SJeremy Fitzhardinge BUG_ON(preemptible()); 2772829b449SJeremy Fitzhardinge 278c6ae41e7SAlex Shi if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) { 279b407fc57SJeremy Fitzhardinge arch_leave_lazy_mmu_mode(); 280224101edSJeremy Fitzhardinge set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES); 281b407fc57SJeremy Fitzhardinge } 282b1df07bdSGlauber de Oliveira Costa enter_lazy(PARAVIRT_LAZY_CPU); 283b1df07bdSGlauber de Oliveira Costa } 284b1df07bdSGlauber de Oliveira Costa 285224101edSJeremy Fitzhardinge void paravirt_end_context_switch(struct task_struct *next) 286b1df07bdSGlauber de Oliveira Costa { 2872829b449SJeremy Fitzhardinge BUG_ON(preemptible()); 2882829b449SJeremy Fitzhardinge 289b407fc57SJeremy Fitzhardinge leave_lazy(PARAVIRT_LAZY_CPU); 290b407fc57SJeremy Fitzhardinge 291224101edSJeremy Fitzhardinge if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES)) 292b407fc57SJeremy Fitzhardinge arch_enter_lazy_mmu_mode(); 293b1df07bdSGlauber de Oliveira Costa } 294b1df07bdSGlauber de Oliveira Costa 295b1df07bdSGlauber de Oliveira Costa enum paravirt_lazy_mode paravirt_get_lazy_mode(void) 296b1df07bdSGlauber de Oliveira Costa { 297b8bcfe99SJeremy Fitzhardinge if (in_interrupt()) 298b8bcfe99SJeremy Fitzhardinge return PARAVIRT_LAZY_NONE; 299b8bcfe99SJeremy Fitzhardinge 300c6ae41e7SAlex Shi return this_cpu_read(paravirt_lazy_mode); 301b1df07bdSGlauber de Oliveira Costa } 302b1df07bdSGlauber de Oliveira Costa 303b1df07bdSGlauber de Oliveira Costa struct pv_info pv_info = { 304b1df07bdSGlauber de Oliveira Costa .name = "bare hardware", 305b1df07bdSGlauber de Oliveira Costa .paravirt_enabled = 0, 306b1df07bdSGlauber de Oliveira Costa .kernel_rpl = 0, 307b1df07bdSGlauber de Oliveira Costa .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */ 308318f5a2aSAndy Lutomirski 309318f5a2aSAndy Lutomirski #ifdef CONFIG_X86_64 310318f5a2aSAndy Lutomirski .extra_user_64bit_cs = __USER_CS, 311318f5a2aSAndy Lutomirski #endif 312b1df07bdSGlauber de Oliveira Costa }; 313b1df07bdSGlauber de Oliveira Costa 314b1df07bdSGlauber de Oliveira Costa struct pv_init_ops pv_init_ops = { 315b1df07bdSGlauber de Oliveira Costa .patch = native_patch, 316b1df07bdSGlauber de Oliveira Costa }; 317b1df07bdSGlauber de Oliveira Costa 318b1df07bdSGlauber de Oliveira Costa struct pv_time_ops pv_time_ops = { 319b1df07bdSGlauber de Oliveira Costa .sched_clock = native_sched_clock, 3203c404b57SGlauber Costa .steal_clock = native_steal_clock, 321b1df07bdSGlauber de Oliveira Costa }; 322b1df07bdSGlauber de Oliveira Costa 3239a55fdbeSAndi Kleen __visible struct pv_irq_ops pv_irq_ops = { 324ecb93d1cSJeremy Fitzhardinge .save_fl = __PV_IS_CALLEE_SAVE(native_save_fl), 325ecb93d1cSJeremy Fitzhardinge .restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl), 326ecb93d1cSJeremy Fitzhardinge .irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable), 327ecb93d1cSJeremy Fitzhardinge .irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable), 328b1df07bdSGlauber de Oliveira Costa .safe_halt = native_safe_halt, 329b1df07bdSGlauber de Oliveira Costa .halt = native_halt, 330fab58420SJeremy Fitzhardinge #ifdef CONFIG_X86_64 331fab58420SJeremy Fitzhardinge .adjust_exception_frame = paravirt_nop, 332fab58420SJeremy Fitzhardinge #endif 333b1df07bdSGlauber de Oliveira Costa }; 334b1df07bdSGlauber de Oliveira Costa 3359a55fdbeSAndi Kleen __visible struct pv_cpu_ops pv_cpu_ops = { 336b1df07bdSGlauber de Oliveira Costa .cpuid = native_cpuid, 337b1df07bdSGlauber de Oliveira Costa .get_debugreg = native_get_debugreg, 338b1df07bdSGlauber de Oliveira Costa .set_debugreg = native_set_debugreg, 339b1df07bdSGlauber de Oliveira Costa .clts = native_clts, 340b1df07bdSGlauber de Oliveira Costa .read_cr0 = native_read_cr0, 341b1df07bdSGlauber de Oliveira Costa .write_cr0 = native_write_cr0, 342b1df07bdSGlauber de Oliveira Costa .read_cr4 = native_read_cr4, 343b1df07bdSGlauber de Oliveira Costa .read_cr4_safe = native_read_cr4_safe, 344b1df07bdSGlauber de Oliveira Costa .write_cr4 = native_write_cr4, 34588b4755fSGlauber de Oliveira Costa #ifdef CONFIG_X86_64 34688b4755fSGlauber de Oliveira Costa .read_cr8 = native_read_cr8, 34788b4755fSGlauber de Oliveira Costa .write_cr8 = native_write_cr8, 34888b4755fSGlauber de Oliveira Costa #endif 349b1df07bdSGlauber de Oliveira Costa .wbinvd = native_wbinvd, 350b1df07bdSGlauber de Oliveira Costa .read_msr = native_read_msr_safe, 351b1df07bdSGlauber de Oliveira Costa .write_msr = native_write_msr_safe, 352b1df07bdSGlauber de Oliveira Costa .read_tsc = native_read_tsc, 353b1df07bdSGlauber de Oliveira Costa .read_pmc = native_read_pmc, 354e5aaac44SGlauber de Oliveira Costa .read_tscp = native_read_tscp, 355b1df07bdSGlauber de Oliveira Costa .load_tr_desc = native_load_tr_desc, 356b1df07bdSGlauber de Oliveira Costa .set_ldt = native_set_ldt, 357b1df07bdSGlauber de Oliveira Costa .load_gdt = native_load_gdt, 358b1df07bdSGlauber de Oliveira Costa .load_idt = native_load_idt, 359b1df07bdSGlauber de Oliveira Costa .store_idt = native_store_idt, 360b1df07bdSGlauber de Oliveira Costa .store_tr = native_store_tr, 361b1df07bdSGlauber de Oliveira Costa .load_tls = native_load_tls, 3629f9d489aSJeremy Fitzhardinge #ifdef CONFIG_X86_64 3639f9d489aSJeremy Fitzhardinge .load_gs_index = native_load_gs_index, 3649f9d489aSJeremy Fitzhardinge #endif 365b1df07bdSGlauber de Oliveira Costa .write_ldt_entry = native_write_ldt_entry, 366b1df07bdSGlauber de Oliveira Costa .write_gdt_entry = native_write_gdt_entry, 367b1df07bdSGlauber de Oliveira Costa .write_idt_entry = native_write_idt_entry, 36838ffbe66SJeremy Fitzhardinge 36938ffbe66SJeremy Fitzhardinge .alloc_ldt = paravirt_nop, 37038ffbe66SJeremy Fitzhardinge .free_ldt = paravirt_nop, 37138ffbe66SJeremy Fitzhardinge 372b1df07bdSGlauber de Oliveira Costa .load_sp0 = native_load_sp0, 373b1df07bdSGlauber de Oliveira Costa 374102d0a4bSJeremy Fitzhardinge #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION) 375d75cd22fSJeremy Fitzhardinge .irq_enable_sysexit = native_irq_enable_sysexit, 376102d0a4bSJeremy Fitzhardinge #endif 3772be29982SJeremy Fitzhardinge #ifdef CONFIG_X86_64 378102d0a4bSJeremy Fitzhardinge #ifdef CONFIG_IA32_EMULATION 3792be29982SJeremy Fitzhardinge .usergs_sysret32 = native_usergs_sysret32, 380102d0a4bSJeremy Fitzhardinge #endif 3812be29982SJeremy Fitzhardinge .usergs_sysret64 = native_usergs_sysret64, 382d75cd22fSJeremy Fitzhardinge #endif 383b1df07bdSGlauber de Oliveira Costa .iret = native_iret, 384e801f864SGlauber de Oliveira Costa .swapgs = native_swapgs, 385b1df07bdSGlauber de Oliveira Costa 386b1df07bdSGlauber de Oliveira Costa .set_iopl_mask = native_set_iopl_mask, 387b1df07bdSGlauber de Oliveira Costa .io_delay = native_io_delay, 388b1df07bdSGlauber de Oliveira Costa 389224101edSJeremy Fitzhardinge .start_context_switch = paravirt_nop, 390224101edSJeremy Fitzhardinge .end_context_switch = paravirt_nop, 391b1df07bdSGlauber de Oliveira Costa }; 392b1df07bdSGlauber de Oliveira Costa 39380271972SMasami Hiramatsu /* At this point, native_get/set_debugreg has real function entries */ 394376e2424SMasami Hiramatsu NOKPROBE_SYMBOL(native_get_debugreg); 39580271972SMasami Hiramatsu NOKPROBE_SYMBOL(native_set_debugreg); 39680271972SMasami Hiramatsu NOKPROBE_SYMBOL(native_load_idt); 397376e2424SMasami Hiramatsu 398b1df07bdSGlauber de Oliveira Costa struct pv_apic_ops pv_apic_ops = { 399b1df07bdSGlauber de Oliveira Costa #ifdef CONFIG_X86_LOCAL_APIC 400b1df07bdSGlauber de Oliveira Costa .startup_ipi_hook = paravirt_nop, 401b1df07bdSGlauber de Oliveira Costa #endif 402b1df07bdSGlauber de Oliveira Costa }; 403b1df07bdSGlauber de Oliveira Costa 40441edafdbSJeremy Fitzhardinge #if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE) 40541edafdbSJeremy Fitzhardinge /* 32-bit pagetable entries */ 406da5de7c2SJeremy Fitzhardinge #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32) 40741edafdbSJeremy Fitzhardinge #else 40841edafdbSJeremy Fitzhardinge /* 64-bit pagetable entries */ 409da5de7c2SJeremy Fitzhardinge #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64) 41041edafdbSJeremy Fitzhardinge #endif 41141edafdbSJeremy Fitzhardinge 412b1df07bdSGlauber de Oliveira Costa struct pv_mmu_ops pv_mmu_ops = { 413b1df07bdSGlauber de Oliveira Costa 414b1df07bdSGlauber de Oliveira Costa .read_cr2 = native_read_cr2, 415b1df07bdSGlauber de Oliveira Costa .write_cr2 = native_write_cr2, 416b1df07bdSGlauber de Oliveira Costa .read_cr3 = native_read_cr3, 417b1df07bdSGlauber de Oliveira Costa .write_cr3 = native_write_cr3, 418b1df07bdSGlauber de Oliveira Costa 419b1df07bdSGlauber de Oliveira Costa .flush_tlb_user = native_flush_tlb, 420b1df07bdSGlauber de Oliveira Costa .flush_tlb_kernel = native_flush_tlb_global, 421b1df07bdSGlauber de Oliveira Costa .flush_tlb_single = native_flush_tlb_single, 422b1df07bdSGlauber de Oliveira Costa .flush_tlb_others = native_flush_tlb_others, 423b1df07bdSGlauber de Oliveira Costa 424eba0045fSJeremy Fitzhardinge .pgd_alloc = __paravirt_pgd_alloc, 425eba0045fSJeremy Fitzhardinge .pgd_free = paravirt_nop, 426eba0045fSJeremy Fitzhardinge 4276944a9c8SJeremy Fitzhardinge .alloc_pte = paravirt_nop, 4286944a9c8SJeremy Fitzhardinge .alloc_pmd = paravirt_nop, 4292761fa09SJeremy Fitzhardinge .alloc_pud = paravirt_nop, 4306944a9c8SJeremy Fitzhardinge .release_pte = paravirt_nop, 4316944a9c8SJeremy Fitzhardinge .release_pmd = paravirt_nop, 4322761fa09SJeremy Fitzhardinge .release_pud = paravirt_nop, 433b1df07bdSGlauber de Oliveira Costa 434b1df07bdSGlauber de Oliveira Costa .set_pte = native_set_pte, 435b1df07bdSGlauber de Oliveira Costa .set_pte_at = native_set_pte_at, 436b1df07bdSGlauber de Oliveira Costa .set_pmd = native_set_pmd, 437331127f7SAndrea Arcangeli .set_pmd_at = native_set_pmd_at, 438b1df07bdSGlauber de Oliveira Costa .pte_update = paravirt_nop, 439b1df07bdSGlauber de Oliveira Costa .pte_update_defer = paravirt_nop, 440331127f7SAndrea Arcangeli .pmd_update = paravirt_nop, 441331127f7SAndrea Arcangeli .pmd_update_defer = paravirt_nop, 442b1df07bdSGlauber de Oliveira Costa 44308b882c6SJeremy Fitzhardinge .ptep_modify_prot_start = __ptep_modify_prot_start, 44408b882c6SJeremy Fitzhardinge .ptep_modify_prot_commit = __ptep_modify_prot_commit, 44508b882c6SJeremy Fitzhardinge 446f95f2f7bSEduardo Habkost #if PAGETABLE_LEVELS >= 3 447b1df07bdSGlauber de Oliveira Costa #ifdef CONFIG_X86_PAE 448b1df07bdSGlauber de Oliveira Costa .set_pte_atomic = native_set_pte_atomic, 449b1df07bdSGlauber de Oliveira Costa .pte_clear = native_pte_clear, 450b1df07bdSGlauber de Oliveira Costa .pmd_clear = native_pmd_clear, 451f95f2f7bSEduardo Habkost #endif 452f95f2f7bSEduardo Habkost .set_pud = native_set_pud, 453da5de7c2SJeremy Fitzhardinge 454da5de7c2SJeremy Fitzhardinge .pmd_val = PTE_IDENT, 455da5de7c2SJeremy Fitzhardinge .make_pmd = PTE_IDENT, 456f95f2f7bSEduardo Habkost 457f95f2f7bSEduardo Habkost #if PAGETABLE_LEVELS == 4 458da5de7c2SJeremy Fitzhardinge .pud_val = PTE_IDENT, 459da5de7c2SJeremy Fitzhardinge .make_pud = PTE_IDENT, 460da5de7c2SJeremy Fitzhardinge 461f95f2f7bSEduardo Habkost .set_pgd = native_set_pgd, 462b1df07bdSGlauber de Oliveira Costa #endif 463f95f2f7bSEduardo Habkost #endif /* PAGETABLE_LEVELS >= 3 */ 464b1df07bdSGlauber de Oliveira Costa 465da5de7c2SJeremy Fitzhardinge .pte_val = PTE_IDENT, 466da5de7c2SJeremy Fitzhardinge .pgd_val = PTE_IDENT, 467b1df07bdSGlauber de Oliveira Costa 468da5de7c2SJeremy Fitzhardinge .make_pte = PTE_IDENT, 469da5de7c2SJeremy Fitzhardinge .make_pgd = PTE_IDENT, 470b1df07bdSGlauber de Oliveira Costa 471b1df07bdSGlauber de Oliveira Costa .dup_mmap = paravirt_nop, 472b1df07bdSGlauber de Oliveira Costa .exit_mmap = paravirt_nop, 473b1df07bdSGlauber de Oliveira Costa .activate_mm = paravirt_nop, 474b1df07bdSGlauber de Oliveira Costa 475b1df07bdSGlauber de Oliveira Costa .lazy_mode = { 476b1df07bdSGlauber de Oliveira Costa .enter = paravirt_nop, 477b1df07bdSGlauber de Oliveira Costa .leave = paravirt_nop, 478511ba86eSBoris Ostrovsky .flush = paravirt_nop, 479b1df07bdSGlauber de Oliveira Costa }, 480aeaaa59cSJeremy Fitzhardinge 481aeaaa59cSJeremy Fitzhardinge .set_fixmap = native_set_fixmap, 482b1df07bdSGlauber de Oliveira Costa }; 483b1df07bdSGlauber de Oliveira Costa 484b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL_GPL(pv_time_ops); 485b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL (pv_cpu_ops); 486b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL (pv_mmu_ops); 487b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL_GPL(pv_apic_ops); 488b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL_GPL(pv_info); 489b1df07bdSGlauber de Oliveira Costa EXPORT_SYMBOL (pv_irq_ops); 490