1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* Paravirtualization interfaces 3 Copyright (C) 2006 Rusty Russell IBM Corporation 4 5 6 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc 7 */ 8 9 #include <linux/errno.h> 10 #include <linux/init.h> 11 #include <linux/export.h> 12 #include <linux/efi.h> 13 #include <linux/bcd.h> 14 #include <linux/highmem.h> 15 #include <linux/kprobes.h> 16 #include <linux/pgtable.h> 17 #include <linux/static_call.h> 18 19 #include <asm/bug.h> 20 #include <asm/paravirt.h> 21 #include <asm/debugreg.h> 22 #include <asm/desc.h> 23 #include <asm/setup.h> 24 #include <asm/time.h> 25 #include <asm/pgalloc.h> 26 #include <asm/irq.h> 27 #include <asm/delay.h> 28 #include <asm/fixmap.h> 29 #include <asm/apic.h> 30 #include <asm/tlbflush.h> 31 #include <asm/timer.h> 32 #include <asm/special_insns.h> 33 #include <asm/tlb.h> 34 #include <asm/io_bitmap.h> 35 36 /* 37 * nop stub, which must not clobber anything *including the stack* to 38 * avoid confusing the entry prologues. 39 */ 40 DEFINE_PARAVIRT_ASM(_paravirt_nop, "", .entry.text); 41 42 /* stub always returning 0. */ 43 DEFINE_PARAVIRT_ASM(paravirt_ret0, "xor %eax,%eax", .entry.text); 44 45 void __init default_banner(void) 46 { 47 printk(KERN_INFO "Booting paravirtualized kernel on %s\n", 48 pv_info.name); 49 } 50 51 /* Undefined instruction for dealing with missing ops pointers. */ 52 noinstr void paravirt_BUG(void) 53 { 54 BUG(); 55 } 56 57 static unsigned paravirt_patch_call(void *insn_buff, const void *target, 58 unsigned long addr, unsigned len) 59 { 60 __text_gen_insn(insn_buff, CALL_INSN_OPCODE, 61 (void *)addr, target, CALL_INSN_SIZE); 62 return CALL_INSN_SIZE; 63 } 64 65 #ifdef CONFIG_PARAVIRT_XXL 66 /* identity function, which can be inlined */ 67 u64 notrace _paravirt_ident_64(u64 x) 68 { 69 return x; 70 } 71 #endif 72 73 DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key); 74 75 void __init native_pv_lock_init(void) 76 { 77 if (!boot_cpu_has(X86_FEATURE_HYPERVISOR)) 78 static_branch_disable(&virt_spin_lock_key); 79 } 80 81 unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, 82 unsigned int len) 83 { 84 /* 85 * Neat trick to map patch type back to the call within the 86 * corresponding structure. 87 */ 88 void *opfunc = *((void **)&pv_ops + type); 89 unsigned ret; 90 91 if (opfunc == NULL) 92 /* If there's no function, patch it with paravirt_BUG() */ 93 ret = paravirt_patch_call(insn_buff, paravirt_BUG, addr, len); 94 else if (opfunc == _paravirt_nop) 95 ret = 0; 96 else 97 /* Otherwise call the function. */ 98 ret = paravirt_patch_call(insn_buff, opfunc, addr, len); 99 100 return ret; 101 } 102 103 struct static_key paravirt_steal_enabled; 104 struct static_key paravirt_steal_rq_enabled; 105 106 static u64 native_steal_clock(int cpu) 107 { 108 return 0; 109 } 110 111 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); 112 DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); 113 114 void paravirt_set_sched_clock(u64 (*func)(void)) 115 { 116 static_call_update(pv_sched_clock, func); 117 } 118 119 /* These are in entry.S */ 120 static struct resource reserve_ioports = { 121 .start = 0, 122 .end = IO_SPACE_LIMIT, 123 .name = "paravirt-ioport", 124 .flags = IORESOURCE_IO | IORESOURCE_BUSY, 125 }; 126 127 /* 128 * Reserve the whole legacy IO space to prevent any legacy drivers 129 * from wasting time probing for their hardware. This is a fairly 130 * brute-force approach to disabling all non-virtual drivers. 131 * 132 * Note that this must be called very early to have any effect. 133 */ 134 int paravirt_disable_iospace(void) 135 { 136 return request_resource(&ioport_resource, &reserve_ioports); 137 } 138 139 static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE; 140 141 static inline void enter_lazy(enum paravirt_lazy_mode mode) 142 { 143 BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE); 144 145 this_cpu_write(paravirt_lazy_mode, mode); 146 } 147 148 static void leave_lazy(enum paravirt_lazy_mode mode) 149 { 150 BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode); 151 152 this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE); 153 } 154 155 void paravirt_enter_lazy_mmu(void) 156 { 157 enter_lazy(PARAVIRT_LAZY_MMU); 158 } 159 160 void paravirt_leave_lazy_mmu(void) 161 { 162 leave_lazy(PARAVIRT_LAZY_MMU); 163 } 164 165 void paravirt_flush_lazy_mmu(void) 166 { 167 preempt_disable(); 168 169 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) { 170 arch_leave_lazy_mmu_mode(); 171 arch_enter_lazy_mmu_mode(); 172 } 173 174 preempt_enable(); 175 } 176 177 #ifdef CONFIG_PARAVIRT_XXL 178 void paravirt_start_context_switch(struct task_struct *prev) 179 { 180 BUG_ON(preemptible()); 181 182 if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) { 183 arch_leave_lazy_mmu_mode(); 184 set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES); 185 } 186 enter_lazy(PARAVIRT_LAZY_CPU); 187 } 188 189 void paravirt_end_context_switch(struct task_struct *next) 190 { 191 BUG_ON(preemptible()); 192 193 leave_lazy(PARAVIRT_LAZY_CPU); 194 195 if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES)) 196 arch_enter_lazy_mmu_mode(); 197 } 198 199 static noinstr unsigned long pv_native_read_cr2(void) 200 { 201 return native_read_cr2(); 202 } 203 204 static noinstr void pv_native_write_cr2(unsigned long val) 205 { 206 native_write_cr2(val); 207 } 208 209 static noinstr unsigned long pv_native_get_debugreg(int regno) 210 { 211 return native_get_debugreg(regno); 212 } 213 214 static noinstr void pv_native_set_debugreg(int regno, unsigned long val) 215 { 216 native_set_debugreg(regno, val); 217 } 218 219 static noinstr void pv_native_irq_enable(void) 220 { 221 native_irq_enable(); 222 } 223 224 static noinstr void pv_native_irq_disable(void) 225 { 226 native_irq_disable(); 227 } 228 #endif 229 230 enum paravirt_lazy_mode paravirt_get_lazy_mode(void) 231 { 232 if (in_interrupt()) 233 return PARAVIRT_LAZY_NONE; 234 235 return this_cpu_read(paravirt_lazy_mode); 236 } 237 238 struct pv_info pv_info = { 239 .name = "bare hardware", 240 #ifdef CONFIG_PARAVIRT_XXL 241 .extra_user_64bit_cs = __USER_CS, 242 #endif 243 }; 244 245 /* 64-bit pagetable entries */ 246 #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64) 247 248 struct paravirt_patch_template pv_ops = { 249 /* Cpu ops. */ 250 .cpu.io_delay = native_io_delay, 251 252 #ifdef CONFIG_PARAVIRT_XXL 253 .cpu.cpuid = native_cpuid, 254 .cpu.get_debugreg = pv_native_get_debugreg, 255 .cpu.set_debugreg = pv_native_set_debugreg, 256 .cpu.read_cr0 = native_read_cr0, 257 .cpu.write_cr0 = native_write_cr0, 258 .cpu.write_cr4 = native_write_cr4, 259 .cpu.wbinvd = native_wbinvd, 260 .cpu.read_msr = native_read_msr, 261 .cpu.write_msr = native_write_msr, 262 .cpu.read_msr_safe = native_read_msr_safe, 263 .cpu.write_msr_safe = native_write_msr_safe, 264 .cpu.read_pmc = native_read_pmc, 265 .cpu.load_tr_desc = native_load_tr_desc, 266 .cpu.set_ldt = native_set_ldt, 267 .cpu.load_gdt = native_load_gdt, 268 .cpu.load_idt = native_load_idt, 269 .cpu.store_tr = native_store_tr, 270 .cpu.load_tls = native_load_tls, 271 .cpu.load_gs_index = native_load_gs_index, 272 .cpu.write_ldt_entry = native_write_ldt_entry, 273 .cpu.write_gdt_entry = native_write_gdt_entry, 274 .cpu.write_idt_entry = native_write_idt_entry, 275 276 .cpu.alloc_ldt = paravirt_nop, 277 .cpu.free_ldt = paravirt_nop, 278 279 .cpu.load_sp0 = native_load_sp0, 280 281 #ifdef CONFIG_X86_IOPL_IOPERM 282 .cpu.invalidate_io_bitmap = native_tss_invalidate_io_bitmap, 283 .cpu.update_io_bitmap = native_tss_update_io_bitmap, 284 #endif 285 286 .cpu.start_context_switch = paravirt_nop, 287 .cpu.end_context_switch = paravirt_nop, 288 289 /* Irq ops. */ 290 .irq.save_fl = __PV_IS_CALLEE_SAVE(native_save_fl), 291 .irq.irq_disable = __PV_IS_CALLEE_SAVE(pv_native_irq_disable), 292 .irq.irq_enable = __PV_IS_CALLEE_SAVE(pv_native_irq_enable), 293 .irq.safe_halt = native_safe_halt, 294 .irq.halt = native_halt, 295 #endif /* CONFIG_PARAVIRT_XXL */ 296 297 /* Mmu ops. */ 298 .mmu.flush_tlb_user = native_flush_tlb_local, 299 .mmu.flush_tlb_kernel = native_flush_tlb_global, 300 .mmu.flush_tlb_one_user = native_flush_tlb_one_user, 301 .mmu.flush_tlb_multi = native_flush_tlb_multi, 302 .mmu.tlb_remove_table = 303 (void (*)(struct mmu_gather *, void *))tlb_remove_page, 304 305 .mmu.exit_mmap = paravirt_nop, 306 .mmu.notify_page_enc_status_changed = paravirt_nop, 307 308 #ifdef CONFIG_PARAVIRT_XXL 309 .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(pv_native_read_cr2), 310 .mmu.write_cr2 = pv_native_write_cr2, 311 .mmu.read_cr3 = __native_read_cr3, 312 .mmu.write_cr3 = native_write_cr3, 313 314 .mmu.pgd_alloc = __paravirt_pgd_alloc, 315 .mmu.pgd_free = paravirt_nop, 316 317 .mmu.alloc_pte = paravirt_nop, 318 .mmu.alloc_pmd = paravirt_nop, 319 .mmu.alloc_pud = paravirt_nop, 320 .mmu.alloc_p4d = paravirt_nop, 321 .mmu.release_pte = paravirt_nop, 322 .mmu.release_pmd = paravirt_nop, 323 .mmu.release_pud = paravirt_nop, 324 .mmu.release_p4d = paravirt_nop, 325 326 .mmu.set_pte = native_set_pte, 327 .mmu.set_pmd = native_set_pmd, 328 329 .mmu.ptep_modify_prot_start = __ptep_modify_prot_start, 330 .mmu.ptep_modify_prot_commit = __ptep_modify_prot_commit, 331 332 .mmu.set_pud = native_set_pud, 333 334 .mmu.pmd_val = PTE_IDENT, 335 .mmu.make_pmd = PTE_IDENT, 336 337 .mmu.pud_val = PTE_IDENT, 338 .mmu.make_pud = PTE_IDENT, 339 340 .mmu.set_p4d = native_set_p4d, 341 342 #if CONFIG_PGTABLE_LEVELS >= 5 343 .mmu.p4d_val = PTE_IDENT, 344 .mmu.make_p4d = PTE_IDENT, 345 346 .mmu.set_pgd = native_set_pgd, 347 #endif /* CONFIG_PGTABLE_LEVELS >= 5 */ 348 349 .mmu.pte_val = PTE_IDENT, 350 .mmu.pgd_val = PTE_IDENT, 351 352 .mmu.make_pte = PTE_IDENT, 353 .mmu.make_pgd = PTE_IDENT, 354 355 .mmu.dup_mmap = paravirt_nop, 356 .mmu.activate_mm = paravirt_nop, 357 358 .mmu.lazy_mode = { 359 .enter = paravirt_nop, 360 .leave = paravirt_nop, 361 .flush = paravirt_nop, 362 }, 363 364 .mmu.set_fixmap = native_set_fixmap, 365 #endif /* CONFIG_PARAVIRT_XXL */ 366 367 #if defined(CONFIG_PARAVIRT_SPINLOCKS) 368 /* Lock ops. */ 369 #ifdef CONFIG_SMP 370 .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, 371 .lock.queued_spin_unlock = 372 PV_CALLEE_SAVE(__native_queued_spin_unlock), 373 .lock.wait = paravirt_nop, 374 .lock.kick = paravirt_nop, 375 .lock.vcpu_is_preempted = 376 PV_CALLEE_SAVE(__native_vcpu_is_preempted), 377 #endif /* SMP */ 378 #endif 379 }; 380 381 #ifdef CONFIG_PARAVIRT_XXL 382 NOKPROBE_SYMBOL(native_load_idt); 383 #endif 384 385 EXPORT_SYMBOL(pv_ops); 386 EXPORT_SYMBOL_GPL(pv_info); 387