1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_PARAVIRT_H 3 #define _ASM_X86_PARAVIRT_H 4 /* Various instructions on x86 need to be replaced for 5 * para-virtualization: those hooks are defined here. */ 6 7 #include <asm/paravirt_types.h> 8 9 #ifdef CONFIG_PARAVIRT 10 #include <asm/pgtable_types.h> 11 #include <asm/asm.h> 12 #include <asm/nospec-branch.h> 13 14 #ifndef __ASSEMBLY__ 15 #include <linux/bug.h> 16 #include <linux/types.h> 17 #include <linux/cpumask.h> 18 #include <linux/static_call_types.h> 19 #include <asm/frame.h> 20 21 u64 dummy_steal_clock(int cpu); 22 u64 dummy_sched_clock(void); 23 24 DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); 25 DECLARE_STATIC_CALL(pv_sched_clock, dummy_sched_clock); 26 27 void paravirt_set_sched_clock(u64 (*func)(void)); 28 29 static __always_inline u64 paravirt_sched_clock(void) 30 { 31 return static_call(pv_sched_clock)(); 32 } 33 34 struct static_key; 35 extern struct static_key paravirt_steal_enabled; 36 extern struct static_key paravirt_steal_rq_enabled; 37 38 __visible void __native_queued_spin_unlock(struct qspinlock *lock); 39 bool pv_is_native_spin_unlock(void); 40 __visible bool __native_vcpu_is_preempted(long cpu); 41 bool pv_is_native_vcpu_is_preempted(void); 42 43 static inline u64 paravirt_steal_clock(int cpu) 44 { 45 return static_call(pv_steal_clock)(cpu); 46 } 47 48 #ifdef CONFIG_PARAVIRT_SPINLOCKS 49 void __init paravirt_set_cap(void); 50 #endif 51 52 /* The paravirtualized I/O functions */ 53 static inline void slow_down_io(void) 54 { 55 PVOP_VCALL0(cpu.io_delay); 56 #ifdef REALLY_SLOW_IO 57 PVOP_VCALL0(cpu.io_delay); 58 PVOP_VCALL0(cpu.io_delay); 59 PVOP_VCALL0(cpu.io_delay); 60 #endif 61 } 62 63 void native_flush_tlb_local(void); 64 void native_flush_tlb_global(void); 65 void native_flush_tlb_one_user(unsigned long addr); 66 void native_flush_tlb_multi(const struct cpumask *cpumask, 67 const struct flush_tlb_info *info); 68 69 static inline void __flush_tlb_local(void) 70 { 71 PVOP_VCALL0(mmu.flush_tlb_user); 72 } 73 74 static inline void __flush_tlb_global(void) 75 { 76 PVOP_VCALL0(mmu.flush_tlb_kernel); 77 } 78 79 static inline void __flush_tlb_one_user(unsigned long addr) 80 { 81 PVOP_VCALL1(mmu.flush_tlb_one_user, addr); 82 } 83 84 static inline void __flush_tlb_multi(const struct cpumask *cpumask, 85 const struct flush_tlb_info *info) 86 { 87 PVOP_VCALL2(mmu.flush_tlb_multi, cpumask, info); 88 } 89 90 static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table) 91 { 92 PVOP_VCALL2(mmu.tlb_remove_table, tlb, table); 93 } 94 95 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 96 { 97 PVOP_VCALL1(mmu.exit_mmap, mm); 98 } 99 100 static inline void notify_page_enc_status_changed(unsigned long pfn, 101 int npages, bool enc) 102 { 103 PVOP_VCALL3(mmu.notify_page_enc_status_changed, pfn, npages, enc); 104 } 105 106 #ifdef CONFIG_PARAVIRT_XXL 107 static inline void load_sp0(unsigned long sp0) 108 { 109 PVOP_VCALL1(cpu.load_sp0, sp0); 110 } 111 112 /* The paravirtualized CPUID instruction. */ 113 static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 114 unsigned int *ecx, unsigned int *edx) 115 { 116 PVOP_VCALL4(cpu.cpuid, eax, ebx, ecx, edx); 117 } 118 119 /* 120 * These special macros can be used to get or set a debugging register 121 */ 122 static __always_inline unsigned long paravirt_get_debugreg(int reg) 123 { 124 return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg); 125 } 126 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) 127 static __always_inline void set_debugreg(unsigned long val, int reg) 128 { 129 PVOP_VCALL2(cpu.set_debugreg, reg, val); 130 } 131 132 static inline unsigned long read_cr0(void) 133 { 134 return PVOP_CALL0(unsigned long, cpu.read_cr0); 135 } 136 137 static inline void write_cr0(unsigned long x) 138 { 139 PVOP_VCALL1(cpu.write_cr0, x); 140 } 141 142 static __always_inline unsigned long read_cr2(void) 143 { 144 return PVOP_ALT_CALLEE0(unsigned long, mmu.read_cr2, 145 "mov %%cr2, %%rax;", 146 ALT_NOT(X86_FEATURE_XENPV)); 147 } 148 149 static __always_inline void write_cr2(unsigned long x) 150 { 151 PVOP_VCALL1(mmu.write_cr2, x); 152 } 153 154 static inline unsigned long __read_cr3(void) 155 { 156 return PVOP_ALT_CALL0(unsigned long, mmu.read_cr3, 157 "mov %%cr3, %%rax;", ALT_NOT(X86_FEATURE_XENPV)); 158 } 159 160 static inline void write_cr3(unsigned long x) 161 { 162 PVOP_ALT_VCALL1(mmu.write_cr3, x, 163 "mov %%rdi, %%cr3", ALT_NOT(X86_FEATURE_XENPV)); 164 } 165 166 static inline void __write_cr4(unsigned long x) 167 { 168 PVOP_VCALL1(cpu.write_cr4, x); 169 } 170 171 static __always_inline void arch_safe_halt(void) 172 { 173 PVOP_VCALL0(irq.safe_halt); 174 } 175 176 static inline void halt(void) 177 { 178 PVOP_VCALL0(irq.halt); 179 } 180 181 extern noinstr void pv_native_wbinvd(void); 182 183 static __always_inline void wbinvd(void) 184 { 185 PVOP_ALT_VCALL0(cpu.wbinvd, "wbinvd", ALT_NOT(X86_FEATURE_XENPV)); 186 } 187 188 static inline u64 paravirt_read_msr(unsigned msr) 189 { 190 return PVOP_CALL1(u64, cpu.read_msr, msr); 191 } 192 193 static inline void paravirt_write_msr(unsigned msr, 194 unsigned low, unsigned high) 195 { 196 PVOP_VCALL3(cpu.write_msr, msr, low, high); 197 } 198 199 static inline u64 paravirt_read_msr_safe(unsigned msr, int *err) 200 { 201 return PVOP_CALL2(u64, cpu.read_msr_safe, msr, err); 202 } 203 204 static inline int paravirt_write_msr_safe(unsigned msr, 205 unsigned low, unsigned high) 206 { 207 return PVOP_CALL3(int, cpu.write_msr_safe, msr, low, high); 208 } 209 210 #define rdmsr(msr, val1, val2) \ 211 do { \ 212 u64 _l = paravirt_read_msr(msr); \ 213 val1 = (u32)_l; \ 214 val2 = _l >> 32; \ 215 } while (0) 216 217 #define wrmsr(msr, val1, val2) \ 218 do { \ 219 paravirt_write_msr(msr, val1, val2); \ 220 } while (0) 221 222 #define rdmsrl(msr, val) \ 223 do { \ 224 val = paravirt_read_msr(msr); \ 225 } while (0) 226 227 static inline void wrmsrl(unsigned msr, u64 val) 228 { 229 wrmsr(msr, (u32)val, (u32)(val>>32)); 230 } 231 232 #define wrmsr_safe(msr, a, b) paravirt_write_msr_safe(msr, a, b) 233 234 /* rdmsr with exception handling */ 235 #define rdmsr_safe(msr, a, b) \ 236 ({ \ 237 int _err; \ 238 u64 _l = paravirt_read_msr_safe(msr, &_err); \ 239 (*a) = (u32)_l; \ 240 (*b) = _l >> 32; \ 241 _err; \ 242 }) 243 244 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) 245 { 246 int err; 247 248 *p = paravirt_read_msr_safe(msr, &err); 249 return err; 250 } 251 252 static inline unsigned long long paravirt_read_pmc(int counter) 253 { 254 return PVOP_CALL1(u64, cpu.read_pmc, counter); 255 } 256 257 #define rdpmc(counter, low, high) \ 258 do { \ 259 u64 _l = paravirt_read_pmc(counter); \ 260 low = (u32)_l; \ 261 high = _l >> 32; \ 262 } while (0) 263 264 #define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter)) 265 266 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) 267 { 268 PVOP_VCALL2(cpu.alloc_ldt, ldt, entries); 269 } 270 271 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) 272 { 273 PVOP_VCALL2(cpu.free_ldt, ldt, entries); 274 } 275 276 static inline void load_TR_desc(void) 277 { 278 PVOP_VCALL0(cpu.load_tr_desc); 279 } 280 static inline void load_gdt(const struct desc_ptr *dtr) 281 { 282 PVOP_VCALL1(cpu.load_gdt, dtr); 283 } 284 static inline void load_idt(const struct desc_ptr *dtr) 285 { 286 PVOP_VCALL1(cpu.load_idt, dtr); 287 } 288 static inline void set_ldt(const void *addr, unsigned entries) 289 { 290 PVOP_VCALL2(cpu.set_ldt, addr, entries); 291 } 292 static inline unsigned long paravirt_store_tr(void) 293 { 294 return PVOP_CALL0(unsigned long, cpu.store_tr); 295 } 296 297 #define store_tr(tr) ((tr) = paravirt_store_tr()) 298 static inline void load_TLS(struct thread_struct *t, unsigned cpu) 299 { 300 PVOP_VCALL2(cpu.load_tls, t, cpu); 301 } 302 303 static inline void load_gs_index(unsigned int gs) 304 { 305 PVOP_VCALL1(cpu.load_gs_index, gs); 306 } 307 308 static inline void write_ldt_entry(struct desc_struct *dt, int entry, 309 const void *desc) 310 { 311 PVOP_VCALL3(cpu.write_ldt_entry, dt, entry, desc); 312 } 313 314 static inline void write_gdt_entry(struct desc_struct *dt, int entry, 315 void *desc, int type) 316 { 317 PVOP_VCALL4(cpu.write_gdt_entry, dt, entry, desc, type); 318 } 319 320 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) 321 { 322 PVOP_VCALL3(cpu.write_idt_entry, dt, entry, g); 323 } 324 325 #ifdef CONFIG_X86_IOPL_IOPERM 326 static inline void tss_invalidate_io_bitmap(void) 327 { 328 PVOP_VCALL0(cpu.invalidate_io_bitmap); 329 } 330 331 static inline void tss_update_io_bitmap(void) 332 { 333 PVOP_VCALL0(cpu.update_io_bitmap); 334 } 335 #endif 336 337 static inline void paravirt_activate_mm(struct mm_struct *prev, 338 struct mm_struct *next) 339 { 340 PVOP_VCALL2(mmu.activate_mm, prev, next); 341 } 342 343 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, 344 struct mm_struct *mm) 345 { 346 PVOP_VCALL2(mmu.dup_mmap, oldmm, mm); 347 } 348 349 static inline int paravirt_pgd_alloc(struct mm_struct *mm) 350 { 351 return PVOP_CALL1(int, mmu.pgd_alloc, mm); 352 } 353 354 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) 355 { 356 PVOP_VCALL2(mmu.pgd_free, mm, pgd); 357 } 358 359 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) 360 { 361 PVOP_VCALL2(mmu.alloc_pte, mm, pfn); 362 } 363 static inline void paravirt_release_pte(unsigned long pfn) 364 { 365 PVOP_VCALL1(mmu.release_pte, pfn); 366 } 367 368 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) 369 { 370 PVOP_VCALL2(mmu.alloc_pmd, mm, pfn); 371 } 372 373 static inline void paravirt_release_pmd(unsigned long pfn) 374 { 375 PVOP_VCALL1(mmu.release_pmd, pfn); 376 } 377 378 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) 379 { 380 PVOP_VCALL2(mmu.alloc_pud, mm, pfn); 381 } 382 static inline void paravirt_release_pud(unsigned long pfn) 383 { 384 PVOP_VCALL1(mmu.release_pud, pfn); 385 } 386 387 static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) 388 { 389 PVOP_VCALL2(mmu.alloc_p4d, mm, pfn); 390 } 391 392 static inline void paravirt_release_p4d(unsigned long pfn) 393 { 394 PVOP_VCALL1(mmu.release_p4d, pfn); 395 } 396 397 static inline pte_t __pte(pteval_t val) 398 { 399 return (pte_t) { PVOP_ALT_CALLEE1(pteval_t, mmu.make_pte, val, 400 "mov %%rdi, %%rax", 401 ALT_NOT(X86_FEATURE_XENPV)) }; 402 } 403 404 static inline pteval_t pte_val(pte_t pte) 405 { 406 return PVOP_ALT_CALLEE1(pteval_t, mmu.pte_val, pte.pte, 407 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 408 } 409 410 static inline pgd_t __pgd(pgdval_t val) 411 { 412 return (pgd_t) { PVOP_ALT_CALLEE1(pgdval_t, mmu.make_pgd, val, 413 "mov %%rdi, %%rax", 414 ALT_NOT(X86_FEATURE_XENPV)) }; 415 } 416 417 static inline pgdval_t pgd_val(pgd_t pgd) 418 { 419 return PVOP_ALT_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd, 420 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 421 } 422 423 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 424 static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, 425 pte_t *ptep) 426 { 427 pteval_t ret; 428 429 ret = PVOP_CALL3(pteval_t, mmu.ptep_modify_prot_start, vma, addr, ptep); 430 431 return (pte_t) { .pte = ret }; 432 } 433 434 static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, 435 pte_t *ptep, pte_t old_pte, pte_t pte) 436 { 437 438 PVOP_VCALL4(mmu.ptep_modify_prot_commit, vma, addr, ptep, pte.pte); 439 } 440 441 static inline void set_pte(pte_t *ptep, pte_t pte) 442 { 443 PVOP_VCALL2(mmu.set_pte, ptep, pte.pte); 444 } 445 446 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 447 { 448 PVOP_VCALL2(mmu.set_pmd, pmdp, native_pmd_val(pmd)); 449 } 450 451 static inline pmd_t __pmd(pmdval_t val) 452 { 453 return (pmd_t) { PVOP_ALT_CALLEE1(pmdval_t, mmu.make_pmd, val, 454 "mov %%rdi, %%rax", 455 ALT_NOT(X86_FEATURE_XENPV)) }; 456 } 457 458 static inline pmdval_t pmd_val(pmd_t pmd) 459 { 460 return PVOP_ALT_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd, 461 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 462 } 463 464 static inline void set_pud(pud_t *pudp, pud_t pud) 465 { 466 PVOP_VCALL2(mmu.set_pud, pudp, native_pud_val(pud)); 467 } 468 469 static inline pud_t __pud(pudval_t val) 470 { 471 pudval_t ret; 472 473 ret = PVOP_ALT_CALLEE1(pudval_t, mmu.make_pud, val, 474 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 475 476 return (pud_t) { ret }; 477 } 478 479 static inline pudval_t pud_val(pud_t pud) 480 { 481 return PVOP_ALT_CALLEE1(pudval_t, mmu.pud_val, pud.pud, 482 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 483 } 484 485 static inline void pud_clear(pud_t *pudp) 486 { 487 set_pud(pudp, native_make_pud(0)); 488 } 489 490 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) 491 { 492 p4dval_t val = native_p4d_val(p4d); 493 494 PVOP_VCALL2(mmu.set_p4d, p4dp, val); 495 } 496 497 #if CONFIG_PGTABLE_LEVELS >= 5 498 499 static inline p4d_t __p4d(p4dval_t val) 500 { 501 p4dval_t ret = PVOP_ALT_CALLEE1(p4dval_t, mmu.make_p4d, val, 502 "mov %%rdi, %%rax", 503 ALT_NOT(X86_FEATURE_XENPV)); 504 505 return (p4d_t) { ret }; 506 } 507 508 static inline p4dval_t p4d_val(p4d_t p4d) 509 { 510 return PVOP_ALT_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d, 511 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 512 } 513 514 static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd) 515 { 516 PVOP_VCALL2(mmu.set_pgd, pgdp, native_pgd_val(pgd)); 517 } 518 519 #define set_pgd(pgdp, pgdval) do { \ 520 if (pgtable_l5_enabled()) \ 521 __set_pgd(pgdp, pgdval); \ 522 else \ 523 set_p4d((p4d_t *)(pgdp), (p4d_t) { (pgdval).pgd }); \ 524 } while (0) 525 526 #define pgd_clear(pgdp) do { \ 527 if (pgtable_l5_enabled()) \ 528 set_pgd(pgdp, native_make_pgd(0)); \ 529 } while (0) 530 531 #endif /* CONFIG_PGTABLE_LEVELS == 5 */ 532 533 static inline void p4d_clear(p4d_t *p4dp) 534 { 535 set_p4d(p4dp, native_make_p4d(0)); 536 } 537 538 static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 539 { 540 set_pte(ptep, pte); 541 } 542 543 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 544 pte_t *ptep) 545 { 546 set_pte(ptep, native_make_pte(0)); 547 } 548 549 static inline void pmd_clear(pmd_t *pmdp) 550 { 551 set_pmd(pmdp, native_make_pmd(0)); 552 } 553 554 #define __HAVE_ARCH_START_CONTEXT_SWITCH 555 static inline void arch_start_context_switch(struct task_struct *prev) 556 { 557 PVOP_VCALL1(cpu.start_context_switch, prev); 558 } 559 560 static inline void arch_end_context_switch(struct task_struct *next) 561 { 562 PVOP_VCALL1(cpu.end_context_switch, next); 563 } 564 565 #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 566 static inline void arch_enter_lazy_mmu_mode(void) 567 { 568 PVOP_VCALL0(mmu.lazy_mode.enter); 569 } 570 571 static inline void arch_leave_lazy_mmu_mode(void) 572 { 573 PVOP_VCALL0(mmu.lazy_mode.leave); 574 } 575 576 static inline void arch_flush_lazy_mmu_mode(void) 577 { 578 PVOP_VCALL0(mmu.lazy_mode.flush); 579 } 580 581 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, 582 phys_addr_t phys, pgprot_t flags) 583 { 584 pv_ops.mmu.set_fixmap(idx, phys, flags); 585 } 586 #endif 587 588 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) 589 590 static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, 591 u32 val) 592 { 593 PVOP_VCALL2(lock.queued_spin_lock_slowpath, lock, val); 594 } 595 596 static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) 597 { 598 PVOP_ALT_VCALLEE1(lock.queued_spin_unlock, lock, 599 "movb $0, (%%" _ASM_ARG1 ");", 600 ALT_NOT(X86_FEATURE_PVUNLOCK)); 601 } 602 603 static __always_inline void pv_wait(u8 *ptr, u8 val) 604 { 605 PVOP_VCALL2(lock.wait, ptr, val); 606 } 607 608 static __always_inline void pv_kick(int cpu) 609 { 610 PVOP_VCALL1(lock.kick, cpu); 611 } 612 613 static __always_inline bool pv_vcpu_is_preempted(long cpu) 614 { 615 return PVOP_ALT_CALLEE1(bool, lock.vcpu_is_preempted, cpu, 616 "xor %%" _ASM_AX ", %%" _ASM_AX ";", 617 ALT_NOT(X86_FEATURE_VCPUPREEMPT)); 618 } 619 620 void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); 621 bool __raw_callee_save___native_vcpu_is_preempted(long cpu); 622 623 #endif /* SMP && PARAVIRT_SPINLOCKS */ 624 625 #ifdef CONFIG_X86_32 626 /* save and restore all caller-save registers, except return value */ 627 #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;" 628 #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;" 629 #else 630 /* save and restore all caller-save registers, except return value */ 631 #define PV_SAVE_ALL_CALLER_REGS \ 632 "push %rcx;" \ 633 "push %rdx;" \ 634 "push %rsi;" \ 635 "push %rdi;" \ 636 "push %r8;" \ 637 "push %r9;" \ 638 "push %r10;" \ 639 "push %r11;" 640 #define PV_RESTORE_ALL_CALLER_REGS \ 641 "pop %r11;" \ 642 "pop %r10;" \ 643 "pop %r9;" \ 644 "pop %r8;" \ 645 "pop %rdi;" \ 646 "pop %rsi;" \ 647 "pop %rdx;" \ 648 "pop %rcx;" 649 #endif 650 651 /* 652 * Generate a thunk around a function which saves all caller-save 653 * registers except for the return value. This allows C functions to 654 * be called from assembler code where fewer than normal registers are 655 * available. It may also help code generation around calls from C 656 * code if the common case doesn't use many registers. 657 * 658 * When a callee is wrapped in a thunk, the caller can assume that all 659 * arg regs and all scratch registers are preserved across the 660 * call. The return value in rax/eax will not be saved, even for void 661 * functions. 662 */ 663 #define PV_THUNK_NAME(func) "__raw_callee_save_" #func 664 #define __PV_CALLEE_SAVE_REGS_THUNK(func, section) \ 665 extern typeof(func) __raw_callee_save_##func; \ 666 \ 667 asm(".pushsection " section ", \"ax\";" \ 668 ".globl " PV_THUNK_NAME(func) ";" \ 669 ".type " PV_THUNK_NAME(func) ", @function;" \ 670 ASM_FUNC_ALIGN \ 671 PV_THUNK_NAME(func) ":" \ 672 ASM_ENDBR \ 673 FRAME_BEGIN \ 674 PV_SAVE_ALL_CALLER_REGS \ 675 "call " #func ";" \ 676 PV_RESTORE_ALL_CALLER_REGS \ 677 FRAME_END \ 678 ASM_RET \ 679 ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \ 680 ".popsection") 681 682 #define PV_CALLEE_SAVE_REGS_THUNK(func) \ 683 __PV_CALLEE_SAVE_REGS_THUNK(func, ".text") 684 685 /* Get a reference to a callee-save function */ 686 #define PV_CALLEE_SAVE(func) \ 687 ((struct paravirt_callee_save) { __raw_callee_save_##func }) 688 689 /* Promise that "func" already uses the right calling convention */ 690 #define __PV_IS_CALLEE_SAVE(func) \ 691 ((struct paravirt_callee_save) { func }) 692 693 #ifdef CONFIG_PARAVIRT_XXL 694 static __always_inline unsigned long arch_local_save_flags(void) 695 { 696 return PVOP_ALT_CALLEE0(unsigned long, irq.save_fl, "pushf; pop %%rax;", 697 ALT_NOT(X86_FEATURE_XENPV)); 698 } 699 700 static __always_inline void arch_local_irq_disable(void) 701 { 702 PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT(X86_FEATURE_XENPV)); 703 } 704 705 static __always_inline void arch_local_irq_enable(void) 706 { 707 PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT(X86_FEATURE_XENPV)); 708 } 709 710 static __always_inline unsigned long arch_local_irq_save(void) 711 { 712 unsigned long f; 713 714 f = arch_local_save_flags(); 715 arch_local_irq_disable(); 716 return f; 717 } 718 #endif 719 720 721 /* Make sure as little as possible of this mess escapes. */ 722 #undef PARAVIRT_CALL 723 #undef __PVOP_CALL 724 #undef __PVOP_VCALL 725 #undef PVOP_VCALL0 726 #undef PVOP_CALL0 727 #undef PVOP_VCALL1 728 #undef PVOP_CALL1 729 #undef PVOP_VCALL2 730 #undef PVOP_CALL2 731 #undef PVOP_VCALL3 732 #undef PVOP_CALL3 733 #undef PVOP_VCALL4 734 #undef PVOP_CALL4 735 736 #define DEFINE_PARAVIRT_ASM(func, instr, sec) \ 737 asm (".pushsection " #sec ", \"ax\"\n" \ 738 ".global " #func "\n\t" \ 739 ".type " #func ", @function\n\t" \ 740 ASM_FUNC_ALIGN "\n" \ 741 #func ":\n\t" \ 742 ASM_ENDBR \ 743 instr "\n\t" \ 744 ASM_RET \ 745 ".size " #func ", . - " #func "\n\t" \ 746 ".popsection") 747 748 extern void default_banner(void); 749 750 #else /* __ASSEMBLY__ */ 751 752 #define _PVSITE(ptype, ops, word, algn) \ 753 771:; \ 754 ops; \ 755 772:; \ 756 .pushsection .parainstructions,"a"; \ 757 .align algn; \ 758 word 771b; \ 759 .byte ptype; \ 760 .byte 772b-771b; \ 761 _ASM_ALIGN; \ 762 .popsection 763 764 765 #ifdef CONFIG_X86_64 766 #ifdef CONFIG_PARAVIRT_XXL 767 768 #define PARA_PATCH(off) ((off) / 8) 769 #define PARA_SITE(ptype, ops) _PVSITE(ptype, ops, .quad, 8) 770 #define PARA_INDIRECT(addr) *addr(%rip) 771 772 #ifdef CONFIG_DEBUG_ENTRY 773 .macro PARA_IRQ_save_fl 774 PARA_SITE(PARA_PATCH(PV_IRQ_save_fl), 775 ANNOTATE_RETPOLINE_SAFE; 776 call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl);) 777 .endm 778 779 #define SAVE_FLAGS ALTERNATIVE "PARA_IRQ_save_fl;", "pushf; pop %rax;", \ 780 ALT_NOT(X86_FEATURE_XENPV) 781 #endif 782 #endif /* CONFIG_PARAVIRT_XXL */ 783 #endif /* CONFIG_X86_64 */ 784 785 #endif /* __ASSEMBLY__ */ 786 #else /* CONFIG_PARAVIRT */ 787 # define default_banner x86_init_noop 788 #endif /* !CONFIG_PARAVIRT */ 789 790 #ifndef __ASSEMBLY__ 791 #ifndef CONFIG_PARAVIRT_XXL 792 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, 793 struct mm_struct *mm) 794 { 795 } 796 #endif 797 798 #ifndef CONFIG_PARAVIRT 799 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 800 { 801 } 802 #endif 803 804 #ifndef CONFIG_PARAVIRT_SPINLOCKS 805 static inline void paravirt_set_cap(void) 806 { 807 } 808 #endif 809 #endif /* __ASSEMBLY__ */ 810 #endif /* _ASM_X86_PARAVIRT_H */ 811