1 #ifndef _ASM_X86_PARAVIRT_H 2 #define _ASM_X86_PARAVIRT_H 3 /* Various instructions on x86 need to be replaced for 4 * para-virtualization: those hooks are defined here. */ 5 6 #ifdef CONFIG_PARAVIRT 7 #include <asm/pgtable_types.h> 8 #include <asm/asm.h> 9 10 #include <asm/paravirt_types.h> 11 12 #ifndef __ASSEMBLY__ 13 #include <linux/bug.h> 14 #include <linux/types.h> 15 #include <linux/cpumask.h> 16 #include <asm/frame.h> 17 18 static inline int paravirt_enabled(void) 19 { 20 return pv_info.paravirt_enabled; 21 } 22 23 static inline int paravirt_has_feature(unsigned int feature) 24 { 25 WARN_ON_ONCE(!pv_info.paravirt_enabled); 26 return (pv_info.features & feature); 27 } 28 29 static inline void load_sp0(struct tss_struct *tss, 30 struct thread_struct *thread) 31 { 32 PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread); 33 } 34 35 /* The paravirtualized CPUID instruction. */ 36 static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 37 unsigned int *ecx, unsigned int *edx) 38 { 39 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx); 40 } 41 42 /* 43 * These special macros can be used to get or set a debugging register 44 */ 45 static inline unsigned long paravirt_get_debugreg(int reg) 46 { 47 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg); 48 } 49 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) 50 static inline void set_debugreg(unsigned long val, int reg) 51 { 52 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val); 53 } 54 55 static inline void clts(void) 56 { 57 PVOP_VCALL0(pv_cpu_ops.clts); 58 } 59 60 static inline unsigned long read_cr0(void) 61 { 62 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0); 63 } 64 65 static inline void write_cr0(unsigned long x) 66 { 67 PVOP_VCALL1(pv_cpu_ops.write_cr0, x); 68 } 69 70 static inline unsigned long read_cr2(void) 71 { 72 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2); 73 } 74 75 static inline void write_cr2(unsigned long x) 76 { 77 PVOP_VCALL1(pv_mmu_ops.write_cr2, x); 78 } 79 80 static inline unsigned long read_cr3(void) 81 { 82 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3); 83 } 84 85 static inline void write_cr3(unsigned long x) 86 { 87 PVOP_VCALL1(pv_mmu_ops.write_cr3, x); 88 } 89 90 static inline unsigned long __read_cr4(void) 91 { 92 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4); 93 } 94 static inline unsigned long __read_cr4_safe(void) 95 { 96 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe); 97 } 98 99 static inline void __write_cr4(unsigned long x) 100 { 101 PVOP_VCALL1(pv_cpu_ops.write_cr4, x); 102 } 103 104 #ifdef CONFIG_X86_64 105 static inline unsigned long read_cr8(void) 106 { 107 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8); 108 } 109 110 static inline void write_cr8(unsigned long x) 111 { 112 PVOP_VCALL1(pv_cpu_ops.write_cr8, x); 113 } 114 #endif 115 116 static inline void arch_safe_halt(void) 117 { 118 PVOP_VCALL0(pv_irq_ops.safe_halt); 119 } 120 121 static inline void halt(void) 122 { 123 PVOP_VCALL0(pv_irq_ops.halt); 124 } 125 126 static inline void wbinvd(void) 127 { 128 PVOP_VCALL0(pv_cpu_ops.wbinvd); 129 } 130 131 #define get_kernel_rpl() (pv_info.kernel_rpl) 132 133 static inline u64 paravirt_read_msr(unsigned msr, int *err) 134 { 135 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err); 136 } 137 138 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) 139 { 140 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high); 141 } 142 143 /* These should all do BUG_ON(_err), but our headers are too tangled. */ 144 #define rdmsr(msr, val1, val2) \ 145 do { \ 146 int _err; \ 147 u64 _l = paravirt_read_msr(msr, &_err); \ 148 val1 = (u32)_l; \ 149 val2 = _l >> 32; \ 150 } while (0) 151 152 #define wrmsr(msr, val1, val2) \ 153 do { \ 154 paravirt_write_msr(msr, val1, val2); \ 155 } while (0) 156 157 #define rdmsrl(msr, val) \ 158 do { \ 159 int _err; \ 160 val = paravirt_read_msr(msr, &_err); \ 161 } while (0) 162 163 static inline void wrmsrl(unsigned msr, u64 val) 164 { 165 wrmsr(msr, (u32)val, (u32)(val>>32)); 166 } 167 168 #define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b) 169 170 /* rdmsr with exception handling */ 171 #define rdmsr_safe(msr, a, b) \ 172 ({ \ 173 int _err; \ 174 u64 _l = paravirt_read_msr(msr, &_err); \ 175 (*a) = (u32)_l; \ 176 (*b) = _l >> 32; \ 177 _err; \ 178 }) 179 180 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) 181 { 182 int err; 183 184 *p = paravirt_read_msr(msr, &err); 185 return err; 186 } 187 188 static inline unsigned long long paravirt_sched_clock(void) 189 { 190 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); 191 } 192 193 struct static_key; 194 extern struct static_key paravirt_steal_enabled; 195 extern struct static_key paravirt_steal_rq_enabled; 196 197 static inline u64 paravirt_steal_clock(int cpu) 198 { 199 return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu); 200 } 201 202 static inline unsigned long long paravirt_read_pmc(int counter) 203 { 204 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); 205 } 206 207 #define rdpmc(counter, low, high) \ 208 do { \ 209 u64 _l = paravirt_read_pmc(counter); \ 210 low = (u32)_l; \ 211 high = _l >> 32; \ 212 } while (0) 213 214 #define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter)) 215 216 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) 217 { 218 PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries); 219 } 220 221 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) 222 { 223 PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries); 224 } 225 226 static inline void load_TR_desc(void) 227 { 228 PVOP_VCALL0(pv_cpu_ops.load_tr_desc); 229 } 230 static inline void load_gdt(const struct desc_ptr *dtr) 231 { 232 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr); 233 } 234 static inline void load_idt(const struct desc_ptr *dtr) 235 { 236 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr); 237 } 238 static inline void set_ldt(const void *addr, unsigned entries) 239 { 240 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries); 241 } 242 static inline void store_idt(struct desc_ptr *dtr) 243 { 244 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr); 245 } 246 static inline unsigned long paravirt_store_tr(void) 247 { 248 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr); 249 } 250 #define store_tr(tr) ((tr) = paravirt_store_tr()) 251 static inline void load_TLS(struct thread_struct *t, unsigned cpu) 252 { 253 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu); 254 } 255 256 #ifdef CONFIG_X86_64 257 static inline void load_gs_index(unsigned int gs) 258 { 259 PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs); 260 } 261 #endif 262 263 static inline void write_ldt_entry(struct desc_struct *dt, int entry, 264 const void *desc) 265 { 266 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc); 267 } 268 269 static inline void write_gdt_entry(struct desc_struct *dt, int entry, 270 void *desc, int type) 271 { 272 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type); 273 } 274 275 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) 276 { 277 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g); 278 } 279 static inline void set_iopl_mask(unsigned mask) 280 { 281 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask); 282 } 283 284 /* The paravirtualized I/O functions */ 285 static inline void slow_down_io(void) 286 { 287 pv_cpu_ops.io_delay(); 288 #ifdef REALLY_SLOW_IO 289 pv_cpu_ops.io_delay(); 290 pv_cpu_ops.io_delay(); 291 pv_cpu_ops.io_delay(); 292 #endif 293 } 294 295 static inline void paravirt_activate_mm(struct mm_struct *prev, 296 struct mm_struct *next) 297 { 298 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next); 299 } 300 301 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, 302 struct mm_struct *mm) 303 { 304 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm); 305 } 306 307 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 308 { 309 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm); 310 } 311 312 static inline void __flush_tlb(void) 313 { 314 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user); 315 } 316 static inline void __flush_tlb_global(void) 317 { 318 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel); 319 } 320 static inline void __flush_tlb_single(unsigned long addr) 321 { 322 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr); 323 } 324 325 static inline void flush_tlb_others(const struct cpumask *cpumask, 326 struct mm_struct *mm, 327 unsigned long start, 328 unsigned long end) 329 { 330 PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end); 331 } 332 333 static inline int paravirt_pgd_alloc(struct mm_struct *mm) 334 { 335 return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm); 336 } 337 338 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) 339 { 340 PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd); 341 } 342 343 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) 344 { 345 PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn); 346 } 347 static inline void paravirt_release_pte(unsigned long pfn) 348 { 349 PVOP_VCALL1(pv_mmu_ops.release_pte, pfn); 350 } 351 352 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) 353 { 354 PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn); 355 } 356 357 static inline void paravirt_release_pmd(unsigned long pfn) 358 { 359 PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn); 360 } 361 362 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) 363 { 364 PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn); 365 } 366 static inline void paravirt_release_pud(unsigned long pfn) 367 { 368 PVOP_VCALL1(pv_mmu_ops.release_pud, pfn); 369 } 370 371 static inline void pte_update(struct mm_struct *mm, unsigned long addr, 372 pte_t *ptep) 373 { 374 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep); 375 } 376 377 static inline pte_t __pte(pteval_t val) 378 { 379 pteval_t ret; 380 381 if (sizeof(pteval_t) > sizeof(long)) 382 ret = PVOP_CALLEE2(pteval_t, 383 pv_mmu_ops.make_pte, 384 val, (u64)val >> 32); 385 else 386 ret = PVOP_CALLEE1(pteval_t, 387 pv_mmu_ops.make_pte, 388 val); 389 390 return (pte_t) { .pte = ret }; 391 } 392 393 static inline pteval_t pte_val(pte_t pte) 394 { 395 pteval_t ret; 396 397 if (sizeof(pteval_t) > sizeof(long)) 398 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val, 399 pte.pte, (u64)pte.pte >> 32); 400 else 401 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val, 402 pte.pte); 403 404 return ret; 405 } 406 407 static inline pgd_t __pgd(pgdval_t val) 408 { 409 pgdval_t ret; 410 411 if (sizeof(pgdval_t) > sizeof(long)) 412 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd, 413 val, (u64)val >> 32); 414 else 415 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd, 416 val); 417 418 return (pgd_t) { ret }; 419 } 420 421 static inline pgdval_t pgd_val(pgd_t pgd) 422 { 423 pgdval_t ret; 424 425 if (sizeof(pgdval_t) > sizeof(long)) 426 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val, 427 pgd.pgd, (u64)pgd.pgd >> 32); 428 else 429 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val, 430 pgd.pgd); 431 432 return ret; 433 } 434 435 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 436 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, 437 pte_t *ptep) 438 { 439 pteval_t ret; 440 441 ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start, 442 mm, addr, ptep); 443 444 return (pte_t) { .pte = ret }; 445 } 446 447 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, 448 pte_t *ptep, pte_t pte) 449 { 450 if (sizeof(pteval_t) > sizeof(long)) 451 /* 5 arg words */ 452 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte); 453 else 454 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit, 455 mm, addr, ptep, pte.pte); 456 } 457 458 static inline void set_pte(pte_t *ptep, pte_t pte) 459 { 460 if (sizeof(pteval_t) > sizeof(long)) 461 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, 462 pte.pte, (u64)pte.pte >> 32); 463 else 464 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, 465 pte.pte); 466 } 467 468 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 469 pte_t *ptep, pte_t pte) 470 { 471 if (sizeof(pteval_t) > sizeof(long)) 472 /* 5 arg words */ 473 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte); 474 else 475 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte); 476 } 477 478 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, 479 pmd_t *pmdp, pmd_t pmd) 480 { 481 if (sizeof(pmdval_t) > sizeof(long)) 482 /* 5 arg words */ 483 pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd); 484 else 485 PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp, 486 native_pmd_val(pmd)); 487 } 488 489 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 490 { 491 pmdval_t val = native_pmd_val(pmd); 492 493 if (sizeof(pmdval_t) > sizeof(long)) 494 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32); 495 else 496 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val); 497 } 498 499 #if CONFIG_PGTABLE_LEVELS >= 3 500 static inline pmd_t __pmd(pmdval_t val) 501 { 502 pmdval_t ret; 503 504 if (sizeof(pmdval_t) > sizeof(long)) 505 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd, 506 val, (u64)val >> 32); 507 else 508 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd, 509 val); 510 511 return (pmd_t) { ret }; 512 } 513 514 static inline pmdval_t pmd_val(pmd_t pmd) 515 { 516 pmdval_t ret; 517 518 if (sizeof(pmdval_t) > sizeof(long)) 519 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val, 520 pmd.pmd, (u64)pmd.pmd >> 32); 521 else 522 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val, 523 pmd.pmd); 524 525 return ret; 526 } 527 528 static inline void set_pud(pud_t *pudp, pud_t pud) 529 { 530 pudval_t val = native_pud_val(pud); 531 532 if (sizeof(pudval_t) > sizeof(long)) 533 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp, 534 val, (u64)val >> 32); 535 else 536 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp, 537 val); 538 } 539 #if CONFIG_PGTABLE_LEVELS == 4 540 static inline pud_t __pud(pudval_t val) 541 { 542 pudval_t ret; 543 544 if (sizeof(pudval_t) > sizeof(long)) 545 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud, 546 val, (u64)val >> 32); 547 else 548 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud, 549 val); 550 551 return (pud_t) { ret }; 552 } 553 554 static inline pudval_t pud_val(pud_t pud) 555 { 556 pudval_t ret; 557 558 if (sizeof(pudval_t) > sizeof(long)) 559 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val, 560 pud.pud, (u64)pud.pud >> 32); 561 else 562 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val, 563 pud.pud); 564 565 return ret; 566 } 567 568 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 569 { 570 pgdval_t val = native_pgd_val(pgd); 571 572 if (sizeof(pgdval_t) > sizeof(long)) 573 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp, 574 val, (u64)val >> 32); 575 else 576 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp, 577 val); 578 } 579 580 static inline void pgd_clear(pgd_t *pgdp) 581 { 582 set_pgd(pgdp, __pgd(0)); 583 } 584 585 static inline void pud_clear(pud_t *pudp) 586 { 587 set_pud(pudp, __pud(0)); 588 } 589 590 #endif /* CONFIG_PGTABLE_LEVELS == 4 */ 591 592 #endif /* CONFIG_PGTABLE_LEVELS >= 3 */ 593 594 #ifdef CONFIG_X86_PAE 595 /* Special-case pte-setting operations for PAE, which can't update a 596 64-bit pte atomically */ 597 static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 598 { 599 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep, 600 pte.pte, pte.pte >> 32); 601 } 602 603 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 604 pte_t *ptep) 605 { 606 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep); 607 } 608 609 static inline void pmd_clear(pmd_t *pmdp) 610 { 611 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp); 612 } 613 #else /* !CONFIG_X86_PAE */ 614 static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 615 { 616 set_pte(ptep, pte); 617 } 618 619 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 620 pte_t *ptep) 621 { 622 set_pte_at(mm, addr, ptep, __pte(0)); 623 } 624 625 static inline void pmd_clear(pmd_t *pmdp) 626 { 627 set_pmd(pmdp, __pmd(0)); 628 } 629 #endif /* CONFIG_X86_PAE */ 630 631 #define __HAVE_ARCH_START_CONTEXT_SWITCH 632 static inline void arch_start_context_switch(struct task_struct *prev) 633 { 634 PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev); 635 } 636 637 static inline void arch_end_context_switch(struct task_struct *next) 638 { 639 PVOP_VCALL1(pv_cpu_ops.end_context_switch, next); 640 } 641 642 #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 643 static inline void arch_enter_lazy_mmu_mode(void) 644 { 645 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter); 646 } 647 648 static inline void arch_leave_lazy_mmu_mode(void) 649 { 650 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave); 651 } 652 653 static inline void arch_flush_lazy_mmu_mode(void) 654 { 655 PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush); 656 } 657 658 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, 659 phys_addr_t phys, pgprot_t flags) 660 { 661 pv_mmu_ops.set_fixmap(idx, phys, flags); 662 } 663 664 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) 665 666 #ifdef CONFIG_QUEUED_SPINLOCKS 667 668 static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, 669 u32 val) 670 { 671 PVOP_VCALL2(pv_lock_ops.queued_spin_lock_slowpath, lock, val); 672 } 673 674 static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) 675 { 676 PVOP_VCALLEE1(pv_lock_ops.queued_spin_unlock, lock); 677 } 678 679 static __always_inline void pv_wait(u8 *ptr, u8 val) 680 { 681 PVOP_VCALL2(pv_lock_ops.wait, ptr, val); 682 } 683 684 static __always_inline void pv_kick(int cpu) 685 { 686 PVOP_VCALL1(pv_lock_ops.kick, cpu); 687 } 688 689 #else /* !CONFIG_QUEUED_SPINLOCKS */ 690 691 static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock, 692 __ticket_t ticket) 693 { 694 PVOP_VCALLEE2(pv_lock_ops.lock_spinning, lock, ticket); 695 } 696 697 static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock, 698 __ticket_t ticket) 699 { 700 PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket); 701 } 702 703 #endif /* CONFIG_QUEUED_SPINLOCKS */ 704 705 #endif /* SMP && PARAVIRT_SPINLOCKS */ 706 707 #ifdef CONFIG_X86_32 708 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;" 709 #define PV_RESTORE_REGS "popl %edx; popl %ecx;" 710 711 /* save and restore all caller-save registers, except return value */ 712 #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;" 713 #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;" 714 715 #define PV_FLAGS_ARG "0" 716 #define PV_EXTRA_CLOBBERS 717 #define PV_VEXTRA_CLOBBERS 718 #else 719 /* save and restore all caller-save registers, except return value */ 720 #define PV_SAVE_ALL_CALLER_REGS \ 721 "push %rcx;" \ 722 "push %rdx;" \ 723 "push %rsi;" \ 724 "push %rdi;" \ 725 "push %r8;" \ 726 "push %r9;" \ 727 "push %r10;" \ 728 "push %r11;" 729 #define PV_RESTORE_ALL_CALLER_REGS \ 730 "pop %r11;" \ 731 "pop %r10;" \ 732 "pop %r9;" \ 733 "pop %r8;" \ 734 "pop %rdi;" \ 735 "pop %rsi;" \ 736 "pop %rdx;" \ 737 "pop %rcx;" 738 739 /* We save some registers, but all of them, that's too much. We clobber all 740 * caller saved registers but the argument parameter */ 741 #define PV_SAVE_REGS "pushq %%rdi;" 742 #define PV_RESTORE_REGS "popq %%rdi;" 743 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi" 744 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi" 745 #define PV_FLAGS_ARG "D" 746 #endif 747 748 /* 749 * Generate a thunk around a function which saves all caller-save 750 * registers except for the return value. This allows C functions to 751 * be called from assembler code where fewer than normal registers are 752 * available. It may also help code generation around calls from C 753 * code if the common case doesn't use many registers. 754 * 755 * When a callee is wrapped in a thunk, the caller can assume that all 756 * arg regs and all scratch registers are preserved across the 757 * call. The return value in rax/eax will not be saved, even for void 758 * functions. 759 */ 760 #define PV_THUNK_NAME(func) "__raw_callee_save_" #func 761 #define PV_CALLEE_SAVE_REGS_THUNK(func) \ 762 extern typeof(func) __raw_callee_save_##func; \ 763 \ 764 asm(".pushsection .text;" \ 765 ".globl " PV_THUNK_NAME(func) ";" \ 766 ".type " PV_THUNK_NAME(func) ", @function;" \ 767 PV_THUNK_NAME(func) ":" \ 768 FRAME_BEGIN \ 769 PV_SAVE_ALL_CALLER_REGS \ 770 "call " #func ";" \ 771 PV_RESTORE_ALL_CALLER_REGS \ 772 FRAME_END \ 773 "ret;" \ 774 ".popsection") 775 776 /* Get a reference to a callee-save function */ 777 #define PV_CALLEE_SAVE(func) \ 778 ((struct paravirt_callee_save) { __raw_callee_save_##func }) 779 780 /* Promise that "func" already uses the right calling convention */ 781 #define __PV_IS_CALLEE_SAVE(func) \ 782 ((struct paravirt_callee_save) { func }) 783 784 static inline notrace unsigned long arch_local_save_flags(void) 785 { 786 return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl); 787 } 788 789 static inline notrace void arch_local_irq_restore(unsigned long f) 790 { 791 PVOP_VCALLEE1(pv_irq_ops.restore_fl, f); 792 } 793 794 static inline notrace void arch_local_irq_disable(void) 795 { 796 PVOP_VCALLEE0(pv_irq_ops.irq_disable); 797 } 798 799 static inline notrace void arch_local_irq_enable(void) 800 { 801 PVOP_VCALLEE0(pv_irq_ops.irq_enable); 802 } 803 804 static inline notrace unsigned long arch_local_irq_save(void) 805 { 806 unsigned long f; 807 808 f = arch_local_save_flags(); 809 arch_local_irq_disable(); 810 return f; 811 } 812 813 814 /* Make sure as little as possible of this mess escapes. */ 815 #undef PARAVIRT_CALL 816 #undef __PVOP_CALL 817 #undef __PVOP_VCALL 818 #undef PVOP_VCALL0 819 #undef PVOP_CALL0 820 #undef PVOP_VCALL1 821 #undef PVOP_CALL1 822 #undef PVOP_VCALL2 823 #undef PVOP_CALL2 824 #undef PVOP_VCALL3 825 #undef PVOP_CALL3 826 #undef PVOP_VCALL4 827 #undef PVOP_CALL4 828 829 extern void default_banner(void); 830 831 #else /* __ASSEMBLY__ */ 832 833 #define _PVSITE(ptype, clobbers, ops, word, algn) \ 834 771:; \ 835 ops; \ 836 772:; \ 837 .pushsection .parainstructions,"a"; \ 838 .align algn; \ 839 word 771b; \ 840 .byte ptype; \ 841 .byte 772b-771b; \ 842 .short clobbers; \ 843 .popsection 844 845 846 #define COND_PUSH(set, mask, reg) \ 847 .if ((~(set)) & mask); push %reg; .endif 848 #define COND_POP(set, mask, reg) \ 849 .if ((~(set)) & mask); pop %reg; .endif 850 851 #ifdef CONFIG_X86_64 852 853 #define PV_SAVE_REGS(set) \ 854 COND_PUSH(set, CLBR_RAX, rax); \ 855 COND_PUSH(set, CLBR_RCX, rcx); \ 856 COND_PUSH(set, CLBR_RDX, rdx); \ 857 COND_PUSH(set, CLBR_RSI, rsi); \ 858 COND_PUSH(set, CLBR_RDI, rdi); \ 859 COND_PUSH(set, CLBR_R8, r8); \ 860 COND_PUSH(set, CLBR_R9, r9); \ 861 COND_PUSH(set, CLBR_R10, r10); \ 862 COND_PUSH(set, CLBR_R11, r11) 863 #define PV_RESTORE_REGS(set) \ 864 COND_POP(set, CLBR_R11, r11); \ 865 COND_POP(set, CLBR_R10, r10); \ 866 COND_POP(set, CLBR_R9, r9); \ 867 COND_POP(set, CLBR_R8, r8); \ 868 COND_POP(set, CLBR_RDI, rdi); \ 869 COND_POP(set, CLBR_RSI, rsi); \ 870 COND_POP(set, CLBR_RDX, rdx); \ 871 COND_POP(set, CLBR_RCX, rcx); \ 872 COND_POP(set, CLBR_RAX, rax) 873 874 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8) 875 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8) 876 #define PARA_INDIRECT(addr) *addr(%rip) 877 #else 878 #define PV_SAVE_REGS(set) \ 879 COND_PUSH(set, CLBR_EAX, eax); \ 880 COND_PUSH(set, CLBR_EDI, edi); \ 881 COND_PUSH(set, CLBR_ECX, ecx); \ 882 COND_PUSH(set, CLBR_EDX, edx) 883 #define PV_RESTORE_REGS(set) \ 884 COND_POP(set, CLBR_EDX, edx); \ 885 COND_POP(set, CLBR_ECX, ecx); \ 886 COND_POP(set, CLBR_EDI, edi); \ 887 COND_POP(set, CLBR_EAX, eax) 888 889 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4) 890 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4) 891 #define PARA_INDIRECT(addr) *%cs:addr 892 #endif 893 894 #define INTERRUPT_RETURN \ 895 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \ 896 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret)) 897 898 #define DISABLE_INTERRUPTS(clobbers) \ 899 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \ 900 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ 901 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \ 902 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) 903 904 #define ENABLE_INTERRUPTS(clobbers) \ 905 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \ 906 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ 907 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \ 908 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) 909 910 #ifdef CONFIG_X86_32 911 #define GET_CR0_INTO_EAX \ 912 push %ecx; push %edx; \ 913 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \ 914 pop %edx; pop %ecx 915 #else /* !CONFIG_X86_32 */ 916 917 /* 918 * If swapgs is used while the userspace stack is still current, 919 * there's no way to call a pvop. The PV replacement *must* be 920 * inlined, or the swapgs instruction must be trapped and emulated. 921 */ 922 #define SWAPGS_UNSAFE_STACK \ 923 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ 924 swapgs) 925 926 /* 927 * Note: swapgs is very special, and in practise is either going to be 928 * implemented with a single "swapgs" instruction or something very 929 * special. Either way, we don't need to save any registers for 930 * it. 931 */ 932 #define SWAPGS \ 933 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ 934 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \ 935 ) 936 937 #define GET_CR2_INTO_RAX \ 938 call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2) 939 940 #define PARAVIRT_ADJUST_EXCEPTION_FRAME \ 941 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \ 942 CLBR_NONE, \ 943 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame)) 944 945 #define USERGS_SYSRET64 \ 946 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \ 947 CLBR_NONE, \ 948 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64)) 949 #endif /* CONFIG_X86_32 */ 950 951 #endif /* __ASSEMBLY__ */ 952 #else /* CONFIG_PARAVIRT */ 953 # define default_banner x86_init_noop 954 #ifndef __ASSEMBLY__ 955 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, 956 struct mm_struct *mm) 957 { 958 } 959 960 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 961 { 962 } 963 #endif /* __ASSEMBLY__ */ 964 #endif /* !CONFIG_PARAVIRT */ 965 #endif /* _ASM_X86_PARAVIRT_H */ 966