1 #ifndef _ASM_X86_PARAVIRT_H 2 #define _ASM_X86_PARAVIRT_H 3 /* Various instructions on x86 need to be replaced for 4 * para-virtualization: those hooks are defined here. */ 5 6 #ifdef CONFIG_PARAVIRT 7 #include <asm/pgtable_types.h> 8 #include <asm/asm.h> 9 10 #include <asm/paravirt_types.h> 11 12 #ifndef __ASSEMBLY__ 13 #include <linux/bug.h> 14 #include <linux/types.h> 15 #include <linux/cpumask.h> 16 17 static inline int paravirt_enabled(void) 18 { 19 return pv_info.paravirt_enabled; 20 } 21 22 static inline int paravirt_has_feature(unsigned int feature) 23 { 24 WARN_ON_ONCE(!pv_info.paravirt_enabled); 25 return (pv_info.features & feature); 26 } 27 28 static inline void load_sp0(struct tss_struct *tss, 29 struct thread_struct *thread) 30 { 31 PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread); 32 } 33 34 /* The paravirtualized CPUID instruction. */ 35 static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 36 unsigned int *ecx, unsigned int *edx) 37 { 38 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx); 39 } 40 41 /* 42 * These special macros can be used to get or set a debugging register 43 */ 44 static inline unsigned long paravirt_get_debugreg(int reg) 45 { 46 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg); 47 } 48 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) 49 static inline void set_debugreg(unsigned long val, int reg) 50 { 51 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val); 52 } 53 54 static inline void clts(void) 55 { 56 PVOP_VCALL0(pv_cpu_ops.clts); 57 } 58 59 static inline unsigned long read_cr0(void) 60 { 61 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0); 62 } 63 64 static inline void write_cr0(unsigned long x) 65 { 66 PVOP_VCALL1(pv_cpu_ops.write_cr0, x); 67 } 68 69 static inline unsigned long read_cr2(void) 70 { 71 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2); 72 } 73 74 static inline void write_cr2(unsigned long x) 75 { 76 PVOP_VCALL1(pv_mmu_ops.write_cr2, x); 77 } 78 79 static inline unsigned long read_cr3(void) 80 { 81 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3); 82 } 83 84 static inline void write_cr3(unsigned long x) 85 { 86 PVOP_VCALL1(pv_mmu_ops.write_cr3, x); 87 } 88 89 static inline unsigned long __read_cr4(void) 90 { 91 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4); 92 } 93 static inline unsigned long __read_cr4_safe(void) 94 { 95 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe); 96 } 97 98 static inline void __write_cr4(unsigned long x) 99 { 100 PVOP_VCALL1(pv_cpu_ops.write_cr4, x); 101 } 102 103 #ifdef CONFIG_X86_64 104 static inline unsigned long read_cr8(void) 105 { 106 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8); 107 } 108 109 static inline void write_cr8(unsigned long x) 110 { 111 PVOP_VCALL1(pv_cpu_ops.write_cr8, x); 112 } 113 #endif 114 115 static inline void arch_safe_halt(void) 116 { 117 PVOP_VCALL0(pv_irq_ops.safe_halt); 118 } 119 120 static inline void halt(void) 121 { 122 PVOP_VCALL0(pv_irq_ops.halt); 123 } 124 125 static inline void wbinvd(void) 126 { 127 PVOP_VCALL0(pv_cpu_ops.wbinvd); 128 } 129 130 #define get_kernel_rpl() (pv_info.kernel_rpl) 131 132 static inline u64 paravirt_read_msr(unsigned msr, int *err) 133 { 134 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err); 135 } 136 137 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) 138 { 139 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high); 140 } 141 142 /* These should all do BUG_ON(_err), but our headers are too tangled. */ 143 #define rdmsr(msr, val1, val2) \ 144 do { \ 145 int _err; \ 146 u64 _l = paravirt_read_msr(msr, &_err); \ 147 val1 = (u32)_l; \ 148 val2 = _l >> 32; \ 149 } while (0) 150 151 #define wrmsr(msr, val1, val2) \ 152 do { \ 153 paravirt_write_msr(msr, val1, val2); \ 154 } while (0) 155 156 #define rdmsrl(msr, val) \ 157 do { \ 158 int _err; \ 159 val = paravirt_read_msr(msr, &_err); \ 160 } while (0) 161 162 static inline void wrmsrl(unsigned msr, u64 val) 163 { 164 wrmsr(msr, (u32)val, (u32)(val>>32)); 165 } 166 167 #define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b) 168 169 /* rdmsr with exception handling */ 170 #define rdmsr_safe(msr, a, b) \ 171 ({ \ 172 int _err; \ 173 u64 _l = paravirt_read_msr(msr, &_err); \ 174 (*a) = (u32)_l; \ 175 (*b) = _l >> 32; \ 176 _err; \ 177 }) 178 179 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) 180 { 181 int err; 182 183 *p = paravirt_read_msr(msr, &err); 184 return err; 185 } 186 187 static inline unsigned long long paravirt_sched_clock(void) 188 { 189 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); 190 } 191 192 struct static_key; 193 extern struct static_key paravirt_steal_enabled; 194 extern struct static_key paravirt_steal_rq_enabled; 195 196 static inline u64 paravirt_steal_clock(int cpu) 197 { 198 return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu); 199 } 200 201 static inline unsigned long long paravirt_read_pmc(int counter) 202 { 203 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); 204 } 205 206 #define rdpmc(counter, low, high) \ 207 do { \ 208 u64 _l = paravirt_read_pmc(counter); \ 209 low = (u32)_l; \ 210 high = _l >> 32; \ 211 } while (0) 212 213 #define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter)) 214 215 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) 216 { 217 PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries); 218 } 219 220 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) 221 { 222 PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries); 223 } 224 225 static inline void load_TR_desc(void) 226 { 227 PVOP_VCALL0(pv_cpu_ops.load_tr_desc); 228 } 229 static inline void load_gdt(const struct desc_ptr *dtr) 230 { 231 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr); 232 } 233 static inline void load_idt(const struct desc_ptr *dtr) 234 { 235 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr); 236 } 237 static inline void set_ldt(const void *addr, unsigned entries) 238 { 239 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries); 240 } 241 static inline void store_idt(struct desc_ptr *dtr) 242 { 243 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr); 244 } 245 static inline unsigned long paravirt_store_tr(void) 246 { 247 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr); 248 } 249 #define store_tr(tr) ((tr) = paravirt_store_tr()) 250 static inline void load_TLS(struct thread_struct *t, unsigned cpu) 251 { 252 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu); 253 } 254 255 #ifdef CONFIG_X86_64 256 static inline void load_gs_index(unsigned int gs) 257 { 258 PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs); 259 } 260 #endif 261 262 static inline void write_ldt_entry(struct desc_struct *dt, int entry, 263 const void *desc) 264 { 265 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc); 266 } 267 268 static inline void write_gdt_entry(struct desc_struct *dt, int entry, 269 void *desc, int type) 270 { 271 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type); 272 } 273 274 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) 275 { 276 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g); 277 } 278 static inline void set_iopl_mask(unsigned mask) 279 { 280 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask); 281 } 282 283 /* The paravirtualized I/O functions */ 284 static inline void slow_down_io(void) 285 { 286 pv_cpu_ops.io_delay(); 287 #ifdef REALLY_SLOW_IO 288 pv_cpu_ops.io_delay(); 289 pv_cpu_ops.io_delay(); 290 pv_cpu_ops.io_delay(); 291 #endif 292 } 293 294 static inline void paravirt_activate_mm(struct mm_struct *prev, 295 struct mm_struct *next) 296 { 297 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next); 298 } 299 300 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, 301 struct mm_struct *mm) 302 { 303 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm); 304 } 305 306 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 307 { 308 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm); 309 } 310 311 static inline void __flush_tlb(void) 312 { 313 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user); 314 } 315 static inline void __flush_tlb_global(void) 316 { 317 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel); 318 } 319 static inline void __flush_tlb_single(unsigned long addr) 320 { 321 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr); 322 } 323 324 static inline void flush_tlb_others(const struct cpumask *cpumask, 325 struct mm_struct *mm, 326 unsigned long start, 327 unsigned long end) 328 { 329 PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end); 330 } 331 332 static inline int paravirt_pgd_alloc(struct mm_struct *mm) 333 { 334 return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm); 335 } 336 337 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) 338 { 339 PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd); 340 } 341 342 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) 343 { 344 PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn); 345 } 346 static inline void paravirt_release_pte(unsigned long pfn) 347 { 348 PVOP_VCALL1(pv_mmu_ops.release_pte, pfn); 349 } 350 351 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) 352 { 353 PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn); 354 } 355 356 static inline void paravirt_release_pmd(unsigned long pfn) 357 { 358 PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn); 359 } 360 361 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) 362 { 363 PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn); 364 } 365 static inline void paravirt_release_pud(unsigned long pfn) 366 { 367 PVOP_VCALL1(pv_mmu_ops.release_pud, pfn); 368 } 369 370 static inline void pte_update(struct mm_struct *mm, unsigned long addr, 371 pte_t *ptep) 372 { 373 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep); 374 } 375 376 static inline pte_t __pte(pteval_t val) 377 { 378 pteval_t ret; 379 380 if (sizeof(pteval_t) > sizeof(long)) 381 ret = PVOP_CALLEE2(pteval_t, 382 pv_mmu_ops.make_pte, 383 val, (u64)val >> 32); 384 else 385 ret = PVOP_CALLEE1(pteval_t, 386 pv_mmu_ops.make_pte, 387 val); 388 389 return (pte_t) { .pte = ret }; 390 } 391 392 static inline pteval_t pte_val(pte_t pte) 393 { 394 pteval_t ret; 395 396 if (sizeof(pteval_t) > sizeof(long)) 397 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val, 398 pte.pte, (u64)pte.pte >> 32); 399 else 400 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val, 401 pte.pte); 402 403 return ret; 404 } 405 406 static inline pgd_t __pgd(pgdval_t val) 407 { 408 pgdval_t ret; 409 410 if (sizeof(pgdval_t) > sizeof(long)) 411 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd, 412 val, (u64)val >> 32); 413 else 414 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd, 415 val); 416 417 return (pgd_t) { ret }; 418 } 419 420 static inline pgdval_t pgd_val(pgd_t pgd) 421 { 422 pgdval_t ret; 423 424 if (sizeof(pgdval_t) > sizeof(long)) 425 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val, 426 pgd.pgd, (u64)pgd.pgd >> 32); 427 else 428 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val, 429 pgd.pgd); 430 431 return ret; 432 } 433 434 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 435 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, 436 pte_t *ptep) 437 { 438 pteval_t ret; 439 440 ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start, 441 mm, addr, ptep); 442 443 return (pte_t) { .pte = ret }; 444 } 445 446 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, 447 pte_t *ptep, pte_t pte) 448 { 449 if (sizeof(pteval_t) > sizeof(long)) 450 /* 5 arg words */ 451 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte); 452 else 453 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit, 454 mm, addr, ptep, pte.pte); 455 } 456 457 static inline void set_pte(pte_t *ptep, pte_t pte) 458 { 459 if (sizeof(pteval_t) > sizeof(long)) 460 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, 461 pte.pte, (u64)pte.pte >> 32); 462 else 463 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, 464 pte.pte); 465 } 466 467 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 468 pte_t *ptep, pte_t pte) 469 { 470 if (sizeof(pteval_t) > sizeof(long)) 471 /* 5 arg words */ 472 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte); 473 else 474 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte); 475 } 476 477 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, 478 pmd_t *pmdp, pmd_t pmd) 479 { 480 if (sizeof(pmdval_t) > sizeof(long)) 481 /* 5 arg words */ 482 pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd); 483 else 484 PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp, 485 native_pmd_val(pmd)); 486 } 487 488 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 489 { 490 pmdval_t val = native_pmd_val(pmd); 491 492 if (sizeof(pmdval_t) > sizeof(long)) 493 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32); 494 else 495 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val); 496 } 497 498 #if CONFIG_PGTABLE_LEVELS >= 3 499 static inline pmd_t __pmd(pmdval_t val) 500 { 501 pmdval_t ret; 502 503 if (sizeof(pmdval_t) > sizeof(long)) 504 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd, 505 val, (u64)val >> 32); 506 else 507 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd, 508 val); 509 510 return (pmd_t) { ret }; 511 } 512 513 static inline pmdval_t pmd_val(pmd_t pmd) 514 { 515 pmdval_t ret; 516 517 if (sizeof(pmdval_t) > sizeof(long)) 518 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val, 519 pmd.pmd, (u64)pmd.pmd >> 32); 520 else 521 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val, 522 pmd.pmd); 523 524 return ret; 525 } 526 527 static inline void set_pud(pud_t *pudp, pud_t pud) 528 { 529 pudval_t val = native_pud_val(pud); 530 531 if (sizeof(pudval_t) > sizeof(long)) 532 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp, 533 val, (u64)val >> 32); 534 else 535 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp, 536 val); 537 } 538 #if CONFIG_PGTABLE_LEVELS == 4 539 static inline pud_t __pud(pudval_t val) 540 { 541 pudval_t ret; 542 543 if (sizeof(pudval_t) > sizeof(long)) 544 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud, 545 val, (u64)val >> 32); 546 else 547 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud, 548 val); 549 550 return (pud_t) { ret }; 551 } 552 553 static inline pudval_t pud_val(pud_t pud) 554 { 555 pudval_t ret; 556 557 if (sizeof(pudval_t) > sizeof(long)) 558 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val, 559 pud.pud, (u64)pud.pud >> 32); 560 else 561 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val, 562 pud.pud); 563 564 return ret; 565 } 566 567 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 568 { 569 pgdval_t val = native_pgd_val(pgd); 570 571 if (sizeof(pgdval_t) > sizeof(long)) 572 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp, 573 val, (u64)val >> 32); 574 else 575 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp, 576 val); 577 } 578 579 static inline void pgd_clear(pgd_t *pgdp) 580 { 581 set_pgd(pgdp, __pgd(0)); 582 } 583 584 static inline void pud_clear(pud_t *pudp) 585 { 586 set_pud(pudp, __pud(0)); 587 } 588 589 #endif /* CONFIG_PGTABLE_LEVELS == 4 */ 590 591 #endif /* CONFIG_PGTABLE_LEVELS >= 3 */ 592 593 #ifdef CONFIG_X86_PAE 594 /* Special-case pte-setting operations for PAE, which can't update a 595 64-bit pte atomically */ 596 static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 597 { 598 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep, 599 pte.pte, pte.pte >> 32); 600 } 601 602 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 603 pte_t *ptep) 604 { 605 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep); 606 } 607 608 static inline void pmd_clear(pmd_t *pmdp) 609 { 610 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp); 611 } 612 #else /* !CONFIG_X86_PAE */ 613 static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 614 { 615 set_pte(ptep, pte); 616 } 617 618 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 619 pte_t *ptep) 620 { 621 set_pte_at(mm, addr, ptep, __pte(0)); 622 } 623 624 static inline void pmd_clear(pmd_t *pmdp) 625 { 626 set_pmd(pmdp, __pmd(0)); 627 } 628 #endif /* CONFIG_X86_PAE */ 629 630 #define __HAVE_ARCH_START_CONTEXT_SWITCH 631 static inline void arch_start_context_switch(struct task_struct *prev) 632 { 633 PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev); 634 } 635 636 static inline void arch_end_context_switch(struct task_struct *next) 637 { 638 PVOP_VCALL1(pv_cpu_ops.end_context_switch, next); 639 } 640 641 #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 642 static inline void arch_enter_lazy_mmu_mode(void) 643 { 644 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter); 645 } 646 647 static inline void arch_leave_lazy_mmu_mode(void) 648 { 649 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave); 650 } 651 652 static inline void arch_flush_lazy_mmu_mode(void) 653 { 654 PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush); 655 } 656 657 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, 658 phys_addr_t phys, pgprot_t flags) 659 { 660 pv_mmu_ops.set_fixmap(idx, phys, flags); 661 } 662 663 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) 664 665 #ifdef CONFIG_QUEUED_SPINLOCKS 666 667 static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, 668 u32 val) 669 { 670 PVOP_VCALL2(pv_lock_ops.queued_spin_lock_slowpath, lock, val); 671 } 672 673 static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) 674 { 675 PVOP_VCALLEE1(pv_lock_ops.queued_spin_unlock, lock); 676 } 677 678 static __always_inline void pv_wait(u8 *ptr, u8 val) 679 { 680 PVOP_VCALL2(pv_lock_ops.wait, ptr, val); 681 } 682 683 static __always_inline void pv_kick(int cpu) 684 { 685 PVOP_VCALL1(pv_lock_ops.kick, cpu); 686 } 687 688 #else /* !CONFIG_QUEUED_SPINLOCKS */ 689 690 static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock, 691 __ticket_t ticket) 692 { 693 PVOP_VCALLEE2(pv_lock_ops.lock_spinning, lock, ticket); 694 } 695 696 static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock, 697 __ticket_t ticket) 698 { 699 PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket); 700 } 701 702 #endif /* CONFIG_QUEUED_SPINLOCKS */ 703 704 #endif /* SMP && PARAVIRT_SPINLOCKS */ 705 706 #ifdef CONFIG_X86_32 707 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;" 708 #define PV_RESTORE_REGS "popl %edx; popl %ecx;" 709 710 /* save and restore all caller-save registers, except return value */ 711 #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;" 712 #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;" 713 714 #define PV_FLAGS_ARG "0" 715 #define PV_EXTRA_CLOBBERS 716 #define PV_VEXTRA_CLOBBERS 717 #else 718 /* save and restore all caller-save registers, except return value */ 719 #define PV_SAVE_ALL_CALLER_REGS \ 720 "push %rcx;" \ 721 "push %rdx;" \ 722 "push %rsi;" \ 723 "push %rdi;" \ 724 "push %r8;" \ 725 "push %r9;" \ 726 "push %r10;" \ 727 "push %r11;" 728 #define PV_RESTORE_ALL_CALLER_REGS \ 729 "pop %r11;" \ 730 "pop %r10;" \ 731 "pop %r9;" \ 732 "pop %r8;" \ 733 "pop %rdi;" \ 734 "pop %rsi;" \ 735 "pop %rdx;" \ 736 "pop %rcx;" 737 738 /* We save some registers, but all of them, that's too much. We clobber all 739 * caller saved registers but the argument parameter */ 740 #define PV_SAVE_REGS "pushq %%rdi;" 741 #define PV_RESTORE_REGS "popq %%rdi;" 742 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi" 743 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi" 744 #define PV_FLAGS_ARG "D" 745 #endif 746 747 /* 748 * Generate a thunk around a function which saves all caller-save 749 * registers except for the return value. This allows C functions to 750 * be called from assembler code where fewer than normal registers are 751 * available. It may also help code generation around calls from C 752 * code if the common case doesn't use many registers. 753 * 754 * When a callee is wrapped in a thunk, the caller can assume that all 755 * arg regs and all scratch registers are preserved across the 756 * call. The return value in rax/eax will not be saved, even for void 757 * functions. 758 */ 759 #define PV_CALLEE_SAVE_REGS_THUNK(func) \ 760 extern typeof(func) __raw_callee_save_##func; \ 761 \ 762 asm(".pushsection .text;" \ 763 ".globl __raw_callee_save_" #func " ; " \ 764 "__raw_callee_save_" #func ": " \ 765 PV_SAVE_ALL_CALLER_REGS \ 766 "call " #func ";" \ 767 PV_RESTORE_ALL_CALLER_REGS \ 768 "ret;" \ 769 ".popsection") 770 771 /* Get a reference to a callee-save function */ 772 #define PV_CALLEE_SAVE(func) \ 773 ((struct paravirt_callee_save) { __raw_callee_save_##func }) 774 775 /* Promise that "func" already uses the right calling convention */ 776 #define __PV_IS_CALLEE_SAVE(func) \ 777 ((struct paravirt_callee_save) { func }) 778 779 static inline notrace unsigned long arch_local_save_flags(void) 780 { 781 return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl); 782 } 783 784 static inline notrace void arch_local_irq_restore(unsigned long f) 785 { 786 PVOP_VCALLEE1(pv_irq_ops.restore_fl, f); 787 } 788 789 static inline notrace void arch_local_irq_disable(void) 790 { 791 PVOP_VCALLEE0(pv_irq_ops.irq_disable); 792 } 793 794 static inline notrace void arch_local_irq_enable(void) 795 { 796 PVOP_VCALLEE0(pv_irq_ops.irq_enable); 797 } 798 799 static inline notrace unsigned long arch_local_irq_save(void) 800 { 801 unsigned long f; 802 803 f = arch_local_save_flags(); 804 arch_local_irq_disable(); 805 return f; 806 } 807 808 809 /* Make sure as little as possible of this mess escapes. */ 810 #undef PARAVIRT_CALL 811 #undef __PVOP_CALL 812 #undef __PVOP_VCALL 813 #undef PVOP_VCALL0 814 #undef PVOP_CALL0 815 #undef PVOP_VCALL1 816 #undef PVOP_CALL1 817 #undef PVOP_VCALL2 818 #undef PVOP_CALL2 819 #undef PVOP_VCALL3 820 #undef PVOP_CALL3 821 #undef PVOP_VCALL4 822 #undef PVOP_CALL4 823 824 extern void default_banner(void); 825 826 #else /* __ASSEMBLY__ */ 827 828 #define _PVSITE(ptype, clobbers, ops, word, algn) \ 829 771:; \ 830 ops; \ 831 772:; \ 832 .pushsection .parainstructions,"a"; \ 833 .align algn; \ 834 word 771b; \ 835 .byte ptype; \ 836 .byte 772b-771b; \ 837 .short clobbers; \ 838 .popsection 839 840 841 #define COND_PUSH(set, mask, reg) \ 842 .if ((~(set)) & mask); push %reg; .endif 843 #define COND_POP(set, mask, reg) \ 844 .if ((~(set)) & mask); pop %reg; .endif 845 846 #ifdef CONFIG_X86_64 847 848 #define PV_SAVE_REGS(set) \ 849 COND_PUSH(set, CLBR_RAX, rax); \ 850 COND_PUSH(set, CLBR_RCX, rcx); \ 851 COND_PUSH(set, CLBR_RDX, rdx); \ 852 COND_PUSH(set, CLBR_RSI, rsi); \ 853 COND_PUSH(set, CLBR_RDI, rdi); \ 854 COND_PUSH(set, CLBR_R8, r8); \ 855 COND_PUSH(set, CLBR_R9, r9); \ 856 COND_PUSH(set, CLBR_R10, r10); \ 857 COND_PUSH(set, CLBR_R11, r11) 858 #define PV_RESTORE_REGS(set) \ 859 COND_POP(set, CLBR_R11, r11); \ 860 COND_POP(set, CLBR_R10, r10); \ 861 COND_POP(set, CLBR_R9, r9); \ 862 COND_POP(set, CLBR_R8, r8); \ 863 COND_POP(set, CLBR_RDI, rdi); \ 864 COND_POP(set, CLBR_RSI, rsi); \ 865 COND_POP(set, CLBR_RDX, rdx); \ 866 COND_POP(set, CLBR_RCX, rcx); \ 867 COND_POP(set, CLBR_RAX, rax) 868 869 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8) 870 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8) 871 #define PARA_INDIRECT(addr) *addr(%rip) 872 #else 873 #define PV_SAVE_REGS(set) \ 874 COND_PUSH(set, CLBR_EAX, eax); \ 875 COND_PUSH(set, CLBR_EDI, edi); \ 876 COND_PUSH(set, CLBR_ECX, ecx); \ 877 COND_PUSH(set, CLBR_EDX, edx) 878 #define PV_RESTORE_REGS(set) \ 879 COND_POP(set, CLBR_EDX, edx); \ 880 COND_POP(set, CLBR_ECX, ecx); \ 881 COND_POP(set, CLBR_EDI, edi); \ 882 COND_POP(set, CLBR_EAX, eax) 883 884 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4) 885 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4) 886 #define PARA_INDIRECT(addr) *%cs:addr 887 #endif 888 889 #define INTERRUPT_RETURN \ 890 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \ 891 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret)) 892 893 #define DISABLE_INTERRUPTS(clobbers) \ 894 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \ 895 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ 896 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \ 897 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) 898 899 #define ENABLE_INTERRUPTS(clobbers) \ 900 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \ 901 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ 902 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \ 903 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) 904 905 #ifdef CONFIG_X86_32 906 #define GET_CR0_INTO_EAX \ 907 push %ecx; push %edx; \ 908 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \ 909 pop %edx; pop %ecx 910 #else /* !CONFIG_X86_32 */ 911 912 /* 913 * If swapgs is used while the userspace stack is still current, 914 * there's no way to call a pvop. The PV replacement *must* be 915 * inlined, or the swapgs instruction must be trapped and emulated. 916 */ 917 #define SWAPGS_UNSAFE_STACK \ 918 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ 919 swapgs) 920 921 /* 922 * Note: swapgs is very special, and in practise is either going to be 923 * implemented with a single "swapgs" instruction or something very 924 * special. Either way, we don't need to save any registers for 925 * it. 926 */ 927 #define SWAPGS \ 928 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ 929 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \ 930 ) 931 932 #define GET_CR2_INTO_RAX \ 933 call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2) 934 935 #define PARAVIRT_ADJUST_EXCEPTION_FRAME \ 936 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \ 937 CLBR_NONE, \ 938 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame)) 939 940 #define USERGS_SYSRET64 \ 941 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \ 942 CLBR_NONE, \ 943 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64)) 944 #endif /* CONFIG_X86_32 */ 945 946 #endif /* __ASSEMBLY__ */ 947 #else /* CONFIG_PARAVIRT */ 948 # define default_banner x86_init_noop 949 #ifndef __ASSEMBLY__ 950 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, 951 struct mm_struct *mm) 952 { 953 } 954 955 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 956 { 957 } 958 #endif /* __ASSEMBLY__ */ 959 #endif /* !CONFIG_PARAVIRT */ 960 #endif /* _ASM_X86_PARAVIRT_H */ 961