1 #ifndef _ASM_X86_PARAVIRT_H 2 #define _ASM_X86_PARAVIRT_H 3 /* Various instructions on x86 need to be replaced for 4 * para-virtualization: those hooks are defined here. */ 5 6 #ifdef CONFIG_PARAVIRT 7 #include <asm/pgtable_types.h> 8 #include <asm/asm.h> 9 10 #include <asm/paravirt_types.h> 11 12 #ifndef __ASSEMBLY__ 13 #include <linux/types.h> 14 #include <linux/cpumask.h> 15 16 static inline int paravirt_enabled(void) 17 { 18 return pv_info.paravirt_enabled; 19 } 20 21 static inline void load_sp0(struct tss_struct *tss, 22 struct thread_struct *thread) 23 { 24 PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread); 25 } 26 27 /* The paravirtualized CPUID instruction. */ 28 static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 29 unsigned int *ecx, unsigned int *edx) 30 { 31 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx); 32 } 33 34 /* 35 * These special macros can be used to get or set a debugging register 36 */ 37 static inline unsigned long paravirt_get_debugreg(int reg) 38 { 39 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg); 40 } 41 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) 42 static inline void set_debugreg(unsigned long val, int reg) 43 { 44 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val); 45 } 46 47 static inline void clts(void) 48 { 49 PVOP_VCALL0(pv_cpu_ops.clts); 50 } 51 52 static inline unsigned long read_cr0(void) 53 { 54 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0); 55 } 56 57 static inline void write_cr0(unsigned long x) 58 { 59 PVOP_VCALL1(pv_cpu_ops.write_cr0, x); 60 } 61 62 static inline unsigned long read_cr2(void) 63 { 64 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2); 65 } 66 67 static inline void write_cr2(unsigned long x) 68 { 69 PVOP_VCALL1(pv_mmu_ops.write_cr2, x); 70 } 71 72 static inline unsigned long read_cr3(void) 73 { 74 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3); 75 } 76 77 static inline void write_cr3(unsigned long x) 78 { 79 PVOP_VCALL1(pv_mmu_ops.write_cr3, x); 80 } 81 82 static inline unsigned long read_cr4(void) 83 { 84 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4); 85 } 86 static inline unsigned long read_cr4_safe(void) 87 { 88 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe); 89 } 90 91 static inline void write_cr4(unsigned long x) 92 { 93 PVOP_VCALL1(pv_cpu_ops.write_cr4, x); 94 } 95 96 #ifdef CONFIG_X86_64 97 static inline unsigned long read_cr8(void) 98 { 99 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8); 100 } 101 102 static inline void write_cr8(unsigned long x) 103 { 104 PVOP_VCALL1(pv_cpu_ops.write_cr8, x); 105 } 106 #endif 107 108 static inline void raw_safe_halt(void) 109 { 110 PVOP_VCALL0(pv_irq_ops.safe_halt); 111 } 112 113 static inline void halt(void) 114 { 115 PVOP_VCALL0(pv_irq_ops.safe_halt); 116 } 117 118 static inline void wbinvd(void) 119 { 120 PVOP_VCALL0(pv_cpu_ops.wbinvd); 121 } 122 123 #define get_kernel_rpl() (pv_info.kernel_rpl) 124 125 static inline u64 paravirt_read_msr(unsigned msr, int *err) 126 { 127 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err); 128 } 129 130 static inline int paravirt_rdmsr_regs(u32 *regs) 131 { 132 return PVOP_CALL1(int, pv_cpu_ops.rdmsr_regs, regs); 133 } 134 135 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) 136 { 137 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high); 138 } 139 140 static inline int paravirt_wrmsr_regs(u32 *regs) 141 { 142 return PVOP_CALL1(int, pv_cpu_ops.wrmsr_regs, regs); 143 } 144 145 /* These should all do BUG_ON(_err), but our headers are too tangled. */ 146 #define rdmsr(msr, val1, val2) \ 147 do { \ 148 int _err; \ 149 u64 _l = paravirt_read_msr(msr, &_err); \ 150 val1 = (u32)_l; \ 151 val2 = _l >> 32; \ 152 } while (0) 153 154 #define wrmsr(msr, val1, val2) \ 155 do { \ 156 paravirt_write_msr(msr, val1, val2); \ 157 } while (0) 158 159 #define rdmsrl(msr, val) \ 160 do { \ 161 int _err; \ 162 val = paravirt_read_msr(msr, &_err); \ 163 } while (0) 164 165 #define wrmsrl(msr, val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32) 166 #define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b) 167 168 /* rdmsr with exception handling */ 169 #define rdmsr_safe(msr, a, b) \ 170 ({ \ 171 int _err; \ 172 u64 _l = paravirt_read_msr(msr, &_err); \ 173 (*a) = (u32)_l; \ 174 (*b) = _l >> 32; \ 175 _err; \ 176 }) 177 178 #define rdmsr_safe_regs(regs) paravirt_rdmsr_regs(regs) 179 #define wrmsr_safe_regs(regs) paravirt_wrmsr_regs(regs) 180 181 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) 182 { 183 int err; 184 185 *p = paravirt_read_msr(msr, &err); 186 return err; 187 } 188 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p) 189 { 190 u32 gprs[8] = { 0 }; 191 int err; 192 193 gprs[1] = msr; 194 gprs[7] = 0x9c5a203a; 195 196 err = paravirt_rdmsr_regs(gprs); 197 198 *p = gprs[0] | ((u64)gprs[2] << 32); 199 200 return err; 201 } 202 203 static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val) 204 { 205 u32 gprs[8] = { 0 }; 206 207 gprs[0] = (u32)val; 208 gprs[1] = msr; 209 gprs[2] = val >> 32; 210 gprs[7] = 0x9c5a203a; 211 212 return paravirt_wrmsr_regs(gprs); 213 } 214 215 static inline u64 paravirt_read_tsc(void) 216 { 217 return PVOP_CALL0(u64, pv_cpu_ops.read_tsc); 218 } 219 220 #define rdtscl(low) \ 221 do { \ 222 u64 _l = paravirt_read_tsc(); \ 223 low = (int)_l; \ 224 } while (0) 225 226 #define rdtscll(val) (val = paravirt_read_tsc()) 227 228 static inline unsigned long long paravirt_sched_clock(void) 229 { 230 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); 231 } 232 233 static inline unsigned long long paravirt_read_pmc(int counter) 234 { 235 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); 236 } 237 238 #define rdpmc(counter, low, high) \ 239 do { \ 240 u64 _l = paravirt_read_pmc(counter); \ 241 low = (u32)_l; \ 242 high = _l >> 32; \ 243 } while (0) 244 245 static inline unsigned long long paravirt_rdtscp(unsigned int *aux) 246 { 247 return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux); 248 } 249 250 #define rdtscp(low, high, aux) \ 251 do { \ 252 int __aux; \ 253 unsigned long __val = paravirt_rdtscp(&__aux); \ 254 (low) = (u32)__val; \ 255 (high) = (u32)(__val >> 32); \ 256 (aux) = __aux; \ 257 } while (0) 258 259 #define rdtscpll(val, aux) \ 260 do { \ 261 unsigned long __aux; \ 262 val = paravirt_rdtscp(&__aux); \ 263 (aux) = __aux; \ 264 } while (0) 265 266 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) 267 { 268 PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries); 269 } 270 271 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) 272 { 273 PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries); 274 } 275 276 static inline void load_TR_desc(void) 277 { 278 PVOP_VCALL0(pv_cpu_ops.load_tr_desc); 279 } 280 static inline void load_gdt(const struct desc_ptr *dtr) 281 { 282 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr); 283 } 284 static inline void load_idt(const struct desc_ptr *dtr) 285 { 286 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr); 287 } 288 static inline void set_ldt(const void *addr, unsigned entries) 289 { 290 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries); 291 } 292 static inline void store_gdt(struct desc_ptr *dtr) 293 { 294 PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr); 295 } 296 static inline void store_idt(struct desc_ptr *dtr) 297 { 298 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr); 299 } 300 static inline unsigned long paravirt_store_tr(void) 301 { 302 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr); 303 } 304 #define store_tr(tr) ((tr) = paravirt_store_tr()) 305 static inline void load_TLS(struct thread_struct *t, unsigned cpu) 306 { 307 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu); 308 } 309 310 #ifdef CONFIG_X86_64 311 static inline void load_gs_index(unsigned int gs) 312 { 313 PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs); 314 } 315 #endif 316 317 static inline void write_ldt_entry(struct desc_struct *dt, int entry, 318 const void *desc) 319 { 320 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc); 321 } 322 323 static inline void write_gdt_entry(struct desc_struct *dt, int entry, 324 void *desc, int type) 325 { 326 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type); 327 } 328 329 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) 330 { 331 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g); 332 } 333 static inline void set_iopl_mask(unsigned mask) 334 { 335 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask); 336 } 337 338 /* The paravirtualized I/O functions */ 339 static inline void slow_down_io(void) 340 { 341 pv_cpu_ops.io_delay(); 342 #ifdef REALLY_SLOW_IO 343 pv_cpu_ops.io_delay(); 344 pv_cpu_ops.io_delay(); 345 pv_cpu_ops.io_delay(); 346 #endif 347 } 348 349 #ifdef CONFIG_SMP 350 static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, 351 unsigned long start_esp) 352 { 353 PVOP_VCALL3(pv_apic_ops.startup_ipi_hook, 354 phys_apicid, start_eip, start_esp); 355 } 356 #endif 357 358 static inline void paravirt_activate_mm(struct mm_struct *prev, 359 struct mm_struct *next) 360 { 361 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next); 362 } 363 364 static inline void arch_dup_mmap(struct mm_struct *oldmm, 365 struct mm_struct *mm) 366 { 367 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm); 368 } 369 370 static inline void arch_exit_mmap(struct mm_struct *mm) 371 { 372 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm); 373 } 374 375 static inline void __flush_tlb(void) 376 { 377 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user); 378 } 379 static inline void __flush_tlb_global(void) 380 { 381 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel); 382 } 383 static inline void __flush_tlb_single(unsigned long addr) 384 { 385 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr); 386 } 387 388 static inline void flush_tlb_others(const struct cpumask *cpumask, 389 struct mm_struct *mm, 390 unsigned long va) 391 { 392 PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va); 393 } 394 395 static inline int paravirt_pgd_alloc(struct mm_struct *mm) 396 { 397 return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm); 398 } 399 400 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) 401 { 402 PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd); 403 } 404 405 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) 406 { 407 PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn); 408 } 409 static inline void paravirt_release_pte(unsigned long pfn) 410 { 411 PVOP_VCALL1(pv_mmu_ops.release_pte, pfn); 412 } 413 414 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) 415 { 416 PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn); 417 } 418 419 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn, 420 unsigned long start, unsigned long count) 421 { 422 PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count); 423 } 424 static inline void paravirt_release_pmd(unsigned long pfn) 425 { 426 PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn); 427 } 428 429 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) 430 { 431 PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn); 432 } 433 static inline void paravirt_release_pud(unsigned long pfn) 434 { 435 PVOP_VCALL1(pv_mmu_ops.release_pud, pfn); 436 } 437 438 static inline void pte_update(struct mm_struct *mm, unsigned long addr, 439 pte_t *ptep) 440 { 441 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep); 442 } 443 444 static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr, 445 pte_t *ptep) 446 { 447 PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep); 448 } 449 450 static inline pte_t __pte(pteval_t val) 451 { 452 pteval_t ret; 453 454 if (sizeof(pteval_t) > sizeof(long)) 455 ret = PVOP_CALLEE2(pteval_t, 456 pv_mmu_ops.make_pte, 457 val, (u64)val >> 32); 458 else 459 ret = PVOP_CALLEE1(pteval_t, 460 pv_mmu_ops.make_pte, 461 val); 462 463 return (pte_t) { .pte = ret }; 464 } 465 466 static inline pteval_t pte_val(pte_t pte) 467 { 468 pteval_t ret; 469 470 if (sizeof(pteval_t) > sizeof(long)) 471 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val, 472 pte.pte, (u64)pte.pte >> 32); 473 else 474 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val, 475 pte.pte); 476 477 return ret; 478 } 479 480 static inline pgd_t __pgd(pgdval_t val) 481 { 482 pgdval_t ret; 483 484 if (sizeof(pgdval_t) > sizeof(long)) 485 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd, 486 val, (u64)val >> 32); 487 else 488 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd, 489 val); 490 491 return (pgd_t) { ret }; 492 } 493 494 static inline pgdval_t pgd_val(pgd_t pgd) 495 { 496 pgdval_t ret; 497 498 if (sizeof(pgdval_t) > sizeof(long)) 499 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val, 500 pgd.pgd, (u64)pgd.pgd >> 32); 501 else 502 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val, 503 pgd.pgd); 504 505 return ret; 506 } 507 508 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 509 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, 510 pte_t *ptep) 511 { 512 pteval_t ret; 513 514 ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start, 515 mm, addr, ptep); 516 517 return (pte_t) { .pte = ret }; 518 } 519 520 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, 521 pte_t *ptep, pte_t pte) 522 { 523 if (sizeof(pteval_t) > sizeof(long)) 524 /* 5 arg words */ 525 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte); 526 else 527 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit, 528 mm, addr, ptep, pte.pte); 529 } 530 531 static inline void set_pte(pte_t *ptep, pte_t pte) 532 { 533 if (sizeof(pteval_t) > sizeof(long)) 534 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, 535 pte.pte, (u64)pte.pte >> 32); 536 else 537 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, 538 pte.pte); 539 } 540 541 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 542 pte_t *ptep, pte_t pte) 543 { 544 if (sizeof(pteval_t) > sizeof(long)) 545 /* 5 arg words */ 546 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte); 547 else 548 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte); 549 } 550 551 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 552 { 553 pmdval_t val = native_pmd_val(pmd); 554 555 if (sizeof(pmdval_t) > sizeof(long)) 556 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32); 557 else 558 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val); 559 } 560 561 #if PAGETABLE_LEVELS >= 3 562 static inline pmd_t __pmd(pmdval_t val) 563 { 564 pmdval_t ret; 565 566 if (sizeof(pmdval_t) > sizeof(long)) 567 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd, 568 val, (u64)val >> 32); 569 else 570 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd, 571 val); 572 573 return (pmd_t) { ret }; 574 } 575 576 static inline pmdval_t pmd_val(pmd_t pmd) 577 { 578 pmdval_t ret; 579 580 if (sizeof(pmdval_t) > sizeof(long)) 581 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val, 582 pmd.pmd, (u64)pmd.pmd >> 32); 583 else 584 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val, 585 pmd.pmd); 586 587 return ret; 588 } 589 590 static inline void set_pud(pud_t *pudp, pud_t pud) 591 { 592 pudval_t val = native_pud_val(pud); 593 594 if (sizeof(pudval_t) > sizeof(long)) 595 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp, 596 val, (u64)val >> 32); 597 else 598 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp, 599 val); 600 } 601 #if PAGETABLE_LEVELS == 4 602 static inline pud_t __pud(pudval_t val) 603 { 604 pudval_t ret; 605 606 if (sizeof(pudval_t) > sizeof(long)) 607 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud, 608 val, (u64)val >> 32); 609 else 610 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud, 611 val); 612 613 return (pud_t) { ret }; 614 } 615 616 static inline pudval_t pud_val(pud_t pud) 617 { 618 pudval_t ret; 619 620 if (sizeof(pudval_t) > sizeof(long)) 621 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val, 622 pud.pud, (u64)pud.pud >> 32); 623 else 624 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val, 625 pud.pud); 626 627 return ret; 628 } 629 630 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 631 { 632 pgdval_t val = native_pgd_val(pgd); 633 634 if (sizeof(pgdval_t) > sizeof(long)) 635 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp, 636 val, (u64)val >> 32); 637 else 638 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp, 639 val); 640 } 641 642 static inline void pgd_clear(pgd_t *pgdp) 643 { 644 set_pgd(pgdp, __pgd(0)); 645 } 646 647 static inline void pud_clear(pud_t *pudp) 648 { 649 set_pud(pudp, __pud(0)); 650 } 651 652 #endif /* PAGETABLE_LEVELS == 4 */ 653 654 #endif /* PAGETABLE_LEVELS >= 3 */ 655 656 #ifdef CONFIG_X86_PAE 657 /* Special-case pte-setting operations for PAE, which can't update a 658 64-bit pte atomically */ 659 static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 660 { 661 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep, 662 pte.pte, pte.pte >> 32); 663 } 664 665 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 666 pte_t *ptep) 667 { 668 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep); 669 } 670 671 static inline void pmd_clear(pmd_t *pmdp) 672 { 673 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp); 674 } 675 #else /* !CONFIG_X86_PAE */ 676 static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 677 { 678 set_pte(ptep, pte); 679 } 680 681 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 682 pte_t *ptep) 683 { 684 set_pte_at(mm, addr, ptep, __pte(0)); 685 } 686 687 static inline void pmd_clear(pmd_t *pmdp) 688 { 689 set_pmd(pmdp, __pmd(0)); 690 } 691 #endif /* CONFIG_X86_PAE */ 692 693 #define __HAVE_ARCH_START_CONTEXT_SWITCH 694 static inline void arch_start_context_switch(struct task_struct *prev) 695 { 696 PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev); 697 } 698 699 static inline void arch_end_context_switch(struct task_struct *next) 700 { 701 PVOP_VCALL1(pv_cpu_ops.end_context_switch, next); 702 } 703 704 #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 705 static inline void arch_enter_lazy_mmu_mode(void) 706 { 707 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter); 708 } 709 710 static inline void arch_leave_lazy_mmu_mode(void) 711 { 712 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave); 713 } 714 715 void arch_flush_lazy_mmu_mode(void); 716 717 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, 718 phys_addr_t phys, pgprot_t flags) 719 { 720 pv_mmu_ops.set_fixmap(idx, phys, flags); 721 } 722 723 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) 724 725 static inline int arch_spin_is_locked(struct arch_spinlock *lock) 726 { 727 return PVOP_CALL1(int, pv_lock_ops.spin_is_locked, lock); 728 } 729 730 static inline int arch_spin_is_contended(struct arch_spinlock *lock) 731 { 732 return PVOP_CALL1(int, pv_lock_ops.spin_is_contended, lock); 733 } 734 #define arch_spin_is_contended arch_spin_is_contended 735 736 static __always_inline void arch_spin_lock(struct arch_spinlock *lock) 737 { 738 PVOP_VCALL1(pv_lock_ops.spin_lock, lock); 739 } 740 741 static __always_inline void arch_spin_lock_flags(struct arch_spinlock *lock, 742 unsigned long flags) 743 { 744 PVOP_VCALL2(pv_lock_ops.spin_lock_flags, lock, flags); 745 } 746 747 static __always_inline int arch_spin_trylock(struct arch_spinlock *lock) 748 { 749 return PVOP_CALL1(int, pv_lock_ops.spin_trylock, lock); 750 } 751 752 static __always_inline void arch_spin_unlock(struct arch_spinlock *lock) 753 { 754 PVOP_VCALL1(pv_lock_ops.spin_unlock, lock); 755 } 756 757 #endif 758 759 #ifdef CONFIG_X86_32 760 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;" 761 #define PV_RESTORE_REGS "popl %edx; popl %ecx;" 762 763 /* save and restore all caller-save registers, except return value */ 764 #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;" 765 #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;" 766 767 #define PV_FLAGS_ARG "0" 768 #define PV_EXTRA_CLOBBERS 769 #define PV_VEXTRA_CLOBBERS 770 #else 771 /* save and restore all caller-save registers, except return value */ 772 #define PV_SAVE_ALL_CALLER_REGS \ 773 "push %rcx;" \ 774 "push %rdx;" \ 775 "push %rsi;" \ 776 "push %rdi;" \ 777 "push %r8;" \ 778 "push %r9;" \ 779 "push %r10;" \ 780 "push %r11;" 781 #define PV_RESTORE_ALL_CALLER_REGS \ 782 "pop %r11;" \ 783 "pop %r10;" \ 784 "pop %r9;" \ 785 "pop %r8;" \ 786 "pop %rdi;" \ 787 "pop %rsi;" \ 788 "pop %rdx;" \ 789 "pop %rcx;" 790 791 /* We save some registers, but all of them, that's too much. We clobber all 792 * caller saved registers but the argument parameter */ 793 #define PV_SAVE_REGS "pushq %%rdi;" 794 #define PV_RESTORE_REGS "popq %%rdi;" 795 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi" 796 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi" 797 #define PV_FLAGS_ARG "D" 798 #endif 799 800 /* 801 * Generate a thunk around a function which saves all caller-save 802 * registers except for the return value. This allows C functions to 803 * be called from assembler code where fewer than normal registers are 804 * available. It may also help code generation around calls from C 805 * code if the common case doesn't use many registers. 806 * 807 * When a callee is wrapped in a thunk, the caller can assume that all 808 * arg regs and all scratch registers are preserved across the 809 * call. The return value in rax/eax will not be saved, even for void 810 * functions. 811 */ 812 #define PV_CALLEE_SAVE_REGS_THUNK(func) \ 813 extern typeof(func) __raw_callee_save_##func; \ 814 static void *__##func##__ __used = func; \ 815 \ 816 asm(".pushsection .text;" \ 817 "__raw_callee_save_" #func ": " \ 818 PV_SAVE_ALL_CALLER_REGS \ 819 "call " #func ";" \ 820 PV_RESTORE_ALL_CALLER_REGS \ 821 "ret;" \ 822 ".popsection") 823 824 /* Get a reference to a callee-save function */ 825 #define PV_CALLEE_SAVE(func) \ 826 ((struct paravirt_callee_save) { __raw_callee_save_##func }) 827 828 /* Promise that "func" already uses the right calling convention */ 829 #define __PV_IS_CALLEE_SAVE(func) \ 830 ((struct paravirt_callee_save) { func }) 831 832 static inline unsigned long __raw_local_save_flags(void) 833 { 834 return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl); 835 } 836 837 static inline void raw_local_irq_restore(unsigned long f) 838 { 839 PVOP_VCALLEE1(pv_irq_ops.restore_fl, f); 840 } 841 842 static inline void raw_local_irq_disable(void) 843 { 844 PVOP_VCALLEE0(pv_irq_ops.irq_disable); 845 } 846 847 static inline void raw_local_irq_enable(void) 848 { 849 PVOP_VCALLEE0(pv_irq_ops.irq_enable); 850 } 851 852 static inline unsigned long __raw_local_irq_save(void) 853 { 854 unsigned long f; 855 856 f = __raw_local_save_flags(); 857 raw_local_irq_disable(); 858 return f; 859 } 860 861 862 /* Make sure as little as possible of this mess escapes. */ 863 #undef PARAVIRT_CALL 864 #undef __PVOP_CALL 865 #undef __PVOP_VCALL 866 #undef PVOP_VCALL0 867 #undef PVOP_CALL0 868 #undef PVOP_VCALL1 869 #undef PVOP_CALL1 870 #undef PVOP_VCALL2 871 #undef PVOP_CALL2 872 #undef PVOP_VCALL3 873 #undef PVOP_CALL3 874 #undef PVOP_VCALL4 875 #undef PVOP_CALL4 876 877 extern void default_banner(void); 878 879 #else /* __ASSEMBLY__ */ 880 881 #define _PVSITE(ptype, clobbers, ops, word, algn) \ 882 771:; \ 883 ops; \ 884 772:; \ 885 .pushsection .parainstructions,"a"; \ 886 .align algn; \ 887 word 771b; \ 888 .byte ptype; \ 889 .byte 772b-771b; \ 890 .short clobbers; \ 891 .popsection 892 893 894 #define COND_PUSH(set, mask, reg) \ 895 .if ((~(set)) & mask); push %reg; .endif 896 #define COND_POP(set, mask, reg) \ 897 .if ((~(set)) & mask); pop %reg; .endif 898 899 #ifdef CONFIG_X86_64 900 901 #define PV_SAVE_REGS(set) \ 902 COND_PUSH(set, CLBR_RAX, rax); \ 903 COND_PUSH(set, CLBR_RCX, rcx); \ 904 COND_PUSH(set, CLBR_RDX, rdx); \ 905 COND_PUSH(set, CLBR_RSI, rsi); \ 906 COND_PUSH(set, CLBR_RDI, rdi); \ 907 COND_PUSH(set, CLBR_R8, r8); \ 908 COND_PUSH(set, CLBR_R9, r9); \ 909 COND_PUSH(set, CLBR_R10, r10); \ 910 COND_PUSH(set, CLBR_R11, r11) 911 #define PV_RESTORE_REGS(set) \ 912 COND_POP(set, CLBR_R11, r11); \ 913 COND_POP(set, CLBR_R10, r10); \ 914 COND_POP(set, CLBR_R9, r9); \ 915 COND_POP(set, CLBR_R8, r8); \ 916 COND_POP(set, CLBR_RDI, rdi); \ 917 COND_POP(set, CLBR_RSI, rsi); \ 918 COND_POP(set, CLBR_RDX, rdx); \ 919 COND_POP(set, CLBR_RCX, rcx); \ 920 COND_POP(set, CLBR_RAX, rax) 921 922 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8) 923 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8) 924 #define PARA_INDIRECT(addr) *addr(%rip) 925 #else 926 #define PV_SAVE_REGS(set) \ 927 COND_PUSH(set, CLBR_EAX, eax); \ 928 COND_PUSH(set, CLBR_EDI, edi); \ 929 COND_PUSH(set, CLBR_ECX, ecx); \ 930 COND_PUSH(set, CLBR_EDX, edx) 931 #define PV_RESTORE_REGS(set) \ 932 COND_POP(set, CLBR_EDX, edx); \ 933 COND_POP(set, CLBR_ECX, ecx); \ 934 COND_POP(set, CLBR_EDI, edi); \ 935 COND_POP(set, CLBR_EAX, eax) 936 937 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4) 938 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4) 939 #define PARA_INDIRECT(addr) *%cs:addr 940 #endif 941 942 #define INTERRUPT_RETURN \ 943 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \ 944 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret)) 945 946 #define DISABLE_INTERRUPTS(clobbers) \ 947 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \ 948 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ 949 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \ 950 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) 951 952 #define ENABLE_INTERRUPTS(clobbers) \ 953 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \ 954 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ 955 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \ 956 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) 957 958 #define USERGS_SYSRET32 \ 959 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \ 960 CLBR_NONE, \ 961 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32)) 962 963 #ifdef CONFIG_X86_32 964 #define GET_CR0_INTO_EAX \ 965 push %ecx; push %edx; \ 966 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \ 967 pop %edx; pop %ecx 968 969 #define ENABLE_INTERRUPTS_SYSEXIT \ 970 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \ 971 CLBR_NONE, \ 972 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit)) 973 974 975 #else /* !CONFIG_X86_32 */ 976 977 /* 978 * If swapgs is used while the userspace stack is still current, 979 * there's no way to call a pvop. The PV replacement *must* be 980 * inlined, or the swapgs instruction must be trapped and emulated. 981 */ 982 #define SWAPGS_UNSAFE_STACK \ 983 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ 984 swapgs) 985 986 /* 987 * Note: swapgs is very special, and in practise is either going to be 988 * implemented with a single "swapgs" instruction or something very 989 * special. Either way, we don't need to save any registers for 990 * it. 991 */ 992 #define SWAPGS \ 993 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ 994 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \ 995 ) 996 997 #define GET_CR2_INTO_RCX \ 998 call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \ 999 movq %rax, %rcx; \ 1000 xorq %rax, %rax; 1001 1002 #define PARAVIRT_ADJUST_EXCEPTION_FRAME \ 1003 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \ 1004 CLBR_NONE, \ 1005 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame)) 1006 1007 #define USERGS_SYSRET64 \ 1008 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \ 1009 CLBR_NONE, \ 1010 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64)) 1011 1012 #define ENABLE_INTERRUPTS_SYSEXIT32 \ 1013 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \ 1014 CLBR_NONE, \ 1015 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit)) 1016 #endif /* CONFIG_X86_32 */ 1017 1018 #endif /* __ASSEMBLY__ */ 1019 #else /* CONFIG_PARAVIRT */ 1020 # define default_banner x86_init_noop 1021 #endif /* !CONFIG_PARAVIRT */ 1022 #endif /* _ASM_X86_PARAVIRT_H */ 1023