1 /* 2 * Core of Xen paravirt_ops implementation. 3 * 4 * This file contains the xen_paravirt_ops structure itself, and the 5 * implementations for: 6 * - privileged instructions 7 * - interrupt flags 8 * - segment operations 9 * - booting and setup 10 * 11 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 12 */ 13 14 #include <linux/kernel.h> 15 #include <linux/init.h> 16 #include <linux/smp.h> 17 #include <linux/preempt.h> 18 #include <linux/hardirq.h> 19 #include <linux/percpu.h> 20 #include <linux/delay.h> 21 #include <linux/start_kernel.h> 22 #include <linux/sched.h> 23 #include <linux/bootmem.h> 24 #include <linux/module.h> 25 #include <linux/mm.h> 26 #include <linux/page-flags.h> 27 #include <linux/highmem.h> 28 #include <linux/console.h> 29 30 #include <xen/interface/xen.h> 31 #include <xen/interface/physdev.h> 32 #include <xen/interface/vcpu.h> 33 #include <xen/features.h> 34 #include <xen/page.h> 35 #include <xen/hvc-console.h> 36 37 #include <asm/paravirt.h> 38 #include <asm/apic.h> 39 #include <asm/page.h> 40 #include <asm/xen/hypercall.h> 41 #include <asm/xen/hypervisor.h> 42 #include <asm/fixmap.h> 43 #include <asm/processor.h> 44 #include <asm/msr-index.h> 45 #include <asm/setup.h> 46 #include <asm/desc.h> 47 #include <asm/pgtable.h> 48 #include <asm/tlbflush.h> 49 #include <asm/reboot.h> 50 51 #include "xen-ops.h" 52 #include "mmu.h" 53 #include "multicalls.h" 54 55 EXPORT_SYMBOL_GPL(hypercall_page); 56 57 DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu); 58 DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info); 59 60 enum xen_domain_type xen_domain_type = XEN_NATIVE; 61 EXPORT_SYMBOL_GPL(xen_domain_type); 62 63 /* 64 * Identity map, in addition to plain kernel map. This needs to be 65 * large enough to allocate page table pages to allocate the rest. 66 * Each page can map 2MB. 67 */ 68 static pte_t level1_ident_pgt[PTRS_PER_PTE * 4] __page_aligned_bss; 69 70 #ifdef CONFIG_X86_64 71 /* l3 pud for userspace vsyscall mapping */ 72 static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss; 73 #endif /* CONFIG_X86_64 */ 74 75 /* 76 * Note about cr3 (pagetable base) values: 77 * 78 * xen_cr3 contains the current logical cr3 value; it contains the 79 * last set cr3. This may not be the current effective cr3, because 80 * its update may be being lazily deferred. However, a vcpu looking 81 * at its own cr3 can use this value knowing that it everything will 82 * be self-consistent. 83 * 84 * xen_current_cr3 contains the actual vcpu cr3; it is set once the 85 * hypercall to set the vcpu cr3 is complete (so it may be a little 86 * out of date, but it will never be set early). If one vcpu is 87 * looking at another vcpu's cr3 value, it should use this variable. 88 */ 89 DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */ 90 DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */ 91 92 struct start_info *xen_start_info; 93 EXPORT_SYMBOL_GPL(xen_start_info); 94 95 struct shared_info xen_dummy_shared_info; 96 97 /* 98 * Point at some empty memory to start with. We map the real shared_info 99 * page as soon as fixmap is up and running. 100 */ 101 struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info; 102 103 /* 104 * Flag to determine whether vcpu info placement is available on all 105 * VCPUs. We assume it is to start with, and then set it to zero on 106 * the first failure. This is because it can succeed on some VCPUs 107 * and not others, since it can involve hypervisor memory allocation, 108 * or because the guest failed to guarantee all the appropriate 109 * constraints on all VCPUs (ie buffer can't cross a page boundary). 110 * 111 * Note that any particular CPU may be using a placed vcpu structure, 112 * but we can only optimise if the all are. 113 * 114 * 0: not available, 1: available 115 */ 116 static int have_vcpu_info_placement = 117 #ifdef CONFIG_X86_32 118 1 119 #else 120 0 121 #endif 122 ; 123 124 125 static void xen_vcpu_setup(int cpu) 126 { 127 struct vcpu_register_vcpu_info info; 128 int err; 129 struct vcpu_info *vcpup; 130 131 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info); 132 per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu]; 133 134 if (!have_vcpu_info_placement) 135 return; /* already tested, not available */ 136 137 vcpup = &per_cpu(xen_vcpu_info, cpu); 138 139 info.mfn = virt_to_mfn(vcpup); 140 info.offset = offset_in_page(vcpup); 141 142 printk(KERN_DEBUG "trying to map vcpu_info %d at %p, mfn %llx, offset %d\n", 143 cpu, vcpup, info.mfn, info.offset); 144 145 /* Check to see if the hypervisor will put the vcpu_info 146 structure where we want it, which allows direct access via 147 a percpu-variable. */ 148 err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info); 149 150 if (err) { 151 printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err); 152 have_vcpu_info_placement = 0; 153 } else { 154 /* This cpu is using the registered vcpu info, even if 155 later ones fail to. */ 156 per_cpu(xen_vcpu, cpu) = vcpup; 157 158 printk(KERN_DEBUG "cpu %d using vcpu_info at %p\n", 159 cpu, vcpup); 160 } 161 } 162 163 /* 164 * On restore, set the vcpu placement up again. 165 * If it fails, then we're in a bad state, since 166 * we can't back out from using it... 167 */ 168 void xen_vcpu_restore(void) 169 { 170 if (have_vcpu_info_placement) { 171 int cpu; 172 173 for_each_online_cpu(cpu) { 174 bool other_cpu = (cpu != smp_processor_id()); 175 176 if (other_cpu && 177 HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL)) 178 BUG(); 179 180 xen_vcpu_setup(cpu); 181 182 if (other_cpu && 183 HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL)) 184 BUG(); 185 } 186 187 BUG_ON(!have_vcpu_info_placement); 188 } 189 } 190 191 static void __init xen_banner(void) 192 { 193 unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL); 194 struct xen_extraversion extra; 195 HYPERVISOR_xen_version(XENVER_extraversion, &extra); 196 197 printk(KERN_INFO "Booting paravirtualized kernel on %s\n", 198 pv_info.name); 199 printk(KERN_INFO "Xen version: %d.%d%s%s\n", 200 version >> 16, version & 0xffff, extra.extraversion, 201 xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : ""); 202 } 203 204 static void xen_cpuid(unsigned int *ax, unsigned int *bx, 205 unsigned int *cx, unsigned int *dx) 206 { 207 unsigned maskedx = ~0; 208 209 /* 210 * Mask out inconvenient features, to try and disable as many 211 * unsupported kernel subsystems as possible. 212 */ 213 if (*ax == 1) 214 maskedx = ~((1 << X86_FEATURE_APIC) | /* disable APIC */ 215 (1 << X86_FEATURE_ACPI) | /* disable ACPI */ 216 (1 << X86_FEATURE_MCE) | /* disable MCE */ 217 (1 << X86_FEATURE_MCA) | /* disable MCA */ 218 (1 << X86_FEATURE_ACC)); /* thermal monitoring */ 219 220 asm(XEN_EMULATE_PREFIX "cpuid" 221 : "=a" (*ax), 222 "=b" (*bx), 223 "=c" (*cx), 224 "=d" (*dx) 225 : "0" (*ax), "2" (*cx)); 226 *dx &= maskedx; 227 } 228 229 static void xen_set_debugreg(int reg, unsigned long val) 230 { 231 HYPERVISOR_set_debugreg(reg, val); 232 } 233 234 static unsigned long xen_get_debugreg(int reg) 235 { 236 return HYPERVISOR_get_debugreg(reg); 237 } 238 239 static void xen_leave_lazy(void) 240 { 241 paravirt_leave_lazy(paravirt_get_lazy_mode()); 242 xen_mc_flush(); 243 } 244 245 static unsigned long xen_store_tr(void) 246 { 247 return 0; 248 } 249 250 /* 251 * Set the page permissions for a particular virtual address. If the 252 * address is a vmalloc mapping (or other non-linear mapping), then 253 * find the linear mapping of the page and also set its protections to 254 * match. 255 */ 256 static void set_aliased_prot(void *v, pgprot_t prot) 257 { 258 int level; 259 pte_t *ptep; 260 pte_t pte; 261 unsigned long pfn; 262 struct page *page; 263 264 ptep = lookup_address((unsigned long)v, &level); 265 BUG_ON(ptep == NULL); 266 267 pfn = pte_pfn(*ptep); 268 page = pfn_to_page(pfn); 269 270 pte = pfn_pte(pfn, prot); 271 272 if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0)) 273 BUG(); 274 275 if (!PageHighMem(page)) { 276 void *av = __va(PFN_PHYS(pfn)); 277 278 if (av != v) 279 if (HYPERVISOR_update_va_mapping((unsigned long)av, pte, 0)) 280 BUG(); 281 } else 282 kmap_flush_unused(); 283 } 284 285 static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries) 286 { 287 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE; 288 int i; 289 290 for(i = 0; i < entries; i += entries_per_page) 291 set_aliased_prot(ldt + i, PAGE_KERNEL_RO); 292 } 293 294 static void xen_free_ldt(struct desc_struct *ldt, unsigned entries) 295 { 296 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE; 297 int i; 298 299 for(i = 0; i < entries; i += entries_per_page) 300 set_aliased_prot(ldt + i, PAGE_KERNEL); 301 } 302 303 static void xen_set_ldt(const void *addr, unsigned entries) 304 { 305 struct mmuext_op *op; 306 struct multicall_space mcs = xen_mc_entry(sizeof(*op)); 307 308 op = mcs.args; 309 op->cmd = MMUEXT_SET_LDT; 310 op->arg1.linear_addr = (unsigned long)addr; 311 op->arg2.nr_ents = entries; 312 313 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); 314 315 xen_mc_issue(PARAVIRT_LAZY_CPU); 316 } 317 318 static void xen_load_gdt(const struct desc_ptr *dtr) 319 { 320 unsigned long *frames; 321 unsigned long va = dtr->address; 322 unsigned int size = dtr->size + 1; 323 unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; 324 int f; 325 struct multicall_space mcs; 326 327 /* A GDT can be up to 64k in size, which corresponds to 8192 328 8-byte entries, or 16 4k pages.. */ 329 330 BUG_ON(size > 65536); 331 BUG_ON(va & ~PAGE_MASK); 332 333 mcs = xen_mc_entry(sizeof(*frames) * pages); 334 frames = mcs.args; 335 336 for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) { 337 frames[f] = virt_to_mfn(va); 338 make_lowmem_page_readonly((void *)va); 339 } 340 341 MULTI_set_gdt(mcs.mc, frames, size / sizeof(struct desc_struct)); 342 343 xen_mc_issue(PARAVIRT_LAZY_CPU); 344 } 345 346 static void load_TLS_descriptor(struct thread_struct *t, 347 unsigned int cpu, unsigned int i) 348 { 349 struct desc_struct *gdt = get_cpu_gdt_table(cpu); 350 xmaddr_t maddr = virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]); 351 struct multicall_space mc = __xen_mc_entry(0); 352 353 MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]); 354 } 355 356 static void xen_load_tls(struct thread_struct *t, unsigned int cpu) 357 { 358 /* 359 * XXX sleazy hack: If we're being called in a lazy-cpu zone, 360 * it means we're in a context switch, and %gs has just been 361 * saved. This means we can zero it out to prevent faults on 362 * exit from the hypervisor if the next process has no %gs. 363 * Either way, it has been saved, and the new value will get 364 * loaded properly. This will go away as soon as Xen has been 365 * modified to not save/restore %gs for normal hypercalls. 366 * 367 * On x86_64, this hack is not used for %gs, because gs points 368 * to KERNEL_GS_BASE (and uses it for PDA references), so we 369 * must not zero %gs on x86_64 370 * 371 * For x86_64, we need to zero %fs, otherwise we may get an 372 * exception between the new %fs descriptor being loaded and 373 * %fs being effectively cleared at __switch_to(). 374 */ 375 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) { 376 #ifdef CONFIG_X86_32 377 loadsegment(gs, 0); 378 #else 379 loadsegment(fs, 0); 380 #endif 381 } 382 383 xen_mc_batch(); 384 385 load_TLS_descriptor(t, cpu, 0); 386 load_TLS_descriptor(t, cpu, 1); 387 load_TLS_descriptor(t, cpu, 2); 388 389 xen_mc_issue(PARAVIRT_LAZY_CPU); 390 } 391 392 #ifdef CONFIG_X86_64 393 static void xen_load_gs_index(unsigned int idx) 394 { 395 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx)) 396 BUG(); 397 } 398 #endif 399 400 static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum, 401 const void *ptr) 402 { 403 xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]); 404 u64 entry = *(u64 *)ptr; 405 406 preempt_disable(); 407 408 xen_mc_flush(); 409 if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry)) 410 BUG(); 411 412 preempt_enable(); 413 } 414 415 static int cvt_gate_to_trap(int vector, const gate_desc *val, 416 struct trap_info *info) 417 { 418 if (val->type != 0xf && val->type != 0xe) 419 return 0; 420 421 info->vector = vector; 422 info->address = gate_offset(*val); 423 info->cs = gate_segment(*val); 424 info->flags = val->dpl; 425 /* interrupt gates clear IF */ 426 if (val->type == 0xe) 427 info->flags |= 4; 428 429 return 1; 430 } 431 432 /* Locations of each CPU's IDT */ 433 static DEFINE_PER_CPU(struct desc_ptr, idt_desc); 434 435 /* Set an IDT entry. If the entry is part of the current IDT, then 436 also update Xen. */ 437 static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g) 438 { 439 unsigned long p = (unsigned long)&dt[entrynum]; 440 unsigned long start, end; 441 442 preempt_disable(); 443 444 start = __get_cpu_var(idt_desc).address; 445 end = start + __get_cpu_var(idt_desc).size + 1; 446 447 xen_mc_flush(); 448 449 native_write_idt_entry(dt, entrynum, g); 450 451 if (p >= start && (p + 8) <= end) { 452 struct trap_info info[2]; 453 454 info[1].address = 0; 455 456 if (cvt_gate_to_trap(entrynum, g, &info[0])) 457 if (HYPERVISOR_set_trap_table(info)) 458 BUG(); 459 } 460 461 preempt_enable(); 462 } 463 464 static void xen_convert_trap_info(const struct desc_ptr *desc, 465 struct trap_info *traps) 466 { 467 unsigned in, out, count; 468 469 count = (desc->size+1) / sizeof(gate_desc); 470 BUG_ON(count > 256); 471 472 for (in = out = 0; in < count; in++) { 473 gate_desc *entry = (gate_desc*)(desc->address) + in; 474 475 if (cvt_gate_to_trap(in, entry, &traps[out])) 476 out++; 477 } 478 traps[out].address = 0; 479 } 480 481 void xen_copy_trap_info(struct trap_info *traps) 482 { 483 const struct desc_ptr *desc = &__get_cpu_var(idt_desc); 484 485 xen_convert_trap_info(desc, traps); 486 } 487 488 /* Load a new IDT into Xen. In principle this can be per-CPU, so we 489 hold a spinlock to protect the static traps[] array (static because 490 it avoids allocation, and saves stack space). */ 491 static void xen_load_idt(const struct desc_ptr *desc) 492 { 493 static DEFINE_SPINLOCK(lock); 494 static struct trap_info traps[257]; 495 496 spin_lock(&lock); 497 498 __get_cpu_var(idt_desc) = *desc; 499 500 xen_convert_trap_info(desc, traps); 501 502 xen_mc_flush(); 503 if (HYPERVISOR_set_trap_table(traps)) 504 BUG(); 505 506 spin_unlock(&lock); 507 } 508 509 /* Write a GDT descriptor entry. Ignore LDT descriptors, since 510 they're handled differently. */ 511 static void xen_write_gdt_entry(struct desc_struct *dt, int entry, 512 const void *desc, int type) 513 { 514 preempt_disable(); 515 516 switch (type) { 517 case DESC_LDT: 518 case DESC_TSS: 519 /* ignore */ 520 break; 521 522 default: { 523 xmaddr_t maddr = virt_to_machine(&dt[entry]); 524 525 xen_mc_flush(); 526 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc)) 527 BUG(); 528 } 529 530 } 531 532 preempt_enable(); 533 } 534 535 static void xen_load_sp0(struct tss_struct *tss, 536 struct thread_struct *thread) 537 { 538 struct multicall_space mcs = xen_mc_entry(0); 539 MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0); 540 xen_mc_issue(PARAVIRT_LAZY_CPU); 541 } 542 543 static void xen_set_iopl_mask(unsigned mask) 544 { 545 struct physdev_set_iopl set_iopl; 546 547 /* Force the change at ring 0. */ 548 set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3; 549 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 550 } 551 552 static void xen_io_delay(void) 553 { 554 } 555 556 #ifdef CONFIG_X86_LOCAL_APIC 557 static u32 xen_apic_read(u32 reg) 558 { 559 return 0; 560 } 561 562 static void xen_apic_write(u32 reg, u32 val) 563 { 564 /* Warn to see if there's any stray references */ 565 WARN_ON(1); 566 } 567 568 static u64 xen_apic_icr_read(void) 569 { 570 return 0; 571 } 572 573 static void xen_apic_icr_write(u32 low, u32 id) 574 { 575 /* Warn to see if there's any stray references */ 576 WARN_ON(1); 577 } 578 579 static void xen_apic_wait_icr_idle(void) 580 { 581 return; 582 } 583 584 static u32 xen_safe_apic_wait_icr_idle(void) 585 { 586 return 0; 587 } 588 589 static struct apic_ops xen_basic_apic_ops = { 590 .read = xen_apic_read, 591 .write = xen_apic_write, 592 .icr_read = xen_apic_icr_read, 593 .icr_write = xen_apic_icr_write, 594 .wait_icr_idle = xen_apic_wait_icr_idle, 595 .safe_wait_icr_idle = xen_safe_apic_wait_icr_idle, 596 }; 597 598 #endif 599 600 static void xen_flush_tlb(void) 601 { 602 struct mmuext_op *op; 603 struct multicall_space mcs; 604 605 preempt_disable(); 606 607 mcs = xen_mc_entry(sizeof(*op)); 608 609 op = mcs.args; 610 op->cmd = MMUEXT_TLB_FLUSH_LOCAL; 611 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); 612 613 xen_mc_issue(PARAVIRT_LAZY_MMU); 614 615 preempt_enable(); 616 } 617 618 static void xen_flush_tlb_single(unsigned long addr) 619 { 620 struct mmuext_op *op; 621 struct multicall_space mcs; 622 623 preempt_disable(); 624 625 mcs = xen_mc_entry(sizeof(*op)); 626 op = mcs.args; 627 op->cmd = MMUEXT_INVLPG_LOCAL; 628 op->arg1.linear_addr = addr & PAGE_MASK; 629 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); 630 631 xen_mc_issue(PARAVIRT_LAZY_MMU); 632 633 preempt_enable(); 634 } 635 636 static void xen_flush_tlb_others(const cpumask_t *cpus, struct mm_struct *mm, 637 unsigned long va) 638 { 639 struct { 640 struct mmuext_op op; 641 cpumask_t mask; 642 } *args; 643 cpumask_t cpumask = *cpus; 644 struct multicall_space mcs; 645 646 /* 647 * A couple of (to be removed) sanity checks: 648 * 649 * - current CPU must not be in mask 650 * - mask must exist :) 651 */ 652 BUG_ON(cpus_empty(cpumask)); 653 BUG_ON(cpu_isset(smp_processor_id(), cpumask)); 654 BUG_ON(!mm); 655 656 /* If a CPU which we ran on has gone down, OK. */ 657 cpus_and(cpumask, cpumask, cpu_online_map); 658 if (cpus_empty(cpumask)) 659 return; 660 661 mcs = xen_mc_entry(sizeof(*args)); 662 args = mcs.args; 663 args->mask = cpumask; 664 args->op.arg2.vcpumask = &args->mask; 665 666 if (va == TLB_FLUSH_ALL) { 667 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI; 668 } else { 669 args->op.cmd = MMUEXT_INVLPG_MULTI; 670 args->op.arg1.linear_addr = va; 671 } 672 673 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF); 674 675 xen_mc_issue(PARAVIRT_LAZY_MMU); 676 } 677 678 static void xen_clts(void) 679 { 680 struct multicall_space mcs; 681 682 mcs = xen_mc_entry(0); 683 684 MULTI_fpu_taskswitch(mcs.mc, 0); 685 686 xen_mc_issue(PARAVIRT_LAZY_CPU); 687 } 688 689 static void xen_write_cr0(unsigned long cr0) 690 { 691 struct multicall_space mcs; 692 693 /* Only pay attention to cr0.TS; everything else is 694 ignored. */ 695 mcs = xen_mc_entry(0); 696 697 MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0); 698 699 xen_mc_issue(PARAVIRT_LAZY_CPU); 700 } 701 702 static void xen_write_cr2(unsigned long cr2) 703 { 704 x86_read_percpu(xen_vcpu)->arch.cr2 = cr2; 705 } 706 707 static unsigned long xen_read_cr2(void) 708 { 709 return x86_read_percpu(xen_vcpu)->arch.cr2; 710 } 711 712 static unsigned long xen_read_cr2_direct(void) 713 { 714 return x86_read_percpu(xen_vcpu_info.arch.cr2); 715 } 716 717 static void xen_write_cr4(unsigned long cr4) 718 { 719 cr4 &= ~X86_CR4_PGE; 720 cr4 &= ~X86_CR4_PSE; 721 722 native_write_cr4(cr4); 723 } 724 725 static unsigned long xen_read_cr3(void) 726 { 727 return x86_read_percpu(xen_cr3); 728 } 729 730 static void set_current_cr3(void *v) 731 { 732 x86_write_percpu(xen_current_cr3, (unsigned long)v); 733 } 734 735 static void __xen_write_cr3(bool kernel, unsigned long cr3) 736 { 737 struct mmuext_op *op; 738 struct multicall_space mcs; 739 unsigned long mfn; 740 741 if (cr3) 742 mfn = pfn_to_mfn(PFN_DOWN(cr3)); 743 else 744 mfn = 0; 745 746 WARN_ON(mfn == 0 && kernel); 747 748 mcs = __xen_mc_entry(sizeof(*op)); 749 750 op = mcs.args; 751 op->cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR; 752 op->arg1.mfn = mfn; 753 754 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); 755 756 if (kernel) { 757 x86_write_percpu(xen_cr3, cr3); 758 759 /* Update xen_current_cr3 once the batch has actually 760 been submitted. */ 761 xen_mc_callback(set_current_cr3, (void *)cr3); 762 } 763 } 764 765 static void xen_write_cr3(unsigned long cr3) 766 { 767 BUG_ON(preemptible()); 768 769 xen_mc_batch(); /* disables interrupts */ 770 771 /* Update while interrupts are disabled, so its atomic with 772 respect to ipis */ 773 x86_write_percpu(xen_cr3, cr3); 774 775 __xen_write_cr3(true, cr3); 776 777 #ifdef CONFIG_X86_64 778 { 779 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3)); 780 if (user_pgd) 781 __xen_write_cr3(false, __pa(user_pgd)); 782 else 783 __xen_write_cr3(false, 0); 784 } 785 #endif 786 787 xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */ 788 } 789 790 static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high) 791 { 792 int ret; 793 794 ret = 0; 795 796 switch(msr) { 797 #ifdef CONFIG_X86_64 798 unsigned which; 799 u64 base; 800 801 case MSR_FS_BASE: which = SEGBASE_FS; goto set; 802 case MSR_KERNEL_GS_BASE: which = SEGBASE_GS_USER; goto set; 803 case MSR_GS_BASE: which = SEGBASE_GS_KERNEL; goto set; 804 805 set: 806 base = ((u64)high << 32) | low; 807 if (HYPERVISOR_set_segment_base(which, base) != 0) 808 ret = -EFAULT; 809 break; 810 #endif 811 812 case MSR_STAR: 813 case MSR_CSTAR: 814 case MSR_LSTAR: 815 case MSR_SYSCALL_MASK: 816 case MSR_IA32_SYSENTER_CS: 817 case MSR_IA32_SYSENTER_ESP: 818 case MSR_IA32_SYSENTER_EIP: 819 /* Fast syscall setup is all done in hypercalls, so 820 these are all ignored. Stub them out here to stop 821 Xen console noise. */ 822 break; 823 824 default: 825 ret = native_write_msr_safe(msr, low, high); 826 } 827 828 return ret; 829 } 830 831 /* Early in boot, while setting up the initial pagetable, assume 832 everything is pinned. */ 833 static __init void xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn) 834 { 835 #ifdef CONFIG_FLATMEM 836 BUG_ON(mem_map); /* should only be used early */ 837 #endif 838 make_lowmem_page_readonly(__va(PFN_PHYS(pfn))); 839 } 840 841 /* Early release_pte assumes that all pts are pinned, since there's 842 only init_mm and anything attached to that is pinned. */ 843 static void xen_release_pte_init(unsigned long pfn) 844 { 845 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); 846 } 847 848 static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn) 849 { 850 struct mmuext_op op; 851 op.cmd = cmd; 852 op.arg1.mfn = pfn_to_mfn(pfn); 853 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF)) 854 BUG(); 855 } 856 857 /* This needs to make sure the new pte page is pinned iff its being 858 attached to a pinned pagetable. */ 859 static void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn, unsigned level) 860 { 861 struct page *page = pfn_to_page(pfn); 862 863 if (PagePinned(virt_to_page(mm->pgd))) { 864 SetPagePinned(page); 865 866 if (!PageHighMem(page)) { 867 make_lowmem_page_readonly(__va(PFN_PHYS((unsigned long)pfn))); 868 if (level == PT_PTE && USE_SPLIT_PTLOCKS) 869 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn); 870 } else 871 /* make sure there are no stray mappings of 872 this page */ 873 kmap_flush_unused(); 874 vm_unmap_aliases(); 875 } 876 } 877 878 static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn) 879 { 880 xen_alloc_ptpage(mm, pfn, PT_PTE); 881 } 882 883 static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn) 884 { 885 xen_alloc_ptpage(mm, pfn, PT_PMD); 886 } 887 888 static int xen_pgd_alloc(struct mm_struct *mm) 889 { 890 pgd_t *pgd = mm->pgd; 891 int ret = 0; 892 893 BUG_ON(PagePinned(virt_to_page(pgd))); 894 895 #ifdef CONFIG_X86_64 896 { 897 struct page *page = virt_to_page(pgd); 898 pgd_t *user_pgd; 899 900 BUG_ON(page->private != 0); 901 902 ret = -ENOMEM; 903 904 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 905 page->private = (unsigned long)user_pgd; 906 907 if (user_pgd != NULL) { 908 user_pgd[pgd_index(VSYSCALL_START)] = 909 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE); 910 ret = 0; 911 } 912 913 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd)))); 914 } 915 #endif 916 917 return ret; 918 } 919 920 static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd) 921 { 922 #ifdef CONFIG_X86_64 923 pgd_t *user_pgd = xen_get_user_pgd(pgd); 924 925 if (user_pgd) 926 free_page((unsigned long)user_pgd); 927 #endif 928 } 929 930 /* This should never happen until we're OK to use struct page */ 931 static void xen_release_ptpage(unsigned long pfn, unsigned level) 932 { 933 struct page *page = pfn_to_page(pfn); 934 935 if (PagePinned(page)) { 936 if (!PageHighMem(page)) { 937 if (level == PT_PTE && USE_SPLIT_PTLOCKS) 938 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn); 939 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); 940 } 941 ClearPagePinned(page); 942 } 943 } 944 945 static void xen_release_pte(unsigned long pfn) 946 { 947 xen_release_ptpage(pfn, PT_PTE); 948 } 949 950 static void xen_release_pmd(unsigned long pfn) 951 { 952 xen_release_ptpage(pfn, PT_PMD); 953 } 954 955 #if PAGETABLE_LEVELS == 4 956 static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn) 957 { 958 xen_alloc_ptpage(mm, pfn, PT_PUD); 959 } 960 961 static void xen_release_pud(unsigned long pfn) 962 { 963 xen_release_ptpage(pfn, PT_PUD); 964 } 965 #endif 966 967 #ifdef CONFIG_HIGHPTE 968 static void *xen_kmap_atomic_pte(struct page *page, enum km_type type) 969 { 970 pgprot_t prot = PAGE_KERNEL; 971 972 if (PagePinned(page)) 973 prot = PAGE_KERNEL_RO; 974 975 if (0 && PageHighMem(page)) 976 printk("mapping highpte %lx type %d prot %s\n", 977 page_to_pfn(page), type, 978 (unsigned long)pgprot_val(prot) & _PAGE_RW ? "WRITE" : "READ"); 979 980 return kmap_atomic_prot(page, type, prot); 981 } 982 #endif 983 984 #ifdef CONFIG_X86_32 985 static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte) 986 { 987 /* If there's an existing pte, then don't allow _PAGE_RW to be set */ 988 if (pte_val_ma(*ptep) & _PAGE_PRESENT) 989 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) & 990 pte_val_ma(pte)); 991 992 return pte; 993 } 994 995 /* Init-time set_pte while constructing initial pagetables, which 996 doesn't allow RO pagetable pages to be remapped RW */ 997 static __init void xen_set_pte_init(pte_t *ptep, pte_t pte) 998 { 999 pte = mask_rw_pte(ptep, pte); 1000 1001 xen_set_pte(ptep, pte); 1002 } 1003 #endif 1004 1005 static __init void xen_pagetable_setup_start(pgd_t *base) 1006 { 1007 } 1008 1009 void xen_setup_shared_info(void) 1010 { 1011 if (!xen_feature(XENFEAT_auto_translated_physmap)) { 1012 set_fixmap(FIX_PARAVIRT_BOOTMAP, 1013 xen_start_info->shared_info); 1014 1015 HYPERVISOR_shared_info = 1016 (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP); 1017 } else 1018 HYPERVISOR_shared_info = 1019 (struct shared_info *)__va(xen_start_info->shared_info); 1020 1021 #ifndef CONFIG_SMP 1022 /* In UP this is as good a place as any to set up shared info */ 1023 xen_setup_vcpu_info_placement(); 1024 #endif 1025 1026 xen_setup_mfn_list_list(); 1027 } 1028 1029 static __init void xen_pagetable_setup_done(pgd_t *base) 1030 { 1031 xen_setup_shared_info(); 1032 } 1033 1034 static __init void xen_post_allocator_init(void) 1035 { 1036 pv_mmu_ops.set_pte = xen_set_pte; 1037 pv_mmu_ops.set_pmd = xen_set_pmd; 1038 pv_mmu_ops.set_pud = xen_set_pud; 1039 #if PAGETABLE_LEVELS == 4 1040 pv_mmu_ops.set_pgd = xen_set_pgd; 1041 #endif 1042 1043 /* This will work as long as patching hasn't happened yet 1044 (which it hasn't) */ 1045 pv_mmu_ops.alloc_pte = xen_alloc_pte; 1046 pv_mmu_ops.alloc_pmd = xen_alloc_pmd; 1047 pv_mmu_ops.release_pte = xen_release_pte; 1048 pv_mmu_ops.release_pmd = xen_release_pmd; 1049 #if PAGETABLE_LEVELS == 4 1050 pv_mmu_ops.alloc_pud = xen_alloc_pud; 1051 pv_mmu_ops.release_pud = xen_release_pud; 1052 #endif 1053 1054 #ifdef CONFIG_X86_64 1055 SetPagePinned(virt_to_page(level3_user_vsyscall)); 1056 #endif 1057 xen_mark_init_mm_pinned(); 1058 } 1059 1060 /* This is called once we have the cpu_possible_map */ 1061 void xen_setup_vcpu_info_placement(void) 1062 { 1063 int cpu; 1064 1065 for_each_possible_cpu(cpu) 1066 xen_vcpu_setup(cpu); 1067 1068 /* xen_vcpu_setup managed to place the vcpu_info within the 1069 percpu area for all cpus, so make use of it */ 1070 if (have_vcpu_info_placement) { 1071 printk(KERN_INFO "Xen: using vcpu_info placement\n"); 1072 1073 pv_irq_ops.save_fl = xen_save_fl_direct; 1074 pv_irq_ops.restore_fl = xen_restore_fl_direct; 1075 pv_irq_ops.irq_disable = xen_irq_disable_direct; 1076 pv_irq_ops.irq_enable = xen_irq_enable_direct; 1077 pv_mmu_ops.read_cr2 = xen_read_cr2_direct; 1078 } 1079 } 1080 1081 static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf, 1082 unsigned long addr, unsigned len) 1083 { 1084 char *start, *end, *reloc; 1085 unsigned ret; 1086 1087 start = end = reloc = NULL; 1088 1089 #define SITE(op, x) \ 1090 case PARAVIRT_PATCH(op.x): \ 1091 if (have_vcpu_info_placement) { \ 1092 start = (char *)xen_##x##_direct; \ 1093 end = xen_##x##_direct_end; \ 1094 reloc = xen_##x##_direct_reloc; \ 1095 } \ 1096 goto patch_site 1097 1098 switch (type) { 1099 SITE(pv_irq_ops, irq_enable); 1100 SITE(pv_irq_ops, irq_disable); 1101 SITE(pv_irq_ops, save_fl); 1102 SITE(pv_irq_ops, restore_fl); 1103 #undef SITE 1104 1105 patch_site: 1106 if (start == NULL || (end-start) > len) 1107 goto default_patch; 1108 1109 ret = paravirt_patch_insns(insnbuf, len, start, end); 1110 1111 /* Note: because reloc is assigned from something that 1112 appears to be an array, gcc assumes it's non-null, 1113 but doesn't know its relationship with start and 1114 end. */ 1115 if (reloc > start && reloc < end) { 1116 int reloc_off = reloc - start; 1117 long *relocp = (long *)(insnbuf + reloc_off); 1118 long delta = start - (char *)addr; 1119 1120 *relocp += delta; 1121 } 1122 break; 1123 1124 default_patch: 1125 default: 1126 ret = paravirt_patch_default(type, clobbers, insnbuf, 1127 addr, len); 1128 break; 1129 } 1130 1131 return ret; 1132 } 1133 1134 static void xen_set_fixmap(unsigned idx, unsigned long phys, pgprot_t prot) 1135 { 1136 pte_t pte; 1137 1138 phys >>= PAGE_SHIFT; 1139 1140 switch (idx) { 1141 case FIX_BTMAP_END ... FIX_BTMAP_BEGIN: 1142 #ifdef CONFIG_X86_F00F_BUG 1143 case FIX_F00F_IDT: 1144 #endif 1145 #ifdef CONFIG_X86_32 1146 case FIX_WP_TEST: 1147 case FIX_VDSO: 1148 # ifdef CONFIG_HIGHMEM 1149 case FIX_KMAP_BEGIN ... FIX_KMAP_END: 1150 # endif 1151 #else 1152 case VSYSCALL_LAST_PAGE ... VSYSCALL_FIRST_PAGE: 1153 #endif 1154 #ifdef CONFIG_X86_LOCAL_APIC 1155 case FIX_APIC_BASE: /* maps dummy local APIC */ 1156 #endif 1157 pte = pfn_pte(phys, prot); 1158 break; 1159 1160 default: 1161 pte = mfn_pte(phys, prot); 1162 break; 1163 } 1164 1165 __native_set_fixmap(idx, pte); 1166 1167 #ifdef CONFIG_X86_64 1168 /* Replicate changes to map the vsyscall page into the user 1169 pagetable vsyscall mapping. */ 1170 if (idx >= VSYSCALL_LAST_PAGE && idx <= VSYSCALL_FIRST_PAGE) { 1171 unsigned long vaddr = __fix_to_virt(idx); 1172 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte); 1173 } 1174 #endif 1175 } 1176 1177 static const struct pv_info xen_info __initdata = { 1178 .paravirt_enabled = 1, 1179 .shared_kernel_pmd = 0, 1180 1181 .name = "Xen", 1182 }; 1183 1184 static const struct pv_init_ops xen_init_ops __initdata = { 1185 .patch = xen_patch, 1186 1187 .banner = xen_banner, 1188 .memory_setup = xen_memory_setup, 1189 .arch_setup = xen_arch_setup, 1190 .post_allocator_init = xen_post_allocator_init, 1191 }; 1192 1193 static const struct pv_time_ops xen_time_ops __initdata = { 1194 .time_init = xen_time_init, 1195 1196 .set_wallclock = xen_set_wallclock, 1197 .get_wallclock = xen_get_wallclock, 1198 .get_tsc_khz = xen_tsc_khz, 1199 .sched_clock = xen_sched_clock, 1200 }; 1201 1202 static const struct pv_cpu_ops xen_cpu_ops __initdata = { 1203 .cpuid = xen_cpuid, 1204 1205 .set_debugreg = xen_set_debugreg, 1206 .get_debugreg = xen_get_debugreg, 1207 1208 .clts = xen_clts, 1209 1210 .read_cr0 = native_read_cr0, 1211 .write_cr0 = xen_write_cr0, 1212 1213 .read_cr4 = native_read_cr4, 1214 .read_cr4_safe = native_read_cr4_safe, 1215 .write_cr4 = xen_write_cr4, 1216 1217 .wbinvd = native_wbinvd, 1218 1219 .read_msr = native_read_msr_safe, 1220 .write_msr = xen_write_msr_safe, 1221 .read_tsc = native_read_tsc, 1222 .read_pmc = native_read_pmc, 1223 1224 .iret = xen_iret, 1225 .irq_enable_sysexit = xen_sysexit, 1226 #ifdef CONFIG_X86_64 1227 .usergs_sysret32 = xen_sysret32, 1228 .usergs_sysret64 = xen_sysret64, 1229 #endif 1230 1231 .load_tr_desc = paravirt_nop, 1232 .set_ldt = xen_set_ldt, 1233 .load_gdt = xen_load_gdt, 1234 .load_idt = xen_load_idt, 1235 .load_tls = xen_load_tls, 1236 #ifdef CONFIG_X86_64 1237 .load_gs_index = xen_load_gs_index, 1238 #endif 1239 1240 .alloc_ldt = xen_alloc_ldt, 1241 .free_ldt = xen_free_ldt, 1242 1243 .store_gdt = native_store_gdt, 1244 .store_idt = native_store_idt, 1245 .store_tr = xen_store_tr, 1246 1247 .write_ldt_entry = xen_write_ldt_entry, 1248 .write_gdt_entry = xen_write_gdt_entry, 1249 .write_idt_entry = xen_write_idt_entry, 1250 .load_sp0 = xen_load_sp0, 1251 1252 .set_iopl_mask = xen_set_iopl_mask, 1253 .io_delay = xen_io_delay, 1254 1255 /* Xen takes care of %gs when switching to usermode for us */ 1256 .swapgs = paravirt_nop, 1257 1258 .lazy_mode = { 1259 .enter = paravirt_enter_lazy_cpu, 1260 .leave = xen_leave_lazy, 1261 }, 1262 }; 1263 1264 static const struct pv_apic_ops xen_apic_ops __initdata = { 1265 #ifdef CONFIG_X86_LOCAL_APIC 1266 .setup_boot_clock = paravirt_nop, 1267 .setup_secondary_clock = paravirt_nop, 1268 .startup_ipi_hook = paravirt_nop, 1269 #endif 1270 }; 1271 1272 static const struct pv_mmu_ops xen_mmu_ops __initdata = { 1273 .pagetable_setup_start = xen_pagetable_setup_start, 1274 .pagetable_setup_done = xen_pagetable_setup_done, 1275 1276 .read_cr2 = xen_read_cr2, 1277 .write_cr2 = xen_write_cr2, 1278 1279 .read_cr3 = xen_read_cr3, 1280 .write_cr3 = xen_write_cr3, 1281 1282 .flush_tlb_user = xen_flush_tlb, 1283 .flush_tlb_kernel = xen_flush_tlb, 1284 .flush_tlb_single = xen_flush_tlb_single, 1285 .flush_tlb_others = xen_flush_tlb_others, 1286 1287 .pte_update = paravirt_nop, 1288 .pte_update_defer = paravirt_nop, 1289 1290 .pgd_alloc = xen_pgd_alloc, 1291 .pgd_free = xen_pgd_free, 1292 1293 .alloc_pte = xen_alloc_pte_init, 1294 .release_pte = xen_release_pte_init, 1295 .alloc_pmd = xen_alloc_pte_init, 1296 .alloc_pmd_clone = paravirt_nop, 1297 .release_pmd = xen_release_pte_init, 1298 1299 #ifdef CONFIG_HIGHPTE 1300 .kmap_atomic_pte = xen_kmap_atomic_pte, 1301 #endif 1302 1303 #ifdef CONFIG_X86_64 1304 .set_pte = xen_set_pte, 1305 #else 1306 .set_pte = xen_set_pte_init, 1307 #endif 1308 .set_pte_at = xen_set_pte_at, 1309 .set_pmd = xen_set_pmd_hyper, 1310 1311 .ptep_modify_prot_start = __ptep_modify_prot_start, 1312 .ptep_modify_prot_commit = __ptep_modify_prot_commit, 1313 1314 .pte_val = xen_pte_val, 1315 .pte_flags = native_pte_flags, 1316 .pgd_val = xen_pgd_val, 1317 1318 .make_pte = xen_make_pte, 1319 .make_pgd = xen_make_pgd, 1320 1321 #ifdef CONFIG_X86_PAE 1322 .set_pte_atomic = xen_set_pte_atomic, 1323 .set_pte_present = xen_set_pte_at, 1324 .pte_clear = xen_pte_clear, 1325 .pmd_clear = xen_pmd_clear, 1326 #endif /* CONFIG_X86_PAE */ 1327 .set_pud = xen_set_pud_hyper, 1328 1329 .make_pmd = xen_make_pmd, 1330 .pmd_val = xen_pmd_val, 1331 1332 #if PAGETABLE_LEVELS == 4 1333 .pud_val = xen_pud_val, 1334 .make_pud = xen_make_pud, 1335 .set_pgd = xen_set_pgd_hyper, 1336 1337 .alloc_pud = xen_alloc_pte_init, 1338 .release_pud = xen_release_pte_init, 1339 #endif /* PAGETABLE_LEVELS == 4 */ 1340 1341 .activate_mm = xen_activate_mm, 1342 .dup_mmap = xen_dup_mmap, 1343 .exit_mmap = xen_exit_mmap, 1344 1345 .lazy_mode = { 1346 .enter = paravirt_enter_lazy_mmu, 1347 .leave = xen_leave_lazy, 1348 }, 1349 1350 .set_fixmap = xen_set_fixmap, 1351 }; 1352 1353 static void xen_reboot(int reason) 1354 { 1355 struct sched_shutdown r = { .reason = reason }; 1356 1357 #ifdef CONFIG_SMP 1358 smp_send_stop(); 1359 #endif 1360 1361 if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r)) 1362 BUG(); 1363 } 1364 1365 static void xen_restart(char *msg) 1366 { 1367 xen_reboot(SHUTDOWN_reboot); 1368 } 1369 1370 static void xen_emergency_restart(void) 1371 { 1372 xen_reboot(SHUTDOWN_reboot); 1373 } 1374 1375 static void xen_machine_halt(void) 1376 { 1377 xen_reboot(SHUTDOWN_poweroff); 1378 } 1379 1380 static void xen_crash_shutdown(struct pt_regs *regs) 1381 { 1382 xen_reboot(SHUTDOWN_crash); 1383 } 1384 1385 static const struct machine_ops __initdata xen_machine_ops = { 1386 .restart = xen_restart, 1387 .halt = xen_machine_halt, 1388 .power_off = xen_machine_halt, 1389 .shutdown = xen_machine_halt, 1390 .crash_shutdown = xen_crash_shutdown, 1391 .emergency_restart = xen_emergency_restart, 1392 }; 1393 1394 1395 static void __init xen_reserve_top(void) 1396 { 1397 #ifdef CONFIG_X86_32 1398 unsigned long top = HYPERVISOR_VIRT_START; 1399 struct xen_platform_parameters pp; 1400 1401 if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0) 1402 top = pp.virt_start; 1403 1404 reserve_top_address(-top); 1405 #endif /* CONFIG_X86_32 */ 1406 } 1407 1408 /* 1409 * Like __va(), but returns address in the kernel mapping (which is 1410 * all we have until the physical memory mapping has been set up. 1411 */ 1412 static void *__ka(phys_addr_t paddr) 1413 { 1414 #ifdef CONFIG_X86_64 1415 return (void *)(paddr + __START_KERNEL_map); 1416 #else 1417 return __va(paddr); 1418 #endif 1419 } 1420 1421 /* Convert a machine address to physical address */ 1422 static unsigned long m2p(phys_addr_t maddr) 1423 { 1424 phys_addr_t paddr; 1425 1426 maddr &= PTE_PFN_MASK; 1427 paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT; 1428 1429 return paddr; 1430 } 1431 1432 /* Convert a machine address to kernel virtual */ 1433 static void *m2v(phys_addr_t maddr) 1434 { 1435 return __ka(m2p(maddr)); 1436 } 1437 1438 static void set_page_prot(void *addr, pgprot_t prot) 1439 { 1440 unsigned long pfn = __pa(addr) >> PAGE_SHIFT; 1441 pte_t pte = pfn_pte(pfn, prot); 1442 1443 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0)) 1444 BUG(); 1445 } 1446 1447 static __init void xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn) 1448 { 1449 unsigned pmdidx, pteidx; 1450 unsigned ident_pte; 1451 unsigned long pfn; 1452 1453 ident_pte = 0; 1454 pfn = 0; 1455 for(pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) { 1456 pte_t *pte_page; 1457 1458 /* Reuse or allocate a page of ptes */ 1459 if (pmd_present(pmd[pmdidx])) 1460 pte_page = m2v(pmd[pmdidx].pmd); 1461 else { 1462 /* Check for free pte pages */ 1463 if (ident_pte == ARRAY_SIZE(level1_ident_pgt)) 1464 break; 1465 1466 pte_page = &level1_ident_pgt[ident_pte]; 1467 ident_pte += PTRS_PER_PTE; 1468 1469 pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE); 1470 } 1471 1472 /* Install mappings */ 1473 for(pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) { 1474 pte_t pte; 1475 1476 if (pfn > max_pfn_mapped) 1477 max_pfn_mapped = pfn; 1478 1479 if (!pte_none(pte_page[pteidx])) 1480 continue; 1481 1482 pte = pfn_pte(pfn, PAGE_KERNEL_EXEC); 1483 pte_page[pteidx] = pte; 1484 } 1485 } 1486 1487 for(pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE) 1488 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO); 1489 1490 set_page_prot(pmd, PAGE_KERNEL_RO); 1491 } 1492 1493 #ifdef CONFIG_X86_64 1494 static void convert_pfn_mfn(void *v) 1495 { 1496 pte_t *pte = v; 1497 int i; 1498 1499 /* All levels are converted the same way, so just treat them 1500 as ptes. */ 1501 for(i = 0; i < PTRS_PER_PTE; i++) 1502 pte[i] = xen_make_pte(pte[i].pte); 1503 } 1504 1505 /* 1506 * Set up the inital kernel pagetable. 1507 * 1508 * We can construct this by grafting the Xen provided pagetable into 1509 * head_64.S's preconstructed pagetables. We copy the Xen L2's into 1510 * level2_ident_pgt, level2_kernel_pgt and level2_fixmap_pgt. This 1511 * means that only the kernel has a physical mapping to start with - 1512 * but that's enough to get __va working. We need to fill in the rest 1513 * of the physical mapping once some sort of allocator has been set 1514 * up. 1515 */ 1516 static __init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) 1517 { 1518 pud_t *l3; 1519 pmd_t *l2; 1520 1521 /* Zap identity mapping */ 1522 init_level4_pgt[0] = __pgd(0); 1523 1524 /* Pre-constructed entries are in pfn, so convert to mfn */ 1525 convert_pfn_mfn(init_level4_pgt); 1526 convert_pfn_mfn(level3_ident_pgt); 1527 convert_pfn_mfn(level3_kernel_pgt); 1528 1529 l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd); 1530 l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud); 1531 1532 memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD); 1533 memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD); 1534 1535 l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd); 1536 l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud); 1537 memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD); 1538 1539 /* Set up identity map */ 1540 xen_map_identity_early(level2_ident_pgt, max_pfn); 1541 1542 /* Make pagetable pieces RO */ 1543 set_page_prot(init_level4_pgt, PAGE_KERNEL_RO); 1544 set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO); 1545 set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO); 1546 set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO); 1547 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO); 1548 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO); 1549 1550 /* Pin down new L4 */ 1551 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE, 1552 PFN_DOWN(__pa_symbol(init_level4_pgt))); 1553 1554 /* Unpin Xen-provided one */ 1555 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); 1556 1557 /* Switch over */ 1558 pgd = init_level4_pgt; 1559 1560 /* 1561 * At this stage there can be no user pgd, and no page 1562 * structure to attach it to, so make sure we just set kernel 1563 * pgd. 1564 */ 1565 xen_mc_batch(); 1566 __xen_write_cr3(true, __pa(pgd)); 1567 xen_mc_issue(PARAVIRT_LAZY_CPU); 1568 1569 reserve_early(__pa(xen_start_info->pt_base), 1570 __pa(xen_start_info->pt_base + 1571 xen_start_info->nr_pt_frames * PAGE_SIZE), 1572 "XEN PAGETABLES"); 1573 1574 return pgd; 1575 } 1576 #else /* !CONFIG_X86_64 */ 1577 static pmd_t level2_kernel_pgt[PTRS_PER_PMD] __page_aligned_bss; 1578 1579 static __init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) 1580 { 1581 pmd_t *kernel_pmd; 1582 1583 init_pg_tables_start = __pa(pgd); 1584 init_pg_tables_end = __pa(pgd) + xen_start_info->nr_pt_frames*PAGE_SIZE; 1585 max_pfn_mapped = PFN_DOWN(init_pg_tables_end + 512*1024); 1586 1587 kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd); 1588 memcpy(level2_kernel_pgt, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD); 1589 1590 xen_map_identity_early(level2_kernel_pgt, max_pfn); 1591 1592 memcpy(swapper_pg_dir, pgd, sizeof(pgd_t) * PTRS_PER_PGD); 1593 set_pgd(&swapper_pg_dir[KERNEL_PGD_BOUNDARY], 1594 __pgd(__pa(level2_kernel_pgt) | _PAGE_PRESENT)); 1595 1596 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO); 1597 set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO); 1598 set_page_prot(empty_zero_page, PAGE_KERNEL_RO); 1599 1600 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); 1601 1602 xen_write_cr3(__pa(swapper_pg_dir)); 1603 1604 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(swapper_pg_dir))); 1605 1606 return swapper_pg_dir; 1607 } 1608 #endif /* CONFIG_X86_64 */ 1609 1610 /* First C function to be called on Xen boot */ 1611 asmlinkage void __init xen_start_kernel(void) 1612 { 1613 pgd_t *pgd; 1614 1615 if (!xen_start_info) 1616 return; 1617 1618 xen_domain_type = XEN_PV_DOMAIN; 1619 1620 BUG_ON(memcmp(xen_start_info->magic, "xen-3", 5) != 0); 1621 1622 xen_setup_features(); 1623 1624 /* Install Xen paravirt ops */ 1625 pv_info = xen_info; 1626 pv_init_ops = xen_init_ops; 1627 pv_time_ops = xen_time_ops; 1628 pv_cpu_ops = xen_cpu_ops; 1629 pv_apic_ops = xen_apic_ops; 1630 pv_mmu_ops = xen_mmu_ops; 1631 1632 xen_init_irq_ops(); 1633 1634 #ifdef CONFIG_X86_LOCAL_APIC 1635 /* 1636 * set up the basic apic ops. 1637 */ 1638 apic_ops = &xen_basic_apic_ops; 1639 #endif 1640 1641 if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) { 1642 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start; 1643 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit; 1644 } 1645 1646 machine_ops = xen_machine_ops; 1647 1648 #ifdef CONFIG_X86_64 1649 /* Disable until direct per-cpu data access. */ 1650 have_vcpu_info_placement = 0; 1651 x86_64_init_pda(); 1652 #endif 1653 1654 xen_smp_init(); 1655 1656 /* Get mfn list */ 1657 if (!xen_feature(XENFEAT_auto_translated_physmap)) 1658 xen_build_dynamic_phys_to_machine(); 1659 1660 pgd = (pgd_t *)xen_start_info->pt_base; 1661 1662 /* Prevent unwanted bits from being set in PTEs. */ 1663 __supported_pte_mask &= ~_PAGE_GLOBAL; 1664 if (!xen_initial_domain()) 1665 __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD); 1666 1667 /* Don't do the full vcpu_info placement stuff until we have a 1668 possible map and a non-dummy shared_info. */ 1669 per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0]; 1670 1671 xen_raw_console_write("mapping kernel into physical memory\n"); 1672 pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages); 1673 1674 init_mm.pgd = pgd; 1675 1676 /* keep using Xen gdt for now; no urgent need to change it */ 1677 1678 pv_info.kernel_rpl = 1; 1679 if (xen_feature(XENFEAT_supervisor_mode_kernel)) 1680 pv_info.kernel_rpl = 0; 1681 1682 /* set the limit of our address space */ 1683 xen_reserve_top(); 1684 1685 #ifdef CONFIG_X86_32 1686 /* set up basic CPUID stuff */ 1687 cpu_detect(&new_cpu_data); 1688 new_cpu_data.hard_math = 1; 1689 new_cpu_data.x86_capability[0] = cpuid_edx(1); 1690 #endif 1691 1692 /* Poke various useful things into boot_params */ 1693 boot_params.hdr.type_of_loader = (9 << 4) | 0; 1694 boot_params.hdr.ramdisk_image = xen_start_info->mod_start 1695 ? __pa(xen_start_info->mod_start) : 0; 1696 boot_params.hdr.ramdisk_size = xen_start_info->mod_len; 1697 boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line); 1698 1699 if (!xen_initial_domain()) { 1700 add_preferred_console("xenboot", 0, NULL); 1701 add_preferred_console("tty", 0, NULL); 1702 add_preferred_console("hvc", 0, NULL); 1703 } 1704 1705 xen_raw_console_write("about to get started...\n"); 1706 1707 /* Start the world */ 1708 #ifdef CONFIG_X86_32 1709 i386_start_kernel(); 1710 #else 1711 x86_64_start_reservations((char *)__pa_symbol(&boot_params)); 1712 #endif 1713 } 1714