1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Core of Xen paravirt_ops implementation. 4 * 5 * This file contains the xen_paravirt_ops structure itself, and the 6 * implementations for: 7 * - privileged instructions 8 * - interrupt flags 9 * - segment operations 10 * - booting and setup 11 * 12 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 13 */ 14 15 #include <linux/cpu.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/smp.h> 19 #include <linux/preempt.h> 20 #include <linux/hardirq.h> 21 #include <linux/percpu.h> 22 #include <linux/delay.h> 23 #include <linux/start_kernel.h> 24 #include <linux/sched.h> 25 #include <linux/kprobes.h> 26 #include <linux/kstrtox.h> 27 #include <linux/memblock.h> 28 #include <linux/export.h> 29 #include <linux/mm.h> 30 #include <linux/page-flags.h> 31 #include <linux/pci.h> 32 #include <linux/gfp.h> 33 #include <linux/edd.h> 34 #include <linux/reboot.h> 35 #include <linux/virtio_anchor.h> 36 #include <linux/stackprotector.h> 37 38 #include <xen/xen.h> 39 #include <xen/events.h> 40 #include <xen/interface/xen.h> 41 #include <xen/interface/version.h> 42 #include <xen/interface/physdev.h> 43 #include <xen/interface/vcpu.h> 44 #include <xen/interface/memory.h> 45 #include <xen/interface/nmi.h> 46 #include <xen/interface/xen-mca.h> 47 #include <xen/features.h> 48 #include <xen/page.h> 49 #include <xen/hvc-console.h> 50 #include <xen/acpi.h> 51 52 #include <asm/paravirt.h> 53 #include <asm/apic.h> 54 #include <asm/page.h> 55 #include <asm/xen/pci.h> 56 #include <asm/xen/hypercall.h> 57 #include <asm/xen/hypervisor.h> 58 #include <asm/xen/cpuid.h> 59 #include <asm/fixmap.h> 60 #include <asm/processor.h> 61 #include <asm/proto.h> 62 #include <asm/msr-index.h> 63 #include <asm/traps.h> 64 #include <asm/setup.h> 65 #include <asm/desc.h> 66 #include <asm/pgalloc.h> 67 #include <asm/tlbflush.h> 68 #include <asm/reboot.h> 69 #include <asm/hypervisor.h> 70 #include <asm/mach_traps.h> 71 #include <asm/mtrr.h> 72 #include <asm/mwait.h> 73 #include <asm/pci_x86.h> 74 #include <asm/cpu.h> 75 #ifdef CONFIG_X86_IOPL_IOPERM 76 #include <asm/io_bitmap.h> 77 #endif 78 79 #ifdef CONFIG_ACPI 80 #include <linux/acpi.h> 81 #include <asm/acpi.h> 82 #include <acpi/proc_cap_intel.h> 83 #include <acpi/processor.h> 84 #include <xen/interface/platform.h> 85 #endif 86 87 #include "xen-ops.h" 88 #include "mmu.h" 89 #include "smp.h" 90 #include "multicalls.h" 91 #include "pmu.h" 92 93 #include "../kernel/cpu/cpu.h" /* get_cpu_cap() */ 94 95 void *xen_initial_gdt; 96 97 static int xen_cpu_up_prepare_pv(unsigned int cpu); 98 static int xen_cpu_dead_pv(unsigned int cpu); 99 100 struct tls_descs { 101 struct desc_struct desc[3]; 102 }; 103 104 /* 105 * Updating the 3 TLS descriptors in the GDT on every task switch is 106 * surprisingly expensive so we avoid updating them if they haven't 107 * changed. Since Xen writes different descriptors than the one 108 * passed in the update_descriptor hypercall we keep shadow copies to 109 * compare against. 110 */ 111 static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc); 112 113 static __read_mostly bool xen_msr_safe = IS_ENABLED(CONFIG_XEN_PV_MSR_SAFE); 114 115 static int __init parse_xen_msr_safe(char *str) 116 { 117 if (str) 118 return kstrtobool(str, &xen_msr_safe); 119 return -EINVAL; 120 } 121 early_param("xen_msr_safe", parse_xen_msr_safe); 122 123 /* Get MTRR settings from Xen and put them into mtrr_state. */ 124 static void __init xen_set_mtrr_data(void) 125 { 126 #ifdef CONFIG_MTRR 127 struct xen_platform_op op = { 128 .cmd = XENPF_read_memtype, 129 .interface_version = XENPF_INTERFACE_VERSION, 130 }; 131 unsigned int reg; 132 unsigned long mask; 133 uint32_t eax, width; 134 static struct mtrr_var_range var[MTRR_MAX_VAR_RANGES] __initdata; 135 136 /* Get physical address width (only 64-bit cpus supported). */ 137 width = 36; 138 eax = cpuid_eax(0x80000000); 139 if ((eax >> 16) == 0x8000 && eax >= 0x80000008) { 140 eax = cpuid_eax(0x80000008); 141 width = eax & 0xff; 142 } 143 144 for (reg = 0; reg < MTRR_MAX_VAR_RANGES; reg++) { 145 op.u.read_memtype.reg = reg; 146 if (HYPERVISOR_platform_op(&op)) 147 break; 148 149 /* 150 * Only called in dom0, which has all RAM PFNs mapped at 151 * RAM MFNs, and all PCI space etc. is identity mapped. 152 * This means we can treat MFN == PFN regarding MTRR settings. 153 */ 154 var[reg].base_lo = op.u.read_memtype.type; 155 var[reg].base_lo |= op.u.read_memtype.mfn << PAGE_SHIFT; 156 var[reg].base_hi = op.u.read_memtype.mfn >> (32 - PAGE_SHIFT); 157 mask = ~((op.u.read_memtype.nr_mfns << PAGE_SHIFT) - 1); 158 mask &= (1UL << width) - 1; 159 if (mask) 160 mask |= MTRR_PHYSMASK_V; 161 var[reg].mask_lo = mask; 162 var[reg].mask_hi = mask >> 32; 163 } 164 165 /* Only overwrite MTRR state if any MTRR could be got from Xen. */ 166 if (reg) 167 mtrr_overwrite_state(var, reg, MTRR_TYPE_UNCACHABLE); 168 #endif 169 } 170 171 static void __init xen_pv_init_platform(void) 172 { 173 /* PV guests can't operate virtio devices without grants. */ 174 if (IS_ENABLED(CONFIG_XEN_VIRTIO)) 175 virtio_set_mem_acc_cb(xen_virtio_restricted_mem_acc); 176 177 populate_extra_pte(fix_to_virt(FIX_PARAVIRT_BOOTMAP)); 178 179 set_fixmap(FIX_PARAVIRT_BOOTMAP, xen_start_info->shared_info); 180 HYPERVISOR_shared_info = (void *)fix_to_virt(FIX_PARAVIRT_BOOTMAP); 181 182 /* xen clock uses per-cpu vcpu_info, need to init it for boot cpu */ 183 xen_vcpu_info_reset(0); 184 185 /* pvclock is in shared info area */ 186 xen_init_time_ops(); 187 188 if (xen_initial_domain()) 189 xen_set_mtrr_data(); 190 else 191 mtrr_overwrite_state(NULL, 0, MTRR_TYPE_WRBACK); 192 } 193 194 static void __init xen_pv_guest_late_init(void) 195 { 196 #ifndef CONFIG_SMP 197 /* Setup shared vcpu info for non-smp configurations */ 198 xen_setup_vcpu_info_placement(); 199 #endif 200 } 201 202 static __read_mostly unsigned int cpuid_leaf5_ecx_val; 203 static __read_mostly unsigned int cpuid_leaf5_edx_val; 204 205 static void xen_cpuid(unsigned int *ax, unsigned int *bx, 206 unsigned int *cx, unsigned int *dx) 207 { 208 unsigned maskebx = ~0; 209 210 /* 211 * Mask out inconvenient features, to try and disable as many 212 * unsupported kernel subsystems as possible. 213 */ 214 switch (*ax) { 215 case CPUID_MWAIT_LEAF: 216 /* Synthesize the values.. */ 217 *ax = 0; 218 *bx = 0; 219 *cx = cpuid_leaf5_ecx_val; 220 *dx = cpuid_leaf5_edx_val; 221 return; 222 223 case 0xb: 224 /* Suppress extended topology stuff */ 225 maskebx = 0; 226 break; 227 } 228 229 asm(XEN_EMULATE_PREFIX "cpuid" 230 : "=a" (*ax), 231 "=b" (*bx), 232 "=c" (*cx), 233 "=d" (*dx) 234 : "0" (*ax), "2" (*cx)); 235 236 *bx &= maskebx; 237 } 238 239 static bool __init xen_check_mwait(void) 240 { 241 #ifdef CONFIG_ACPI 242 struct xen_platform_op op = { 243 .cmd = XENPF_set_processor_pminfo, 244 .u.set_pminfo.id = -1, 245 .u.set_pminfo.type = XEN_PM_PDC, 246 }; 247 uint32_t buf[3]; 248 unsigned int ax, bx, cx, dx; 249 unsigned int mwait_mask; 250 251 /* We need to determine whether it is OK to expose the MWAIT 252 * capability to the kernel to harvest deeper than C3 states from ACPI 253 * _CST using the processor_harvest_xen.c module. For this to work, we 254 * need to gather the MWAIT_LEAF values (which the cstate.c code 255 * checks against). The hypervisor won't expose the MWAIT flag because 256 * it would break backwards compatibility; so we will find out directly 257 * from the hardware and hypercall. 258 */ 259 if (!xen_initial_domain()) 260 return false; 261 262 /* 263 * When running under platform earlier than Xen4.2, do not expose 264 * mwait, to avoid the risk of loading native acpi pad driver 265 */ 266 if (!xen_running_on_version_or_later(4, 2)) 267 return false; 268 269 ax = 1; 270 cx = 0; 271 272 native_cpuid(&ax, &bx, &cx, &dx); 273 274 mwait_mask = (1 << (X86_FEATURE_EST % 32)) | 275 (1 << (X86_FEATURE_MWAIT % 32)); 276 277 if ((cx & mwait_mask) != mwait_mask) 278 return false; 279 280 /* We need to emulate the MWAIT_LEAF and for that we need both 281 * ecx and edx. The hypercall provides only partial information. 282 */ 283 284 ax = CPUID_MWAIT_LEAF; 285 bx = 0; 286 cx = 0; 287 dx = 0; 288 289 native_cpuid(&ax, &bx, &cx, &dx); 290 291 /* Ask the Hypervisor whether to clear ACPI_PROC_CAP_C_C2C3_FFH. If so, 292 * don't expose MWAIT_LEAF and let ACPI pick the IOPORT version of C3. 293 */ 294 buf[0] = ACPI_PDC_REVISION_ID; 295 buf[1] = 1; 296 buf[2] = (ACPI_PROC_CAP_C_CAPABILITY_SMP | ACPI_PROC_CAP_EST_CAPABILITY_SWSMP); 297 298 set_xen_guest_handle(op.u.set_pminfo.pdc, buf); 299 300 if ((HYPERVISOR_platform_op(&op) == 0) && 301 (buf[2] & (ACPI_PROC_CAP_C_C1_FFH | ACPI_PROC_CAP_C_C2C3_FFH))) { 302 cpuid_leaf5_ecx_val = cx; 303 cpuid_leaf5_edx_val = dx; 304 } 305 return true; 306 #else 307 return false; 308 #endif 309 } 310 311 static bool __init xen_check_xsave(void) 312 { 313 unsigned int cx, xsave_mask; 314 315 cx = cpuid_ecx(1); 316 317 xsave_mask = (1 << (X86_FEATURE_XSAVE % 32)) | 318 (1 << (X86_FEATURE_OSXSAVE % 32)); 319 320 /* Xen will set CR4.OSXSAVE if supported and not disabled by force */ 321 return (cx & xsave_mask) == xsave_mask; 322 } 323 324 static void __init xen_init_capabilities(void) 325 { 326 setup_force_cpu_cap(X86_FEATURE_XENPV); 327 setup_clear_cpu_cap(X86_FEATURE_DCA); 328 setup_clear_cpu_cap(X86_FEATURE_APERFMPERF); 329 setup_clear_cpu_cap(X86_FEATURE_MTRR); 330 setup_clear_cpu_cap(X86_FEATURE_ACC); 331 setup_clear_cpu_cap(X86_FEATURE_X2APIC); 332 setup_clear_cpu_cap(X86_FEATURE_SME); 333 setup_clear_cpu_cap(X86_FEATURE_LKGS); 334 335 /* 336 * Xen PV would need some work to support PCID: CR3 handling as well 337 * as xen_flush_tlb_others() would need updating. 338 */ 339 setup_clear_cpu_cap(X86_FEATURE_PCID); 340 341 if (!xen_initial_domain()) 342 setup_clear_cpu_cap(X86_FEATURE_ACPI); 343 344 if (xen_check_mwait()) 345 setup_force_cpu_cap(X86_FEATURE_MWAIT); 346 else 347 setup_clear_cpu_cap(X86_FEATURE_MWAIT); 348 349 if (!xen_check_xsave()) { 350 setup_clear_cpu_cap(X86_FEATURE_XSAVE); 351 setup_clear_cpu_cap(X86_FEATURE_OSXSAVE); 352 } 353 } 354 355 static noinstr void xen_set_debugreg(int reg, unsigned long val) 356 { 357 HYPERVISOR_set_debugreg(reg, val); 358 } 359 360 static noinstr unsigned long xen_get_debugreg(int reg) 361 { 362 return HYPERVISOR_get_debugreg(reg); 363 } 364 365 static void xen_end_context_switch(struct task_struct *next) 366 { 367 xen_mc_flush(); 368 paravirt_end_context_switch(next); 369 } 370 371 static unsigned long xen_store_tr(void) 372 { 373 return 0; 374 } 375 376 /* 377 * Set the page permissions for a particular virtual address. If the 378 * address is a vmalloc mapping (or other non-linear mapping), then 379 * find the linear mapping of the page and also set its protections to 380 * match. 381 */ 382 static void set_aliased_prot(void *v, pgprot_t prot) 383 { 384 int level; 385 pte_t *ptep; 386 pte_t pte; 387 unsigned long pfn; 388 unsigned char dummy; 389 void *va; 390 391 ptep = lookup_address((unsigned long)v, &level); 392 BUG_ON(ptep == NULL); 393 394 pfn = pte_pfn(*ptep); 395 pte = pfn_pte(pfn, prot); 396 397 /* 398 * Careful: update_va_mapping() will fail if the virtual address 399 * we're poking isn't populated in the page tables. We don't 400 * need to worry about the direct map (that's always in the page 401 * tables), but we need to be careful about vmap space. In 402 * particular, the top level page table can lazily propagate 403 * entries between processes, so if we've switched mms since we 404 * vmapped the target in the first place, we might not have the 405 * top-level page table entry populated. 406 * 407 * We disable preemption because we want the same mm active when 408 * we probe the target and when we issue the hypercall. We'll 409 * have the same nominal mm, but if we're a kernel thread, lazy 410 * mm dropping could change our pgd. 411 * 412 * Out of an abundance of caution, this uses __get_user() to fault 413 * in the target address just in case there's some obscure case 414 * in which the target address isn't readable. 415 */ 416 417 preempt_disable(); 418 419 copy_from_kernel_nofault(&dummy, v, 1); 420 421 if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0)) 422 BUG(); 423 424 va = __va(PFN_PHYS(pfn)); 425 426 if (va != v && HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0)) 427 BUG(); 428 429 preempt_enable(); 430 } 431 432 static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries) 433 { 434 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE; 435 int i; 436 437 /* 438 * We need to mark the all aliases of the LDT pages RO. We 439 * don't need to call vm_flush_aliases(), though, since that's 440 * only responsible for flushing aliases out the TLBs, not the 441 * page tables, and Xen will flush the TLB for us if needed. 442 * 443 * To avoid confusing future readers: none of this is necessary 444 * to load the LDT. The hypervisor only checks this when the 445 * LDT is faulted in due to subsequent descriptor access. 446 */ 447 448 for (i = 0; i < entries; i += entries_per_page) 449 set_aliased_prot(ldt + i, PAGE_KERNEL_RO); 450 } 451 452 static void xen_free_ldt(struct desc_struct *ldt, unsigned entries) 453 { 454 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE; 455 int i; 456 457 for (i = 0; i < entries; i += entries_per_page) 458 set_aliased_prot(ldt + i, PAGE_KERNEL); 459 } 460 461 static void xen_set_ldt(const void *addr, unsigned entries) 462 { 463 struct mmuext_op *op; 464 struct multicall_space mcs = xen_mc_entry(sizeof(*op)); 465 466 trace_xen_cpu_set_ldt(addr, entries); 467 468 op = mcs.args; 469 op->cmd = MMUEXT_SET_LDT; 470 op->arg1.linear_addr = (unsigned long)addr; 471 op->arg2.nr_ents = entries; 472 473 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); 474 475 xen_mc_issue(PARAVIRT_LAZY_CPU); 476 } 477 478 static void xen_load_gdt(const struct desc_ptr *dtr) 479 { 480 unsigned long va = dtr->address; 481 unsigned int size = dtr->size + 1; 482 unsigned long pfn, mfn; 483 int level; 484 pte_t *ptep; 485 void *virt; 486 487 /* @size should be at most GDT_SIZE which is smaller than PAGE_SIZE. */ 488 BUG_ON(size > PAGE_SIZE); 489 BUG_ON(va & ~PAGE_MASK); 490 491 /* 492 * The GDT is per-cpu and is in the percpu data area. 493 * That can be virtually mapped, so we need to do a 494 * page-walk to get the underlying MFN for the 495 * hypercall. The page can also be in the kernel's 496 * linear range, so we need to RO that mapping too. 497 */ 498 ptep = lookup_address(va, &level); 499 BUG_ON(ptep == NULL); 500 501 pfn = pte_pfn(*ptep); 502 mfn = pfn_to_mfn(pfn); 503 virt = __va(PFN_PHYS(pfn)); 504 505 make_lowmem_page_readonly((void *)va); 506 make_lowmem_page_readonly(virt); 507 508 if (HYPERVISOR_set_gdt(&mfn, size / sizeof(struct desc_struct))) 509 BUG(); 510 } 511 512 /* 513 * load_gdt for early boot, when the gdt is only mapped once 514 */ 515 static void __init xen_load_gdt_boot(const struct desc_ptr *dtr) 516 { 517 unsigned long va = dtr->address; 518 unsigned int size = dtr->size + 1; 519 unsigned long pfn, mfn; 520 pte_t pte; 521 522 /* @size should be at most GDT_SIZE which is smaller than PAGE_SIZE. */ 523 BUG_ON(size > PAGE_SIZE); 524 BUG_ON(va & ~PAGE_MASK); 525 526 pfn = virt_to_pfn((void *)va); 527 mfn = pfn_to_mfn(pfn); 528 529 pte = pfn_pte(pfn, PAGE_KERNEL_RO); 530 531 if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0)) 532 BUG(); 533 534 if (HYPERVISOR_set_gdt(&mfn, size / sizeof(struct desc_struct))) 535 BUG(); 536 } 537 538 static inline bool desc_equal(const struct desc_struct *d1, 539 const struct desc_struct *d2) 540 { 541 return !memcmp(d1, d2, sizeof(*d1)); 542 } 543 544 static void load_TLS_descriptor(struct thread_struct *t, 545 unsigned int cpu, unsigned int i) 546 { 547 struct desc_struct *shadow = &per_cpu(shadow_tls_desc, cpu).desc[i]; 548 struct desc_struct *gdt; 549 xmaddr_t maddr; 550 struct multicall_space mc; 551 552 if (desc_equal(shadow, &t->tls_array[i])) 553 return; 554 555 *shadow = t->tls_array[i]; 556 557 gdt = get_cpu_gdt_rw(cpu); 558 maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]); 559 mc = __xen_mc_entry(0); 560 561 MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]); 562 } 563 564 static void xen_load_tls(struct thread_struct *t, unsigned int cpu) 565 { 566 /* 567 * In lazy mode we need to zero %fs, otherwise we may get an 568 * exception between the new %fs descriptor being loaded and 569 * %fs being effectively cleared at __switch_to(). 570 */ 571 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) 572 loadsegment(fs, 0); 573 574 xen_mc_batch(); 575 576 load_TLS_descriptor(t, cpu, 0); 577 load_TLS_descriptor(t, cpu, 1); 578 load_TLS_descriptor(t, cpu, 2); 579 580 xen_mc_issue(PARAVIRT_LAZY_CPU); 581 } 582 583 static void xen_load_gs_index(unsigned int idx) 584 { 585 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx)) 586 BUG(); 587 } 588 589 static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum, 590 const void *ptr) 591 { 592 xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]); 593 u64 entry = *(u64 *)ptr; 594 595 trace_xen_cpu_write_ldt_entry(dt, entrynum, entry); 596 597 preempt_disable(); 598 599 xen_mc_flush(); 600 if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry)) 601 BUG(); 602 603 preempt_enable(); 604 } 605 606 void noist_exc_debug(struct pt_regs *regs); 607 608 DEFINE_IDTENTRY_RAW(xenpv_exc_nmi) 609 { 610 /* On Xen PV, NMI doesn't use IST. The C part is the same as native. */ 611 exc_nmi(regs); 612 } 613 614 DEFINE_IDTENTRY_RAW_ERRORCODE(xenpv_exc_double_fault) 615 { 616 /* On Xen PV, DF doesn't use IST. The C part is the same as native. */ 617 exc_double_fault(regs, error_code); 618 } 619 620 DEFINE_IDTENTRY_RAW(xenpv_exc_debug) 621 { 622 /* 623 * There's no IST on Xen PV, but we still need to dispatch 624 * to the correct handler. 625 */ 626 if (user_mode(regs)) 627 noist_exc_debug(regs); 628 else 629 exc_debug(regs); 630 } 631 632 DEFINE_IDTENTRY_RAW(exc_xen_unknown_trap) 633 { 634 /* This should never happen and there is no way to handle it. */ 635 instrumentation_begin(); 636 pr_err("Unknown trap in Xen PV mode."); 637 BUG(); 638 instrumentation_end(); 639 } 640 641 #ifdef CONFIG_X86_MCE 642 DEFINE_IDTENTRY_RAW(xenpv_exc_machine_check) 643 { 644 /* 645 * There's no IST on Xen PV, but we still need to dispatch 646 * to the correct handler. 647 */ 648 if (user_mode(regs)) 649 noist_exc_machine_check(regs); 650 else 651 exc_machine_check(regs); 652 } 653 #endif 654 655 struct trap_array_entry { 656 void (*orig)(void); 657 void (*xen)(void); 658 bool ist_okay; 659 }; 660 661 #define TRAP_ENTRY(func, ist_ok) { \ 662 .orig = asm_##func, \ 663 .xen = xen_asm_##func, \ 664 .ist_okay = ist_ok } 665 666 #define TRAP_ENTRY_REDIR(func, ist_ok) { \ 667 .orig = asm_##func, \ 668 .xen = xen_asm_xenpv_##func, \ 669 .ist_okay = ist_ok } 670 671 static struct trap_array_entry trap_array[] = { 672 TRAP_ENTRY_REDIR(exc_debug, true ), 673 TRAP_ENTRY_REDIR(exc_double_fault, true ), 674 #ifdef CONFIG_X86_MCE 675 TRAP_ENTRY_REDIR(exc_machine_check, true ), 676 #endif 677 TRAP_ENTRY_REDIR(exc_nmi, true ), 678 TRAP_ENTRY(exc_int3, false ), 679 TRAP_ENTRY(exc_overflow, false ), 680 #ifdef CONFIG_IA32_EMULATION 681 { entry_INT80_compat, xen_entry_INT80_compat, false }, 682 #endif 683 TRAP_ENTRY(exc_page_fault, false ), 684 TRAP_ENTRY(exc_divide_error, false ), 685 TRAP_ENTRY(exc_bounds, false ), 686 TRAP_ENTRY(exc_invalid_op, false ), 687 TRAP_ENTRY(exc_device_not_available, false ), 688 TRAP_ENTRY(exc_coproc_segment_overrun, false ), 689 TRAP_ENTRY(exc_invalid_tss, false ), 690 TRAP_ENTRY(exc_segment_not_present, false ), 691 TRAP_ENTRY(exc_stack_segment, false ), 692 TRAP_ENTRY(exc_general_protection, false ), 693 TRAP_ENTRY(exc_spurious_interrupt_bug, false ), 694 TRAP_ENTRY(exc_coprocessor_error, false ), 695 TRAP_ENTRY(exc_alignment_check, false ), 696 TRAP_ENTRY(exc_simd_coprocessor_error, false ), 697 #ifdef CONFIG_X86_CET 698 TRAP_ENTRY(exc_control_protection, false ), 699 #endif 700 }; 701 702 static bool __ref get_trap_addr(void **addr, unsigned int ist) 703 { 704 unsigned int nr; 705 bool ist_okay = false; 706 bool found = false; 707 708 /* 709 * Replace trap handler addresses by Xen specific ones. 710 * Check for known traps using IST and whitelist them. 711 * The debugger ones are the only ones we care about. 712 * Xen will handle faults like double_fault, so we should never see 713 * them. Warn if there's an unexpected IST-using fault handler. 714 */ 715 for (nr = 0; nr < ARRAY_SIZE(trap_array); nr++) { 716 struct trap_array_entry *entry = trap_array + nr; 717 718 if (*addr == entry->orig) { 719 *addr = entry->xen; 720 ist_okay = entry->ist_okay; 721 found = true; 722 break; 723 } 724 } 725 726 if (nr == ARRAY_SIZE(trap_array) && 727 *addr >= (void *)early_idt_handler_array[0] && 728 *addr < (void *)early_idt_handler_array[NUM_EXCEPTION_VECTORS]) { 729 nr = (*addr - (void *)early_idt_handler_array[0]) / 730 EARLY_IDT_HANDLER_SIZE; 731 *addr = (void *)xen_early_idt_handler_array[nr]; 732 found = true; 733 } 734 735 if (!found) 736 *addr = (void *)xen_asm_exc_xen_unknown_trap; 737 738 if (WARN_ON(found && ist != 0 && !ist_okay)) 739 return false; 740 741 return true; 742 } 743 744 static int cvt_gate_to_trap(int vector, const gate_desc *val, 745 struct trap_info *info) 746 { 747 unsigned long addr; 748 749 if (val->bits.type != GATE_TRAP && val->bits.type != GATE_INTERRUPT) 750 return 0; 751 752 info->vector = vector; 753 754 addr = gate_offset(val); 755 if (!get_trap_addr((void **)&addr, val->bits.ist)) 756 return 0; 757 info->address = addr; 758 759 info->cs = gate_segment(val); 760 info->flags = val->bits.dpl; 761 /* interrupt gates clear IF */ 762 if (val->bits.type == GATE_INTERRUPT) 763 info->flags |= 1 << 2; 764 765 return 1; 766 } 767 768 /* Locations of each CPU's IDT */ 769 static DEFINE_PER_CPU(struct desc_ptr, idt_desc); 770 771 /* Set an IDT entry. If the entry is part of the current IDT, then 772 also update Xen. */ 773 static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g) 774 { 775 unsigned long p = (unsigned long)&dt[entrynum]; 776 unsigned long start, end; 777 778 trace_xen_cpu_write_idt_entry(dt, entrynum, g); 779 780 preempt_disable(); 781 782 start = __this_cpu_read(idt_desc.address); 783 end = start + __this_cpu_read(idt_desc.size) + 1; 784 785 xen_mc_flush(); 786 787 native_write_idt_entry(dt, entrynum, g); 788 789 if (p >= start && (p + 8) <= end) { 790 struct trap_info info[2]; 791 792 info[1].address = 0; 793 794 if (cvt_gate_to_trap(entrynum, g, &info[0])) 795 if (HYPERVISOR_set_trap_table(info)) 796 BUG(); 797 } 798 799 preempt_enable(); 800 } 801 802 static unsigned xen_convert_trap_info(const struct desc_ptr *desc, 803 struct trap_info *traps, bool full) 804 { 805 unsigned in, out, count; 806 807 count = (desc->size+1) / sizeof(gate_desc); 808 BUG_ON(count > 256); 809 810 for (in = out = 0; in < count; in++) { 811 gate_desc *entry = (gate_desc *)(desc->address) + in; 812 813 if (cvt_gate_to_trap(in, entry, &traps[out]) || full) 814 out++; 815 } 816 817 return out; 818 } 819 820 void xen_copy_trap_info(struct trap_info *traps) 821 { 822 const struct desc_ptr *desc = this_cpu_ptr(&idt_desc); 823 824 xen_convert_trap_info(desc, traps, true); 825 } 826 827 /* Load a new IDT into Xen. In principle this can be per-CPU, so we 828 hold a spinlock to protect the static traps[] array (static because 829 it avoids allocation, and saves stack space). */ 830 static void xen_load_idt(const struct desc_ptr *desc) 831 { 832 static DEFINE_SPINLOCK(lock); 833 static struct trap_info traps[257]; 834 static const struct trap_info zero = { }; 835 unsigned out; 836 837 trace_xen_cpu_load_idt(desc); 838 839 spin_lock(&lock); 840 841 memcpy(this_cpu_ptr(&idt_desc), desc, sizeof(idt_desc)); 842 843 out = xen_convert_trap_info(desc, traps, false); 844 traps[out] = zero; 845 846 xen_mc_flush(); 847 if (HYPERVISOR_set_trap_table(traps)) 848 BUG(); 849 850 spin_unlock(&lock); 851 } 852 853 /* Write a GDT descriptor entry. Ignore LDT descriptors, since 854 they're handled differently. */ 855 static void xen_write_gdt_entry(struct desc_struct *dt, int entry, 856 const void *desc, int type) 857 { 858 trace_xen_cpu_write_gdt_entry(dt, entry, desc, type); 859 860 preempt_disable(); 861 862 switch (type) { 863 case DESC_LDT: 864 case DESC_TSS: 865 /* ignore */ 866 break; 867 868 default: { 869 xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]); 870 871 xen_mc_flush(); 872 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc)) 873 BUG(); 874 } 875 876 } 877 878 preempt_enable(); 879 } 880 881 /* 882 * Version of write_gdt_entry for use at early boot-time needed to 883 * update an entry as simply as possible. 884 */ 885 static void __init xen_write_gdt_entry_boot(struct desc_struct *dt, int entry, 886 const void *desc, int type) 887 { 888 trace_xen_cpu_write_gdt_entry(dt, entry, desc, type); 889 890 switch (type) { 891 case DESC_LDT: 892 case DESC_TSS: 893 /* ignore */ 894 break; 895 896 default: { 897 xmaddr_t maddr = virt_to_machine(&dt[entry]); 898 899 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc)) 900 dt[entry] = *(struct desc_struct *)desc; 901 } 902 903 } 904 } 905 906 static void xen_load_sp0(unsigned long sp0) 907 { 908 struct multicall_space mcs; 909 910 mcs = xen_mc_entry(0); 911 MULTI_stack_switch(mcs.mc, __KERNEL_DS, sp0); 912 xen_mc_issue(PARAVIRT_LAZY_CPU); 913 this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0); 914 } 915 916 #ifdef CONFIG_X86_IOPL_IOPERM 917 static void xen_invalidate_io_bitmap(void) 918 { 919 struct physdev_set_iobitmap iobitmap = { 920 .bitmap = NULL, 921 .nr_ports = 0, 922 }; 923 924 native_tss_invalidate_io_bitmap(); 925 HYPERVISOR_physdev_op(PHYSDEVOP_set_iobitmap, &iobitmap); 926 } 927 928 static void xen_update_io_bitmap(void) 929 { 930 struct physdev_set_iobitmap iobitmap; 931 struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw); 932 933 native_tss_update_io_bitmap(); 934 935 iobitmap.bitmap = (uint8_t *)(&tss->x86_tss) + 936 tss->x86_tss.io_bitmap_base; 937 if (tss->x86_tss.io_bitmap_base == IO_BITMAP_OFFSET_INVALID) 938 iobitmap.nr_ports = 0; 939 else 940 iobitmap.nr_ports = IO_BITMAP_BITS; 941 942 HYPERVISOR_physdev_op(PHYSDEVOP_set_iobitmap, &iobitmap); 943 } 944 #endif 945 946 static void xen_io_delay(void) 947 { 948 } 949 950 static DEFINE_PER_CPU(unsigned long, xen_cr0_value); 951 952 static unsigned long xen_read_cr0(void) 953 { 954 unsigned long cr0 = this_cpu_read(xen_cr0_value); 955 956 if (unlikely(cr0 == 0)) { 957 cr0 = native_read_cr0(); 958 this_cpu_write(xen_cr0_value, cr0); 959 } 960 961 return cr0; 962 } 963 964 static void xen_write_cr0(unsigned long cr0) 965 { 966 struct multicall_space mcs; 967 968 this_cpu_write(xen_cr0_value, cr0); 969 970 /* Only pay attention to cr0.TS; everything else is 971 ignored. */ 972 mcs = xen_mc_entry(0); 973 974 MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0); 975 976 xen_mc_issue(PARAVIRT_LAZY_CPU); 977 } 978 979 static void xen_write_cr4(unsigned long cr4) 980 { 981 cr4 &= ~(X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PCE); 982 983 native_write_cr4(cr4); 984 } 985 986 static u64 xen_do_read_msr(unsigned int msr, int *err) 987 { 988 u64 val = 0; /* Avoid uninitialized value for safe variant. */ 989 990 if (pmu_msr_read(msr, &val, err)) 991 return val; 992 993 if (err) 994 val = native_read_msr_safe(msr, err); 995 else 996 val = native_read_msr(msr); 997 998 switch (msr) { 999 case MSR_IA32_APICBASE: 1000 val &= ~X2APIC_ENABLE; 1001 break; 1002 } 1003 return val; 1004 } 1005 1006 static void set_seg(unsigned int which, unsigned int low, unsigned int high, 1007 int *err) 1008 { 1009 u64 base = ((u64)high << 32) | low; 1010 1011 if (HYPERVISOR_set_segment_base(which, base) == 0) 1012 return; 1013 1014 if (err) 1015 *err = -EIO; 1016 else 1017 WARN(1, "Xen set_segment_base(%u, %llx) failed\n", which, base); 1018 } 1019 1020 /* 1021 * Support write_msr_safe() and write_msr() semantics. 1022 * With err == NULL write_msr() semantics are selected. 1023 * Supplying an err pointer requires err to be pre-initialized with 0. 1024 */ 1025 static void xen_do_write_msr(unsigned int msr, unsigned int low, 1026 unsigned int high, int *err) 1027 { 1028 switch (msr) { 1029 case MSR_FS_BASE: 1030 set_seg(SEGBASE_FS, low, high, err); 1031 break; 1032 1033 case MSR_KERNEL_GS_BASE: 1034 set_seg(SEGBASE_GS_USER, low, high, err); 1035 break; 1036 1037 case MSR_GS_BASE: 1038 set_seg(SEGBASE_GS_KERNEL, low, high, err); 1039 break; 1040 1041 case MSR_STAR: 1042 case MSR_CSTAR: 1043 case MSR_LSTAR: 1044 case MSR_SYSCALL_MASK: 1045 case MSR_IA32_SYSENTER_CS: 1046 case MSR_IA32_SYSENTER_ESP: 1047 case MSR_IA32_SYSENTER_EIP: 1048 /* Fast syscall setup is all done in hypercalls, so 1049 these are all ignored. Stub them out here to stop 1050 Xen console noise. */ 1051 break; 1052 1053 default: 1054 if (!pmu_msr_write(msr, low, high, err)) { 1055 if (err) 1056 *err = native_write_msr_safe(msr, low, high); 1057 else 1058 native_write_msr(msr, low, high); 1059 } 1060 } 1061 } 1062 1063 static u64 xen_read_msr_safe(unsigned int msr, int *err) 1064 { 1065 return xen_do_read_msr(msr, err); 1066 } 1067 1068 static int xen_write_msr_safe(unsigned int msr, unsigned int low, 1069 unsigned int high) 1070 { 1071 int err = 0; 1072 1073 xen_do_write_msr(msr, low, high, &err); 1074 1075 return err; 1076 } 1077 1078 static u64 xen_read_msr(unsigned int msr) 1079 { 1080 int err; 1081 1082 return xen_do_read_msr(msr, xen_msr_safe ? &err : NULL); 1083 } 1084 1085 static void xen_write_msr(unsigned int msr, unsigned low, unsigned high) 1086 { 1087 int err; 1088 1089 xen_do_write_msr(msr, low, high, xen_msr_safe ? &err : NULL); 1090 } 1091 1092 /* This is called once we have the cpu_possible_mask */ 1093 void __init xen_setup_vcpu_info_placement(void) 1094 { 1095 int cpu; 1096 1097 for_each_possible_cpu(cpu) { 1098 /* Set up direct vCPU id mapping for PV guests. */ 1099 per_cpu(xen_vcpu_id, cpu) = cpu; 1100 xen_vcpu_setup(cpu); 1101 } 1102 1103 pv_ops.irq.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct); 1104 pv_ops.irq.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct); 1105 pv_ops.irq.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct); 1106 pv_ops.mmu.read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2_direct); 1107 } 1108 1109 static const struct pv_info xen_info __initconst = { 1110 .extra_user_64bit_cs = FLAT_USER_CS64, 1111 .name = "Xen", 1112 }; 1113 1114 static const typeof(pv_ops) xen_cpu_ops __initconst = { 1115 .cpu = { 1116 .cpuid = xen_cpuid, 1117 1118 .set_debugreg = xen_set_debugreg, 1119 .get_debugreg = xen_get_debugreg, 1120 1121 .read_cr0 = xen_read_cr0, 1122 .write_cr0 = xen_write_cr0, 1123 1124 .write_cr4 = xen_write_cr4, 1125 1126 .wbinvd = pv_native_wbinvd, 1127 1128 .read_msr = xen_read_msr, 1129 .write_msr = xen_write_msr, 1130 1131 .read_msr_safe = xen_read_msr_safe, 1132 .write_msr_safe = xen_write_msr_safe, 1133 1134 .read_pmc = xen_read_pmc, 1135 1136 .load_tr_desc = paravirt_nop, 1137 .set_ldt = xen_set_ldt, 1138 .load_gdt = xen_load_gdt, 1139 .load_idt = xen_load_idt, 1140 .load_tls = xen_load_tls, 1141 .load_gs_index = xen_load_gs_index, 1142 1143 .alloc_ldt = xen_alloc_ldt, 1144 .free_ldt = xen_free_ldt, 1145 1146 .store_tr = xen_store_tr, 1147 1148 .write_ldt_entry = xen_write_ldt_entry, 1149 .write_gdt_entry = xen_write_gdt_entry, 1150 .write_idt_entry = xen_write_idt_entry, 1151 .load_sp0 = xen_load_sp0, 1152 1153 #ifdef CONFIG_X86_IOPL_IOPERM 1154 .invalidate_io_bitmap = xen_invalidate_io_bitmap, 1155 .update_io_bitmap = xen_update_io_bitmap, 1156 #endif 1157 .io_delay = xen_io_delay, 1158 1159 .start_context_switch = paravirt_start_context_switch, 1160 .end_context_switch = xen_end_context_switch, 1161 }, 1162 }; 1163 1164 static void xen_restart(char *msg) 1165 { 1166 xen_reboot(SHUTDOWN_reboot); 1167 } 1168 1169 static void xen_machine_halt(void) 1170 { 1171 xen_reboot(SHUTDOWN_poweroff); 1172 } 1173 1174 static void xen_machine_power_off(void) 1175 { 1176 do_kernel_power_off(); 1177 xen_reboot(SHUTDOWN_poweroff); 1178 } 1179 1180 static void xen_crash_shutdown(struct pt_regs *regs) 1181 { 1182 xen_reboot(SHUTDOWN_crash); 1183 } 1184 1185 static const struct machine_ops xen_machine_ops __initconst = { 1186 .restart = xen_restart, 1187 .halt = xen_machine_halt, 1188 .power_off = xen_machine_power_off, 1189 .shutdown = xen_machine_halt, 1190 .crash_shutdown = xen_crash_shutdown, 1191 .emergency_restart = xen_emergency_restart, 1192 }; 1193 1194 static unsigned char xen_get_nmi_reason(void) 1195 { 1196 unsigned char reason = 0; 1197 1198 /* Construct a value which looks like it came from port 0x61. */ 1199 if (test_bit(_XEN_NMIREASON_io_error, 1200 &HYPERVISOR_shared_info->arch.nmi_reason)) 1201 reason |= NMI_REASON_IOCHK; 1202 if (test_bit(_XEN_NMIREASON_pci_serr, 1203 &HYPERVISOR_shared_info->arch.nmi_reason)) 1204 reason |= NMI_REASON_SERR; 1205 1206 return reason; 1207 } 1208 1209 static void __init xen_boot_params_init_edd(void) 1210 { 1211 #if IS_ENABLED(CONFIG_EDD) 1212 struct xen_platform_op op; 1213 struct edd_info *edd_info; 1214 u32 *mbr_signature; 1215 unsigned nr; 1216 int ret; 1217 1218 edd_info = boot_params.eddbuf; 1219 mbr_signature = boot_params.edd_mbr_sig_buffer; 1220 1221 op.cmd = XENPF_firmware_info; 1222 1223 op.u.firmware_info.type = XEN_FW_DISK_INFO; 1224 for (nr = 0; nr < EDDMAXNR; nr++) { 1225 struct edd_info *info = edd_info + nr; 1226 1227 op.u.firmware_info.index = nr; 1228 info->params.length = sizeof(info->params); 1229 set_xen_guest_handle(op.u.firmware_info.u.disk_info.edd_params, 1230 &info->params); 1231 ret = HYPERVISOR_platform_op(&op); 1232 if (ret) 1233 break; 1234 1235 #define C(x) info->x = op.u.firmware_info.u.disk_info.x 1236 C(device); 1237 C(version); 1238 C(interface_support); 1239 C(legacy_max_cylinder); 1240 C(legacy_max_head); 1241 C(legacy_sectors_per_track); 1242 #undef C 1243 } 1244 boot_params.eddbuf_entries = nr; 1245 1246 op.u.firmware_info.type = XEN_FW_DISK_MBR_SIGNATURE; 1247 for (nr = 0; nr < EDD_MBR_SIG_MAX; nr++) { 1248 op.u.firmware_info.index = nr; 1249 ret = HYPERVISOR_platform_op(&op); 1250 if (ret) 1251 break; 1252 mbr_signature[nr] = op.u.firmware_info.u.disk_mbr_signature.mbr_signature; 1253 } 1254 boot_params.edd_mbr_sig_buf_entries = nr; 1255 #endif 1256 } 1257 1258 /* 1259 * Set up the GDT and segment registers for -fstack-protector. Until 1260 * we do this, we have to be careful not to call any stack-protected 1261 * function, which is most of the kernel. 1262 */ 1263 static void __init xen_setup_gdt(int cpu) 1264 { 1265 pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry_boot; 1266 pv_ops.cpu.load_gdt = xen_load_gdt_boot; 1267 1268 switch_gdt_and_percpu_base(cpu); 1269 1270 pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry; 1271 pv_ops.cpu.load_gdt = xen_load_gdt; 1272 } 1273 1274 static void __init xen_dom0_set_legacy_features(void) 1275 { 1276 x86_platform.legacy.rtc = 1; 1277 } 1278 1279 static void __init xen_domu_set_legacy_features(void) 1280 { 1281 x86_platform.legacy.rtc = 0; 1282 } 1283 1284 extern void early_xen_iret_patch(void); 1285 1286 /* First C function to be called on Xen boot */ 1287 asmlinkage __visible void __init xen_start_kernel(struct start_info *si) 1288 { 1289 struct physdev_set_iopl set_iopl; 1290 unsigned long initrd_start = 0; 1291 int rc; 1292 1293 if (!si) 1294 return; 1295 1296 clear_bss(); 1297 1298 xen_start_info = si; 1299 1300 __text_gen_insn(&early_xen_iret_patch, 1301 JMP32_INSN_OPCODE, &early_xen_iret_patch, &xen_iret, 1302 JMP32_INSN_SIZE); 1303 1304 xen_domain_type = XEN_PV_DOMAIN; 1305 xen_start_flags = xen_start_info->flags; 1306 1307 xen_setup_features(); 1308 1309 /* Install Xen paravirt ops */ 1310 pv_info = xen_info; 1311 pv_ops.cpu = xen_cpu_ops.cpu; 1312 xen_init_irq_ops(); 1313 1314 /* 1315 * Setup xen_vcpu early because it is needed for 1316 * local_irq_disable(), irqs_disabled(), e.g. in printk(). 1317 * 1318 * Don't do the full vcpu_info placement stuff until we have 1319 * the cpu_possible_mask and a non-dummy shared_info. 1320 */ 1321 xen_vcpu_info_reset(0); 1322 1323 x86_platform.get_nmi_reason = xen_get_nmi_reason; 1324 x86_platform.realmode_reserve = x86_init_noop; 1325 x86_platform.realmode_init = x86_init_noop; 1326 1327 x86_init.resources.memory_setup = xen_memory_setup; 1328 x86_init.irqs.intr_mode_select = x86_init_noop; 1329 x86_init.irqs.intr_mode_init = x86_64_probe_apic; 1330 x86_init.oem.arch_setup = xen_arch_setup; 1331 x86_init.oem.banner = xen_banner; 1332 x86_init.hyper.init_platform = xen_pv_init_platform; 1333 x86_init.hyper.guest_late_init = xen_pv_guest_late_init; 1334 1335 /* 1336 * Set up some pagetable state before starting to set any ptes. 1337 */ 1338 1339 xen_setup_machphys_mapping(); 1340 xen_init_mmu_ops(); 1341 1342 /* Prevent unwanted bits from being set in PTEs. */ 1343 __supported_pte_mask &= ~_PAGE_GLOBAL; 1344 __default_kernel_pte_mask &= ~_PAGE_GLOBAL; 1345 1346 /* Get mfn list */ 1347 xen_build_dynamic_phys_to_machine(); 1348 1349 /* Work out if we support NX */ 1350 get_cpu_cap(&boot_cpu_data); 1351 x86_configure_nx(); 1352 1353 /* 1354 * Set up kernel GDT and segment registers, mainly so that 1355 * -fstack-protector code can be executed. 1356 */ 1357 xen_setup_gdt(0); 1358 1359 /* Determine virtual and physical address sizes */ 1360 get_cpu_address_sizes(&boot_cpu_data); 1361 1362 /* Let's presume PV guests always boot on vCPU with id 0. */ 1363 per_cpu(xen_vcpu_id, 0) = 0; 1364 1365 idt_setup_early_handler(); 1366 1367 xen_init_capabilities(); 1368 1369 /* 1370 * set up the basic apic ops. 1371 */ 1372 xen_init_apic(); 1373 1374 machine_ops = xen_machine_ops; 1375 1376 /* 1377 * The only reliable way to retain the initial address of the 1378 * percpu gdt_page is to remember it here, so we can go and 1379 * mark it RW later, when the initial percpu area is freed. 1380 */ 1381 xen_initial_gdt = &per_cpu(gdt_page, 0); 1382 1383 xen_smp_init(); 1384 1385 #ifdef CONFIG_ACPI_NUMA 1386 /* 1387 * The pages we from Xen are not related to machine pages, so 1388 * any NUMA information the kernel tries to get from ACPI will 1389 * be meaningless. Prevent it from trying. 1390 */ 1391 disable_srat(); 1392 #endif 1393 WARN_ON(xen_cpuhp_setup(xen_cpu_up_prepare_pv, xen_cpu_dead_pv)); 1394 1395 local_irq_disable(); 1396 early_boot_irqs_disabled = true; 1397 1398 xen_raw_console_write("mapping kernel into physical memory\n"); 1399 xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base, 1400 xen_start_info->nr_pages); 1401 xen_reserve_special_pages(); 1402 1403 /* 1404 * We used to do this in xen_arch_setup, but that is too late 1405 * on AMD were early_cpu_init (run before ->arch_setup()) calls 1406 * early_amd_init which pokes 0xcf8 port. 1407 */ 1408 set_iopl.iopl = 1; 1409 rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 1410 if (rc != 0) 1411 xen_raw_printk("physdev_op failed %d\n", rc); 1412 1413 1414 if (xen_start_info->mod_start) { 1415 if (xen_start_info->flags & SIF_MOD_START_PFN) 1416 initrd_start = PFN_PHYS(xen_start_info->mod_start); 1417 else 1418 initrd_start = __pa(xen_start_info->mod_start); 1419 } 1420 1421 /* Poke various useful things into boot_params */ 1422 boot_params.hdr.type_of_loader = (9 << 4) | 0; 1423 boot_params.hdr.ramdisk_image = initrd_start; 1424 boot_params.hdr.ramdisk_size = xen_start_info->mod_len; 1425 boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line); 1426 boot_params.hdr.hardware_subarch = X86_SUBARCH_XEN; 1427 1428 if (!xen_initial_domain()) { 1429 if (pci_xen) 1430 x86_init.pci.arch_init = pci_xen_init; 1431 x86_platform.set_legacy_features = 1432 xen_domu_set_legacy_features; 1433 } else { 1434 const struct dom0_vga_console_info *info = 1435 (void *)((char *)xen_start_info + 1436 xen_start_info->console.dom0.info_off); 1437 struct xen_platform_op op = { 1438 .cmd = XENPF_firmware_info, 1439 .interface_version = XENPF_INTERFACE_VERSION, 1440 .u.firmware_info.type = XEN_FW_KBD_SHIFT_FLAGS, 1441 }; 1442 1443 x86_platform.set_legacy_features = 1444 xen_dom0_set_legacy_features; 1445 xen_init_vga(info, xen_start_info->console.dom0.info_size, 1446 &boot_params.screen_info); 1447 xen_start_info->console.domU.mfn = 0; 1448 xen_start_info->console.domU.evtchn = 0; 1449 1450 if (HYPERVISOR_platform_op(&op) == 0) 1451 boot_params.kbd_status = op.u.firmware_info.u.kbd_shift_flags; 1452 1453 /* Make sure ACS will be enabled */ 1454 pci_request_acs(); 1455 1456 xen_acpi_sleep_register(); 1457 1458 xen_boot_params_init_edd(); 1459 1460 #ifdef CONFIG_ACPI 1461 /* 1462 * Disable selecting "Firmware First mode" for correctable 1463 * memory errors, as this is the duty of the hypervisor to 1464 * decide. 1465 */ 1466 acpi_disable_cmcff = 1; 1467 #endif 1468 } 1469 1470 xen_add_preferred_consoles(); 1471 1472 #ifdef CONFIG_PCI 1473 /* PCI BIOS service won't work from a PV guest. */ 1474 pci_probe &= ~PCI_PROBE_BIOS; 1475 #endif 1476 xen_raw_console_write("about to get started...\n"); 1477 1478 /* We need this for printk timestamps */ 1479 xen_setup_runstate_info(0); 1480 1481 xen_efi_init(&boot_params); 1482 1483 /* Start the world */ 1484 cr4_init_shadow(); /* 32b kernel does this in i386_start_kernel() */ 1485 x86_64_start_reservations((char *)__pa_symbol(&boot_params)); 1486 } 1487 1488 static int xen_cpu_up_prepare_pv(unsigned int cpu) 1489 { 1490 int rc; 1491 1492 if (per_cpu(xen_vcpu, cpu) == NULL) 1493 return -ENODEV; 1494 1495 xen_setup_timer(cpu); 1496 1497 rc = xen_smp_intr_init(cpu); 1498 if (rc) { 1499 WARN(1, "xen_smp_intr_init() for CPU %d failed: %d\n", 1500 cpu, rc); 1501 return rc; 1502 } 1503 1504 rc = xen_smp_intr_init_pv(cpu); 1505 if (rc) { 1506 WARN(1, "xen_smp_intr_init_pv() for CPU %d failed: %d\n", 1507 cpu, rc); 1508 return rc; 1509 } 1510 1511 return 0; 1512 } 1513 1514 static int xen_cpu_dead_pv(unsigned int cpu) 1515 { 1516 xen_smp_intr_free(cpu); 1517 xen_smp_intr_free_pv(cpu); 1518 1519 xen_teardown_timer(cpu); 1520 1521 return 0; 1522 } 1523 1524 static uint32_t __init xen_platform_pv(void) 1525 { 1526 if (xen_pv_domain()) 1527 return xen_cpuid_base(); 1528 1529 return 0; 1530 } 1531 1532 const __initconst struct hypervisor_x86 x86_hyper_xen_pv = { 1533 .name = "Xen PV", 1534 .detect = xen_platform_pv, 1535 .type = X86_HYPER_XEN_PV, 1536 .runtime.pin_vcpu = xen_pin_vcpu, 1537 .ignore_nopv = true, 1538 }; 1539