1 /* 2 * X86 specific Hyper-V initialization code. 3 * 4 * Copyright (C) 2016, Microsoft, Inc. 5 * 6 * Author : K. Y. Srinivasan <kys@microsoft.com> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License version 2 as published 10 * by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, but 13 * WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 15 * NON INFRINGEMENT. See the GNU General Public License for more 16 * details. 17 * 18 */ 19 20 #include <linux/efi.h> 21 #include <linux/types.h> 22 #include <asm/apic.h> 23 #include <asm/desc.h> 24 #include <asm/hypervisor.h> 25 #include <asm/hyperv-tlfs.h> 26 #include <asm/mshyperv.h> 27 #include <linux/version.h> 28 #include <linux/vmalloc.h> 29 #include <linux/mm.h> 30 #include <linux/clockchips.h> 31 #include <linux/hyperv.h> 32 #include <linux/slab.h> 33 #include <linux/cpuhotplug.h> 34 35 #ifdef CONFIG_HYPERV_TSCPAGE 36 37 static struct ms_hyperv_tsc_page *tsc_pg; 38 39 struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 40 { 41 return tsc_pg; 42 } 43 EXPORT_SYMBOL_GPL(hv_get_tsc_page); 44 45 static u64 read_hv_clock_tsc(struct clocksource *arg) 46 { 47 u64 current_tick = hv_read_tsc_page(tsc_pg); 48 49 if (current_tick == U64_MAX) 50 rdmsrl(HV_X64_MSR_TIME_REF_COUNT, current_tick); 51 52 return current_tick; 53 } 54 55 static struct clocksource hyperv_cs_tsc = { 56 .name = "hyperv_clocksource_tsc_page", 57 .rating = 400, 58 .read = read_hv_clock_tsc, 59 .mask = CLOCKSOURCE_MASK(64), 60 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 61 }; 62 #endif 63 64 static u64 read_hv_clock_msr(struct clocksource *arg) 65 { 66 u64 current_tick; 67 /* 68 * Read the partition counter to get the current tick count. This count 69 * is set to 0 when the partition is created and is incremented in 70 * 100 nanosecond units. 71 */ 72 rdmsrl(HV_X64_MSR_TIME_REF_COUNT, current_tick); 73 return current_tick; 74 } 75 76 static struct clocksource hyperv_cs_msr = { 77 .name = "hyperv_clocksource_msr", 78 .rating = 400, 79 .read = read_hv_clock_msr, 80 .mask = CLOCKSOURCE_MASK(64), 81 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 82 }; 83 84 void *hv_hypercall_pg; 85 EXPORT_SYMBOL_GPL(hv_hypercall_pg); 86 struct clocksource *hyperv_cs; 87 EXPORT_SYMBOL_GPL(hyperv_cs); 88 89 u32 *hv_vp_index; 90 EXPORT_SYMBOL_GPL(hv_vp_index); 91 92 struct hv_vp_assist_page **hv_vp_assist_page; 93 EXPORT_SYMBOL_GPL(hv_vp_assist_page); 94 95 void __percpu **hyperv_pcpu_input_arg; 96 EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg); 97 98 u32 hv_max_vp_index; 99 EXPORT_SYMBOL_GPL(hv_max_vp_index); 100 101 static int hv_cpu_init(unsigned int cpu) 102 { 103 u64 msr_vp_index; 104 struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()]; 105 void **input_arg; 106 struct page *pg; 107 108 input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); 109 pg = alloc_page(GFP_KERNEL); 110 if (unlikely(!pg)) 111 return -ENOMEM; 112 *input_arg = page_address(pg); 113 114 hv_get_vp_index(msr_vp_index); 115 116 hv_vp_index[smp_processor_id()] = msr_vp_index; 117 118 if (msr_vp_index > hv_max_vp_index) 119 hv_max_vp_index = msr_vp_index; 120 121 if (!hv_vp_assist_page) 122 return 0; 123 124 if (!*hvp) 125 *hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL); 126 127 if (*hvp) { 128 u64 val; 129 130 val = vmalloc_to_pfn(*hvp); 131 val = (val << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT) | 132 HV_X64_MSR_VP_ASSIST_PAGE_ENABLE; 133 134 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, val); 135 } 136 137 return 0; 138 } 139 140 static void (*hv_reenlightenment_cb)(void); 141 142 static void hv_reenlightenment_notify(struct work_struct *dummy) 143 { 144 struct hv_tsc_emulation_status emu_status; 145 146 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 147 148 /* Don't issue the callback if TSC accesses are not emulated */ 149 if (hv_reenlightenment_cb && emu_status.inprogress) 150 hv_reenlightenment_cb(); 151 } 152 static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify); 153 154 void hyperv_stop_tsc_emulation(void) 155 { 156 u64 freq; 157 struct hv_tsc_emulation_status emu_status; 158 159 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 160 emu_status.inprogress = 0; 161 wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 162 163 rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq); 164 tsc_khz = div64_u64(freq, 1000); 165 } 166 EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation); 167 168 static inline bool hv_reenlightenment_available(void) 169 { 170 /* 171 * Check for required features and priviliges to make TSC frequency 172 * change notifications work. 173 */ 174 return ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS && 175 ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE && 176 ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT; 177 } 178 179 __visible void __irq_entry hyperv_reenlightenment_intr(struct pt_regs *regs) 180 { 181 entering_ack_irq(); 182 183 inc_irq_stat(irq_hv_reenlightenment_count); 184 185 schedule_delayed_work(&hv_reenlightenment_work, HZ/10); 186 187 exiting_irq(); 188 } 189 190 void set_hv_tscchange_cb(void (*cb)(void)) 191 { 192 struct hv_reenlightenment_control re_ctrl = { 193 .vector = HYPERV_REENLIGHTENMENT_VECTOR, 194 .enabled = 1, 195 .target_vp = hv_vp_index[smp_processor_id()] 196 }; 197 struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1}; 198 199 if (!hv_reenlightenment_available()) { 200 pr_warn("Hyper-V: reenlightenment support is unavailable\n"); 201 return; 202 } 203 204 hv_reenlightenment_cb = cb; 205 206 /* Make sure callback is registered before we write to MSRs */ 207 wmb(); 208 209 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 210 wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl)); 211 } 212 EXPORT_SYMBOL_GPL(set_hv_tscchange_cb); 213 214 void clear_hv_tscchange_cb(void) 215 { 216 struct hv_reenlightenment_control re_ctrl; 217 218 if (!hv_reenlightenment_available()) 219 return; 220 221 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 222 re_ctrl.enabled = 0; 223 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 224 225 hv_reenlightenment_cb = NULL; 226 } 227 EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb); 228 229 static int hv_cpu_die(unsigned int cpu) 230 { 231 struct hv_reenlightenment_control re_ctrl; 232 unsigned int new_cpu; 233 unsigned long flags; 234 void **input_arg; 235 void *input_pg = NULL; 236 237 local_irq_save(flags); 238 input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); 239 input_pg = *input_arg; 240 *input_arg = NULL; 241 local_irq_restore(flags); 242 free_page((unsigned long)input_pg); 243 244 if (hv_vp_assist_page && hv_vp_assist_page[cpu]) 245 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, 0); 246 247 if (hv_reenlightenment_cb == NULL) 248 return 0; 249 250 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 251 if (re_ctrl.target_vp == hv_vp_index[cpu]) { 252 /* Reassign to some other online CPU */ 253 new_cpu = cpumask_any_but(cpu_online_mask, cpu); 254 255 re_ctrl.target_vp = hv_vp_index[new_cpu]; 256 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 257 } 258 259 return 0; 260 } 261 262 static int __init hv_pci_init(void) 263 { 264 int gen2vm = efi_enabled(EFI_BOOT); 265 266 /* 267 * For Generation-2 VM, we exit from pci_arch_init() by returning 0. 268 * The purpose is to suppress the harmless warning: 269 * "PCI: Fatal: No config space access function found" 270 */ 271 if (gen2vm) 272 return 0; 273 274 /* For Generation-1 VM, we'll proceed in pci_arch_init(). */ 275 return 1; 276 } 277 278 /* 279 * This function is to be invoked early in the boot sequence after the 280 * hypervisor has been detected. 281 * 282 * 1. Setup the hypercall page. 283 * 2. Register Hyper-V specific clocksource. 284 * 3. Setup Hyper-V specific APIC entry points. 285 */ 286 void __init hyperv_init(void) 287 { 288 u64 guest_id, required_msrs; 289 union hv_x64_msr_hypercall_contents hypercall_msr; 290 int cpuhp, i; 291 292 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 293 return; 294 295 /* Absolutely required MSRs */ 296 required_msrs = HV_X64_MSR_HYPERCALL_AVAILABLE | 297 HV_X64_MSR_VP_INDEX_AVAILABLE; 298 299 if ((ms_hyperv.features & required_msrs) != required_msrs) 300 return; 301 302 /* 303 * Allocate the per-CPU state for the hypercall input arg. 304 * If this allocation fails, we will not be able to setup 305 * (per-CPU) hypercall input page and thus this failure is 306 * fatal on Hyper-V. 307 */ 308 hyperv_pcpu_input_arg = alloc_percpu(void *); 309 310 BUG_ON(hyperv_pcpu_input_arg == NULL); 311 312 /* Allocate percpu VP index */ 313 hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index), 314 GFP_KERNEL); 315 if (!hv_vp_index) 316 return; 317 318 for (i = 0; i < num_possible_cpus(); i++) 319 hv_vp_index[i] = VP_INVAL; 320 321 hv_vp_assist_page = kcalloc(num_possible_cpus(), 322 sizeof(*hv_vp_assist_page), GFP_KERNEL); 323 if (!hv_vp_assist_page) { 324 ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED; 325 goto free_vp_index; 326 } 327 328 cpuhp = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online", 329 hv_cpu_init, hv_cpu_die); 330 if (cpuhp < 0) 331 goto free_vp_assist_page; 332 333 /* 334 * Setup the hypercall page and enable hypercalls. 335 * 1. Register the guest ID 336 * 2. Enable the hypercall and register the hypercall page 337 */ 338 guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0); 339 wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 340 341 hv_hypercall_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX); 342 if (hv_hypercall_pg == NULL) { 343 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 344 goto remove_cpuhp_state; 345 } 346 347 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 348 hypercall_msr.enable = 1; 349 hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); 350 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 351 352 hv_apic_init(); 353 354 x86_init.pci.arch_init = hv_pci_init; 355 356 /* 357 * Register Hyper-V specific clocksource. 358 */ 359 #ifdef CONFIG_HYPERV_TSCPAGE 360 if (ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE) { 361 union hv_x64_msr_hypercall_contents tsc_msr; 362 363 tsc_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL); 364 if (!tsc_pg) 365 goto register_msr_cs; 366 367 hyperv_cs = &hyperv_cs_tsc; 368 369 rdmsrl(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64); 370 371 tsc_msr.enable = 1; 372 tsc_msr.guest_physical_address = vmalloc_to_pfn(tsc_pg); 373 374 wrmsrl(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64); 375 376 hyperv_cs_tsc.archdata.vclock_mode = VCLOCK_HVCLOCK; 377 378 clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); 379 return; 380 } 381 register_msr_cs: 382 #endif 383 /* 384 * For 32 bit guests just use the MSR based mechanism for reading 385 * the partition counter. 386 */ 387 388 hyperv_cs = &hyperv_cs_msr; 389 if (ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE) 390 clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); 391 392 return; 393 394 remove_cpuhp_state: 395 cpuhp_remove_state(cpuhp); 396 free_vp_assist_page: 397 kfree(hv_vp_assist_page); 398 hv_vp_assist_page = NULL; 399 free_vp_index: 400 kfree(hv_vp_index); 401 hv_vp_index = NULL; 402 } 403 404 /* 405 * This routine is called before kexec/kdump, it does the required cleanup. 406 */ 407 void hyperv_cleanup(void) 408 { 409 union hv_x64_msr_hypercall_contents hypercall_msr; 410 411 /* Reset our OS id */ 412 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 413 414 /* 415 * Reset hypercall page reference before reset the page, 416 * let hypercall operations fail safely rather than 417 * panic the kernel for using invalid hypercall page 418 */ 419 hv_hypercall_pg = NULL; 420 421 /* Reset the hypercall page */ 422 hypercall_msr.as_uint64 = 0; 423 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 424 425 /* Reset the TSC page */ 426 hypercall_msr.as_uint64 = 0; 427 wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64); 428 } 429 EXPORT_SYMBOL_GPL(hyperv_cleanup); 430 431 void hyperv_report_panic(struct pt_regs *regs, long err) 432 { 433 static bool panic_reported; 434 u64 guest_id; 435 436 /* 437 * We prefer to report panic on 'die' chain as we have proper 438 * registers to report, but if we miss it (e.g. on BUG()) we need 439 * to report it on 'panic'. 440 */ 441 if (panic_reported) 442 return; 443 panic_reported = true; 444 445 rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 446 447 wrmsrl(HV_X64_MSR_CRASH_P0, err); 448 wrmsrl(HV_X64_MSR_CRASH_P1, guest_id); 449 wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip); 450 wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax); 451 wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp); 452 453 /* 454 * Let Hyper-V know there is crash data available 455 */ 456 wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY); 457 } 458 EXPORT_SYMBOL_GPL(hyperv_report_panic); 459 460 /** 461 * hyperv_report_panic_msg - report panic message to Hyper-V 462 * @pa: physical address of the panic page containing the message 463 * @size: size of the message in the page 464 */ 465 void hyperv_report_panic_msg(phys_addr_t pa, size_t size) 466 { 467 /* 468 * P3 to contain the physical address of the panic page & P4 to 469 * contain the size of the panic data in that page. Rest of the 470 * registers are no-op when the NOTIFY_MSG flag is set. 471 */ 472 wrmsrl(HV_X64_MSR_CRASH_P0, 0); 473 wrmsrl(HV_X64_MSR_CRASH_P1, 0); 474 wrmsrl(HV_X64_MSR_CRASH_P2, 0); 475 wrmsrl(HV_X64_MSR_CRASH_P3, pa); 476 wrmsrl(HV_X64_MSR_CRASH_P4, size); 477 478 /* 479 * Let Hyper-V know there is crash data available along with 480 * the panic message. 481 */ 482 wrmsrl(HV_X64_MSR_CRASH_CTL, 483 (HV_CRASH_CTL_CRASH_NOTIFY | HV_CRASH_CTL_CRASH_NOTIFY_MSG)); 484 } 485 EXPORT_SYMBOL_GPL(hyperv_report_panic_msg); 486 487 bool hv_is_hyperv_initialized(void) 488 { 489 union hv_x64_msr_hypercall_contents hypercall_msr; 490 491 /* 492 * Ensure that we're really on Hyper-V, and not a KVM or Xen 493 * emulation of Hyper-V 494 */ 495 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 496 return false; 497 498 /* 499 * Verify that earlier initialization succeeded by checking 500 * that the hypercall page is setup 501 */ 502 hypercall_msr.as_uint64 = 0; 503 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 504 505 return hypercall_msr.enable; 506 } 507 EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized); 508