1 /* 2 * X86 specific Hyper-V initialization code. 3 * 4 * Copyright (C) 2016, Microsoft, Inc. 5 * 6 * Author : K. Y. Srinivasan <kys@microsoft.com> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License version 2 as published 10 * by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, but 13 * WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 15 * NON INFRINGEMENT. See the GNU General Public License for more 16 * details. 17 * 18 */ 19 20 #include <linux/efi.h> 21 #include <linux/types.h> 22 #include <asm/apic.h> 23 #include <asm/desc.h> 24 #include <asm/hypervisor.h> 25 #include <asm/hyperv-tlfs.h> 26 #include <asm/mshyperv.h> 27 #include <linux/version.h> 28 #include <linux/vmalloc.h> 29 #include <linux/mm.h> 30 #include <linux/clockchips.h> 31 #include <linux/hyperv.h> 32 #include <linux/slab.h> 33 #include <linux/cpuhotplug.h> 34 35 #ifdef CONFIG_HYPERV_TSCPAGE 36 37 static struct ms_hyperv_tsc_page *tsc_pg; 38 39 struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 40 { 41 return tsc_pg; 42 } 43 EXPORT_SYMBOL_GPL(hv_get_tsc_page); 44 45 static u64 read_hv_clock_tsc(struct clocksource *arg) 46 { 47 u64 current_tick = hv_read_tsc_page(tsc_pg); 48 49 if (current_tick == U64_MAX) 50 rdmsrl(HV_X64_MSR_TIME_REF_COUNT, current_tick); 51 52 return current_tick; 53 } 54 55 static struct clocksource hyperv_cs_tsc = { 56 .name = "hyperv_clocksource_tsc_page", 57 .rating = 400, 58 .read = read_hv_clock_tsc, 59 .mask = CLOCKSOURCE_MASK(64), 60 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 61 }; 62 #endif 63 64 static u64 read_hv_clock_msr(struct clocksource *arg) 65 { 66 u64 current_tick; 67 /* 68 * Read the partition counter to get the current tick count. This count 69 * is set to 0 when the partition is created and is incremented in 70 * 100 nanosecond units. 71 */ 72 rdmsrl(HV_X64_MSR_TIME_REF_COUNT, current_tick); 73 return current_tick; 74 } 75 76 static struct clocksource hyperv_cs_msr = { 77 .name = "hyperv_clocksource_msr", 78 .rating = 400, 79 .read = read_hv_clock_msr, 80 .mask = CLOCKSOURCE_MASK(64), 81 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 82 }; 83 84 void *hv_hypercall_pg; 85 EXPORT_SYMBOL_GPL(hv_hypercall_pg); 86 struct clocksource *hyperv_cs; 87 EXPORT_SYMBOL_GPL(hyperv_cs); 88 89 u32 *hv_vp_index; 90 EXPORT_SYMBOL_GPL(hv_vp_index); 91 92 struct hv_vp_assist_page **hv_vp_assist_page; 93 EXPORT_SYMBOL_GPL(hv_vp_assist_page); 94 95 void __percpu **hyperv_pcpu_input_arg; 96 EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg); 97 98 u32 hv_max_vp_index; 99 EXPORT_SYMBOL_GPL(hv_max_vp_index); 100 101 static int hv_cpu_init(unsigned int cpu) 102 { 103 u64 msr_vp_index; 104 struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()]; 105 void **input_arg; 106 107 input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); 108 *input_arg = page_address(alloc_page(GFP_KERNEL)); 109 110 hv_get_vp_index(msr_vp_index); 111 112 hv_vp_index[smp_processor_id()] = msr_vp_index; 113 114 if (msr_vp_index > hv_max_vp_index) 115 hv_max_vp_index = msr_vp_index; 116 117 if (!hv_vp_assist_page) 118 return 0; 119 120 if (!*hvp) 121 *hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL); 122 123 if (*hvp) { 124 u64 val; 125 126 val = vmalloc_to_pfn(*hvp); 127 val = (val << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT) | 128 HV_X64_MSR_VP_ASSIST_PAGE_ENABLE; 129 130 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, val); 131 } 132 133 return 0; 134 } 135 136 static void (*hv_reenlightenment_cb)(void); 137 138 static void hv_reenlightenment_notify(struct work_struct *dummy) 139 { 140 struct hv_tsc_emulation_status emu_status; 141 142 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 143 144 /* Don't issue the callback if TSC accesses are not emulated */ 145 if (hv_reenlightenment_cb && emu_status.inprogress) 146 hv_reenlightenment_cb(); 147 } 148 static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify); 149 150 void hyperv_stop_tsc_emulation(void) 151 { 152 u64 freq; 153 struct hv_tsc_emulation_status emu_status; 154 155 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 156 emu_status.inprogress = 0; 157 wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 158 159 rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq); 160 tsc_khz = div64_u64(freq, 1000); 161 } 162 EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation); 163 164 static inline bool hv_reenlightenment_available(void) 165 { 166 /* 167 * Check for required features and priviliges to make TSC frequency 168 * change notifications work. 169 */ 170 return ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS && 171 ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE && 172 ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT; 173 } 174 175 __visible void __irq_entry hyperv_reenlightenment_intr(struct pt_regs *regs) 176 { 177 entering_ack_irq(); 178 179 inc_irq_stat(irq_hv_reenlightenment_count); 180 181 schedule_delayed_work(&hv_reenlightenment_work, HZ/10); 182 183 exiting_irq(); 184 } 185 186 void set_hv_tscchange_cb(void (*cb)(void)) 187 { 188 struct hv_reenlightenment_control re_ctrl = { 189 .vector = HYPERV_REENLIGHTENMENT_VECTOR, 190 .enabled = 1, 191 .target_vp = hv_vp_index[smp_processor_id()] 192 }; 193 struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1}; 194 195 if (!hv_reenlightenment_available()) { 196 pr_warn("Hyper-V: reenlightenment support is unavailable\n"); 197 return; 198 } 199 200 hv_reenlightenment_cb = cb; 201 202 /* Make sure callback is registered before we write to MSRs */ 203 wmb(); 204 205 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 206 wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl)); 207 } 208 EXPORT_SYMBOL_GPL(set_hv_tscchange_cb); 209 210 void clear_hv_tscchange_cb(void) 211 { 212 struct hv_reenlightenment_control re_ctrl; 213 214 if (!hv_reenlightenment_available()) 215 return; 216 217 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 218 re_ctrl.enabled = 0; 219 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 220 221 hv_reenlightenment_cb = NULL; 222 } 223 EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb); 224 225 static int hv_cpu_die(unsigned int cpu) 226 { 227 struct hv_reenlightenment_control re_ctrl; 228 unsigned int new_cpu; 229 unsigned long flags; 230 void **input_arg; 231 void *input_pg = NULL; 232 233 local_irq_save(flags); 234 input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); 235 input_pg = *input_arg; 236 *input_arg = NULL; 237 local_irq_restore(flags); 238 free_page((unsigned long)input_pg); 239 240 if (hv_vp_assist_page && hv_vp_assist_page[cpu]) 241 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, 0); 242 243 if (hv_reenlightenment_cb == NULL) 244 return 0; 245 246 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 247 if (re_ctrl.target_vp == hv_vp_index[cpu]) { 248 /* Reassign to some other online CPU */ 249 new_cpu = cpumask_any_but(cpu_online_mask, cpu); 250 251 re_ctrl.target_vp = hv_vp_index[new_cpu]; 252 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 253 } 254 255 return 0; 256 } 257 258 static int __init hv_pci_init(void) 259 { 260 int gen2vm = efi_enabled(EFI_BOOT); 261 262 /* 263 * For Generation-2 VM, we exit from pci_arch_init() by returning 0. 264 * The purpose is to suppress the harmless warning: 265 * "PCI: Fatal: No config space access function found" 266 */ 267 if (gen2vm) 268 return 0; 269 270 /* For Generation-1 VM, we'll proceed in pci_arch_init(). */ 271 return 1; 272 } 273 274 /* 275 * This function is to be invoked early in the boot sequence after the 276 * hypervisor has been detected. 277 * 278 * 1. Setup the hypercall page. 279 * 2. Register Hyper-V specific clocksource. 280 * 3. Setup Hyper-V specific APIC entry points. 281 */ 282 void __init hyperv_init(void) 283 { 284 u64 guest_id, required_msrs; 285 union hv_x64_msr_hypercall_contents hypercall_msr; 286 int cpuhp, i; 287 288 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 289 return; 290 291 /* Absolutely required MSRs */ 292 required_msrs = HV_X64_MSR_HYPERCALL_AVAILABLE | 293 HV_X64_MSR_VP_INDEX_AVAILABLE; 294 295 if ((ms_hyperv.features & required_msrs) != required_msrs) 296 return; 297 298 /* 299 * Allocate the per-CPU state for the hypercall input arg. 300 * If this allocation fails, we will not be able to setup 301 * (per-CPU) hypercall input page and thus this failure is 302 * fatal on Hyper-V. 303 */ 304 hyperv_pcpu_input_arg = alloc_percpu(void *); 305 306 BUG_ON(hyperv_pcpu_input_arg == NULL); 307 308 /* Allocate percpu VP index */ 309 hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index), 310 GFP_KERNEL); 311 if (!hv_vp_index) 312 return; 313 314 for (i = 0; i < num_possible_cpus(); i++) 315 hv_vp_index[i] = VP_INVAL; 316 317 hv_vp_assist_page = kcalloc(num_possible_cpus(), 318 sizeof(*hv_vp_assist_page), GFP_KERNEL); 319 if (!hv_vp_assist_page) { 320 ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED; 321 goto free_vp_index; 322 } 323 324 cpuhp = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online", 325 hv_cpu_init, hv_cpu_die); 326 if (cpuhp < 0) 327 goto free_vp_assist_page; 328 329 /* 330 * Setup the hypercall page and enable hypercalls. 331 * 1. Register the guest ID 332 * 2. Enable the hypercall and register the hypercall page 333 */ 334 guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0); 335 wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 336 337 hv_hypercall_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX); 338 if (hv_hypercall_pg == NULL) { 339 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 340 goto remove_cpuhp_state; 341 } 342 343 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 344 hypercall_msr.enable = 1; 345 hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); 346 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 347 348 hv_apic_init(); 349 350 x86_init.pci.arch_init = hv_pci_init; 351 352 /* 353 * Register Hyper-V specific clocksource. 354 */ 355 #ifdef CONFIG_HYPERV_TSCPAGE 356 if (ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE) { 357 union hv_x64_msr_hypercall_contents tsc_msr; 358 359 tsc_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL); 360 if (!tsc_pg) 361 goto register_msr_cs; 362 363 hyperv_cs = &hyperv_cs_tsc; 364 365 rdmsrl(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64); 366 367 tsc_msr.enable = 1; 368 tsc_msr.guest_physical_address = vmalloc_to_pfn(tsc_pg); 369 370 wrmsrl(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64); 371 372 hyperv_cs_tsc.archdata.vclock_mode = VCLOCK_HVCLOCK; 373 374 clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); 375 return; 376 } 377 register_msr_cs: 378 #endif 379 /* 380 * For 32 bit guests just use the MSR based mechanism for reading 381 * the partition counter. 382 */ 383 384 hyperv_cs = &hyperv_cs_msr; 385 if (ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE) 386 clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); 387 388 return; 389 390 remove_cpuhp_state: 391 cpuhp_remove_state(cpuhp); 392 free_vp_assist_page: 393 kfree(hv_vp_assist_page); 394 hv_vp_assist_page = NULL; 395 free_vp_index: 396 kfree(hv_vp_index); 397 hv_vp_index = NULL; 398 } 399 400 /* 401 * This routine is called before kexec/kdump, it does the required cleanup. 402 */ 403 void hyperv_cleanup(void) 404 { 405 union hv_x64_msr_hypercall_contents hypercall_msr; 406 407 /* Reset our OS id */ 408 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 409 410 /* 411 * Reset hypercall page reference before reset the page, 412 * let hypercall operations fail safely rather than 413 * panic the kernel for using invalid hypercall page 414 */ 415 hv_hypercall_pg = NULL; 416 417 /* Reset the hypercall page */ 418 hypercall_msr.as_uint64 = 0; 419 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 420 421 /* Reset the TSC page */ 422 hypercall_msr.as_uint64 = 0; 423 wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64); 424 } 425 EXPORT_SYMBOL_GPL(hyperv_cleanup); 426 427 void hyperv_report_panic(struct pt_regs *regs, long err) 428 { 429 static bool panic_reported; 430 u64 guest_id; 431 432 /* 433 * We prefer to report panic on 'die' chain as we have proper 434 * registers to report, but if we miss it (e.g. on BUG()) we need 435 * to report it on 'panic'. 436 */ 437 if (panic_reported) 438 return; 439 panic_reported = true; 440 441 rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 442 443 wrmsrl(HV_X64_MSR_CRASH_P0, err); 444 wrmsrl(HV_X64_MSR_CRASH_P1, guest_id); 445 wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip); 446 wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax); 447 wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp); 448 449 /* 450 * Let Hyper-V know there is crash data available 451 */ 452 wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY); 453 } 454 EXPORT_SYMBOL_GPL(hyperv_report_panic); 455 456 /** 457 * hyperv_report_panic_msg - report panic message to Hyper-V 458 * @pa: physical address of the panic page containing the message 459 * @size: size of the message in the page 460 */ 461 void hyperv_report_panic_msg(phys_addr_t pa, size_t size) 462 { 463 /* 464 * P3 to contain the physical address of the panic page & P4 to 465 * contain the size of the panic data in that page. Rest of the 466 * registers are no-op when the NOTIFY_MSG flag is set. 467 */ 468 wrmsrl(HV_X64_MSR_CRASH_P0, 0); 469 wrmsrl(HV_X64_MSR_CRASH_P1, 0); 470 wrmsrl(HV_X64_MSR_CRASH_P2, 0); 471 wrmsrl(HV_X64_MSR_CRASH_P3, pa); 472 wrmsrl(HV_X64_MSR_CRASH_P4, size); 473 474 /* 475 * Let Hyper-V know there is crash data available along with 476 * the panic message. 477 */ 478 wrmsrl(HV_X64_MSR_CRASH_CTL, 479 (HV_CRASH_CTL_CRASH_NOTIFY | HV_CRASH_CTL_CRASH_NOTIFY_MSG)); 480 } 481 EXPORT_SYMBOL_GPL(hyperv_report_panic_msg); 482 483 bool hv_is_hyperv_initialized(void) 484 { 485 union hv_x64_msr_hypercall_contents hypercall_msr; 486 487 /* 488 * Ensure that we're really on Hyper-V, and not a KVM or Xen 489 * emulation of Hyper-V 490 */ 491 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 492 return false; 493 494 /* 495 * Verify that earlier initialization succeeded by checking 496 * that the hypercall page is setup 497 */ 498 hypercall_msr.as_uint64 = 0; 499 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 500 501 return hypercall_msr.enable; 502 } 503 EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized); 504