1 /* 2 * X86 specific Hyper-V initialization code. 3 * 4 * Copyright (C) 2016, Microsoft, Inc. 5 * 6 * Author : K. Y. Srinivasan <kys@microsoft.com> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License version 2 as published 10 * by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, but 13 * WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 15 * NON INFRINGEMENT. See the GNU General Public License for more 16 * details. 17 * 18 */ 19 20 #include <linux/types.h> 21 #include <asm/apic.h> 22 #include <asm/desc.h> 23 #include <asm/hypervisor.h> 24 #include <asm/hyperv-tlfs.h> 25 #include <asm/mshyperv.h> 26 #include <linux/version.h> 27 #include <linux/vmalloc.h> 28 #include <linux/mm.h> 29 #include <linux/clockchips.h> 30 #include <linux/hyperv.h> 31 #include <linux/slab.h> 32 #include <linux/cpuhotplug.h> 33 34 #ifdef CONFIG_HYPERV_TSCPAGE 35 36 static struct ms_hyperv_tsc_page *tsc_pg; 37 38 struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 39 { 40 return tsc_pg; 41 } 42 EXPORT_SYMBOL_GPL(hv_get_tsc_page); 43 44 static u64 read_hv_clock_tsc(struct clocksource *arg) 45 { 46 u64 current_tick = hv_read_tsc_page(tsc_pg); 47 48 if (current_tick == U64_MAX) 49 rdmsrl(HV_X64_MSR_TIME_REF_COUNT, current_tick); 50 51 return current_tick; 52 } 53 54 static struct clocksource hyperv_cs_tsc = { 55 .name = "hyperv_clocksource_tsc_page", 56 .rating = 400, 57 .read = read_hv_clock_tsc, 58 .mask = CLOCKSOURCE_MASK(64), 59 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 60 }; 61 #endif 62 63 static u64 read_hv_clock_msr(struct clocksource *arg) 64 { 65 u64 current_tick; 66 /* 67 * Read the partition counter to get the current tick count. This count 68 * is set to 0 when the partition is created and is incremented in 69 * 100 nanosecond units. 70 */ 71 rdmsrl(HV_X64_MSR_TIME_REF_COUNT, current_tick); 72 return current_tick; 73 } 74 75 static struct clocksource hyperv_cs_msr = { 76 .name = "hyperv_clocksource_msr", 77 .rating = 400, 78 .read = read_hv_clock_msr, 79 .mask = CLOCKSOURCE_MASK(64), 80 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 81 }; 82 83 void *hv_hypercall_pg; 84 EXPORT_SYMBOL_GPL(hv_hypercall_pg); 85 struct clocksource *hyperv_cs; 86 EXPORT_SYMBOL_GPL(hyperv_cs); 87 88 u32 *hv_vp_index; 89 EXPORT_SYMBOL_GPL(hv_vp_index); 90 91 struct hv_vp_assist_page **hv_vp_assist_page; 92 EXPORT_SYMBOL_GPL(hv_vp_assist_page); 93 94 void __percpu **hyperv_pcpu_input_arg; 95 EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg); 96 97 u32 hv_max_vp_index; 98 99 static int hv_cpu_init(unsigned int cpu) 100 { 101 u64 msr_vp_index; 102 struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()]; 103 void **input_arg; 104 105 input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); 106 *input_arg = page_address(alloc_page(GFP_KERNEL)); 107 108 hv_get_vp_index(msr_vp_index); 109 110 hv_vp_index[smp_processor_id()] = msr_vp_index; 111 112 if (msr_vp_index > hv_max_vp_index) 113 hv_max_vp_index = msr_vp_index; 114 115 if (!hv_vp_assist_page) 116 return 0; 117 118 if (!*hvp) 119 *hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL); 120 121 if (*hvp) { 122 u64 val; 123 124 val = vmalloc_to_pfn(*hvp); 125 val = (val << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT) | 126 HV_X64_MSR_VP_ASSIST_PAGE_ENABLE; 127 128 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, val); 129 } 130 131 return 0; 132 } 133 134 static void (*hv_reenlightenment_cb)(void); 135 136 static void hv_reenlightenment_notify(struct work_struct *dummy) 137 { 138 struct hv_tsc_emulation_status emu_status; 139 140 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 141 142 /* Don't issue the callback if TSC accesses are not emulated */ 143 if (hv_reenlightenment_cb && emu_status.inprogress) 144 hv_reenlightenment_cb(); 145 } 146 static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify); 147 148 void hyperv_stop_tsc_emulation(void) 149 { 150 u64 freq; 151 struct hv_tsc_emulation_status emu_status; 152 153 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 154 emu_status.inprogress = 0; 155 wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); 156 157 rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq); 158 tsc_khz = div64_u64(freq, 1000); 159 } 160 EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation); 161 162 static inline bool hv_reenlightenment_available(void) 163 { 164 /* 165 * Check for required features and priviliges to make TSC frequency 166 * change notifications work. 167 */ 168 return ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS && 169 ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE && 170 ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT; 171 } 172 173 __visible void __irq_entry hyperv_reenlightenment_intr(struct pt_regs *regs) 174 { 175 entering_ack_irq(); 176 177 inc_irq_stat(irq_hv_reenlightenment_count); 178 179 schedule_delayed_work(&hv_reenlightenment_work, HZ/10); 180 181 exiting_irq(); 182 } 183 184 void set_hv_tscchange_cb(void (*cb)(void)) 185 { 186 struct hv_reenlightenment_control re_ctrl = { 187 .vector = HYPERV_REENLIGHTENMENT_VECTOR, 188 .enabled = 1, 189 .target_vp = hv_vp_index[smp_processor_id()] 190 }; 191 struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1}; 192 193 if (!hv_reenlightenment_available()) { 194 pr_warn("Hyper-V: reenlightenment support is unavailable\n"); 195 return; 196 } 197 198 hv_reenlightenment_cb = cb; 199 200 /* Make sure callback is registered before we write to MSRs */ 201 wmb(); 202 203 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 204 wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl)); 205 } 206 EXPORT_SYMBOL_GPL(set_hv_tscchange_cb); 207 208 void clear_hv_tscchange_cb(void) 209 { 210 struct hv_reenlightenment_control re_ctrl; 211 212 if (!hv_reenlightenment_available()) 213 return; 214 215 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 216 re_ctrl.enabled = 0; 217 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); 218 219 hv_reenlightenment_cb = NULL; 220 } 221 EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb); 222 223 static int hv_cpu_die(unsigned int cpu) 224 { 225 struct hv_reenlightenment_control re_ctrl; 226 unsigned int new_cpu; 227 unsigned long flags; 228 void **input_arg; 229 void *input_pg = NULL; 230 231 local_irq_save(flags); 232 input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); 233 input_pg = *input_arg; 234 *input_arg = NULL; 235 local_irq_restore(flags); 236 free_page((unsigned long)input_pg); 237 238 if (hv_vp_assist_page && hv_vp_assist_page[cpu]) 239 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, 0); 240 241 if (hv_reenlightenment_cb == NULL) 242 return 0; 243 244 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 245 if (re_ctrl.target_vp == hv_vp_index[cpu]) { 246 /* Reassign to some other online CPU */ 247 new_cpu = cpumask_any_but(cpu_online_mask, cpu); 248 249 re_ctrl.target_vp = hv_vp_index[new_cpu]; 250 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); 251 } 252 253 return 0; 254 } 255 256 /* 257 * This function is to be invoked early in the boot sequence after the 258 * hypervisor has been detected. 259 * 260 * 1. Setup the hypercall page. 261 * 2. Register Hyper-V specific clocksource. 262 * 3. Setup Hyper-V specific APIC entry points. 263 */ 264 void __init hyperv_init(void) 265 { 266 u64 guest_id, required_msrs; 267 union hv_x64_msr_hypercall_contents hypercall_msr; 268 int cpuhp; 269 270 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 271 return; 272 273 /* Absolutely required MSRs */ 274 required_msrs = HV_X64_MSR_HYPERCALL_AVAILABLE | 275 HV_X64_MSR_VP_INDEX_AVAILABLE; 276 277 if ((ms_hyperv.features & required_msrs) != required_msrs) 278 return; 279 280 /* 281 * Allocate the per-CPU state for the hypercall input arg. 282 * If this allocation fails, we will not be able to setup 283 * (per-CPU) hypercall input page and thus this failure is 284 * fatal on Hyper-V. 285 */ 286 hyperv_pcpu_input_arg = alloc_percpu(void *); 287 288 BUG_ON(hyperv_pcpu_input_arg == NULL); 289 290 /* Allocate percpu VP index */ 291 hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index), 292 GFP_KERNEL); 293 if (!hv_vp_index) 294 return; 295 296 hv_vp_assist_page = kcalloc(num_possible_cpus(), 297 sizeof(*hv_vp_assist_page), GFP_KERNEL); 298 if (!hv_vp_assist_page) { 299 ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED; 300 goto free_vp_index; 301 } 302 303 cpuhp = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online", 304 hv_cpu_init, hv_cpu_die); 305 if (cpuhp < 0) 306 goto free_vp_assist_page; 307 308 /* 309 * Setup the hypercall page and enable hypercalls. 310 * 1. Register the guest ID 311 * 2. Enable the hypercall and register the hypercall page 312 */ 313 guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0); 314 wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 315 316 hv_hypercall_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX); 317 if (hv_hypercall_pg == NULL) { 318 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 319 goto remove_cpuhp_state; 320 } 321 322 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 323 hypercall_msr.enable = 1; 324 hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); 325 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 326 327 hv_apic_init(); 328 329 /* 330 * Register Hyper-V specific clocksource. 331 */ 332 #ifdef CONFIG_HYPERV_TSCPAGE 333 if (ms_hyperv.features & HV_X64_MSR_REFERENCE_TSC_AVAILABLE) { 334 union hv_x64_msr_hypercall_contents tsc_msr; 335 336 tsc_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL); 337 if (!tsc_pg) 338 goto register_msr_cs; 339 340 hyperv_cs = &hyperv_cs_tsc; 341 342 rdmsrl(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64); 343 344 tsc_msr.enable = 1; 345 tsc_msr.guest_physical_address = vmalloc_to_pfn(tsc_pg); 346 347 wrmsrl(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64); 348 349 hyperv_cs_tsc.archdata.vclock_mode = VCLOCK_HVCLOCK; 350 351 clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); 352 return; 353 } 354 register_msr_cs: 355 #endif 356 /* 357 * For 32 bit guests just use the MSR based mechanism for reading 358 * the partition counter. 359 */ 360 361 hyperv_cs = &hyperv_cs_msr; 362 if (ms_hyperv.features & HV_X64_MSR_TIME_REF_COUNT_AVAILABLE) 363 clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); 364 365 return; 366 367 remove_cpuhp_state: 368 cpuhp_remove_state(cpuhp); 369 free_vp_assist_page: 370 kfree(hv_vp_assist_page); 371 hv_vp_assist_page = NULL; 372 free_vp_index: 373 kfree(hv_vp_index); 374 hv_vp_index = NULL; 375 } 376 377 /* 378 * This routine is called before kexec/kdump, it does the required cleanup. 379 */ 380 void hyperv_cleanup(void) 381 { 382 union hv_x64_msr_hypercall_contents hypercall_msr; 383 384 /* Reset our OS id */ 385 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); 386 387 /* Reset the hypercall page */ 388 hypercall_msr.as_uint64 = 0; 389 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 390 391 /* Reset the TSC page */ 392 hypercall_msr.as_uint64 = 0; 393 wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64); 394 } 395 EXPORT_SYMBOL_GPL(hyperv_cleanup); 396 397 void hyperv_report_panic(struct pt_regs *regs, long err) 398 { 399 static bool panic_reported; 400 u64 guest_id; 401 402 /* 403 * We prefer to report panic on 'die' chain as we have proper 404 * registers to report, but if we miss it (e.g. on BUG()) we need 405 * to report it on 'panic'. 406 */ 407 if (panic_reported) 408 return; 409 panic_reported = true; 410 411 rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); 412 413 wrmsrl(HV_X64_MSR_CRASH_P0, err); 414 wrmsrl(HV_X64_MSR_CRASH_P1, guest_id); 415 wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip); 416 wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax); 417 wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp); 418 419 /* 420 * Let Hyper-V know there is crash data available 421 */ 422 wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY); 423 } 424 EXPORT_SYMBOL_GPL(hyperv_report_panic); 425 426 bool hv_is_hyperv_initialized(void) 427 { 428 union hv_x64_msr_hypercall_contents hypercall_msr; 429 430 /* 431 * Ensure that we're really on Hyper-V, and not a KVM or Xen 432 * emulation of Hyper-V 433 */ 434 if (x86_hyper_type != X86_HYPER_MS_HYPERV) 435 return false; 436 437 /* 438 * Verify that earlier initialization succeeded by checking 439 * that the hypercall page is setup 440 */ 441 hypercall_msr.as_uint64 = 0; 442 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); 443 444 return hypercall_msr.enable; 445 } 446 EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized); 447