1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KVM Microsoft Hyper-V emulation 4 * 5 * derived from arch/x86/kvm/x86.c 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright (C) 2008 Qumranet, Inc. 9 * Copyright IBM Corporation, 2008 10 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 11 * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com> 12 * 13 * Authors: 14 * Avi Kivity <avi@qumranet.com> 15 * Yaniv Kamay <yaniv@qumranet.com> 16 * Amit Shah <amit.shah@qumranet.com> 17 * Ben-Ami Yassour <benami@il.ibm.com> 18 * Andrey Smetanin <asmetanin@virtuozzo.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include "x86.h" 23 #include "lapic.h" 24 #include "ioapic.h" 25 #include "cpuid.h" 26 #include "hyperv.h" 27 #include "mmu.h" 28 #include "xen.h" 29 30 #include <linux/cpu.h> 31 #include <linux/kvm_host.h> 32 #include <linux/highmem.h> 33 #include <linux/sched/cputime.h> 34 #include <linux/spinlock.h> 35 #include <linux/eventfd.h> 36 37 #include <asm/apicdef.h> 38 #include <asm/mshyperv.h> 39 #include <trace/events/kvm.h> 40 41 #include "trace.h" 42 #include "irq.h" 43 #include "fpu.h" 44 45 #define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, HV_VCPUS_PER_SPARSE_BANK) 46 47 /* 48 * As per Hyper-V TLFS, extended hypercalls start from 0x8001 49 * (HvExtCallQueryCapabilities). Response of this hypercalls is a 64 bit value 50 * where each bit tells which extended hypercall is available besides 51 * HvExtCallQueryCapabilities. 52 * 53 * 0x8001 - First extended hypercall, HvExtCallQueryCapabilities, no bit 54 * assigned. 55 * 56 * 0x8002 - Bit 0 57 * 0x8003 - Bit 1 58 * .. 59 * 0x8041 - Bit 63 60 * 61 * Therefore, HV_EXT_CALL_MAX = 0x8001 + 64 62 */ 63 #define HV_EXT_CALL_MAX (HV_EXT_CALL_QUERY_CAPABILITIES + 64) 64 65 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, 66 bool vcpu_kick); 67 68 static inline u64 synic_read_sint(struct kvm_vcpu_hv_synic *synic, int sint) 69 { 70 return atomic64_read(&synic->sint[sint]); 71 } 72 73 static inline int synic_get_sint_vector(u64 sint_value) 74 { 75 if (sint_value & HV_SYNIC_SINT_MASKED) 76 return -1; 77 return sint_value & HV_SYNIC_SINT_VECTOR_MASK; 78 } 79 80 static bool synic_has_vector_connected(struct kvm_vcpu_hv_synic *synic, 81 int vector) 82 { 83 int i; 84 85 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 86 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector) 87 return true; 88 } 89 return false; 90 } 91 92 static bool synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic *synic, 93 int vector) 94 { 95 int i; 96 u64 sint_value; 97 98 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 99 sint_value = synic_read_sint(synic, i); 100 if (synic_get_sint_vector(sint_value) == vector && 101 sint_value & HV_SYNIC_SINT_AUTO_EOI) 102 return true; 103 } 104 return false; 105 } 106 107 static void synic_update_vector(struct kvm_vcpu_hv_synic *synic, 108 int vector) 109 { 110 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 111 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 112 bool auto_eoi_old, auto_eoi_new; 113 114 if (vector < HV_SYNIC_FIRST_VALID_VECTOR) 115 return; 116 117 if (synic_has_vector_connected(synic, vector)) 118 __set_bit(vector, synic->vec_bitmap); 119 else 120 __clear_bit(vector, synic->vec_bitmap); 121 122 auto_eoi_old = !bitmap_empty(synic->auto_eoi_bitmap, 256); 123 124 if (synic_has_vector_auto_eoi(synic, vector)) 125 __set_bit(vector, synic->auto_eoi_bitmap); 126 else 127 __clear_bit(vector, synic->auto_eoi_bitmap); 128 129 auto_eoi_new = !bitmap_empty(synic->auto_eoi_bitmap, 256); 130 131 if (auto_eoi_old == auto_eoi_new) 132 return; 133 134 if (!enable_apicv) 135 return; 136 137 down_write(&vcpu->kvm->arch.apicv_update_lock); 138 139 if (auto_eoi_new) 140 hv->synic_auto_eoi_used++; 141 else 142 hv->synic_auto_eoi_used--; 143 144 /* 145 * Inhibit APICv if any vCPU is using SynIC's AutoEOI, which relies on 146 * the hypervisor to manually inject IRQs. 147 */ 148 __kvm_set_or_clear_apicv_inhibit(vcpu->kvm, 149 APICV_INHIBIT_REASON_HYPERV, 150 !!hv->synic_auto_eoi_used); 151 152 up_write(&vcpu->kvm->arch.apicv_update_lock); 153 } 154 155 static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint, 156 u64 data, bool host) 157 { 158 int vector, old_vector; 159 bool masked; 160 161 vector = data & HV_SYNIC_SINT_VECTOR_MASK; 162 masked = data & HV_SYNIC_SINT_MASKED; 163 164 /* 165 * Valid vectors are 16-255, however, nested Hyper-V attempts to write 166 * default '0x10000' value on boot and this should not #GP. We need to 167 * allow zero-initing the register from host as well. 168 */ 169 if (vector < HV_SYNIC_FIRST_VALID_VECTOR && !host && !masked) 170 return 1; 171 /* 172 * Guest may configure multiple SINTs to use the same vector, so 173 * we maintain a bitmap of vectors handled by synic, and a 174 * bitmap of vectors with auto-eoi behavior. The bitmaps are 175 * updated here, and atomically queried on fast paths. 176 */ 177 old_vector = synic_read_sint(synic, sint) & HV_SYNIC_SINT_VECTOR_MASK; 178 179 atomic64_set(&synic->sint[sint], data); 180 181 synic_update_vector(synic, old_vector); 182 183 synic_update_vector(synic, vector); 184 185 /* Load SynIC vectors into EOI exit bitmap */ 186 kvm_make_request(KVM_REQ_SCAN_IOAPIC, hv_synic_to_vcpu(synic)); 187 return 0; 188 } 189 190 static struct kvm_vcpu *get_vcpu_by_vpidx(struct kvm *kvm, u32 vpidx) 191 { 192 struct kvm_vcpu *vcpu = NULL; 193 unsigned long i; 194 195 if (vpidx >= KVM_MAX_VCPUS) 196 return NULL; 197 198 vcpu = kvm_get_vcpu(kvm, vpidx); 199 if (vcpu && kvm_hv_get_vpindex(vcpu) == vpidx) 200 return vcpu; 201 kvm_for_each_vcpu(i, vcpu, kvm) 202 if (kvm_hv_get_vpindex(vcpu) == vpidx) 203 return vcpu; 204 return NULL; 205 } 206 207 static struct kvm_vcpu_hv_synic *synic_get(struct kvm *kvm, u32 vpidx) 208 { 209 struct kvm_vcpu *vcpu; 210 struct kvm_vcpu_hv_synic *synic; 211 212 vcpu = get_vcpu_by_vpidx(kvm, vpidx); 213 if (!vcpu || !to_hv_vcpu(vcpu)) 214 return NULL; 215 synic = to_hv_synic(vcpu); 216 return (synic->active) ? synic : NULL; 217 } 218 219 static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint) 220 { 221 struct kvm *kvm = vcpu->kvm; 222 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 223 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 224 struct kvm_vcpu_hv_stimer *stimer; 225 int gsi, idx; 226 227 trace_kvm_hv_notify_acked_sint(vcpu->vcpu_id, sint); 228 229 /* Try to deliver pending Hyper-V SynIC timers messages */ 230 for (idx = 0; idx < ARRAY_SIZE(hv_vcpu->stimer); idx++) { 231 stimer = &hv_vcpu->stimer[idx]; 232 if (stimer->msg_pending && stimer->config.enable && 233 !stimer->config.direct_mode && 234 stimer->config.sintx == sint) 235 stimer_mark_pending(stimer, false); 236 } 237 238 idx = srcu_read_lock(&kvm->irq_srcu); 239 gsi = atomic_read(&synic->sint_to_gsi[sint]); 240 if (gsi != -1) 241 kvm_notify_acked_gsi(kvm, gsi); 242 srcu_read_unlock(&kvm->irq_srcu, idx); 243 } 244 245 static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr) 246 { 247 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 248 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 249 250 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNIC; 251 hv_vcpu->exit.u.synic.msr = msr; 252 hv_vcpu->exit.u.synic.control = synic->control; 253 hv_vcpu->exit.u.synic.evt_page = synic->evt_page; 254 hv_vcpu->exit.u.synic.msg_page = synic->msg_page; 255 256 kvm_make_request(KVM_REQ_HV_EXIT, vcpu); 257 } 258 259 static int synic_set_msr(struct kvm_vcpu_hv_synic *synic, 260 u32 msr, u64 data, bool host) 261 { 262 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 263 int ret; 264 265 if (!synic->active && (!host || data)) 266 return 1; 267 268 trace_kvm_hv_synic_set_msr(vcpu->vcpu_id, msr, data, host); 269 270 ret = 0; 271 switch (msr) { 272 case HV_X64_MSR_SCONTROL: 273 synic->control = data; 274 if (!host) 275 synic_exit(synic, msr); 276 break; 277 case HV_X64_MSR_SVERSION: 278 if (!host) { 279 ret = 1; 280 break; 281 } 282 synic->version = data; 283 break; 284 case HV_X64_MSR_SIEFP: 285 if ((data & HV_SYNIC_SIEFP_ENABLE) && !host && 286 !synic->dont_zero_synic_pages) 287 if (kvm_clear_guest(vcpu->kvm, 288 data & PAGE_MASK, PAGE_SIZE)) { 289 ret = 1; 290 break; 291 } 292 synic->evt_page = data; 293 if (!host) 294 synic_exit(synic, msr); 295 break; 296 case HV_X64_MSR_SIMP: 297 if ((data & HV_SYNIC_SIMP_ENABLE) && !host && 298 !synic->dont_zero_synic_pages) 299 if (kvm_clear_guest(vcpu->kvm, 300 data & PAGE_MASK, PAGE_SIZE)) { 301 ret = 1; 302 break; 303 } 304 synic->msg_page = data; 305 if (!host) 306 synic_exit(synic, msr); 307 break; 308 case HV_X64_MSR_EOM: { 309 int i; 310 311 if (!synic->active) 312 break; 313 314 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) 315 kvm_hv_notify_acked_sint(vcpu, i); 316 break; 317 } 318 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 319 ret = synic_set_sint(synic, msr - HV_X64_MSR_SINT0, data, host); 320 break; 321 default: 322 ret = 1; 323 break; 324 } 325 return ret; 326 } 327 328 static bool kvm_hv_is_syndbg_enabled(struct kvm_vcpu *vcpu) 329 { 330 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 331 332 return hv_vcpu->cpuid_cache.syndbg_cap_eax & 333 HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; 334 } 335 336 static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu) 337 { 338 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 339 340 if (vcpu->run->hyperv.u.syndbg.msr == HV_X64_MSR_SYNDBG_CONTROL) 341 hv->hv_syndbg.control.status = 342 vcpu->run->hyperv.u.syndbg.status; 343 return 1; 344 } 345 346 static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr) 347 { 348 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 349 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 350 351 hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNDBG; 352 hv_vcpu->exit.u.syndbg.msr = msr; 353 hv_vcpu->exit.u.syndbg.control = syndbg->control.control; 354 hv_vcpu->exit.u.syndbg.send_page = syndbg->control.send_page; 355 hv_vcpu->exit.u.syndbg.recv_page = syndbg->control.recv_page; 356 hv_vcpu->exit.u.syndbg.pending_page = syndbg->control.pending_page; 357 vcpu->arch.complete_userspace_io = 358 kvm_hv_syndbg_complete_userspace; 359 360 kvm_make_request(KVM_REQ_HV_EXIT, vcpu); 361 } 362 363 static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 364 { 365 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 366 367 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) 368 return 1; 369 370 trace_kvm_hv_syndbg_set_msr(vcpu->vcpu_id, 371 to_hv_vcpu(vcpu)->vp_index, msr, data); 372 switch (msr) { 373 case HV_X64_MSR_SYNDBG_CONTROL: 374 syndbg->control.control = data; 375 if (!host) 376 syndbg_exit(vcpu, msr); 377 break; 378 case HV_X64_MSR_SYNDBG_STATUS: 379 syndbg->control.status = data; 380 break; 381 case HV_X64_MSR_SYNDBG_SEND_BUFFER: 382 syndbg->control.send_page = data; 383 break; 384 case HV_X64_MSR_SYNDBG_RECV_BUFFER: 385 syndbg->control.recv_page = data; 386 break; 387 case HV_X64_MSR_SYNDBG_PENDING_BUFFER: 388 syndbg->control.pending_page = data; 389 if (!host) 390 syndbg_exit(vcpu, msr); 391 break; 392 case HV_X64_MSR_SYNDBG_OPTIONS: 393 syndbg->options = data; 394 break; 395 default: 396 break; 397 } 398 399 return 0; 400 } 401 402 static int syndbg_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 403 { 404 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 405 406 if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) 407 return 1; 408 409 switch (msr) { 410 case HV_X64_MSR_SYNDBG_CONTROL: 411 *pdata = syndbg->control.control; 412 break; 413 case HV_X64_MSR_SYNDBG_STATUS: 414 *pdata = syndbg->control.status; 415 break; 416 case HV_X64_MSR_SYNDBG_SEND_BUFFER: 417 *pdata = syndbg->control.send_page; 418 break; 419 case HV_X64_MSR_SYNDBG_RECV_BUFFER: 420 *pdata = syndbg->control.recv_page; 421 break; 422 case HV_X64_MSR_SYNDBG_PENDING_BUFFER: 423 *pdata = syndbg->control.pending_page; 424 break; 425 case HV_X64_MSR_SYNDBG_OPTIONS: 426 *pdata = syndbg->options; 427 break; 428 default: 429 break; 430 } 431 432 trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id, kvm_hv_get_vpindex(vcpu), msr, *pdata); 433 434 return 0; 435 } 436 437 static int synic_get_msr(struct kvm_vcpu_hv_synic *synic, u32 msr, u64 *pdata, 438 bool host) 439 { 440 int ret; 441 442 if (!synic->active && !host) 443 return 1; 444 445 ret = 0; 446 switch (msr) { 447 case HV_X64_MSR_SCONTROL: 448 *pdata = synic->control; 449 break; 450 case HV_X64_MSR_SVERSION: 451 *pdata = synic->version; 452 break; 453 case HV_X64_MSR_SIEFP: 454 *pdata = synic->evt_page; 455 break; 456 case HV_X64_MSR_SIMP: 457 *pdata = synic->msg_page; 458 break; 459 case HV_X64_MSR_EOM: 460 *pdata = 0; 461 break; 462 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 463 *pdata = atomic64_read(&synic->sint[msr - HV_X64_MSR_SINT0]); 464 break; 465 default: 466 ret = 1; 467 break; 468 } 469 return ret; 470 } 471 472 static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint) 473 { 474 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 475 struct kvm_lapic_irq irq; 476 int ret, vector; 477 478 if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 479 return -EINVAL; 480 481 if (sint >= ARRAY_SIZE(synic->sint)) 482 return -EINVAL; 483 484 vector = synic_get_sint_vector(synic_read_sint(synic, sint)); 485 if (vector < 0) 486 return -ENOENT; 487 488 memset(&irq, 0, sizeof(irq)); 489 irq.shorthand = APIC_DEST_SELF; 490 irq.dest_mode = APIC_DEST_PHYSICAL; 491 irq.delivery_mode = APIC_DM_FIXED; 492 irq.vector = vector; 493 irq.level = 1; 494 495 ret = kvm_irq_delivery_to_apic(vcpu->kvm, vcpu->arch.apic, &irq, NULL); 496 trace_kvm_hv_synic_set_irq(vcpu->vcpu_id, sint, irq.vector, ret); 497 return ret; 498 } 499 500 int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vpidx, u32 sint) 501 { 502 struct kvm_vcpu_hv_synic *synic; 503 504 synic = synic_get(kvm, vpidx); 505 if (!synic) 506 return -EINVAL; 507 508 return synic_set_irq(synic, sint); 509 } 510 511 void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector) 512 { 513 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 514 int i; 515 516 trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector); 517 518 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) 519 if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector) 520 kvm_hv_notify_acked_sint(vcpu, i); 521 } 522 523 static int kvm_hv_set_sint_gsi(struct kvm *kvm, u32 vpidx, u32 sint, int gsi) 524 { 525 struct kvm_vcpu_hv_synic *synic; 526 527 synic = synic_get(kvm, vpidx); 528 if (!synic) 529 return -EINVAL; 530 531 if (sint >= ARRAY_SIZE(synic->sint_to_gsi)) 532 return -EINVAL; 533 534 atomic_set(&synic->sint_to_gsi[sint], gsi); 535 return 0; 536 } 537 538 void kvm_hv_irq_routing_update(struct kvm *kvm) 539 { 540 struct kvm_irq_routing_table *irq_rt; 541 struct kvm_kernel_irq_routing_entry *e; 542 u32 gsi; 543 544 irq_rt = srcu_dereference_check(kvm->irq_routing, &kvm->irq_srcu, 545 lockdep_is_held(&kvm->irq_lock)); 546 547 for (gsi = 0; gsi < irq_rt->nr_rt_entries; gsi++) { 548 hlist_for_each_entry(e, &irq_rt->map[gsi], link) { 549 if (e->type == KVM_IRQ_ROUTING_HV_SINT) 550 kvm_hv_set_sint_gsi(kvm, e->hv_sint.vcpu, 551 e->hv_sint.sint, gsi); 552 } 553 } 554 } 555 556 static void synic_init(struct kvm_vcpu_hv_synic *synic) 557 { 558 int i; 559 560 memset(synic, 0, sizeof(*synic)); 561 synic->version = HV_SYNIC_VERSION_1; 562 for (i = 0; i < ARRAY_SIZE(synic->sint); i++) { 563 atomic64_set(&synic->sint[i], HV_SYNIC_SINT_MASKED); 564 atomic_set(&synic->sint_to_gsi[i], -1); 565 } 566 } 567 568 static u64 get_time_ref_counter(struct kvm *kvm) 569 { 570 struct kvm_hv *hv = to_kvm_hv(kvm); 571 struct kvm_vcpu *vcpu; 572 u64 tsc; 573 574 /* 575 * Fall back to get_kvmclock_ns() when TSC page hasn't been set up, 576 * is broken, disabled or being updated. 577 */ 578 if (hv->hv_tsc_page_status != HV_TSC_PAGE_SET) 579 return div_u64(get_kvmclock_ns(kvm), 100); 580 581 vcpu = kvm_get_vcpu(kvm, 0); 582 tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 583 return mul_u64_u64_shr(tsc, hv->tsc_ref.tsc_scale, 64) 584 + hv->tsc_ref.tsc_offset; 585 } 586 587 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, 588 bool vcpu_kick) 589 { 590 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 591 592 set_bit(stimer->index, 593 to_hv_vcpu(vcpu)->stimer_pending_bitmap); 594 kvm_make_request(KVM_REQ_HV_STIMER, vcpu); 595 if (vcpu_kick) 596 kvm_vcpu_kick(vcpu); 597 } 598 599 static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer) 600 { 601 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 602 603 trace_kvm_hv_stimer_cleanup(hv_stimer_to_vcpu(stimer)->vcpu_id, 604 stimer->index); 605 606 hrtimer_cancel(&stimer->timer); 607 clear_bit(stimer->index, 608 to_hv_vcpu(vcpu)->stimer_pending_bitmap); 609 stimer->msg_pending = false; 610 stimer->exp_time = 0; 611 } 612 613 static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer) 614 { 615 struct kvm_vcpu_hv_stimer *stimer; 616 617 stimer = container_of(timer, struct kvm_vcpu_hv_stimer, timer); 618 trace_kvm_hv_stimer_callback(hv_stimer_to_vcpu(stimer)->vcpu_id, 619 stimer->index); 620 stimer_mark_pending(stimer, true); 621 622 return HRTIMER_NORESTART; 623 } 624 625 /* 626 * stimer_start() assumptions: 627 * a) stimer->count is not equal to 0 628 * b) stimer->config has HV_STIMER_ENABLE flag 629 */ 630 static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) 631 { 632 u64 time_now; 633 ktime_t ktime_now; 634 635 time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm); 636 ktime_now = ktime_get(); 637 638 if (stimer->config.periodic) { 639 if (stimer->exp_time) { 640 if (time_now >= stimer->exp_time) { 641 u64 remainder; 642 643 div64_u64_rem(time_now - stimer->exp_time, 644 stimer->count, &remainder); 645 stimer->exp_time = 646 time_now + (stimer->count - remainder); 647 } 648 } else 649 stimer->exp_time = time_now + stimer->count; 650 651 trace_kvm_hv_stimer_start_periodic( 652 hv_stimer_to_vcpu(stimer)->vcpu_id, 653 stimer->index, 654 time_now, stimer->exp_time); 655 656 hrtimer_start(&stimer->timer, 657 ktime_add_ns(ktime_now, 658 100 * (stimer->exp_time - time_now)), 659 HRTIMER_MODE_ABS); 660 return 0; 661 } 662 stimer->exp_time = stimer->count; 663 if (time_now >= stimer->count) { 664 /* 665 * Expire timer according to Hypervisor Top-Level Functional 666 * specification v4(15.3.1): 667 * "If a one shot is enabled and the specified count is in 668 * the past, it will expire immediately." 669 */ 670 stimer_mark_pending(stimer, false); 671 return 0; 672 } 673 674 trace_kvm_hv_stimer_start_one_shot(hv_stimer_to_vcpu(stimer)->vcpu_id, 675 stimer->index, 676 time_now, stimer->count); 677 678 hrtimer_start(&stimer->timer, 679 ktime_add_ns(ktime_now, 100 * (stimer->count - time_now)), 680 HRTIMER_MODE_ABS); 681 return 0; 682 } 683 684 static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, 685 bool host) 686 { 687 union hv_stimer_config new_config = {.as_uint64 = config}, 688 old_config = {.as_uint64 = stimer->config.as_uint64}; 689 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 690 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 691 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 692 693 if (!synic->active && (!host || config)) 694 return 1; 695 696 if (unlikely(!host && hv_vcpu->enforce_cpuid && new_config.direct_mode && 697 !(hv_vcpu->cpuid_cache.features_edx & 698 HV_STIMER_DIRECT_MODE_AVAILABLE))) 699 return 1; 700 701 trace_kvm_hv_stimer_set_config(hv_stimer_to_vcpu(stimer)->vcpu_id, 702 stimer->index, config, host); 703 704 stimer_cleanup(stimer); 705 if (old_config.enable && 706 !new_config.direct_mode && new_config.sintx == 0) 707 new_config.enable = 0; 708 stimer->config.as_uint64 = new_config.as_uint64; 709 710 if (stimer->config.enable) 711 stimer_mark_pending(stimer, false); 712 713 return 0; 714 } 715 716 static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count, 717 bool host) 718 { 719 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 720 struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); 721 722 if (!synic->active && (!host || count)) 723 return 1; 724 725 trace_kvm_hv_stimer_set_count(hv_stimer_to_vcpu(stimer)->vcpu_id, 726 stimer->index, count, host); 727 728 stimer_cleanup(stimer); 729 stimer->count = count; 730 if (!host) { 731 if (stimer->count == 0) 732 stimer->config.enable = 0; 733 else if (stimer->config.auto_enable) 734 stimer->config.enable = 1; 735 } 736 737 if (stimer->config.enable) 738 stimer_mark_pending(stimer, false); 739 740 return 0; 741 } 742 743 static int stimer_get_config(struct kvm_vcpu_hv_stimer *stimer, u64 *pconfig) 744 { 745 *pconfig = stimer->config.as_uint64; 746 return 0; 747 } 748 749 static int stimer_get_count(struct kvm_vcpu_hv_stimer *stimer, u64 *pcount) 750 { 751 *pcount = stimer->count; 752 return 0; 753 } 754 755 static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, 756 struct hv_message *src_msg, bool no_retry) 757 { 758 struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); 759 int msg_off = offsetof(struct hv_message_page, sint_message[sint]); 760 gfn_t msg_page_gfn; 761 struct hv_message_header hv_hdr; 762 int r; 763 764 if (!(synic->msg_page & HV_SYNIC_SIMP_ENABLE)) 765 return -ENOENT; 766 767 msg_page_gfn = synic->msg_page >> PAGE_SHIFT; 768 769 /* 770 * Strictly following the spec-mandated ordering would assume setting 771 * .msg_pending before checking .message_type. However, this function 772 * is only called in vcpu context so the entire update is atomic from 773 * guest POV and thus the exact order here doesn't matter. 774 */ 775 r = kvm_vcpu_read_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_type, 776 msg_off + offsetof(struct hv_message, 777 header.message_type), 778 sizeof(hv_hdr.message_type)); 779 if (r < 0) 780 return r; 781 782 if (hv_hdr.message_type != HVMSG_NONE) { 783 if (no_retry) 784 return 0; 785 786 hv_hdr.message_flags.msg_pending = 1; 787 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, 788 &hv_hdr.message_flags, 789 msg_off + 790 offsetof(struct hv_message, 791 header.message_flags), 792 sizeof(hv_hdr.message_flags)); 793 if (r < 0) 794 return r; 795 return -EAGAIN; 796 } 797 798 r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, src_msg, msg_off, 799 sizeof(src_msg->header) + 800 src_msg->header.payload_size); 801 if (r < 0) 802 return r; 803 804 r = synic_set_irq(synic, sint); 805 if (r < 0) 806 return r; 807 if (r == 0) 808 return -EFAULT; 809 return 0; 810 } 811 812 static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer) 813 { 814 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 815 struct hv_message *msg = &stimer->msg; 816 struct hv_timer_message_payload *payload = 817 (struct hv_timer_message_payload *)&msg->u.payload; 818 819 /* 820 * To avoid piling up periodic ticks, don't retry message 821 * delivery for them (within "lazy" lost ticks policy). 822 */ 823 bool no_retry = stimer->config.periodic; 824 825 payload->expiration_time = stimer->exp_time; 826 payload->delivery_time = get_time_ref_counter(vcpu->kvm); 827 return synic_deliver_msg(to_hv_synic(vcpu), 828 stimer->config.sintx, msg, 829 no_retry); 830 } 831 832 static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer) 833 { 834 struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); 835 struct kvm_lapic_irq irq = { 836 .delivery_mode = APIC_DM_FIXED, 837 .vector = stimer->config.apic_vector 838 }; 839 840 if (lapic_in_kernel(vcpu)) 841 return !kvm_apic_set_irq(vcpu, &irq, NULL); 842 return 0; 843 } 844 845 static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer) 846 { 847 int r, direct = stimer->config.direct_mode; 848 849 stimer->msg_pending = true; 850 if (!direct) 851 r = stimer_send_msg(stimer); 852 else 853 r = stimer_notify_direct(stimer); 854 trace_kvm_hv_stimer_expiration(hv_stimer_to_vcpu(stimer)->vcpu_id, 855 stimer->index, direct, r); 856 if (!r) { 857 stimer->msg_pending = false; 858 if (!(stimer->config.periodic)) 859 stimer->config.enable = 0; 860 } 861 } 862 863 void kvm_hv_process_stimers(struct kvm_vcpu *vcpu) 864 { 865 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 866 struct kvm_vcpu_hv_stimer *stimer; 867 u64 time_now, exp_time; 868 int i; 869 870 if (!hv_vcpu) 871 return; 872 873 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 874 if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) { 875 stimer = &hv_vcpu->stimer[i]; 876 if (stimer->config.enable) { 877 exp_time = stimer->exp_time; 878 879 if (exp_time) { 880 time_now = 881 get_time_ref_counter(vcpu->kvm); 882 if (time_now >= exp_time) 883 stimer_expiration(stimer); 884 } 885 886 if ((stimer->config.enable) && 887 stimer->count) { 888 if (!stimer->msg_pending) 889 stimer_start(stimer); 890 } else 891 stimer_cleanup(stimer); 892 } 893 } 894 } 895 896 void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu) 897 { 898 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 899 int i; 900 901 if (!hv_vcpu) 902 return; 903 904 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 905 stimer_cleanup(&hv_vcpu->stimer[i]); 906 907 kfree(hv_vcpu); 908 vcpu->arch.hyperv = NULL; 909 } 910 911 bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu) 912 { 913 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 914 915 if (!hv_vcpu) 916 return false; 917 918 if (!(hv_vcpu->hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) 919 return false; 920 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 921 } 922 EXPORT_SYMBOL_GPL(kvm_hv_assist_page_enabled); 923 924 int kvm_hv_get_assist_page(struct kvm_vcpu *vcpu) 925 { 926 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 927 928 if (!hv_vcpu || !kvm_hv_assist_page_enabled(vcpu)) 929 return -EFAULT; 930 931 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, 932 &hv_vcpu->vp_assist_page, sizeof(struct hv_vp_assist_page)); 933 } 934 EXPORT_SYMBOL_GPL(kvm_hv_get_assist_page); 935 936 static void stimer_prepare_msg(struct kvm_vcpu_hv_stimer *stimer) 937 { 938 struct hv_message *msg = &stimer->msg; 939 struct hv_timer_message_payload *payload = 940 (struct hv_timer_message_payload *)&msg->u.payload; 941 942 memset(&msg->header, 0, sizeof(msg->header)); 943 msg->header.message_type = HVMSG_TIMER_EXPIRED; 944 msg->header.payload_size = sizeof(*payload); 945 946 payload->timer_index = stimer->index; 947 payload->expiration_time = 0; 948 payload->delivery_time = 0; 949 } 950 951 static void stimer_init(struct kvm_vcpu_hv_stimer *stimer, int timer_index) 952 { 953 memset(stimer, 0, sizeof(*stimer)); 954 stimer->index = timer_index; 955 hrtimer_init(&stimer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 956 stimer->timer.function = stimer_timer_callback; 957 stimer_prepare_msg(stimer); 958 } 959 960 int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) 961 { 962 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 963 int i; 964 965 if (hv_vcpu) 966 return 0; 967 968 hv_vcpu = kzalloc(sizeof(struct kvm_vcpu_hv), GFP_KERNEL_ACCOUNT); 969 if (!hv_vcpu) 970 return -ENOMEM; 971 972 vcpu->arch.hyperv = hv_vcpu; 973 hv_vcpu->vcpu = vcpu; 974 975 synic_init(&hv_vcpu->synic); 976 977 bitmap_zero(hv_vcpu->stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); 978 for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) 979 stimer_init(&hv_vcpu->stimer[i], i); 980 981 hv_vcpu->vp_index = vcpu->vcpu_idx; 982 983 for (i = 0; i < HV_NR_TLB_FLUSH_FIFOS; i++) { 984 INIT_KFIFO(hv_vcpu->tlb_flush_fifo[i].entries); 985 spin_lock_init(&hv_vcpu->tlb_flush_fifo[i].write_lock); 986 } 987 988 return 0; 989 } 990 991 int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages) 992 { 993 struct kvm_vcpu_hv_synic *synic; 994 int r; 995 996 r = kvm_hv_vcpu_init(vcpu); 997 if (r) 998 return r; 999 1000 synic = to_hv_synic(vcpu); 1001 1002 synic->active = true; 1003 synic->dont_zero_synic_pages = dont_zero_synic_pages; 1004 synic->control = HV_SYNIC_CONTROL_ENABLE; 1005 return 0; 1006 } 1007 1008 static bool kvm_hv_msr_partition_wide(u32 msr) 1009 { 1010 bool r = false; 1011 1012 switch (msr) { 1013 case HV_X64_MSR_GUEST_OS_ID: 1014 case HV_X64_MSR_HYPERCALL: 1015 case HV_X64_MSR_REFERENCE_TSC: 1016 case HV_X64_MSR_TIME_REF_COUNT: 1017 case HV_X64_MSR_CRASH_CTL: 1018 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1019 case HV_X64_MSR_RESET: 1020 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1021 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1022 case HV_X64_MSR_TSC_EMULATION_STATUS: 1023 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1024 case HV_X64_MSR_SYNDBG_OPTIONS: 1025 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1026 r = true; 1027 break; 1028 } 1029 1030 return r; 1031 } 1032 1033 static int kvm_hv_msr_get_crash_data(struct kvm *kvm, u32 index, u64 *pdata) 1034 { 1035 struct kvm_hv *hv = to_kvm_hv(kvm); 1036 size_t size = ARRAY_SIZE(hv->hv_crash_param); 1037 1038 if (WARN_ON_ONCE(index >= size)) 1039 return -EINVAL; 1040 1041 *pdata = hv->hv_crash_param[array_index_nospec(index, size)]; 1042 return 0; 1043 } 1044 1045 static int kvm_hv_msr_get_crash_ctl(struct kvm *kvm, u64 *pdata) 1046 { 1047 struct kvm_hv *hv = to_kvm_hv(kvm); 1048 1049 *pdata = hv->hv_crash_ctl; 1050 return 0; 1051 } 1052 1053 static int kvm_hv_msr_set_crash_ctl(struct kvm *kvm, u64 data) 1054 { 1055 struct kvm_hv *hv = to_kvm_hv(kvm); 1056 1057 hv->hv_crash_ctl = data & HV_CRASH_CTL_CRASH_NOTIFY; 1058 1059 return 0; 1060 } 1061 1062 static int kvm_hv_msr_set_crash_data(struct kvm *kvm, u32 index, u64 data) 1063 { 1064 struct kvm_hv *hv = to_kvm_hv(kvm); 1065 size_t size = ARRAY_SIZE(hv->hv_crash_param); 1066 1067 if (WARN_ON_ONCE(index >= size)) 1068 return -EINVAL; 1069 1070 hv->hv_crash_param[array_index_nospec(index, size)] = data; 1071 return 0; 1072 } 1073 1074 /* 1075 * The kvmclock and Hyper-V TSC page use similar formulas, and converting 1076 * between them is possible: 1077 * 1078 * kvmclock formula: 1079 * nsec = (ticks - tsc_timestamp) * tsc_to_system_mul * 2^(tsc_shift-32) 1080 * + system_time 1081 * 1082 * Hyper-V formula: 1083 * nsec/100 = ticks * scale / 2^64 + offset 1084 * 1085 * When tsc_timestamp = system_time = 0, offset is zero in the Hyper-V formula. 1086 * By dividing the kvmclock formula by 100 and equating what's left we get: 1087 * ticks * scale / 2^64 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1088 * scale / 2^64 = tsc_to_system_mul * 2^(tsc_shift-32) / 100 1089 * scale = tsc_to_system_mul * 2^(32+tsc_shift) / 100 1090 * 1091 * Now expand the kvmclock formula and divide by 100: 1092 * nsec = ticks * tsc_to_system_mul * 2^(tsc_shift-32) 1093 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) 1094 * + system_time 1095 * nsec/100 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1096 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) / 100 1097 * + system_time / 100 1098 * 1099 * Replace tsc_to_system_mul * 2^(tsc_shift-32) / 100 by scale / 2^64: 1100 * nsec/100 = ticks * scale / 2^64 1101 * - tsc_timestamp * scale / 2^64 1102 * + system_time / 100 1103 * 1104 * Equate with the Hyper-V formula so that ticks * scale / 2^64 cancels out: 1105 * offset = system_time / 100 - tsc_timestamp * scale / 2^64 1106 * 1107 * These two equivalencies are implemented in this function. 1108 */ 1109 static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock, 1110 struct ms_hyperv_tsc_page *tsc_ref) 1111 { 1112 u64 max_mul; 1113 1114 if (!(hv_clock->flags & PVCLOCK_TSC_STABLE_BIT)) 1115 return false; 1116 1117 /* 1118 * check if scale would overflow, if so we use the time ref counter 1119 * tsc_to_system_mul * 2^(tsc_shift+32) / 100 >= 2^64 1120 * tsc_to_system_mul / 100 >= 2^(32-tsc_shift) 1121 * tsc_to_system_mul >= 100 * 2^(32-tsc_shift) 1122 */ 1123 max_mul = 100ull << (32 - hv_clock->tsc_shift); 1124 if (hv_clock->tsc_to_system_mul >= max_mul) 1125 return false; 1126 1127 /* 1128 * Otherwise compute the scale and offset according to the formulas 1129 * derived above. 1130 */ 1131 tsc_ref->tsc_scale = 1132 mul_u64_u32_div(1ULL << (32 + hv_clock->tsc_shift), 1133 hv_clock->tsc_to_system_mul, 1134 100); 1135 1136 tsc_ref->tsc_offset = hv_clock->system_time; 1137 do_div(tsc_ref->tsc_offset, 100); 1138 tsc_ref->tsc_offset -= 1139 mul_u64_u64_shr(hv_clock->tsc_timestamp, tsc_ref->tsc_scale, 64); 1140 return true; 1141 } 1142 1143 /* 1144 * Don't touch TSC page values if the guest has opted for TSC emulation after 1145 * migration. KVM doesn't fully support reenlightenment notifications and TSC 1146 * access emulation and Hyper-V is known to expect the values in TSC page to 1147 * stay constant before TSC access emulation is disabled from guest side 1148 * (HV_X64_MSR_TSC_EMULATION_STATUS). KVM userspace is expected to preserve TSC 1149 * frequency and guest visible TSC value across migration (and prevent it when 1150 * TSC scaling is unsupported). 1151 */ 1152 static inline bool tsc_page_update_unsafe(struct kvm_hv *hv) 1153 { 1154 return (hv->hv_tsc_page_status != HV_TSC_PAGE_GUEST_CHANGED) && 1155 hv->hv_tsc_emulation_control; 1156 } 1157 1158 void kvm_hv_setup_tsc_page(struct kvm *kvm, 1159 struct pvclock_vcpu_time_info *hv_clock) 1160 { 1161 struct kvm_hv *hv = to_kvm_hv(kvm); 1162 u32 tsc_seq; 1163 u64 gfn; 1164 1165 BUILD_BUG_ON(sizeof(tsc_seq) != sizeof(hv->tsc_ref.tsc_sequence)); 1166 BUILD_BUG_ON(offsetof(struct ms_hyperv_tsc_page, tsc_sequence) != 0); 1167 1168 mutex_lock(&hv->hv_lock); 1169 1170 if (hv->hv_tsc_page_status == HV_TSC_PAGE_BROKEN || 1171 hv->hv_tsc_page_status == HV_TSC_PAGE_SET || 1172 hv->hv_tsc_page_status == HV_TSC_PAGE_UNSET) 1173 goto out_unlock; 1174 1175 if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE)) 1176 goto out_unlock; 1177 1178 gfn = hv->hv_tsc_page >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT; 1179 /* 1180 * Because the TSC parameters only vary when there is a 1181 * change in the master clock, do not bother with caching. 1182 */ 1183 if (unlikely(kvm_read_guest(kvm, gfn_to_gpa(gfn), 1184 &tsc_seq, sizeof(tsc_seq)))) 1185 goto out_err; 1186 1187 if (tsc_seq && tsc_page_update_unsafe(hv)) { 1188 if (kvm_read_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref))) 1189 goto out_err; 1190 1191 hv->hv_tsc_page_status = HV_TSC_PAGE_SET; 1192 goto out_unlock; 1193 } 1194 1195 /* 1196 * While we're computing and writing the parameters, force the 1197 * guest to use the time reference count MSR. 1198 */ 1199 hv->tsc_ref.tsc_sequence = 0; 1200 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), 1201 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence))) 1202 goto out_err; 1203 1204 if (!compute_tsc_page_parameters(hv_clock, &hv->tsc_ref)) 1205 goto out_err; 1206 1207 /* Ensure sequence is zero before writing the rest of the struct. */ 1208 smp_wmb(); 1209 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref))) 1210 goto out_err; 1211 1212 /* 1213 * Now switch to the TSC page mechanism by writing the sequence. 1214 */ 1215 tsc_seq++; 1216 if (tsc_seq == 0xFFFFFFFF || tsc_seq == 0) 1217 tsc_seq = 1; 1218 1219 /* Write the struct entirely before the non-zero sequence. */ 1220 smp_wmb(); 1221 1222 hv->tsc_ref.tsc_sequence = tsc_seq; 1223 if (kvm_write_guest(kvm, gfn_to_gpa(gfn), 1224 &hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence))) 1225 goto out_err; 1226 1227 hv->hv_tsc_page_status = HV_TSC_PAGE_SET; 1228 goto out_unlock; 1229 1230 out_err: 1231 hv->hv_tsc_page_status = HV_TSC_PAGE_BROKEN; 1232 out_unlock: 1233 mutex_unlock(&hv->hv_lock); 1234 } 1235 1236 void kvm_hv_request_tsc_page_update(struct kvm *kvm) 1237 { 1238 struct kvm_hv *hv = to_kvm_hv(kvm); 1239 1240 mutex_lock(&hv->hv_lock); 1241 1242 if (hv->hv_tsc_page_status == HV_TSC_PAGE_SET && 1243 !tsc_page_update_unsafe(hv)) 1244 hv->hv_tsc_page_status = HV_TSC_PAGE_HOST_CHANGED; 1245 1246 mutex_unlock(&hv->hv_lock); 1247 } 1248 1249 static bool hv_check_msr_access(struct kvm_vcpu_hv *hv_vcpu, u32 msr) 1250 { 1251 if (!hv_vcpu->enforce_cpuid) 1252 return true; 1253 1254 switch (msr) { 1255 case HV_X64_MSR_GUEST_OS_ID: 1256 case HV_X64_MSR_HYPERCALL: 1257 return hv_vcpu->cpuid_cache.features_eax & 1258 HV_MSR_HYPERCALL_AVAILABLE; 1259 case HV_X64_MSR_VP_RUNTIME: 1260 return hv_vcpu->cpuid_cache.features_eax & 1261 HV_MSR_VP_RUNTIME_AVAILABLE; 1262 case HV_X64_MSR_TIME_REF_COUNT: 1263 return hv_vcpu->cpuid_cache.features_eax & 1264 HV_MSR_TIME_REF_COUNT_AVAILABLE; 1265 case HV_X64_MSR_VP_INDEX: 1266 return hv_vcpu->cpuid_cache.features_eax & 1267 HV_MSR_VP_INDEX_AVAILABLE; 1268 case HV_X64_MSR_RESET: 1269 return hv_vcpu->cpuid_cache.features_eax & 1270 HV_MSR_RESET_AVAILABLE; 1271 case HV_X64_MSR_REFERENCE_TSC: 1272 return hv_vcpu->cpuid_cache.features_eax & 1273 HV_MSR_REFERENCE_TSC_AVAILABLE; 1274 case HV_X64_MSR_SCONTROL: 1275 case HV_X64_MSR_SVERSION: 1276 case HV_X64_MSR_SIEFP: 1277 case HV_X64_MSR_SIMP: 1278 case HV_X64_MSR_EOM: 1279 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1280 return hv_vcpu->cpuid_cache.features_eax & 1281 HV_MSR_SYNIC_AVAILABLE; 1282 case HV_X64_MSR_STIMER0_CONFIG: 1283 case HV_X64_MSR_STIMER1_CONFIG: 1284 case HV_X64_MSR_STIMER2_CONFIG: 1285 case HV_X64_MSR_STIMER3_CONFIG: 1286 case HV_X64_MSR_STIMER0_COUNT: 1287 case HV_X64_MSR_STIMER1_COUNT: 1288 case HV_X64_MSR_STIMER2_COUNT: 1289 case HV_X64_MSR_STIMER3_COUNT: 1290 return hv_vcpu->cpuid_cache.features_eax & 1291 HV_MSR_SYNTIMER_AVAILABLE; 1292 case HV_X64_MSR_EOI: 1293 case HV_X64_MSR_ICR: 1294 case HV_X64_MSR_TPR: 1295 case HV_X64_MSR_VP_ASSIST_PAGE: 1296 return hv_vcpu->cpuid_cache.features_eax & 1297 HV_MSR_APIC_ACCESS_AVAILABLE; 1298 case HV_X64_MSR_TSC_FREQUENCY: 1299 case HV_X64_MSR_APIC_FREQUENCY: 1300 return hv_vcpu->cpuid_cache.features_eax & 1301 HV_ACCESS_FREQUENCY_MSRS; 1302 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1303 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1304 case HV_X64_MSR_TSC_EMULATION_STATUS: 1305 return hv_vcpu->cpuid_cache.features_eax & 1306 HV_ACCESS_REENLIGHTENMENT; 1307 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1308 return hv_vcpu->cpuid_cache.features_eax & 1309 HV_ACCESS_TSC_INVARIANT; 1310 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1311 case HV_X64_MSR_CRASH_CTL: 1312 return hv_vcpu->cpuid_cache.features_edx & 1313 HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; 1314 case HV_X64_MSR_SYNDBG_OPTIONS: 1315 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1316 return hv_vcpu->cpuid_cache.features_edx & 1317 HV_FEATURE_DEBUG_MSRS_AVAILABLE; 1318 default: 1319 break; 1320 } 1321 1322 return false; 1323 } 1324 1325 static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, 1326 bool host) 1327 { 1328 struct kvm *kvm = vcpu->kvm; 1329 struct kvm_hv *hv = to_kvm_hv(kvm); 1330 1331 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) 1332 return 1; 1333 1334 switch (msr) { 1335 case HV_X64_MSR_GUEST_OS_ID: 1336 hv->hv_guest_os_id = data; 1337 /* setting guest os id to zero disables hypercall page */ 1338 if (!hv->hv_guest_os_id) 1339 hv->hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE; 1340 break; 1341 case HV_X64_MSR_HYPERCALL: { 1342 u8 instructions[9]; 1343 int i = 0; 1344 u64 addr; 1345 1346 /* if guest os id is not set hypercall should remain disabled */ 1347 if (!hv->hv_guest_os_id) 1348 break; 1349 if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) { 1350 hv->hv_hypercall = data; 1351 break; 1352 } 1353 1354 /* 1355 * If Xen and Hyper-V hypercalls are both enabled, disambiguate 1356 * the same way Xen itself does, by setting the bit 31 of EAX 1357 * which is RsvdZ in the 32-bit Hyper-V hypercall ABI and just 1358 * going to be clobbered on 64-bit. 1359 */ 1360 if (kvm_xen_hypercall_enabled(kvm)) { 1361 /* orl $0x80000000, %eax */ 1362 instructions[i++] = 0x0d; 1363 instructions[i++] = 0x00; 1364 instructions[i++] = 0x00; 1365 instructions[i++] = 0x00; 1366 instructions[i++] = 0x80; 1367 } 1368 1369 /* vmcall/vmmcall */ 1370 static_call(kvm_x86_patch_hypercall)(vcpu, instructions + i); 1371 i += 3; 1372 1373 /* ret */ 1374 ((unsigned char *)instructions)[i++] = 0xc3; 1375 1376 addr = data & HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK; 1377 if (kvm_vcpu_write_guest(vcpu, addr, instructions, i)) 1378 return 1; 1379 hv->hv_hypercall = data; 1380 break; 1381 } 1382 case HV_X64_MSR_REFERENCE_TSC: 1383 hv->hv_tsc_page = data; 1384 if (hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE) { 1385 if (!host) 1386 hv->hv_tsc_page_status = HV_TSC_PAGE_GUEST_CHANGED; 1387 else 1388 hv->hv_tsc_page_status = HV_TSC_PAGE_HOST_CHANGED; 1389 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 1390 } else { 1391 hv->hv_tsc_page_status = HV_TSC_PAGE_UNSET; 1392 } 1393 break; 1394 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1395 return kvm_hv_msr_set_crash_data(kvm, 1396 msr - HV_X64_MSR_CRASH_P0, 1397 data); 1398 case HV_X64_MSR_CRASH_CTL: 1399 if (host) 1400 return kvm_hv_msr_set_crash_ctl(kvm, data); 1401 1402 if (data & HV_CRASH_CTL_CRASH_NOTIFY) { 1403 vcpu_debug(vcpu, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n", 1404 hv->hv_crash_param[0], 1405 hv->hv_crash_param[1], 1406 hv->hv_crash_param[2], 1407 hv->hv_crash_param[3], 1408 hv->hv_crash_param[4]); 1409 1410 /* Send notification about crash to user space */ 1411 kvm_make_request(KVM_REQ_HV_CRASH, vcpu); 1412 } 1413 break; 1414 case HV_X64_MSR_RESET: 1415 if (data == 1) { 1416 vcpu_debug(vcpu, "hyper-v reset requested\n"); 1417 kvm_make_request(KVM_REQ_HV_RESET, vcpu); 1418 } 1419 break; 1420 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1421 hv->hv_reenlightenment_control = data; 1422 break; 1423 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1424 hv->hv_tsc_emulation_control = data; 1425 break; 1426 case HV_X64_MSR_TSC_EMULATION_STATUS: 1427 if (data && !host) 1428 return 1; 1429 1430 hv->hv_tsc_emulation_status = data; 1431 break; 1432 case HV_X64_MSR_TIME_REF_COUNT: 1433 /* read-only, but still ignore it if host-initiated */ 1434 if (!host) 1435 return 1; 1436 break; 1437 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1438 /* Only bit 0 is supported */ 1439 if (data & ~HV_EXPOSE_INVARIANT_TSC) 1440 return 1; 1441 1442 /* The feature can't be disabled from the guest */ 1443 if (!host && hv->hv_invtsc_control && !data) 1444 return 1; 1445 1446 hv->hv_invtsc_control = data; 1447 break; 1448 case HV_X64_MSR_SYNDBG_OPTIONS: 1449 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1450 return syndbg_set_msr(vcpu, msr, data, host); 1451 default: 1452 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1453 return 1; 1454 } 1455 return 0; 1456 } 1457 1458 /* Calculate cpu time spent by current task in 100ns units */ 1459 static u64 current_task_runtime_100ns(void) 1460 { 1461 u64 utime, stime; 1462 1463 task_cputime_adjusted(current, &utime, &stime); 1464 1465 return div_u64(utime + stime, 100); 1466 } 1467 1468 static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 1469 { 1470 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1471 1472 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) 1473 return 1; 1474 1475 switch (msr) { 1476 case HV_X64_MSR_VP_INDEX: { 1477 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1478 u32 new_vp_index = (u32)data; 1479 1480 if (!host || new_vp_index >= KVM_MAX_VCPUS) 1481 return 1; 1482 1483 if (new_vp_index == hv_vcpu->vp_index) 1484 return 0; 1485 1486 /* 1487 * The VP index is initialized to vcpu_index by 1488 * kvm_hv_vcpu_postcreate so they initially match. Now the 1489 * VP index is changing, adjust num_mismatched_vp_indexes if 1490 * it now matches or no longer matches vcpu_idx. 1491 */ 1492 if (hv_vcpu->vp_index == vcpu->vcpu_idx) 1493 atomic_inc(&hv->num_mismatched_vp_indexes); 1494 else if (new_vp_index == vcpu->vcpu_idx) 1495 atomic_dec(&hv->num_mismatched_vp_indexes); 1496 1497 hv_vcpu->vp_index = new_vp_index; 1498 break; 1499 } 1500 case HV_X64_MSR_VP_ASSIST_PAGE: { 1501 u64 gfn; 1502 unsigned long addr; 1503 1504 if (!(data & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) { 1505 hv_vcpu->hv_vapic = data; 1506 if (kvm_lapic_set_pv_eoi(vcpu, 0, 0)) 1507 return 1; 1508 break; 1509 } 1510 gfn = data >> HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT; 1511 addr = kvm_vcpu_gfn_to_hva(vcpu, gfn); 1512 if (kvm_is_error_hva(addr)) 1513 return 1; 1514 1515 /* 1516 * Clear apic_assist portion of struct hv_vp_assist_page 1517 * only, there can be valuable data in the rest which needs 1518 * to be preserved e.g. on migration. 1519 */ 1520 if (__put_user(0, (u32 __user *)addr)) 1521 return 1; 1522 hv_vcpu->hv_vapic = data; 1523 kvm_vcpu_mark_page_dirty(vcpu, gfn); 1524 if (kvm_lapic_set_pv_eoi(vcpu, 1525 gfn_to_gpa(gfn) | KVM_MSR_ENABLED, 1526 sizeof(struct hv_vp_assist_page))) 1527 return 1; 1528 break; 1529 } 1530 case HV_X64_MSR_EOI: 1531 return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data); 1532 case HV_X64_MSR_ICR: 1533 return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data); 1534 case HV_X64_MSR_TPR: 1535 return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data); 1536 case HV_X64_MSR_VP_RUNTIME: 1537 if (!host) 1538 return 1; 1539 hv_vcpu->runtime_offset = data - current_task_runtime_100ns(); 1540 break; 1541 case HV_X64_MSR_SCONTROL: 1542 case HV_X64_MSR_SVERSION: 1543 case HV_X64_MSR_SIEFP: 1544 case HV_X64_MSR_SIMP: 1545 case HV_X64_MSR_EOM: 1546 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1547 return synic_set_msr(to_hv_synic(vcpu), msr, data, host); 1548 case HV_X64_MSR_STIMER0_CONFIG: 1549 case HV_X64_MSR_STIMER1_CONFIG: 1550 case HV_X64_MSR_STIMER2_CONFIG: 1551 case HV_X64_MSR_STIMER3_CONFIG: { 1552 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2; 1553 1554 return stimer_set_config(to_hv_stimer(vcpu, timer_index), 1555 data, host); 1556 } 1557 case HV_X64_MSR_STIMER0_COUNT: 1558 case HV_X64_MSR_STIMER1_COUNT: 1559 case HV_X64_MSR_STIMER2_COUNT: 1560 case HV_X64_MSR_STIMER3_COUNT: { 1561 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2; 1562 1563 return stimer_set_count(to_hv_stimer(vcpu, timer_index), 1564 data, host); 1565 } 1566 case HV_X64_MSR_TSC_FREQUENCY: 1567 case HV_X64_MSR_APIC_FREQUENCY: 1568 /* read-only, but still ignore it if host-initiated */ 1569 if (!host) 1570 return 1; 1571 break; 1572 default: 1573 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 1574 return 1; 1575 } 1576 1577 return 0; 1578 } 1579 1580 static int kvm_hv_get_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, 1581 bool host) 1582 { 1583 u64 data = 0; 1584 struct kvm *kvm = vcpu->kvm; 1585 struct kvm_hv *hv = to_kvm_hv(kvm); 1586 1587 if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) 1588 return 1; 1589 1590 switch (msr) { 1591 case HV_X64_MSR_GUEST_OS_ID: 1592 data = hv->hv_guest_os_id; 1593 break; 1594 case HV_X64_MSR_HYPERCALL: 1595 data = hv->hv_hypercall; 1596 break; 1597 case HV_X64_MSR_TIME_REF_COUNT: 1598 data = get_time_ref_counter(kvm); 1599 break; 1600 case HV_X64_MSR_REFERENCE_TSC: 1601 data = hv->hv_tsc_page; 1602 break; 1603 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 1604 return kvm_hv_msr_get_crash_data(kvm, 1605 msr - HV_X64_MSR_CRASH_P0, 1606 pdata); 1607 case HV_X64_MSR_CRASH_CTL: 1608 return kvm_hv_msr_get_crash_ctl(kvm, pdata); 1609 case HV_X64_MSR_RESET: 1610 data = 0; 1611 break; 1612 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 1613 data = hv->hv_reenlightenment_control; 1614 break; 1615 case HV_X64_MSR_TSC_EMULATION_CONTROL: 1616 data = hv->hv_tsc_emulation_control; 1617 break; 1618 case HV_X64_MSR_TSC_EMULATION_STATUS: 1619 data = hv->hv_tsc_emulation_status; 1620 break; 1621 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 1622 data = hv->hv_invtsc_control; 1623 break; 1624 case HV_X64_MSR_SYNDBG_OPTIONS: 1625 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 1626 return syndbg_get_msr(vcpu, msr, pdata, host); 1627 default: 1628 kvm_pr_unimpl_rdmsr(vcpu, msr); 1629 return 1; 1630 } 1631 1632 *pdata = data; 1633 return 0; 1634 } 1635 1636 static int kvm_hv_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, 1637 bool host) 1638 { 1639 u64 data = 0; 1640 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1641 1642 if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) 1643 return 1; 1644 1645 switch (msr) { 1646 case HV_X64_MSR_VP_INDEX: 1647 data = hv_vcpu->vp_index; 1648 break; 1649 case HV_X64_MSR_EOI: 1650 return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata); 1651 case HV_X64_MSR_ICR: 1652 return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata); 1653 case HV_X64_MSR_TPR: 1654 return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata); 1655 case HV_X64_MSR_VP_ASSIST_PAGE: 1656 data = hv_vcpu->hv_vapic; 1657 break; 1658 case HV_X64_MSR_VP_RUNTIME: 1659 data = current_task_runtime_100ns() + hv_vcpu->runtime_offset; 1660 break; 1661 case HV_X64_MSR_SCONTROL: 1662 case HV_X64_MSR_SVERSION: 1663 case HV_X64_MSR_SIEFP: 1664 case HV_X64_MSR_SIMP: 1665 case HV_X64_MSR_EOM: 1666 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 1667 return synic_get_msr(to_hv_synic(vcpu), msr, pdata, host); 1668 case HV_X64_MSR_STIMER0_CONFIG: 1669 case HV_X64_MSR_STIMER1_CONFIG: 1670 case HV_X64_MSR_STIMER2_CONFIG: 1671 case HV_X64_MSR_STIMER3_CONFIG: { 1672 int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2; 1673 1674 return stimer_get_config(to_hv_stimer(vcpu, timer_index), 1675 pdata); 1676 } 1677 case HV_X64_MSR_STIMER0_COUNT: 1678 case HV_X64_MSR_STIMER1_COUNT: 1679 case HV_X64_MSR_STIMER2_COUNT: 1680 case HV_X64_MSR_STIMER3_COUNT: { 1681 int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2; 1682 1683 return stimer_get_count(to_hv_stimer(vcpu, timer_index), 1684 pdata); 1685 } 1686 case HV_X64_MSR_TSC_FREQUENCY: 1687 data = (u64)vcpu->arch.virtual_tsc_khz * 1000; 1688 break; 1689 case HV_X64_MSR_APIC_FREQUENCY: 1690 data = APIC_BUS_FREQUENCY; 1691 break; 1692 default: 1693 kvm_pr_unimpl_rdmsr(vcpu, msr); 1694 return 1; 1695 } 1696 *pdata = data; 1697 return 0; 1698 } 1699 1700 int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) 1701 { 1702 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1703 1704 if (!host && !vcpu->arch.hyperv_enabled) 1705 return 1; 1706 1707 if (kvm_hv_vcpu_init(vcpu)) 1708 return 1; 1709 1710 if (kvm_hv_msr_partition_wide(msr)) { 1711 int r; 1712 1713 mutex_lock(&hv->hv_lock); 1714 r = kvm_hv_set_msr_pw(vcpu, msr, data, host); 1715 mutex_unlock(&hv->hv_lock); 1716 return r; 1717 } else 1718 return kvm_hv_set_msr(vcpu, msr, data, host); 1719 } 1720 1721 int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 1722 { 1723 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 1724 1725 if (!host && !vcpu->arch.hyperv_enabled) 1726 return 1; 1727 1728 if (kvm_hv_vcpu_init(vcpu)) 1729 return 1; 1730 1731 if (kvm_hv_msr_partition_wide(msr)) { 1732 int r; 1733 1734 mutex_lock(&hv->hv_lock); 1735 r = kvm_hv_get_msr_pw(vcpu, msr, pdata, host); 1736 mutex_unlock(&hv->hv_lock); 1737 return r; 1738 } else 1739 return kvm_hv_get_msr(vcpu, msr, pdata, host); 1740 } 1741 1742 static void sparse_set_to_vcpu_mask(struct kvm *kvm, u64 *sparse_banks, 1743 u64 valid_bank_mask, unsigned long *vcpu_mask) 1744 { 1745 struct kvm_hv *hv = to_kvm_hv(kvm); 1746 bool has_mismatch = atomic_read(&hv->num_mismatched_vp_indexes); 1747 u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS]; 1748 struct kvm_vcpu *vcpu; 1749 int bank, sbank = 0; 1750 unsigned long i; 1751 u64 *bitmap; 1752 1753 BUILD_BUG_ON(sizeof(vp_bitmap) > 1754 sizeof(*vcpu_mask) * BITS_TO_LONGS(KVM_MAX_VCPUS)); 1755 1756 /* 1757 * If vp_index == vcpu_idx for all vCPUs, fill vcpu_mask directly, else 1758 * fill a temporary buffer and manually test each vCPU's VP index. 1759 */ 1760 if (likely(!has_mismatch)) 1761 bitmap = (u64 *)vcpu_mask; 1762 else 1763 bitmap = vp_bitmap; 1764 1765 /* 1766 * Each set of 64 VPs is packed into sparse_banks, with valid_bank_mask 1767 * having a '1' for each bank that exists in sparse_banks. Sets must 1768 * be in ascending order, i.e. bank0..bankN. 1769 */ 1770 memset(bitmap, 0, sizeof(vp_bitmap)); 1771 for_each_set_bit(bank, (unsigned long *)&valid_bank_mask, 1772 KVM_HV_MAX_SPARSE_VCPU_SET_BITS) 1773 bitmap[bank] = sparse_banks[sbank++]; 1774 1775 if (likely(!has_mismatch)) 1776 return; 1777 1778 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 1779 kvm_for_each_vcpu(i, vcpu, kvm) { 1780 if (test_bit(kvm_hv_get_vpindex(vcpu), (unsigned long *)vp_bitmap)) 1781 __set_bit(i, vcpu_mask); 1782 } 1783 } 1784 1785 static bool hv_is_vp_in_sparse_set(u32 vp_id, u64 valid_bank_mask, u64 sparse_banks[]) 1786 { 1787 int valid_bit_nr = vp_id / HV_VCPUS_PER_SPARSE_BANK; 1788 unsigned long sbank; 1789 1790 if (!test_bit(valid_bit_nr, (unsigned long *)&valid_bank_mask)) 1791 return false; 1792 1793 /* 1794 * The index into the sparse bank is the number of preceding bits in 1795 * the valid mask. Optimize for VMs with <64 vCPUs by skipping the 1796 * fancy math if there can't possibly be preceding bits. 1797 */ 1798 if (valid_bit_nr) 1799 sbank = hweight64(valid_bank_mask & GENMASK_ULL(valid_bit_nr - 1, 0)); 1800 else 1801 sbank = 0; 1802 1803 return test_bit(vp_id % HV_VCPUS_PER_SPARSE_BANK, 1804 (unsigned long *)&sparse_banks[sbank]); 1805 } 1806 1807 struct kvm_hv_hcall { 1808 /* Hypercall input data */ 1809 u64 param; 1810 u64 ingpa; 1811 u64 outgpa; 1812 u16 code; 1813 u16 var_cnt; 1814 u16 rep_cnt; 1815 u16 rep_idx; 1816 bool fast; 1817 bool rep; 1818 sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS]; 1819 1820 /* 1821 * Current read offset when KVM reads hypercall input data gradually, 1822 * either offset in bytes from 'ingpa' for regular hypercalls or the 1823 * number of already consumed 'XMM halves' for 'fast' hypercalls. 1824 */ 1825 union { 1826 gpa_t data_offset; 1827 int consumed_xmm_halves; 1828 }; 1829 }; 1830 1831 1832 static int kvm_hv_get_hc_data(struct kvm *kvm, struct kvm_hv_hcall *hc, 1833 u16 orig_cnt, u16 cnt_cap, u64 *data) 1834 { 1835 /* 1836 * Preserve the original count when ignoring entries via a "cap", KVM 1837 * still needs to validate the guest input (though the non-XMM path 1838 * punts on the checks). 1839 */ 1840 u16 cnt = min(orig_cnt, cnt_cap); 1841 int i, j; 1842 1843 if (hc->fast) { 1844 /* 1845 * Each XMM holds two sparse banks, but do not count halves that 1846 * have already been consumed for hypercall parameters. 1847 */ 1848 if (orig_cnt > 2 * HV_HYPERCALL_MAX_XMM_REGISTERS - hc->consumed_xmm_halves) 1849 return HV_STATUS_INVALID_HYPERCALL_INPUT; 1850 1851 for (i = 0; i < cnt; i++) { 1852 j = i + hc->consumed_xmm_halves; 1853 if (j % 2) 1854 data[i] = sse128_hi(hc->xmm[j / 2]); 1855 else 1856 data[i] = sse128_lo(hc->xmm[j / 2]); 1857 } 1858 return 0; 1859 } 1860 1861 return kvm_read_guest(kvm, hc->ingpa + hc->data_offset, data, 1862 cnt * sizeof(*data)); 1863 } 1864 1865 static u64 kvm_get_sparse_vp_set(struct kvm *kvm, struct kvm_hv_hcall *hc, 1866 u64 *sparse_banks) 1867 { 1868 if (hc->var_cnt > HV_MAX_SPARSE_VCPU_BANKS) 1869 return -EINVAL; 1870 1871 /* Cap var_cnt to ignore banks that cannot contain a legal VP index. */ 1872 return kvm_hv_get_hc_data(kvm, hc, hc->var_cnt, KVM_HV_MAX_SPARSE_VCPU_SET_BITS, 1873 sparse_banks); 1874 } 1875 1876 static int kvm_hv_get_tlb_flush_entries(struct kvm *kvm, struct kvm_hv_hcall *hc, u64 entries[]) 1877 { 1878 return kvm_hv_get_hc_data(kvm, hc, hc->rep_cnt, hc->rep_cnt, entries); 1879 } 1880 1881 static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu, 1882 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo, 1883 u64 *entries, int count) 1884 { 1885 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1886 u64 flush_all_entry = KVM_HV_TLB_FLUSHALL_ENTRY; 1887 1888 if (!hv_vcpu) 1889 return; 1890 1891 spin_lock(&tlb_flush_fifo->write_lock); 1892 1893 /* 1894 * All entries should fit on the fifo leaving one free for 'flush all' 1895 * entry in case another request comes in. In case there's not enough 1896 * space, just put 'flush all' entry there. 1897 */ 1898 if (count && entries && count < kfifo_avail(&tlb_flush_fifo->entries)) { 1899 WARN_ON(kfifo_in(&tlb_flush_fifo->entries, entries, count) != count); 1900 goto out_unlock; 1901 } 1902 1903 /* 1904 * Note: full fifo always contains 'flush all' entry, no need to check the 1905 * return value. 1906 */ 1907 kfifo_in(&tlb_flush_fifo->entries, &flush_all_entry, 1); 1908 1909 out_unlock: 1910 spin_unlock(&tlb_flush_fifo->write_lock); 1911 } 1912 1913 int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) 1914 { 1915 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; 1916 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1917 u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; 1918 int i, j, count; 1919 gva_t gva; 1920 1921 if (!tdp_enabled || !hv_vcpu) 1922 return -EINVAL; 1923 1924 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode(vcpu)); 1925 1926 count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); 1927 1928 for (i = 0; i < count; i++) { 1929 if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) 1930 goto out_flush_all; 1931 1932 /* 1933 * Lower 12 bits of 'address' encode the number of additional 1934 * pages to flush. 1935 */ 1936 gva = entries[i] & PAGE_MASK; 1937 for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) 1938 static_call(kvm_x86_flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE); 1939 1940 ++vcpu->stat.tlb_flush; 1941 } 1942 return 0; 1943 1944 out_flush_all: 1945 kfifo_reset_out(&tlb_flush_fifo->entries); 1946 1947 /* Fall back to full flush. */ 1948 return -ENOSPC; 1949 } 1950 1951 static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 1952 { 1953 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 1954 u64 *sparse_banks = hv_vcpu->sparse_banks; 1955 struct kvm *kvm = vcpu->kvm; 1956 struct hv_tlb_flush_ex flush_ex; 1957 struct hv_tlb_flush flush; 1958 DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); 1959 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; 1960 /* 1961 * Normally, there can be no more than 'KVM_HV_TLB_FLUSH_FIFO_SIZE' 1962 * entries on the TLB flush fifo. The last entry, however, needs to be 1963 * always left free for 'flush all' entry which gets placed when 1964 * there is not enough space to put all the requested entries. 1965 */ 1966 u64 __tlb_flush_entries[KVM_HV_TLB_FLUSH_FIFO_SIZE - 1]; 1967 u64 *tlb_flush_entries; 1968 u64 valid_bank_mask; 1969 struct kvm_vcpu *v; 1970 unsigned long i; 1971 bool all_cpus; 1972 1973 /* 1974 * The Hyper-V TLFS doesn't allow more than HV_MAX_SPARSE_VCPU_BANKS 1975 * sparse banks. Fail the build if KVM's max allowed number of 1976 * vCPUs (>4096) exceeds this limit. 1977 */ 1978 BUILD_BUG_ON(KVM_HV_MAX_SPARSE_VCPU_SET_BITS > HV_MAX_SPARSE_VCPU_BANKS); 1979 1980 /* 1981 * 'Slow' hypercall's first parameter is the address in guest's memory 1982 * where hypercall parameters are placed. This is either a GPA or a 1983 * nested GPA when KVM is handling the call from L2 ('direct' TLB 1984 * flush). Translate the address here so the memory can be uniformly 1985 * read with kvm_read_guest(). 1986 */ 1987 if (!hc->fast && is_guest_mode(vcpu)) { 1988 hc->ingpa = translate_nested_gpa(vcpu, hc->ingpa, 0, NULL); 1989 if (unlikely(hc->ingpa == INVALID_GPA)) 1990 return HV_STATUS_INVALID_HYPERCALL_INPUT; 1991 } 1992 1993 if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST || 1994 hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE) { 1995 if (hc->fast) { 1996 flush.address_space = hc->ingpa; 1997 flush.flags = hc->outgpa; 1998 flush.processor_mask = sse128_lo(hc->xmm[0]); 1999 hc->consumed_xmm_halves = 1; 2000 } else { 2001 if (unlikely(kvm_read_guest(kvm, hc->ingpa, 2002 &flush, sizeof(flush)))) 2003 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2004 hc->data_offset = sizeof(flush); 2005 } 2006 2007 trace_kvm_hv_flush_tlb(flush.processor_mask, 2008 flush.address_space, flush.flags, 2009 is_guest_mode(vcpu)); 2010 2011 valid_bank_mask = BIT_ULL(0); 2012 sparse_banks[0] = flush.processor_mask; 2013 2014 /* 2015 * Work around possible WS2012 bug: it sends hypercalls 2016 * with processor_mask = 0x0 and HV_FLUSH_ALL_PROCESSORS clear, 2017 * while also expecting us to flush something and crashing if 2018 * we don't. Let's treat processor_mask == 0 same as 2019 * HV_FLUSH_ALL_PROCESSORS. 2020 */ 2021 all_cpus = (flush.flags & HV_FLUSH_ALL_PROCESSORS) || 2022 flush.processor_mask == 0; 2023 } else { 2024 if (hc->fast) { 2025 flush_ex.address_space = hc->ingpa; 2026 flush_ex.flags = hc->outgpa; 2027 memcpy(&flush_ex.hv_vp_set, 2028 &hc->xmm[0], sizeof(hc->xmm[0])); 2029 hc->consumed_xmm_halves = 2; 2030 } else { 2031 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex, 2032 sizeof(flush_ex)))) 2033 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2034 hc->data_offset = sizeof(flush_ex); 2035 } 2036 2037 trace_kvm_hv_flush_tlb_ex(flush_ex.hv_vp_set.valid_bank_mask, 2038 flush_ex.hv_vp_set.format, 2039 flush_ex.address_space, 2040 flush_ex.flags, is_guest_mode(vcpu)); 2041 2042 valid_bank_mask = flush_ex.hv_vp_set.valid_bank_mask; 2043 all_cpus = flush_ex.hv_vp_set.format != 2044 HV_GENERIC_SET_SPARSE_4K; 2045 2046 if (hc->var_cnt != hweight64(valid_bank_mask)) 2047 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2048 2049 if (!all_cpus) { 2050 if (!hc->var_cnt) 2051 goto ret_success; 2052 2053 if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks)) 2054 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2055 } 2056 2057 /* 2058 * Hyper-V TLFS doesn't explicitly forbid non-empty sparse vCPU 2059 * banks (and, thus, non-zero 'var_cnt') for the 'all vCPUs' 2060 * case (HV_GENERIC_SET_ALL). Always adjust data_offset and 2061 * consumed_xmm_halves to make sure TLB flush entries are read 2062 * from the correct offset. 2063 */ 2064 if (hc->fast) 2065 hc->consumed_xmm_halves += hc->var_cnt; 2066 else 2067 hc->data_offset += hc->var_cnt * sizeof(sparse_banks[0]); 2068 } 2069 2070 if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE || 2071 hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX || 2072 hc->rep_cnt > ARRAY_SIZE(__tlb_flush_entries)) { 2073 tlb_flush_entries = NULL; 2074 } else { 2075 if (kvm_hv_get_tlb_flush_entries(kvm, hc, __tlb_flush_entries)) 2076 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2077 tlb_flush_entries = __tlb_flush_entries; 2078 } 2079 2080 /* 2081 * vcpu->arch.cr3 may not be up-to-date for running vCPUs so we can't 2082 * analyze it here, flush TLB regardless of the specified address space. 2083 */ 2084 if (all_cpus && !is_guest_mode(vcpu)) { 2085 kvm_for_each_vcpu(i, v, kvm) { 2086 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, false); 2087 hv_tlb_flush_enqueue(v, tlb_flush_fifo, 2088 tlb_flush_entries, hc->rep_cnt); 2089 } 2090 2091 kvm_make_all_cpus_request(kvm, KVM_REQ_HV_TLB_FLUSH); 2092 } else if (!is_guest_mode(vcpu)) { 2093 sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask, vcpu_mask); 2094 2095 for_each_set_bit(i, vcpu_mask, KVM_MAX_VCPUS) { 2096 v = kvm_get_vcpu(kvm, i); 2097 if (!v) 2098 continue; 2099 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, false); 2100 hv_tlb_flush_enqueue(v, tlb_flush_fifo, 2101 tlb_flush_entries, hc->rep_cnt); 2102 } 2103 2104 kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); 2105 } else { 2106 struct kvm_vcpu_hv *hv_v; 2107 2108 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 2109 2110 kvm_for_each_vcpu(i, v, kvm) { 2111 hv_v = to_hv_vcpu(v); 2112 2113 /* 2114 * The following check races with nested vCPUs entering/exiting 2115 * and/or migrating between L1's vCPUs, however the only case when 2116 * KVM *must* flush the TLB is when the target L2 vCPU keeps 2117 * running on the same L1 vCPU from the moment of the request until 2118 * kvm_hv_flush_tlb() returns. TLB is fully flushed in all other 2119 * cases, e.g. when the target L2 vCPU migrates to a different L1 2120 * vCPU or when the corresponding L1 vCPU temporary switches to a 2121 * different L2 vCPU while the request is being processed. 2122 */ 2123 if (!hv_v || hv_v->nested.vm_id != hv_vcpu->nested.vm_id) 2124 continue; 2125 2126 if (!all_cpus && 2127 !hv_is_vp_in_sparse_set(hv_v->nested.vp_id, valid_bank_mask, 2128 sparse_banks)) 2129 continue; 2130 2131 __set_bit(i, vcpu_mask); 2132 tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, true); 2133 hv_tlb_flush_enqueue(v, tlb_flush_fifo, 2134 tlb_flush_entries, hc->rep_cnt); 2135 } 2136 2137 kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); 2138 } 2139 2140 ret_success: 2141 /* We always do full TLB flush, set 'Reps completed' = 'Rep Count' */ 2142 return (u64)HV_STATUS_SUCCESS | 2143 ((u64)hc->rep_cnt << HV_HYPERCALL_REP_COMP_OFFSET); 2144 } 2145 2146 static void kvm_hv_send_ipi_to_many(struct kvm *kvm, u32 vector, 2147 u64 *sparse_banks, u64 valid_bank_mask) 2148 { 2149 struct kvm_lapic_irq irq = { 2150 .delivery_mode = APIC_DM_FIXED, 2151 .vector = vector 2152 }; 2153 struct kvm_vcpu *vcpu; 2154 unsigned long i; 2155 2156 kvm_for_each_vcpu(i, vcpu, kvm) { 2157 if (sparse_banks && 2158 !hv_is_vp_in_sparse_set(kvm_hv_get_vpindex(vcpu), 2159 valid_bank_mask, sparse_banks)) 2160 continue; 2161 2162 /* We fail only when APIC is disabled */ 2163 kvm_apic_set_irq(vcpu, &irq, NULL); 2164 } 2165 } 2166 2167 static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2168 { 2169 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2170 u64 *sparse_banks = hv_vcpu->sparse_banks; 2171 struct kvm *kvm = vcpu->kvm; 2172 struct hv_send_ipi_ex send_ipi_ex; 2173 struct hv_send_ipi send_ipi; 2174 u64 valid_bank_mask; 2175 u32 vector; 2176 bool all_cpus; 2177 2178 if (hc->code == HVCALL_SEND_IPI) { 2179 if (!hc->fast) { 2180 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi, 2181 sizeof(send_ipi)))) 2182 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2183 sparse_banks[0] = send_ipi.cpu_mask; 2184 vector = send_ipi.vector; 2185 } else { 2186 /* 'reserved' part of hv_send_ipi should be 0 */ 2187 if (unlikely(hc->ingpa >> 32 != 0)) 2188 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2189 sparse_banks[0] = hc->outgpa; 2190 vector = (u32)hc->ingpa; 2191 } 2192 all_cpus = false; 2193 valid_bank_mask = BIT_ULL(0); 2194 2195 trace_kvm_hv_send_ipi(vector, sparse_banks[0]); 2196 } else { 2197 if (!hc->fast) { 2198 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex, 2199 sizeof(send_ipi_ex)))) 2200 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2201 } else { 2202 send_ipi_ex.vector = (u32)hc->ingpa; 2203 send_ipi_ex.vp_set.format = hc->outgpa; 2204 send_ipi_ex.vp_set.valid_bank_mask = sse128_lo(hc->xmm[0]); 2205 } 2206 2207 trace_kvm_hv_send_ipi_ex(send_ipi_ex.vector, 2208 send_ipi_ex.vp_set.format, 2209 send_ipi_ex.vp_set.valid_bank_mask); 2210 2211 vector = send_ipi_ex.vector; 2212 valid_bank_mask = send_ipi_ex.vp_set.valid_bank_mask; 2213 all_cpus = send_ipi_ex.vp_set.format == HV_GENERIC_SET_ALL; 2214 2215 if (hc->var_cnt != hweight64(valid_bank_mask)) 2216 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2217 2218 if (all_cpus) 2219 goto check_and_send_ipi; 2220 2221 if (!hc->var_cnt) 2222 goto ret_success; 2223 2224 if (!hc->fast) 2225 hc->data_offset = offsetof(struct hv_send_ipi_ex, 2226 vp_set.bank_contents); 2227 else 2228 hc->consumed_xmm_halves = 1; 2229 2230 if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks)) 2231 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2232 } 2233 2234 check_and_send_ipi: 2235 if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR)) 2236 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2237 2238 if (all_cpus) 2239 kvm_hv_send_ipi_to_many(kvm, vector, NULL, 0); 2240 else 2241 kvm_hv_send_ipi_to_many(kvm, vector, sparse_banks, valid_bank_mask); 2242 2243 ret_success: 2244 return HV_STATUS_SUCCESS; 2245 } 2246 2247 void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu, bool hyperv_enabled) 2248 { 2249 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2250 struct kvm_cpuid_entry2 *entry; 2251 2252 vcpu->arch.hyperv_enabled = hyperv_enabled; 2253 2254 if (!hv_vcpu) { 2255 /* 2256 * KVM should have already allocated kvm_vcpu_hv if Hyper-V is 2257 * enabled in CPUID. 2258 */ 2259 WARN_ON_ONCE(vcpu->arch.hyperv_enabled); 2260 return; 2261 } 2262 2263 memset(&hv_vcpu->cpuid_cache, 0, sizeof(hv_vcpu->cpuid_cache)); 2264 2265 if (!vcpu->arch.hyperv_enabled) 2266 return; 2267 2268 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_FEATURES); 2269 if (entry) { 2270 hv_vcpu->cpuid_cache.features_eax = entry->eax; 2271 hv_vcpu->cpuid_cache.features_ebx = entry->ebx; 2272 hv_vcpu->cpuid_cache.features_edx = entry->edx; 2273 } 2274 2275 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_ENLIGHTMENT_INFO); 2276 if (entry) { 2277 hv_vcpu->cpuid_cache.enlightenments_eax = entry->eax; 2278 hv_vcpu->cpuid_cache.enlightenments_ebx = entry->ebx; 2279 } 2280 2281 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES); 2282 if (entry) 2283 hv_vcpu->cpuid_cache.syndbg_cap_eax = entry->eax; 2284 2285 entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_NESTED_FEATURES); 2286 if (entry) { 2287 hv_vcpu->cpuid_cache.nested_eax = entry->eax; 2288 hv_vcpu->cpuid_cache.nested_ebx = entry->ebx; 2289 } 2290 } 2291 2292 int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce) 2293 { 2294 struct kvm_vcpu_hv *hv_vcpu; 2295 int ret = 0; 2296 2297 if (!to_hv_vcpu(vcpu)) { 2298 if (enforce) { 2299 ret = kvm_hv_vcpu_init(vcpu); 2300 if (ret) 2301 return ret; 2302 } else { 2303 return 0; 2304 } 2305 } 2306 2307 hv_vcpu = to_hv_vcpu(vcpu); 2308 hv_vcpu->enforce_cpuid = enforce; 2309 2310 return ret; 2311 } 2312 2313 static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result) 2314 { 2315 bool longmode; 2316 2317 longmode = is_64_bit_hypercall(vcpu); 2318 if (longmode) 2319 kvm_rax_write(vcpu, result); 2320 else { 2321 kvm_rdx_write(vcpu, result >> 32); 2322 kvm_rax_write(vcpu, result & 0xffffffff); 2323 } 2324 } 2325 2326 static int kvm_hv_hypercall_complete(struct kvm_vcpu *vcpu, u64 result) 2327 { 2328 u32 tlb_lock_count = 0; 2329 int ret; 2330 2331 if (hv_result_success(result) && is_guest_mode(vcpu) && 2332 kvm_hv_is_tlb_flush_hcall(vcpu) && 2333 kvm_read_guest(vcpu->kvm, to_hv_vcpu(vcpu)->nested.pa_page_gpa, 2334 &tlb_lock_count, sizeof(tlb_lock_count))) 2335 result = HV_STATUS_INVALID_HYPERCALL_INPUT; 2336 2337 trace_kvm_hv_hypercall_done(result); 2338 kvm_hv_hypercall_set_result(vcpu, result); 2339 ++vcpu->stat.hypercalls; 2340 2341 ret = kvm_skip_emulated_instruction(vcpu); 2342 2343 if (tlb_lock_count) 2344 kvm_x86_ops.nested_ops->hv_inject_synthetic_vmexit_post_tlb_flush(vcpu); 2345 2346 return ret; 2347 } 2348 2349 static int kvm_hv_hypercall_complete_userspace(struct kvm_vcpu *vcpu) 2350 { 2351 return kvm_hv_hypercall_complete(vcpu, vcpu->run->hyperv.u.hcall.result); 2352 } 2353 2354 static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) 2355 { 2356 struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); 2357 struct eventfd_ctx *eventfd; 2358 2359 if (unlikely(!hc->fast)) { 2360 int ret; 2361 gpa_t gpa = hc->ingpa; 2362 2363 if ((gpa & (__alignof__(hc->ingpa) - 1)) || 2364 offset_in_page(gpa) + sizeof(hc->ingpa) > PAGE_SIZE) 2365 return HV_STATUS_INVALID_ALIGNMENT; 2366 2367 ret = kvm_vcpu_read_guest(vcpu, gpa, 2368 &hc->ingpa, sizeof(hc->ingpa)); 2369 if (ret < 0) 2370 return HV_STATUS_INVALID_ALIGNMENT; 2371 } 2372 2373 /* 2374 * Per spec, bits 32-47 contain the extra "flag number". However, we 2375 * have no use for it, and in all known usecases it is zero, so just 2376 * report lookup failure if it isn't. 2377 */ 2378 if (hc->ingpa & 0xffff00000000ULL) 2379 return HV_STATUS_INVALID_PORT_ID; 2380 /* remaining bits are reserved-zero */ 2381 if (hc->ingpa & ~KVM_HYPERV_CONN_ID_MASK) 2382 return HV_STATUS_INVALID_HYPERCALL_INPUT; 2383 2384 /* the eventfd is protected by vcpu->kvm->srcu, but conn_to_evt isn't */ 2385 rcu_read_lock(); 2386 eventfd = idr_find(&hv->conn_to_evt, hc->ingpa); 2387 rcu_read_unlock(); 2388 if (!eventfd) 2389 return HV_STATUS_INVALID_PORT_ID; 2390 2391 eventfd_signal(eventfd, 1); 2392 return HV_STATUS_SUCCESS; 2393 } 2394 2395 static bool is_xmm_fast_hypercall(struct kvm_hv_hcall *hc) 2396 { 2397 switch (hc->code) { 2398 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2399 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2400 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2401 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2402 case HVCALL_SEND_IPI_EX: 2403 return true; 2404 } 2405 2406 return false; 2407 } 2408 2409 static void kvm_hv_hypercall_read_xmm(struct kvm_hv_hcall *hc) 2410 { 2411 int reg; 2412 2413 kvm_fpu_get(); 2414 for (reg = 0; reg < HV_HYPERCALL_MAX_XMM_REGISTERS; reg++) 2415 _kvm_read_sse_reg(reg, &hc->xmm[reg]); 2416 kvm_fpu_put(); 2417 } 2418 2419 static bool hv_check_hypercall_access(struct kvm_vcpu_hv *hv_vcpu, u16 code) 2420 { 2421 if (!hv_vcpu->enforce_cpuid) 2422 return true; 2423 2424 switch (code) { 2425 case HVCALL_NOTIFY_LONG_SPIN_WAIT: 2426 return hv_vcpu->cpuid_cache.enlightenments_ebx && 2427 hv_vcpu->cpuid_cache.enlightenments_ebx != U32_MAX; 2428 case HVCALL_POST_MESSAGE: 2429 return hv_vcpu->cpuid_cache.features_ebx & HV_POST_MESSAGES; 2430 case HVCALL_SIGNAL_EVENT: 2431 return hv_vcpu->cpuid_cache.features_ebx & HV_SIGNAL_EVENTS; 2432 case HVCALL_POST_DEBUG_DATA: 2433 case HVCALL_RETRIEVE_DEBUG_DATA: 2434 case HVCALL_RESET_DEBUG_SESSION: 2435 /* 2436 * Return 'true' when SynDBG is disabled so the resulting code 2437 * will be HV_STATUS_INVALID_HYPERCALL_CODE. 2438 */ 2439 return !kvm_hv_is_syndbg_enabled(hv_vcpu->vcpu) || 2440 hv_vcpu->cpuid_cache.features_ebx & HV_DEBUGGING; 2441 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2442 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2443 if (!(hv_vcpu->cpuid_cache.enlightenments_eax & 2444 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 2445 return false; 2446 fallthrough; 2447 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2448 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2449 return hv_vcpu->cpuid_cache.enlightenments_eax & 2450 HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; 2451 case HVCALL_SEND_IPI_EX: 2452 if (!(hv_vcpu->cpuid_cache.enlightenments_eax & 2453 HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 2454 return false; 2455 fallthrough; 2456 case HVCALL_SEND_IPI: 2457 return hv_vcpu->cpuid_cache.enlightenments_eax & 2458 HV_X64_CLUSTER_IPI_RECOMMENDED; 2459 case HV_EXT_CALL_QUERY_CAPABILITIES ... HV_EXT_CALL_MAX: 2460 return hv_vcpu->cpuid_cache.features_ebx & 2461 HV_ENABLE_EXTENDED_HYPERCALLS; 2462 default: 2463 break; 2464 } 2465 2466 return true; 2467 } 2468 2469 int kvm_hv_hypercall(struct kvm_vcpu *vcpu) 2470 { 2471 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 2472 struct kvm_hv_hcall hc; 2473 u64 ret = HV_STATUS_SUCCESS; 2474 2475 /* 2476 * hypercall generates UD from non zero cpl and real mode 2477 * per HYPER-V spec 2478 */ 2479 if (static_call(kvm_x86_get_cpl)(vcpu) != 0 || !is_protmode(vcpu)) { 2480 kvm_queue_exception(vcpu, UD_VECTOR); 2481 return 1; 2482 } 2483 2484 #ifdef CONFIG_X86_64 2485 if (is_64_bit_hypercall(vcpu)) { 2486 hc.param = kvm_rcx_read(vcpu); 2487 hc.ingpa = kvm_rdx_read(vcpu); 2488 hc.outgpa = kvm_r8_read(vcpu); 2489 } else 2490 #endif 2491 { 2492 hc.param = ((u64)kvm_rdx_read(vcpu) << 32) | 2493 (kvm_rax_read(vcpu) & 0xffffffff); 2494 hc.ingpa = ((u64)kvm_rbx_read(vcpu) << 32) | 2495 (kvm_rcx_read(vcpu) & 0xffffffff); 2496 hc.outgpa = ((u64)kvm_rdi_read(vcpu) << 32) | 2497 (kvm_rsi_read(vcpu) & 0xffffffff); 2498 } 2499 2500 hc.code = hc.param & 0xffff; 2501 hc.var_cnt = (hc.param & HV_HYPERCALL_VARHEAD_MASK) >> HV_HYPERCALL_VARHEAD_OFFSET; 2502 hc.fast = !!(hc.param & HV_HYPERCALL_FAST_BIT); 2503 hc.rep_cnt = (hc.param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff; 2504 hc.rep_idx = (hc.param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff; 2505 hc.rep = !!(hc.rep_cnt || hc.rep_idx); 2506 2507 trace_kvm_hv_hypercall(hc.code, hc.fast, hc.var_cnt, hc.rep_cnt, 2508 hc.rep_idx, hc.ingpa, hc.outgpa); 2509 2510 if (unlikely(!hv_check_hypercall_access(hv_vcpu, hc.code))) { 2511 ret = HV_STATUS_ACCESS_DENIED; 2512 goto hypercall_complete; 2513 } 2514 2515 if (unlikely(hc.param & HV_HYPERCALL_RSVD_MASK)) { 2516 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2517 goto hypercall_complete; 2518 } 2519 2520 if (hc.fast && is_xmm_fast_hypercall(&hc)) { 2521 if (unlikely(hv_vcpu->enforce_cpuid && 2522 !(hv_vcpu->cpuid_cache.features_edx & 2523 HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE))) { 2524 kvm_queue_exception(vcpu, UD_VECTOR); 2525 return 1; 2526 } 2527 2528 kvm_hv_hypercall_read_xmm(&hc); 2529 } 2530 2531 switch (hc.code) { 2532 case HVCALL_NOTIFY_LONG_SPIN_WAIT: 2533 if (unlikely(hc.rep || hc.var_cnt)) { 2534 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2535 break; 2536 } 2537 kvm_vcpu_on_spin(vcpu, true); 2538 break; 2539 case HVCALL_SIGNAL_EVENT: 2540 if (unlikely(hc.rep || hc.var_cnt)) { 2541 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2542 break; 2543 } 2544 ret = kvm_hvcall_signal_event(vcpu, &hc); 2545 if (ret != HV_STATUS_INVALID_PORT_ID) 2546 break; 2547 fallthrough; /* maybe userspace knows this conn_id */ 2548 case HVCALL_POST_MESSAGE: 2549 /* don't bother userspace if it has no way to handle it */ 2550 if (unlikely(hc.rep || hc.var_cnt || !to_hv_synic(vcpu)->active)) { 2551 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2552 break; 2553 } 2554 goto hypercall_userspace_exit; 2555 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: 2556 if (unlikely(hc.var_cnt)) { 2557 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2558 break; 2559 } 2560 fallthrough; 2561 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: 2562 if (unlikely(!hc.rep_cnt || hc.rep_idx)) { 2563 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2564 break; 2565 } 2566 ret = kvm_hv_flush_tlb(vcpu, &hc); 2567 break; 2568 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: 2569 if (unlikely(hc.var_cnt)) { 2570 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2571 break; 2572 } 2573 fallthrough; 2574 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: 2575 if (unlikely(hc.rep)) { 2576 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2577 break; 2578 } 2579 ret = kvm_hv_flush_tlb(vcpu, &hc); 2580 break; 2581 case HVCALL_SEND_IPI: 2582 if (unlikely(hc.var_cnt)) { 2583 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2584 break; 2585 } 2586 fallthrough; 2587 case HVCALL_SEND_IPI_EX: 2588 if (unlikely(hc.rep)) { 2589 ret = HV_STATUS_INVALID_HYPERCALL_INPUT; 2590 break; 2591 } 2592 ret = kvm_hv_send_ipi(vcpu, &hc); 2593 break; 2594 case HVCALL_POST_DEBUG_DATA: 2595 case HVCALL_RETRIEVE_DEBUG_DATA: 2596 if (unlikely(hc.fast)) { 2597 ret = HV_STATUS_INVALID_PARAMETER; 2598 break; 2599 } 2600 fallthrough; 2601 case HVCALL_RESET_DEBUG_SESSION: { 2602 struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu); 2603 2604 if (!kvm_hv_is_syndbg_enabled(vcpu)) { 2605 ret = HV_STATUS_INVALID_HYPERCALL_CODE; 2606 break; 2607 } 2608 2609 if (!(syndbg->options & HV_X64_SYNDBG_OPTION_USE_HCALLS)) { 2610 ret = HV_STATUS_OPERATION_DENIED; 2611 break; 2612 } 2613 goto hypercall_userspace_exit; 2614 } 2615 case HV_EXT_CALL_QUERY_CAPABILITIES ... HV_EXT_CALL_MAX: 2616 if (unlikely(hc.fast)) { 2617 ret = HV_STATUS_INVALID_PARAMETER; 2618 break; 2619 } 2620 goto hypercall_userspace_exit; 2621 default: 2622 ret = HV_STATUS_INVALID_HYPERCALL_CODE; 2623 break; 2624 } 2625 2626 hypercall_complete: 2627 return kvm_hv_hypercall_complete(vcpu, ret); 2628 2629 hypercall_userspace_exit: 2630 vcpu->run->exit_reason = KVM_EXIT_HYPERV; 2631 vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL; 2632 vcpu->run->hyperv.u.hcall.input = hc.param; 2633 vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa; 2634 vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa; 2635 vcpu->arch.complete_userspace_io = kvm_hv_hypercall_complete_userspace; 2636 return 0; 2637 } 2638 2639 void kvm_hv_init_vm(struct kvm *kvm) 2640 { 2641 struct kvm_hv *hv = to_kvm_hv(kvm); 2642 2643 mutex_init(&hv->hv_lock); 2644 idr_init(&hv->conn_to_evt); 2645 } 2646 2647 void kvm_hv_destroy_vm(struct kvm *kvm) 2648 { 2649 struct kvm_hv *hv = to_kvm_hv(kvm); 2650 struct eventfd_ctx *eventfd; 2651 int i; 2652 2653 idr_for_each_entry(&hv->conn_to_evt, eventfd, i) 2654 eventfd_ctx_put(eventfd); 2655 idr_destroy(&hv->conn_to_evt); 2656 } 2657 2658 static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd) 2659 { 2660 struct kvm_hv *hv = to_kvm_hv(kvm); 2661 struct eventfd_ctx *eventfd; 2662 int ret; 2663 2664 eventfd = eventfd_ctx_fdget(fd); 2665 if (IS_ERR(eventfd)) 2666 return PTR_ERR(eventfd); 2667 2668 mutex_lock(&hv->hv_lock); 2669 ret = idr_alloc(&hv->conn_to_evt, eventfd, conn_id, conn_id + 1, 2670 GFP_KERNEL_ACCOUNT); 2671 mutex_unlock(&hv->hv_lock); 2672 2673 if (ret >= 0) 2674 return 0; 2675 2676 if (ret == -ENOSPC) 2677 ret = -EEXIST; 2678 eventfd_ctx_put(eventfd); 2679 return ret; 2680 } 2681 2682 static int kvm_hv_eventfd_deassign(struct kvm *kvm, u32 conn_id) 2683 { 2684 struct kvm_hv *hv = to_kvm_hv(kvm); 2685 struct eventfd_ctx *eventfd; 2686 2687 mutex_lock(&hv->hv_lock); 2688 eventfd = idr_remove(&hv->conn_to_evt, conn_id); 2689 mutex_unlock(&hv->hv_lock); 2690 2691 if (!eventfd) 2692 return -ENOENT; 2693 2694 synchronize_srcu(&kvm->srcu); 2695 eventfd_ctx_put(eventfd); 2696 return 0; 2697 } 2698 2699 int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args) 2700 { 2701 if ((args->flags & ~KVM_HYPERV_EVENTFD_DEASSIGN) || 2702 (args->conn_id & ~KVM_HYPERV_CONN_ID_MASK)) 2703 return -EINVAL; 2704 2705 if (args->flags == KVM_HYPERV_EVENTFD_DEASSIGN) 2706 return kvm_hv_eventfd_deassign(kvm, args->conn_id); 2707 return kvm_hv_eventfd_assign(kvm, args->conn_id, args->fd); 2708 } 2709 2710 int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, 2711 struct kvm_cpuid_entry2 __user *entries) 2712 { 2713 uint16_t evmcs_ver = 0; 2714 struct kvm_cpuid_entry2 cpuid_entries[] = { 2715 { .function = HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS }, 2716 { .function = HYPERV_CPUID_INTERFACE }, 2717 { .function = HYPERV_CPUID_VERSION }, 2718 { .function = HYPERV_CPUID_FEATURES }, 2719 { .function = HYPERV_CPUID_ENLIGHTMENT_INFO }, 2720 { .function = HYPERV_CPUID_IMPLEMENT_LIMITS }, 2721 { .function = HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS }, 2722 { .function = HYPERV_CPUID_SYNDBG_INTERFACE }, 2723 { .function = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES }, 2724 { .function = HYPERV_CPUID_NESTED_FEATURES }, 2725 }; 2726 int i, nent = ARRAY_SIZE(cpuid_entries); 2727 2728 if (kvm_x86_ops.nested_ops->get_evmcs_version) 2729 evmcs_ver = kvm_x86_ops.nested_ops->get_evmcs_version(vcpu); 2730 2731 if (cpuid->nent < nent) 2732 return -E2BIG; 2733 2734 if (cpuid->nent > nent) 2735 cpuid->nent = nent; 2736 2737 for (i = 0; i < nent; i++) { 2738 struct kvm_cpuid_entry2 *ent = &cpuid_entries[i]; 2739 u32 signature[3]; 2740 2741 switch (ent->function) { 2742 case HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS: 2743 memcpy(signature, "Linux KVM Hv", 12); 2744 2745 ent->eax = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES; 2746 ent->ebx = signature[0]; 2747 ent->ecx = signature[1]; 2748 ent->edx = signature[2]; 2749 break; 2750 2751 case HYPERV_CPUID_INTERFACE: 2752 ent->eax = HYPERV_CPUID_SIGNATURE_EAX; 2753 break; 2754 2755 case HYPERV_CPUID_VERSION: 2756 /* 2757 * We implement some Hyper-V 2016 functions so let's use 2758 * this version. 2759 */ 2760 ent->eax = 0x00003839; 2761 ent->ebx = 0x000A0000; 2762 break; 2763 2764 case HYPERV_CPUID_FEATURES: 2765 ent->eax |= HV_MSR_VP_RUNTIME_AVAILABLE; 2766 ent->eax |= HV_MSR_TIME_REF_COUNT_AVAILABLE; 2767 ent->eax |= HV_MSR_SYNIC_AVAILABLE; 2768 ent->eax |= HV_MSR_SYNTIMER_AVAILABLE; 2769 ent->eax |= HV_MSR_APIC_ACCESS_AVAILABLE; 2770 ent->eax |= HV_MSR_HYPERCALL_AVAILABLE; 2771 ent->eax |= HV_MSR_VP_INDEX_AVAILABLE; 2772 ent->eax |= HV_MSR_RESET_AVAILABLE; 2773 ent->eax |= HV_MSR_REFERENCE_TSC_AVAILABLE; 2774 ent->eax |= HV_ACCESS_FREQUENCY_MSRS; 2775 ent->eax |= HV_ACCESS_REENLIGHTENMENT; 2776 ent->eax |= HV_ACCESS_TSC_INVARIANT; 2777 2778 ent->ebx |= HV_POST_MESSAGES; 2779 ent->ebx |= HV_SIGNAL_EVENTS; 2780 ent->ebx |= HV_ENABLE_EXTENDED_HYPERCALLS; 2781 2782 ent->edx |= HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE; 2783 ent->edx |= HV_FEATURE_FREQUENCY_MSRS_AVAILABLE; 2784 ent->edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; 2785 2786 ent->ebx |= HV_DEBUGGING; 2787 ent->edx |= HV_X64_GUEST_DEBUGGING_AVAILABLE; 2788 ent->edx |= HV_FEATURE_DEBUG_MSRS_AVAILABLE; 2789 ent->edx |= HV_FEATURE_EXT_GVA_RANGES_FLUSH; 2790 2791 /* 2792 * Direct Synthetic timers only make sense with in-kernel 2793 * LAPIC 2794 */ 2795 if (!vcpu || lapic_in_kernel(vcpu)) 2796 ent->edx |= HV_STIMER_DIRECT_MODE_AVAILABLE; 2797 2798 break; 2799 2800 case HYPERV_CPUID_ENLIGHTMENT_INFO: 2801 ent->eax |= HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; 2802 ent->eax |= HV_X64_APIC_ACCESS_RECOMMENDED; 2803 ent->eax |= HV_X64_RELAXED_TIMING_RECOMMENDED; 2804 ent->eax |= HV_X64_CLUSTER_IPI_RECOMMENDED; 2805 ent->eax |= HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED; 2806 if (evmcs_ver) 2807 ent->eax |= HV_X64_ENLIGHTENED_VMCS_RECOMMENDED; 2808 if (!cpu_smt_possible()) 2809 ent->eax |= HV_X64_NO_NONARCH_CORESHARING; 2810 2811 ent->eax |= HV_DEPRECATING_AEOI_RECOMMENDED; 2812 /* 2813 * Default number of spinlock retry attempts, matches 2814 * HyperV 2016. 2815 */ 2816 ent->ebx = 0x00000FFF; 2817 2818 break; 2819 2820 case HYPERV_CPUID_IMPLEMENT_LIMITS: 2821 /* Maximum number of virtual processors */ 2822 ent->eax = KVM_MAX_VCPUS; 2823 /* 2824 * Maximum number of logical processors, matches 2825 * HyperV 2016. 2826 */ 2827 ent->ebx = 64; 2828 2829 break; 2830 2831 case HYPERV_CPUID_NESTED_FEATURES: 2832 ent->eax = evmcs_ver; 2833 ent->eax |= HV_X64_NESTED_DIRECT_FLUSH; 2834 ent->eax |= HV_X64_NESTED_MSR_BITMAP; 2835 ent->ebx |= HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL; 2836 break; 2837 2838 case HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS: 2839 memcpy(signature, "Linux KVM Hv", 12); 2840 2841 ent->eax = 0; 2842 ent->ebx = signature[0]; 2843 ent->ecx = signature[1]; 2844 ent->edx = signature[2]; 2845 break; 2846 2847 case HYPERV_CPUID_SYNDBG_INTERFACE: 2848 memcpy(signature, "VS#1\0\0\0\0\0\0\0\0", 12); 2849 ent->eax = signature[0]; 2850 break; 2851 2852 case HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES: 2853 ent->eax |= HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; 2854 break; 2855 2856 default: 2857 break; 2858 } 2859 } 2860 2861 if (copy_to_user(entries, cpuid_entries, 2862 nent * sizeof(struct kvm_cpuid_entry2))) 2863 return -EFAULT; 2864 2865 return 0; 2866 } 2867