1 2 /* 3 * Local APIC virtualization 4 * 5 * Copyright (C) 2006 Qumranet, Inc. 6 * Copyright (C) 2007 Novell 7 * Copyright (C) 2007 Intel 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Dor Laor <dor.laor@qumranet.com> 12 * Gregory Haskins <ghaskins@novell.com> 13 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 14 * 15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 16 * 17 * This work is licensed under the terms of the GNU GPL, version 2. See 18 * the COPYING file in the top-level directory. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/mm.h> 24 #include <linux/highmem.h> 25 #include <linux/smp.h> 26 #include <linux/hrtimer.h> 27 #include <linux/io.h> 28 #include <linux/export.h> 29 #include <linux/math64.h> 30 #include <linux/slab.h> 31 #include <asm/processor.h> 32 #include <asm/msr.h> 33 #include <asm/page.h> 34 #include <asm/current.h> 35 #include <asm/apicdef.h> 36 #include <asm/delay.h> 37 #include <linux/atomic.h> 38 #include <linux/jump_label.h> 39 #include "kvm_cache_regs.h" 40 #include "irq.h" 41 #include "trace.h" 42 #include "x86.h" 43 #include "cpuid.h" 44 #include "hyperv.h" 45 46 #ifndef CONFIG_X86_64 47 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 48 #else 49 #define mod_64(x, y) ((x) % (y)) 50 #endif 51 52 #define PRId64 "d" 53 #define PRIx64 "llx" 54 #define PRIu64 "u" 55 #define PRIo64 "o" 56 57 /* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ 58 #define apic_debug(fmt, arg...) 59 60 /* 14 is the version for Xeon and Pentium 8.4.8*/ 61 #define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16)) 62 #define LAPIC_MMIO_LENGTH (1 << 12) 63 /* followed define is not in apicdef.h */ 64 #define APIC_SHORT_MASK 0xc0000 65 #define APIC_DEST_NOSHORT 0x0 66 #define APIC_DEST_MASK 0x800 67 #define MAX_APIC_VECTOR 256 68 #define APIC_VECTORS_PER_REG 32 69 70 #define APIC_BROADCAST 0xFF 71 #define X2APIC_BROADCAST 0xFFFFFFFFul 72 73 static inline int apic_test_vector(int vec, void *bitmap) 74 { 75 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 76 } 77 78 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 79 { 80 struct kvm_lapic *apic = vcpu->arch.apic; 81 82 return apic_test_vector(vector, apic->regs + APIC_ISR) || 83 apic_test_vector(vector, apic->regs + APIC_IRR); 84 } 85 86 static inline void apic_clear_vector(int vec, void *bitmap) 87 { 88 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 89 } 90 91 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 92 { 93 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 94 } 95 96 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 97 { 98 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 99 } 100 101 struct static_key_deferred apic_hw_disabled __read_mostly; 102 struct static_key_deferred apic_sw_disabled __read_mostly; 103 104 static inline int apic_enabled(struct kvm_lapic *apic) 105 { 106 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 107 } 108 109 #define LVT_MASK \ 110 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 111 112 #define LINT_MASK \ 113 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 114 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 115 116 static inline u8 kvm_xapic_id(struct kvm_lapic *apic) 117 { 118 return kvm_lapic_get_reg(apic, APIC_ID) >> 24; 119 } 120 121 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) 122 { 123 return apic->vcpu->vcpu_id; 124 } 125 126 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, 127 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { 128 switch (map->mode) { 129 case KVM_APIC_MODE_X2APIC: { 130 u32 offset = (dest_id >> 16) * 16; 131 u32 max_apic_id = map->max_apic_id; 132 133 if (offset <= max_apic_id) { 134 u8 cluster_size = min(max_apic_id - offset + 1, 16U); 135 136 *cluster = &map->phys_map[offset]; 137 *mask = dest_id & (0xffff >> (16 - cluster_size)); 138 } else { 139 *mask = 0; 140 } 141 142 return true; 143 } 144 case KVM_APIC_MODE_XAPIC_FLAT: 145 *cluster = map->xapic_flat_map; 146 *mask = dest_id & 0xff; 147 return true; 148 case KVM_APIC_MODE_XAPIC_CLUSTER: 149 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; 150 *mask = dest_id & 0xf; 151 return true; 152 default: 153 /* Not optimized. */ 154 return false; 155 } 156 } 157 158 static void kvm_apic_map_free(struct rcu_head *rcu) 159 { 160 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu); 161 162 kvfree(map); 163 } 164 165 static void recalculate_apic_map(struct kvm *kvm) 166 { 167 struct kvm_apic_map *new, *old = NULL; 168 struct kvm_vcpu *vcpu; 169 int i; 170 u32 max_id = 255; /* enough space for any xAPIC ID */ 171 172 mutex_lock(&kvm->arch.apic_map_lock); 173 174 kvm_for_each_vcpu(i, vcpu, kvm) 175 if (kvm_apic_present(vcpu)) 176 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); 177 178 new = kvzalloc(sizeof(struct kvm_apic_map) + 179 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), GFP_KERNEL); 180 181 if (!new) 182 goto out; 183 184 new->max_apic_id = max_id; 185 186 kvm_for_each_vcpu(i, vcpu, kvm) { 187 struct kvm_lapic *apic = vcpu->arch.apic; 188 struct kvm_lapic **cluster; 189 u16 mask; 190 u32 ldr; 191 u8 xapic_id; 192 u32 x2apic_id; 193 194 if (!kvm_apic_present(vcpu)) 195 continue; 196 197 xapic_id = kvm_xapic_id(apic); 198 x2apic_id = kvm_x2apic_id(apic); 199 200 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */ 201 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) && 202 x2apic_id <= new->max_apic_id) 203 new->phys_map[x2apic_id] = apic; 204 /* 205 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around, 206 * prevent them from masking VCPUs with APIC ID <= 0xff. 207 */ 208 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) 209 new->phys_map[xapic_id] = apic; 210 211 ldr = kvm_lapic_get_reg(apic, APIC_LDR); 212 213 if (apic_x2apic_mode(apic)) { 214 new->mode |= KVM_APIC_MODE_X2APIC; 215 } else if (ldr) { 216 ldr = GET_APIC_LOGICAL_ID(ldr); 217 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT) 218 new->mode |= KVM_APIC_MODE_XAPIC_FLAT; 219 else 220 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER; 221 } 222 223 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask)) 224 continue; 225 226 if (mask) 227 cluster[ffs(mask) - 1] = apic; 228 } 229 out: 230 old = rcu_dereference_protected(kvm->arch.apic_map, 231 lockdep_is_held(&kvm->arch.apic_map_lock)); 232 rcu_assign_pointer(kvm->arch.apic_map, new); 233 mutex_unlock(&kvm->arch.apic_map_lock); 234 235 if (old) 236 call_rcu(&old->rcu, kvm_apic_map_free); 237 238 kvm_make_scan_ioapic_request(kvm); 239 } 240 241 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 242 { 243 bool enabled = val & APIC_SPIV_APIC_ENABLED; 244 245 kvm_lapic_set_reg(apic, APIC_SPIV, val); 246 247 if (enabled != apic->sw_enabled) { 248 apic->sw_enabled = enabled; 249 if (enabled) { 250 static_key_slow_dec_deferred(&apic_sw_disabled); 251 recalculate_apic_map(apic->vcpu->kvm); 252 } else 253 static_key_slow_inc(&apic_sw_disabled.key); 254 } 255 } 256 257 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) 258 { 259 kvm_lapic_set_reg(apic, APIC_ID, id << 24); 260 recalculate_apic_map(apic->vcpu->kvm); 261 } 262 263 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 264 { 265 kvm_lapic_set_reg(apic, APIC_LDR, id); 266 recalculate_apic_map(apic->vcpu->kvm); 267 } 268 269 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id) 270 { 271 return ((id >> 4) << 16) | (1 << (id & 0xf)); 272 } 273 274 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) 275 { 276 u32 ldr = kvm_apic_calc_x2apic_ldr(id); 277 278 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); 279 280 kvm_lapic_set_reg(apic, APIC_ID, id); 281 kvm_lapic_set_reg(apic, APIC_LDR, ldr); 282 recalculate_apic_map(apic->vcpu->kvm); 283 } 284 285 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 286 { 287 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 288 } 289 290 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) 291 { 292 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; 293 } 294 295 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 296 { 297 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; 298 } 299 300 static inline int apic_lvtt_period(struct kvm_lapic *apic) 301 { 302 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; 303 } 304 305 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 306 { 307 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; 308 } 309 310 static inline int apic_lvt_nmi_mode(u32 lvt_val) 311 { 312 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 313 } 314 315 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 316 { 317 struct kvm_lapic *apic = vcpu->arch.apic; 318 struct kvm_cpuid_entry2 *feat; 319 u32 v = APIC_VERSION; 320 321 if (!lapic_in_kernel(vcpu)) 322 return; 323 324 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); 325 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31)))) 326 v |= APIC_LVR_DIRECTED_EOI; 327 kvm_lapic_set_reg(apic, APIC_LVR, v); 328 } 329 330 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = { 331 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ 332 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ 333 LVT_MASK | APIC_MODE_MASK, /* LVTPC */ 334 LINT_MASK, LINT_MASK, /* LVT0-1 */ 335 LVT_MASK /* LVTERR */ 336 }; 337 338 static int find_highest_vector(void *bitmap) 339 { 340 int vec; 341 u32 *reg; 342 343 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 344 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 345 reg = bitmap + REG_POS(vec); 346 if (*reg) 347 return __fls(*reg) + vec; 348 } 349 350 return -1; 351 } 352 353 static u8 count_vectors(void *bitmap) 354 { 355 int vec; 356 u32 *reg; 357 u8 count = 0; 358 359 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 360 reg = bitmap + REG_POS(vec); 361 count += hweight32(*reg); 362 } 363 364 return count; 365 } 366 367 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr) 368 { 369 u32 i, vec; 370 u32 pir_val, irr_val, prev_irr_val; 371 int max_updated_irr; 372 373 max_updated_irr = -1; 374 *max_irr = -1; 375 376 for (i = vec = 0; i <= 7; i++, vec += 32) { 377 pir_val = READ_ONCE(pir[i]); 378 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10)); 379 if (pir_val) { 380 prev_irr_val = irr_val; 381 irr_val |= xchg(&pir[i], 0); 382 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val; 383 if (prev_irr_val != irr_val) { 384 max_updated_irr = 385 __fls(irr_val ^ prev_irr_val) + vec; 386 } 387 } 388 if (irr_val) 389 *max_irr = __fls(irr_val) + vec; 390 } 391 392 return ((max_updated_irr != -1) && 393 (max_updated_irr == *max_irr)); 394 } 395 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); 396 397 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr) 398 { 399 struct kvm_lapic *apic = vcpu->arch.apic; 400 401 return __kvm_apic_update_irr(pir, apic->regs, max_irr); 402 } 403 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 404 405 static inline int apic_search_irr(struct kvm_lapic *apic) 406 { 407 return find_highest_vector(apic->regs + APIC_IRR); 408 } 409 410 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 411 { 412 int result; 413 414 /* 415 * Note that irr_pending is just a hint. It will be always 416 * true with virtual interrupt delivery enabled. 417 */ 418 if (!apic->irr_pending) 419 return -1; 420 421 result = apic_search_irr(apic); 422 ASSERT(result == -1 || result >= 16); 423 424 return result; 425 } 426 427 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 428 { 429 struct kvm_vcpu *vcpu; 430 431 vcpu = apic->vcpu; 432 433 if (unlikely(vcpu->arch.apicv_active)) { 434 /* need to update RVI */ 435 apic_clear_vector(vec, apic->regs + APIC_IRR); 436 kvm_x86_ops->hwapic_irr_update(vcpu, 437 apic_find_highest_irr(apic)); 438 } else { 439 apic->irr_pending = false; 440 apic_clear_vector(vec, apic->regs + APIC_IRR); 441 if (apic_search_irr(apic) != -1) 442 apic->irr_pending = true; 443 } 444 } 445 446 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 447 { 448 struct kvm_vcpu *vcpu; 449 450 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 451 return; 452 453 vcpu = apic->vcpu; 454 455 /* 456 * With APIC virtualization enabled, all caching is disabled 457 * because the processor can modify ISR under the hood. Instead 458 * just set SVI. 459 */ 460 if (unlikely(vcpu->arch.apicv_active)) 461 kvm_x86_ops->hwapic_isr_update(vcpu, vec); 462 else { 463 ++apic->isr_count; 464 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 465 /* 466 * ISR (in service register) bit is set when injecting an interrupt. 467 * The highest vector is injected. Thus the latest bit set matches 468 * the highest bit in ISR. 469 */ 470 apic->highest_isr_cache = vec; 471 } 472 } 473 474 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 475 { 476 int result; 477 478 /* 479 * Note that isr_count is always 1, and highest_isr_cache 480 * is always -1, with APIC virtualization enabled. 481 */ 482 if (!apic->isr_count) 483 return -1; 484 if (likely(apic->highest_isr_cache != -1)) 485 return apic->highest_isr_cache; 486 487 result = find_highest_vector(apic->regs + APIC_ISR); 488 ASSERT(result == -1 || result >= 16); 489 490 return result; 491 } 492 493 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 494 { 495 struct kvm_vcpu *vcpu; 496 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 497 return; 498 499 vcpu = apic->vcpu; 500 501 /* 502 * We do get here for APIC virtualization enabled if the guest 503 * uses the Hyper-V APIC enlightenment. In this case we may need 504 * to trigger a new interrupt delivery by writing the SVI field; 505 * on the other hand isr_count and highest_isr_cache are unused 506 * and must be left alone. 507 */ 508 if (unlikely(vcpu->arch.apicv_active)) 509 kvm_x86_ops->hwapic_isr_update(vcpu, 510 apic_find_highest_isr(apic)); 511 else { 512 --apic->isr_count; 513 BUG_ON(apic->isr_count < 0); 514 apic->highest_isr_cache = -1; 515 } 516 } 517 518 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 519 { 520 /* This may race with setting of irr in __apic_accept_irq() and 521 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 522 * will cause vmexit immediately and the value will be recalculated 523 * on the next vmentry. 524 */ 525 return apic_find_highest_irr(vcpu->arch.apic); 526 } 527 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); 528 529 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 530 int vector, int level, int trig_mode, 531 struct dest_map *dest_map); 532 533 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 534 struct dest_map *dest_map) 535 { 536 struct kvm_lapic *apic = vcpu->arch.apic; 537 538 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 539 irq->level, irq->trig_mode, dest_map); 540 } 541 542 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 543 { 544 545 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 546 sizeof(val)); 547 } 548 549 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 550 { 551 552 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 553 sizeof(*val)); 554 } 555 556 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 557 { 558 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 559 } 560 561 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu) 562 { 563 u8 val; 564 if (pv_eoi_get_user(vcpu, &val) < 0) 565 apic_debug("Can't read EOI MSR value: 0x%llx\n", 566 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 567 return val & 0x1; 568 } 569 570 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 571 { 572 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) { 573 apic_debug("Can't set EOI MSR value: 0x%llx\n", 574 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 575 return; 576 } 577 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 578 } 579 580 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu) 581 { 582 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) { 583 apic_debug("Can't clear EOI MSR value: 0x%llx\n", 584 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 585 return; 586 } 587 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 588 } 589 590 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) 591 { 592 int highest_irr; 593 if (apic->vcpu->arch.apicv_active) 594 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu); 595 else 596 highest_irr = apic_find_highest_irr(apic); 597 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) 598 return -1; 599 return highest_irr; 600 } 601 602 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr) 603 { 604 u32 tpr, isrv, ppr, old_ppr; 605 int isr; 606 607 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI); 608 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI); 609 isr = apic_find_highest_isr(apic); 610 isrv = (isr != -1) ? isr : 0; 611 612 if ((tpr & 0xf0) >= (isrv & 0xf0)) 613 ppr = tpr & 0xff; 614 else 615 ppr = isrv & 0xf0; 616 617 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x", 618 apic, ppr, isr, isrv); 619 620 *new_ppr = ppr; 621 if (old_ppr != ppr) 622 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr); 623 624 return ppr < old_ppr; 625 } 626 627 static void apic_update_ppr(struct kvm_lapic *apic) 628 { 629 u32 ppr; 630 631 if (__apic_update_ppr(apic, &ppr) && 632 apic_has_interrupt_for_ppr(apic, ppr) != -1) 633 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 634 } 635 636 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu) 637 { 638 apic_update_ppr(vcpu->arch.apic); 639 } 640 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr); 641 642 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 643 { 644 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr); 645 apic_update_ppr(apic); 646 } 647 648 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda) 649 { 650 return mda == (apic_x2apic_mode(apic) ? 651 X2APIC_BROADCAST : APIC_BROADCAST); 652 } 653 654 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) 655 { 656 if (kvm_apic_broadcast(apic, mda)) 657 return true; 658 659 if (apic_x2apic_mode(apic)) 660 return mda == kvm_x2apic_id(apic); 661 662 /* 663 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if 664 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and 665 * this allows unique addressing of VCPUs with APIC ID over 0xff. 666 * The 0xff condition is needed because writeable xAPIC ID. 667 */ 668 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic)) 669 return true; 670 671 return mda == kvm_xapic_id(apic); 672 } 673 674 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 675 { 676 u32 logical_id; 677 678 if (kvm_apic_broadcast(apic, mda)) 679 return true; 680 681 logical_id = kvm_lapic_get_reg(apic, APIC_LDR); 682 683 if (apic_x2apic_mode(apic)) 684 return ((logical_id >> 16) == (mda >> 16)) 685 && (logical_id & mda & 0xffff) != 0; 686 687 logical_id = GET_APIC_LOGICAL_ID(logical_id); 688 689 switch (kvm_lapic_get_reg(apic, APIC_DFR)) { 690 case APIC_DFR_FLAT: 691 return (logical_id & mda) != 0; 692 case APIC_DFR_CLUSTER: 693 return ((logical_id >> 4) == (mda >> 4)) 694 && (logical_id & mda & 0xf) != 0; 695 default: 696 apic_debug("Bad DFR vcpu %d: %08x\n", 697 apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR)); 698 return false; 699 } 700 } 701 702 /* The KVM local APIC implementation has two quirks: 703 * 704 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs 705 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID. 706 * KVM doesn't do that aliasing. 707 * 708 * - in-kernel IOAPIC messages have to be delivered directly to 709 * x2APIC, because the kernel does not support interrupt remapping. 710 * In order to support broadcast without interrupt remapping, x2APIC 711 * rewrites the destination of non-IPI messages from APIC_BROADCAST 712 * to X2APIC_BROADCAST. 713 * 714 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is 715 * important when userspace wants to use x2APIC-format MSIs, because 716 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7". 717 */ 718 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id, 719 struct kvm_lapic *source, struct kvm_lapic *target) 720 { 721 bool ipi = source != NULL; 722 723 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && 724 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target)) 725 return X2APIC_BROADCAST; 726 727 return dest_id; 728 } 729 730 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 731 int short_hand, unsigned int dest, int dest_mode) 732 { 733 struct kvm_lapic *target = vcpu->arch.apic; 734 u32 mda = kvm_apic_mda(vcpu, dest, source, target); 735 736 apic_debug("target %p, source %p, dest 0x%x, " 737 "dest_mode 0x%x, short_hand 0x%x\n", 738 target, source, dest, dest_mode, short_hand); 739 740 ASSERT(target); 741 switch (short_hand) { 742 case APIC_DEST_NOSHORT: 743 if (dest_mode == APIC_DEST_PHYSICAL) 744 return kvm_apic_match_physical_addr(target, mda); 745 else 746 return kvm_apic_match_logical_addr(target, mda); 747 case APIC_DEST_SELF: 748 return target == source; 749 case APIC_DEST_ALLINC: 750 return true; 751 case APIC_DEST_ALLBUT: 752 return target != source; 753 default: 754 apic_debug("kvm: apic: Bad dest shorthand value %x\n", 755 short_hand); 756 return false; 757 } 758 } 759 EXPORT_SYMBOL_GPL(kvm_apic_match_dest); 760 761 int kvm_vector_to_index(u32 vector, u32 dest_vcpus, 762 const unsigned long *bitmap, u32 bitmap_size) 763 { 764 u32 mod; 765 int i, idx = -1; 766 767 mod = vector % dest_vcpus; 768 769 for (i = 0; i <= mod; i++) { 770 idx = find_next_bit(bitmap, bitmap_size, idx + 1); 771 BUG_ON(idx == bitmap_size); 772 } 773 774 return idx; 775 } 776 777 static void kvm_apic_disabled_lapic_found(struct kvm *kvm) 778 { 779 if (!kvm->arch.disabled_lapic_found) { 780 kvm->arch.disabled_lapic_found = true; 781 printk(KERN_INFO 782 "Disabled LAPIC found during irq injection\n"); 783 } 784 } 785 786 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src, 787 struct kvm_lapic_irq *irq, struct kvm_apic_map *map) 788 { 789 if (kvm->arch.x2apic_broadcast_quirk_disabled) { 790 if ((irq->dest_id == APIC_BROADCAST && 791 map->mode != KVM_APIC_MODE_X2APIC)) 792 return true; 793 if (irq->dest_id == X2APIC_BROADCAST) 794 return true; 795 } else { 796 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src); 797 if (irq->dest_id == (x2apic_ipi ? 798 X2APIC_BROADCAST : APIC_BROADCAST)) 799 return true; 800 } 801 802 return false; 803 } 804 805 /* Return true if the interrupt can be handled by using *bitmap as index mask 806 * for valid destinations in *dst array. 807 * Return false if kvm_apic_map_get_dest_lapic did nothing useful. 808 * Note: we may have zero kvm_lapic destinations when we return true, which 809 * means that the interrupt should be dropped. In this case, *bitmap would be 810 * zero and *dst undefined. 811 */ 812 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm, 813 struct kvm_lapic **src, struct kvm_lapic_irq *irq, 814 struct kvm_apic_map *map, struct kvm_lapic ***dst, 815 unsigned long *bitmap) 816 { 817 int i, lowest; 818 819 if (irq->shorthand == APIC_DEST_SELF && src) { 820 *dst = src; 821 *bitmap = 1; 822 return true; 823 } else if (irq->shorthand) 824 return false; 825 826 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map)) 827 return false; 828 829 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 830 if (irq->dest_id > map->max_apic_id) { 831 *bitmap = 0; 832 } else { 833 *dst = &map->phys_map[irq->dest_id]; 834 *bitmap = 1; 835 } 836 return true; 837 } 838 839 *bitmap = 0; 840 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, 841 (u16 *)bitmap)) 842 return false; 843 844 if (!kvm_lowest_prio_delivery(irq)) 845 return true; 846 847 if (!kvm_vector_hashing_enabled()) { 848 lowest = -1; 849 for_each_set_bit(i, bitmap, 16) { 850 if (!(*dst)[i]) 851 continue; 852 if (lowest < 0) 853 lowest = i; 854 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, 855 (*dst)[lowest]->vcpu) < 0) 856 lowest = i; 857 } 858 } else { 859 if (!*bitmap) 860 return true; 861 862 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), 863 bitmap, 16); 864 865 if (!(*dst)[lowest]) { 866 kvm_apic_disabled_lapic_found(kvm); 867 *bitmap = 0; 868 return true; 869 } 870 } 871 872 *bitmap = (lowest >= 0) ? 1 << lowest : 0; 873 874 return true; 875 } 876 877 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 878 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map) 879 { 880 struct kvm_apic_map *map; 881 unsigned long bitmap; 882 struct kvm_lapic **dst = NULL; 883 int i; 884 bool ret; 885 886 *r = -1; 887 888 if (irq->shorthand == APIC_DEST_SELF) { 889 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 890 return true; 891 } 892 893 rcu_read_lock(); 894 map = rcu_dereference(kvm->arch.apic_map); 895 896 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap); 897 if (ret) 898 for_each_set_bit(i, &bitmap, 16) { 899 if (!dst[i]) 900 continue; 901 if (*r < 0) 902 *r = 0; 903 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 904 } 905 906 rcu_read_unlock(); 907 return ret; 908 } 909 910 /* 911 * This routine tries to handler interrupts in posted mode, here is how 912 * it deals with different cases: 913 * - For single-destination interrupts, handle it in posted mode 914 * - Else if vector hashing is enabled and it is a lowest-priority 915 * interrupt, handle it in posted mode and use the following mechanism 916 * to find the destinaiton vCPU. 917 * 1. For lowest-priority interrupts, store all the possible 918 * destination vCPUs in an array. 919 * 2. Use "guest vector % max number of destination vCPUs" to find 920 * the right destination vCPU in the array for the lowest-priority 921 * interrupt. 922 * - Otherwise, use remapped mode to inject the interrupt. 923 */ 924 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq, 925 struct kvm_vcpu **dest_vcpu) 926 { 927 struct kvm_apic_map *map; 928 unsigned long bitmap; 929 struct kvm_lapic **dst = NULL; 930 bool ret = false; 931 932 if (irq->shorthand) 933 return false; 934 935 rcu_read_lock(); 936 map = rcu_dereference(kvm->arch.apic_map); 937 938 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) && 939 hweight16(bitmap) == 1) { 940 unsigned long i = find_first_bit(&bitmap, 16); 941 942 if (dst[i]) { 943 *dest_vcpu = dst[i]->vcpu; 944 ret = true; 945 } 946 } 947 948 rcu_read_unlock(); 949 return ret; 950 } 951 952 /* 953 * Add a pending IRQ into lapic. 954 * Return 1 if successfully added and 0 if discarded. 955 */ 956 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 957 int vector, int level, int trig_mode, 958 struct dest_map *dest_map) 959 { 960 int result = 0; 961 struct kvm_vcpu *vcpu = apic->vcpu; 962 963 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 964 trig_mode, vector); 965 switch (delivery_mode) { 966 case APIC_DM_LOWEST: 967 vcpu->arch.apic_arb_prio++; 968 case APIC_DM_FIXED: 969 if (unlikely(trig_mode && !level)) 970 break; 971 972 /* FIXME add logic for vcpu on reset */ 973 if (unlikely(!apic_enabled(apic))) 974 break; 975 976 result = 1; 977 978 if (dest_map) { 979 __set_bit(vcpu->vcpu_id, dest_map->map); 980 dest_map->vectors[vcpu->vcpu_id] = vector; 981 } 982 983 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { 984 if (trig_mode) 985 kvm_lapic_set_vector(vector, apic->regs + APIC_TMR); 986 else 987 apic_clear_vector(vector, apic->regs + APIC_TMR); 988 } 989 990 if (vcpu->arch.apicv_active) 991 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector); 992 else { 993 kvm_lapic_set_irr(vector, apic); 994 995 kvm_make_request(KVM_REQ_EVENT, vcpu); 996 kvm_vcpu_kick(vcpu); 997 } 998 break; 999 1000 case APIC_DM_REMRD: 1001 result = 1; 1002 vcpu->arch.pv.pv_unhalted = 1; 1003 kvm_make_request(KVM_REQ_EVENT, vcpu); 1004 kvm_vcpu_kick(vcpu); 1005 break; 1006 1007 case APIC_DM_SMI: 1008 result = 1; 1009 kvm_make_request(KVM_REQ_SMI, vcpu); 1010 kvm_vcpu_kick(vcpu); 1011 break; 1012 1013 case APIC_DM_NMI: 1014 result = 1; 1015 kvm_inject_nmi(vcpu); 1016 kvm_vcpu_kick(vcpu); 1017 break; 1018 1019 case APIC_DM_INIT: 1020 if (!trig_mode || level) { 1021 result = 1; 1022 /* assumes that there are only KVM_APIC_INIT/SIPI */ 1023 apic->pending_events = (1UL << KVM_APIC_INIT); 1024 /* make sure pending_events is visible before sending 1025 * the request */ 1026 smp_wmb(); 1027 kvm_make_request(KVM_REQ_EVENT, vcpu); 1028 kvm_vcpu_kick(vcpu); 1029 } else { 1030 apic_debug("Ignoring de-assert INIT to vcpu %d\n", 1031 vcpu->vcpu_id); 1032 } 1033 break; 1034 1035 case APIC_DM_STARTUP: 1036 apic_debug("SIPI to vcpu %d vector 0x%02x\n", 1037 vcpu->vcpu_id, vector); 1038 result = 1; 1039 apic->sipi_vector = vector; 1040 /* make sure sipi_vector is visible for the receiver */ 1041 smp_wmb(); 1042 set_bit(KVM_APIC_SIPI, &apic->pending_events); 1043 kvm_make_request(KVM_REQ_EVENT, vcpu); 1044 kvm_vcpu_kick(vcpu); 1045 break; 1046 1047 case APIC_DM_EXTINT: 1048 /* 1049 * Should only be called by kvm_apic_local_deliver() with LVT0, 1050 * before NMI watchdog was enabled. Already handled by 1051 * kvm_apic_accept_pic_intr(). 1052 */ 1053 break; 1054 1055 default: 1056 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 1057 delivery_mode); 1058 break; 1059 } 1060 return result; 1061 } 1062 1063 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 1064 { 1065 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 1066 } 1067 1068 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) 1069 { 1070 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); 1071 } 1072 1073 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 1074 { 1075 int trigger_mode; 1076 1077 /* Eoi the ioapic only if the ioapic doesn't own the vector. */ 1078 if (!kvm_ioapic_handles_vector(apic, vector)) 1079 return; 1080 1081 /* Request a KVM exit to inform the userspace IOAPIC. */ 1082 if (irqchip_split(apic->vcpu->kvm)) { 1083 apic->vcpu->arch.pending_ioapic_eoi = vector; 1084 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); 1085 return; 1086 } 1087 1088 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 1089 trigger_mode = IOAPIC_LEVEL_TRIG; 1090 else 1091 trigger_mode = IOAPIC_EDGE_TRIG; 1092 1093 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 1094 } 1095 1096 static int apic_set_eoi(struct kvm_lapic *apic) 1097 { 1098 int vector = apic_find_highest_isr(apic); 1099 1100 trace_kvm_eoi(apic, vector); 1101 1102 /* 1103 * Not every write EOI will has corresponding ISR, 1104 * one example is when Kernel check timer on setup_IO_APIC 1105 */ 1106 if (vector == -1) 1107 return vector; 1108 1109 apic_clear_isr(vector, apic); 1110 apic_update_ppr(apic); 1111 1112 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap)) 1113 kvm_hv_synic_send_eoi(apic->vcpu, vector); 1114 1115 kvm_ioapic_send_eoi(apic, vector); 1116 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1117 return vector; 1118 } 1119 1120 /* 1121 * this interface assumes a trap-like exit, which has already finished 1122 * desired side effect including vISR and vPPR update. 1123 */ 1124 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 1125 { 1126 struct kvm_lapic *apic = vcpu->arch.apic; 1127 1128 trace_kvm_eoi(apic, vector); 1129 1130 kvm_ioapic_send_eoi(apic, vector); 1131 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1132 } 1133 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 1134 1135 static void apic_send_ipi(struct kvm_lapic *apic) 1136 { 1137 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR); 1138 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2); 1139 struct kvm_lapic_irq irq; 1140 1141 irq.vector = icr_low & APIC_VECTOR_MASK; 1142 irq.delivery_mode = icr_low & APIC_MODE_MASK; 1143 irq.dest_mode = icr_low & APIC_DEST_MASK; 1144 irq.level = (icr_low & APIC_INT_ASSERT) != 0; 1145 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 1146 irq.shorthand = icr_low & APIC_SHORT_MASK; 1147 irq.msi_redir_hint = false; 1148 if (apic_x2apic_mode(apic)) 1149 irq.dest_id = icr_high; 1150 else 1151 irq.dest_id = GET_APIC_DEST_FIELD(icr_high); 1152 1153 trace_kvm_apic_ipi(icr_low, irq.dest_id); 1154 1155 apic_debug("icr_high 0x%x, icr_low 0x%x, " 1156 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, " 1157 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, " 1158 "msi_redir_hint 0x%x\n", 1159 icr_high, icr_low, irq.shorthand, irq.dest_id, 1160 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode, 1161 irq.vector, irq.msi_redir_hint); 1162 1163 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 1164 } 1165 1166 static u32 apic_get_tmcct(struct kvm_lapic *apic) 1167 { 1168 ktime_t remaining, now; 1169 s64 ns; 1170 u32 tmcct; 1171 1172 ASSERT(apic != NULL); 1173 1174 /* if initial count is 0, current count should also be 0 */ 1175 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 || 1176 apic->lapic_timer.period == 0) 1177 return 0; 1178 1179 now = ktime_get(); 1180 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1181 if (ktime_to_ns(remaining) < 0) 1182 remaining = 0; 1183 1184 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 1185 tmcct = div64_u64(ns, 1186 (APIC_BUS_CYCLE_NS * apic->divide_count)); 1187 1188 return tmcct; 1189 } 1190 1191 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 1192 { 1193 struct kvm_vcpu *vcpu = apic->vcpu; 1194 struct kvm_run *run = vcpu->run; 1195 1196 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 1197 run->tpr_access.rip = kvm_rip_read(vcpu); 1198 run->tpr_access.is_write = write; 1199 } 1200 1201 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 1202 { 1203 if (apic->vcpu->arch.tpr_access_reporting) 1204 __report_tpr_access(apic, write); 1205 } 1206 1207 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 1208 { 1209 u32 val = 0; 1210 1211 if (offset >= LAPIC_MMIO_LENGTH) 1212 return 0; 1213 1214 switch (offset) { 1215 case APIC_ARBPRI: 1216 apic_debug("Access APIC ARBPRI register which is for P6\n"); 1217 break; 1218 1219 case APIC_TMCCT: /* Timer CCR */ 1220 if (apic_lvtt_tscdeadline(apic)) 1221 return 0; 1222 1223 val = apic_get_tmcct(apic); 1224 break; 1225 case APIC_PROCPRI: 1226 apic_update_ppr(apic); 1227 val = kvm_lapic_get_reg(apic, offset); 1228 break; 1229 case APIC_TASKPRI: 1230 report_tpr_access(apic, false); 1231 /* fall thru */ 1232 default: 1233 val = kvm_lapic_get_reg(apic, offset); 1234 break; 1235 } 1236 1237 return val; 1238 } 1239 1240 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 1241 { 1242 return container_of(dev, struct kvm_lapic, dev); 1243 } 1244 1245 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 1246 void *data) 1247 { 1248 unsigned char alignment = offset & 0xf; 1249 u32 result; 1250 /* this bitmask has a bit cleared for each reserved register */ 1251 static const u64 rmask = 0x43ff01ffffffe70cULL; 1252 1253 if ((alignment + len) > 4) { 1254 apic_debug("KVM_APIC_READ: alignment error %x %d\n", 1255 offset, len); 1256 return 1; 1257 } 1258 1259 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) { 1260 apic_debug("KVM_APIC_READ: read reserved register %x\n", 1261 offset); 1262 return 1; 1263 } 1264 1265 result = __apic_read(apic, offset & ~0xf); 1266 1267 trace_kvm_apic_read(offset, result); 1268 1269 switch (len) { 1270 case 1: 1271 case 2: 1272 case 4: 1273 memcpy(data, (char *)&result + alignment, len); 1274 break; 1275 default: 1276 printk(KERN_ERR "Local APIC read with len = %x, " 1277 "should be 1,2, or 4 instead\n", len); 1278 break; 1279 } 1280 return 0; 1281 } 1282 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read); 1283 1284 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1285 { 1286 return kvm_apic_hw_enabled(apic) && 1287 addr >= apic->base_address && 1288 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1289 } 1290 1291 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1292 gpa_t address, int len, void *data) 1293 { 1294 struct kvm_lapic *apic = to_lapic(this); 1295 u32 offset = address - apic->base_address; 1296 1297 if (!apic_mmio_in_range(apic, address)) 1298 return -EOPNOTSUPP; 1299 1300 kvm_lapic_reg_read(apic, offset, len, data); 1301 1302 return 0; 1303 } 1304 1305 static void update_divide_count(struct kvm_lapic *apic) 1306 { 1307 u32 tmp1, tmp2, tdcr; 1308 1309 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR); 1310 tmp1 = tdcr & 0xf; 1311 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1312 apic->divide_count = 0x1 << (tmp2 & 0x7); 1313 1314 apic_debug("timer divide count is 0x%x\n", 1315 apic->divide_count); 1316 } 1317 1318 static void limit_periodic_timer_frequency(struct kvm_lapic *apic) 1319 { 1320 /* 1321 * Do not allow the guest to program periodic timers with small 1322 * interval, since the hrtimers are not throttled by the host 1323 * scheduler. 1324 */ 1325 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1326 s64 min_period = min_timer_period_us * 1000LL; 1327 1328 if (apic->lapic_timer.period < min_period) { 1329 pr_info_ratelimited( 1330 "kvm: vcpu %i: requested %lld ns " 1331 "lapic timer period limited to %lld ns\n", 1332 apic->vcpu->vcpu_id, 1333 apic->lapic_timer.period, min_period); 1334 apic->lapic_timer.period = min_period; 1335 } 1336 } 1337 } 1338 1339 static void apic_update_lvtt(struct kvm_lapic *apic) 1340 { 1341 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & 1342 apic->lapic_timer.timer_mode_mask; 1343 1344 if (apic->lapic_timer.timer_mode != timer_mode) { 1345 if (apic_lvtt_tscdeadline(apic) != (timer_mode == 1346 APIC_LVT_TIMER_TSCDEADLINE)) { 1347 hrtimer_cancel(&apic->lapic_timer.timer); 1348 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 1349 apic->lapic_timer.period = 0; 1350 apic->lapic_timer.tscdeadline = 0; 1351 } 1352 apic->lapic_timer.timer_mode = timer_mode; 1353 limit_periodic_timer_frequency(apic); 1354 } 1355 } 1356 1357 static void apic_timer_expired(struct kvm_lapic *apic) 1358 { 1359 struct kvm_vcpu *vcpu = apic->vcpu; 1360 struct swait_queue_head *q = &vcpu->wq; 1361 struct kvm_timer *ktimer = &apic->lapic_timer; 1362 1363 if (atomic_read(&apic->lapic_timer.pending)) 1364 return; 1365 1366 atomic_inc(&apic->lapic_timer.pending); 1367 kvm_set_pending_timer(vcpu); 1368 1369 /* 1370 * For x86, the atomic_inc() is serialized, thus 1371 * using swait_active() is safe. 1372 */ 1373 if (swait_active(q)) 1374 swake_up(q); 1375 1376 if (apic_lvtt_tscdeadline(apic)) 1377 ktimer->expired_tscdeadline = ktimer->tscdeadline; 1378 } 1379 1380 /* 1381 * On APICv, this test will cause a busy wait 1382 * during a higher-priority task. 1383 */ 1384 1385 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu) 1386 { 1387 struct kvm_lapic *apic = vcpu->arch.apic; 1388 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT); 1389 1390 if (kvm_apic_hw_enabled(apic)) { 1391 int vec = reg & APIC_VECTOR_MASK; 1392 void *bitmap = apic->regs + APIC_ISR; 1393 1394 if (vcpu->arch.apicv_active) 1395 bitmap = apic->regs + APIC_IRR; 1396 1397 if (apic_test_vector(vec, bitmap)) 1398 return true; 1399 } 1400 return false; 1401 } 1402 1403 void wait_lapic_expire(struct kvm_vcpu *vcpu) 1404 { 1405 struct kvm_lapic *apic = vcpu->arch.apic; 1406 u64 guest_tsc, tsc_deadline; 1407 1408 if (!lapic_in_kernel(vcpu)) 1409 return; 1410 1411 if (apic->lapic_timer.expired_tscdeadline == 0) 1412 return; 1413 1414 if (!lapic_timer_int_injected(vcpu)) 1415 return; 1416 1417 tsc_deadline = apic->lapic_timer.expired_tscdeadline; 1418 apic->lapic_timer.expired_tscdeadline = 0; 1419 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1420 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); 1421 1422 /* __delay is delay_tsc whenever the hardware has TSC, thus always. */ 1423 if (guest_tsc < tsc_deadline) 1424 __delay(min(tsc_deadline - guest_tsc, 1425 nsec_to_cycles(vcpu, lapic_timer_advance_ns))); 1426 } 1427 1428 static void start_sw_tscdeadline(struct kvm_lapic *apic) 1429 { 1430 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline; 1431 u64 ns = 0; 1432 ktime_t expire; 1433 struct kvm_vcpu *vcpu = apic->vcpu; 1434 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1435 unsigned long flags; 1436 ktime_t now; 1437 1438 if (unlikely(!tscdeadline || !this_tsc_khz)) 1439 return; 1440 1441 local_irq_save(flags); 1442 1443 now = ktime_get(); 1444 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1445 if (likely(tscdeadline > guest_tsc)) { 1446 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1447 do_div(ns, this_tsc_khz); 1448 expire = ktime_add_ns(now, ns); 1449 expire = ktime_sub_ns(expire, lapic_timer_advance_ns); 1450 hrtimer_start(&apic->lapic_timer.timer, 1451 expire, HRTIMER_MODE_ABS_PINNED); 1452 } else 1453 apic_timer_expired(apic); 1454 1455 local_irq_restore(flags); 1456 } 1457 1458 static void start_sw_period(struct kvm_lapic *apic) 1459 { 1460 if (!apic->lapic_timer.period) 1461 return; 1462 1463 if (apic_lvtt_oneshot(apic) && 1464 ktime_after(ktime_get(), 1465 apic->lapic_timer.target_expiration)) { 1466 apic_timer_expired(apic); 1467 return; 1468 } 1469 1470 hrtimer_start(&apic->lapic_timer.timer, 1471 apic->lapic_timer.target_expiration, 1472 HRTIMER_MODE_ABS_PINNED); 1473 } 1474 1475 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) 1476 { 1477 ktime_t now, remaining; 1478 u64 ns_remaining_old, ns_remaining_new; 1479 1480 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1481 * APIC_BUS_CYCLE_NS * apic->divide_count; 1482 limit_periodic_timer_frequency(apic); 1483 1484 now = ktime_get(); 1485 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1486 if (ktime_to_ns(remaining) < 0) 1487 remaining = 0; 1488 1489 ns_remaining_old = ktime_to_ns(remaining); 1490 ns_remaining_new = mul_u64_u32_div(ns_remaining_old, 1491 apic->divide_count, old_divisor); 1492 1493 apic->lapic_timer.tscdeadline += 1494 nsec_to_cycles(apic->vcpu, ns_remaining_new) - 1495 nsec_to_cycles(apic->vcpu, ns_remaining_old); 1496 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); 1497 } 1498 1499 static bool set_target_expiration(struct kvm_lapic *apic) 1500 { 1501 ktime_t now; 1502 u64 tscl = rdtsc(); 1503 1504 now = ktime_get(); 1505 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1506 * APIC_BUS_CYCLE_NS * apic->divide_count; 1507 1508 if (!apic->lapic_timer.period) { 1509 apic->lapic_timer.tscdeadline = 0; 1510 return false; 1511 } 1512 1513 limit_periodic_timer_frequency(apic); 1514 1515 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" 1516 PRIx64 ", " 1517 "timer initial count 0x%x, period %lldns, " 1518 "expire @ 0x%016" PRIx64 ".\n", __func__, 1519 APIC_BUS_CYCLE_NS, ktime_to_ns(now), 1520 kvm_lapic_get_reg(apic, APIC_TMICT), 1521 apic->lapic_timer.period, 1522 ktime_to_ns(ktime_add_ns(now, 1523 apic->lapic_timer.period))); 1524 1525 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 1526 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); 1527 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period); 1528 1529 return true; 1530 } 1531 1532 static void advance_periodic_target_expiration(struct kvm_lapic *apic) 1533 { 1534 apic->lapic_timer.tscdeadline += 1535 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); 1536 apic->lapic_timer.target_expiration = 1537 ktime_add_ns(apic->lapic_timer.target_expiration, 1538 apic->lapic_timer.period); 1539 } 1540 1541 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) 1542 { 1543 if (!lapic_in_kernel(vcpu)) 1544 return false; 1545 1546 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; 1547 } 1548 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use); 1549 1550 static void cancel_hv_timer(struct kvm_lapic *apic) 1551 { 1552 WARN_ON(preemptible()); 1553 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1554 kvm_x86_ops->cancel_hv_timer(apic->vcpu); 1555 apic->lapic_timer.hv_timer_in_use = false; 1556 } 1557 1558 static bool start_hv_timer(struct kvm_lapic *apic) 1559 { 1560 struct kvm_timer *ktimer = &apic->lapic_timer; 1561 int r; 1562 1563 WARN_ON(preemptible()); 1564 if (!kvm_x86_ops->set_hv_timer) 1565 return false; 1566 1567 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 1568 return false; 1569 1570 if (!ktimer->tscdeadline) 1571 return false; 1572 1573 r = kvm_x86_ops->set_hv_timer(apic->vcpu, ktimer->tscdeadline); 1574 if (r < 0) 1575 return false; 1576 1577 ktimer->hv_timer_in_use = true; 1578 hrtimer_cancel(&ktimer->timer); 1579 1580 /* 1581 * Also recheck ktimer->pending, in case the sw timer triggered in 1582 * the window. For periodic timer, leave the hv timer running for 1583 * simplicity, and the deadline will be recomputed on the next vmexit. 1584 */ 1585 if (!apic_lvtt_period(apic) && (r || atomic_read(&ktimer->pending))) { 1586 if (r) 1587 apic_timer_expired(apic); 1588 return false; 1589 } 1590 1591 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, true); 1592 return true; 1593 } 1594 1595 static void start_sw_timer(struct kvm_lapic *apic) 1596 { 1597 struct kvm_timer *ktimer = &apic->lapic_timer; 1598 1599 WARN_ON(preemptible()); 1600 if (apic->lapic_timer.hv_timer_in_use) 1601 cancel_hv_timer(apic); 1602 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 1603 return; 1604 1605 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1606 start_sw_period(apic); 1607 else if (apic_lvtt_tscdeadline(apic)) 1608 start_sw_tscdeadline(apic); 1609 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); 1610 } 1611 1612 static void restart_apic_timer(struct kvm_lapic *apic) 1613 { 1614 preempt_disable(); 1615 if (!start_hv_timer(apic)) 1616 start_sw_timer(apic); 1617 preempt_enable(); 1618 } 1619 1620 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) 1621 { 1622 struct kvm_lapic *apic = vcpu->arch.apic; 1623 1624 preempt_disable(); 1625 /* If the preempt notifier has already run, it also called apic_timer_expired */ 1626 if (!apic->lapic_timer.hv_timer_in_use) 1627 goto out; 1628 WARN_ON(swait_active(&vcpu->wq)); 1629 cancel_hv_timer(apic); 1630 apic_timer_expired(apic); 1631 1632 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1633 advance_periodic_target_expiration(apic); 1634 restart_apic_timer(apic); 1635 } 1636 out: 1637 preempt_enable(); 1638 } 1639 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); 1640 1641 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) 1642 { 1643 restart_apic_timer(vcpu->arch.apic); 1644 } 1645 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer); 1646 1647 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) 1648 { 1649 struct kvm_lapic *apic = vcpu->arch.apic; 1650 1651 preempt_disable(); 1652 /* Possibly the TSC deadline timer is not enabled yet */ 1653 if (apic->lapic_timer.hv_timer_in_use) 1654 start_sw_timer(apic); 1655 preempt_enable(); 1656 } 1657 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer); 1658 1659 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) 1660 { 1661 struct kvm_lapic *apic = vcpu->arch.apic; 1662 1663 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1664 restart_apic_timer(apic); 1665 } 1666 1667 static void start_apic_timer(struct kvm_lapic *apic) 1668 { 1669 atomic_set(&apic->lapic_timer.pending, 0); 1670 1671 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1672 && !set_target_expiration(apic)) 1673 return; 1674 1675 restart_apic_timer(apic); 1676 } 1677 1678 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 1679 { 1680 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val); 1681 1682 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { 1683 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; 1684 if (lvt0_in_nmi_mode) { 1685 apic_debug("Receive NMI setting on APIC_LVT0 " 1686 "for cpu %d\n", apic->vcpu->vcpu_id); 1687 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1688 } else 1689 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1690 } 1691 } 1692 1693 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 1694 { 1695 int ret = 0; 1696 1697 trace_kvm_apic_write(reg, val); 1698 1699 switch (reg) { 1700 case APIC_ID: /* Local APIC ID */ 1701 if (!apic_x2apic_mode(apic)) 1702 kvm_apic_set_xapic_id(apic, val >> 24); 1703 else 1704 ret = 1; 1705 break; 1706 1707 case APIC_TASKPRI: 1708 report_tpr_access(apic, true); 1709 apic_set_tpr(apic, val & 0xff); 1710 break; 1711 1712 case APIC_EOI: 1713 apic_set_eoi(apic); 1714 break; 1715 1716 case APIC_LDR: 1717 if (!apic_x2apic_mode(apic)) 1718 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 1719 else 1720 ret = 1; 1721 break; 1722 1723 case APIC_DFR: 1724 if (!apic_x2apic_mode(apic)) { 1725 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); 1726 recalculate_apic_map(apic->vcpu->kvm); 1727 } else 1728 ret = 1; 1729 break; 1730 1731 case APIC_SPIV: { 1732 u32 mask = 0x3ff; 1733 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 1734 mask |= APIC_SPIV_DIRECTED_EOI; 1735 apic_set_spiv(apic, val & mask); 1736 if (!(val & APIC_SPIV_APIC_ENABLED)) { 1737 int i; 1738 u32 lvt_val; 1739 1740 for (i = 0; i < KVM_APIC_LVT_NUM; i++) { 1741 lvt_val = kvm_lapic_get_reg(apic, 1742 APIC_LVTT + 0x10 * i); 1743 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, 1744 lvt_val | APIC_LVT_MASKED); 1745 } 1746 apic_update_lvtt(apic); 1747 atomic_set(&apic->lapic_timer.pending, 0); 1748 1749 } 1750 break; 1751 } 1752 case APIC_ICR: 1753 /* No delay here, so we always clear the pending bit */ 1754 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12)); 1755 apic_send_ipi(apic); 1756 break; 1757 1758 case APIC_ICR2: 1759 if (!apic_x2apic_mode(apic)) 1760 val &= 0xff000000; 1761 kvm_lapic_set_reg(apic, APIC_ICR2, val); 1762 break; 1763 1764 case APIC_LVT0: 1765 apic_manage_nmi_watchdog(apic, val); 1766 case APIC_LVTTHMR: 1767 case APIC_LVTPC: 1768 case APIC_LVT1: 1769 case APIC_LVTERR: 1770 /* TODO: Check vector */ 1771 if (!kvm_apic_sw_enabled(apic)) 1772 val |= APIC_LVT_MASKED; 1773 1774 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; 1775 kvm_lapic_set_reg(apic, reg, val); 1776 1777 break; 1778 1779 case APIC_LVTT: 1780 if (!kvm_apic_sw_enabled(apic)) 1781 val |= APIC_LVT_MASKED; 1782 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 1783 kvm_lapic_set_reg(apic, APIC_LVTT, val); 1784 apic_update_lvtt(apic); 1785 break; 1786 1787 case APIC_TMICT: 1788 if (apic_lvtt_tscdeadline(apic)) 1789 break; 1790 1791 hrtimer_cancel(&apic->lapic_timer.timer); 1792 kvm_lapic_set_reg(apic, APIC_TMICT, val); 1793 start_apic_timer(apic); 1794 break; 1795 1796 case APIC_TDCR: { 1797 uint32_t old_divisor = apic->divide_count; 1798 1799 if (val & 4) 1800 apic_debug("KVM_WRITE:TDCR %x\n", val); 1801 kvm_lapic_set_reg(apic, APIC_TDCR, val); 1802 update_divide_count(apic); 1803 if (apic->divide_count != old_divisor && 1804 apic->lapic_timer.period) { 1805 hrtimer_cancel(&apic->lapic_timer.timer); 1806 update_target_expiration(apic, old_divisor); 1807 restart_apic_timer(apic); 1808 } 1809 break; 1810 } 1811 case APIC_ESR: 1812 if (apic_x2apic_mode(apic) && val != 0) { 1813 apic_debug("KVM_WRITE:ESR not zero %x\n", val); 1814 ret = 1; 1815 } 1816 break; 1817 1818 case APIC_SELF_IPI: 1819 if (apic_x2apic_mode(apic)) { 1820 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff)); 1821 } else 1822 ret = 1; 1823 break; 1824 default: 1825 ret = 1; 1826 break; 1827 } 1828 if (ret) 1829 apic_debug("Local APIC Write to read-only register %x\n", reg); 1830 return ret; 1831 } 1832 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write); 1833 1834 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1835 gpa_t address, int len, const void *data) 1836 { 1837 struct kvm_lapic *apic = to_lapic(this); 1838 unsigned int offset = address - apic->base_address; 1839 u32 val; 1840 1841 if (!apic_mmio_in_range(apic, address)) 1842 return -EOPNOTSUPP; 1843 1844 /* 1845 * APIC register must be aligned on 128-bits boundary. 1846 * 32/64/128 bits registers must be accessed thru 32 bits. 1847 * Refer SDM 8.4.1 1848 */ 1849 if (len != 4 || (offset & 0xf)) { 1850 /* Don't shout loud, $infamous_os would cause only noise. */ 1851 apic_debug("apic write: bad size=%d %lx\n", len, (long)address); 1852 return 0; 1853 } 1854 1855 val = *(u32*)data; 1856 1857 /* too common printing */ 1858 if (offset != APIC_EOI) 1859 apic_debug("%s: offset 0x%x with length 0x%x, and value is " 1860 "0x%x\n", __func__, offset, len, val); 1861 1862 kvm_lapic_reg_write(apic, offset & 0xff0, val); 1863 1864 return 0; 1865 } 1866 1867 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 1868 { 1869 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 1870 } 1871 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 1872 1873 /* emulate APIC access in a trap manner */ 1874 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 1875 { 1876 u32 val = 0; 1877 1878 /* hw has done the conditional check and inst decode */ 1879 offset &= 0xff0; 1880 1881 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val); 1882 1883 /* TODO: optimize to just emulate side effect w/o one more write */ 1884 kvm_lapic_reg_write(vcpu->arch.apic, offset, val); 1885 } 1886 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 1887 1888 void kvm_free_lapic(struct kvm_vcpu *vcpu) 1889 { 1890 struct kvm_lapic *apic = vcpu->arch.apic; 1891 1892 if (!vcpu->arch.apic) 1893 return; 1894 1895 hrtimer_cancel(&apic->lapic_timer.timer); 1896 1897 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 1898 static_key_slow_dec_deferred(&apic_hw_disabled); 1899 1900 if (!apic->sw_enabled) 1901 static_key_slow_dec_deferred(&apic_sw_disabled); 1902 1903 if (apic->regs) 1904 free_page((unsigned long)apic->regs); 1905 1906 kfree(apic); 1907 } 1908 1909 /* 1910 *---------------------------------------------------------------------- 1911 * LAPIC interface 1912 *---------------------------------------------------------------------- 1913 */ 1914 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 1915 { 1916 struct kvm_lapic *apic = vcpu->arch.apic; 1917 1918 if (!lapic_in_kernel(vcpu) || 1919 !apic_lvtt_tscdeadline(apic)) 1920 return 0; 1921 1922 return apic->lapic_timer.tscdeadline; 1923 } 1924 1925 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 1926 { 1927 struct kvm_lapic *apic = vcpu->arch.apic; 1928 1929 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) || 1930 apic_lvtt_period(apic)) 1931 return; 1932 1933 hrtimer_cancel(&apic->lapic_timer.timer); 1934 apic->lapic_timer.tscdeadline = data; 1935 start_apic_timer(apic); 1936 } 1937 1938 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 1939 { 1940 struct kvm_lapic *apic = vcpu->arch.apic; 1941 1942 apic_set_tpr(apic, ((cr8 & 0x0f) << 4) 1943 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4)); 1944 } 1945 1946 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 1947 { 1948 u64 tpr; 1949 1950 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 1951 1952 return (tpr & 0xf0) >> 4; 1953 } 1954 1955 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 1956 { 1957 u64 old_value = vcpu->arch.apic_base; 1958 struct kvm_lapic *apic = vcpu->arch.apic; 1959 1960 if (!apic) 1961 value |= MSR_IA32_APICBASE_BSP; 1962 1963 vcpu->arch.apic_base = value; 1964 1965 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) 1966 kvm_update_cpuid(vcpu); 1967 1968 if (!apic) 1969 return; 1970 1971 /* update jump label if enable bit changes */ 1972 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 1973 if (value & MSR_IA32_APICBASE_ENABLE) { 1974 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 1975 static_key_slow_dec_deferred(&apic_hw_disabled); 1976 } else { 1977 static_key_slow_inc(&apic_hw_disabled.key); 1978 recalculate_apic_map(vcpu->kvm); 1979 } 1980 } 1981 1982 if ((old_value ^ value) & X2APIC_ENABLE) { 1983 if (value & X2APIC_ENABLE) { 1984 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); 1985 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true); 1986 } else 1987 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false); 1988 } 1989 1990 apic->base_address = apic->vcpu->arch.apic_base & 1991 MSR_IA32_APICBASE_BASE; 1992 1993 if ((value & MSR_IA32_APICBASE_ENABLE) && 1994 apic->base_address != APIC_DEFAULT_PHYS_BASE) 1995 pr_warn_once("APIC base relocation is unsupported by KVM"); 1996 1997 /* with FSB delivery interrupt, we can restart APIC functionality */ 1998 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " 1999 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); 2000 2001 } 2002 2003 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) 2004 { 2005 struct kvm_lapic *apic = vcpu->arch.apic; 2006 int i; 2007 2008 if (!apic) 2009 return; 2010 2011 apic_debug("%s\n", __func__); 2012 2013 /* Stop the timer in case it's a reset to an active apic */ 2014 hrtimer_cancel(&apic->lapic_timer.timer); 2015 2016 if (!init_event) { 2017 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE | 2018 MSR_IA32_APICBASE_ENABLE); 2019 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2020 } 2021 kvm_apic_set_version(apic->vcpu); 2022 2023 for (i = 0; i < KVM_APIC_LVT_NUM; i++) 2024 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED); 2025 apic_update_lvtt(apic); 2026 if (kvm_vcpu_is_reset_bsp(vcpu) && 2027 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) 2028 kvm_lapic_set_reg(apic, APIC_LVT0, 2029 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 2030 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2031 2032 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU); 2033 apic_set_spiv(apic, 0xff); 2034 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0); 2035 if (!apic_x2apic_mode(apic)) 2036 kvm_apic_set_ldr(apic, 0); 2037 kvm_lapic_set_reg(apic, APIC_ESR, 0); 2038 kvm_lapic_set_reg(apic, APIC_ICR, 0); 2039 kvm_lapic_set_reg(apic, APIC_ICR2, 0); 2040 kvm_lapic_set_reg(apic, APIC_TDCR, 0); 2041 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 2042 for (i = 0; i < 8; i++) { 2043 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 2044 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 2045 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 2046 } 2047 apic->irr_pending = vcpu->arch.apicv_active; 2048 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0; 2049 apic->highest_isr_cache = -1; 2050 update_divide_count(apic); 2051 atomic_set(&apic->lapic_timer.pending, 0); 2052 if (kvm_vcpu_is_bsp(vcpu)) 2053 kvm_lapic_set_base(vcpu, 2054 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP); 2055 vcpu->arch.pv_eoi.msr_val = 0; 2056 apic_update_ppr(apic); 2057 if (vcpu->arch.apicv_active) { 2058 kvm_x86_ops->apicv_post_state_restore(vcpu); 2059 kvm_x86_ops->hwapic_irr_update(vcpu, -1); 2060 kvm_x86_ops->hwapic_isr_update(vcpu, -1); 2061 } 2062 2063 vcpu->arch.apic_arb_prio = 0; 2064 vcpu->arch.apic_attention = 0; 2065 2066 apic_debug("%s: vcpu=%p, id=0x%x, base_msr=" 2067 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__, 2068 vcpu, kvm_lapic_get_reg(apic, APIC_ID), 2069 vcpu->arch.apic_base, apic->base_address); 2070 } 2071 2072 /* 2073 *---------------------------------------------------------------------- 2074 * timer interface 2075 *---------------------------------------------------------------------- 2076 */ 2077 2078 static bool lapic_is_periodic(struct kvm_lapic *apic) 2079 { 2080 return apic_lvtt_period(apic); 2081 } 2082 2083 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 2084 { 2085 struct kvm_lapic *apic = vcpu->arch.apic; 2086 2087 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT)) 2088 return atomic_read(&apic->lapic_timer.pending); 2089 2090 return 0; 2091 } 2092 2093 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 2094 { 2095 u32 reg = kvm_lapic_get_reg(apic, lvt_type); 2096 int vector, mode, trig_mode; 2097 2098 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 2099 vector = reg & APIC_VECTOR_MASK; 2100 mode = reg & APIC_MODE_MASK; 2101 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 2102 return __apic_accept_irq(apic, mode, vector, 1, trig_mode, 2103 NULL); 2104 } 2105 return 0; 2106 } 2107 2108 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 2109 { 2110 struct kvm_lapic *apic = vcpu->arch.apic; 2111 2112 if (apic) 2113 kvm_apic_local_deliver(apic, APIC_LVT0); 2114 } 2115 2116 static const struct kvm_io_device_ops apic_mmio_ops = { 2117 .read = apic_mmio_read, 2118 .write = apic_mmio_write, 2119 }; 2120 2121 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 2122 { 2123 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 2124 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 2125 2126 apic_timer_expired(apic); 2127 2128 if (lapic_is_periodic(apic)) { 2129 advance_periodic_target_expiration(apic); 2130 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 2131 return HRTIMER_RESTART; 2132 } else 2133 return HRTIMER_NORESTART; 2134 } 2135 2136 int kvm_create_lapic(struct kvm_vcpu *vcpu) 2137 { 2138 struct kvm_lapic *apic; 2139 2140 ASSERT(vcpu != NULL); 2141 apic_debug("apic_init %d\n", vcpu->vcpu_id); 2142 2143 apic = kzalloc(sizeof(*apic), GFP_KERNEL); 2144 if (!apic) 2145 goto nomem; 2146 2147 vcpu->arch.apic = apic; 2148 2149 apic->regs = (void *)get_zeroed_page(GFP_KERNEL); 2150 if (!apic->regs) { 2151 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 2152 vcpu->vcpu_id); 2153 goto nomem_free_apic; 2154 } 2155 apic->vcpu = vcpu; 2156 2157 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 2158 HRTIMER_MODE_ABS_PINNED); 2159 apic->lapic_timer.timer.function = apic_timer_fn; 2160 2161 /* 2162 * APIC is created enabled. This will prevent kvm_lapic_set_base from 2163 * thinking that APIC satet has changed. 2164 */ 2165 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 2166 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 2167 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 2168 2169 return 0; 2170 nomem_free_apic: 2171 kfree(apic); 2172 nomem: 2173 return -ENOMEM; 2174 } 2175 2176 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 2177 { 2178 struct kvm_lapic *apic = vcpu->arch.apic; 2179 u32 ppr; 2180 2181 if (!apic_enabled(apic)) 2182 return -1; 2183 2184 __apic_update_ppr(apic, &ppr); 2185 return apic_has_interrupt_for_ppr(apic, ppr); 2186 } 2187 2188 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 2189 { 2190 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); 2191 int r = 0; 2192 2193 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 2194 r = 1; 2195 if ((lvt0 & APIC_LVT_MASKED) == 0 && 2196 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 2197 r = 1; 2198 return r; 2199 } 2200 2201 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 2202 { 2203 struct kvm_lapic *apic = vcpu->arch.apic; 2204 2205 if (atomic_read(&apic->lapic_timer.pending) > 0) { 2206 kvm_apic_local_deliver(apic, APIC_LVTT); 2207 if (apic_lvtt_tscdeadline(apic)) 2208 apic->lapic_timer.tscdeadline = 0; 2209 if (apic_lvtt_oneshot(apic)) { 2210 apic->lapic_timer.tscdeadline = 0; 2211 apic->lapic_timer.target_expiration = 0; 2212 } 2213 atomic_set(&apic->lapic_timer.pending, 0); 2214 } 2215 } 2216 2217 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 2218 { 2219 int vector = kvm_apic_has_interrupt(vcpu); 2220 struct kvm_lapic *apic = vcpu->arch.apic; 2221 u32 ppr; 2222 2223 if (vector == -1) 2224 return -1; 2225 2226 /* 2227 * We get here even with APIC virtualization enabled, if doing 2228 * nested virtualization and L1 runs with the "acknowledge interrupt 2229 * on exit" mode. Then we cannot inject the interrupt via RVI, 2230 * because the process would deliver it through the IDT. 2231 */ 2232 2233 apic_clear_irr(vector, apic); 2234 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) { 2235 /* 2236 * For auto-EOI interrupts, there might be another pending 2237 * interrupt above PPR, so check whether to raise another 2238 * KVM_REQ_EVENT. 2239 */ 2240 apic_update_ppr(apic); 2241 } else { 2242 /* 2243 * For normal interrupts, PPR has been raised and there cannot 2244 * be a higher-priority pending interrupt---except if there was 2245 * a concurrent interrupt injection, but that would have 2246 * triggered KVM_REQ_EVENT already. 2247 */ 2248 apic_set_isr(vector, apic); 2249 __apic_update_ppr(apic, &ppr); 2250 } 2251 2252 return vector; 2253 } 2254 2255 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, 2256 struct kvm_lapic_state *s, bool set) 2257 { 2258 if (apic_x2apic_mode(vcpu->arch.apic)) { 2259 u32 *id = (u32 *)(s->regs + APIC_ID); 2260 u32 *ldr = (u32 *)(s->regs + APIC_LDR); 2261 2262 if (vcpu->kvm->arch.x2apic_format) { 2263 if (*id != vcpu->vcpu_id) 2264 return -EINVAL; 2265 } else { 2266 if (set) 2267 *id >>= 24; 2268 else 2269 *id <<= 24; 2270 } 2271 2272 /* In x2APIC mode, the LDR is fixed and based on the id */ 2273 if (set) 2274 *ldr = kvm_apic_calc_x2apic_ldr(*id); 2275 } 2276 2277 return 0; 2278 } 2279 2280 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2281 { 2282 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); 2283 return kvm_apic_state_fixup(vcpu, s, false); 2284 } 2285 2286 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2287 { 2288 struct kvm_lapic *apic = vcpu->arch.apic; 2289 int r; 2290 2291 2292 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 2293 /* set SPIV separately to get count of SW disabled APICs right */ 2294 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 2295 2296 r = kvm_apic_state_fixup(vcpu, s, true); 2297 if (r) 2298 return r; 2299 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); 2300 2301 recalculate_apic_map(vcpu->kvm); 2302 kvm_apic_set_version(vcpu); 2303 2304 apic_update_ppr(apic); 2305 hrtimer_cancel(&apic->lapic_timer.timer); 2306 apic_update_lvtt(apic); 2307 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2308 update_divide_count(apic); 2309 start_apic_timer(apic); 2310 apic->irr_pending = true; 2311 apic->isr_count = vcpu->arch.apicv_active ? 2312 1 : count_vectors(apic->regs + APIC_ISR); 2313 apic->highest_isr_cache = -1; 2314 if (vcpu->arch.apicv_active) { 2315 kvm_x86_ops->apicv_post_state_restore(vcpu); 2316 kvm_x86_ops->hwapic_irr_update(vcpu, 2317 apic_find_highest_irr(apic)); 2318 kvm_x86_ops->hwapic_isr_update(vcpu, 2319 apic_find_highest_isr(apic)); 2320 } 2321 kvm_make_request(KVM_REQ_EVENT, vcpu); 2322 if (ioapic_in_kernel(vcpu->kvm)) 2323 kvm_rtc_eoi_tracking_restore_one(vcpu); 2324 2325 vcpu->arch.apic_arb_prio = 0; 2326 2327 return 0; 2328 } 2329 2330 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 2331 { 2332 struct hrtimer *timer; 2333 2334 if (!lapic_in_kernel(vcpu)) 2335 return; 2336 2337 timer = &vcpu->arch.apic->lapic_timer.timer; 2338 if (hrtimer_cancel(timer)) 2339 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 2340 } 2341 2342 /* 2343 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 2344 * 2345 * Detect whether guest triggered PV EOI since the 2346 * last entry. If yes, set EOI on guests's behalf. 2347 * Clear PV EOI in guest memory in any case. 2348 */ 2349 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 2350 struct kvm_lapic *apic) 2351 { 2352 bool pending; 2353 int vector; 2354 /* 2355 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 2356 * and KVM_PV_EOI_ENABLED in guest memory as follows: 2357 * 2358 * KVM_APIC_PV_EOI_PENDING is unset: 2359 * -> host disabled PV EOI. 2360 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 2361 * -> host enabled PV EOI, guest did not execute EOI yet. 2362 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 2363 * -> host enabled PV EOI, guest executed EOI. 2364 */ 2365 BUG_ON(!pv_eoi_enabled(vcpu)); 2366 pending = pv_eoi_get_pending(vcpu); 2367 /* 2368 * Clear pending bit in any case: it will be set again on vmentry. 2369 * While this might not be ideal from performance point of view, 2370 * this makes sure pv eoi is only enabled when we know it's safe. 2371 */ 2372 pv_eoi_clr_pending(vcpu); 2373 if (pending) 2374 return; 2375 vector = apic_set_eoi(apic); 2376 trace_kvm_pv_eoi(apic, vector); 2377 } 2378 2379 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 2380 { 2381 u32 data; 2382 2383 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 2384 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 2385 2386 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2387 return; 2388 2389 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2390 sizeof(u32))) 2391 return; 2392 2393 apic_set_tpr(vcpu->arch.apic, data & 0xff); 2394 } 2395 2396 /* 2397 * apic_sync_pv_eoi_to_guest - called before vmentry 2398 * 2399 * Detect whether it's safe to enable PV EOI and 2400 * if yes do so. 2401 */ 2402 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 2403 struct kvm_lapic *apic) 2404 { 2405 if (!pv_eoi_enabled(vcpu) || 2406 /* IRR set or many bits in ISR: could be nested. */ 2407 apic->irr_pending || 2408 /* Cache not set: could be safe but we don't bother. */ 2409 apic->highest_isr_cache == -1 || 2410 /* Need EOI to update ioapic. */ 2411 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { 2412 /* 2413 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 2414 * so we need not do anything here. 2415 */ 2416 return; 2417 } 2418 2419 pv_eoi_set_pending(apic->vcpu); 2420 } 2421 2422 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 2423 { 2424 u32 data, tpr; 2425 int max_irr, max_isr; 2426 struct kvm_lapic *apic = vcpu->arch.apic; 2427 2428 apic_sync_pv_eoi_to_guest(vcpu, apic); 2429 2430 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2431 return; 2432 2433 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff; 2434 max_irr = apic_find_highest_irr(apic); 2435 if (max_irr < 0) 2436 max_irr = 0; 2437 max_isr = apic_find_highest_isr(apic); 2438 if (max_isr < 0) 2439 max_isr = 0; 2440 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 2441 2442 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2443 sizeof(u32)); 2444 } 2445 2446 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 2447 { 2448 if (vapic_addr) { 2449 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 2450 &vcpu->arch.apic->vapic_cache, 2451 vapic_addr, sizeof(u32))) 2452 return -EINVAL; 2453 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2454 } else { 2455 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2456 } 2457 2458 vcpu->arch.apic->vapic_addr = vapic_addr; 2459 return 0; 2460 } 2461 2462 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 2463 { 2464 struct kvm_lapic *apic = vcpu->arch.apic; 2465 u32 reg = (msr - APIC_BASE_MSR) << 4; 2466 2467 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2468 return 1; 2469 2470 if (reg == APIC_ICR2) 2471 return 1; 2472 2473 /* if this is ICR write vector before command */ 2474 if (reg == APIC_ICR) 2475 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2476 return kvm_lapic_reg_write(apic, reg, (u32)data); 2477 } 2478 2479 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 2480 { 2481 struct kvm_lapic *apic = vcpu->arch.apic; 2482 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0; 2483 2484 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2485 return 1; 2486 2487 if (reg == APIC_DFR || reg == APIC_ICR2) { 2488 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n", 2489 reg); 2490 return 1; 2491 } 2492 2493 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2494 return 1; 2495 if (reg == APIC_ICR) 2496 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2497 2498 *data = (((u64)high) << 32) | low; 2499 2500 return 0; 2501 } 2502 2503 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 2504 { 2505 struct kvm_lapic *apic = vcpu->arch.apic; 2506 2507 if (!lapic_in_kernel(vcpu)) 2508 return 1; 2509 2510 /* if this is ICR write vector before command */ 2511 if (reg == APIC_ICR) 2512 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2513 return kvm_lapic_reg_write(apic, reg, (u32)data); 2514 } 2515 2516 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 2517 { 2518 struct kvm_lapic *apic = vcpu->arch.apic; 2519 u32 low, high = 0; 2520 2521 if (!lapic_in_kernel(vcpu)) 2522 return 1; 2523 2524 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2525 return 1; 2526 if (reg == APIC_ICR) 2527 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2528 2529 *data = (((u64)high) << 32) | low; 2530 2531 return 0; 2532 } 2533 2534 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data) 2535 { 2536 u64 addr = data & ~KVM_MSR_ENABLED; 2537 if (!IS_ALIGNED(addr, 4)) 2538 return 1; 2539 2540 vcpu->arch.pv_eoi.msr_val = data; 2541 if (!pv_eoi_enabled(vcpu)) 2542 return 0; 2543 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data, 2544 addr, sizeof(u8)); 2545 } 2546 2547 void kvm_apic_accept_events(struct kvm_vcpu *vcpu) 2548 { 2549 struct kvm_lapic *apic = vcpu->arch.apic; 2550 u8 sipi_vector; 2551 unsigned long pe; 2552 2553 if (!lapic_in_kernel(vcpu) || !apic->pending_events) 2554 return; 2555 2556 /* 2557 * INITs are latched while in SMM. Because an SMM CPU cannot 2558 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs 2559 * and delay processing of INIT until the next RSM. 2560 */ 2561 if (is_smm(vcpu)) { 2562 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); 2563 if (test_bit(KVM_APIC_SIPI, &apic->pending_events)) 2564 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 2565 return; 2566 } 2567 2568 pe = xchg(&apic->pending_events, 0); 2569 if (test_bit(KVM_APIC_INIT, &pe)) { 2570 kvm_vcpu_reset(vcpu, true); 2571 if (kvm_vcpu_is_bsp(apic->vcpu)) 2572 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2573 else 2574 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 2575 } 2576 if (test_bit(KVM_APIC_SIPI, &pe) && 2577 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 2578 /* evaluate pending_events before reading the vector */ 2579 smp_rmb(); 2580 sipi_vector = apic->sipi_vector; 2581 apic_debug("vcpu %d received sipi with vector # %x\n", 2582 vcpu->vcpu_id, sipi_vector); 2583 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector); 2584 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2585 } 2586 } 2587 2588 void kvm_lapic_init(void) 2589 { 2590 /* do not patch jump label more than once per second */ 2591 jump_label_rate_limit(&apic_hw_disabled, HZ); 2592 jump_label_rate_limit(&apic_sw_disabled, HZ); 2593 } 2594 2595 void kvm_lapic_exit(void) 2596 { 2597 static_key_deferred_flush(&apic_hw_disabled); 2598 static_key_deferred_flush(&apic_sw_disabled); 2599 } 2600