1 2 /* 3 * Local APIC virtualization 4 * 5 * Copyright (C) 2006 Qumranet, Inc. 6 * Copyright (C) 2007 Novell 7 * Copyright (C) 2007 Intel 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Dor Laor <dor.laor@qumranet.com> 12 * Gregory Haskins <ghaskins@novell.com> 13 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 14 * 15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 16 * 17 * This work is licensed under the terms of the GNU GPL, version 2. See 18 * the COPYING file in the top-level directory. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/mm.h> 24 #include <linux/highmem.h> 25 #include <linux/smp.h> 26 #include <linux/hrtimer.h> 27 #include <linux/io.h> 28 #include <linux/export.h> 29 #include <linux/math64.h> 30 #include <linux/slab.h> 31 #include <asm/processor.h> 32 #include <asm/msr.h> 33 #include <asm/page.h> 34 #include <asm/current.h> 35 #include <asm/apicdef.h> 36 #include <asm/delay.h> 37 #include <linux/atomic.h> 38 #include <linux/jump_label.h> 39 #include "kvm_cache_regs.h" 40 #include "irq.h" 41 #include "trace.h" 42 #include "x86.h" 43 #include "cpuid.h" 44 #include "hyperv.h" 45 46 #ifndef CONFIG_X86_64 47 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 48 #else 49 #define mod_64(x, y) ((x) % (y)) 50 #endif 51 52 #define PRId64 "d" 53 #define PRIx64 "llx" 54 #define PRIu64 "u" 55 #define PRIo64 "o" 56 57 /* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ 58 #define apic_debug(fmt, arg...) 59 60 /* 14 is the version for Xeon and Pentium 8.4.8*/ 61 #define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16)) 62 #define LAPIC_MMIO_LENGTH (1 << 12) 63 /* followed define is not in apicdef.h */ 64 #define APIC_SHORT_MASK 0xc0000 65 #define APIC_DEST_NOSHORT 0x0 66 #define APIC_DEST_MASK 0x800 67 #define MAX_APIC_VECTOR 256 68 #define APIC_VECTORS_PER_REG 32 69 70 #define APIC_BROADCAST 0xFF 71 #define X2APIC_BROADCAST 0xFFFFFFFFul 72 73 static inline int apic_test_vector(int vec, void *bitmap) 74 { 75 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 76 } 77 78 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 79 { 80 struct kvm_lapic *apic = vcpu->arch.apic; 81 82 return apic_test_vector(vector, apic->regs + APIC_ISR) || 83 apic_test_vector(vector, apic->regs + APIC_IRR); 84 } 85 86 static inline void apic_clear_vector(int vec, void *bitmap) 87 { 88 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 89 } 90 91 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 92 { 93 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 94 } 95 96 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 97 { 98 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 99 } 100 101 struct static_key_deferred apic_hw_disabled __read_mostly; 102 struct static_key_deferred apic_sw_disabled __read_mostly; 103 104 static inline int apic_enabled(struct kvm_lapic *apic) 105 { 106 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 107 } 108 109 #define LVT_MASK \ 110 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 111 112 #define LINT_MASK \ 113 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 114 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 115 116 static inline u8 kvm_xapic_id(struct kvm_lapic *apic) 117 { 118 return kvm_lapic_get_reg(apic, APIC_ID) >> 24; 119 } 120 121 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) 122 { 123 return apic->vcpu->vcpu_id; 124 } 125 126 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, 127 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { 128 switch (map->mode) { 129 case KVM_APIC_MODE_X2APIC: { 130 u32 offset = (dest_id >> 16) * 16; 131 u32 max_apic_id = map->max_apic_id; 132 133 if (offset <= max_apic_id) { 134 u8 cluster_size = min(max_apic_id - offset + 1, 16U); 135 136 *cluster = &map->phys_map[offset]; 137 *mask = dest_id & (0xffff >> (16 - cluster_size)); 138 } else { 139 *mask = 0; 140 } 141 142 return true; 143 } 144 case KVM_APIC_MODE_XAPIC_FLAT: 145 *cluster = map->xapic_flat_map; 146 *mask = dest_id & 0xff; 147 return true; 148 case KVM_APIC_MODE_XAPIC_CLUSTER: 149 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; 150 *mask = dest_id & 0xf; 151 return true; 152 default: 153 /* Not optimized. */ 154 return false; 155 } 156 } 157 158 static void kvm_apic_map_free(struct rcu_head *rcu) 159 { 160 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu); 161 162 kvfree(map); 163 } 164 165 static void recalculate_apic_map(struct kvm *kvm) 166 { 167 struct kvm_apic_map *new, *old = NULL; 168 struct kvm_vcpu *vcpu; 169 int i; 170 u32 max_id = 255; /* enough space for any xAPIC ID */ 171 172 mutex_lock(&kvm->arch.apic_map_lock); 173 174 kvm_for_each_vcpu(i, vcpu, kvm) 175 if (kvm_apic_present(vcpu)) 176 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); 177 178 new = kvzalloc(sizeof(struct kvm_apic_map) + 179 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), GFP_KERNEL); 180 181 if (!new) 182 goto out; 183 184 new->max_apic_id = max_id; 185 186 kvm_for_each_vcpu(i, vcpu, kvm) { 187 struct kvm_lapic *apic = vcpu->arch.apic; 188 struct kvm_lapic **cluster; 189 u16 mask; 190 u32 ldr; 191 u8 xapic_id; 192 u32 x2apic_id; 193 194 if (!kvm_apic_present(vcpu)) 195 continue; 196 197 xapic_id = kvm_xapic_id(apic); 198 x2apic_id = kvm_x2apic_id(apic); 199 200 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */ 201 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) && 202 x2apic_id <= new->max_apic_id) 203 new->phys_map[x2apic_id] = apic; 204 /* 205 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around, 206 * prevent them from masking VCPUs with APIC ID <= 0xff. 207 */ 208 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) 209 new->phys_map[xapic_id] = apic; 210 211 ldr = kvm_lapic_get_reg(apic, APIC_LDR); 212 213 if (apic_x2apic_mode(apic)) { 214 new->mode |= KVM_APIC_MODE_X2APIC; 215 } else if (ldr) { 216 ldr = GET_APIC_LOGICAL_ID(ldr); 217 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT) 218 new->mode |= KVM_APIC_MODE_XAPIC_FLAT; 219 else 220 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER; 221 } 222 223 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask)) 224 continue; 225 226 if (mask) 227 cluster[ffs(mask) - 1] = apic; 228 } 229 out: 230 old = rcu_dereference_protected(kvm->arch.apic_map, 231 lockdep_is_held(&kvm->arch.apic_map_lock)); 232 rcu_assign_pointer(kvm->arch.apic_map, new); 233 mutex_unlock(&kvm->arch.apic_map_lock); 234 235 if (old) 236 call_rcu(&old->rcu, kvm_apic_map_free); 237 238 kvm_make_scan_ioapic_request(kvm); 239 } 240 241 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 242 { 243 bool enabled = val & APIC_SPIV_APIC_ENABLED; 244 245 kvm_lapic_set_reg(apic, APIC_SPIV, val); 246 247 if (enabled != apic->sw_enabled) { 248 apic->sw_enabled = enabled; 249 if (enabled) { 250 static_key_slow_dec_deferred(&apic_sw_disabled); 251 recalculate_apic_map(apic->vcpu->kvm); 252 } else 253 static_key_slow_inc(&apic_sw_disabled.key); 254 } 255 } 256 257 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) 258 { 259 kvm_lapic_set_reg(apic, APIC_ID, id << 24); 260 recalculate_apic_map(apic->vcpu->kvm); 261 } 262 263 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 264 { 265 kvm_lapic_set_reg(apic, APIC_LDR, id); 266 recalculate_apic_map(apic->vcpu->kvm); 267 } 268 269 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id) 270 { 271 return ((id >> 4) << 16) | (1 << (id & 0xf)); 272 } 273 274 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) 275 { 276 u32 ldr = kvm_apic_calc_x2apic_ldr(id); 277 278 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); 279 280 kvm_lapic_set_reg(apic, APIC_ID, id); 281 kvm_lapic_set_reg(apic, APIC_LDR, ldr); 282 recalculate_apic_map(apic->vcpu->kvm); 283 } 284 285 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 286 { 287 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 288 } 289 290 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) 291 { 292 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; 293 } 294 295 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 296 { 297 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; 298 } 299 300 static inline int apic_lvtt_period(struct kvm_lapic *apic) 301 { 302 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; 303 } 304 305 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 306 { 307 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; 308 } 309 310 static inline int apic_lvt_nmi_mode(u32 lvt_val) 311 { 312 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 313 } 314 315 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 316 { 317 struct kvm_lapic *apic = vcpu->arch.apic; 318 struct kvm_cpuid_entry2 *feat; 319 u32 v = APIC_VERSION; 320 321 if (!lapic_in_kernel(vcpu)) 322 return; 323 324 /* 325 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) 326 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with 327 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC 328 * version first and level-triggered interrupts never get EOIed in 329 * IOAPIC. 330 */ 331 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); 332 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) && 333 !ioapic_in_kernel(vcpu->kvm)) 334 v |= APIC_LVR_DIRECTED_EOI; 335 kvm_lapic_set_reg(apic, APIC_LVR, v); 336 } 337 338 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = { 339 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ 340 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ 341 LVT_MASK | APIC_MODE_MASK, /* LVTPC */ 342 LINT_MASK, LINT_MASK, /* LVT0-1 */ 343 LVT_MASK /* LVTERR */ 344 }; 345 346 static int find_highest_vector(void *bitmap) 347 { 348 int vec; 349 u32 *reg; 350 351 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 352 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 353 reg = bitmap + REG_POS(vec); 354 if (*reg) 355 return __fls(*reg) + vec; 356 } 357 358 return -1; 359 } 360 361 static u8 count_vectors(void *bitmap) 362 { 363 int vec; 364 u32 *reg; 365 u8 count = 0; 366 367 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 368 reg = bitmap + REG_POS(vec); 369 count += hweight32(*reg); 370 } 371 372 return count; 373 } 374 375 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr) 376 { 377 u32 i, vec; 378 u32 pir_val, irr_val, prev_irr_val; 379 int max_updated_irr; 380 381 max_updated_irr = -1; 382 *max_irr = -1; 383 384 for (i = vec = 0; i <= 7; i++, vec += 32) { 385 pir_val = READ_ONCE(pir[i]); 386 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10)); 387 if (pir_val) { 388 prev_irr_val = irr_val; 389 irr_val |= xchg(&pir[i], 0); 390 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val; 391 if (prev_irr_val != irr_val) { 392 max_updated_irr = 393 __fls(irr_val ^ prev_irr_val) + vec; 394 } 395 } 396 if (irr_val) 397 *max_irr = __fls(irr_val) + vec; 398 } 399 400 return ((max_updated_irr != -1) && 401 (max_updated_irr == *max_irr)); 402 } 403 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); 404 405 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr) 406 { 407 struct kvm_lapic *apic = vcpu->arch.apic; 408 409 return __kvm_apic_update_irr(pir, apic->regs, max_irr); 410 } 411 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 412 413 static inline int apic_search_irr(struct kvm_lapic *apic) 414 { 415 return find_highest_vector(apic->regs + APIC_IRR); 416 } 417 418 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 419 { 420 int result; 421 422 /* 423 * Note that irr_pending is just a hint. It will be always 424 * true with virtual interrupt delivery enabled. 425 */ 426 if (!apic->irr_pending) 427 return -1; 428 429 result = apic_search_irr(apic); 430 ASSERT(result == -1 || result >= 16); 431 432 return result; 433 } 434 435 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 436 { 437 struct kvm_vcpu *vcpu; 438 439 vcpu = apic->vcpu; 440 441 if (unlikely(vcpu->arch.apicv_active)) { 442 /* need to update RVI */ 443 apic_clear_vector(vec, apic->regs + APIC_IRR); 444 kvm_x86_ops->hwapic_irr_update(vcpu, 445 apic_find_highest_irr(apic)); 446 } else { 447 apic->irr_pending = false; 448 apic_clear_vector(vec, apic->regs + APIC_IRR); 449 if (apic_search_irr(apic) != -1) 450 apic->irr_pending = true; 451 } 452 } 453 454 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 455 { 456 struct kvm_vcpu *vcpu; 457 458 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 459 return; 460 461 vcpu = apic->vcpu; 462 463 /* 464 * With APIC virtualization enabled, all caching is disabled 465 * because the processor can modify ISR under the hood. Instead 466 * just set SVI. 467 */ 468 if (unlikely(vcpu->arch.apicv_active)) 469 kvm_x86_ops->hwapic_isr_update(vcpu, vec); 470 else { 471 ++apic->isr_count; 472 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 473 /* 474 * ISR (in service register) bit is set when injecting an interrupt. 475 * The highest vector is injected. Thus the latest bit set matches 476 * the highest bit in ISR. 477 */ 478 apic->highest_isr_cache = vec; 479 } 480 } 481 482 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 483 { 484 int result; 485 486 /* 487 * Note that isr_count is always 1, and highest_isr_cache 488 * is always -1, with APIC virtualization enabled. 489 */ 490 if (!apic->isr_count) 491 return -1; 492 if (likely(apic->highest_isr_cache != -1)) 493 return apic->highest_isr_cache; 494 495 result = find_highest_vector(apic->regs + APIC_ISR); 496 ASSERT(result == -1 || result >= 16); 497 498 return result; 499 } 500 501 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 502 { 503 struct kvm_vcpu *vcpu; 504 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 505 return; 506 507 vcpu = apic->vcpu; 508 509 /* 510 * We do get here for APIC virtualization enabled if the guest 511 * uses the Hyper-V APIC enlightenment. In this case we may need 512 * to trigger a new interrupt delivery by writing the SVI field; 513 * on the other hand isr_count and highest_isr_cache are unused 514 * and must be left alone. 515 */ 516 if (unlikely(vcpu->arch.apicv_active)) 517 kvm_x86_ops->hwapic_isr_update(vcpu, 518 apic_find_highest_isr(apic)); 519 else { 520 --apic->isr_count; 521 BUG_ON(apic->isr_count < 0); 522 apic->highest_isr_cache = -1; 523 } 524 } 525 526 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 527 { 528 /* This may race with setting of irr in __apic_accept_irq() and 529 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 530 * will cause vmexit immediately and the value will be recalculated 531 * on the next vmentry. 532 */ 533 return apic_find_highest_irr(vcpu->arch.apic); 534 } 535 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); 536 537 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 538 int vector, int level, int trig_mode, 539 struct dest_map *dest_map); 540 541 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 542 struct dest_map *dest_map) 543 { 544 struct kvm_lapic *apic = vcpu->arch.apic; 545 546 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 547 irq->level, irq->trig_mode, dest_map); 548 } 549 550 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 551 { 552 553 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 554 sizeof(val)); 555 } 556 557 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 558 { 559 560 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 561 sizeof(*val)); 562 } 563 564 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 565 { 566 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 567 } 568 569 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu) 570 { 571 u8 val; 572 if (pv_eoi_get_user(vcpu, &val) < 0) 573 apic_debug("Can't read EOI MSR value: 0x%llx\n", 574 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 575 return val & 0x1; 576 } 577 578 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 579 { 580 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) { 581 apic_debug("Can't set EOI MSR value: 0x%llx\n", 582 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 583 return; 584 } 585 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 586 } 587 588 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu) 589 { 590 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) { 591 apic_debug("Can't clear EOI MSR value: 0x%llx\n", 592 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 593 return; 594 } 595 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 596 } 597 598 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) 599 { 600 int highest_irr; 601 if (apic->vcpu->arch.apicv_active) 602 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu); 603 else 604 highest_irr = apic_find_highest_irr(apic); 605 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) 606 return -1; 607 return highest_irr; 608 } 609 610 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr) 611 { 612 u32 tpr, isrv, ppr, old_ppr; 613 int isr; 614 615 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI); 616 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI); 617 isr = apic_find_highest_isr(apic); 618 isrv = (isr != -1) ? isr : 0; 619 620 if ((tpr & 0xf0) >= (isrv & 0xf0)) 621 ppr = tpr & 0xff; 622 else 623 ppr = isrv & 0xf0; 624 625 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x", 626 apic, ppr, isr, isrv); 627 628 *new_ppr = ppr; 629 if (old_ppr != ppr) 630 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr); 631 632 return ppr < old_ppr; 633 } 634 635 static void apic_update_ppr(struct kvm_lapic *apic) 636 { 637 u32 ppr; 638 639 if (__apic_update_ppr(apic, &ppr) && 640 apic_has_interrupt_for_ppr(apic, ppr) != -1) 641 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 642 } 643 644 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu) 645 { 646 apic_update_ppr(vcpu->arch.apic); 647 } 648 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr); 649 650 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 651 { 652 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr); 653 apic_update_ppr(apic); 654 } 655 656 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda) 657 { 658 return mda == (apic_x2apic_mode(apic) ? 659 X2APIC_BROADCAST : APIC_BROADCAST); 660 } 661 662 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) 663 { 664 if (kvm_apic_broadcast(apic, mda)) 665 return true; 666 667 if (apic_x2apic_mode(apic)) 668 return mda == kvm_x2apic_id(apic); 669 670 /* 671 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if 672 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and 673 * this allows unique addressing of VCPUs with APIC ID over 0xff. 674 * The 0xff condition is needed because writeable xAPIC ID. 675 */ 676 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic)) 677 return true; 678 679 return mda == kvm_xapic_id(apic); 680 } 681 682 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 683 { 684 u32 logical_id; 685 686 if (kvm_apic_broadcast(apic, mda)) 687 return true; 688 689 logical_id = kvm_lapic_get_reg(apic, APIC_LDR); 690 691 if (apic_x2apic_mode(apic)) 692 return ((logical_id >> 16) == (mda >> 16)) 693 && (logical_id & mda & 0xffff) != 0; 694 695 logical_id = GET_APIC_LOGICAL_ID(logical_id); 696 697 switch (kvm_lapic_get_reg(apic, APIC_DFR)) { 698 case APIC_DFR_FLAT: 699 return (logical_id & mda) != 0; 700 case APIC_DFR_CLUSTER: 701 return ((logical_id >> 4) == (mda >> 4)) 702 && (logical_id & mda & 0xf) != 0; 703 default: 704 apic_debug("Bad DFR vcpu %d: %08x\n", 705 apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR)); 706 return false; 707 } 708 } 709 710 /* The KVM local APIC implementation has two quirks: 711 * 712 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs 713 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID. 714 * KVM doesn't do that aliasing. 715 * 716 * - in-kernel IOAPIC messages have to be delivered directly to 717 * x2APIC, because the kernel does not support interrupt remapping. 718 * In order to support broadcast without interrupt remapping, x2APIC 719 * rewrites the destination of non-IPI messages from APIC_BROADCAST 720 * to X2APIC_BROADCAST. 721 * 722 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is 723 * important when userspace wants to use x2APIC-format MSIs, because 724 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7". 725 */ 726 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id, 727 struct kvm_lapic *source, struct kvm_lapic *target) 728 { 729 bool ipi = source != NULL; 730 731 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && 732 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target)) 733 return X2APIC_BROADCAST; 734 735 return dest_id; 736 } 737 738 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 739 int short_hand, unsigned int dest, int dest_mode) 740 { 741 struct kvm_lapic *target = vcpu->arch.apic; 742 u32 mda = kvm_apic_mda(vcpu, dest, source, target); 743 744 apic_debug("target %p, source %p, dest 0x%x, " 745 "dest_mode 0x%x, short_hand 0x%x\n", 746 target, source, dest, dest_mode, short_hand); 747 748 ASSERT(target); 749 switch (short_hand) { 750 case APIC_DEST_NOSHORT: 751 if (dest_mode == APIC_DEST_PHYSICAL) 752 return kvm_apic_match_physical_addr(target, mda); 753 else 754 return kvm_apic_match_logical_addr(target, mda); 755 case APIC_DEST_SELF: 756 return target == source; 757 case APIC_DEST_ALLINC: 758 return true; 759 case APIC_DEST_ALLBUT: 760 return target != source; 761 default: 762 apic_debug("kvm: apic: Bad dest shorthand value %x\n", 763 short_hand); 764 return false; 765 } 766 } 767 EXPORT_SYMBOL_GPL(kvm_apic_match_dest); 768 769 int kvm_vector_to_index(u32 vector, u32 dest_vcpus, 770 const unsigned long *bitmap, u32 bitmap_size) 771 { 772 u32 mod; 773 int i, idx = -1; 774 775 mod = vector % dest_vcpus; 776 777 for (i = 0; i <= mod; i++) { 778 idx = find_next_bit(bitmap, bitmap_size, idx + 1); 779 BUG_ON(idx == bitmap_size); 780 } 781 782 return idx; 783 } 784 785 static void kvm_apic_disabled_lapic_found(struct kvm *kvm) 786 { 787 if (!kvm->arch.disabled_lapic_found) { 788 kvm->arch.disabled_lapic_found = true; 789 printk(KERN_INFO 790 "Disabled LAPIC found during irq injection\n"); 791 } 792 } 793 794 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src, 795 struct kvm_lapic_irq *irq, struct kvm_apic_map *map) 796 { 797 if (kvm->arch.x2apic_broadcast_quirk_disabled) { 798 if ((irq->dest_id == APIC_BROADCAST && 799 map->mode != KVM_APIC_MODE_X2APIC)) 800 return true; 801 if (irq->dest_id == X2APIC_BROADCAST) 802 return true; 803 } else { 804 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src); 805 if (irq->dest_id == (x2apic_ipi ? 806 X2APIC_BROADCAST : APIC_BROADCAST)) 807 return true; 808 } 809 810 return false; 811 } 812 813 /* Return true if the interrupt can be handled by using *bitmap as index mask 814 * for valid destinations in *dst array. 815 * Return false if kvm_apic_map_get_dest_lapic did nothing useful. 816 * Note: we may have zero kvm_lapic destinations when we return true, which 817 * means that the interrupt should be dropped. In this case, *bitmap would be 818 * zero and *dst undefined. 819 */ 820 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm, 821 struct kvm_lapic **src, struct kvm_lapic_irq *irq, 822 struct kvm_apic_map *map, struct kvm_lapic ***dst, 823 unsigned long *bitmap) 824 { 825 int i, lowest; 826 827 if (irq->shorthand == APIC_DEST_SELF && src) { 828 *dst = src; 829 *bitmap = 1; 830 return true; 831 } else if (irq->shorthand) 832 return false; 833 834 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map)) 835 return false; 836 837 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 838 if (irq->dest_id > map->max_apic_id) { 839 *bitmap = 0; 840 } else { 841 *dst = &map->phys_map[irq->dest_id]; 842 *bitmap = 1; 843 } 844 return true; 845 } 846 847 *bitmap = 0; 848 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, 849 (u16 *)bitmap)) 850 return false; 851 852 if (!kvm_lowest_prio_delivery(irq)) 853 return true; 854 855 if (!kvm_vector_hashing_enabled()) { 856 lowest = -1; 857 for_each_set_bit(i, bitmap, 16) { 858 if (!(*dst)[i]) 859 continue; 860 if (lowest < 0) 861 lowest = i; 862 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, 863 (*dst)[lowest]->vcpu) < 0) 864 lowest = i; 865 } 866 } else { 867 if (!*bitmap) 868 return true; 869 870 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), 871 bitmap, 16); 872 873 if (!(*dst)[lowest]) { 874 kvm_apic_disabled_lapic_found(kvm); 875 *bitmap = 0; 876 return true; 877 } 878 } 879 880 *bitmap = (lowest >= 0) ? 1 << lowest : 0; 881 882 return true; 883 } 884 885 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 886 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map) 887 { 888 struct kvm_apic_map *map; 889 unsigned long bitmap; 890 struct kvm_lapic **dst = NULL; 891 int i; 892 bool ret; 893 894 *r = -1; 895 896 if (irq->shorthand == APIC_DEST_SELF) { 897 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 898 return true; 899 } 900 901 rcu_read_lock(); 902 map = rcu_dereference(kvm->arch.apic_map); 903 904 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap); 905 if (ret) 906 for_each_set_bit(i, &bitmap, 16) { 907 if (!dst[i]) 908 continue; 909 if (*r < 0) 910 *r = 0; 911 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 912 } 913 914 rcu_read_unlock(); 915 return ret; 916 } 917 918 /* 919 * This routine tries to handler interrupts in posted mode, here is how 920 * it deals with different cases: 921 * - For single-destination interrupts, handle it in posted mode 922 * - Else if vector hashing is enabled and it is a lowest-priority 923 * interrupt, handle it in posted mode and use the following mechanism 924 * to find the destinaiton vCPU. 925 * 1. For lowest-priority interrupts, store all the possible 926 * destination vCPUs in an array. 927 * 2. Use "guest vector % max number of destination vCPUs" to find 928 * the right destination vCPU in the array for the lowest-priority 929 * interrupt. 930 * - Otherwise, use remapped mode to inject the interrupt. 931 */ 932 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq, 933 struct kvm_vcpu **dest_vcpu) 934 { 935 struct kvm_apic_map *map; 936 unsigned long bitmap; 937 struct kvm_lapic **dst = NULL; 938 bool ret = false; 939 940 if (irq->shorthand) 941 return false; 942 943 rcu_read_lock(); 944 map = rcu_dereference(kvm->arch.apic_map); 945 946 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) && 947 hweight16(bitmap) == 1) { 948 unsigned long i = find_first_bit(&bitmap, 16); 949 950 if (dst[i]) { 951 *dest_vcpu = dst[i]->vcpu; 952 ret = true; 953 } 954 } 955 956 rcu_read_unlock(); 957 return ret; 958 } 959 960 /* 961 * Add a pending IRQ into lapic. 962 * Return 1 if successfully added and 0 if discarded. 963 */ 964 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 965 int vector, int level, int trig_mode, 966 struct dest_map *dest_map) 967 { 968 int result = 0; 969 struct kvm_vcpu *vcpu = apic->vcpu; 970 971 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 972 trig_mode, vector); 973 switch (delivery_mode) { 974 case APIC_DM_LOWEST: 975 vcpu->arch.apic_arb_prio++; 976 case APIC_DM_FIXED: 977 if (unlikely(trig_mode && !level)) 978 break; 979 980 /* FIXME add logic for vcpu on reset */ 981 if (unlikely(!apic_enabled(apic))) 982 break; 983 984 result = 1; 985 986 if (dest_map) { 987 __set_bit(vcpu->vcpu_id, dest_map->map); 988 dest_map->vectors[vcpu->vcpu_id] = vector; 989 } 990 991 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { 992 if (trig_mode) 993 kvm_lapic_set_vector(vector, apic->regs + APIC_TMR); 994 else 995 apic_clear_vector(vector, apic->regs + APIC_TMR); 996 } 997 998 if (vcpu->arch.apicv_active) 999 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector); 1000 else { 1001 kvm_lapic_set_irr(vector, apic); 1002 1003 kvm_make_request(KVM_REQ_EVENT, vcpu); 1004 kvm_vcpu_kick(vcpu); 1005 } 1006 break; 1007 1008 case APIC_DM_REMRD: 1009 result = 1; 1010 vcpu->arch.pv.pv_unhalted = 1; 1011 kvm_make_request(KVM_REQ_EVENT, vcpu); 1012 kvm_vcpu_kick(vcpu); 1013 break; 1014 1015 case APIC_DM_SMI: 1016 result = 1; 1017 kvm_make_request(KVM_REQ_SMI, vcpu); 1018 kvm_vcpu_kick(vcpu); 1019 break; 1020 1021 case APIC_DM_NMI: 1022 result = 1; 1023 kvm_inject_nmi(vcpu); 1024 kvm_vcpu_kick(vcpu); 1025 break; 1026 1027 case APIC_DM_INIT: 1028 if (!trig_mode || level) { 1029 result = 1; 1030 /* assumes that there are only KVM_APIC_INIT/SIPI */ 1031 apic->pending_events = (1UL << KVM_APIC_INIT); 1032 /* make sure pending_events is visible before sending 1033 * the request */ 1034 smp_wmb(); 1035 kvm_make_request(KVM_REQ_EVENT, vcpu); 1036 kvm_vcpu_kick(vcpu); 1037 } else { 1038 apic_debug("Ignoring de-assert INIT to vcpu %d\n", 1039 vcpu->vcpu_id); 1040 } 1041 break; 1042 1043 case APIC_DM_STARTUP: 1044 apic_debug("SIPI to vcpu %d vector 0x%02x\n", 1045 vcpu->vcpu_id, vector); 1046 result = 1; 1047 apic->sipi_vector = vector; 1048 /* make sure sipi_vector is visible for the receiver */ 1049 smp_wmb(); 1050 set_bit(KVM_APIC_SIPI, &apic->pending_events); 1051 kvm_make_request(KVM_REQ_EVENT, vcpu); 1052 kvm_vcpu_kick(vcpu); 1053 break; 1054 1055 case APIC_DM_EXTINT: 1056 /* 1057 * Should only be called by kvm_apic_local_deliver() with LVT0, 1058 * before NMI watchdog was enabled. Already handled by 1059 * kvm_apic_accept_pic_intr(). 1060 */ 1061 break; 1062 1063 default: 1064 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 1065 delivery_mode); 1066 break; 1067 } 1068 return result; 1069 } 1070 1071 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 1072 { 1073 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 1074 } 1075 1076 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) 1077 { 1078 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); 1079 } 1080 1081 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 1082 { 1083 int trigger_mode; 1084 1085 /* Eoi the ioapic only if the ioapic doesn't own the vector. */ 1086 if (!kvm_ioapic_handles_vector(apic, vector)) 1087 return; 1088 1089 /* Request a KVM exit to inform the userspace IOAPIC. */ 1090 if (irqchip_split(apic->vcpu->kvm)) { 1091 apic->vcpu->arch.pending_ioapic_eoi = vector; 1092 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); 1093 return; 1094 } 1095 1096 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 1097 trigger_mode = IOAPIC_LEVEL_TRIG; 1098 else 1099 trigger_mode = IOAPIC_EDGE_TRIG; 1100 1101 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 1102 } 1103 1104 static int apic_set_eoi(struct kvm_lapic *apic) 1105 { 1106 int vector = apic_find_highest_isr(apic); 1107 1108 trace_kvm_eoi(apic, vector); 1109 1110 /* 1111 * Not every write EOI will has corresponding ISR, 1112 * one example is when Kernel check timer on setup_IO_APIC 1113 */ 1114 if (vector == -1) 1115 return vector; 1116 1117 apic_clear_isr(vector, apic); 1118 apic_update_ppr(apic); 1119 1120 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap)) 1121 kvm_hv_synic_send_eoi(apic->vcpu, vector); 1122 1123 kvm_ioapic_send_eoi(apic, vector); 1124 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1125 return vector; 1126 } 1127 1128 /* 1129 * this interface assumes a trap-like exit, which has already finished 1130 * desired side effect including vISR and vPPR update. 1131 */ 1132 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 1133 { 1134 struct kvm_lapic *apic = vcpu->arch.apic; 1135 1136 trace_kvm_eoi(apic, vector); 1137 1138 kvm_ioapic_send_eoi(apic, vector); 1139 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1140 } 1141 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 1142 1143 static void apic_send_ipi(struct kvm_lapic *apic) 1144 { 1145 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR); 1146 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2); 1147 struct kvm_lapic_irq irq; 1148 1149 irq.vector = icr_low & APIC_VECTOR_MASK; 1150 irq.delivery_mode = icr_low & APIC_MODE_MASK; 1151 irq.dest_mode = icr_low & APIC_DEST_MASK; 1152 irq.level = (icr_low & APIC_INT_ASSERT) != 0; 1153 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 1154 irq.shorthand = icr_low & APIC_SHORT_MASK; 1155 irq.msi_redir_hint = false; 1156 if (apic_x2apic_mode(apic)) 1157 irq.dest_id = icr_high; 1158 else 1159 irq.dest_id = GET_APIC_DEST_FIELD(icr_high); 1160 1161 trace_kvm_apic_ipi(icr_low, irq.dest_id); 1162 1163 apic_debug("icr_high 0x%x, icr_low 0x%x, " 1164 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, " 1165 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, " 1166 "msi_redir_hint 0x%x\n", 1167 icr_high, icr_low, irq.shorthand, irq.dest_id, 1168 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode, 1169 irq.vector, irq.msi_redir_hint); 1170 1171 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 1172 } 1173 1174 static u32 apic_get_tmcct(struct kvm_lapic *apic) 1175 { 1176 ktime_t remaining, now; 1177 s64 ns; 1178 u32 tmcct; 1179 1180 ASSERT(apic != NULL); 1181 1182 /* if initial count is 0, current count should also be 0 */ 1183 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 || 1184 apic->lapic_timer.period == 0) 1185 return 0; 1186 1187 now = ktime_get(); 1188 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1189 if (ktime_to_ns(remaining) < 0) 1190 remaining = 0; 1191 1192 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 1193 tmcct = div64_u64(ns, 1194 (APIC_BUS_CYCLE_NS * apic->divide_count)); 1195 1196 return tmcct; 1197 } 1198 1199 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 1200 { 1201 struct kvm_vcpu *vcpu = apic->vcpu; 1202 struct kvm_run *run = vcpu->run; 1203 1204 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 1205 run->tpr_access.rip = kvm_rip_read(vcpu); 1206 run->tpr_access.is_write = write; 1207 } 1208 1209 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 1210 { 1211 if (apic->vcpu->arch.tpr_access_reporting) 1212 __report_tpr_access(apic, write); 1213 } 1214 1215 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 1216 { 1217 u32 val = 0; 1218 1219 if (offset >= LAPIC_MMIO_LENGTH) 1220 return 0; 1221 1222 switch (offset) { 1223 case APIC_ARBPRI: 1224 apic_debug("Access APIC ARBPRI register which is for P6\n"); 1225 break; 1226 1227 case APIC_TMCCT: /* Timer CCR */ 1228 if (apic_lvtt_tscdeadline(apic)) 1229 return 0; 1230 1231 val = apic_get_tmcct(apic); 1232 break; 1233 case APIC_PROCPRI: 1234 apic_update_ppr(apic); 1235 val = kvm_lapic_get_reg(apic, offset); 1236 break; 1237 case APIC_TASKPRI: 1238 report_tpr_access(apic, false); 1239 /* fall thru */ 1240 default: 1241 val = kvm_lapic_get_reg(apic, offset); 1242 break; 1243 } 1244 1245 return val; 1246 } 1247 1248 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 1249 { 1250 return container_of(dev, struct kvm_lapic, dev); 1251 } 1252 1253 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 1254 void *data) 1255 { 1256 unsigned char alignment = offset & 0xf; 1257 u32 result; 1258 /* this bitmask has a bit cleared for each reserved register */ 1259 static const u64 rmask = 0x43ff01ffffffe70cULL; 1260 1261 if ((alignment + len) > 4) { 1262 apic_debug("KVM_APIC_READ: alignment error %x %d\n", 1263 offset, len); 1264 return 1; 1265 } 1266 1267 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) { 1268 apic_debug("KVM_APIC_READ: read reserved register %x\n", 1269 offset); 1270 return 1; 1271 } 1272 1273 result = __apic_read(apic, offset & ~0xf); 1274 1275 trace_kvm_apic_read(offset, result); 1276 1277 switch (len) { 1278 case 1: 1279 case 2: 1280 case 4: 1281 memcpy(data, (char *)&result + alignment, len); 1282 break; 1283 default: 1284 printk(KERN_ERR "Local APIC read with len = %x, " 1285 "should be 1,2, or 4 instead\n", len); 1286 break; 1287 } 1288 return 0; 1289 } 1290 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read); 1291 1292 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1293 { 1294 return kvm_apic_hw_enabled(apic) && 1295 addr >= apic->base_address && 1296 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1297 } 1298 1299 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1300 gpa_t address, int len, void *data) 1301 { 1302 struct kvm_lapic *apic = to_lapic(this); 1303 u32 offset = address - apic->base_address; 1304 1305 if (!apic_mmio_in_range(apic, address)) 1306 return -EOPNOTSUPP; 1307 1308 kvm_lapic_reg_read(apic, offset, len, data); 1309 1310 return 0; 1311 } 1312 1313 static void update_divide_count(struct kvm_lapic *apic) 1314 { 1315 u32 tmp1, tmp2, tdcr; 1316 1317 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR); 1318 tmp1 = tdcr & 0xf; 1319 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1320 apic->divide_count = 0x1 << (tmp2 & 0x7); 1321 1322 apic_debug("timer divide count is 0x%x\n", 1323 apic->divide_count); 1324 } 1325 1326 static void limit_periodic_timer_frequency(struct kvm_lapic *apic) 1327 { 1328 /* 1329 * Do not allow the guest to program periodic timers with small 1330 * interval, since the hrtimers are not throttled by the host 1331 * scheduler. 1332 */ 1333 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1334 s64 min_period = min_timer_period_us * 1000LL; 1335 1336 if (apic->lapic_timer.period < min_period) { 1337 pr_info_ratelimited( 1338 "kvm: vcpu %i: requested %lld ns " 1339 "lapic timer period limited to %lld ns\n", 1340 apic->vcpu->vcpu_id, 1341 apic->lapic_timer.period, min_period); 1342 apic->lapic_timer.period = min_period; 1343 } 1344 } 1345 } 1346 1347 static void apic_update_lvtt(struct kvm_lapic *apic) 1348 { 1349 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & 1350 apic->lapic_timer.timer_mode_mask; 1351 1352 if (apic->lapic_timer.timer_mode != timer_mode) { 1353 if (apic_lvtt_tscdeadline(apic) != (timer_mode == 1354 APIC_LVT_TIMER_TSCDEADLINE)) { 1355 hrtimer_cancel(&apic->lapic_timer.timer); 1356 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 1357 apic->lapic_timer.period = 0; 1358 apic->lapic_timer.tscdeadline = 0; 1359 } 1360 apic->lapic_timer.timer_mode = timer_mode; 1361 limit_periodic_timer_frequency(apic); 1362 } 1363 } 1364 1365 static void apic_timer_expired(struct kvm_lapic *apic) 1366 { 1367 struct kvm_vcpu *vcpu = apic->vcpu; 1368 struct swait_queue_head *q = &vcpu->wq; 1369 struct kvm_timer *ktimer = &apic->lapic_timer; 1370 1371 if (atomic_read(&apic->lapic_timer.pending)) 1372 return; 1373 1374 atomic_inc(&apic->lapic_timer.pending); 1375 kvm_set_pending_timer(vcpu); 1376 1377 /* 1378 * For x86, the atomic_inc() is serialized, thus 1379 * using swait_active() is safe. 1380 */ 1381 if (swait_active(q)) 1382 swake_up(q); 1383 1384 if (apic_lvtt_tscdeadline(apic)) 1385 ktimer->expired_tscdeadline = ktimer->tscdeadline; 1386 } 1387 1388 /* 1389 * On APICv, this test will cause a busy wait 1390 * during a higher-priority task. 1391 */ 1392 1393 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu) 1394 { 1395 struct kvm_lapic *apic = vcpu->arch.apic; 1396 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT); 1397 1398 if (kvm_apic_hw_enabled(apic)) { 1399 int vec = reg & APIC_VECTOR_MASK; 1400 void *bitmap = apic->regs + APIC_ISR; 1401 1402 if (vcpu->arch.apicv_active) 1403 bitmap = apic->regs + APIC_IRR; 1404 1405 if (apic_test_vector(vec, bitmap)) 1406 return true; 1407 } 1408 return false; 1409 } 1410 1411 void wait_lapic_expire(struct kvm_vcpu *vcpu) 1412 { 1413 struct kvm_lapic *apic = vcpu->arch.apic; 1414 u64 guest_tsc, tsc_deadline; 1415 1416 if (!lapic_in_kernel(vcpu)) 1417 return; 1418 1419 if (apic->lapic_timer.expired_tscdeadline == 0) 1420 return; 1421 1422 if (!lapic_timer_int_injected(vcpu)) 1423 return; 1424 1425 tsc_deadline = apic->lapic_timer.expired_tscdeadline; 1426 apic->lapic_timer.expired_tscdeadline = 0; 1427 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1428 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); 1429 1430 /* __delay is delay_tsc whenever the hardware has TSC, thus always. */ 1431 if (guest_tsc < tsc_deadline) 1432 __delay(min(tsc_deadline - guest_tsc, 1433 nsec_to_cycles(vcpu, lapic_timer_advance_ns))); 1434 } 1435 1436 static void start_sw_tscdeadline(struct kvm_lapic *apic) 1437 { 1438 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline; 1439 u64 ns = 0; 1440 ktime_t expire; 1441 struct kvm_vcpu *vcpu = apic->vcpu; 1442 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1443 unsigned long flags; 1444 ktime_t now; 1445 1446 if (unlikely(!tscdeadline || !this_tsc_khz)) 1447 return; 1448 1449 local_irq_save(flags); 1450 1451 now = ktime_get(); 1452 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1453 if (likely(tscdeadline > guest_tsc)) { 1454 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1455 do_div(ns, this_tsc_khz); 1456 expire = ktime_add_ns(now, ns); 1457 expire = ktime_sub_ns(expire, lapic_timer_advance_ns); 1458 hrtimer_start(&apic->lapic_timer.timer, 1459 expire, HRTIMER_MODE_ABS_PINNED); 1460 } else 1461 apic_timer_expired(apic); 1462 1463 local_irq_restore(flags); 1464 } 1465 1466 static void start_sw_period(struct kvm_lapic *apic) 1467 { 1468 if (!apic->lapic_timer.period) 1469 return; 1470 1471 if (apic_lvtt_oneshot(apic) && 1472 ktime_after(ktime_get(), 1473 apic->lapic_timer.target_expiration)) { 1474 apic_timer_expired(apic); 1475 return; 1476 } 1477 1478 hrtimer_start(&apic->lapic_timer.timer, 1479 apic->lapic_timer.target_expiration, 1480 HRTIMER_MODE_ABS_PINNED); 1481 } 1482 1483 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) 1484 { 1485 ktime_t now, remaining; 1486 u64 ns_remaining_old, ns_remaining_new; 1487 1488 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1489 * APIC_BUS_CYCLE_NS * apic->divide_count; 1490 limit_periodic_timer_frequency(apic); 1491 1492 now = ktime_get(); 1493 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1494 if (ktime_to_ns(remaining) < 0) 1495 remaining = 0; 1496 1497 ns_remaining_old = ktime_to_ns(remaining); 1498 ns_remaining_new = mul_u64_u32_div(ns_remaining_old, 1499 apic->divide_count, old_divisor); 1500 1501 apic->lapic_timer.tscdeadline += 1502 nsec_to_cycles(apic->vcpu, ns_remaining_new) - 1503 nsec_to_cycles(apic->vcpu, ns_remaining_old); 1504 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); 1505 } 1506 1507 static bool set_target_expiration(struct kvm_lapic *apic) 1508 { 1509 ktime_t now; 1510 u64 tscl = rdtsc(); 1511 1512 now = ktime_get(); 1513 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1514 * APIC_BUS_CYCLE_NS * apic->divide_count; 1515 1516 if (!apic->lapic_timer.period) { 1517 apic->lapic_timer.tscdeadline = 0; 1518 return false; 1519 } 1520 1521 limit_periodic_timer_frequency(apic); 1522 1523 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" 1524 PRIx64 ", " 1525 "timer initial count 0x%x, period %lldns, " 1526 "expire @ 0x%016" PRIx64 ".\n", __func__, 1527 APIC_BUS_CYCLE_NS, ktime_to_ns(now), 1528 kvm_lapic_get_reg(apic, APIC_TMICT), 1529 apic->lapic_timer.period, 1530 ktime_to_ns(ktime_add_ns(now, 1531 apic->lapic_timer.period))); 1532 1533 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 1534 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); 1535 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period); 1536 1537 return true; 1538 } 1539 1540 static void advance_periodic_target_expiration(struct kvm_lapic *apic) 1541 { 1542 apic->lapic_timer.tscdeadline += 1543 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); 1544 apic->lapic_timer.target_expiration = 1545 ktime_add_ns(apic->lapic_timer.target_expiration, 1546 apic->lapic_timer.period); 1547 } 1548 1549 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) 1550 { 1551 if (!lapic_in_kernel(vcpu)) 1552 return false; 1553 1554 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; 1555 } 1556 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use); 1557 1558 static void cancel_hv_timer(struct kvm_lapic *apic) 1559 { 1560 WARN_ON(preemptible()); 1561 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1562 kvm_x86_ops->cancel_hv_timer(apic->vcpu); 1563 apic->lapic_timer.hv_timer_in_use = false; 1564 } 1565 1566 static bool start_hv_timer(struct kvm_lapic *apic) 1567 { 1568 struct kvm_timer *ktimer = &apic->lapic_timer; 1569 int r; 1570 1571 WARN_ON(preemptible()); 1572 if (!kvm_x86_ops->set_hv_timer) 1573 return false; 1574 1575 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 1576 return false; 1577 1578 if (!ktimer->tscdeadline) 1579 return false; 1580 1581 r = kvm_x86_ops->set_hv_timer(apic->vcpu, ktimer->tscdeadline); 1582 if (r < 0) 1583 return false; 1584 1585 ktimer->hv_timer_in_use = true; 1586 hrtimer_cancel(&ktimer->timer); 1587 1588 /* 1589 * Also recheck ktimer->pending, in case the sw timer triggered in 1590 * the window. For periodic timer, leave the hv timer running for 1591 * simplicity, and the deadline will be recomputed on the next vmexit. 1592 */ 1593 if (!apic_lvtt_period(apic) && (r || atomic_read(&ktimer->pending))) { 1594 if (r) 1595 apic_timer_expired(apic); 1596 return false; 1597 } 1598 1599 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, true); 1600 return true; 1601 } 1602 1603 static void start_sw_timer(struct kvm_lapic *apic) 1604 { 1605 struct kvm_timer *ktimer = &apic->lapic_timer; 1606 1607 WARN_ON(preemptible()); 1608 if (apic->lapic_timer.hv_timer_in_use) 1609 cancel_hv_timer(apic); 1610 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 1611 return; 1612 1613 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1614 start_sw_period(apic); 1615 else if (apic_lvtt_tscdeadline(apic)) 1616 start_sw_tscdeadline(apic); 1617 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); 1618 } 1619 1620 static void restart_apic_timer(struct kvm_lapic *apic) 1621 { 1622 preempt_disable(); 1623 if (!start_hv_timer(apic)) 1624 start_sw_timer(apic); 1625 preempt_enable(); 1626 } 1627 1628 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) 1629 { 1630 struct kvm_lapic *apic = vcpu->arch.apic; 1631 1632 preempt_disable(); 1633 /* If the preempt notifier has already run, it also called apic_timer_expired */ 1634 if (!apic->lapic_timer.hv_timer_in_use) 1635 goto out; 1636 WARN_ON(swait_active(&vcpu->wq)); 1637 cancel_hv_timer(apic); 1638 apic_timer_expired(apic); 1639 1640 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1641 advance_periodic_target_expiration(apic); 1642 restart_apic_timer(apic); 1643 } 1644 out: 1645 preempt_enable(); 1646 } 1647 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); 1648 1649 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) 1650 { 1651 restart_apic_timer(vcpu->arch.apic); 1652 } 1653 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer); 1654 1655 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) 1656 { 1657 struct kvm_lapic *apic = vcpu->arch.apic; 1658 1659 preempt_disable(); 1660 /* Possibly the TSC deadline timer is not enabled yet */ 1661 if (apic->lapic_timer.hv_timer_in_use) 1662 start_sw_timer(apic); 1663 preempt_enable(); 1664 } 1665 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer); 1666 1667 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) 1668 { 1669 struct kvm_lapic *apic = vcpu->arch.apic; 1670 1671 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1672 restart_apic_timer(apic); 1673 } 1674 1675 static void start_apic_timer(struct kvm_lapic *apic) 1676 { 1677 atomic_set(&apic->lapic_timer.pending, 0); 1678 1679 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1680 && !set_target_expiration(apic)) 1681 return; 1682 1683 restart_apic_timer(apic); 1684 } 1685 1686 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 1687 { 1688 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val); 1689 1690 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { 1691 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; 1692 if (lvt0_in_nmi_mode) { 1693 apic_debug("Receive NMI setting on APIC_LVT0 " 1694 "for cpu %d\n", apic->vcpu->vcpu_id); 1695 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1696 } else 1697 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1698 } 1699 } 1700 1701 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 1702 { 1703 int ret = 0; 1704 1705 trace_kvm_apic_write(reg, val); 1706 1707 switch (reg) { 1708 case APIC_ID: /* Local APIC ID */ 1709 if (!apic_x2apic_mode(apic)) 1710 kvm_apic_set_xapic_id(apic, val >> 24); 1711 else 1712 ret = 1; 1713 break; 1714 1715 case APIC_TASKPRI: 1716 report_tpr_access(apic, true); 1717 apic_set_tpr(apic, val & 0xff); 1718 break; 1719 1720 case APIC_EOI: 1721 apic_set_eoi(apic); 1722 break; 1723 1724 case APIC_LDR: 1725 if (!apic_x2apic_mode(apic)) 1726 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 1727 else 1728 ret = 1; 1729 break; 1730 1731 case APIC_DFR: 1732 if (!apic_x2apic_mode(apic)) { 1733 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); 1734 recalculate_apic_map(apic->vcpu->kvm); 1735 } else 1736 ret = 1; 1737 break; 1738 1739 case APIC_SPIV: { 1740 u32 mask = 0x3ff; 1741 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 1742 mask |= APIC_SPIV_DIRECTED_EOI; 1743 apic_set_spiv(apic, val & mask); 1744 if (!(val & APIC_SPIV_APIC_ENABLED)) { 1745 int i; 1746 u32 lvt_val; 1747 1748 for (i = 0; i < KVM_APIC_LVT_NUM; i++) { 1749 lvt_val = kvm_lapic_get_reg(apic, 1750 APIC_LVTT + 0x10 * i); 1751 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, 1752 lvt_val | APIC_LVT_MASKED); 1753 } 1754 apic_update_lvtt(apic); 1755 atomic_set(&apic->lapic_timer.pending, 0); 1756 1757 } 1758 break; 1759 } 1760 case APIC_ICR: 1761 /* No delay here, so we always clear the pending bit */ 1762 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12)); 1763 apic_send_ipi(apic); 1764 break; 1765 1766 case APIC_ICR2: 1767 if (!apic_x2apic_mode(apic)) 1768 val &= 0xff000000; 1769 kvm_lapic_set_reg(apic, APIC_ICR2, val); 1770 break; 1771 1772 case APIC_LVT0: 1773 apic_manage_nmi_watchdog(apic, val); 1774 case APIC_LVTTHMR: 1775 case APIC_LVTPC: 1776 case APIC_LVT1: 1777 case APIC_LVTERR: 1778 /* TODO: Check vector */ 1779 if (!kvm_apic_sw_enabled(apic)) 1780 val |= APIC_LVT_MASKED; 1781 1782 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; 1783 kvm_lapic_set_reg(apic, reg, val); 1784 1785 break; 1786 1787 case APIC_LVTT: 1788 if (!kvm_apic_sw_enabled(apic)) 1789 val |= APIC_LVT_MASKED; 1790 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 1791 kvm_lapic_set_reg(apic, APIC_LVTT, val); 1792 apic_update_lvtt(apic); 1793 break; 1794 1795 case APIC_TMICT: 1796 if (apic_lvtt_tscdeadline(apic)) 1797 break; 1798 1799 hrtimer_cancel(&apic->lapic_timer.timer); 1800 kvm_lapic_set_reg(apic, APIC_TMICT, val); 1801 start_apic_timer(apic); 1802 break; 1803 1804 case APIC_TDCR: { 1805 uint32_t old_divisor = apic->divide_count; 1806 1807 if (val & 4) 1808 apic_debug("KVM_WRITE:TDCR %x\n", val); 1809 kvm_lapic_set_reg(apic, APIC_TDCR, val); 1810 update_divide_count(apic); 1811 if (apic->divide_count != old_divisor && 1812 apic->lapic_timer.period) { 1813 hrtimer_cancel(&apic->lapic_timer.timer); 1814 update_target_expiration(apic, old_divisor); 1815 restart_apic_timer(apic); 1816 } 1817 break; 1818 } 1819 case APIC_ESR: 1820 if (apic_x2apic_mode(apic) && val != 0) { 1821 apic_debug("KVM_WRITE:ESR not zero %x\n", val); 1822 ret = 1; 1823 } 1824 break; 1825 1826 case APIC_SELF_IPI: 1827 if (apic_x2apic_mode(apic)) { 1828 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff)); 1829 } else 1830 ret = 1; 1831 break; 1832 default: 1833 ret = 1; 1834 break; 1835 } 1836 if (ret) 1837 apic_debug("Local APIC Write to read-only register %x\n", reg); 1838 return ret; 1839 } 1840 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write); 1841 1842 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1843 gpa_t address, int len, const void *data) 1844 { 1845 struct kvm_lapic *apic = to_lapic(this); 1846 unsigned int offset = address - apic->base_address; 1847 u32 val; 1848 1849 if (!apic_mmio_in_range(apic, address)) 1850 return -EOPNOTSUPP; 1851 1852 /* 1853 * APIC register must be aligned on 128-bits boundary. 1854 * 32/64/128 bits registers must be accessed thru 32 bits. 1855 * Refer SDM 8.4.1 1856 */ 1857 if (len != 4 || (offset & 0xf)) { 1858 /* Don't shout loud, $infamous_os would cause only noise. */ 1859 apic_debug("apic write: bad size=%d %lx\n", len, (long)address); 1860 return 0; 1861 } 1862 1863 val = *(u32*)data; 1864 1865 /* too common printing */ 1866 if (offset != APIC_EOI) 1867 apic_debug("%s: offset 0x%x with length 0x%x, and value is " 1868 "0x%x\n", __func__, offset, len, val); 1869 1870 kvm_lapic_reg_write(apic, offset & 0xff0, val); 1871 1872 return 0; 1873 } 1874 1875 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 1876 { 1877 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 1878 } 1879 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 1880 1881 /* emulate APIC access in a trap manner */ 1882 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 1883 { 1884 u32 val = 0; 1885 1886 /* hw has done the conditional check and inst decode */ 1887 offset &= 0xff0; 1888 1889 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val); 1890 1891 /* TODO: optimize to just emulate side effect w/o one more write */ 1892 kvm_lapic_reg_write(vcpu->arch.apic, offset, val); 1893 } 1894 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 1895 1896 void kvm_free_lapic(struct kvm_vcpu *vcpu) 1897 { 1898 struct kvm_lapic *apic = vcpu->arch.apic; 1899 1900 if (!vcpu->arch.apic) 1901 return; 1902 1903 hrtimer_cancel(&apic->lapic_timer.timer); 1904 1905 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 1906 static_key_slow_dec_deferred(&apic_hw_disabled); 1907 1908 if (!apic->sw_enabled) 1909 static_key_slow_dec_deferred(&apic_sw_disabled); 1910 1911 if (apic->regs) 1912 free_page((unsigned long)apic->regs); 1913 1914 kfree(apic); 1915 } 1916 1917 /* 1918 *---------------------------------------------------------------------- 1919 * LAPIC interface 1920 *---------------------------------------------------------------------- 1921 */ 1922 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 1923 { 1924 struct kvm_lapic *apic = vcpu->arch.apic; 1925 1926 if (!lapic_in_kernel(vcpu) || 1927 !apic_lvtt_tscdeadline(apic)) 1928 return 0; 1929 1930 return apic->lapic_timer.tscdeadline; 1931 } 1932 1933 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 1934 { 1935 struct kvm_lapic *apic = vcpu->arch.apic; 1936 1937 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) || 1938 apic_lvtt_period(apic)) 1939 return; 1940 1941 hrtimer_cancel(&apic->lapic_timer.timer); 1942 apic->lapic_timer.tscdeadline = data; 1943 start_apic_timer(apic); 1944 } 1945 1946 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 1947 { 1948 struct kvm_lapic *apic = vcpu->arch.apic; 1949 1950 apic_set_tpr(apic, ((cr8 & 0x0f) << 4) 1951 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4)); 1952 } 1953 1954 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 1955 { 1956 u64 tpr; 1957 1958 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 1959 1960 return (tpr & 0xf0) >> 4; 1961 } 1962 1963 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 1964 { 1965 u64 old_value = vcpu->arch.apic_base; 1966 struct kvm_lapic *apic = vcpu->arch.apic; 1967 1968 if (!apic) 1969 value |= MSR_IA32_APICBASE_BSP; 1970 1971 vcpu->arch.apic_base = value; 1972 1973 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) 1974 kvm_update_cpuid(vcpu); 1975 1976 if (!apic) 1977 return; 1978 1979 /* update jump label if enable bit changes */ 1980 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 1981 if (value & MSR_IA32_APICBASE_ENABLE) { 1982 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 1983 static_key_slow_dec_deferred(&apic_hw_disabled); 1984 } else { 1985 static_key_slow_inc(&apic_hw_disabled.key); 1986 recalculate_apic_map(vcpu->kvm); 1987 } 1988 } 1989 1990 if ((old_value ^ value) & X2APIC_ENABLE) { 1991 if (value & X2APIC_ENABLE) { 1992 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); 1993 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true); 1994 } else 1995 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false); 1996 } 1997 1998 apic->base_address = apic->vcpu->arch.apic_base & 1999 MSR_IA32_APICBASE_BASE; 2000 2001 if ((value & MSR_IA32_APICBASE_ENABLE) && 2002 apic->base_address != APIC_DEFAULT_PHYS_BASE) 2003 pr_warn_once("APIC base relocation is unsupported by KVM"); 2004 2005 /* with FSB delivery interrupt, we can restart APIC functionality */ 2006 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " 2007 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); 2008 2009 } 2010 2011 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) 2012 { 2013 struct kvm_lapic *apic = vcpu->arch.apic; 2014 int i; 2015 2016 if (!apic) 2017 return; 2018 2019 apic_debug("%s\n", __func__); 2020 2021 /* Stop the timer in case it's a reset to an active apic */ 2022 hrtimer_cancel(&apic->lapic_timer.timer); 2023 2024 if (!init_event) { 2025 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE | 2026 MSR_IA32_APICBASE_ENABLE); 2027 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2028 } 2029 kvm_apic_set_version(apic->vcpu); 2030 2031 for (i = 0; i < KVM_APIC_LVT_NUM; i++) 2032 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED); 2033 apic_update_lvtt(apic); 2034 if (kvm_vcpu_is_reset_bsp(vcpu) && 2035 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) 2036 kvm_lapic_set_reg(apic, APIC_LVT0, 2037 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 2038 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2039 2040 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU); 2041 apic_set_spiv(apic, 0xff); 2042 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0); 2043 if (!apic_x2apic_mode(apic)) 2044 kvm_apic_set_ldr(apic, 0); 2045 kvm_lapic_set_reg(apic, APIC_ESR, 0); 2046 kvm_lapic_set_reg(apic, APIC_ICR, 0); 2047 kvm_lapic_set_reg(apic, APIC_ICR2, 0); 2048 kvm_lapic_set_reg(apic, APIC_TDCR, 0); 2049 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 2050 for (i = 0; i < 8; i++) { 2051 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 2052 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 2053 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 2054 } 2055 apic->irr_pending = vcpu->arch.apicv_active; 2056 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0; 2057 apic->highest_isr_cache = -1; 2058 update_divide_count(apic); 2059 atomic_set(&apic->lapic_timer.pending, 0); 2060 if (kvm_vcpu_is_bsp(vcpu)) 2061 kvm_lapic_set_base(vcpu, 2062 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP); 2063 vcpu->arch.pv_eoi.msr_val = 0; 2064 apic_update_ppr(apic); 2065 if (vcpu->arch.apicv_active) { 2066 kvm_x86_ops->apicv_post_state_restore(vcpu); 2067 kvm_x86_ops->hwapic_irr_update(vcpu, -1); 2068 kvm_x86_ops->hwapic_isr_update(vcpu, -1); 2069 } 2070 2071 vcpu->arch.apic_arb_prio = 0; 2072 vcpu->arch.apic_attention = 0; 2073 2074 apic_debug("%s: vcpu=%p, id=0x%x, base_msr=" 2075 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__, 2076 vcpu, kvm_lapic_get_reg(apic, APIC_ID), 2077 vcpu->arch.apic_base, apic->base_address); 2078 } 2079 2080 /* 2081 *---------------------------------------------------------------------- 2082 * timer interface 2083 *---------------------------------------------------------------------- 2084 */ 2085 2086 static bool lapic_is_periodic(struct kvm_lapic *apic) 2087 { 2088 return apic_lvtt_period(apic); 2089 } 2090 2091 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 2092 { 2093 struct kvm_lapic *apic = vcpu->arch.apic; 2094 2095 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT)) 2096 return atomic_read(&apic->lapic_timer.pending); 2097 2098 return 0; 2099 } 2100 2101 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 2102 { 2103 u32 reg = kvm_lapic_get_reg(apic, lvt_type); 2104 int vector, mode, trig_mode; 2105 2106 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 2107 vector = reg & APIC_VECTOR_MASK; 2108 mode = reg & APIC_MODE_MASK; 2109 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 2110 return __apic_accept_irq(apic, mode, vector, 1, trig_mode, 2111 NULL); 2112 } 2113 return 0; 2114 } 2115 2116 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 2117 { 2118 struct kvm_lapic *apic = vcpu->arch.apic; 2119 2120 if (apic) 2121 kvm_apic_local_deliver(apic, APIC_LVT0); 2122 } 2123 2124 static const struct kvm_io_device_ops apic_mmio_ops = { 2125 .read = apic_mmio_read, 2126 .write = apic_mmio_write, 2127 }; 2128 2129 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 2130 { 2131 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 2132 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 2133 2134 apic_timer_expired(apic); 2135 2136 if (lapic_is_periodic(apic)) { 2137 advance_periodic_target_expiration(apic); 2138 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 2139 return HRTIMER_RESTART; 2140 } else 2141 return HRTIMER_NORESTART; 2142 } 2143 2144 int kvm_create_lapic(struct kvm_vcpu *vcpu) 2145 { 2146 struct kvm_lapic *apic; 2147 2148 ASSERT(vcpu != NULL); 2149 apic_debug("apic_init %d\n", vcpu->vcpu_id); 2150 2151 apic = kzalloc(sizeof(*apic), GFP_KERNEL); 2152 if (!apic) 2153 goto nomem; 2154 2155 vcpu->arch.apic = apic; 2156 2157 apic->regs = (void *)get_zeroed_page(GFP_KERNEL); 2158 if (!apic->regs) { 2159 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 2160 vcpu->vcpu_id); 2161 goto nomem_free_apic; 2162 } 2163 apic->vcpu = vcpu; 2164 2165 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 2166 HRTIMER_MODE_ABS_PINNED); 2167 apic->lapic_timer.timer.function = apic_timer_fn; 2168 2169 /* 2170 * APIC is created enabled. This will prevent kvm_lapic_set_base from 2171 * thinking that APIC satet has changed. 2172 */ 2173 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 2174 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 2175 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 2176 2177 return 0; 2178 nomem_free_apic: 2179 kfree(apic); 2180 nomem: 2181 return -ENOMEM; 2182 } 2183 2184 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 2185 { 2186 struct kvm_lapic *apic = vcpu->arch.apic; 2187 u32 ppr; 2188 2189 if (!apic_enabled(apic)) 2190 return -1; 2191 2192 __apic_update_ppr(apic, &ppr); 2193 return apic_has_interrupt_for_ppr(apic, ppr); 2194 } 2195 2196 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 2197 { 2198 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); 2199 int r = 0; 2200 2201 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 2202 r = 1; 2203 if ((lvt0 & APIC_LVT_MASKED) == 0 && 2204 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 2205 r = 1; 2206 return r; 2207 } 2208 2209 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 2210 { 2211 struct kvm_lapic *apic = vcpu->arch.apic; 2212 2213 if (atomic_read(&apic->lapic_timer.pending) > 0) { 2214 kvm_apic_local_deliver(apic, APIC_LVTT); 2215 if (apic_lvtt_tscdeadline(apic)) 2216 apic->lapic_timer.tscdeadline = 0; 2217 if (apic_lvtt_oneshot(apic)) { 2218 apic->lapic_timer.tscdeadline = 0; 2219 apic->lapic_timer.target_expiration = 0; 2220 } 2221 atomic_set(&apic->lapic_timer.pending, 0); 2222 } 2223 } 2224 2225 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 2226 { 2227 int vector = kvm_apic_has_interrupt(vcpu); 2228 struct kvm_lapic *apic = vcpu->arch.apic; 2229 u32 ppr; 2230 2231 if (vector == -1) 2232 return -1; 2233 2234 /* 2235 * We get here even with APIC virtualization enabled, if doing 2236 * nested virtualization and L1 runs with the "acknowledge interrupt 2237 * on exit" mode. Then we cannot inject the interrupt via RVI, 2238 * because the process would deliver it through the IDT. 2239 */ 2240 2241 apic_clear_irr(vector, apic); 2242 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) { 2243 /* 2244 * For auto-EOI interrupts, there might be another pending 2245 * interrupt above PPR, so check whether to raise another 2246 * KVM_REQ_EVENT. 2247 */ 2248 apic_update_ppr(apic); 2249 } else { 2250 /* 2251 * For normal interrupts, PPR has been raised and there cannot 2252 * be a higher-priority pending interrupt---except if there was 2253 * a concurrent interrupt injection, but that would have 2254 * triggered KVM_REQ_EVENT already. 2255 */ 2256 apic_set_isr(vector, apic); 2257 __apic_update_ppr(apic, &ppr); 2258 } 2259 2260 return vector; 2261 } 2262 2263 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, 2264 struct kvm_lapic_state *s, bool set) 2265 { 2266 if (apic_x2apic_mode(vcpu->arch.apic)) { 2267 u32 *id = (u32 *)(s->regs + APIC_ID); 2268 u32 *ldr = (u32 *)(s->regs + APIC_LDR); 2269 2270 if (vcpu->kvm->arch.x2apic_format) { 2271 if (*id != vcpu->vcpu_id) 2272 return -EINVAL; 2273 } else { 2274 if (set) 2275 *id >>= 24; 2276 else 2277 *id <<= 24; 2278 } 2279 2280 /* In x2APIC mode, the LDR is fixed and based on the id */ 2281 if (set) 2282 *ldr = kvm_apic_calc_x2apic_ldr(*id); 2283 } 2284 2285 return 0; 2286 } 2287 2288 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2289 { 2290 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); 2291 return kvm_apic_state_fixup(vcpu, s, false); 2292 } 2293 2294 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2295 { 2296 struct kvm_lapic *apic = vcpu->arch.apic; 2297 int r; 2298 2299 2300 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 2301 /* set SPIV separately to get count of SW disabled APICs right */ 2302 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 2303 2304 r = kvm_apic_state_fixup(vcpu, s, true); 2305 if (r) 2306 return r; 2307 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); 2308 2309 recalculate_apic_map(vcpu->kvm); 2310 kvm_apic_set_version(vcpu); 2311 2312 apic_update_ppr(apic); 2313 hrtimer_cancel(&apic->lapic_timer.timer); 2314 apic_update_lvtt(apic); 2315 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2316 update_divide_count(apic); 2317 start_apic_timer(apic); 2318 apic->irr_pending = true; 2319 apic->isr_count = vcpu->arch.apicv_active ? 2320 1 : count_vectors(apic->regs + APIC_ISR); 2321 apic->highest_isr_cache = -1; 2322 if (vcpu->arch.apicv_active) { 2323 kvm_x86_ops->apicv_post_state_restore(vcpu); 2324 kvm_x86_ops->hwapic_irr_update(vcpu, 2325 apic_find_highest_irr(apic)); 2326 kvm_x86_ops->hwapic_isr_update(vcpu, 2327 apic_find_highest_isr(apic)); 2328 } 2329 kvm_make_request(KVM_REQ_EVENT, vcpu); 2330 if (ioapic_in_kernel(vcpu->kvm)) 2331 kvm_rtc_eoi_tracking_restore_one(vcpu); 2332 2333 vcpu->arch.apic_arb_prio = 0; 2334 2335 return 0; 2336 } 2337 2338 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 2339 { 2340 struct hrtimer *timer; 2341 2342 if (!lapic_in_kernel(vcpu)) 2343 return; 2344 2345 timer = &vcpu->arch.apic->lapic_timer.timer; 2346 if (hrtimer_cancel(timer)) 2347 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 2348 } 2349 2350 /* 2351 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 2352 * 2353 * Detect whether guest triggered PV EOI since the 2354 * last entry. If yes, set EOI on guests's behalf. 2355 * Clear PV EOI in guest memory in any case. 2356 */ 2357 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 2358 struct kvm_lapic *apic) 2359 { 2360 bool pending; 2361 int vector; 2362 /* 2363 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 2364 * and KVM_PV_EOI_ENABLED in guest memory as follows: 2365 * 2366 * KVM_APIC_PV_EOI_PENDING is unset: 2367 * -> host disabled PV EOI. 2368 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 2369 * -> host enabled PV EOI, guest did not execute EOI yet. 2370 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 2371 * -> host enabled PV EOI, guest executed EOI. 2372 */ 2373 BUG_ON(!pv_eoi_enabled(vcpu)); 2374 pending = pv_eoi_get_pending(vcpu); 2375 /* 2376 * Clear pending bit in any case: it will be set again on vmentry. 2377 * While this might not be ideal from performance point of view, 2378 * this makes sure pv eoi is only enabled when we know it's safe. 2379 */ 2380 pv_eoi_clr_pending(vcpu); 2381 if (pending) 2382 return; 2383 vector = apic_set_eoi(apic); 2384 trace_kvm_pv_eoi(apic, vector); 2385 } 2386 2387 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 2388 { 2389 u32 data; 2390 2391 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 2392 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 2393 2394 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2395 return; 2396 2397 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2398 sizeof(u32))) 2399 return; 2400 2401 apic_set_tpr(vcpu->arch.apic, data & 0xff); 2402 } 2403 2404 /* 2405 * apic_sync_pv_eoi_to_guest - called before vmentry 2406 * 2407 * Detect whether it's safe to enable PV EOI and 2408 * if yes do so. 2409 */ 2410 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 2411 struct kvm_lapic *apic) 2412 { 2413 if (!pv_eoi_enabled(vcpu) || 2414 /* IRR set or many bits in ISR: could be nested. */ 2415 apic->irr_pending || 2416 /* Cache not set: could be safe but we don't bother. */ 2417 apic->highest_isr_cache == -1 || 2418 /* Need EOI to update ioapic. */ 2419 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { 2420 /* 2421 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 2422 * so we need not do anything here. 2423 */ 2424 return; 2425 } 2426 2427 pv_eoi_set_pending(apic->vcpu); 2428 } 2429 2430 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 2431 { 2432 u32 data, tpr; 2433 int max_irr, max_isr; 2434 struct kvm_lapic *apic = vcpu->arch.apic; 2435 2436 apic_sync_pv_eoi_to_guest(vcpu, apic); 2437 2438 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2439 return; 2440 2441 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff; 2442 max_irr = apic_find_highest_irr(apic); 2443 if (max_irr < 0) 2444 max_irr = 0; 2445 max_isr = apic_find_highest_isr(apic); 2446 if (max_isr < 0) 2447 max_isr = 0; 2448 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 2449 2450 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2451 sizeof(u32)); 2452 } 2453 2454 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 2455 { 2456 if (vapic_addr) { 2457 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 2458 &vcpu->arch.apic->vapic_cache, 2459 vapic_addr, sizeof(u32))) 2460 return -EINVAL; 2461 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2462 } else { 2463 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2464 } 2465 2466 vcpu->arch.apic->vapic_addr = vapic_addr; 2467 return 0; 2468 } 2469 2470 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 2471 { 2472 struct kvm_lapic *apic = vcpu->arch.apic; 2473 u32 reg = (msr - APIC_BASE_MSR) << 4; 2474 2475 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2476 return 1; 2477 2478 if (reg == APIC_ICR2) 2479 return 1; 2480 2481 /* if this is ICR write vector before command */ 2482 if (reg == APIC_ICR) 2483 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2484 return kvm_lapic_reg_write(apic, reg, (u32)data); 2485 } 2486 2487 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 2488 { 2489 struct kvm_lapic *apic = vcpu->arch.apic; 2490 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0; 2491 2492 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2493 return 1; 2494 2495 if (reg == APIC_DFR || reg == APIC_ICR2) { 2496 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n", 2497 reg); 2498 return 1; 2499 } 2500 2501 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2502 return 1; 2503 if (reg == APIC_ICR) 2504 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2505 2506 *data = (((u64)high) << 32) | low; 2507 2508 return 0; 2509 } 2510 2511 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 2512 { 2513 struct kvm_lapic *apic = vcpu->arch.apic; 2514 2515 if (!lapic_in_kernel(vcpu)) 2516 return 1; 2517 2518 /* if this is ICR write vector before command */ 2519 if (reg == APIC_ICR) 2520 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2521 return kvm_lapic_reg_write(apic, reg, (u32)data); 2522 } 2523 2524 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 2525 { 2526 struct kvm_lapic *apic = vcpu->arch.apic; 2527 u32 low, high = 0; 2528 2529 if (!lapic_in_kernel(vcpu)) 2530 return 1; 2531 2532 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2533 return 1; 2534 if (reg == APIC_ICR) 2535 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2536 2537 *data = (((u64)high) << 32) | low; 2538 2539 return 0; 2540 } 2541 2542 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data) 2543 { 2544 u64 addr = data & ~KVM_MSR_ENABLED; 2545 if (!IS_ALIGNED(addr, 4)) 2546 return 1; 2547 2548 vcpu->arch.pv_eoi.msr_val = data; 2549 if (!pv_eoi_enabled(vcpu)) 2550 return 0; 2551 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data, 2552 addr, sizeof(u8)); 2553 } 2554 2555 void kvm_apic_accept_events(struct kvm_vcpu *vcpu) 2556 { 2557 struct kvm_lapic *apic = vcpu->arch.apic; 2558 u8 sipi_vector; 2559 unsigned long pe; 2560 2561 if (!lapic_in_kernel(vcpu) || !apic->pending_events) 2562 return; 2563 2564 /* 2565 * INITs are latched while in SMM. Because an SMM CPU cannot 2566 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs 2567 * and delay processing of INIT until the next RSM. 2568 */ 2569 if (is_smm(vcpu)) { 2570 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); 2571 if (test_bit(KVM_APIC_SIPI, &apic->pending_events)) 2572 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 2573 return; 2574 } 2575 2576 pe = xchg(&apic->pending_events, 0); 2577 if (test_bit(KVM_APIC_INIT, &pe)) { 2578 kvm_vcpu_reset(vcpu, true); 2579 if (kvm_vcpu_is_bsp(apic->vcpu)) 2580 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2581 else 2582 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 2583 } 2584 if (test_bit(KVM_APIC_SIPI, &pe) && 2585 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 2586 /* evaluate pending_events before reading the vector */ 2587 smp_rmb(); 2588 sipi_vector = apic->sipi_vector; 2589 apic_debug("vcpu %d received sipi with vector # %x\n", 2590 vcpu->vcpu_id, sipi_vector); 2591 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector); 2592 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2593 } 2594 } 2595 2596 void kvm_lapic_init(void) 2597 { 2598 /* do not patch jump label more than once per second */ 2599 jump_label_rate_limit(&apic_hw_disabled, HZ); 2600 jump_label_rate_limit(&apic_sw_disabled, HZ); 2601 } 2602 2603 void kvm_lapic_exit(void) 2604 { 2605 static_key_deferred_flush(&apic_hw_disabled); 2606 static_key_deferred_flush(&apic_sw_disabled); 2607 } 2608