1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Local APIC virtualization 5 * 6 * Copyright (C) 2006 Qumranet, Inc. 7 * Copyright (C) 2007 Novell 8 * Copyright (C) 2007 Intel 9 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Dor Laor <dor.laor@qumranet.com> 13 * Gregory Haskins <ghaskins@novell.com> 14 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 15 * 16 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 17 */ 18 19 #include <linux/kvm_host.h> 20 #include <linux/kvm.h> 21 #include <linux/mm.h> 22 #include <linux/highmem.h> 23 #include <linux/smp.h> 24 #include <linux/hrtimer.h> 25 #include <linux/io.h> 26 #include <linux/export.h> 27 #include <linux/math64.h> 28 #include <linux/slab.h> 29 #include <asm/processor.h> 30 #include <asm/msr.h> 31 #include <asm/page.h> 32 #include <asm/current.h> 33 #include <asm/apicdef.h> 34 #include <asm/delay.h> 35 #include <linux/atomic.h> 36 #include <linux/jump_label.h> 37 #include "kvm_cache_regs.h" 38 #include "irq.h" 39 #include "trace.h" 40 #include "x86.h" 41 #include "cpuid.h" 42 #include "hyperv.h" 43 44 #ifndef CONFIG_X86_64 45 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 46 #else 47 #define mod_64(x, y) ((x) % (y)) 48 #endif 49 50 #define PRId64 "d" 51 #define PRIx64 "llx" 52 #define PRIu64 "u" 53 #define PRIo64 "o" 54 55 /* 14 is the version for Xeon and Pentium 8.4.8*/ 56 #define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16)) 57 #define LAPIC_MMIO_LENGTH (1 << 12) 58 /* followed define is not in apicdef.h */ 59 #define APIC_SHORT_MASK 0xc0000 60 #define APIC_DEST_NOSHORT 0x0 61 #define APIC_DEST_MASK 0x800 62 #define MAX_APIC_VECTOR 256 63 #define APIC_VECTORS_PER_REG 32 64 65 #define APIC_BROADCAST 0xFF 66 #define X2APIC_BROADCAST 0xFFFFFFFFul 67 68 #define LAPIC_TIMER_ADVANCE_ADJUST_DONE 100 69 #define LAPIC_TIMER_ADVANCE_ADJUST_INIT 1000 70 /* step-by-step approximation to mitigate fluctuation */ 71 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8 72 73 static inline int apic_test_vector(int vec, void *bitmap) 74 { 75 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 76 } 77 78 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 79 { 80 struct kvm_lapic *apic = vcpu->arch.apic; 81 82 return apic_test_vector(vector, apic->regs + APIC_ISR) || 83 apic_test_vector(vector, apic->regs + APIC_IRR); 84 } 85 86 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 87 { 88 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 89 } 90 91 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 92 { 93 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 94 } 95 96 struct static_key_deferred apic_hw_disabled __read_mostly; 97 struct static_key_deferred apic_sw_disabled __read_mostly; 98 99 static inline int apic_enabled(struct kvm_lapic *apic) 100 { 101 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 102 } 103 104 #define LVT_MASK \ 105 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 106 107 #define LINT_MASK \ 108 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 109 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 110 111 static inline u8 kvm_xapic_id(struct kvm_lapic *apic) 112 { 113 return kvm_lapic_get_reg(apic, APIC_ID) >> 24; 114 } 115 116 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) 117 { 118 return apic->vcpu->vcpu_id; 119 } 120 121 bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu) 122 { 123 return pi_inject_timer && kvm_vcpu_apicv_active(vcpu); 124 } 125 EXPORT_SYMBOL_GPL(kvm_can_post_timer_interrupt); 126 127 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu) 128 { 129 return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE; 130 } 131 132 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, 133 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { 134 switch (map->mode) { 135 case KVM_APIC_MODE_X2APIC: { 136 u32 offset = (dest_id >> 16) * 16; 137 u32 max_apic_id = map->max_apic_id; 138 139 if (offset <= max_apic_id) { 140 u8 cluster_size = min(max_apic_id - offset + 1, 16U); 141 142 offset = array_index_nospec(offset, map->max_apic_id + 1); 143 *cluster = &map->phys_map[offset]; 144 *mask = dest_id & (0xffff >> (16 - cluster_size)); 145 } else { 146 *mask = 0; 147 } 148 149 return true; 150 } 151 case KVM_APIC_MODE_XAPIC_FLAT: 152 *cluster = map->xapic_flat_map; 153 *mask = dest_id & 0xff; 154 return true; 155 case KVM_APIC_MODE_XAPIC_CLUSTER: 156 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; 157 *mask = dest_id & 0xf; 158 return true; 159 default: 160 /* Not optimized. */ 161 return false; 162 } 163 } 164 165 static void kvm_apic_map_free(struct rcu_head *rcu) 166 { 167 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu); 168 169 kvfree(map); 170 } 171 172 static void recalculate_apic_map(struct kvm *kvm) 173 { 174 struct kvm_apic_map *new, *old = NULL; 175 struct kvm_vcpu *vcpu; 176 int i; 177 u32 max_id = 255; /* enough space for any xAPIC ID */ 178 179 mutex_lock(&kvm->arch.apic_map_lock); 180 181 kvm_for_each_vcpu(i, vcpu, kvm) 182 if (kvm_apic_present(vcpu)) 183 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); 184 185 new = kvzalloc(sizeof(struct kvm_apic_map) + 186 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), 187 GFP_KERNEL_ACCOUNT); 188 189 if (!new) 190 goto out; 191 192 new->max_apic_id = max_id; 193 194 kvm_for_each_vcpu(i, vcpu, kvm) { 195 struct kvm_lapic *apic = vcpu->arch.apic; 196 struct kvm_lapic **cluster; 197 u16 mask; 198 u32 ldr; 199 u8 xapic_id; 200 u32 x2apic_id; 201 202 if (!kvm_apic_present(vcpu)) 203 continue; 204 205 xapic_id = kvm_xapic_id(apic); 206 x2apic_id = kvm_x2apic_id(apic); 207 208 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */ 209 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) && 210 x2apic_id <= new->max_apic_id) 211 new->phys_map[x2apic_id] = apic; 212 /* 213 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around, 214 * prevent them from masking VCPUs with APIC ID <= 0xff. 215 */ 216 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) 217 new->phys_map[xapic_id] = apic; 218 219 ldr = kvm_lapic_get_reg(apic, APIC_LDR); 220 221 if (apic_x2apic_mode(apic)) { 222 new->mode |= KVM_APIC_MODE_X2APIC; 223 } else if (ldr) { 224 ldr = GET_APIC_LOGICAL_ID(ldr); 225 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT) 226 new->mode |= KVM_APIC_MODE_XAPIC_FLAT; 227 else 228 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER; 229 } 230 231 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask)) 232 continue; 233 234 if (mask) 235 cluster[ffs(mask) - 1] = apic; 236 } 237 out: 238 old = rcu_dereference_protected(kvm->arch.apic_map, 239 lockdep_is_held(&kvm->arch.apic_map_lock)); 240 rcu_assign_pointer(kvm->arch.apic_map, new); 241 mutex_unlock(&kvm->arch.apic_map_lock); 242 243 if (old) 244 call_rcu(&old->rcu, kvm_apic_map_free); 245 246 kvm_make_scan_ioapic_request(kvm); 247 } 248 249 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 250 { 251 bool enabled = val & APIC_SPIV_APIC_ENABLED; 252 253 kvm_lapic_set_reg(apic, APIC_SPIV, val); 254 255 if (enabled != apic->sw_enabled) { 256 apic->sw_enabled = enabled; 257 if (enabled) 258 static_key_slow_dec_deferred(&apic_sw_disabled); 259 else 260 static_key_slow_inc(&apic_sw_disabled.key); 261 } 262 } 263 264 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) 265 { 266 kvm_lapic_set_reg(apic, APIC_ID, id << 24); 267 recalculate_apic_map(apic->vcpu->kvm); 268 } 269 270 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 271 { 272 kvm_lapic_set_reg(apic, APIC_LDR, id); 273 recalculate_apic_map(apic->vcpu->kvm); 274 } 275 276 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id) 277 { 278 return ((id >> 4) << 16) | (1 << (id & 0xf)); 279 } 280 281 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) 282 { 283 u32 ldr = kvm_apic_calc_x2apic_ldr(id); 284 285 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); 286 287 kvm_lapic_set_reg(apic, APIC_ID, id); 288 kvm_lapic_set_reg(apic, APIC_LDR, ldr); 289 recalculate_apic_map(apic->vcpu->kvm); 290 } 291 292 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 293 { 294 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 295 } 296 297 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) 298 { 299 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; 300 } 301 302 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 303 { 304 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; 305 } 306 307 static inline int apic_lvtt_period(struct kvm_lapic *apic) 308 { 309 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; 310 } 311 312 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 313 { 314 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; 315 } 316 317 static inline int apic_lvt_nmi_mode(u32 lvt_val) 318 { 319 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 320 } 321 322 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 323 { 324 struct kvm_lapic *apic = vcpu->arch.apic; 325 struct kvm_cpuid_entry2 *feat; 326 u32 v = APIC_VERSION; 327 328 if (!lapic_in_kernel(vcpu)) 329 return; 330 331 /* 332 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) 333 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with 334 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC 335 * version first and level-triggered interrupts never get EOIed in 336 * IOAPIC. 337 */ 338 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); 339 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) && 340 !ioapic_in_kernel(vcpu->kvm)) 341 v |= APIC_LVR_DIRECTED_EOI; 342 kvm_lapic_set_reg(apic, APIC_LVR, v); 343 } 344 345 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = { 346 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ 347 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ 348 LVT_MASK | APIC_MODE_MASK, /* LVTPC */ 349 LINT_MASK, LINT_MASK, /* LVT0-1 */ 350 LVT_MASK /* LVTERR */ 351 }; 352 353 static int find_highest_vector(void *bitmap) 354 { 355 int vec; 356 u32 *reg; 357 358 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 359 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 360 reg = bitmap + REG_POS(vec); 361 if (*reg) 362 return __fls(*reg) + vec; 363 } 364 365 return -1; 366 } 367 368 static u8 count_vectors(void *bitmap) 369 { 370 int vec; 371 u32 *reg; 372 u8 count = 0; 373 374 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 375 reg = bitmap + REG_POS(vec); 376 count += hweight32(*reg); 377 } 378 379 return count; 380 } 381 382 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr) 383 { 384 u32 i, vec; 385 u32 pir_val, irr_val, prev_irr_val; 386 int max_updated_irr; 387 388 max_updated_irr = -1; 389 *max_irr = -1; 390 391 for (i = vec = 0; i <= 7; i++, vec += 32) { 392 pir_val = READ_ONCE(pir[i]); 393 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10)); 394 if (pir_val) { 395 prev_irr_val = irr_val; 396 irr_val |= xchg(&pir[i], 0); 397 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val; 398 if (prev_irr_val != irr_val) { 399 max_updated_irr = 400 __fls(irr_val ^ prev_irr_val) + vec; 401 } 402 } 403 if (irr_val) 404 *max_irr = __fls(irr_val) + vec; 405 } 406 407 return ((max_updated_irr != -1) && 408 (max_updated_irr == *max_irr)); 409 } 410 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); 411 412 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr) 413 { 414 struct kvm_lapic *apic = vcpu->arch.apic; 415 416 return __kvm_apic_update_irr(pir, apic->regs, max_irr); 417 } 418 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 419 420 static inline int apic_search_irr(struct kvm_lapic *apic) 421 { 422 return find_highest_vector(apic->regs + APIC_IRR); 423 } 424 425 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 426 { 427 int result; 428 429 /* 430 * Note that irr_pending is just a hint. It will be always 431 * true with virtual interrupt delivery enabled. 432 */ 433 if (!apic->irr_pending) 434 return -1; 435 436 result = apic_search_irr(apic); 437 ASSERT(result == -1 || result >= 16); 438 439 return result; 440 } 441 442 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 443 { 444 struct kvm_vcpu *vcpu; 445 446 vcpu = apic->vcpu; 447 448 if (unlikely(vcpu->arch.apicv_active)) { 449 /* need to update RVI */ 450 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); 451 kvm_x86_ops->hwapic_irr_update(vcpu, 452 apic_find_highest_irr(apic)); 453 } else { 454 apic->irr_pending = false; 455 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); 456 if (apic_search_irr(apic) != -1) 457 apic->irr_pending = true; 458 } 459 } 460 461 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 462 { 463 struct kvm_vcpu *vcpu; 464 465 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 466 return; 467 468 vcpu = apic->vcpu; 469 470 /* 471 * With APIC virtualization enabled, all caching is disabled 472 * because the processor can modify ISR under the hood. Instead 473 * just set SVI. 474 */ 475 if (unlikely(vcpu->arch.apicv_active)) 476 kvm_x86_ops->hwapic_isr_update(vcpu, vec); 477 else { 478 ++apic->isr_count; 479 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 480 /* 481 * ISR (in service register) bit is set when injecting an interrupt. 482 * The highest vector is injected. Thus the latest bit set matches 483 * the highest bit in ISR. 484 */ 485 apic->highest_isr_cache = vec; 486 } 487 } 488 489 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 490 { 491 int result; 492 493 /* 494 * Note that isr_count is always 1, and highest_isr_cache 495 * is always -1, with APIC virtualization enabled. 496 */ 497 if (!apic->isr_count) 498 return -1; 499 if (likely(apic->highest_isr_cache != -1)) 500 return apic->highest_isr_cache; 501 502 result = find_highest_vector(apic->regs + APIC_ISR); 503 ASSERT(result == -1 || result >= 16); 504 505 return result; 506 } 507 508 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 509 { 510 struct kvm_vcpu *vcpu; 511 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 512 return; 513 514 vcpu = apic->vcpu; 515 516 /* 517 * We do get here for APIC virtualization enabled if the guest 518 * uses the Hyper-V APIC enlightenment. In this case we may need 519 * to trigger a new interrupt delivery by writing the SVI field; 520 * on the other hand isr_count and highest_isr_cache are unused 521 * and must be left alone. 522 */ 523 if (unlikely(vcpu->arch.apicv_active)) 524 kvm_x86_ops->hwapic_isr_update(vcpu, 525 apic_find_highest_isr(apic)); 526 else { 527 --apic->isr_count; 528 BUG_ON(apic->isr_count < 0); 529 apic->highest_isr_cache = -1; 530 } 531 } 532 533 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 534 { 535 /* This may race with setting of irr in __apic_accept_irq() and 536 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 537 * will cause vmexit immediately and the value will be recalculated 538 * on the next vmentry. 539 */ 540 return apic_find_highest_irr(vcpu->arch.apic); 541 } 542 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); 543 544 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 545 int vector, int level, int trig_mode, 546 struct dest_map *dest_map); 547 548 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 549 struct dest_map *dest_map) 550 { 551 struct kvm_lapic *apic = vcpu->arch.apic; 552 553 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 554 irq->level, irq->trig_mode, dest_map); 555 } 556 557 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, 558 unsigned long ipi_bitmap_high, u32 min, 559 unsigned long icr, int op_64_bit) 560 { 561 int i; 562 struct kvm_apic_map *map; 563 struct kvm_vcpu *vcpu; 564 struct kvm_lapic_irq irq = {0}; 565 int cluster_size = op_64_bit ? 64 : 32; 566 int count = 0; 567 568 irq.vector = icr & APIC_VECTOR_MASK; 569 irq.delivery_mode = icr & APIC_MODE_MASK; 570 irq.level = (icr & APIC_INT_ASSERT) != 0; 571 irq.trig_mode = icr & APIC_INT_LEVELTRIG; 572 573 if (icr & APIC_DEST_MASK) 574 return -KVM_EINVAL; 575 if (icr & APIC_SHORT_MASK) 576 return -KVM_EINVAL; 577 578 rcu_read_lock(); 579 map = rcu_dereference(kvm->arch.apic_map); 580 581 if (unlikely(!map)) { 582 count = -EOPNOTSUPP; 583 goto out; 584 } 585 586 if (min > map->max_apic_id) 587 goto out; 588 /* Bits above cluster_size are masked in the caller. */ 589 for_each_set_bit(i, &ipi_bitmap_low, 590 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { 591 if (map->phys_map[min + i]) { 592 vcpu = map->phys_map[min + i]->vcpu; 593 count += kvm_apic_set_irq(vcpu, &irq, NULL); 594 } 595 } 596 597 min += cluster_size; 598 599 if (min > map->max_apic_id) 600 goto out; 601 602 for_each_set_bit(i, &ipi_bitmap_high, 603 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { 604 if (map->phys_map[min + i]) { 605 vcpu = map->phys_map[min + i]->vcpu; 606 count += kvm_apic_set_irq(vcpu, &irq, NULL); 607 } 608 } 609 610 out: 611 rcu_read_unlock(); 612 return count; 613 } 614 615 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 616 { 617 618 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 619 sizeof(val)); 620 } 621 622 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 623 { 624 625 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 626 sizeof(*val)); 627 } 628 629 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 630 { 631 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 632 } 633 634 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu) 635 { 636 u8 val; 637 if (pv_eoi_get_user(vcpu, &val) < 0) 638 printk(KERN_WARNING "Can't read EOI MSR value: 0x%llx\n", 639 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 640 return val & 0x1; 641 } 642 643 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 644 { 645 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) { 646 printk(KERN_WARNING "Can't set EOI MSR value: 0x%llx\n", 647 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 648 return; 649 } 650 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 651 } 652 653 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu) 654 { 655 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) { 656 printk(KERN_WARNING "Can't clear EOI MSR value: 0x%llx\n", 657 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 658 return; 659 } 660 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 661 } 662 663 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) 664 { 665 int highest_irr; 666 if (apic->vcpu->arch.apicv_active) 667 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu); 668 else 669 highest_irr = apic_find_highest_irr(apic); 670 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) 671 return -1; 672 return highest_irr; 673 } 674 675 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr) 676 { 677 u32 tpr, isrv, ppr, old_ppr; 678 int isr; 679 680 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI); 681 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI); 682 isr = apic_find_highest_isr(apic); 683 isrv = (isr != -1) ? isr : 0; 684 685 if ((tpr & 0xf0) >= (isrv & 0xf0)) 686 ppr = tpr & 0xff; 687 else 688 ppr = isrv & 0xf0; 689 690 *new_ppr = ppr; 691 if (old_ppr != ppr) 692 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr); 693 694 return ppr < old_ppr; 695 } 696 697 static void apic_update_ppr(struct kvm_lapic *apic) 698 { 699 u32 ppr; 700 701 if (__apic_update_ppr(apic, &ppr) && 702 apic_has_interrupt_for_ppr(apic, ppr) != -1) 703 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 704 } 705 706 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu) 707 { 708 apic_update_ppr(vcpu->arch.apic); 709 } 710 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr); 711 712 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 713 { 714 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr); 715 apic_update_ppr(apic); 716 } 717 718 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda) 719 { 720 return mda == (apic_x2apic_mode(apic) ? 721 X2APIC_BROADCAST : APIC_BROADCAST); 722 } 723 724 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) 725 { 726 if (kvm_apic_broadcast(apic, mda)) 727 return true; 728 729 if (apic_x2apic_mode(apic)) 730 return mda == kvm_x2apic_id(apic); 731 732 /* 733 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if 734 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and 735 * this allows unique addressing of VCPUs with APIC ID over 0xff. 736 * The 0xff condition is needed because writeable xAPIC ID. 737 */ 738 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic)) 739 return true; 740 741 return mda == kvm_xapic_id(apic); 742 } 743 744 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 745 { 746 u32 logical_id; 747 748 if (kvm_apic_broadcast(apic, mda)) 749 return true; 750 751 logical_id = kvm_lapic_get_reg(apic, APIC_LDR); 752 753 if (apic_x2apic_mode(apic)) 754 return ((logical_id >> 16) == (mda >> 16)) 755 && (logical_id & mda & 0xffff) != 0; 756 757 logical_id = GET_APIC_LOGICAL_ID(logical_id); 758 759 switch (kvm_lapic_get_reg(apic, APIC_DFR)) { 760 case APIC_DFR_FLAT: 761 return (logical_id & mda) != 0; 762 case APIC_DFR_CLUSTER: 763 return ((logical_id >> 4) == (mda >> 4)) 764 && (logical_id & mda & 0xf) != 0; 765 default: 766 return false; 767 } 768 } 769 770 /* The KVM local APIC implementation has two quirks: 771 * 772 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs 773 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID. 774 * KVM doesn't do that aliasing. 775 * 776 * - in-kernel IOAPIC messages have to be delivered directly to 777 * x2APIC, because the kernel does not support interrupt remapping. 778 * In order to support broadcast without interrupt remapping, x2APIC 779 * rewrites the destination of non-IPI messages from APIC_BROADCAST 780 * to X2APIC_BROADCAST. 781 * 782 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is 783 * important when userspace wants to use x2APIC-format MSIs, because 784 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7". 785 */ 786 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id, 787 struct kvm_lapic *source, struct kvm_lapic *target) 788 { 789 bool ipi = source != NULL; 790 791 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && 792 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target)) 793 return X2APIC_BROADCAST; 794 795 return dest_id; 796 } 797 798 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 799 int short_hand, unsigned int dest, int dest_mode) 800 { 801 struct kvm_lapic *target = vcpu->arch.apic; 802 u32 mda = kvm_apic_mda(vcpu, dest, source, target); 803 804 ASSERT(target); 805 switch (short_hand) { 806 case APIC_DEST_NOSHORT: 807 if (dest_mode == APIC_DEST_PHYSICAL) 808 return kvm_apic_match_physical_addr(target, mda); 809 else 810 return kvm_apic_match_logical_addr(target, mda); 811 case APIC_DEST_SELF: 812 return target == source; 813 case APIC_DEST_ALLINC: 814 return true; 815 case APIC_DEST_ALLBUT: 816 return target != source; 817 default: 818 return false; 819 } 820 } 821 EXPORT_SYMBOL_GPL(kvm_apic_match_dest); 822 823 int kvm_vector_to_index(u32 vector, u32 dest_vcpus, 824 const unsigned long *bitmap, u32 bitmap_size) 825 { 826 u32 mod; 827 int i, idx = -1; 828 829 mod = vector % dest_vcpus; 830 831 for (i = 0; i <= mod; i++) { 832 idx = find_next_bit(bitmap, bitmap_size, idx + 1); 833 BUG_ON(idx == bitmap_size); 834 } 835 836 return idx; 837 } 838 839 static void kvm_apic_disabled_lapic_found(struct kvm *kvm) 840 { 841 if (!kvm->arch.disabled_lapic_found) { 842 kvm->arch.disabled_lapic_found = true; 843 printk(KERN_INFO 844 "Disabled LAPIC found during irq injection\n"); 845 } 846 } 847 848 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src, 849 struct kvm_lapic_irq *irq, struct kvm_apic_map *map) 850 { 851 if (kvm->arch.x2apic_broadcast_quirk_disabled) { 852 if ((irq->dest_id == APIC_BROADCAST && 853 map->mode != KVM_APIC_MODE_X2APIC)) 854 return true; 855 if (irq->dest_id == X2APIC_BROADCAST) 856 return true; 857 } else { 858 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src); 859 if (irq->dest_id == (x2apic_ipi ? 860 X2APIC_BROADCAST : APIC_BROADCAST)) 861 return true; 862 } 863 864 return false; 865 } 866 867 /* Return true if the interrupt can be handled by using *bitmap as index mask 868 * for valid destinations in *dst array. 869 * Return false if kvm_apic_map_get_dest_lapic did nothing useful. 870 * Note: we may have zero kvm_lapic destinations when we return true, which 871 * means that the interrupt should be dropped. In this case, *bitmap would be 872 * zero and *dst undefined. 873 */ 874 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm, 875 struct kvm_lapic **src, struct kvm_lapic_irq *irq, 876 struct kvm_apic_map *map, struct kvm_lapic ***dst, 877 unsigned long *bitmap) 878 { 879 int i, lowest; 880 881 if (irq->shorthand == APIC_DEST_SELF && src) { 882 *dst = src; 883 *bitmap = 1; 884 return true; 885 } else if (irq->shorthand) 886 return false; 887 888 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map)) 889 return false; 890 891 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 892 if (irq->dest_id > map->max_apic_id) { 893 *bitmap = 0; 894 } else { 895 u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1); 896 *dst = &map->phys_map[dest_id]; 897 *bitmap = 1; 898 } 899 return true; 900 } 901 902 *bitmap = 0; 903 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, 904 (u16 *)bitmap)) 905 return false; 906 907 if (!kvm_lowest_prio_delivery(irq)) 908 return true; 909 910 if (!kvm_vector_hashing_enabled()) { 911 lowest = -1; 912 for_each_set_bit(i, bitmap, 16) { 913 if (!(*dst)[i]) 914 continue; 915 if (lowest < 0) 916 lowest = i; 917 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, 918 (*dst)[lowest]->vcpu) < 0) 919 lowest = i; 920 } 921 } else { 922 if (!*bitmap) 923 return true; 924 925 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), 926 bitmap, 16); 927 928 if (!(*dst)[lowest]) { 929 kvm_apic_disabled_lapic_found(kvm); 930 *bitmap = 0; 931 return true; 932 } 933 } 934 935 *bitmap = (lowest >= 0) ? 1 << lowest : 0; 936 937 return true; 938 } 939 940 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 941 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map) 942 { 943 struct kvm_apic_map *map; 944 unsigned long bitmap; 945 struct kvm_lapic **dst = NULL; 946 int i; 947 bool ret; 948 949 *r = -1; 950 951 if (irq->shorthand == APIC_DEST_SELF) { 952 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 953 return true; 954 } 955 956 rcu_read_lock(); 957 map = rcu_dereference(kvm->arch.apic_map); 958 959 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap); 960 if (ret) { 961 *r = 0; 962 for_each_set_bit(i, &bitmap, 16) { 963 if (!dst[i]) 964 continue; 965 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 966 } 967 } 968 969 rcu_read_unlock(); 970 return ret; 971 } 972 973 /* 974 * This routine tries to handler interrupts in posted mode, here is how 975 * it deals with different cases: 976 * - For single-destination interrupts, handle it in posted mode 977 * - Else if vector hashing is enabled and it is a lowest-priority 978 * interrupt, handle it in posted mode and use the following mechanism 979 * to find the destinaiton vCPU. 980 * 1. For lowest-priority interrupts, store all the possible 981 * destination vCPUs in an array. 982 * 2. Use "guest vector % max number of destination vCPUs" to find 983 * the right destination vCPU in the array for the lowest-priority 984 * interrupt. 985 * - Otherwise, use remapped mode to inject the interrupt. 986 */ 987 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq, 988 struct kvm_vcpu **dest_vcpu) 989 { 990 struct kvm_apic_map *map; 991 unsigned long bitmap; 992 struct kvm_lapic **dst = NULL; 993 bool ret = false; 994 995 if (irq->shorthand) 996 return false; 997 998 rcu_read_lock(); 999 map = rcu_dereference(kvm->arch.apic_map); 1000 1001 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) && 1002 hweight16(bitmap) == 1) { 1003 unsigned long i = find_first_bit(&bitmap, 16); 1004 1005 if (dst[i]) { 1006 *dest_vcpu = dst[i]->vcpu; 1007 ret = true; 1008 } 1009 } 1010 1011 rcu_read_unlock(); 1012 return ret; 1013 } 1014 1015 /* 1016 * Add a pending IRQ into lapic. 1017 * Return 1 if successfully added and 0 if discarded. 1018 */ 1019 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 1020 int vector, int level, int trig_mode, 1021 struct dest_map *dest_map) 1022 { 1023 int result = 0; 1024 struct kvm_vcpu *vcpu = apic->vcpu; 1025 1026 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 1027 trig_mode, vector); 1028 switch (delivery_mode) { 1029 case APIC_DM_LOWEST: 1030 vcpu->arch.apic_arb_prio++; 1031 /* fall through */ 1032 case APIC_DM_FIXED: 1033 if (unlikely(trig_mode && !level)) 1034 break; 1035 1036 /* FIXME add logic for vcpu on reset */ 1037 if (unlikely(!apic_enabled(apic))) 1038 break; 1039 1040 result = 1; 1041 1042 if (dest_map) { 1043 __set_bit(vcpu->vcpu_id, dest_map->map); 1044 dest_map->vectors[vcpu->vcpu_id] = vector; 1045 } 1046 1047 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { 1048 if (trig_mode) 1049 kvm_lapic_set_vector(vector, 1050 apic->regs + APIC_TMR); 1051 else 1052 kvm_lapic_clear_vector(vector, 1053 apic->regs + APIC_TMR); 1054 } 1055 1056 if (vcpu->arch.apicv_active) 1057 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector); 1058 else { 1059 kvm_lapic_set_irr(vector, apic); 1060 1061 kvm_make_request(KVM_REQ_EVENT, vcpu); 1062 kvm_vcpu_kick(vcpu); 1063 } 1064 break; 1065 1066 case APIC_DM_REMRD: 1067 result = 1; 1068 vcpu->arch.pv.pv_unhalted = 1; 1069 kvm_make_request(KVM_REQ_EVENT, vcpu); 1070 kvm_vcpu_kick(vcpu); 1071 break; 1072 1073 case APIC_DM_SMI: 1074 result = 1; 1075 kvm_make_request(KVM_REQ_SMI, vcpu); 1076 kvm_vcpu_kick(vcpu); 1077 break; 1078 1079 case APIC_DM_NMI: 1080 result = 1; 1081 kvm_inject_nmi(vcpu); 1082 kvm_vcpu_kick(vcpu); 1083 break; 1084 1085 case APIC_DM_INIT: 1086 if (!trig_mode || level) { 1087 result = 1; 1088 /* assumes that there are only KVM_APIC_INIT/SIPI */ 1089 apic->pending_events = (1UL << KVM_APIC_INIT); 1090 /* make sure pending_events is visible before sending 1091 * the request */ 1092 smp_wmb(); 1093 kvm_make_request(KVM_REQ_EVENT, vcpu); 1094 kvm_vcpu_kick(vcpu); 1095 } 1096 break; 1097 1098 case APIC_DM_STARTUP: 1099 result = 1; 1100 apic->sipi_vector = vector; 1101 /* make sure sipi_vector is visible for the receiver */ 1102 smp_wmb(); 1103 set_bit(KVM_APIC_SIPI, &apic->pending_events); 1104 kvm_make_request(KVM_REQ_EVENT, vcpu); 1105 kvm_vcpu_kick(vcpu); 1106 break; 1107 1108 case APIC_DM_EXTINT: 1109 /* 1110 * Should only be called by kvm_apic_local_deliver() with LVT0, 1111 * before NMI watchdog was enabled. Already handled by 1112 * kvm_apic_accept_pic_intr(). 1113 */ 1114 break; 1115 1116 default: 1117 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 1118 delivery_mode); 1119 break; 1120 } 1121 return result; 1122 } 1123 1124 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 1125 { 1126 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 1127 } 1128 1129 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) 1130 { 1131 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); 1132 } 1133 1134 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 1135 { 1136 int trigger_mode; 1137 1138 /* Eoi the ioapic only if the ioapic doesn't own the vector. */ 1139 if (!kvm_ioapic_handles_vector(apic, vector)) 1140 return; 1141 1142 /* Request a KVM exit to inform the userspace IOAPIC. */ 1143 if (irqchip_split(apic->vcpu->kvm)) { 1144 apic->vcpu->arch.pending_ioapic_eoi = vector; 1145 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); 1146 return; 1147 } 1148 1149 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 1150 trigger_mode = IOAPIC_LEVEL_TRIG; 1151 else 1152 trigger_mode = IOAPIC_EDGE_TRIG; 1153 1154 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 1155 } 1156 1157 static int apic_set_eoi(struct kvm_lapic *apic) 1158 { 1159 int vector = apic_find_highest_isr(apic); 1160 1161 trace_kvm_eoi(apic, vector); 1162 1163 /* 1164 * Not every write EOI will has corresponding ISR, 1165 * one example is when Kernel check timer on setup_IO_APIC 1166 */ 1167 if (vector == -1) 1168 return vector; 1169 1170 apic_clear_isr(vector, apic); 1171 apic_update_ppr(apic); 1172 1173 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap)) 1174 kvm_hv_synic_send_eoi(apic->vcpu, vector); 1175 1176 kvm_ioapic_send_eoi(apic, vector); 1177 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1178 return vector; 1179 } 1180 1181 /* 1182 * this interface assumes a trap-like exit, which has already finished 1183 * desired side effect including vISR and vPPR update. 1184 */ 1185 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 1186 { 1187 struct kvm_lapic *apic = vcpu->arch.apic; 1188 1189 trace_kvm_eoi(apic, vector); 1190 1191 kvm_ioapic_send_eoi(apic, vector); 1192 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1193 } 1194 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 1195 1196 static void apic_send_ipi(struct kvm_lapic *apic) 1197 { 1198 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR); 1199 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2); 1200 struct kvm_lapic_irq irq; 1201 1202 irq.vector = icr_low & APIC_VECTOR_MASK; 1203 irq.delivery_mode = icr_low & APIC_MODE_MASK; 1204 irq.dest_mode = icr_low & APIC_DEST_MASK; 1205 irq.level = (icr_low & APIC_INT_ASSERT) != 0; 1206 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 1207 irq.shorthand = icr_low & APIC_SHORT_MASK; 1208 irq.msi_redir_hint = false; 1209 if (apic_x2apic_mode(apic)) 1210 irq.dest_id = icr_high; 1211 else 1212 irq.dest_id = GET_APIC_DEST_FIELD(icr_high); 1213 1214 trace_kvm_apic_ipi(icr_low, irq.dest_id); 1215 1216 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 1217 } 1218 1219 static u32 apic_get_tmcct(struct kvm_lapic *apic) 1220 { 1221 ktime_t remaining, now; 1222 s64 ns; 1223 u32 tmcct; 1224 1225 ASSERT(apic != NULL); 1226 1227 /* if initial count is 0, current count should also be 0 */ 1228 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 || 1229 apic->lapic_timer.period == 0) 1230 return 0; 1231 1232 now = ktime_get(); 1233 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1234 if (ktime_to_ns(remaining) < 0) 1235 remaining = 0; 1236 1237 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 1238 tmcct = div64_u64(ns, 1239 (APIC_BUS_CYCLE_NS * apic->divide_count)); 1240 1241 return tmcct; 1242 } 1243 1244 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 1245 { 1246 struct kvm_vcpu *vcpu = apic->vcpu; 1247 struct kvm_run *run = vcpu->run; 1248 1249 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 1250 run->tpr_access.rip = kvm_rip_read(vcpu); 1251 run->tpr_access.is_write = write; 1252 } 1253 1254 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 1255 { 1256 if (apic->vcpu->arch.tpr_access_reporting) 1257 __report_tpr_access(apic, write); 1258 } 1259 1260 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 1261 { 1262 u32 val = 0; 1263 1264 if (offset >= LAPIC_MMIO_LENGTH) 1265 return 0; 1266 1267 switch (offset) { 1268 case APIC_ARBPRI: 1269 break; 1270 1271 case APIC_TMCCT: /* Timer CCR */ 1272 if (apic_lvtt_tscdeadline(apic)) 1273 return 0; 1274 1275 val = apic_get_tmcct(apic); 1276 break; 1277 case APIC_PROCPRI: 1278 apic_update_ppr(apic); 1279 val = kvm_lapic_get_reg(apic, offset); 1280 break; 1281 case APIC_TASKPRI: 1282 report_tpr_access(apic, false); 1283 /* fall thru */ 1284 default: 1285 val = kvm_lapic_get_reg(apic, offset); 1286 break; 1287 } 1288 1289 return val; 1290 } 1291 1292 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 1293 { 1294 return container_of(dev, struct kvm_lapic, dev); 1295 } 1296 1297 #define APIC_REG_MASK(reg) (1ull << ((reg) >> 4)) 1298 #define APIC_REGS_MASK(first, count) \ 1299 (APIC_REG_MASK(first) * ((1ull << (count)) - 1)) 1300 1301 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 1302 void *data) 1303 { 1304 unsigned char alignment = offset & 0xf; 1305 u32 result; 1306 /* this bitmask has a bit cleared for each reserved register */ 1307 u64 valid_reg_mask = 1308 APIC_REG_MASK(APIC_ID) | 1309 APIC_REG_MASK(APIC_LVR) | 1310 APIC_REG_MASK(APIC_TASKPRI) | 1311 APIC_REG_MASK(APIC_PROCPRI) | 1312 APIC_REG_MASK(APIC_LDR) | 1313 APIC_REG_MASK(APIC_DFR) | 1314 APIC_REG_MASK(APIC_SPIV) | 1315 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) | 1316 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) | 1317 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) | 1318 APIC_REG_MASK(APIC_ESR) | 1319 APIC_REG_MASK(APIC_ICR) | 1320 APIC_REG_MASK(APIC_ICR2) | 1321 APIC_REG_MASK(APIC_LVTT) | 1322 APIC_REG_MASK(APIC_LVTTHMR) | 1323 APIC_REG_MASK(APIC_LVTPC) | 1324 APIC_REG_MASK(APIC_LVT0) | 1325 APIC_REG_MASK(APIC_LVT1) | 1326 APIC_REG_MASK(APIC_LVTERR) | 1327 APIC_REG_MASK(APIC_TMICT) | 1328 APIC_REG_MASK(APIC_TMCCT) | 1329 APIC_REG_MASK(APIC_TDCR); 1330 1331 /* ARBPRI is not valid on x2APIC */ 1332 if (!apic_x2apic_mode(apic)) 1333 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI); 1334 1335 if (offset > 0x3f0 || !(valid_reg_mask & APIC_REG_MASK(offset))) 1336 return 1; 1337 1338 result = __apic_read(apic, offset & ~0xf); 1339 1340 trace_kvm_apic_read(offset, result); 1341 1342 switch (len) { 1343 case 1: 1344 case 2: 1345 case 4: 1346 memcpy(data, (char *)&result + alignment, len); 1347 break; 1348 default: 1349 printk(KERN_ERR "Local APIC read with len = %x, " 1350 "should be 1,2, or 4 instead\n", len); 1351 break; 1352 } 1353 return 0; 1354 } 1355 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read); 1356 1357 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1358 { 1359 return addr >= apic->base_address && 1360 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1361 } 1362 1363 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1364 gpa_t address, int len, void *data) 1365 { 1366 struct kvm_lapic *apic = to_lapic(this); 1367 u32 offset = address - apic->base_address; 1368 1369 if (!apic_mmio_in_range(apic, address)) 1370 return -EOPNOTSUPP; 1371 1372 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 1373 if (!kvm_check_has_quirk(vcpu->kvm, 1374 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 1375 return -EOPNOTSUPP; 1376 1377 memset(data, 0xff, len); 1378 return 0; 1379 } 1380 1381 kvm_lapic_reg_read(apic, offset, len, data); 1382 1383 return 0; 1384 } 1385 1386 static void update_divide_count(struct kvm_lapic *apic) 1387 { 1388 u32 tmp1, tmp2, tdcr; 1389 1390 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR); 1391 tmp1 = tdcr & 0xf; 1392 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1393 apic->divide_count = 0x1 << (tmp2 & 0x7); 1394 } 1395 1396 static void limit_periodic_timer_frequency(struct kvm_lapic *apic) 1397 { 1398 /* 1399 * Do not allow the guest to program periodic timers with small 1400 * interval, since the hrtimers are not throttled by the host 1401 * scheduler. 1402 */ 1403 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1404 s64 min_period = min_timer_period_us * 1000LL; 1405 1406 if (apic->lapic_timer.period < min_period) { 1407 pr_info_ratelimited( 1408 "kvm: vcpu %i: requested %lld ns " 1409 "lapic timer period limited to %lld ns\n", 1410 apic->vcpu->vcpu_id, 1411 apic->lapic_timer.period, min_period); 1412 apic->lapic_timer.period = min_period; 1413 } 1414 } 1415 } 1416 1417 static void apic_update_lvtt(struct kvm_lapic *apic) 1418 { 1419 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & 1420 apic->lapic_timer.timer_mode_mask; 1421 1422 if (apic->lapic_timer.timer_mode != timer_mode) { 1423 if (apic_lvtt_tscdeadline(apic) != (timer_mode == 1424 APIC_LVT_TIMER_TSCDEADLINE)) { 1425 hrtimer_cancel(&apic->lapic_timer.timer); 1426 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 1427 apic->lapic_timer.period = 0; 1428 apic->lapic_timer.tscdeadline = 0; 1429 } 1430 apic->lapic_timer.timer_mode = timer_mode; 1431 limit_periodic_timer_frequency(apic); 1432 } 1433 } 1434 1435 /* 1436 * On APICv, this test will cause a busy wait 1437 * during a higher-priority task. 1438 */ 1439 1440 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu) 1441 { 1442 struct kvm_lapic *apic = vcpu->arch.apic; 1443 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT); 1444 1445 if (kvm_apic_hw_enabled(apic)) { 1446 int vec = reg & APIC_VECTOR_MASK; 1447 void *bitmap = apic->regs + APIC_ISR; 1448 1449 if (vcpu->arch.apicv_active) 1450 bitmap = apic->regs + APIC_IRR; 1451 1452 if (apic_test_vector(vec, bitmap)) 1453 return true; 1454 } 1455 return false; 1456 } 1457 1458 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles) 1459 { 1460 u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns; 1461 1462 /* 1463 * If the guest TSC is running at a different ratio than the host, then 1464 * convert the delay to nanoseconds to achieve an accurate delay. Note 1465 * that __delay() uses delay_tsc whenever the hardware has TSC, thus 1466 * always for VMX enabled hardware. 1467 */ 1468 if (vcpu->arch.tsc_scaling_ratio == kvm_default_tsc_scaling_ratio) { 1469 __delay(min(guest_cycles, 1470 nsec_to_cycles(vcpu, timer_advance_ns))); 1471 } else { 1472 u64 delay_ns = guest_cycles * 1000000ULL; 1473 do_div(delay_ns, vcpu->arch.virtual_tsc_khz); 1474 ndelay(min_t(u32, delay_ns, timer_advance_ns)); 1475 } 1476 } 1477 1478 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu, 1479 s64 advance_expire_delta) 1480 { 1481 struct kvm_lapic *apic = vcpu->arch.apic; 1482 u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns; 1483 u64 ns; 1484 1485 /* too early */ 1486 if (advance_expire_delta < 0) { 1487 ns = -advance_expire_delta * 1000000ULL; 1488 do_div(ns, vcpu->arch.virtual_tsc_khz); 1489 timer_advance_ns -= min((u32)ns, 1490 timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP); 1491 } else { 1492 /* too late */ 1493 ns = advance_expire_delta * 1000000ULL; 1494 do_div(ns, vcpu->arch.virtual_tsc_khz); 1495 timer_advance_ns += min((u32)ns, 1496 timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP); 1497 } 1498 1499 if (abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_DONE) 1500 apic->lapic_timer.timer_advance_adjust_done = true; 1501 if (unlikely(timer_advance_ns > 5000)) { 1502 timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT; 1503 apic->lapic_timer.timer_advance_adjust_done = false; 1504 } 1505 apic->lapic_timer.timer_advance_ns = timer_advance_ns; 1506 } 1507 1508 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu) 1509 { 1510 struct kvm_lapic *apic = vcpu->arch.apic; 1511 u64 guest_tsc, tsc_deadline; 1512 1513 if (apic->lapic_timer.expired_tscdeadline == 0) 1514 return; 1515 1516 tsc_deadline = apic->lapic_timer.expired_tscdeadline; 1517 apic->lapic_timer.expired_tscdeadline = 0; 1518 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1519 apic->lapic_timer.advance_expire_delta = guest_tsc - tsc_deadline; 1520 1521 if (guest_tsc < tsc_deadline) 1522 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc); 1523 1524 if (unlikely(!apic->lapic_timer.timer_advance_adjust_done)) 1525 adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta); 1526 } 1527 1528 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu) 1529 { 1530 if (lapic_timer_int_injected(vcpu)) 1531 __kvm_wait_lapic_expire(vcpu); 1532 } 1533 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire); 1534 1535 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic) 1536 { 1537 struct kvm_timer *ktimer = &apic->lapic_timer; 1538 1539 kvm_apic_local_deliver(apic, APIC_LVTT); 1540 if (apic_lvtt_tscdeadline(apic)) 1541 ktimer->tscdeadline = 0; 1542 if (apic_lvtt_oneshot(apic)) { 1543 ktimer->tscdeadline = 0; 1544 ktimer->target_expiration = 0; 1545 } 1546 } 1547 1548 static void apic_timer_expired(struct kvm_lapic *apic) 1549 { 1550 struct kvm_vcpu *vcpu = apic->vcpu; 1551 struct kvm_timer *ktimer = &apic->lapic_timer; 1552 1553 if (atomic_read(&apic->lapic_timer.pending)) 1554 return; 1555 1556 if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) 1557 ktimer->expired_tscdeadline = ktimer->tscdeadline; 1558 1559 if (kvm_use_posted_timer_interrupt(apic->vcpu)) { 1560 if (apic->lapic_timer.timer_advance_ns) 1561 __kvm_wait_lapic_expire(vcpu); 1562 kvm_apic_inject_pending_timer_irqs(apic); 1563 return; 1564 } 1565 1566 atomic_inc(&apic->lapic_timer.pending); 1567 kvm_set_pending_timer(vcpu); 1568 } 1569 1570 static void start_sw_tscdeadline(struct kvm_lapic *apic) 1571 { 1572 struct kvm_timer *ktimer = &apic->lapic_timer; 1573 u64 guest_tsc, tscdeadline = ktimer->tscdeadline; 1574 u64 ns = 0; 1575 ktime_t expire; 1576 struct kvm_vcpu *vcpu = apic->vcpu; 1577 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1578 unsigned long flags; 1579 ktime_t now; 1580 1581 if (unlikely(!tscdeadline || !this_tsc_khz)) 1582 return; 1583 1584 local_irq_save(flags); 1585 1586 now = ktime_get(); 1587 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1588 1589 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1590 do_div(ns, this_tsc_khz); 1591 1592 if (likely(tscdeadline > guest_tsc) && 1593 likely(ns > apic->lapic_timer.timer_advance_ns)) { 1594 expire = ktime_add_ns(now, ns); 1595 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns); 1596 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS); 1597 } else 1598 apic_timer_expired(apic); 1599 1600 local_irq_restore(flags); 1601 } 1602 1603 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) 1604 { 1605 ktime_t now, remaining; 1606 u64 ns_remaining_old, ns_remaining_new; 1607 1608 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1609 * APIC_BUS_CYCLE_NS * apic->divide_count; 1610 limit_periodic_timer_frequency(apic); 1611 1612 now = ktime_get(); 1613 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1614 if (ktime_to_ns(remaining) < 0) 1615 remaining = 0; 1616 1617 ns_remaining_old = ktime_to_ns(remaining); 1618 ns_remaining_new = mul_u64_u32_div(ns_remaining_old, 1619 apic->divide_count, old_divisor); 1620 1621 apic->lapic_timer.tscdeadline += 1622 nsec_to_cycles(apic->vcpu, ns_remaining_new) - 1623 nsec_to_cycles(apic->vcpu, ns_remaining_old); 1624 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); 1625 } 1626 1627 static bool set_target_expiration(struct kvm_lapic *apic) 1628 { 1629 ktime_t now; 1630 u64 tscl = rdtsc(); 1631 1632 now = ktime_get(); 1633 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) 1634 * APIC_BUS_CYCLE_NS * apic->divide_count; 1635 1636 if (!apic->lapic_timer.period) { 1637 apic->lapic_timer.tscdeadline = 0; 1638 return false; 1639 } 1640 1641 limit_periodic_timer_frequency(apic); 1642 1643 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 1644 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); 1645 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period); 1646 1647 return true; 1648 } 1649 1650 static void advance_periodic_target_expiration(struct kvm_lapic *apic) 1651 { 1652 ktime_t now = ktime_get(); 1653 u64 tscl = rdtsc(); 1654 ktime_t delta; 1655 1656 /* 1657 * Synchronize both deadlines to the same time source or 1658 * differences in the periods (caused by differences in the 1659 * underlying clocks or numerical approximation errors) will 1660 * cause the two to drift apart over time as the errors 1661 * accumulate. 1662 */ 1663 apic->lapic_timer.target_expiration = 1664 ktime_add_ns(apic->lapic_timer.target_expiration, 1665 apic->lapic_timer.period); 1666 delta = ktime_sub(apic->lapic_timer.target_expiration, now); 1667 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 1668 nsec_to_cycles(apic->vcpu, delta); 1669 } 1670 1671 static void start_sw_period(struct kvm_lapic *apic) 1672 { 1673 if (!apic->lapic_timer.period) 1674 return; 1675 1676 if (ktime_after(ktime_get(), 1677 apic->lapic_timer.target_expiration)) { 1678 apic_timer_expired(apic); 1679 1680 if (apic_lvtt_oneshot(apic)) 1681 return; 1682 1683 advance_periodic_target_expiration(apic); 1684 } 1685 1686 hrtimer_start(&apic->lapic_timer.timer, 1687 apic->lapic_timer.target_expiration, 1688 HRTIMER_MODE_ABS); 1689 } 1690 1691 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) 1692 { 1693 if (!lapic_in_kernel(vcpu)) 1694 return false; 1695 1696 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; 1697 } 1698 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use); 1699 1700 static void cancel_hv_timer(struct kvm_lapic *apic) 1701 { 1702 WARN_ON(preemptible()); 1703 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1704 kvm_x86_ops->cancel_hv_timer(apic->vcpu); 1705 apic->lapic_timer.hv_timer_in_use = false; 1706 } 1707 1708 static bool start_hv_timer(struct kvm_lapic *apic) 1709 { 1710 struct kvm_timer *ktimer = &apic->lapic_timer; 1711 struct kvm_vcpu *vcpu = apic->vcpu; 1712 bool expired; 1713 1714 WARN_ON(preemptible()); 1715 if (!kvm_x86_ops->set_hv_timer) 1716 return false; 1717 1718 if (!ktimer->tscdeadline) 1719 return false; 1720 1721 if (kvm_x86_ops->set_hv_timer(vcpu, ktimer->tscdeadline, &expired)) 1722 return false; 1723 1724 ktimer->hv_timer_in_use = true; 1725 hrtimer_cancel(&ktimer->timer); 1726 1727 /* 1728 * To simplify handling the periodic timer, leave the hv timer running 1729 * even if the deadline timer has expired, i.e. rely on the resulting 1730 * VM-Exit to recompute the periodic timer's target expiration. 1731 */ 1732 if (!apic_lvtt_period(apic)) { 1733 /* 1734 * Cancel the hv timer if the sw timer fired while the hv timer 1735 * was being programmed, or if the hv timer itself expired. 1736 */ 1737 if (atomic_read(&ktimer->pending)) { 1738 cancel_hv_timer(apic); 1739 } else if (expired) { 1740 apic_timer_expired(apic); 1741 cancel_hv_timer(apic); 1742 } 1743 } 1744 1745 trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use); 1746 1747 return true; 1748 } 1749 1750 static void start_sw_timer(struct kvm_lapic *apic) 1751 { 1752 struct kvm_timer *ktimer = &apic->lapic_timer; 1753 1754 WARN_ON(preemptible()); 1755 if (apic->lapic_timer.hv_timer_in_use) 1756 cancel_hv_timer(apic); 1757 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 1758 return; 1759 1760 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1761 start_sw_period(apic); 1762 else if (apic_lvtt_tscdeadline(apic)) 1763 start_sw_tscdeadline(apic); 1764 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); 1765 } 1766 1767 static void restart_apic_timer(struct kvm_lapic *apic) 1768 { 1769 preempt_disable(); 1770 1771 if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending)) 1772 goto out; 1773 1774 if (!start_hv_timer(apic)) 1775 start_sw_timer(apic); 1776 out: 1777 preempt_enable(); 1778 } 1779 1780 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) 1781 { 1782 struct kvm_lapic *apic = vcpu->arch.apic; 1783 1784 preempt_disable(); 1785 /* If the preempt notifier has already run, it also called apic_timer_expired */ 1786 if (!apic->lapic_timer.hv_timer_in_use) 1787 goto out; 1788 WARN_ON(swait_active(&vcpu->wq)); 1789 cancel_hv_timer(apic); 1790 apic_timer_expired(apic); 1791 1792 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1793 advance_periodic_target_expiration(apic); 1794 restart_apic_timer(apic); 1795 } 1796 out: 1797 preempt_enable(); 1798 } 1799 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); 1800 1801 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) 1802 { 1803 restart_apic_timer(vcpu->arch.apic); 1804 } 1805 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer); 1806 1807 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) 1808 { 1809 struct kvm_lapic *apic = vcpu->arch.apic; 1810 1811 preempt_disable(); 1812 /* Possibly the TSC deadline timer is not enabled yet */ 1813 if (apic->lapic_timer.hv_timer_in_use) 1814 start_sw_timer(apic); 1815 preempt_enable(); 1816 } 1817 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer); 1818 1819 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) 1820 { 1821 struct kvm_lapic *apic = vcpu->arch.apic; 1822 1823 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 1824 restart_apic_timer(apic); 1825 } 1826 1827 static void start_apic_timer(struct kvm_lapic *apic) 1828 { 1829 atomic_set(&apic->lapic_timer.pending, 0); 1830 1831 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 1832 && !set_target_expiration(apic)) 1833 return; 1834 1835 restart_apic_timer(apic); 1836 } 1837 1838 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 1839 { 1840 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val); 1841 1842 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { 1843 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; 1844 if (lvt0_in_nmi_mode) { 1845 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1846 } else 1847 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 1848 } 1849 } 1850 1851 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 1852 { 1853 int ret = 0; 1854 1855 trace_kvm_apic_write(reg, val); 1856 1857 switch (reg) { 1858 case APIC_ID: /* Local APIC ID */ 1859 if (!apic_x2apic_mode(apic)) 1860 kvm_apic_set_xapic_id(apic, val >> 24); 1861 else 1862 ret = 1; 1863 break; 1864 1865 case APIC_TASKPRI: 1866 report_tpr_access(apic, true); 1867 apic_set_tpr(apic, val & 0xff); 1868 break; 1869 1870 case APIC_EOI: 1871 apic_set_eoi(apic); 1872 break; 1873 1874 case APIC_LDR: 1875 if (!apic_x2apic_mode(apic)) 1876 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 1877 else 1878 ret = 1; 1879 break; 1880 1881 case APIC_DFR: 1882 if (!apic_x2apic_mode(apic)) { 1883 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); 1884 recalculate_apic_map(apic->vcpu->kvm); 1885 } else 1886 ret = 1; 1887 break; 1888 1889 case APIC_SPIV: { 1890 u32 mask = 0x3ff; 1891 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 1892 mask |= APIC_SPIV_DIRECTED_EOI; 1893 apic_set_spiv(apic, val & mask); 1894 if (!(val & APIC_SPIV_APIC_ENABLED)) { 1895 int i; 1896 u32 lvt_val; 1897 1898 for (i = 0; i < KVM_APIC_LVT_NUM; i++) { 1899 lvt_val = kvm_lapic_get_reg(apic, 1900 APIC_LVTT + 0x10 * i); 1901 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, 1902 lvt_val | APIC_LVT_MASKED); 1903 } 1904 apic_update_lvtt(apic); 1905 atomic_set(&apic->lapic_timer.pending, 0); 1906 1907 } 1908 break; 1909 } 1910 case APIC_ICR: 1911 /* No delay here, so we always clear the pending bit */ 1912 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12)); 1913 apic_send_ipi(apic); 1914 break; 1915 1916 case APIC_ICR2: 1917 if (!apic_x2apic_mode(apic)) 1918 val &= 0xff000000; 1919 kvm_lapic_set_reg(apic, APIC_ICR2, val); 1920 break; 1921 1922 case APIC_LVT0: 1923 apic_manage_nmi_watchdog(apic, val); 1924 /* fall through */ 1925 case APIC_LVTTHMR: 1926 case APIC_LVTPC: 1927 case APIC_LVT1: 1928 case APIC_LVTERR: 1929 /* TODO: Check vector */ 1930 if (!kvm_apic_sw_enabled(apic)) 1931 val |= APIC_LVT_MASKED; 1932 1933 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; 1934 kvm_lapic_set_reg(apic, reg, val); 1935 1936 break; 1937 1938 case APIC_LVTT: 1939 if (!kvm_apic_sw_enabled(apic)) 1940 val |= APIC_LVT_MASKED; 1941 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 1942 kvm_lapic_set_reg(apic, APIC_LVTT, val); 1943 apic_update_lvtt(apic); 1944 break; 1945 1946 case APIC_TMICT: 1947 if (apic_lvtt_tscdeadline(apic)) 1948 break; 1949 1950 hrtimer_cancel(&apic->lapic_timer.timer); 1951 kvm_lapic_set_reg(apic, APIC_TMICT, val); 1952 start_apic_timer(apic); 1953 break; 1954 1955 case APIC_TDCR: { 1956 uint32_t old_divisor = apic->divide_count; 1957 1958 kvm_lapic_set_reg(apic, APIC_TDCR, val); 1959 update_divide_count(apic); 1960 if (apic->divide_count != old_divisor && 1961 apic->lapic_timer.period) { 1962 hrtimer_cancel(&apic->lapic_timer.timer); 1963 update_target_expiration(apic, old_divisor); 1964 restart_apic_timer(apic); 1965 } 1966 break; 1967 } 1968 case APIC_ESR: 1969 if (apic_x2apic_mode(apic) && val != 0) 1970 ret = 1; 1971 break; 1972 1973 case APIC_SELF_IPI: 1974 if (apic_x2apic_mode(apic)) { 1975 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff)); 1976 } else 1977 ret = 1; 1978 break; 1979 default: 1980 ret = 1; 1981 break; 1982 } 1983 1984 return ret; 1985 } 1986 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write); 1987 1988 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1989 gpa_t address, int len, const void *data) 1990 { 1991 struct kvm_lapic *apic = to_lapic(this); 1992 unsigned int offset = address - apic->base_address; 1993 u32 val; 1994 1995 if (!apic_mmio_in_range(apic, address)) 1996 return -EOPNOTSUPP; 1997 1998 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 1999 if (!kvm_check_has_quirk(vcpu->kvm, 2000 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 2001 return -EOPNOTSUPP; 2002 2003 return 0; 2004 } 2005 2006 /* 2007 * APIC register must be aligned on 128-bits boundary. 2008 * 32/64/128 bits registers must be accessed thru 32 bits. 2009 * Refer SDM 8.4.1 2010 */ 2011 if (len != 4 || (offset & 0xf)) 2012 return 0; 2013 2014 val = *(u32*)data; 2015 2016 kvm_lapic_reg_write(apic, offset & 0xff0, val); 2017 2018 return 0; 2019 } 2020 2021 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 2022 { 2023 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 2024 } 2025 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 2026 2027 /* emulate APIC access in a trap manner */ 2028 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 2029 { 2030 u32 val = 0; 2031 2032 /* hw has done the conditional check and inst decode */ 2033 offset &= 0xff0; 2034 2035 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val); 2036 2037 /* TODO: optimize to just emulate side effect w/o one more write */ 2038 kvm_lapic_reg_write(vcpu->arch.apic, offset, val); 2039 } 2040 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 2041 2042 void kvm_free_lapic(struct kvm_vcpu *vcpu) 2043 { 2044 struct kvm_lapic *apic = vcpu->arch.apic; 2045 2046 if (!vcpu->arch.apic) 2047 return; 2048 2049 hrtimer_cancel(&apic->lapic_timer.timer); 2050 2051 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 2052 static_key_slow_dec_deferred(&apic_hw_disabled); 2053 2054 if (!apic->sw_enabled) 2055 static_key_slow_dec_deferred(&apic_sw_disabled); 2056 2057 if (apic->regs) 2058 free_page((unsigned long)apic->regs); 2059 2060 kfree(apic); 2061 } 2062 2063 /* 2064 *---------------------------------------------------------------------- 2065 * LAPIC interface 2066 *---------------------------------------------------------------------- 2067 */ 2068 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 2069 { 2070 struct kvm_lapic *apic = vcpu->arch.apic; 2071 2072 if (!lapic_in_kernel(vcpu) || 2073 !apic_lvtt_tscdeadline(apic)) 2074 return 0; 2075 2076 return apic->lapic_timer.tscdeadline; 2077 } 2078 2079 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 2080 { 2081 struct kvm_lapic *apic = vcpu->arch.apic; 2082 2083 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) || 2084 apic_lvtt_period(apic)) 2085 return; 2086 2087 hrtimer_cancel(&apic->lapic_timer.timer); 2088 apic->lapic_timer.tscdeadline = data; 2089 start_apic_timer(apic); 2090 } 2091 2092 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 2093 { 2094 struct kvm_lapic *apic = vcpu->arch.apic; 2095 2096 apic_set_tpr(apic, ((cr8 & 0x0f) << 4) 2097 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4)); 2098 } 2099 2100 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 2101 { 2102 u64 tpr; 2103 2104 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 2105 2106 return (tpr & 0xf0) >> 4; 2107 } 2108 2109 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 2110 { 2111 u64 old_value = vcpu->arch.apic_base; 2112 struct kvm_lapic *apic = vcpu->arch.apic; 2113 2114 if (!apic) 2115 value |= MSR_IA32_APICBASE_BSP; 2116 2117 vcpu->arch.apic_base = value; 2118 2119 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) 2120 kvm_update_cpuid(vcpu); 2121 2122 if (!apic) 2123 return; 2124 2125 /* update jump label if enable bit changes */ 2126 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 2127 if (value & MSR_IA32_APICBASE_ENABLE) { 2128 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2129 static_key_slow_dec_deferred(&apic_hw_disabled); 2130 } else { 2131 static_key_slow_inc(&apic_hw_disabled.key); 2132 recalculate_apic_map(vcpu->kvm); 2133 } 2134 } 2135 2136 if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE)) 2137 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); 2138 2139 if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) 2140 kvm_x86_ops->set_virtual_apic_mode(vcpu); 2141 2142 apic->base_address = apic->vcpu->arch.apic_base & 2143 MSR_IA32_APICBASE_BASE; 2144 2145 if ((value & MSR_IA32_APICBASE_ENABLE) && 2146 apic->base_address != APIC_DEFAULT_PHYS_BASE) 2147 pr_warn_once("APIC base relocation is unsupported by KVM"); 2148 } 2149 2150 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) 2151 { 2152 struct kvm_lapic *apic = vcpu->arch.apic; 2153 int i; 2154 2155 if (!apic) 2156 return; 2157 2158 /* Stop the timer in case it's a reset to an active apic */ 2159 hrtimer_cancel(&apic->lapic_timer.timer); 2160 2161 if (!init_event) { 2162 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE | 2163 MSR_IA32_APICBASE_ENABLE); 2164 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2165 } 2166 kvm_apic_set_version(apic->vcpu); 2167 2168 for (i = 0; i < KVM_APIC_LVT_NUM; i++) 2169 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED); 2170 apic_update_lvtt(apic); 2171 if (kvm_vcpu_is_reset_bsp(vcpu) && 2172 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) 2173 kvm_lapic_set_reg(apic, APIC_LVT0, 2174 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 2175 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2176 2177 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU); 2178 apic_set_spiv(apic, 0xff); 2179 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0); 2180 if (!apic_x2apic_mode(apic)) 2181 kvm_apic_set_ldr(apic, 0); 2182 kvm_lapic_set_reg(apic, APIC_ESR, 0); 2183 kvm_lapic_set_reg(apic, APIC_ICR, 0); 2184 kvm_lapic_set_reg(apic, APIC_ICR2, 0); 2185 kvm_lapic_set_reg(apic, APIC_TDCR, 0); 2186 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 2187 for (i = 0; i < 8; i++) { 2188 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 2189 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 2190 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 2191 } 2192 apic->irr_pending = vcpu->arch.apicv_active; 2193 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0; 2194 apic->highest_isr_cache = -1; 2195 update_divide_count(apic); 2196 atomic_set(&apic->lapic_timer.pending, 0); 2197 if (kvm_vcpu_is_bsp(vcpu)) 2198 kvm_lapic_set_base(vcpu, 2199 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP); 2200 vcpu->arch.pv_eoi.msr_val = 0; 2201 apic_update_ppr(apic); 2202 if (vcpu->arch.apicv_active) { 2203 kvm_x86_ops->apicv_post_state_restore(vcpu); 2204 kvm_x86_ops->hwapic_irr_update(vcpu, -1); 2205 kvm_x86_ops->hwapic_isr_update(vcpu, -1); 2206 } 2207 2208 vcpu->arch.apic_arb_prio = 0; 2209 vcpu->arch.apic_attention = 0; 2210 } 2211 2212 /* 2213 *---------------------------------------------------------------------- 2214 * timer interface 2215 *---------------------------------------------------------------------- 2216 */ 2217 2218 static bool lapic_is_periodic(struct kvm_lapic *apic) 2219 { 2220 return apic_lvtt_period(apic); 2221 } 2222 2223 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 2224 { 2225 struct kvm_lapic *apic = vcpu->arch.apic; 2226 2227 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT)) 2228 return atomic_read(&apic->lapic_timer.pending); 2229 2230 return 0; 2231 } 2232 2233 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 2234 { 2235 u32 reg = kvm_lapic_get_reg(apic, lvt_type); 2236 int vector, mode, trig_mode; 2237 2238 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 2239 vector = reg & APIC_VECTOR_MASK; 2240 mode = reg & APIC_MODE_MASK; 2241 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 2242 return __apic_accept_irq(apic, mode, vector, 1, trig_mode, 2243 NULL); 2244 } 2245 return 0; 2246 } 2247 2248 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 2249 { 2250 struct kvm_lapic *apic = vcpu->arch.apic; 2251 2252 if (apic) 2253 kvm_apic_local_deliver(apic, APIC_LVT0); 2254 } 2255 2256 static const struct kvm_io_device_ops apic_mmio_ops = { 2257 .read = apic_mmio_read, 2258 .write = apic_mmio_write, 2259 }; 2260 2261 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 2262 { 2263 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 2264 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 2265 2266 apic_timer_expired(apic); 2267 2268 if (lapic_is_periodic(apic)) { 2269 advance_periodic_target_expiration(apic); 2270 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 2271 return HRTIMER_RESTART; 2272 } else 2273 return HRTIMER_NORESTART; 2274 } 2275 2276 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns) 2277 { 2278 struct kvm_lapic *apic; 2279 2280 ASSERT(vcpu != NULL); 2281 2282 apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT); 2283 if (!apic) 2284 goto nomem; 2285 2286 vcpu->arch.apic = apic; 2287 2288 apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); 2289 if (!apic->regs) { 2290 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 2291 vcpu->vcpu_id); 2292 goto nomem_free_apic; 2293 } 2294 apic->vcpu = vcpu; 2295 2296 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 2297 HRTIMER_MODE_ABS); 2298 apic->lapic_timer.timer.function = apic_timer_fn; 2299 if (timer_advance_ns == -1) { 2300 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT; 2301 apic->lapic_timer.timer_advance_adjust_done = false; 2302 } else { 2303 apic->lapic_timer.timer_advance_ns = timer_advance_ns; 2304 apic->lapic_timer.timer_advance_adjust_done = true; 2305 } 2306 2307 2308 /* 2309 * APIC is created enabled. This will prevent kvm_lapic_set_base from 2310 * thinking that APIC state has changed. 2311 */ 2312 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 2313 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 2314 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 2315 2316 return 0; 2317 nomem_free_apic: 2318 kfree(apic); 2319 vcpu->arch.apic = NULL; 2320 nomem: 2321 return -ENOMEM; 2322 } 2323 2324 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 2325 { 2326 struct kvm_lapic *apic = vcpu->arch.apic; 2327 u32 ppr; 2328 2329 if (!kvm_apic_hw_enabled(apic)) 2330 return -1; 2331 2332 __apic_update_ppr(apic, &ppr); 2333 return apic_has_interrupt_for_ppr(apic, ppr); 2334 } 2335 2336 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 2337 { 2338 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); 2339 int r = 0; 2340 2341 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 2342 r = 1; 2343 if ((lvt0 & APIC_LVT_MASKED) == 0 && 2344 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 2345 r = 1; 2346 return r; 2347 } 2348 2349 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 2350 { 2351 struct kvm_lapic *apic = vcpu->arch.apic; 2352 2353 if (atomic_read(&apic->lapic_timer.pending) > 0) { 2354 kvm_apic_inject_pending_timer_irqs(apic); 2355 atomic_set(&apic->lapic_timer.pending, 0); 2356 } 2357 } 2358 2359 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 2360 { 2361 int vector = kvm_apic_has_interrupt(vcpu); 2362 struct kvm_lapic *apic = vcpu->arch.apic; 2363 u32 ppr; 2364 2365 if (vector == -1) 2366 return -1; 2367 2368 /* 2369 * We get here even with APIC virtualization enabled, if doing 2370 * nested virtualization and L1 runs with the "acknowledge interrupt 2371 * on exit" mode. Then we cannot inject the interrupt via RVI, 2372 * because the process would deliver it through the IDT. 2373 */ 2374 2375 apic_clear_irr(vector, apic); 2376 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) { 2377 /* 2378 * For auto-EOI interrupts, there might be another pending 2379 * interrupt above PPR, so check whether to raise another 2380 * KVM_REQ_EVENT. 2381 */ 2382 apic_update_ppr(apic); 2383 } else { 2384 /* 2385 * For normal interrupts, PPR has been raised and there cannot 2386 * be a higher-priority pending interrupt---except if there was 2387 * a concurrent interrupt injection, but that would have 2388 * triggered KVM_REQ_EVENT already. 2389 */ 2390 apic_set_isr(vector, apic); 2391 __apic_update_ppr(apic, &ppr); 2392 } 2393 2394 return vector; 2395 } 2396 2397 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, 2398 struct kvm_lapic_state *s, bool set) 2399 { 2400 if (apic_x2apic_mode(vcpu->arch.apic)) { 2401 u32 *id = (u32 *)(s->regs + APIC_ID); 2402 u32 *ldr = (u32 *)(s->regs + APIC_LDR); 2403 2404 if (vcpu->kvm->arch.x2apic_format) { 2405 if (*id != vcpu->vcpu_id) 2406 return -EINVAL; 2407 } else { 2408 if (set) 2409 *id >>= 24; 2410 else 2411 *id <<= 24; 2412 } 2413 2414 /* In x2APIC mode, the LDR is fixed and based on the id */ 2415 if (set) 2416 *ldr = kvm_apic_calc_x2apic_ldr(*id); 2417 } 2418 2419 return 0; 2420 } 2421 2422 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2423 { 2424 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); 2425 return kvm_apic_state_fixup(vcpu, s, false); 2426 } 2427 2428 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2429 { 2430 struct kvm_lapic *apic = vcpu->arch.apic; 2431 int r; 2432 2433 2434 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 2435 /* set SPIV separately to get count of SW disabled APICs right */ 2436 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 2437 2438 r = kvm_apic_state_fixup(vcpu, s, true); 2439 if (r) 2440 return r; 2441 memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s)); 2442 2443 recalculate_apic_map(vcpu->kvm); 2444 kvm_apic_set_version(vcpu); 2445 2446 apic_update_ppr(apic); 2447 hrtimer_cancel(&apic->lapic_timer.timer); 2448 apic_update_lvtt(apic); 2449 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2450 update_divide_count(apic); 2451 start_apic_timer(apic); 2452 apic->irr_pending = true; 2453 apic->isr_count = vcpu->arch.apicv_active ? 2454 1 : count_vectors(apic->regs + APIC_ISR); 2455 apic->highest_isr_cache = -1; 2456 if (vcpu->arch.apicv_active) { 2457 kvm_x86_ops->apicv_post_state_restore(vcpu); 2458 kvm_x86_ops->hwapic_irr_update(vcpu, 2459 apic_find_highest_irr(apic)); 2460 kvm_x86_ops->hwapic_isr_update(vcpu, 2461 apic_find_highest_isr(apic)); 2462 } 2463 kvm_make_request(KVM_REQ_EVENT, vcpu); 2464 if (ioapic_in_kernel(vcpu->kvm)) 2465 kvm_rtc_eoi_tracking_restore_one(vcpu); 2466 2467 vcpu->arch.apic_arb_prio = 0; 2468 2469 return 0; 2470 } 2471 2472 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 2473 { 2474 struct hrtimer *timer; 2475 2476 if (!lapic_in_kernel(vcpu) || 2477 kvm_can_post_timer_interrupt(vcpu)) 2478 return; 2479 2480 timer = &vcpu->arch.apic->lapic_timer.timer; 2481 if (hrtimer_cancel(timer)) 2482 hrtimer_start_expires(timer, HRTIMER_MODE_ABS); 2483 } 2484 2485 /* 2486 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 2487 * 2488 * Detect whether guest triggered PV EOI since the 2489 * last entry. If yes, set EOI on guests's behalf. 2490 * Clear PV EOI in guest memory in any case. 2491 */ 2492 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 2493 struct kvm_lapic *apic) 2494 { 2495 bool pending; 2496 int vector; 2497 /* 2498 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 2499 * and KVM_PV_EOI_ENABLED in guest memory as follows: 2500 * 2501 * KVM_APIC_PV_EOI_PENDING is unset: 2502 * -> host disabled PV EOI. 2503 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 2504 * -> host enabled PV EOI, guest did not execute EOI yet. 2505 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 2506 * -> host enabled PV EOI, guest executed EOI. 2507 */ 2508 BUG_ON(!pv_eoi_enabled(vcpu)); 2509 pending = pv_eoi_get_pending(vcpu); 2510 /* 2511 * Clear pending bit in any case: it will be set again on vmentry. 2512 * While this might not be ideal from performance point of view, 2513 * this makes sure pv eoi is only enabled when we know it's safe. 2514 */ 2515 pv_eoi_clr_pending(vcpu); 2516 if (pending) 2517 return; 2518 vector = apic_set_eoi(apic); 2519 trace_kvm_pv_eoi(apic, vector); 2520 } 2521 2522 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 2523 { 2524 u32 data; 2525 2526 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 2527 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 2528 2529 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2530 return; 2531 2532 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2533 sizeof(u32))) 2534 return; 2535 2536 apic_set_tpr(vcpu->arch.apic, data & 0xff); 2537 } 2538 2539 /* 2540 * apic_sync_pv_eoi_to_guest - called before vmentry 2541 * 2542 * Detect whether it's safe to enable PV EOI and 2543 * if yes do so. 2544 */ 2545 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 2546 struct kvm_lapic *apic) 2547 { 2548 if (!pv_eoi_enabled(vcpu) || 2549 /* IRR set or many bits in ISR: could be nested. */ 2550 apic->irr_pending || 2551 /* Cache not set: could be safe but we don't bother. */ 2552 apic->highest_isr_cache == -1 || 2553 /* Need EOI to update ioapic. */ 2554 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { 2555 /* 2556 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 2557 * so we need not do anything here. 2558 */ 2559 return; 2560 } 2561 2562 pv_eoi_set_pending(apic->vcpu); 2563 } 2564 2565 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 2566 { 2567 u32 data, tpr; 2568 int max_irr, max_isr; 2569 struct kvm_lapic *apic = vcpu->arch.apic; 2570 2571 apic_sync_pv_eoi_to_guest(vcpu, apic); 2572 2573 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 2574 return; 2575 2576 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff; 2577 max_irr = apic_find_highest_irr(apic); 2578 if (max_irr < 0) 2579 max_irr = 0; 2580 max_isr = apic_find_highest_isr(apic); 2581 if (max_isr < 0) 2582 max_isr = 0; 2583 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 2584 2585 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 2586 sizeof(u32)); 2587 } 2588 2589 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 2590 { 2591 if (vapic_addr) { 2592 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 2593 &vcpu->arch.apic->vapic_cache, 2594 vapic_addr, sizeof(u32))) 2595 return -EINVAL; 2596 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2597 } else { 2598 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 2599 } 2600 2601 vcpu->arch.apic->vapic_addr = vapic_addr; 2602 return 0; 2603 } 2604 2605 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 2606 { 2607 struct kvm_lapic *apic = vcpu->arch.apic; 2608 u32 reg = (msr - APIC_BASE_MSR) << 4; 2609 2610 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2611 return 1; 2612 2613 if (reg == APIC_ICR2) 2614 return 1; 2615 2616 /* if this is ICR write vector before command */ 2617 if (reg == APIC_ICR) 2618 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2619 return kvm_lapic_reg_write(apic, reg, (u32)data); 2620 } 2621 2622 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 2623 { 2624 struct kvm_lapic *apic = vcpu->arch.apic; 2625 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0; 2626 2627 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 2628 return 1; 2629 2630 if (reg == APIC_DFR || reg == APIC_ICR2) 2631 return 1; 2632 2633 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2634 return 1; 2635 if (reg == APIC_ICR) 2636 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2637 2638 *data = (((u64)high) << 32) | low; 2639 2640 return 0; 2641 } 2642 2643 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 2644 { 2645 struct kvm_lapic *apic = vcpu->arch.apic; 2646 2647 if (!lapic_in_kernel(vcpu)) 2648 return 1; 2649 2650 /* if this is ICR write vector before command */ 2651 if (reg == APIC_ICR) 2652 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 2653 return kvm_lapic_reg_write(apic, reg, (u32)data); 2654 } 2655 2656 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 2657 { 2658 struct kvm_lapic *apic = vcpu->arch.apic; 2659 u32 low, high = 0; 2660 2661 if (!lapic_in_kernel(vcpu)) 2662 return 1; 2663 2664 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 2665 return 1; 2666 if (reg == APIC_ICR) 2667 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high); 2668 2669 *data = (((u64)high) << 32) | low; 2670 2671 return 0; 2672 } 2673 2674 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len) 2675 { 2676 u64 addr = data & ~KVM_MSR_ENABLED; 2677 struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data; 2678 unsigned long new_len; 2679 2680 if (!IS_ALIGNED(addr, 4)) 2681 return 1; 2682 2683 vcpu->arch.pv_eoi.msr_val = data; 2684 if (!pv_eoi_enabled(vcpu)) 2685 return 0; 2686 2687 if (addr == ghc->gpa && len <= ghc->len) 2688 new_len = ghc->len; 2689 else 2690 new_len = len; 2691 2692 return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); 2693 } 2694 2695 void kvm_apic_accept_events(struct kvm_vcpu *vcpu) 2696 { 2697 struct kvm_lapic *apic = vcpu->arch.apic; 2698 u8 sipi_vector; 2699 unsigned long pe; 2700 2701 if (!lapic_in_kernel(vcpu) || !apic->pending_events) 2702 return; 2703 2704 /* 2705 * INITs are latched while in SMM. Because an SMM CPU cannot 2706 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs 2707 * and delay processing of INIT until the next RSM. 2708 */ 2709 if (is_smm(vcpu)) { 2710 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); 2711 if (test_bit(KVM_APIC_SIPI, &apic->pending_events)) 2712 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 2713 return; 2714 } 2715 2716 pe = xchg(&apic->pending_events, 0); 2717 if (test_bit(KVM_APIC_INIT, &pe)) { 2718 kvm_vcpu_reset(vcpu, true); 2719 if (kvm_vcpu_is_bsp(apic->vcpu)) 2720 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2721 else 2722 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 2723 } 2724 if (test_bit(KVM_APIC_SIPI, &pe) && 2725 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 2726 /* evaluate pending_events before reading the vector */ 2727 smp_rmb(); 2728 sipi_vector = apic->sipi_vector; 2729 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector); 2730 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2731 } 2732 } 2733 2734 void kvm_lapic_init(void) 2735 { 2736 /* do not patch jump label more than once per second */ 2737 jump_label_rate_limit(&apic_hw_disabled, HZ); 2738 jump_label_rate_limit(&apic_sw_disabled, HZ); 2739 } 2740 2741 void kvm_lapic_exit(void) 2742 { 2743 static_key_deferred_flush(&apic_hw_disabled); 2744 static_key_deferred_flush(&apic_sw_disabled); 2745 } 2746