1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Local APIC virtualization 5 * 6 * Copyright (C) 2006 Qumranet, Inc. 7 * Copyright (C) 2007 Novell 8 * Copyright (C) 2007 Intel 9 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Dor Laor <dor.laor@qumranet.com> 13 * Gregory Haskins <ghaskins@novell.com> 14 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 15 * 16 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 17 */ 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/kvm_host.h> 21 #include <linux/kvm.h> 22 #include <linux/mm.h> 23 #include <linux/highmem.h> 24 #include <linux/smp.h> 25 #include <linux/hrtimer.h> 26 #include <linux/io.h> 27 #include <linux/export.h> 28 #include <linux/math64.h> 29 #include <linux/slab.h> 30 #include <asm/processor.h> 31 #include <asm/mce.h> 32 #include <asm/msr.h> 33 #include <asm/page.h> 34 #include <asm/current.h> 35 #include <asm/apicdef.h> 36 #include <asm/delay.h> 37 #include <linux/atomic.h> 38 #include <linux/jump_label.h> 39 #include "kvm_cache_regs.h" 40 #include "irq.h" 41 #include "ioapic.h" 42 #include "trace.h" 43 #include "x86.h" 44 #include "cpuid.h" 45 #include "hyperv.h" 46 #include "smm.h" 47 48 #ifndef CONFIG_X86_64 49 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 50 #else 51 #define mod_64(x, y) ((x) % (y)) 52 #endif 53 54 /* 14 is the version for Xeon and Pentium 8.4.8*/ 55 #define APIC_VERSION 0x14UL 56 #define LAPIC_MMIO_LENGTH (1 << 12) 57 /* followed define is not in apicdef.h */ 58 #define MAX_APIC_VECTOR 256 59 #define APIC_VECTORS_PER_REG 32 60 61 static bool lapic_timer_advance_dynamic __read_mostly; 62 #define LAPIC_TIMER_ADVANCE_ADJUST_MIN 100 /* clock cycles */ 63 #define LAPIC_TIMER_ADVANCE_ADJUST_MAX 10000 /* clock cycles */ 64 #define LAPIC_TIMER_ADVANCE_NS_INIT 1000 65 #define LAPIC_TIMER_ADVANCE_NS_MAX 5000 66 /* step-by-step approximation to mitigate fluctuation */ 67 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8 68 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data); 69 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data); 70 71 static inline void __kvm_lapic_set_reg(char *regs, int reg_off, u32 val) 72 { 73 *((u32 *) (regs + reg_off)) = val; 74 } 75 76 static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) 77 { 78 __kvm_lapic_set_reg(apic->regs, reg_off, val); 79 } 80 81 static __always_inline u64 __kvm_lapic_get_reg64(char *regs, int reg) 82 { 83 BUILD_BUG_ON(reg != APIC_ICR); 84 return *((u64 *) (regs + reg)); 85 } 86 87 static __always_inline u64 kvm_lapic_get_reg64(struct kvm_lapic *apic, int reg) 88 { 89 return __kvm_lapic_get_reg64(apic->regs, reg); 90 } 91 92 static __always_inline void __kvm_lapic_set_reg64(char *regs, int reg, u64 val) 93 { 94 BUILD_BUG_ON(reg != APIC_ICR); 95 *((u64 *) (regs + reg)) = val; 96 } 97 98 static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic, 99 int reg, u64 val) 100 { 101 __kvm_lapic_set_reg64(apic->regs, reg, val); 102 } 103 104 static inline int apic_test_vector(int vec, void *bitmap) 105 { 106 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 107 } 108 109 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 110 { 111 struct kvm_lapic *apic = vcpu->arch.apic; 112 113 return apic_test_vector(vector, apic->regs + APIC_ISR) || 114 apic_test_vector(vector, apic->regs + APIC_IRR); 115 } 116 117 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 118 { 119 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 120 } 121 122 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 123 { 124 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 125 } 126 127 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ); 128 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ); 129 130 static inline int apic_enabled(struct kvm_lapic *apic) 131 { 132 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 133 } 134 135 #define LVT_MASK \ 136 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 137 138 #define LINT_MASK \ 139 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 140 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 141 142 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) 143 { 144 return apic->vcpu->vcpu_id; 145 } 146 147 static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu) 148 { 149 return pi_inject_timer && kvm_vcpu_apicv_active(vcpu) && 150 (kvm_mwait_in_guest(vcpu->kvm) || kvm_hlt_in_guest(vcpu->kvm)); 151 } 152 153 bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu) 154 { 155 return kvm_x86_ops.set_hv_timer 156 && !(kvm_mwait_in_guest(vcpu->kvm) || 157 kvm_can_post_timer_interrupt(vcpu)); 158 } 159 160 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu) 161 { 162 return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE; 163 } 164 165 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id) 166 { 167 return ((id >> 4) << 16) | (1 << (id & 0xf)); 168 } 169 170 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, 171 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { 172 switch (map->logical_mode) { 173 case KVM_APIC_MODE_SW_DISABLED: 174 /* Arbitrarily use the flat map so that @cluster isn't NULL. */ 175 *cluster = map->xapic_flat_map; 176 *mask = 0; 177 return true; 178 case KVM_APIC_MODE_X2APIC: { 179 u32 offset = (dest_id >> 16) * 16; 180 u32 max_apic_id = map->max_apic_id; 181 182 if (offset <= max_apic_id) { 183 u8 cluster_size = min(max_apic_id - offset + 1, 16U); 184 185 offset = array_index_nospec(offset, map->max_apic_id + 1); 186 *cluster = &map->phys_map[offset]; 187 *mask = dest_id & (0xffff >> (16 - cluster_size)); 188 } else { 189 *mask = 0; 190 } 191 192 return true; 193 } 194 case KVM_APIC_MODE_XAPIC_FLAT: 195 *cluster = map->xapic_flat_map; 196 *mask = dest_id & 0xff; 197 return true; 198 case KVM_APIC_MODE_XAPIC_CLUSTER: 199 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; 200 *mask = dest_id & 0xf; 201 return true; 202 case KVM_APIC_MODE_MAP_DISABLED: 203 return false; 204 default: 205 WARN_ON_ONCE(1); 206 return false; 207 } 208 } 209 210 static void kvm_apic_map_free(struct rcu_head *rcu) 211 { 212 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu); 213 214 kvfree(map); 215 } 216 217 static int kvm_recalculate_phys_map(struct kvm_apic_map *new, 218 struct kvm_vcpu *vcpu, 219 bool *xapic_id_mismatch) 220 { 221 struct kvm_lapic *apic = vcpu->arch.apic; 222 u32 x2apic_id = kvm_x2apic_id(apic); 223 u32 xapic_id = kvm_xapic_id(apic); 224 u32 physical_id; 225 226 /* 227 * For simplicity, KVM always allocates enough space for all possible 228 * xAPIC IDs. Yell, but don't kill the VM, as KVM can continue on 229 * without the optimized map. 230 */ 231 if (WARN_ON_ONCE(xapic_id > new->max_apic_id)) 232 return -EINVAL; 233 234 /* 235 * Bail if a vCPU was added and/or enabled its APIC between allocating 236 * the map and doing the actual calculations for the map. Note, KVM 237 * hardcodes the x2APIC ID to vcpu_id, i.e. there's no TOCTOU bug if 238 * the compiler decides to reload x2apic_id after this check. 239 */ 240 if (x2apic_id > new->max_apic_id) 241 return -E2BIG; 242 243 /* 244 * Deliberately truncate the vCPU ID when detecting a mismatched APIC 245 * ID to avoid false positives if the vCPU ID, i.e. x2APIC ID, is a 246 * 32-bit value. Any unwanted aliasing due to truncation results will 247 * be detected below. 248 */ 249 if (!apic_x2apic_mode(apic) && xapic_id != (u8)vcpu->vcpu_id) 250 *xapic_id_mismatch = true; 251 252 /* 253 * Apply KVM's hotplug hack if userspace has enable 32-bit APIC IDs. 254 * Allow sending events to vCPUs by their x2APIC ID even if the target 255 * vCPU is in legacy xAPIC mode, and silently ignore aliased xAPIC IDs 256 * (the x2APIC ID is truncated to 8 bits, causing IDs > 0xff to wrap 257 * and collide). 258 * 259 * Honor the architectural (and KVM's non-optimized) behavior if 260 * userspace has not enabled 32-bit x2APIC IDs. Each APIC is supposed 261 * to process messages independently. If multiple vCPUs have the same 262 * effective APIC ID, e.g. due to the x2APIC wrap or because the guest 263 * manually modified its xAPIC IDs, events targeting that ID are 264 * supposed to be recognized by all vCPUs with said ID. 265 */ 266 if (vcpu->kvm->arch.x2apic_format) { 267 /* See also kvm_apic_match_physical_addr(). */ 268 if (apic_x2apic_mode(apic) || x2apic_id > 0xff) 269 new->phys_map[x2apic_id] = apic; 270 271 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) 272 new->phys_map[xapic_id] = apic; 273 } else { 274 /* 275 * Disable the optimized map if the physical APIC ID is already 276 * mapped, i.e. is aliased to multiple vCPUs. The optimized 277 * map requires a strict 1:1 mapping between IDs and vCPUs. 278 */ 279 if (apic_x2apic_mode(apic)) 280 physical_id = x2apic_id; 281 else 282 physical_id = xapic_id; 283 284 if (new->phys_map[physical_id]) 285 return -EINVAL; 286 287 new->phys_map[physical_id] = apic; 288 } 289 290 return 0; 291 } 292 293 static void kvm_recalculate_logical_map(struct kvm_apic_map *new, 294 struct kvm_vcpu *vcpu) 295 { 296 struct kvm_lapic *apic = vcpu->arch.apic; 297 enum kvm_apic_logical_mode logical_mode; 298 struct kvm_lapic **cluster; 299 u16 mask; 300 u32 ldr; 301 302 if (new->logical_mode == KVM_APIC_MODE_MAP_DISABLED) 303 return; 304 305 if (!kvm_apic_sw_enabled(apic)) 306 return; 307 308 ldr = kvm_lapic_get_reg(apic, APIC_LDR); 309 if (!ldr) 310 return; 311 312 if (apic_x2apic_mode(apic)) { 313 logical_mode = KVM_APIC_MODE_X2APIC; 314 } else { 315 ldr = GET_APIC_LOGICAL_ID(ldr); 316 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT) 317 logical_mode = KVM_APIC_MODE_XAPIC_FLAT; 318 else 319 logical_mode = KVM_APIC_MODE_XAPIC_CLUSTER; 320 } 321 322 /* 323 * To optimize logical mode delivery, all software-enabled APICs must 324 * be configured for the same mode. 325 */ 326 if (new->logical_mode == KVM_APIC_MODE_SW_DISABLED) { 327 new->logical_mode = logical_mode; 328 } else if (new->logical_mode != logical_mode) { 329 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; 330 return; 331 } 332 333 /* 334 * In x2APIC mode, the LDR is read-only and derived directly from the 335 * x2APIC ID, thus is guaranteed to be addressable. KVM reuses 336 * kvm_apic_map.phys_map to optimize logical mode x2APIC interrupts by 337 * reversing the LDR calculation to get cluster of APICs, i.e. no 338 * additional work is required. 339 */ 340 if (apic_x2apic_mode(apic)) { 341 WARN_ON_ONCE(ldr != kvm_apic_calc_x2apic_ldr(kvm_x2apic_id(apic))); 342 return; 343 } 344 345 if (WARN_ON_ONCE(!kvm_apic_map_get_logical_dest(new, ldr, 346 &cluster, &mask))) { 347 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; 348 return; 349 } 350 351 if (!mask) 352 return; 353 354 ldr = ffs(mask) - 1; 355 if (!is_power_of_2(mask) || cluster[ldr]) 356 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; 357 else 358 cluster[ldr] = apic; 359 } 360 361 /* 362 * CLEAN -> DIRTY and UPDATE_IN_PROGRESS -> DIRTY changes happen without a lock. 363 * 364 * DIRTY -> UPDATE_IN_PROGRESS and UPDATE_IN_PROGRESS -> CLEAN happen with 365 * apic_map_lock_held. 366 */ 367 enum { 368 CLEAN, 369 UPDATE_IN_PROGRESS, 370 DIRTY 371 }; 372 373 void kvm_recalculate_apic_map(struct kvm *kvm) 374 { 375 struct kvm_apic_map *new, *old = NULL; 376 struct kvm_vcpu *vcpu; 377 unsigned long i; 378 u32 max_id = 255; /* enough space for any xAPIC ID */ 379 bool xapic_id_mismatch; 380 int r; 381 382 /* Read kvm->arch.apic_map_dirty before kvm->arch.apic_map. */ 383 if (atomic_read_acquire(&kvm->arch.apic_map_dirty) == CLEAN) 384 return; 385 386 WARN_ONCE(!irqchip_in_kernel(kvm), 387 "Dirty APIC map without an in-kernel local APIC"); 388 389 mutex_lock(&kvm->arch.apic_map_lock); 390 391 retry: 392 /* 393 * Read kvm->arch.apic_map_dirty before kvm->arch.apic_map (if clean) 394 * or the APIC registers (if dirty). Note, on retry the map may have 395 * not yet been marked dirty by whatever task changed a vCPU's x2APIC 396 * ID, i.e. the map may still show up as in-progress. In that case 397 * this task still needs to retry and complete its calculation. 398 */ 399 if (atomic_cmpxchg_acquire(&kvm->arch.apic_map_dirty, 400 DIRTY, UPDATE_IN_PROGRESS) == CLEAN) { 401 /* Someone else has updated the map. */ 402 mutex_unlock(&kvm->arch.apic_map_lock); 403 return; 404 } 405 406 /* 407 * Reset the mismatch flag between attempts so that KVM does the right 408 * thing if a vCPU changes its xAPIC ID, but do NOT reset max_id, i.e. 409 * keep max_id strictly increasing. Disallowing max_id from shrinking 410 * ensures KVM won't get stuck in an infinite loop, e.g. if the vCPU 411 * with the highest x2APIC ID is toggling its APIC on and off. 412 */ 413 xapic_id_mismatch = false; 414 415 kvm_for_each_vcpu(i, vcpu, kvm) 416 if (kvm_apic_present(vcpu)) 417 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); 418 419 new = kvzalloc(sizeof(struct kvm_apic_map) + 420 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), 421 GFP_KERNEL_ACCOUNT); 422 423 if (!new) 424 goto out; 425 426 new->max_apic_id = max_id; 427 new->logical_mode = KVM_APIC_MODE_SW_DISABLED; 428 429 kvm_for_each_vcpu(i, vcpu, kvm) { 430 if (!kvm_apic_present(vcpu)) 431 continue; 432 433 r = kvm_recalculate_phys_map(new, vcpu, &xapic_id_mismatch); 434 if (r) { 435 kvfree(new); 436 new = NULL; 437 if (r == -E2BIG) { 438 cond_resched(); 439 goto retry; 440 } 441 442 goto out; 443 } 444 445 kvm_recalculate_logical_map(new, vcpu); 446 } 447 out: 448 /* 449 * The optimized map is effectively KVM's internal version of APICv, 450 * and all unwanted aliasing that results in disabling the optimized 451 * map also applies to APICv. 452 */ 453 if (!new) 454 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED); 455 else 456 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED); 457 458 if (!new || new->logical_mode == KVM_APIC_MODE_MAP_DISABLED) 459 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED); 460 else 461 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED); 462 463 if (xapic_id_mismatch) 464 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED); 465 else 466 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED); 467 468 old = rcu_dereference_protected(kvm->arch.apic_map, 469 lockdep_is_held(&kvm->arch.apic_map_lock)); 470 rcu_assign_pointer(kvm->arch.apic_map, new); 471 /* 472 * Write kvm->arch.apic_map before clearing apic->apic_map_dirty. 473 * If another update has come in, leave it DIRTY. 474 */ 475 atomic_cmpxchg_release(&kvm->arch.apic_map_dirty, 476 UPDATE_IN_PROGRESS, CLEAN); 477 mutex_unlock(&kvm->arch.apic_map_lock); 478 479 if (old) 480 call_rcu(&old->rcu, kvm_apic_map_free); 481 482 kvm_make_scan_ioapic_request(kvm); 483 } 484 485 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 486 { 487 bool enabled = val & APIC_SPIV_APIC_ENABLED; 488 489 kvm_lapic_set_reg(apic, APIC_SPIV, val); 490 491 if (enabled != apic->sw_enabled) { 492 apic->sw_enabled = enabled; 493 if (enabled) 494 static_branch_slow_dec_deferred(&apic_sw_disabled); 495 else 496 static_branch_inc(&apic_sw_disabled.key); 497 498 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 499 } 500 501 /* Check if there are APF page ready requests pending */ 502 if (enabled) 503 kvm_make_request(KVM_REQ_APF_READY, apic->vcpu); 504 } 505 506 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) 507 { 508 kvm_lapic_set_reg(apic, APIC_ID, id << 24); 509 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 510 } 511 512 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 513 { 514 kvm_lapic_set_reg(apic, APIC_LDR, id); 515 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 516 } 517 518 static inline void kvm_apic_set_dfr(struct kvm_lapic *apic, u32 val) 519 { 520 kvm_lapic_set_reg(apic, APIC_DFR, val); 521 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 522 } 523 524 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) 525 { 526 u32 ldr = kvm_apic_calc_x2apic_ldr(id); 527 528 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); 529 530 kvm_lapic_set_reg(apic, APIC_ID, id); 531 kvm_lapic_set_reg(apic, APIC_LDR, ldr); 532 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 533 } 534 535 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 536 { 537 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 538 } 539 540 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 541 { 542 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; 543 } 544 545 static inline int apic_lvtt_period(struct kvm_lapic *apic) 546 { 547 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; 548 } 549 550 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 551 { 552 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; 553 } 554 555 static inline int apic_lvt_nmi_mode(u32 lvt_val) 556 { 557 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 558 } 559 560 static inline bool kvm_lapic_lvt_supported(struct kvm_lapic *apic, int lvt_index) 561 { 562 return apic->nr_lvt_entries > lvt_index; 563 } 564 565 static inline int kvm_apic_calc_nr_lvt_entries(struct kvm_vcpu *vcpu) 566 { 567 return KVM_APIC_MAX_NR_LVT_ENTRIES - !(vcpu->arch.mcg_cap & MCG_CMCI_P); 568 } 569 570 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 571 { 572 struct kvm_lapic *apic = vcpu->arch.apic; 573 u32 v = 0; 574 575 if (!lapic_in_kernel(vcpu)) 576 return; 577 578 v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16); 579 580 /* 581 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) 582 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with 583 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC 584 * version first and level-triggered interrupts never get EOIed in 585 * IOAPIC. 586 */ 587 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) && 588 !ioapic_in_kernel(vcpu->kvm)) 589 v |= APIC_LVR_DIRECTED_EOI; 590 kvm_lapic_set_reg(apic, APIC_LVR, v); 591 } 592 593 void kvm_apic_after_set_mcg_cap(struct kvm_vcpu *vcpu) 594 { 595 int nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu); 596 struct kvm_lapic *apic = vcpu->arch.apic; 597 int i; 598 599 if (!lapic_in_kernel(vcpu) || nr_lvt_entries == apic->nr_lvt_entries) 600 return; 601 602 /* Initialize/mask any "new" LVT entries. */ 603 for (i = apic->nr_lvt_entries; i < nr_lvt_entries; i++) 604 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED); 605 606 apic->nr_lvt_entries = nr_lvt_entries; 607 608 /* The number of LVT entries is reflected in the version register. */ 609 kvm_apic_set_version(vcpu); 610 } 611 612 static const unsigned int apic_lvt_mask[KVM_APIC_MAX_NR_LVT_ENTRIES] = { 613 [LVT_TIMER] = LVT_MASK, /* timer mode mask added at runtime */ 614 [LVT_THERMAL_MONITOR] = LVT_MASK | APIC_MODE_MASK, 615 [LVT_PERFORMANCE_COUNTER] = LVT_MASK | APIC_MODE_MASK, 616 [LVT_LINT0] = LINT_MASK, 617 [LVT_LINT1] = LINT_MASK, 618 [LVT_ERROR] = LVT_MASK, 619 [LVT_CMCI] = LVT_MASK | APIC_MODE_MASK 620 }; 621 622 static int find_highest_vector(void *bitmap) 623 { 624 int vec; 625 u32 *reg; 626 627 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 628 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 629 reg = bitmap + REG_POS(vec); 630 if (*reg) 631 return __fls(*reg) + vec; 632 } 633 634 return -1; 635 } 636 637 static u8 count_vectors(void *bitmap) 638 { 639 int vec; 640 u32 *reg; 641 u8 count = 0; 642 643 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 644 reg = bitmap + REG_POS(vec); 645 count += hweight32(*reg); 646 } 647 648 return count; 649 } 650 651 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr) 652 { 653 u32 i, vec; 654 u32 pir_val, irr_val, prev_irr_val; 655 int max_updated_irr; 656 657 max_updated_irr = -1; 658 *max_irr = -1; 659 660 for (i = vec = 0; i <= 7; i++, vec += 32) { 661 u32 *p_irr = (u32 *)(regs + APIC_IRR + i * 0x10); 662 663 irr_val = *p_irr; 664 pir_val = READ_ONCE(pir[i]); 665 666 if (pir_val) { 667 pir_val = xchg(&pir[i], 0); 668 669 prev_irr_val = irr_val; 670 do { 671 irr_val = prev_irr_val | pir_val; 672 } while (prev_irr_val != irr_val && 673 !try_cmpxchg(p_irr, &prev_irr_val, irr_val)); 674 675 if (prev_irr_val != irr_val) 676 max_updated_irr = __fls(irr_val ^ prev_irr_val) + vec; 677 } 678 if (irr_val) 679 *max_irr = __fls(irr_val) + vec; 680 } 681 682 return ((max_updated_irr != -1) && 683 (max_updated_irr == *max_irr)); 684 } 685 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); 686 687 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr) 688 { 689 struct kvm_lapic *apic = vcpu->arch.apic; 690 bool irr_updated = __kvm_apic_update_irr(pir, apic->regs, max_irr); 691 692 if (unlikely(!apic->apicv_active && irr_updated)) 693 apic->irr_pending = true; 694 return irr_updated; 695 } 696 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 697 698 static inline int apic_search_irr(struct kvm_lapic *apic) 699 { 700 return find_highest_vector(apic->regs + APIC_IRR); 701 } 702 703 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 704 { 705 int result; 706 707 /* 708 * Note that irr_pending is just a hint. It will be always 709 * true with virtual interrupt delivery enabled. 710 */ 711 if (!apic->irr_pending) 712 return -1; 713 714 result = apic_search_irr(apic); 715 ASSERT(result == -1 || result >= 16); 716 717 return result; 718 } 719 720 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 721 { 722 if (unlikely(apic->apicv_active)) { 723 /* need to update RVI */ 724 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); 725 static_call_cond(kvm_x86_hwapic_irr_update)(apic->vcpu, 726 apic_find_highest_irr(apic)); 727 } else { 728 apic->irr_pending = false; 729 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); 730 if (apic_search_irr(apic) != -1) 731 apic->irr_pending = true; 732 } 733 } 734 735 void kvm_apic_clear_irr(struct kvm_vcpu *vcpu, int vec) 736 { 737 apic_clear_irr(vec, vcpu->arch.apic); 738 } 739 EXPORT_SYMBOL_GPL(kvm_apic_clear_irr); 740 741 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 742 { 743 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 744 return; 745 746 /* 747 * With APIC virtualization enabled, all caching is disabled 748 * because the processor can modify ISR under the hood. Instead 749 * just set SVI. 750 */ 751 if (unlikely(apic->apicv_active)) 752 static_call_cond(kvm_x86_hwapic_isr_update)(vec); 753 else { 754 ++apic->isr_count; 755 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 756 /* 757 * ISR (in service register) bit is set when injecting an interrupt. 758 * The highest vector is injected. Thus the latest bit set matches 759 * the highest bit in ISR. 760 */ 761 apic->highest_isr_cache = vec; 762 } 763 } 764 765 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 766 { 767 int result; 768 769 /* 770 * Note that isr_count is always 1, and highest_isr_cache 771 * is always -1, with APIC virtualization enabled. 772 */ 773 if (!apic->isr_count) 774 return -1; 775 if (likely(apic->highest_isr_cache != -1)) 776 return apic->highest_isr_cache; 777 778 result = find_highest_vector(apic->regs + APIC_ISR); 779 ASSERT(result == -1 || result >= 16); 780 781 return result; 782 } 783 784 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 785 { 786 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 787 return; 788 789 /* 790 * We do get here for APIC virtualization enabled if the guest 791 * uses the Hyper-V APIC enlightenment. In this case we may need 792 * to trigger a new interrupt delivery by writing the SVI field; 793 * on the other hand isr_count and highest_isr_cache are unused 794 * and must be left alone. 795 */ 796 if (unlikely(apic->apicv_active)) 797 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic)); 798 else { 799 --apic->isr_count; 800 BUG_ON(apic->isr_count < 0); 801 apic->highest_isr_cache = -1; 802 } 803 } 804 805 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 806 { 807 /* This may race with setting of irr in __apic_accept_irq() and 808 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 809 * will cause vmexit immediately and the value will be recalculated 810 * on the next vmentry. 811 */ 812 return apic_find_highest_irr(vcpu->arch.apic); 813 } 814 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); 815 816 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 817 int vector, int level, int trig_mode, 818 struct dest_map *dest_map); 819 820 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 821 struct dest_map *dest_map) 822 { 823 struct kvm_lapic *apic = vcpu->arch.apic; 824 825 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 826 irq->level, irq->trig_mode, dest_map); 827 } 828 829 static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map, 830 struct kvm_lapic_irq *irq, u32 min) 831 { 832 int i, count = 0; 833 struct kvm_vcpu *vcpu; 834 835 if (min > map->max_apic_id) 836 return 0; 837 838 for_each_set_bit(i, ipi_bitmap, 839 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { 840 if (map->phys_map[min + i]) { 841 vcpu = map->phys_map[min + i]->vcpu; 842 count += kvm_apic_set_irq(vcpu, irq, NULL); 843 } 844 } 845 846 return count; 847 } 848 849 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, 850 unsigned long ipi_bitmap_high, u32 min, 851 unsigned long icr, int op_64_bit) 852 { 853 struct kvm_apic_map *map; 854 struct kvm_lapic_irq irq = {0}; 855 int cluster_size = op_64_bit ? 64 : 32; 856 int count; 857 858 if (icr & (APIC_DEST_MASK | APIC_SHORT_MASK)) 859 return -KVM_EINVAL; 860 861 irq.vector = icr & APIC_VECTOR_MASK; 862 irq.delivery_mode = icr & APIC_MODE_MASK; 863 irq.level = (icr & APIC_INT_ASSERT) != 0; 864 irq.trig_mode = icr & APIC_INT_LEVELTRIG; 865 866 rcu_read_lock(); 867 map = rcu_dereference(kvm->arch.apic_map); 868 869 count = -EOPNOTSUPP; 870 if (likely(map)) { 871 count = __pv_send_ipi(&ipi_bitmap_low, map, &irq, min); 872 min += cluster_size; 873 count += __pv_send_ipi(&ipi_bitmap_high, map, &irq, min); 874 } 875 876 rcu_read_unlock(); 877 return count; 878 } 879 880 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 881 { 882 883 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 884 sizeof(val)); 885 } 886 887 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 888 { 889 890 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 891 sizeof(*val)); 892 } 893 894 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 895 { 896 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 897 } 898 899 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 900 { 901 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) 902 return; 903 904 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 905 } 906 907 static bool pv_eoi_test_and_clr_pending(struct kvm_vcpu *vcpu) 908 { 909 u8 val; 910 911 if (pv_eoi_get_user(vcpu, &val) < 0) 912 return false; 913 914 val &= KVM_PV_EOI_ENABLED; 915 916 if (val && pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) 917 return false; 918 919 /* 920 * Clear pending bit in any case: it will be set again on vmentry. 921 * While this might not be ideal from performance point of view, 922 * this makes sure pv eoi is only enabled when we know it's safe. 923 */ 924 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 925 926 return val; 927 } 928 929 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) 930 { 931 int highest_irr; 932 if (kvm_x86_ops.sync_pir_to_irr) 933 highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu); 934 else 935 highest_irr = apic_find_highest_irr(apic); 936 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) 937 return -1; 938 return highest_irr; 939 } 940 941 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr) 942 { 943 u32 tpr, isrv, ppr, old_ppr; 944 int isr; 945 946 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI); 947 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI); 948 isr = apic_find_highest_isr(apic); 949 isrv = (isr != -1) ? isr : 0; 950 951 if ((tpr & 0xf0) >= (isrv & 0xf0)) 952 ppr = tpr & 0xff; 953 else 954 ppr = isrv & 0xf0; 955 956 *new_ppr = ppr; 957 if (old_ppr != ppr) 958 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr); 959 960 return ppr < old_ppr; 961 } 962 963 static void apic_update_ppr(struct kvm_lapic *apic) 964 { 965 u32 ppr; 966 967 if (__apic_update_ppr(apic, &ppr) && 968 apic_has_interrupt_for_ppr(apic, ppr) != -1) 969 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 970 } 971 972 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu) 973 { 974 apic_update_ppr(vcpu->arch.apic); 975 } 976 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr); 977 978 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 979 { 980 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr); 981 apic_update_ppr(apic); 982 } 983 984 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda) 985 { 986 return mda == (apic_x2apic_mode(apic) ? 987 X2APIC_BROADCAST : APIC_BROADCAST); 988 } 989 990 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) 991 { 992 if (kvm_apic_broadcast(apic, mda)) 993 return true; 994 995 /* 996 * Hotplug hack: Accept interrupts for vCPUs in xAPIC mode as if they 997 * were in x2APIC mode if the target APIC ID can't be encoded as an 998 * xAPIC ID. This allows unique addressing of hotplugged vCPUs (which 999 * start in xAPIC mode) with an APIC ID that is unaddressable in xAPIC 1000 * mode. Match the x2APIC ID if and only if the target APIC ID can't 1001 * be encoded in xAPIC to avoid spurious matches against a vCPU that 1002 * changed its (addressable) xAPIC ID (which is writable). 1003 */ 1004 if (apic_x2apic_mode(apic) || mda > 0xff) 1005 return mda == kvm_x2apic_id(apic); 1006 1007 return mda == kvm_xapic_id(apic); 1008 } 1009 1010 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 1011 { 1012 u32 logical_id; 1013 1014 if (kvm_apic_broadcast(apic, mda)) 1015 return true; 1016 1017 logical_id = kvm_lapic_get_reg(apic, APIC_LDR); 1018 1019 if (apic_x2apic_mode(apic)) 1020 return ((logical_id >> 16) == (mda >> 16)) 1021 && (logical_id & mda & 0xffff) != 0; 1022 1023 logical_id = GET_APIC_LOGICAL_ID(logical_id); 1024 1025 switch (kvm_lapic_get_reg(apic, APIC_DFR)) { 1026 case APIC_DFR_FLAT: 1027 return (logical_id & mda) != 0; 1028 case APIC_DFR_CLUSTER: 1029 return ((logical_id >> 4) == (mda >> 4)) 1030 && (logical_id & mda & 0xf) != 0; 1031 default: 1032 return false; 1033 } 1034 } 1035 1036 /* The KVM local APIC implementation has two quirks: 1037 * 1038 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs 1039 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID. 1040 * KVM doesn't do that aliasing. 1041 * 1042 * - in-kernel IOAPIC messages have to be delivered directly to 1043 * x2APIC, because the kernel does not support interrupt remapping. 1044 * In order to support broadcast without interrupt remapping, x2APIC 1045 * rewrites the destination of non-IPI messages from APIC_BROADCAST 1046 * to X2APIC_BROADCAST. 1047 * 1048 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is 1049 * important when userspace wants to use x2APIC-format MSIs, because 1050 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7". 1051 */ 1052 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id, 1053 struct kvm_lapic *source, struct kvm_lapic *target) 1054 { 1055 bool ipi = source != NULL; 1056 1057 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && 1058 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target)) 1059 return X2APIC_BROADCAST; 1060 1061 return dest_id; 1062 } 1063 1064 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 1065 int shorthand, unsigned int dest, int dest_mode) 1066 { 1067 struct kvm_lapic *target = vcpu->arch.apic; 1068 u32 mda = kvm_apic_mda(vcpu, dest, source, target); 1069 1070 ASSERT(target); 1071 switch (shorthand) { 1072 case APIC_DEST_NOSHORT: 1073 if (dest_mode == APIC_DEST_PHYSICAL) 1074 return kvm_apic_match_physical_addr(target, mda); 1075 else 1076 return kvm_apic_match_logical_addr(target, mda); 1077 case APIC_DEST_SELF: 1078 return target == source; 1079 case APIC_DEST_ALLINC: 1080 return true; 1081 case APIC_DEST_ALLBUT: 1082 return target != source; 1083 default: 1084 return false; 1085 } 1086 } 1087 EXPORT_SYMBOL_GPL(kvm_apic_match_dest); 1088 1089 int kvm_vector_to_index(u32 vector, u32 dest_vcpus, 1090 const unsigned long *bitmap, u32 bitmap_size) 1091 { 1092 u32 mod; 1093 int i, idx = -1; 1094 1095 mod = vector % dest_vcpus; 1096 1097 for (i = 0; i <= mod; i++) { 1098 idx = find_next_bit(bitmap, bitmap_size, idx + 1); 1099 BUG_ON(idx == bitmap_size); 1100 } 1101 1102 return idx; 1103 } 1104 1105 static void kvm_apic_disabled_lapic_found(struct kvm *kvm) 1106 { 1107 if (!kvm->arch.disabled_lapic_found) { 1108 kvm->arch.disabled_lapic_found = true; 1109 pr_info("Disabled LAPIC found during irq injection\n"); 1110 } 1111 } 1112 1113 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src, 1114 struct kvm_lapic_irq *irq, struct kvm_apic_map *map) 1115 { 1116 if (kvm->arch.x2apic_broadcast_quirk_disabled) { 1117 if ((irq->dest_id == APIC_BROADCAST && 1118 map->logical_mode != KVM_APIC_MODE_X2APIC)) 1119 return true; 1120 if (irq->dest_id == X2APIC_BROADCAST) 1121 return true; 1122 } else { 1123 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src); 1124 if (irq->dest_id == (x2apic_ipi ? 1125 X2APIC_BROADCAST : APIC_BROADCAST)) 1126 return true; 1127 } 1128 1129 return false; 1130 } 1131 1132 /* Return true if the interrupt can be handled by using *bitmap as index mask 1133 * for valid destinations in *dst array. 1134 * Return false if kvm_apic_map_get_dest_lapic did nothing useful. 1135 * Note: we may have zero kvm_lapic destinations when we return true, which 1136 * means that the interrupt should be dropped. In this case, *bitmap would be 1137 * zero and *dst undefined. 1138 */ 1139 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm, 1140 struct kvm_lapic **src, struct kvm_lapic_irq *irq, 1141 struct kvm_apic_map *map, struct kvm_lapic ***dst, 1142 unsigned long *bitmap) 1143 { 1144 int i, lowest; 1145 1146 if (irq->shorthand == APIC_DEST_SELF && src) { 1147 *dst = src; 1148 *bitmap = 1; 1149 return true; 1150 } else if (irq->shorthand) 1151 return false; 1152 1153 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map)) 1154 return false; 1155 1156 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 1157 if (irq->dest_id > map->max_apic_id) { 1158 *bitmap = 0; 1159 } else { 1160 u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1); 1161 *dst = &map->phys_map[dest_id]; 1162 *bitmap = 1; 1163 } 1164 return true; 1165 } 1166 1167 *bitmap = 0; 1168 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, 1169 (u16 *)bitmap)) 1170 return false; 1171 1172 if (!kvm_lowest_prio_delivery(irq)) 1173 return true; 1174 1175 if (!kvm_vector_hashing_enabled()) { 1176 lowest = -1; 1177 for_each_set_bit(i, bitmap, 16) { 1178 if (!(*dst)[i]) 1179 continue; 1180 if (lowest < 0) 1181 lowest = i; 1182 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, 1183 (*dst)[lowest]->vcpu) < 0) 1184 lowest = i; 1185 } 1186 } else { 1187 if (!*bitmap) 1188 return true; 1189 1190 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), 1191 bitmap, 16); 1192 1193 if (!(*dst)[lowest]) { 1194 kvm_apic_disabled_lapic_found(kvm); 1195 *bitmap = 0; 1196 return true; 1197 } 1198 } 1199 1200 *bitmap = (lowest >= 0) ? 1 << lowest : 0; 1201 1202 return true; 1203 } 1204 1205 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 1206 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map) 1207 { 1208 struct kvm_apic_map *map; 1209 unsigned long bitmap; 1210 struct kvm_lapic **dst = NULL; 1211 int i; 1212 bool ret; 1213 1214 *r = -1; 1215 1216 if (irq->shorthand == APIC_DEST_SELF) { 1217 if (KVM_BUG_ON(!src, kvm)) { 1218 *r = 0; 1219 return true; 1220 } 1221 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 1222 return true; 1223 } 1224 1225 rcu_read_lock(); 1226 map = rcu_dereference(kvm->arch.apic_map); 1227 1228 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap); 1229 if (ret) { 1230 *r = 0; 1231 for_each_set_bit(i, &bitmap, 16) { 1232 if (!dst[i]) 1233 continue; 1234 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 1235 } 1236 } 1237 1238 rcu_read_unlock(); 1239 return ret; 1240 } 1241 1242 /* 1243 * This routine tries to handle interrupts in posted mode, here is how 1244 * it deals with different cases: 1245 * - For single-destination interrupts, handle it in posted mode 1246 * - Else if vector hashing is enabled and it is a lowest-priority 1247 * interrupt, handle it in posted mode and use the following mechanism 1248 * to find the destination vCPU. 1249 * 1. For lowest-priority interrupts, store all the possible 1250 * destination vCPUs in an array. 1251 * 2. Use "guest vector % max number of destination vCPUs" to find 1252 * the right destination vCPU in the array for the lowest-priority 1253 * interrupt. 1254 * - Otherwise, use remapped mode to inject the interrupt. 1255 */ 1256 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq, 1257 struct kvm_vcpu **dest_vcpu) 1258 { 1259 struct kvm_apic_map *map; 1260 unsigned long bitmap; 1261 struct kvm_lapic **dst = NULL; 1262 bool ret = false; 1263 1264 if (irq->shorthand) 1265 return false; 1266 1267 rcu_read_lock(); 1268 map = rcu_dereference(kvm->arch.apic_map); 1269 1270 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) && 1271 hweight16(bitmap) == 1) { 1272 unsigned long i = find_first_bit(&bitmap, 16); 1273 1274 if (dst[i]) { 1275 *dest_vcpu = dst[i]->vcpu; 1276 ret = true; 1277 } 1278 } 1279 1280 rcu_read_unlock(); 1281 return ret; 1282 } 1283 1284 /* 1285 * Add a pending IRQ into lapic. 1286 * Return 1 if successfully added and 0 if discarded. 1287 */ 1288 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 1289 int vector, int level, int trig_mode, 1290 struct dest_map *dest_map) 1291 { 1292 int result = 0; 1293 struct kvm_vcpu *vcpu = apic->vcpu; 1294 1295 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 1296 trig_mode, vector); 1297 switch (delivery_mode) { 1298 case APIC_DM_LOWEST: 1299 vcpu->arch.apic_arb_prio++; 1300 fallthrough; 1301 case APIC_DM_FIXED: 1302 if (unlikely(trig_mode && !level)) 1303 break; 1304 1305 /* FIXME add logic for vcpu on reset */ 1306 if (unlikely(!apic_enabled(apic))) 1307 break; 1308 1309 result = 1; 1310 1311 if (dest_map) { 1312 __set_bit(vcpu->vcpu_id, dest_map->map); 1313 dest_map->vectors[vcpu->vcpu_id] = vector; 1314 } 1315 1316 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { 1317 if (trig_mode) 1318 kvm_lapic_set_vector(vector, 1319 apic->regs + APIC_TMR); 1320 else 1321 kvm_lapic_clear_vector(vector, 1322 apic->regs + APIC_TMR); 1323 } 1324 1325 static_call(kvm_x86_deliver_interrupt)(apic, delivery_mode, 1326 trig_mode, vector); 1327 break; 1328 1329 case APIC_DM_REMRD: 1330 result = 1; 1331 vcpu->arch.pv.pv_unhalted = 1; 1332 kvm_make_request(KVM_REQ_EVENT, vcpu); 1333 kvm_vcpu_kick(vcpu); 1334 break; 1335 1336 case APIC_DM_SMI: 1337 if (!kvm_inject_smi(vcpu)) { 1338 kvm_vcpu_kick(vcpu); 1339 result = 1; 1340 } 1341 break; 1342 1343 case APIC_DM_NMI: 1344 result = 1; 1345 kvm_inject_nmi(vcpu); 1346 kvm_vcpu_kick(vcpu); 1347 break; 1348 1349 case APIC_DM_INIT: 1350 if (!trig_mode || level) { 1351 result = 1; 1352 /* assumes that there are only KVM_APIC_INIT/SIPI */ 1353 apic->pending_events = (1UL << KVM_APIC_INIT); 1354 kvm_make_request(KVM_REQ_EVENT, vcpu); 1355 kvm_vcpu_kick(vcpu); 1356 } 1357 break; 1358 1359 case APIC_DM_STARTUP: 1360 result = 1; 1361 apic->sipi_vector = vector; 1362 /* make sure sipi_vector is visible for the receiver */ 1363 smp_wmb(); 1364 set_bit(KVM_APIC_SIPI, &apic->pending_events); 1365 kvm_make_request(KVM_REQ_EVENT, vcpu); 1366 kvm_vcpu_kick(vcpu); 1367 break; 1368 1369 case APIC_DM_EXTINT: 1370 /* 1371 * Should only be called by kvm_apic_local_deliver() with LVT0, 1372 * before NMI watchdog was enabled. Already handled by 1373 * kvm_apic_accept_pic_intr(). 1374 */ 1375 break; 1376 1377 default: 1378 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 1379 delivery_mode); 1380 break; 1381 } 1382 return result; 1383 } 1384 1385 /* 1386 * This routine identifies the destination vcpus mask meant to receive the 1387 * IOAPIC interrupts. It either uses kvm_apic_map_get_dest_lapic() to find 1388 * out the destination vcpus array and set the bitmap or it traverses to 1389 * each available vcpu to identify the same. 1390 */ 1391 void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq, 1392 unsigned long *vcpu_bitmap) 1393 { 1394 struct kvm_lapic **dest_vcpu = NULL; 1395 struct kvm_lapic *src = NULL; 1396 struct kvm_apic_map *map; 1397 struct kvm_vcpu *vcpu; 1398 unsigned long bitmap, i; 1399 int vcpu_idx; 1400 bool ret; 1401 1402 rcu_read_lock(); 1403 map = rcu_dereference(kvm->arch.apic_map); 1404 1405 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu, 1406 &bitmap); 1407 if (ret) { 1408 for_each_set_bit(i, &bitmap, 16) { 1409 if (!dest_vcpu[i]) 1410 continue; 1411 vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx; 1412 __set_bit(vcpu_idx, vcpu_bitmap); 1413 } 1414 } else { 1415 kvm_for_each_vcpu(i, vcpu, kvm) { 1416 if (!kvm_apic_present(vcpu)) 1417 continue; 1418 if (!kvm_apic_match_dest(vcpu, NULL, 1419 irq->shorthand, 1420 irq->dest_id, 1421 irq->dest_mode)) 1422 continue; 1423 __set_bit(i, vcpu_bitmap); 1424 } 1425 } 1426 rcu_read_unlock(); 1427 } 1428 1429 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 1430 { 1431 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 1432 } 1433 1434 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) 1435 { 1436 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); 1437 } 1438 1439 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 1440 { 1441 int trigger_mode; 1442 1443 /* Eoi the ioapic only if the ioapic doesn't own the vector. */ 1444 if (!kvm_ioapic_handles_vector(apic, vector)) 1445 return; 1446 1447 /* Request a KVM exit to inform the userspace IOAPIC. */ 1448 if (irqchip_split(apic->vcpu->kvm)) { 1449 apic->vcpu->arch.pending_ioapic_eoi = vector; 1450 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); 1451 return; 1452 } 1453 1454 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 1455 trigger_mode = IOAPIC_LEVEL_TRIG; 1456 else 1457 trigger_mode = IOAPIC_EDGE_TRIG; 1458 1459 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 1460 } 1461 1462 static int apic_set_eoi(struct kvm_lapic *apic) 1463 { 1464 int vector = apic_find_highest_isr(apic); 1465 1466 trace_kvm_eoi(apic, vector); 1467 1468 /* 1469 * Not every write EOI will has corresponding ISR, 1470 * one example is when Kernel check timer on setup_IO_APIC 1471 */ 1472 if (vector == -1) 1473 return vector; 1474 1475 apic_clear_isr(vector, apic); 1476 apic_update_ppr(apic); 1477 1478 if (to_hv_vcpu(apic->vcpu) && 1479 test_bit(vector, to_hv_synic(apic->vcpu)->vec_bitmap)) 1480 kvm_hv_synic_send_eoi(apic->vcpu, vector); 1481 1482 kvm_ioapic_send_eoi(apic, vector); 1483 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1484 return vector; 1485 } 1486 1487 /* 1488 * this interface assumes a trap-like exit, which has already finished 1489 * desired side effect including vISR and vPPR update. 1490 */ 1491 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 1492 { 1493 struct kvm_lapic *apic = vcpu->arch.apic; 1494 1495 trace_kvm_eoi(apic, vector); 1496 1497 kvm_ioapic_send_eoi(apic, vector); 1498 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1499 } 1500 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 1501 1502 void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high) 1503 { 1504 struct kvm_lapic_irq irq; 1505 1506 /* KVM has no delay and should always clear the BUSY/PENDING flag. */ 1507 WARN_ON_ONCE(icr_low & APIC_ICR_BUSY); 1508 1509 irq.vector = icr_low & APIC_VECTOR_MASK; 1510 irq.delivery_mode = icr_low & APIC_MODE_MASK; 1511 irq.dest_mode = icr_low & APIC_DEST_MASK; 1512 irq.level = (icr_low & APIC_INT_ASSERT) != 0; 1513 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 1514 irq.shorthand = icr_low & APIC_SHORT_MASK; 1515 irq.msi_redir_hint = false; 1516 if (apic_x2apic_mode(apic)) 1517 irq.dest_id = icr_high; 1518 else 1519 irq.dest_id = GET_XAPIC_DEST_FIELD(icr_high); 1520 1521 trace_kvm_apic_ipi(icr_low, irq.dest_id); 1522 1523 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 1524 } 1525 EXPORT_SYMBOL_GPL(kvm_apic_send_ipi); 1526 1527 static u32 apic_get_tmcct(struct kvm_lapic *apic) 1528 { 1529 ktime_t remaining, now; 1530 s64 ns; 1531 1532 ASSERT(apic != NULL); 1533 1534 /* if initial count is 0, current count should also be 0 */ 1535 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 || 1536 apic->lapic_timer.period == 0) 1537 return 0; 1538 1539 now = ktime_get(); 1540 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1541 if (ktime_to_ns(remaining) < 0) 1542 remaining = 0; 1543 1544 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 1545 return div64_u64(ns, (APIC_BUS_CYCLE_NS * apic->divide_count)); 1546 } 1547 1548 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 1549 { 1550 struct kvm_vcpu *vcpu = apic->vcpu; 1551 struct kvm_run *run = vcpu->run; 1552 1553 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 1554 run->tpr_access.rip = kvm_rip_read(vcpu); 1555 run->tpr_access.is_write = write; 1556 } 1557 1558 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 1559 { 1560 if (apic->vcpu->arch.tpr_access_reporting) 1561 __report_tpr_access(apic, write); 1562 } 1563 1564 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 1565 { 1566 u32 val = 0; 1567 1568 if (offset >= LAPIC_MMIO_LENGTH) 1569 return 0; 1570 1571 switch (offset) { 1572 case APIC_ARBPRI: 1573 break; 1574 1575 case APIC_TMCCT: /* Timer CCR */ 1576 if (apic_lvtt_tscdeadline(apic)) 1577 return 0; 1578 1579 val = apic_get_tmcct(apic); 1580 break; 1581 case APIC_PROCPRI: 1582 apic_update_ppr(apic); 1583 val = kvm_lapic_get_reg(apic, offset); 1584 break; 1585 case APIC_TASKPRI: 1586 report_tpr_access(apic, false); 1587 fallthrough; 1588 default: 1589 val = kvm_lapic_get_reg(apic, offset); 1590 break; 1591 } 1592 1593 return val; 1594 } 1595 1596 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 1597 { 1598 return container_of(dev, struct kvm_lapic, dev); 1599 } 1600 1601 #define APIC_REG_MASK(reg) (1ull << ((reg) >> 4)) 1602 #define APIC_REGS_MASK(first, count) \ 1603 (APIC_REG_MASK(first) * ((1ull << (count)) - 1)) 1604 1605 u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic) 1606 { 1607 /* Leave bits '0' for reserved and write-only registers. */ 1608 u64 valid_reg_mask = 1609 APIC_REG_MASK(APIC_ID) | 1610 APIC_REG_MASK(APIC_LVR) | 1611 APIC_REG_MASK(APIC_TASKPRI) | 1612 APIC_REG_MASK(APIC_PROCPRI) | 1613 APIC_REG_MASK(APIC_LDR) | 1614 APIC_REG_MASK(APIC_SPIV) | 1615 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) | 1616 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) | 1617 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) | 1618 APIC_REG_MASK(APIC_ESR) | 1619 APIC_REG_MASK(APIC_ICR) | 1620 APIC_REG_MASK(APIC_LVTT) | 1621 APIC_REG_MASK(APIC_LVTTHMR) | 1622 APIC_REG_MASK(APIC_LVTPC) | 1623 APIC_REG_MASK(APIC_LVT0) | 1624 APIC_REG_MASK(APIC_LVT1) | 1625 APIC_REG_MASK(APIC_LVTERR) | 1626 APIC_REG_MASK(APIC_TMICT) | 1627 APIC_REG_MASK(APIC_TMCCT) | 1628 APIC_REG_MASK(APIC_TDCR); 1629 1630 if (kvm_lapic_lvt_supported(apic, LVT_CMCI)) 1631 valid_reg_mask |= APIC_REG_MASK(APIC_LVTCMCI); 1632 1633 /* ARBPRI, DFR, and ICR2 are not valid in x2APIC mode. */ 1634 if (!apic_x2apic_mode(apic)) 1635 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI) | 1636 APIC_REG_MASK(APIC_DFR) | 1637 APIC_REG_MASK(APIC_ICR2); 1638 1639 return valid_reg_mask; 1640 } 1641 EXPORT_SYMBOL_GPL(kvm_lapic_readable_reg_mask); 1642 1643 static int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 1644 void *data) 1645 { 1646 unsigned char alignment = offset & 0xf; 1647 u32 result; 1648 1649 /* 1650 * WARN if KVM reads ICR in x2APIC mode, as it's an 8-byte register in 1651 * x2APIC and needs to be manually handled by the caller. 1652 */ 1653 WARN_ON_ONCE(apic_x2apic_mode(apic) && offset == APIC_ICR); 1654 1655 if (alignment + len > 4) 1656 return 1; 1657 1658 if (offset > 0x3f0 || 1659 !(kvm_lapic_readable_reg_mask(apic) & APIC_REG_MASK(offset))) 1660 return 1; 1661 1662 result = __apic_read(apic, offset & ~0xf); 1663 1664 trace_kvm_apic_read(offset, result); 1665 1666 switch (len) { 1667 case 1: 1668 case 2: 1669 case 4: 1670 memcpy(data, (char *)&result + alignment, len); 1671 break; 1672 default: 1673 printk(KERN_ERR "Local APIC read with len = %x, " 1674 "should be 1,2, or 4 instead\n", len); 1675 break; 1676 } 1677 return 0; 1678 } 1679 1680 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1681 { 1682 return addr >= apic->base_address && 1683 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1684 } 1685 1686 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1687 gpa_t address, int len, void *data) 1688 { 1689 struct kvm_lapic *apic = to_lapic(this); 1690 u32 offset = address - apic->base_address; 1691 1692 if (!apic_mmio_in_range(apic, address)) 1693 return -EOPNOTSUPP; 1694 1695 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 1696 if (!kvm_check_has_quirk(vcpu->kvm, 1697 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 1698 return -EOPNOTSUPP; 1699 1700 memset(data, 0xff, len); 1701 return 0; 1702 } 1703 1704 kvm_lapic_reg_read(apic, offset, len, data); 1705 1706 return 0; 1707 } 1708 1709 static void update_divide_count(struct kvm_lapic *apic) 1710 { 1711 u32 tmp1, tmp2, tdcr; 1712 1713 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR); 1714 tmp1 = tdcr & 0xf; 1715 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1716 apic->divide_count = 0x1 << (tmp2 & 0x7); 1717 } 1718 1719 static void limit_periodic_timer_frequency(struct kvm_lapic *apic) 1720 { 1721 /* 1722 * Do not allow the guest to program periodic timers with small 1723 * interval, since the hrtimers are not throttled by the host 1724 * scheduler. 1725 */ 1726 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1727 s64 min_period = min_timer_period_us * 1000LL; 1728 1729 if (apic->lapic_timer.period < min_period) { 1730 pr_info_ratelimited( 1731 "vcpu %i: requested %lld ns " 1732 "lapic timer period limited to %lld ns\n", 1733 apic->vcpu->vcpu_id, 1734 apic->lapic_timer.period, min_period); 1735 apic->lapic_timer.period = min_period; 1736 } 1737 } 1738 } 1739 1740 static void cancel_hv_timer(struct kvm_lapic *apic); 1741 1742 static void cancel_apic_timer(struct kvm_lapic *apic) 1743 { 1744 hrtimer_cancel(&apic->lapic_timer.timer); 1745 preempt_disable(); 1746 if (apic->lapic_timer.hv_timer_in_use) 1747 cancel_hv_timer(apic); 1748 preempt_enable(); 1749 atomic_set(&apic->lapic_timer.pending, 0); 1750 } 1751 1752 static void apic_update_lvtt(struct kvm_lapic *apic) 1753 { 1754 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & 1755 apic->lapic_timer.timer_mode_mask; 1756 1757 if (apic->lapic_timer.timer_mode != timer_mode) { 1758 if (apic_lvtt_tscdeadline(apic) != (timer_mode == 1759 APIC_LVT_TIMER_TSCDEADLINE)) { 1760 cancel_apic_timer(apic); 1761 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 1762 apic->lapic_timer.period = 0; 1763 apic->lapic_timer.tscdeadline = 0; 1764 } 1765 apic->lapic_timer.timer_mode = timer_mode; 1766 limit_periodic_timer_frequency(apic); 1767 } 1768 } 1769 1770 /* 1771 * On APICv, this test will cause a busy wait 1772 * during a higher-priority task. 1773 */ 1774 1775 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu) 1776 { 1777 struct kvm_lapic *apic = vcpu->arch.apic; 1778 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT); 1779 1780 if (kvm_apic_hw_enabled(apic)) { 1781 int vec = reg & APIC_VECTOR_MASK; 1782 void *bitmap = apic->regs + APIC_ISR; 1783 1784 if (apic->apicv_active) 1785 bitmap = apic->regs + APIC_IRR; 1786 1787 if (apic_test_vector(vec, bitmap)) 1788 return true; 1789 } 1790 return false; 1791 } 1792 1793 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles) 1794 { 1795 u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns; 1796 1797 /* 1798 * If the guest TSC is running at a different ratio than the host, then 1799 * convert the delay to nanoseconds to achieve an accurate delay. Note 1800 * that __delay() uses delay_tsc whenever the hardware has TSC, thus 1801 * always for VMX enabled hardware. 1802 */ 1803 if (vcpu->arch.tsc_scaling_ratio == kvm_caps.default_tsc_scaling_ratio) { 1804 __delay(min(guest_cycles, 1805 nsec_to_cycles(vcpu, timer_advance_ns))); 1806 } else { 1807 u64 delay_ns = guest_cycles * 1000000ULL; 1808 do_div(delay_ns, vcpu->arch.virtual_tsc_khz); 1809 ndelay(min_t(u32, delay_ns, timer_advance_ns)); 1810 } 1811 } 1812 1813 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu, 1814 s64 advance_expire_delta) 1815 { 1816 struct kvm_lapic *apic = vcpu->arch.apic; 1817 u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns; 1818 u64 ns; 1819 1820 /* Do not adjust for tiny fluctuations or large random spikes. */ 1821 if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX || 1822 abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN) 1823 return; 1824 1825 /* too early */ 1826 if (advance_expire_delta < 0) { 1827 ns = -advance_expire_delta * 1000000ULL; 1828 do_div(ns, vcpu->arch.virtual_tsc_khz); 1829 timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP; 1830 } else { 1831 /* too late */ 1832 ns = advance_expire_delta * 1000000ULL; 1833 do_div(ns, vcpu->arch.virtual_tsc_khz); 1834 timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP; 1835 } 1836 1837 if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX)) 1838 timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT; 1839 apic->lapic_timer.timer_advance_ns = timer_advance_ns; 1840 } 1841 1842 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu) 1843 { 1844 struct kvm_lapic *apic = vcpu->arch.apic; 1845 u64 guest_tsc, tsc_deadline; 1846 1847 tsc_deadline = apic->lapic_timer.expired_tscdeadline; 1848 apic->lapic_timer.expired_tscdeadline = 0; 1849 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1850 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); 1851 1852 if (lapic_timer_advance_dynamic) { 1853 adjust_lapic_timer_advance(vcpu, guest_tsc - tsc_deadline); 1854 /* 1855 * If the timer fired early, reread the TSC to account for the 1856 * overhead of the above adjustment to avoid waiting longer 1857 * than is necessary. 1858 */ 1859 if (guest_tsc < tsc_deadline) 1860 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1861 } 1862 1863 if (guest_tsc < tsc_deadline) 1864 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc); 1865 } 1866 1867 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu) 1868 { 1869 if (lapic_in_kernel(vcpu) && 1870 vcpu->arch.apic->lapic_timer.expired_tscdeadline && 1871 vcpu->arch.apic->lapic_timer.timer_advance_ns && 1872 lapic_timer_int_injected(vcpu)) 1873 __kvm_wait_lapic_expire(vcpu); 1874 } 1875 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire); 1876 1877 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic) 1878 { 1879 struct kvm_timer *ktimer = &apic->lapic_timer; 1880 1881 kvm_apic_local_deliver(apic, APIC_LVTT); 1882 if (apic_lvtt_tscdeadline(apic)) { 1883 ktimer->tscdeadline = 0; 1884 } else if (apic_lvtt_oneshot(apic)) { 1885 ktimer->tscdeadline = 0; 1886 ktimer->target_expiration = 0; 1887 } 1888 } 1889 1890 static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn) 1891 { 1892 struct kvm_vcpu *vcpu = apic->vcpu; 1893 struct kvm_timer *ktimer = &apic->lapic_timer; 1894 1895 if (atomic_read(&apic->lapic_timer.pending)) 1896 return; 1897 1898 if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) 1899 ktimer->expired_tscdeadline = ktimer->tscdeadline; 1900 1901 if (!from_timer_fn && apic->apicv_active) { 1902 WARN_ON(kvm_get_running_vcpu() != vcpu); 1903 kvm_apic_inject_pending_timer_irqs(apic); 1904 return; 1905 } 1906 1907 if (kvm_use_posted_timer_interrupt(apic->vcpu)) { 1908 /* 1909 * Ensure the guest's timer has truly expired before posting an 1910 * interrupt. Open code the relevant checks to avoid querying 1911 * lapic_timer_int_injected(), which will be false since the 1912 * interrupt isn't yet injected. Waiting until after injecting 1913 * is not an option since that won't help a posted interrupt. 1914 */ 1915 if (vcpu->arch.apic->lapic_timer.expired_tscdeadline && 1916 vcpu->arch.apic->lapic_timer.timer_advance_ns) 1917 __kvm_wait_lapic_expire(vcpu); 1918 kvm_apic_inject_pending_timer_irqs(apic); 1919 return; 1920 } 1921 1922 atomic_inc(&apic->lapic_timer.pending); 1923 kvm_make_request(KVM_REQ_UNBLOCK, vcpu); 1924 if (from_timer_fn) 1925 kvm_vcpu_kick(vcpu); 1926 } 1927 1928 static void start_sw_tscdeadline(struct kvm_lapic *apic) 1929 { 1930 struct kvm_timer *ktimer = &apic->lapic_timer; 1931 u64 guest_tsc, tscdeadline = ktimer->tscdeadline; 1932 u64 ns = 0; 1933 ktime_t expire; 1934 struct kvm_vcpu *vcpu = apic->vcpu; 1935 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1936 unsigned long flags; 1937 ktime_t now; 1938 1939 if (unlikely(!tscdeadline || !this_tsc_khz)) 1940 return; 1941 1942 local_irq_save(flags); 1943 1944 now = ktime_get(); 1945 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1946 1947 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1948 do_div(ns, this_tsc_khz); 1949 1950 if (likely(tscdeadline > guest_tsc) && 1951 likely(ns > apic->lapic_timer.timer_advance_ns)) { 1952 expire = ktime_add_ns(now, ns); 1953 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns); 1954 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD); 1955 } else 1956 apic_timer_expired(apic, false); 1957 1958 local_irq_restore(flags); 1959 } 1960 1961 static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict) 1962 { 1963 return (u64)tmict * APIC_BUS_CYCLE_NS * (u64)apic->divide_count; 1964 } 1965 1966 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) 1967 { 1968 ktime_t now, remaining; 1969 u64 ns_remaining_old, ns_remaining_new; 1970 1971 apic->lapic_timer.period = 1972 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); 1973 limit_periodic_timer_frequency(apic); 1974 1975 now = ktime_get(); 1976 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1977 if (ktime_to_ns(remaining) < 0) 1978 remaining = 0; 1979 1980 ns_remaining_old = ktime_to_ns(remaining); 1981 ns_remaining_new = mul_u64_u32_div(ns_remaining_old, 1982 apic->divide_count, old_divisor); 1983 1984 apic->lapic_timer.tscdeadline += 1985 nsec_to_cycles(apic->vcpu, ns_remaining_new) - 1986 nsec_to_cycles(apic->vcpu, ns_remaining_old); 1987 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); 1988 } 1989 1990 static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg) 1991 { 1992 ktime_t now; 1993 u64 tscl = rdtsc(); 1994 s64 deadline; 1995 1996 now = ktime_get(); 1997 apic->lapic_timer.period = 1998 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); 1999 2000 if (!apic->lapic_timer.period) { 2001 apic->lapic_timer.tscdeadline = 0; 2002 return false; 2003 } 2004 2005 limit_periodic_timer_frequency(apic); 2006 deadline = apic->lapic_timer.period; 2007 2008 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { 2009 if (unlikely(count_reg != APIC_TMICT)) { 2010 deadline = tmict_to_ns(apic, 2011 kvm_lapic_get_reg(apic, count_reg)); 2012 if (unlikely(deadline <= 0)) { 2013 if (apic_lvtt_period(apic)) 2014 deadline = apic->lapic_timer.period; 2015 else 2016 deadline = 0; 2017 } 2018 else if (unlikely(deadline > apic->lapic_timer.period)) { 2019 pr_info_ratelimited( 2020 "vcpu %i: requested lapic timer restore with " 2021 "starting count register %#x=%u (%lld ns) > initial count (%lld ns). " 2022 "Using initial count to start timer.\n", 2023 apic->vcpu->vcpu_id, 2024 count_reg, 2025 kvm_lapic_get_reg(apic, count_reg), 2026 deadline, apic->lapic_timer.period); 2027 kvm_lapic_set_reg(apic, count_reg, 0); 2028 deadline = apic->lapic_timer.period; 2029 } 2030 } 2031 } 2032 2033 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 2034 nsec_to_cycles(apic->vcpu, deadline); 2035 apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline); 2036 2037 return true; 2038 } 2039 2040 static void advance_periodic_target_expiration(struct kvm_lapic *apic) 2041 { 2042 ktime_t now = ktime_get(); 2043 u64 tscl = rdtsc(); 2044 ktime_t delta; 2045 2046 /* 2047 * Synchronize both deadlines to the same time source or 2048 * differences in the periods (caused by differences in the 2049 * underlying clocks or numerical approximation errors) will 2050 * cause the two to drift apart over time as the errors 2051 * accumulate. 2052 */ 2053 apic->lapic_timer.target_expiration = 2054 ktime_add_ns(apic->lapic_timer.target_expiration, 2055 apic->lapic_timer.period); 2056 delta = ktime_sub(apic->lapic_timer.target_expiration, now); 2057 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 2058 nsec_to_cycles(apic->vcpu, delta); 2059 } 2060 2061 static void start_sw_period(struct kvm_lapic *apic) 2062 { 2063 if (!apic->lapic_timer.period) 2064 return; 2065 2066 if (ktime_after(ktime_get(), 2067 apic->lapic_timer.target_expiration)) { 2068 apic_timer_expired(apic, false); 2069 2070 if (apic_lvtt_oneshot(apic)) 2071 return; 2072 2073 advance_periodic_target_expiration(apic); 2074 } 2075 2076 hrtimer_start(&apic->lapic_timer.timer, 2077 apic->lapic_timer.target_expiration, 2078 HRTIMER_MODE_ABS_HARD); 2079 } 2080 2081 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) 2082 { 2083 if (!lapic_in_kernel(vcpu)) 2084 return false; 2085 2086 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; 2087 } 2088 2089 static void cancel_hv_timer(struct kvm_lapic *apic) 2090 { 2091 WARN_ON(preemptible()); 2092 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 2093 static_call(kvm_x86_cancel_hv_timer)(apic->vcpu); 2094 apic->lapic_timer.hv_timer_in_use = false; 2095 } 2096 2097 static bool start_hv_timer(struct kvm_lapic *apic) 2098 { 2099 struct kvm_timer *ktimer = &apic->lapic_timer; 2100 struct kvm_vcpu *vcpu = apic->vcpu; 2101 bool expired; 2102 2103 WARN_ON(preemptible()); 2104 if (!kvm_can_use_hv_timer(vcpu)) 2105 return false; 2106 2107 if (!ktimer->tscdeadline) 2108 return false; 2109 2110 if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired)) 2111 return false; 2112 2113 ktimer->hv_timer_in_use = true; 2114 hrtimer_cancel(&ktimer->timer); 2115 2116 /* 2117 * To simplify handling the periodic timer, leave the hv timer running 2118 * even if the deadline timer has expired, i.e. rely on the resulting 2119 * VM-Exit to recompute the periodic timer's target expiration. 2120 */ 2121 if (!apic_lvtt_period(apic)) { 2122 /* 2123 * Cancel the hv timer if the sw timer fired while the hv timer 2124 * was being programmed, or if the hv timer itself expired. 2125 */ 2126 if (atomic_read(&ktimer->pending)) { 2127 cancel_hv_timer(apic); 2128 } else if (expired) { 2129 apic_timer_expired(apic, false); 2130 cancel_hv_timer(apic); 2131 } 2132 } 2133 2134 trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use); 2135 2136 return true; 2137 } 2138 2139 static void start_sw_timer(struct kvm_lapic *apic) 2140 { 2141 struct kvm_timer *ktimer = &apic->lapic_timer; 2142 2143 WARN_ON(preemptible()); 2144 if (apic->lapic_timer.hv_timer_in_use) 2145 cancel_hv_timer(apic); 2146 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 2147 return; 2148 2149 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 2150 start_sw_period(apic); 2151 else if (apic_lvtt_tscdeadline(apic)) 2152 start_sw_tscdeadline(apic); 2153 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); 2154 } 2155 2156 static void restart_apic_timer(struct kvm_lapic *apic) 2157 { 2158 preempt_disable(); 2159 2160 if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending)) 2161 goto out; 2162 2163 if (!start_hv_timer(apic)) 2164 start_sw_timer(apic); 2165 out: 2166 preempt_enable(); 2167 } 2168 2169 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) 2170 { 2171 struct kvm_lapic *apic = vcpu->arch.apic; 2172 2173 preempt_disable(); 2174 /* If the preempt notifier has already run, it also called apic_timer_expired */ 2175 if (!apic->lapic_timer.hv_timer_in_use) 2176 goto out; 2177 WARN_ON(kvm_vcpu_is_blocking(vcpu)); 2178 apic_timer_expired(apic, false); 2179 cancel_hv_timer(apic); 2180 2181 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 2182 advance_periodic_target_expiration(apic); 2183 restart_apic_timer(apic); 2184 } 2185 out: 2186 preempt_enable(); 2187 } 2188 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); 2189 2190 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) 2191 { 2192 restart_apic_timer(vcpu->arch.apic); 2193 } 2194 2195 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) 2196 { 2197 struct kvm_lapic *apic = vcpu->arch.apic; 2198 2199 preempt_disable(); 2200 /* Possibly the TSC deadline timer is not enabled yet */ 2201 if (apic->lapic_timer.hv_timer_in_use) 2202 start_sw_timer(apic); 2203 preempt_enable(); 2204 } 2205 2206 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) 2207 { 2208 struct kvm_lapic *apic = vcpu->arch.apic; 2209 2210 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 2211 restart_apic_timer(apic); 2212 } 2213 2214 static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg) 2215 { 2216 atomic_set(&apic->lapic_timer.pending, 0); 2217 2218 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 2219 && !set_target_expiration(apic, count_reg)) 2220 return; 2221 2222 restart_apic_timer(apic); 2223 } 2224 2225 static void start_apic_timer(struct kvm_lapic *apic) 2226 { 2227 __start_apic_timer(apic, APIC_TMICT); 2228 } 2229 2230 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 2231 { 2232 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val); 2233 2234 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { 2235 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; 2236 if (lvt0_in_nmi_mode) { 2237 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 2238 } else 2239 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 2240 } 2241 } 2242 2243 static int get_lvt_index(u32 reg) 2244 { 2245 if (reg == APIC_LVTCMCI) 2246 return LVT_CMCI; 2247 if (reg < APIC_LVTT || reg > APIC_LVTERR) 2248 return -1; 2249 return array_index_nospec( 2250 (reg - APIC_LVTT) >> 4, KVM_APIC_MAX_NR_LVT_ENTRIES); 2251 } 2252 2253 static int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 2254 { 2255 int ret = 0; 2256 2257 trace_kvm_apic_write(reg, val); 2258 2259 switch (reg) { 2260 case APIC_ID: /* Local APIC ID */ 2261 if (!apic_x2apic_mode(apic)) { 2262 kvm_apic_set_xapic_id(apic, val >> 24); 2263 } else { 2264 ret = 1; 2265 } 2266 break; 2267 2268 case APIC_TASKPRI: 2269 report_tpr_access(apic, true); 2270 apic_set_tpr(apic, val & 0xff); 2271 break; 2272 2273 case APIC_EOI: 2274 apic_set_eoi(apic); 2275 break; 2276 2277 case APIC_LDR: 2278 if (!apic_x2apic_mode(apic)) 2279 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 2280 else 2281 ret = 1; 2282 break; 2283 2284 case APIC_DFR: 2285 if (!apic_x2apic_mode(apic)) 2286 kvm_apic_set_dfr(apic, val | 0x0FFFFFFF); 2287 else 2288 ret = 1; 2289 break; 2290 2291 case APIC_SPIV: { 2292 u32 mask = 0x3ff; 2293 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 2294 mask |= APIC_SPIV_DIRECTED_EOI; 2295 apic_set_spiv(apic, val & mask); 2296 if (!(val & APIC_SPIV_APIC_ENABLED)) { 2297 int i; 2298 2299 for (i = 0; i < apic->nr_lvt_entries; i++) { 2300 kvm_lapic_set_reg(apic, APIC_LVTx(i), 2301 kvm_lapic_get_reg(apic, APIC_LVTx(i)) | APIC_LVT_MASKED); 2302 } 2303 apic_update_lvtt(apic); 2304 atomic_set(&apic->lapic_timer.pending, 0); 2305 2306 } 2307 break; 2308 } 2309 case APIC_ICR: 2310 WARN_ON_ONCE(apic_x2apic_mode(apic)); 2311 2312 /* No delay here, so we always clear the pending bit */ 2313 val &= ~APIC_ICR_BUSY; 2314 kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2)); 2315 kvm_lapic_set_reg(apic, APIC_ICR, val); 2316 break; 2317 case APIC_ICR2: 2318 if (apic_x2apic_mode(apic)) 2319 ret = 1; 2320 else 2321 kvm_lapic_set_reg(apic, APIC_ICR2, val & 0xff000000); 2322 break; 2323 2324 case APIC_LVT0: 2325 apic_manage_nmi_watchdog(apic, val); 2326 fallthrough; 2327 case APIC_LVTTHMR: 2328 case APIC_LVTPC: 2329 case APIC_LVT1: 2330 case APIC_LVTERR: 2331 case APIC_LVTCMCI: { 2332 u32 index = get_lvt_index(reg); 2333 if (!kvm_lapic_lvt_supported(apic, index)) { 2334 ret = 1; 2335 break; 2336 } 2337 if (!kvm_apic_sw_enabled(apic)) 2338 val |= APIC_LVT_MASKED; 2339 val &= apic_lvt_mask[index]; 2340 kvm_lapic_set_reg(apic, reg, val); 2341 break; 2342 } 2343 2344 case APIC_LVTT: 2345 if (!kvm_apic_sw_enabled(apic)) 2346 val |= APIC_LVT_MASKED; 2347 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 2348 kvm_lapic_set_reg(apic, APIC_LVTT, val); 2349 apic_update_lvtt(apic); 2350 break; 2351 2352 case APIC_TMICT: 2353 if (apic_lvtt_tscdeadline(apic)) 2354 break; 2355 2356 cancel_apic_timer(apic); 2357 kvm_lapic_set_reg(apic, APIC_TMICT, val); 2358 start_apic_timer(apic); 2359 break; 2360 2361 case APIC_TDCR: { 2362 uint32_t old_divisor = apic->divide_count; 2363 2364 kvm_lapic_set_reg(apic, APIC_TDCR, val & 0xb); 2365 update_divide_count(apic); 2366 if (apic->divide_count != old_divisor && 2367 apic->lapic_timer.period) { 2368 hrtimer_cancel(&apic->lapic_timer.timer); 2369 update_target_expiration(apic, old_divisor); 2370 restart_apic_timer(apic); 2371 } 2372 break; 2373 } 2374 case APIC_ESR: 2375 if (apic_x2apic_mode(apic) && val != 0) 2376 ret = 1; 2377 break; 2378 2379 case APIC_SELF_IPI: 2380 /* 2381 * Self-IPI exists only when x2APIC is enabled. Bits 7:0 hold 2382 * the vector, everything else is reserved. 2383 */ 2384 if (!apic_x2apic_mode(apic) || (val & ~APIC_VECTOR_MASK)) 2385 ret = 1; 2386 else 2387 kvm_apic_send_ipi(apic, APIC_DEST_SELF | val, 0); 2388 break; 2389 default: 2390 ret = 1; 2391 break; 2392 } 2393 2394 /* 2395 * Recalculate APIC maps if necessary, e.g. if the software enable bit 2396 * was toggled, the APIC ID changed, etc... The maps are marked dirty 2397 * on relevant changes, i.e. this is a nop for most writes. 2398 */ 2399 kvm_recalculate_apic_map(apic->vcpu->kvm); 2400 2401 return ret; 2402 } 2403 2404 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 2405 gpa_t address, int len, const void *data) 2406 { 2407 struct kvm_lapic *apic = to_lapic(this); 2408 unsigned int offset = address - apic->base_address; 2409 u32 val; 2410 2411 if (!apic_mmio_in_range(apic, address)) 2412 return -EOPNOTSUPP; 2413 2414 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 2415 if (!kvm_check_has_quirk(vcpu->kvm, 2416 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 2417 return -EOPNOTSUPP; 2418 2419 return 0; 2420 } 2421 2422 /* 2423 * APIC register must be aligned on 128-bits boundary. 2424 * 32/64/128 bits registers must be accessed thru 32 bits. 2425 * Refer SDM 8.4.1 2426 */ 2427 if (len != 4 || (offset & 0xf)) 2428 return 0; 2429 2430 val = *(u32*)data; 2431 2432 kvm_lapic_reg_write(apic, offset & 0xff0, val); 2433 2434 return 0; 2435 } 2436 2437 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 2438 { 2439 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 2440 } 2441 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 2442 2443 /* emulate APIC access in a trap manner */ 2444 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 2445 { 2446 struct kvm_lapic *apic = vcpu->arch.apic; 2447 u64 val; 2448 2449 /* 2450 * ICR is a single 64-bit register when x2APIC is enabled. For legacy 2451 * xAPIC, ICR writes need to go down the common (slightly slower) path 2452 * to get the upper half from ICR2. 2453 */ 2454 if (apic_x2apic_mode(apic) && offset == APIC_ICR) { 2455 val = kvm_lapic_get_reg64(apic, APIC_ICR); 2456 kvm_apic_send_ipi(apic, (u32)val, (u32)(val >> 32)); 2457 trace_kvm_apic_write(APIC_ICR, val); 2458 } else { 2459 /* TODO: optimize to just emulate side effect w/o one more write */ 2460 val = kvm_lapic_get_reg(apic, offset); 2461 kvm_lapic_reg_write(apic, offset, (u32)val); 2462 } 2463 } 2464 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 2465 2466 void kvm_free_lapic(struct kvm_vcpu *vcpu) 2467 { 2468 struct kvm_lapic *apic = vcpu->arch.apic; 2469 2470 if (!vcpu->arch.apic) 2471 return; 2472 2473 hrtimer_cancel(&apic->lapic_timer.timer); 2474 2475 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 2476 static_branch_slow_dec_deferred(&apic_hw_disabled); 2477 2478 if (!apic->sw_enabled) 2479 static_branch_slow_dec_deferred(&apic_sw_disabled); 2480 2481 if (apic->regs) 2482 free_page((unsigned long)apic->regs); 2483 2484 kfree(apic); 2485 } 2486 2487 /* 2488 *---------------------------------------------------------------------- 2489 * LAPIC interface 2490 *---------------------------------------------------------------------- 2491 */ 2492 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 2493 { 2494 struct kvm_lapic *apic = vcpu->arch.apic; 2495 2496 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic)) 2497 return 0; 2498 2499 return apic->lapic_timer.tscdeadline; 2500 } 2501 2502 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 2503 { 2504 struct kvm_lapic *apic = vcpu->arch.apic; 2505 2506 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic)) 2507 return; 2508 2509 hrtimer_cancel(&apic->lapic_timer.timer); 2510 apic->lapic_timer.tscdeadline = data; 2511 start_apic_timer(apic); 2512 } 2513 2514 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 2515 { 2516 apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4); 2517 } 2518 2519 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 2520 { 2521 u64 tpr; 2522 2523 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 2524 2525 return (tpr & 0xf0) >> 4; 2526 } 2527 2528 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 2529 { 2530 u64 old_value = vcpu->arch.apic_base; 2531 struct kvm_lapic *apic = vcpu->arch.apic; 2532 2533 vcpu->arch.apic_base = value; 2534 2535 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) 2536 kvm_update_cpuid_runtime(vcpu); 2537 2538 if (!apic) 2539 return; 2540 2541 /* update jump label if enable bit changes */ 2542 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 2543 if (value & MSR_IA32_APICBASE_ENABLE) { 2544 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2545 static_branch_slow_dec_deferred(&apic_hw_disabled); 2546 /* Check if there are APF page ready requests pending */ 2547 kvm_make_request(KVM_REQ_APF_READY, vcpu); 2548 } else { 2549 static_branch_inc(&apic_hw_disabled.key); 2550 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 2551 } 2552 } 2553 2554 if ((old_value ^ value) & X2APIC_ENABLE) { 2555 if (value & X2APIC_ENABLE) 2556 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); 2557 else if (value & MSR_IA32_APICBASE_ENABLE) 2558 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2559 } 2560 2561 if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) { 2562 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 2563 static_call_cond(kvm_x86_set_virtual_apic_mode)(vcpu); 2564 } 2565 2566 apic->base_address = apic->vcpu->arch.apic_base & 2567 MSR_IA32_APICBASE_BASE; 2568 2569 if ((value & MSR_IA32_APICBASE_ENABLE) && 2570 apic->base_address != APIC_DEFAULT_PHYS_BASE) { 2571 kvm_set_apicv_inhibit(apic->vcpu->kvm, 2572 APICV_INHIBIT_REASON_APIC_BASE_MODIFIED); 2573 } 2574 } 2575 2576 void kvm_apic_update_apicv(struct kvm_vcpu *vcpu) 2577 { 2578 struct kvm_lapic *apic = vcpu->arch.apic; 2579 2580 if (apic->apicv_active) { 2581 /* irr_pending is always true when apicv is activated. */ 2582 apic->irr_pending = true; 2583 apic->isr_count = 1; 2584 } else { 2585 /* 2586 * Don't clear irr_pending, searching the IRR can race with 2587 * updates from the CPU as APICv is still active from hardware's 2588 * perspective. The flag will be cleared as appropriate when 2589 * KVM injects the interrupt. 2590 */ 2591 apic->isr_count = count_vectors(apic->regs + APIC_ISR); 2592 } 2593 apic->highest_isr_cache = -1; 2594 } 2595 2596 int kvm_alloc_apic_access_page(struct kvm *kvm) 2597 { 2598 struct page *page; 2599 void __user *hva; 2600 int ret = 0; 2601 2602 mutex_lock(&kvm->slots_lock); 2603 if (kvm->arch.apic_access_memslot_enabled || 2604 kvm->arch.apic_access_memslot_inhibited) 2605 goto out; 2606 2607 hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 2608 APIC_DEFAULT_PHYS_BASE, PAGE_SIZE); 2609 if (IS_ERR(hva)) { 2610 ret = PTR_ERR(hva); 2611 goto out; 2612 } 2613 2614 page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 2615 if (is_error_page(page)) { 2616 ret = -EFAULT; 2617 goto out; 2618 } 2619 2620 /* 2621 * Do not pin the page in memory, so that memory hot-unplug 2622 * is able to migrate it. 2623 */ 2624 put_page(page); 2625 kvm->arch.apic_access_memslot_enabled = true; 2626 out: 2627 mutex_unlock(&kvm->slots_lock); 2628 return ret; 2629 } 2630 EXPORT_SYMBOL_GPL(kvm_alloc_apic_access_page); 2631 2632 void kvm_inhibit_apic_access_page(struct kvm_vcpu *vcpu) 2633 { 2634 struct kvm *kvm = vcpu->kvm; 2635 2636 if (!kvm->arch.apic_access_memslot_enabled) 2637 return; 2638 2639 kvm_vcpu_srcu_read_unlock(vcpu); 2640 2641 mutex_lock(&kvm->slots_lock); 2642 2643 if (kvm->arch.apic_access_memslot_enabled) { 2644 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0); 2645 /* 2646 * Clear "enabled" after the memslot is deleted so that a 2647 * different vCPU doesn't get a false negative when checking 2648 * the flag out of slots_lock. No additional memory barrier is 2649 * needed as modifying memslots requires waiting other vCPUs to 2650 * drop SRCU (see above), and false positives are ok as the 2651 * flag is rechecked after acquiring slots_lock. 2652 */ 2653 kvm->arch.apic_access_memslot_enabled = false; 2654 2655 /* 2656 * Mark the memslot as inhibited to prevent reallocating the 2657 * memslot during vCPU creation, e.g. if a vCPU is hotplugged. 2658 */ 2659 kvm->arch.apic_access_memslot_inhibited = true; 2660 } 2661 2662 mutex_unlock(&kvm->slots_lock); 2663 2664 kvm_vcpu_srcu_read_lock(vcpu); 2665 } 2666 2667 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) 2668 { 2669 struct kvm_lapic *apic = vcpu->arch.apic; 2670 u64 msr_val; 2671 int i; 2672 2673 if (!init_event) { 2674 msr_val = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE; 2675 if (kvm_vcpu_is_reset_bsp(vcpu)) 2676 msr_val |= MSR_IA32_APICBASE_BSP; 2677 kvm_lapic_set_base(vcpu, msr_val); 2678 } 2679 2680 if (!apic) 2681 return; 2682 2683 /* Stop the timer in case it's a reset to an active apic */ 2684 hrtimer_cancel(&apic->lapic_timer.timer); 2685 2686 /* The xAPIC ID is set at RESET even if the APIC was already enabled. */ 2687 if (!init_event) 2688 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2689 kvm_apic_set_version(apic->vcpu); 2690 2691 for (i = 0; i < apic->nr_lvt_entries; i++) 2692 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED); 2693 apic_update_lvtt(apic); 2694 if (kvm_vcpu_is_reset_bsp(vcpu) && 2695 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) 2696 kvm_lapic_set_reg(apic, APIC_LVT0, 2697 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 2698 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2699 2700 kvm_apic_set_dfr(apic, 0xffffffffU); 2701 apic_set_spiv(apic, 0xff); 2702 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0); 2703 if (!apic_x2apic_mode(apic)) 2704 kvm_apic_set_ldr(apic, 0); 2705 kvm_lapic_set_reg(apic, APIC_ESR, 0); 2706 if (!apic_x2apic_mode(apic)) { 2707 kvm_lapic_set_reg(apic, APIC_ICR, 0); 2708 kvm_lapic_set_reg(apic, APIC_ICR2, 0); 2709 } else { 2710 kvm_lapic_set_reg64(apic, APIC_ICR, 0); 2711 } 2712 kvm_lapic_set_reg(apic, APIC_TDCR, 0); 2713 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 2714 for (i = 0; i < 8; i++) { 2715 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 2716 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 2717 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 2718 } 2719 kvm_apic_update_apicv(vcpu); 2720 update_divide_count(apic); 2721 atomic_set(&apic->lapic_timer.pending, 0); 2722 2723 vcpu->arch.pv_eoi.msr_val = 0; 2724 apic_update_ppr(apic); 2725 if (apic->apicv_active) { 2726 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu); 2727 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, -1); 2728 static_call_cond(kvm_x86_hwapic_isr_update)(-1); 2729 } 2730 2731 vcpu->arch.apic_arb_prio = 0; 2732 vcpu->arch.apic_attention = 0; 2733 2734 kvm_recalculate_apic_map(vcpu->kvm); 2735 } 2736 2737 /* 2738 *---------------------------------------------------------------------- 2739 * timer interface 2740 *---------------------------------------------------------------------- 2741 */ 2742 2743 static bool lapic_is_periodic(struct kvm_lapic *apic) 2744 { 2745 return apic_lvtt_period(apic); 2746 } 2747 2748 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 2749 { 2750 struct kvm_lapic *apic = vcpu->arch.apic; 2751 2752 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT)) 2753 return atomic_read(&apic->lapic_timer.pending); 2754 2755 return 0; 2756 } 2757 2758 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 2759 { 2760 u32 reg = kvm_lapic_get_reg(apic, lvt_type); 2761 int vector, mode, trig_mode; 2762 int r; 2763 2764 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 2765 vector = reg & APIC_VECTOR_MASK; 2766 mode = reg & APIC_MODE_MASK; 2767 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 2768 2769 r = __apic_accept_irq(apic, mode, vector, 1, trig_mode, NULL); 2770 if (r && lvt_type == APIC_LVTPC) 2771 kvm_lapic_set_reg(apic, APIC_LVTPC, reg | APIC_LVT_MASKED); 2772 return r; 2773 } 2774 return 0; 2775 } 2776 2777 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 2778 { 2779 struct kvm_lapic *apic = vcpu->arch.apic; 2780 2781 if (apic) 2782 kvm_apic_local_deliver(apic, APIC_LVT0); 2783 } 2784 2785 static const struct kvm_io_device_ops apic_mmio_ops = { 2786 .read = apic_mmio_read, 2787 .write = apic_mmio_write, 2788 }; 2789 2790 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 2791 { 2792 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 2793 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 2794 2795 apic_timer_expired(apic, true); 2796 2797 if (lapic_is_periodic(apic)) { 2798 advance_periodic_target_expiration(apic); 2799 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 2800 return HRTIMER_RESTART; 2801 } else 2802 return HRTIMER_NORESTART; 2803 } 2804 2805 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns) 2806 { 2807 struct kvm_lapic *apic; 2808 2809 ASSERT(vcpu != NULL); 2810 2811 apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT); 2812 if (!apic) 2813 goto nomem; 2814 2815 vcpu->arch.apic = apic; 2816 2817 apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); 2818 if (!apic->regs) { 2819 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 2820 vcpu->vcpu_id); 2821 goto nomem_free_apic; 2822 } 2823 apic->vcpu = vcpu; 2824 2825 apic->nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu); 2826 2827 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 2828 HRTIMER_MODE_ABS_HARD); 2829 apic->lapic_timer.timer.function = apic_timer_fn; 2830 if (timer_advance_ns == -1) { 2831 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT; 2832 lapic_timer_advance_dynamic = true; 2833 } else { 2834 apic->lapic_timer.timer_advance_ns = timer_advance_ns; 2835 lapic_timer_advance_dynamic = false; 2836 } 2837 2838 /* 2839 * Stuff the APIC ENABLE bit in lieu of temporarily incrementing 2840 * apic_hw_disabled; the full RESET value is set by kvm_lapic_reset(). 2841 */ 2842 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 2843 static_branch_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 2844 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 2845 2846 return 0; 2847 nomem_free_apic: 2848 kfree(apic); 2849 vcpu->arch.apic = NULL; 2850 nomem: 2851 return -ENOMEM; 2852 } 2853 2854 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 2855 { 2856 struct kvm_lapic *apic = vcpu->arch.apic; 2857 u32 ppr; 2858 2859 if (!kvm_apic_present(vcpu)) 2860 return -1; 2861 2862 __apic_update_ppr(apic, &ppr); 2863 return apic_has_interrupt_for_ppr(apic, ppr); 2864 } 2865 EXPORT_SYMBOL_GPL(kvm_apic_has_interrupt); 2866 2867 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 2868 { 2869 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); 2870 2871 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 2872 return 1; 2873 if ((lvt0 & APIC_LVT_MASKED) == 0 && 2874 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 2875 return 1; 2876 return 0; 2877 } 2878 2879 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 2880 { 2881 struct kvm_lapic *apic = vcpu->arch.apic; 2882 2883 if (atomic_read(&apic->lapic_timer.pending) > 0) { 2884 kvm_apic_inject_pending_timer_irqs(apic); 2885 atomic_set(&apic->lapic_timer.pending, 0); 2886 } 2887 } 2888 2889 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 2890 { 2891 int vector = kvm_apic_has_interrupt(vcpu); 2892 struct kvm_lapic *apic = vcpu->arch.apic; 2893 u32 ppr; 2894 2895 if (vector == -1) 2896 return -1; 2897 2898 /* 2899 * We get here even with APIC virtualization enabled, if doing 2900 * nested virtualization and L1 runs with the "acknowledge interrupt 2901 * on exit" mode. Then we cannot inject the interrupt via RVI, 2902 * because the process would deliver it through the IDT. 2903 */ 2904 2905 apic_clear_irr(vector, apic); 2906 if (to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap)) { 2907 /* 2908 * For auto-EOI interrupts, there might be another pending 2909 * interrupt above PPR, so check whether to raise another 2910 * KVM_REQ_EVENT. 2911 */ 2912 apic_update_ppr(apic); 2913 } else { 2914 /* 2915 * For normal interrupts, PPR has been raised and there cannot 2916 * be a higher-priority pending interrupt---except if there was 2917 * a concurrent interrupt injection, but that would have 2918 * triggered KVM_REQ_EVENT already. 2919 */ 2920 apic_set_isr(vector, apic); 2921 __apic_update_ppr(apic, &ppr); 2922 } 2923 2924 return vector; 2925 } 2926 2927 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, 2928 struct kvm_lapic_state *s, bool set) 2929 { 2930 if (apic_x2apic_mode(vcpu->arch.apic)) { 2931 u32 *id = (u32 *)(s->regs + APIC_ID); 2932 u32 *ldr = (u32 *)(s->regs + APIC_LDR); 2933 u64 icr; 2934 2935 if (vcpu->kvm->arch.x2apic_format) { 2936 if (*id != vcpu->vcpu_id) 2937 return -EINVAL; 2938 } else { 2939 if (set) 2940 *id >>= 24; 2941 else 2942 *id <<= 24; 2943 } 2944 2945 /* 2946 * In x2APIC mode, the LDR is fixed and based on the id. And 2947 * ICR is internally a single 64-bit register, but needs to be 2948 * split to ICR+ICR2 in userspace for backwards compatibility. 2949 */ 2950 if (set) { 2951 *ldr = kvm_apic_calc_x2apic_ldr(*id); 2952 2953 icr = __kvm_lapic_get_reg(s->regs, APIC_ICR) | 2954 (u64)__kvm_lapic_get_reg(s->regs, APIC_ICR2) << 32; 2955 __kvm_lapic_set_reg64(s->regs, APIC_ICR, icr); 2956 } else { 2957 icr = __kvm_lapic_get_reg64(s->regs, APIC_ICR); 2958 __kvm_lapic_set_reg(s->regs, APIC_ICR2, icr >> 32); 2959 } 2960 } 2961 2962 return 0; 2963 } 2964 2965 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2966 { 2967 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); 2968 2969 /* 2970 * Get calculated timer current count for remaining timer period (if 2971 * any) and store it in the returned register set. 2972 */ 2973 __kvm_lapic_set_reg(s->regs, APIC_TMCCT, 2974 __apic_read(vcpu->arch.apic, APIC_TMCCT)); 2975 2976 return kvm_apic_state_fixup(vcpu, s, false); 2977 } 2978 2979 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2980 { 2981 struct kvm_lapic *apic = vcpu->arch.apic; 2982 int r; 2983 2984 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 2985 /* set SPIV separately to get count of SW disabled APICs right */ 2986 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 2987 2988 r = kvm_apic_state_fixup(vcpu, s, true); 2989 if (r) { 2990 kvm_recalculate_apic_map(vcpu->kvm); 2991 return r; 2992 } 2993 memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s)); 2994 2995 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 2996 kvm_recalculate_apic_map(vcpu->kvm); 2997 kvm_apic_set_version(vcpu); 2998 2999 apic_update_ppr(apic); 3000 cancel_apic_timer(apic); 3001 apic->lapic_timer.expired_tscdeadline = 0; 3002 apic_update_lvtt(apic); 3003 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 3004 update_divide_count(apic); 3005 __start_apic_timer(apic, APIC_TMCCT); 3006 kvm_lapic_set_reg(apic, APIC_TMCCT, 0); 3007 kvm_apic_update_apicv(vcpu); 3008 if (apic->apicv_active) { 3009 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu); 3010 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, apic_find_highest_irr(apic)); 3011 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic)); 3012 } 3013 kvm_make_request(KVM_REQ_EVENT, vcpu); 3014 if (ioapic_in_kernel(vcpu->kvm)) 3015 kvm_rtc_eoi_tracking_restore_one(vcpu); 3016 3017 vcpu->arch.apic_arb_prio = 0; 3018 3019 return 0; 3020 } 3021 3022 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 3023 { 3024 struct hrtimer *timer; 3025 3026 if (!lapic_in_kernel(vcpu) || 3027 kvm_can_post_timer_interrupt(vcpu)) 3028 return; 3029 3030 timer = &vcpu->arch.apic->lapic_timer.timer; 3031 if (hrtimer_cancel(timer)) 3032 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD); 3033 } 3034 3035 /* 3036 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 3037 * 3038 * Detect whether guest triggered PV EOI since the 3039 * last entry. If yes, set EOI on guests's behalf. 3040 * Clear PV EOI in guest memory in any case. 3041 */ 3042 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 3043 struct kvm_lapic *apic) 3044 { 3045 int vector; 3046 /* 3047 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 3048 * and KVM_PV_EOI_ENABLED in guest memory as follows: 3049 * 3050 * KVM_APIC_PV_EOI_PENDING is unset: 3051 * -> host disabled PV EOI. 3052 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 3053 * -> host enabled PV EOI, guest did not execute EOI yet. 3054 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 3055 * -> host enabled PV EOI, guest executed EOI. 3056 */ 3057 BUG_ON(!pv_eoi_enabled(vcpu)); 3058 3059 if (pv_eoi_test_and_clr_pending(vcpu)) 3060 return; 3061 vector = apic_set_eoi(apic); 3062 trace_kvm_pv_eoi(apic, vector); 3063 } 3064 3065 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 3066 { 3067 u32 data; 3068 3069 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 3070 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 3071 3072 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 3073 return; 3074 3075 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 3076 sizeof(u32))) 3077 return; 3078 3079 apic_set_tpr(vcpu->arch.apic, data & 0xff); 3080 } 3081 3082 /* 3083 * apic_sync_pv_eoi_to_guest - called before vmentry 3084 * 3085 * Detect whether it's safe to enable PV EOI and 3086 * if yes do so. 3087 */ 3088 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 3089 struct kvm_lapic *apic) 3090 { 3091 if (!pv_eoi_enabled(vcpu) || 3092 /* IRR set or many bits in ISR: could be nested. */ 3093 apic->irr_pending || 3094 /* Cache not set: could be safe but we don't bother. */ 3095 apic->highest_isr_cache == -1 || 3096 /* Need EOI to update ioapic. */ 3097 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { 3098 /* 3099 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 3100 * so we need not do anything here. 3101 */ 3102 return; 3103 } 3104 3105 pv_eoi_set_pending(apic->vcpu); 3106 } 3107 3108 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 3109 { 3110 u32 data, tpr; 3111 int max_irr, max_isr; 3112 struct kvm_lapic *apic = vcpu->arch.apic; 3113 3114 apic_sync_pv_eoi_to_guest(vcpu, apic); 3115 3116 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 3117 return; 3118 3119 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff; 3120 max_irr = apic_find_highest_irr(apic); 3121 if (max_irr < 0) 3122 max_irr = 0; 3123 max_isr = apic_find_highest_isr(apic); 3124 if (max_isr < 0) 3125 max_isr = 0; 3126 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 3127 3128 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 3129 sizeof(u32)); 3130 } 3131 3132 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 3133 { 3134 if (vapic_addr) { 3135 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 3136 &vcpu->arch.apic->vapic_cache, 3137 vapic_addr, sizeof(u32))) 3138 return -EINVAL; 3139 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 3140 } else { 3141 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 3142 } 3143 3144 vcpu->arch.apic->vapic_addr = vapic_addr; 3145 return 0; 3146 } 3147 3148 int kvm_x2apic_icr_write(struct kvm_lapic *apic, u64 data) 3149 { 3150 data &= ~APIC_ICR_BUSY; 3151 3152 kvm_apic_send_ipi(apic, (u32)data, (u32)(data >> 32)); 3153 kvm_lapic_set_reg64(apic, APIC_ICR, data); 3154 trace_kvm_apic_write(APIC_ICR, data); 3155 return 0; 3156 } 3157 3158 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data) 3159 { 3160 u32 low; 3161 3162 if (reg == APIC_ICR) { 3163 *data = kvm_lapic_get_reg64(apic, APIC_ICR); 3164 return 0; 3165 } 3166 3167 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 3168 return 1; 3169 3170 *data = low; 3171 3172 return 0; 3173 } 3174 3175 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data) 3176 { 3177 /* 3178 * ICR is a 64-bit register in x2APIC mode (and Hyper-V PV vAPIC) and 3179 * can be written as such, all other registers remain accessible only 3180 * through 32-bit reads/writes. 3181 */ 3182 if (reg == APIC_ICR) 3183 return kvm_x2apic_icr_write(apic, data); 3184 3185 /* Bits 63:32 are reserved in all other registers. */ 3186 if (data >> 32) 3187 return 1; 3188 3189 return kvm_lapic_reg_write(apic, reg, (u32)data); 3190 } 3191 3192 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 3193 { 3194 struct kvm_lapic *apic = vcpu->arch.apic; 3195 u32 reg = (msr - APIC_BASE_MSR) << 4; 3196 3197 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 3198 return 1; 3199 3200 return kvm_lapic_msr_write(apic, reg, data); 3201 } 3202 3203 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 3204 { 3205 struct kvm_lapic *apic = vcpu->arch.apic; 3206 u32 reg = (msr - APIC_BASE_MSR) << 4; 3207 3208 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 3209 return 1; 3210 3211 return kvm_lapic_msr_read(apic, reg, data); 3212 } 3213 3214 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 3215 { 3216 if (!lapic_in_kernel(vcpu)) 3217 return 1; 3218 3219 return kvm_lapic_msr_write(vcpu->arch.apic, reg, data); 3220 } 3221 3222 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 3223 { 3224 if (!lapic_in_kernel(vcpu)) 3225 return 1; 3226 3227 return kvm_lapic_msr_read(vcpu->arch.apic, reg, data); 3228 } 3229 3230 int kvm_lapic_set_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len) 3231 { 3232 u64 addr = data & ~KVM_MSR_ENABLED; 3233 struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data; 3234 unsigned long new_len; 3235 int ret; 3236 3237 if (!IS_ALIGNED(addr, 4)) 3238 return 1; 3239 3240 if (data & KVM_MSR_ENABLED) { 3241 if (addr == ghc->gpa && len <= ghc->len) 3242 new_len = ghc->len; 3243 else 3244 new_len = len; 3245 3246 ret = kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); 3247 if (ret) 3248 return ret; 3249 } 3250 3251 vcpu->arch.pv_eoi.msr_val = data; 3252 3253 return 0; 3254 } 3255 3256 int kvm_apic_accept_events(struct kvm_vcpu *vcpu) 3257 { 3258 struct kvm_lapic *apic = vcpu->arch.apic; 3259 u8 sipi_vector; 3260 int r; 3261 3262 if (!kvm_apic_has_pending_init_or_sipi(vcpu)) 3263 return 0; 3264 3265 if (is_guest_mode(vcpu)) { 3266 r = kvm_check_nested_events(vcpu); 3267 if (r < 0) 3268 return r == -EBUSY ? 0 : r; 3269 /* 3270 * Continue processing INIT/SIPI even if a nested VM-Exit 3271 * occurred, e.g. pending SIPIs should be dropped if INIT+SIPI 3272 * are blocked as a result of transitioning to VMX root mode. 3273 */ 3274 } 3275 3276 /* 3277 * INITs are blocked while CPU is in specific states (SMM, VMX root 3278 * mode, SVM with GIF=0), while SIPIs are dropped if the CPU isn't in 3279 * wait-for-SIPI (WFS). 3280 */ 3281 if (!kvm_apic_init_sipi_allowed(vcpu)) { 3282 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); 3283 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 3284 return 0; 3285 } 3286 3287 if (test_and_clear_bit(KVM_APIC_INIT, &apic->pending_events)) { 3288 kvm_vcpu_reset(vcpu, true); 3289 if (kvm_vcpu_is_bsp(apic->vcpu)) 3290 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3291 else 3292 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 3293 } 3294 if (test_and_clear_bit(KVM_APIC_SIPI, &apic->pending_events)) { 3295 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 3296 /* evaluate pending_events before reading the vector */ 3297 smp_rmb(); 3298 sipi_vector = apic->sipi_vector; 3299 static_call(kvm_x86_vcpu_deliver_sipi_vector)(vcpu, sipi_vector); 3300 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3301 } 3302 } 3303 return 0; 3304 } 3305 3306 void kvm_lapic_exit(void) 3307 { 3308 static_key_deferred_flush(&apic_hw_disabled); 3309 WARN_ON(static_branch_unlikely(&apic_hw_disabled.key)); 3310 static_key_deferred_flush(&apic_sw_disabled); 3311 WARN_ON(static_branch_unlikely(&apic_sw_disabled.key)); 3312 } 3313